fix gpload multi-level partition table and special char in columns issue (#10686)

fix match column condition to resovle primary key conflict when using the gpload
merge mode to import data to the Multi-level partition table
fix fail when special char and capital letters in column names
Co-authored-by: NXiaoxiaoHe <hxiaoxiao@vmware.com>

gpMgmt/bin/gpload.py

@@ -1888,7 +1888,7 @@ class gpload:
 
                 # Mark this column as having no mapping, which is important
                 # for do_insert()
-                self.from_columns.append([key.lower(),d[key].lower(),None, False])
+                self.from_columns.append([quote_ident(key),d[key].lower(),None, False])
         else:
             self.from_columns = self.into_columns
             self.from_cols_from_user = False
@@ -2758,7 +2758,7 @@ class gpload:
         sql += 'FROM %s) AS from_table ' % self.staging_table_name
         sql += 'LEFT OUTER JOIN %s into_table ' % self.get_qualified_tablename()
         sql += 'ON %s '%' AND '.join(match)
-        where = self.map_stuff('gpload:output:match_columns',lambda x,y:'into_table.%s IS NULL'%x,0)
+        where = self.map_stuff('gpload:output:match_columns',lambda x,y:'(into_table.%s IS NULL OR CAST(into_table.%s AS varchar) = \'\')'%(x,x),0)
         sql += 'WHERE %s ' % ' AND '.join(where)
         sql += 'AND gpload_row_number=1)'
 

gpMgmt/bin/gpload_test/gpload2/TEST.py

@@ -98,9 +98,9 @@ d = mkpath('config')
 if not os.path.exists(d):
     os.mkdir(d)
 
-def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False):
+def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False, config='config/config_file'):
 
-    f = open(mkpath('config/config_file'),'w')
+    f = open(mkpath(config),'w')
     f.write("VERSION: 1.0.0.1")
     if database:
         f.write("\nDATABASE: "+database)
@@ -131,6 +131,11 @@ def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',
         f.write("\n           - s_n7: double precision")
         f.write("\n           - s_n8: text")
         f.write("\n           - s_n9: text")
+    if columns_flag == '2':
+        f.write("\n    - COLUMNS:")
+        f.write("\n           - 'Field1': bigint")
+        f.write("\n           - 'Field#2': text")
+
     if format:
         f.write("\n    - FORMAT: "+format)
     if log_errors:
@@ -445,7 +450,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
 
     def test_00_gpload_formatOpts_setup(self):
         "0  gpload setup"
-        for num in range(1,40):
+        for num in range(1,42):
            f = open(mkpath('query%d.sql' % num),'w')
            f.write("\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n"+"\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n")
            f.close()
@@ -763,6 +768,41 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase):
         write_config_file(mode='insert',reuse_flag='false',fast_match='false',file='data_file.txt',table='texttable1', error_limit='1000', fill=True)
         self.doTest(39)
 
+    def test_40_gpload_merge_mode_with_multi_pk(self):
+	"40  gpload merge mode with multiple pk"
+        file = mkpath('setup.sql')
+        runfile(file)
+        copy_data('external_file_pk.txt','data_file.txt')
+        write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file.txt',table='testpk')
+        copy_data('external_file_pk2.txt','data_file2.txt')
+        write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file2.txt',table='testpk',config='config/config_file2')
+        f = open(mkpath('query40.sql'),'w')
+        f.write("""\! psql -d reuse_gptest -c "create table testpk (n1 integer, s1 integer, s2 varchar(128), n2 integer, primary key(n1,s1,s2))\
+                partition by range (s1)\
+                subpartition by list(s2)\
+                SUBPARTITION TEMPLATE\
+                ( SUBPARTITION usa VALUES ('usa'),\
+                        SUBPARTITION asia VALUES ('asia'),\
+                        SUBPARTITION europe VALUES ('europe'),\
+                        DEFAULT SUBPARTITION other_regions)\
+                (start (1) end (13) every (1),\
+                default partition others)\
+                ;"\n""")
+        f.write("\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n")
+        f.write("\! gpload -f "+mkpath('config/config_file2')+ " -d reuse_gptest\n")
+        f.write("\! psql -d reuse_gptest -c 'drop table testpk;'\n")
+        f.close()
+        self.doTest(40)
+
+    def test_41_gpload_special_char(self):
+        "41 gpload special char"
+        file = mkpath('setup.sql')
+        runfile(file)
+        copy_data('external_file_15.txt','data_file.txt')
+        write_config_file(mode='insert',reuse_flag='true',fast_match='false', file='data_file.txt',table='testSpecialChar',columns_flag='2', delimiter=";")
+        self.doTest(41)
+
+
 if __name__ == '__main__':
     suite = unittest.TestLoader().loadTestsFromTestCase(GPLoad_FormatOpts_TestCase)
     runner = unittest.TextTestRunner(verbosity=2)

gpMgmt/bin/gpload_test/gpload2/data/external_file_15.txt

+1;Line 1
+2;2nd line
+3;test
+4;
+5;Vide
+6;Field 2
+8;Line 8
+9;Line 9

gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt

+1|1|usa|1
+2|2|usa|2
+3|2|asia|3
+4|2|usa|4

gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt

+1|1|usa|1
+2|2|usa|2
+3|2|asia|56
+4|2|usa|4
+5|12|asia|5

gpMgmt/bin/gpload_test/gpload2/data_file.csv

-1997,Ford,E350,&ac, abs, moon&,3000.00
-1999,Chevy,&Venture &&Extended Edition&&&,&&,4900.00
-1999,Chevy,&Venture &&Extended Edition, Very Large&&&,,5000.00
-1996,Jeep,Grand Cherokee,&MUST SELL!
-air, moon roof, loaded&,4799.00
+123,abc,abc,abc,1.23
+234,bcd,bcd,bcd,2.34

gpMgmt/bin/gpload_test/gpload2/query40.ans

+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_others" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_others_2_prt_usa" for table "testpk_1_prt_others"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_others_2_prt_asia" for table "testpk_1_prt_others"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_others_2_prt_europe" for table "testpk_1_prt_others"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_others_2_prt_other_regions" for table "testpk_1_prt_others"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_2" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_2_2_prt_usa" for table "testpk_1_prt_2"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_2_2_prt_asia" for table "testpk_1_prt_2"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_2_2_prt_europe" for table "testpk_1_prt_2"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_2_2_prt_other_regions" for table "testpk_1_prt_2"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_3" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_3_2_prt_usa" for table "testpk_1_prt_3"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_3_2_prt_asia" for table "testpk_1_prt_3"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_3_2_prt_europe" for table "testpk_1_prt_3"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_3_2_prt_other_regions" for table "testpk_1_prt_3"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_4" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_4_2_prt_usa" for table "testpk_1_prt_4"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_4_2_prt_asia" for table "testpk_1_prt_4"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_4_2_prt_europe" for table "testpk_1_prt_4"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_4_2_prt_other_regions" for table "testpk_1_prt_4"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_5" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_5_2_prt_usa" for table "testpk_1_prt_5"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_5_2_prt_asia" for table "testpk_1_prt_5"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_5_2_prt_europe" for table "testpk_1_prt_5"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_5_2_prt_other_regions" for table "testpk_1_prt_5"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_6" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_6_2_prt_usa" for table "testpk_1_prt_6"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_6_2_prt_asia" for table "testpk_1_prt_6"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_6_2_prt_europe" for table "testpk_1_prt_6"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_6_2_prt_other_regions" for table "testpk_1_prt_6"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_7" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_7_2_prt_usa" for table "testpk_1_prt_7"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_7_2_prt_asia" for table "testpk_1_prt_7"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_7_2_prt_europe" for table "testpk_1_prt_7"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_7_2_prt_other_regions" for table "testpk_1_prt_7"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_8" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_8_2_prt_usa" for table "testpk_1_prt_8"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_8_2_prt_asia" for table "testpk_1_prt_8"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_8_2_prt_europe" for table "testpk_1_prt_8"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_8_2_prt_other_regions" for table "testpk_1_prt_8"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_9" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_9_2_prt_usa" for table "testpk_1_prt_9"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_9_2_prt_asia" for table "testpk_1_prt_9"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_9_2_prt_europe" for table "testpk_1_prt_9"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_9_2_prt_other_regions" for table "testpk_1_prt_9"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_10" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_10_2_prt_usa" for table "testpk_1_prt_10"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_10_2_prt_asia" for table "testpk_1_prt_10"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_10_2_prt_europe" for table "testpk_1_prt_10"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_10_2_prt_other_regions" for table "testpk_1_prt_10"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_11" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_11_2_prt_usa" for table "testpk_1_prt_11"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_11_2_prt_asia" for table "testpk_1_prt_11"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_11_2_prt_europe" for table "testpk_1_prt_11"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_11_2_prt_other_regions" for table "testpk_1_prt_11"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_12" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_12_2_prt_usa" for table "testpk_1_prt_12"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_12_2_prt_asia" for table "testpk_1_prt_12"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_12_2_prt_europe" for table "testpk_1_prt_12"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_12_2_prt_other_regions" for table "testpk_1_prt_12"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_13" for table "testpk"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_13_2_prt_usa" for table "testpk_1_prt_13"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_13_2_prt_asia" for table "testpk_1_prt_13"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_13_2_prt_europe" for table "testpk_1_prt_13"
+NOTICE:  CREATE TABLE will create partition "testpk_1_prt_13_2_prt_other_regions" for table "testpk_1_prt_13"
+CREATE TABLE
+2020-08-21 14:09:30|INFO|gpload session started 2020-08-21 14:09:30
+2020-08-21 14:09:30|INFO|setting schema 'public' for table 'testpk'
+2020-08-21 14:09:30|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
+2020-08-21 14:09:30|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20
+2020-08-21 14:09:30|INFO|did not find an external table to reuse. creating ext_gpload_reusable_e068ae78_e374_11ea_bc95_00505698707d
+2020-08-21 14:09:31|INFO|running time: 0.19 seconds
+2020-08-21 14:09:31|INFO|rows Inserted          = 4
+2020-08-21 14:09:31|INFO|rows Updated           = 0
+2020-08-21 14:09:31|INFO|data formatting errors = 0
+2020-08-21 14:09:31|INFO|gpload succeeded
+2020-08-21 14:09:31|INFO|gpload session started 2020-08-21 14:09:31
+2020-08-21 14:09:31|INFO|setting schema 'public' for table 'testpk'
+2020-08-21 14:09:31|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file2.txt" -t 30
+2020-08-21 14:09:31|INFO|reusing staging table staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20
+2020-08-21 14:09:31|INFO|did not find an external table to reuse. creating ext_gpload_reusable_e0aab714_e374_11ea_a28c_00505698707d
+2020-08-21 14:09:31|INFO|running time: 0.19 seconds
+2020-08-21 14:09:31|INFO|rows Inserted          = 1
+2020-08-21 14:09:31|INFO|rows Updated           = 4
+2020-08-21 14:09:31|INFO|data formatting errors = 0
+2020-08-21 14:09:31|INFO|gpload succeeded
+DROP TABLE

gpMgmt/bin/gpload_test/gpload2/query41.ans

+2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06
+2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar'
+2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
+2020-08-20 16:13:06|INFO|did not find an external table to reuse. creating ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d
+2020-08-20 16:13:06|INFO|running time: 0.08 seconds
+2020-08-20 16:13:06|INFO|rows Inserted          = 8
+2020-08-20 16:13:06|INFO|rows Updated           = 0
+2020-08-20 16:13:06|INFO|data formatting errors = 0
+2020-08-20 16:13:06|INFO|gpload succeeded
+2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06
+2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar'
+2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30
+2020-08-20 16:13:06|INFO|reusing external table ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d
+2020-08-20 16:13:06|INFO|running time: 0.07 seconds
+2020-08-20 16:13:06|INFO|rows Inserted          = 8
+2020-08-20 16:13:06|INFO|rows Updated           = 0
+2020-08-20 16:13:06|INFO|data formatting errors = 0
+2020-08-20 16:13:06|INFO|gpload succeeded

gpMgmt/bin/gpload_test/gpload2/setup.ans

@@ -15,6 +15,9 @@ DROP TABLE IF EXISTS csvtable;
 DROP TABLE
 DROP TABLE IF EXISTS texttable1;
 DROP TABLE
+DROP TABLE IF EXISTS testSpecialChar;
+NOTICE:  table "testspecialchar" does not exist, skipping
+DROP TABLE
 reset client_min_messages;
 RESET
 CREATE TABLE texttable (
@@ -35,3 +38,5 @@ CREATE TABLE test.csvtable (
 	    year int, make text, model text, decription text, price decimal)
             DISTRIBUTED BY (year);
 CREATE TABLE
+create table testSpecialChar("Field1" bigint, "Field#2" text) distributed by ("Field1");
+CREATE TABLE

gpMgmt/bin/gpload_test/gpload2/setup.sql

@@ -11,6 +11,7 @@ DROP EXTERNAL TABLE IF EXISTS temp_gpload_staging_table;
 DROP TABLE IF EXISTS texttable;
 DROP TABLE IF EXISTS csvtable;
 DROP TABLE IF EXISTS texttable1;
+DROP TABLE IF EXISTS testSpecialChar;
 reset client_min_messages;
 CREATE TABLE texttable (
             s1 text, s2 text, s3 text, dt timestamp,
@@ -26,3 +27,4 @@ CREATE TABLE csvtable (
 CREATE TABLE test.csvtable (
 	    year int, make text, model text, decription text, price decimal)
             DISTRIBUTED BY (year);
+create table testSpecialChar("Field1" bigint, "Field#2" text) distributed by ("Field1");