diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index 0aaf210f061a9d6370cd36799f3e3c1ca021ebb7..ed7e4b8b294ef8a695c3a46e843de2e22f290176 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -2518,9 +2518,9 @@ class gpload: # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table, # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True. sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name) - cols = map(lambda a:'%s %s' % (a[0], a[1]), target_columns) + cols = map(lambda a:'"%s" %s' % (a[0], a[1]), target_columns) sql += "(%s)" % ','.join(cols) - sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols) + #sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols) self.log(self.LOG, sql) if not self.options.D: diff --git a/gpMgmt/bin/gpload_test/.gitignore b/gpMgmt/bin/gpload_test/.gitignore index 97f8f218f184a7981bf8cbebd3e94b80a5df0be9..4aa755af8b6dc4891e3c4548fe63416a6ccccc4b 100644 --- a/gpMgmt/bin/gpload_test/.gitignore +++ b/gpMgmt/bin/gpload_test/.gitignore @@ -4,6 +4,7 @@ gpload/config gpload2/config gpload2/setup.out gpload2/data_file.txt +gpload2/data_file2.txt GPTest.pm gpstringsubs.pl gpdiff.pl diff --git a/gpMgmt/bin/gpload_test/gpload2/TEST.py b/gpMgmt/bin/gpload_test/gpload2/TEST.py index 8cdeacd08f59bb5910eafc697b1857b29455a061..6ae37f6ef3fe3f60fb2b497c207ff11b4ea6a089 100755 --- a/gpMgmt/bin/gpload_test/gpload2/TEST.py +++ b/gpMgmt/bin/gpload_test/gpload2/TEST.py @@ -98,8 +98,8 @@ d = mkpath('config') if not os.path.exists(d): os.mkdir(d) -def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False, config='config/config_file'): +def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False, config='config/config_file', match_columns='true', update_columns='n2'): f = open(mkpath(config),'w') f.write("VERSION: 1.0.0.1") if database: @@ -164,11 +164,16 @@ def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0', if mode == 'merge': f.write("\n - MODE: "+'merge') f.write("\n - UPDATE_COLUMNS:") - f.write("\n - n2") - f.write("\n - MATCH_COLUMNS:") - f.write("\n - n1") - f.write("\n - s1") - f.write("\n - s2") + f.write("\n - "+update_columns) + if match_columns=='true': + f.write("\n - MATCH_COLUMNS:") + f.write("\n - n1") + f.write("\n - s1") + f.write("\n - s2") + if match_columns=='2': + f.write("\n - MATCH_COLUMNS:") + f.write("\n - '\"Field1\"'") + f.write("\n - '\"Field#2\"'") if mapping=='1': f.write("\n - MAPPING:") f.write("\n s1: s_s1") @@ -800,6 +805,11 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): runfile(file) copy_data('external_file_15.txt','data_file.txt') write_config_file(mode='insert',reuse_flag='true',fast_match='false', file='data_file.txt',table='testSpecialChar',columns_flag='2', delimiter=";") + copy_data('external_file_16.txt','data_file2.txt') + write_config_file(update_columns='\'"Field#2"\'',config='config/config_file2', mode='merge',reuse_flag='true',fast_match='false', file='data_file2.txt',table='testSpecialChar',columns_flag='2', delimiter=";",match_columns='2') + f = open(mkpath('query41.sql'),'a') + f.write("\! gpload -f "+mkpath('config/config_file2')+ " -d reuse_gptest\n") + f.close() self.doTest(41) diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt new file mode 100644 index 0000000000000000000000000000000000000000..06cd5435dceb2f8af4973eaa325774f1cfa795b7 --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt @@ -0,0 +1,8 @@ +1;Line 1 +2;2nd line +3;test +4; +5;Vide +6;Field 2 +8;new line +9;Line 10 \ No newline at end of file diff --git a/gpMgmt/bin/gpload_test/gpload2/query41.ans b/gpMgmt/bin/gpload_test/gpload2/query41.ans index 08546b38484256e4cd64f7175e390fd97733e84e..c3926cec21290c8b2d77f5e4719b7a3d3e10e1b5 100644 --- a/gpMgmt/bin/gpload_test/gpload2/query41.ans +++ b/gpMgmt/bin/gpload_test/gpload2/query41.ans @@ -1,18 +1,28 @@ -2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06 -2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar' -2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2020-08-20 16:13:06|INFO|did not find an external table to reuse. creating ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d -2020-08-20 16:13:06|INFO|running time: 0.08 seconds -2020-08-20 16:13:06|INFO|rows Inserted = 8 -2020-08-20 16:13:06|INFO|rows Updated = 0 -2020-08-20 16:13:06|INFO|data formatting errors = 0 -2020-08-20 16:13:06|INFO|gpload succeeded -2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06 -2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar' -2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2020-08-20 16:13:06|INFO|reusing external table ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d -2020-08-20 16:13:06|INFO|running time: 0.07 seconds -2020-08-20 16:13:06|INFO|rows Inserted = 8 -2020-08-20 16:13:06|INFO|rows Updated = 0 -2020-08-20 16:13:06|INFO|data formatting errors = 0 -2020-08-20 16:13:06|INFO|gpload succeeded +2020-09-01 11:10:22|INFO|gpload session started 2020-09-01 11:10:22 +2020-09-01 11:10:22|INFO|setting schema 'public' for table 'testspecialchar' +2020-09-01 11:10:22|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-09-01 11:10:22|INFO|did not find an external table to reuse. creating ext_gpload_reusable_ac968900_ec00_11ea_9fbd_00505698707d +2020-09-01 11:10:22|INFO|running time: 0.08 seconds +2020-09-01 11:10:22|INFO|rows Inserted = 8 +2020-09-01 11:10:22|INFO|rows Updated = 0 +2020-09-01 11:10:22|INFO|data formatting errors = 0 +2020-09-01 11:10:22|INFO|gpload succeeded +2020-09-01 11:10:23|INFO|gpload session started 2020-09-01 11:10:23 +2020-09-01 11:10:23|INFO|setting schema 'public' for table 'testspecialchar' +2020-09-01 11:10:23|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-09-01 11:10:23|INFO|reusing external table ext_gpload_reusable_ac968900_ec00_11ea_9fbd_00505698707d +2020-09-01 11:10:23|INFO|running time: 0.06 seconds +2020-09-01 11:10:23|INFO|rows Inserted = 8 +2020-09-01 11:10:23|INFO|rows Updated = 0 +2020-09-01 11:10:23|INFO|data formatting errors = 0 +2020-09-01 11:10:23|INFO|gpload succeeded +2020-09-01 11:10:23|INFO|gpload session started 2020-09-01 11:10:23 +2020-09-01 11:10:23|INFO|setting schema 'public' for table 'testspecialchar' +2020-09-01 11:10:23|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file2.txt" -t 30 +2020-09-01 11:10:23|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_a1101b5024707ea34f55e778f329e548 +2020-09-01 11:10:23|INFO|did not find an external table to reuse. creating ext_gpload_reusable_ace96f80_ec00_11ea_bd1e_00505698707d +2020-09-01 11:10:23|INFO|running time: 0.10 seconds +2020-09-01 11:10:23|INFO|rows Inserted = 2 +2020-09-01 11:10:23|INFO|rows Updated = 12 +2020-09-01 11:10:23|INFO|data formatting errors = 0 +2020-09-01 11:10:23|INFO|gpload succeeded \ No newline at end of file