From bad7a3924735444e7f704d59b0dea2b0ad9c4652 Mon Sep 17 00:00:00 2001 From: xiaoxiao <53000479+xiaoxiaoHe-E@users.noreply.github.com> Date: Mon, 7 Sep 2020 21:20:55 +0800 Subject: [PATCH] Revert "fix gpload fail when capital letters in column name in merge mode (#10751)" This reverts commit 4ba11904a8d8b040f98fe89db553296c9f0ba8ff. --- gpMgmt/bin/gpload.py | 4 +- gpMgmt/bin/gpload_test/.gitignore | 1 - gpMgmt/bin/gpload_test/gpload2/TEST.py | 22 +++------ .../gpload2/data/external_file_16.txt | 8 ---- gpMgmt/bin/gpload_test/gpload2/query41.ans | 46 ++++++++----------- 5 files changed, 26 insertions(+), 55 deletions(-) delete mode 100644 gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index ed7e4b8b29..0aaf210f06 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -2518,9 +2518,9 @@ class gpload: # MPP-14667 - self.reuse_tables should change one, and only one, aspect of how we build the following table, # and that is, whether it's a temp table or not. In other words, is_temp_table = '' iff self.reuse_tables == True. sql = 'CREATE %sTABLE %s ' % (is_temp_table, self.staging_table_name) - cols = map(lambda a:'"%s" %s' % (a[0], a[1]), target_columns) + cols = map(lambda a:'%s %s' % (a[0], a[1]), target_columns) sql += "(%s)" % ','.join(cols) - #sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols) + sql += " DISTRIBUTED BY (%s)" % ', '.join(distcols) self.log(self.LOG, sql) if not self.options.D: diff --git a/gpMgmt/bin/gpload_test/.gitignore b/gpMgmt/bin/gpload_test/.gitignore index 4aa755af8b..97f8f218f1 100644 --- a/gpMgmt/bin/gpload_test/.gitignore +++ b/gpMgmt/bin/gpload_test/.gitignore @@ -4,7 +4,6 @@ gpload/config gpload2/config gpload2/setup.out gpload2/data_file.txt -gpload2/data_file2.txt GPTest.pm gpstringsubs.pl gpdiff.pl diff --git a/gpMgmt/bin/gpload_test/gpload2/TEST.py b/gpMgmt/bin/gpload_test/gpload2/TEST.py index 6ae37f6ef3..8cdeacd08f 100755 --- a/gpMgmt/bin/gpload_test/gpload2/TEST.py +++ b/gpMgmt/bin/gpload_test/gpload2/TEST.py @@ -98,8 +98,8 @@ d = mkpath('config') if not os.path.exists(d): os.mkdir(d) +def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False, config='config/config_file'): -def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, fill=False, config='config/config_file', match_columns='true', update_columns='n2'): f = open(mkpath(config),'w') f.write("VERSION: 1.0.0.1") if database: @@ -164,16 +164,11 @@ def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0', if mode == 'merge': f.write("\n - MODE: "+'merge') f.write("\n - UPDATE_COLUMNS:") - f.write("\n - "+update_columns) - if match_columns=='true': - f.write("\n - MATCH_COLUMNS:") - f.write("\n - n1") - f.write("\n - s1") - f.write("\n - s2") - if match_columns=='2': - f.write("\n - MATCH_COLUMNS:") - f.write("\n - '\"Field1\"'") - f.write("\n - '\"Field#2\"'") + f.write("\n - n2") + f.write("\n - MATCH_COLUMNS:") + f.write("\n - n1") + f.write("\n - s1") + f.write("\n - s2") if mapping=='1': f.write("\n - MAPPING:") f.write("\n s1: s_s1") @@ -805,11 +800,6 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): runfile(file) copy_data('external_file_15.txt','data_file.txt') write_config_file(mode='insert',reuse_flag='true',fast_match='false', file='data_file.txt',table='testSpecialChar',columns_flag='2', delimiter=";") - copy_data('external_file_16.txt','data_file2.txt') - write_config_file(update_columns='\'"Field#2"\'',config='config/config_file2', mode='merge',reuse_flag='true',fast_match='false', file='data_file2.txt',table='testSpecialChar',columns_flag='2', delimiter=";",match_columns='2') - f = open(mkpath('query41.sql'),'a') - f.write("\! gpload -f "+mkpath('config/config_file2')+ " -d reuse_gptest\n") - f.close() self.doTest(41) diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt deleted file mode 100644 index 06cd5435dc..0000000000 --- a/gpMgmt/bin/gpload_test/gpload2/data/external_file_16.txt +++ /dev/null @@ -1,8 +0,0 @@ -1;Line 1 -2;2nd line -3;test -4; -5;Vide -6;Field 2 -8;new line -9;Line 10 \ No newline at end of file diff --git a/gpMgmt/bin/gpload_test/gpload2/query41.ans b/gpMgmt/bin/gpload_test/gpload2/query41.ans index c3926cec21..08546b3848 100644 --- a/gpMgmt/bin/gpload_test/gpload2/query41.ans +++ b/gpMgmt/bin/gpload_test/gpload2/query41.ans @@ -1,28 +1,18 @@ -2020-09-01 11:10:22|INFO|gpload session started 2020-09-01 11:10:22 -2020-09-01 11:10:22|INFO|setting schema 'public' for table 'testspecialchar' -2020-09-01 11:10:22|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2020-09-01 11:10:22|INFO|did not find an external table to reuse. creating ext_gpload_reusable_ac968900_ec00_11ea_9fbd_00505698707d -2020-09-01 11:10:22|INFO|running time: 0.08 seconds -2020-09-01 11:10:22|INFO|rows Inserted = 8 -2020-09-01 11:10:22|INFO|rows Updated = 0 -2020-09-01 11:10:22|INFO|data formatting errors = 0 -2020-09-01 11:10:22|INFO|gpload succeeded -2020-09-01 11:10:23|INFO|gpload session started 2020-09-01 11:10:23 -2020-09-01 11:10:23|INFO|setting schema 'public' for table 'testspecialchar' -2020-09-01 11:10:23|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 -2020-09-01 11:10:23|INFO|reusing external table ext_gpload_reusable_ac968900_ec00_11ea_9fbd_00505698707d -2020-09-01 11:10:23|INFO|running time: 0.06 seconds -2020-09-01 11:10:23|INFO|rows Inserted = 8 -2020-09-01 11:10:23|INFO|rows Updated = 0 -2020-09-01 11:10:23|INFO|data formatting errors = 0 -2020-09-01 11:10:23|INFO|gpload succeeded -2020-09-01 11:10:23|INFO|gpload session started 2020-09-01 11:10:23 -2020-09-01 11:10:23|INFO|setting schema 'public' for table 'testspecialchar' -2020-09-01 11:10:23|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file2.txt" -t 30 -2020-09-01 11:10:23|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_a1101b5024707ea34f55e778f329e548 -2020-09-01 11:10:23|INFO|did not find an external table to reuse. creating ext_gpload_reusable_ace96f80_ec00_11ea_bd1e_00505698707d -2020-09-01 11:10:23|INFO|running time: 0.10 seconds -2020-09-01 11:10:23|INFO|rows Inserted = 2 -2020-09-01 11:10:23|INFO|rows Updated = 12 -2020-09-01 11:10:23|INFO|data formatting errors = 0 -2020-09-01 11:10:23|INFO|gpload succeeded \ No newline at end of file +2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06 +2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar' +2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-20 16:13:06|INFO|did not find an external table to reuse. creating ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d +2020-08-20 16:13:06|INFO|running time: 0.08 seconds +2020-08-20 16:13:06|INFO|rows Inserted = 8 +2020-08-20 16:13:06|INFO|rows Updated = 0 +2020-08-20 16:13:06|INFO|data formatting errors = 0 +2020-08-20 16:13:06|INFO|gpload succeeded +2020-08-20 16:13:06|INFO|gpload session started 2020-08-20 16:13:06 +2020-08-20 16:13:06|INFO|setting schema 'public' for table 'testspecialchar' +2020-08-20 16:13:06|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-20 16:13:06|INFO|reusing external table ext_gpload_reusable_f9d2769a_e2bc_11ea_9270_00505698707d +2020-08-20 16:13:06|INFO|running time: 0.07 seconds +2020-08-20 16:13:06|INFO|rows Inserted = 8 +2020-08-20 16:13:06|INFO|rows Updated = 0 +2020-08-20 16:13:06|INFO|data formatting errors = 0 +2020-08-20 16:13:06|INFO|gpload succeeded -- GitLab