From 4dfee0f3bb581dc4e0fa83d1573b90725341e6de Mon Sep 17 00:00:00 2001 From: "Huiliang.Liu" Date: Mon, 17 Aug 2020 16:47:08 +0800 Subject: [PATCH] Add merge mode with multiple pk test case --- gpMgmt/bin/gpload_test/gpload2/TEST.py | 21 ++++- .../gpload2/data/external_file_pk.txt | 4 + .../gpload2/data/external_file_pk2.txt | 5 ++ gpMgmt/bin/gpload_test/gpload2/query39.ans | 30 ++++++++ gpMgmt/bin/gpload_test/gpload2/setup.ans | 77 +++++++++++++++++++ gpMgmt/bin/gpload_test/gpload2/setup.sql | 11 +++ 6 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt create mode 100644 gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt create mode 100644 gpMgmt/bin/gpload_test/gpload2/query39.ans diff --git a/gpMgmt/bin/gpload_test/gpload2/TEST.py b/gpMgmt/bin/gpload_test/gpload2/TEST.py index e874f3b109..6942acdd58 100755 --- a/gpMgmt/bin/gpload_test/gpload2/TEST.py +++ b/gpMgmt/bin/gpload_test/gpload2/TEST.py @@ -98,9 +98,9 @@ d = mkpath('config') if not os.path.exists(d): os.mkdir(d) -def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True): +def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, config='config/config_file'): - f = open(mkpath('config/config_file'),'w') + f = open(mkpath(config),'w') f.write("VERSION: 1.0.0.1") if database: f.write("\nDATABASE: "+database) @@ -443,7 +443,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): def test_00_gpload_formatOpts_setup(self): "0 gpload setup" - for num in range(1,39): + for num in range(1,40): f = open(mkpath('query%d.sql' % num),'w') f.write("\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n"+"\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n") f.close() @@ -753,6 +753,21 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): write_config_file(mode='insert',reuse_flag='true',fast_match='false',file='data_file.txt',error_table="err_table",error_limit='1000',preload=False) self.doTest(38) + def test_39_gpload_merge_mode_with_multi_pk(self): + "39 gpload merge mode with multiple pk" + file = mkpath('setup.sql') + runfile(file) + copy_data('external_file_pk.txt','data_file.txt') + write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file.txt',table='testpk') + copy_data('external_file_pk2.txt','data_file2.txt') + write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file2.txt',table='testpk',config='config/config_file2') + f = open(mkpath('query39.sql'),'a') + f.write("\! gpload -f "+mkpath('config/config_file2')+ " -d reuse_gptest\n") + f.close() + self.doTest(39) + + + if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(GPLoad_FormatOpts_TestCase) runner = unittest.TextTestRunner(verbosity=2) diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt new file mode 100644 index 0000000000..a8338136bb --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt @@ -0,0 +1,4 @@ +1|1|usa|1 +2|2|usa|2 +3|2|asia|3 +4|2|usa|4 diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt new file mode 100644 index 0000000000..1534ebb5e7 --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt @@ -0,0 +1,5 @@ +1|1|usa|1 +2|2|usa|2 +3|2|asia|56 +4|2|usa|4 +5|12|asia|5 diff --git a/gpMgmt/bin/gpload_test/gpload2/query39.ans b/gpMgmt/bin/gpload_test/gpload2/query39.ans new file mode 100644 index 0000000000..8991b1200e --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/query39.ans @@ -0,0 +1,30 @@ +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-17 16:12:26|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|did not find an external table to reuse. creating ext_gpload_reusable_62d0fa4e_e061_11ea_aee0_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.12 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 4 +2020-08-17 16:12:26|INFO|rows Updated = 0 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-17 16:12:26|INFO|reusing staging table staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|reusing external table ext_gpload_reusable_62d0fa4e_e061_11ea_aee0_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.11 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 0 +2020-08-17 16:12:26|INFO|rows Updated = 4 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file2.txt" -t 30 +2020-08-17 16:12:26|INFO|reusing staging table staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|did not find an external table to reuse. creating ext_gpload_reusable_631ae46a_e061_11ea_993b_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.16 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 1 +2020-08-17 16:12:26|INFO|rows Updated = 4 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded diff --git a/gpMgmt/bin/gpload_test/gpload2/setup.ans b/gpMgmt/bin/gpload_test/gpload2/setup.ans index 8750621631..fb48223feb 100644 --- a/gpMgmt/bin/gpload_test/gpload2/setup.ans +++ b/gpMgmt/bin/gpload_test/gpload2/setup.ans @@ -26,3 +26,80 @@ CREATE TABLE test.csvtable ( year int, make text, model text, decription text, price decimal) DISTRIBUTED BY (year); CREATE TABLE +create table testpk (n1 integer, s1 integer, s2 varchar(128), n2 integer, primary key(n1,s1,s2)) +partition by range (s1) + subpartition by list(s2) + SUBPARTITION TEMPLATE + ( SUBPARTITION usa VALUES ('usa'), + SUBPARTITION asia VALUES ('asia'), + SUBPARTITION europe VALUES ('europe'), + DEFAULT SUBPARTITION other_regions) +(start (1) end (13) every (1), +default partition others) +; +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_usa" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_asia" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_europe" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_other_regions" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_usa" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_asia" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_europe" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_other_regions" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_usa" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_asia" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_europe" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_other_regions" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_usa" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_asia" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_europe" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_other_regions" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_usa" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_asia" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_europe" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_other_regions" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_usa" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_asia" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_europe" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_other_regions" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_usa" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_asia" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_europe" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_other_regions" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_usa" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_asia" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_europe" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_other_regions" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_usa" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_asia" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_europe" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_other_regions" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_usa" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_asia" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_europe" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_other_regions" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_usa" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_asia" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_europe" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_other_regions" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_usa" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_asia" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_europe" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_other_regions" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_usa" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_asia" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_europe" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_other_regions" for table "testpk_1_prt_13" +CREATE TABLE diff --git a/gpMgmt/bin/gpload_test/gpload2/setup.sql b/gpMgmt/bin/gpload_test/gpload2/setup.sql index b5a8608ce2..37585b24e6 100644 --- a/gpMgmt/bin/gpload_test/gpload2/setup.sql +++ b/gpMgmt/bin/gpload_test/gpload2/setup.sql @@ -19,3 +19,14 @@ CREATE TABLE csvtable ( CREATE TABLE test.csvtable ( year int, make text, model text, decription text, price decimal) DISTRIBUTED BY (year); +create table testpk (n1 integer, s1 integer, s2 varchar(128), n2 integer, primary key(n1,s1,s2)) +partition by range (s1) + subpartition by list(s2) + SUBPARTITION TEMPLATE + ( SUBPARTITION usa VALUES ('usa'), + SUBPARTITION asia VALUES ('asia'), + SUBPARTITION europe VALUES ('europe'), + DEFAULT SUBPARTITION other_regions) +(start (1) end (13) every (1), +default partition others) +; -- GitLab