diff --git a/gpMgmt/bin/gpload_test/gpload2/TEST.py b/gpMgmt/bin/gpload_test/gpload2/TEST.py index e874f3b109d33adf92f0e2ef1516c0920720f4d0..6942acdd58c45488b3c769e0c867b11acde820eb 100755 --- a/gpMgmt/bin/gpload_test/gpload2/TEST.py +++ b/gpMgmt/bin/gpload_test/gpload2/TEST.py @@ -98,9 +98,9 @@ d = mkpath('config') if not os.path.exists(d): os.mkdir(d) -def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True): +def write_config_file(mode='insert', reuse_flag='',columns_flag='0',mapping='0',portNum='8081',database='reuse_gptest',host='localhost',formatOpts='text',file='data/external_file_01.txt',table='texttable',format='text',delimiter="'|'",escape='',quote='',truncate='False',log_errors=None, error_limit='0',error_table=None,externalSchema=None,staging_table=None,fast_match='false', encoding=None, preload=True, config='config/config_file'): - f = open(mkpath('config/config_file'),'w') + f = open(mkpath(config),'w') f.write("VERSION: 1.0.0.1") if database: f.write("\nDATABASE: "+database) @@ -443,7 +443,7 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): def test_00_gpload_formatOpts_setup(self): "0 gpload setup" - for num in range(1,39): + for num in range(1,40): f = open(mkpath('query%d.sql' % num),'w') f.write("\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n"+"\! gpload -f "+mkpath('config/config_file')+ " -d reuse_gptest\n") f.close() @@ -753,6 +753,21 @@ class GPLoad_FormatOpts_TestCase(unittest.TestCase): write_config_file(mode='insert',reuse_flag='true',fast_match='false',file='data_file.txt',error_table="err_table",error_limit='1000',preload=False) self.doTest(38) + def test_39_gpload_merge_mode_with_multi_pk(self): + "39 gpload merge mode with multiple pk" + file = mkpath('setup.sql') + runfile(file) + copy_data('external_file_pk.txt','data_file.txt') + write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file.txt',table='testpk') + copy_data('external_file_pk2.txt','data_file2.txt') + write_config_file(mode='merge',reuse_flag='true',fast_match='false',file='data_file2.txt',table='testpk',config='config/config_file2') + f = open(mkpath('query39.sql'),'a') + f.write("\! gpload -f "+mkpath('config/config_file2')+ " -d reuse_gptest\n") + f.close() + self.doTest(39) + + + if __name__ == '__main__': suite = unittest.TestLoader().loadTestsFromTestCase(GPLoad_FormatOpts_TestCase) runner = unittest.TextTestRunner(verbosity=2) diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt new file mode 100644 index 0000000000000000000000000000000000000000..a8338136bbd94aa2ab779371e208123b563de07a --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk.txt @@ -0,0 +1,4 @@ +1|1|usa|1 +2|2|usa|2 +3|2|asia|3 +4|2|usa|4 diff --git a/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt new file mode 100644 index 0000000000000000000000000000000000000000..1534ebb5e73716054d84f3a00905a30c0aafddbf --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/data/external_file_pk2.txt @@ -0,0 +1,5 @@ +1|1|usa|1 +2|2|usa|2 +3|2|asia|56 +4|2|usa|4 +5|12|asia|5 diff --git a/gpMgmt/bin/gpload_test/gpload2/query39.ans b/gpMgmt/bin/gpload_test/gpload2/query39.ans new file mode 100644 index 0000000000000000000000000000000000000000..8991b1200efaa283a74de2150f7deb91834d1b72 --- /dev/null +++ b/gpMgmt/bin/gpload_test/gpload2/query39.ans @@ -0,0 +1,30 @@ +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-17 16:12:26|INFO|did not find a staging table to reuse. creating staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|did not find an external table to reuse. creating ext_gpload_reusable_62d0fa4e_e061_11ea_aee0_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.12 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 4 +2020-08-17 16:12:26|INFO|rows Updated = 0 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file.txt" -t 30 +2020-08-17 16:12:26|INFO|reusing staging table staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|reusing external table ext_gpload_reusable_62d0fa4e_e061_11ea_aee0_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.11 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 0 +2020-08-17 16:12:26|INFO|rows Updated = 4 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded +2020-08-17 16:12:26|INFO|gpload session started 2020-08-17 16:12:26 +2020-08-17 16:12:26|INFO|setting schema 'public' for table 'testpk' +2020-08-17 16:12:26|INFO|started gpfdist -p 8081 -P 8082 -f "/home/gpadmin/workspace/gpdb/gpMgmt/bin/gpload_test/gpload2/data_file2.txt" -t 30 +2020-08-17 16:12:26|INFO|reusing staging table staging_gpload_reusable_9cfc886e8f484d05619f4e7cb992ce20 +2020-08-17 16:12:26|INFO|did not find an external table to reuse. creating ext_gpload_reusable_631ae46a_e061_11ea_993b_00505698d059 +2020-08-17 16:12:26|INFO|running time: 0.16 seconds +2020-08-17 16:12:26|INFO|rows Inserted = 1 +2020-08-17 16:12:26|INFO|rows Updated = 4 +2020-08-17 16:12:26|INFO|data formatting errors = 0 +2020-08-17 16:12:26|INFO|gpload succeeded diff --git a/gpMgmt/bin/gpload_test/gpload2/setup.ans b/gpMgmt/bin/gpload_test/gpload2/setup.ans index 87506216312f7e29a36717ed4abe9a87b3bbfd6c..fb48223feb1b09d5bd1e299205fb20eeeff61bbb 100644 --- a/gpMgmt/bin/gpload_test/gpload2/setup.ans +++ b/gpMgmt/bin/gpload_test/gpload2/setup.ans @@ -26,3 +26,80 @@ CREATE TABLE test.csvtable ( year int, make text, model text, decription text, price decimal) DISTRIBUTED BY (year); CREATE TABLE +create table testpk (n1 integer, s1 integer, s2 varchar(128), n2 integer, primary key(n1,s1,s2)) +partition by range (s1) + subpartition by list(s2) + SUBPARTITION TEMPLATE + ( SUBPARTITION usa VALUES ('usa'), + SUBPARTITION asia VALUES ('asia'), + SUBPARTITION europe VALUES ('europe'), + DEFAULT SUBPARTITION other_regions) +(start (1) end (13) every (1), +default partition others) +; +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_usa" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_asia" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_europe" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_others_2_prt_other_regions" for table "testpk_1_prt_others" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_usa" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_asia" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_europe" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_2_2_prt_other_regions" for table "testpk_1_prt_2" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_usa" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_asia" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_europe" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_3_2_prt_other_regions" for table "testpk_1_prt_3" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_usa" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_asia" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_europe" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_4_2_prt_other_regions" for table "testpk_1_prt_4" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_usa" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_asia" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_europe" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_5_2_prt_other_regions" for table "testpk_1_prt_5" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_usa" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_asia" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_europe" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_6_2_prt_other_regions" for table "testpk_1_prt_6" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_usa" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_asia" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_europe" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_7_2_prt_other_regions" for table "testpk_1_prt_7" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_usa" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_asia" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_europe" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_8_2_prt_other_regions" for table "testpk_1_prt_8" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_usa" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_asia" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_europe" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_9_2_prt_other_regions" for table "testpk_1_prt_9" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_usa" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_asia" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_europe" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_10_2_prt_other_regions" for table "testpk_1_prt_10" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_usa" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_asia" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_europe" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_11_2_prt_other_regions" for table "testpk_1_prt_11" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_usa" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_asia" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_europe" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_12_2_prt_other_regions" for table "testpk_1_prt_12" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13" for table "testpk" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_usa" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_asia" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_europe" for table "testpk_1_prt_13" +NOTICE: CREATE TABLE will create partition "testpk_1_prt_13_2_prt_other_regions" for table "testpk_1_prt_13" +CREATE TABLE diff --git a/gpMgmt/bin/gpload_test/gpload2/setup.sql b/gpMgmt/bin/gpload_test/gpload2/setup.sql index b5a8608ce28a8970d91131f99b0e35e545d3c9f3..37585b24e64e60149a7fc7dba14d6b1fbf89670d 100644 --- a/gpMgmt/bin/gpload_test/gpload2/setup.sql +++ b/gpMgmt/bin/gpload_test/gpload2/setup.sql @@ -19,3 +19,14 @@ CREATE TABLE csvtable ( CREATE TABLE test.csvtable ( year int, make text, model text, decription text, price decimal) DISTRIBUTED BY (year); +create table testpk (n1 integer, s1 integer, s2 varchar(128), n2 integer, primary key(n1,s1,s2)) +partition by range (s1) + subpartition by list(s2) + SUBPARTITION TEMPLATE + ( SUBPARTITION usa VALUES ('usa'), + SUBPARTITION asia VALUES ('asia'), + SUBPARTITION europe VALUES ('europe'), + DEFAULT SUBPARTITION other_regions) +(start (1) end (13) every (1), +default partition others) +;