Fix a flaky test for gdd/dist-deadlock-upsert (#10302)

* Fix a flaky test for gdd/dist-deadlock-upsert

When to run GDD probe is undermined, but it is import for the test
gdd/dist-deadlock-upsert. If the GDD probe runs immediately after
the 2 inter-dead-locked transactions, one of the transactions will
be killed. The isolation2 framework consider the transaction being
blocked if the transaction doesn't finished in 0.5 second. So, if
the killed transaction is too early to be aborted, the test framework
sees no dead lock.
Analyzed-by: NGang Xiong <gxiong@pivotal.io>

* rm sleep

src/test/isolation2/expected/gdd/dist-deadlock-upsert.out

@@ -29,12 +29,27 @@ INSERT 1
 20: INSERT INTO t_upsert VALUES (segid(1,1), segid(1,1)) on conflict (id, val) do update set val = 888;
 INSERT 1
 
+select gp_inject_fault('gdd_probe', 'suspend', dbid) from gp_segment_configuration where content=-1 and role='p';
+ gp_inject_fault 
+-----------------
+ Success:        
+(1 row)
+select gp_wait_until_triggered_fault('gdd_probe', 1, dbid) from gp_segment_configuration where content=-1 and role='p';
+ gp_wait_until_triggered_fault 
+-------------------------------
+ Success:                      
+(1 row)
 -- seg 0: con20 ==> con10, xid lock
 20&: INSERT INTO t_upsert VALUES (segid(0,1), segid(0,1)) on conflict (id, val) do update set val = 666;  <waiting ...>
 
 -- seg 1: con10 ~~> con20, tuple lock
 10&: INSERT INTO t_upsert VALUES (segid(1,1), segid(1,1)) on conflict (id, val) do update set val = 777;  <waiting ...>
 
+select gp_inject_fault('gdd_probe', 'reset', dbid) from gp_segment_configuration where content=-1 and role='p';
+ gp_inject_fault 
+-----------------
+ Success:        
+(1 row)
 -- con20 will be cancelled by gdd
 20<:  <... completed>
 ERROR:  canceling statement due to user request: "cancelled by global deadlock detector"

src/test/isolation2/isolation2_schedule

@@ -25,7 +25,8 @@ test: packcore
 # Tests on global deadlock detector
 test: gdd/prepare
 test: gdd/concurrent_update
-test: gdd/dist-deadlock-01 gdd/dist-deadlock-04 gdd/dist-deadlock-05 gdd/dist-deadlock-06 gdd/dist-deadlock-07 gdd/dist-deadlock-102 gdd/dist-deadlock-103 gdd/dist-deadlock-104 gdd/dist-deadlock-106 gdd/dist-deadlock-upsert gdd/non-lock-105
+test: gdd/dist-deadlock-01 gdd/dist-deadlock-04 gdd/dist-deadlock-05 gdd/dist-deadlock-06 gdd/dist-deadlock-07 gdd/dist-deadlock-102 gdd/dist-deadlock-103 gdd/dist-deadlock-104 gdd/dist-deadlock-106 gdd/non-lock-105
+test: gdd/dist-deadlock-upsert
 # until we can improve below flaky case please keep it disabled
 ignore: gdd/non-lock-107
 # keep this in a separate group

src/test/isolation2/sql/gdd/dist-deadlock-upsert.sql

@@ -19,12 +19,18 @@ INSERT INTO t_upsert (id, val) SELECT i, i FROM generate_series(1, 100) i;
 
 20: INSERT INTO t_upsert VALUES (segid(1,1), segid(1,1)) on conflict (id, val) do update set val = 888;
 
+select gp_inject_fault('gdd_probe', 'suspend', dbid)
+  from gp_segment_configuration where content=-1 and role='p';
+select gp_wait_until_triggered_fault('gdd_probe', 1, dbid)
+  from gp_segment_configuration where content=-1 and role='p';
 -- seg 0: con20 ==> con10, xid lock
 20&: INSERT INTO t_upsert VALUES (segid(0,1), segid(0,1)) on conflict (id, val) do update set val = 666;
 
 -- seg 1: con10 ~~> con20, tuple lock
 10&: INSERT INTO t_upsert VALUES (segid(1,1), segid(1,1)) on conflict (id, val) do update set val = 777;
 
+select gp_inject_fault('gdd_probe', 'reset', dbid)
+  from gp_segment_configuration where content=-1 and role='p';
 -- con20 will be cancelled by gdd
 20<:
 20q: