Ensure the current probe does not cause flake

Wait until we've inserted the fts_probe skip fault, AND we have
observed the fault being hit. This ensures that we've completed the
in-progress fts probe before continuing on with a test.

Previously, if we don't wait and our test continues forward to inject
a panic, then then in-progress fts probe would do its job and promote
the mirror.

We lower the minimum allowed gp_fts_probe_interval to 1 second, which
allows us to configure the interval to one second during our tests and
speeds up the tests significantly.
Reported-by: NAsim R P <apraveen@pivotal.io>

src/test/regress/input/gp_tablespace_with_faults.source

@@ -55,10 +55,26 @@ $$ LANGUAGE plpythonu;
 
 
 create or replace function disable_fts() returns void as $$
+	declare
+		master_content_id integer = -1;
+		master_primary_role character = 'p';
 	begin
 		-- intentionally skip fts during these tests
 		-- we know we're going to be inducing errors and panics on primaries
-		perform gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port) from gp_segment_configuration where content = -1 and role = 'p';
+		perform gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port)
+			from gp_segment_configuration
+			where content = master_content_id
+			and role = master_primary_role;
+
+		perform gp_request_fts_probe_scan();
+
+		-- also, ensure that the fts loop observes the probe being skipped before
+		-- continuing with the test because it could still be in the middle of a
+		-- previous probe
+		perform gp_wait_until_triggered_fault2('fts_probe', 1, dbid, hostname, port)
+			from gp_segment_configuration
+			where content = master_content_id
+			and role = master_primary_role;
 	end;
 $$ language plpgsql;
 

src/test/regress/output/gp_tablespace_with_faults.source

@@ -53,10 +53,26 @@ create or replace function setup_tablespace_location_dir_for_test(tablespace_loc
 	os.mkdir(tablespace_location_dir);
 $$ LANGUAGE plpythonu;
 create or replace function disable_fts() returns void as $$
+	declare
+		master_content_id integer = -1;
+		master_primary_role character = 'p';
 	begin
 		-- intentionally skip fts during these tests
 		-- we know we're going to be inducing errors and panics on primaries
-		perform gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port) from gp_segment_configuration where content = -1 and role = 'p';
+		perform gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port)
+			from gp_segment_configuration
+			where content = master_content_id
+			and role = master_primary_role;
+
+		perform gp_request_fts_probe_scan();
+
+		-- also, ensure that the fts loop observes the probe being skipped before
+		-- continuing with the test because it could still be in the middle of a
+		-- previous probe
+		perform gp_wait_until_triggered_fault2('fts_probe', 1, dbid, hostname, port)
+			from gp_segment_configuration
+			where content = master_content_id
+			and role = master_primary_role;
 	end;
 $$ language plpgsql;
 create or replace function setup(content_id integer, fault_name text, fault_action_type text, tablespace_dir text) returns void as $$