Notice reject messages of external tables executed on master (#11110)

With reject limit, if reject limit is not reached,
there should be some messages that how many rows
are rejected as `execute on segments` does.

Co-authored-by Mingli Zhang <zmingli@vmware.com>
Co-authored-by Adam Lee <adlee@vmware.com>
Co-authored-by: NMingli Zhang <zmingli@vmware.com>

gpcontrib/gp_exttable_fdw/gp_exttable_fdw.c

@@ -36,6 +36,8 @@
 #include "catalog/pg_foreign_server.h"
 #include "catalog/pg_foreign_table.h"
 #include "commands/defrem.h"
+#include "commands/copy.h"
+#include "cdb/cdbsreh.h"
 #include "foreign/fdwapi.h"
 #include "funcapi.h"
 #include "nodes/execnodes.h"
@@ -834,6 +836,22 @@ exttable_EndForeignScan(ForeignScanState *node)
 	if (node->ss.ps.squelched)
 		external_stopscan(fdw_state->ess_ScanDesc);
 
+	/*
+	 * report Sreh results if external web table execute on master with reject limit.
+	 * if external web table execute on segment, these messages are printed
+	 * in cdbdisp_sumRejectedRows()
+	*/
+	if (Gp_role == GP_ROLE_DISPATCH) {
+		CopyState cstate = fdw_state->ess_ScanDesc->fs_pstate;
+		if (cstate && cstate->cdbsreh)
+		{
+			CdbSreh	 *cdbsreh = cstate->cdbsreh;
+			uint64	total_rejected_from_qd = cdbsreh->rejectcount;
+			if (total_rejected_from_qd > 0)
+				ReportSrehResults(cdbsreh, total_rejected_from_qd);
+		}
+	}
+
 	external_endscan(fdw_state->ess_ScanDesc);
 }
 

src/test/regress/input/external_table.source

@@ -3504,3 +3504,37 @@ DROP EXTERNAL TABLE test_delimiter;
 -- Test multiple character delimiter
 
 CREATE EXTERNAL TABLE test_delimiter(data text) LOCATION('gpfdist://127.0.0.1/test_delimiter.txt') FORMAT 'csv' (DELIMITER 'ab');
+
+-- Test notice reject message when execute on master
+
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_err_limit (c1 int) EXECUTE 'for i in `seq 1 2`; do echo 1 3;done; echo 22' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 3;
+
+SELECT * FROM web_exec_on_master_with_err_limit;
+
+DROP EXTERNAL TABLE web_exec_on_master_with_err_limit;
+
+-- Test notice reject message when execute on master with multiple commands
+
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_multiple_commands (c1 int) EXECUTE 'for i in `seq 1 `; do echo 2 1;
+done; echo 2 ; echo 1 2; echo 2;  echo 3 4' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 6;
+
+SELECT * FROM  web_exec_on_master_with_multiple_commands;
+
+DROP EXTERNAL TABLE web_exec_on_master_with_multiple_commands;
+
+-- Test notice reject message when execute on segments with multiple commands
+
+CREATE EXTERNAL WEB TABLE web_exec_on_segments_with_multiple_commands (c1 int) EXECUTE 'for i in `seq 1 `; do echo 2 1;
+done; echo 2 ; echo 1 2; echo 2;  echo 3 4' FORMAT 'TEXT' SEGMENT REJECT LIMIT 6;
+
+SELECT * FROM  web_exec_on_segments_with_multiple_commands;
+
+DROP EXTERNAL TABLE web_exec_on_segments_with_multiple_commands;
+
+-- Test reject number reached when execute on master
+
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_err_limit_reached (c1 int) EXECUTE 'for i in `seq 1 3`; do echo 1 3;done; echo 22' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 3;
+
+SELECT * FROM web_exec_on_master_with_err_limit_reached;
+
+DROP EXTERNAL TABLE web_exec_on_master_with_err_limit_reached;

src/test/regress/output/external_table.source

@@ -578,6 +578,7 @@ EXECUTE E'cat @abs_srcdir@/data/exttab.data' ON MASTER
 FORMAT 'TEXT' (DELIMITER '|')
 SEGMENT REJECT LIMIT 20;
 SELECT * FROM exttab_basic_error_1;
+NOTICE:  found 10 data formatting errors (10 or more input rows), rejected related input data
  i 
 ---
 (0 rows)
@@ -4760,3 +4761,48 @@ DROP EXTERNAL TABLE test_delimiter;
 -- Test multiple character delimiter
 CREATE EXTERNAL TABLE test_delimiter(data text) LOCATION('gpfdist://127.0.0.1/test_delimiter.txt') FORMAT 'csv' (DELIMITER 'ab');
 ERROR:  COPY delimiter must be a single one-byte character, or 'off'
+-- Test notice reject message when execute on master
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_err_limit (c1 int) EXECUTE 'for i in `seq 1 2`; do echo 1 3;done; echo 22' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 3;
+SELECT * FROM web_exec_on_master_with_err_limit;
+NOTICE:  found 2 data formatting errors (2 or more input rows), rejected related input data
+ c1 
+----
+ 22
+(1 row)
+
+DROP EXTERNAL TABLE web_exec_on_master_with_err_limit;
+-- Test notice reject message when execute on master with multiple commands
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_multiple_commands (c1 int) EXECUTE 'for i in `seq 1 `; do echo 2 1;
+done; echo 2 ; echo 1 2; echo 2;  echo 3 4' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 6;
+SELECT * FROM  web_exec_on_master_with_multiple_commands;
+NOTICE:  found 3 data formatting errors (3 or more input rows), rejected related input data
+ c1 
+----
+  2
+  2
+(2 rows)
+
+DROP EXTERNAL TABLE web_exec_on_master_with_multiple_commands;
+-- Test notice reject message when execute on segments with multiple commands
+CREATE EXTERNAL WEB TABLE web_exec_on_segments_with_multiple_commands (c1 int) EXECUTE 'for i in `seq 1 `; do echo 2 1;
+done; echo 2 ; echo 1 2; echo 2;  echo 3 4' FORMAT 'TEXT' SEGMENT REJECT LIMIT 6;
+SELECT * FROM  web_exec_on_segments_with_multiple_commands;
+NOTICE:  found 9 data formatting errors (9 or more input rows), rejected related input data
+ c1 
+----
+  2
+  2
+  2
+  2
+  2
+  2
+(6 rows)
+
+DROP EXTERNAL TABLE web_exec_on_segments_with_multiple_commands;
+-- Test reject number reached when execute on master
+CREATE EXTERNAL WEB TABLE web_exec_on_master_with_err_limit_reached (c1 int) EXECUTE 'for i in `seq 1 3`; do echo 1 3;done; echo 22' ON MASTER FORMAT 'TEXT' SEGMENT REJECT LIMIT 3;
+SELECT * FROM web_exec_on_master_with_err_limit_reached;
+ERROR:  segment reject limit reached, aborting operation
+DETAIL:  Last error was: invalid input syntax for type integer: "1 3", column c1
+CONTEXT:  External table web_exec_on_master_with_err_limit_reached, line 3 of execute:for i in `seq 1 3`; do echo 1 3;done; echo 22, column c1
+DROP EXTERNAL TABLE web_exec_on_master_with_err_limit_reached;