Avoid generating core files during testing. (#10304)

We had some negative tests that need to panic and thus generating core files
finally if the system is configured with corefile dump. Long ago we did
optimization to avoid generating core files in some cases. Now we found other
new scenarios that could be further optimized.

1. avoid core file generation with setrlimit() in the FATAL fault inject cod.
Some times FATAL is upgraded to PANIC (e.g.  critical section, fail when doing
QD prepare related work). So we could avoid generating core file for this
scenario also. Note even if the FATAL is not upgraded, it's fine mostly to
avoid core file generation since the process will quit soon.  With the code
change, We avoid two core files from test isolation2:crash_recovery_dtm.

2. We previously had sanity check dbid/segidx in QE:HandleFtsMessage(), and
panic if there is inconsistency when cassert is enabled, but it seems that we
really do not need to panic since the root cause of the failure is quite
straightforward, and the call stack is quite simple: PostgresMain() ->
HandleFtsMessage(), and also that part of code does not invovle shared memory
so no need to worry about shared memory mess (else we might want a core file to
check). Downgrading the log level to FATAL. This avoids 6 core files from test
isolation2:segwalrep/recoverseg_from_file.
Reviewed-by: NAshwin Agrawal <aagrawal@pivotal.io>

src/backend/cdb/cdbutil.c

@@ -22,6 +22,11 @@
 
 #include "postgres.h"
 
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/time.h>
+#include <sys/resource.h>
+#endif
+
 #include <sys/param.h>			/* for MAXHOSTNAMELEN */
 #include "access/genam.h"
 #include "catalog/gp_segment_config.h"
@@ -1768,3 +1773,21 @@ IsOnConflictUpdate(PlannedStmt *ps)
 
 	return ((ModifyTable *)plan)->onConflictAction == ONCONFLICT_UPDATE;
 }
+
+/*
+ * Avoid core file generation for this PANIC. It helps to avoid
+ * filling up disks during tests and also saves time.
+ */
+void
+AvoidCorefileGeneration()
+{
+#if defined(HAVE_GETRLIMIT) && defined(RLIMIT_CORE)
+	struct rlimit lim;
+	getrlimit(RLIMIT_CORE, &lim);
+	lim.rlim_cur = 0;
+	if (setrlimit(RLIMIT_CORE, &lim) != 0)
+		elog(NOTICE,
+			 "setrlimit failed for RLIMIT_CORE soft limit to zero. errno: %d (%m).",
+			 errno);
+#endif
+}

src/backend/fts/ftsmessagehandler.c

@@ -416,7 +416,7 @@ HandleFtsMessage(const char* query_string)
 	}
 
 #ifdef USE_ASSERT_CHECKING
-	error_level = PANIC;
+	error_level = FATAL;
 #else
 	error_level = WARNING;
 #endif

src/backend/tcop/postgres.c

@@ -83,6 +83,7 @@
 #include "utils/timestamp.h"
 #include "mb/pg_wchar.h"
 
+#include "cdb/cdbutil.h"
 #include "cdb/cdbvars.h"
 #include "cdb/cdbsrlz.h"
 #include "cdb/cdbtm.h"
@@ -1444,19 +1445,7 @@ exec_mpp_dtx_protocol_command(DtxProtocolCommand dtxProtocolCommand,
 	if (Debug_dtm_action == DEBUG_DTM_ACTION_PANIC_BEGIN_COMMAND &&
 		CheckDebugDtmActionProtocol(dtxProtocolCommand, contextInfo))
 	{
-			/*
-			 * Avoid core file generation for this PANIC. It helps to avoid
-			 * filling up disks during tests and also saves time.
-			 */
-#if defined(HAVE_GETRLIMIT) && defined(RLIMIT_CORE)
-			struct rlimit lim;
-			getrlimit(RLIMIT_CORE, &lim);
-			lim.rlim_cur = 0;
-			if (setrlimit(RLIMIT_CORE, &lim) != 0)
-				elog(NOTICE,
-					 "setrlimit failed for RLIMIT_CORE soft limit to zero. errno: %d (%m).",
-					 errno);
-#endif
+		AvoidCorefileGeneration();
 		elog(PANIC,"PANIC for debug_dtm_action = %d, debug_dtm_action_protocol = %s",
 			 Debug_dtm_action, DtxProtocolCommandToString(dtxProtocolCommand));
 	}

src/backend/utils/misc/faultinjector.c

@@ -22,9 +22,6 @@
 #include "postgres.h"
 
 #include <signal.h>
-#ifdef HAVE_SYS_RESOURCE_H
-#include <sys/resource.h>
-#endif
 #include "access/xact.h"
 #include "cdb/cdbutil.h"
 #include "libpq/libpq.h"
@@ -357,6 +354,15 @@ FaultInjector_InjectFaultIfSet(
 			break;
 
 		case FaultInjectorTypeFatal:
+			/*
+			 * Sometimes Fatal is upgraded to Panic (e.g. when it is called in
+			 * critical section or when it is called during QD prepare
+			 * handling).  We should avoid core file generation for this
+			 * scenario, just like what we do for the FaultInjectorTypePanic
+			 * case.  Even FATAL is not upgraded to PANIC the process will quit
+			 * soon, it does not affect subsequent code.
+			 */
+			AvoidCorefileGeneration();
 			ereport(FATAL, 
 					(errcode(ERRCODE_FAULT_INJECT),
 					 errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
@@ -365,19 +371,7 @@ FaultInjector_InjectFaultIfSet(
 			break;
 
 		case FaultInjectorTypePanic:
-			/*
-			 * Avoid core file generation for this PANIC. It helps to avoid
-			 * filling up disks during tests and also saves time.
-			 */
-#if defined(HAVE_GETRLIMIT) && defined(RLIMIT_CORE)
-			;struct rlimit lim;
-			getrlimit(RLIMIT_CORE, &lim);
-			lim.rlim_cur = 0;
-			if (setrlimit(RLIMIT_CORE, &lim) != 0)
-				elog(NOTICE,
-					 "setrlimit failed for RLIMIT_CORE soft limit to zero. errno: %d (%m).",
-					 errno);
-#endif
+			AvoidCorefileGeneration();
 			ereport(PANIC, 
 					(errcode(ERRCODE_FAULT_INJECT),
 					 errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
@@ -465,20 +459,7 @@ FaultInjector_InjectFaultIfSet(
 			
 		case FaultInjectorTypeSegv:
 		{
-			/*
-			 * Avoid core file generation for this PANIC. It helps to avoid
-			 * filling up disks during tests and also saves time.
-			 */
-#if defined(HAVE_GETRLIMIT) && defined(RLIMIT_CORE)
-			struct rlimit lim;
-			getrlimit(RLIMIT_CORE, &lim);
-			lim.rlim_cur = 0;
-			if (setrlimit(RLIMIT_CORE, &lim) != 0)
-				elog(NOTICE,
-					 "setrlimit failed for RLIMIT_CORE soft limit to zero. errno: %d (%m).",
-					 errno);
-#endif
-
+			AvoidCorefileGeneration();
 			*(volatile int *) 0 = 1234;
 			break;
 		}

src/include/cdb/cdbutil.h

@@ -212,6 +212,8 @@ extern int getgpsegmentCount(void);
 
 extern bool IsOnConflictUpdate(PlannedStmt *ps);
 
+extern void AvoidCorefileGeneration(void);
+
 #define ELOG_DISPATCHER_DEBUG(...) do { \
        if (gp_log_gang >= GPVARS_VERBOSITY_DEBUG) elog(LOG, __VA_ARGS__); \
     } while(false);