Add EXPLAIN CODEGEN to print generated IR to the client

This will aid tremendously in debugging codegen because the generated IR
will be easily available to be seen. Rest of the functionality of
EXPLAIN is maintained. Also we only perform the compilation step if it
not a EXPLAIN-only query. That is EXPLAIN CODEGEN with perform code
generation only and will not compile the resultant code. EXPLAIN ANALYZE
will do both, and also execute the query.

src/backend/codegen/codegen_manager.cc

@@ -102,3 +102,13 @@ bool CodegenManager::InvalidateGeneratedFunctions() {
   assert(false);
   return false;
 }
+
+const std::string& CodegenManager::GetExplainString() {
+  return explain_string_;
+}
+
+void CodegenManager::AccumulateExplainString() {
+  explain_string_.clear();
+  llvm::raw_string_ostream out(explain_string_);
+  codegen_utils_->PrintUnderlyingModules(out);
+}

src/backend/codegen/codegen_wrapper.cc

@@ -17,6 +17,10 @@
 
 #include "codegen/utils/gp_codegen_utils.h"
 
+extern "C" {
+#include "lib/stringinfo.h"
+}
+
 using gpcodegen::CodegenManager;
 using gpcodegen::BaseCodegen;
 using gpcodegen::ExecVariableListCodegen;
@@ -61,6 +65,25 @@ unsigned int CodeGeneratorManagerNotifyParameterChange(void* manager) {
   return 0;
 }
 
+void CodeGeneratorManagerAccumulateExplainString(void* manager) {
+  if (!codegen) {
+    return;
+  }
+  assert(nullptr != manager);
+  static_cast<CodegenManager*>(manager)->AccumulateExplainString();
+}
+
+char* CodeGeneratorManagerGetExplainString(void* manager) {
+  if (!codegen) {
+    return nullptr;
+  }
+  StringInfo return_string = makeStringInfo();
+  appendStringInfoString(
+      return_string,
+      static_cast<CodegenManager*>(manager)->GetExplainString().c_str());
+  return return_string->data;
+}
+
 void CodeGeneratorManagerDestroy(void* manager) {
   delete (static_cast<CodegenManager*>(manager));
 }

src/backend/codegen/include/codegen/codegen_manager.h

@@ -99,6 +99,17 @@ class CodegenManager {
     return enrolled_code_generators_.size();
   }
 
+  /*
+   * @brief Accumulate the explain string with a dump of all the underlying LLVM
+   *        modules
+   */
+  void AccumulateExplainString();
+
+  /*
+   * @brief Return the previous accumulated explain string
+   */
+  const std::string& GetExplainString();
+
  private:
   // GpCodegenUtils provides a facade to LLVM subsystem.
   std::unique_ptr<gpcodegen::GpCodegenUtils> codegen_utils_;
@@ -108,6 +119,9 @@ class CodegenManager {
   // List of all enrolled code generators.
   std::vector<std::unique_ptr<CodegenInterface>> enrolled_code_generators_;
 
+  // Holds the dumped IR of all underlying modules for EXPLAIN CODEGEN queries
+  std::string explain_string_;
+
   DISALLOW_COPY_AND_ASSIGN(CodegenManager);
 };
 

src/backend/codegen/include/codegen/utils/codegen_utils.h

@@ -483,6 +483,13 @@ class CodegenUtils {
     return llvm::InlineFunction(llvm::CallSite(call_inst), info);
   }
 
+  /*
+   * @brief Dump the IR of all underlying LLVM modules.
+   *
+   * @param out  Stream to send the output to - as required by llvm::Value::print
+   */
+  void PrintUnderlyingModules(llvm::raw_ostream& out); // NOLINT
+
  protected:
   /**
    * @return LLVMContext

src/backend/codegen/include/codegen/utils/gp_codegen_utils.h

@@ -169,7 +169,6 @@ class GpCodegenUtils : public CodegenUtils {
   llvm::Value* CreateCppTypeToDatumCast(llvm::Value* value,
                                         bool is_src_unsigned = false);
 };
-
 }  // namespace gpcodegen
 
 #endif  // GPCODEGEN_GP_CODEGEN_UTILS_H

src/backend/codegen/utils/codegen_utils.cc

@@ -263,6 +263,22 @@ bool CodegenUtils::PrepareForExecution(const OptimizationLevel cpu_opt_level,
   return true;
 }
 
+void CodegenUtils::PrintUnderlyingModules(llvm::raw_ostream& out) {
+  // Print the main module
+  out << "==== MAIN MODULE ====" << "\n";
+  out.flush();
+  module()->print(out, nullptr);
+
+  // Print auxiliary modules
+  out << "==== AUXILIARY MODULES ====" << "\n";
+  out.flush();
+  for (std::unique_ptr<llvm::Module>& auxiliary_module : auxiliary_modules_) {
+      auxiliary_module->print(out, nullptr);
+  }
+  out << "==== END MODULES ====" << "\n\n";
+  out.flush();
+}
+
 llvm::GlobalVariable* CodegenUtils::AddExternalGlobalVariable(
     llvm::Type* type,
     const void* address) {

src/backend/commands/explain.c

@@ -86,6 +86,9 @@ static void ExplainDXL(Query *query, ExplainStmt *stmt,
 							const char *queryString,
 							ParamListInfo params, TupOutputState *tstate);
 #endif
+#ifdef USE_CODEGEN
+static void ExplainCodegen(PlanState *planstate, TupOutputState *tstate);
+#endif
 static double elapsed_time(instr_time *starttime);
 static ErrorData *explain_defer_error(ExplainState *es);
 static void explain_outNode(StringInfo str,
@@ -117,7 +120,6 @@ show_motion_keys(Plan *plan, List *hashExpr, int nkeys, AttrNumber *keycols,
 static void
 show_static_part_selection(PartitionSelector *ps, Sequence *parent, StringInfo str, int indent, ExplainState *es);
 
-
 /*
  * ExplainQuery -
  *	  execute an EXPLAIN command
@@ -316,6 +318,32 @@ ExplainOneUtility(Node *utilityStmt, ExplainStmt *stmt,
 							   "Utility statements have no plan structure");
 }
 
+#ifdef USE_CODEGEN
+/*
+ * ExplainCodegen -
+ * 		given a PlanState tree, traverse its nodes, collect any accumulated
+ * 		explain strings from the state's CodegenManager, and print to EXPLAIN
+ * 		output
+ * 		NB: This method does not recurse into sub plans at this point.
+ */
+static void
+ExplainCodegen(PlanState *planstate, TupOutputState *tstate) {
+	if (NULL == planstate) {
+		return;
+	}
+
+	Assert(NULL != tstate);
+
+	ExplainCodegen(planstate->lefttree, tstate);
+
+	char* str = CodeGeneratorManagerGetExplainString(planstate->CodegenManager);
+	Assert(NULL != str);
+	do_text_output_oneline(tstate, str);
+
+	ExplainCodegen(planstate->righttree, tstate);
+}
+#endif
+
 /*
  * ExplainOnePlan -
  *		given a planned query, execute it if needed, and then print
@@ -399,11 +427,23 @@ ExplainOnePlan(PlannedStmt *plannedstmt, ParamListInfo params,
 		{
 			queryDesc->plannedstmt->query_mem = ResourceQueueGetQueryMemoryLimit(queryDesc->plannedstmt, GetResQueueId());			
 		}
-    }
+	}
+
+#ifdef USE_CODEGEN
+	if (stmt->codegen && codegen && Gp_segment == -1) {
+		eflags |= EXEC_FLAG_EXPLAIN_CODEGEN;
+	}
+#endif
 
 	/* call ExecutorStart to prepare the plan for execution */
 	ExecutorStart(queryDesc, eflags);
 
+#ifdef USE_CODEGEN
+	if (stmt->codegen && codegen && Gp_segment == -1) {
+		ExplainCodegen(queryDesc->planstate, tstate);
+	}
+#endif
+
     estate = queryDesc->estate;
 
     /* CDB: Find slice table entry for the root slice. */

src/backend/executor/execProcnode.c

@@ -761,7 +761,12 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
 		SAVE_EXECUTOR_MEMORY_ACCOUNT(result, curMemoryAccount);
 		result->CodegenManager = CodegenManager;
 		CodeGeneratorManagerGenerateCode(CodegenManager);
-		CodeGeneratorManagerPrepareGeneratedFunctions(CodegenManager);
+		if (eflags & EXEC_FLAG_EXPLAIN_CODEGEN) {
+			CodeGeneratorManagerAccumulateExplainString(CodegenManager);
+		}
+		if (!(eflags & EXEC_FLAG_EXPLAIN_ONLY)) {
+			CodeGeneratorManagerPrepareGeneratedFunctions(CodegenManager);
+		}
 	}
 	}
 	END_CODE_GENERATOR_MANAGER();

src/backend/parser/gram.y

@@ -336,6 +336,7 @@ static Node *makeIsNotDistinctFromNode(Node *expr, int position);
 %type <boolean> opt_freeze opt_default opt_ordered opt_recheck
 %type <boolean> opt_rootonly_all
 %type <boolean> opt_dxl
+%type <boolean> codegen
 %type <defelt>	opt_binary opt_oids copy_delimiter
 
 %type <boolean> copy_from skip_external_partition
@@ -490,7 +491,7 @@ static Node *makeIsNotDistinctFromNode(Node *expr, int position);
 
 	CACHE CALLED CASCADE CASCADED CASE CAST CHAIN CHAR_P
 	CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE
-	CLUSTER COALESCE COLLATE COLUMN COMMENT COMMIT
+	CLUSTER COALESCE CODEGEN COLLATE COLUMN COMMENT COMMIT
 	COMMITTED CONCURRENTLY CONFIGURATION CONNECTION CONSTRAINT CONSTRAINTS CONTAINS
 	CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE CREATEDB
 	CREATEEXTTABLE
@@ -8499,7 +8500,7 @@ opt_name_list:
  *
  *****************************************************************************/
 
-ExplainStmt: EXPLAIN opt_analyze opt_verbose opt_dxl opt_force ExplainableStmt
+ExplainStmt: EXPLAIN opt_analyze opt_verbose opt_dxl opt_force codegen ExplainableStmt
 				{
 					ExplainStmt *n = makeNode(ExplainStmt);
 					n->analyze = $2;
@@ -8509,7 +8510,8 @@ ExplainStmt: EXPLAIN opt_analyze opt_verbose opt_dxl opt_force ExplainableStmt
 						ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), 
 								errmsg("cannot use force with explain statement")
 							       ));
-					n->query = $6;
+					n->codegen = $6;
+					n->query = $7;
 					$$ = (Node *)n;
 				}
 		;
@@ -8540,6 +8542,11 @@ opt_analyze:
 			| /* EMPTY */			{ $$ = FALSE; }
 		;
 
+codegen:
+			CODEGEN				{ $$ = TRUE; }
+			| /* EMPTY */			{ $$ = FALSE; }
+		;
+
 /*****************************************************************************
  *
  *		QUERY:
@@ -12616,6 +12623,7 @@ unreserved_keyword:
 			| CLASS
 			| CLOSE
 			| CLUSTER
+			| CODEGEN
 			| COMMENT
 			| COMMIT
 			| COMMITTED

src/include/codegen/codegen_wrapper.h

@@ -47,6 +47,8 @@ typedef Datum (*SlotGetAttrFn) (struct TupleTableSlot *slot, int attnum, bool *i
 #define CodeGeneratorManagerGenerateCode(manager);
 #define CodeGeneratorManagerPrepareGeneratedFunctions(manager) 1
 #define CodeGeneratorManagerNotifyParameterChange(manager) 1
+#define CodeGeneratorManagerAccumulateExplainString(manager) 1
+#define CodeGeneratorManagerGetExplainString(manager) 1
 #define CodeGeneratorManagerDestroy(manager);
 #define GetActiveCodeGeneratorManager() NULL
 #define SetActiveCodeGeneratorManager(manager);
@@ -127,6 +129,19 @@ CodeGeneratorManagerNotifyParameterChange(void* manager);
 void
 CodeGeneratorManagerDestroy(void* manager);
 
+/*
+ * Accumulate the explain string with a dump of all the underlying LLVM modules
+ */
+void
+CodeGeneratorManagerAccumulateExplainString(void* manager);
+
+/*
+ * Return a copy in CurrentMemoryContext of the previously accumulated explain
+ * string
+ */
+char*
+CodeGeneratorManagerGetExplainString(void* manager);
+
 /*
  * Get the active code generator manager
  */
@@ -158,7 +173,6 @@ ExecEvalExprCodegenEnroll(ExecEvalExprFn regular_func_ptr,
                           struct ExprContext *econtext,
                           struct TupleTableSlot* slot);
 
-
 #ifdef __cplusplus
 }  // extern "C"
 #endif

src/include/executor/executor.h

@@ -52,6 +52,7 @@ struct ChunkTransportState;             /* #include "cdb/cdbinterconnect.h" */
 #define EXEC_FLAG_REWIND		0x0002	/* expect rescan */
 #define EXEC_FLAG_BACKWARD		0x0004	/* need backward scan */
 #define EXEC_FLAG_MARK			0x0008	/* need mark/restore */
+#define EXEC_FLAG_EXPLAIN_CODEGEN	0x0010	/* EXPLAIN CODEGEN */
 
 
 /*

src/include/nodes/parsenodes.h

@@ -2624,6 +2624,7 @@ typedef struct ExplainStmt
 	bool		verbose;		/* print plan info */
 	bool		analyze;		/* get statistics by executing plan */
 	bool		dxl;			/* display plan in dxl format */
+	bool		codegen;		/* display generated IR codegen */
 } ExplainStmt;
 
 /* ----------------------

src/include/parser/kwlist.h

@@ -78,6 +78,7 @@ PG_KEYWORD("class", CLASS, UNRESERVED_KEYWORD)
 PG_KEYWORD("close", CLOSE, UNRESERVED_KEYWORD)
 PG_KEYWORD("cluster", CLUSTER, UNRESERVED_KEYWORD)
 PG_KEYWORD("coalesce", COALESCE, COL_NAME_KEYWORD)
+PG_KEYWORD("codegen", CODEGEN, UNRESERVED_KEYWORD)
 PG_KEYWORD("collate", COLLATE, RESERVED_KEYWORD)
 PG_KEYWORD("column", COLUMN, RESERVED_KEYWORD)
 PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD)

src/test/unit/mock/gpcodegen_mock.c

@@ -50,6 +50,26 @@ CodeGeneratorManagerDestroy(void* manager)
 	elog(ERROR, "mock implementation of CodeGeneratorManager_Destroy called");
 }
 
+/*
+ * Accumulate the explain string with a dump of all the underlying LLVM modules
+ */
+void
+CodeGeneratorManagerAccumulateExplainString(void* manager)
+{
+	elog(ERROR, "mock implementation of CodeGeneratorManager_AccumulateExplainString called");
+}
+
+/*
+ * Return a copy in CurrentMemoryContext of the previously accumulated explain
+ * string
+ */
+char*
+CodeGeneratorManagerGetExplainString(void* manager)
+{
+	elog(ERROR, "mock implementation of CodeGeneratorManager_GetExplainString called");
+	return NULL;
+}
+
 // get the active code generator manager
 void*
 GetActiveCodeGeneratorManager()