Detect window functions plus correlated subquery in target list and fall back (#10265)

We found that when we have window functions and also correlated subqueries in the
same target list, the CQueryMutators::NormalizeWindowProjList method would leave
the varattno attributes of outer references in the subquery unchanged. That needs
to be changed, since we are producing a different RTE for the query.

We will eventually create a fix. For now, this PR just searches for the
problem and triggers a fallback when we see it, to avoid incorrect results.
Co-authored-by: NAbhijit Subramanya <asubramanya@pivotal.io>
Co-authored-by: NHans Zeller <hzeller@vmware.com>

src/backend/gpopt/gpdbwrappers.cpp

@@ -494,6 +494,23 @@ gpdb::WalkExpressionTree
 	return false;
 }
 
+bool
+gpdb::WalkQueryTree
+	(
+	Query *node,
+	bool (*walker) (),
+	void *context,
+	int flags
+	)
+{
+	GP_WRAP_START;
+	{
+		return query_tree_walker(node, walker, context, flags);
+	}
+	GP_WRAP_END;
+	return false;
+}
+
 Oid
 gpdb::ExprType
 	(

src/backend/gpopt/translate/CQueryMutators.cpp

@@ -134,7 +134,7 @@ CQueryMutators::ShouldFallback
 		return false;
 	}
 
-	return gpdb::WalkExpressionTree(node, (FallbackWalkerFn) CQueryMutators::ShouldFallback, context);
+	return gpdb::WalkExpressionTree(node, (ExprWalkerFn) CQueryMutators::ShouldFallback, context);
 }
 
 
@@ -1582,6 +1582,13 @@ CQueryMutators::NormalizeWindowProjList
 		}
 		else
 		{
+			if (HasOuterRefs((Node *) target_entry->expr, NULL))
+			{
+				// We currently don't support this situation, so fall back.
+				GPOS_RAISE(gpdxl::ExmaDXL,
+						   gpdxl::ExmiExpr2DXLUnsupportedFeature,
+						   GPOS_WSZ_LIT("Window functions and correlated subqueries appear together in select list"));
+			}
 			// normalize target list entry
 			Expr *pexprNew = (Expr*) RunWindowProjListMutator( (Node*) target_entry->expr, &context);
 			TargetEntry *new_target_entry = gpdb::MakeTargetEntry(pexprNew, ulResNoNew, target_entry->resname, target_entry->resjunk);
@@ -1788,4 +1795,42 @@ CQueryMutators::ReassignSortClause
 	derived_table_query->limitCount = NULL;
 }
 
+//---------------------------------------------------------------------------
+//	CQueryMutators::HasOuterRefs
+//
+//	Search the tree for any outer refs.
+//---------------------------------------------------------------------------
+BOOL
+CQueryMutators::HasOuterRefs
+	(
+	Node *node,
+	void *context
+	)
+{
+	if (NULL == node)
+	{
+		return false;
+	}
+
+	if (IsA(node, Var))
+	{
+		Var *var = (Var *) node;
+		if (0 < var->varlevelsup)
+		{
+			// we found an outer ref
+			return true;
+		}
+
+		return false;
+	}
+	if (IsA(node, Query))
+	{
+		return gpdb::WalkQueryTree((Query *) node, (ExprWalkerFn) CQueryMutators::HasOuterRefs, context, 0);
+	}
+
+	return gpdb::WalkExpressionTree(node, (ExprWalkerFn) CQueryMutators::HasOuterRefs, context);
+}
+
+
+
 // EOF

src/include/gpopt/gpdbwrappers.h

@@ -491,6 +491,9 @@ namespace gpdb {
 	// expression tree walker
 	bool WalkExpressionTree(Node *node, bool(*walker)(), void *context);
 
+	// query tree walkers, descend into subqueries
+	bool WalkQueryTree(Query *node, bool (*walker) (), void *context, int flags);
+
 	// query or expression tree walker
 	bool WalkQueryOrExpressionTree(Node *node, bool(*walker)(), void *context, int flags);
 

src/include/gpopt/translate/CQueryMutators.h

@@ -53,7 +53,7 @@ namespace gpdxl
 	class CQueryMutators
 	{
 		typedef Node *(*MutatorWalkerFn) ();
-		typedef BOOL (*FallbackWalkerFn) ();
+		typedef BOOL (*ExprWalkerFn) ();
 
 		typedef struct SContextGrpbyPlMutator
 		{
@@ -270,6 +270,10 @@ namespace gpdxl
 			// reassign the sorting clause from the derived table to the new top-level query
 			static
 			void ReassignSortClause(Query *top_level_query, Query *derive_table_query);
+
+			static
+			BOOL HasOuterRefs(Node *node, void *context);
+
 	};
 }
 #endif // GPDXL_CWalkerUtils_H

src/test/regress/expected/gporca_optimizer.out

@@ -6547,6 +6547,8 @@ insert into orca_w2 select i, i from generate_series(2, 4) i;
 insert into orca_w3 select i, i from generate_series(3, 5) i;
 -- outer ref in subquery in target list and window func in target list
 select (select b from orca_w3 where a = orca_w1.a) as one, row_number() over(partition by orca_w1.a) as two from orca_w2, orca_w1;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  one | two 
 -----+-----
      |   1
@@ -6562,6 +6564,8 @@ select (select b from orca_w3 where a = orca_w1.a) as one, row_number() over(par
 
 -- aggref in subquery with window func in target list
 select orca_w1.a, (select sum(orca_w2.a) from orca_w2 where orca_w1.b = orca_w2.b), count(*), rank() over (order by orca_w1.b) from orca_w1 group by orca_w1.a, orca_w1.b order by orca_w1.a;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  a | sum | count | rank 
 ---+-----+-------+------
  1 |     |     1 |    1
@@ -6580,6 +6584,8 @@ select orca_w1.a, (select rank() over (order by orca_w1.b) from orca_w2 where or
 
 -- window function with empty partition clause inside subquery inside target list with outer ref
 select (select rank() over() from orca_w3 where a = orca_w1.a) as one, row_number() over(partition by orca_w1.a) as two from orca_w1, orca_w2;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  one | two 
 -----+-----
      |   1
@@ -6602,6 +6608,8 @@ select (select a from orca_w3 where a = orca_w1.a) as one from orca_w1 where orc
 
 -- window function in subquery inside target list with outer ref in partition clause
 select (select rank() over(partition by orca_w2.a) from orca_w3 where a = orca_w1.a) as one, row_number() over(partition by orca_w1.a) as two from orca_w1, orca_w2 order by orca_w1.a;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  one | two 
 -----+-----
      |   2
@@ -6617,6 +6625,8 @@ select (select rank() over(partition by orca_w2.a) from orca_w3 where a = orca_w
 
 -- window function in subquery inside target list with outer ref in order clause
 select (select rank() over(order by orca_w2.a) from orca_w3 where a = orca_w1.a) as one, row_number() over(partition by orca_w1.a) as two from orca_w1, orca_w2 order by orca_w1.a;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  one | two 
 -----+-----
      |   2
@@ -6632,6 +6642,8 @@ select (select rank() over(order by orca_w2.a) from orca_w3 where a = orca_w1.a)
 
 -- window function with outer ref in arguments
 select (select sum(orca_w1.a + a) over(order by b) + 1 from orca_w2 where orca_w1.a = orca_w2.a) from orca_w1 order by orca_w1.a;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  ?column? 
 ----------
          
@@ -6641,6 +6653,8 @@ select (select sum(orca_w1.a + a) over(order by b) + 1 from orca_w2 where orca_w
 
 -- window function with outer ref in window clause and arguments 
 select (select sum(orca_w1.a + a) over(order by b + orca_w1.a) + 1 from orca_w2 where orca_w1.a = orca_w2.a) from orca_w1 order by orca_w1.a;
+INFO:  GPORCA failed to produce a plan, falling back to planner
+DETAIL:  Feature not supported: Window functions and correlated subqueries appear together in select list
  ?column? 
 ----------