WL#15540 Transform correlated scalar subqueries: allow column references as function arguments

WL#13520 "Transform correlated scalar subqueries" has a requirement
(#9), which states that the equality expression operands must be
simple column references, e.g. in the below transformation example
from WL#13520, t2.a and t1.a — the operands of the WHERE clause
equality predicate — are simple references. This WL lifts this
restriction and allows the column references to be arguments of
functions.

SELECT * FROM t1 WHERE (SELECT a FROM t2 WHERE t2.a=t1.a) > 0;

->

(with non-strict grouping and a built-in assert to signal any
cardinality error):

SELECT t1.a AS a
FROM t1
    JOIN
    (SELECT t2.a AS a,
            COUNT(0) AS cnt
     FROM t2
     WHERE (t2.a > 0)
     GROUP BY t2.a) derived
WHERE t1.a = derived.a AND
      reject_if(derived.cnt > 1)

The outer reference, t1.a, can already be embedded as a function
argument, but the non-outer (inner for short) column that we place in
the GROUP BY above, t2.a, cannot.

With this WL, we will also be able to transform queries like

SELECT * FROM t1 WHERE (SELECT func(t2.a) FROM t2 WHERE func(t2.a) = t1.a ) > 0;

->

SELECT t1.*
FROM t1
     JOIN
     ( SELECT func(t2.a) AS a,
              COUNT(0) AS cnt
       FROM t2
       WHERE (t2.a > 0)
       GROUP BY func(t2.a)) derived
WHERE derived.a = t1.a AND
      reject_if(derived.cnt > 1)

This transformation is needed by Heatwave/RAPID, which doesn't support
subqueries in the WHERE clause.

Limitations: due to a phase problem in the resolver, functional
dependencies are not always handled correctly: if the transformed
subquery has explicit grouping, functional dependency analysis may be
too pessimistic and give an error when the transform is used (but not
without). See worklog for details.  A work-around is to wrap the
offending column in ANY_VALUE.

Change-Id: I06fc0bbf8931f87ca964dc784329f4b2e24e048f

Author: Dag Wanvik

mysql-test/r/subquery_scalar_to_derived_correlated.result

@@ -11,20 +11,29 @@ INSERT INTO t2 VALUES(1), (2);
 CREATE TABLE t0 AS SELECT *FROM t1;
 CREATE TABLE t3(a INT, b INT);
 INSERT INTO t3 VALUES(1, 3), (2, 3);
+CREATE VIEW v3 AS SELECT * FROM t3;
 ANALYZE TABLE t1, t2, t0, t3;
 
 --echo #
 --echo # example supported query
 --echo #
 
-# inner field present in SELECT list (t2.a)
+--echo # inner field present in SELECT list (t2.a)
 let $query =
 SELECT * FROM t1 WHERE(SELECT a FROM t2 WHERE t2.a = t1.a) > 0;
 
 eval $query;
 eval EXPLAIN $query;
 
-# inner field not present in SELECT list (t3.a)
+--echo # function(inner) present in select list
+let $query =
+SELECT * FROM t1 WHERE(SELECT -a FROM t2 WHERE -t2.a = -t1.a) < 0;
+
+--sorted_result
+eval $query;
+eval EXPLAIN $query;
+
+--echo # inner field not present in SELECT list (t3.a)
 let $query =
 SELECT * FROM t1 WHERE(SELECT b FROM t3 WHERE t3.a = t1.a) > 0;
 
@@ -36,6 +45,88 @@ SELECT * FROM t1 WHERE(SELECT ABS(a) FROM t2 WHERE t2.a = t1.a) > 0;
 
 eval $query;
 eval EXPLAIN $query;
+
+--echo # function(inner) not present in SELECT list
+let $query =
+SELECT * FROM t1 WHERE(SELECT a FROM t2 WHERE -t2.a = -t1.a) > 0;
+
+--sorted_result
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT a FROM t2 WHERE -t2.a = -t1.a AND t2.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT -t2.a FROM t2 WHERE -t2.a = -t1.a AND t2.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT ABS(t2.a) FROM t2 WHERE -t2.a = -t1.a AND t2.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+--echo # Could have been transformed, since t1.a selected is equal to t2.a
+--echo # (functionally dependent here, but this analysis is not done at the
+--echo # moment.
+let $query =
+SELECT * FROM t1 WHERE(SELECT t1.a FROM t2 WHERE -t2.a = -t1.a AND t2.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+# multiple inner fields in a function
+let $query =
+SELECT * FROM t1 WHERE(SELECT t3.a FROM t3 WHERE t3.a + t3.b = t1.a ) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT ABS(t3.a) FROM t3 WHERE t3.a + t3.b = t1.a ) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+--echo # We don't group on abs(t3.a), since abs(t3.a) is functionally dependent
+--echo # on t3.a.
+let $query =
+SELECT * FROM t1 WHERE(SELECT ABS(t3.a) FROM t3 WHERE t3.a + t3.b = t1.a AND t3.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+--echo # Not transformed, non-deterministic function
+let $query =
+SELECT * FROM t1 WHERE(SELECT ABS(t3.a) + ROUND(RAND()*10) FROM t3 WHERE t3.a + t3.b = t1.a AND t3.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT v3.a FROM v3 WHERE v3.a = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT v3.a FROM v3 WHERE v3.a + v3.b = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
+let $query =
+SELECT * FROM t1 WHERE(SELECT v3.a + v3.b FROM v3 WHERE v3.a + v3.b = t1.a) > 0;
+
+eval $query;
+eval EXPLAIN $query;
+
 --echo #
 --echo # example supported query: more than one correlated field
 --echo #
@@ -151,16 +242,24 @@ eval EXPLAIN $query;
 --echo # add GROUP BY
 --echo #
 INSERT INTO t2 VALUES (2);
-let $query=
+let $query_field=
 SELECT * FROM t1 WHERE (SELECT a FROM t2 WHERE t2.a = t1.a) > 0;
+let $query_func=
+SELECT * FROM t1 WHERE (SELECT a FROM t2 WHERE ABS(t2.a) = t1.a) > 0;
 
 --error ER_SUBQUERY_NO_1_ROW
-eval $query;
-eval EXPLAIN $query;
+eval $query_field;
+eval EXPLAIN $query_field;
+--error ER_SUBQUERY_NO_1_ROW
+eval $query_func;
+eval EXPLAIN $query_func;
 SET optimizer_switch = 'subquery_to_derived=default';
 --error ER_SUBQUERY_NO_1_ROW
-eval $query;
-eval EXPLAIN $query;
+eval $query_field;
+eval EXPLAIN $query_field;
+--error ER_SUBQUERY_NO_1_ROW
+eval $query_func;
+eval EXPLAIN $query_func;
 
 SET optimizer_switch = 'subquery_to_derived=on';
 
@@ -181,6 +280,21 @@ SELECT * FROM t1 WHERE (SELECT COUNT(a) FROM t3 WHERE t3.a = t1.a GROUP BY b) >
 eval $query;
 eval EXPLAIN $query;
 
+let $query=
+SELECT * FROM t1 WHERE (SELECT COUNT(a) FROM t3 WHERE ANY_VALUE(t3.a) = t1.a GROUP BY b) > 0;
+eval $query;
+eval EXPLAIN $query;
+
+let $query=
+SELECT * FROM t1 WHERE (SELECT COUNT(a) FROM t3 WHERE ABS(t3.a) = t1.a GROUP BY b) > 0;
+--error ER_WRONG_FIELD_WITH_GROUP
+eval $query;
+
+let $query=
+SELECT * FROM t1 WHERE (SELECT COUNT(a) FROM t3 WHERE ABS(ANY_VALUE(t3.a)) = t1.a GROUP BY b) > 0;
+eval $query;
+eval EXPLAIN $query;
+
 --echo #
 --echo # Test case that used to yield wrong result before we corrected
 --echo # computations of slice positions to accommodate non-hidden fields
@@ -209,6 +323,7 @@ eval EXPLAIN $query;
 
 DROP TABLE p, l;
 
+DROP VIEW v3;
 DROP TABLE t0, t1, t2, t3;
 
 --echo #
@@ -240,15 +355,23 @@ INSERT INTO t1 VALUES (1), (2), (3), (4);
 INSERT INTO t2 VALUES (1, 3, 3), (2, 3, 3);
 ANALYZE TABLE t1, t2;
 
-let $query=
+let $query_field=
 SELECT * FROM t1 WHERE (SELECT a FROM t2 WHERE t2.b = t1.a and t2.c = t1.a GROUP BY b) > 0;
+let $query_func=
+SELECT * FROM t1 WHERE (SELECT ABS(a) FROM t2 WHERE ABS(t2.b) = t1.a and ABS(t2.c) = t1.a GROUP BY b) > 0;
 --error 1055
-eval EXPLAIN $query;
+eval EXPLAIN $query_field;
+--error 1055
+eval EXPLAIN $query_func;
+
 SET sql_mode='';
-eval EXPLAIN $query;
-eval $query;
+eval EXPLAIN $query_field;
+eval $query_field;
+eval EXPLAIN $query_func;
+eval $query_func;
 DELETE FROM t2 WHERE a = 2;
-eval $query;
+eval $query_field;
+eval $query_func;
 SET sql_mode=default;
 
 let $query=
@@ -258,10 +381,37 @@ eval EXPLAIN $query;
 
 # bug found during review of Bug#35508108: used to get cardinality error
 let $query=
-SELECT * FROM t1 WHERE (SELECT b FROM t2 WHERE t2.b = t1.a and t2.c = t1.a GROUP BY b) > 0;
+SELECT * FROM t1 WHERE (SELECT b FROM t2 WHERE t2.b = t1.a AND t2.c = t1.a GROUP BY b) > 0;
 eval $query;
 eval EXPLAIN $query;
 
+--echo # This query gets imprecise full group by checking: accepted without transform
+let $query=
+SELECT * FROM t1 WHERE (SELECT ABS(b) FROM t2 WHERE ABS(t2.b) = t1.a AND t2.c + 1 = t1.a + 1 GROUP BY b) > 0;
+--error ER_WRONG_FIELD_WITH_GROUP
+eval $query;
+
+--echo # Ok without transform
+SET optimizer_switch = 'subquery_to_derived=off';
+eval $query;
+SET optimizer_switch = 'subquery_to_derived=on';
+
+--echo # Remedied with an ANY_VALUE:
+let $query=
+SELECT * FROM t1 WHERE (SELECT ABS(b) FROM t2 WHERE ABS(t2.b) = t1.a AND ANY_VALUE(t2.c) + 1 = t1.a + 1 GROUP BY b) > 0;
+eval $query;
+eval EXPLAIN $query;
+
+--echo # Wrong both with transform and without
+let $query=
+SELECT * FROM t1 WHERE (SELECT abs(a-b) FROM t2 WHERE ABS(a-b) = t1.a AND t2.c + 1 = t1.a + 1 GROUP BY a+b) > 0;
+--error ER_WRONG_FIELD_WITH_GROUP
+eval $query;
+SET optimizer_switch = 'subquery_to_derived=off';
+--error ER_WRONG_FIELD_WITH_GROUP
+eval $query;
+SET optimizer_switch = 'subquery_to_derived=on';
+
 DROP TABLE t1, t2;
 
 --echo #
@@ -275,7 +425,6 @@ INSERT INTO t3 VALUES (1, 3), (2, 3), (1, 4);
 ANALYZE TABLE t1, t3;
 let $query =
 SELECT * FROM t1 WHERE (SELECT COUNT(a) FROM t3 WHERE t3.a = t1.a GROUP BY b) > 0;
-
 --error ER_SUBQUERY_NO_1_ROW
 eval $query;
 eval EXPLAIN $query;
@@ -841,6 +990,46 @@ eval EXPLAIN $query;
 
 DROP TABLE t1, t2;
 
+--echo Somewhat realistic example
+CREATE TABLE employees(employee_id INT,
+                       name VARCHAR(255),
+                       salary DECIMAL(8,2),
+                       department VARCHAR(255));
+INSERT INTO employees
+       VALUES (1, 'Bob', 250000, 'English'),
+              (2, 'Charles', 250000, 'English'),
+              (3, 'Kari', 250000, 'English'),
+              (4, 'Per', 250000, 'ENGLISH'),
+              (6, 'Ole', 300000, 'English'),
+              (7, 'Heinrich', 250000, 'English'),
+              (8, 'James', 250000, 'Math'),
+              (9, 'Dag', 300000, 'MATHEMATICS'),
+              (10, 'Norvald', 250000, 'Math'),
+              (11, 'Edward', 250000, 'Math'),
+              (12, 'Rose', 250000, 'MATH'),
+              (13, 'Sally', 250000, 'MATH');
+
+--echo # Sloppy department data doesn't give right answer for maths dept.
+--echo # WL#13520:
+SELECT employee_id, name
+   FROM employees oemp
+  WHERE salary > (
+        SELECT AVG(salary)
+          FROM employees
+         WHERE department = oemp.department);
+--echo # In addition to default case insignificance, use only first four letters
+--echo # to get desired answer
+--echo # WL#15540
+SELECT employee_id, name
+   FROM employees oemp
+  WHERE salary > (
+        SELECT AVG(salary)
+          FROM employees
+         WHERE SUBSTRING(department,1,4) = SUBSTRING(oemp.department,1,4));
+
+DROP TABLE employees;
+
+
 --echo #
 --echo # Bug#36060557 WL#15540: Expression containing Non-deterministic function
 --echo #              UUID() is transformed

mysql-test/t/subquery_scalar_to_derived_correlated.test

@@ -569,9 +569,26 @@ class Group_check : public Item_tree_walker {
         whole_tables_fd(0),
         recheck_nullable_keys(0),
         mat_tables(root),
-        failed_ident(nullptr) {}
+        failed_ident(nullptr) {
+    if (select->m_no_of_added_exprs > 0) {
+      // Temporarily shorten GROUP BY list for full group by checking of
+      // non-field expressions stemming from transform of correlated scalar
+      // subquery to join with derived table, cf.
+      // Query_block::m_no_of_added_exprs
+      select->group_list.split_after(
+          select->group_list.elements - select->m_no_of_added_exprs,
+          &m_added_by_transform);
+    }
+  }
 
-  ~Group_check() { std::destroy_n(mat_tables.data(), mat_tables.size()); }
+  ~Group_check() {
+    std::destroy_n(mat_tables.data(), mat_tables.size());
+    if (select->m_no_of_added_exprs > 0) {
+      // restore GROUP BY list
+      select->group_list.push_back(&m_added_by_transform);
+      m_added_by_transform.clear();
+    }
+  }
   Group_check(const Group_check &) = delete;
   Group_check &operator=(const Group_check &) = delete;
 
@@ -641,6 +658,8 @@ class Group_check : public Item_tree_walker {
   Mem_root_array<Group_check *> mat_tables;
   /// Identifier which triggered an error
   Item_ident *failed_ident;
+  /// GROUP BY list non single column expressions entries added by transform
+  SQL_I_List<ORDER> m_added_by_transform;
 
   bool is_fd_on_source(Item *item);
   bool is_child() const { return table != nullptr; }

sql/aggregate_check.h

@@ -987,8 +987,7 @@ bool Item_field::collect_item_field_or_ref_processor(uchar *arg) {
       if (already_collected->eq(this, true)) return false;
     }
   }
-  info->m_items->push_back(this);
-  return false;
+  return info->m_items->push_back(this);
 }
 
 bool Item_field::collect_item_field_or_view_ref_processor(uchar *arg) {
@@ -1010,8 +1009,7 @@ bool Item_field::collect_item_field_or_view_ref_processor(uchar *arg) {
     }
   }
   m_protected_by_any_value = info->m_any_value_level > 0;
-  info->m_item_fields_or_view_refs->push_back(this);
-  return false;
+  return info->m_item_fields_or_view_refs->push_back(this);
 }
 
 bool Item_field::add_field_to_set_processor(uchar *arg) {
@@ -9020,7 +9018,10 @@ bool Item_view_ref::collect_item_field_or_view_ref_processor(uchar *arg) {
                     : ((depended_from == info->m_transformed_block)  // 3
                            ? this
                            : nullptr));
-  if (item != nullptr) info->m_item_fields_or_view_refs->push_back(item);
+  bool error = false;
+  if (item != nullptr)
+    error = info->m_item_fields_or_view_refs->push_back(item);
+  if (error) return true;
   info->stop_at(this);
   return false;
 }

sql/item.cc

@@ -3190,6 +3190,16 @@ class Item : public Parse_tree_node {
           m_default_value(default_value) {}
   };
 
+  struct Item_func_call_replacement : Item_replacement {
+    Item_func *m_target;  ///< The function call to be replaced
+    Item_field *m_item;   ///< The replacement field
+    Item_func_call_replacement(Item_func *func_target, Item_field *item,
+                               Query_block *select)
+        : Item_replacement(select, select),
+          m_target(func_target),
+          m_item(item) {}
+  };
+
   struct Item_view_ref_replacement : Item_replacement {
     Item *m_target;  ///< The item identifying the view_ref to be replaced
     Field *m_field;  ///< The replacement field
@@ -3221,6 +3231,7 @@ class Item : public Parse_tree_node {
     operation of the original transformed query block.
   */
   virtual Item *replace_item_field(uchar *) { return this; }
+  virtual Item *replace_func_call(uchar *) { return this; }
   virtual Item *replace_item_view_ref(uchar *) { return this; }
   virtual Item *replace_aggregate(uchar *) { return this; }
   virtual Item *replace_outer_ref(uchar *) { return this; }