Bug#34306497 - Low limit heuristic used unnecessarily with descending scans

Nimita Joshi · Nimita Joshi · commit 2010aeb8592f · 2023-08-22T09:31:53.000+05:30
Issue:
The low limit heuristic is not working well for order by desc as it
chooses the wrong index.

Analysis:
Because of the low limit we relook at the access method that is chosen
which in this case is the index. If the chosen range scan can provide
the order we skip the heuristic. At this point since const optimization
is not done the optimizer thinks the current range scan cannot provide
order so it relooks at other options. Since the primary key can provide
the order it chooses that one.

Fix:
Since const optimisation is not done when we check if an index can
provide order, the key part of the index and the order by field did not
match. The fix is to check if the index can provide the order by
skipping constant key parts preceding to the specified order by
expression. Later we check if the best range scan can provide order by
reversing it.

Change-Id: I1f967670c195f49fe39284e17e2d3c70030ad466
diff --git a/mysql-test/r/myisam_explain_json_non_select_all.result b/mysql-test/r/myisam_explain_json_non_select_all.result
@@ -4264,6 +4264,7 @@ EXPLAIN
         "filtered": "100.00",
         "index_condition": "((`test`.`t2`.`i` > 10) and (`test`.`t2`.`i` <= 18))",
         "backward_index_scan": true,
+        "using_MRR": true,
         "cost_info": {
           "read_cost": "1.46",
           "eval_cost": "0.80",
@@ -5301,6 +5302,7 @@ EXPLAIN
         "filtered": "100.00",
         "index_condition": "((`test`.`t2`.`i` > 10) and (`test`.`t2`.`i` <= 18))",
         "backward_index_scan": true,
+        "using_MRR": true,
         "cost_info": {
           "read_cost": "1.46",
           "eval_cost": "0.80",
diff --git a/mysql-test/r/order_by_limit.result b/mysql-test/r/order_by_limit.result
@@ -555,3 +555,88 @@ Warnings:
 Note	1003	/* select#1 */ select `test`.`test`.`a` AS `a`,`test`.`test`.`b` AS `b`,`test`.`test`.`c` AS `c`,`test`.`test`.`d` AS `d` from `test`.`test` where ((`test`.`test`.`c` = DATE'2022-06-13') and (`test`.`test`.`a` > 222)) order by `test`.`test`.`c`,`test`.`test`.`a` limit 10
 DROP PROCEDURE insertProc;
 DROP TABLE test;
+#
+# Bug#34306497 - Low limit heuristic used unnecessarily with descending scans
+#
+CREATE TABLE t1 (f1 INTEGER, f2 INTEGER, PRIMARY KEY (f1), KEY(f2, f1));
+INSERT INTO t1 (
+WITH RECURSIVE
+a(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM a WHERE i < 9 ),
+b(i) AS (SELECT x.i + y.i * 10 + z.i * 100 FROM a x, a y, a z)
+SELECT b.i, b.i %2 FROM b ORDER BY i);
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+EXPLAIN SELECT * FROM t1 WHERE f2 = 1 AND f1 <= 100 ORDER BY f1 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	NULL	range	PRIMARY,f2	f2	9	NULL	50	100.00	Using where; Backward index scan; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t1`.`f1` AS `f1`,`test`.`t1`.`f2` AS `f2` from `test`.`t1` where ((`test`.`t1`.`f2` = 1) and (`test`.`t1`.`f1` <= 100)) order by `test`.`t1`.`f1` desc limit 1
+SELECT * FROM t1 WHERE f2 = 1 AND f1 <= 100 ORDER BY f1 DESC LIMIT 1;
+f1	f2
+99	1
+CREATE TABLE t2 (
+f1 INTEGER,
+f2 INTEGER,
+f3 INTEGER,
+f4 INTEGER,
+f5 INTEGER,
+PRIMARY KEY (f1), KEY(f2,f3,f4,f5,f1));
+INSERT INTO t2 (
+WITH RECURSIVE a (i) AS (SELECT 0 UNION ALL SELECT i+1 FROM a WHERE i < 9 ),
+b (i) AS (SELECT x.i + y.i * 10 + z.i * 100 FROM a x, a y, a z)
+SELECT b.i, b.i%2, b.i%3, b.i%4, b.i%5 FROM b ORDER BY i);
+ANALYZE TABLE t2;
+Table	Op	Msg_type	Msg_text
+test.t2	analyze	status	OK
+EXPLAIN SELECT * FROM t2
+WHERE f3 = 1 AND f2 = 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f4 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	f2	f2	20	NULL	33	100.00	Using where; Backward index scan; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t2`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t2`.`f3` AS `f3`,`test`.`t2`.`f4` AS `f4`,`test`.`t2`.`f5` AS `f5` from `test`.`t2` where ((`test`.`t2`.`f4` = 3) and (`test`.`t2`.`f2` = 1) and (`test`.`t2`.`f3` = 1) and (`test`.`t2`.`f5` in (2,3))) order by `test`.`t2`.`f4` desc limit 1
+SELECT * FROM t2
+WHERE f3 = 1 AND f2 = 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f4 DESC LIMIT 1;
+f1	f2	f3	f4	f5
+943	1	1	3	3
+EXPLAIN SELECT * FROM t2
+WHERE f2 = 1 AND f3 = 2 AND f4 = 3 AND f5 IN(2,3) ORDER BY f3,f4 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	f2	f2	20	NULL	33	100.00	Using where; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t2`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t2`.`f3` AS `f3`,`test`.`t2`.`f4` AS `f4`,`test`.`t2`.`f5` AS `f5` from `test`.`t2` where ((`test`.`t2`.`f4` = 3) and (`test`.`t2`.`f3` = 2) and (`test`.`t2`.`f2` = 1) and (`test`.`t2`.`f5` in (2,3))) order by `test`.`t2`.`f3`,`test`.`t2`.`f4` desc limit 1
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 = 2 AND f4 = 3 AND f5 IN(2,3) ORDER BY f3,f4 DESC LIMIT 1;
+f1	f2	f3	f4	f5
+47	1	2	3	2
+EXPLAIN SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f2,f3 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	f2	f2	10	NULL	166	2.00	Using where; Backward index scan; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t2`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t2`.`f3` AS `f3`,`test`.`t2`.`f4` AS `f4`,`test`.`t2`.`f5` AS `f5` from `test`.`t2` where ((`test`.`t2`.`f4` = 3) and (`test`.`t2`.`f2` = 1) and (`test`.`t2`.`f3` > 1) and (`test`.`t2`.`f5` in (2,3))) order by `test`.`t2`.`f2`,`test`.`t2`.`f3` desc limit 1
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f2,f3 DESC LIMIT 1;
+f1	f2	f3	f4	f5
+983	1	2	3	3
+EXPLAIN SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2,f3,f5 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	f2	f2	10	NULL	166	10.00	Using where; Using index; Using filesort
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t2`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t2`.`f3` AS `f3`,`test`.`t2`.`f4` AS `f4`,`test`.`t2`.`f5` AS `f5` from `test`.`t2` where ((`test`.`t2`.`f4` = 3) and (`test`.`t2`.`f2` = 1) and (`test`.`t2`.`f3` > 1)) order by `test`.`t2`.`f2`,`test`.`t2`.`f3`,`test`.`t2`.`f5` desc limit 1
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2,f3,f5 DESC LIMIT 1;
+f1	f2	f3	f4	f5
+59	1	2	3	4
+EXPLAIN SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2 DESC ,f3 DESC ,f5 DESC LIMIT 1;
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	f2	f2	10	NULL	166	10.00	Using where; Backward index scan; Using index
+Warnings:
+Note	1003	/* select#1 */ select `test`.`t2`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t2`.`f3` AS `f3`,`test`.`t2`.`f4` AS `f4`,`test`.`t2`.`f5` AS `f5` from `test`.`t2` where ((`test`.`t2`.`f4` = 3) and (`test`.`t2`.`f2` = 1) and (`test`.`t2`.`f3` > 1)) order by `test`.`t2`.`f2` desc,`test`.`t2`.`f3` desc,`test`.`t2`.`f5` desc limit 1
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2 DESC ,f3 DESC ,f5 DESC LIMIT 1;
+f1	f2	f3	f4	f5
+959	1	2	3	4
+DROP TABLE t1,t2;
diff --git a/mysql-test/t/order_by_limit.test b/mysql-test/t/order_by_limit.test
@@ -436,3 +436,70 @@ EXPLAIN SELECT * FROM test  WHERE c = '2022-06-13' AND a > 222 ORDER BY c, a LIM
 
 DROP PROCEDURE insertProc;
 DROP TABLE test;
+
+--echo #
+--echo # Bug#34306497 - Low limit heuristic used unnecessarily with descending scans
+--echo #
+
+CREATE TABLE t1 (f1 INTEGER, f2 INTEGER, PRIMARY KEY (f1), KEY(f2, f1));
+
+INSERT INTO t1 (
+WITH RECURSIVE
+a(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM a WHERE i < 9 ),
+b(i) AS (SELECT x.i + y.i * 10 + z.i * 100 FROM a x, a y, a z)
+SELECT b.i, b.i %2 FROM b ORDER BY i);
+
+ANALYZE TABLE t1;
+
+let query =
+SELECT * FROM t1 WHERE f2 = 1 AND f1 <= 100 ORDER BY f1 DESC LIMIT 1;
+
+eval EXPLAIN $query;
+eval $query;
+
+CREATE TABLE t2 (
+ f1 INTEGER,
+ f2 INTEGER,
+ f3 INTEGER,
+ f4 INTEGER,
+ f5 INTEGER,
+PRIMARY KEY (f1), KEY(f2,f3,f4,f5,f1));
+
+INSERT INTO t2 (
+WITH RECURSIVE a (i) AS (SELECT 0 UNION ALL SELECT i+1 FROM a WHERE i < 9 ),
+b (i) AS (SELECT x.i + y.i * 10 + z.i * 100 FROM a x, a y, a z)
+SELECT b.i, b.i%2, b.i%3, b.i%4, b.i%5 FROM b ORDER BY i);
+
+ANALYZE TABLE t2;
+
+let query =
+SELECT * FROM t2
+WHERE f3 = 1 AND f2 = 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f4 DESC LIMIT 1;
+eval EXPLAIN $query;
+eval $query;
+
+let query =
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 = 2 AND f4 = 3 AND f5 IN(2,3) ORDER BY f3,f4 DESC LIMIT 1;
+eval EXPLAIN $query;
+eval $query;
+
+let query =
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 AND f5 IN(2,3) ORDER BY f2,f3 DESC LIMIT 1;
+eval EXPLAIN $query;
+eval $query;
+
+let query =
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2,f3,f5 DESC LIMIT 1;
+eval EXPLAIN $query;
+eval $query;
+
+let query =
+SELECT * FROM t2
+WHERE f2 = 1 AND f3 > 1 AND f4 = 3 ORDER BY f2 DESC ,f3 DESC ,f5 DESC LIMIT 1;
+eval EXPLAIN $query;
+eval $query;
+
+DROP TABLE t1,t2;
diff --git a/sql/sql_optimizer.cc b/sql/sql_optimizer.cc
@@ -1819,11 +1819,13 @@ int test_if_order_by_key(ORDER_with_src *order_src, TABLE *table, uint idx,
     /*
       Skip key parts that are constants in the WHERE clause if these are
       already removed in the ORDER expression by check_field_is_const().
+      If they are not removed in the ORDER expression yet, then we skip
+      the constant keyparts that are not part of the ORDER expression.
     */
-    if (order_src->is_const_optimized()) {
-      for (; const_key_parts & 1 && key_part < key_part_end;
-           const_key_parts >>= 1)
-        key_part++;
+    for (; const_key_parts & 1 && key_part < key_part_end &&
+           (order_src->is_const_optimized() || key_part->field != field);
+         const_key_parts >>= 1) {
+      key_part++;
     }
 
     /* Avoid usage of prefix index for sorting a partition table */
@@ -1850,11 +1852,13 @@ int test_if_order_by_key(ORDER_with_src *order_src, TABLE *table, uint idx,
         /*
           Skip key parts that are constants in the WHERE clause if these are
           already removed in the ORDER expression by check_field_is_const().
+          If they are not removed in the ORDER expression yet, then we skip
+          the constant keyparts that are not part of the ORDER expression.
         */
-        if (order_src->is_const_optimized()) {
-          for (; const_key_parts & 1 && key_part < key_part_end;
-               const_key_parts >>= 1)
-            key_part++;
+        for (; const_key_parts & 1 && key_part < key_part_end &&
+               (order_src->is_const_optimized() || key_part->field != field);
+             const_key_parts >>= 1) {
+          key_part++;
         }
         /*
          The primary and secondary key parts were all const (i.e. there's
@@ -9776,7 +9780,9 @@ static bool make_join_query_block(JOIN *join, Item *cond) {
                 */
                 if (read_direction == 1 ||
                     (read_direction == -1 &&
-                     is_reverse_sorted_range(tab->range_scan()))) {
+                     reverse_sort_possible(tab->range_scan()) &&
+                     !make_reverse(get_used_key_parts(tab->range_scan()),
+                                   tab->range_scan()))) {
                   recheck_reason = DONT_RECHECK;
                 }
               }
