Bug#35483044 Hypergraph: Invalid row estimate for filter on 'Index range scan'

Jan Wedvik · Jan Wedvik · commit 94f69e9fcc2e · 2023-10-19T15:23:15.000+02:00
This commit improves row estimates for the following cases:

- field &lt;=&gt; NULL : If field is non-nullable, then the selectivity is 0.
  This fixes the error described in the bug report, where the selectivity
  of such a term was estimated to be 0.1.

- field=NULL : The selectivity is 0.

- field &lt;=&gt; other_expression : Use the same algorithm as for
  'field=other_expression', meaning that we use histograms or index
  statistics if applicable.

This commit also changes mysql-test/include/index_merge1.inc so that
the code for populating table t0 is pure SQL rather than perl.
This makes it easier to run part of the index_merge_... tests in
isolation, by copying the relevant SQL code from a .result file.

Change-Id: Ibba50f61a60edc0184293edb7317c5be6e082223
diff --git a/mysql-test/include/index_merge1.inc b/mysql-test/include/index_merge1.inc
@@ -49,19 +49,25 @@ create table t0
   INDEX i8(key8)
 );
 
---disable_query_log
-insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+DELIMITER |;
 
-let $1=9;
-set @d=2;
-while ($1)
-{
-  eval insert into t0 select key1+@d, key2+@d, key3+@d, key4+@d, key5+@d,
-                             key6+@d, key7+@d, key8-@d from t0;
-  eval set @d=@d*2;
-  dec $1;
-}
---enable_query_log
+create procedure populate_t0()
+begin
+  declare d int default 2;
+  insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+
+  while d<=POW(2,9) do
+    insert into t0 select key1+d, key2+d, key3+d, key4+d, key5+d,
+      key6+d, key7+d, key8-d from t0;
+    set d = d*2;
+  end while;
+end|
+
+DELIMITER ;|
+
+call populate_t0();
+
+drop procedure populate_t0;
 
 analyze table t0;
 
diff --git a/mysql-test/r/filter_single_col_idx_big_myisam.result b/mysql-test/r/filter_single_col_idx_big_myisam.result
@@ -1219,7 +1219,7 @@ Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`col1_idx` AS `col1_idx`,`test`.`t1`.`col2_idx` AS `col2_idx`,`test`.`t1`.`col3` AS `col3`,`test`.`t1`.`col4` AS `col4`,`test`.`t1`.`vc` AS `vc`,`test`.`t1`.`vc_ft` AS `vc_ft` from `test`.`t1` where (`test`.`t1`.`col3` is null)
 EXPLAIN SELECT * FROM t1 WHERE t1.col3 = NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	1029	10.00	Using where
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	1029	0.10	Using where
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`col1_idx` AS `col1_idx`,`test`.`t1`.`col2_idx` AS `col2_idx`,`test`.`t1`.`col3` AS `col3`,`test`.`t1`.`col4` AS `col4`,`test`.`t1`.`vc` AS `vc`,`test`.`t1`.`vc_ft` AS `vc_ft` from `test`.`t1` where (`test`.`t1`.`col3` = NULL)
 EXPLAIN SELECT * FROM t1 WHERE t1.col3>NULL;
@@ -1283,7 +1283,7 @@ Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`col1_idx` AS `col1_idx`,`test`.`t1`.`col2_idx` AS `col2_idx`,`test`.`t1`.`col3` AS `col3`,`test`.`t1`.`col4` AS `col4`,`test`.`t1`.`vc` AS `vc`,`test`.`t1`.`vc_ft` AS `vc_ft` from `test`.`t1` where false
 EXPLAIN SELECT * FROM t1 WHERE t1.col4 = NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	1029	10.00	Using where
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	1029	0.10	Using where
 Warnings:
 Note	1003	/* select#1 */ select `test`.`t1`.`col1_idx` AS `col1_idx`,`test`.`t1`.`col2_idx` AS `col2_idx`,`test`.`t1`.`col3` AS `col3`,`test`.`t1`.`col4` AS `col4`,`test`.`t1`.`vc` AS `vc`,`test`.`t1`.`vc_ft` AS `vc_ft` from `test`.`t1` where (`test`.`t1`.`col4` = NULL)
 EXPLAIN SELECT * FROM t1 WHERE t1.col4>NULL;
diff --git a/mysql-test/r/filter_single_col_idx_small_myisam.result b/mysql-test/r/filter_single_col_idx_small_myisam.result
@@ -139,7 +139,7 @@ id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered
 1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	10.00	Using where
 EXPLAIN SELECT * FROM t1 WHERE t1.col3 = NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	10.00	Using where
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	0.78	Using where
 EXPLAIN SELECT * FROM t1 WHERE t1.col3>NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	33.33	Using where
@@ -179,7 +179,7 @@ id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered
 1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
 EXPLAIN SELECT * FROM t1 WHERE t1.col4 = NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	10.00	Using where
+1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	0.78	Using where
 EXPLAIN SELECT * FROM t1 WHERE t1.col4>NULL;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t1	NULL	ALL	NULL	NULL	NULL	NULL	128	33.33	Using where
diff --git a/mysql-test/r/hypergraph_bugs.result b/mysql-test/r/hypergraph_bugs.result
@@ -1185,3 +1185,98 @@ EXPLAIN
         -> Covering index lookup on x2 using k1 (a=x1.b)  (rows=6)
 
 DROP TABLE t1;
+#
+# Bug#35483044 Hypergraph: Invalid row estimate for filter on
+# 'Index range scan'
+#
+CREATE TABLE t1 (
+a INT PRIMARY KEY,
+b INT NOT NULL,
+c INT,
+KEY k_b(b),
+KEY k_c(c)
+);
+INSERT INTO t1 WITH RECURSIVE qn(n) AS
+(SELECT 1 UNION ALL SELECT n+1 FROM qn WHERE n<30) SELECT n, n/2, n/2 FROM qn;
+ANALYZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c=NULL;
+EXPLAIN
+-> Filter: ((t1.b = 5) or (t1.c = NULL))  (rows=2)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c<=>NULL;
+EXPLAIN
+-> Deduplicate rows sorted by row ID  (rows=4.8)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+    -> Index range scan on t1 using k_c over (c = NULL)  (rows=1)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c IS NULL;
+EXPLAIN
+-> Deduplicate rows sorted by row ID  (rows=4.8)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+    -> Index range scan on t1 using k_c over (c = NULL)  (rows=1)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=NULL OR c=5;
+EXPLAIN
+-> Filter: ((t1.b = NULL) or (t1.c = 5))  (rows=2)
+    -> Index range scan on t1 using k_c over (c = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b<=>NULL OR c=5;
+EXPLAIN
+-> Filter: ((t1.b <=> NULL) or (t1.c = 5))  (rows=2)
+    -> Index range scan on t1 using k_c over (c = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b<=>FLOOR(RAND(0));
+EXPLAIN
+-> Filter: (t1.b <=> floor(rand(0)))  (rows=2)
+    -> Table scan on t1  (rows=30)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE c<=>FLOOR(RAND(0));
+EXPLAIN
+-> Filter: (t1.c <=> floor(rand(0)))  (rows=2)
+    -> Table scan on t1  (rows=30)
+
+ANALYZE TABLE t1 UPDATE HISTOGRAM ON b,c;
+Table	Op	Msg_type	Msg_text
+test.t1	histogram	status	Histogram statistics created for column 'b'.
+test.t1	histogram	status	Histogram statistics created for column 'c'.
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c=NULL;
+EXPLAIN
+-> Filter: ((t1.b = 5) or (t1.c = NULL))  (rows=2)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c<=>NULL;
+EXPLAIN
+-> Deduplicate rows sorted by row ID  (rows=2.03)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+    -> Index range scan on t1 using k_c over (c = NULL)  (rows=1)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=5 OR c IS NULL;
+EXPLAIN
+-> Deduplicate rows sorted by row ID  (rows=2.03)
+    -> Index range scan on t1 using k_b over (b = 5)  (rows=2)
+    -> Index range scan on t1 using k_c over (c = NULL)  (rows=1)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b=NULL OR c=5;
+EXPLAIN
+-> Filter: ((t1.b = NULL) or (t1.c = 5))  (rows=2)
+    -> Index range scan on t1 using k_c over (c = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b<=>NULL OR c=5;
+EXPLAIN
+-> Filter: ((t1.b <=> NULL) or (t1.c = 5))  (rows=2)
+    -> Index range scan on t1 using k_c over (c = 5)  (rows=2)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE b<=>FLOOR(RAND(0));
+EXPLAIN
+-> Filter: (t1.b <=> floor(rand(0)))  (rows=2)
+    -> Table scan on t1  (rows=30)
+
+EXPLAIN FORMAT=TREE SELECT * FROM t1 WHERE c<=>FLOOR(RAND(0));
+EXPLAIN
+-> Filter: (t1.c <=> floor(rand(0)))  (rows=2)
+    -> Table scan on t1  (rows=30)
+
+DROP TABLE t1;
diff --git a/mysql-test/r/index_merge_innodb.result b/mysql-test/r/index_merge_innodb.result
@@ -23,6 +23,18 @@ INDEX i6(key6),
 INDEX i7(key7),
 INDEX i8(key8)
 );
+create procedure populate_t0()
+begin
+declare d int default 2;
+insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+while d<=POW(2,9) do
+insert into t0 select key1+d, key2+d, key3+d, key4+d, key5+d,
+key6+d, key7+d, key8-d from t0;
+set d = d*2;
+end while;
+end|
+call populate_t0();
+drop procedure populate_t0;
 analyze table t0;
 Table	Op	Msg_type	Msg_text
 test.t0	analyze	status	OK
diff --git a/mysql-test/r/index_merge_innodb_hypergraph.result b/mysql-test/r/index_merge_innodb_hypergraph.result
@@ -23,6 +23,18 @@ INDEX i6(key6),
 INDEX i7(key7),
 INDEX i8(key8)
 );
+create procedure populate_t0()
+begin
+declare d int default 2;
+insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+while d<=POW(2,9) do
+insert into t0 select key1+d, key2+d, key3+d, key4+d, key5+d,
+key6+d, key7+d, key8-d from t0;
+set d = d*2;
+end while;
+end|
+call populate_t0();
+drop procedure populate_t0;
 analyze table t0;
 Table	Op	Msg_type	Msg_text
 test.t0	analyze	status	OK
@@ -114,7 +126,7 @@ EXPLAIN
 #    verify fallback to "range" if there is only one non-confluent condition
 explain select * from t0 where key2 = 45 or key1 <=> null;
 EXPLAIN
--> Filter: ((t0.key2 = 45) or (t0.key1 <=> NULL))  (rows=103)
+-> Filter: ((t0.key2 = 45) or (t0.key1 <=> NULL))  (rows=1)
     -> Index range scan on t0 using i2 over (key2 = 45)  (rows=1)
 
 explain select * from t0 where key2 = 45 or key1 is not null;
@@ -128,7 +140,7 @@ EXPLAIN
 #   the last conj. is always false and will be discarded
 explain select * from t0 where key2=10 or key3=3 or key4 <=> null;
 EXPLAIN
--> Deduplicate rows sorted by row ID  (rows=104)
+-> Deduplicate rows sorted by row ID  (rows=2)
     -> Index range scan on t0 using i2 over (key2 = 10)  (rows=1)
     -> Index range scan on t0 using i3 over (key3 = 3)  (rows=1)
 
@@ -143,14 +155,14 @@ EXPLAIN
 explain select key1 from t0 where (key1 <=> null) or (key2 < 2) or
 (key3=10) or (key4 <=> null);
 EXPLAIN
--> Sort-deduplicate by row ID  (rows=472)
+-> Sort-deduplicate by row ID  (rows=342)
     -> Index range scan on t0 using i2 over (key2 < 2)  (rows=1)
     -> Index range scan on t0 using i3 over (key3 = 10)  (rows=1)
 
 explain select key1 from t0 where (key1 <=> null) or (key1 < 5) or
 (key3=10) or (key4 <=> null);
 EXPLAIN
--> Sort-deduplicate by row ID  (rows=472)
+-> Sort-deduplicate by row ID  (rows=342)
     -> Index range scan on t0 using i1 over (key1 < 5)  (rows=4)
     -> Index range scan on t0 using i3 over (key3 = 10)  (rows=1)
 
diff --git a/mysql-test/r/index_merge_myisam.result b/mysql-test/r/index_merge_myisam.result
@@ -23,6 +23,18 @@ INDEX i6(key6),
 INDEX i7(key7),
 INDEX i8(key8)
 );
+create procedure populate_t0()
+begin
+declare d int default 2;
+insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+while d<=POW(2,9) do
+insert into t0 select key1+d, key2+d, key3+d, key4+d, key5+d,
+key6+d, key7+d, key8-d from t0;
+set d = d*2;
+end while;
+end|
+call populate_t0();
+drop procedure populate_t0;
 analyze table t0;
 Table	Op	Msg_type	Msg_text
 test.t0	analyze	status	OK
diff --git a/mysql-test/r/index_merge_myisam_hypergraph.result b/mysql-test/r/index_merge_myisam_hypergraph.result
@@ -23,6 +23,18 @@ INDEX i6(key6),
 INDEX i7(key7),
 INDEX i8(key8)
 );
+create procedure populate_t0()
+begin
+declare d int default 2;
+insert into t0 values (1,1,1,1,1,1,1,1023),(2,2,2,2,2,2,2,1022);
+while d<=POW(2,9) do
+insert into t0 select key1+d, key2+d, key3+d, key4+d, key5+d,
+key6+d, key7+d, key8-d from t0;
+set d = d*2;
+end while;
+end|
+call populate_t0();
+drop procedure populate_t0;
 analyze table t0;
 Table	Op	Msg_type	Msg_text
 test.t0	analyze	status	OK
@@ -114,7 +126,7 @@ EXPLAIN
 #    verify fallback to "range" if there is only one non-confluent condition
 explain select * from t0 where key2 = 45 or key1 <=> null;
 EXPLAIN
--> Filter: ((t0.key2 = 45) or (t0.key1 <=> NULL))  (rows=103)
+-> Filter: ((t0.key2 = 45) or (t0.key1 <=> NULL))  (rows=1)
     -> Index range scan on t0 using i2 over (key2 = 45)  (rows=1)
 
 explain select * from t0 where key2 = 45 or key1 is not null;
@@ -128,7 +140,7 @@ EXPLAIN
 #   the last conj. is always false and will be discarded
 explain select * from t0 where key2=10 or key3=3 or key4 <=> null;
 EXPLAIN
--> Deduplicate rows sorted by row ID  (rows=104)
+-> Deduplicate rows sorted by row ID  (rows=2)
     -> Index range scan on t0 using i2 over (key2 = 10)  (rows=1)
     -> Index range scan on t0 using i3 over (key3 = 3)  (rows=1)
 
@@ -143,14 +155,14 @@ EXPLAIN
 explain select key1 from t0 where (key1 <=> null) or (key2 < 2) or
 (key3=10) or (key4 <=> null);
 EXPLAIN
--> Sort-deduplicate by row ID  (rows=472)
+-> Sort-deduplicate by row ID  (rows=342)
     -> Index range scan on t0 using i2 over (key2 < 2)  (rows=2)
     -> Index range scan on t0 using i3 over (key3 = 10)  (rows=1)
 
 explain select key1 from t0 where (key1 <=> null) or (key1 < 5) or
 (key3=10) or (key4 <=> null);
 EXPLAIN
--> Sort-deduplicate by row ID  (rows=472)
+-> Sort-deduplicate by row ID  (rows=342)
     -> Index range scan on t0 using i1 over (key1 < 5)  (rows=5)
     -> Index range scan on t0 using i3 over (key3 = 10)  (rows=1)
 
diff --git a/mysql-test/t/hypergraph_bugs.test b/mysql-test/t/hypergraph_bugs.test
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
