Fix for bug#49897: crash in ptr_compare when char(0) NOT NULL

column is used for ORDER BY

Problem: filesort isn't meant for null length sort data
(e.g. char(0)), that leads to a server crash.

Fix: disregard sort order if sort data record length is 0 (nothing
to sort).

Author: Ramil Kalimullin

mysql-test/r/select.result

@@ -4540,4 +4540,68 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	SIMPLE	z	system	NULL	NULL	NULL	NULL	1	
 Warnings:
 Note	1003	select '2001-01-01 00:00:00' AS `a`,'2001-01-01 00:00:00' AS `a`,'2001-01-01 00:00:00' AS `a` from `test`.`t1` `x` join `test`.`t1` `y` join `test`.`t1` `z` where 1
+DROP TABLE t1;
+#
+# Bug #49897: crash in ptr_compare when char(0) NOT NULL 
+# column is used for ORDER BY
+#
+SET @old_sort_buffer_size= @@session.sort_buffer_size;
+SET @@sort_buffer_size= 40000;
+CREATE TABLE t1(a CHAR(0) NOT NULL);
+INSERT INTO t1 VALUES (0), (0), (0);
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+EXPLAIN SELECT a FROM t1 ORDER BY a;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	24492	
+SELECT a FROM t1 ORDER BY a;
+DROP TABLE t1;
+CREATE TABLE t1(a CHAR(0) NOT NULL, b CHAR(0) NOT NULL, c int);
+INSERT INTO t1 VALUES (0, 0, 0), (0, 0, 2), (0, 0, 1);
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+EXPLAIN SELECT a FROM t1 ORDER BY a LIMIT 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	24492	
+SELECT a FROM t1 ORDER BY a LIMIT 5;
+a
+
+
+
+
+
+EXPLAIN SELECT * FROM t1 ORDER BY a, b LIMIT 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	24492	
+SELECT * FROM t1 ORDER BY a, b LIMIT 5;
+a	b	c
+		0
+		2
+		1
+		0
+		2
+EXPLAIN SELECT * FROM t1 ORDER BY a, b, c LIMIT 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	24492	Using filesort
+SELECT * FROM t1 ORDER BY a, b, c LIMIT 5;
+a	b	c
+		0
+		0
+		0
+		0
+		0
+EXPLAIN SELECT * FROM t1 ORDER BY c, a LIMIT 5;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	24492	Using filesort
+SELECT * FROM t1 ORDER BY c, a LIMIT 5;
+a	b	c
+		0
+		0
+		0
+		0
+		0
+SET @@sort_buffer_size= @old_sort_buffer_size;
+DROP TABLE t1;
 End of 5.0 tests

mysql-test/t/select.test

@@ -3844,6 +3844,46 @@ EXPLAIN EXTENDED SELECT x.a, y.a, z.a FROM t1 x
   JOIN t1 y ON x.a=y.a 
   JOIN t1 z ON y.a=z.a 
   WHERE x.a='2001-01-01' AND z.a='2001-01-01 00:00:00';
+DROP TABLE t1;
+
+
+--echo #
+--echo # Bug #49897: crash in ptr_compare when char(0) NOT NULL 
+--echo # column is used for ORDER BY
+--echo #
+SET @old_sort_buffer_size= @@session.sort_buffer_size;
+SET @@sort_buffer_size= 40000;
+
+CREATE TABLE t1(a CHAR(0) NOT NULL);
+--disable_warnings
+INSERT INTO t1 VALUES (0), (0), (0);
+--enable_warnings
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a FROM t1 t11, t1 t12;
+EXPLAIN SELECT a FROM t1 ORDER BY a;
+--disable_result_log
+SELECT a FROM t1 ORDER BY a;
+--enable_result_log
+DROP TABLE t1;
 
+CREATE TABLE t1(a CHAR(0) NOT NULL, b CHAR(0) NOT NULL, c int);
+--disable_warnings
+INSERT INTO t1 VALUES (0, 0, 0), (0, 0, 2), (0, 0, 1);
+--enable_warnings
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+INSERT INTO t1 SELECT t11.a, t11.b, t11.c FROM t1 t11, t1 t12;
+EXPLAIN SELECT a FROM t1 ORDER BY a LIMIT 5;
+SELECT a FROM t1 ORDER BY a LIMIT 5;
+EXPLAIN SELECT * FROM t1 ORDER BY a, b LIMIT 5;
+SELECT * FROM t1 ORDER BY a, b LIMIT 5;
+EXPLAIN SELECT * FROM t1 ORDER BY a, b, c LIMIT 5;
+SELECT * FROM t1 ORDER BY a, b, c LIMIT 5;
+EXPLAIN SELECT * FROM t1 ORDER BY c, a LIMIT 5;
+SELECT * FROM t1 ORDER BY c, a LIMIT 5;
+
+SET @@sort_buffer_size= @old_sort_buffer_size;
+DROP TABLE t1;
 
 --echo End of 5.0 tests

sql/filesort.cc

@@ -134,6 +134,8 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
   error= 1;
   bzero((char*) &param,sizeof(param));
   param.sort_length= sortlength(thd, sortorder, s_length, &multi_byte_charset);
+  /* filesort cannot handle zero-length records. */
+  DBUG_ASSERT(param.sort_length);
   param.ref_length= table->file->ref_length;
   param.addon_field= 0;
   param.addon_length= 0;

sql/sql_select.cc

@@ -521,13 +521,26 @@ JOIN::prepare(Item ***rref_pointer_array,
 
   if (order)
   {
+    bool real_order= FALSE;
     ORDER *ord;
     for (ord= order; ord; ord= ord->next)
     {
       Item *item= *ord->item;
+      /*
+        Disregard sort order if there's only "{VAR}CHAR(0) NOT NULL" fields
+        there. Such fields don't contain any data to sort.
+      */
+      if (!real_order &&
+          (item->type() != Item::Item::FIELD_ITEM ||
+           ((Item_field *) item)->field->maybe_null() ||
+           ((Item_field *) item)->field->sort_length()))
+        real_order= TRUE;
+
       if (item->with_sum_func && item->type() != Item::SUM_FUNC_ITEM)
         item->split_sum_func(thd, ref_pointer_array, all_fields);
     }
+    if (!real_order)
+      order= NULL;
   }
 
   if (having && having->with_sum_func)