Bug#36562979: MySQL crash at VarintParse64()

The hash join buffer could in some cases have too little space for a
row to be written, so that data could be written outside of the
allocated buffer and cause erroneous behaviour.

This happened if the hash join buffer stored row ids from a table in
the MEMORY engine. The reason was that the maximum size per row was
calculated before the row id size (that is, handler::ref_length) had
been initialized in the MEMORY engine handler, so the default
ref_length for the handler class was used instead, which is
sizeof(my_off_t). This is usually half the correct size, which is
sizeof(HP_HEAP_POSITION).

Fixed by initializing ref_length already when the MEMORY engine
handler is constructed, so that it always has the correct value. Also
add more assertions to detect the problem earlier.

Change-Id: I6a7a675f2117b78c5b279bbba1abd8da641fad6e
(cherry picked from commit 05b23e971e0e779406043c15cc8b4b417a246c91)

Author: kahatlen

sql/iterators/hash_join_buffer.cc

@@ -139,11 +139,15 @@ HashJoinRowBuffer::StoreLinkedImmutableStringFromTableBuffers(
   }
 
   ret = LinkedImmutableString::EncodeHeader(next_ptr, &dptr);
+
+  const char *const start_of_row [[maybe_unused]] = dptr;
   dptr = pointer_cast<char *>(
       StoreFromTableBuffersRaw(m_tables, pointer_cast<uchar *>(dptr)));
+  assert(dptr <= start_of_row + row_size_upper_bound);
 
+  const size_t actual_length = dptr - start_of_value;
+  assert(actual_length <= required_value_bytes);
   if (!committed) {
-    const size_t actual_length = dptr - pointer_cast<char *>(start_of_value);
     m_mem_root.RawCommit(actual_length);
   }
   return ret;

storage/heap/ha_heap.cc

@@ -77,11 +77,9 @@ static handler *heap_create_handler(handlerton *hton, TABLE_SHARE *table, bool,
 *****************************************************************************/
 
 ha_heap::ha_heap(handlerton *hton, TABLE_SHARE *table_arg)
-    : handler(hton, table_arg),
-      file(nullptr),
-      records_changed(0),
-      key_stat_version(0),
-      single_instance(false) {}
+    : handler(hton, table_arg) {
+  ref_length = sizeof(HP_HEAP_POSITION);
+}
 
 /*
   Hash index statistics is updated (copied from HP_KEYDEF::hash_buckets to
@@ -131,7 +129,6 @@ int ha_heap::open(const char *name, int mode, uint test_if_locked,
     }
   }
 
-  ref_length = sizeof(HP_HEAP_POSITION);
   /*
     We cannot run update_key_stats() here because we do not have a
     lock on the table. The 'records' count might just be changed

storage/heap/ha_heap.h

@@ -32,13 +32,13 @@
 #include "sql/table.h"
 
 class ha_heap : public handler {
-  HP_INFO *file;
-  HP_SHARE *internal_share;
+  HP_INFO *file{nullptr};
+  HP_SHARE *internal_share{nullptr};
   /* number of records changed since last statistics update */
-  uint records_changed;
-  uint key_stat_version;
+  uint records_changed{0};
+  uint key_stat_version{0};
   /// True if only one ha_heap is to exist for the table.
-  bool single_instance;
+  bool single_instance{false};
 
  public:
   ha_heap(handlerton *hton, TABLE_SHARE *table);