Bug#36879147 [InnoDB] FULLTEXT index limits FTS_DOC_ID to max unsigned 32-bit value

Backport to 5.7.49.

Issue:
FTS_DOC_ID is a 64-bit field and can have values 2^32 and higher. However
current implementation only supports 32-bit value range. This limtation
takes the form of:
 - Assertions
 - Use of unsigned long type which resolves to 32-bit on some platforms
 - VLC (variable length coding) implementation supporting up to 35 buts

Fix:
Support 64-bit doc IDs:
 - Remove assertions
 - Replace use of unsigned long for doc ID deltas with uint64_t
 - Extend VLC functions to support full unsigned 64-bit range

Change-Id: Ifb56b33c5ec75e578391612eb371c41fc6aeef31

Author: Andrzej Jarzabek

storage/innobase/fts/fts0fts.cc

@@ -1280,7 +1280,6 @@ fts_cache_node_add_positions(
 	ulint		enc_len;
 	ulint		last_pos;
 	byte*		ptr_start;
-	ulint		doc_id_delta;
 
 #ifdef UNIV_DEBUG
 	if (cache) {
@@ -1291,7 +1290,7 @@ fts_cache_node_add_positions(
 	ut_ad(doc_id >= node->last_doc_id);
 
 	/* Calculate the space required to store the ilist. */
-	doc_id_delta = (ulint)(doc_id - node->last_doc_id);
+	const uint64_t doc_id_delta = doc_id - node->last_doc_id;
 	enc_len = fts_get_encoded_len(doc_id_delta);
 
 	last_pos = 0;

storage/innobase/fts/fts0opt.cc

@@ -1153,7 +1153,7 @@ fts_optimize_encode_node(
 	/* Calculate the space required to store the ilist. */
 	ut_ad(doc_id > node->last_doc_id);
 	doc_id_delta = doc_id - node->last_doc_id;
-	enc_len = fts_get_encoded_len(static_cast<ulint>(doc_id_delta));
+	enc_len = fts_get_encoded_len(doc_id_delta);
 
 	/* Calculate the size of the encoded pos array. */
 	while (*src) {
@@ -1197,9 +1197,8 @@ fts_optimize_encode_node(
 	src = enc->src_ilist_ptr;
 	dst = node->ilist + node->ilist_size;
 
-	/* Encode the doc id. Cast to ulint, the delta should be small and
-	therefore no loss of precision. */
-	dst += fts_encode_int((ulint) doc_id_delta, dst);
+	/* Encode the doc id. */
+	dst += fts_encode_int(doc_id_delta, dst);
 
 	/* Copy the encoded pos array. */
 	memcpy(dst, src, pos_enc_len);
@@ -1243,10 +1242,9 @@ fts_optimize_node(
 	while (copied < src_node->ilist_size
 	       && dst_node->ilist_size < FTS_ILIST_MAX_SIZE) {
 
-		doc_id_t	delta;
 		doc_id_t	del_doc_id = FTS_NULL_DOC_ID;
 
-		delta = fts_decode_vlc(&enc->src_ilist_ptr);
+		doc_id_t delta = fts_decode_vlc(&enc->src_ilist_ptr);
 
 test_again:
 		/* Check whether the doc id is in the delete list, if

storage/innobase/fts/fts0que.cc

@@ -52,6 +52,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang
 #endif
 
 #include <iomanip>
+#include <limits>
 #include <vector>
 
 #define FTS_ELEM(t, n, i, j) (t[(i) * n + (j)])
@@ -3154,14 +3155,17 @@ fts_query_find_doc_id(
 		ulint		freq = 0;
 		ulint		min_pos = 0;
 		ulint		last_pos = 0;
-		ulint		pos = fts_decode_vlc(&ptr);
+		const uint64_t  delta = fts_decode_vlc(&ptr);
 
 		/* Add the delta. */
-		doc_id += pos;
+		doc_id += delta;
 
 		while (*ptr) {
 			++freq;
-			last_pos += fts_decode_vlc(&ptr);
+                        const uint64_t pos_delta = fts_decode_vlc(&ptr);
+                        ut_ad(uint64_t(last_pos) + pos_delta <=
+                              std::numeric_limits<ulint>::max());
+                        last_pos += static_cast<ulint>(pos_delta);
 
 			/* Only if min_pos is not set and the current
 			term exists in a position greater than the
@@ -3234,15 +3238,15 @@ fts_query_filter_doc_ids(
 		fts_doc_freq_t*	doc_freq;
 		fts_match_t*	match = NULL;
 		ulint		last_pos = 0;
-		ulint		pos = fts_decode_vlc(&ptr);
+		const uint64_t	delta = fts_decode_vlc(&ptr);
 
 		/* Some sanity checks. */
 		if (doc_id == 0) {
-			ut_a(pos == node->first_doc_id);
+			ut_a(delta == node->first_doc_id);
 		}
 
 		/* Add the delta. */
-		doc_id += pos;
+		doc_id += delta;
 
 		if (calc_doc_count) {
 			word_freq->doc_count++;
@@ -3272,7 +3276,10 @@ fts_query_filter_doc_ids(
 
 		/* Unpack the positions within the document. */
 		while (*ptr) {
-			last_pos += fts_decode_vlc(&ptr);
+                        const uint64_t decoded_pos = fts_decode_vlc(&ptr);
+                        ut_ad(uint64_t(last_pos) + decoded_pos
+                              <= std::numeric_limits<ulint>::max());
+                        last_pos += static_cast<ulint>(decoded_pos);
 
 			/* Collect the matching word positions, for phrase
 			matching later. */

storage/innobase/handler/i_s.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2023, Oracle and/or its affiliates.
+Copyright (c) 2007, 2024, Oracle and/or its affiliates.
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License, version 2.0,
@@ -3377,13 +3377,15 @@ i_s_fts_index_cache_fill_one_index(
 			ptr = node->ilist;
 
 			while (decoded < node->ilist_size) {
-				ulint	pos = fts_decode_vlc(&ptr);
+				const uint64_t  delta = fts_decode_vlc(&ptr);
 
-				doc_id += pos;
+				doc_id += delta;
 
 				/* Get position info */
 				while (*ptr) {
-					pos = fts_decode_vlc(&ptr);
+                                        const uint64_t decoded_pos = fts_decode_vlc(&ptr);
+                                        ut_ad(decoded_pos <= std::numeric_limits<ulint>::max());
+                                        const ulint pos = static_cast<ulint>(decoded_pos);
 
 					OK(field_store_string(
 						   fields[I_S_FTS_WORD],
@@ -3758,13 +3760,15 @@ i_s_fts_index_table_fill_one_fetch(
 			ptr = node->ilist;
 
 			while (decoded < node->ilist_size) {
-				ulint	pos = fts_decode_vlc(&ptr);
+				const uint64_t  delta = fts_decode_vlc(&ptr);
 
-				doc_id += pos;
+				doc_id += delta;
 
 				/* Get position info */
 				while (*ptr) {
-					pos = fts_decode_vlc(&ptr);
+                                        const uint64_t decoded_pos = fts_decode_vlc(&ptr);
+                                        ut_ad(decoded_pos <= std::numeric_limits<ulint>::max());
+                                        const ulint pos = static_cast<ulint>(decoded_pos);
 
 					OK(field_store_string(
 						   fields[I_S_FTS_WORD],

storage/innobase/include/fts0types.h

@@ -35,6 +35,7 @@ Created 2007-03-27 Sunny Bains
 #ifndef INNOBASE_FTS0TYPES_H
 #define INNOBASE_FTS0TYPES_H
 
+#include <stdint.h>
 #include "univ.i"
 #include "fts0fts.h"
 #include "fut0fut.h"
@@ -307,7 +308,7 @@ extern const fts_index_selector_t fts_index_selector[];
 /******************************************************************//**
 Decode and return the integer that was encoded using our VLC scheme.*/
 UNIV_INLINE
-ulint
+uint64_t
 fts_decode_vlc(
 /*===========*/
 			/*!< out: value decoded */
@@ -331,22 +332,22 @@ fts_string_dup(
 /******************************************************************//**
 Return length of val if it were encoded using our VLC scheme. */
 UNIV_INLINE
-ulint
+unsigned int
 fts_get_encoded_len(
 /*================*/
 						/*!< out: length of value
 						 encoded, in bytes */
-	ulint		val);			/*!< in: value to encode */
+	uint64_t	val);			/*!< in: value to encode */
 
 /******************************************************************//**
 Encode an integer using our VLC scheme and return the length in bytes. */
 UNIV_INLINE
-ulint
+unsigned int
 fts_encode_int(
 /*===========*/
 						/*!< out: length of value
 						encoded, in bytes */
-	ulint		val,			/*!< in: value to encode */
+	uint64_t	val,			/*!< in: value to encode */
 	byte*		buf);			/*!< in: buffer, must have
 						enough space */
 

storage/innobase/include/fts0vlc.ic

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2023, Oracle and/or its affiliates.
+Copyright (c) 2007, 2024, Oracle and/or its affiliates.
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License, version 2.0,
@@ -35,101 +35,97 @@ Created 2007-03-27 Sunny Bains
 #ifndef INNOBASE_FTS0VLC_IC
 #define INNOBASE_FTS0VLC_IC
 
+#include <stddef.h>
+#include <stdint.h>
 #include "fts0types.h"
 
 /******************************************************************//**
 Return length of val if it were encoded using our VLC scheme.
-FIXME: We will need to be able encode 8 bytes value
 @return length of value encoded, in bytes */
 UNIV_INLINE
-ulint
+unsigned int
 fts_get_encoded_len(
 /*================*/
-	ulint	val)	/* in: value to encode */
+	uint64_t val)	/* in: value to encode */
 {
-	if (val <= 127) {
-		return(1);
-	} else if (val <= 16383) {
-		return(2);
-	} else if (val <= 2097151) {
-		return(3);
-	} else if (val <= 268435455) {
-		return(4);
-	} else {
-		/* Possibly we should care that on 64-bit machines ulint can
-		contain values that we can't encode in 5 bytes, but
-		fts_encode_int doesn't handle them either so it doesn't much
-		matter. */
-
-		return(5);
-	}
+        unsigned int length = 1;
+        for (;;)
+        {
+                val >>= 7;
+                if (val != 0)
+                {
+                        ++length;
+                }
+                else
+                {
+                  break;
+                }
+        }
+        return length;
 }
 
 /******************************************************************//**
 Encode an integer using our VLC scheme and return the length in bytes.
 @return length of value encoded, in bytes */
 UNIV_INLINE
-ulint
+unsigned int
 fts_encode_int(
 /*===========*/
-	ulint	val,	/* in: value to encode */
+	uint64_t val,	/* in: value to encode */
 	byte*	buf)	/* in: buffer, must have enough space */
 {
-	ulint	len;
-
-	if (val <= 127) {
-		*buf = (byte) val;
-
-		len = 1;
-	} else if (val <= 16383) {
-		*buf++ = (byte)(val >> 7);
-		*buf = (byte)(val & 0x7F);
-
-		len = 2;
-	} else if (val <= 2097151) {
-		*buf++ = (byte)(val >> 14);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 3;
-	} else if (val <= 268435455) {
-		*buf++ = (byte)(val >> 21);
-		*buf++ = (byte)((val >> 14) & 0x7F);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 4;
-	} else {
-		/* Best to keep the limitations of the 32/64 bit versions
-		identical, at least for the time being. */
-		ut_ad(val <= 4294967295u);
-
-		*buf++ = (byte)(val >> 28);
-		*buf++ = (byte)((val >> 21) & 0x7F);
-		*buf++ = (byte)((val >> 14) & 0x7F);
-		*buf++ = (byte)((val >> 7) & 0x7F);
-		*buf = (byte)(val & 0x7F);
-
-		len = 5;
+        const unsigned int max_length = 10;
+        /* skip leading zeros */
+        unsigned int count = max_length - 1;
+        while (count > 0)
+        {
+                /* We split the value into 7 bit batches); so val >= 2^63 need 10 bytes,
+                   2^63 > val >= 2^56 needs 9 bytes, 2^56 > val >= 2^49 needs 8 bytes etc.
+                */
+                if (val >= uint64_t(1) << (count * 7))
+                {
+                        break;
+                }
+                --count;
 	}
 
-	/* High-bit on means "last byte in the encoded integer". */
-	*buf |= 0x80;
-
-	return(len);
+        unsigned int length = count + 1;
+
+        byte *bufptr= buf;
+
+        for (;;)
+        {
+                *bufptr = (byte)((val >> (7 * count)) & 0x7f);
+                if (count == 0)
+                {
+                        /* High-bit on means "last byte in the encoded integer". */
+                        *bufptr |= 0x80;
+                        break;
+                }
+                else
+                {
+                        --count;
+                        ++bufptr;
+                }
+        }
+
+        ut_ad(length <= max_length);
+        ut_a(bufptr - buf == ptrdiff_t(length) - 1);
+
+        return length;
 }
 
 /******************************************************************//**
 Decode and return the integer that was encoded using our VLC scheme.
 @return value decoded */
 UNIV_INLINE
-ulint
+uint64_t
 fts_decode_vlc(
 /*===========*/
 	byte**	ptr)	/* in: ptr to decode from, this ptr is
 			incremented by the number of bytes decoded */
 {
-	ulint	val = 0;
+	uint64_t        val = 0;
 
 	for (;;) {
 		byte	b = **ptr;
@@ -145,7 +141,7 @@ fts_decode_vlc(
 		}
 	}
 
-	return(val);
+	return val;
 }
 
 #endif

unittest/gunit/innodb/CMakeLists.txt

@@ -34,6 +34,7 @@ INCLUDE(${CMAKE_SOURCE_DIR}/storage/innobase/innodb.cmake)
 
 SET(TESTS
   #example
+  fts0vlc
   ha_innodb
   mem0mem
   ut0crc32