Bug #25080442 PARTITIONED TABLES USE MORE MEMORY IN 5.7 THAN 5.6

Aditya A · Aditya A · commit 906f7fa20629 · 2017-02-28T08:02:59.000+05:30
Additional features in innodb have added new variables in memory
structures which increases the total memory usage. In dict_index_t
in-memory structure which is allocated for each index these three
additional members are added in mysql-5.7.


last_ops_cur_t* last_ins_cur :- cache the last insert position.
last_ops_cur_t* last_sel_cur :- cache the last selected position
rec_cache_t     rec_cache    :- cache the field that needs to be
                                re-computed on each insert.

These three caches are only used for temporary or intrinsic tables
but memory for them is allocated for all tables which increases
the overall usage of memory

Fix
---
This fix optimizes the memory usage by allocating memory for these
caches only when they are used .
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
@@ -8470,7 +8470,9 @@ ha_innobase::index_end(void)
 {
 	DBUG_ENTER("index_end");
 
-	m_prebuilt->index->last_sel_cur->release();
+	if(m_prebuilt->index->last_sel_cur) {
+		m_prebuilt->index->last_sel_cur->release();
+	}
 
 	active_index = MAX_KEY;
 
@@ -12442,8 +12444,11 @@ ha_innobase::delete_table(
 
 	if (handler != NULL) {
 		for (dict_index_t* index = UT_LIST_GET_FIRST(handler->indexes);
-		     index != NULL;
+		     index != NULL && index->last_ins_cur;
 		     index = UT_LIST_GET_NEXT(indexes, index)) {
+			/* last_ins_cur and last_sel_cur are allocated
+			together ,therfore checking only last_ins_cur
+			before releasing mtr */
 			index->last_ins_cur->release();
 			index->last_sel_cur->release();
 		}
diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -2109,6 +2109,13 @@ bool
 dict_table_have_virtual_index(
 	dict_table_t*	table);
 
+/** Allocate memory for intrinsic cache elements in the index
+ * @param[in]      index   index object */
+UNIV_INLINE
+void
+dict_allocate_mem_intrinsic_cache(
+                dict_index_t*           index);
+
 #endif /* !UNIV_HOTBACKUP */
 
 #ifndef UNIV_NONINL
diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1938,5 +1938,25 @@ dict_table_have_virtual_index(
 
 	return(false);
 }
+/** Allocate memory for intrinsic cache elements in the index
+@param[in]      index   index object */
+UNIV_INLINE
+void
+dict_allocate_mem_intrinsic_cache(
+                dict_index_t*           index)
+{
+
+        index->last_ins_cur =
+                static_cast<last_ops_cur_t*>(mem_heap_alloc(
+                        index->heap, sizeof(last_ops_cur_t)));
+
+        new (index->last_ins_cur) last_ops_cur_t();
+
+        index->last_sel_cur =
+                static_cast<last_ops_cur_t*>(mem_heap_alloc(
+                        index->heap, sizeof(last_ops_cur_t)));
+
+        new (index->last_sel_cur) last_ops_cur_t();
+}
 
 #endif /* !UNIV_HOTBACKUP */
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -850,10 +850,6 @@ class last_ops_cur_t
 system clustered index when there is no primary key. */
 const char innobase_index_reserve_name[] = "GEN_CLUST_INDEX";
 
-/* Estimated number of offsets in records (based on columns)
-to start with. */
-#define OFFS_IN_REC_NORMAL_SIZE		100
-
 /** Data structure for an index.  Most fields will be
 initialized to 0, NULL or FALSE in dict_mem_index_create(). */
 struct dict_index_t{
diff --git a/storage/innobase/include/dict0mem.ic b/storage/innobase/include/dict0mem.ic
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -72,33 +72,10 @@ dict_mem_fill_index_struct(
 	index->allow_duplicates = false;
 	index->nulls_equal = false;
 	index->disable_ahi = false;
-
+	index->last_ins_cur = NULL;
+	index->last_sel_cur = NULL;
 	new (&index->rec_cache) rec_cache_t();
 
-	if (heap != NULL) {
-		index->last_ins_cur =
-			static_cast<last_ops_cur_t*>(mem_heap_alloc(
-				heap, sizeof(last_ops_cur_t)));
-
-		new (index->last_ins_cur) last_ops_cur_t();
-
-		index->last_sel_cur =
-			static_cast<last_ops_cur_t*>(mem_heap_alloc(
-				heap, sizeof(last_ops_cur_t)));
-
-		new (index->last_sel_cur) last_ops_cur_t();
-
-		index->rec_cache.offsets =
-			static_cast<ulint*>(mem_heap_alloc(
-				heap, sizeof(ulint) * OFFS_IN_REC_NORMAL_SIZE));
-
-		index->rec_cache.sz_of_offsets = OFFS_IN_REC_NORMAL_SIZE;
-	} else {
-		index->last_ins_cur = NULL;
-		index->last_sel_cur = NULL;
-		index->rec_cache.offsets = NULL;
-	}
-
 #ifdef UNIV_DEBUG
 	index->magic_n = DICT_INDEX_MAGIC_N;
 #endif /* UNIV_DEBUG */
diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1994, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -88,7 +88,7 @@ significant bit denotes that the tail of a field is stored off-page. */
 
 /* Number of elements that should be initially allocated for the
 offsets[] array, first passed to rec_get_offsets() */
-#define REC_OFFS_NORMAL_SIZE	OFFS_IN_REC_NORMAL_SIZE
+#define REC_OFFS_NORMAL_SIZE	100
 #define REC_OFFS_SMALL_SIZE	10
 
 /******************************************************//**
diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2579,7 +2579,9 @@ row_ins_clust_index_entry_low(
 		doesn't fit the provided slot then existing record is added
 		to free list and new record is inserted. This also means
 		cursor that we have cached for SELECT is now invalid. */
-		index->last_sel_cur->invalid = true;
+		if(index->last_sel_cur) {
+			index->last_sel_cur->invalid = true;
+		}
 
 		err = row_ins_clust_index_entry_by_modify(
 			&pcur, flags, mode, &offsets, &offsets_heap,
@@ -3132,7 +3134,9 @@ row_ins_sec_index_entry_low(
 		is doesn't fit the provided slot then existing record is added
 		to free list and new record is inserted. This also means
 		cursor that we have cached for SELECT is now invalid. */
-		index->last_sel_cur->invalid = true;
+		if(index->last_sel_cur) {
+			index->last_sel_cur->invalid = true;
+		}
 
 		/* There is already an index entry with a long enough common
 		prefix, we must convert the insert into a modify of an
@@ -3311,6 +3315,12 @@ row_ins_clust_index_entry(
 
 	if (dict_table_is_intrinsic(index->table)
 	    && dict_index_is_auto_gen_clust(index)) {
+
+		/* Check if the memory allocated for intrinsic cache*/
+		if(!index->last_ins_cur) {
+			dict_allocate_mem_intrinsic_cache(index);
+		}
+
 		err = row_ins_sorted_clust_index_entry(
 			BTR_MODIFY_LEAF, index, entry, n_ext, thr);
 	} else {
@@ -3331,6 +3341,9 @@ row_ins_clust_index_entry(
 	/* Try then pessimistic descent to the B-tree */
 	if (!dict_table_is_intrinsic(index->table)) {
 		log_free_check();
+	} else if(!index->last_sel_cur) {
+		dict_allocate_mem_intrinsic_cache(index);
+		index->last_sel_cur->invalid = true;
 	} else {
 		index->last_sel_cur->invalid = true;
 	}
@@ -3410,6 +3423,9 @@ row_ins_sec_index_entry(
 
 		if (!dict_table_is_intrinsic(index->table)) {
 			log_free_check();
+		} else if(!index->last_sel_cur) {
+			dict_allocate_mem_intrinsic_cache(index);
+			index->last_sel_cur->invalid = true;
 		} else {
 			index->last_sel_cur->invalid = true;
 		}
diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc
@@ -2285,7 +2285,9 @@ row_update_for_mysql_using_cursor(
 				node->upd_ext ? node->upd_ext->n_ext : 0,
 				false);
 			/* Commit the open mtr as we are processing UPDATE. */
-			index->last_ins_cur->release();
+			if (index->last_ins_cur) {
+				index->last_ins_cur->release();
+			}
 		} else {
 			err = row_ins_sec_index_entry(index, entry, thr, false);
 		}
@@ -2329,7 +2331,9 @@ row_del_upd_for_mysql_using_cursor(
 	to change. */
 	thr = que_fork_get_first_thr(prebuilt->upd_graph);
 	clust_index = dict_table_get_first_index(prebuilt->table);
-	clust_index->last_ins_cur->release();
+	if (clust_index->last_ins_cur) {
+		clust_index->last_ins_cur->release();
+	}
 
 	/* Step-1: Select the appropriate cursor that will help build
 	the original row and updated row. */
@@ -2731,15 +2735,20 @@ row_delete_all_rows(
 	dict_table_t*	table)
 {
 	dberr_t		err = DB_SUCCESS;
+	dict_index_t*	index;
+
 
+	index = dict_table_get_first_index(table);
 	/* Step-0: If there is cached insert position along with mtr
 	commit it before starting delete/update action. */
-	dict_table_get_first_index(table)->last_ins_cur->release();
+	if (index->last_ins_cur) {
+		index->last_ins_cur->release();
+	}
 
 	/* Step-1: Now truncate all the indexes and re-create them.
 	Note: This is ddl action even though delete all rows is
 	DML action. Any error during this action is ir-reversible. */
-	for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes);
+	for (index = UT_LIST_GET_FIRST(table->indexes);
 	     index != NULL && err == DB_SUCCESS;
 	     index = UT_LIST_GET_NEXT(indexes, index)) {
 
diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2008, Google Inc.
 
 Portions of this file contain modifications contributed and copyrighted by
@@ -4257,9 +4257,15 @@ row_search_no_mvcc(
 	ut_ad(index && pcur && search_tuple);
 
 	/* Step-0: Re-use the cached mtr. */
-	mtr_t*		mtr = &index->last_sel_cur->mtr;
+	mtr_t*		mtr;
 	dict_index_t*	clust_index = dict_table_get_first_index(index->table);
 
+	if(!index->last_sel_cur) {
+		dict_allocate_mem_intrinsic_cache(index);
+	}
+
+	mtr = &index->last_sel_cur->mtr;
+
 	/* Step-1: Build the select graph. */
 	if (direction == 0 && prebuilt->sel_graph == NULL) {
 		row_prebuild_sel_graph(prebuilt);
