Bug #18592390 QUERY TO I_S.TABLES AND I_S.COLUMNS LEADS TO HUGE MEMORY USAGE

Description: On an example MySQL instance with 28k empty
InnoDB tables, a specific query to information_schema.tables
and information_schema.columns leads to memory consumption
over 38GB RSS.

Analysis: In get_all_tables() call, we fill the I_S tables
from frm files and storage engine. As part of that process
we call make_table_name_list() and allocate memory for all
the 28k frm file names in the THD mem_root through
make_lex_string_root(). Since it has been called around
28k * 28k times there is a huge memory getting hogged in
THD mem_root. This causes the RSS to grow to 38GB.

Fix: As part of fix we are creating a temporary mem_root
in get_all_tables and passing it to fill_fiels(). There we
replace the THD mem_root with the temporary mem_root and
allocates the file names in temporary mem_root and frees
it once we fill the I_S tables in get_all_tables and
re-assign the original mem_root back to THD mem_root.

Note: Checked the massif out put with the fix now the memory growth is just around 580MB at peak.

Author: V S Murthy Sidagam

sql/sql_show.cc

@@ -404,13 +404,14 @@ bool mysqld_show_privileges(THD *thd)
 
 find_files_result
 find_files(THD *thd, List<LEX_STRING> *files, const char *db,
-           const char *path, const char *wild, bool dir)
+           const char *path, const char *wild, bool dir, MEM_ROOT *tmp_mem_root)
 {
   uint i;
   char *ext;
   MY_DIR *dirp;
   FILEINFO *file;
   LEX_STRING *file_name= 0;
+  MEM_ROOT **root_ptr= NULL, *old_root= NULL;
   uint file_name_len;
 #ifndef NO_EMBEDDED_ACCESS_CHECKS
   uint col_access=thd->col_access;
@@ -440,6 +441,13 @@ find_files(THD *thd, List<LEX_STRING> *files, const char *db,
     DBUG_RETURN(FIND_FILES_DIR);
   }
 
+  if (tmp_mem_root)
+  {
+    root_ptr= my_pthread_getspecific_ptr(MEM_ROOT**, THR_MALLOC);
+    old_root= *root_ptr;
+    *root_ptr= tmp_mem_root;
+  }
+
   for (i=0 ; i < (uint) dirp->number_off_files  ; i++)
   {
     char uname[NAME_LEN + 1];                   /* Unencoded name */
@@ -519,8 +527,11 @@ find_files(THD *thd, List<LEX_STRING> *files, const char *db,
         continue;
     }
 #endif
-    if (!(file_name= 
-          thd->make_lex_string(file_name, uname, file_name_len, TRUE)) ||
+    if (!(file_name= tmp_mem_root ?
+                     make_lex_string_root(tmp_mem_root, file_name, uname,
+                                          file_name_len, TRUE) :
+                     thd->make_lex_string(file_name, uname,
+                                          file_name_len, TRUE)) ||
         files->push_back(file_name))
     {
       my_dirend(dirp);
@@ -532,6 +543,9 @@ find_files(THD *thd, List<LEX_STRING> *files, const char *db,
 
   (void) ha_find_files(thd, db, path, wild, dir, files);
 
+  if (tmp_mem_root)
+    *root_ptr= old_root;
+
   DBUG_RETURN(FIND_FILES_OK);
 }
 
@@ -2882,7 +2896,7 @@ enum enum_schema_tables get_schema_table_idx(ST_SCHEMA_TABLE *schema_table)
 
 int make_db_list(THD *thd, List<LEX_STRING> *files,
                  LOOKUP_FIELD_VALUES *lookup_field_vals,
-                 bool *with_i_schema)
+                 bool *with_i_schema, MEM_ROOT *tmp_mem_root)
 {
   LEX_STRING *i_s_name_copy= 0;
   i_s_name_copy= thd->make_lex_string(i_s_name_copy,
@@ -2906,7 +2920,8 @@ int make_db_list(THD *thd, List<LEX_STRING> *files,
         return 1;
     }
     return (find_files(thd, files, NullS, mysql_data_home,
-                       lookup_field_vals->db_value.str, 1) != FIND_FILES_OK);
+                       lookup_field_vals->db_value.str, 1, tmp_mem_root) !=
+                      FIND_FILES_OK);
   }
 
 
@@ -2948,7 +2963,7 @@ int make_db_list(THD *thd, List<LEX_STRING> *files,
     return 1;
   *with_i_schema= 1;
   return (find_files(thd, files, NullS,
-                     mysql_data_home, NullS, 1) != FIND_FILES_OK);
+                     mysql_data_home, NullS, 1, tmp_mem_root) != FIND_FILES_OK);
 }
 
 
@@ -3056,7 +3071,8 @@ int schema_tables_add(THD *thd, List<LEX_STRING> *files, const char *wild)
 static int
 make_table_name_list(THD *thd, List<LEX_STRING> *table_names, LEX *lex,
                      LOOKUP_FIELD_VALUES *lookup_field_vals,
-                     bool with_i_schema, LEX_STRING *db_name)
+                     bool with_i_schema, LEX_STRING *db_name,
+                     MEM_ROOT *tmp_mem_root)
 {
   char path[FN_REFLEN + 1];
   build_table_filename(path, sizeof(path) - 1, db_name->str, "", "", 0);
@@ -3110,7 +3126,8 @@ make_table_name_list(THD *thd, List<LEX_STRING> *table_names, LEX *lex,
                               lookup_field_vals->table_value.str));
 
   find_files_result res= find_files(thd, table_names, db_name->str, path,
-                                    lookup_field_vals->table_value.str, 0);
+                                    lookup_field_vals->table_value.str, 0,
+                                    tmp_mem_root);
   if (res != FIND_FILES_OK)
   {
     /*
@@ -3776,6 +3793,9 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
   bool can_deadlock;
   DBUG_ENTER("get_all_tables");
 
+  MEM_ROOT tmp_mem_root;
+  init_sql_alloc(&tmp_mem_root, TABLE_ALLOC_BLOCK_SIZE, 0);
+
   /*
     In cases when SELECT from I_S table being filled by this call is
     part of statement which also uses other tables or is being executed
@@ -3867,7 +3887,7 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
     goto err;
   }
 
-  if (make_db_list(thd, &db_names, &lookup_field_vals, &with_i_schema))
+  if (make_db_list(thd, &db_names, &lookup_field_vals, &with_i_schema, &tmp_mem_root))
     goto err;
   it.rewind(); /* To get access to new elements in basis list */
   while ((db_name= it++))
@@ -3885,7 +3905,7 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
       List<LEX_STRING> table_names;
       int res= make_table_name_list(thd, &table_names, lex,
                                     &lookup_field_vals,
-                                    with_i_schema, db_name);
+                                    with_i_schema, db_name, &tmp_mem_root);
       if (res == 2)   /* Not fatal error, continue */
         continue;
       if (res)
@@ -3972,9 +3992,10 @@ int get_all_tables(THD *thd, TABLE_LIST *tables, COND *cond)
       with_i_schema= 0;
     }
   }
-
   error= 0;
 err:
+
+  free_root(&tmp_mem_root, MYF(0));
   thd->restore_backup_open_tables_state(&open_tables_state_backup);
 
   DBUG_RETURN(error);
@@ -4000,6 +4021,27 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond)
     Returning error status in this case leads to client hangup.
   */
 
+  /*
+   * A temporary class is created to free tmp_mem_root when we return from
+   * this function, since we have 'return' from this function from many
+   * places. This is just to avoid goto.
+   */
+  class free_tmp_mem_root
+  {
+  public:
+    free_tmp_mem_root()
+    {
+      init_sql_alloc(&tmp_mem_root, TABLE_ALLOC_BLOCK_SIZE, 0);
+    }
+    ~free_tmp_mem_root()
+    {
+      free_root(&tmp_mem_root, MYF(0));
+    }
+    MEM_ROOT tmp_mem_root;
+  };
+
+  free_tmp_mem_root dummy_member;
+
   LOOKUP_FIELD_VALUES lookup_field_vals;
   List<LEX_STRING> db_names;
   LEX_STRING *db_name;
@@ -4013,11 +4055,12 @@ int fill_schema_schemata(THD *thd, TABLE_LIST *tables, COND *cond)
 
   if (get_lookup_field_values(thd, cond, tables, &lookup_field_vals))
     DBUG_RETURN(0);
+
   DBUG_PRINT("INDEX VALUES",("db_name='%s', table_name='%s'",
                              lookup_field_vals.db_value.str,
                              lookup_field_vals.table_value.str));
   if (make_db_list(thd, &db_names, &lookup_field_vals,
-                   &with_i_schema))
+                   &with_i_schema, &dummy_member.tmp_mem_root))
     DBUG_RETURN(1);
 
   /*

sql/sql_show.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2005, 2015, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -82,7 +82,8 @@ enum find_files_result {
 #define IS_FILES_EXTRA               37
 
 find_files_result find_files(THD *thd, List<LEX_STRING> *files, const char *db,
-                             const char *path, const char *wild, bool dir);
+                             const char *path, const char *wild, bool dir,
+                             MEM_ROOT *tmp_mem_root);
 
 int store_create_info(THD *thd, TABLE_LIST *table_list, String *packet,
                       HA_CREATE_INFO  *create_info_arg, bool show_database);