Bug#30242141 MYSQLD HANGS INDEFINITELY AT STARTUP IF IT IS UNABLE TO ACQUIRE GSL

PROBLEM
=======
When doing alter, mysqld holds the Global Schema Lock(GSL) for the entire
time.

Until cluster-7.6, when a mysqld tried to connect to the cluster, it would
try to acquire the GSL during setup, if GSL is already acquired by another
mysqld, it waits for a particular amount of seconds, specified by the
ndb-wait-setup config and then starts the mysqld with out NDB.

In 8.0, well before the server calls the hook, which preforms the wait for
ndb-wait-setup, it makes some update to the DD, as a part of that, it tries
to acquire mdl on the table. This gets sent down to the NDB plugin via
ndbcluster_notify_exclusive_mdl() and it is stuck trying to acquire the GSL
on the table. So, even the time specified in ndb-wait-setup expires, mysqld
will not start and keeps waiting for GSL.

FIX
===
Fix by delaying the registration of all handler callback functions that
takes GSL until NDB startup is initiated.

Change-Id: Ibcdc93663e1730ceb6abc022ad0804ce796f397a

Author: dineshsuryaprakash

mysql-test/suite/ndb/r/ndb_waiting_for_gsl.result

@@ -0,0 +1,7 @@
+CREATE TABLE t1 (a int) engine ndb;
+ALTER TABLE t1 ALGORITHM = COPY, ADD COLUMN b INT;;
+# restart
+ERROR HY000: Can't create destination table for copying alter table
+# restart
+drop table t1;
+include/assert_grep.inc [Checking if ndb-wait-setup timeout]

mysql-test/suite/ndb/t/ndb_waiting_for_gsl.cnf

@@ -0,0 +1,5 @@
+!include ../my.cnf
+
+[mysqld]
+
+ndb-wait-setup=60

mysql-test/suite/ndb/t/ndb_waiting_for_gsl.test

@@ -0,0 +1,40 @@
+--source include/have_debug.inc
+--source include/have_multi_ndb.inc
+--source have_ndb_error_insert.inc
+
+# Delay alter table infinitely using error insert
+--connection server1
+--let $ndb_desc_file= $MYSQLTEST_VARDIR/log/mysqld.2.1.err
+CREATE TABLE t1 (a int) engine ndb;
+# error insert 6217, will keep Dbdict::execALTER_TABLE_REQ() in an loop with a interval of 1 sec.
+# As the signal is not allowed to excuted, ALTER will be struck with holding GSL for some period of time.
+--exec $NDB_MGM -e'all error 6217' > /dev/null
+--send ALTER TABLE t1 ALGORITHM = COPY, ADD COLUMN b INT;
+
+
+# As GSL is held by alter in Server1, restart Server2 and check if ndb-wait-setup timeout occurs
+# Without patch, restart will hang for more than ndb-wait-setup
+--connection server2
+--disable_query_log
+call mtr.add_suppression("NDB: Tables not available after [0-9]* seconds. Consider increasing --ndb-wait-setup value");
+--enable_query_log
+--let $mysqld_name=mysqld.2.1
+--source include/restart_mysqld.inc
+
+# Reap for alter in Server1, and thereby release GSL
+--connection server1
+--error 1005
+--reap
+
+# Restart Server2 and check if it starts with NDB
+--connection server2
+--exec $NDB_MGM -e'all error 0' > /dev/null
+--let $mysqld_name=mysqld.2.1
+--source include/restart_mysqld.inc
+drop table t1;
+
+--let $assert_text= Checking if ndb-wait-setup timeout
+--let $assert_select=Tables not available after
+--let $assert_file= $ndb_desc_file
+--let $assert_count= 1
+--source include/assert_grep.inc

storage/ndb/plugin/ha_ndbcluster.cc

@@ -866,6 +866,12 @@ ulong opt_ndb_slave_conflict_role;
 static int handle_conflict_op_error(NdbTransaction *trans, const NdbError &err,
                                     const NdbOperation *op);
 
+static bool ndbcluster_notify_alter_table(THD *, const MDL_key *,
+                                          ha_notification_type);
+
+static bool ndbcluster_notify_exclusive_mdl(THD *, const MDL_key *,
+                                            ha_notification_type, bool *);
+
 static int handle_row_conflict(
     NDB_CONFLICT_FN_SHARE *cfn_share, const char *tab_name,
     const char *handling_type, const NdbRecord *key_rec,
@@ -10156,6 +10162,10 @@ int rename_table_impl(THD *thd, Ndb *ndb,
     }
   });
 
+  Thd_ndb *thd_ndb = get_thd_ndb(thd);
+  if (!thd_ndb->has_required_global_schema_lock("ha_ndbcluster::rename_table"))
+    return HA_ERR_NO_CONNECTION;
+
   NDBDICT *dict = ndb->getDictionary();
   NDBDICT::List index_list;
   if (my_strcasecmp(system_charset_info, new_dbname, old_dbname)) {
@@ -10206,7 +10216,6 @@ int rename_table_impl(THD *thd, Ndb *ndb,
   NDB_SHARE_KEY *new_key = NDB_SHARE::create_key(to);
   (void)NDB_SHARE::rename_share(share, new_key);
 
-  Thd_ndb *thd_ndb = get_thd_ndb(thd);
   Ndb_DDL_transaction_ctx *ddl_ctx = thd_ndb->get_ddl_transaction_ctx(false);
   const bool rollback_in_progress =
       (ddl_ctx != nullptr && ddl_ctx->rollback_in_progress());
@@ -10499,10 +10508,6 @@ int ha_ndbcluster::rename_table(const char *from, const char *to,
     return HA_WRONG_CREATE_OPTION;
   }
 
-  Thd_ndb *thd_ndb = get_thd_ndb(thd);
-  if (!thd_ndb->has_required_global_schema_lock("ha_ndbcluster::rename_table"))
-    return HA_ERR_NO_CONNECTION;
-
   // Open the table which is to be renamed(aka. the old)
   Ndb *ndb = get_ndb(thd);
   ndb->setDatabaseName(old_dbname);
@@ -10901,10 +10906,6 @@ int ha_ndbcluster::delete_table(const char *path, const dd::Table *) {
   }
 
   Thd_ndb *thd_ndb = get_thd_ndb(thd);
-  if (!thd_ndb->has_required_global_schema_lock(
-          "ha_ndbcluster::delete_table")) {
-    return HA_ERR_NO_CONNECTION;
-  }
 
   if (ndb_name_is_temp(m_tabname)) {
     const char *orig_table_name =
@@ -10930,6 +10931,11 @@ int ha_ndbcluster::delete_table(const char *path, const dd::Table *) {
       return 0;
     }
 
+    if (!thd_ndb->has_required_global_schema_lock(
+            "ha_ndbcluster::delete_table")) {
+      return HA_ERR_NO_CONNECTION;
+    }
+
     /* This the final phase of a copy alter. Delay the drop of the table with
        temp name until after commit so that when required, a rollback would be
        possible. Log it in the ddl_ctx and return. It will be dropped after
@@ -11885,6 +11891,8 @@ static int ndb_wait_setup_func(ulong max_wait) {
 */
 
 static int ndb_wait_setup_server_startup(void *) {
+  ndbcluster_hton->notify_alter_table = ndbcluster_notify_alter_table;
+  ndbcluster_hton->notify_exclusive_mdl = ndbcluster_notify_exclusive_mdl;
   // Signal components that server is started
   ndb_index_stat_thread.set_server_started();
   ndbcluster_binlog_set_server_started();
@@ -12227,9 +12235,9 @@ static int ndbcluster_init(void *handlerton_ptr) {
 
   hton->check_fk_column_compat = ndbcluster_check_fk_column_compat;
   hton->pre_dd_shutdown = ndbcluster_pre_dd_shutdown;
-  hton->notify_alter_table = ndbcluster_notify_alter_table;
-  hton->notify_exclusive_mdl = ndbcluster_notify_exclusive_mdl;
 
+  // notify_alter_table and notify_exclusive_mdl will be registered latter
+  // SO, that GSL will not be held unnecessary for non-ndb tables.
   hton->post_ddl = ndbcluster_post_ddl;
 
   // Initialize NdbApi

storage/ndb/plugin/ndb_schema_dist.cc

@@ -29,8 +29,10 @@
 #include <mutex>
 
 #include "my_dbug.h"
+#include "mysqld_error.h"
 #include "ndbapi/ndb_cluster_connection.hpp"
 #include "sql/query_options.h"  // OPTION_BIN_LOG
+#include "sql/sql_error.h"
 #include "sql/sql_thd_internal_api.h"
 #include "storage/ndb/plugin/ndb_anyvalue.h"
 #include "storage/ndb/plugin/ndb_dist_priv_util.h"
@@ -117,7 +119,8 @@ bool Ndb_schema_dist_client::prepare(const char *db, const char *tabname) {
       DBUG_EVALUATE_IF("ndb_schema_dist_not_ready_early", true, false)) {
     // The NDB_SHARE for mysql.ndb_schema hasn't been created or not setup
     // yet -> schema distribution is not ready
-    m_thd_ndb->push_warning("Schema distribution is not ready");
+    push_warning(m_thd, Sql_condition::SL_WARNING, ER_GET_ERRMSG,
+                 "Schema distribution is not ready");
     return false;
   }
 
@@ -247,7 +250,7 @@ Ndb_schema_dist_client::~Ndb_schema_dist_client() {
     NDB_SHARE::release_reference(m_share, "ndb_schema_dist_client");
   }
 
-  if (m_thd_ndb->is_slave_thread()) {
+  if (m_thd_ndb && m_thd_ndb->is_slave_thread()) {
     // Copy-out slave thread statistics
     // NOTE! This is just a "convenient place" to call this
     // function, it could be moved to "end of statement"(if there