WL#15524 Patches #9 and #10: --ndb-mgm-tls in ndbd and ndb_mgmd

When a node starts up, it might connect to a management server to
obtain the cluster configuration. The component that does this is
known as ConfigRetriever. This patch adds an instance of TlsKeyManager
to ConfigRetriever for use in MGM TLS by both sorts of ndbd processes:
the data node and the angel process.

The data node (but not the angel) also has a TlsKeyManager in the global
TransporterRegistry. In the data node, these two TlsKeyManagers are
initialized with the same node type and TLS search path, so they will
both use the same certificate.

In a data node, ConfigRetriever's MGM connection is the one that
will get turned in to a transporter. In the existing MTR test
tls_off_certs, the behavior changes so that the data nodes now have
TLS connections to the MGM node.

An NDB management server might also be a client of some other NDB
management server. This patch adds support for the --ndb-mgm-tls
option to ndb_mgmd, for setting its MGM TLS requirement level when
it acts as a client.

This patch extends the test testMgmd -n NdbdWithCertificate to run
with [MGM]RequireTls set to true.

Change-Id: I9ab68bef61b80a18edbf1375366c81ed874d7026

Author: jdduncan

mysql-test/suite/ndb_tls/tls_off_certs.test

@@ -3,13 +3,10 @@
 
 # Data node certs exist and are visible in ndbinfo
 
-# Expect 2 node certificates.
-# ndbinfo is aware of DB certs that belong to each data node, but it
-# is not aware of any API or MGM certs because no data node has a TLS
-# connection to an MGM or API node.
+# Expect 3 node certificates, for DB, DB, and MGM.
 #
 SELECT * FROM ndbinfo.certificates order by Node_id;
 
-# Expect all connections unencrypted 
+# Expect encrypted connections to MGMD (node 3) only
 SELECT node_id, remote_node_id, encrypted from ndbinfo.transporters
 WHERE status = 'CONNECTED' ORDER BY node_id, remote_node_id;

storage/ndb/include/mgmcommon/ConfigRetriever.hpp

@@ -28,6 +28,7 @@
 #include <ndb_types.h>
 #include <mgmapi.h>
 #include "mgmcommon/NdbMgm.hpp"
+#include "util/TlsKeyManager.hpp"
 #include <BaseString.hpp>
 
 /**
@@ -42,6 +43,9 @@ class ConfigRetriever {
                   int timeout_ms = 30000);
   ~ConfigRetriever();
 
+  /* Initialize a built-in TlsKeyManager to use for MGM TLS. */
+  void init_mgm_tls(const char * tls_search_path, Node::Type, int req_level);
+
   int do_connect(int no_retries, int retry_delay_in_seconds, int verbose);
   int disconnect();
   bool is_connected();
@@ -98,7 +102,9 @@ class ConfigRetriever {
   void end_session(bool end) { m_end_session= end; }
 
   Uint32 get_configuration_nodeid() const;
+  ssl_ctx_st * ssl_ctx() const { return m_tlsKeyManager.ctx(); }
 private:
+  TlsKeyManager m_tlsKeyManager;
   BaseString errorString;
   enum ErrorType {
     CR_NO_ERROR = 0,
@@ -113,6 +119,7 @@ class ConfigRetriever {
   Uint32 m_version;
   ndb_mgm_node_type m_node_type;
   NdbMgmHandle m_handle;
+  int m_tls_req_level{0};
 };
 
 #endif

storage/ndb/src/common/mgmcommon/ConfigRetriever.cpp

@@ -110,6 +110,16 @@ ConfigRetriever::~ConfigRetriever()
   DBUG_VOID_RETURN;
 }
 
+void
+ConfigRetriever::init_mgm_tls(const char * tls_search_path,
+                              Node::Type node_type,
+                              int mgm_tls_req_level)
+{
+  m_tlsKeyManager.init_mgm_client(tls_search_path, node_type);
+  ndb_mgm_set_ssl_ctx(m_handle, m_tlsKeyManager.ctx());
+  m_tls_req_level = mgm_tls_req_level;
+}
+
 Uint32 
 ConfigRetriever::get_configuration_nodeid() const
 {
@@ -138,7 +148,8 @@ int
 ConfigRetriever::do_connect(int no_retries,
 			    int retry_delay_in_seconds, int verbose)
 {
-  if (ndb_mgm_connect(m_handle, no_retries, retry_delay_in_seconds, verbose) == 0)
+  if (ndb_mgm_connect_tls(m_handle, no_retries, retry_delay_in_seconds,
+                          verbose, m_tls_req_level) == 0)
   {
     return 0;
   }
@@ -486,7 +497,7 @@ ConfigRetriever::allocNodeId(int no_retries, int retry_delay_in_seconds,
   while (1)
   {
     if (ndb_mgm_is_connected(m_handle) == 1 ||
-        ndb_mgm_connect(m_handle, 0, 0, verbose) == 0)
+        ndb_mgm_connect_tls(m_handle, 0, 0, verbose, m_tls_req_level) == 0)
     {
       int res =
         ndb_mgm_alloc_nodeid(m_handle, m_version, m_node_type,
@@ -496,8 +507,9 @@ ConfigRetriever::allocNodeId(int no_retries, int retry_delay_in_seconds,
     }
 
     error = ndb_mgm_get_latest_error(m_handle);
-    if (no_retries == 0 ||                        /* No more retries */
-        error == NDB_MGM_ALLOCID_CONFIG_MISMATCH) /* Fatal error */
+    if (no_retries == 0 ||                           /* No more retries */
+        error == NDB_MGM_ALLOCID_CONFIG_MISMATCH ||
+        error == NDB_MGM_AUTH_REQUIRES_TLS)          /* Fatal errors */
     {
       break;
     }

storage/ndb/src/kernel/angel.cpp

@@ -36,7 +36,7 @@
 #include <portlib/ndb_daemon.h>
 #include <portlib/NdbSleep.h>
 #include <portlib/NdbDir.hpp>
-
+#include "util/TlsKeyManager.hpp"
 #include <ConfigRetriever.hpp>
 
 #include <NdbTCP.h>
@@ -212,7 +212,8 @@ static void
 reportShutdown(const ndb_mgm_configuration* config,
                NodeId nodeid, int error_exit,
                bool restart, bool nostart, bool initial,
-               Uint32 error, Uint32 signum, Uint32 sphase)
+               Uint32 error, Uint32 signum, Uint32 sphase,
+               ssl_ctx_st * tls, int tls_req_level)
 {
   // Only allow "initial" and "nostart" to be set if "restart" is set
   assert(restart ||
@@ -279,8 +280,9 @@ reportShutdown(const ndb_mgm_configuration* config,
       continue;
     }
 
+    ndb_mgm_set_ssl_ctx(h, tls);
     if (ndb_mgm_set_connectstring(h, connect_str.c_str()) ||
-        ndb_mgm_connect(h, 1, 0, 0) ||
+        ndb_mgm_connect_tls(h, 1, 0, 0, tls_req_level) ||
         ndb_mgm_report_event(h, theData, length))
     {
       g_eventLogger->warning("Unable to report shutdown reason "
@@ -613,7 +615,9 @@ angel_run(const char* progname,
           bool no_start,
           bool daemon,
           int connnect_retries,
-          int connect_delay)
+          int connect_delay,
+          const char * tls_search_path,
+          int mgm_tls_level)
 {
   ConfigRetriever retriever(connect_str,
                             force_nodeid,
@@ -627,6 +631,8 @@ angel_run(const char* progname,
     angel_exit(1);
   }
 
+  retriever.init_mgm_tls(tls_search_path, Node::Type::DB, mgm_tls_level);
+
   const int verbose = 1;
   if (retriever.do_connect(connnect_retries, connect_delay, verbose) != 0)
   {
@@ -879,7 +885,8 @@ angel_run(const char* progname,
       case NRT_Default:
         g_eventLogger->info("Angel shutting down");
         reportShutdown(config.get(), nodeid, 0, 0, false, false,
-                       child_error, child_signal, child_sphase);
+                       child_error, child_signal, child_sphase,
+                       retriever.ssl_ctx(), mgm_tls_level);
         angel_exit(0);
         break;
       case NRT_NoStart_Restart:
@@ -903,7 +910,8 @@ angel_run(const char* progname,
            */
           reportShutdown(config.get(), nodeid,
                          error_exit, 0, false, false,
-                         child_error, child_signal, child_sphase);
+                         child_error, child_signal, child_sphase,
+                         retriever.ssl_ctx(), mgm_tls_level);
           angel_exit(0);
         }
         [[fallthrough]];
@@ -932,7 +940,8 @@ angel_run(const char* progname,
          */
         reportShutdown(config.get(), nodeid,
                        error_exit, 0, false, false,
-                       child_error, child_signal, child_sphase);
+                       child_error, child_signal, child_sphase,
+                       retriever.ssl_ctx(), mgm_tls_level);
         angel_exit(0);
       }
       else
@@ -957,7 +966,8 @@ angel_run(const char* progname,
                              "not restarting again", failed_startups_counter);
         reportShutdown(config.get(), nodeid,
                        error_exit, 0, false, false,
-                       child_error, child_signal, child_sphase);
+                       child_error, child_signal, child_sphase,
+                       retriever.ssl_ctx(), mgm_tls_level);
         angel_exit(0);
       }
       g_eventLogger->info("Angel detected startup failure, count: %u",
@@ -975,7 +985,8 @@ angel_run(const char* progname,
                    error_exit, 1,
                    no_start,
                    initial,
-                   child_error, child_signal, child_sphase);
+                   child_error, child_signal, child_sphase,
+                   retriever.ssl_ctx(), mgm_tls_level);
     g_eventLogger->info("Child has terminated (pid %jd). "
                         "Angel restarting child process",
                         child_pid);

storage/ndb/src/kernel/angel.hpp

@@ -39,7 +39,9 @@ angel_run(const char* progname,
           bool no_start,
           bool daemon,
           int connnect_retries,
-          int connect_delay);
+          int connect_delay,
+          const char * tls_search_path,
+          int mgm_tls_level);
 
 void
 angel_stop(void);

storage/ndb/src/kernel/main.cpp

@@ -82,6 +82,7 @@ static struct my_option my_long_options[] =
   NdbStdOpt::connect_retry_delay, //used
   NdbStdOpt::connect_retries, // used
   NdbStdOpt::tls_search_path,
+  NdbStdOpt::mgm_tls,
   NDB_STD_OPT_DEBUG
   { "core-file", NDB_OPT_NOSHORT, "Write core on errors.",\
     &opt_core, nullptr, nullptr, GET_BOOL, NO_ARG,
@@ -271,7 +272,7 @@ real_main(int argc, char** argv)
              opt_ndb_connectstring, opt_ndb_nodeid, opt_bind_address,
              opt_no_start, opt_initial, opt_initialstart,
              opt_allocated_nodeid, opt_connect_retries, opt_connect_retry_delay,
-             opt_logbuffer_size, opt_tls_search_path);
+             opt_logbuffer_size, opt_tls_search_path, opt_mgm_tls);
   }
 
   /**
@@ -290,7 +291,9 @@ real_main(int argc, char** argv)
             opt_no_start,
             opt_daemon,
             opt_connect_retries,
-            opt_connect_retry_delay);
+            opt_connect_retry_delay,
+            opt_tls_search_path,
+            opt_mgm_tls);
 
   return 1; // Never reached
 }

storage/ndb/src/kernel/ndbd.cpp

@@ -44,7 +44,6 @@
 
 #include <TransporterRegistry.hpp>
 
-#include <ConfigRetriever.hpp>
 #include <LogLevel.hpp>
 
 #if defined NDB_SOLARIS
@@ -1034,7 +1033,7 @@ ndbd_run(bool foreground, int report_fd,
          const char* connect_str, int force_nodeid, const char* bind_address,
          bool no_start, bool initial, bool initialstart,
          unsigned allocated_nodeid, int connect_retries, int connect_delay,
-         size_t logbuffer_size, const char * tls_search_path)
+         size_t logbuffer_size, const char * tls_search_path, int mgm_tls_req)
 {
   log_memusage("ndbd_run");
   LogBuffer* logBuf = new LogBuffer(logbuffer_size);
@@ -1151,9 +1150,9 @@ ndbd_run(bool foreground, int report_fd,
     already assigned the nodeid, either by the operator or by the angel
     process.
   */
-  theConfig->fetch_configuration(connect_str, force_nodeid, bind_address,
-                                 allocated_nodeid, connect_retries,
-                                 connect_delay);
+  theConfig->fetch_configuration(connect_str, force_nodeid,
+      bind_address, allocated_nodeid, connect_retries, connect_delay,
+      tls_search_path, opt_mgm_tls);
 
   /**
     Set the NDB DataDir, this is where we will locate log files and data
@@ -1369,8 +1368,7 @@ ndbd_run(bool foreground, int report_fd,
     ndbd_exit(-1);
   }
   // Re-use the mgm handle as a transporter
-  if(!globalTransporterRegistry.connect_client(
-		 theConfig->get_config_retriever()->get_mgmHandlePtr()))
+  if(!globalTransporterRegistry.connect_client(theConfig->get_mgm_handle_ptr()))
       ERROR_SET(fatal, NDBD_EXIT_CONNECTION_SETUP_FAILED,
                 "Failed to convert mgm connection to a transporter",
                 __FILE__);

storage/ndb/src/kernel/ndbd.hpp

@@ -33,7 +33,7 @@ ndbd_run(bool foreground, int report_fd,
          const char* connect_str, int force_nodeid, const char* bind_address,
          bool no_start, bool initial, bool initialstart,
          unsigned allocated_nodeid, int connect_retries, int connect_delay,
-         size_t logbuffer_size, const char * tls_search_path);
+         size_t logbuffer_size, const char * tls_search_path, int mgm_tls_req);
 
 enum NdbShutdownType {
   NST_Normal,

storage/ndb/src/kernel/vm/Configuration.cpp

@@ -132,7 +132,8 @@ Configuration::fetch_configuration(const char* _connect_string,
                                    int force_nodeid,
                                    const char* _bind_address,
                                    NodeId allocated_nodeid,
-                                   int connect_retries, int connect_delay)
+                                   int connect_retries, int connect_delay,
+                                   const char * tls_search_path, int mgm_tls)
 {
   /**
    * Fetch configuration from management server
@@ -159,6 +160,8 @@ Configuration::fetch_configuration(const char* _connect_string,
 	      m_config_retriever->getErrorString());
   }
 
+  m_config_retriever->init_mgm_tls(tls_search_path, Node::Type::DB, mgm_tls);
+
   if(m_config_retriever->do_connect(connect_retries, connect_delay, 1) == -1){
     const char * s = m_config_retriever->getErrorString();
     if(s == 0)
@@ -1656,5 +1659,11 @@ Configuration::initThreadArray()
   NdbMutex_Unlock(threadIdMutex);
 }
 
+NdbMgmHandle *
+Configuration::get_mgm_handle_ptr()
+{
+  return m_config_retriever->get_mgmHandlePtr();
+}
+
 template class Vector<struct ThreadInfo>;
 

storage/ndb/src/kernel/vm/Configuration.hpp

@@ -75,7 +75,9 @@ class Configuration {
   void fetch_configuration(const char* _connect_string, int force_nodeid,
                            const char* _bind_adress,
                            NodeId allocated_nodeid,
-                           int connect_retries, int connect_delay);
+                           int connect_retries, int connect_delay,
+                           const char * tls_search_path, int mgm_tls_level);
+
   void setupConfiguration();
   void closeConfiguration(bool end_session= true);
 
@@ -149,6 +151,7 @@ class Configuration {
   const ndb_mgm_configuration_iterator * getOwnConfigIterator() const;
 
   ConfigRetriever* get_config_retriever() { return m_config_retriever; }
+  NdbMgmHandle * get_mgm_handle_ptr();
 
   class LogLevel * m_logLevel;
   ndb_mgm_configuration_iterator * getClusterConfigIterator() const;

storage/ndb/src/mgmsrv/ConfigManager.cpp

@@ -2230,6 +2230,9 @@ ConfigManager::fetch_config(void)
 {
   DBUG_ENTER("ConfigManager::fetch_config");
 
+  m_config_retriever.init_mgm_tls(m_opts.tls_search_path,
+                                  Node::Type::MGMD, m_opts.mgm_tls);
+
   while(true)
   {
     /* Loop until config loaded from other mgmd(s) */

storage/ndb/src/mgmsrv/MgmtSrvr.cpp

@@ -255,6 +255,7 @@ MgmtSrvr::MgmtSrvr(const MgmtOpts& opts) :
   _ownReference(0),
   m_config_manager(NULL),
   m_tls_search_path(opts.tls_search_path),
+  m_client_tls_req(opts.mgm_tls),
   m_need_restart(false),
   theFacade(NULL),
   _isStopThread(false),

storage/ndb/src/mgmsrv/MgmtSrvr.hpp

@@ -116,6 +116,7 @@ class MgmtSrvr : private ConfigSubscriber, public trp_client {
     int initial;
     NodeBitmask nowait_nodes;
     const char* tls_search_path;
+    int mgm_tls;
 
     MgmtOpts() : configdir(MYSQLCLUSTERDIR),
                  tls_search_path(NDB_TLS_SEARCH_PATH) {}
@@ -458,7 +459,8 @@ class MgmtSrvr : private ConfigSubscriber, public trp_client {
 
   const char * m_tls_search_path { nullptr };
 
-  bool m_require_tls { false };
+  int m_client_tls_req;         // TLS requirement level as MGM client ...
+  bool m_require_tls { false }; // ... and as MGM server.
   bool m_require_cert { false };
 
   bool m_need_restart;

storage/ndb/src/mgmsrv/main.cpp

@@ -118,6 +118,7 @@ static struct my_option my_long_options[] =
   NdbStdOpt::mgmd_host,
   NdbStdOpt::connectstring,
   NdbStdOpt::tls_search_path,
+  NdbStdOpt::mgm_tls,
   NDB_STD_OPT_DEBUG
   { "config-file", 'f', "Specify cluster configuration file",
     &opts.config_filename, nullptr, nullptr, GET_STR, REQUIRED_ARG,
@@ -464,6 +465,7 @@ static int mgmd_main(int argc, char** argv)
   }
 
   opts.tls_search_path = opt_tls_search_path;
+  opts.mgm_tls = opt_mgm_tls;
 
   /* Setup use of event logger */
   g_eventLogger->setCategory(opt_logname);