WL#15130 Socket-level TLS patch mysql#2: SSL Socket Table

The SSL socket table is a big table of pointers to SSL objects,
indexed by socket number.

When an NdbSocket is initialized from an ndb_socket_t, it
will fetch any saved SSL * associated with the socket from the
SSL socket table.

If SSL_TABLE_ABORT_ON_HIT in ssl_socket_table.h is set to 1,
debug builds will abort when an SSL pointer is found but not
expected. This is a intended as a tool to help find and fix
code that might lose the association between a socket and its
SSL.

The table is protected by a read-write lock. As the code base
evolves toward use of NdbSocket with correct life cycles,
the contention on the lock should become less and less. Once
SecureSockets are correctly used everywhere, the table can be
disabled.

Change-Id: Ibb9b858146b8d983ca99f05b61ba0abee11b3ee6

Author: jdduncan

storage/ndb/include/util/NdbSocket.h

@@ -28,6 +28,7 @@
 #include "portlib/ndb_socket.h"
 #include "portlib/ndb_socket_poller.h"
 #include "portlib/NdbMutex.h"
+#include "util/ssl_socket_table.h"
 
 static constexpr int TLS_BUSY_TRY_AGAIN = -2;
 
@@ -36,8 +37,8 @@ class NdbSocket {
   enum class From { New, Existing };
 
   NdbSocket() = default;
-  NdbSocket(ndb_socket_t ndbsocket, From /*fromType*/) {
-    // ssl = socket_table_get_ssl(ndbsocket.s, (fromType == From::Existing));
+  NdbSocket(ndb_socket_t ndbsocket, From fromType) {
+    ssl = socket_table_get_ssl(ndbsocket.s, (fromType == From::Existing));
     init_from_native(ndbsocket.s);
   }
   /* NdbSockets cannot be copied, except using NdbSocket::transfer() */
@@ -199,7 +200,7 @@ class NdbSocket {
 
 inline
 void NdbSocket::init_from_new(ndb_socket_t ndbsocket) {
-  // assert(! socket_table_get_ssl(ndbsocket.s, false));
+  assert(! socket_table_get_ssl(ndbsocket.s, false));
   init_from_native(ndbsocket.s);
 }
 

storage/ndb/include/util/ssl_socket_table.h

@@ -0,0 +1,67 @@
+/*
+   Copyright (c) 2022, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+*/
+#ifndef NDB_UTIL_SSL_SOCKET_TABLE_H
+#define NDB_UTIL_SSL_SOCKET_TABLE_H
+
+/* The implementation uses a static-allocated table of size
+   NDB_SSL_FIXED_TABLE_SIZE, plus a dynamic unordered map for
+   descriptor values larger than NDB_SSL_FIXED_TABLE_SIZE.
+
+   In a data node, dynamic allocation is not allowed, but we have
+   full understanding of the code base. NDB_SSL_FIXED_TABLE_SIZE
+   must be big enough so that all lookups in the data node use the
+   fixed table.
+
+   In an API node, dynamic allocation is okay, but the application-level
+   code could require any arbitrary number of file descriptors. In this
+   case the overflow table may be used.
+*/
+#define NDB_SSL_FIXED_TABLE_SIZE 4192
+
+/* Enable or disable the socket table with 0 or 1 here */
+#define NDB_USE_SSL_SOCKET_TABLE 1
+
+#if NDB_USE_SSL_SOCKET_TABLE
+
+/* Set SSL for socket */
+void socket_table_set_ssl(socket_t, struct ssl_st *);
+
+/* Clear stored SSL for socket. */
+void socket_table_clear_ssl(socket_t);
+
+/* Get SSL associated with socket, or nullptr if none.
+   If an SSL is found when expected is false, this will abort in debug builds.
+*/
+struct ssl_st * socket_table_get_ssl(socket_t, bool expected);
+
+#else
+
+/* Socket table is disabled */
+#define socket_table_set_ssl(A, B)
+#define socket_table_clear_ssl(A)
+#define socket_table_get_ssl(A, ...) nullptr
+
+#endif   /* NDB_USE_SSL_SOCKET_TABLE */
+
+#endif   /* NDB_UTIL_SSL_SOCKET_TABLE_H */

storage/ndb/src/common/util/CMakeLists.txt

@@ -69,6 +69,7 @@ ADD_CONVENIENCE_LIBRARY(ndbgeneral
   parse_mask.cpp
   random.cpp
   require.cpp
+  ssl_socket_table.cpp
   socket_io.cpp
   uucode.cpp
   version.cpp

storage/ndb/src/common/util/NdbSocket.cpp

@@ -127,7 +127,7 @@ bool NdbSocket::associate(SSL * new_ssl)
 {
   if(ssl) return false; // already associated
   if(! SSL_set_fd(new_ssl, s.s)) return false;
-  // if(! socket_table_set_ssl(s.s, new_ssl)) return false;
+  socket_table_set_ssl(s.s, new_ssl);
   ssl = new_ssl;
   return true;
 }
@@ -186,7 +186,7 @@ bool NdbSocket::disable_locking() {
 void NdbSocket::ssl_close() {
   set_nonblocking(false);  // set to blocking
   SSL_shutdown(ssl);       // wait for close
-  // socket_table_clear_ssl(s.s);
+  socket_table_clear_ssl(s.s);
   SSL_free(ssl);
 }
 

storage/ndb/src/common/util/ssl_socket_table.cpp

@@ -0,0 +1,112 @@
+/*
+   Copyright (c) 2022, Oracle and/or its affiliates.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License, version 2.0,
+   as published by the Free Software Foundation.
+
+   This program is also distributed with certain software (including
+   but not limited to OpenSSL) that is licensed under separate terms,
+   as designated in a particular file or component or in included license
+   documentation.  The authors of MySQL hereby grant you an additional
+   permission to link the program and your derivative works with the
+   separately licensed software that they have included with MySQL.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License, version 2.0, for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+*/
+#include <assert.h>
+#include <mutex>
+#include <shared_mutex>
+#include <unordered_map>
+
+#include "portlib/ndb_socket.h"
+#include "util/ssl_socket_table.h"
+
+#if NDB_USE_SSL_SOCKET_TABLE
+
+static constexpr int ssl_table_size = NDB_SSL_FIXED_TABLE_SIZE;
+
+static struct ssl_st * ssl_table[ssl_table_size];
+std::unordered_map<uint, struct ssl_st *> ssl_overflow_table;
+std::shared_mutex ssl_table_mutex;  // a read/write lock
+
+
+/* A Windows socket_t is an offset into a table of 32-bit values. So, the first
+   Unix fds are 0, 1, and 2; but the first Win32 fds are 0x0, 0x4, 0x8.
+
+   You could right-shift the Win32 fd two places (to divide it by four) in
+   calculating the index. This would make more efficient use of the space in
+   the fixed table, but with the overflow table available it is not strictly
+   necessary.
+*/
+#define socket_to_index(s) (uint)s
+
+
+void socket_table_set_ssl(socket_t s, struct ssl_st *ssl)
+{
+  assert(s != INVALID_SOCKET);
+  uint fd = socket_to_index(s);
+
+  std::unique_lock write_lock(ssl_table_mutex);     // unique lock
+
+  if(fd < ssl_table_size)
+  {
+    assert(ssl_table[fd] == nullptr);
+    ssl_table[fd] = ssl;
+  }
+  else
+  {
+    assert(ssl_overflow_table.count(fd) == 0);
+    ssl_overflow_table[fd] = ssl;
+  }
+}
+
+void socket_table_clear_ssl(socket_t s)
+{
+  assert(s != INVALID_SOCKET);
+  uint fd = socket_to_index(s);
+
+  std::unique_lock write_lock(ssl_table_mutex);     // unique lock
+
+  if(fd < ssl_table_size)
+  {
+    assert(ssl_table[fd]);
+    ssl_table[fd] = nullptr;
+  }
+  else
+  {
+    assert(ssl_overflow_table.count(fd) == 1);
+    ssl_overflow_table.erase(fd);
+  }
+}
+
+struct ssl_st * socket_table_get_ssl(socket_t s, bool expected [[maybe_unused]])
+{
+  struct ssl_st * ptr = nullptr;
+  if(s == INVALID_SOCKET) return ptr;
+  uint fd = socket_to_index(s);
+
+  std::shared_lock read_lock(ssl_table_mutex);      // shared lock
+
+  if(fd < ssl_table_size)
+  {
+    ptr = ssl_table[fd];
+  }
+  else
+  {
+    auto it = ssl_overflow_table.find(fd);
+    if(it != ssl_overflow_table.end()) ptr = it->second;
+  }
+
+  assert(expected || ! ptr);
+  return ptr;
+}
+
+#endif /* NDB_USE_SSL_SOCKET_TABLE */