Bug#35157953 MySQL Router stops accepting connections - 8.0.32

This patch:
- adds more logging info to help debug the issue
- replaces use of ENOENT with custom error code for scenario
  when the Router discovers there are no destinations available

Change-Id: I544973fdd01d1c8012abc353c5d1214444b1a43d

Author: Andrzej Religa

router/src/routing/src/classic_connect.cc

@@ -153,7 +153,8 @@ ConnectProcessor::init_destination() {
   } else {
     if (!last_ec_) {
       // no backends
-      last_ec_ = make_error_code(std::errc::no_such_file_or_directory);
+      log_debug("init_destination(): the destinations list is empty");
+      last_ec_ = make_error_code(DestinationsErrc::kNoDestinations);
     }
 
     stage(Stage::Error);
@@ -194,6 +195,9 @@ stdx::expected<Processor::Result, std::error_code> ConnectProcessor::resolve() {
       destination->hostname(), std::to_string(destination->port()));
 
   if (!resolve_res) {
+    log_debug("resolve(%s,%d) failed: %s:%s", destination->hostname().c_str(),
+              destination->port(), resolve_res.error().category().name(),
+              resolve_res.error().message().c_str());
     destination->connect_status(resolve_res.error());
 
     stage(Stage::NextDestination);
@@ -463,6 +467,10 @@ stdx::expected<Processor::Result, std::error_code> ConnectProcessor::connect() {
 
       return Result::SendableToServer;
     } else {
+      log_debug("connect(%s, %d) failed: %s:%s",
+                server_endpoint_.address().to_string().c_str(),
+                server_endpoint_.port(), connect_res.error().category().name(),
+                connect_res.error().message().c_str());
       connection()->connect_error_code(ec);
 
       stage(Stage::ConnectFinish);
@@ -651,7 +659,10 @@ stdx::expected<Processor::Result, std::error_code> ConnectProcessor::error() {
 
   connection()->connect_error_code(ec);
 
-  if (ec == std::errc::no_such_file_or_directory) {
+  log_debug("ConnectProcessor::error(): %s:%s", ec.category().name(),
+            ec.message().c_str());
+
+  if (ec == DestinationsErrc::kNoDestinations) {
     log_error("no backend available to connect to");
   } else {
     log_fatal_error_code("connecting to backend failed", ec);
@@ -664,7 +675,7 @@ stdx::expected<Processor::Result, std::error_code> ConnectProcessor::error() {
     //
     // don't retry as router may run into an infinite loop.
     ConnectionPoolComponent::get_instance().clear();
-  } else if (ec == std::errc::no_such_file_or_directory &&
+  } else if (ec == DestinationsErrc::kNoDestinations &&
              connection()->get_destination_id().empty()) {
     // if there are no destinations for a fresh connect, close the
     // acceptor-ports

router/src/routing/src/classic_connect.h

@@ -42,7 +42,14 @@ class ConnectProcessor : public Processor {
         io_ctx_{conn->socket_splicer()->client_conn().connection()->io_ctx()},
         destinations_{conn->current_destinations()},
         destinations_it_{destinations_.begin()},
-        on_error_(std::move(on_error)) {}
+        on_error_(std::move(on_error)) {
+    // this is needed to shut down accepting port with next-available strategy
+    // despite there are destinations available
+    if (conn->destinations()->get_strategy() ==
+        routing::RoutingStrategy::kNextAvailable) {
+      last_ec_ = make_error_code(DestinationsErrc::kNoDestinations);
+    }
+  }
 
   using server_protocol_type = net::ip::tcp;
 
@@ -94,8 +101,7 @@ class ConnectProcessor : public Processor {
   net::ip::tcp::resolver::results_type endpoints_;
   net::ip::tcp::resolver::results_type::iterator endpoints_it_;
 
-  std::error_code last_ec_{
-      make_error_code(std::errc::no_such_file_or_directory)};
+  std::error_code last_ec_{make_error_code(DestinationsErrc::kNotSet)};
 
   std::function<void(const classic_protocol::message::server::Error &err)>
       on_error_;

router/src/routing/src/connection.cc

@@ -47,7 +47,7 @@ stdx::expected<void, std::error_code> ConnectorBase::init_destination() {
     // no backends
     log_warning("%d: no connectable destinations :(", __LINE__);
     return stdx::make_unexpected(
-        make_error_code(std::errc::no_such_file_or_directory));
+        make_error_code(DestinationsErrc::kNoDestinations));
   }
 }
 

router/src/routing/src/connection.h

@@ -219,8 +219,7 @@ class ConnectorBase {
   net::ip::tcp::resolver::results_type endpoints_;
   net::ip::tcp::resolver::results_type::iterator endpoints_it_;
 
-  std::error_code last_ec_{
-      make_error_code(std::errc::no_such_file_or_directory)};
+  std::error_code last_ec_{make_error_code(DestinationsErrc::kNotSet)};
 
   Function func_{Function::kInitDestination};
 

router/src/routing/src/dest_first_available.h

@@ -40,6 +40,10 @@ class DestFirstAvailable final : public RouteDestination {
   // mark index as invalid
   void mark_ndx_invalid(size_t ndx) noexcept { valid_ndx_ = ndx + 1; }
 
+  routing::RoutingStrategy get_strategy() override {
+    return routing::RoutingStrategy::kFirstAvailable;
+  }
+
  private:
   size_t valid_ndx_{};
 };

router/src/routing/src/dest_metadata_cache.cc

@@ -502,8 +502,12 @@ std::optional<Destinations> DestMetadataCacheGroup::refresh_destinations(
       auto const *primary_member = dynamic_cast<MetadataCacheDestination *>(
           previous_dests.begin()->get());
 
-      if (primary_member->last_error_code() ==
-          make_error_condition(std::errc::timed_out)) {
+      const auto err = primary_member->last_error_code();
+      log_debug("refresh_destinations(): %s:%s", err.category().name(),
+                err.message().c_str());
+
+      if (err == make_error_condition(std::errc::timed_out) ||
+          err == make_error_condition(std::errc::no_such_file_or_directory)) {
         return std::nullopt;
       }
 

router/src/routing/src/dest_metadata_cache.h

@@ -113,6 +113,8 @@ class DestMetadataCacheGroup final
     cache_api()->handle_sockets_acceptors_on_md_refresh();
   }
 
+  routing::RoutingStrategy get_strategy() override { return routing_strategy_; }
+
  private:
   /** @brief The Metadata Cache to use
    *

router/src/routing/src/dest_next_available.h

@@ -44,6 +44,10 @@ class DestNextAvailable final : public RouteDestination {
     }
   }
 
+  routing::RoutingStrategy get_strategy() override {
+    return routing::RoutingStrategy::kNextAvailable;
+  }
+
  private:
   size_t valid_ndx_{};
 };

router/src/routing/src/dest_round_robin.h

@@ -52,6 +52,10 @@ class DestRoundRobin : public RouteDestination {
 
   Destinations destinations() override;
 
+  routing::RoutingStrategy get_strategy() override {
+    return routing::RoutingStrategy::kRoundRobin;
+  }
+
  protected:
   // MUST take the RouteDestination Mutex
   size_t start_pos_{};

router/src/routing/src/destination.cc

@@ -157,3 +157,27 @@ std::optional<Destinations> RouteDestination::refresh_destinations(
     const Destinations &) {
   return std::nullopt;
 }
+
+const std::error_category &destinations_error_category() noexcept {
+  class destinations_category_impl : public std::error_category {
+   public:
+    const char *name() const noexcept override { return "destinations"; }
+    std::string message(int ev) const override {
+      switch (static_cast<DestinationsErrc>(ev)) {
+        case DestinationsErrc::kNotSet:
+          return "not set";
+        case DestinationsErrc::kNoDestinations:
+          return "no destinations";
+        default:
+          return "(unrecognized error)";
+      }
+    }
+  };
+
+  static destinations_category_impl instance;
+  return instance;
+}
+
+std::error_code make_error_code(DestinationsErrc e) {
+  return {static_cast<int>(e), destinations_error_category()};
+}

router/src/routing/src/destination.h

@@ -206,6 +206,10 @@ class RouteDestination : public DestinationNodesStateNotifier {
   RouteDestination &operator=(const RouteDestination &other) = delete;
   RouteDestination &operator=(RouteDestination &&other) = delete;
 
+  /** @brief Return our routing strategy
+   */
+  virtual routing::RoutingStrategy get_strategy() = 0;
+
   /** @brief Adds a destination
    *
    * Adds a destination using the given address and port number.
@@ -318,4 +322,16 @@ class RouteDestination : public DestinationNodesStateNotifier {
   Protocol::Type protocol_;
 };
 
+enum class DestinationsErrc {
+  kNotSet = 1,
+  kNoDestinations = 2,
+};
+
+namespace std {
+template <>
+struct is_error_code_enum<DestinationsErrc> : true_type {};
+}  // namespace std
+
+std::error_code make_error_code(DestinationsErrc);
+
 #endif  // ROUTING_DESTINATION_INCLUDED

router/src/routing/src/x_connection.cc

@@ -805,7 +805,7 @@ void MysqlRoutingXConnection::connect() {
     // close the server side.
     this->connector().socket().close();
 
-    if (ec == std::errc::no_such_file_or_directory) {
+    if (ec == DestinationsErrc::kNoDestinations) {
       MySQLRoutingComponent::get_instance()
           .api(context().get_id())
           .stop_socket_acceptors();