CDRIVER-699 obey cooldownTimeMS

Server Description And Monitoring Spec change: "After a single-threaded client
gets a network error trying to check a server, the client skips re-checking the
server until cooldownMS has passed."

Additional timing fixes in topology: don't quit with a timeout until *after*
checking if a suitable server was discovered, and don't sleep
minHeartbeatFrequencyMS unless needed.

Author: ajdavis

src/mongoc/mongoc-topology-private.h

@@ -26,6 +26,7 @@
 
 #define MONGOC_TOPOLOGY_MIN_HEARTBEAT_FREQUENCY_MS 500
 #define MONGOC_TOPOLOGY_SOCKET_CHECK_INTERVAL_MS 5000
+#define MONGOC_TOPOLOGY_COOLDOWN_MS 5000
 #define MONGOC_TOPOLOGY_SERVER_SELECTION_TIMEOUT_MS 30000
 #define MONGOC_TOPOLOGY_HEARTBEAT_FREQUENCY_MS_MULTI_THREADED 10000
 #define MONGOC_TOPOLOGY_HEARTBEAT_FREQUENCY_MS_SINGLE_THREADED 60000

src/mongoc/mongoc-topology-scanner-private.h

@@ -45,6 +45,7 @@ typedef struct mongoc_topology_scanner_node
    mongoc_stream_t                *stream;
    int64_t                         timestamp;
    int64_t                         last_used;
+   int64_t                         last_failed;
    bool                            has_auth;
    mongoc_host_list_t              host;
    struct addrinfo                *dns_results;
@@ -93,7 +94,8 @@ mongoc_topology_scanner_node_destroy (mongoc_topology_scanner_node_t *node,
 
 void
 mongoc_topology_scanner_start (mongoc_topology_scanner_t *ts,
-                               int32_t                    timeout_msec);
+                               int32_t timeout_msec,
+                               bool obey_cooldown);
 
 bool
 mongoc_topology_scanner_work (mongoc_topology_scanner_t *ts,

src/mongoc/mongoc-topology-scanner.c

@@ -29,6 +29,7 @@
 #include "mongoc-async-private.h"
 #include "mongoc-async-cmd-private.h"
 #include "utlist.h"
+#include "mongoc-topology-private.h"
 
 #undef MONGOC_LOG_DOMAIN
 #define MONGOC_LOG_DOMAIN "topology_scanner"
@@ -97,6 +98,7 @@ mongoc_topology_scanner_add (mongoc_topology_scanner_t *ts,
 
    node->id = id;
    node->ts = ts;
+   node->last_failed = -1;
 
    DL_APPEND(ts->nodes, node);
 
@@ -177,6 +179,7 @@ mongoc_topology_scanner_ismaster_handler (mongoc_async_cmd_result_t async_status
                                           bson_error_t             *error)
 {
    mongoc_topology_scanner_node_t *node;
+   int64_t now = bson_get_monotonic_time ();
 
    bson_return_if_fail (data);
 
@@ -189,9 +192,12 @@ mongoc_topology_scanner_ismaster_handler (mongoc_async_cmd_result_t async_status
        async_status == MONGOC_ASYNC_CMD_TIMEOUT) {
       mongoc_stream_failed (node->stream);
       node->stream = NULL;
+      node->last_failed = now;
+   } else {
+      node->last_failed = -1;
    }
 
-   node->last_used = bson_get_monotonic_time ();
+   node->last_used = now;
 
    node->ts->cb (node->id, ismaster_response, rtt_msec,
                  node->ts->cb_data, error);
@@ -372,28 +378,52 @@ mongoc_topology_scanner_node_setup (mongoc_topology_scanner_node_t *node)
  *      should be called once before calling mongoc_topology_scanner_work()
  *      repeatedly to complete the scan.
  *
+ *      If "obey_cooldown" is true, this is a single-threaded blocking scan
+ *      that must obey the Server Discovery And Monitoring Spec's cooldownMS:
+ *
+ *      "After a single-threaded client gets a network error trying to check
+ *      a server, the client skips re-checking the server until cooldownMS has
+ *      passed.
+ *
+ *      "This avoids spending connectTimeoutMS on each unavailable server
+ *      during each scan.
+ *
+ *      "This value MUST be 5000 ms, and it MUST NOT be configurable."
+ *
  *--------------------------------------------------------------------------
  */
 
 void
 mongoc_topology_scanner_start (mongoc_topology_scanner_t *ts,
-                               int32_t                timeout_msec)
+                               int32_t timeout_msec,
+                               bool obey_cooldown)
 {
    mongoc_topology_scanner_node_t *node, *tmp;
-
+   int64_t cooldown;
    bson_return_if_fail (ts);
 
    if (ts->in_progress) {
       return;
    }
 
+   if (obey_cooldown) {
+      cooldown = bson_get_monotonic_time ()
+                 - 1000 * MONGOC_TOPOLOGY_COOLDOWN_MS;
+   } else {
+      cooldown = INT64_MIN;
+   }
+
    DL_FOREACH_SAFE (ts->nodes, node, tmp)
    {
-      if (mongoc_topology_scanner_node_setup (node)) {
-         node->cmd = mongoc_async_cmd (ts->async, node->stream, ts->setup,
-                                       node->host.host, "admin", &ts->ismaster_cmd,
-                                       &mongoc_topology_scanner_ismaster_handler, node,
-                                       timeout_msec);
+      if (node->last_failed < cooldown) {
+         if (mongoc_topology_scanner_node_setup (node)) {
+            node->cmd = mongoc_async_cmd (
+               ts->async, node->stream, ts->setup,
+               node->host.host, "admin",
+               &ts->ismaster_cmd,
+               &mongoc_topology_scanner_ismaster_handler,
+               node, timeout_msec);
+         }
       }
    }
 }

src/mongoc/mongoc-topology.c

@@ -16,9 +16,7 @@
 
 #include "mongoc-error.h"
 #include "mongoc-topology-private.h"
-#include "mongoc-topology-scanner-private.h"
 #include "mongoc-uri-private.h"
-#include "mongoc-trace.h"
 
 #include "utlist.h"
 
@@ -267,6 +265,31 @@ _mongoc_topology_time_to_scan (mongoc_topology_t *topology) {
           topology->heartbeat_msec * 1000;
 }
 
+/*
+ *--------------------------------------------------------------------------
+ *
+ * _mongoc_topology_sleep_min_heartbeat --
+ *
+ *       Wait until we're allowed to rescan.
+ *
+ *       Server Discovery And Monitoring Spec: "If a client frequently
+ *       rechecks a server, it MUST wait at least minHeartbeatFrequencyMS
+ *       milliseconds  since the previous check to avoid pointless effort.
+ *       This value MUST be 500 ms, and it MUST NOT be configurable."
+ *
+ *--------------------------------------------------------------------------
+ */
+static void
+_mongoc_topology_sleep_min_heartbeat (mongoc_topology_t *topology) {
+   int64_t next_scan = topology->last_scan
+                       + MONGOC_TOPOLOGY_MIN_HEARTBEAT_FREQUENCY_MS * 1000;
+   int64_t sleep_usec = next_scan - bson_get_monotonic_time ();
+
+   if (sleep_usec > 0) {
+      usleep (sleep_usec);
+   }
+}
+
 /*
  *--------------------------------------------------------------------------
  *
@@ -317,7 +340,10 @@ _mongoc_topology_run_scanner (mongoc_topology_t *topology,
  */
 static void
 _mongoc_topology_do_blocking_scan (mongoc_topology_t *topology) {
-   mongoc_topology_scanner_start (topology->scanner, topology->timeout_msec);
+   mongoc_topology_scanner_start (topology->scanner,
+                                  topology->timeout_msec,
+                                  true);
+
    while (_mongoc_topology_run_scanner (topology, topology->timeout_msec)) {}
    topology->last_scan = bson_get_monotonic_time ();
 }
@@ -372,12 +398,6 @@ mongoc_topology_select (mongoc_topology_t         *topology,
 
       /* until we find a server or timeout */
       for (;;) {
-         /* error if we've timed out */
-         now = bson_get_monotonic_time();
-         if (now >= expire_at) {
-            goto TIMEOUT;
-         }
-
          /* attempt to select a server */
          selected_server = mongoc_topology_description_select(&topology->description,
                                                               optype,
@@ -389,9 +409,15 @@ mongoc_topology_select (mongoc_topology_t         *topology,
             return mongoc_server_description_new_copy(selected_server);
          }
 
+         /* error if we've timed out */
+         now = bson_get_monotonic_time();
+         if (now >= expire_at) {
+            goto TIMEOUT;
+         }
+
          /* rescan */
-         usleep (MONGOC_TOPOLOGY_MIN_HEARTBEAT_FREQUENCY_MS * 1000);
-         _mongoc_topology_do_blocking_scan(topology);
+         _mongoc_topology_sleep_min_heartbeat (topology);
+         _mongoc_topology_do_blocking_scan (topology);
       }
    }
 
@@ -628,7 +654,9 @@ void * _mongoc_topology_run_background (void *data)
 
          /* if we can start scanning, do so immediately */
          if (timeout <= 0) {
-            mongoc_topology_scanner_start (topology->scanner, topology->timeout_msec);
+            mongoc_topology_scanner_start (topology->scanner,
+                                           topology->timeout_msec,
+                                           false);
             break;
          } else {
             /* otherwise wait until someone:

tests/test-mongoc-topology-scanner.c

@@ -90,7 +90,7 @@ _test_topology_scanner(bool with_ssl)
    }
 
    for (i = 0; i < 3; i++) {
-      mongoc_topology_scanner_start (topology_scanner, TIMEOUT);
+      mongoc_topology_scanner_start (topology_scanner, TIMEOUT, false);
 
       more_to_do = mongoc_topology_scanner_work (topology_scanner, TIMEOUT);