CDRIVER-4192 Support retryable handshake network errors (#1141)

* Move legacy retryable reads test files into legacy subdirectory

* Move legacy retryable writes test files into legacy subdirectory

* Sync unified retryable reads test files with 08230607

* Sync unified retryable writes test files with 08230607

* Sync unified transactions test files with 08230607

* Bump default task timeout from 40 minutes to to 1 hour

* Address TSAN warnings in test_add_and_scan_failure

* Address -Wunused-const-variable for gHexCharPairs

* Improve error message when expected OP_MSG request is not received

* Improve assertion message for monotonic clock time comparison

* Address null pointer warnings by scan-build

* CDRIVER-4517 Sync unified retryable writes tests files with b9cdbbbe

* CDRIVER-4517 Sync unified retryable reads tests files with b9cdbbbe

* Declare _mongoc_cluster_stream_for_server as static

* Refactor _mongoc_bson_init_with_transient_txn_error -> _mongoc_add_transient_txn_error

* Separate aggregates with writes from mongoc_cluster_stream_for_reads

* Add _mongoc_error_is_auth

* Declare _mongoc_write_error_append_retryable_label in mongoc-error-private.h

* Remove always-true handshake_complete parameter from _handle_network_error

* Assert preconditions for mongoc_cluster_stream_for_server

* Permit empty arguments field for operation_list_collections

* Retry when encountering a network error establishing an initial connection to a server

* Update tests to account for retryable handshake network failures

* Add unified test runner support for listDatabaseNames

* Skip retryable reads tests that require unsupported optional helpers

Author: eramongodb

.evergreen/config.yml

@@ -492,10 +492,8 @@ def __init__(self, *args, **kwargs):
         elif self.coverage:
             self.add_tags('test-coverage')
             self.add_tags(self.version)
-            self.options['exec_timeout_secs'] = 3600
         elif self.sanitizer == "asan":
             self.add_tags('test-asan', self.version)
-            self.options['exec_timeout_secs'] = 3600
         elif self.sanitizer == "tsan":
             self.add_tags('tsan')
             self.add_tags(self.version)
@@ -855,15 +853,13 @@ def __init__(self, *args, **kwargs):
     NamedTask(
         'test-coverage-latest-server-dns',
         tags=['test-coverage'],
-        exec_timeout_secs=3600,
         commands=[func('debug-compile-coverage-notest-nosasl-openssl'),
                   bootstrap(TOPOLOGY='replica_set', AUTH='auth', SSL='ssl'),
                   run_tests(AUTH='auth', SSL='ssl', DNS='on'),
                   func('update codecov.io')]),
     NamedTask(
         'authentication-tests-memcheck',
         tags=['authentication-tests', 'valgrind'],
-        exec_timeout_seconds=3600,
         commands=[
             shell_mongoc("""
                 VALGRIND=ON DEBUG=ON CC='${CC}' MARCH='${MARCH}' SASL=AUTO \
@@ -1120,7 +1116,6 @@ def __init__(self, *args, **kwargs):
                 "unimplemented configuration for LoadBalancedTask")
 
         self.add_tags(self.version)
-        self.options['exec_timeout_secs'] = 3600
 
     # Return the task name.
     # Example: test-loadbalanced-asan-auth-openssl-latest

build/evergreen_config_lib/tasks.py

@@ -51,8 +51,8 @@
 config = OD([
     ('stepback', True),
     ('command_type', 'system'),
-    # 40 minute max except valgrind tasks, which get 2 hours.
-    ('exec_timeout_secs', 2400),
+    # 60 minute max except valgrind tasks, which get 2 hours.
+    ('exec_timeout_secs', 3600),
     ('functions', all_functions),
     ('pre', [
         OD([('func', 'fetch source')]),

build/generate-evergreen-config.py

@@ -32,7 +32,7 @@
  * to hex formatted ASCII. Performing two characters at a time roughly
  * reduces the number of operations by one-half.
  */
-static const uint16_t gHexCharPairs[] = {
+BSON_MAYBE_UNUSED static const uint16_t gHexCharPairs[] = {
 #if BSON_BYTE_ORDER == BSON_BIG_ENDIAN
    12336, 12337, 12338, 12339, 12340, 12341, 12342, 12343, 12344, 12345, 12385,
    12386, 12387, 12388, 12389, 12390, 12592, 12593, 12594, 12595, 12596, 12597,

src/libbson/src/bson/bson-oid.c

@@ -294,13 +294,8 @@ _make_cursor (mongoc_change_stream_t *stream)
       goto cleanup;
    }
 
-   server_stream =
-      mongoc_cluster_stream_for_reads (&stream->client->cluster,
-                                       stream->read_prefs,
-                                       cs,
-                                       &reply,
-                                       /* Not aggregate-with-write */ false,
-                                       &stream->err);
+   server_stream = mongoc_cluster_stream_for_reads (
+      &stream->client->cluster, stream->read_prefs, cs, &reply, &stream->err);
    if (!server_stream) {
       bson_destroy (&stream->err_doc);
       bson_copy_to (&reply, &stream->err_doc);

src/libmongoc/src/mongoc/mongoc-change-stream.c

@@ -1832,7 +1832,6 @@ _mongoc_client_retryable_read_command_with_stream (
                                           parts->read_prefs,
                                           parts->assembled.session,
                                           NULL,
-                                          /* Not aggregate-with-write */ false,
                                           &ignored_error);
 
       if (retry_server_stream && retry_server_stream->sd->max_wire_version >=
@@ -1923,12 +1922,7 @@ mongoc_client_command_simple (mongoc_client_t *client,
     * preference argument."
     */
    server_stream =
-      mongoc_cluster_stream_for_reads (cluster,
-                                       read_prefs,
-                                       NULL,
-                                       reply,
-                                       /* Not aggregate-with-write */ false,
-                                       error);
+      mongoc_cluster_stream_for_reads (cluster, read_prefs, NULL, reply, error);
 
    if (server_stream) {
       ret = _mongoc_client_command_with_stream (
@@ -2088,12 +2082,7 @@ _mongoc_client_command_with_opts (mongoc_client_t *client,
          mongoc_cluster_stream_for_writes (cluster, cs, reply_ptr, error);
    } else {
       server_stream =
-         mongoc_cluster_stream_for_reads (cluster,
-                                          prefs,
-                                          cs,
-                                          reply_ptr,
-                                          /* Not aggregate-with-write */ false,
-                                          error);
+         mongoc_cluster_stream_for_reads (cluster, prefs, cs, reply_ptr, error);
    }
 
    if (!server_stream) {

src/libmongoc/src/mongoc/mongoc-client.c

@@ -123,7 +123,6 @@ mongoc_cluster_stream_for_reads (mongoc_cluster_t *cluster,
                                  const mongoc_read_prefs_t *read_prefs,
                                  mongoc_client_session_t *cs,
                                  bson_t *reply,
-                                 bool is_aggr_with_write,
                                  bson_error_t *error);
 
 /**
@@ -143,6 +142,26 @@ mongoc_cluster_stream_for_writes (mongoc_cluster_t *cluster,
                                   bson_t *reply,
                                   bson_error_t *error);
 
+/**
+ * @brief Obtain a server stream appropriate for aggregate operations with
+ * writes on the cluster.
+ *
+ * Returns a new stream (that must be freed) or NULL and sets an error via
+ * `error`.
+ *
+ * @note The returned stream must be released via
+ * `mongoc_server_stream_cleanup`.
+ *
+ * @note May add nodes and/or update the cluster's topology.
+ */
+mongoc_server_stream_t *
+mongoc_cluster_stream_for_aggr_with_write (
+   mongoc_cluster_t *cluster,
+   const mongoc_read_prefs_t *read_prefs,
+   mongoc_client_session_t *cs,
+   bson_t *reply,
+   bson_error_t *error);
+
 /**
  * @brief Obtain a server stream associated with the cluster node associated
  * with the given server ID.