CDRIVER-4048 support loadBalanced URI option (#808)

- Resyncs the URI options tests and the initial DNS seedlist discovery tests
- Add the loadBalanced URI option
- Validates the loadBalanced URI option after TXT records are applied
- Disable SRV polling if loadBalanced is enabled
- Add prose test to ensure SRV polling is disabled
- Consolidates DNS related tests under /initial_dns_seedlist_discovery/

Author: kevinAlbs

.evergreen/config.yml

@@ -754,6 +754,41 @@ functions:
         export IAM_AUTH_ECS_ACCOUNT=${iam_auth_ecs_account}
         export IAM_AUTH_ECS_SECRET_ACCESS_KEY=${iam_auth_ecs_secret_access_key}
         sh ./.evergreen/run-aws-tests.sh ${TESTCASE}
+  clone drivers-evergreen-tools:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        if [ ! -d "drivers-evergreen-tools" ]; then
+            git clone [email protected]:mongodb-labs/drivers-evergreen-tools.git --depth=1
+        fi
+  start load balancer:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        export DRIVERS_TOOLS=./drivers-evergreen-tools
+        export MONGODB_URI="${MONGODB_URI}"
+        bash $DRIVERS_TOOLS/.evergreen/run-load-balancer.sh start
+  - command: expansions.update
+    params:
+      file: lb-expansion.yml
+  stop load balancer:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        # Only run if a load balancer was started.
+        if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then
+            echo "OK - no load balancer running"
+            exit 0
+        fi
+        export DRIVERS_TOOLS=./drivers-evergreen-tools
+        export MONGODB_URI="foo" # TODO: DRIVERS-1833 remove this.
+        $DRIVERS_TOOLS/.evergreen/run-load-balancer.sh stop
 pre:
 - func: fetch source
 - func: windows fix
@@ -765,6 +800,7 @@ post:
 - func: upload mo artifacts
 - func: upload test results
 - func: cleanup
+- func: stop load balancer
 timeout:
 - func: backtrace
 tasks:
@@ -19123,6 +19159,28 @@ tasks:
       AUTH: noauth
       DNS: 'on'
       SSL: ssl
+- name: test-dns-loadbalanced-openssl
+  depends_on:
+    name: debug-compile-sasl-openssl
+  commands:
+  - func: fetch build
+    vars:
+      BUILD_NAME: debug-compile-sasl-openssl
+  - func: bootstrap mongo-orchestration
+    vars:
+      AUTH: noauth
+      SSL: ssl
+      TOPOLOGY: sharded_cluster
+      VERSION: latest
+  - func: clone drivers-evergreen-tools
+  - func: start load balancer
+    vars:
+      MONGODB_URI: mongodb://localhost:27017,localhost:27018
+  - func: run tests
+    vars:
+      AUTH: noauth
+      DNS: loadbalanced
+      SSL: ssl
 - name: test-dns-auth-openssl
   depends_on:
     name: debug-compile-sasl-openssl
@@ -26840,6 +26898,7 @@ buildvariants:
   - .4.0 .openssl !.nosasl .server
   - test-dns-openssl
   - test-dns-auth-openssl
+  - test-dns-loadbalanced-openssl
 - name: gcc54
   display_name: GCC 5.4 (Ubuntu 16.04)
   expansions:

.evergreen/run-tests.sh

@@ -49,12 +49,14 @@ if [ "$IPV4_ONLY" != "on" ]; then
    export MONGOC_CHECK_IPV6="on"
 fi
 
-if [ "$DNS" = "dns-auth" ]; then
-   export MONGOC_TEST_DNS=on
-   TEST_ARGS="$TEST_ARGS -l /initial_dns_auth/*"
-elif [ "$DNS" != "nodns" ]; then
-   export MONGOC_TEST_DNS=on
-   TEST_ARGS="$TEST_ARGS -l /initial_dns_seedlist_discovery*"
+# TODO (CDRIVER-4045): consolidate DNS tests into regular test tasks.
+if [ "$DNS" != "nodns" ]; then
+   TEST_ARGS="$TEST_ARGS -l /initial_dns_seedlist_discovery/*"
+   if [ "$DNS" = "loadbalanced" ]; then
+      export MONGOC_TEST_DNS_LOADBALANCED=on
+   else
+      export MONGOC_TEST_DNS=on
+   fi
 fi
 
 if [ "$CC" = "mingw" ]; then

.gitignore

@@ -21,6 +21,7 @@ Makefile
 test-results.json
 VERSION_CURRENT
 VERSION_RELEASED
+*.pyc
 
 # Windows things
 *.sdf

CONTRIBUTING.md

@@ -215,7 +215,9 @@ The tests for mongodb+srv:// connection strings require some setup, see the
 Initial DNS Seedlist Discovery Spec. By default these connection strings are
 NOT tested, enable them with:
 
-* `MONGOC_TEST_DNS=on`
+* `MONGOC_TEST_DNS=on` assumes a replica set is running with TLS enabled on ports 27017, 27018, and 27019.
+
+* `MONGOC_TEST_DNS_LOADBALANCED=on` assumes a load balanced sharded cluster is running with mongoses on ports 27017 and 27018 and TLS enabled. The load balancer can be listening on any port.
 
 The mock server timeout threshold for future functions can be set with:
 

build/evergreen_config_lib/functions.py

@@ -543,5 +543,35 @@
         export IAM_AUTH_ECS_SECRET_ACCESS_KEY=${iam_auth_ecs_secret_access_key}
         sh ./.evergreen/run-aws-tests.sh ${TESTCASE}
         ''')
-    ))
+    )),
+    ('clone drivers-evergreen-tools', Function(
+        shell_exec(r'''
+        if [ ! -d "drivers-evergreen-tools" ]; then
+            git clone [email protected]:mongodb-labs/drivers-evergreen-tools.git --depth=1
+        fi
+        ''', test=False)
+    )),
+    ('start load balancer', Function(
+        shell_exec(r'''
+        export DRIVERS_TOOLS=./drivers-evergreen-tools
+        export MONGODB_URI="${MONGODB_URI}"
+        bash $DRIVERS_TOOLS/.evergreen/run-load-balancer.sh start
+        ''', test=False),
+        OD ([
+            ("command", "expansions.update"),
+            ("params", OD([("file", "lb-expansion.yml")]))
+        ]),
+    )),
+    ('stop load balancer', Function(
+        shell_exec(r'''
+        # Only run if a load balancer was started.
+        if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then
+            echo "OK - no load balancer running"
+            exit 0
+        fi
+        export DRIVERS_TOOLS=./drivers-evergreen-tools
+        export MONGODB_URI="foo" # TODO: DRIVERS-1833 remove this.
+        $DRIVERS_TOOLS/.evergreen/run-load-balancer.sh stop
+        ''', test=False),
+    )),
 ])

build/evergreen_config_lib/tasks.py

@@ -626,7 +626,9 @@ def _check_allowed(self):
 
 class DNSTask(MatrixTask):
     axes = OD([('auth', [False, True]),
-               ('ssl', ['openssl', 'winssl', 'darwinssl'])])
+               ('loadbalanced', [False, True]),
+               ('ssl', ['openssl', 'winssl', 'darwinssl'])
+               ])
 
     name_prefix = 'test-dns'
 
@@ -648,20 +650,33 @@ def to_dict(self):
         commands.append(
             func('fetch build', BUILD_NAME=self.depends_on['name']))
 
-        orchestration = bootstrap(TOPOLOGY='replica_set',
+        orchestration = bootstrap(TOPOLOGY='sharded_cluster' if self.loadbalanced else 'replica_set',
                                   AUTH='auth' if self.auth else 'noauth',
                                   SSL='ssl')
 
         if self.auth:
             orchestration['vars']['AUTHSOURCE'] = 'thisDB'
 
         commands.append(orchestration)
+
+        dns = 'on'
+        if self.loadbalanced:
+            dns = 'loadbalanced'
+            commands.append (func("clone drivers-evergreen-tools"))
+            commands.append (func("start load balancer", MONGODB_URI="mongodb://localhost:27017,localhost:27018"))
+        elif self.auth:
+            dns = 'dns-auth'
         commands.append(run_tests(SSL='ssl',
                                   AUTH=self.display('auth'),
-                                  DNS='dns-auth' if self.auth else 'on'))
+                                  DNS=dns))
 
         return task
 
+    def _check_allowed(self):
+        prohibit (self.loadbalanced and self.auth)
+        # Load balancer tests only run on some Linux hosts in Evergreen until CDRIVER-4041 is resolved.
+        prohibit (self.loadbalanced and self.ssl in ["darwinssl", "winssl"])
+
 
 all_tasks = chain(all_tasks, DNSTask.matrix())
 

build/evergreen_config_lib/variants.py

@@ -325,7 +325,8 @@ def days(n):
              '.4.2 .openssl !.nosasl .server',
              '.4.0 .openssl !.nosasl .server',
              'test-dns-openssl',
-             'test-dns-auth-openssl'
+             'test-dns-auth-openssl',
+             'test-dns-loadbalanced-openssl'
              ],
             {'CC': 'gcc'}),
     Variant('gcc54',

build/generate-evergreen-config.py

@@ -60,6 +60,7 @@
         OD([('func', 'upload mo artifacts')]),
         OD([('func', 'upload test results')]),
         OD([('func', 'cleanup')]),
+        OD([('func', 'stop load balancer')]),
     ]),
     ('timeout', [
         OD([('func', 'backtrace')])

src/libmongoc/doc/mongoc_uri_t.rst

@@ -101,6 +101,7 @@ MONGOC_URI_SOCKETTIMEOUTMS                 sockettimeoutms                   300
 MONGOC_URI_TIMEOUTMS                       timeoutms                         Empty (no timeout)                The time limit for the full execution of an operation.
 MONGOC_URI_REPLICASET                      replicaset                        Empty (no replicaset)             The name of the Replica Set that the driver should connect to.
 MONGOC_URI_ZLIBCOMPRESSIONLEVEL            zlibcompressionlevel              -1                                When the MONGOC_URI_COMPRESSORS includes "zlib" this options configures the zlib compression level, when the zlib compressor is used to compress client data.
+MONGOC_URI_LOADBALANCED                    loadbalanced                      false                             If true, this indicates the driver is connecting to a MongoDB cluster behind a load balancer.
 ========================================== ================================= ================================= ============================================================================================================================================================================================================================================
 
 Setting any of the \*timeoutMS options above to ``0`` will be interpreted as "use the default value".

src/libmongoc/src/mongoc/mongoc-client-private.h

@@ -148,22 +148,10 @@ typedef enum {
 BSON_STATIC_ASSERT2 (mongoc_cmd_rw,
                      MONGOC_CMD_RW == (MONGOC_CMD_READ | MONGOC_CMD_WRITE));
 
-typedef enum { MONGOC_RR_SRV, MONGOC_RR_TXT } mongoc_rr_type_t;
-
-typedef struct _mongoc_rr_data_t {
-   /* Number of records returned by DNS. */
-   uint32_t count;
-
-   /* Set to lowest TTL found when polling SRV records. */
-   uint32_t min_ttl;
-
-   /* Set to the resulting host list when polling SRV records */
-   mongoc_host_list_t *hosts;
-
-   /* Set to the TXT record when polling for TXT */
-   char *txt_record_opts;
-} mongoc_rr_data_t;
 
+/* TODO (CDRIVER-4052): Move MONGOC_RR_DEFAULT_BUFFER_SIZE and _mongoc_client_get_rr to
+ * mongoc-topology-private.h or in a separate file. There is no reason these
+ * should be in mongoc-client. */
 #define MONGOC_RR_DEFAULT_BUFFER_SIZE 1024
 bool
 _mongoc_client_get_rr (const char *service,

src/libmongoc/src/mongoc/mongoc-topology-private.h

@@ -46,6 +46,28 @@ typedef enum {
 struct _mongoc_background_monitor_t;
 struct _mongoc_client_pool_t;
 
+typedef enum { MONGOC_RR_SRV, MONGOC_RR_TXT } mongoc_rr_type_t;
+
+typedef struct _mongoc_rr_data_t {
+   /* Number of records returned by DNS. */
+   uint32_t count;
+
+   /* Set to lowest TTL found when polling SRV records. */
+   uint32_t min_ttl;
+
+   /* Set to the resulting host list when polling SRV records */
+   mongoc_host_list_t *hosts;
+
+   /* Set to the TXT record when polling for TXT */
+   char *txt_record_opts;
+} mongoc_rr_data_t;
+
+typedef bool (*_mongoc_rr_resolver_fn) (const char *service,
+                                        mongoc_rr_type_t rr_type,
+                                        mongoc_rr_data_t *rr_data,
+                                        size_t initial_buffer_size,
+                                        bson_error_t *error);
+
 typedef struct _mongoc_topology_t {
    mongoc_topology_description_t description;
    /* topology->uri is initialized as a copy of the client/pool's URI.
@@ -104,6 +126,9 @@ typedef struct _mongoc_topology_t {
    mongoc_set_t *server_monitors;
    mongoc_set_t *rtt_monitors;
    bson_mutex_t apm_mutex;
+
+   /* This is overridable for SRV polling tests to mock DNS records. */
+   _mongoc_rr_resolver_fn rr_resolver;
 } mongoc_topology_t;
 
 mongoc_topology_t *
@@ -229,4 +254,24 @@ mongoc_topology_rescan_srv (mongoc_topology_t *topology);
 
 bool
 mongoc_topology_should_rescan_srv (mongoc_topology_t *topology);
+
+/* _mongoc_topology_set_rr_resolver is called by tests to mock DNS responses for
+ * SRV polling.
+ * This is necessarily called after initial seedlist discovery completes in
+ * mongoc_topology_new.
+ * Callers should call this before monitoring starts.
+ * Callers must lock topology->mutex.
+ */
+void
+_mongoc_topology_set_rr_resolver (mongoc_topology_t *topology,
+                                  _mongoc_rr_resolver_fn rr_resolver);
+
+/* _mongoc_topology_set_srv_polling_rescan_interval_ms is called by tests to
+ * shorten the rescan interval.
+ * Callers should call this before monitoring starts.
+ * Callers must lock topology->mutex.
+ */
+void
+_mongoc_topology_set_srv_polling_rescan_interval_ms (
+   mongoc_topology_t *topology, int64_t val);
 #endif