GODRIVER-1931 Run tests against LBs in Evergreen

Author: Divjot Arora

.evergreen/config.yml

@@ -518,6 +518,47 @@ functions:
           PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
+  run-load-balancer-tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src/go.mongodb.org/mongo-driver
+        script: |
+          ${PREPARE_SHELL}
+
+          if [ ${SSL} = "ssl" ]; then
+              export MONGO_GO_DRIVER_CA_FILE="$PROJECT_DIRECTORY/data/certificates/ca.pem"
+              if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+                  export MONGO_GO_DRIVER_CA_FILE=$(cygpath -m $MONGO_GO_DRIVER_CA_FILE)
+              fi
+          fi
+
+          # Verify that the required LB URI env vars are set to ensure that the test runner can correctly connect to
+          # the LBs.
+          if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then
+            echo "SINGLE_MONGOS_LB_URI must be set for testing against LBs"
+            exit 1
+          fi
+          if [ -z "${MULTI_MONGOS_LB_URI}" ]; then
+            echo "MULTI_MONGOS_LB_URI must be set for testing against LBs"
+            exit 1
+          fi
+
+          # Per the LB testing spec, the URI of an LB fronting a single mongos should be used to configure internal
+          # testing Client instances, so we set MONGODB_URI to SINGLE_MONGOS_LB_URI.
+
+          export GOFLAGS=-mod=vendor
+          set +o xtrace
+          AUTH="${AUTH}" \
+          SSL="${SSL}" \
+          MONGODB_URI="${SINGLE_MONGOS_LB_URI}" \
+          SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
+          MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
+          TOPOLOGY="${TOPOLOGY}" \
+          MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
+          BUILD_TAGS="-tags cse" \
+          make evg-test-load-balancers
+
   run-atlas-data-lake-test:
     - command: shell.exec
       type: test
@@ -612,6 +653,21 @@ functions:
           -v \
           --fault revoked
 
+  run-load-balancer:
+    - command: shell.exec
+      params:
+        script: |
+          DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
+    - command: expansions.update
+      params:
+        file: lb-expansion.yml
+
+  stop-load-balancer:
+    - command: shell.exec
+      params:
+        script: |
+          DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
+
   add-aws-auth-variables-to-file:
     - command: shell.exec
       type: test
@@ -866,6 +922,7 @@ post:
       files:
         - "src/go.mongodb.org/mongo-driver/*.suite"
   - func: upload-mo-artifacts
+  - func: stop-load-balancer
   - func: cleanup
 
 tasks:
@@ -1379,6 +1436,28 @@ tasks:
       - func: bootstrap-mongohoused
       - func: run-atlas-data-lake-test
 
+  - name: test-load-balancer-noauth-nossl
+    tags: ["load-balancer"]
+    commands:
+      - func: bootstrap-mongo-orchestration
+        vars:
+          TOPOLOGY: "sharded_cluster"
+          AUTH: "noauth"
+          SSL: "nossl"
+      - func: run-load-balancer
+      - func: run-load-balancer-tests
+
+  - name: test-load-balancer-auth-ssl
+    tags: ["load-balancer"]
+    commands:
+      - func: bootstrap-mongo-orchestration
+        vars:
+          TOPOLOGY: "sharded_cluster"
+          AUTH: "auth"
+          SSL: "ssl"
+      - func: run-load-balancer
+      - func: run-load-balancer-tests
+
   - name: test-replicaset-noauth-nossl
     tags: ["test", "replicaset"]
     commands:
@@ -1974,3 +2053,10 @@ buildvariants:
     display_name: "KMS TLS ${version} ${os-ssl-40}"
     tasks:
       - name: ".kms-tls"
+
+  - matrix_name: "load-balancer-test"
+    # The LB software is only available on Ubuntu 18.04, so we don't test on all OSes.
+    matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-15"] }
+    display_name: "Load Balancer Support ${version} ${os-ssl-40}"
+    tasks:
+      - name: ".load-balancer"

Makefile

@@ -165,6 +165,14 @@ evg-test-versioned-api:
 		go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s $$TEST_PKG >> test.suite ; \
 	done
 
+.PHONY: evg-test-load-balancers
+evg-test-load-balancers:
+	go test $(BUILD_TAGS) ./mongo/integration -run TestUnifiedSpecs/retryable-reads -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration -run TestRetryableWritesSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration -run TestChangeStreamSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BULID_TAGS) ./mongo/integration -run TestInitialDNSSeedlistDiscoverySpec/load_balanced -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+
 .PHONY: evg-test-kms
 evg-test-kms:
 	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_tests >> test.suite

internal/const.go

@@ -8,3 +8,8 @@ package internal // import "go.mongodb.org/mongo-driver/internal"
 
 // Version is the current version of the driver.
 var Version = "local build"
+
+// SetServiceID enables a mode in which the driver mocks server support for returning a "serviceId" field in "hello"
+// command responses by using the value of "topologyVersion.processId". This is used for testing load balancer
+// support until an upstream service can support running behind a load balancer.
+var SetServiceID = false

mongo/description/server.go

@@ -266,6 +266,10 @@ func NewServer(addr address.Address, response bson.Raw) Server {
 				desc.LastError = err
 				return desc
 			}
+
+			if internal.SetServiceID {
+				desc.ServiceID = &desc.TopologyVersion.ProcessID
+			}
 		}
 	}
 

mongo/integration/main_test.go

@@ -9,12 +9,22 @@ package integration
 import (
 	"log"
 	"os"
+	"strings"
 	"testing"
 
+	"go.mongodb.org/mongo-driver/internal"
 	"go.mongodb.org/mongo-driver/mongo/integration/mtest"
 )
 
 func TestMain(m *testing.M) {
+	// If the cluster is behind a load balancer, enable the SetServiceID flag to mock server-side LB support.
+	if strings.Contains(os.Getenv("MONGODB_URI"), "loadBalanced=true") {
+		internal.SetServiceID = true
+		defer func() {
+			internal.SetServiceID = false
+		}()
+	}
+
 	if err := mtest.Setup(); err != nil {
 		log.Fatal(err)
 	}

mongo/integration/mtest/global_state.go

@@ -35,6 +35,18 @@ func ClusterURI() string {
 	return testContext.connString.Original
 }
 
+// SingleMongosLoadBalancerURI returns the URI for a load balancer fronting a single mongos. This will only be set
+// if the cluster is load balanced.
+func SingleMongosLoadBalancerURI() string {
+	return testContext.singleMongosLoadBalancerURI
+}
+
+// MultiMongosLoadBalancerURI returns the URI for a load balancer fronting multiple mongoses. This will only be set
+// if the cluster is load balanced.
+func MultiMongosLoadBalancerURI() string {
+	return testContext.multiMongosLoadBalancerURI
+}
+
 // ClusterConnString returns the parsed ConnString for the cluster.
 func ClusterConnString() connstring.ConnString {
 	return testContext.connString

mongo/integration/mtest/setup.go

@@ -43,15 +43,17 @@ var testContext struct {
 	// shardedReplicaSet will be true if we're connected to a sharded cluster and each shard is backed by a replica set.
 	// We track this as a separate boolean rather than setting topoKind to ShardedReplicaSet because a general
 	// "Sharded" constraint in a test should match both Sharded and ShardedReplicaSet.
-	shardedReplicaSet bool
-	client            *mongo.Client // client used for setup and teardown
-	serverVersion     string
-	authEnabled       bool
-	sslEnabled        bool
-	enterpriseServer  bool
-	dataLake          bool
-	requireAPIVersion bool
-	serverParameters  bson.Raw
+	shardedReplicaSet           bool
+	client                      *mongo.Client // client used for setup and teardown
+	serverVersion               string
+	authEnabled                 bool
+	sslEnabled                  bool
+	enterpriseServer            bool
+	dataLake                    bool
+	requireAPIVersion           bool
+	serverParameters            bson.Raw
+	singleMongosLoadBalancerURI string
+	multiMongosLoadBalancerURI  string
 }
 
 func setupClient(cs connstring.ConnString, opts *options.ClientOptions) (*mongo.Client, error) {
@@ -75,7 +77,7 @@ func Setup(setupOpts ...*SetupOptions) error {
 	case opts.URI != nil:
 		testContext.connString, err = connstring.ParseAndValidate(*opts.URI)
 	default:
-		testContext.connString, err = getConnString()
+		testContext.connString, err = getClusterConnString()
 	}
 	if err != nil {
 		return fmt.Errorf("error getting connection string: %v", err)
@@ -176,6 +178,22 @@ func Setup(setupOpts ...*SetupOptions) error {
 		}
 	}
 
+	// For load balanced clusters, retrieve the required LB URIs and add additional information (e.g. TLS options) to
+	// them if necessary.
+	if testContext.topoKind == LoadBalanced {
+		singleMongosURI := os.Getenv("SINGLE_MONGOS_LB_URI")
+		if singleMongosURI == "" {
+			return errors.New("SINGLE_MONGOS_LB_URI must be set when running against load balanced clusters")
+		}
+		testContext.singleMongosLoadBalancerURI = addNecessaryParamsToURI(singleMongosURI)
+
+		multiMongosURI := os.Getenv("MULTI_MONGOS_LB_URI")
+		if multiMongosURI == "" {
+			return errors.New("MULTI_MONGOS_LB_URI must be set when running against load balanced clusters")
+		}
+		testContext.multiMongosLoadBalancerURI = addNecessaryParamsToURI(multiMongosURI)
+	}
+
 	testContext.authEnabled = os.Getenv("AUTH") == "auth"
 	testContext.sslEnabled = os.Getenv("SSL") == "ssl"
 	biRes, err := testContext.client.Database("admin").RunCommand(Background, bson.D{{"buildInfo", 1}}).DecodeBytes()
@@ -282,17 +300,21 @@ func addCompressors(uri string) string {
 	return addOptions(uri, "compressors=", comp)
 }
 
-// ConnString gets the globally configured connection string.
-func getConnString() (connstring.ConnString, error) {
+// getClusterConnString gets the globally configured connection string.
+func getClusterConnString() (connstring.ConnString, error) {
 	uri := os.Getenv("MONGODB_URI")
 	if uri == "" {
 		uri = "mongodb://localhost:27017"
 	}
-	uri = addTLSConfig(uri)
-	uri = addCompressors(uri)
+	uri = addNecessaryParamsToURI(uri)
 	return connstring.ParseAndValidate(uri)
 }
 
+func addNecessaryParamsToURI(uri string) string {
+	uri = addTLSConfig(uri)
+	return addCompressors(uri)
+}
+
 // CompareServerVersions compares two version number strings (i.e. positive integers separated by
 // periods). Comparisons are done to the lesser precision of the two versions. For example, 3.2 is
 // considered equal to 3.2.11, whereas 3.2.0 is considered less than 3.2.11.

mongo/integration/unified/client_entity.go

@@ -9,7 +9,6 @@ package unified
 import (
 	"context"
 	"fmt"
-	"os"
 	"sync/atomic"
 	"time"
 
@@ -57,10 +56,7 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp
 
 	// Construct a ClientOptions instance by first applying the cluster URI and then the URIOptions map to ensure that
 	// the options specified in the test file take precedence.
-	uri, err := getURIForClient(entityOptions)
-	if err != nil {
-		return nil, err
-	}
+	uri := getURIForClient(entityOptions)
 	clientOpts := options.Client().ApplyURI(uri)
 	if entityOptions.URIOptions != nil {
 		if err := setClientOptionsFromURIOptions(clientOpts, entityOptions.URIOptions); err != nil {
@@ -126,26 +122,19 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp
 	return entity, nil
 }
 
-func getURIForClient(opts *entityOptions) (string, error) {
+func getURIForClient(opts *entityOptions) string {
 	if mtest.ClusterTopologyKind() != mtest.LoadBalanced {
-		return mtest.ClusterURI(), nil
+		return mtest.ClusterURI()
 	}
 
-	// For load-balanced deployments, UseMultipleMongoses is used to determine the load balancer URI. If set to
-	// false, the LB fronts a single server. If unset or explicitly true, the LB fronts multiple mongos servers.
-	var uriEnvVar string
+	// For load-balanced deployments, UseMultipleMongoses is used to determine the load balancer URI. If set to false,
+	// the LB fronts a single server. If unset or explicitly true, the LB fronts multiple mongos servers.
 	switch {
 	case opts.UseMultipleMongoses != nil && !*opts.UseMultipleMongoses:
-		uriEnvVar = "SINGLE_MONGOS_LB_URI"
+		return mtest.SingleMongosLoadBalancerURI()
 	default:
-		uriEnvVar = "MULTI_MONGOS_LB_URI"
-	}
-
-	uri := os.Getenv(uriEnvVar)
-	if uri == "" {
-		return "", fmt.Errorf("expected load balancer URI to be in environment variable %q, but is not set", uriEnvVar)
+		return mtest.MultiMongosLoadBalancerURI()
 	}
-	return uri, nil
 }
 
 func (c *clientEntity) stopListeningForEvents() {
@@ -344,8 +333,14 @@ func setClientOptionsFromURIOptions(clientOpts *options.ClientOptions, uriOpts b
 
 	for key, value := range uriOpts {
 		switch key {
+		case "appname":
+			clientOpts.SetAppName(value.(string))
 		case "heartbeatFrequencyMS":
 			clientOpts.SetHeartbeatInterval(time.Duration(value.(int32)) * time.Millisecond)
+		case "loadBalanced":
+			clientOpts.SetLoadBalanced(value.(bool))
+		case "maxPoolSize":
+			clientOpts.SetMaxPoolSize(uint64(value.(int32)))
 		case "readConcernLevel":
 			clientOpts.SetReadConcern(readconcern.New(readconcern.Level(value.(string))))
 		case "retryReads":
@@ -355,6 +350,8 @@ func setClientOptionsFromURIOptions(clientOpts *options.ClientOptions, uriOpts b
 		case "w":
 			wc.W = value
 			wcSet = true
+		case "waitQueueTimeoutMS":
+			return newSkipTestError("the waitQueueTimeoutMS client option is not supported")
 		default:
 			return fmt.Errorf("unrecognized URI option %s", key)
 		}

mongo/integration/unified/collection_operation_execution.go

@@ -748,6 +748,38 @@ func executeInsertOne(ctx context.Context, operation *operation) (*operationResu
 	return newDocumentResult(raw, err), nil
 }
 
+func executeListIndexes(ctx context.Context, operation *operation) (*operationResult, error) {
+	coll, err := entities(ctx).collection(operation.Object)
+	if err != nil {
+		return nil, err
+	}
+
+	opts := options.ListIndexes()
+	elems, _ := operation.Arguments.Elements()
+	for _, elem := range elems {
+		key := elem.Key()
+		val := elem.Value()
+
+		switch key {
+		case "batchSize":
+			opts.SetBatchSize(val.Int32())
+		default:
+			return nil, fmt.Errorf("unrecognized listIndexes option: %q", key)
+		}
+	}
+
+	cursor, err := coll.Indexes().List(ctx, opts)
+	if err != nil {
+		return newErrorResult(err), nil
+	}
+
+	var docs []bson.Raw
+	if err := cursor.All(ctx, &docs); err != nil {
+		return newErrorResult(err), nil
+	}
+	return newCursorResult(docs), nil
+}
+
 func executeReplaceOne(ctx context.Context, operation *operation) (*operationResult, error) {
 	coll, err := entities(ctx).collection(operation.Object)
 	if err != nil {