CXX-1848 Add sharded cluster test coverage to EVG config (#1007)

* Move legacy transaction spec tests into subdirectory

* Sync legacy transaction spec tests with 56454749 (CXX-2406)

* Sync legacy transactions convenient API spec tests with e9280056 (CXX-1976)

* Remove references to obsolete "sharded-replicaset" topology

* Fix inconsistent behavior of index_view::create_*() on sharded clusters

* The failCommand operation on mongos requires 4.1+

* Multi-document transactions require 4.2+

* Fix legacy test runner outcome checks missing readPreference=primary

* Fix flaky tailable await cursors tests due to extreme maxAwaitTimeMS

* Remove unused configureFailPoint operation in legacy test runner

* Ensure configureFailPoint events are ignored

* Improve apm_checker::print_all() output and const-correctness

* Capture APM events in result assertion failures

* Use Catch2 substring matcher instead of index comparison

* Fix operation name match for assertSessionUnpinned

* Update session pin assertion operations

* Do not assume session object name when not session0

* Add transactions mongos pinning prose tests

* Improve failure messages for APM event checks in legacy test runner

* Add Catch2 sections separating legacy transactions and convenient API tests

* Handle client errors reported as logic_error in legacy test runner

* Skip legacy transactions test requiring explicit readPreference with runCommand

* Update legacy transactions test runner to conform to latest spec

* Implement assertSessionTransactionState in legacy test runner

* Implement targetedFailPoint in legacy test runner

* Sync unified transaction spec tests with 474ddfcc (CXX-2221, CXX-2233, CXX-2235, CXX-2393, CXX-2552, CXX-2690)

* Fix unified test runner outcome checks missing readPreference=primary and readConcern=local

* Add transactions spec tests to unified test runner

* Fix test client URI hostname specification in unified test runner

* Implement targetedFailPoint in unified test runner

* Implement session pin assertions in unified test runner

* Add sharded cluster tasks to EVG config

* Use scope guards to disable fail points in unified test runner

* Remove obsolete version checks in examples (CXX-1547)

* Fix server version comparisons

* Workaround possibly unexpected multi and upsert fields

* Fix match actions when comparing APM events in legacy test runner

* Add support for missing sessionOptions fields in legacy test runner

Author: eramongodb

.mci.yml

@@ -476,7 +476,8 @@ functions:
                   export RETRYABLE_READS_UNIFIED_TESTS_PATH="$(pwd)/../data/retryable-reads/unified"
                   export RETRYABLE_WRITES_UNIFIED_TESTS_PATH="$(pwd)/../data/retryable-writes/unified"
                   export SESSION_UNIFIED_TESTS_PATH="$(pwd)/../data/sessions/unified"
-                  export TRANSACTIONS_TESTS_PATH="$(pwd)/../data/transactions"
+                  export TRANSACTIONS_LEGACY_TESTS_PATH="$(pwd)/../data/transactions/legacy"
+                  export TRANSACTIONS_UNIFIED_TESTS_PATH="$(pwd)/../data/transactions/unified"
                   export UNIFIED_FORMAT_TESTS_PATH=$(pwd)/../data/unified-format
                   export URI_OPTIONS_TESTS_PATH="$(pwd)/../data/uri-options"
                   export VERSIONED_API_TESTS_PATH=$(pwd)/../data/versioned-api
@@ -988,6 +989,21 @@ tasks:
         - func: "install_c_driver"
         - func: "test"
 
+    - name: compile_and_test_with_shared_libs_sharded_cluster
+      commands:
+        - func: "setup"
+        - func: "start_mongod"
+          vars:
+              TOPOLOGY: "sharded_cluster"
+        - func: "fetch_c_driver_source"
+        - func: "compile"
+          vars:
+              RUN_DISTCHECK: 1
+        - func: "clone_drivers-evergreen-tools"
+        - func: "run_kms_servers"
+        - func: "install_c_driver"
+        - func: "test"
+
     # Auto downloading the C driver in the C++ build does not currently include
     # support for libmongocrypt, therefore it is not configured with
     # -DENABLE_CLIENT_SIDE_ENCRYPTION=ON. For now, CSFLE tests will need to have
@@ -1006,6 +1022,20 @@ tasks:
         - func: "run_kms_servers"
         - func: "test"
 
+    - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
+      commands:
+        - func: "setup"
+        - func: "start_mongod"
+          vars:
+              TOPOLOGY: "sharded_cluster"
+        - func: "install_c_driver"
+        - func: "compile"
+          vars:
+              RUN_DISTCHECK: 1
+        - func: "clone_drivers-evergreen-tools"
+        - func: "run_kms_servers"
+        - func: "test"
+
     - name: uninstall_check
       commands:
       - func: "setup"
@@ -1314,6 +1344,12 @@ buildvariants:
       tasks:
         - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
 
+    - matrix_name: "integration (sharded cluster)"
+      matrix_spec: {os: "ubuntu-1804", mongodb_version: "*"}
+      display_name: "${os} sharded cluster (MongoDB ${mongodb_version})"
+      tasks:
+        - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
+
     - matrix_name: "auth"
       matrix_spec: {os: "*", mongodb_version: "latest"}
       display_name: "${os} ${mongodb_version} Auth"
@@ -1327,15 +1363,6 @@ buildvariants:
           - name: test_versioned_api
           - name: test_versioned_api_accept_version_two
 
-    # Add matrix for specification test requirement of mongocryptd:
-    # "Drivers MUST run all tests with mongocryptd on at least one platform for all tested server versions (4.2+)."
-    - matrix_name: "mongocryptd"
-      matrix_spec: {os: "ubuntu-1804", mongodb_version: ["4.2", "4.4", "5.0", "latest"], use_mongocryptd: "true"}
-      display_name: "${os} (MongoDB ${mongodb_version}) with mongocryptd"
-      tasks:
-          - name: compile_and_test_with_shared_libs
-          - name: compile_and_test_with_shared_libs_replica_set
-
     #######################################
     #         Linux Buildvariants         #
     #######################################
@@ -1358,6 +1385,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - rhel90-large
@@ -1381,6 +1409,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - rhel90-arm64-large
@@ -1403,6 +1432,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - debian11-large
@@ -1426,6 +1456,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_sharded_cluster
           - name: build_example_with_add_subdirectory
             distros:
             - debian11-large
@@ -1447,6 +1478,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - debian10-large
@@ -1468,6 +1500,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_sharded_cluster
           - name: build_example_with_add_subdirectory
             distros:
             - debian10-large
@@ -1479,11 +1512,11 @@ buildvariants:
       matrix_spec:
         os: "ubuntu-1804"
         mongodb_version: ["4.2", "4.4", "5.0", "latest"]
+        use_mongocryptd: "true"
       display_name: "${os} (MongoDB ${mongodb_version}) with mongocryptd"
       tasks:
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
-      expansions:
-        use_mongocryptd: true
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
 
     - name: ubuntu2004-release-latest
       display_name: "Ubuntu 20.04 Release (MongoDB Latest)"
@@ -1501,6 +1534,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu2004-large
@@ -1522,6 +1556,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_sharded_cluster
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu2004-large
@@ -1544,6 +1579,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+          - name: compile_and_test_with_shared_libs_sharded_cluster_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu1804-build
@@ -1565,6 +1601,7 @@ buildvariants:
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
           - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_sharded_cluster
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu1804-build

data/transactions/abort.json renamed to data/transactions/legacy/abort.json

@@ -10,7 +10,8 @@
       "minServerVersion": "4.1.8",
       "topology": [
         "sharded"
-      ]
+      ],
+      "serverless": "forbid"
     }
   ],
   "database_name": "transaction-tests",
@@ -102,7 +103,7 @@
       }
     },
     {
-      "description": "NotMaster errors contain transient label",
+      "description": "NotWritablePrimary errors contain transient label",
       "failPoint": {
         "configureFailPoint": "failCommand",
         "mode": {
@@ -962,12 +963,12 @@
           "failCommands": [
             "commitTransaction"
           ],
+          "errorLabels": [
+            "RetryableWriteError"
+          ],
           "writeConcernError": {
             "code": 91,
-            "errmsg": "Replication is being shut down",
-            "errorLabels": [
-              "RetryableWriteError"
-            ]
+            "errmsg": "Replication is being shut down"
           }
         }
       },

data/transactions/bulk.json renamed to data/transactions/legacy/bulk.json

@@ -0,0 +1,96 @@
+{
+  "runOn": [
+    {
+      "minServerVersion": "4.0",
+      "topology": [
+        "replicaset"
+      ]
+    },
+    {
+      "minServerVersion": "4.1.8",
+      "topology": [
+        "sharded"
+      ]
+    }
+  ],
+  "database_name": "transaction-tests",
+  "collection_name": "test",
+  "data": [],
+  "tests": [
+    {
+      "description": "Client side error in command starting transaction",
+      "operations": [
+        {
+          "name": "startTransaction",
+          "object": "session0"
+        },
+        {
+          "name": "updateOne",
+          "object": "collection",
+          "arguments": {
+            "session": "session0",
+            "filter": {
+              "_id": 1
+            },
+            "update": {
+              "x": 1
+            }
+          },
+          "error": true
+        },
+        {
+          "name": "assertSessionTransactionState",
+          "object": "testRunner",
+          "arguments": {
+            "session": "session0",
+            "state": "starting"
+          }
+        }
+      ]
+    },
+    {
+      "description": "Client side error when transaction is in progress",
+      "operations": [
+        {
+          "name": "startTransaction",
+          "object": "session0"
+        },
+        {
+          "name": "insertOne",
+          "object": "collection",
+          "arguments": {
+            "session": "session0",
+            "document": {
+              "_id": 1
+            }
+          },
+          "result": {
+            "insertedId": 1
+          }
+        },
+        {
+          "name": "updateOne",
+          "object": "collection",
+          "arguments": {
+            "session": "session0",
+            "filter": {
+              "_id": 1
+            },
+            "update": {
+              "x": 1
+            }
+          },
+          "error": true
+        },
+        {
+          "name": "assertSessionTransactionState",
+          "object": "testRunner",
+          "arguments": {
+            "session": "session0",
+            "state": "in_progress"
+          }
+        }
+      ]
+    }
+  ]
+}

data/transactions/causal-consistency.json renamed to data/transactions/legacy/causal-consistency.json

@@ -4,7 +4,8 @@
       "minServerVersion": "4.1.8",
       "topology": [
         "sharded"
-      ]
+      ],
+      "serverless": "forbid"
     }
   ],
   "database_name": "transaction-tests",

data/transactions/commit.json renamed to data/transactions/legacy/commit.json

@@ -4,7 +4,8 @@
       "minServerVersion": "4.1.8",
       "topology": [
         "sharded"
-      ]
+      ],
+      "serverless": "forbid"
     }
   ],
   "database_name": "transaction-tests",
@@ -179,6 +180,9 @@
                 "failCommands": [
                   "commitTransaction"
                 ],
+                "errorLabels": [
+                  "RetryableWriteError"
+                ],
                 "writeConcernError": {
                   "code": 91,
                   "errmsg": "Replication is being shut down"
@@ -303,7 +307,8 @@
               "data": {
                 "failCommands": [
                   "commitTransaction",
-                  "isMaster"
+                  "isMaster",
+                  "hello"
                 ],
                 "closeConnection": true
               }

data/transactions/count.json renamed to data/transactions/legacy/count.json

@@ -4,7 +4,8 @@
       "minServerVersion": "4.1.8",
       "topology": [
         "sharded"
-      ]
+      ],
+      "serverless": "forbid"
     }
   ],
   "database_name": "transaction-tests",
@@ -875,7 +876,7 @@
                 "failCommands": [
                   "commitTransaction"
                 ],
-                "errorCode": 50
+                "errorCode": 51
               }
             }
           }
@@ -887,7 +888,7 @@
             "errorLabelsOmit": [
               "TransientTransactionError"
             ],
-            "errorCode": 50
+            "errorCode": 51
           }
         },
         {
@@ -1106,7 +1107,8 @@
               "data": {
                 "failCommands": [
                   "insert",
-                  "isMaster"
+                  "isMaster",
+                  "hello"
                 ],
                 "closeConnection": true
               }