Auto download C Driver (#967)

CXX-2695
- Auto download the C driver if it is not already installed.
- Make auto downloading the C driver the default for CI.
- Libmongocrypt is not yet supported for auto downloading, so use the classic method of installing the C driver separately for tasks that require libmongocrypt.

---------

Co-authored-by: vector-of-bool <[email protected]>
Co-authored-by: Roberto C. Sánchez <[email protected]>
Co-authored-by: Kevin Albertson <[email protected]>

Author: 4 people

.mci.yml

@@ -283,6 +283,18 @@ functions:
             add_expansions_to_env: true
             script: mongo-cxx-driver/.evergreen/install_c_driver.sh
 
+    # fetch_c_driver_source may be used to fetch the C driver source without installing the C driver.
+    # This can be used when only CI scripts are needed.
+    "fetch_c_driver_source":
+      - command: shell.exec
+        params:
+            shell: bash
+            working_dir: "."
+            script: |
+                set -o errexit
+                set -o pipefail
+                git clone --depth 1 https://github.com/mongodb/mongo-c-driver mongoc
+
     "lint":
         - command: shell.exec
           params:
@@ -832,45 +844,49 @@ tasks:
     - name: clang-tidy
       commands:
         - func: "setup"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
         - func: "clang-tidy"
 
     - name: compile_and_test_with_shared_libs
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
           vars:
               RUN_DISTCHECK: 1
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        # Call "install_c_driver" before "test" to build static C driver libraries. Example projects require static C driver libraries.
+        - func: "install_c_driver"
         - func: "test"
 
     - name: compile_and_test_with_shared_libs_extra_alignment
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
           vars:
               BSON_EXTRA_ALIGNMENT: 1
         - func: "compile"
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        - func: "install_c_driver"
         - func: "test"
 
     - name: compile_and_test_with_shared_libs_cxx20
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
           vars:
               RUN_DISTCHECK: 1
               REQUIRED_CXX_STANDARD: 20
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        - func: "install_c_driver"
         - func: "test"
           vars:
               REQUIRED_CXX_STANDARD: 20
@@ -880,14 +896,15 @@ tasks:
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
           vars:
               BSON_EXTRA_ALIGNMENT: 1
         - func: "compile"
           vars:
               REQUIRED_CXX_STANDARD: 20
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        - func: "install_c_driver"
         - func: "test"
           vars:
               REQUIRED_CXX_STANDARD: 20
@@ -897,13 +914,13 @@ tasks:
       commands:
       - func: "setup"
       - func: "start_mongod"
-      - func: "install_c_driver"
+      - func: "fetch_c_driver_source"
       - func: "compile"
 
     - name: compile_without_tests
       commands:
       - func: "setup"
-      - func: "install_c_driver"
+      - func: "fetch_c_driver_source"
       - func: "compile"
         vars:
             ENABLE_TESTS: OFF
@@ -914,7 +931,7 @@ tasks:
         - func: "start_mongod"
           vars:
               AUTH: auth
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
         - func: "test auth"
         - func: "test atlas connectivity"
@@ -923,13 +940,14 @@ tasks:
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
           vars:
               USE_STATIC_LIBS: 1
               RUN_DISTCHECK: 1
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        - func: "install_c_driver"
         - func: "test"
           vars:
               USE_STATIC_LIBS: 1
@@ -938,19 +956,39 @@ tasks:
       commands:
         - func: "setup"
         - func: "start_mongod"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
           vars:
               BSON_EXTRA_ALIGNMENT: 1
         - func: "compile"
           vars:
               USE_STATIC_LIBS: 1
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
+        - func: "install_c_driver"
         - func: "test"
           vars:
               USE_STATIC_LIBS: 1
 
     - name: compile_and_test_with_shared_libs_replica_set
+      commands:
+        - func: "setup"
+        - func: "start_mongod"
+          vars:
+              TOPOLOGY: "replica_set"
+        - func: "fetch_c_driver_source"
+        - func: "compile"
+          vars:
+              RUN_DISTCHECK: 1
+        - func: "clone_drivers-evergreen-tools"
+        - func: "run_kms_servers"
+        - func: "install_c_driver"
+        - func: "test"
+
+    # Auto downloading the C driver in the C++ build does not currently include
+    # support for libmongocrypt, therefore it is not configured with
+    # -DENABLE_CLIENT_SIDE_ENCRYPTION=ON. For now, CSFLE tests will need to have
+    # a manually configured C driver, hence the need to seperate this task.
+    - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
       commands:
         - func: "setup"
         - func: "start_mongod"
@@ -967,7 +1005,7 @@ tasks:
     - name: uninstall_check
       commands:
       - func: "setup"
-      - func: "install_c_driver"
+      - func: "fetch_c_driver_source"
       - func: "compile"
       - command: expansions.update
         params:
@@ -987,7 +1025,7 @@ tasks:
     - name: uninstall_check_windows
       commands:
       - func: "setup"
-      - func: "install_c_driver"
+      - func: "fetch_c_driver_source"
       - func: "compile"
       - command: expansions.update
         params:
@@ -1023,7 +1061,7 @@ tasks:
             export CMAKE="$(find_cmake_latest)"
             cd build
             $CMAKE ..
-            $CMAKE --build . -- -j 8
+            $CMAKE --build . -- -j $(nproc)
             ./hello_mongocxx
 
     - name: debian-package-build
@@ -1098,7 +1136,7 @@ tasks:
     - name: test_mongohouse
       commands:
         - func: "setup"
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
         - func: "build_mongohouse"
         - func: "run_mongohouse"
@@ -1113,7 +1151,7 @@ tasks:
               # Authentication with versioned API should already be tested
               # in the C driver.
               AUTH: noauth
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
@@ -1128,7 +1166,7 @@ tasks:
           vars:
               ORCHESTRATION_FILE: versioned-api-testing.json
               AUTH: noauth
-        - func: "install_c_driver"
+        - func: "fetch_c_driver_source"
         - func: "compile"
         - func: "clone_drivers-evergreen-tools"
         - func: "run_kms_servers"
@@ -1220,7 +1258,7 @@ buildvariants:
       matrix_spec: {os: "ubuntu-1804", mongodb_version: "*"}
       display_name: "${os} replica set (MongoDB ${mongodb_version})"
       tasks:
-        - name: compile_and_test_with_shared_libs_replica_set
+        - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
 
     - matrix_name: "auth"
       matrix_spec: {os: "*", mongodb_version: "latest"}
@@ -1265,7 +1303,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment_cxx20
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - rhel90-large
@@ -1288,7 +1326,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment_cxx20
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - rhel90-arm64-large
@@ -1310,7 +1348,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment_cxx20
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - debian11-large
@@ -1354,7 +1392,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - debian10-large
@@ -1381,6 +1419,18 @@ buildvariants:
             - debian10-large
           - name: uninstall_check
 
+    # Add matrix for specification test requirement of mongocryptd:
+    # "Drivers MUST run all tests with mongocryptd on at least one platform for all tested server versions (4.2+)."
+    - matrix_name: "mongocryptd"
+      matrix_spec:
+        os: "ubuntu-1804"
+        mongodb_version: ["4.2", "4.4", "5.0", "latest"]
+      display_name: "${os} (MongoDB ${mongodb_version}) with mongocryptd"
+      tasks:
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
+      expansions:
+        use_mongocryptd: true
+
     - name: ubuntu2004-release-latest
       display_name: "Ubuntu 20.04 Release (MongoDB Latest)"
       expansions:
@@ -1396,7 +1446,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu2004-large
@@ -1439,7 +1489,7 @@ buildvariants:
           - name: compile_and_test_with_shared_libs_extra_alignment
           - name: compile_and_test_with_static_libs
           - name: compile_and_test_with_static_libs_extra_alignment
-          - name: compile_and_test_with_shared_libs_replica_set
+          - name: compile_and_test_with_shared_libs_replica_set_with_libmongocrypt
           - name: build_example_with_add_subdirectory
             distros:
             - ubuntu1804-build