CDRIVER-4435 Add support for GCP service accounts in GCP KMS

.evergreen/compile-test-gcpkms.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Working directory is expected to be mongo-c-driver repo.
+ROOT=$(pwd)
+INSTALL_DIR=$ROOT/install
+. .evergreen/find-cmake.sh
+echo "Installing libmongocrypt ... begin"
+git clone --depth=1 https://github.com/mongodb/libmongocrypt --branch 1.6.0
+$CMAKE -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+    -DBUILD_TESTING=OFF \
+    "-H$ROOT/libmongocrypt" \
+    "-B$ROOT/libmongocrypt"
+$CMAKE --build "$ROOT/libmongocrypt" --target install
+echo "Installing libmongocrypt ... end"
+
+echo "Compile test-gcpkms ... begin"
+# Disable unnecessary dependencies. test-gcpkms is copied to a remote host for testing, which may not have all dependent libraries.
+$CMAKE \
+    -DENABLE_SASL=OFF \
+    -DENABLE_SNAPPY=OFF \
+    -DENABLE_ZSTD=OFF \
+    -DENABLE_ZLIB=OFF \
+    -DENABLE_ICU=OFF \
+    -DENABLE_SRV=OFF \
+    -DENABLE_CLIENT_SIDE_ENCRYPTION=ON \
+    -DCMAKE_PREFIX_PATH=$INSTALL_DIR \
+    .
+$CMAKE --build . --target test-gcpkms
+echo "Compile test-gcpkms ... end"

.evergreen/compile-unix.sh

@@ -229,7 +229,7 @@ pkg-config --modversion libssl || true
 
 if [ "$COMPILE_LIBMONGOCRYPT" = "ON" ]; then
    # Build libmongocrypt, using the previously fetched installed source.
-   git clone https://github.com/mongodb/libmongocrypt --branch 1.5.2
+   git clone https://github.com/mongodb/libmongocrypt --branch 1.6.0
 
    mkdir libmongocrypt/cmake-build
    cd libmongocrypt/cmake-build

.evergreen/compile-windows.sh

@@ -120,7 +120,7 @@ fi
 
 if [ "$COMPILE_LIBMONGOCRYPT" = "ON" ]; then
    # Build libmongocrypt, using the previously fetched installed source.
-   git clone https://github.com/mongodb/libmongocrypt --branch 1.5.2
+   git clone https://github.com/mongodb/libmongocrypt --branch 1.6.0
    mkdir libmongocrypt/cmake-build
    cd libmongocrypt/cmake-build
    "$CMAKE" -G "$CC" "-DCMAKE_PREFIX_PATH=${INSTALL_DIR}/lib/cmake" -DENABLE_SHARED_BSON=ON -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" ../

.evergreen/config.yml

@@ -24180,6 +24180,67 @@ tasks:
         KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \
         EXPECT_ERROR='Error from Azure IMDS server' \
           ./mongoc/src/libmongoc/test-azurekms
+- name: testgcpkms-task
+  commands:
+  - func: fetch source
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        echo "Building test-gcpkms ... begin"
+        pushd mongoc
+        ./.evergreen/compile-test-gcpkms.sh
+        popd
+        echo "Building test-gcpkms ... end"
+        echo "Copying files ... begin"
+        export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+        export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+        export GCPKMS_ZONE=${GCPKMS_ZONE}
+        export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        mkdir testgcpkms
+        cp ./mongoc/src/libmongoc/test-gcpkms ./mongoc/install/lib/libmongocrypt.* testgcpkms
+        tar czf testgcpkms.tgz testgcpkms/*
+        GCPKMS_SRC="testgcpkms.tgz" GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
+        echo "Copying files ... end"
+        echo "Untarring file ... begin"
+        GCPKMS_CMD="tar xf testgcpkms.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+        echo "Untarring file ... end"
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+        export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+        export GCPKMS_ZONE=${GCPKMS_ZONE}
+        export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        GCPKMS_CMD="LD_LIBRARY_PATH=./testgcpkms MONGODB_URI='mongodb://localhost:27017' ./testgcpkms/test-gcpkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+- name: testgcpkms-fail-task
+  commands:
+  - func: fetch source
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        pushd mongoc
+        ./.evergreen/compile-test-gcpkms.sh
+        popd
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+        export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+        export GCPKMS_ZONE=${GCPKMS_ZONE}
+        export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+        LD_LIBRARY_PATH=$(pwd)/install MONGODB_URI='mongodb://localhost:27017' EXPECT_ERROR='Failed to connect to: metadata.google.internal' ./mongoc/src/libmongoc/test-gcpkms
 task_groups:
 - name: testazurekms_task_group
   setup_group:
@@ -24221,6 +24282,39 @@ task_groups:
   setup_group_timeout_secs: 1800
   tasks:
   - testazurekms-task
+- name: testgcpkms_task_group
+  setup_group:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
+        export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
+        export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
+        export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
+        $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh
+  - command: expansions.update
+    params:
+      file: testgcpkms-expansions.yml
+  teardown_group:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+        export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+        export GCPKMS_ZONE=${GCPKMS_ZONE}
+        export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+        $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
+  setup_group_can_fail_task: true
+  setup_group_timeout_secs: 1800
+  tasks:
+  - testgcpkms-task
 buildvariants:
 - name: releng
   display_name: '**Release Archive Creator'
@@ -24990,3 +25084,10 @@ buildvariants:
   - testazurekms_task_group
   - testazurekms-fail-task
   batchtime: 20160
+- name: testgcpkms-variant
+  display_name: GCP KMS
+  run_on: debian11-small
+  tasks:
+  - testgcpkms_task_group
+  - testgcpkms-fail-task
+  batchtime: 20160

build/cmake/LoadTests.cmake

@@ -27,9 +27,9 @@ endif ()
 string (REPLACE "\n" ";" lines "${tests_out}")
 
 # TODO: Allow individual test cases to specify the fixtures they want.
-set (all_fixtures "mongoc/fixtures/fake_imds")
+set (all_fixtures "mongoc/fixtures/fake_kms_provider_server")
 set (all_env
-    MCD_TEST_AZURE_IMDS_HOST=localhost:14987  # Refer: Fixtures.cmake
+    TEST_KMS_PROVIDER_HOST=localhost:14987  # Refer: Fixtures.cmake
     )
 
 # Generate the test definitions

build/cmake/TestFixtures.cmake

@@ -44,6 +44,6 @@ mongo_define_subprocess_fixture(
     SPAWN_WAIT 0.2
     COMMAND
         "$<TARGET_FILE:Python3::Interpreter>" -u --
-        "${_MONGOC_BUILD_SCRIPT_DIR}/bottle.py" fake_azure:imds
+        "${_MONGOC_BUILD_SCRIPT_DIR}/bottle.py" fake_kms_provider_server:kms_provider
             --bind localhost:14987  # Port 14987 chosen arbitrarily
     )

build/evergreen_config_lib/testgcpkms.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+#
+#Copyright 2022 - present MongoDB, Inc.
+#
+#Licensed under the Apache License, Version 2.0(the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http: // www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+from collections import OrderedDict as OD
+
+from evergreen_config_generator.functions import (shell_exec, func)
+from evergreen_config_generator.tasks import (NamedTask)
+from evergreen_config_generator.variants import (Variant)
+from evergreen_config_generator.taskgroups import (TaskGroup)
+
+def _create_tasks():
+    passtask = NamedTask (task_name="testgcpkms-task")
+
+    passtask.commands = [
+        func("fetch source"),
+        shell_exec (r'''
+            echo "Building test-gcpkms ... begin"
+            pushd mongoc
+            ./.evergreen/compile-test-gcpkms.sh
+            popd
+            echo "Building test-gcpkms ... end"
+            echo "Copying files ... begin"
+            export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+            export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+            export GCPKMS_ZONE=${GCPKMS_ZONE}
+            export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            mkdir testgcpkms
+            cp ./mongoc/src/libmongoc/test-gcpkms ./mongoc/install/lib/libmongocrypt.* testgcpkms
+            tar czf testgcpkms.tgz testgcpkms/*
+            GCPKMS_SRC="testgcpkms.tgz" GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
+            echo "Copying files ... end"
+            echo "Untarring file ... begin"
+            GCPKMS_CMD="tar xf testgcpkms.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+            echo "Untarring file ... end"
+            ''', test=False),
+           shell_exec (r'''
+            export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+            export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+            export GCPKMS_ZONE=${GCPKMS_ZONE}
+            export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            GCPKMS_CMD="LD_LIBRARY_PATH=./testgcpkms MONGODB_URI='mongodb://localhost:27017' ./testgcpkms/test-gcpkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+            ''')]
+
+    failtask = NamedTask(task_name="testgcpkms-fail-task")
+    failtask.commands = [
+        func("fetch source"),
+        shell_exec (r'''
+            pushd mongoc
+            ./.evergreen/compile-test-gcpkms.sh
+            popd''', test=False),
+         shell_exec (r'''
+            export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+            export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+            export GCPKMS_ZONE=${GCPKMS_ZONE}
+            export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+            LD_LIBRARY_PATH=$(pwd)/install MONGODB_URI='mongodb://localhost:27017' EXPECT_ERROR='Failed to connect to: metadata.google.internal' ./mongoc/src/libmongoc/test-gcpkms''')]
+
+    return [passtask, failtask]
+
+def _create_variant():
+        return Variant(
+        name="testgcpkms-variant",
+        display_name="GCP KMS",
+        # GCP Virtual Machine created is Debian 11.
+        run_on="debian11-small", tasks=[
+            "testgcpkms_task_group",
+            "testgcpkms-fail-task"
+            ], batchtime=20160)  # Use a batchtime of 14 days as suggested by the CSFLE test README
+
+def _create_task_group():
+    task_group = TaskGroup(name="testgcpkms_task_group")
+    task_group.setup_group_can_fail_task = True
+    task_group.setup_group_timeout_secs = 1800  # 30 minutes
+    task_group.setup_group = [
+        # Create and set up a GCE instance using driver tools script
+        shell_exec(r'''
+            git clone --depth=1 https://github.com/mongodb-labs/drivers-evergreen-tools.git
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json
+            export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json
+            export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
+            export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}"
+            $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh''', test=False),
+
+           # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_PROJECT, and GCPKMS_ZONE expansions.
+            OD([('command', 'expansions.update'),
+            ('params', OD([
+                ('file', 'testgcpkms-expansions.yml')]))])]
+
+    task_group.teardown_group = [
+        shell_exec(r'''
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            export GCPKMS_GCLOUD=${GCPKMS_GCLOUD}
+            export GCPKMS_PROJECT=${GCPKMS_PROJECT}
+            export GCPKMS_ZONE=${GCPKMS_ZONE}
+            export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
+            $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh''', test=False)
+    ]
+    task_group.tasks= ["testgcpkms-task"]
+    return task_group
+
+def testgcpkms_generate(all_tasks, all_variants, all_task_groups):
+    all_tasks.extend(_create_tasks())
+    all_variants.append(_create_variant())
+    all_task_groups.append(_create_task_group())

build/fake_azure.py → build/fake_kms_provider_server.py

@@ -8,8 +8,8 @@
 import bottle
 from bottle import Bottle, HTTPResponse
 
-imds = Bottle(autojson=True)
-"""An Azure IMDS server"""
+kms_provider = Bottle(autojson=True)
+"""A mock server for Azure IMDS and GCP metadata"""
 
 from typing import TYPE_CHECKING, Any, Callable, Iterable, cast, overload
 
@@ -83,8 +83,37 @@ def wrapped():
 def test_params() -> 'dict[str, str]':
     return parse_qs(request.headers.get('X-MongoDB-HTTP-TestParams', ''))
 
+@kms_provider.get('/computeMetadata/v1/instance/service-accounts/default/token')
+@handle_asserts
+def get_gcp_token():
+    metadata_header = request.headers.get("Metadata-Flavor")
+    assert metadata_header == 'Google'
+
+    case = test_params().get('case')
+    print('Case is:', case)
+    if case == '404': 
+        return HTTPResponse(status=404)
+
+    if case == 'bad-json':
+        return b'{"access-token": }'
+
+    if case == 'empty-json':
+        return b'{}'
+
+    if case == 'giant':
+        return _gen_giant()
+
+    if case == 'slow':
+        return _slow()
+
+    assert case in (None, ''), 'Unknown HTTP test case "{}"'.format(case)
+
+    return {
+        'access_token' : 'google-cookie',
+        'token_type' : 'Bearer'
+    }
 
-@imds.get('/metadata/identity/oauth2/token')
+@kms_provider.get('/metadata/identity/oauth2/token')
 @handle_asserts
 def get_oauth2_token():
     api_version = request.query['api-version']
@@ -145,7 +174,7 @@ def _slow() -> Iterable[bytes]:
 
 if __name__ == '__main__':
     print(
-        'RECOMMENDED: Run this script using bottle.py (e.g. [{} {}/bottle.py fake_azure:imds])'
+        'RECOMMENDED: Run this script using bottle.py (e.g. [{} {}/bottle.py fake_kms_provider_server:kms_provider])'
         .format(sys.executable,
                 Path(__file__).resolve().parent))
-    imds.run()
+    kms_provider.run()

build/generate-evergreen-config.py

@@ -42,9 +42,11 @@
 from evergreen_config_lib.tasks import all_tasks
 from evergreen_config_lib.variants import all_variants
 from evergreen_config_lib.taskgroups import all_task_groups
+from evergreen_config_lib.testgcpkms import testgcpkms_generate
 from evergreen_config_lib.testazurekms import testazurekms_generate
 
 testazurekms_generate (all_tasks, all_variants, all_task_groups)
+testgcpkms_generate(all_tasks, all_variants, all_task_groups)
 
 config = OD([
     ('stepback', True),