CDRIVER-4454 add integration tests for Azure KMS

.evergreen/compile-test-azurekms.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# Working directory is expected to be mongo-c-driver repo.
+ROOT=$(pwd)
+INSTALL_DIR=$ROOT/install
+. .evergreen/find-cmake.sh
+echo "Installing libmongocrypt ... begin"
+git clone https://github.com/mongodb/libmongocrypt --branch 1.6.0
+$CMAKE -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \
+    -DBUILD_TESTING=OFF \
+    "-H$ROOT/libmongocrypt" \
+    "-B$ROOT/libmongocrypt"
+$CMAKE --build "$ROOT/libmongocrypt" --target install
+echo "Installing libmongocrypt ... end"
+
+echo "Compile test-azurekms ... begin"
+# Disable unnecessary dependencies. test-azurekms is copied to a remote host for testing, which may not have all dependent libraries.
+$CMAKE \
+    -DENABLE_SASL=OFF \
+    -DENABLE_SNAPPY=OFF \
+    -DENABLE_ZSTD=OFF \
+    -DENABLE_ZLIB=OFF \
+    -DENABLE_ICU=OFF \
+    -DENABLE_SRV=OFF \
+    -DENABLE_CLIENT_SIDE_ENCRYPTION=ON \
+    -DCMAKE_PREFIX_PATH=$INSTALL_DIR \
+    .
+$CMAKE --build . --target test-azurekms
+echo "Compile test-azurekms ... end"

.evergreen/config.yml

@@ -24116,6 +24116,111 @@ tasks:
       AUTH: noauth
       LOADBALANCED: loadbalanced
       SSL: nossl
+- name: testazurekms-task
+  commands:
+  - func: fetch source
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        echo "Building test-azurekms ... begin"
+        pushd mongoc
+        ./.evergreen/compile-test-azurekms.sh
+        popd
+        echo "Building test-azurekms ... end"
+        echo "Copying files ... begin"
+        export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+        export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+        export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        mkdir testazurekms
+        cp ./mongoc/src/libmongoc/test-azurekms ./mongoc/install/lib/libmongocrypt.* testazurekms
+        tar czf testazurekms.tgz testazurekms/*
+        AZUREKMS_SRC="testazurekms.tgz" \
+        AZUREKMS_DST="./" \
+          $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
+        echo "Copying files ... end"
+        echo "Untarring file ... begin"
+        AZUREKMS_CMD="tar xf testazurekms.tgz" \
+          $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+        echo "Untarring file ... end"
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+        export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+        export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./testazurekms/test-azurekms" \
+          $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+- name: testazurekms-fail-task
+  commands:
+  - func: fetch source
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        pushd mongoc
+        ./.evergreen/compile-test-azurekms.sh
+        popd
+  - command: shell.exec
+    type: test
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        LD_LIBRARY_PATH=$PWD/install \
+        MONGODB_URI='mongodb://localhost:27017' \
+        KEY_NAME='${testazurekms_keyname}' \
+        KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \
+        EXPECT_ERROR='Error from Azure IMDS server' \
+          ./mongoc/src/libmongoc/test-azurekms
+task_groups:
+- name: testazurekms_task_group
+  setup_group:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git --depth=1 drivers-evergreen-tools
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+        echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey
+        echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey
+        # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open".
+        chmod 600 /tmp/testazurekms_privatekey
+        export AZUREKMS_CLIENTID=${testazurekms_clientid}
+        export AZUREKMS_TENANTID=${testazurekms_tenantid}
+        export AZUREKMS_SECRET=${testazurekms_secret}
+        export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
+        export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+        export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey
+        export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+        export AZUREKMS_SCOPE=${testazurekms_scope}
+        export AZUREKMS_VMNAME_PREFIX=CDRIVER
+        $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh
+  - command: expansions.update
+    params:
+      file: testazurekms-expansions.yml
+  teardown_group:
+  - command: shell.exec
+    params:
+      shell: bash
+      script: |-
+        set -o errexit
+        DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+            export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+            $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh
+  setup_group_can_fail_task: true
+  setup_group_timeout_secs: 1800
+  tasks:
+  - testazurekms-task
 buildvariants:
 - name: releng
   display_name: '**Release Archive Creator'
@@ -24878,3 +24983,10 @@ buildvariants:
   run_on: macos-1100-arm64
   tasks:
   - debug-compile-sasl-darwinssl
+- name: testazurekms-variant
+  display_name: Azure KMS
+  run_on: debian10-small
+  tasks:
+  - testazurekms_task_group
+  - testazurekms-fail-task
+  batchtime: 20160

.evergreen/find-cmake.sh

@@ -3,6 +3,8 @@ set -o errexit  # Exit the script with error if any of the commands fail
 
 find_cmake ()
 {
+  CMAKE=${CMAKE:-}
+  IGNORE_SYSTEM_CMAKE=${IGNORE_SYSTEM_CMAKE:-}
   if [ ! -z "$CMAKE" ]; then
     return 0
   elif [ -f "/Applications/cmake-3.2.2-Darwin-x86_64/CMake.app/Contents/bin/cmake" ]; then

build/evergreen_config_generator/taskgroups.py

@@ -0,0 +1,48 @@
+# Copyright 2018-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from evergreen_config_generator import ConfigObject
+
+
+class TaskGroup(ConfigObject):
+    def __init__(self, name):
+        super(TaskGroup, self).__init__()
+        self._task_group_name = name
+
+    @property
+    def name(self):
+        return self._task_group_name
+
+    def to_dict(self):
+        v = super(TaskGroup, self).to_dict()
+        # See possible TaskGroup attributes from the Evergreen wiki:
+        # https://github.com/evergreen-ci/evergreen/wiki/Project-Configuration-Files#task-groups
+        attrs = [
+            'setup_group',
+            'teardown_group',
+            'setup_task',
+            'teardown_task',
+            'max_hosts',
+            'timeout',
+            'setup_group_can_fail_task',
+            'setup_group_timeout_secs',
+            'setup_group_can_fail_task',
+            'share_processes',
+            'tasks'
+        ]        
+
+        for i in attrs:
+            if getattr(self, i, None):
+                v[i] = getattr(self, i)
+        return v

build/evergreen_config_lib/taskgroups.py

@@ -0,0 +1,15 @@
+# Copyright 2018-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+all_task_groups = []

build/evergreen_config_lib/testazurekms.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+#
+# Copyright 2022-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from collections import OrderedDict as OD
+
+from evergreen_config_generator.functions import (shell_exec, func)
+from evergreen_config_generator.tasks import (NamedTask)
+from evergreen_config_generator.variants import (Variant)
+from evergreen_config_generator.taskgroups import (TaskGroup)
+
+def _create_tasks():
+    # passtask is expected to run on a remote Azure VM and succeed at obtaining credentials.
+    passtask = NamedTask (task_name="testazurekms-task")
+    passtask.commands = [
+        func("fetch source"),
+        shell_exec (r'''
+          echo "Building test-azurekms ... begin"
+          pushd mongoc
+          ./.evergreen/compile-test-azurekms.sh
+          popd
+          echo "Building test-azurekms ... end"
+
+          echo "Copying files ... begin"
+          export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+          export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+          export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+          DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+          mkdir testazurekms
+          cp ./mongoc/src/libmongoc/test-azurekms ./mongoc/install/lib/libmongocrypt.* testazurekms
+          tar czf testazurekms.tgz testazurekms/*
+          AZUREKMS_SRC="testazurekms.tgz" \
+          AZUREKMS_DST="./" \
+            $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
+          echo "Copying files ... end"
+
+          echo "Untarring file ... begin"
+          AZUREKMS_CMD="tar xf testazurekms.tgz" \
+            $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+          echo "Untarring file ... end"
+        ''', test=False),
+        shell_exec (r'''
+          export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+          export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+          export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+          DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+          AZUREKMS_CMD="LD_LIBRARY_PATH=./testazurekms MONGODB_URI='mongodb://localhost:27017' KEY_NAME='${testazurekms_keyname}' KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' ./testazurekms/test-azurekms" \
+            $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+        ''')
+    ]
+
+    failtask = NamedTask (task_name="testazurekms-fail-task")
+    failtask.commands = [
+        func("fetch source"),
+        shell_exec (r'''
+          pushd mongoc
+          ./.evergreen/compile-test-azurekms.sh
+          popd
+        ''', test=False),
+        shell_exec (r'''
+          LD_LIBRARY_PATH=$PWD/install \
+          MONGODB_URI='mongodb://localhost:27017' \
+          KEY_NAME='${testazurekms_keyname}' \
+          KEY_VAULT_ENDPOINT='${testazurekms_keyvaultendpoint}' \
+          EXPECT_ERROR='Error from Azure IMDS server' \
+            ./mongoc/src/libmongoc/test-azurekms
+        ''')
+    ]
+    return [passtask, failtask]
+
+
+def _create_variant():
+    return Variant(
+        name="testazurekms-variant",
+        display_name="Azure KMS",
+        # Azure Virtual Machine created is Debian 10.
+        run_on="debian10-small", tasks=[
+            "testazurekms_task_group",
+            "testazurekms-fail-task"
+        ], batchtime=20160)  # Use a batchtime of 14 days as suggested by the CSFLE test README
+
+
+def _create_task_group():
+    task_group = TaskGroup(name="testazurekms_task_group")
+    task_group.setup_group_can_fail_task = True
+    task_group.setup_group_timeout_secs = 1800  # 30 minutes
+    task_group.setup_group = [
+        shell_exec(r'''
+            git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git --depth=1 drivers-evergreen-tools
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+            echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey
+            echo '${testazurekms_privatekey}' > /tmp/testazurekms_privatekey
+            # Set 600 permissions on private key file. Otherwise ssh / scp may error with permissions "are too open".
+            chmod 600 /tmp/testazurekms_privatekey
+            export AZUREKMS_CLIENTID=${testazurekms_clientid}
+            export AZUREKMS_TENANTID=${testazurekms_tenantid}
+            export AZUREKMS_SECRET=${testazurekms_secret}
+            export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS
+            export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+            export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey
+            export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey
+            export AZUREKMS_SCOPE=${testazurekms_scope}
+            export AZUREKMS_VMNAME_PREFIX=CDRIVER
+            $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh
+            ''', test=False),
+        # Load the AZUREKMS_VMNAME expansion.
+        OD([('command', 'expansions.update'),
+            ('params', OD([
+                ('file', 'testazurekms-expansions.yml'),
+            ]))])
+    ]
+
+    task_group.teardown_group = [
+        shell_exec(r'''
+            DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools
+                export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
+                export AZUREKMS_RESOURCEGROUP=${testazurekms_resourcegroup}
+                $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh
+            ''', test=False)
+    ]
+    task_group.tasks = ["testazurekms-task"]
+    return task_group
+
+
+def testazurekms_generate(all_tasks, all_variants, all_task_groups):
+    all_tasks.extend(_create_tasks())
+    all_variants.append(_create_variant())
+    all_task_groups.append(_create_task_group())

build/generate-evergreen-config.py

@@ -41,6 +41,10 @@
 from evergreen_config_lib.functions import all_functions
 from evergreen_config_lib.tasks import all_tasks
 from evergreen_config_lib.variants import all_variants
+from evergreen_config_lib.taskgroups import all_task_groups
+from evergreen_config_lib.testazurekms import testazurekms_generate
+
+testazurekms_generate (all_tasks, all_variants, all_task_groups)
 
 config = OD([
     ('stepback', True),
@@ -66,6 +70,7 @@
         OD([('func', 'backtrace')])
     ]),
     ('tasks', all_tasks),
+    ('task_groups', all_task_groups),
     ('buildvariants', all_variants),
 ])
 

src/libmongoc/CMakeLists.txt

@@ -1059,6 +1059,7 @@ endif ()
 mongoc_add_test (test-libmongoc FALSE ${test-libmongoc-sources})
 mongoc_add_test (test-mongoc-gssapi FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongoc-gssapi.c)
 mongoc_add_test (test-mongoc-cache FALSE ${PROJECT_SOURCE_DIR}/tests/test-mongoc-cache.c)
+mongoc_add_test (test-azurekms FALSE ${PROJECT_SOURCE_DIR}/tests/test-azurekms.c)
 
 if (ENABLE_TESTS)
    # "make test" doesn't compile tests, so we create "make check" which compiles