CDRIVER-4206 KMIP support (#881)

Co-authored-by: Ezra Chung <[email protected]>

Author: kevinAlbs

.evergreen/config.yml

@@ -439,6 +439,8 @@ functions:
           export MONGOC_TEST_AZURE_CLIENT_SECRET="${client_side_encryption_azure_client_secret}"
           export MONGOC_TEST_GCP_EMAIL="${client_side_encryption_gcp_email}"
           export MONGOC_TEST_GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
+          export MONGOC_TEST_CSFLE_TLS_CA_FILE=../drivers-evergreen-tools/.evergreen/x509gen/ca.pem
+          export MONGOC_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=../drivers-evergreen-tools/.evergreen/x509gen/client.pem
         fi
         export LOADBALANCED=${LOADBALANCED}
         export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
@@ -781,6 +783,8 @@ functions:
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 7999 &
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+        python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+        python -u kms_kmip_server.py &
         echo "Starting mock KMS servers... done."
   start load balancer:
   - command: shell.exec

.evergreen/run-tests.sh

@@ -139,6 +139,8 @@ if [ "$CLIENT_SIDE_ENCRYPTION" = "on" ]; then
    wait_for_kms_server 7999
    wait_for_kms_server 8000
    wait_for_kms_server 8001
+   wait_for_kms_server 8002
+   wait_for_kms_server 5698
    echo "Waiting for mock KMS servers to start... done."
 fi
 

CONTRIBUTING.md

@@ -251,14 +251,26 @@ start mongocryptd on port 27020 and set the following:
 
 * `MONGOC_TEST_MONGOCRYPTD_BYPASS_SPAWN=on`
 
-KMS TLS tests for Client-Side Field Level Encryption require mock KMS servers to be running in the background according to the instructions given in the Client Side Encryption Tests specification.
-The set of mock KMS servers running in the background and their corresponding port number, CA file, and cert file must be as follows:
-
-| Port | CA File | Cert File |
-| --- | --- | --- |
-| 7999 | ca.pem | server.pem |
-| 8000 | ca.pem | expired.pem |
-| 8001 | ca.pem | wrong-host.pem |
+KMS TLS tests for Client-Side Field Level Encryption require mock KMS servers to be running in the background.
+
+The [Setup instructions](https://github.com/mongodb/specifications/tree/master/source/client-side-encryption/tests#setup-3) given in the Client Side Encryption Tests specification provide additional information.
+
+The mock server scripts are located in the [mongodb-labs/drivers-evergreen-tools](https://github.com/mongodb-labs/drivers-evergreen-tools) in the [csfle directory](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/csfle). The mock servers use certificates located in the [x509gen](https://github.com/mongodb-labs/drivers-evergreen-tools/tree/master/.evergreen/x509gen) directory.
+
+The set of mock KMS servers running in the background and their corresponding invocation command must be as follows:
+
+| Port | CA File | Cert File | Command |
+| --- | --- | --- | --- |
+| 7999 | ca.pem | server.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 7999
+| 8000 | ca.pem | expired.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000
+| 8001 | ca.pem | wrong-host.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001
+| 8002 | ca.pem | server.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert
+| 5698 | ca.pem | server.pem | python -u kms_kmip_server.py
+
+The path to `ca.pem` and `client.pem` must be passed through the following environment variables:
+
+* `MONGOC_TEST_CSFLE_TLS_CA_FILE=<string>`
+* `MONGOC_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=<string>`
 
 KMS TLS tests for Client-Side Field Level Encryption can be skipped by defining:
 

build/evergreen_config_lib/functions.py

@@ -310,6 +310,9 @@
 
           export MONGOC_TEST_GCP_EMAIL="${client_side_encryption_gcp_email}"
           export MONGOC_TEST_GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
+
+          export MONGOC_TEST_CSFLE_TLS_CA_FILE=../drivers-evergreen-tools/.evergreen/x509gen/ca.pem
+          export MONGOC_TEST_CSFLE_TLS_CERTIFICATE_KEY_FILE=../drivers-evergreen-tools/.evergreen/x509gen/client.pem
         fi
         export LOADBALANCED=${LOADBALANCED}
         export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
@@ -564,6 +567,8 @@
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 7999 &
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
         python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+        python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &
+        python -u kms_kmip_server.py &
         echo "Starting mock KMS servers... done."
         ''', test=False, background=True),
     )),

src/libmongoc/CMakeLists.txt

@@ -1101,6 +1101,7 @@ if (MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION)
    mongoc_add_example (client-side-encryption-server-schema TRUE ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-server-schema.c ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-helpers.c)
    mongoc_add_example (client-side-encryption-explicit TRUE ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-explicit.c ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-helpers.c)
    mongoc_add_example (client-side-encryption-auto-decryption TRUE ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-auto-decryption.c ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-helpers.c)
+   mongoc_add_example (client-side-encryption-doc-snippets TRUE ${PROJECT_SOURCE_DIR}/examples/client-side-encryption-doc-snippets.c)
 endif ()
 
 file (COPY ${PROJECT_SOURCE_DIR}/tests/binary DESTINATION ${PROJECT_BINARY_DIR}/tests)

src/libmongoc/doc/includes/tls-options.txt

@@ -10,7 +10,7 @@
    * - MONGOC_URI_TLSCERTIFICATEKEYFILE
      - tlscertificatekeyfile
      - Path to PEM formatted Private Key, with its Public Certificate concatenated at the end.
-   * - MONGOC_URI_TLSCERTIFICATEKEYPASSWORD
+   * - MONGOC_URI_TLSCERTIFICATEKEYFILEPASSWORD
      - tlscertificatekeypassword
      - The password, if any, to use to unlock encrypted Private Key.
    * - MONGOC_URI_TLSCAFILE

src/libmongoc/doc/mongoc_auto_encryption_opts_set_kms_providers.rst

@@ -19,23 +19,52 @@ Parameters
 * ``opts``: The :symbol:`mongoc_auto_encryption_opts_t`
 * ``kms_providers``: A :symbol:`bson_t` containing configuration for an external Key Management Service (KMS).
 
-``kms_providers`` is a BSON document containing configuration for each KMS provider. Currently ``aws`` or ``local`` are supported. At least one must be specified.
+``kms_providers`` is a BSON document containing configuration for each KMS provider. Currently ``aws``, ``local``, ``azure``, ``gcp``, and ``kmip`` are supported. At least one must be specified.
 
 The format for "aws" is as follows:
 
 .. code-block:: javascript
 
    aws: {
-      accessKeyId: <string>,
-      secretAccessKey: <string>
+      accessKeyId: String,
+      secretAccessKey: String
    }
 
 The format for "local" is as follows:
 
 .. code-block:: javascript
 
    local: {
-      key: <96 byte BSON binary of subtype 0> // The master key used to encrypt/decrypt data keys.
+      key: <96 byte BSON binary of subtype 0> or String /* The master key used to encrypt/decrypt data keys. May be passed as a base64 encoded string. */
+   }
+
+The format for "azure" is as follows:
+
+.. code-block:: javascript
+
+   azure: {
+      tenantId: String,
+      clientId: String,
+      clientSecret: String,
+      identityPlatformEndpoint: Optional<String> /* Defaults to login.microsoftonline.com */
+   }
+
+The format for "gcp" is as follows:
+
+.. code-block:: javascript
+
+   gcp: {
+      email: String,
+      privateKey: byte[] or String, /* May be passed as a base64 encoded string. */
+      endpoint: Optional<String> /* Defaults to oauth2.googleapis.com */
+   }
+
+The format for "kmip" is as follows:
+
+.. code-block:: javascript
+
+   kmip: {
+      endpoint: String
    }
 
 .. seealso::

src/libmongoc/doc/mongoc_auto_encryption_opts_set_tls_opts.rst

@@ -0,0 +1,55 @@
+:man_page: mongoc_auto_encryption_opts_set_tls_opts
+
+mongoc_auto_encryption_opts_set_tls_opts()
+==========================================
+
+Synopsis
+--------
+
+.. code-block:: c
+
+   void
+   mongoc_auto_encryption_opts_set_tls_opts (
+      mongoc_auto_encryption_opts_t *opts, const bson_t *tls_opts);
+
+
+Parameters
+----------
+
+* ``opts``: The :symbol:`mongoc_auto_encryption_opts_t`
+* ``tls_opts``: A :symbol:`bson_t` mapping a Key Management Service (KMS) provider name to a BSON document with TLS options.
+
+``tls_opts`` is a BSON document of the following form:
+
+.. code-block:: javascript
+
+   <KMS provider name>: {
+      tlsCaFile: Optional<String>
+      tlsCertificateKeyFile: Optional<String>
+      tlsCertificateKeyFilePassword: Optional<String>
+   }
+
+The KMS providers ``aws``, ``azure``, ``gcp``, and ``kmip`` are supported as keys in the ``tls_opts`` document.
+
+``tls_opts`` maps the KMS provider name to a BSON document for TLS options.
+
+The BSON document for TLS options may contain the following keys:
+
+- ``MONGOC_URI_TLSCERTIFICATEKEYFILE``
+- ``MONGOC_URI_TLSCERTIFICATEKEYFILEPASSWORD``
+- ``MONGOC_URI_TLSCAFILE``
+
+.. literalinclude:: ../examples/client-side-encryption-doc-snippets.c
+   :caption: Example use
+   :start-after: BEGIN:mongoc_auto_encryption_opts_set_tls_opts
+   :end-before: END:mongoc_auto_encryption_opts_set_tls_opts
+   :dedent: 6
+
+See :doc:`configuring_tls` for a description of the behavior of these options.
+
+.. seealso::
+
+  | :symbol:`mongoc_client_enable_auto_encryption()`
+
+  | The guide for :doc:`Using Client-Side Field Level Encryption <using_client_side_encryption>`
+

src/libmongoc/doc/mongoc_auto_encryption_opts_t.rst

@@ -34,4 +34,5 @@ Synopsis
     mongoc_auto_encryption_opts_set_schema_map
     mongoc_auto_encryption_opts_set_bypass_auto_encryption
     mongoc_auto_encryption_opts_set_extra
+    mongoc_auto_encryption_opts_set_tls_opts
 

src/libmongoc/doc/mongoc_client_encryption_datakey_opts_set_masterkey.rst

@@ -23,16 +23,48 @@ Parameters
 Description
 -----------
 
-Setting the masterkey is required if using AWS KMS, and ``masterkey`` must have the form:
+Setting the masterkey is required when creating a data key with the KMS providers: ``aws``, ``azure``, ``gcp``, and ``kmip``.
+
+Setting the masterkey is prohibited with the KMS provider ``local``.
+
+The format of ``masterkey`` for "aws" is as follows:
 
 .. code-block:: javascript
 
    {
-      region: <string>, // Required.
-      key: <string>, // Required. The Amazon Resource Name (ARN) to the AWS customer master key (CMK).
-      endpoint: <string> // Optional. An alternate host identifier to send KMS requests to. May include port number.
+      region: String,
+      key: String, /* The Amazon Resource Name (ARN) to the AWS customer master key (CMK). */
+      endpoint: Optional<String> /* An alternate host identifier to send KMS requests to. May include port number. Defaults to "kms.<region>.amazonaws.com" */
    }
 
-The value of "endpoint" is a host name with optional port number separated by a colon. E.g. "kms.us-east-1.amazonaws.com" or "kms.us-east-1.amazonaws.com:443"
+The format of ``masterkey`` for "azure" is as follows:
 
-This function is only applicable for the "aws" KMS provider. It is not applicable for creating data keys with the "local" KMS provider (as configured in :symbol:`mongoc_client_encryption_opts_set_kms_providers()`).
+.. code-block:: javascript
+
+   {
+      keyVaultEndpoint: String, /* Host with optional port. Example: "example.vault.azure.net". */
+      keyName: String,
+      keyVersion: Optional<String> /* A specific version of the named key, defaults to using the key's primary version. */
+   }
+
+The format of ``masterkey`` for "gcp" is as follows:
+
+.. code-block:: javascript
+
+   {
+      projectId: String,
+      location: String,
+      keyRing: String,
+      keyName: String,
+      keyVersion: Optional<String>, /* A specific version of the named key, defaults to using the key's primary version. */
+      endpoint: Optional<String> /* Host with optional port. Defaults to "cloudkms.googleapis.com". */
+   }
+
+The format of ``masterkey`` for "kmip" is as follows:
+
+.. code-block:: javascript
+
+   {
+      keyId: Optional<String>,
+      endpoint: Optional<String> /* Host with optional port. */
+   }

src/libmongoc/doc/mongoc_client_encryption_opts_set_kms_providers.rst

@@ -18,23 +18,52 @@ Parameters
 * ``opts``: The :symbol:`mongoc_client_encryption_opts_t`
 * ``kms_providers``: A :symbol:`bson_t` containing configuration for an external Key Management Service (KMS).
 
-``kms_providers`` is a BSON document containing configuration for each KMS provider. Currently ``aws`` or ``local`` are supported. At least one must be specified.
+``kms_providers`` is a BSON document containing configuration for each KMS provider. Currently ``aws``, ``local``, ``azure``, ``gcp``, and ``kmip`` are supported. At least one must be specified.
 
 The format for "aws" is as follows:
 
 .. code-block:: javascript
 
    aws: {
-      accessKeyId: <string>,
-      secretAccessKey: <string>
+      accessKeyId: String,
+      secretAccessKey: String
    }
 
 The format for "local" is as follows:
 
 .. code-block:: javascript
 
    local: {
-      key: <96 byte BSON binary of subtype 0> // The master key used to encrypt/decrypt data keys.
+      key: <96 byte BSON binary of subtype 0> or String /* The master key used to encrypt/decrypt data keys. May be passed as a base64 encoded string. */
+   }
+
+The format for "azure" is as follows:
+
+.. code-block:: javascript
+
+   azure: {
+      tenantId: String,
+      clientId: String,
+      clientSecret: String,
+      identityPlatformEndpoint: Optional<String> /* Defaults to login.microsoftonline.com */
+   }
+
+The format for "gcp" is as follows:
+
+.. code-block:: javascript
+
+   gcp: {
+      email: String,
+      privateKey: byte[] or String, /* May be passed as a base64 encoded string. */
+      endpoint: Optional<String> /* Defaults to oauth2.googleapis.com */
+   }
+
+The format for "kmip" is as follows:
+
+.. code-block:: javascript
+
+   kmip: {
+      endpoint: String
    }
 
 

src/libmongoc/doc/mongoc_client_encryption_opts_set_tls_opts.rst

@@ -0,0 +1,53 @@
+:man_page: mongoc_client_encryption_opts_set_tls_opts
+
+mongoc_client_encryption_opts_set_tls_opts()
+============================================
+
+Synopsis
+--------
+
+.. code-block:: c
+
+   void
+   mongoc_client_encryption_opts_set_tls_opts (
+      mongoc_client_encryption_opts_t *opts, const bson_t *tls_opts);
+
+
+Parameters
+----------
+
+* ``opts``: The :symbol:`mongoc_client_encryption_opts_t`
+* ``tls_opts``: A :symbol:`bson_t` mapping a Key Management Service (KMS) provider name to a BSON document with TLS options.
+
+``tls_opts`` is a BSON document of the following form:
+
+.. code-block:: javascript
+
+   <KMS provider name>: {
+      tlsCaFile: Optional<String>
+      tlsCertificateKeyFile: Optional<String>
+      tlsCertificateKeyFilePassword: Optional<String>
+   }
+
+The KMS providers ``aws``, ``azure``, ``gcp``, and ``kmip`` are supported as keys in the ``tls_opts`` document.
+
+``tls_opts`` maps the KMS provider name to a BSON document for TLS options.
+
+The BSON document for TLS options may contain the following keys:
+
+- ``MONGOC_URI_TLSCERTIFICATEKEYFILE``
+- ``MONGOC_URI_TLSCERTIFICATEKEYFILEPASSWORD``
+- ``MONGOC_URI_TLSCAFILE``
+
+.. literalinclude:: ../examples/client-side-encryption-doc-snippets.c
+   :caption: Example use
+   :start-after: BEGIN:mongoc_client_encryption_opts_set_tls_opts
+   :end-before: END:mongoc_client_encryption_opts_set_tls_opts
+   :dedent: 6
+
+See :doc:`configuring_tls` for a description of the behavior of these options.
+
+.. seealso::
+
+  | The guide for :doc:`Using Client-Side Field Level Encryption <using_client_side_encryption>`
+

src/libmongoc/doc/mongoc_client_encryption_opts_t.rst

@@ -26,6 +26,7 @@ Used to set options for :symbol:`mongoc_client_encryption_new()`.
     mongoc_client_encryption_opts_set_keyvault_client
     mongoc_client_encryption_opts_set_keyvault_namespace
     mongoc_client_encryption_opts_set_kms_providers
+    mongoc_client_encryption_opts_set_tls_opts
 
 .. seealso::