RUBY-2833 Add KMIP Support (#2383)

Author: comandeo-mongo

.evergreen/config-atlas.yml

@@ -420,6 +420,20 @@ functions:
           ${PREPARE_SHELL}
           SERVERLESS=1 SSL=ssl RVM_RUBY="${RVM_RUBY}" SINGLE_MONGOS="${SINGLE_MONGOS}" MULTI_ATLASPROXY_SERVERLESS_URI="${MULTI_ATLASPROXY_SERVERLESS_URI}" SINGLE_ATLASPROXY_SERVERLESS_URI="${SINGLE_ATLASPROXY_SERVERLESS_URI}" .evergreen/run-tests-serverless.sh
 
+  "start kmip server":
+    - command: shell.exec
+      params:
+        background: true
+        shell: bash
+        working_dir: "src"
+        script: |
+          . .evergreen/csfle/activate_venv.sh
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/expired.pem --port 8000 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/wrong-host.pem --port 8001 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert &
+          python -u .evergreen/csfle/kms_kmip_server.py &
+
 pre:
   - func: "fetch source"
   - func: "fetch egos"

.evergreen/config.yml

@@ -437,6 +437,20 @@ functions:
           ${PREPARE_SHELL}
           SERVERLESS=1 SSL=ssl RVM_RUBY="${RVM_RUBY}" SINGLE_MONGOS="${SINGLE_MONGOS}" MULTI_ATLASPROXY_SERVERLESS_URI="${MULTI_ATLASPROXY_SERVERLESS_URI}" SINGLE_ATLASPROXY_SERVERLESS_URI="${SINGLE_ATLASPROXY_SERVERLESS_URI}" .evergreen/run-tests-serverless.sh
 
+  "start kmip server":
+    - command: shell.exec
+      params:
+        background: true
+        shell: bash
+        working_dir: "src"
+        script: |
+          . .evergreen/csfle/activate_venv.sh
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/expired.pem --port 8000 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/wrong-host.pem --port 8001 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert &
+          python -u .evergreen/csfle/kms_kmip_server.py &
+
 pre:
   - func: "fetch source"
   - func: "fetch egos"
@@ -471,6 +485,7 @@ tasks:
   - name: "test-fle"
     commands:
       - func: "export FLE credentials"
+      - func: "start kmip server"
       - func: "run tests"
   - name: "test-aws-auth"
     commands:

.evergreen/config/common.yml.erb

@@ -502,6 +502,20 @@ functions:
           ${PREPARE_SHELL}
           SERVERLESS=1 SSL=ssl RVM_RUBY="${RVM_RUBY}" SINGLE_MONGOS="${SINGLE_MONGOS}" MULTI_ATLASPROXY_SERVERLESS_URI="${MULTI_ATLASPROXY_SERVERLESS_URI}" SINGLE_ATLASPROXY_SERVERLESS_URI="${SINGLE_ATLASPROXY_SERVERLESS_URI}" .evergreen/run-tests-serverless.sh
 
+  "start kmip server":
+    - command: shell.exec
+      params:
+        background: true
+        shell: bash
+        working_dir: "src"
+        script: |
+          . .evergreen/csfle/activate_venv.sh
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 7999 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/expired.pem --port 8000 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/wrong-host.pem --port 8001 &
+          python -u .evergreen/csfle/kms_http_server.py --ca_file .evergreen/x509gen/ca.pem --cert_file .evergreen/x509gen/server.pem --port 8002 --require_client_cert &
+          python -u .evergreen/csfle/kms_kmip_server.py &
+
 pre:
   - func: "fetch source"
   - func: "fetch egos"
@@ -547,6 +561,7 @@ tasks:
   - name: "test-fle"
     commands:
       - func: "export FLE credentials"
+      - func: "start kmip server"
       - func: "run tests"
   - name: "test-aws-auth"
     commands:

.evergreen/csfle

@@ -0,0 +1 @@
+../.mod/drivers-evergreen-tools/.evergreen/csfle

.evergreen/run-tests.sh

@@ -139,6 +139,24 @@ if test -n "$FLE"; then
 
   export LIBMONGOCRYPT_PATH=`pwd`/rhel-70-64-bit/nocrypto/lib64/libmongocrypt.so
   test -f "$LIBMONGOCRYPT_PATH"
+
+  echo "Waiting for mock KMS servers to start..."
+   wait_for_kms_server() {
+      for i in $(seq 60); do
+         if curl -s "localhost:$1"; test $? -ne 7; then
+            return 0
+         else
+            sleep 1
+         fi
+      done
+      echo "Could not detect mock KMS server on port $1"
+      return 1
+   }
+   wait_for_kms_server 8000
+   wait_for_kms_server 8001
+   wait_for_kms_server 8002
+   wait_for_kms_server 5698
+   echo "Waiting for mock KMS servers to start... done."
 fi
 
 if test -n "$OCSP_CONNECTIVITY"; then

.evergreen/x509gen

@@ -0,0 +1 @@
+../.mod/drivers-evergreen-tools/.evergreen/x509gen/

docs/reference/client-side-encryption.txt

@@ -403,6 +403,12 @@ Please note that GCP private key can be in different formats. Ruby driver
 supports DER encoded RSA private key as base64 encoded string. For MRI Ruby
 the driver additionally support PEM encoded RSA private key.
 
+If you have created a master key using a Key Management Interoperability
+Protocol (KMIP) compatible key management server, note the server host and port,
+and key id. You will use that information to generate a data key. You may also
+need certificate authority certificate(s), as well as  and your client
+certificate and private key to authenticate to KMIP server.
+
 
 .. code-block:: ruby
 
@@ -430,6 +436,18 @@ the driver additionally support PEM encoded RSA private key.
         # :private_key value should be GCP private key as base64 encoded
         # DER RSA private key, or PEM RSA private key, if you are using MRI Ruby.
         private_key: 'GCP-PRIVATE-KEY',
+      },
+      kmip: {
+        # KMIP server endpoint may include port.
+        endpoint: 'KMIP-SERVER-HOST'
+      },
+      # TLS options to connect to KMIP server.
+      kms_tls_options: {
+        kmip: {
+          ssl_ca_cert: 'PATH-TO-CA-FILE',
+          ssl_cert: 'PATH-TO-CLIENT-CERT-FILE',
+          ssl_key: 'PATH-TO-CLIENT-KEY-FILE'
+        }
       }
     }
   )
@@ -477,6 +495,11 @@ For more information about creating a data key, see the
 :drivers:`Create a Data Encryption Key </security/client-side-field-level-encryption-guide/#b.-create-a-data-encryption-key>`
 section of the MongoDB manual.
 
+For a list of possible KMS TLS options
+see :manual:`create client reference </reference/config-database/>`.
+``Mongo::ClientEncryption`` constructor accepts same ``ssl_`` options as
+``Mongo::Client``.
+
 Auto-Encryption Options
 =======================
 Automatic encryption can be configured on a ``Mongo::Client`` using the
@@ -523,6 +546,50 @@ keys are stored in the ``admin`` database in the ``datakeys`` collection:
 
 There is no default key vault namespace, and this option must be provided.
 
+``:kms_providers``
+~~~~~~~~~~~~~~~~~~
+A Hash that contains KMP provider names as keys, and provider options as values.
+
+.. code-block:: ruby
+
+  Mongo::Client.new(['localhost:27017],
+    auto_encryption_options: {
+      key_vault_namespace: 'admin.datakeys',
+      kms_providers: {
+        aws: {
+          access_key_id: 'IAM-ACCESS-KEY-ID',
+          secret_access_key: 'IAM-SECRET-ACCESS-KEY'
+        }
+      }
+    }
+  )
+
+``:kms_tls_options``
+~~~~~~~~~~~~~~~~~~
+A Hash that contains KMP provider names as keys, and TLS options to connect to
+corresponding providers.
+
+.. code-block:: ruby
+
+  Mongo::Client.new(['localhost:27017],
+    auto_encryption_options: {
+      key_vault_namespace: 'admin.datakeys',
+      kms_providers: {
+        kmip: {
+          endpoint: 'KMIP-SERVER-HOST'
+        }
+      },
+      kms_tls_options: {
+        kmip: {
+          ssl_ca_cert: 'PATH-TO-CA-FILE',
+          ssl_cert: 'PATH-TO-CLIENT-CERT-FILE',
+          ssl_key: 'PATH-TO-CLIENT-KEY-FILE'
+        }
+      }
+    }
+  )
+
+
 ``:schema_map``
 ~~~~~~~~~~~~~~~
 A schema map is a Hash with information about which fields to automatically

docs/release-notes.txt

@@ -17,6 +17,14 @@ comprehensive list of changes in each version of the driver and the
 for the complete list of changes, including those internal to the driver and
 its test suite.
 
+2.18
+====
+
+This release includes the following new features:
+
+- Added support for Azure Key Vault, Google Cloud Key Management, and any
+  KMIP compliant Key Management System to be used as master key storage for
+  client side encryption.
 
 2.17
 ====
@@ -148,7 +156,7 @@ functionality has been added:
   cross-driver mechanims to discover deployment topology or force direct
   connection.
 - Support for :ref:`MONGODB-AWS authentication mechanism <aws-auth>`.
-- When SCRAM authentication is used with 4.4 and newer servers, the driver will 
+- When SCRAM authentication is used with 4.4 and newer servers, the driver will
   complete authentication with fewer network roundtrips.
 - The driver creates an additional monitoring connection for 4.4 and newer
   servers, permitting the server to notify the driver when its state changes.
@@ -265,7 +273,7 @@ This release adds the following new features:
 
 The following smaller improvements have been made:
 
-- Support for the ``startAfter`` option in the ``$changeStream``        
+- Support for the ``startAfter`` option in the ``$changeStream``
   aggregation pipeline stage.
 - Field order of BSON documents sent to the server changed for better logging.
 - Certificate paths with unescaped slashes can now be specified in

lib/mongo/client.rb

@@ -429,9 +429,13 @@ def hash
     #     instance containing the encryption key vault
     #   - :key_vault_namespace => String, the namespace of the key vault in the
     #     format database.collection
-    #   - :kms_providers => Hash, A hash of key management service configuration
-    #     information. Valid hash keys are :local or :aws. There may be more
-    #     than one kms provider specified.
+    #   - :kms_providers => Hash, A hash of key management service (KMS) configuration
+    #     information. Valid hash keys are :aws, :azure, :gcp, :kmip, :local.
+    #     There may be more than one kms provider specified.
+    #   - :kms_tls_options => Hash, A hash of TLS options to authenticate to
+    #     KMS providers, usually used for KMIP servers. Valid hash keys
+    #     are :aws, :azure, :gcp, :kmip, :local. There may be more than one
+    #     kms provider specified.
     #   - :schema_map => Hash | nil, JSONSchema for one or more collections
     #     specifying which fields should be encrypted.
     #     - Note: Schemas supplied in the schema_map only apply to configuring

lib/mongo/crypt/encryption_io.rb

@@ -265,7 +265,7 @@ def with_ssl_socket(endpoint, tls_options)
         )
         yield(mongo_socket.socket)
       rescue => e
-        raise Error::KmsError, "Error decrypting data key: #{e.class}: #{e.message}"
+        raise Error::KmsError, "Error when connecting to KMS provider: #{e.class}: #{e.message}"
       ensure
         mongo_socket&.close
       end

lib/mongo/crypt/kms.rb

@@ -113,4 +113,5 @@ def validate_tls_options(options)
 require 'mongo/crypt/kms/aws'
 require 'mongo/crypt/kms/azure'
 require 'mongo/crypt/kms/gcp'
+require 'mongo/crypt/kms/kmip'
 require 'mongo/crypt/kms/local'

lib/mongo/crypt/kms/credentials.rb

@@ -49,11 +49,17 @@ def initialize(kms_providers)
           if kms_providers.key?(:gcp)
             @gcp = GCP::Credentials.new(kms_providers[:gcp])
           end
+          if kms_providers.key?(:kmip)
+            @kmip = KMIP::Credentials.new(kms_providers[:kmip])
+          end
           if kms_providers.key?(:local)
             @local = Local::Credentials.new(kms_providers[:local])
           end
-          if @aws.nil? && @azure.nil? && @gcp.nil? && @local.nil?
-            raise ArgumentError.new("KMS providers options must have one of the following keys: :aws, :azure, :gcp, :local")
+          if @aws.nil? && @azure.nil? && @gcp.nil? && @kmip.nil? && @local.nil?
+            raise ArgumentError.new(
+              "KMS providers options must have one of the following keys: " +
+              ":aws, :azure, :gcp, :kmip, :local"
+            )
           end
         end
 
@@ -65,6 +71,7 @@ def to_document
             bson[:aws] = @aws.to_document if @aws
             bson[:azure] = @azure.to_document if @azure
             bson[:gcp] = @gcp.to_document if @gcp
+            bson[:kmip] = @kmip.to_document if @kmip
             bson[:local] = @local.to_document if @local
           end
         end