CDRIVER-4369 Add FLE 2 API to AutoEncryptionOpts (#989)

* do not return error on "ns not found" error in drop with encryptedFields

* use assert_match_bson, not ASSERT(match_bson

* pass encrypted_fields_map to libmongocrypt

* add mongoc_auto_encryption_opts_set_bypass_query_analysis

* require libmongocrypt 1.5.0

* update csfle test dependency to 6.0.0-rc4

* do not spawn mongocryptd if bypassQueryAnalysis is true

Author: kevinAlbs

src/libmongoc/CMakeLists.txt

@@ -441,10 +441,10 @@ elseif (NOT ENABLE_CLIENT_SIDE_ENCRYPTION STREQUAL OFF)
       find_package (mongocrypt QUIET)
    endif ()
 
-   if (mongocrypt_FOUND AND "${mongocrypt_VERSION}" VERSION_LESS 1.3.0)
+   if (mongocrypt_FOUND AND "${mongocrypt_VERSION}" VERSION_LESS 1.5.0)
       message ("--   libmongocrypt found at ${LIBMONGOCRYPT_LIBRARY}")
       message ("--   libmongocrypt version ${mongocrypt_VERSION} found")
-      message ("--   libmongocrypt version 1.3.0 is required to enable Client-Side Field Level Encryption Support.")
+      message ("--   libmongocrypt version 1.5.0 is required to enable Client-Side Field Level Encryption Support.")
       set (REQUIRED_MONGOCRYPT_VERSION_FOUND OFF)
    elseif (mongocrypt_FOUND)
       set (REQUIRED_MONGOCRYPT_VERSION_FOUND ON)
@@ -877,7 +877,7 @@ if (MONGOC_TEST_USE_CSFLE)
       COMMAND
          "${_PYTHON3_EXE}" -u "${mongo-c-driver_SOURCE_DIR}/build/mongodl.py"
             --component csfle
-            --version 5.3.1
+            --version 6.0.0-rc4
             --edition enterprise
             --out "${CMAKE_CURRENT_BINARY_DIR}"
             --only "**/mongo_csfle_v1.*"

src/libmongoc/doc/mongoc_auto_encryption_opts_set_bypass_query_analysis.rst

@@ -0,0 +1,32 @@
+:man_page: mongoc_auto_encryption_opts_set_bypass_query_analysis
+
+mongoc_auto_encryption_opts_set_bypass_query_analysis()
+=======================================================
+
+Synopsis
+--------
+
+.. code-block:: c
+
+   void
+   mongoc_auto_encryption_opts_set_bypass_query_analysis (
+      mongoc_auto_encryption_opts_t *opts, bool bypass_query_analysis);
+
+
+Parameters
+----------
+
+* ``opts``: The :symbol:`mongoc_auto_encryption_opts_t`
+* ``bypass_query_analysis``: A boolean.
+
+
+``bypass_query_analysis`` disables automatic analysis of outgoing commands.
+``bypass_query_analysis`` is useful for encrypting indexed fields without the ``csfle`` shared library or ``mongocryptd`` process.
+Set ``bypass_query_analysis`` to true to use explicit encryption on indexed fields.
+
+.. seealso::
+
+  | :symbol:`mongoc_client_enable_auto_encryption()`
+
+  | The guide for :doc:`Using Client-Side Field Level Encryption <using_client_side_encryption>`
+

src/libmongoc/doc/mongoc_auto_encryption_opts_t.rst

@@ -36,4 +36,5 @@ Synopsis
     mongoc_auto_encryption_opts_set_extra
     mongoc_auto_encryption_opts_set_tls_opts
     mongoc_auto_encryption_opts_set_encrypted_fields_map
+    mongoc_auto_encryption_opts_set_bypass_query_analysis
 

src/libmongoc/doc/mongoc_collection_drop_with_opts.rst

@@ -59,6 +59,8 @@ If the collection does not exist, the server responds with an "ns not found" err
 
 In MongoDB 3.0 and older, the "ns not found" error code is the generic MONGOC_ERROR_QUERY_FAILURE; in this case check whether the error message is equal to the string "ns not found".
 
+The ``encryptedFields`` document in ``opts`` may be used to create a collection used for :doc:`Using Client-Side Field Level Encryption <using_client_side_encryption>`. If ``encryptedFields`` is specifed, the "ns not found" error is not returned.
+
 Errors
 ------
 

src/libmongoc/src/mongoc/mongoc-client-side-encryption.c

@@ -44,6 +44,7 @@ struct _mongoc_auto_encryption_opts_t {
    bson_t *schema_map;
    bson_t *encrypted_fields_map;
    bool bypass_auto_encryption;
+   bool bypass_query_analysis;
    bson_t *extra;
 };
 
@@ -181,6 +182,16 @@ mongoc_auto_encryption_opts_set_bypass_auto_encryption (
    opts->bypass_auto_encryption = bypass_auto_encryption;
 }
 
+void
+mongoc_auto_encryption_opts_set_bypass_query_analysis (
+   mongoc_auto_encryption_opts_t *opts, bool bypass_query_analysis)
+{
+   if (!opts) {
+      return;
+   }
+   opts->bypass_query_analysis = bypass_query_analysis;
+}
+
 void
 mongoc_auto_encryption_opts_set_extra (mongoc_auto_encryption_opts_t *opts,
                                        const bson_t *extra)
@@ -1331,18 +1342,22 @@ _mongoc_cse_client_enable_auto_encryption (mongoc_client_t *client,
    client->topology->crypt =
       _mongoc_crypt_new (opts->kms_providers,
                          opts->schema_map,
+                         opts->encrypted_fields_map,
                          opts->tls_opts,
                          client->topology->csfle_override_path,
                          client->topology->csfle_required,
                          opts->bypass_auto_encryption,
+                         opts->bypass_query_analysis,
                          error);
    if (!client->topology->crypt) {
       GOTO (fail);
    }
 
    client->topology->bypass_auto_encryption = opts->bypass_auto_encryption;
+   client->topology->bypass_query_analysis = opts->bypass_query_analysis;
 
-   if (!client->topology->bypass_auto_encryption) {
+   if (!client->topology->bypass_auto_encryption &&
+       !client->topology->bypass_query_analysis) {
       if (!client->topology->mongocryptd_bypass_spawn) {
          if (!_spawn_mongocryptd (client->topology->mongocryptd_spawn_path,
                                   client->topology->mongocryptd_spawn_args,
@@ -1485,18 +1500,21 @@ _mongoc_cse_client_pool_enable_auto_encryption (
 
    topology->crypt = _mongoc_crypt_new (opts->kms_providers,
                                         opts->schema_map,
+                                        opts->encrypted_fields_map,
                                         opts->tls_opts,
                                         topology->csfle_override_path,
                                         topology->csfle_required,
                                         opts->bypass_auto_encryption,
+                                        opts->bypass_query_analysis,
                                         error);
    if (!topology->crypt) {
       GOTO (fail);
    }
 
    topology->bypass_auto_encryption = opts->bypass_auto_encryption;
+   topology->bypass_query_analysis = opts->bypass_query_analysis;
 
-   if (!topology->bypass_auto_encryption) {
+   if (!topology->bypass_auto_encryption && !topology->bypass_query_analysis) {
       if (!topology->mongocryptd_bypass_spawn) {
          if (!_spawn_mongocryptd (topology->mongocryptd_spawn_path,
                                   topology->mongocryptd_spawn_args,
@@ -1584,10 +1602,12 @@ mongoc_client_encryption_new (mongoc_client_encryption_opts_t *opts,
    client_encryption->crypt =
       _mongoc_crypt_new (opts->kms_providers,
                          NULL /* schema_map */,
+                         NULL /* encrypted_fields_map */,
                          opts->tls_opts,
                          NULL /* No csfle path */,
                          false /* csfle not requried */,
                          true, /* bypassAutoEncryption (We are explicit) */
+                         false /* bypass_query_analysis. Not applicable. */,
                          error);
    if (!client_encryption->crypt) {
       goto fail;

src/libmongoc/src/mongoc/mongoc-client-side-encryption.h

@@ -72,6 +72,10 @@ MONGOC_EXPORT (void)
 mongoc_auto_encryption_opts_set_bypass_auto_encryption (
    mongoc_auto_encryption_opts_t *opts, bool bypass_auto_encryption);
 
+MONGOC_EXPORT (void)
+mongoc_auto_encryption_opts_set_bypass_query_analysis (
+   mongoc_auto_encryption_opts_t *opts, bool bypass_query_analysis);
+
 MONGOC_EXPORT (void)
 mongoc_auto_encryption_opts_set_extra (mongoc_auto_encryption_opts_t *opts,
                                        const bson_t *extra);

src/libmongoc/src/mongoc/mongoc-collection.c

@@ -1067,7 +1067,11 @@ drop_with_opts_with_encryptedFields (mongoc_collection_t *collection,
 
    /* Drop data collection. */
    if (!drop_with_opts (collection, opts, error)) {
-      goto fail;
+      if (error->code == MONGOC_SERVER_ERR_NS_NOT_FOUND) {
+         memset (error, 0, sizeof (bson_error_t));
+      } else {
+         goto fail;
+      }
    }
 
    /* Drop ESC collection. */
@@ -1080,7 +1084,11 @@ drop_with_opts_with_encryptedFields (mongoc_collection_t *collection,
    escCollection = mongoc_client_get_collection (
       collection->client, collection->db, escName);
    if (!drop_with_opts (escCollection, NULL /* opts */, error)) {
-      goto fail;
+      if (error->code == MONGOC_SERVER_ERR_NS_NOT_FOUND) {
+         memset (error, 0, sizeof (bson_error_t));
+      } else {
+         goto fail;
+      }
    }
 
    /* Drop ECC collection. */
@@ -1093,7 +1101,11 @@ drop_with_opts_with_encryptedFields (mongoc_collection_t *collection,
    eccCollection = mongoc_client_get_collection (
       collection->client, collection->db, eccName);
    if (!drop_with_opts (eccCollection, NULL /* opts */, error)) {
-      goto fail;
+      if (error->code == MONGOC_SERVER_ERR_NS_NOT_FOUND) {
+         memset (error, 0, sizeof (bson_error_t));
+      } else {
+         goto fail;
+      }
    }
 
    /* Drop ECOC collection. */
@@ -1106,7 +1118,11 @@ drop_with_opts_with_encryptedFields (mongoc_collection_t *collection,
    ecocCollection = mongoc_client_get_collection (
       collection->client, collection->db, ecocName);
    if (!drop_with_opts (ecocCollection, NULL /* opts */, error)) {
-      goto fail;
+      if (error->code == MONGOC_SERVER_ERR_NS_NOT_FOUND) {
+         memset (error, 0, sizeof (bson_error_t));
+      } else {
+         goto fail;
+      }
    }
 
    ok = true;

src/libmongoc/src/mongoc/mongoc-crypt-private.h

@@ -36,10 +36,12 @@ Creates a new handle into libmongocrypt.
 _mongoc_crypt_t *
 _mongoc_crypt_new (const bson_t *kms_providers,
                    const bson_t *schema_map,
+                   const bson_t *encrypted_fields_map,
                    const bson_t *tls_opts,
                    const char *csfle_override_path,
                    bool csfle_required,
                    bool bypass_auto_encryption,
+                   bool bypass_query_analysis,
                    bson_error_t *error);
 
 void

src/libmongoc/src/mongoc/mongoc-crypt.c

@@ -913,15 +913,18 @@ _parse_all_tls_opts (_mongoc_crypt_t *crypt,
 _mongoc_crypt_t *
 _mongoc_crypt_new (const bson_t *kms_providers,
                    const bson_t *schema_map,
+                   const bson_t *encrypted_fields_map,
                    const bson_t *tls_opts,
                    const char *csfle_override_path,
                    bool csfle_required,
                    bool bypass_auto_encryption,
+                   bool bypass_query_analysis,
                    bson_error_t *error)
 {
    _mongoc_crypt_t *crypt;
    mongocrypt_binary_t *local_masterkey_bin = NULL;
    mongocrypt_binary_t *schema_map_bin = NULL;
+   mongocrypt_binary_t *encrypted_fields_map_bin = NULL;
    mongocrypt_binary_t *kms_providers_bin = NULL;
    bool success = false;
 
@@ -952,6 +955,17 @@ _mongoc_crypt_new (const bson_t *kms_providers,
       }
    }
 
+   if (encrypted_fields_map) {
+      encrypted_fields_map_bin = mongocrypt_binary_new_from_data (
+         (uint8_t *) bson_get_data (encrypted_fields_map),
+         encrypted_fields_map->len);
+      if (!mongocrypt_setopt_encrypted_field_config_map (
+             crypt->handle, encrypted_fields_map_bin)) {
+         _crypt_check_error (crypt->handle, error, true);
+         goto fail;
+      }
+   }
+
    if (!bypass_auto_encryption) {
       mongocrypt_setopt_append_csfle_search_path (crypt->handle, "$SYSTEM");
       if (!_crypt_check_error (crypt->handle, error, false)) {
@@ -967,6 +981,13 @@ _mongoc_crypt_new (const bson_t *kms_providers,
       }
    }
 
+   if (bypass_query_analysis) {
+      mongocrypt_setopt_bypass_query_analysis (crypt->handle);
+      if (!_crypt_check_error (crypt->handle, error, false)) {
+         goto fail;
+      }
+   }
+
    if (!mongocrypt_init (crypt->handle)) {
       _crypt_check_error (crypt->handle, error, true);
       goto fail;
@@ -994,6 +1015,7 @@ _mongoc_crypt_new (const bson_t *kms_providers,
    success = true;
 fail:
    mongocrypt_binary_destroy (local_masterkey_bin);
+   mongocrypt_binary_destroy (encrypted_fields_map_bin);
    mongocrypt_binary_destroy (schema_map_bin);
    mongocrypt_binary_destroy (kms_providers_bin);
 

src/libmongoc/src/mongoc/mongoc-error-private.h

@@ -50,7 +50,8 @@ typedef enum {
    MONGOC_SERVER_ERR_STALECONFIG = 13388,
    MONGOC_SERVER_ERR_NOTPRIMARYNOSECONDARYOK = 13435,
    MONGOC_SERVER_ERR_NOTPRIMARYORSECONDARY = 13436,
-   MONGOC_SERVER_ERR_LEGACYNOTPRIMARY = 10058
+   MONGOC_SERVER_ERR_LEGACYNOTPRIMARY = 10058,
+   MONGOC_SERVER_ERR_NS_NOT_FOUND = 26
 } mongoc_server_err_t;
 
 mongoc_read_err_type_t

src/libmongoc/src/mongoc/mongoc-topology-private.h

@@ -185,6 +185,7 @@ typedef struct _mongoc_topology_t {
    bool mongocryptd_bypass_spawn;
    char *mongocryptd_spawn_path;
    bson_t *mongocryptd_spawn_args;
+   bool bypass_query_analysis;
 #endif
 
    // Corresponds to extraOptions.csflePath