MONGOCRYPT-588 support bulkWrite (#745)

Support processing `bulkWrite`:

* add `MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB` state to support obtaining remote schema on a target database differing from the command database

* add `mongocrypt_setopt_use_need_mongo_collinfo_with_db_state`

Improve test runner:

* identify missing key in error message

* propagate error message when matching BSON arrays

* retain dots in keys when matching

* fix test data for `fle2-explain`

Author: kevinAlbs

integrating.md

@@ -137,6 +137,23 @@ A result from a listCollections cursor.
 
 auto encrypt
 
+#### State: `MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB` ####
+
+**libmongocrypt needs**...
+
+Results from a listCollections cursor from a specified database.
+
+**Driver needs to...**
+
+1.  Run listCollections on the encrypted MongoClient with the filter
+    provided by `mongocrypt_ctx_mongo_op` on the database provided by `mongocrypt_ctx_mongo_db`.
+2.  Return the first result (if any) with `mongocrypt_ctx_mongo_feed` or proceed to the next step if nothing was returned.
+3.  Call `mongocrypt_ctx_mongo_done`
+
+**Applies to...**
+
+A context initialized with `mongocrypt_ctx_encrypt_init` for automatic encryption. This state is only entered when `mongocrypt_setopt_use_need_mongo_collinfo_with_db_state` is called to opt-in.
+
 #### State: `MONGOCRYPT_CTX_NEED_MONGO_MARKINGS` ####
 
 **libmongocrypt needs**...

src/mongocrypt-ctx-encrypt.c

@@ -89,6 +89,7 @@ typedef struct __mongocrypt_ctx_opts_t {
 
 /* All derived contexts may override these methods. */
 typedef struct {
+    const char *(*mongo_db_collinfo)(mongocrypt_ctx_t *ctx);
     bool (*mongo_op_collinfo)(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *out);
     bool (*mongo_feed_collinfo)(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *in);
     bool (*mongo_done_collinfo)(mongocrypt_ctx_t *ctx);
@@ -134,9 +135,22 @@ bool _mongocrypt_ctx_fail_w_msg(mongocrypt_ctx_t *ctx, const char *msg);
 typedef struct {
     mongocrypt_ctx_t parent;
     bool explicit;
-    char *coll_name;
-    char *db_name;
-    char *ns;
+
+    // `cmd_db` is the command database (appended as `$db`).
+    char *cmd_db;
+
+    // `target_ns` is the target namespace "<target_db>.<target_coll>" for the operation. May be associated with
+    // jsonSchema (CSFLE) or encryptedFields (QE). For `bulkWrite`, the target namespace database may differ from
+    // `cmd_db`.
+    char *target_ns;
+
+    // `target_db` is the target database for the operation. For `bulkWrite`, the target namespace database may differ
+    // from `cmd_db`. If `target_db` is NULL, the target namespace database is the same as `cmd_db`.
+    char *target_db;
+
+    // `target_coll` is the target namespace collection name.
+    char *target_coll;
+
     _mongocrypt_buffer_t list_collections_filter;
     _mongocrypt_buffer_t schema;
     /* TODO CDRIVER-3150: audit + rename these buffers.
@@ -161,13 +175,19 @@ typedef struct {
      * schema, and there were siblings. */
     bool collinfo_has_siblings;
     /* encrypted_field_config is set when:
-     * 1. <db_name>.<coll_name> is present in an encrypted_field_config_map.
+     * 1. `target_ns` is present in an encrypted_field_config_map.
      * 2. (TODO MONGOCRYPT-414) The collection has encryptedFields in the
      * response to listCollections. encrypted_field_config is true if and only if
      * encryption is using FLE 2.0.
+     * 3. The `bulkWrite` command is processed and needs an empty encryptedFields to be processed by query analysis.
+     * (`bulkWrite` does not support empty JSON schema).
      */
     _mongocrypt_buffer_t encrypted_field_config;
     mc_EncryptedFieldConfig_t efc;
+    // `used_empty_encryptedFields` is true if the collection has no JSON schema or encryptedFields,
+    // yet an empty encryptedFields was constructed to support query analysis.
+    // When true, an empty encryptedFields is sent to query analysis, but not appended to the final command.
+    bool used_empty_encryptedFields;
     /* bypass_query_analysis is set to true to skip the
      * MONGOCRYPT_CTX_NEED_MONGO_MARKINGS state. */
     bool bypass_query_analysis;

src/mongocrypt-ctx-private.h

@@ -386,6 +386,7 @@ bool mongocrypt_ctx_mongo_op(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *out) {
     }
 
     switch (ctx->state) {
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: CHECK_AND_CALL(mongo_op_collinfo, ctx, out);
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS: CHECK_AND_CALL(mongo_op_markings, ctx, out);
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS: CHECK_AND_CALL(mongo_op_keys, ctx, out);
@@ -398,6 +399,38 @@ bool mongocrypt_ctx_mongo_op(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *out) {
     }
 }
 
+const char *mongocrypt_ctx_mongo_db(mongocrypt_ctx_t *ctx) {
+    if (!ctx) {
+        return NULL;
+    }
+    if (!ctx->initialized) {
+        _mongocrypt_ctx_fail_w_msg(ctx, "ctx NULL or uninitialized");
+        return NULL;
+    }
+
+    switch (ctx->state) {
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB: {
+        if (!ctx->vtable.mongo_db_collinfo) {
+            _mongocrypt_ctx_fail_w_msg(ctx, "not applicable to context");
+            return NULL;
+        }
+        return ctx->vtable.mongo_db_collinfo(ctx);
+    }
+    case MONGOCRYPT_CTX_ERROR: return false;
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO:
+    case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS:
+    case MONGOCRYPT_CTX_NEED_MONGO_KEYS:
+    case MONGOCRYPT_CTX_DONE:
+    case MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS:
+    case MONGOCRYPT_CTX_NEED_KMS:
+    case MONGOCRYPT_CTX_READY:
+    default: {
+        _mongocrypt_ctx_fail_w_msg(ctx, "wrong state");
+        return NULL;
+    }
+    }
+}
+
 bool mongocrypt_ctx_mongo_feed(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *in) {
     if (!ctx) {
         return false;
@@ -419,6 +452,7 @@ bool mongocrypt_ctx_mongo_feed(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *in) {
     }
 
     switch (ctx->state) {
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: CHECK_AND_CALL(mongo_feed_collinfo, ctx, in);
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS: CHECK_AND_CALL(mongo_feed_markings, ctx, in);
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS: CHECK_AND_CALL(mongo_feed_keys, ctx, in);
@@ -440,6 +474,7 @@ bool mongocrypt_ctx_mongo_done(mongocrypt_ctx_t *ctx) {
     }
 
     switch (ctx->state) {
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO: CHECK_AND_CALL(mongo_done_collinfo, ctx);
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS: CHECK_AND_CALL(mongo_done_markings, ctx);
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS: CHECK_AND_CALL(mongo_done_keys, ctx);
@@ -483,6 +518,7 @@ mongocrypt_kms_ctx_t *mongocrypt_ctx_next_kms_ctx(mongocrypt_ctx_t *ctx) {
     case MONGOCRYPT_CTX_ERROR: return NULL;
     case MONGOCRYPT_CTX_DONE:
     case MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS:
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO:
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS:
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS:
@@ -554,6 +590,7 @@ bool mongocrypt_ctx_kms_done(mongocrypt_ctx_t *ctx) {
     case MONGOCRYPT_CTX_ERROR: return false;
     case MONGOCRYPT_CTX_DONE:
     case MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS:
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO:
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS:
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS:
@@ -584,6 +621,7 @@ bool mongocrypt_ctx_finalize(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *out) {
     case MONGOCRYPT_CTX_DONE:
     case MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS:
     case MONGOCRYPT_CTX_NEED_KMS:
+    case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB:
     case MONGOCRYPT_CTX_NEED_MONGO_COLLINFO:
     case MONGOCRYPT_CTX_NEED_MONGO_KEYS:
     case MONGOCRYPT_CTX_NEED_MONGO_MARKINGS:

src/mongocrypt-ctx.c

@@ -115,6 +115,7 @@ typedef struct {
     mstr crypt_shared_lib_override_path;
 
     bool use_need_kms_credentials_state;
+    bool use_need_mongo_collinfo_with_db_state;
     bool bypass_query_analysis;
 
     // When creating new encrypted payloads,

src/mongocrypt-opts-private.h

@@ -1143,6 +1143,12 @@ void mongocrypt_setopt_use_need_kms_credentials_state(mongocrypt_t *crypt) {
     crypt->opts.use_need_kms_credentials_state = true;
 }
 
+void mongocrypt_setopt_use_need_mongo_collinfo_with_db_state(mongocrypt_t *crypt) {
+    BSON_ASSERT_PARAM(crypt);
+
+    crypt->opts.use_need_mongo_collinfo_with_db_state = true;
+}
+
 void mongocrypt_setopt_set_crypt_shared_lib_path_override(mongocrypt_t *crypt, const char *path) {
     BSON_ASSERT_PARAM(crypt);
     BSON_ASSERT_PARAM(path);

src/mongocrypt.c

@@ -473,6 +473,18 @@ void mongocrypt_setopt_set_crypt_shared_lib_path_override(mongocrypt_t *crypt, c
 MONGOCRYPT_EXPORT
 void mongocrypt_setopt_use_need_kms_credentials_state(mongocrypt_t *crypt);
 
+/**
+ * @brief Opt-into handling the MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB state.
+ *
+ * A context enters the MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB state when
+ * processing a `bulkWrite` command. The target database of the `bulkWrite` may differ from the command database
+ * ("admin").
+ *
+ * @param[in] crypt The @ref mongocrypt_t object to update
+ */
+MONGOCRYPT_EXPORT
+void mongocrypt_setopt_use_need_mongo_collinfo_with_db_state(mongocrypt_t *crypt);
+
 /**
  * Initialize new @ref mongocrypt_t object.
  *
@@ -962,9 +974,10 @@ bool mongocrypt_ctx_rewrap_many_datakey_init(mongocrypt_ctx_t *ctx, mongocrypt_b
  */
 typedef enum {
     MONGOCRYPT_CTX_ERROR = 0,
-    MONGOCRYPT_CTX_NEED_MONGO_COLLINFO = 1, /* run on main MongoClient */
-    MONGOCRYPT_CTX_NEED_MONGO_MARKINGS = 2, /* run on mongocryptd. */
-    MONGOCRYPT_CTX_NEED_MONGO_KEYS = 3,     /* run on key vault */
+    MONGOCRYPT_CTX_NEED_MONGO_COLLINFO = 1,         /* run on main MongoClient */
+    MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB = 8, /* run on main MongoClient */
+    MONGOCRYPT_CTX_NEED_MONGO_MARKINGS = 2,         /* run on mongocryptd. */
+    MONGOCRYPT_CTX_NEED_MONGO_KEYS = 3,             /* run on key vault */
     MONGOCRYPT_CTX_NEED_KMS = 4,
     MONGOCRYPT_CTX_NEED_KMS_CREDENTIALS = 7, /* fetch/renew KMS credentials */
     MONGOCRYPT_CTX_READY = 5,                /* ready for encryption/decryption */
@@ -985,7 +998,7 @@ mongocrypt_ctx_state_t mongocrypt_ctx_state(mongocrypt_ctx_t *ctx);
  * is in MONGOCRYPT_CTX_NEED_MONGO_* states.
  *
  * @p op_bson is a BSON document to be used for the operation.
- * - For MONGOCRYPT_CTX_NEED_MONGO_COLLINFO it is a listCollections filter.
+ * - For MONGOCRYPT_CTX_NEED_MONGO_COLLINFO(_WITH_DB) it is a listCollections filter.
  * - For MONGOCRYPT_CTX_NEED_MONGO_KEYS it is a find filter.
  * - For MONGOCRYPT_CTX_NEED_MONGO_MARKINGS it is a command to send to
  * mongocryptd.
@@ -1003,13 +1016,29 @@ mongocrypt_ctx_state_t mongocrypt_ctx_state(mongocrypt_ctx_t *ctx);
 MONGOCRYPT_EXPORT
 bool mongocrypt_ctx_mongo_op(mongocrypt_ctx_t *ctx, mongocrypt_binary_t *op_bson);
 
+/**
+ * Get the database to run the mongo operation.
+ *
+ * Only applies when mongocrypt_ctx_t is in the state:
+ * MONGOCRYPT_CTX_NEED_MONGO_COLLINFO_WITH_DB.
+ *
+ * The lifetime of the returned string is tied to the lifetime of @p ctx. It is
+ * valid until @ref mongocrypt_ctx_destroy is called.
+ *
+ * @param[in] ctx The @ref mongocrypt_ctx_t object.
+ * @returns A string or NULL. If NULL, an error status is set. Retrieve it with
+ * @ref mongocrypt_ctx_status
+ */
+MONGOCRYPT_EXPORT
+const char *mongocrypt_ctx_mongo_db(mongocrypt_ctx_t *ctx);
+
 /**
  * Feed a BSON reply or result when mongocrypt_ctx_t is in
  * MONGOCRYPT_CTX_NEED_MONGO_* states. This may be called multiple times
  * depending on the operation.
  *
  * reply is a BSON document result being fed back for this operation.
- * - For MONGOCRYPT_CTX_NEED_MONGO_COLLINFO it is a doc from a listCollections
+ * - For MONGOCRYPT_CTX_NEED_MONGO_COLLINFO(_WITH_DB) it is a doc from a listCollections
  * cursor. (Note, if listCollections returned no result, do not call this
  * function.)
  * - For MONGOCRYPT_CTX_NEED_MONGO_KEYS it is a doc from a find cursor.

src/mongocrypt.h

@@ -0,0 +1,53 @@
+{
+   "bulkWrite": {
+      "$numberInt": "1"
+   },
+   "ops": [
+      {
+         "insert": {
+            "$numberInt": "0"
+         },
+         "document": {
+            "plainText": "sample",
+            "encrypted": {
+               "$numberInt": "123"
+            }
+         }
+      }
+   ],
+   "$db": "admin",
+   "nsInfo": [
+      {
+         "ns": "db.test",
+         "encryptionInformation": {
+            "type": {
+               "$numberInt": "1"
+            },
+            "schema": {
+               "db.test": {
+                  "escCollection": "enxcol_.test.esc",
+                  "ecocCollection": "enxcol_.test.ecoc",
+                  "fields": [
+                     {
+                        "keyId": {
+                           "$binary": {
+                              "base64": "YWFhYWFhYWFhYWFhYWFhYQ==",
+                              "subType": "04"
+                           }
+                        },
+                        "path": "encrypted",
+                        "bsonType": "int",
+                        "queries": {
+                           "queryType": "equality",
+                           "contention": {
+                              "$numberLong": "0"
+                           }
+                        }
+                     }
+                  ]
+               }
+            }
+         }
+      }
+   ]
+}

test/data/bulkWrite/bypassQueryAnalysis/payload.json

@@ -0,0 +1,23 @@
+{
+    "bulkWrite": {
+        "$numberInt": "1"
+    },
+    "ops": [
+        {
+            "insert": 0,
+            "document": {
+                "plainText": "sample",
+                "encrypted": {
+                    "$numberInt": "123"
+                }
+            }
+        }
+    ],
+    "nsInfo": [
+        {
+            "ns": "db.test"
+        }
+    ],
+    "jsonSchema": {},
+    "isRemoteSchema": false
+}

test/data/bulkWrite/jsonSchema/cmd-to-mongocryptd.json

@@ -0,0 +1,20 @@
+{
+   "bulkWrite": 1,
+   "ops": [
+      {
+         "insert": 0,
+         "document": {
+            "plainText": "sample",
+            "encrypted": {
+               "$numberInt": "123"
+            }
+         }
+      }
+   ],
+   "nsInfo": [
+      {
+         "ns": "db.test"
+      }
+   ],
+   "$db": "admin"
+}