CDRIVER-5861 support $lookup in CSFLE and QE (#1880)

* update table for test mock servers

* remove trailing colons in error message

* update libmongocrypt

* pass all `listCollections` results to libmongocrypt

Accounts for updated protocol in MONGOCRYPT-723.

* fail test on error in `get_bson_from_json_file`

* add test wire version checks for server 8.1

* use `const` in `ASSERT_EQUAL_BSON`

* implement prose tests

* apply majority write concern

* drop individual collections

To match latest revision of prose tests

* fix param name

Author: kevinAlbs

.evergreen/scripts/compile-libmongocrypt.sh

@@ -10,14 +10,20 @@ compile_libmongocrypt() {
   # `.evergreen/scripts/kms-divergence-check.sh` to ensure that there is no
   # divergence in the copied files.
 
-  git clone -q --depth=1 https://github.com/mongodb/libmongocrypt --branch 1.12.0 || return
+  # Clone libmongocrypt and check-out commit for MONGOCRYPT-723.
+  # TODO: once libmongocrypt 1.13.0 is released, updated to:
+  # git clone -q --depth=1 https://github.com/mongodb/libmongocrypt --branch 1.13.0 || return
+  git clone -q https://github.com/mongodb/libmongocrypt || return
+  cd libmongocrypt
+  git checkout 33fdf65cce5a0c0cdd293c64ed40e4a8205c3ce0
+  cd ..
 
   declare -a crypt_cmake_flags=(
     "-DMONGOCRYPT_MONGOC_DIR=${mongoc_dir}"
     "-DBUILD_TESTING=OFF"
     "-DENABLE_ONLINE_TESTS=OFF"
     "-DENABLE_MONGOC=OFF"
-    "-DBUILD_VERSION=1.12.0"
+    "-DBUILD_VERSION=1.13.0-pre"
   )
 
   . "$(dirname "${BASH_SOURCE[0]}")/find-ccache.sh"

CONTRIBUTING.md

@@ -289,8 +289,9 @@ The set of mock KMS servers running in the background and their corresponding in
 | 8999 | ca.pem | server.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8999
 | 9000 | ca.pem | expired.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000
 | 9001 | ca.pem | wrong-host.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001
-| 9002 | ca.pem | server.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port --require_client_cert 9002
+| 9002 | ca.pem | server.pem | python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --require_client_cert --port 9002
 | 5698 | ca.pem | server.pem | python -u kms_kmip_server.py
+| 9003 | ca.pem | server.pem | python kms_failpoint_server.py --port 9003
 
 The path to `ca.pem` and `client.pem` must be passed through the following environment variables:
 

src/libmongoc/src/mongoc/mongoc-crypt.c

@@ -170,7 +170,7 @@ _prefix_mongocryptd_error (bson_error_t *error)
    char buf[sizeof (error->message)];
 
    // Truncation is OK.
-   int req = bson_snprintf (buf, sizeof (buf), "mongocryptd error: %s:", error->message);
+   int req = bson_snprintf (buf, sizeof (buf), "mongocryptd error: %s", error->message);
    BSON_ASSERT (req > 0);
    memcpy (error->message, buf, sizeof (buf));
 }
@@ -181,7 +181,7 @@ _prefix_keyvault_error (bson_error_t *error)
    char buf[sizeof (error->message)];
 
    // Truncation is OK.
-   int req = bson_snprintf (buf, sizeof (buf), "key vault error: %s:", error->message);
+   int req = bson_snprintf (buf, sizeof (buf), "key vault error: %s", error->message);
    BSON_ASSERT (req > 0);
    memcpy (error->message, buf, sizeof (buf));
 }
@@ -342,15 +342,18 @@ _state_need_mongo_collinfo (_state_machine_t *state_machine, bson_error_t *error
       goto fail;
    }
 
-   /* 2. Return the first result (if any) with mongocrypt_ctx_mongo_feed or
+   /* 2. Return all results (if any) with mongocrypt_ctx_mongo_feed or
     * proceed to the next step if nothing was returned. */
-   if (mongoc_cursor_next (cursor, &collinfo_bson)) {
+   while (mongoc_cursor_next (cursor, &collinfo_bson)) {
       collinfo_bin = mongocrypt_binary_new_from_data ((uint8_t *) bson_get_data (collinfo_bson), collinfo_bson->len);
       if (!mongocrypt_ctx_mongo_feed (state_machine->ctx, collinfo_bin)) {
          _ctx_check_error (state_machine->ctx, error, true);
          goto fail;
       }
-   } else if (mongoc_cursor_error (cursor, error)) {
+      mongocrypt_binary_destroy (collinfo_bin);
+      collinfo_bin = NULL;
+   }
+   if (mongoc_cursor_error (cursor, error)) {
       goto fail;
    }
 
@@ -1397,6 +1400,10 @@ _mongoc_crypt_new (const bson_t *kms_providers,
    crypt->kmsid_to_tlsopts = mcd_mapof_kmsid_to_tlsopts_new ();
    crypt->handle = mongocrypt_new ();
    mongocrypt_setopt_retry_kms (crypt->handle, true);
+   if (!mongocrypt_setopt_enable_multiple_collinfo (crypt->handle)) {
+      _crypt_check_error (crypt->handle, error, true);
+      goto fail;
+   }
 
    // Stash away a copy of the user's kmsProviders in case we need to lazily
    // load credentials.

src/libmongoc/tests/client_side_encryption_prose/lookup/key-doc.json

@@ -0,0 +1,30 @@
+{
+    "_id": {
+        "$binary": {
+            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+            "subType": "04"
+        }
+    },
+    "keyMaterial": {
+        "$binary": {
+            "base64": "sHe0kz57YW7v8g9VP9sf/+K1ex4JqKc5rf/URX3n3p8XdZ6+15uXPaSayC6adWbNxkFskuMCOifDoTT+rkqMtFkDclOy884RuGGtUysq3X7zkAWYTKi8QAfKkajvVbZl2y23UqgVasdQu3OVBQCrH/xY00nNAs/52e958nVjBuzQkSb1T8pKJAyjZsHJ60+FtnfafDZSTAIBJYn7UWBCwQ==",
+            "subType": "00"
+        }
+    },
+    "creationDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "updateDate": {
+        "$date": {
+            "$numberLong": "1648914851981"
+        }
+    },
+    "status": {
+        "$numberInt": "0"
+    },
+    "masterKey": {
+        "provider": "local"
+    }
+}

src/libmongoc/tests/client_side_encryption_prose/lookup/schema-csfle.json

@@ -0,0 +1,19 @@
+{
+    "properties": {
+        "csfle": {
+            "encrypt": {
+                "keyId": [
+                    {
+                        "$binary": {
+                            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                            "subType": "04"
+                        }
+                    }
+                ],
+                "bsonType": "string",
+                "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+            }
+        }
+    },
+    "bsonType": "object"
+}

src/libmongoc/tests/client_side_encryption_prose/lookup/schema-csfle2.json

@@ -0,0 +1,19 @@
+{
+    "properties": {
+        "csfle2": {
+            "encrypt": {
+                "keyId": [
+                    {
+                        "$binary": {
+                            "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                            "subType": "04"
+                        }
+                    }
+                ],
+                "bsonType": "string",
+                "algorithm": "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
+            }
+        }
+    },
+    "bsonType": "object"
+}

src/libmongoc/tests/client_side_encryption_prose/lookup/schema-qe.json

@@ -0,0 +1,20 @@
+{
+    "escCollection": "enxcol_.qe.esc",
+    "ecocCollection": "enxcol_.qe.ecoc",
+    "fields": [
+        {
+            "keyId": {
+                "$binary": {
+                    "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                    "subType": "04"
+                }
+            },
+            "path": "qe",
+            "bsonType": "string",
+            "queries": {
+                "queryType": "equality",
+                "contention": 0
+            }
+        }
+    ]
+}

src/libmongoc/tests/client_side_encryption_prose/lookup/schema-qe2.json

@@ -0,0 +1,20 @@
+{
+    "escCollection": "enxcol_.qe2.esc",
+    "ecocCollection": "enxcol_.qe2.ecoc",
+    "fields": [
+        {
+            "keyId": {
+                "$binary": {
+                    "base64": "EjRWeBI0mHYSNBI0VniQEg==",
+                    "subType": "04"
+                }
+            },
+            "path": "qe2",
+            "bsonType": "string",
+            "queries": {
+                "queryType": "equality",
+                "contention": 0
+            }
+        }
+    ]
+}

src/libmongoc/tests/json-test.c

@@ -663,27 +663,24 @@ get_bson_from_json_file (char *filename)
 
    file = fopen (filename, "rb");
    if (!file) {
-      return NULL;
+      test_error ("Failed to open JSON file: %s", filename);
    }
 
    /* get file length */
    fseek (file, 0, SEEK_END);
    length = ftell (file);
    fseek (file, 0, SEEK_SET);
    if (length < 1) {
-      return NULL;
+      test_error ("Failed to read length of JSON file: %s", filename);
    }
 
    /* read entire file into buffer */
    buffer = (const char *) bson_malloc0 (length);
    if (fread ((void *) buffer, 1, length, file) != length) {
-      test_error ("Failed to read JSON file into buffer");
+      test_error ("Failed to read JSON file into buffer: %s", filename);
    }
 
    fclose (file);
-   if (!buffer) {
-      return NULL;
-   }
 
    /* convert to bson */
    data = bson_new_from_json ((const uint8_t *) buffer, length, &error);

src/libmongoc/tests/test-conveniences.h

@@ -209,7 +209,7 @@ match_json (const bson_t *doc,
 
 #define ASSERT_EQUAL_BSON(expected, actual)                                   \
    do {                                                                       \
-      bson_t *_expected_bson = expected, *_actual_bson = actual;              \
+      const bson_t *_expected_bson = expected, *_actual_bson = actual;        \
       char *_expected_str, *_actual_str;                                      \
       _expected_str = bson_as_canonical_extended_json (_expected_bson, NULL); \
       _actual_str = bson_as_canonical_extended_json (_actual_bson, NULL);     \

src/libmongoc/tests/test-libmongoc.c

@@ -2273,6 +2273,8 @@ WIRE_VERSION_CHECKS (23)
 WIRE_VERSION_CHECKS (24)
 /* wire version 25 begins with the 8.0 release. */
 WIRE_VERSION_CHECKS (25)
+/* wire version 26 begins with the 8.1 release. */
+WIRE_VERSION_CHECKS (26)
 
 int
 test_framework_skip_if_no_dual_ip_hostname (void)

src/libmongoc/tests/test-libmongoc.h

@@ -216,6 +216,8 @@ WIRE_VERSION_CHECK_DECLS (23)
 WIRE_VERSION_CHECK_DECLS (24)
 /* wire version 25 begins with the 8.0 release. */
 WIRE_VERSION_CHECK_DECLS (25)
+/* wire version 26 begins with the 8.1 release. */
+WIRE_VERSION_CHECK_DECLS (26)
 
 #undef WIRE_VERSION_CHECK_DECLS