fix cursor validation

Author: baileympearson

.evergreen/insert_malformed_utf8.py

@@ -0,0 +1,29 @@
+# test script for inserting invalid data into a db
+
+from pymongo import MongoClient
+from bson import BSON
+from bson.raw_bson import RawBSONDocument
+import os
+
+# Connect to MongoDB
+client = MongoClient(os.environ["MONGODB_URI"])
+db = client['test']
+collection = db['invalidutf8']
+
+# template document
+document = {
+    'field': 'asdf'
+}
+
+# first, insert once with invalid value
+raw_bson = BSON.encode(document)
+raw_bson = raw_bson.replace(b'asdf', b'a\x80df')
+collection.drop()
+collection.insert_one(RawBSONDocument(raw_bson))
+# then again with invalid key
+raw_bson = raw_bson.replace(b'field', b'fe\x80\xc2\xa9')
+collection.insert_one(RawBSONDocument(raw_bson))
+# then insert another doc that serves as a control mechanism
+collection.insert_one({ 'control': 'normal\\string\\with\\backslashes', 'nested': { 'a': 'b' }, 'array': [1,2,3] })
+
+print("Inserted document with an invalid UTF-8 string")

.evergreen/run-invalid-utf8-tests.sh

@@ -0,0 +1 @@
+#! /bin/bash

src/cmap/wire_protocol/on_demand/document.ts

@@ -10,6 +10,7 @@ import {
   getInt32LE,
   ObjectId,
   parseToElementsToArray,
+  pluckBSONSerializeOptions,
   Timestamp,
   toUTF8
 } from '../../../bson';
@@ -330,11 +331,23 @@ export class OnDemandDocument {
    * @param options - BSON deserialization options
    */
   public toObject(options?: BSONSerializeOptions): Record<string, any> {
-    return BSON.deserialize(this.bson, {
-      ...options,
+    const exactBSONOptions = {
+      ...pluckBSONSerializeOptions(options ?? {}),
+      validation: this.parseBsonSerializationOptions(options),
       index: this.offset,
       allowObjectSmallerThanBufferSize: true
-    });
+    };
+    return BSON.deserialize(this.bson, exactBSONOptions);
+  }
+
+  private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): {
+    utf8: { writeErrors: false } | false;
+  } {
+    const enableUtf8Validation = options?.enableUtf8Validation;
+    if (enableUtf8Validation === false) {
+      return { utf8: false };
+    }
+    return { utf8: { writeErrors: false } };
   }
 
   /** Returns this document's bytes only */

src/cmap/wire_protocol/responses.ts

@@ -166,24 +166,6 @@ export class MongoDBResponse extends OnDemandDocument {
     }
     return this.clusterTime ?? null;
   }
-
-  public override toObject(options?: BSONSerializeOptions): Record<string, any> {
-    const exactBSONOptions = {
-      ...pluckBSONSerializeOptions(options ?? {}),
-      validation: this.parseBsonSerializationOptions(options)
-    };
-    return super.toObject(exactBSONOptions);
-  }
-
-  private parseBsonSerializationOptions(options?: { enableUtf8Validation?: boolean }): {
-    utf8: { writeErrors: false } | false;
-  } {
-    const enableUtf8Validation = options?.enableUtf8Validation;
-    if (enableUtf8Validation === false) {
-      return { utf8: false };
-    }
-    return { utf8: { writeErrors: false } };
-  }
 }
 
 /** @internal */

test/integration/node-specific/bson-options/utf8_validation.test.ts

@@ -4,7 +4,7 @@ import * as sinon from 'sinon';
 import {
   BSON,
   BSONError,
-  Collection,
+  type Collection,
   type MongoClient,
   MongoDBResponse,
   MongoError,
@@ -164,8 +164,8 @@ describe('utf8 validation with cursors', function () {
   beforeEach(async function () {
     client = this.configuration.newClient();
     await client.connect();
-    const db = client.db("test");
-    collection = db.collection("invalidutf8");
+    const db = client.db('test');
+    collection = db.collection('invalidutf8');
   });
 
   afterEach(async function () {
@@ -174,7 +174,7 @@ describe('utf8 validation with cursors', function () {
 
   context('when utf-8 validation is explicitly disabled', function () {
     it('documents can be read using a for-await loop without errors', async function () {
-      for await (const doc of collection.find({}, { enableUtf8Validation: false }));
+      for await (const _doc of collection.find({}, { enableUtf8Validation: false }));
     });
     it('documents can be read using next() without errors', async function () {
       const cursor = collection.find({}, { enableUtf8Validation: false });
@@ -200,10 +200,10 @@ describe('utf8 validation with cursors', function () {
       while (await cursor.hasNext()) {
         await cursor.tryNext();
       }
-    })
-  })
+    });
+  });
 
-  async function expectReject(fn: () => Promise<void>, options?: { regex?: RegExp, errorClass }) {
+  async function expectReject(fn: () => Promise<void>, options?: { regex?: RegExp; errorClass }) {
     const regex = options?.regex ?? /.*/;
     const errorClass = options?.errorClass ?? MongoError;
     try {
@@ -217,95 +217,113 @@ describe('utf8 validation with cursors', function () {
 
   context('when utf-8 validation is explicitly enabled', function () {
     it('a for-await loop throw a BSON error', async function () {
-      await expectReject(async () => {
-        for await (const doc of collection.find({}, { enableUtf8Validation: true }));
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ })
+      await expectReject(
+        async () => {
+          for await (const doc of collection.find({}, { enableUtf8Validation: true }));
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      );
     });
     it('next() throws a BSON error', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({}, { enableUtf8Validation: true });
-
-        while (await cursor.hasNext()) {
-          await cursor.next();
-        }
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({}, { enableUtf8Validation: true });
+
+          while (await cursor.hasNext()) {
+            await cursor.next();
+          }
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
     });
 
     it('toArray() throws a BSON error', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({}, { enableUtf8Validation: true });
-        await cursor.toArray();
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({}, { enableUtf8Validation: true });
+          await cursor.toArray();
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
-
     });
 
     it('.stream() throws a BSONError', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({}, { enableUtf8Validation: true });
-        await cursor.stream().toArray();
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({}, { enableUtf8Validation: true });
+          await cursor.stream().toArray();
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
     });
 
     it('tryNext() throws a BSONError', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({}, { enableUtf8Validation: true });
-
-        while (await cursor.hasNext()) {
-          await cursor.tryNext();
-        }
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({}, { enableUtf8Validation: true });
+
+          while (await cursor.hasNext()) {
+            await cursor.tryNext();
+          }
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
-
-    })
-  })
+    });
+  });
 
   context('utf-8 validation defaults to enabled', function () {
     it('a for-await loop throw a BSON error', async function () {
-      await expectReject(async () => {
-        for await (const doc of collection.find({}));
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ })
+      await expectReject(
+        async () => {
+          for await (const doc of collection.find({}));
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      );
     });
     it('next() throws a BSON error', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({});
-
-        while (await cursor.hasNext()) {
-          await cursor.next();
-        }
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({});
+
+          while (await cursor.hasNext()) {
+            await cursor.next();
+          }
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
     });
 
     it('toArray() throws a BSON error', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({});
-        await cursor.toArray();
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({});
+          await cursor.toArray();
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
-
     });
 
     it('.stream() throws a BSONError', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({});
-        await cursor.stream().toArray();
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({});
+          await cursor.stream().toArray();
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
     });
 
     it('tryNext() throws a BSONError', async function () {
-      await expectReject(async () => {
-        const cursor = collection.find({});
-
-        while (await cursor.hasNext()) {
-          await cursor.tryNext();
-        }
-      }, { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
+      await expectReject(
+        async () => {
+          const cursor = collection.find({}, { enableUtf8Validation: true });
+
+          while (await cursor.hasNext()) {
+            await cursor.tryNext();
+          }
+        },
+        { errorClass: BSONError, regex: /Invalid UTF-8 string in BSON document/ }
       );
-
-    })
-  })
-})
+    });
+  });
+});