Merge branch 'master' of github.com:mongodb/mongo-python-driver into PYTHON-3460

Author: blink1073

.evergreen/utils.sh

@@ -30,7 +30,10 @@ createvirtualenv () {
     fi
 
     python -m pip install --upgrade pip
-    python -m pip install --upgrade setuptools wheel unittest-xml-reporting
+    python -m pip install --upgrade setuptools wheel
+    # lxml only has wheels for macos 10.15+
+    python -m pip install unittest-xml-reporting || true
+
 }
 
 # Usage:

.github/workflows/test-python.yml

@@ -58,19 +58,14 @@ jobs:
           cache-dependency-path: 'setup.py'
       - name: Install dependencies
         run: |
-          python -m pip install -U pip mypy==0.990
+          python -m pip install -U pip mypy==1.2
           pip install -e ".[zstd, encryption, ocsp]"
       - name: Run mypy
         run: |
           mypy --install-types --non-interactive bson gridfs tools pymongo
-          # Test overshadowed codec_options.py file
-          mypy --install-types --non-interactive bson/codec_options.py
           mypy --install-types --non-interactive --disable-error-code var-annotated --disable-error-code attr-defined --disable-error-code union-attr --disable-error-code assignment --disable-error-code no-redef --disable-error-code index --allow-redefinition --allow-untyped-globals --exclude "test/mypy_fails/*.*" test
           python -m pip install -U typing_extensions
           mypy --install-types --non-interactive test/test_typing.py test/test_typing_strict.py
-      - name: Run mypy strict
-        run: |
-          mypy --strict test/test_typing_strict.py
       - name: Run pyright
         run: |
           python -m pip install -U pip pyright==1.1.290

bson/__init__.py

@@ -1115,7 +1115,7 @@ def decode_all(
     if not isinstance(opts, CodecOptions):
         raise _CODEC_OPTIONS_TYPE_ERROR
 
-    return _decode_all(data, opts)  # type: ignore[arg-type]
+    return _decode_all(data, opts)  # type:ignore[arg-type]
 
 
 def _decode_selective(rawdoc: Any, fields: Any, codec_options: Any) -> Mapping[Any, Any]:

bson/codec_options.py

@@ -100,7 +100,7 @@ def _decimal_to_128(value: _VALUE_OPTIONS) -> Tuple[int, int]:
         if significand & (1 << i):
             high |= 1 << (i - 64)
 
-    biased_exponent = exponent + _EXPONENT_BIAS
+    biased_exponent = exponent + _EXPONENT_BIAS  # type: ignore[operator]
 
     if high >> 49 == 1:
         high = high & 0x7FFFFFFFFFFF

bson/codec_options.pyi

@@ -66,7 +66,7 @@ def __init__(
         self.update(kwargs)
 
     def __new__(cls: Type["SON[_Key, _Value]"], *args: Any, **kwargs: Any) -> "SON[_Key, _Value]":
-        instance = super(SON, cls).__new__(cls, *args, **kwargs)
+        instance = super(SON, cls).__new__(cls, *args, **kwargs)  # type: ignore[type-var]
         instance.__keys = []
         return instance
 
@@ -115,7 +115,7 @@ def clear(self) -> None:
         self.__keys = []
         super(SON, self).clear()
 
-    def setdefault(self, key: _Key, default: _Value) -> _Value:  # type: ignore[override]
+    def setdefault(self, key: _Key, default: _Value) -> _Value:
         try:
             return self[key]
         except KeyError:

bson/decimal128.py

@@ -8,9 +8,14 @@ Changes in Version 4.4
   or keys to :meth:`~pymongo.collection.Collection.create_index`.
 - pymongocrypt 1.5.0 or later is now required for client side field level
   encryption support.
-- Improved support for Pyright to improve typing support for IDEs like Visual Studio Code or Visual Studio.
+- Improved support for Pyright to improve typing support for IDEs like Visual Studio Code
+  or Visual Studio.
 - Improved support for type-checking with MyPy "strict" mode (`--strict`).
 - Added support for Python 3.11.
+- pymongocrypt 1.6.0 or later is now required for Client Side Field Level Encryption (CSFLE)
+  and Queryable Encryption (QE) support. MongoDB Server 7.0 introduced a backwards breaking
+  change to the QE protocol. Users taking advantage of the QE beta must now upgrade to
+  MongoDB 7.0+ and PyMongo 4.4+.
 
 Issues Resolved
 ...............

bson/son.py

@@ -364,13 +364,13 @@ data key and create a collection with the
 Automatic Queryable Encryption (Beta)
 `````````````````````````````````````
 
-PyMongo 4.2 brings beta support for Queryable Encryption with MongoDB >=6.0.
+PyMongo 4.4 brings beta support for Queryable Encryption with MongoDB >=7.0.
 
 Queryable Encryption is the second version of Client-Side Field Level Encryption.
 Data is encrypted client-side. Queryable Encryption supports indexed encrypted fields,
 which are further processed server-side.
 
-You must have MongoDB 6.0 Enterprise to preview the capability.
+You must have MongoDB 7.0 Enterprise to preview the capability.
 
 Automatic encryption in Queryable Encryption is configured with an ``encrypted_fields`` mapping, as demonstrated by the following example::
 
@@ -396,7 +396,6 @@ Automatic encryption in Queryable Encryption is configured with an ``encrypted_f
   encrypted_fields_map = {
       "default.encryptedCollection": {
         "escCollection": "encryptedCollection.esc",
-        "eccCollection": "encryptedCollection.ecc",
         "ecocCollection": "encryptedCollection.ecoc",
         "fields": [
           {
@@ -429,7 +428,7 @@ automatically encrypted and decrypted.
 Explicit Queryable Encryption (Beta)
 ````````````````````````````````````
 
-PyMongo 4.2 brings beta support for Queryable Encryption with MongoDB >=6.0.
+PyMongo 4.4 brings beta support for Queryable Encryption with MongoDB >=7.0.
 
 Queryable Encryption is the second version of Client-Side Field Level Encryption.
 Data is encrypted client-side. Queryable Encryption supports indexed encrypted fields,
@@ -487,7 +486,6 @@ using an ``encrypted_fields`` mapping, as demonstrated by the following example:
 
         encrypted_fields = {
           "escCollection": "enxcol_.default.esc",
-          "eccCollection": "enxcol_.default.ecc",
           "ecocCollection": "enxcol_.default.ecoc",
           "fields": [
             {

doc/changelog.rst

@@ -43,7 +43,7 @@
 from pymongo.change_stream import CollectionChangeStream
 from pymongo.collation import validate_collation_or_none
 from pymongo.command_cursor import CommandCursor, RawBatchCommandCursor
-from pymongo.common import _ecc_coll_name, _ecoc_coll_name, _esc_coll_name
+from pymongo.common import _ecoc_coll_name, _esc_coll_name
 from pymongo.cursor import Cursor, RawBatchCursor
 from pymongo.errors import (
     ConfigurationError,
@@ -232,8 +232,9 @@ def __init__(
             if encrypted_fields:
                 common.validate_is_mapping("encrypted_fields", encrypted_fields)
                 opts = {"clusteredIndex": {"key": {"_id": 1}, "unique": True}}
-                self.__create(_esc_coll_name(encrypted_fields, name), opts, None, session)
-                self.__create(_ecc_coll_name(encrypted_fields, name), opts, None, session)
+                self.__create(
+                    _esc_coll_name(encrypted_fields, name), opts, None, session, qev2_required=True
+                )
                 self.__create(_ecoc_coll_name(encrypted_fields, name), opts, None, session)
                 self.__create(name, kwargs, collation, session, encrypted_fields=encrypted_fields)
                 self.create_index([("__safeContent__", ASCENDING)], session)
@@ -305,7 +306,9 @@ def _command(
                 user_fields=user_fields,
             )
 
-    def __create(self, name, options, collation, session, encrypted_fields=None):
+    def __create(
+        self, name, options, collation, session, encrypted_fields=None, qev2_required=False
+    ):
         """Sends a create command with the given options."""
         cmd = SON([("create", name)])
         if encrypted_fields:
@@ -316,6 +319,13 @@ def __create(self, name, options, collation, session, encrypted_fields=None):
                 options["size"] = float(options["size"])
             cmd.update(options)
         with self._socket_for_writes(session) as sock_info:
+            if qev2_required and sock_info.max_wire_version < 21:
+                raise ConfigurationError(
+                    "Driver support of Queryable Encryption is incompatible with server. "
+                    "Upgrade server to use Queryable Encryption. "
+                    f"Got maxWireVersion {sock_info.max_wire_version} but need maxWireVersion >= 21 (MongoDB >=7.0)"
+                )
+
             self._command(
                 sock_info,
                 cmd,
@@ -2020,11 +2030,11 @@ def create_index(
             pairs specifying the index to create
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
-            arguments
           - `comment` (optional): A user-provided comment to attach to this
             command.
           - `**kwargs` (optional): any additional index creation
             options (see the above list) should be passed as keyword
+            arguments.
 
         .. versionchanged:: 4.4
            Allow passing a list containing (key, direction) pairs
@@ -2072,14 +2082,11 @@ def drop_indexes(
         :Parameters:
           - `session` (optional): a
             :class:`~pymongo.client_session.ClientSession`.
-            arguments
           - `comment` (optional): A user-provided comment to attach to this
             command.
           - `**kwargs` (optional): optional arguments to the createIndexes
             command (like maxTimeMS) can be passed as keyword arguments.
 
-
-
         .. note:: The :attr:`~pymongo.collection.Collection.write_concern` of
            this collection is automatically applied to this operation.
 
@@ -2090,7 +2097,6 @@ def drop_indexes(
         .. versionchanged:: 3.4
            Apply this collection's write concern automatically to this operation
            when connected to MongoDB >= 3.4.
-
         """
         if comment is not None:
             kwargs["comment"] = comment

doc/examples/encryption.rst

@@ -859,10 +859,6 @@ def _esc_coll_name(encrypted_fields, name):
     return encrypted_fields.get("escCollection", f"enxcol_.{name}.esc")
 
 
-def _ecc_coll_name(encrypted_fields, name):
-    return encrypted_fields.get("eccCollection", f"enxcol_.{name}.ecc")
-
-
 def _ecoc_coll_name(encrypted_fields, name):
     return encrypted_fields.get("ecocCollection", f"enxcol_.{name}.ecoc")
 

pymongo/collection.py

@@ -39,7 +39,7 @@
 from pymongo.change_stream import DatabaseChangeStream
 from pymongo.collection import Collection
 from pymongo.command_cursor import CommandCursor
-from pymongo.common import _ecc_coll_name, _ecoc_coll_name, _esc_coll_name
+from pymongo.common import _ecoc_coll_name, _esc_coll_name
 from pymongo.errors import CollectionInvalid, InvalidName
 from pymongo.read_preferences import ReadPreference, _ServerMode
 from pymongo.typings import _CollationIn, _DocumentType, _DocumentTypeArg, _Pipeline
@@ -394,7 +394,6 @@ def create_collection(
 
                 {
                   "escCollection": "enxcol_.encryptedCollection.esc",
-                  "eccCollection": "enxcol_.encryptedCollection.ecc",
                   "ecocCollection": "enxcol_.encryptedCollection.ecoc",
                   "fields": [
                       {
@@ -1009,7 +1008,6 @@ def drop_collection(
 
                 {
                   "escCollection": "enxcol_.encryptedCollection.esc",
-                  "eccCollection": "enxcol_.encryptedCollection.ecc",
                   "ecocCollection": "enxcol_.encryptedCollection.ecoc",
                   "fields": [
                       {
@@ -1061,9 +1059,6 @@ def drop_collection(
             self._drop_helper(
                 _esc_coll_name(encrypted_fields, name), session=session, comment=comment
             )
-            self._drop_helper(
-                _ecc_coll_name(encrypted_fields, name), session=session, comment=comment
-            )
             self._drop_helper(
                 _ecoc_coll_name(encrypted_fields, name), session=session, comment=comment
             )