Merge branch 'master' into PYTHON-4706

Author: blink1073

.evergreen/config.yml

@@ -0,0 +1,53 @@
+#!/bin/bash -ex
+
+# Get the current unique version of this checkout
+# shellcheck disable=SC2154
+if [ "$is_patch" = "true" ]; then
+    # shellcheck disable=SC2154
+    CURRENT_VERSION="$(git describe)-patch-$version_id"
+else
+    CURRENT_VERSION=latest
+fi
+
+PROJECT_DIRECTORY="$(pwd)"
+DRIVERS_TOOLS="$(dirname $PROJECT_DIRECTORY)/drivers-tools"
+
+# Python has cygwin path problems on Windows. Detect prospective mongo-orchestration home directory
+if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+    DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
+    PROJECT_DIRECTORY=$(cygpath -m $PROJECT_DIRECTORY)
+fi
+
+SCRIPT_DIR="$PROJECT_DIRECTORY/.evergreen/scripts"
+
+if [ -f "$SCRIPT_DIR/env.sh" ]; then
+  echo "Reading $SCRIPT_DIR/env.sh file"
+  . "$SCRIPT_DIR/env.sh"
+  exit 0
+fi
+
+export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
+export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
+
+cat <<EOT > $SCRIPT_DIR/env.sh
+set -o errexit
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export CURRENT_VERSION="$CURRENT_VERSION"
+export SKIP_LEGACY_SHELL=1
+export DRIVERS_TOOLS="$DRIVERS_TOOLS"
+export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
+export MONGODB_BINARIES="$MONGODB_BINARIES"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+
+export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
+export PATH="$MONGODB_BINARIES:$PATH"
+# shellcheck disable=SC2154
+export PROJECT="$project"
+export PIP_QUIET=1
+EOT
+
+# Add these expansions to make it easier to call out tests scripts from the EVG yaml
+cat <<EOT > expansion.yml
+DRIVERS_TOOLS: "$DRIVERS_TOOLS"
+PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+EOT

.evergreen/scripts/configure-env.sh

@@ -209,4 +209,4 @@ jobs:
             ls
             which python
             pip install -e ".[test]"
-            PYMONGO_MUST_CONNECT=1 pytest -v test/test_client_context.py
+            PYMONGO_MUST_CONNECT=1 pytest -v -k client_context

.github/workflows/test-python.yml

@@ -248,3 +248,10 @@ you are attempting to validate new spec tests in PyMongo.
 ## Making a Release
 
 Follow the [Python Driver Release Process Uncyclo](https://wiki.corp.mongodb.com/display/DRIVERS/Python+Driver+Release+Process).
+
+## Converting a test to async
+The `tools/convert_test_to_async.py` script takes in an existing synchronous test file and outputs a
+partially-converted asynchronous version of the same name to the `test/asynchronous` directory.
+Use this generated file as a starting point for the completed conversion.
+
+The script is used like so: `python tools/convert_test_to_async.py [test_file.py]`

CONTRIBUTING.md

@@ -383,10 +383,11 @@ static int millis_from_datetime_ms(PyObject* dt, long long* out){
 static PyObject* decode_datetime(PyObject* self, long long millis, const codec_options_t* options){
     PyObject* naive = NULL;
     PyObject* replace = NULL;
-    PyObject* args = NULL;
-    PyObject* kwargs = NULL;
     PyObject* value = NULL;
     struct module_state *state = GETSTATE(self);
+    if (!state) {
+        goto invalid;
+    }
     if (options->datetime_conversion == DATETIME_MS){
         return datetime_ms_from_millis(self, millis);
     }
@@ -414,8 +415,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -433,8 +434,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -487,8 +488,6 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
 invalid:
     Py_XDECREF(naive);
     Py_XDECREF(replace);
-    Py_XDECREF(args);
-    Py_XDECREF(kwargs);
     return value;
 }
 

bson/_cbsonmodule.c

@@ -4,13 +4,18 @@ Changelog
 Changes in Version 4.9.0
 -------------------------
 
+.. warning:: Driver support for MongoDB 3.6 reached end of life in April 2024.
+   PyMongo 4.9 will be the last release to support MongoDB 3.6.
+
 PyMongo 4.9 brings a number of improvements including:
 
 - Added support for MongoDB 8.0.
 - A new asynchronous API with full asyncio support.
 - Added support for In-Use Encryption range queries with MongoDB 8.0.
   Added :attr:`~pymongo.encryption.Algorithm.RANGE`.
   ``sparsity`` and ``trim_factor`` are now optional in :class:`~pymongo.encryption_options.RangeOpts`.
+- Added support for the "delegated" option for the KMIP ``master_key`` in
+  :meth:`~pymongo.encryption.ClientEncryption.create_data_key`.
 - pymongocrypt>=1.10 is now required for :ref:`In-Use Encryption` support.
 - Added :meth:`~pymongo.cursor.Cursor.to_list` to :class:`~pymongo.cursor.Cursor`,
   :class:`~pymongo.command_cursor.CommandCursor`,

doc/changelog.rst

@@ -1176,24 +1176,6 @@ def __getattr__(self, name: str) -> Any:
         raise AttributeError("GridIn object has no attribute '%s'" % name)
 
     def __setattr__(self, name: str, value: Any) -> None:
-        # For properties of this instance like _buffer, or descriptors set on
-        # the class like filename, use regular __setattr__
-        if name in self.__dict__ or name in self.__class__.__dict__:
-            object.__setattr__(self, name, value)
-        else:
-            if _IS_SYNC:
-                # All other attributes are part of the document in db.fs.files.
-                # Store them to be sent to server on close() or if closed, send
-                # them now.
-                self._file[name] = value
-                if self._closed:
-                    self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {name: value}})
-            else:
-                raise AttributeError(
-                    "AsyncGridIn does not support __setattr__. Use AsyncGridIn.set() instead"
-                )
-
-    async def set(self, name: str, value: Any) -> None:
         # For properties of this instance like _buffer, or descriptors set on
         # the class like filename, use regular __setattr__
         if name in self.__dict__ or name in self.__class__.__dict__:
@@ -1204,9 +1186,17 @@ async def set(self, name: str, value: Any) -> None:
             # them now.
             self._file[name] = value
             if self._closed:
-                await self._coll.files.update_one(
-                    {"_id": self._file["_id"]}, {"$set": {name: value}}
-                )
+                if _IS_SYNC:
+                    self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {name: value}})
+                else:
+                    raise AttributeError(
+                        "AsyncGridIn does not support __setattr__ after being closed(). Set the attribute before closing the file or use AsyncGridIn.set() instead"
+                    )
+
+    async def set(self, name: str, value: Any) -> None:
+        self._file[name] = value
+        if self._closed:
+            await self._coll.files.update_one({"_id": self._file["_id"]}, {"$set": {name: value}})
 
     async def _flush_data(self, data: Any, force: bool = False) -> None:
         """Flush `data` to a chunk."""
@@ -1400,7 +1390,11 @@ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> Any:
         return False
 
 
-class AsyncGridOut(io.IOBase):
+GRIDOUT_BASE_CLASS = io.IOBase if _IS_SYNC else object  # type: Any
+
+
+class AsyncGridOut(GRIDOUT_BASE_CLASS):  # type: ignore
+
     """Class to read data out of GridFS."""
 
     def __init__(
@@ -1460,6 +1454,8 @@ def __init__(
         self._position = 0
         self._file = file_document
         self._session = session
+        if not _IS_SYNC:
+            self.closed = False
 
     _id: Any = _a_grid_out_property("_id", "The ``'_id'`` value for this file.")
     filename: str = _a_grid_out_property("filename", "Name of this file.")
@@ -1486,16 +1482,43 @@ def __init__(
     _file: Any
     _chunk_iter: Any
 
-    async def __anext__(self) -> bytes:
-        return super().__next__()
+    if not _IS_SYNC:
+        closed: bool
 
-    def __next__(self) -> bytes:  # noqa: F811, RUF100
-        if _IS_SYNC:
-            return super().__next__()
-        else:
-            raise TypeError(
-                "AsyncGridOut does not support synchronous iteration. Use `async for` instead"
-            )
+        async def __anext__(self) -> bytes:
+            line = await self.readline()
+            if line:
+                return line
+            raise StopAsyncIteration()
+
+        async def to_list(self) -> list[bytes]:
+            return [x async for x in self]  # noqa: C416, RUF100
+
+        async def readline(self, size: int = -1) -> bytes:
+            """Read one line or up to `size` bytes from the file.
+
+            :param size: the maximum number of bytes to read
+            """
+            return await self._read_size_or_line(size=size, line=True)
+
+        async def readlines(self, size: int = -1) -> list[bytes]:
+            """Read one line or up to `size` bytes from the file.
+
+            :param size: the maximum number of bytes to read
+            """
+            await self.open()
+            lines = []
+            remainder = int(self.length) - self._position
+            bytes_read = 0
+            while remainder > 0:
+                line = await self._read_size_or_line(line=True)
+                bytes_read += len(line)
+                lines.append(line)
+                remainder = int(self.length) - self._position
+                if 0 < size < bytes_read:
+                    break
+
+            return lines
 
     async def open(self) -> None:
         if not self._file:
@@ -1616,18 +1639,11 @@ async def read(self, size: int = -1) -> bytes:
         """
         return await self._read_size_or_line(size=size)
 
-    async def readline(self, size: int = -1) -> bytes:  # type: ignore[override]
-        """Read one line or up to `size` bytes from the file.
-
-        :param size: the maximum number of bytes to read
-        """
-        return await self._read_size_or_line(size=size, line=True)
-
     def tell(self) -> int:
         """Return the current position of this file."""
         return self._position
 
-    async def seek(self, pos: int, whence: int = _SEEK_SET) -> int:  # type: ignore[override]
+    async def seek(self, pos: int, whence: int = _SEEK_SET) -> int:
         """Set the current position of this file.
 
         :param pos: the position (or offset if using relative
@@ -1690,12 +1706,15 @@ def __aiter__(self) -> AsyncGridOut:
         """
         return self
 
-    async def close(self) -> None:  # type: ignore[override]
+    async def close(self) -> None:
         """Make GridOut more generically file-like."""
         if self._chunk_iter:
             await self._chunk_iter.close()
             self._chunk_iter = None
-        super().close()
+        if _IS_SYNC:
+            super().close()
+        else:
+            self.closed = True
 
     def write(self, value: Any) -> NoReturn:
         raise io.UnsupportedOperation("write")

gridfs/asynchronous/grid_file.py

@@ -38,14 +38,31 @@ def _a_grid_in_property(
 ) -> Any:
     """Create a GridIn property."""
 
+    warn_str = ""
+    if docstring.startswith("DEPRECATED,"):
+        warn_str = (
+            f"GridIn property '{field_name}' is deprecated and will be removed in PyMongo 5.0"
+        )
+
     def getter(self: Any) -> Any:
+        if warn_str:
+            warnings.warn(warn_str, stacklevel=2, category=DeprecationWarning)
         if closed_only and not self._closed:
             raise AttributeError("can only get %r on a closed file" % field_name)
         # Protect against PHP-237
         if field_name == "length":
             return self._file.get(field_name, 0)
         return self._file.get(field_name, None)
 
+    def setter(self: Any, value: Any) -> Any:
+        if warn_str:
+            warnings.warn(warn_str, stacklevel=2, category=DeprecationWarning)
+        if self._closed:
+            raise InvalidOperation(
+                "AsyncGridIn does not support __setattr__ after being closed(). Set the attribute before closing the file or use AsyncGridIn.set() instead"
+            )
+        self._file[field_name] = value
+
     if read_only:
         docstring += "\n\nThis attribute is read-only."
     elif closed_only:
@@ -56,6 +73,8 @@ def getter(self: Any) -> Any:
             "has been called.",
         )
 
+    if not read_only and not closed_only:
+        return property(getter, setter, doc=docstring)
     return property(getter, doc=docstring)