PYTHON-1307 Remove SONManipulator APIs

doc/api/pymongo/collection.rst

@@ -47,8 +47,8 @@
       .. automethod:: aggregate
       .. automethod:: aggregate_raw_batches
       .. automethod:: watch
-      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None)
-      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+      .. automethod:: find(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
+      .. automethod:: find_raw_batches(filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)
       .. automethod:: find_one(filter=None, *args, **kwargs)
       .. automethod:: find_one_and_delete
       .. automethod:: find_one_and_replace(filter, replacement, projection=None, sort=None, return_document=ReturnDocument.BEFORE, hint=None, session=None, **kwargs)

doc/api/pymongo/cursor.rst

@@ -15,7 +15,7 @@
       .. autoattribute:: EXHAUST
          :annotation:
 
-   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, manipulate=True, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.Cursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, session=None, allow_disk_use=None)
       :members:
 
       .. describe:: c[index]
@@ -24,4 +24,4 @@
 
       .. automethod:: __getitem__
 
-   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None)
+   .. autoclass:: pymongo.cursor.RawBatchCursor(collection, filter=None, projection=None, skip=0, limit=0, no_cursor_timeout=False, cursor_type=CursorType.NON_TAILABLE, sort=None, allow_partial_results=False, oplog_replay=False, modifiers=None, batch_size=0, collation=None, hint=None, max_scan=None, max_time_ms=None, max=None, min=None, return_key=False, show_record_id=False, snapshot=False, comment=None, allow_disk_use=None)

doc/api/pymongo/index.rst

@@ -47,7 +47,6 @@ Sub-modules:
    read_concern
    read_preferences
    results
-   son_manipulator
    server_api
    uri_parser
    write_concern

doc/changelog.rst

@@ -30,11 +30,11 @@ Breaking Changes in 4.0
 - Removed :meth:`pymongo.collection.Collection.parallel_scan`.
 - Removed :meth:`pymongo.collection.Collection.ensure_index`.
 - Removed :meth:`pymongo.collection.Collection.reindex`.
-- Removed :meth:`pymongo.collection.Collection.save`
-- Removed :meth:`pymongo.collection.Collection.insert`
-- Removed :meth:`pymongo.collection.Collection.update`
-- Removed :meth:`pymongo.collection.Collection.remove`
-- Removed :meth:`pymongo.collection.Collection.find_and_modify`
+- Removed :meth:`pymongo.collection.Collection.save`.
+- Removed :meth:`pymongo.collection.Collection.insert`.
+- Removed :meth:`pymongo.collection.Collection.update`.
+- Removed :meth:`pymongo.collection.Collection.remove`.
+- Removed :meth:`pymongo.collection.Collection.find_and_modify`.
 - Removed :meth:`pymongo.mongo_client.MongoClient.close_cursor`. Use
   :meth:`pymongo.cursor.Cursor.close` instead.
 - Removed :meth:`pymongo.mongo_client.MongoClient.kill_cursors`.
@@ -43,6 +43,21 @@ Breaking Changes in 4.0
 - Removed :meth:`pymongo.mongo_client.MongoClient.set_cursor_manager`.
 - Removed :mod:`pymongo.thread_util`.
 - Removed :class:`~pymongo.mongo_replica_set_client.MongoReplicaSetClient`.
+- Removed :mod:`pymongo.son_manipulator`,
+  :class:`pymongo.son_manipulator.SONManipulator`,
+  :class:`pymongo.son_manipulator.ObjectIdInjector`,
+  :class:`pymongo.son_manipulator.ObjectIdShuffler`,
+  :class:`pymongo.son_manipulator.AutoReference`,
+  :class:`pymongo.son_manipulator.NamespaceInjector`,
+  :meth:`pymongo.database.Database.add_son_manipulator`,
+  :attr:`pymongo.database.Database.outgoing_copying_manipulators`,
+  :attr:`pymongo.database.Database.outgoing_manipulators`,
+  :attr:`pymongo.database.Database.incoming_copying_manipulators`, and
+  :attr:`pymongo.database.Database.incoming_manipulators`.
+- Removed the ``manipulate`` parameter from
+  :meth:`~pymongo.collection.Collection.find`,
+  :meth:`~pymongo.collection.Collection.find_one`, and
+  :meth:`~pymongo.cursor.Cursor`.
 
 Notable improvements
 ....................

doc/migrate-to-pymongo4.rst

@@ -274,6 +274,38 @@ can be changed to this::
 
 .. _reIndex command: https://docs.mongodb.com/manual/reference/command/reIndex/
 
+SONManipulator is removed
+-------------------------
+
+Removed :mod:`pymongo.son_manipulator`,
+:class:`pymongo.son_manipulator.SONManipulator`,
+:class:`pymongo.son_manipulator.ObjectIdInjector`,
+:class:`pymongo.son_manipulator.ObjectIdShuffler`,
+:class:`pymongo.son_manipulator.AutoReference`,
+:class:`pymongo.son_manipulator.NamespaceInjector`,
+:meth:`pymongo.database.Database.add_son_manipulator`,
+:attr:`pymongo.database.Database.outgoing_copying_manipulators`,
+:attr:`pymongo.database.Database.outgoing_manipulators`,
+:attr:`pymongo.database.Database.incoming_copying_manipulators`, and
+:attr:`pymongo.database.Database.incoming_manipulators`.
+
+Removed the ``manipulate`` parameter from
+:meth:`~pymongo.collection.Collection.find`,
+:meth:`~pymongo.collection.Collection.find_one`, and
+:meth:`~pymongo.cursor.Cursor`.
+
+The :class:`pymongo.son_manipulator.SONManipulator` API has limitations as a
+technique for transforming your data and was deprecated in PyMongo 3.0.
+Instead, it is more flexible and straightforward to transform outgoing
+documents in your own code before passing them to PyMongo, and transform
+incoming documents after receiving them from PyMongo.
+
+Alternatively, if your application uses the ``SONManipulator`` API to convert
+custom types to BSON, the :class:`~bson.codec_options.TypeCodec` and
+:class:`~bson.codec_options.TypeRegistry` APIs may be a suitable alternative.
+For more information, see the
+:doc:`custom type example <examples/custom_type>`.
+
 Removed features with no migration path
 ---------------------------------------
 

pymongo/bulk.py

@@ -478,9 +478,9 @@ def execute_no_results(self, sock_info, generator):
                             sock_info,
                             operation['q'],
                             doc,
-                            operation['upsert'],
-                            check_keys,
-                            operation['multi'],
+                            upsert=operation['upsert'],
+                            check_keys=check_keys,
+                            multi=operation['multi'],
                             write_concern=write_concern,
                             op_id=op_id,
                             ordered=self.ordered,

pymongo/collection.py

@@ -554,15 +554,9 @@ def _legacy_write(self, sock_info, name, cmd, op_id,
 
     def _insert_one(
             self, doc, ordered,
-            check_keys, manipulate, write_concern, op_id, bypass_doc_val,
+            check_keys, write_concern, op_id, bypass_doc_val,
             session):
         """Internal helper for inserting a single document."""
-        if manipulate:
-            doc = self.__database._apply_incoming_manipulators(doc, self)
-            if not isinstance(doc, RawBSONDocument) and '_id' not in doc:
-                doc['_id'] = ObjectId()
-            doc = self.__database._apply_incoming_copying_manipulators(doc,
-                                                                       self)
         write_concern = write_concern or self.write_concern
         acknowledged = write_concern.acknowledged
         command = SON([('insert', self.name),
@@ -646,7 +640,7 @@ def insert_one(self, document, bypass_document_validation=False,
         write_concern = self._write_concern_for(session)
         return InsertOneResult(
             self._insert_one(
-                document, ordered=True, check_keys=True, manipulate=False,
+                document, ordered=True, check_keys=True,
                 write_concern=write_concern, op_id=None,
                 bypass_doc_val=bypass_document_validation, session=session),
             write_concern.acknowledged)
@@ -712,14 +706,12 @@ def gen():
         return InsertManyResult(inserted_ids, write_concern.acknowledged)
 
     def _update(self, sock_info, criteria, document, upsert=False,
-                check_keys=True, multi=False, manipulate=False,
+                check_keys=True, multi=False,
                 write_concern=None, op_id=None, ordered=True,
                 bypass_doc_val=False, collation=None, array_filters=None,
                 hint=None, session=None, retryable_write=False):
         """Internal update / replace helper."""
         common.validate_boolean("upsert", upsert)
-        if manipulate:
-            document = self.__database._fix_incoming(document, self)
         collation = validate_collation_or_none(collation)
         write_concern = write_concern or self.write_concern
         acknowledged = write_concern.acknowledged
@@ -801,15 +793,15 @@ def _update(self, sock_info, criteria, document, upsert=False,
 
     def _update_retryable(
             self, criteria, document, upsert=False,
-            check_keys=True, multi=False, manipulate=False,
+            check_keys=True, multi=False,
             write_concern=None, op_id=None, ordered=True,
             bypass_doc_val=False, collation=None, array_filters=None,
             hint=None, session=None):
         """Internal update / replace helper."""
         def _update(session, sock_info, retryable_write):
             return self._update(
                 sock_info, criteria, document, upsert=upsert,
-                check_keys=check_keys, multi=multi, manipulate=manipulate,
+                check_keys=check_keys, multi=multi,
                 write_concern=write_concern, op_id=op_id, ordered=ordered,
                 bypass_doc_val=bypass_doc_val, collation=collation,
                 array_filters=array_filters, hint=hint, session=session,
@@ -1346,8 +1338,6 @@ def find(self, *args, **kwargs):
             oplogReplay query flag. Default: False.
           - `batch_size` (optional): Limits the number of documents returned in
             a single batch.
-          - `manipulate` (optional): **DEPRECATED** - If True, apply any
-            outgoing SON manipulators before returning. Default: True.
           - `collation` (optional): An instance of
             :class:`~pymongo.collation.Collation`. This option is only supported
             on MongoDB 3.4 and above.

pymongo/command_cursor.py

@@ -268,8 +268,7 @@ def _try_next(self, get_more_allowed):
         if not len(self.__data) and not self.__killed and get_more_allowed:
             self._refresh()
         if len(self.__data):
-            coll = self.__collection
-            return coll.database._fix_outgoing(self.__data.popleft(), coll)
+            return self.__data.popleft()
         else:
             return None
 

pymongo/cursor.py

@@ -108,7 +108,7 @@ def __init__(self, collection, filter=None, projection=None, skip=0,
                  limit=0, no_cursor_timeout=False,
                  cursor_type=CursorType.NON_TAILABLE,
                  sort=None, allow_partial_results=False, oplog_replay=False,
-                 modifiers=None, batch_size=0, manipulate=True,
+                 modifiers=None, batch_size=0,
                  collation=None, hint=None, max_scan=None, max_time_ms=None,
                  max=None, min=None, return_key=False, show_record_id=False,
                  snapshot=False, comment=None, session=None,
@@ -181,7 +181,6 @@ def __init__(self, collection, filter=None, projection=None, skip=0,
         self.__max_await_time_ms = None
         self.__max = max
         self.__min = min
-        self.__manipulate = manipulate
         self.__collation = validate_collation_or_none(collation)
         self.__return_key = return_key
         self.__show_record_id = show_record_id
@@ -281,7 +280,7 @@ def _clone(self, deepcopy=True, base=None):
         values_to_clone = ("spec", "projection", "skip", "limit",
                            "max_time_ms", "max_await_time_ms", "comment",
                            "max", "min", "ordering", "explain", "hint",
-                           "batch_size", "max_scan", "manipulate",
+                           "batch_size", "max_scan",
                            "query_flags", "modifiers", "collation", "empty",
                            "show_record_id", "return_key", "allow_disk_use",
                            "snapshot", "exhaust")
@@ -1198,12 +1197,7 @@ def next(self):
         if self.__empty:
             raise StopIteration
         if len(self.__data) or self._refresh():
-            if self.__manipulate:
-                _db = self.__collection.database
-                return _db._fix_outgoing(self.__data.popleft(),
-                                         self.__collection)
-            else:
-                return self.__data.popleft()
+            return self.__data.popleft()
         else:
             raise StopIteration
 
@@ -1277,15 +1271,8 @@ def __init__(self, *args, **kwargs):
 
         .. mongodoc:: cursors
         """
-        manipulate = kwargs.get('manipulate')
-        kwargs['manipulate'] = False
         super(RawBatchCursor, self).__init__(*args, **kwargs)
 
-        # Throw only after cursor's initialized, to prevent errors in __del__.
-        if manipulate:
-            raise InvalidOperation(
-                "Cannot use RawBatchCursor with manipulate=True")
-
     def _unpack_response(self, response, cursor_id, codec_options,
                          user_fields=None, legacy_response=False):
         return response.raw_response(cursor_id)