PYTHON-2680 Make breaking changes to DBRef BSON+JSON encoding/decoding TODO: C extensions

PYTHON-2841 resync bson corpus tests

Author: ShaneHarvey

bson/__init__.py

@@ -202,7 +202,8 @@ def _get_object(data, view, position, obj_end, opts, dummy):
     obj = _elements_to_dict(data, view, position + 4, end, opts)
 
     position += obj_size
-    if "$ref" in obj:
+    # If DBRef validation fails, return a normal doc.
+    if isinstance(obj.get('$ref'), str) and "$id" in obj and isinstance(obj.get('$db'), (str, type(None))):
         return (DBRef(obj.pop("$ref"), obj.pop("$id", None),
                       obj.pop("$db", None), obj), position)
     return obj, position

bson/json_util.py

@@ -121,9 +121,6 @@
     "x": re.X,
 }
 
-# Dollar-prefixed keys which may appear in DBRefs.
-_DBREF_KEYS = frozenset(['$id', '$ref', '$db'])
-
 
 class DatetimeRepresentation:
     LEGACY = 0
@@ -436,7 +433,7 @@ def object_pairs_hook(pairs, json_options=DEFAULT_JSON_OPTIONS):
 def object_hook(dct, json_options=DEFAULT_JSON_OPTIONS):
     if "$oid" in dct:
         return _parse_canonical_oid(dct)
-    if "$ref" in dct:
+    if isinstance(dct.get('$ref'), str) and "$id" in dct and isinstance(dct.get('$db'), (str, type(None))):
         return _parse_canonical_dbref(dct)
     if "$date" in dct:
         return _parse_canonical_datetime(dct, json_options)
@@ -649,10 +646,6 @@ def _parse_canonical_regex(doc):
 
 def _parse_canonical_dbref(doc):
     """Decode a JSON DBRef to bson.dbref.DBRef."""
-    for key in doc:
-        if key.startswith('$') and key not in _DBREF_KEYS:
-            # Other keys start with $, so dct cannot be parsed as a DBRef.
-            return doc
     return DBRef(doc.pop('$ref'), doc.pop('$id'),
                  database=doc.pop('$db', None), **doc)
 

test/bson_corpus/dbref.json

@@ -25,7 +25,8 @@
         {
             "description": "Document with key names similar to those of a DBRef",
             "canonical_bson": "3e0000000224726566000c0000006e6f742d612d646272656600072469640058921b3e6e32ab156a22b59e022462616e616e6100050000007065656c0000",
-            "canonical_extjson": "{\"$ref\": \"not-a-dbref\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$banana\": \"peel\"}"
+            "canonical_extjson": "{\"$ref\": \"not-a-dbref\", \"$id\": {\"$oid\": \"58921b3e6e32ab156a22b59e\"}, \"$banana\": \"peel\"}",
+            "lossy": true
         },
         {
             "description": "DBRef with additional dollar-prefixed and dotted fields",

test/test_bson_corpus.py

@@ -135,7 +135,8 @@ def run_test(self):
                 # Make sure we can parse the legacy (default) JSON format.
                 legacy_json = json_util.dumps(
                     decoded_bson, json_options=json_util.LEGACY_JSON_OPTIONS)
-                self.assertEqual(decode_extjson(legacy_json), decoded_bson)
+                self.assertEqual(
+                    decode_extjson(legacy_json), decoded_bson, description)
 
             if deprecated:
                 if 'converted_bson' in valid_case:
@@ -157,7 +158,7 @@ def run_test(self):
             if not (sys.platform.startswith("java") and
                     description == 'NaN with payload'):
                 # Test round-tripping canonical bson.
-                self.assertEqual(encode_bson(decoded_bson), cB)
+                self.assertEqual(encode_bson(decoded_bson), cB, description)
             self.assertJsonEqual(encode_extjson(decoded_bson), cEJ)
 
             # Test round-tripping canonical extended json.