Adding "include_paths", affeceed_root_keys, get_root_key, better

reporting of encoding errors

Author: seperman

deepdiff/deephash.py

@@ -88,11 +88,11 @@ def prepare_string_for_hashing(
                 err = er
         if not encoded:
             obj_decoded = obj.decode('utf-8', errors='ignore')
-            start = min(err.start - 10, 0)
+            start = max(err.start - 20, 0)
             start_prefix = ''
             if start > 0:
                 start_prefix = '...'
-            end = err.end + 10
+            end = err.end + 20
             end_suffix = '...'
             if end >= len(obj):
                 end = len(obj)
@@ -329,8 +329,13 @@ def _skip_this(self, obj, parent):
         skip = False
         if self.exclude_paths and parent in self.exclude_paths:
             skip = True
-        if self.include_paths and parent not in self.include_paths:
-            skip = True
+        if self.include_paths and parent != 'root':
+            if parent not in self.include_paths:
+                skip = True
+                for prefix in self.include_paths:
+                    if parent.startswith(prefix):
+                        skip = False
+                        break
         elif self.exclude_regex_paths and any(
                 [exclude_regex_path.search(parent) for exclude_regex_path in self.exclude_regex_paths]):
             skip = True

deepdiff/diff.py

@@ -438,8 +438,13 @@ def _skip_this(self, level):
         skip = False
         if self.exclude_paths and level_path in self.exclude_paths:
             skip = True
-        if self.include_paths and level_path not in self.include_paths:
-            skip = True
+        if self.include_paths and level_path != 'root':
+            if level_path not in self.include_paths:
+                skip = True
+                for prefix in self.include_paths:
+                    if level_path.startswith(prefix):
+                        skip = False
+                        break
         elif self.exclude_regex_paths and any(
                 [exclude_regex_path.search(level_path) for exclude_regex_path in self.exclude_regex_paths]):
             skip = True
@@ -1445,6 +1450,22 @@ def affected_paths(self):
                     result |= OrderedSet(value.keys())
         return result
 
+    @property
+    def affected_root_keys(self):
+        """
+        Get the list of root keys that were affected.
+        Whether a value was changed or they were added or removed.
+        """
+        result = OrderedSet()
+        for key in REPORT_KEYS:
+            value = self.tree.get(key)
+            if value:
+                if isinstance(value, PrettyOrderedSet):
+                    result |= OrderedSet([i.get_root_key() for i in value])
+                else:
+                    result |= OrderedSet([i.get_root_key() for i in value.keys()])
+        return result
+
 
 if __name__ == "__main__":  # pragma: no cover
     import doctest

deepdiff/helper.py

@@ -5,6 +5,7 @@
 import uuid
 import logging
 import warnings
+import string
 import time
 from ast import literal_eval
 from decimal import Decimal, localcontext
@@ -93,6 +94,8 @@ class np_type:
 py4 = py_major_version == 4
 
 
+NUMERICS = frozenset(string.digits)
+
 # we used to use OrderedDictPlus when dictionaries in Python were not ordered.
 dict_ = dict
 
@@ -254,9 +257,14 @@ def add_root_to_paths(paths):
         if path.startswith('root'):
             result.add(path)
         else:
-            result.add(f"root.{path}")
-            result.add(f"root[{path}]")
-            result.add(f"root['{path}']")
+            if path.isdigit():
+                result.add(f"root['{path}']")
+                result.add(f"root[{path}]")
+            elif path[0].isdigit():
+                result.add(f"root['{path}']")
+            else:
+                result.add(f"root.{path}")
+                result.add(f"root['{path}']")
     return result
 
 

deepdiff/model.py

@@ -621,6 +621,21 @@ def all_down(self):
     def _format_result(root, result):
         return None if result is None else "{}{}".format(root, result)
 
+    def get_root_key(self, use_t2=False):
+        """
+        Get the path's root key value for this change
+
+        For example if the path to the element that is reported to have a change in value is root['X'][0]
+        then get_root_key should return 'X'
+        """
+        root_level = self.all_up
+        if(use_t2):
+            next_rel = root_level.t2_child_rel
+        else:
+            next_rel = root_level.t1_child_rel or root_level.t2_child_rel  # next relationship object to get a formatted param from
+
+        return next_rel.param
+
     def path(self, root="root", force=None, get_parent_too=False, use_t2=False, output_format='str'):
         """
         A python syntax string describing how to descend to this level, assuming the top level object is called root.

deepdiff/serialization.py

@@ -3,6 +3,7 @@
 import io
 import os
 import json
+import uuid
 import logging
 import re  # NOQA
 import builtins  # NOQA
@@ -70,6 +71,7 @@ class UnsupportedFormatErr(TypeError):
     'datetime.time',
     'datetime.timedelta',
     'decimal.Decimal',
+    'uuid.UUID',
     'ordered_set.OrderedSet',
     'collections.namedtuple',
     'collections.OrderedDict',
@@ -502,6 +504,7 @@ def _save_content(content, path, file_type, keep_backup=True):
     type: lambda x: x.__name__,
     bytes: lambda x: x.decode('utf-8'),
     datetime.datetime: lambda x: x.isoformat(),
+    uuid.UUID: lambda x: str(x),
 }
 
 

docs/exclude_paths.rst

@@ -44,6 +44,21 @@ Example
     {}
 
 
+When passing include_paths, all the children of that path will be included too.
+
+Example
+    >>> t1 = {
+    ...     "foo": {"bar": "potato"},
+    ...     "ingredients": ["no meat", "no eggs", "no dairy"]
+    ... }
+    >>> t2 = {
+    ...     "foo": {"bar": "banana"},
+    ...     "ingredients": ["bread", "cheese"]
+    ... }
+    >>> DeepDiff(t1, t2, include_paths="foo")
+    {'values_changed': {"root['foo']['bar']": {'new_value': 'banana', 'old_value': 'potato'}}}
+
+
 .. _exclude_regex_paths_label:
 
 Exclude Regex Paths

setup.py

@@ -26,7 +26,7 @@ def get_reqs(filename):
     long_description = file.read()
 
 
-setup(name='deepdiff6',
+setup(name='deepdiff',
       version=version,
       description='Deep Difference and Search of any Python object/data.',
       url='https://github.com/seperman/deepdiff',

tests/test_cache.py

@@ -26,6 +26,7 @@ def test_cache_deeply_nested_a1(self, nested_a_t1, nested_a_t2, nested_a_result,
         diff_of_diff = DeepDiff(nested_a_result, diff.to_dict(), ignore_order=False)
         assert not diff_of_diff
         assert nested_a_affected_paths == diff.affected_paths
+        assert [0, 1] == diff.affected_root_keys
 
     @pytest.mark.slow
     def test_cache_deeply_nested_a2(self, nested_a_t1, nested_a_t2, nested_a_result):

tests/test_diff_text.py

@@ -1335,9 +1335,28 @@ def test_include_path3_with_just_key_names(self):
             "ingredients": ["no meat", "no eggs", "no dairy"]
         }
         t2 = {"for life": "vegan"}
-        ddiff = DeepDiff(t2, t1, include_paths={"for_life"})
+        ddiff = DeepDiff(t1, t2, include_paths={"for_life"})
         assert {} == ddiff
 
+    def test_include_path4_nested(self):
+        t1 = {
+            "foo": {"bar": "potato"},
+            "ingredients": ["no meat", "no eggs", "no dairy"]
+        }
+        t2 = {
+            "foo": {"bar": "banana"},
+            "ingredients": ["bread", "cheese"]
+        }
+        ddiff = DeepDiff(t1, t2, include_paths="foo")
+        assert {
+            'values_changed': {
+                "root['foo']['bar']": {
+                    'new_value': 'banana',
+                    'old_value': 'potato'
+                }
+            }
+        } == ddiff
+
     def test_skip_path4(self):
         t1 = {
             "for life": "vegan",
@@ -1423,6 +1442,7 @@ def exclude_obj_callback_strict(obj, path):
         result = {'values_changed': {"root['x']": {'new_value': 12, 'old_value': 10}}}
         assert result == ddiff
         assert {"root['x']"} == ddiff.affected_paths
+        assert {"x"} == ddiff.affected_root_keys
 
     def test_skip_str_type_in_dictionary(self):
         t1 = {1: {2: "a"}}

tests/test_hash.py

@@ -568,16 +568,18 @@ def test_skip_str_type_in_dict_on_list(self):
         assert 1 in t1_hash
         assert t1_hash[dic1] == t2_hash[dic2]
 
-    def test_skip_path(self):
+    def test_skip_path_in_hash(self):
         dic1 = {1: "a"}
         t1 = [dic1, 2]
         dic2 = {}
         t2 = [dic2, 2]
         t1_hash = DeepHashPrep(t1, exclude_paths=['root[0]'])
         t2_hash = DeepHashPrep(t2, exclude_paths='root[0]')
+        t2_hash_again = DeepHashPrep(t2, include_paths='1')
         assert 1 not in t1_hash
         assert 2 in t1_hash
         assert t1_hash[2] == t2_hash[2]
+        assert t1_hash[2] == t2_hash_again[2]
 
     def test_skip_path2(self):
 
@@ -596,6 +598,23 @@ def test_skip_path2(self):
         t2_hash = DeepHashPrep(t2, exclude_paths=exclude_paths)
         assert t1_hash[t1] == t2_hash[t2]
 
+    def test_hash_include_path_nested(self):
+
+        obj10 = {'a': 1, 'b': 'f', 'e': "1111", 'foo': {'bar': 'baz'}}
+        obj11 = {'c': 1, 'd': 'f', 'e': 'Cool'}
+
+        obj20 = {'a': 1, 'b': 'f', 'e': 'Cool', 'foo': {'bar': 'baz'}}
+        obj21 = {'c': 1, 'd': 'f', 'e': "2222"}
+
+        t1 = [obj10, obj11]
+        t2 = [obj20, obj21]
+
+        include_paths = ["root[0]['foo']['bar']"]
+
+        t1_hash = DeepHashPrep(t1, include_paths=include_paths)
+        t2_hash = DeepHashPrep(t2, include_paths=include_paths)
+        assert t1_hash[t1] == t2_hash[t2]
+
     def test_skip_regex_path(self):
         dic1 = {1: "a"}
         t1 = [dic1, 2]
@@ -805,18 +824,24 @@ def test_combine_hashes_lists(self, items, prefix, expected):
         "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].")
 
     EXPECTED_MESSAGE2 = (
-        "'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte in 'p of flo...'. "
+        "'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte in ' cup of flour'. "
         "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].")
 
+    EXPECTED_MESSAGE3 = (
+        "'utf-8' codec can't decode byte 0xc3 in position 34: invalid continuation byte in '...up of potatos. Then ( cup of flour'. Please either pass ignore_encoding_errors=True or "
+        "pass the encoding via encodings=['utf-8', '...']."
+    )
+
     @pytest.mark.parametrize('test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message', [
         (1, b'\xc3\x28', None, False, UnicodeDecodeError, EXPECTED_MESSAGE1),
         (2, b'\xc3\x28', ['utf-8'], False, UnicodeDecodeError, EXPECTED_MESSAGE1),
         (3, b'\xc3\x28', ['utf-8'], True, {b'\xc3(': '640da73f0d9b268a0a7ae884d77063d1193f43a651352f9032d99a8fe1705546'}, None),
         (4, b"\xbc cup of flour", ['utf-8'], False, UnicodeDecodeError, EXPECTED_MESSAGE2),
         (5, b"\xbc cup of flour", ['utf-8'], True, {b'\xbc cup of flour': '86ac12eb5e35db88cf93baca1d62098023b2d93d634e75fb4e37657e514f3d51'}, None),
         (6, b"\xbc cup of flour", ['utf-8', 'latin-1'], False, {b'\xbc cup of flour': 'cfc354ae2232a8983bf59b2004f44fcb4036f57df1d08b9cde9950adea3f8d3e'}, None),
+        (7, b"First have a cup of potatos. Then \xc3\x28 cup of flour", None, False, UnicodeDecodeError, EXPECTED_MESSAGE3),
     ])
-    def test_encodings(self, test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message):
+    def test_hash_encodings(self, test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message):
         if UnicodeDecodeError == expected_result:
             with pytest.raises(expected_result) as exc_info:
                 DeepHash(item, encodings=encodings, ignore_encoding_errors=ignore_encoding_errors)

tests/test_helper.py

@@ -9,7 +9,7 @@
     cartesian_product_of_shape, literal_eval_extended,
     not_found, OrderedSetPlus, diff_numpy_array, cartesian_product_numpy,
     get_truncate_datetime, datetime_normalize,
-    detailed__dict__, ENUM_IGNORE_KEYS,
+    detailed__dict__, ENUM_IGNORE_KEYS, add_root_to_paths,
 )
 
 
@@ -288,3 +288,12 @@ def test_datetime_normalize(self, truncate_datetime, obj, expected):
     def test_detailed__dict__(self, obj, ignore_keys, expected):
         result = detailed__dict__(obj, ignore_private_variables=True, ignore_keys=ignore_keys)
         assert expected == result, f"test_detailed__dict__ failed for {obj}"
+
+    @pytest.mark.parametrize('test_num, value, expected', [
+        (1, ['ab'], {'root.ab', "root['ab']"}),
+        (2, ['11'], {"root['11']", 'root[11]'}),
+        (3, ['1a'], {"root['1a']"}),
+    ])
+    def test_add_root_to_paths(self, test_num, value, expected):
+        result = add_root_to_paths(value)
+        assert expected == result, f"test_add_root_to_paths #{test_num} failed."

tests/test_ignore_order.py

@@ -1130,7 +1130,7 @@ class TestDecodingErrorIgnoreOrder:
         "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].")
 
     EXPECTED_MESSAGE2 = (
-        "'utf-8' codec can't decode byte 0xbc in position 0: Can not produce a hash for root: invalid start byte in 'p of flo...'. "
+        "'utf-8' codec can't decode byte 0xbc in position 0: Can not produce a hash for root: invalid start byte in ' cup of flour'. "
         "Please either pass ignore_encoding_errors=True or pass the encoding via encodings=['utf-8', '...'].")
 
     @pytest.mark.parametrize('test_num, item, encodings, ignore_encoding_errors, expected_result, expected_message', [

tests/test_serialization.py

@@ -317,7 +317,7 @@ def test_pretty_form_method(self, expected, verbose_level):
 
     @pytest.mark.parametrize('test_num, value', [
         (1, {'10': None}),
-        (2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}})
+        (2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}}),
     ])
     def test_json_dumps_and_loads(self, test_num, value):
         serialized = json_dumps(value)