Merge pull request #337 from seperman/dev

DeepDiff 6

Author: seperman

.github/ISSUE_TEMPLATE/bug_report.md

@@ -21,6 +21,8 @@ A clear and concise description of what you expected to happen.
 **OS, DeepDiff version and Python version (please complete the following information):**
  - OS: [e.g. Ubuntu]
  - Version [e.g. 20LTS]
+ - Python Version [e.g. 3.9.12]
+ - DeepDiff Version [e.g. 5.8.0]
 
 **Additional context**
 Add any other context about the problem here.

.github/workflows/main.yaml

@@ -14,7 +14,6 @@ jobs:
       matrix:
         python-version: [3.7, 3.8, 3.9, "3.10"]
         architecture: ["x64"]
-
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }} on ${{ matrix.architecture }}
@@ -32,20 +31,30 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
+    - name: Install dependencies py3.7
+      if: matrix.python-version == 3.7
+      run: pip install -r requirements-dev-3.7.txt
     - name: Install dependencies
+      if: matrix.python-version != 3.7
       run: pip install -r requirements-dev.txt
     - name: Lint with flake8
+      if: matrix.python-version == 3.10
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 deepdiff --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 deepdiff --count --exit-zero --max-complexity=26 --max-line-lengt=250 --statistics
-    - name: Test with pytest
+    - name: Test with pytest and get the coverage
+      if: matrix.python-version == 3.10
+      run: |
+        pytest --cov-report=xml --cov=deepdiff tests/ --runslow
+    - name: Test with pytest and no coverage report
+      if: matrix.python-version != 3.10
       run: |
-        pytest --cov-report=xml --cov=deepdiff tests/  --runslow
+        pytest
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v1
-      if: matrix.python-version == 3.8
+      if: matrix.python-version == 3.10
       with:
         file: ./coverage.xml
         env_vars: OS,PYTHON

AUTHORS.md

@@ -44,3 +44,5 @@ Authors in order of the timeline of their contributions:
 - Håvard Thom [havardthom](https://github.com/havardthom) for adding UUID support.
 - Dhanvantari Tilak [Dhanvantari](https://github.com/Dhanvantari) for Bug-Fix: `TypeError in _get_numbers_distance() when ignore_order = True`.
 - Yael Mintz [yaelmi3](https://github.com/yaelmi3) for detailed pretty print when verbose_level=2.
+- Mikhail Khviyuzov [mskhviyu](https://github.com/mskhviyu) for Exclude obj callback strict.
+- [dtorres-sf](https://github.com/dtorres-sf) for the fix for diffing using iterable_compare_func with nested objects.

README.md

@@ -41,11 +41,13 @@ DeepDiff 5-8-0 includes bug fixes and improvements:
 
 ### Install from PyPi:
 
-`pip install deepdiff`
+`pip install deepdiff6`
 
 If you want to use DeepDiff from commandline:
 
-`pip install "deepdiff[cli]"`
+`pip install "deepdiff6[cli]"`
+
+> Note: prior to DeepDiff 6, we used `pip install deepdiff` to install DeepDiff. DeepDiff 6 is being published with a different package name on Pypi temporarily until further notice.
 
 ### Importing
 
@@ -55,7 +57,7 @@ If you want to use DeepDiff from commandline:
 >>> from deepdiff import DeepHash  # For hashing objects based on their contents
 ```
 
-Note: if you want to use DeepDiff via commandline, make sure to run `pip install "deepdiff[cli]"`. Then you can access the commands via:
+Note: if you want to use DeepDiff via commandline, make sure to run `pip install "deepdiff6[cli]"`. Then you can access the commands via:
 
 - DeepDiff
     - `$ deep diff --help`

deepdiff/delta.py

@@ -263,13 +263,21 @@ def _del_elem(self, parent, parent_to_obj_elem, parent_to_obj_action,
     def _do_iterable_item_added(self):
         iterable_item_added = self.diff.get('iterable_item_added', {})
         iterable_item_moved = self.diff.get('iterable_item_moved')
+
+        # First we need to create a placeholder for moved items.
+        # This will then get replaced below after we go through added items.
+        # Without this items can get double added because moved store the new_value and does not need item_added replayed
         if iterable_item_moved:
-            added_dict = {v["new_path"]: v["value"] for k, v in iterable_item_moved.items()}
+            added_dict = {v["new_path"]: None for k, v in iterable_item_moved.items()}
             iterable_item_added.update(added_dict)
 
         if iterable_item_added:
             self._do_item_added(iterable_item_added, insert=True)
 
+        if iterable_item_moved:
+            added_dict = {v["new_path"]: v["value"] for k, v in iterable_item_moved.items()}
+            self._do_item_added(added_dict, insert=False)
+
     def _do_dictionary_item_added(self):
         dictionary_item_added = self.diff.get('dictionary_item_added')
         if dictionary_item_added:

deepdiff/diff.py

@@ -7,6 +7,7 @@
 # However the docstring expects it in a specific order in order to pass!
 import difflib
 import logging
+from enum import Enum
 from copy import deepcopy
 from math import isclose as is_close
 from collections.abc import Mapping, Iterable
@@ -21,7 +22,7 @@
                              number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans,
                              np_ndarray, get_numpy_ndarray_rows, OrderedSetPlus, RepeatedTimer,
                              TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__,
-                             np, get_truncate_datetime, dict_, CannotCompare)
+                             np, get_truncate_datetime, dict_, CannotCompare, ENUM_IGNORE_KEYS)
 from deepdiff.serialization import SerializationMixin
 from deepdiff.distance import DistanceMixin
 from deepdiff.model import (
@@ -116,6 +117,7 @@ def __init__(self,
                  cutoff_intersection_for_pairs=CUTOFF_INTERSECTION_FOR_PAIRS_DEFAULT,
                  encodings=None,
                  exclude_obj_callback=None,
+                 exclude_obj_callback_strict=None,
                  exclude_paths=None,
                  exclude_regex_paths=None,
                  exclude_types=None,
@@ -194,6 +196,7 @@ def __init__(self,
             self.type_check_func = type_is_subclass_of_type_group if ignore_type_subclasses else type_in_type_group
             self.ignore_string_case = ignore_string_case
             self.exclude_obj_callback = exclude_obj_callback
+            self.exclude_obj_callback_strict = exclude_obj_callback_strict
             self.number_to_string = number_to_string_func or number_to_string
             self.iterable_compare_func = iterable_compare_func
             self.ignore_private_variables = ignore_private_variables
@@ -386,8 +389,19 @@ def unmangle(attribute):
 
         return {i: getattr(object, unmangle(i)) for i in all_slots}
 
-    def _diff_obj(self, level, parents_ids=frozenset(),
-                  is_namedtuple=False):
+    def _diff_enum(self, level, parents_ids=frozenset()):
+        t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables, ignore_keys=ENUM_IGNORE_KEYS)
+        t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables, ignore_keys=ENUM_IGNORE_KEYS)
+
+        self._diff_dict(
+            level,
+            parents_ids,
+            print_as_attribute=True,
+            override=True,
+            override_t1=t1,
+            override_t2=t2)
+
+    def _diff_obj(self, level, parents_ids=frozenset(), is_namedtuple=False):
         """Difference of 2 objects"""
         try:
             if is_namedtuple:
@@ -429,6 +443,10 @@ def _skip_this(self, level):
         elif self.exclude_obj_callback and \
                 (self.exclude_obj_callback(level.t1, level.path()) or self.exclude_obj_callback(level.t2, level.path())):
             skip = True
+        elif self.exclude_obj_callback_strict and \
+                (self.exclude_obj_callback_strict(level.t1, level.path()) and
+                 self.exclude_obj_callback_strict(level.t2, level.path())):
+            skip = True
 
         return skip
 
@@ -709,7 +727,7 @@ def _diff_iterable_in_order(self, level, parents_ids=frozenset(), _original_type
                     x,
                     y,
                     child_relationship_class=child_relationship_class,
-                    child_relationship_param=i)
+                    child_relationship_param=j)
                 self._diff(next_level, parents_ids_added)
 
     def _diff_str(self, level):
@@ -1350,6 +1368,9 @@ def _diff(self, level, parents_ids=frozenset(), _original_type=None):
         elif isinstance(level.t1, Iterable):
             self._diff_iterable(level, parents_ids, _original_type=_original_type)
 
+        elif isinstance(level.t1, Enum):
+            self._diff_enum(level, parents_ids)
+
         else:
             self._diff_obj(level, parents_ids)
 

deepdiff/helper.py

@@ -123,6 +123,8 @@ class np_type:
 TEXT_VIEW = 'text'
 DELTA_VIEW = '_delta'
 
+ENUM_IGNORE_KEYS = frozenset(['_name_', '_value_', '_sort_order_'])
+
 
 def short_repr(item, max_length=15):
     """Short representation of item if it is too long"""
@@ -495,7 +497,7 @@ def _eval_date(params):
 
 def literal_eval_extended(item):
     """
-    An extend version of literal_eval
+    An extended version of literal_eval
     """
     try:
         return literal_eval(item)
@@ -584,21 +586,27 @@ def get_homogeneous_numpy_compatible_type_of_seq(seq):
         return False
 
 
-def detailed__dict__(obj, ignore_private_variables=True):
+def detailed__dict__(obj, ignore_private_variables=True, ignore_keys=frozenset()):
     """
     Get the detailed dictionary of an object.
 
     This is used so we retrieve object properties too.
     """
     result = obj.__dict__.copy()  # A shallow copy
     private_var_prefix = f"_{obj.__class__.__name__}__"  # The semi private variables in Python get this prefix
+    for key in ignore_keys:
+        if key in result or (
+            ignore_private_variables and key.startswith('__') and not key.startswith(private_var_prefix)
+        ):
+            del result[key]
     for key in dir(obj):
-        if key not in result and (
+        if key not in result and key not in ignore_keys and (
                 not ignore_private_variables or (
                     ignore_private_variables and not key.startswith('__') and not key.startswith(private_var_prefix)
                 )
         ):
             value = getattr(obj, key)
             if not callable(value):
+                print(f"{key}: {value}")
                 result[key] = value
     return result

docs/authors.rst

@@ -56,6 +56,8 @@ Authors in order of the timeline of their contributions:
 -  Dhanvantari Tilak `Dhanvantari`_ for Bug-Fix:
    ``TypeError in _get_numbers_distance() when ignore_order = True``.
 -  Yael Mintz `yaelmi3`_ for detailed pretty print when verbose_level=2.
+-  Mikhail Khviyuzov `mskhviyu`_ for Exclude obj callback strict.
+-  `dtorres-sf`_ for the fix for diffing using iterable_compare_func with nested objects.
 
 .. _Sep Dehpour (Seperman): http://www.zepworks.com
 .. _Victor Hahn Castell: http://hahncastell.de
@@ -96,7 +98,7 @@ Authors in order of the timeline of their contributions:
 .. _havardthom: https://github.com/havardthom
 .. _Dhanvantari: https://github.com/Dhanvantari
 .. _yaelmi3: https://github.com/yaelmi3
-
+.. _mskhviyu: https://github.com/mskhviyu
 
 Thank you for contributing to DeepDiff!
 

docs/ignore_types_or_values.rst

@@ -272,6 +272,18 @@ exclude_obj_callback: function, default = None
     >>> DeepDiff(t1, t2, exclude_obj_callback=exclude_obj_callback)
     {}
 
+exclude_obj_callback_strict: function, default = None
+    A function works the same way as exclude_obj_callback, but excludes elements from the result only if the function returns True for both elements
+
+    >>> def exclude_obj_callback_strict(obj, path):
+    ...         return True if isinstance(obj, int) and obj > 10 else False
+    ...
+    >>> t1 = {"x": 10, "y": "b", "z": "c"}
+    >>> t2 = {"x": 12, "y": "b", "z": "c"}
+    >>> DeepDiff(t1, t2, exclude_obj_callback=exclude_obj_callback_strict)
+    {}
+    >>> DeepDiff(t1, t2, exclude_obj_callback_strict=exclude_obj_callback_strict)
+    {'values_changed': {"root['x']": {'new_value': 12, 'old_value': 10}}}
 
 .. _truncate_datetime_label:
 

docs/index.rst

@@ -75,14 +75,17 @@ Installation
 
 Install from PyPi::
 
-    pip install deepdiff
+    pip install deepdiff6
 
 If you want to use DeepDiff from commandline::
 
-    pip install "deepdiff[cli]"
+    pip install "deepdiff6[cli]"
 
 Read about DeepDiff optimizations at :ref:`optimizations_label`
 
+Note: prior to DeepDiff 6, we used pip install deepdiff to install DeepDiff::
+    DeepDiff 6 is being published with a different package name on Pypi temporarily until further notice.
+
 
 Importing
 ~~~~~~~~~
@@ -97,7 +100,7 @@ Importing
 
 
 Note: if you want to use DeepDiff via commandline, make sure to run:: 
-    pip install "deepdiff[cli]"
+    pip install "deepdiff6[cli]"
 
 Then you can access the commands via:
 

requirements-cli.txt

@@ -1,4 +1,4 @@
-click==8.0.3
-pyyaml==5.4.1
+click==8.1.3
+pyyaml==6.0
 toml==0.10.2
-clevercsv==0.7.1
+clevercsv==0.7.4

requirements-dev-3.7.txt

@@ -0,0 +1,9 @@
+wheel==0.37.0
+-r requirements.txt
+-r requirements-cli.txt
+bump2version==1.0.1
+jsonpickle==2.2.0
+ipdb==0.13.9
+numpy==1.21.6
+pytest==7.1.2
+python-dotenv==0.20.0

requirements-dev.txt

@@ -2,14 +2,14 @@ wheel==0.37.0
 -r requirements.txt
 -r requirements-cli.txt
 bump2version==1.0.1
-jsonpickle==2.0.0
-coverage==6.0.2
+jsonpickle==2.2.0
+coverage==6.4.3
 ipdb==0.13.9
-numpy==1.21.2
-pytest==6.2.5
+numpy==1.23.1
+pytest==7.1.2
 pytest-cov==3.0.0
-python-dotenv==0.19.1
-watchdog==2.1.6
-Sphinx==4.2.0
+python-dotenv==0.20.0
+watchdog==2.1.9
+Sphinx==5.1.1
 sphinx-sitemap==2.2.0
-flake8==4.0.1
+flake8==5.0.4

setup.py

@@ -34,11 +34,11 @@ def get_reqs(filename):
       author='Seperman',
       author_email='[email protected]',
       license='MIT',
-      packages=['deepdiff'],
+      packages=['deepdiff6'],
       zip_safe=True,
       test_suite="tests",
       include_package_data=True,
-      tests_require=['mock'],  # 'numpy==1.11.2' numpy is needed but comes already installed with travis
+      tests_require=['mock'],
       long_description=long_description,
       long_description_content_type='text/markdown',
       install_requires=reqs,

tests/fixtures/compare_func_result1.json

@@ -6,11 +6,11 @@
     "root['Cars'][3]['production']"
   ],
   "values_changed": {
-    "root['Cars'][0]['dealers'][1]['quantity']": {
+    "root['Cars'][2]['dealers'][0]['quantity']": {
       "new_value": 50,
       "old_value": 20
     },
-    "root['Cars'][2]['model_numbers'][2]": {
+    "root['Cars'][1]['model_numbers'][2]": {
       "new_value": 3,
       "old_value": 4
     },
@@ -20,20 +20,20 @@
     }
   },
   "iterable_item_added": {
-    "root['Cars'][0]['dealers'][1]": {
+    "root['Cars'][2]['dealers'][1]": {
       "id": 200,
       "address": "200 Fake St",
       "quantity": 10
     },
-    "root['Cars'][2]['model_numbers'][3]": 4,
+    "root['Cars'][1]['model_numbers'][3]": 4,
     "root['Cars'][0]": {
       "id": "7",
       "make": "Toyota",
       "model": "8Runner"
     }
   },
   "iterable_item_removed": {
-    "root['Cars'][0]['dealers'][0]": {
+    "root['Cars'][2]['dealers'][0]": {
       "id": 103,
       "address": "103 Fake St",
       "quantity": 50