Merge pull request #360 from seperman/dev

6.2.2

Author: seperman

.github/workflows/main.yaml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
         architecture: ["x64"]
     steps:
     - uses: actions/checkout@v2

AUTHORS.md

@@ -47,3 +47,4 @@ Authors in order of the timeline of their contributions:
 - Mikhail Khviyuzov [mskhviyu](https://github.com/mskhviyu) for Exclude obj callback strict.
 - [dtorres-sf](https://github.com/dtorres-sf) for the fix for diffing using iterable_compare_func with nested objects.
 - [Enric Pou](https://github.com/epou) for bug fix of ValueError when using Decimal 0.x
+- [Uwe Fladrich](https://github.com/uwefladrich) for fixing bug when diff'ing non-sequence iterables

CHANGELOG.md

@@ -1,5 +1,8 @@
 # DeepDiff Change log
 
+- v6-2-2
+    - Enum test fix for python 3.11
+    - Adding support for dateutils rrules
 - v6-2-1
     - Removed the print statements.
 - v6-2-0

deepdiff/diff.py

@@ -11,7 +11,7 @@
 from enum import Enum
 from copy import deepcopy
 from math import isclose as is_close
-from collections.abc import Mapping, Iterable
+from collections.abc import Mapping, Iterable, Sequence
 from collections import defaultdict
 from itertools import zip_longest
 from ordered_set import OrderedSet
@@ -23,7 +23,7 @@
                              number_to_string, datetime_normalize, KEY_TO_VAL_STR, booleans,
                              np_ndarray, get_numpy_ndarray_rows, OrderedSetPlus, RepeatedTimer,
                              TEXT_VIEW, TREE_VIEW, DELTA_VIEW, detailed__dict__, add_root_to_paths,
-                             np, get_truncate_datetime, dict_, CannotCompare, ENUM_IGNORE_KEYS)
+                             np, get_truncate_datetime, dict_, CannotCompare, ENUM_INCLUDE_KEYS)
 from deepdiff.serialization import SerializationMixin
 from deepdiff.distance import DistanceMixin
 from deepdiff.model import (
@@ -395,8 +395,8 @@ def unmangle(attribute):
         return {i: getattr(object, unmangle(i)) for i in all_slots}
 
     def _diff_enum(self, level, parents_ids=frozenset(), local_tree=None):
-        t1 = detailed__dict__(level.t1, ignore_private_variables=self.ignore_private_variables, ignore_keys=ENUM_IGNORE_KEYS)
-        t2 = detailed__dict__(level.t2, ignore_private_variables=self.ignore_private_variables, ignore_keys=ENUM_IGNORE_KEYS)
+        t1 = detailed__dict__(level.t1, include_keys=ENUM_INCLUDE_KEYS)
+        t2 = detailed__dict__(level.t2, include_keys=ENUM_INCLUDE_KEYS)
 
         self._diff_dict(
             level,
@@ -727,7 +727,13 @@ def _diff_iterable_in_order(self, level, parents_ids=frozenset(), _original_type
         else:
             child_relationship_class = NonSubscriptableIterableRelationship
 
-        if self._all_values_basic_hashable(level.t1) and self._all_values_basic_hashable(level.t2) and self.iterable_compare_func is None:
+        if (
+            isinstance(level.t1, Sequence)
+            and isinstance(level.t2, Sequence)
+            and self._all_values_basic_hashable(level.t1)
+            and self._all_values_basic_hashable(level.t2)
+            and self.iterable_compare_func is None
+        ):
             local_tree_pass = TreeResult()
             self._diff_ordered_iterable_by_difflib(
                 level,

deepdiff/helper.py

@@ -133,7 +133,7 @@ class np_type:
 TEXT_VIEW = 'text'
 DELTA_VIEW = '_delta'
 
-ENUM_IGNORE_KEYS = frozenset(['_name_', '_value_', '_sort_order_'])
+ENUM_INCLUDE_KEYS = ['__objclass__', 'name', 'value']
 
 
 def short_repr(item, max_length=15):
@@ -630,26 +630,37 @@ def get_homogeneous_numpy_compatible_type_of_seq(seq):
         return False
 
 
-def detailed__dict__(obj, ignore_private_variables=True, ignore_keys=frozenset()):
+def detailed__dict__(obj, ignore_private_variables=True, ignore_keys=frozenset(), include_keys=None):
     """
     Get the detailed dictionary of an object.
 
     This is used so we retrieve object properties too.
     """
-    result = obj.__dict__.copy()  # A shallow copy
-    private_var_prefix = f"_{obj.__class__.__name__}__"  # The semi private variables in Python get this prefix
-    for key in ignore_keys:
-        if key in result or (
-            ignore_private_variables and key.startswith('__') and not key.startswith(private_var_prefix)
-        ):
-            del result[key]
-    for key in dir(obj):
-        if key not in result and key not in ignore_keys and (
-                not ignore_private_variables or (
-                    ignore_private_variables and not key.startswith('__') and not key.startswith(private_var_prefix)
-                )
-        ):
-            value = getattr(obj, key)
-            if not callable(value):
-                result[key] = value
+    if include_keys:
+        result = {}
+        for key in include_keys:
+            try:
+                value = getattr(obj, key)
+            except Exception:
+                pass
+            else:
+                if not callable(value) or key == '__objclass__':  # We don't want to compare functions, however for backward compatibility, __objclass__ needs to be reported.
+                    result[key] = value
+    else:
+        result = obj.__dict__.copy()  # A shallow copy
+        private_var_prefix = f"_{obj.__class__.__name__}__"  # The semi private variables in Python get this prefix
+        for key in ignore_keys:
+            if key in result or (
+                ignore_private_variables and key.startswith('__') and not key.startswith(private_var_prefix)
+            ):
+                del result[key]
+        for key in dir(obj):
+            if key not in result and key not in ignore_keys and (
+                    not ignore_private_variables or (
+                        ignore_private_variables and not key.startswith('__') and not key.startswith(private_var_prefix)
+                    )
+            ):
+                value = getattr(obj, key)
+                if not callable(value):
+                    result[key] = value
     return result

deepdiff/serialization.py

@@ -21,7 +21,9 @@
     toml = None  # pragma: no cover.
 try:
     import clevercsv
+    csv = None
 except ImportError:  # pragma: no cover.
+    import csv
     clevercsv = None  # pragma: no cover.
 from copy import deepcopy
 from functools import partial
@@ -424,9 +426,11 @@ def load_path_content(path, file_type=None):
             content = the_file.read()
             content = pickle_load(content)
     elif file_type in {'csv', 'tsv'}:
-        if clevercsv is None:  # pragma: no cover.
-            raise ImportError('CleverCSV needs to be installed.')  # pragma: no cover.
-        content = clevercsv.read_dicts(path)
+        if clevercsv:  # pragma: no cover.
+            content = clevercsv.read_dicts(path)
+        else:
+            with open(path, 'r') as the_file:
+                content = list(csv.DictReader(the_file))
         logger.info(f"NOTE: CSV content was empty in {path}")
 
         # Everything in csv is string but we try to automatically convert any numbers we find
@@ -485,11 +489,13 @@ def _save_content(content, path, file_type, keep_backup=True):
         with open(path, 'wb') as the_file:
             content = pickle_dump(content, file_obj=the_file)
     elif file_type in {'csv', 'tsv'}:
-        if clevercsv is None:  # pragma: no cover.
-            raise ImportError('CleverCSV needs to be installed.')  # pragma: no cover.
+        if clevercsv:  # pragma: no cover.
+            dict_writer = clevercsv.DictWriter
+        else:
+            dict_writer = csv.DictWriter
         with open(path, 'w', newline='') as csvfile:
             fieldnames = list(content[0].keys())
-            writer = clevercsv.DictWriter(csvfile, fieldnames=fieldnames)
+            writer = dict_writer(csvfile, fieldnames=fieldnames)
             writer.writeheader()
             writer.writerows(content)
     else:

docs/authors.rst

@@ -60,6 +60,8 @@ Authors in order of the timeline of their contributions:
 -  `dtorres-sf`_ for the fix for diffing using iterable_compare_func with nested objects.
 -  `Enric Pou <https://github.com/epou>`__ for bug fix of ValueError
    when using Decimal 0.x
+- `Uwe Fladrich <https://github.com/uwefladrich>`__ for fixing bug when diff'ing non-sequence iterables
+
 
 .. _Sep Dehpour (Seperman): http://www.zepworks.com
 .. _Victor Hahn Castell: http://hahncastell.de

docs/changelog.rst

@@ -5,6 +5,39 @@ Changelog
 
 DeepDiff Changelog
 
+-  v6-2-2
+
+   -  Enum test fix for python 3.11
+   -  Adding support for dateutils rrules
+
+-  v6-2-1
+
+   -  Removed the print statements.
+
+-  v6-2-0
+
+   -  Major improvement in the diff report for lists when items are all
+      hashable and the order of items is important.
+
+-  v6-1-0
+
+   -  DeepDiff.affected_paths can be used to get the list of all paths
+      where a change, addition, or deletion was reported for.
+   -  DeepDiff.affected_root_keys can be used to get the list of all
+      paths where a change, addition, or deletion was reported for.
+   -  Bugfix: ValueError when using Decimal 0.x #339 by `Enric
+      Pou <https://github.com/epou>`__
+   -  Serialization of UUID
+
+-  v6-0-0
+
+   -  `Exclude obj callback
+      strict <https://github.com/seperman/deepdiff/pull/320/files>`__
+      parameter is added to DeepDiff by Mikhail Khviyuzov
+      `mskhviyu <https://github.com/mskhviyu>`__.
+   -  A fix for diffing using ``iterable_compare_func`` with nested
+      objects by `dtorres-sf <https://github.com/dtorres-sf>`__ who
+      originally contributed this feature.
 -  v5-7-0:
 
    -  https://github.com/seperman/deepdiff/pull/284 Bug-Fix: TypeError

requirements-cli.txt

@@ -1,4 +1,2 @@
 click==8.1.3
 pyyaml==6.0
-toml==0.10.2
-clevercsv==0.7.4

requirements-dev-3.7.txt

@@ -7,3 +7,4 @@ ipdb==0.13.9
 numpy==1.21.6
 pytest==7.1.2
 python-dotenv==0.20.0
+python-dateutil==2.8.2

requirements-dev.txt

@@ -1,15 +1,16 @@
-wheel==0.37.0
+wheel==0.38.4
 -r requirements.txt
 -r requirements-cli.txt
 bump2version==1.0.1
-jsonpickle==2.2.0
-coverage==6.4.3
+jsonpickle==3.0.0
+coverage==6.5.0
 ipdb==0.13.9
-numpy==1.23.1
-pytest==7.1.2
-pytest-cov==3.0.0
-python-dotenv==0.20.0
-watchdog==2.1.9
-Sphinx==5.1.1
-sphinx-sitemap==2.2.0
-flake8==5.0.4
+numpy==1.23.5
+pytest==7.2.0
+pytest-cov==4.0.0
+python-dotenv==0.21.0
+watchdog==2.2.0
+Sphinx==5.3.0
+sphinx-sitemap==2.2.1
+flake8==6.0.0
+python-dateutil==2.8.2

tests/test_diff_text.py

@@ -1713,3 +1713,34 @@ def __thing2(self):
         }
 
         assert expected2 == diff2
+
+    def test_diffs_rrules(self):
+
+        from dateutil.rrule import MONTHLY, rrule
+
+        d = DeepDiff(
+            rrule(freq=MONTHLY, count=5, dtstart=datetime.datetime(2014, 12, 31)),
+            rrule(freq=MONTHLY, count=4, dtstart=datetime.datetime(2011, 12, 31)),
+        )
+
+        assert d == {
+            "values_changed": {
+                "root[0]": {
+                    "new_value": datetime.datetime(2011, 12, 31, 0, 0),
+                    "old_value": datetime.datetime(2014, 12, 31, 0, 0),
+                },
+                "root[1]": {
+                    "new_value": datetime.datetime(2012, 1, 31, 0, 0),
+                    "old_value": datetime.datetime(2015, 1, 31, 0, 0),
+                },
+                "root[2]": {
+                    "new_value": datetime.datetime(2012, 3, 31, 0, 0),
+                    "old_value": datetime.datetime(2015, 3, 31, 0, 0),
+                },
+                "root[3]": {
+                    "new_value": datetime.datetime(2012, 5, 31, 0, 0),
+                    "old_value": datetime.datetime(2015, 5, 31, 0, 0),
+                },
+            },
+            "iterable_item_removed": {"root[4]": datetime.datetime(2015, 7, 31, 0, 0)},
+        }

tests/test_helper.py

@@ -9,7 +9,7 @@
     cartesian_product_of_shape, literal_eval_extended,
     not_found, OrderedSetPlus, diff_numpy_array, cartesian_product_numpy,
     get_truncate_datetime, datetime_normalize,
-    detailed__dict__, ENUM_IGNORE_KEYS, add_root_to_paths,
+    detailed__dict__, ENUM_INCLUDE_KEYS, add_root_to_paths,
 )
 
 
@@ -278,15 +278,15 @@ def test_datetime_normalize(self, truncate_datetime, obj, expected):
         result = datetime_normalize(truncate_datetime, obj)
         assert expected == result
 
-    @pytest.mark.parametrize('obj, ignore_keys, expected', [
+    @pytest.mark.parametrize('obj, include_keys, expected', [
         (
             MyEnum.A,
-            ENUM_IGNORE_KEYS,
+            ENUM_INCLUDE_KEYS,
             {'__objclass__': MyEnum, 'name': 'A', 'value': 1},
         )
     ])
-    def test_detailed__dict__(self, obj, ignore_keys, expected):
-        result = detailed__dict__(obj, ignore_private_variables=True, ignore_keys=ignore_keys)
+    def test_detailed__dict__(self, obj, include_keys, expected):
+        result = detailed__dict__(obj, ignore_private_variables=True, include_keys=include_keys)
         assert expected == result, f"test_detailed__dict__ failed for {obj}"
 
     @pytest.mark.parametrize('test_num, value, expected', [