Merge pull request #423 from seperman/dev

6.6.0

Author: seperman

.github/workflows/main.yaml

@@ -14,6 +14,9 @@ jobs:
       matrix:
         python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
         architecture: ["x64"]
+        include:
+          - python-version: "3.10"
+            numpy-version: "2.0.dev"
     steps:
     - uses: actions/checkout@v2
     - name: Setup Python ${{ matrix.python-version }} on ${{ matrix.architecture }}
@@ -37,6 +40,9 @@ jobs:
     - name: Install dependencies
       if: matrix.python-version != 3.7
       run: pip install -r requirements-dev.txt
+    - name: Install Numpy Dev
+      if: ${{ matrix.numpy-version }}
+      run: pip install -I --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple "numpy>=0.0.dev0" 
     - name: Lint with flake8
       if: matrix.python-version == 3.11
       run: |

AUTHORS.md

@@ -58,3 +58,4 @@ Authors in order of the timeline of their contributions:
 - [kor4ik](https://github.com/kor4ik) for the bugfix for `include_paths` for nested dictionaries.
 - [martin-kokos](https://github.com/martin-kokos) for using tomli and tomli-w for dealing with tomli files.
 - [Alex Sauer-Budge](https://github.com/amsb) for the bugfix for `datetime.date`.
+- [William Jamieson](https://github.com/WilliamJamieson) for [NumPy 2.0 compatibility](https://github.com/seperman/deepdiff/pull/422)

README.md

@@ -1,4 +1,4 @@
-# DeepDiff v 6.5.0
+# DeepDiff v 6.6.0
 
 ![Downloads](https://img.shields.io/pypi/dm/deepdiff.svg?style=flat)
 ![Python Versions](https://img.shields.io/pypi/pyversions/deepdiff.svg?style=flat)
@@ -17,12 +17,17 @@
 
 Tested on Python 3.7+ and PyPy3.
 
-- **[Documentation](https://zepworks.com/deepdiff/6.5.0/)**
+- **[Documentation](https://zepworks.com/deepdiff/6.6.0/)**
 
 ## What is new?
 
 Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.
 
+DeepDiff 6-6-0
+
+- [Serialize To Flat Dicts]()
+- [NumPy 2.0 compatibility](https://github.com/seperman/deepdiff/pull/422) by [William Jamieson](https://github.com/WilliamJamieson)
+
 DeepDiff 6-5-0
 
 - [parse_path](https://zepworks.com/deepdiff/current/faq.html#q-how-do-i-parse-deepdiff-result-paths)
@@ -62,6 +67,13 @@ Install optional packages:
 
 <https://zepworks.com/deepdiff/current/>
 
+### A message from Sep, the creator of DeepDiff
+
+> 👋 Hi there,
+> If you find value in DeepDiff, you might be interested in another tool I've crafted: [Qluster](https://qluster.ai/solution). <br/>
+> As an engineer, I understand the frustration of wrestling with **unruly data** in pipelines. <br/>
+> I developed **Qluster** to empower product managers and ops teams to control and resolve data issues autonomously and **stop bugging the engineers**! 🛠️
+
 # ChangeLog
 
 Please take a look at the [CHANGELOG](CHANGELOG.md) file.
@@ -70,7 +82,6 @@ Please take a look at the [CHANGELOG](CHANGELOG.md) file.
 
 :mega: **Please fill out our [fast 5-question survey](https://forms.gle/E6qXexcgjoKnSzjB8)** so that we can learn how & why you use DeepDiff, and what improvements we should make. Thank you! :dancers:
 
-
 # Contribute
 
 1. Please make your PR against the dev branch
@@ -86,11 +97,11 @@ Thank you!
 
 How to cite this library (APA style):
 
-    Dehpour, S. (2023). DeepDiff (Version 6.5.0) [Software]. Available from https://github.com/seperman/deepdiff.
+    Dehpour, S. (2023). DeepDiff (Version 6.6.0) [Software]. Available from https://github.com/seperman/deepdiff.
 
 How to cite this library (Chicago style):
 
-    Dehpour, Sep. 2023. DeepDiff (version 6.5.0).
+    Dehpour, Sep. 2023. DeepDiff (version 6.6.0).
 
 # Authors
 

deepdiff/__init__.py

@@ -1,6 +1,6 @@
 """This module offers the DeepDiff, DeepSearch, grep, Delta and DeepHash classes."""
 # flake8: noqa
-__version__ = '6.5.0'
+__version__ = '6.6.0'
 import logging
 
 if __name__ == '__main__':

deepdiff/delta.py

@@ -1,4 +1,5 @@
 import logging
+from functools import partial
 from collections.abc import Mapping
 from copy import deepcopy
 from ordered_set import OrderedSet
@@ -9,7 +10,7 @@
     np_ndarray, np_array_factory, numpy_dtypes, get_doc,
     not_found, numpy_dtype_string_to_type, dict_,
 )
-from deepdiff.path import _path_to_elements, _get_nested_obj, _get_nested_obj_and_force, GET, GETATTR
+from deepdiff.path import _path_to_elements, _get_nested_obj, _get_nested_obj_and_force, GET, GETATTR, parse_path
 from deepdiff.anyset import AnySet
 
 
@@ -591,6 +592,155 @@ def dumps(self):
     def to_dict(self):
         return dict(self.diff)
 
+    @staticmethod
+    def _get_flat_row(action, info, _parse_path, keys_and_funcs):
+        for path, details in info.items():
+            row = {'path': _parse_path(path), 'action': action}
+            for key, new_key, func in keys_and_funcs:
+                if key in details:
+                    if func:
+                        row[new_key] = func(details[key])
+                    else:
+                        row[new_key] = details[key]
+            yield row
+
+    def to_flat_dicts(self, include_action_in_path=False, report_type_changes=True):
+        """
+        Returns a flat list of actions that is easily machine readable.
+
+        For example:
+            {'iterable_item_added': {'root[3]': 5, 'root[2]': 3}}
+
+        Becomes:
+            [
+                {'path': [3], 'value': 5, 'action': 'iterable_item_added'},
+                {'path': [2], 'value': 3, 'action': 'iterable_item_added'},
+            ]
+
+        
+        **Parameters**
+
+        include_action_in_path : Boolean, default=False
+            When False, we translate DeepDiff's paths like root[3].attribute1 into a [3, 'attribute1'].
+            When True, we include the action to retrieve the item in the path: [(3, 'GET'), ('attribute1', 'GETATTR')]
+
+        report_type_changes : Boolean, default=True
+            If False, we don't report the type change. Instead we report the value change.
+
+        Example:
+            t1 = {"a": None}
+            t2 = {"a": 1}
+
+            dump = Delta(DeepDiff(t1, t2)).dumps()
+            delta = Delta(dump)
+            assert t2 == delta + t1
+
+            flat_result = delta.to_flat_dicts()
+            flat_expected = [{'path': ['a'], 'action': 'type_changes', 'value': 1, 'new_type': int, 'old_type': type(None)}]
+            assert flat_expected == flat_result
+
+            flat_result2 = delta.to_flat_dicts(report_type_changes=False)
+            flat_expected2 = [{'path': ['a'], 'action': 'values_changed', 'value': 1}]
+
+        **List of actions**
+
+        Here are the list of actions that the flat dictionary can return.
+            iterable_item_added
+            iterable_item_removed
+            values_changed
+            type_changes
+            set_item_added
+            set_item_removed
+            dictionary_item_added
+            dictionary_item_removed
+            attribute_added
+            attribute_removed
+        """
+        result = []
+        if include_action_in_path:
+            _parse_path = partial(parse_path, include_actions=True)
+        else:
+            _parse_path = parse_path
+        if report_type_changes:
+            keys_and_funcs = [
+                ('value', 'value', None),
+                ('new_value', 'value', None),
+                ('old_value', 'old_value', None),
+                ('new_type', 'type', None),
+                ('old_type', 'old_type', None),
+                ('new_path', 'new_path', _parse_path),
+            ]
+            action_mapping = {}
+        else:
+            keys_and_funcs = [
+                ('value', 'value', None),
+                ('new_value', 'value', None),
+                ('old_value', 'old_value', None),
+                ('new_path', 'new_path', _parse_path),
+            ]
+            action_mapping = {'type_changes': 'values_changed'}
+
+        FLATTENING_NEW_ACTION_MAP = {
+            'iterable_items_added_at_indexes': 'iterable_item_added',
+            'iterable_items_removed_at_indexes': 'iterable_item_removed',
+        }
+        for action, info in self.diff.items():
+            if action in FLATTENING_NEW_ACTION_MAP:
+                new_action = FLATTENING_NEW_ACTION_MAP[action]
+                for path, index_to_value in info.items():
+                    path = _parse_path(path)
+                    for index, value in index_to_value.items():
+                        path2 = path.copy()
+                        if include_action_in_path:
+                            path2.append((index, 'GET'))
+                        else:
+                            path2.append(index)
+                        result.append(
+                            {'path': path2, 'value': value, 'action': new_action}
+                        )
+            elif action in {'set_item_added', 'set_item_removed'}:
+                for path, values in info.items():
+                    path = _parse_path(path)
+                    for value in values:
+                        result.append(
+                            {'path': path, 'value': value, 'action': action}
+                        )
+            elif action == 'dictionary_item_added':
+                for path, value in info.items():
+                    path = _parse_path(path)
+                    if isinstance(value, dict) and len(value) == 1:
+                        new_key = next(iter(value))
+                        path.append(new_key)
+                        value = value[new_key]
+                    elif isinstance(value, (list, tuple)) and len(value) == 1:
+                        value = value[0]
+                        path.append(0)
+                        action = 'iterable_item_added'
+                    elif isinstance(value, set) and len(value) == 1:
+                        value = value.pop()
+                        action = 'set_item_added'
+                    result.append(
+                        {'path': path, 'value': value, 'action': action}
+                    )
+            elif action in {
+                'dictionary_item_removed', 'iterable_item_added',
+                'iterable_item_removed', 'attribute_removed', 'attribute_added'
+            }:
+                for path, value in info.items():
+                    path = _parse_path(path)
+                    result.append(
+                        {'path': path, 'value': value, 'action': action}
+                    )
+            else:
+                for row in self._get_flat_row(
+                    action=action_mapping.get(action, action),
+                    info=info,
+                    _parse_path=_parse_path,
+                    keys_and_funcs=keys_and_funcs,
+                ):
+                    result.append(row)
+        return result
+
 
 if __name__ == "__main__":  # pragma: no cover
     import doctest

deepdiff/diff.py

@@ -181,7 +181,7 @@ def __init__(self,
             self.custom_operators = custom_operators or []
             self.ignore_order = ignore_order
 
-            self.ignore_order_func = ignore_order_func or (lambda *_args, **_kwargs: ignore_order)
+            self.ignore_order_func = ignore_order_func
 
             ignore_type_in_groups = ignore_type_in_groups or []
             if numbers == ignore_type_in_groups or numbers in ignore_type_in_groups:
@@ -649,7 +649,7 @@ def _iterables_subscriptable(t1, t2):
 
     def _diff_iterable(self, level, parents_ids=frozenset(), _original_type=None, local_tree=None):
         """Difference of iterables"""
-        if self.ignore_order_func(level):
+        if (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order:
             self._diff_iterable_with_deephash(level, parents_ids, _original_type=_original_type, local_tree=local_tree)
         else:
             self._diff_iterable_in_order(level, parents_ids, _original_type=_original_type, local_tree=local_tree)
@@ -1103,7 +1103,9 @@ def _get_most_in_common_pairs_in_iterables(
         # And the objects with the same distances are grouped together in an ordered set.
         # It also includes a "max" key that is just the value of the biggest current distance in the
         # most_in_common_pairs dictionary.
-        most_in_common_pairs = defaultdict(lambda: defaultdict(OrderedSetPlus))
+        def defaultdict_orderedset():
+            return defaultdict(OrderedSetPlus)
+        most_in_common_pairs = defaultdict(defaultdict_orderedset)
         pairs = dict_()
 
         pre_calced_distances = None
@@ -1390,7 +1392,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset(), local_tree=None):
             # which means numpy module needs to be available. So np can't be None.
             raise ImportError(CANT_FIND_NUMPY_MSG)  # pragma: no cover
 
-        if not self.ignore_order_func(level):
+        if (self.ignore_order_func and not self.ignore_order_func(level)) or not self.ignore_order:
             # fast checks
             if self.significant_digits is None:
                 if np.array_equal(level.t1, level.t2, equal_nan=self.ignore_nan_inequality):
@@ -1416,7 +1418,7 @@ def _diff_numpy_array(self, level, parents_ids=frozenset(), local_tree=None):
             dimensions = len(shape)
             if dimensions == 1:
                 self._diff_iterable(level, parents_ids, _original_type=_original_type, local_tree=local_tree)
-            elif self.ignore_order_func(level):
+            elif (self.ignore_order_func and self.ignore_order_func(level)) or self.ignore_order:
                 # arrays are converted to python lists so that certain features of DeepDiff can apply on them easier.
                 # They will be converted back to Numpy at their final dimension.
                 level.t1 = level.t1.tolist()

deepdiff/helper.py

@@ -42,11 +42,11 @@ class pydantic_base_model_type:
     np_uintp = np_type  # pragma: no cover.
     np_float32 = np_type  # pragma: no cover.
     np_float64 = np_type  # pragma: no cover.
-    np_float_ = np_type  # pragma: no cover.
+    np_double = np_type  # pragma: no cover.
     np_floating = np_type  # pragma: no cover.
     np_complex64 = np_type  # pragma: no cover.
     np_complex128 = np_type  # pragma: no cover.
-    np_complex_ = np_type  # pragma: no cover.
+    np_cdouble = np_type  # pragma: no cover.
     np_complexfloating = np_type  # pragma: no cover.
 else:
     np_array_factory = np.array
@@ -64,21 +64,21 @@ class pydantic_base_model_type:
     np_uintp = np.uintp
     np_float32 = np.float32
     np_float64 = np.float64
-    np_float_ = np.float_
+    np_double = np.double  # np.float_ is an alias for np.double and is being removed by NumPy 2.0
     np_floating = np.floating
     np_complex64 = np.complex64
     np_complex128 = np.complex128
-    np_complex_ = np.complex_
+    np_cdouble = np.cdouble  # np.complex_ is an alias for np.cdouble and is being removed by NumPy 2.0
     np_complexfloating = np.complexfloating
 
 numpy_numbers = (
     np_int8, np_int16, np_int32, np_int64, np_uint8,
     np_uint16, np_uint32, np_uint64, np_intp, np_uintp,
-    np_float32, np_float64, np_float_, np_floating, np_complex64,
-    np_complex128, np_complex_,)
+    np_float32, np_float64, np_double, np_floating, np_complex64,
+    np_complex128, np_cdouble,)
 
 numpy_complex_numbers = (
-    np_complexfloating, np_complex64, np_complex128, np_complex_,
+    np_complexfloating, np_complex64, np_complex128, np_cdouble,
 )
 
 numpy_dtypes = set(numpy_numbers)
@@ -655,7 +655,7 @@ def diff_numpy_array(A, B):
     By Divakar
     https://stackoverflow.com/a/52417967/1497443
     """
-    return A[~np.in1d(A, B)]
+    return A[~np.isin(A, B)]
 
 
 PYTHON_TYPE_TO_NUMPY_TYPE = {

deepdiff/serialization.py

@@ -44,7 +44,9 @@
 from copy import deepcopy
 from functools import partial
 from collections.abc import Mapping
-from deepdiff.helper import (strings, get_type, TEXT_VIEW)
+from deepdiff.helper import (
+    strings, get_type, TEXT_VIEW, np_float32, np_float64, np_int32, np_int64
+)
 from deepdiff.model import DeltaResult
 
 logger = logging.getLogger(__name__)
@@ -539,6 +541,10 @@ def _serialize_decimal(value):
     bytes: lambda x: x.decode('utf-8'),
     datetime.datetime: lambda x: x.isoformat(),
     uuid.UUID: lambda x: str(x),
+    np_float32: float,
+    np_float64: float,
+    np_int32: int,
+    np_int64: int
 }
 
 if PydanticBaseModel:

docs/authors.rst

@@ -83,6 +83,7 @@ Authors in order of the timeline of their contributions:
    and tomli-w for dealing with tomli files.
 -  `Alex Sauer-Budge <https://github.com/amsb>`__ for the bugfix for
    ``datetime.date``.
+- `William Jamieson <https://github.com/WilliamJamieson>`__ for `NumPy 2.0 compatibility <https://github.com/seperman/deepdiff/pull/422>`__
 
 .. _Sep Dehpour (Seperman): http://www.zepworks.com
 .. _Victor Hahn Castell: http://hahncastell.de