subtract delta fixed when iterable_compare_func is used. Better handling

of force adding a delta to an object. We change between an empty list.
and an empty dictionary when needed. We find the closest list item when
removing items from iterable and force=True.

Author: seperman

deepdiff/delta.py

@@ -71,6 +71,7 @@ def __init__(
         diff=None,
         delta_path=None,
         delta_file=None,
+        delta_diff=None,
         flat_dict_list=None,
         deserializer=pickle_load,
         log_errors=True,
@@ -81,6 +82,7 @@ def __init__(
         verify_symmetry=None,
         bidirectional=False,
         always_include_values=False,
+        iterable_compare_func_was_used=None,
         force=False,
     ):
         if hasattr(deserializer, '__code__') and 'safe_to_import' in set(deserializer.__code__.co_varnames):
@@ -114,6 +116,8 @@ def _deserializer(obj, safe_to_import=None):
             with open(delta_path, 'rb') as the_file:
                 content = the_file.read()
             self.diff = _deserializer(content, safe_to_import=safe_to_import)
+        elif delta_diff:
+            self.diff = delta_diff
         elif delta_file:
             try:
                 content = delta_file.read()
@@ -128,7 +132,10 @@ def _deserializer(obj, safe_to_import=None):
         self.mutate = mutate
         self.raise_errors = raise_errors
         self.log_errors = log_errors
-        self._numpy_paths = self.diff.pop('_numpy_paths', False)
+        self._numpy_paths = self.diff.get('_numpy_paths', False)
+        # When we create the delta from a list of flat dictionaries, details such as iterable_compare_func_was_used get lost.
+        # That's why we allow iterable_compare_func_was_used to be explicitly set.
+        self._iterable_compare_func_was_used = self.diff.get('_iterable_compare_func_was_used', iterable_compare_func_was_used)
         self.serializer = serializer
         self.deserializer = deserializer
         self.force = force
@@ -198,7 +205,17 @@ def _do_verify_changes(self, path, expected_old_value, current_old_value):
             self._raise_or_log(VERIFICATION_MSG.format(
                 path_str, expected_old_value, current_old_value, VERIFY_BIDIRECTIONAL_MSG))
 
-    def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expected_old_value, elem=None, action=None, forced_old_value=None):
+    def _get_elem_and_compare_to_old_value(
+        self,
+        obj,
+        path_for_err_reporting,
+        expected_old_value,
+        elem=None,
+        action=None,
+        forced_old_value=None,
+        next_element=None,
+    ):
+        # if forced_old_value is not None:
         try:
             if action == GET:
                 current_old_value = obj[elem]
@@ -208,9 +225,21 @@ def _get_elem_and_compare_to_old_value(self, obj, path_for_err_reporting, expect
                 raise DeltaError(INVALID_ACTION_WHEN_CALLING_GET_ELEM.format(action))
         except (KeyError, IndexError, AttributeError, TypeError) as e:
             if self.force:
-                _forced_old_value = {} if forced_old_value is None else forced_old_value
+                if forced_old_value is None:
+                    if next_element is None or isinstance(next_element, str):
+                        _forced_old_value = {}
+                    else:
+                        _forced_old_value = []    
+                else:
+                    _forced_old_value = forced_old_value
                 if action == GET:
-                    obj[elem] = _forced_old_value
+                    if isinstance(obj, list):
+                        if isinstance(elem, int) and elem < len(obj):
+                            obj[elem] = _forced_old_value
+                        else:
+                            obj.append(_forced_old_value)
+                    else:
+                        obj[elem] = _forced_old_value
                 elif action == GETATTR:
                     setattr(obj, elem, _forced_old_value)
                 return _forced_old_value
@@ -277,6 +306,11 @@ def _set_new_value(self, parent, parent_to_obj_elem, parent_to_obj_action,
                 parent, obj, path, parent_to_obj_elem,
                 parent_to_obj_action, elements,
                 to_type=list, from_type=tuple)
+        if elem != 0 and self.force and isinstance(obj, list) and len(obj) == 0:
+            # it must have been a dictionary    
+            obj = {}
+            self._simple_set_elem_value(obj=parent, path_for_err_reporting=path, elem=parent_to_obj_elem,
+                                        value=obj, action=parent_to_obj_action)
         self._simple_set_elem_value(obj=obj, path_for_err_reporting=path, elem=elem,
                                     value=new_value, action=action)
 
@@ -356,6 +390,9 @@ def _do_item_added(self, items, sort=True, insert=False):
         else:
             items = items.items()
 
+        # if getattr(self, 'DEBUG', None):
+        #     import pytest; pytest.set_trace()
+
         for path, new_value in items:
             elem_and_details = self._get_elements_and_details(path)
             if elem_and_details:
@@ -404,14 +441,21 @@ def _get_elements_and_details(self, path):
         try:
             elements = _path_to_elements(path)
             if len(elements) > 1:
-                parent = self.get_nested_obj(obj=self, elements=elements[:-2])
+                elements_subset = elements[:-2]
+                if len(elements_subset) != len(elements):
+                    next_element = elements[-2][0]
+                    next2_element = elements[-1][0]
+                else:
+                    next_element = None
+                parent = self.get_nested_obj(obj=self, elements=elements_subset, next_element=next_element)
                 parent_to_obj_elem, parent_to_obj_action = elements[-2]
                 obj = self._get_elem_and_compare_to_old_value(
                     obj=parent, path_for_err_reporting=path, expected_old_value=None,
-                    elem=parent_to_obj_elem, action=parent_to_obj_action)
+                    elem=parent_to_obj_elem, action=parent_to_obj_action, next_element=next2_element)
             else:
                 parent = parent_to_obj_elem = parent_to_obj_action = None
-                obj = self.get_nested_obj(obj=self, elements=elements[:-1])
+                obj = self
+                # obj = self.get_nested_obj(obj=self, elements=elements[:-1])
             elem, action = elements[-1]
         except Exception as e:
             self._raise_or_log(UNABLE_TO_GET_ITEM_MSG.format(path, e))
@@ -458,6 +502,57 @@ def _do_values_or_type_changed(self, changes, is_type_change=False, verify_chang
                 self._do_verify_changes(path, expected_old_value, current_old_value)
 
     def _do_item_removed(self, items):
+        """
+        Handle removing items.
+        """
+        # Sorting the iterable_item_removed in reverse order based on the paths.
+        # So that we delete a bigger index before a smaller index
+        # if hasattr(self, 'DEBUG'):
+        #     import pytest; pytest.set_trace()
+        for path, expected_old_value in sorted(items.items(), key=self._sort_key_for_item_added, reverse=True):
+            elem_and_details = self._get_elements_and_details(path)
+            if elem_and_details:
+                elements, parent, parent_to_obj_elem, parent_to_obj_action, obj, elem, action = elem_and_details
+            else:
+                continue  # pragma: no cover. Due to cPython peephole optimizer, this line doesn't get covered. https://github.com/nedbat/coveragepy/issues/198
+
+            look_for_expected_old_value = False
+            current_old_value = not_found
+            try:
+                if action == GET:
+                    current_old_value = obj[elem]
+                    look_for_expected_old_value = current_old_value != expected_old_value
+                elif action == GETATTR:
+                    current_old_value = getattr(obj, elem)
+                    look_for_expected_old_value = current_old_value != expected_old_value
+            except (KeyError, IndexError, AttributeError, TypeError):
+                look_for_expected_old_value = True
+
+            if look_for_expected_old_value and isinstance(obj, list) and not self._iterable_compare_func_was_used:
+                # It may return None if it doesn't find it
+                elem = self._find_closest_iterable_element_for_index(obj, elem, expected_old_value)
+                if elem is not None:
+                    current_old_value = expected_old_value
+            if current_old_value is not_found or elem is None:
+                continue
+
+            self._del_elem(parent, parent_to_obj_elem, parent_to_obj_action,
+                           obj, elements, path, elem, action)
+            self._do_verify_changes(path, expected_old_value, current_old_value)
+
+    def _find_closest_iterable_element_for_index(self, obj, elem, expected_old_value):
+        closest_elem = None
+        closest_distance = float('inf')
+        for index, value in enumerate(obj):
+            dist = abs(index - elem)
+            if dist > closest_distance:
+                break
+            if value == expected_old_value and dist < closest_distance:
+                closest_elem = index
+                closest_distance = dist
+        return closest_elem
+
+    def _do_item_removedOLD(self, items):
         """
         Handle removing items.
         """
@@ -695,10 +790,9 @@ def _from_flat_dicts(flat_dict_list):
         Create the delta's diff object from the flat_dict_list
         """
         result = {}
-
-        DEFLATTENING_NEW_ACTION_MAP = {
-            'iterable_item_added': 'iterable_items_added_at_indexes',
-            'iterable_item_removed': 'iterable_items_removed_at_indexes',
+        FLATTENING_NEW_ACTION_MAP = {
+            'unordered_iterable_item_added': 'iterable_items_added_at_indexes',
+            'unordered_iterable_item_removed': 'iterable_items_removed_at_indexes',
         }
         for flat_dict in flat_dict_list:
             index = None
@@ -710,8 +804,8 @@ def _from_flat_dicts(flat_dict_list):
                 raise ValueError("Flat dict need to include the 'action'.")
             if path is None:
                 raise ValueError("Flat dict need to include the 'path'.")
-            if action in DEFLATTENING_NEW_ACTION_MAP:
-                action = DEFLATTENING_NEW_ACTION_MAP[action]
+            if action in FLATTENING_NEW_ACTION_MAP:
+                action = FLATTENING_NEW_ACTION_MAP[action]
                 index = path.pop()
             if action in {'attribute_added', 'attribute_removed'}:
                 root_element = ('root', GETATTR)
@@ -729,8 +823,8 @@ def _from_flat_dicts(flat_dict_list):
                     result[action][path_str] = set()
                 result[action][path_str].add(value)
             elif action in {
-                'dictionary_item_added', 'dictionary_item_removed', 'iterable_item_added',
-                'iterable_item_removed', 'attribute_removed', 'attribute_added'
+                'dictionary_item_added', 'dictionary_item_removed',
+                'attribute_removed', 'attribute_added', 'iterable_item_added', 'iterable_item_removed',
             }:
                 result[action][path_str] = value
             elif action == 'values_changed':
@@ -843,10 +937,12 @@ def to_flat_dicts(self, include_action_in_path=False, report_type_changes=True):
             ]
 
         FLATTENING_NEW_ACTION_MAP = {
-            'iterable_items_added_at_indexes': 'iterable_item_added',
-            'iterable_items_removed_at_indexes': 'iterable_item_removed',
+            'iterable_items_added_at_indexes': 'unordered_iterable_item_added',
+            'iterable_items_removed_at_indexes': 'unordered_iterable_item_removed',
         }
         for action, info in self.diff.items():
+            if action.startswith('_'):
+                continue
             if action in FLATTENING_NEW_ACTION_MAP:
                 new_action = FLATTENING_NEW_ACTION_MAP[action]
                 for path, index_to_value in info.items():

deepdiff/path.py

@@ -115,7 +115,7 @@ def _path_to_elements(path, root_element=DEFAULT_FIRST_ELEMENT):
     return tuple(elements)
 
 
-def _get_nested_obj(obj, elements):
+def _get_nested_obj(obj, elements, next_element=None):
     for (elem, action) in elements:
         if action == GET:
             obj = obj[elem]
@@ -124,21 +124,50 @@ def _get_nested_obj(obj, elements):
     return obj
 
 
-def _get_nested_obj_and_force(obj, elements):
-    for (elem, action) in elements:
+def _guess_type(elements, elem, index, next_element):
+    # If we are not at the last elements
+    if index < len(elements) - 1:
+        # We assume it is a nested dictionary not a nested list
+        return {}
+    if isinstance(next_element, int):
+        return []
+    return {}
+
+
+def _get_nested_obj_and_force(obj, elements, next_element=None):
+    prev_elem = None
+    prev_action = None
+    prev_obj = obj
+    for index, (elem, action) in enumerate(elements):
+        _prev_obj = obj
         if action == GET:
             try:
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except KeyError:
-                obj[elem] = {}
+                obj[elem] = _guess_type(elements, elem, index, next_element)
                 obj = obj[elem]
+                prev_obj = _prev_obj
             except IndexError:
                 if isinstance(obj, list) and isinstance(elem, int) and elem >= len(obj):
                     obj.extend([None] * (elem - len(obj)))
-                    obj.append({})
+                    obj.append(_guess_type(elements, elem, index), next_element)
                     obj = obj[-1]
+                    prev_obj = _prev_obj
+                elif isinstance(obj, list) and len(obj) == 0 and prev_elem:
+                    # We ran into an empty list that should have been a dictionary
+                    # We need to change it from an empty list to a dictionary
+                    obj = {elem: _guess_type(elements, elem, index, next_element)}
+                    if prev_action == GET:
+                        prev_obj[prev_elem] = obj
+                    else:
+                        setattr(prev_obj, prev_elem, obj)
+                    obj = obj[elem]
         elif action == GETATTR:
             obj = getattr(obj, elem)
+            prev_obj = _prev_obj
+        prev_elem = elem
+        prev_action = action
     return obj
 
 
@@ -245,9 +274,10 @@ def stringify_element(param, quote_str=None):
         new_param = []
         for char in param:
             if char in {'"', "'"}:
+                import pytest; pytest.set_trace()
                 new_param.append('\\')
             new_param.append(char)
-        param = ''.join(new_param)
+        result = '"' + ''.join(new_param) + '"'
     elif has_quote:
         result = f'"{param}"'
     elif has_double_quote:

deepdiff/serialization.py

@@ -256,6 +256,9 @@ def _to_delta_dict(self, directed=True, report_repetition_required=True, always_
             # and will be omitted when counting distance. (Look inside the distance module.)
             result['_numpy_paths'] = self._numpy_paths
 
+        if self.iterable_compare_func:
+            result['_iterable_compare_func_was_used'] = True
+
         return deepcopy(dict(result))
 
     def pretty(self):

docs/delta.rst

@@ -21,6 +21,10 @@ delta_path : String, default=None.
 delta_file : File Object, default=None.
     :ref:`delta_file_label` is the file object containing the delta data.
 
+delta_diff : Delta diff, default=None.
+    This is a slightly different diff than the output of DeepDiff. When Delta object is initiated from the DeepDiff output, it transforms the diff into a slightly different structure that is more suitable for delta. You can find that object via delta.diff.
+    It is the same object that is serialized when you create a delta dump. If you already have the delta_diff object, you can pass it to Delta via the delta_diff parameter.
+
 flat_dict_list : List of flat dictionaries, default=None,
     :ref:`flat_dict_list_label` can be used to load the delta object from a list of flat dictionaries.