Using difflib to come up with better diff results when order is

important

Author: seperman

deepdiff/diff.py

@@ -114,11 +114,13 @@ class np_type:
 only_complex_number = (complex,) + numpy_complex_numbers
 only_numbers = (int, float, complex, Decimal) + numpy_numbers
 datetimes = (datetime.datetime, datetime.date, datetime.timedelta, datetime.time)
-uuids = (uuid.UUID)
+uuids = (uuid.UUID, )
 times = (datetime.datetime, datetime.time)
 numbers = only_numbers + datetimes
 booleans = (bool, np_bool_)
 
+basic_types = strings + numbers + uuids + booleans + (type(None), )
+
 IndexedHash = namedtuple('IndexedHash', 'indexes item')
 
 current_dir = os.path.dirname(os.path.abspath(__file__))

deepdiff/helper.py

@@ -96,6 +96,9 @@ def __getitem__(self, item):
             self[item] = PrettyOrderedSet()
         return self.get(item)
 
+    def __len__(self):
+        return sum([len(i) for i in self.values() if isinstance(i, PrettyOrderedSet)])
+
 
 class TextResult(ResultDict):
     ADD_QUOTES_TO_STRINGS = True

deepdiff/model.py

@@ -13,7 +13,9 @@ def parameterize_cases(argnames, cases):
 
     """
     argnames_list = [i.strip() for i in argnames.split(',')]
-    argvalues = [tuple(i[k] for k in argnames_list) for i in cases.values()]
+    if 'test_name' not in argnames_list:
+        argnames_list.append('test_name')
+    argvalues = [tuple(test_name if (k == 'test_name') else test_dict[k] for k in argnames_list) for test_name, test_dict in cases.items()]
     ids = list(cases.keys())
     return {'argnames': argnames, 'argvalues': argvalues, 'ids': ids}
 

tests/__init__.py

@@ -1,59 +1,40 @@
 {
-  "dictionary_item_added": [
-    "root['Cars'][3]['dealers']"
-  ],
-  "dictionary_item_removed": [
-    "root['Cars'][3]['production']"
-  ],
-  "values_changed": {
-    "root['Cars'][2]['dealers'][0]['quantity']": {
-      "new_value": 50,
-      "old_value": 20
-    },
-    "root['Cars'][1]['model_numbers'][2]": {
-      "new_value": 3,
-      "old_value": 4
-    },
-    "root['Cars'][3]['model']": {
-      "new_value": "Supra",
-      "old_value": "supra"
-    }
-  },
-  "iterable_item_added": {
-    "root['Cars'][2]['dealers'][1]": {
-      "id": 200,
-      "address": "200 Fake St",
-      "quantity": 10
+    "dictionary_item_added": [
+        "root['Cars'][3]['dealers']"
+    ],
+    "dictionary_item_removed": [
+        "root['Cars'][3]['production']"
+    ],
+    "values_changed": {
+        "root['Cars'][3]['model']": {
+            "new_value": "Supra",
+            "old_value": "supra"
+        }
     },
-    "root['Cars'][1]['model_numbers'][3]": 4,
-    "root['Cars'][0]": {
-      "id": "7",
-      "make": "Toyota",
-      "model": "8Runner"
-    }
-  },
-  "iterable_item_removed": {
-    "root['Cars'][2]['dealers'][0]": {
-      "id": 103,
-      "address": "103 Fake St",
-      "quantity": 50
+    "iterable_item_added": {
+        "root['Cars'][0]": {
+            "id": "7",
+            "make": "Toyota",
+            "model": "8Runner"
+        }
     },
-    "root['Cars'][1]": {
-      "id": "2",
-      "make": "Toyota",
-      "model": "Highlander",
-      "dealers": [
-        {
-          "id": 123,
-          "address": "123 Fake St",
-          "quantity": 50
-        },
-        {
-          "id": 125,
-          "address": "125 Fake St",
-          "quantity": 20
+    "iterable_item_removed": {
+        "root['Cars'][1]": {
+            "id": "2",
+            "make": "Toyota",
+            "model": "Highlander",
+            "dealers": [
+                {
+                    "id": 123,
+                    "address": "123 Fake St",
+                    "quantity": 50
+                },
+                {
+                    "id": 125,
+                    "address": "125 Fake St",
+                    "quantity": 20
+                }
+            ]
         }
-      ]
     }
-  }
 }

tests/fixtures/compare_func_result1.json

@@ -634,21 +634,28 @@ def test_delta_dict_items_added_retain_order(self):
         'to_delta_kwargs': {},
         'expected_delta_dict': {'iterable_item_removed': {'root[9]': 'a', 'root[10]': 'b', 'root[11]': 'c'}}
     },
+    'delta_case19_value_removed_from_the_middle_of_list': {
+        't1': [0, 1, 2, 3, 4, 5, 6, 7, 8, 'a', 'b', 'c'],
+        't2': [0, 1, 2, 3, 5, 6, 7, 8, 'a', 'b', 'c'],
+        'deepdiff_kwargs': {},
+        'to_delta_kwargs': {'directed': True},
+        'expected_delta_dict': {'iterable_item_removed': {'root[4]': 4}}
+    },
 }
 
 
-DELTA_CASES_PARAMS = parameterize_cases('t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict', DELTA_CASES)
+DELTA_CASES_PARAMS = parameterize_cases('test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict', DELTA_CASES)
 
 
 class TestDelta:
 
     @pytest.mark.parametrize(**DELTA_CASES_PARAMS)
-    def test_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict):
+    def test_delta_cases(self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
-        assert expected_delta_dict == delta_dict
+        assert expected_delta_dict == delta_dict, f"test_delta_cases {test_name} failed."
         delta = Delta(diff, verify_symmetry=False, raise_errors=True)
-        assert t1 + delta == t2
+        assert t1 + delta == t2, f"test_delta_cases {test_name} failed."
 
 
 DELTA_IGNORE_ORDER_CASES = {
@@ -931,15 +938,15 @@ def test_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_de
 }
 
 DELTA_IGNORE_ORDER_CASES_PARAMS = parameterize_cases(
-    't1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta', DELTA_IGNORE_ORDER_CASES)
+    'test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta', DELTA_IGNORE_ORDER_CASES)
 
 
 class TestIgnoreOrderDelta:
 
     @pytest.mark.parametrize(**DELTA_IGNORE_ORDER_CASES_PARAMS)
     def test_ignore_order_delta_cases(
-            self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta, request):
-        test_name = request.node.callspec.id
+            self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_t1_plus_delta, request):
+        # test_name = request.node.callspec.id
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
         assert expected_delta_dict == delta_dict, f"test_ignore_order_delta_cases {test_name} failed"
@@ -1094,31 +1101,31 @@ def test_ignore_order_delta_cases(
 
 
 DELTA_NUMPY_TEST_PARAMS = parameterize_cases(
-    't1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result', DELTA_NUMPY_TEST_CASES)
+    'test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result', DELTA_NUMPY_TEST_CASES)
 
 
 class TestNumpyDelta:
 
     @pytest.mark.parametrize(**DELTA_NUMPY_TEST_PARAMS)
-    def test_numpy_delta_cases(self, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result):
+    def test_numpy_delta_cases(self, test_name, t1, t2, deepdiff_kwargs, to_delta_kwargs, expected_delta_dict, expected_result):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
         delta_dict = diff._to_delta_dict(**to_delta_kwargs)
         if expected_delta_dict:
-            assert expected_delta_dict == delta_dict
+            assert expected_delta_dict == delta_dict, f"test_numpy_delta_cases {test_name} failed."
         delta = Delta(diff, verify_symmetry=False, raise_errors=True)
         if expected_result == 't2':
             result = delta + t1
-            assert np.array_equal(result, t2)
+            assert np.array_equal(result, t2), f"test_numpy_delta_cases {test_name} failed."
         elif expected_result == 't2_via_deepdiff':
             result = delta + t1
             diff = DeepDiff(result, t2, ignore_order=True, report_repetition=True)
-            assert not diff
+            assert not diff, f"test_numpy_delta_cases {test_name} failed."
         elif expected_result is DeltaNumpyOperatorOverrideError:
             with pytest.raises(DeltaNumpyOperatorOverrideError):
-                assert t1 + delta
+                t1 + delta
         else:
             result = delta + t1
-            assert np.array_equal(result, expected_result)
+            assert np.array_equal(result, expected_result), f"test_numpy_delta_cases {test_name} failed."
 
     def test_invalid_numpy_type(self):
 
@@ -1510,11 +1517,27 @@ def test_compare_func_with_duplicates_removed(self):
         t2 = [{'id': 3, 'val': 3}, {'id': 2, 'val': 2}, {'id': 1, 'val': 3}]
         ddiff = DeepDiff(t1, t2, iterable_compare_func=self.compare_func, verbose_level=2)
         expected = {
-            'values_changed': {"root[2]['val']": {'new_value': 3, 'old_value': 1}},
-            'iterable_item_removed': {'root[2]': {'id': 1, 'val': 3}},
-            'iterable_item_moved': {
-                'root[0]': {'new_path': 'root[2]', 'value': {'id': 1, 'val': 3}},
-                'root[3]': {'new_path': 'root[0]', 'value': {'id': 3, 'val': 3}}
+            "iterable_item_removed": {
+                "root[2]": {
+                    "id": 1,
+                    "val": 3
+                }
+            },
+            "iterable_item_moved": {
+                "root[0]": {
+                    "new_path": "root[2]",
+                    "value": {
+                        "id": 1,
+                        "val": 3
+                    }
+                },
+                "root[3]": {
+                    "new_path": "root[0]",
+                    "value": {
+                        "id": 3,
+                        "val": 3
+                    }
+                }
             }
         }
         assert expected == ddiff
@@ -1527,11 +1550,27 @@ def test_compare_func_with_duplicates_added(self):
         t2 = [{'id': 1, 'val': 1}, {'id': 2, 'val': 2}, {'id': 1, 'val': 3}, {'id': 3, 'val': 3}]
         ddiff = DeepDiff(t1, t2, iterable_compare_func=self.compare_func, verbose_level=2)
         expected = {
-            'values_changed': {"root[0]['val']": {'new_value': 1, 'old_value': 3}},
-            'iterable_item_added': {'root[2]': {'id': 1, 'val': 3}},
+            'iterable_item_added': {
+                'root[2]': {
+                    'id': 1,
+                    'val': 3
+                }
+            },
             'iterable_item_moved': {
-                'root[2]': {'new_path': 'root[0]', 'value': {'id': 1, 'val': 1}},
-                'root[0]': {'new_path': 'root[3]', 'value': {'id': 3, 'val': 3}}
+                'root[0]': {
+                    'new_path': 'root[3]',
+                    'value': {
+                        'id': 3,
+                        'val': 3
+                    }
+                },
+                'root[2]': {
+                    'new_path': 'root[0]',
+                    'value': {
+                        'id': 1,
+                        'val': 1
+                    }
+                }
             }
         }
         assert expected == ddiff

tests/test_delta.py

@@ -132,12 +132,12 @@
 }
 
 
-NUMPY_CASES_PARAMS = parameterize_cases('t1, t2, deepdiff_kwargs, expected_result', NUMPY_CASES)
+NUMPY_CASES_PARAMS = parameterize_cases('test_name, t1, t2, deepdiff_kwargs, expected_result', NUMPY_CASES)
 
 
 class TestNumpy:
 
     @pytest.mark.parametrize(**NUMPY_CASES_PARAMS)
-    def test_numpy(self, t1, t2, deepdiff_kwargs, expected_result):
+    def test_numpy(self, test_name, t1, t2, deepdiff_kwargs, expected_result):
         diff = DeepDiff(t1, t2, **deepdiff_kwargs)
-        assert expected_result == diff
+        assert expected_result == diff, f"test_numpy {test_name} failed."

tests/test_diff_numpy.py

@@ -426,44 +426,14 @@ def test_list_difference3(self):
         t1 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 2, 5]}}
         t2 = {1: 1, 2: 2, 3: 3, 4: {"a": "hello", "b": [1, 3, 2, 5]}}
         ddiff = DeepDiff(t1, t2)
-        result = {
-            'values_changed': {
-                "root[4]['b'][2]": {
-                    'new_value': 2,
-                    'old_value': 5
-                },
-                "root[4]['b'][1]": {
-                    'new_value': 3,
-                    'old_value': 2
-                }
-            },
-            'iterable_item_added': {
-                "root[4]['b'][3]": 5
-            }
-        }
-        assert result == ddiff
+        expected = {'iterable_item_added': {"root[4]['b'][1]": 3}}
+        assert expected == ddiff
 
     def test_list_difference4(self):
-        # TODO: Look into Levenshtein algorithm
-        # So that the result is just insertion of "c" in this test.
         t1 = ["a", "b", "d", "e"]
         t2 = ["a", "b", "c", "d", "e"]
         ddiff = DeepDiff(t1, t2)
-        result = {
-            'values_changed': {
-                'root[2]': {
-                    'new_value': 'c',
-                    'old_value': 'd'
-                },
-                'root[3]': {
-                    'new_value': 'd',
-                    'old_value': 'e'
-                }
-            },
-            'iterable_item_added': {
-                'root[4]': 'e'
-            }
-        }
+        result = {'iterable_item_added': {'root[2]': 'c'}}
         assert result == ddiff
 
     def test_list_of_booleans(self):
@@ -1502,6 +1472,35 @@ def test_list_none_item_removed(self):
         assert result == ddiff
         assert {"root[2]"} == ddiff.affected_paths
 
+    def test_list_item_removed_from_the_middle(self):
+        t1 = [0, 1, 2, 3, 'bye', 5, 6, 7, 8, 'a', 'b', 'c']
+        t2 = [0, 1, 2, 3, 5, 6, 7, 8, 'a', 'b', 'c']
+        diff = DeepDiff(t1, t2)
+        result = {'iterable_item_removed': {'root[4]': 'bye'}}
+        assert result == diff
+        assert {"root[4]"} == diff.affected_paths
+        assert {4} == diff.affected_root_keys
+
+    def test_list_item_values_replace_in_the_middle(self):
+        t1 = [0, 1, 2, 3, 'bye', 5, 6, 7, 8, 'a', 'b', 'c']
+        t2 = [0, 1, 2, 3, 'see', 'you', 'later', 5, 6, 7, 8, 'a', 'b', 'c']
+        diff = DeepDiff(t1, t2)
+        result = {
+            'values_changed': {
+                'root[4]': {
+                    'old_value': 'bye',
+                    'new_value': 'see',
+                }
+            },
+            'iterable_item_added': {
+                'root[5]': 'you',
+                'root[6]': 'later'
+            }
+        }
+        assert result == diff
+        assert {'root[5]', 'root[6]', 'root[4]'} == diff.affected_paths
+        assert {4, 5, 6} == diff.affected_root_keys
+
     def test_non_subscriptable_iterable(self):
         def gen1():
             yield 42