BUG: fix nested meta path bug in json_normalize (GH 27220)

Author: another-green

doc/source/whatsnew/v0.25.1.rst

@@ -107,7 +107,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- Fix bug in :meth:`io.json.json_normalize` when nested meta paths with a nested record_path (:issue:`27220`)
 -
 -
 

pandas/io/json/_normalize.py

@@ -288,6 +288,8 @@ def _recursive_extract(data, path, seen_meta, level=0):
         if len(path) > 1:
             for obj in data:
                 for val, key in zip(meta, meta_keys):
+                    # Pull value for all the keys in case meta path and
+                    # record path are on two branches
                     seen_meta[key] = _pull_field(obj, val[0])
 
                 _recursive_extract(obj[path[0]], path[1:], seen_meta, level=level + 1)
@@ -304,15 +306,24 @@ def _recursive_extract(data, path, seen_meta, level=0):
                 # For repeating the metadata later
                 lengths.append(len(recs))
                 for val, key in zip(meta, meta_keys):
+                    # Extract the value of the key when the level
+                    # is at the meta path end
                     if level + 1 > len(val):
                         meta_val = seen_meta[key]
                         meta_vals[key].append(meta_val)
+                    # Extract the value of the key from seen_meta when
+                    # meta path and record path are on two branches
                     elif seen_meta:
                         meta_val = seen_meta[key]
-                        if isinstance(meta_val, list):
-                            meta_vals[key].append(meta_val[ind][val[level]])
-                        else:
-                            meta_vals[key].append(meta_val[val[level]])
+                        meta_vals[key] += [
+                            # The list case
+                            meta_val[ind][val[level]]
+                            if isinstance(meta_val, list)
+                            # The dict case
+                            else meta_val[val[level]]
+                        ]
+                    # At top level, seen_meta is empty, pull from data
+                    # directly and raise KeyError if not found
                     else:
                         try:
                             meta_val = _pull_field(obj, val[level:])

pandas/tests/io/json/test_normalize.py

@@ -304,13 +304,10 @@ def test_nested_meta_path_with_nested_record_path(self, state_data):
             "state": ["Florida"] * 21 + ["Ohio"] * 14,
             "shortname": ["FL"] * 21 + ["OH"] * 14,
             "info.governor": ["Rick Scott"] * 21 + ["John Kasich"] * 14,
-            "population": [12345] * 4
-            + [40000] * 7
-            + [60000] * 10
-            + [1234] * 6
-            + [1337] * 8,
         }
-        expected = DataFrame(ex_data, columns=result.columns)
+        expected = DataFrame(
+            ex_data, columns=[0, "state", "shortname", "info.governor"]
+        )
         tm.assert_frame_equal(result, expected)
 
     def test_meta_name_conflict(self):