BUG: fix nested meta path bug (GH 27220)

doc/source/whatsnew/v0.25.1.rst

@@ -107,7 +107,7 @@ MultiIndex
 I/O
 ^^^
 
--
+- Fix bug in :meth:`io.json.json_normalize` when nested meta paths with a nested record path. (:issue:`27220`)
 -
 -
 

pandas/io/json/_normalize.py

@@ -288,12 +288,13 @@ def _recursive_extract(data, path, seen_meta, level=0):
         if len(path) > 1:
             for obj in data:
                 for val, key in zip(meta, meta_keys):
-                    if level + 1 == len(val):
-                        seen_meta[key] = _pull_field(obj, val[-1])
+                    # Pull value for all the keys in case meta path and
+                    # record path are on two branches
+                    seen_meta[key] = _pull_field(obj, val[0])
 
                 _recursive_extract(obj[path[0]], path[1:], seen_meta, level=level + 1)
         else:
-            for obj in data:
+            for ind, obj in enumerate(data):
                 recs = _pull_field(obj, path[0])
                 recs = [
                     nested_to_record(r, sep=sep, max_level=max_level)
@@ -305,8 +306,24 @@ def _recursive_extract(data, path, seen_meta, level=0):
                 # For repeating the metadata later
                 lengths.append(len(recs))
                 for val, key in zip(meta, meta_keys):
+                    # Extract the value of the key when the level
+                    # is at the meta path end
                     if level + 1 > len(val):
                         meta_val = seen_meta[key]
+                        meta_vals[key].append(meta_val)
+                    # Extract the value of the key from seen_meta when
+                    # meta path and record path are on two branches
+                    elif seen_meta:
+                        meta_val = seen_meta[key]
+                        meta_vals[key] += [
+                            # The list case
+                            meta_val[ind][val[level]]
+                            if isinstance(meta_val, list)
+                            # The dict case
+                            else meta_val[val[level]]
+                        ]
+                    # At top level, seen_meta is empty, pull from data
+                    # directly and raise KeyError if not found
                     else:
                         try:
                             meta_val = _pull_field(obj, val[level:])
@@ -319,7 +336,8 @@ def _recursive_extract(data, path, seen_meta, level=0):
                                     "errors='ignore' as key "
                                     "{err} is not always present".format(err=e)
                                 )
-                    meta_vals[key].append(meta_val)
+                        meta_vals[key].append(meta_val)
+
                 records.extend(recs)
 
     _recursive_extract(data, record_path, {}, level=0)

pandas/tests/io/json/test_normalize.py

@@ -1,4 +1,5 @@
 import json
+import sys
 
 import numpy as np
 import pytest
@@ -287,6 +288,30 @@ def test_shallow_nested(self):
         expected = DataFrame(ex_data, columns=result.columns)
         tm.assert_frame_equal(result, expected)
 
+    @pytest.mark.skipif(sys.version_info < (3, 6), reason="drop support for 3.5 soon")
+    def test_nested_meta_path_with_nested_record_path(self, state_data):
+        # GH 27220
+        result = json_normalize(
+            state_data,
+            ["counties", "name"],
+            ["state", "shortname", ["info", "governor"]],
+            errors="ignore",
+        )
+        ex_data = {
+            0: [
+                i
+                for word in ["Dade", "Broward", "Palm Beach", "Summit", "Cuyahoga"]
+                for i in word
+            ],
+            "state": ["Florida"] * 21 + ["Ohio"] * 14,
+            "shortname": ["FL"] * 21 + ["OH"] * 14,
+            "info.governor": ["Rick Scott"] * 21 + ["John Kasich"] * 14,
+        }
+        expected = DataFrame(
+            ex_data, columns=[0, "state", "shortname", "info.governor"]
+        )
+        tm.assert_frame_equal(result, expected)
+
     def test_meta_name_conflict(self):
         data = [
             {