[libc] Add --json mode for hdrgen

[frobtech]

This adds a feature to hdrgen to emit JSON summaries of header
files for build system integration. For now the summaries have
only the basic information about each header that is relevant for
build and testing purposes: the standards and includes lists.

[github-actions]

✅ With the latest revision this PR passed the Python code formatter.

[llvmbot]

@llvm/pr-subscribers-libc

Author: Roland McGrath (frobtech)

Changes

This adds a feature to hdrgen to emit JSON summaries of header
files for build system integration. For now the summaries have
only the basic information about each header that is relevant for
build and testing purposes: the standards and includes lists.

---

Full diff: https://github.com/llvm/llvm-project/pull/127847.diff

4 Files Affected:

• (modified) libc/utils/hdrgen/header.py (+9)
• (modified) libc/utils/hdrgen/main.py (+53-30)
• (added) libc/utils/hdrgen/tests/expected_output/test_small.json (+14)
• (modified) libc/utils/hdrgen/tests/test_integration.py (+13-2)

diff --git a/libc/utils/hdrgen/header.py b/libc/utils/hdrgen/header.py
index 42a075c4b6c89..11e0234eda1cf 100644
--- a/libc/utils/hdrgen/header.py
+++ b/libc/utils/hdrgen/header.py
@@ -233,3 +233,12 @@ def relpath(file):
         content.append("__END_C_DECLS")
 
         return "\n".join(content)
+
+    def json_data(self):
+        return {
+            "name": self.name,
+            "standards": self.standards,
+            "includes": [
+                str(file) for file in sorted({COMMON_HEADER} | self.includes())
+            ],
+        }
diff --git a/libc/utils/hdrgen/main.py b/libc/utils/hdrgen/main.py
index 27b21ce8ca44b..d5a1c25e7ce20 100755
--- a/libc/utils/hdrgen/main.py
+++ b/libc/utils/hdrgen/main.py
@@ -9,6 +9,7 @@
 # ==------------------------------------------------------------------------==#
 
 import argparse
+import json
 import sys
 from pathlib import Path
 
@@ -23,7 +24,7 @@ def main():
         help="Path to the YAML file containing header specification",
         metavar="FILE",
         type=Path,
-        nargs=1,
+        nargs="+",
     )
     parser.add_argument(
         "-o",
@@ -32,6 +33,11 @@ def main():
         type=Path,
         required=True,
     )
+    parser.add_argument(
+        "--json",
+        help="Write JSON instead of a header, can use multiple YAML files",
+        action="store_true",
+    )
     parser.add_argument(
         "--depfile",
         help="Path to write a depfile",
@@ -52,6 +58,11 @@ def main():
     )
     args = parser.parse_args()
 
+    if not args.json and len(args.yaml_file) != 1:
+        print("Only one YAML file at a time without --json", file=sys.stderr)
+        parser.print_usage(sys.stderr)
+        return 2
+
     files_read = set()
 
     def write_depfile():
@@ -66,35 +77,47 @@ def load_yaml(path):
         files_read.add(path)
         return load_yaml_file(path, HeaderFile, args.entry_point)
 
-    merge_from_files = dict()
-
-    def merge_from(paths):
-        for path in paths:
-            # Load each file exactly once, in case of redundant merges.
-            if path in merge_from_files:
-                continue
-            header = load_yaml(path)
-            merge_from_files[path] = header
-            merge_from(path.parent / f for f in header.merge_yaml_files)
-
-    # Load the main file first.
-    [yaml_file] = args.yaml_file
-    header = load_yaml(yaml_file)
-
-    # Now load all the merge_yaml_files, and any transitive merge_yaml_files.
-    merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
-
-    # Merge in all those files' contents.
-    for merge_from_path, merge_from_header in merge_from_files.items():
-        if merge_from_header.name is not None:
-            print(f"{merge_from_path!s}: Merge file cannot have header field", stderr)
-            return 2
-        header.merge(merge_from_header)
-
-    # The header_template path is relative to the containing YAML file.
-    template = header.template(yaml_file.parent, files_read)
-
-    contents = fill_public_api(header.public_api(), template)
+    def load_header(yaml_file):
+        merge_from_files = dict()
+
+        def merge_from(paths):
+            for path in paths:
+                # Load each file exactly once, in case of redundant merges.
+                if path in merge_from_files:
+                    continue
+                header = load_yaml(path)
+                merge_from_files[path] = header
+                merge_from(path.parent / f for f in header.merge_yaml_files)
+
+        # Load the main file first.
+        header = load_yaml(yaml_file)
+
+        # Now load all the merge_yaml_files, and transitive merge_yaml_files.
+        merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
+
+        # Merge in all those files' contents.
+        for merge_from_path, merge_from_header in merge_from_files.items():
+            if merge_from_header.name is not None:
+                print(
+                    f"{merge_from_path!s}: Merge file cannot have header field",
+                    file=sys.stderr,
+                )
+                return 2
+            header.merge(merge_from_header)
+
+        return header
+
+    if args.json:
+        contents = json.dumps(
+            [load_header(file).json_data() for file in args.yaml_file],
+            indent=2,
+        )
+    else:
+        [yaml_file] = args.yaml_file
+        header = load_header(yaml_file)
+        # The header_template path is relative to the containing YAML file.
+        template = header.template(yaml_file.parent, files_read)
+        contents = fill_public_api(header.public_api(), template)
 
     write_depfile()
 
diff --git a/libc/utils/hdrgen/tests/expected_output/test_small.json b/libc/utils/hdrgen/tests/expected_output/test_small.json
new file mode 100644
index 0000000000000..9cc73d013a679
--- /dev/null
+++ b/libc/utils/hdrgen/tests/expected_output/test_small.json
@@ -0,0 +1,14 @@
+[
+  {
+    "name": "test_small.h",
+    "standards": [],
+    "includes": [
+      "__llvm-libc-common.h",
+      "llvm-libc-macros/test_more-macros.h",
+      "llvm-libc-macros/test_small-macros.h",
+      "llvm-libc-types/float128.h",
+      "llvm-libc-types/type_a.h",
+      "llvm-libc-types/type_b.h"
+    ]
+  }
+]
\ No newline at end of file
diff --git a/libc/utils/hdrgen/tests/test_integration.py b/libc/utils/hdrgen/tests/test_integration.py
index 4f3d2a939520a..0fbd6c8bd9df1 100644
--- a/libc/utils/hdrgen/tests/test_integration.py
+++ b/libc/utils/hdrgen/tests/test_integration.py
@@ -12,14 +12,15 @@ def setUp(self):
         self.main_script = self.source_dir.parent / "main.py"
         self.maxDiff = 80 * 100
 
-    def run_script(self, yaml_file, output_file, entry_points=[]):
+    def run_script(self, yaml_file, output_file, entry_points=[],
+                   switches=[]):
         command = [
             "python3",
             str(self.main_script),
             str(yaml_file),
             "--output",
             str(output_file),
-        ]
+        ] + switches
 
         for entry_point in entry_points:
             command.extend(["--entry-point", entry_point])
@@ -59,6 +60,16 @@ def test_generate_subdir_header(self):
         self.run_script(yaml_file, output_file)
         self.compare_files(output_file, expected_output_file)
 
+    def test_generate_json(self):
+        yaml_file = self.source_dir / "input/test_small.yaml"
+        expected_output_file = self.source_dir / "expected_output/test_small.json"
+        output_file = self.output_dir / "test_small.json"
+
+        self.run_script(yaml_file, output_file, switches=["--json"])
+
+        self.compare_files(output_file, expected_output_file)
+
+
 
 def main():
     parser = argparse.ArgumentParser(description="TestHeaderGenIntegration arguments")

[michaelrj-google]

Overall LGTM

[michaelrj-google]

missing newline at EoF

[frobtech]

That's necessary for the way json.dump emits it. I didn't write special code to add a newline.
I can do that if you think I should.

[michaelrj-google]

ah, fine as-is then.