[libc][hdrgen] Allow to treat hdrgen Python code as a Python module. (#128955)

Move the hdrgen code under a subdirectory to treat it as a Python
module.

This mimics the structure used by llvm/utils/lit and
llvm/utils/mlgo-utils and simplifies integration of hdrgen to the build
system which rely on Python modules. In addition to that, it clarifies
which imports are coming from the hdrgen-specific helpers (e.g. "from
type import ..." becomes "from hdrgen.type import ...".

Leave the entrypoints (top-level main.py and yaml_to_classes.py) as-is:
they can keep being referred by the CMake build system w/o any changes.

Author: vonosmas

libc/docs/dev/header_generation.rst

@@ -63,7 +63,7 @@ To add through the command line:
    examine.
 
 If you want to sort the functions alphabetically you can check out
-libc/utils/hdrgen/yaml_functions_sorted.py.
+``libc/utils/hdrgen/hdrgen/yaml_functions_sorted.py``.
 
 
 Testing
@@ -90,7 +90,7 @@ Common Errors
 
    .. code-block:: none
 
-      "/llvm-project/libc/utils/hdrgen/yaml_to_classes.py", line 67, in yaml_to_classes function_data["return_type"]
+      "/llvm-project/libc/utils/hdrgen/hdrgen/yaml_to_classes.py", line 67, in yaml_to_classes function_data["return_type"]
 
    If you receive this error or any error pertaining to
    ``function_data[function_specific_component]`` while building the headers
@@ -107,9 +107,9 @@ Common Errors
 
      CMake Error at:
      /llvm-project/libc/cmake/modules/LLVMLibCHeaderRules.cmake:86 (message):
-     'add_gen_hdr2' rule requires GEN_HDR to be specified.
+     'add_gen_hdr' rule requires GEN_HDR to be specified.
      Call Stack (most recent call first):
-     /llvm-project/libc/include/CMakeLists.txt:22 (add_gen_header2)
+     /llvm-project/libc/include/CMakeLists.txt:22 (add_gen_header)
      /llvm-project/libc/include/CMakeLists.txt:62 (add_header_macro)
 
    If you receive this error, there is a missing YAML file, h_def file, or
@@ -119,7 +119,6 @@ Common Errors
 
    | ``[header_name]``
    | ``[../libc/include/[yaml_file.yaml]``
-   | ``[header_name.h.def]``
    | ``[header_name.h]``
    | ``DEPENDS``
    |   ``{Necessary Depend Files}``
@@ -148,13 +147,13 @@ Common Errors
 
    .. code-block:: none
 
-     File "/llvm-project/libc/utils/hdrgen/header.py", line 60, in __str__ for
+     File "/llvm-project/libc/utils/hdrgen/hdrgen/header.py", line 60, in __str__ for
      function in self.functions: AttributeError: 'HeaderFile' object has no
      attribute 'functions'
 
    When running ``ninja libc`` in the build directory to generate headers you
    may receive the error above. Essentially this means that in
-   ``libc/utils/hdrgen/header.py`` there is a missing attribute named functions.
+   ``libc/utils/hdrgen/hdrgen/header.py`` there is a missing attribute named functions.
    Make sure all function components are defined within this file and there are
    no missing functions to add these components.
 

libc/utils/hdrgen/hdrgen/__init__.py

@@ -0,0 +1,3 @@
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

libc/utils/hdrgen/enumeration.py renamed to libc/utils/hdrgen/hdrgen/enumeration.py

@@ -8,7 +8,7 @@
 
 import re
 from functools import total_ordering
-from type import Type
+from hdrgen.type import Type
 
 
 # These are the keywords that appear in C type syntax but are not part of the

libc/utils/hdrgen/function.py renamed to libc/utils/hdrgen/hdrgen/function.py

@@ -6,7 +6,7 @@
 #
 # ==-------------------------------------------------------------------------==#
 
-from header import HeaderFile
+from hdrgen.header import HeaderFile
 
 
 class GpuHeaderFile(HeaderFile):

libc/utils/hdrgen/gpu_headers.py renamed to libc/utils/hdrgen/hdrgen/gpu_headers.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+#
+# ===- Generate headers for libc functions  ------------------*- python -*--==#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ==------------------------------------------------------------------------==#
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from hdrgen.header import HeaderFile
+from hdrgen.yaml_to_classes import load_yaml_file, fill_public_api
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate header files from YAML")
+    parser.add_argument(
+        "yaml_file",
+        help="Path to the YAML file containing header specification",
+        metavar="FILE",
+        type=Path,
+        nargs="+",
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Path to write generated header file",
+        type=Path,
+        required=True,
+    )
+    parser.add_argument(
+        "--json",
+        help="Write JSON instead of a header, can use multiple YAML files",
+        action="store_true",
+    )
+    parser.add_argument(
+        "--depfile",
+        help="Path to write a depfile",
+        type=Path,
+    )
+    parser.add_argument(
+        "--write-if-changed",
+        help="Write the output file only if its contents have changed",
+        action="store_true",
+        default=False,
+    )
+    parser.add_argument(
+        "-e",
+        "--entry-point",
+        help="Entry point to include; may be given many times",
+        metavar="SYMBOL",
+        action="append",
+    )
+    args = parser.parse_args()
+
+    if not args.json and len(args.yaml_file) != 1:
+        print("Only one YAML file at a time without --json", file=sys.stderr)
+        parser.print_usage(sys.stderr)
+        return 2
+
+    files_read = set()
+
+    def write_depfile():
+        if not args.depfile:
+            return
+        deps = " ".join(str(f) for f in sorted(files_read))
+        args.depfile.parent.mkdir(parents=True, exist_ok=True)
+        with open(args.depfile, "w") as depfile:
+            depfile.write(f"{args.output}: {deps}\n")
+
+    def load_yaml(path):
+        files_read.add(path)
+        return load_yaml_file(path, HeaderFile, args.entry_point)
+
+    def load_header(yaml_file):
+        merge_from_files = dict()
+
+        def merge_from(paths):
+            for path in paths:
+                # Load each file exactly once, in case of redundant merges.
+                if path in merge_from_files:
+                    continue
+                header = load_yaml(path)
+                merge_from_files[path] = header
+                merge_from(path.parent / f for f in header.merge_yaml_files)
+
+        # Load the main file first.
+        header = load_yaml(yaml_file)
+
+        # Now load all the merge_yaml_files, and transitive merge_yaml_files.
+        merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
+
+        # Merge in all those files' contents.
+        for merge_from_path, merge_from_header in merge_from_files.items():
+            if merge_from_header.name is not None:
+                print(
+                    f"{merge_from_path!s}: Merge file cannot have header field",
+                    file=sys.stderr,
+                )
+                return 2
+            header.merge(merge_from_header)
+
+        return header
+
+    if args.json:
+        contents = json.dumps(
+            [load_header(file).json_data() for file in args.yaml_file],
+            indent=2,
+        )
+    else:
+        [yaml_file] = args.yaml_file
+        header = load_header(yaml_file)
+        # The header_template path is relative to the containing YAML file.
+        template = header.template(yaml_file.parent, files_read)
+        contents = fill_public_api(header.public_api(), template)
+
+    write_depfile()
+
+    if (
+        not args.write_if_changed
+        or not args.output.exists()
+        or args.output.read_text() != contents
+    ):
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(contents)
+
+
+if __name__ == "__main__":
+    sys.exit(main())