[libc] Make template_header optional for hdrgen

[frobtech]

This allows a YAML file to omit template_header and have no
.h.def file. A default template is generated based purely on
the information in the YAML file. This should handle most of the
cases. For now, it's exercised (aside from the hdrgen tests)
only for one of the simplest cases: <ctype.h>.

This includes making the parser notice the "standards" YAML field
at the top (header) level, not just in "functions" lists. The
standards listed for the header overall and for the individual
functions both feed into how a fully-generated header describes
itself in comments. To go with this, files using the default
generated template must stick to a new uniform set of spellings
for the "standards" lists. As more custom template files are
retired, the corresponding YAML files will need all their
standards lists normalized. For now, ctype.yaml is updated
with correct attribution for the POSIX _l extensions.

[llvmbot]

@llvm/pr-subscribers-libc

Author: Roland McGrath (frobtech)

Changes

This allows a YAML file to omit template_header and have no
.h.def file. A default template is generated based purely on
the information in the YAML file. This should handle most of the
cases. For now, it's exercised (aside from the hdrgen tests)
only for one of the simplest cases: <ctype.h>.

This includes making the parser notice the "standards" YAML field
at the top (header) level, not just in "functions" lists. The
standards listed for the header overall and for the individual
functions both feed into how a fully-generated header describes
itself in comments. To go with this, files using the default
generated template must stick to a new uniform set of spellings
for the "standards" lists. As more custom template files are
retired, the corresponding YAML files will need all their
standards lists normalized. For now, ctype.yaml is updated
with correct attribution for the POSIX _l extensions.

---

Full diff: https://github.com/llvm/llvm-project/pull/127259.diff

8 Files Affected:

• (removed) libc/include/ctype.h.def (-16)
• (modified) libc/include/ctype.yaml (+16-18)
• (modified) libc/utils/hdrgen/header.py (+89-1)
• (modified) libc/utils/hdrgen/main.py (+2-8)
• (modified) libc/utils/hdrgen/tests/expected_output/subdir/test.h (+4-3)
• (removed) libc/utils/hdrgen/tests/input/subdir/test.h.def (-16)
• (modified) libc/utils/hdrgen/tests/input/subdir/test.yaml (+7-2)
• (modified) libc/utils/hdrgen/yaml_to_classes.py (+1)

diff --git a/libc/include/ctype.h.def b/libc/include/ctype.h.def
deleted file mode 100644
index a9bb786931ead..0000000000000
--- a/libc/include/ctype.h.def
+++ /dev/null
@@ -1,16 +0,0 @@
-//===-- C standard library header ctype.h --------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_CTYPE_H
-#define LLVM_LIBC_CTYPE_H
-
-#include "__llvm-libc-common.h"
-
-%%public_api()
-
-#endif // LLVM_LIBC_CTYPE_H
diff --git a/libc/include/ctype.yaml b/libc/include/ctype.yaml
index 6238f1b889986..278d08b5fafaa 100644
--- a/libc/include/ctype.yaml
+++ b/libc/include/ctype.yaml
@@ -1,8 +1,6 @@
 header: ctype.h
-header_template: ctype.h.def
-macros: []
-types:
-  - type_name: locale_t
+standards:
+  - stdc
 enums: []
 objects: []
 functions:
@@ -104,98 +102,98 @@ functions:
       - type: int
   - name: isalnum_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isalpha_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isblank_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: iscntrl_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isdigit_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isgraph_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: islower_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isprint_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: ispunct_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isspace_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isupper_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isxdigit_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: tolower_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: toupper_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
diff --git a/libc/utils/hdrgen/header.py b/libc/utils/hdrgen/header.py
index c817c02b83b75..f923909c1ae06 100644
--- a/libc/utils/hdrgen/header.py
+++ b/libc/utils/hdrgen/header.py
@@ -6,6 +6,7 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+import re
 from functools import reduce
 from pathlib import PurePosixPath
 
@@ -32,6 +33,37 @@
     | {f"uint{size}_t": "<stdint.h>" for size in STDINT_SIZES}
 )
 
+NONIDENTIFIER = re.compile("[^a-zA-Z0-9_]+")
+
+COMMON_HEADER = PurePosixPath("__llvm-libc-common.h")
+
+# All the canonical identifiers are in lowercase for easy maintenance.
+# This maps them to the pretty descriptions to generate in header comments.
+LIBRARY_DESCRIPTIONS = {
+    "stdc": "Standard C",
+    "posix": "POSIX",
+    "bsd": "BSD",
+    "gnu": "GNU",
+    "linux": "Linux",
+}
+
+HEADER_TEMPLATE = """\
+//===-- {library} header <{header}> --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef {guard}
+#define {guard}
+
+%%public_api()
+
+#endif // {guard}
+"""
+
 
 class HeaderFile:
     def __init__(self, name):
@@ -42,6 +74,7 @@ def __init__(self, name):
         self.enumerations = []
         self.objects = []
         self.functions = []
+        self.standards = []
 
     def add_macro(self, macro):
         self.macros.append(macro)
@@ -65,6 +98,13 @@ def all_types(self):
             set(self.types),
         )
 
+    def all_standards(self):
+        # FIXME: Only functions have the "standard" field, but all the entity
+        # types should have one too.
+        return set(self.standards).union(
+            *(filter(None, (f.standards for f in self.functions)))
+        )
+
     def includes(self):
         return {
             PurePosixPath("llvm-libc-macros") / macro.header
@@ -77,6 +117,45 @@ def includes(self):
             for typ in self.all_types()
         }
 
+    def header_guard(self):
+        return "LLVM_LIBC_" + "_".join(
+            word.upper() for word in NONIDENTIFIER.split(self.name) if word
+        )
+
+    def library_description(self):
+        # If the header itself is in standard C, just call it that.
+        if "stdc" in self.standards:
+            return LIBRARY_DESCRIPTIONS["stdc"]
+        # If the header itself is in POSIX, just call it that.
+        if "posix" in self.standards:
+            return LIBRARY_DESCRIPTIONS["posix"]
+        # Otherwise, consider the standards for each symbol as well.
+        standards = self.all_standards()
+        # Otherwise, it's described by all those that apply, but ignoring
+        # "stdc" and "posix" since this is not a "stdc" or "posix" header.
+        return " / ".join(
+            sorted(
+                LIBRARY_DESCRIPTIONS[standard]
+                for standard in standards
+                if standard not in {"stdc", "posix"}
+            )
+        )
+
+    def template(self, dir, files_read):
+        if self.template_file is not None:
+            # There's a custom template file, so just read it in and record
+            # that it was read as an input file.
+            template_path = dir / self.template_file
+            files_read.add(template_path)
+            return template_path.read_text()
+
+        # Generate the default template.
+        return HEADER_TEMPLATE.format(
+            library=self.library_description(),
+            header=self.name,
+            guard=self.header_guard(),
+        )
+
     def public_api(self):
         # Python 3.12 has .relative_to(dir, walk_up=True) for this.
         path_prefix = PurePosixPath("../" * (len(PurePosixPath(self.name).parents) - 1))
@@ -84,7 +163,16 @@ def public_api(self):
         def relpath(file):
             return path_prefix / file
 
-        content = [
+        content = []
+
+        if self.template_file is None:
+            # This always goes before all the other includes, which are sorted.
+            # It's implicitly emitted here when using the default template so
+            # it can get the right relative path.  Custom template files should
+            # all have it explicitly with their right particular relative path.
+            content.append('#include "{file!s}"'.format(file=relpath(COMMON_HEADER)))
+
+        content += [
             f"#include {file}"
             for file in sorted(
                 file if isinstance(file, str) else f'"{relpath(file)!s}"'
diff --git a/libc/utils/hdrgen/main.py b/libc/utils/hdrgen/main.py
index 5dd392ab6662e..af57845582e5e 100755
--- a/libc/utils/hdrgen/main.py
+++ b/libc/utils/hdrgen/main.py
@@ -65,16 +65,10 @@ def write_depfile():
 
     header = load_yaml_file(yaml_file, HeaderFile, args.entry_point)
 
-    if not header.template_file:
-        print(f"{yaml_file}: Missing header_template", sys.stderr)
-        return 2
-
     # The header_template path is relative to the containing YAML file.
-    template_path = yaml_file.parent / header.template_file
+    template = header.template(yaml_file.parent, files_read)
 
-    files_read.add(template_path)
-    with open(template_path) as template:
-        contents = fill_public_api(header.public_api(), template.read())
+    contents = fill_public_api(header.public_api(), template)
 
     write_depfile()
 
diff --git a/libc/utils/hdrgen/tests/expected_output/subdir/test.h b/libc/utils/hdrgen/tests/expected_output/subdir/test.h
index 53d28e8430104..e968f92bf4e6d 100644
--- a/libc/utils/hdrgen/tests/expected_output/subdir/test.h
+++ b/libc/utils/hdrgen/tests/expected_output/subdir/test.h
@@ -1,16 +1,15 @@
-//===-- C standard library header subdir/test.h --------------------------===//
+//===-- BSD / GNU header <subdir/test.h> --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===--------------------------------------------------------------------===//
+//===---------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SUBDIR_TEST_H
 #define LLVM_LIBC_SUBDIR_TEST_H
 
 #include "../__llvm-libc-common.h"
-
 #include "../llvm-libc-types/type_a.h"
 #include "../llvm-libc-types/type_b.h"
 
@@ -18,6 +17,8 @@ __BEGIN_C_DECLS
 
 type_a func(type_b) __NOEXCEPT;
 
+void gnufunc(type_a) __NOEXCEPT;
+
 __END_C_DECLS
 
 #endif // LLVM_LIBC_SUBDIR_TEST_H
diff --git a/libc/utils/hdrgen/tests/input/subdir/test.h.def b/libc/utils/hdrgen/tests/input/subdir/test.h.def
deleted file mode 100644
index ded06082a51be..0000000000000
--- a/libc/utils/hdrgen/tests/input/subdir/test.h.def
+++ /dev/null
@@ -1,16 +0,0 @@
-//===-- C standard library header subdir/test.h --------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===--------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SUBDIR_TEST_H
-#define LLVM_LIBC_SUBDIR_TEST_H
-
-#include "../__llvm-libc-common.h"
-
-%%public_api()
-
-#endif // LLVM_LIBC_SUBDIR_TEST_H
diff --git a/libc/utils/hdrgen/tests/input/subdir/test.yaml b/libc/utils/hdrgen/tests/input/subdir/test.yaml
index e68af00849b0e..f325363e09cde 100644
--- a/libc/utils/hdrgen/tests/input/subdir/test.yaml
+++ b/libc/utils/hdrgen/tests/input/subdir/test.yaml
@@ -1,9 +1,14 @@
 header: subdir/test.h
-header_template: test.h.def
+standards:
+  - bsd
 functions:
   - name: func
     return_type: type_a
     arguments:
       - type: type_b
+  - name: gnufunc
+    return_type: void
+    arguments:
+      - type: type_a
     standards:
-      - stdc
+      - gnu
diff --git a/libc/utils/hdrgen/yaml_to_classes.py b/libc/utils/hdrgen/yaml_to_classes.py
index 4a3f48633dc1d..d7a349648340b 100644
--- a/libc/utils/hdrgen/yaml_to_classes.py
+++ b/libc/utils/hdrgen/yaml_to_classes.py
@@ -36,6 +36,7 @@ def yaml_to_classes(yaml_data, header_class, entry_points=None):
     header_name = yaml_data.get("header")
     header = header_class(header_name)
     header.template_file = yaml_data.get("header_template")
+    header.standards = yaml_data.get("standards", [])
 
     for macro_data in yaml_data.get("macros", []):
         header.add_macro(