[mypyc] Reorganization of headers and declarations (#7634)

This splits the headers we generate into internal headers (containing
type declarations) and external headers (containing function and
object declarations) and refactors declarations to be much more driven
by HeaderDeclaration objects. This will allow us to use
HeaderDeclaration information to populate structures for dynamic
linking in separate compilation.

Author: msullivan

mypyc/emit.py

@@ -20,13 +20,30 @@
 
 
 class HeaderDeclaration:
+    """A representation of a declaration in C.
+
+    This is used to generate declarations in header files and
+    (optionally) definitions in source files.
+
+    Attributes:
+      decl: C source code for the declaration.
+      defn: Optionally, C source code for a definition.
+      dependencies: The names of any objects that must be declared prior.
+      is_type: Whether the declaration is of a C type. (C types will be declared in
+               external header files and not marked 'extern'.)
+    """
+
     def __init__(self,
-                 dependencies: Set[str], decl: List[str], defn: Optional[List[str]],
-                 needs_extern: bool = False) -> None:
-        self.dependencies = dependencies
-        self.decl = decl
+                 decl: Union[str, List[str]],
+                 defn: Optional[List[str]] = None,
+                 *,
+                 dependencies: Optional[Set[str]] = None,
+                 is_type: bool = False
+                 ) -> None:
+        self.decl = [decl] if isinstance(decl, str) else decl
         self.defn = defn
-        self.needs_extern = needs_extern
+        self.dependencies = dependencies or set()
+        self.is_type = is_type
 
 
 class EmitterContext:
@@ -220,9 +237,9 @@ def declare_tuple_struct(self, tuple_type: RTuple) -> None:
                     dependencies.add(typ.struct_name)
 
             self.context.declarations[tuple_type.struct_name] = HeaderDeclaration(
-                dependencies,
                 self.tuple_c_declaration(tuple_type),
-                None,
+                dependencies=dependencies,
+                is_type=True,
             )
 
     def emit_inc_ref(self, dest: str, rtype: RType) -> None:

mypyc/emitclass.py

@@ -4,8 +4,8 @@
 from typing import Optional, List, Tuple, Dict, Callable, Mapping, Set
 from collections import OrderedDict
 
-from mypyc.common import NATIVE_PREFIX, PREFIX, REG_PREFIX
-from mypyc.emit import Emitter
+from mypyc.common import PREFIX, NATIVE_PREFIX, REG_PREFIX
+from mypyc.emit import Emitter, HeaderDeclaration
 from mypyc.emitfunc import native_function_header, native_getter_name, native_setter_name
 from mypyc.emitwrapper import (
     generate_dunder_wrapper, generate_hash_wrapper, generate_richcompare_wrapper,
@@ -78,16 +78,20 @@ def generate_slots(cl: ClassIR, table: SlotTable, emitter: Emitter) -> Dict[str,
     return fields
 
 
-def generate_class_type_decl(cl: ClassIR, c_emitter: Emitter, emitter: Emitter) -> None:
-    c_emitter.emit_line('PyTypeObject *{};'.format(emitter.type_struct_name(cl)))
-    emitter.emit_line('extern PyTypeObject *{};'.format(emitter.type_struct_name(cl)))
-    emitter.emit_line()
-    generate_object_struct(cl, emitter)
-    emitter.emit_line()
+def generate_class_type_decl(cl: ClassIR, c_emitter: Emitter,
+                             external_emitter: Emitter,
+                             emitter: Emitter) -> None:
+    context = c_emitter.context
+    name = emitter.type_struct_name(cl)
+    context.declarations[name] = HeaderDeclaration(
+        'PyTypeObject *{};'.format(emitter.type_struct_name(cl)))
+
+    generate_object_struct(cl, external_emitter)
     declare_native_getters_and_setters(cl, emitter)
     generate_full = not cl.is_trait and not cl.builtin_base
     if generate_full:
-        emitter.emit_line('{};'.format(native_function_header(cl.ctor, emitter)))
+        context.declarations[emitter.native_function_name(cl.ctor)] = HeaderDeclaration(
+            '{};'.format(native_function_header(cl.ctor, emitter)))
 
 
 def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None:
@@ -241,30 +245,42 @@ def setter_name(cl: ClassIR, attribute: str, names: NameGenerator) -> str:
 
 
 def generate_object_struct(cl: ClassIR, emitter: Emitter) -> None:
-    emitter.emit_lines('typedef struct {',
-                       'PyObject_HEAD',
-                       'CPyVTableItem *vtable;')
     seen_attrs = set()  # type: Set[Tuple[str, RType]]
+    lines = []  # type: List[str]
+    lines += ['typedef struct {',
+              'PyObject_HEAD',
+              'CPyVTableItem *vtable;']
     for base in reversed(cl.base_mro):
         if not base.is_trait:
             for attr, rtype in base.attributes.items():
                 if (attr, rtype) not in seen_attrs:
-                    emitter.emit_line('{}{};'.format(emitter.ctype_spaced(rtype),
-                                                     emitter.attr(attr)))
+                    lines.append('{}{};'.format(emitter.ctype_spaced(rtype),
+                                                emitter.attr(attr)))
                     seen_attrs.add((attr, rtype))
-    emitter.emit_line('}} {};'.format(cl.struct_name(emitter.names)))
+    lines.append('}} {};'.format(cl.struct_name(emitter.names)))
+    lines.append('')
+    emitter.context.declarations[cl.struct_name(emitter.names)] = HeaderDeclaration(
+        lines,
+        is_type=True
+    )
 
 
 def declare_native_getters_and_setters(cl: ClassIR,
                                        emitter: Emitter) -> None:
+    decls = emitter.context.declarations
     for attr, rtype in cl.attributes.items():
-        emitter.emit_line('{}{}({} *self);'.format(emitter.ctype_spaced(rtype),
-                                                   native_getter_name(cl, attr, emitter.names),
-                                                   cl.struct_name(emitter.names)))
-        emitter.emit_line(
+        getter_name = native_getter_name(cl, attr, emitter.names)
+        setter_name = native_setter_name(cl, attr, emitter.names)
+        decls[getter_name] = HeaderDeclaration(
+            '{}{}({} *self);'.format(emitter.ctype_spaced(rtype),
+                                     getter_name,
+                                     cl.struct_name(emitter.names))
+        )
+        decls[setter_name] = HeaderDeclaration(
             'bool {}({} *self, {}value);'.format(native_setter_name(cl, attr, emitter.names),
                                                  cl.struct_name(emitter.names),
-                                                 emitter.ctype_spaced(rtype)))
+                                                 emitter.ctype_spaced(rtype))
+        )
 
 
 def generate_native_getters_and_setters(cl: ClassIR,

mypyc/emitmodule.py

@@ -82,9 +82,11 @@ def compile_modules_to_c(result: BuildResult, module_names: List[str],
 
 
 def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None:
-    emitter.emit_line('{};'.format(native_function_header(fn.decl, emitter)))
+    emitter.context.declarations[emitter.native_function_name(fn.decl)] = HeaderDeclaration(
+        '{};'.format(native_function_header(fn.decl, emitter)))
     if fn.name != TOP_LEVEL_NAME:
-        emitter.emit_line('{};'.format(wrapper_function_header(fn, emitter.names)))
+        emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
+            '{};'.format(wrapper_function_header(fn, emitter.names)))
 
 
 def encode_as_c_string(s: str) -> Tuple[str, int]:
@@ -127,6 +129,7 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
 
         base_emitter = Emitter(self.context)
         base_emitter.emit_line('#include "__native.h"')
+        base_emitter.emit_line('#include "__native_internal.h"')
         emitter = base_emitter
 
         for (_, literal), identifier in self.literals.items():
@@ -140,6 +143,7 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
             if multi_file:
                 emitter = Emitter(self.context)
                 emitter.emit_line('#include "__native.h"')
+                emitter.emit_line('#include "__native_internal.h"')
 
             self.declare_module(module_name, emitter)
             self.declare_internal_globals(module_name, emitter)
@@ -166,6 +170,33 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
                 name = ('__native_{}.c'.format(emitter.names.private_name(module_name)))
                 file_contents.append((name, ''.join(emitter.fragments)))
 
+        # The external header file contains type declarations while
+        # the internal contains declarations of functions and objects
+        # (which are shared between shared libraries via dynamic
+        # linking tables and not accessed directly.)
+        ext_declarations = Emitter(self.context)
+        ext_declarations.emit_line('#ifndef MYPYC_NATIVE_H')
+        ext_declarations.emit_line('#define MYPYC_NATIVE_H')
+        ext_declarations.emit_line('#include <Python.h>')
+        ext_declarations.emit_line('#include <CPy.h>')
+
+        declarations = Emitter(self.context)
+        declarations.emit_line('#ifndef MYPYC_NATIVE_INTERNAL_H')
+        declarations.emit_line('#define MYPYC_NATIVE_INTERNAL_H')
+        declarations.emit_line('#include <Python.h>')
+        declarations.emit_line('#include <CPy.h>')
+        declarations.emit_line('#include "__native.h"')
+        declarations.emit_line()
+        declarations.emit_line('int CPyGlobalsInit(void);')
+        declarations.emit_line()
+
+        for module_name, module in self.modules:
+            self.declare_finals(module_name, module.final_names, declarations)
+            for cl in module.classes:
+                generate_class_type_decl(cl, emitter, ext_declarations, declarations)
+            for fn in module.functions:
+                generate_function_declaration(fn, declarations)
+
         sorted_decls = self.toposort_declarations()
 
         emitter = base_emitter
@@ -176,33 +207,25 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
 
         emitter.emit_line()
 
-        declarations = Emitter(self.context)
-        declarations.emit_line('#include <Python.h>')
-        declarations.emit_line('#include <CPy.h>')
-        declarations.emit_line()
-        declarations.emit_line('int CPyGlobalsInit(void);')
-        declarations.emit_line()
-
         for declaration in sorted_decls:
-            if declaration.needs_extern:
-                declarations.emit_lines(
+            decls = ext_declarations if declaration.is_type else declarations
+            if not declaration.is_type:
+                decls.emit_lines(
                     'extern {}'.format(declaration.decl[0]), *declaration.decl[1:])
                 emitter.emit_lines(*declaration.decl)
             else:
-                declarations.emit_lines(*declaration.decl)
-
-        for module_name, module in self.modules:
-            self.declare_finals(module_name, module.final_names, declarations)
-            for cl in module.classes:
-                generate_class_type_decl(cl, emitter, declarations)
-            for fn in module.functions:
-                generate_function_declaration(fn, declarations)
+                decls.emit_lines(*declaration.decl)
 
         if self.shared_lib_name:
             self.generate_shared_lib_init(emitter)
 
+        ext_declarations.emit_line('#endif')
+        declarations.emit_line('#endif')
+
         return file_contents + [('__native.c', ''.join(emitter.fragments)),
-                                ('__native.h', ''.join(declarations.fragments))]
+                                ('__native_internal.h', ''.join(declarations.fragments)),
+                                ('__native.h', ''.join(ext_declarations.fragments)),
+                                ]
 
     def generate_shared_lib_init(self, emitter: Emitter) -> None:
         """Generate the init function for a shared library.
@@ -438,17 +461,16 @@ def _toposort_visit(name: str) -> None:
         return result
 
     def declare_global(self, type_spaced: str, name: str,
+                       *,
                        initializer: Optional[str] = None) -> None:
         if not initializer:
             defn = None
         else:
             defn = ['{}{} = {};'.format(type_spaced, name, initializer)]
         if name not in self.context.declarations:
             self.context.declarations[name] = HeaderDeclaration(
-                set(),
-                ['{}{};'.format(type_spaced, name)],
-                defn,
-                needs_extern=True,
+                '{}{};'.format(type_spaced, name),
+                defn=defn,
             )
 
     def declare_internal_globals(self, module_name: str, emitter: Emitter) -> None: