[mypyc] Allow dots in group names (#7941)

Single-module groups will be given a dotted group name equal to the
module name by default.

A dot in a group name will put the group C extension at that
path. Generated C could will also be placed in a directory
corresponding to the full dotted name (still under the build
directory).

The direct motivation here is that bazel wants artifacts to be under
the appropriate directory, but it is cleaner in general I think.

Author: msullivan

mypyc/build.py

@@ -92,6 +92,10 @@ def get_mypy_config(mypy_options: List[str],
     else:
         mypyc_sources = all_sources
 
+    if compiler_options.separate:
+        mypyc_sources = [src for src in mypyc_sources
+                         if src.path and not src.path.endswith('__init__.py')]
+
     if not mypyc_sources:
         return mypyc_sources, all_sources, options
 
@@ -126,7 +130,7 @@ def generate_c_extension_shim(
         dir_name: the directory to place source code
         group_name: the name of the group
     """
-    cname = '%s.c' % exported_name(full_module_name)
+    cname = '%s.c' % full_module_name.replace('.', os.sep)
     cpath = os.path.join(dir_name, cname)
 
     # We load the C extension shim template from a file.
@@ -146,7 +150,7 @@ def generate_c_extension_shim(
 def group_name(modules: List[str]) -> str:
     """Produce a probably unique name for a group from a list of module names."""
     if len(modules) == 1:
-        return exported_name(modules[0])
+        return modules[0]
 
     h = hashlib.sha1()
     h.update(','.join(modules).encode())
@@ -226,7 +230,7 @@ def build_using_shared_lib(sources: List[BuildSource],
     extensions = [get_extension()(
         shared_lib_name(group_name),
         sources=cfiles,
-        include_dirs=[include_dir()],
+        include_dirs=[include_dir(), build_dir],
         depends=deps,
         extra_compile_args=extra_compile_args,
     )]

mypyc/common.py

@@ -36,6 +36,6 @@ def decorator_helper_name(func_name: str) -> str:
 def shared_lib_name(group_name: str) -> str:
     """Given a group name, return the actual name of its extension module.
 
-    (This just adds a prefix.)
+    (This just adds a suffix to the final component.)
     """
-    return 'mypyc_{}'.format(group_name)
+    return '{}__mypyc'.format(group_name)

mypyc/emit.py

@@ -15,7 +15,7 @@
     is_none_rprimitive, is_object_rprimitive, object_rprimitive, is_str_rprimitive, ClassIR,
     FuncDecl, int_rprimitive, is_optional_type, optional_value_type, all_concrete_classes
 )
-from mypyc.namegen import NameGenerator
+from mypyc.namegen import NameGenerator, exported_name
 from mypyc.sametype import is_same_type
 
 
@@ -167,7 +167,7 @@ def get_module_group_prefix(self, module_name: str) -> str:
         target_group_name = groups.get(module_name)
         if target_group_name and target_group_name != self.context.group_name:
             self.context.group_deps.add(target_group_name)
-            return 'exports_{}.'.format(target_group_name)
+            return 'exports_{}.'.format(exported_name(target_group_name))
         else:
             return ''
 

mypyc/emitmodule.py

@@ -437,6 +437,11 @@ def pointerize(decl: str, name: str) -> str:
         return decl.replace(name, '*{}'.format(name))
 
 
+def group_dir(group_name: str) -> str:
+    """Given a group name, return the relative directory path for it. """
+    return os.sep.join(group_name.split('.')[:-1])
+
+
 class GroupGenerator:
     def __init__(self,
                  literals: LiteralsMap,
@@ -477,7 +482,11 @@ def __init__(self,
 
     @property
     def group_suffix(self) -> str:
-        return '_' + self.group_name if self.group_name else ''
+        return '_' + exported_name(self.group_name) if self.group_name else ''
+
+    @property
+    def short_group_suffix(self) -> str:
+        return '_' + exported_name(self.group_name.split('.')[-1]) if self.group_name else ''
 
     def generate_c_for_modules(self) -> List[Tuple[str, str]]:
         file_contents = []
@@ -489,8 +498,8 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
         if self.compiler_options.include_runtime_files:
             base_emitter.emit_line('#include "CPy.c"')
             base_emitter.emit_line('#include "getargs.c"')
-        base_emitter.emit_line('#include "__native{}.h"'.format(self.group_suffix))
-        base_emitter.emit_line('#include "__native_internal{}.h"'.format(self.group_suffix))
+        base_emitter.emit_line('#include "__native{}.h"'.format(self.short_group_suffix))
+        base_emitter.emit_line('#include "__native_internal{}.h"'.format(self.short_group_suffix))
         emitter = base_emitter
 
         for (_, literal), identifier in self.literals.items():
@@ -503,8 +512,9 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
         for module_name, module in self.modules:
             if multi_file:
                 emitter = Emitter(self.context)
-                emitter.emit_line('#include "__native{}.h"'.format(self.group_suffix))
-                emitter.emit_line('#include "__native_internal{}.h"'.format(self.group_suffix))
+                emitter.emit_line('#include "__native{}.h"'.format(self.short_group_suffix))
+                emitter.emit_line(
+                    '#include "__native_internal{}.h"'.format(self.short_group_suffix))
 
             self.declare_module(module_name, emitter)
             self.declare_internal_globals(module_name, emitter)
@@ -544,7 +554,7 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
         declarations.emit_line('#define MYPYC_NATIVE_INTERNAL{}_H'.format(self.group_suffix))
         declarations.emit_line('#include <Python.h>')
         declarations.emit_line('#include <CPy.h>')
-        declarations.emit_line('#include "__native{}.h"'.format(self.group_suffix))
+        declarations.emit_line('#include "__native{}.h"'.format(self.short_group_suffix))
         declarations.emit_line()
         declarations.emit_line('int CPyGlobalsInit(void);')
         declarations.emit_line()
@@ -557,9 +567,13 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
                 generate_function_declaration(fn, declarations)
 
         for lib in sorted(self.context.group_deps):
+            elib = exported_name(lib)
+            short_lib = exported_name(lib.split('.')[-1])
             declarations.emit_lines(
-                '#include "__native_{}.h"'.format(lib),
-                'struct export_table_{} exports_{};'.format(lib, lib)
+                '#include <{}>'.format(
+                    os.path.join(group_dir(lib), "__native_{}.h".format(short_lib))
+                ),
+                'struct export_table_{} exports_{};'.format(elib, elib)
             )
 
         sorted_decls = self.toposort_declarations()
@@ -591,13 +605,15 @@ def generate_c_for_modules(self) -> List[Tuple[str, str]]:
         ext_declarations.emit_line('#endif')
         declarations.emit_line('#endif')
 
-        return file_contents + [('__native{}.c'.format(self.group_suffix),
-                                 ''.join(emitter.fragments)),
-                                ('__native_internal{}.h'.format(self.group_suffix),
-                                 ''.join(declarations.fragments)),
-                                ('__native{}.h'.format(self.group_suffix),
-                                 ''.join(ext_declarations.fragments)),
-                                ]
+        output_dir = group_dir(self.group_name) if self.group_name else ''
+        return file_contents + [
+            (os.path.join(output_dir, '__native{}.c'.format(self.short_group_suffix)),
+             ''.join(emitter.fragments)),
+            (os.path.join(output_dir, '__native_internal{}.h'.format(self.short_group_suffix)),
+             ''.join(declarations.fragments)),
+            (os.path.join(output_dir, '__native{}.h'.format(self.short_group_suffix)),
+             ''.join(ext_declarations.fragments)),
+        ]
 
     def generate_export_table(self, decl_emitter: Emitter, code_emitter: Emitter) -> None:
         """Generate the declaration and definition of the group's export struct.
@@ -683,12 +699,14 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None:
 
         emitter.emit_line()
         emitter.emit_lines(
-            'PyMODINIT_FUNC PyInit_{}(void)'.format(shared_lib_name(self.group_name)),
+            'PyMODINIT_FUNC PyInit_{}(void)'.format(
+                shared_lib_name(self.group_name).split('.')[-1]),
             '{',
             ('static PyModuleDef def = {{ PyModuleDef_HEAD_INIT, "{}", NULL, -1, NULL, NULL }};'
-             .format(self.group_name)),
+             .format(shared_lib_name(self.group_name))),
             'int res;',
             'PyObject *capsule;',
+            'PyObject *tmp;',
             'static PyObject *module;',
             'if (module) {',
             'Py_INCREF(module);',
@@ -733,14 +751,17 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None:
             )
 
         for group in sorted(self.context.group_deps):
+            egroup = exported_name(group)
             emitter.emit_lines(
+                'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);'.format(
+                    shared_lib_name(group)),
                 'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);'.format(
-                    group, group, shared_lib_name(group)),
-                'if (!pexports_{}) {{'.format(group),
+                    egroup, egroup, shared_lib_name(group)),
+                'if (!pexports_{}) {{'.format(egroup),
                 'goto fail;',
                 '}',
                 'memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));'.format(
-                    group=group),
+                    group=egroup),
                 '',
             )
 

mypyc/test-data/run-multimodule.test

@@ -1,19 +1,20 @@
 -- These test cases compile two modules at a time (native and other.py)
 
-[case testMultiModuleBasic]
-from other import g
+[case testMultiModulePackage]
+from p.other import g
 def f(x: int) -> int:
-    from other import h
+    from p.other import h
     return h(g(x + 1))
-[file other.py]
+[file p/__init__.py]
+[file p/other.py]
 def g(x: int) -> int:
     return x + 2
 def h(x: int) -> int:
     return x + 1
 [file driver.py]
 import native
 from native import f
-from other import g
+from p.other import g
 assert f(3) == 7
 assert g(2) == 4
 try:

mypyc/test/test_run.py

@@ -184,7 +184,7 @@ def run_case_step(self, testcase: DataDrivenTestCase, incremental_step: int) ->
             fn = os.path.relpath(fn, test_temp_dir)
 
             if os.path.basename(fn).startswith('other') and fn.endswith('.py'):
-                name = os.path.basename(fn).split('.')[0]
+                name = fn.split('.')[0].replace(os.sep, '.')
                 module_names.append(name)
                 sources.append(build.BuildSource(fn, name, None))
                 to_delete.append(fn)