[mypyc] Make namegen not need to use suffixes to disambiguate (#7676)

Accomplish this by using a mapping that doesn't collide on valid
python identifiers. We map '.' to '___' and the (rare and unlikely,
but possible) '___' to '_3___'. (This collides '___' with '.3',
which is fine.)

This makes naming not reliant on the order of processing things
which is important for separate compilation, since it will turn out
that most of our "internal" names actually do need to be exported
(but we would still like to shorten the module name part).

I also like the output better, even though it is a little more
verbose, since it makes it obvious what is a module separator and what
is an underscore.

Author: msullivan

mypyc/namegen.py

@@ -25,13 +25,10 @@ class NameGenerator:
       respectively. If the modules are 'bar.foo' and 'baz.foo', the
       prefixes will be 'bar_foo_' and 'baz_foo_'.
 
-    * Replace '.' in the Python name with '_' in the C name. This can
-      obviously generate conflicts at C name level. If multiple Python
-      names would map to the same C name using the basic algorithm,
-      add suffixes _2, _3, etc. to make the C names unique.
-
-    * Keep a dictionary of mappings so that we can translate each name
-      consistently within a build.
+    * Replace '.' in the Python name with '___' in the C name. (And
+      replace the unlikely but possible '___' with '___3_'. This
+      collides '___' with '.3_', but this is OK because names
+      may not start with a digit.)
 
     The generated should be internal to a build and thus the mapping is
     arbitrary. Just generating names '1', '2', ... would be correct,
@@ -66,31 +63,19 @@ def private_name(self, module: str, partial_name: Optional[str] = None) -> str:
         """
         # TODO: Support unicode
         if partial_name is None:
-            return self.module_map[module].rstrip('_')
+            return exported_name(self.module_map[module].rstrip('.'))
         if (module, partial_name) in self.translations:
             return self.translations[module, partial_name]
         if module in self.module_map:
             module_prefix = self.module_map[module]
         elif module:
-            module_prefix = module.replace('.', '_') + '_'
+            module_prefix = module + '.'
         else:
             module_prefix = ''
-        candidate = '{}{}'.format(module_prefix, partial_name.replace('.', '_'))
-        actual = self.make_unique(candidate)
+        actual = exported_name('{}{}'.format(module_prefix, partial_name))
         self.translations[module, partial_name] = actual
-        self.used_names.add(actual)
         return actual
 
-    def make_unique(self, name: str) -> str:
-        if name not in self.used_names:
-            return name
-        i = 2
-        while True:
-            candidate = '{}_{}'.format(name, i)
-            if candidate not in self.used_names:
-                return candidate
-            i += 1
-
 
 def exported_name(fullname: str) -> str:
     """Return a C name usable for an exported definition.
@@ -100,8 +85,7 @@ def exported_name(fullname: str) -> str:
     builds.
     """
     # TODO: Support unicode
-    # TODO: Ensure that there are no conflicts?
-    return fullname.replace('.', '___')
+    return fullname.replace('___', '___3_').replace('.', '___')
 
 
 def make_module_translation_map(names: List[str]) -> Dict[str, str]:
@@ -124,5 +108,5 @@ def candidate_suffixes(fullname: str) -> List[str]:
     components = fullname.split('.')
     result = ['']
     for i in range(len(components)):
-        result.append('_'.join(components[-i - 1:]) + '_')
+        result.append('.'.join(components[-i - 1:]) + '.')
     return result

mypyc/ops.py

@@ -1416,11 +1416,12 @@ def __init__(self,
             else:
                 self.bound_sig = FuncSignature(sig.args[1:], sig.ret_type)
 
+    @property
+    def shortname(self) -> str:
+        return self.class_name + '.' + self.name if self.class_name else self.name
+
     def cname(self, names: NameGenerator) -> str:
-        name = self.name
-        if self.class_name:
-            name += '_' + self.class_name
-        return names.private_name(self.module_name, name)
+        return names.private_name(self.module_name, self.shortname)
 
 
 class FuncIR:

mypyc/test-data/run.test

@@ -4345,6 +4345,9 @@ class C:
     def foo(self) -> None:
         def bar(self) -> None:
             pass
+
+def C___foo() -> None: pass
+
 class D:
     def foo(self) -> None:
         def bar(self) -> None:

mypyc/test/test_namegen.py

@@ -7,33 +7,34 @@
 
 class TestNameGen(unittest.TestCase):
     def test_candidate_suffixes(self) -> None:
-        assert candidate_suffixes('foo') == ['', 'foo_']
-        assert candidate_suffixes('foo.bar') == ['', 'bar_', 'foo_bar_']
+        assert candidate_suffixes('foo') == ['', 'foo.']
+        assert candidate_suffixes('foo.bar') == ['', 'bar.', 'foo.bar.']
 
     def test_exported_name(self) -> None:
         assert exported_name('foo') == 'foo'
         assert exported_name('foo.bar') == 'foo___bar'
 
     def test_make_module_translation_map(self) -> None:
         assert make_module_translation_map(
-            ['foo', 'bar']) == {'foo': 'foo_', 'bar': 'bar_'}
+            ['foo', 'bar']) == {'foo': 'foo.', 'bar': 'bar.'}
         assert make_module_translation_map(
-            ['foo.bar', 'foo.baz']) == {'foo.bar': 'bar_', 'foo.baz': 'baz_'}
+            ['foo.bar', 'foo.baz']) == {'foo.bar': 'bar.', 'foo.baz': 'baz.'}
         assert make_module_translation_map(
-            ['zar', 'foo.bar', 'foo.baz']) == {'foo.bar': 'bar_',
-                                               'foo.baz': 'baz_',
-                                               'zar': 'zar_'}
+            ['zar', 'foo.bar', 'foo.baz']) == {'foo.bar': 'bar.',
+                                               'foo.baz': 'baz.',
+                                               'zar': 'zar.'}
         assert make_module_translation_map(
-            ['foo.bar', 'fu.bar', 'foo.baz']) == {'foo.bar': 'foo_bar_',
-                                                  'fu.bar': 'fu_bar_',
-                                                  'foo.baz': 'baz_'}
+            ['foo.bar', 'fu.bar', 'foo.baz']) == {'foo.bar': 'foo.bar.',
+                                                  'fu.bar': 'fu.bar.',
+                                                  'foo.baz': 'baz.'}
 
     def test_name_generator(self) -> None:
         g = NameGenerator(['foo', 'foo.zar'])
-        assert g.private_name('foo', 'f') == 'foo_f'
-        assert g.private_name('foo', 'C.x.y') == 'foo_C_x_y'
-        assert g.private_name('foo', 'C.x.y') == 'foo_C_x_y'
-        assert g.private_name('foo.zar', 'C.x.y') == 'zar_C_x_y'
-        assert g.private_name('foo', 'C.x_y') == 'foo_C_x_y_2'
-        assert g.private_name('foo', 'C_x_y') == 'foo_C_x_y_3'
-        assert g.private_name('foo', 'C_x_y') == 'foo_C_x_y_3'
+        assert g.private_name('foo', 'f') == 'foo___f'
+        assert g.private_name('foo', 'C.x.y') == 'foo___C___x___y'
+        assert g.private_name('foo', 'C.x.y') == 'foo___C___x___y'
+        assert g.private_name('foo.zar', 'C.x.y') == 'zar___C___x___y'
+        assert g.private_name('foo', 'C.x_y') == 'foo___C___x_y'
+        assert g.private_name('foo', 'C_x_y') == 'foo___C_x_y'
+        assert g.private_name('foo', 'C_x_y') == 'foo___C_x_y'
+        assert g.private_name('foo', '___') == 'foo______3_'