[mypyc] Enable native integers outside tests (#14606)

I think that native integers work well enough to enable them outside
tests.

Require a more recent `mypy_extensions` that includes native int types,
including `i64` and `i32`.

Add definitions of `i64` and `i32` to the bundled stubs for
`mypy_extensions`. Fork the stubs, since the definitions only make sense
for mypy/mypyc. They require custom type checking logic. Other tools can
treat these as aliases to `int`, which was implemented here:
python/typeshed#9675

Also fix serialization of native int TypeInfos. Since we patch
`builtins.int` when we process `mypy_extensions`, the patched
information may not be serialized. We'll also need to perform similar
patching during deserialization.

Here is the performance impact to some benchmarks when using `i64`
instead of `int` types (these assume some additional tweaks that should
be ready soon):
* richards: 33% faster
* hexiom: 18% faster
* deltablue: 2.6% faster

Perhaps more importantly, native integers help with upcoming low-level
features, such as packed arrays.

Closes mypyc/mypyc#837. Remaining work can be tracked in separate
issues.

Author: JukkaL

misc/sync-typeshed.py

@@ -35,10 +35,13 @@ def check_state() -> None:
 def update_typeshed(typeshed_dir: str, commit: str | None) -> str:
     """Update contents of local typeshed copy.
 
+    We maintain our own separate mypy_extensions stubs, since it's
+    treated specially by mypy and we make assumptions about what's there.
+    We don't sync mypy_extensions stubs here -- this is done manually.
+
     Return the normalized typeshed commit hash.
     """
     assert os.path.isdir(os.path.join(typeshed_dir, "stdlib"))
-    assert os.path.isdir(os.path.join(typeshed_dir, "stubs"))
     if commit:
         subprocess.run(["git", "checkout", commit], check=True, cwd=typeshed_dir)
     commit = git_head_commit(typeshed_dir)
@@ -48,15 +51,6 @@ def update_typeshed(typeshed_dir: str, commit: str | None) -> str:
     shutil.rmtree(stdlib_dir)
     # Copy new stdlib stubs.
     shutil.copytree(os.path.join(typeshed_dir, "stdlib"), stdlib_dir)
-    # Copy mypy_extensions stubs. We don't want to use a stub package, since it's
-    # treated specially by mypy and we make assumptions about what's there.
-    stubs_dir = os.path.join("mypy", "typeshed", "stubs")
-    shutil.rmtree(stubs_dir)
-    os.makedirs(stubs_dir)
-    shutil.copytree(
-        os.path.join(typeshed_dir, "stubs", "mypy-extensions"),
-        os.path.join(stubs_dir, "mypy-extensions"),
-    )
     shutil.copy(os.path.join(typeshed_dir, "LICENSE"), os.path.join("mypy", "typeshed"))
     return commit
 

mypy-requirements.txt

@@ -1,5 +1,5 @@
 # NOTE: this needs to be kept in sync with the "requires" list in pyproject.toml
 typing_extensions>=3.10
-mypy_extensions>=0.4.3
+mypy_extensions>=1.0.0
 typed_ast>=1.4.0,<2; python_version<'3.8'
 tomli>=1.1.0; python_version<'3.11'

mypy/checkexpr.py

@@ -3354,7 +3354,10 @@ def lookup_definer(typ: Instance, attr_name: str) -> str | None:
             is_subtype(right_type, left_type)
             and isinstance(left_type, Instance)
             and isinstance(right_type, Instance)
-            and left_type.type.alt_promote is not right_type.type
+            and not (
+                left_type.type.alt_promote is not None
+                and left_type.type.alt_promote.type is right_type.type
+            )
             and lookup_definer(left_type, op_name) != lookup_definer(right_type, rev_op_name)
         ):
             # When we do "A() + B()" where B is a subclass of A, we'll actually try calling

mypy/fixup.py

@@ -87,6 +87,13 @@ def visit_type_info(self, info: TypeInfo) -> None:
                 info.declared_metaclass.accept(self.type_fixer)
             if info.metaclass_type:
                 info.metaclass_type.accept(self.type_fixer)
+            if info.alt_promote:
+                info.alt_promote.accept(self.type_fixer)
+                instance = Instance(info, [])
+                # Hack: We may also need to add a backwards promotion (from int to native int),
+                # since it might not be serialized.
+                if instance not in info.alt_promote.type._promote:
+                    info.alt_promote.type._promote.append(instance)
             if info._mro_refs:
                 info.mro = [
                     lookup_fully_qualified_typeinfo(

mypy/meet.py

@@ -167,7 +167,7 @@ def narrow_declared_type(declared: Type, narrowed: Type) -> Type:
         if (
             isinstance(narrowed, Instance)
             and narrowed.type.alt_promote
-            and narrowed.type.alt_promote is declared.type
+            and narrowed.type.alt_promote.type is declared.type
         ):
             # Special case: 'int' can't be narrowed down to a native int type such as
             # i64, since they have different runtime representations.
@@ -715,10 +715,10 @@ def visit_instance(self, t: Instance) -> ProperType:
                         return NoneType()
             else:
                 alt_promote = t.type.alt_promote
-                if alt_promote and alt_promote is self.s.type:
+                if alt_promote and alt_promote.type is self.s.type:
                     return t
                 alt_promote = self.s.type.alt_promote
-                if alt_promote and alt_promote is t.type:
+                if alt_promote and alt_promote.type is t.type:
                     return self.s
                 if is_subtype(t, self.s):
                     return t

mypy/nodes.py

@@ -2932,7 +2932,7 @@ class is generic then it will be a type constructor of higher kind.
     # This results in some unintuitive results, such as that even
     # though i64 is compatible with int and int is compatible with
     # float, i64 is *not* compatible with float.
-    alt_promote: TypeInfo | None
+    alt_promote: mypy.types.Instance | None
 
     # Representation of a Tuple[...] base class, if the class has any
     # (e.g., for named tuples). If this is not None, the actual Type
@@ -3230,6 +3230,7 @@ def serialize(self) -> JsonDict:
             "bases": [b.serialize() for b in self.bases],
             "mro": [c.fullname for c in self.mro],
             "_promote": [p.serialize() for p in self._promote],
+            "alt_promote": None if self.alt_promote is None else self.alt_promote.serialize(),
             "declared_metaclass": (
                 None if self.declared_metaclass is None else self.declared_metaclass.serialize()
             ),
@@ -3266,6 +3267,11 @@ def deserialize(cls, data: JsonDict) -> TypeInfo:
             assert isinstance(t, mypy.types.ProperType)
             _promote.append(t)
         ti._promote = _promote
+        ti.alt_promote = (
+            None
+            if data["alt_promote"] is None
+            else mypy.types.Instance.deserialize(data["alt_promote"])
+        )
         ti.declared_metaclass = (
             None
             if data["declared_metaclass"] is None

mypy/semanal_classprop.py

@@ -181,6 +181,6 @@ def add_type_promotion(
         int_sym = builtin_names["int"]
         assert isinstance(int_sym.node, TypeInfo)
         int_sym.node._promote.append(Instance(defn.info, []))
-        defn.info.alt_promote = int_sym.node
+        defn.info.alt_promote = Instance(int_sym.node, [])
     if promote_targets:
         defn.info._promote.extend(promote_targets)

mypy/subtypes.py

@@ -455,7 +455,7 @@ def visit_instance(self, left: Instance) -> bool:
                 # Special case: Low-level integer types are compatible with 'int'. We can't
                 # use promotions, since 'int' is already promoted to low-level integer types,
                 # and we can't have circular promotions.
-                if left.type.alt_promote is right.type:
+                if left.type.alt_promote and left.type.alt_promote.type is right.type:
                     return True
             rname = right.type.fullname
             # Always try a nominal check if possible,

mypy/typeshed/stubs/mypy-extensions/METADATA.toml

@@ -1,4 +1,4 @@
-version = "0.4.*"
+version = "1.0.*"
 
 [tool.stubtest]
 ignore_missing_stub = false

mypy/typeshed/stubs/mypy-extensions/mypy_extensions.pyi

@@ -1,10 +1,14 @@
+# These stubs are forked from typeshed, since we use some definitions that only make
+# sense in the context of mypy/mypyc (in particular, native int types such as i64).
+
 import abc
 import sys
 from _collections_abc import dict_items, dict_keys, dict_values
 from _typeshed import IdentityFunction, Self
 from collections.abc import Mapping
-from typing import Any, ClassVar, Generic, TypeVar, overload, type_check_only
-from typing_extensions import Never
+from typing import Any, ClassVar, Generic, SupportsInt, TypeVar, overload, type_check_only
+from typing_extensions import Never, SupportsIndex
+from _typeshed import ReadableBuffer, SupportsTrunc
 
 _T = TypeVar("_T")
 _U = TypeVar("_U")
@@ -68,3 +72,77 @@ def trait(cls: _T) -> _T: ...
 def mypyc_attr(*attrs: str, **kwattrs: object) -> IdentityFunction: ...
 
 class FlexibleAlias(Generic[_T, _U]): ...
+
+# Native int types such as i64 are magical and support implicit
+# coercions to/from int using special logic in mypy. We generally only
+# include operations here for which we have specialized primitives.
+
+class i64:
+    @overload
+    def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> i64: ...
+    @overload
+    def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> i64: ...
+
+    def __add__(self, x: i64) -> i64: ...
+    def __radd__(self, x: i64) -> i64: ...
+    def __sub__(self, x: i64) -> i64: ...
+    def __rsub__(self, x: i64) -> i64: ...
+    def __mul__(self, x: i64) -> i64: ...
+    def __rmul__(self, x: i64) -> i64: ...
+    def __floordiv__(self, x: i64) -> i64: ...
+    def __rfloordiv__(self, x: i64) -> i64: ...
+    def __mod__(self, x: i64) -> i64: ...
+    def __rmod__(self, x: i64) -> i64: ...
+    def __and__(self, x: i64) -> i64: ...
+    def __rand__(self, x: i64) -> i64: ...
+    def __or__(self, x: i64) -> i64: ...
+    def __ror__(self, x: i64) -> i64: ...
+    def __xor__(self, x: i64) -> i64: ...
+    def __rxor__(self, x: i64) -> i64: ...
+    def __lshift__(self, x: i64) -> i64: ...
+    def __rlshift__(self, x: i64) -> i64: ...
+    def __rshift__(self, x: i64) -> i64: ...
+    def __rrshift__(self, x: i64) -> i64: ...
+    def __neg__(self) -> i64: ...
+    def __invert__(self) -> i64: ...
+    def __pos__(self) -> i64: ...
+    def __lt__(self, x: i64) -> bool: ...
+    def __le__(self, x: i64) -> bool: ...
+    def __ge__(self, x: i64) -> bool: ...
+    def __gt__(self, x: i64) -> bool: ...
+    def __index__(self) -> int: ...
+
+class i32:
+    @overload
+    def __new__(cls, __x: str | ReadableBuffer | SupportsInt | SupportsIndex | SupportsTrunc = ...) -> i32: ...
+    @overload
+    def __new__(cls, __x: str | bytes | bytearray, base: SupportsIndex) -> i32: ...
+
+    def __add__(self, x: i32) -> i32: ...
+    def __radd__(self, x: i32) -> i32: ...
+    def __sub__(self, x: i32) -> i32: ...
+    def __rsub__(self, x: i32) -> i32: ...
+    def __mul__(self, x: i32) -> i32: ...
+    def __rmul__(self, x: i32) -> i32: ...
+    def __floordiv__(self, x: i32) -> i32: ...
+    def __rfloordiv__(self, x: i32) -> i32: ...
+    def __mod__(self, x: i32) -> i32: ...
+    def __rmod__(self, x: i32) -> i32: ...
+    def __and__(self, x: i32) -> i32: ...
+    def __rand__(self, x: i32) -> i32: ...
+    def __or__(self, x: i32) -> i32: ...
+    def __ror__(self, x: i32) -> i32: ...
+    def __xor__(self, x: i32) -> i32: ...
+    def __rxor__(self, x: i32) -> i32: ...
+    def __lshift__(self, x: i32) -> i32: ...
+    def __rlshift__(self, x: i32) -> i32: ...
+    def __rshift__(self, x: i32) -> i32: ...
+    def __rrshift__(self, x: i32) -> i32: ...
+    def __neg__(self) -> i32: ...
+    def __invert__(self) -> i32: ...
+    def __pos__(self) -> i32: ...
+    def __lt__(self, x: i32) -> bool: ...
+    def __le__(self, x: i32) -> bool: ...
+    def __ge__(self, x: i32) -> bool: ...
+    def __gt__(self, x: i32) -> bool: ...
+    def __index__(self) -> int: ...

mypyc/test-data/run-i32.test

@@ -1,9 +1,7 @@
 [case testI32BasicOps]
 from typing import Any, Tuple
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i32, i64
+from mypy_extensions import i32, i64
 
 from testutil import assertRaises
 

mypyc/test-data/run-i64.test

@@ -1,9 +1,7 @@
 [case testI64BasicOps]
 from typing import List, Any, Tuple, Union
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i64, i32
+from mypy_extensions import i64, i32
 
 from testutil import assertRaises
 
@@ -517,13 +515,9 @@ def test_isinstance() -> None:
 from typing import Any, Tuple
 import sys
 
-from mypy_extensions import mypyc_attr
+from mypy_extensions import mypyc_attr, i64
 from typing_extensions import Final
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i64
-
 from testutil import assertRaises
 
 def maybe_raise(n: i64, error: bool) -> i64:
@@ -911,9 +905,7 @@ from typing_extensions import Final
 
 MAGIC: Final = -113
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i64
+from mypy_extensions import i64
 
 def f(x: i64, y: i64 = 5) -> i64:
     return x + y
@@ -1211,9 +1203,7 @@ def test_magic_default() -> None:
 [case testI64UndefinedLocal]
 from typing_extensions import Final
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i64, i32
+from mypy_extensions import i64, i32
 
 from testutil import assertRaises
 
@@ -1346,9 +1336,7 @@ def test_many_locals() -> None:
 from typing import Any
 from typing_extensions import Final
 
-MYPY = False
-if MYPY:
-    from mypy_extensions import i64, trait
+from mypy_extensions import i64, trait
 
 from testutil import assertRaises
 

pyproject.toml

@@ -7,7 +7,7 @@ requires = [
     "wheel >= 0.30.0",
     # the following is from mypy-requirements.txt
     "typing_extensions>=3.10",
-    "mypy_extensions>=0.4.3",
+    "mypy_extensions>=1.0.0",
     "typed_ast>=1.4.0,<2; python_version<'3.8'",
     "tomli>=1.1.0; python_version<'3.11'",
     # the following is from build-requirements.txt

setup.py

@@ -213,7 +213,7 @@ def run(self):
     install_requires=[
         "typed_ast >= 1.4.0, < 2; python_version<'3.8'",
         "typing_extensions>=3.10",
-        "mypy_extensions >= 0.4.3",
+        "mypy_extensions >= 1.0.0",
         "tomli>=1.1.0; python_version<'3.11'",
     ],
     # Same here.

test-data/unit/check-incremental.test

@@ -6359,3 +6359,16 @@ from m import Foo
 [file m.py]
 from missing_module import Meta  # type: ignore[import]
 class Foo(metaclass=Meta): ...
+
+[case testIncrementalNativeInt]
+import a
+[file a.py]
+from mypy_extensions import i64
+x: i64 = 0
+[file a.py.2]
+from mypy_extensions import i64
+x: i64 = 0
+y: int = x
+[builtins fixtures/tuple.pyi]
+[out]
+[out2]

test-data/unit/pythoneval.test

@@ -1877,3 +1877,27 @@ for value in enum_iter(socket.SocketKind):
 _testEnumIterMetaInference.py:8: note: Revealed type is "typing.Iterator[_E`-1]"
 _testEnumIterMetaInference.py:9: note: Revealed type is "_E`-1"
 _testEnumIterMetaInference.py:13: note: Revealed type is "socket.SocketKind"
+
+[case testNativeIntTypes]
+# Spot check various native int operations with full stubs.
+from mypy_extensions import i64, i32
+
+x: i64 = 0
+y: int = x
+x = i64(0)
+y = int(x)
+i64()
+i64("12")
+i64("ab", 16)
+i64(1.2)
+float(i64(1))
+
+i64(1) + i32(2)  # Error
+reveal_type(x + y)
+reveal_type(y + x)
+a = [0]
+a[x]
+[out]
+_testNativeIntTypes.py:14: error: Unsupported operand types for + ("i64" and "i32")
+_testNativeIntTypes.py:15: note: Revealed type is "mypy_extensions.i64"
+_testNativeIntTypes.py:16: note: Revealed type is "mypy_extensions.i64"