[mypyc] Foundational work to help support native ints (#12884)

Some IR and codegen changes that help with native int support.

This was split off from a branch with a working implementation of
native ints to make reviewing easier. Some tests and primitives 
are missing here and I will include them in follow-up PRs.

Summary of major changes below.

1) Allow ambiguous error returns from functions. Since all values
of `i64` values are valid return values, none can be reserved for 
errors. The approach here is to have the error value overlap a 
valid value, and use `PyErr_Occurred()` as a secondary check 
to make sure it actually was an error.

2) Add `Extend` op which extends a value to a larger integer
type with either zero or sign extension.

3) Improve subtype checking with native int types.

4) Fill in other minor gaps in IR and codegen support for native 
ints.

Work on mypyc/mypyc#837.

Author: JukkaL

mypyc/analysis/dataflow.py

@@ -9,7 +9,7 @@
     BasicBlock, OpVisitor, Assign, AssignMulti, Integer, LoadErrorValue, RegisterOp, Goto, Branch,
     Return, Call, Box, Unbox, Cast, Op, Unreachable, TupleGet, TupleSet, GetAttr, SetAttr,
     LoadLiteral, LoadStatic, InitStatic, MethodCall, RaiseStandardError, CallC, LoadGlobal,
-    Truncate, IntOp, LoadMem, GetElementPtr, LoadAddress, ComparisonOp, SetMem, KeepAlive
+    Truncate, IntOp, LoadMem, GetElementPtr, LoadAddress, ComparisonOp, SetMem, KeepAlive, Extend
 )
 from mypyc.ir.func_ir import all_values
 
@@ -199,6 +199,9 @@ def visit_call_c(self, op: CallC) -> GenAndKill[T]:
     def visit_truncate(self, op: Truncate) -> GenAndKill[T]:
         return self.visit_register_op(op)
 
+    def visit_extend(self, op: Extend) -> GenAndKill[T]:
+        return self.visit_register_op(op)
+
     def visit_load_global(self, op: LoadGlobal) -> GenAndKill[T]:
         return self.visit_register_op(op)
 

mypyc/analysis/ircheck.py

@@ -7,7 +7,7 @@
     InitStatic, TupleGet, TupleSet, IncRef, DecRef, Call, MethodCall, Cast,
     Box, Unbox, RaiseStandardError, CallC, Truncate, LoadGlobal, IntOp, ComparisonOp,
     LoadMem, SetMem, GetElementPtr, LoadAddress, KeepAlive, Register, Integer,
-    BaseAssign
+    BaseAssign, Extend
 )
 from mypyc.ir.rtypes import (
     RType, RPrimitive, RUnion, is_object_rprimitive, RInstance, RArray,
@@ -326,6 +326,9 @@ def visit_call_c(self, op: CallC) -> None:
     def visit_truncate(self, op: Truncate) -> None:
         pass
 
+    def visit_extend(self, op: Extend) -> None:
+        pass
+
     def visit_load_global(self, op: LoadGlobal) -> None:
         pass
 

mypyc/analysis/selfleaks.py

@@ -4,7 +4,8 @@
     OpVisitor, Register, Goto, Assign, AssignMulti, SetMem, Call, MethodCall, LoadErrorValue,
     LoadLiteral, GetAttr, SetAttr, LoadStatic, InitStatic, TupleGet, TupleSet, Box, Unbox,
     Cast, RaiseStandardError, CallC, Truncate, LoadGlobal, IntOp, ComparisonOp, LoadMem,
-    GetElementPtr, LoadAddress, KeepAlive, Branch, Return, Unreachable, RegisterOp, BasicBlock
+    GetElementPtr, LoadAddress, KeepAlive, Branch, Return, Unreachable, RegisterOp, BasicBlock,
+    Extend
 )
 from mypyc.ir.rtypes import RInstance
 from mypyc.analysis.dataflow import MAYBE_ANALYSIS, run_analysis, AnalysisResult, CFG
@@ -115,6 +116,9 @@ def visit_call_c(self, op: CallC) -> GenAndKill:
     def visit_truncate(self, op: Truncate) -> GenAndKill:
         return CLEAN
 
+    def visit_extend(self, op: Extend) -> GenAndKill:
+        return CLEAN
+
     def visit_load_global(self, op: LoadGlobal) -> GenAndKill:
         return CLEAN
 

mypyc/codegen/emit.py

@@ -17,7 +17,8 @@
     is_list_rprimitive, is_dict_rprimitive, is_set_rprimitive, is_tuple_rprimitive,
     is_none_rprimitive, is_object_rprimitive, object_rprimitive, is_str_rprimitive,
     int_rprimitive, is_optional_type, optional_value_type, is_int32_rprimitive,
-    is_int64_rprimitive, is_bit_rprimitive, is_range_rprimitive, is_bytes_rprimitive
+    is_int64_rprimitive, is_bit_rprimitive, is_range_rprimitive, is_bytes_rprimitive,
+    is_fixed_width_rtype
 )
 from mypyc.ir.func_ir import FuncDecl
 from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -479,9 +480,16 @@ def emit_cast(self,
                 return
 
         # TODO: Verify refcount handling.
-        if (is_list_rprimitive(typ) or is_dict_rprimitive(typ) or is_set_rprimitive(typ)
-                or is_str_rprimitive(typ) or is_range_rprimitive(typ) or is_float_rprimitive(typ)
-                or is_int_rprimitive(typ) or is_bool_rprimitive(typ) or is_bit_rprimitive(typ)):
+        if (is_list_rprimitive(typ)
+                or is_dict_rprimitive(typ)
+                or is_set_rprimitive(typ)
+                or is_str_rprimitive(typ)
+                or is_range_rprimitive(typ)
+                or is_float_rprimitive(typ)
+                or is_int_rprimitive(typ)
+                or is_bool_rprimitive(typ)
+                or is_bit_rprimitive(typ)
+                or is_fixed_width_rtype(typ)):
             if declare_dest:
                 self.emit_line(f'PyObject *{dest};')
             if is_list_rprimitive(typ):
@@ -496,12 +504,13 @@ def emit_cast(self,
                 prefix = 'PyRange'
             elif is_float_rprimitive(typ):
                 prefix = 'CPyFloat'
-            elif is_int_rprimitive(typ):
+            elif is_int_rprimitive(typ) or is_fixed_width_rtype(typ):
+                # TODO: Range check for fixed-width types?
                 prefix = 'PyLong'
             elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ):
                 prefix = 'PyBool'
             else:
-                assert False, 'unexpected primitive type'
+                assert False, f'unexpected primitive type: {typ}'
             check = '({}_Check({}))'
             if likely:
                 check = f'(likely{check})'
@@ -765,6 +774,20 @@ def emit_unbox(self,
             self.emit_line(failure)
             self.emit_line('} else')
             self.emit_line(f'    {dest} = 1;')
+        elif is_int64_rprimitive(typ):
+            # Whether we are borrowing or not makes no difference.
+            if declare_dest:
+                self.emit_line(f'int64_t {dest};')
+            self.emit_line(f'{dest} = CPyLong_AsInt64({src});')
+            # TODO: Handle 'optional'
+            # TODO: Handle 'failure'
+        elif is_int32_rprimitive(typ):
+            # Whether we are borrowing or not makes no difference.
+            if declare_dest:
+                self.emit_line('int32_t {};'.format(dest))
+            self.emit_line('{} = CPyLong_AsInt32({});'.format(dest, src))
+            # TODO: Handle 'optional'
+            # TODO: Handle 'failure'
         elif isinstance(typ, RTuple):
             self.declare_tuple_struct(typ)
             if declare_dest:

mypyc/codegen/emitfunc.py

@@ -12,7 +12,8 @@
     LoadStatic, InitStatic, TupleGet, TupleSet, Call, IncRef, DecRef, Box, Cast, Unbox,
     BasicBlock, Value, MethodCall, Unreachable, NAMESPACE_STATIC, NAMESPACE_TYPE, NAMESPACE_MODULE,
     RaiseStandardError, CallC, LoadGlobal, Truncate, IntOp, LoadMem, GetElementPtr,
-    LoadAddress, ComparisonOp, SetMem, Register, LoadLiteral, AssignMulti, KeepAlive, ERR_FALSE
+    LoadAddress, ComparisonOp, SetMem, Register, LoadLiteral, AssignMulti, KeepAlive, Extend,
+    ERR_FALSE
 )
 from mypyc.ir.rtypes import (
     RType, RTuple, RArray, is_tagged, is_int32_rprimitive, is_int64_rprimitive, RStruct,
@@ -210,6 +211,10 @@ def visit_assign(self, op: Assign) -> None:
         # clang whines about self assignment (which we might generate
         # for some casts), so don't emit it.
         if dest != src:
+            # We sometimes assign from an integer prepresentation of a pointer
+            # to a real pointer, and C compilers insist on a cast.
+            if op.src.type.is_unboxed and not op.dest.type.is_unboxed:
+                src = f'(void *){src}'
             self.emit_line(f'{dest} = {src};')
 
     def visit_assign_multi(self, op: AssignMulti) -> None:
@@ -538,6 +543,15 @@ def visit_truncate(self, op: Truncate) -> None:
         # for C backend the generated code are straight assignments
         self.emit_line(f"{dest} = {value};")
 
+    def visit_extend(self, op: Extend) -> None:
+        dest = self.reg(op)
+        value = self.reg(op.src)
+        if op.signed:
+            src_cast = self.emit_signed_int_cast(op.src.type)
+        else:
+            src_cast = self.emit_unsigned_int_cast(op.src.type)
+        self.emit_line("{} = {}{};".format(dest, src_cast, value))
+
     def visit_load_global(self, op: LoadGlobal) -> None:
         dest = self.reg(op)
         ann = ''
@@ -551,6 +565,10 @@ def visit_int_op(self, op: IntOp) -> None:
         dest = self.reg(op)
         lhs = self.reg(op.lhs)
         rhs = self.reg(op.rhs)
+        if op.op == IntOp.RIGHT_SHIFT:
+            # Signed right shift
+            lhs = self.emit_signed_int_cast(op.lhs.type) + lhs
+            rhs = self.emit_signed_int_cast(op.rhs.type) + rhs
         self.emit_line(f'{dest} = {lhs} {op.op_str[op.op]} {rhs};')
 
     def visit_comparison_op(self, op: ComparisonOp) -> None:
@@ -624,7 +642,10 @@ def reg(self, reg: Value) -> str:
             s = str(val)
             if val >= (1 << 31):
                 # Avoid overflowing signed 32-bit int
-                s += 'ULL'
+                if val >= (1 << 63):
+                    s += 'ULL'
+                else:
+                    s += 'LL'
             elif val == -(1 << 63):
                 # Avoid overflowing C integer literal
                 s = '(-9223372036854775807LL - 1)'

mypyc/ir/ops.py

@@ -21,7 +21,7 @@
     RType, RInstance, RTuple, RArray, RVoid, is_bool_rprimitive, is_int_rprimitive,
     is_short_int_rprimitive, is_none_rprimitive, object_rprimitive, bool_rprimitive,
     short_int_rprimitive, int_rprimitive, void_rtype, pointer_rprimitive, is_pointer_rprimitive,
-    bit_rprimitive, is_bit_rprimitive
+    bit_rprimitive, is_bit_rprimitive, is_fixed_width_rtype
 )
 
 if TYPE_CHECKING:
@@ -90,6 +90,9 @@ def terminator(self) -> 'ControlOp':
 ERR_FALSE: Final = 2
 # Always fails
 ERR_ALWAYS: Final = 3
+# Like ERR_MAGIC, but the magic return overlaps with a possible return value, and
+# an extra PyErr_Occurred() check is also required
+ERR_MAGIC_OVERLAPPING: Final = 4
 
 # Hack: using this line number for an op will suppress it in tracebacks
 NO_TRACEBACK_LINE_NO = -10000
@@ -489,14 +492,17 @@ class Call(RegisterOp):
     The call target can be a module-level function or a class.
     """
 
-    error_kind = ERR_MAGIC
-
     def __init__(self, fn: 'FuncDecl', args: Sequence[Value], line: int) -> None:
-        super().__init__(line)
         self.fn = fn
         self.args = list(args)
         assert len(self.args) == len(fn.sig.args)
         self.type = fn.sig.ret_type
+        ret_type = fn.sig.ret_type
+        if not ret_type.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return list(self.args[:])
@@ -508,14 +514,11 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
 class MethodCall(RegisterOp):
     """Native method call obj.method(arg, ...)"""
 
-    error_kind = ERR_MAGIC
-
     def __init__(self,
                  obj: Value,
                  method: str,
                  args: List[Value],
                  line: int = -1) -> None:
-        super().__init__(line)
         self.obj = obj
         self.method = method
         self.args = args
@@ -524,7 +527,13 @@ def __init__(self,
         method_ir = self.receiver_type.class_ir.method_sig(method)
         assert method_ir is not None, "{} doesn't have method {}".format(
             self.receiver_type.name, method)
-        self.type = method_ir.ret_type
+        ret_type = method_ir.ret_type
+        self.type = ret_type
+        if not ret_type.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return self.args[:] + [self.obj]
@@ -605,8 +614,11 @@ def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) ->
         self.attr = attr
         assert isinstance(obj.type, RInstance), 'Attribute access not supported: %s' % obj.type
         self.class_type = obj.type
-        self.type = obj.type.attr_type(attr)
-        self.is_borrowed = borrow
+        attr_type = obj.type.attr_type(attr)
+        self.type = attr_type
+        if is_fixed_width_rtype(attr_type):
+            self.error_kind = ERR_NEVER
+        self.is_borrowed = borrow and attr_type.is_refcounted
 
     def sources(self) -> List[Value]:
         return [self.obj]
@@ -829,12 +841,14 @@ class Unbox(RegisterOp):
     representation. Only supported for types with an unboxed representation.
     """
 
-    error_kind = ERR_MAGIC
-
     def __init__(self, src: Value, typ: RType, line: int) -> None:
-        super().__init__(line)
         self.src = src
         self.type = typ
+        if not typ.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return [self.src]
@@ -924,22 +938,20 @@ class Truncate(RegisterOp):
 
     Truncate a value from type with more bits to type with less bits.
 
-    Both src_type and dst_type should be non-reference counted integer
-    types or bool. Note that int_rprimitive is reference counted so
-    it should never be used here.
+    dst_type and src_type can be native integer types, bools or tagged
+    integers. Tagged integers should have the tag bit unset.
     """
 
     error_kind = ERR_NEVER
 
     def __init__(self,
                  src: Value,
-                 src_type: RType,
                  dst_type: RType,
                  line: int = -1) -> None:
         super().__init__(line)
         self.src = src
-        self.src_type = src_type
         self.type = dst_type
+        self.src_type = src.type
 
     def sources(self) -> List[Value]:
         return [self.src]
@@ -951,6 +963,41 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
         return visitor.visit_truncate(self)
 
 
+class Extend(RegisterOp):
+    """result = extend src from src_type to dst_type
+
+    Extend a value from a type with fewer bits to a type with more bits.
+
+    dst_type and src_type can be native integer types, bools or tagged
+    integers. Tagged integers should have the tag bit unset.
+
+    If 'signed' is true, perform sign extension. Otherwise, the result will be
+    zero extended.
+    """
+
+    error_kind = ERR_NEVER
+
+    def __init__(self,
+                 src: Value,
+                 dst_type: RType,
+                 signed: bool,
+                 line: int = -1) -> None:
+        super().__init__(line)
+        self.src = src
+        self.type = dst_type
+        self.src_type = src.type
+        self.signed = signed
+
+    def sources(self) -> List[Value]:
+        return [self.src]
+
+    def stolen(self) -> List[Value]:
+        return []
+
+    def accept(self, visitor: 'OpVisitor[T]') -> T:
+        return visitor.visit_extend(self)
+
+
 class LoadGlobal(RegisterOp):
     """Load a low-level global variable/pointer.
 
@@ -1035,6 +1082,11 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
         return visitor.visit_int_op(self)
 
 
+# We can't have this in the IntOp class body, because of
+# https://github.com/mypyc/mypyc/issues/932.
+int_op_to_id: Final = {op: op_id for op_id, op in IntOp.op_str.items()}
+
+
 class ComparisonOp(RegisterOp):
     """Low-level comparison op for integers and pointers.
 
@@ -1076,6 +1128,15 @@ class ComparisonOp(RegisterOp):
         UGE: '>=',
     }
 
+    signed_ops: Final = {
+        '==': EQ,
+        '!=': NEQ,
+        '<': SLT,
+        '>': SGT,
+        '<=': SLE,
+        '>=': SGE,
+    }
+
     def __init__(self, lhs: Value, rhs: Value, op: int, line: int = -1) -> None:
         super().__init__(line)
         self.type = bit_rprimitive
@@ -1327,6 +1388,10 @@ def visit_call_c(self, op: CallC) -> T:
     def visit_truncate(self, op: Truncate) -> T:
         raise NotImplementedError
 
+    @abstractmethod
+    def visit_extend(self, op: Extend) -> T:
+        raise NotImplementedError
+
     @abstractmethod
     def visit_load_global(self, op: LoadGlobal) -> T:
         raise NotImplementedError