[mypyc] Foundational work to help support native ints

mypyc/analysis/dataflow.py

@@ -9,7 +9,7 @@
     BasicBlock, OpVisitor, Assign, AssignMulti, Integer, LoadErrorValue, RegisterOp, Goto, Branch,
     Return, Call, Box, Unbox, Cast, Op, Unreachable, TupleGet, TupleSet, GetAttr, SetAttr,
     LoadLiteral, LoadStatic, InitStatic, MethodCall, RaiseStandardError, CallC, LoadGlobal,
-    Truncate, IntOp, LoadMem, GetElementPtr, LoadAddress, ComparisonOp, SetMem, KeepAlive
+    Truncate, IntOp, LoadMem, GetElementPtr, LoadAddress, ComparisonOp, SetMem, KeepAlive, Extend
 )
 from mypyc.ir.func_ir import all_values
 
@@ -199,6 +199,9 @@ def visit_call_c(self, op: CallC) -> GenAndKill[T]:
     def visit_truncate(self, op: Truncate) -> GenAndKill[T]:
         return self.visit_register_op(op)
 
+    def visit_extend(self, op: Extend) -> GenAndKill[T]:
+        return self.visit_register_op(op)
+
     def visit_load_global(self, op: LoadGlobal) -> GenAndKill[T]:
         return self.visit_register_op(op)
 

mypyc/analysis/ircheck.py

@@ -7,7 +7,7 @@
     InitStatic, TupleGet, TupleSet, IncRef, DecRef, Call, MethodCall, Cast,
     Box, Unbox, RaiseStandardError, CallC, Truncate, LoadGlobal, IntOp, ComparisonOp,
     LoadMem, SetMem, GetElementPtr, LoadAddress, KeepAlive, Register, Integer,
-    BaseAssign
+    BaseAssign, Extend
 )
 from mypyc.ir.rtypes import (
     RType, RPrimitive, RUnion, is_object_rprimitive, RInstance, RArray,
@@ -326,6 +326,9 @@ def visit_call_c(self, op: CallC) -> None:
     def visit_truncate(self, op: Truncate) -> None:
         pass
 
+    def visit_extend(self, op: Extend) -> None:
+        pass
+
     def visit_load_global(self, op: LoadGlobal) -> None:
         pass
 

mypyc/analysis/selfleaks.py

@@ -4,7 +4,8 @@
     OpVisitor, Register, Goto, Assign, AssignMulti, SetMem, Call, MethodCall, LoadErrorValue,
     LoadLiteral, GetAttr, SetAttr, LoadStatic, InitStatic, TupleGet, TupleSet, Box, Unbox,
     Cast, RaiseStandardError, CallC, Truncate, LoadGlobal, IntOp, ComparisonOp, LoadMem,
-    GetElementPtr, LoadAddress, KeepAlive, Branch, Return, Unreachable, RegisterOp, BasicBlock
+    GetElementPtr, LoadAddress, KeepAlive, Branch, Return, Unreachable, RegisterOp, BasicBlock,
+    Extend
 )
 from mypyc.ir.rtypes import RInstance
 from mypyc.analysis.dataflow import MAYBE_ANALYSIS, run_analysis, AnalysisResult, CFG
@@ -115,6 +116,9 @@ def visit_call_c(self, op: CallC) -> GenAndKill:
     def visit_truncate(self, op: Truncate) -> GenAndKill:
         return CLEAN
 
+    def visit_extend(self, op: Extend) -> GenAndKill:
+        return CLEAN
+
     def visit_load_global(self, op: LoadGlobal) -> GenAndKill:
         return CLEAN
 

mypyc/codegen/emit.py

@@ -17,7 +17,8 @@
     is_list_rprimitive, is_dict_rprimitive, is_set_rprimitive, is_tuple_rprimitive,
     is_none_rprimitive, is_object_rprimitive, object_rprimitive, is_str_rprimitive,
     int_rprimitive, is_optional_type, optional_value_type, is_int32_rprimitive,
-    is_int64_rprimitive, is_bit_rprimitive, is_range_rprimitive, is_bytes_rprimitive
+    is_int64_rprimitive, is_bit_rprimitive, is_range_rprimitive, is_bytes_rprimitive,
+    is_fixed_width_rtype
 )
 from mypyc.ir.func_ir import FuncDecl
 from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -479,9 +480,16 @@ def emit_cast(self,
                 return
 
         # TODO: Verify refcount handling.
-        if (is_list_rprimitive(typ) or is_dict_rprimitive(typ) or is_set_rprimitive(typ)
-                or is_str_rprimitive(typ) or is_range_rprimitive(typ) or is_float_rprimitive(typ)
-                or is_int_rprimitive(typ) or is_bool_rprimitive(typ) or is_bit_rprimitive(typ)):
+        if (is_list_rprimitive(typ)
+                or is_dict_rprimitive(typ)
+                or is_set_rprimitive(typ)
+                or is_str_rprimitive(typ)
+                or is_range_rprimitive(typ)
+                or is_float_rprimitive(typ)
+                or is_int_rprimitive(typ)
+                or is_bool_rprimitive(typ)
+                or is_bit_rprimitive(typ)
+                or is_fixed_width_rtype(typ)):
             if declare_dest:
                 self.emit_line(f'PyObject *{dest};')
             if is_list_rprimitive(typ):
@@ -496,12 +504,13 @@ def emit_cast(self,
                 prefix = 'PyRange'
             elif is_float_rprimitive(typ):
                 prefix = 'CPyFloat'
-            elif is_int_rprimitive(typ):
+            elif is_int_rprimitive(typ) or is_fixed_width_rtype(typ):
+                # TODO: Range check for fixed-width types?
                 prefix = 'PyLong'
             elif is_bool_rprimitive(typ) or is_bit_rprimitive(typ):
                 prefix = 'PyBool'
             else:
-                assert False, 'unexpected primitive type'
+                assert False, f'unexpected primitive type: {typ}'
             check = '({}_Check({}))'
             if likely:
                 check = f'(likely{check})'
@@ -765,6 +774,20 @@ def emit_unbox(self,
             self.emit_line(failure)
             self.emit_line('} else')
             self.emit_line(f'    {dest} = 1;')
+        elif is_int64_rprimitive(typ):
+            # Whether we are borrowing or not makes no difference.
+            if declare_dest:
+                self.emit_line(f'int64_t {dest};')
+            self.emit_line(f'{dest} = CPyLong_AsInt64({src});')
+            # TODO: Handle 'optional'
+            # TODO: Handle 'failure'
+        elif is_int32_rprimitive(typ):
+            # Whether we are borrowing or not makes no difference.
+            if declare_dest:
+                self.emit_line('int32_t {};'.format(dest))
+            self.emit_line('{} = CPyLong_AsInt32({});'.format(dest, src))
+            # TODO: Handle 'optional'
+            # TODO: Handle 'failure'
         elif isinstance(typ, RTuple):
             self.declare_tuple_struct(typ)
             if declare_dest:

mypyc/codegen/emitfunc.py

@@ -12,7 +12,8 @@
     LoadStatic, InitStatic, TupleGet, TupleSet, Call, IncRef, DecRef, Box, Cast, Unbox,
     BasicBlock, Value, MethodCall, Unreachable, NAMESPACE_STATIC, NAMESPACE_TYPE, NAMESPACE_MODULE,
     RaiseStandardError, CallC, LoadGlobal, Truncate, IntOp, LoadMem, GetElementPtr,
-    LoadAddress, ComparisonOp, SetMem, Register, LoadLiteral, AssignMulti, KeepAlive, ERR_FALSE
+    LoadAddress, ComparisonOp, SetMem, Register, LoadLiteral, AssignMulti, KeepAlive, Extend,
+    ERR_FALSE
 )
 from mypyc.ir.rtypes import (
     RType, RTuple, RArray, is_tagged, is_int32_rprimitive, is_int64_rprimitive, RStruct,
@@ -210,6 +211,10 @@ def visit_assign(self, op: Assign) -> None:
         # clang whines about self assignment (which we might generate
         # for some casts), so don't emit it.
         if dest != src:
+            # We sometimes assign from an integer prepresentation of a pointer
+            # to a real pointer, and C compilers insist on a cast.
+            if op.src.type.is_unboxed and not op.dest.type.is_unboxed:
+                src = f'(void *){src}'
             self.emit_line(f'{dest} = {src};')
 
     def visit_assign_multi(self, op: AssignMulti) -> None:
@@ -537,6 +542,15 @@ def visit_truncate(self, op: Truncate) -> None:
         # for C backend the generated code are straight assignments
         self.emit_line(f"{dest} = {value};")
 
+    def visit_extend(self, op: Extend) -> None:
+        dest = self.reg(op)
+        value = self.reg(op.src)
+        if op.signed:
+            src_cast = self.emit_signed_int_cast(op.src.type)
+        else:
+            src_cast = self.emit_unsigned_int_cast(op.src.type)
+        self.emit_line("{} = {}{};".format(dest, src_cast, value))
+
     def visit_load_global(self, op: LoadGlobal) -> None:
         dest = self.reg(op)
         ann = ''
@@ -550,6 +564,10 @@ def visit_int_op(self, op: IntOp) -> None:
         dest = self.reg(op)
         lhs = self.reg(op.lhs)
         rhs = self.reg(op.rhs)
+        if op.op == IntOp.RIGHT_SHIFT:
+            # Signed right shift
+            lhs = self.emit_signed_int_cast(op.lhs.type) + lhs
+            rhs = self.emit_signed_int_cast(op.rhs.type) + rhs
         self.emit_line(f'{dest} = {lhs} {op.op_str[op.op]} {rhs};')
 
     def visit_comparison_op(self, op: ComparisonOp) -> None:
@@ -623,7 +641,10 @@ def reg(self, reg: Value) -> str:
             s = str(val)
             if val >= (1 << 31):
                 # Avoid overflowing signed 32-bit int
-                s += 'ULL'
+                if val >= (1 << 63):
+                    s += 'ULL'
+                else:
+                    s += 'LL'
             elif val == -(1 << 63):
                 # Avoid overflowing C integer literal
                 s = '(-9223372036854775807LL - 1)'

mypyc/ir/ops.py

@@ -21,7 +21,7 @@
     RType, RInstance, RTuple, RArray, RVoid, is_bool_rprimitive, is_int_rprimitive,
     is_short_int_rprimitive, is_none_rprimitive, object_rprimitive, bool_rprimitive,
     short_int_rprimitive, int_rprimitive, void_rtype, pointer_rprimitive, is_pointer_rprimitive,
-    bit_rprimitive, is_bit_rprimitive
+    bit_rprimitive, is_bit_rprimitive, is_fixed_width_rtype
 )
 
 if TYPE_CHECKING:
@@ -90,6 +90,9 @@ def terminator(self) -> 'ControlOp':
 ERR_FALSE: Final = 2
 # Always fails
 ERR_ALWAYS: Final = 3
+# Like ERR_MAGIC, but the magic return overlaps with a possible return value, and
+# an extra PyErr_Occurred() check is also required
+ERR_MAGIC_OVERLAPPING: Final = 4
 
 # Hack: using this line number for an op will suppress it in tracebacks
 NO_TRACEBACK_LINE_NO = -10000
@@ -489,14 +492,17 @@ class Call(RegisterOp):
     The call target can be a module-level function or a class.
     """
 
-    error_kind = ERR_MAGIC
-
     def __init__(self, fn: 'FuncDecl', args: Sequence[Value], line: int) -> None:
-        super().__init__(line)
         self.fn = fn
         self.args = list(args)
         assert len(self.args) == len(fn.sig.args)
         self.type = fn.sig.ret_type
+        ret_type = fn.sig.ret_type
+        if not ret_type.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return list(self.args[:])
@@ -508,14 +514,11 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
 class MethodCall(RegisterOp):
     """Native method call obj.method(arg, ...)"""
 
-    error_kind = ERR_MAGIC
-
     def __init__(self,
                  obj: Value,
                  method: str,
                  args: List[Value],
                  line: int = -1) -> None:
-        super().__init__(line)
         self.obj = obj
         self.method = method
         self.args = args
@@ -524,7 +527,13 @@ def __init__(self,
         method_ir = self.receiver_type.class_ir.method_sig(method)
         assert method_ir is not None, "{} doesn't have method {}".format(
             self.receiver_type.name, method)
-        self.type = method_ir.ret_type
+        ret_type = method_ir.ret_type
+        self.type = ret_type
+        if not ret_type.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return self.args[:] + [self.obj]
@@ -605,8 +614,11 @@ def __init__(self, obj: Value, attr: str, line: int, *, borrow: bool = False) ->
         self.attr = attr
         assert isinstance(obj.type, RInstance), 'Attribute access not supported: %s' % obj.type
         self.class_type = obj.type
-        self.type = obj.type.attr_type(attr)
-        self.is_borrowed = borrow
+        attr_type = obj.type.attr_type(attr)
+        self.type = attr_type
+        if is_fixed_width_rtype(attr_type):
+            self.error_kind = ERR_NEVER
+        self.is_borrowed = borrow and attr_type.is_refcounted
 
     def sources(self) -> List[Value]:
         return [self.obj]
@@ -829,12 +841,14 @@ class Unbox(RegisterOp):
     representation. Only supported for types with an unboxed representation.
     """
 
-    error_kind = ERR_MAGIC
-
     def __init__(self, src: Value, typ: RType, line: int) -> None:
-        super().__init__(line)
         self.src = src
         self.type = typ
+        if not typ.error_overlap:
+            self.error_kind = ERR_MAGIC
+        else:
+            self.error_kind = ERR_MAGIC_OVERLAPPING
+        super().__init__(line)
 
     def sources(self) -> List[Value]:
         return [self.src]
@@ -924,22 +938,20 @@ class Truncate(RegisterOp):
 
     Truncate a value from type with more bits to type with less bits.
 
-    Both src_type and dst_type should be non-reference counted integer
-    types or bool. Note that int_rprimitive is reference counted so
-    it should never be used here.
+    dst_type and src_type can be native integer types, bools or tagged
+    integers. Tagged integers should have the tag bit unset.
     """
 
     error_kind = ERR_NEVER
 
     def __init__(self,
                  src: Value,
-                 src_type: RType,
                  dst_type: RType,
                  line: int = -1) -> None:
         super().__init__(line)
         self.src = src
-        self.src_type = src_type
         self.type = dst_type
+        self.src_type = src.type
 
     def sources(self) -> List[Value]:
         return [self.src]
@@ -951,6 +963,41 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
         return visitor.visit_truncate(self)
 
 
+class Extend(RegisterOp):
+    """result = extend src from src_type to dst_type
+
+    Extend a value from a type with fewer bits to a type with more bits.
+
+    dst_type and src_type can be native integer types, bools or tagged
+    integers. Tagged integers should have the tag bit unset.
+
+    If 'signed' is true, perform sign extension. Otherwise, the result will be
+    zero extended.
+    """
+
+    error_kind = ERR_NEVER
+
+    def __init__(self,
+                 src: Value,
+                 dst_type: RType,
+                 signed: bool,
+                 line: int = -1) -> None:
+        super().__init__(line)
+        self.src = src
+        self.type = dst_type
+        self.src_type = src.type
+        self.signed = signed
+
+    def sources(self) -> List[Value]:
+        return [self.src]
+
+    def stolen(self) -> List[Value]:
+        return []
+
+    def accept(self, visitor: 'OpVisitor[T]') -> T:
+        return visitor.visit_extend(self)
+
+
 class LoadGlobal(RegisterOp):
     """Load a low-level global variable/pointer.
 
@@ -1035,6 +1082,11 @@ def accept(self, visitor: 'OpVisitor[T]') -> T:
         return visitor.visit_int_op(self)
 
 
+# We can't have this in the IntOp class body, because of
+# https://github.com/mypyc/mypyc/issues/932.
+int_op_to_id: Final = {op: op_id for op_id, op in IntOp.op_str.items()}
+
+
 class ComparisonOp(RegisterOp):
     """Low-level comparison op for integers and pointers.
 
@@ -1076,6 +1128,15 @@ class ComparisonOp(RegisterOp):
         UGE: '>=',
     }
 
+    signed_ops: Final = {
+        '==': EQ,
+        '!=': NEQ,
+        '<': SLT,
+        '>': SGT,
+        '<=': SLE,
+        '>=': SGE,
+    }
+
     def __init__(self, lhs: Value, rhs: Value, op: int, line: int = -1) -> None:
         super().__init__(line)
         self.type = bit_rprimitive
@@ -1327,6 +1388,10 @@ def visit_call_c(self, op: CallC) -> T:
     def visit_truncate(self, op: Truncate) -> T:
         raise NotImplementedError
 
+    @abstractmethod
+    def visit_extend(self, op: Extend) -> T:
+        raise NotImplementedError
+
     @abstractmethod
     def visit_load_global(self, op: LoadGlobal) -> T:
         raise NotImplementedError