[mypyc] new error_kind and branch variant to handle call_negative_bool_emit (#9035)

Related: mypyc/mypyc#734 and mypyc/mypyc#741.

Introducing ERR_NEG_INT error_kind and NEG_INT_EXPR branch variant to support 
checking the return value of a call is non-negative. set.discard is used as an example. 
With some modifications, this would also support negative_int_emit.

Author: TH3CHARLie

mypyc/codegen/emitfunc.py

@@ -13,7 +13,7 @@
     BasicBlock, Value, MethodCall, PrimitiveOp, EmitterInterface, Unreachable, NAMESPACE_STATIC,
     NAMESPACE_TYPE, NAMESPACE_MODULE, RaiseStandardError, CallC, LoadGlobal
 )
-from mypyc.ir.rtypes import RType, RTuple, is_c_int_rprimitive
+from mypyc.ir.rtypes import RType, RTuple, is_int32_rprimitive, is_int64_rprimitive
 from mypyc.ir.func_ir import FuncIR, FuncDecl, FUNC_STATICMETHOD, FUNC_CLASSMETHOD
 from mypyc.ir.class_ir import ClassIR
 
@@ -105,6 +105,9 @@ def visit_branch(self, op: Branch) -> None:
         if op.op == Branch.BOOL_EXPR:
             expr_result = self.reg(op.left)  # right isn't used
             cond = '{}{}'.format(neg, expr_result)
+        elif op.op == Branch.NEG_INT_EXPR:
+            expr_result = self.reg(op.left)
+            cond = '{} < 0'.format(expr_result)
         elif op.op == Branch.IS_ERROR:
             typ = op.left.type
             compare = '!=' if op.negated else '=='
@@ -179,7 +182,7 @@ def visit_assign(self, op: Assign) -> None:
 
     def visit_load_int(self, op: LoadInt) -> None:
         dest = self.reg(op)
-        if is_c_int_rprimitive(op.type):
+        if is_int32_rprimitive(op.type) or is_int64_rprimitive(op.type):
             self.emit_line('%s = %d;' % (dest, op.value))
         else:
             self.emit_line('%s = %d;' % (dest, op.value * 2))

mypyc/common.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Dict, Any
 
 from typing_extensions import Final
@@ -28,6 +29,8 @@
 # Maximal number of subclasses for a class to trigger fast path in isinstance() checks.
 FAST_ISINSTANCE_MAX_SUBCLASSES = 2  # type: Final
 
+IS_32_BIT_PLATFORM = sys.maxsize < (1 << 31)  # type: Final
+
 
 def decorator_helper_name(func_name: str) -> str:
     return '__mypyc_{}_decorator_helper__'.format(func_name)

mypyc/ir/ops.py

@@ -293,6 +293,8 @@ def terminated(self) -> bool:
 ERR_MAGIC = 1  # type: Final
 # Generates false (bool) on exception
 ERR_FALSE = 2  # type: Final
+# Generates negative integer on exception
+ERR_NEG_INT = 3  # type: Final
 
 # Hack: using this line number for an op will suppress it in tracebacks
 NO_TRACEBACK_LINE_NO = -10000
@@ -413,10 +415,12 @@ class Branch(ControlOp):
 
     BOOL_EXPR = 100  # type: Final
     IS_ERROR = 101  # type: Final
+    NEG_INT_EXPR = 102  # type: Final
 
     op_names = {
         BOOL_EXPR: ('%r', 'bool'),
         IS_ERROR: ('is_error(%r)', ''),
+        NEG_INT_EXPR: ('%r < 0', ''),
     }  # type: Final
 
     def __init__(self,

mypyc/ir/rtypes.py

@@ -25,7 +25,7 @@
 
 from typing_extensions import Final, ClassVar, TYPE_CHECKING
 
-from mypyc.common import JsonDict, short_name
+from mypyc.common import JsonDict, short_name, IS_32_BIT_PLATFORM
 from mypyc.namegen import NameGenerator
 
 if TYPE_CHECKING:
@@ -174,7 +174,9 @@ def __init__(self,
         self.is_unboxed = is_unboxed
         self._ctype = ctype
         self.is_refcounted = is_refcounted
-        if ctype in ('CPyTagged', 'Py_ssize_t'):
+        # TODO: For low-level integers, they actually don't have undefined values
+        #       we need to figure out some way to represent here.
+        if ctype in ('CPyTagged', 'int32_t', 'int64_t'):
             self.c_undefined = 'CPY_INT_TAG'
         elif ctype == 'PyObject *':
             # Boxed types use the null pointer as the error value.
@@ -234,9 +236,17 @@ def __repr__(self) -> str:
 short_int_rprimitive = RPrimitive('short_int', is_unboxed=True, is_refcounted=False,
                                   ctype='CPyTagged')  # type: Final
 
-# low level integer (corresponds to C's 'int').
-c_int_rprimitive = RPrimitive('c_int', is_unboxed=True, is_refcounted=False,
-                              ctype='Py_ssize_t')  # type: Final
+# low level integer (corresponds to C's 'int's).
+int32_rprimitive = RPrimitive('int32', is_unboxed=True, is_refcounted=False,
+                              ctype='int32_t')  # type: Final
+int64_rprimitive = RPrimitive('int64', is_unboxed=True, is_refcounted=False,
+                              ctype='int64_t')  # type: Final
+# integer alias
+c_int_rprimitive = int32_rprimitive
+if IS_32_BIT_PLATFORM:
+    c_pyssize_t_rprimitive = int32_rprimitive
+else:
+    c_pyssize_t_rprimitive = int64_rprimitive
 
 # Floats are represent as 'float' PyObject * values. (In the future
 # we'll likely switch to a more efficient, unboxed representation.)
@@ -279,8 +289,12 @@ def is_short_int_rprimitive(rtype: RType) -> bool:
     return rtype is short_int_rprimitive
 
 
-def is_c_int_rprimitive(rtype: RType) -> bool:
-    return rtype is c_int_rprimitive
+def is_int32_rprimitive(rtype: RType) -> bool:
+    return rtype is int32_rprimitive
+
+
+def is_int64_rprimitive(rtype: RType) -> bool:
+    return rtype is int64_rprimitive
 
 
 def is_float_rprimitive(rtype: RType) -> bool:

mypyc/irbuild/ll_builder.py

@@ -26,7 +26,7 @@
 from mypyc.ir.rtypes import (
     RType, RUnion, RInstance, optional_value_type, int_rprimitive, float_rprimitive,
     bool_rprimitive, list_rprimitive, str_rprimitive, is_none_rprimitive, object_rprimitive,
-    c_int_rprimitive
+    c_pyssize_t_rprimitive
 )
 from mypyc.ir.func_ir import FuncDecl, FuncSignature
 from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -884,7 +884,7 @@ def _create_dict(self,
         # keys and values should have the same number of items
         size = len(keys)
         if size > 0:
-            load_size_op = self.add(LoadInt(size, -1, c_int_rprimitive))
+            load_size_op = self.add(LoadInt(size, -1, c_pyssize_t_rprimitive))
             # merge keys and values
             items = [i for t in list(zip(keys, values)) for i in t]
             return self.call_c(dict_build_op, [load_size_op] + items, line)

mypyc/primitives/dict_ops.py

@@ -5,7 +5,7 @@
 from mypyc.ir.ops import EmitterInterface, ERR_FALSE, ERR_MAGIC, ERR_NEVER
 from mypyc.ir.rtypes import (
     dict_rprimitive, object_rprimitive, bool_rprimitive, int_rprimitive,
-    list_rprimitive, dict_next_rtuple_single, dict_next_rtuple_pair, c_int_rprimitive
+    list_rprimitive, dict_next_rtuple_single, dict_next_rtuple_pair, c_pyssize_t_rprimitive
 )
 
 from mypyc.primitives.registry import (
@@ -99,7 +99,7 @@
 # Positional argument is the number of key-value pairs
 # Variable arguments are (key1, value1, ..., keyN, valueN).
 dict_build_op = c_custom_op(
-    arg_types=[c_int_rprimitive],
+    arg_types=[c_pyssize_t_rprimitive],
     return_type=dict_rprimitive,
     c_function_name='CPyDict_Build',
     error_kind=ERR_MAGIC,

mypyc/primitives/set_ops.py

@@ -4,8 +4,10 @@
     func_op, method_op, binary_op, simple_emit, negative_int_emit,
     call_negative_bool_emit, c_function_op, c_method_op
 )
-from mypyc.ir.ops import ERR_MAGIC, ERR_FALSE, ERR_NEVER, EmitterInterface
-from mypyc.ir.rtypes import object_rprimitive, bool_rprimitive, set_rprimitive, int_rprimitive
+from mypyc.ir.ops import ERR_MAGIC, ERR_FALSE, ERR_NEVER, ERR_NEG_INT, EmitterInterface
+from mypyc.ir.rtypes import (
+    object_rprimitive, bool_rprimitive, set_rprimitive, int_rprimitive, c_int_rprimitive
+)
 from typing import List
 
 
@@ -70,13 +72,12 @@ def emit_len(emitter: EmitterInterface, args: List[str], dest: str) -> None:
     error_kind=ERR_FALSE)
 
 # set.discard(obj)
-method_op(
+c_method_op(
     name='discard',
     arg_types=[set_rprimitive, object_rprimitive],
-    result_type=bool_rprimitive,
-    error_kind=ERR_FALSE,
-    emit=call_negative_bool_emit('PySet_Discard')
-)
+    return_type=c_int_rprimitive,
+    c_function_name='PySet_Discard',
+    error_kind=ERR_NEG_INT)
 
 # set.add(obj)
 set_add_op = method_op(

mypyc/test-data/irbuild-basic.test

@@ -1088,7 +1088,7 @@ def call_python_function_with_keyword_arg(x):
     r1 :: object
     r2 :: str
     r3 :: tuple
-    r4 :: c_int
+    r4 :: native_int
     r5 :: object
     r6 :: dict
     r7 :: object
@@ -1113,7 +1113,7 @@ def call_python_method_with_keyword_args(xs, first, second):
     r3 :: str
     r4 :: object
     r5 :: tuple
-    r6 :: c_int
+    r6 :: native_int
     r7 :: object
     r8 :: dict
     r9 :: object
@@ -1122,7 +1122,7 @@ def call_python_method_with_keyword_args(xs, first, second):
     r12 :: object
     r13, r14 :: str
     r15 :: tuple
-    r16 :: c_int
+    r16 :: native_int
     r17, r18 :: object
     r19 :: dict
     r20 :: object
@@ -1690,7 +1690,7 @@ def g():
     r3 :: short_int
     r4 :: str
     r5 :: short_int
-    r6 :: c_int
+    r6 :: native_int
     r7, r8, r9 :: object
     r10, r11 :: dict
     r12 :: str
@@ -1727,7 +1727,7 @@ def h():
     r2 :: short_int
     r3 :: str
     r4 :: short_int
-    r5 :: c_int
+    r5 :: native_int
     r6, r7 :: object
     r8, r9 :: dict
     r10 :: str

mypyc/test-data/irbuild-dict.test

@@ -58,7 +58,7 @@ def f(x):
     x :: object
     r0, r1 :: short_int
     r2 :: str
-    r3 :: c_int
+    r3 :: native_int
     r4, r5 :: object
     r6, d :: dict
     r7 :: None
@@ -204,7 +204,7 @@ def f(x, y):
     r0 :: short_int
     r1 :: str
     r2 :: short_int
-    r3 :: c_int
+    r3 :: native_int
     r4 :: object
     r5 :: dict
     r6 :: bool

mypyc/test-data/irbuild-set.test

@@ -141,14 +141,14 @@ def f():
     r0, x :: set
     r1 :: short_int
     r2 :: object
-    r3 :: bool
+    r3 :: int32
     r4 :: None
 L0:
     r0 = set
     x = r0
     r1 = 1
     r2 = box(short_int, r1)
-    r3 = x.discard(r2) :: set
+    r3 = PySet_Discard(x, r2)
     r4 = None
     return x
 

mypyc/test-data/irbuild-statements.test

@@ -717,7 +717,7 @@ def delDict():
     r1 :: short_int
     r2 :: str
     r3 :: short_int
-    r4 :: c_int
+    r4 :: native_int
     r5, r6 :: object
     r7, d :: dict
     r8 :: str
@@ -746,7 +746,7 @@ def delDictMultiple():
     r5 :: short_int
     r6 :: str
     r7 :: short_int
-    r8 :: c_int
+    r8 :: native_int
     r9, r10, r11, r12 :: object
     r13, d :: dict
     r14, r15 :: str

mypyc/test-data/refcount.test

@@ -740,7 +740,7 @@ def g(x):
     r1 :: object
     r2 :: str
     r3 :: tuple
-    r4 :: c_int
+    r4 :: native_int
     r5 :: object
     r6 :: dict
     r7 :: object

mypyc/test/test_irbuild.py

@@ -6,7 +6,7 @@
 from mypy.test.data import DataDrivenTestCase
 from mypy.errors import CompileError
 
-from mypyc.common import TOP_LEVEL_NAME
+from mypyc.common import TOP_LEVEL_NAME, IS_32_BIT_PLATFORM
 from mypyc.ir.func_ir import format_func
 from mypyc.test.testutil import (
     ICODE_GEN_BUILTINS, use_custom_builtins, MypycDataSuite, build_ir_for_single_file,
@@ -42,7 +42,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         """Perform a runtime checking transformation test case."""
         with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
             expected_output = remove_comment_lines(testcase.output)
-
+            # replace native_int with platform specific ints
+            int_format_str = 'int32' if IS_32_BIT_PLATFORM else 'int64'
+            expected_output = [s.replace('native_int', int_format_str) for s in expected_output]
             try:
                 ir = build_ir_for_single_file(testcase.input, options)
             except CompileError as e:

mypyc/test/test_refcount.py

@@ -10,7 +10,7 @@
 from mypy.test.data import DataDrivenTestCase
 from mypy.errors import CompileError
 
-from mypyc.common import TOP_LEVEL_NAME
+from mypyc.common import TOP_LEVEL_NAME, IS_32_BIT_PLATFORM
 from mypyc.ir.func_ir import format_func
 from mypyc.transform.refcount import insert_ref_count_opcodes
 from mypyc.test.testutil import (
@@ -32,7 +32,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         """Perform a runtime checking transformation test case."""
         with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
             expected_output = remove_comment_lines(testcase.output)
-
+            # replace native_int with platform specific ints
+            int_format_str = 'int32' if IS_32_BIT_PLATFORM else 'int64'
+            expected_output = [s.replace('native_int', int_format_str) for s in expected_output]
             try:
                 ir = build_ir_for_single_file(testcase.input)
             except CompileError as e:

mypyc/transform/exceptions.py

@@ -13,7 +13,7 @@
 
 from mypyc.ir.ops import (
     BasicBlock, LoadErrorValue, Return, Branch, RegisterOp, ERR_NEVER, ERR_MAGIC,
-    ERR_FALSE, NO_TRACEBACK_LINE_NO,
+    ERR_FALSE, ERR_NEG_INT, NO_TRACEBACK_LINE_NO,
 )
 from mypyc.ir.func_ir import FuncIR
 
@@ -74,6 +74,9 @@ def split_blocks_at_errors(blocks: List[BasicBlock],
                     # Op returns a C false value on error.
                     variant = Branch.BOOL_EXPR
                     negated = True
+                elif op.error_kind == ERR_NEG_INT:
+                    variant = Branch.NEG_INT_EXPR
+                    negated = False
                 else:
                     assert False, 'unknown error kind %d' % op.error_kind