[mypyc] Don't coerce types checked with isinstance (#811) (#10245)

Prevent the coercion of things whose types are checked with isinstance
to other types - such coercion is not necessary and can lead to bugs.

Also, add test for avoiding coercion of isinstance() comparators.

Fixes mypyc/mypyc#811.

Author: sinback

mypyc/irbuild/specialize.py

@@ -278,10 +278,15 @@ def gen_inner_stmts() -> None:
 
 @specialize_function('builtins.isinstance')
 def translate_isinstance(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]:
-    # Special case builtins.isinstance
     if (len(expr.args) == 2
             and expr.arg_kinds == [ARG_POS, ARG_POS]
             and isinstance(expr.args[1], (RefExpr, TupleExpr))):
+        # Special case for builtins.isinstance
+        # Prevent coercions on the thing we are checking the instance of - there is no need to
+        # coerce something to a new type before checking what type it is, and the coercion could
+        # lead to bugs.
+        builder.types[expr.args[0]] = AnyType(TypeOfAny.from_error)
+
         irs = builder.flatten_classes(expr.args[1])
         if irs is not None:
             return builder.builder.isinstance_helper(builder.accept(expr.args[0]), irs, expr.line)

mypyc/test-data/irbuild-isinstance.test

@@ -0,0 +1,77 @@
+[case testIsinstanceInt]
+def is_int(value: object) -> bool:
+    return isinstance(value, int)
+
+[out]
+def is_int(value):
+    value, r0 :: object
+    r1 :: int32
+    r2 :: bit
+    r3 :: bool
+L0:
+    r0 = load_address PyLong_Type
+    r1 = PyObject_IsInstance(value, r0)
+    r2 = r1 >= 0 :: signed
+    r3 = truncate r1: int32 to builtins.bool
+    return r3
+
+[case testIsinstanceNotBool1]
+def is_not_bool(value: object) -> bool:
+    return not isinstance(value, bool)
+
+[out]
+def is_not_bool(value):
+    value, r0 :: object
+    r1 :: str
+    r2 :: object
+    r3 :: int32
+    r4 :: bit
+    r5, r6 :: bool
+L0:
+    r0 = builtins :: module
+    r1 = 'bool'
+    r2 = CPyObject_GetAttr(r0, r1)
+    r3 = PyObject_IsInstance(value, r2)
+    r4 = r3 >= 0 :: signed
+    r5 = truncate r3: int32 to builtins.bool
+    r6 = r5 ^ 1
+    return r6
+
+[case testIsinstanceIntAndNotBool]
+# This test is to ensure that 'value' doesn't get coerced to int when we are
+# checking if it's a bool, since an int can never be an instance of a bool
+def is_not_bool_and_is_int(value: object) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool)
+
+[out]
+def is_not_bool_and_is_int(value):
+    value, r0 :: object
+    r1 :: int32
+    r2 :: bit
+    r3, r4 :: bool
+    r5 :: object
+    r6 :: str
+    r7 :: object
+    r8 :: int32
+    r9 :: bit
+    r10, r11 :: bool
+L0:
+    r0 = load_address PyLong_Type
+    r1 = PyObject_IsInstance(value, r0)
+    r2 = r1 >= 0 :: signed
+    r3 = truncate r1: int32 to builtins.bool
+    if r3 goto L2 else goto L1 :: bool
+L1:
+    r4 = r3
+    goto L3
+L2:
+    r5 = builtins :: module
+    r6 = 'bool'
+    r7 = CPyObject_GetAttr(r5, r6)
+    r8 = PyObject_IsInstance(value, r7)
+    r9 = r8 >= 0 :: signed
+    r10 = truncate r8: int32 to builtins.bool
+    r11 = r10 ^ 1
+    r4 = r11
+L3:
+    return r4

mypyc/test-data/run-integers.test

@@ -157,6 +157,15 @@ def check_bitwise(x: int, y: int) -> None:
             check_or(ll, rr)
             check_xor(ll, rr)
 
+[case testIsinstanceIntAndNotBool]
+def test_isinstance_int_and_not_bool(value: object) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool)
+[file driver.py]
+from native import test_isinstance_int_and_not_bool
+assert test_isinstance_int_and_not_bool(True) == False
+assert test_isinstance_int_and_not_bool(1) == True
+
+
 SHIFT = 30
 DIGIT0a = 615729753
 DIGIT0b = 832796681

mypyc/test/test_irbuild.py

@@ -32,6 +32,7 @@
     'irbuild-int.test',
     'irbuild-vectorcall.test',
     'irbuild-unreachable.test',
+    'irbuild-isinstance.test',
 ]