Merge pull request RustPython#3293 from DimitrisJim/recursive_inst_subcls

Fix recursions in isinstance and issubclass.

Author: youknowone

Lib/test/test_isinstance.py

@@ -241,15 +241,11 @@ def test_subclass_tuple(self):
         self.assertEqual(True, issubclass(int, (int, (float, int))))
         self.assertEqual(True, issubclass(str, (str, (Child, str))))
 
-    # TODO: RUSTPYTHON requires issue 2680 for stack depth detection in Rust
-    @unittest.expectedFailure
     def test_subclass_recursion_limit(self):
         # make sure that issubclass raises RecursionError before the C stack is
         # blown
         self.assertRaises(RecursionError, blowstack, issubclass, str, str)
 
-    # TODO: RUSTPYTHON requires issue 2680 for stack depth detection in Rust
-    @unittest.expectedFailure
     def test_isinstance_recursion_limit(self):
         # make sure that issubclass raises RecursionError before the C stack is
         # blown

vm/src/vm.rs

@@ -902,7 +902,7 @@ impl VirtualMachine {
     }
 
     pub fn to_repr(&self, obj: &PyObjectRef) -> PyResult<PyStrRef> {
-        self.with_recursion(" while getting the repr of an object", || {
+        self.with_recursion("while getting the repr of an object", || {
             let repr = self.call_special_method(obj.clone(), "__repr__", ())?;
             repr.try_into_value(self)
         })
@@ -1057,15 +1057,16 @@ impl VirtualMachine {
 
         if let Ok(tuple) = PyTupleRef::try_from_object(self, cls.clone()) {
             for typ in tuple.as_slice().iter() {
-                if self.isinstance(obj, typ)? {
+                if self.with_recursion("in __instancecheck__", || self.isinstance(obj, typ))? {
                     return Ok(true);
                 }
             }
             return Ok(false);
         }
 
         if let Ok(meth) = self.get_special_method(cls.clone(), "__instancecheck__")? {
-            let ret = meth.invoke((obj.clone(),), self)?;
+            let ret =
+                self.with_recursion("in __instancecheck__", || meth.invoke((obj.clone(),), self))?;
             return ret.try_to_bool(self);
         }
 
@@ -1139,15 +1140,17 @@ impl VirtualMachine {
 
         if let Ok(tuple) = PyTupleRef::try_from_object(self, cls.clone()) {
             for typ in tuple.as_slice().iter() {
-                if self.issubclass(subclass, typ)? {
+                if self.with_recursion("in __subclasscheck__", || self.issubclass(subclass, typ))? {
                     return Ok(true);
                 }
             }
             return Ok(false);
         }
 
         if let Ok(meth) = self.get_special_method(cls.clone(), "__subclasscheck__")? {
-            let ret = meth.invoke((subclass.clone(),), self)?;
+            let ret = self.with_recursion("in __subclasscheck__", || {
+                meth.invoke((subclass.clone(),), self)
+            })?;
             return ret.try_to_bool(self);
         }