Stackless issue python#168: make Stackless compatible with old Cython modules

Many extension modules were created by Cython versions before commit
037bcf0 and were compiled with regular C-Python. These modules call
PyEval_EvalFrameEx() with a broken frame object.
This commit add code to recover a broken frame in PyEval_EvalFrameEx().

Author: Anselm Kruis

Objects/frameobject.c

@@ -447,6 +447,13 @@ frame_dealloc(PyFrameObject *f)
         _PyObject_GC_UNTRACK(f);
 
     Py_TRASHCAN_SAFE_BEGIN(f)
+
+#if defined(STACKLESS) && PY_VERSION_HEX < 0x03080000
+    /* Clear the magic for the old Cython frame hack.
+     * See below in PyFrame_New() for a detailed explanation.
+     */
+    f->f_blockstack[0].b_type = 0;
+#endif
     /* Kill all local variables */
     valuestack = f->f_valuestack;
     for (p = f->f_localsplus; p < valuestack; p++)
@@ -770,8 +777,28 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
             PyObject *globals, PyObject *locals)
 {
     PyFrameObject *f = _PyFrame_New_NoTrack(tstate, code, globals, locals);
-    if (f)
+    if (f) {
         _PyObject_GC_TRACK(f);
+#if defined(STACKLESS) && PY_VERSION_HEX < 0x03080000
+        if (code->co_argcount > 0) {
+            /*
+             * A hack for binary compatibility with Cython extension modules, which
+             * were created with an older Cythoncompiled with regular C-Python. These
+             * modules create frames using PyFrame_New then write to frame->f_localsplus
+             * to set the arguments. But C-Python f_localsplus is Stackless f_code.
+             * Therefore we add a copy of f_code and a magic number in the
+             * uninitialized f_blockstack array.
+             * If the blockstack is used, the magic is overwritten.
+             * To make sure, the pointer is aligned correctly, we address it relative to
+             * f_code.
+             * See Stackless issue #168
+             */
+            (&(f->f_code))[-1] = code;
+            /* an arbitrary negative number which is not an opcode */
+            f->f_blockstack[0].b_type = -31683;
+        }
+#endif
+    }
     return f;
 }
 

Python/ceval.c

@@ -4022,16 +4022,49 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
     if (f == NULL)
         return NULL;
 
+    /* The layout of PyFrameObject differs between Stackless and C-Python.
+     * Stackless f->f_execute is C-Python f->f_code. Stackless f->f_code is at
+     * the end, just before f_localsplus.
+     */
     if (PyFrame_Check(f) && f->f_execute == NULL) {
         /* A new frame returned from PyFrame_New() has f->f_execute == NULL.
+         * Set the usual execution function.
          */
         f->f_execute = PyEval_EvalFrameEx_slp;
+
+#if PY_VERSION_HEX < 0x03080000
+        /* Older versions of Cython used to create frames using C-Python layout
+         * of PyFrameObject. As a consequence f_code is overwritten by the first
+         * item of f_localsplus[]. To be able to fix it, we have a copy of
+         * f_code and a signature at the end of the block-stack.
+         * The Py_BUILD_ASSERT_EXPR checks,that our assumptions about the layout
+         * of PyFrameObject are true.
+         * See Stackless issue #168
+         */
+        (void) Py_BUILD_ASSERT_EXPR(offsetof(PyFrameObject, f_code) ==
+               offsetof(PyFrameObject, f_localsplus) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus[0]));
+
+        /* Check for an old Cython frame */
+        if (f->f_iblock == 0 && f->f_lasti == -1 && /* blockstack is empty */
+            f->f_blockstack[0].b_type == -31683 && /* magic is present */
+            /* and f_code has been overwritten */
+            f->f_code != (&(f->f_code))[-1] &&
+            /* and (&(f->f_code))[-1] looks like a valid code object */
+            (&(f->f_code))[-1] && PyCode_Check((&(f->f_code))[-1]) &&
+            /* and there are arguments */
+            (&(f->f_code))[-1]->co_argcount > 0)
+        {
+            PyCodeObject * code = (&(f->f_code))[-1];
+            memmove(f->f_localsplus, f->f_localsplus-1, code->co_argcount * sizeof(f->f_localsplus[0]));
+            f->f_code = code;
+        } else
+#endif
+        if (!(f->f_code != NULL && PyCode_Check(f->f_code))) {
+            PyErr_BadInternalCall();
+            return NULL;
+        }
     } else {
-        /* The layout of PyFrameObject differs between Stackless and C-Python.
-         * Stackless f->f_execute is C-Python f->f_code. Stackless f->f_code is at
-         * the end, just before f_localsplus.
-         *
-         * In order to detect a C-Python frame, we must compare f->f_execute
+        /* In order to detect a broken C-Python frame, we must compare f->f_execute
          * with every valid frame function. Hard to implement completely.
          * Therefore I'll check only for relevant functions.
          * Amend the list as needed.

Stackless/changelog.txt

@@ -13,6 +13,10 @@ What's New in Stackless 3.X.X?
   Fix C-API functions PyEval_EvalFrameEx() and PyEval_EvalFrame().
   They are now compatible with C-Python.
 
+- https://github.com/stackless-dev/stackless/issues/168
+  Make Stackless compatible with old Cython extension modules compiled
+  for regular C-Python.
+
 - https://github.com/stackless-dev/stackless/issues/167
   Replace 'printf(...)' calls by PySys_WriteStderr(...). They are used to emit
   an error message, if there is a pending error while entering Stackless

Stackless/module/stacklessmodule.c

@@ -1131,20 +1131,21 @@ by Stackless Python.\n\
 The function creates a frame from code, globals and args and executes the frame.");
 
 static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObject *kwds) {
-    static char *kwlist[] = {"code", "globals", "args", "alloca", "throw", NULL};
+    static char *kwlist[] = {"code", "globals", "args", "alloca", "throw", "oldcython", NULL};
     PyThreadState *tstate = PyThreadState_GET();
     PyCodeObject *co;
     PyObject *globals, *co_args = NULL;
     Py_ssize_t alloca_size = 0;
     PyObject *exc = NULL;
+    PyObject *oldcython = NULL;
     PyFrameObject *f;
     PyObject *result = NULL;
     void *p;
     Py_ssize_t na;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!nO:test_PyEval_EvalFrameEx", kwlist,
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!|O!nOO!:test_PyEval_EvalFrameEx", kwlist,
             &PyCode_Type, &co, &PyDict_Type, &globals, &PyTuple_Type, &co_args, &alloca_size,
-            &exc))
+            &exc, &PyBool_Type, &oldcython))
         return NULL;
     if (exc && !PyExceptionInstance_Check(exc)) {
         PyErr_SetString(PyExc_TypeError, "exc must be an exception instance");
@@ -1175,6 +1176,13 @@ static PyObject* test_PyEval_EvalFrameEx(PyObject *self, PyObject *args, PyObjec
             goto exit;
         }
         fastlocals = f->f_localsplus;
+        if (oldcython == Py_True) {
+            /* Use the f_localsplus offset from regular C-Python. Old versions of cython used to
+             * access f_localplus directly. Current versions compute the field offset for
+             * f_localsplus at run-time.
+             */
+            fastlocals--;
+        }
         for (i = 0; i < na; i++) {
             PyObject *arg = PyTuple_GetItem(co_args, i);
             if (arg == NULL) {

Stackless/unittests/test_capi.py

@@ -75,6 +75,10 @@ def test_cstack_spilling(self):
         # Force stack spilling. 16384 is the value of CSTACK_WATERMARK from slp_platformselect.h
         self.call_PyEval_EvalFrameEx(None, alloca=16384 * 8)
 
+    def test_oldcython_frame(self):
+        # A test for Stackless issue #168
+        self.assertEqual(self.call_PyEval_EvalFrameEx(47110816, oldcython=True), 47110816)
+
     def test_stack_unwinding(self):
         # Calling the __init__ method of a new-style class involves stack unwinding
         class C(object):