Stackless issue python#198: Improve the soft-switchable extension functions

Better handling of exceptions / NULL result values.
Move the macros to manipulate the try-stackless floag to the API.
Now it is possible to convert existing extension functions to stackless,
without using internal API.
Coming soon: test cases and documentation

Author: Anselm Kruis

Stackless/core/cframeobject.c

@@ -436,7 +436,8 @@ execute_soft_switchable_func(PyFrameObject *f, int exc, PyObject *retval)
     PyObject *ob1, *ob2, *ob3;
     PyStacklessFunctionDeclarationObject *ssfd =
             (PyStacklessFunctionDeclarationObject*)cf->any2;
-    if (retval == NULL) {
+    if (retval == NULL && !PyErr_Occurred()) {
+        PyErr_SetString(PyExc_SystemError, "retval is NULL, but no error occurred");
         SLP_STORE_NEXT_FRAME(ts, cf->f_back);
         return NULL;
     }
@@ -452,7 +453,7 @@ execute_soft_switchable_func(PyFrameObject *f, int exc, PyObject *retval)
     Py_XINCREF(ob3);
     STACKLESS_PROPOSE_ALL(ts);
     /* use Py_SETREF, because we own a ref to retval. */
-    Py_SETREF(retval, ssfd->sfunc(retval, &cf->i, &cf->ob1, &cf->ob2, &cf->ob3, &cf->n, &cf->any1));
+    Py_XSETREF(retval, ssfd->sfunc(retval, &cf->i, &cf->ob1, &cf->ob2, &cf->ob3, &cf->n, &cf->any1));
     STACKLESS_RETRACT();
     STACKLESS_ASSERT();
     Py_XDECREF(ob1);
@@ -484,18 +485,29 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a
 {
     STACKLESS_GETARG();
     PyThreadState *ts = PyThreadState_GET();
+    PyObject *et, *ev, *tb;
 
     assert(ssfd);
     assert(ts->st.main != NULL);
 
+    if (arg == NULL) {
+        if (!PyErr_Occurred()) {
+            PyErr_SetString(PyExc_SystemError, "No error occurred, but arg is NULL");
+            return NULL;
+        }
+        PyErr_Fetch(&et, &ev, &tb);
+    }
+
     if (stackless) {
         /* Only soft-switch, if the caller can handle Py_UnwindToken.
          * If called from Python-code, this is always the case,
          * but if you call this method from a C-function, you don't know. */
         PyCFrameObject *cf;
         cf = slp_cframe_new(execute_soft_switchable_func, 1);
-        if (cf == NULL)
+        if (cf == NULL) {
+            _PyErr_ChainExceptions(et, ev, tb);
             return NULL;
+        }
         cf->any2 = ssfd;
 
         Py_XINCREF(ob1);
@@ -510,9 +522,9 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a
         cf->any1 = any;
         SLP_STORE_NEXT_FRAME(ts, (PyFrameObject *) cf);
         Py_DECREF(cf);
+        Py_XINCREF(arg);
         if (arg == NULL)
-            arg = Py_None;
-        Py_INCREF(arg);
+            PyErr_Restore(et, ev, tb);
         return STACKLESS_PACK(ts, arg);
     } else {
         /*
@@ -521,23 +533,23 @@ PyStackless_CallFunction(PyStacklessFunctionDeclarationObject *ssfd, PyObject *a
         PyObject *retval, *saved_ob1, *saved_ob2, *saved_ob3;
         long step = 0;
 
-        if (arg == NULL)
-            arg = Py_None;
-        Py_INCREF(arg);
+        Py_XINCREF(arg);
 
         saved_ob1 = ob1;
         saved_ob2 = ob2;
         saved_ob3 = ob3;
         Py_XINCREF(saved_ob1);
         Py_XINCREF(saved_ob2);
         Py_XINCREF(saved_ob3);
+        if (arg == NULL)
+            PyErr_Restore(et, ev, tb);
         retval = ssfd->sfunc(arg, &step, &ob1, &ob2, &ob3, &n, &any);
         STACKLESS_ASSERT();
         assert(!STACKLESS_UNWINDING(retval));
         Py_XDECREF(saved_ob1);
         Py_XDECREF(saved_ob2);
         Py_XDECREF(saved_ob3);
-        Py_DECREF(arg);
+        Py_XDECREF(arg);
         return retval;
     }
 }

Stackless/core/stackless_impl.h

@@ -423,32 +423,19 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
 /* macros for setting/resetting the stackless flag */
 
+#ifdef _PyStackless_TRY_STACKLESS
+#undef _PyStackless_TRY_STACKLESS
+#endif
+#define _PyStackless_TRY_STACKLESS (_PyRuntime.st.try_stackless)
+
 #define STACKLESS_POSSIBLE(tstate) \
     ((tstate)->interp->st.enable_softswitch && (tstate)->st.unwinding_retval == NULL)
 
-#define STACKLESS_GETARG() \
-    int stackless = (assert(SLP_CURRENT_FRAME_IS_VALID(PyThreadState_GET())), \
-                     stackless = (_PyRuntime.st.try_stackless), \
-                     (_PyRuntime.st.try_stackless) = 0, \
-                     stackless)
-
-#define STACKLESS_PROMOTE(func) \
-    (stackless ? (_PyRuntime.st.try_stackless) = \
-     Py_TYPE(func)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_CALL : 0)
-
-#define STACKLESS_PROMOTE_FLAG(flag) \
-    (stackless ? (_PyRuntime.st.try_stackless) = (flag) : 0)
-
-#define STACKLESS_PROMOTE_METHOD(obj, meth) do { \
-    if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_EXTENSION) && \
-     Py_TYPE(obj)->tp_as_mapping) \
-        (_PyRuntime.st.try_stackless) = stackless && Py_TYPE(obj)->tp_as_mapping->slpflags.meth; \
-} while (0)
-
-#define STACKLESS_PROMOTE_WRAPPER(descr) \
-    ((_PyRuntime.st.try_stackless) = stackless && (descr)->d_slpmask)
-
-#define STACKLESS_PROMOTE_ALL() ((void)((_PyRuntime.st.try_stackless) = stackless, NULL))
+#ifdef STACKLESS__GETARG_ASSERT
+#undef STACKLESS__GETARG_ASSERT
+#endif
+#define STACKLESS__GETARG_ASSERT \
+    assert(SLP_CURRENT_FRAME_IS_VALID(PyThreadState_GET()))
 
 #define STACKLESS_PROPOSE(tstate, func) {int stackless = STACKLESS_POSSIBLE(tstate); \
                  STACKLESS_PROMOTE(func);}
@@ -459,11 +446,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 #define STACKLESS_PROPOSE_METHOD(tstate, obj, meth) {int stackless = STACKLESS_POSSIBLE(tstate); \
                  STACKLESS_PROMOTE_METHOD(obj, meth);}
 
-#define STACKLESS_PROPOSE_ALL(tstate) (_PyRuntime.st.try_stackless) = STACKLESS_POSSIBLE(tstate)
-
-#define STACKLESS_RETRACT() (_PyRuntime.st.try_stackless) = 0;
-
-#define STACKLESS_ASSERT() assert(!(_PyRuntime.st.try_stackless))
+#define STACKLESS_PROPOSE_ALL(tstate) _PyStackless_TRY_STACKLESS = STACKLESS_POSSIBLE(tstate)
 
 /* this is just a tag to denote which methods are stackless */
 
@@ -477,7 +460,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 /*
 
   How this works:
-  There is one global variable (_PyRuntime.st.try_stackless) which is used
+  There is one global variable _PyStackless_TRY_STACKLESS which is used
   like an implicit parameter. Since we don't have a real parameter,
   the flag is copied into the local variable "stackless" and cleared.
   This is done by the STACKLESS_GETARG() macro, which should be added to
@@ -493,7 +476,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
   STACKLESS_GETARG()
 
-    move the (_PyRuntime.st.try_stackless) flag into the local variable "stackless".
+    move the _PyStackless_TRY_STACKLESS flag into the local variable "stackless".
 
   STACKLESS_PROMOTE_ALL()
 
@@ -506,7 +489,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
     if stackless was set and the function's type has set
     Py_TPFLAGS_HAVE_STACKLESS_CALL, then this flag will be
-    put back into (_PyRuntime.st.try_stackless), and we expect that the
+    put back into _PyStackless_TRY_STACKLESS, and we expect that the
     function handles it correctly.
 
   STACKLESS_PROMOTE_FLAG(flag)
@@ -526,7 +509,7 @@ PyTaskletObject * slp_get_watchdog(PyThreadState *ts, int interrupt);
 
   STACKLESS_ASSERT()
 
-    make sure that (_PyRuntime.st.try_stackless) was cleared. This debug feature
+    make sure that _PyStackless_TRY_STACKLESS was cleared. This debug feature
     tries to ensure that no unexpected nonrecursive call can happen.
 
   Some functions which are known to be stackless by nature
@@ -787,18 +770,10 @@ long slp_parse_thread_id(PyObject *thread_id, unsigned long *id);
 #define SLP_PEEK_NEXT_FRAME(tstate) \
     ((tstate)->frame)
 
-#define STACKLESS_GETARG() int stackless = 0
-#define STACKLESS_PROMOTE(func) stackless = 0
-#define STACKLESS_PROMOTE_FLAG(flag) stackless = 0
-#define STACKLESS_PROMOTE_METHOD(obj, meth) stackless = 0
-#define STACKLESS_PROMOTE_WRAPPER(descr) stackless = 0
-#define STACKLESS_PROMOTE_ALL() stackless = 0
 #define STACKLESS_PROPOSE(tstate, func) assert(1)
 #define STACKLESS_PROPOSE_FLAG(tstate, flag) assert(1)
 #define STACKLESS_PROPOSE_ALL(tstate) assert(1)
 #define STACKLESS_PROPOSE_METHOD(tstate, obj, meth) assert(1)
-#define STACKLESS_RETRACT() assert(1)
-#define STACKLESS_ASSERT() assert(1)
 
 #define STACKLESS_RETVAL(tstate, obj) (obj)
 #define STACKLESS_ASSERT_UNWINDING_VALUE_IS_NOT(tstate, obj, val) assert(1)

Stackless/core/stackless_structs.h

@@ -288,6 +288,18 @@ PyAPI_DATA(PyTypeObject) PyTasklet_Type;
 PyAPI_DATA(PyTypeObject) PyChannel_Type;
 #define PyChannel_Check(op) PyObject_TypeCheck(op, &PyChannel_Type)
 #define PyChannel_CheckExact(op) (Py_TYPE(op) == &PyChannel_Type)
+
+/******************************************************
+  Macros for the stackless protocol
+ ******************************************************/
+
+#ifndef _PyStackless_TRY_STACKLESS
+PyAPI_DATA(int * const) _PyStackless__TryStacklessPtr;
+#define _PyStackless_TRY_STACKLESS (*_PyStackless__TryStacklessPtr)
+#endif
+#ifndef STACKLESS__GETARG_ASSERT
+#define STACKLESS__GETARG_ASSERT ((void)0)
+#endif
 
 #endif /* #ifdef STACKLESS */
 

Stackless/core/stackless_util.c

@@ -3,6 +3,8 @@
 #ifdef STACKLESS
 #include "stackless_impl.h"
 
+int * const _PyStackless__TryStacklessPtr = &_PyStackless_TRY_STACKLESS;
+
 /* Initialize the Stackless runtime state */
 void
 slp_initialize(struct _stackless_runtime_state * state) {

Stackless/stackless_api.h

@@ -390,6 +390,111 @@ PyAPI_FUNC(PyObject *) PyStackless_CallFunction(
 PyAPI_FUNC(int) PyStackless_InitFunctionDeclaration(
         PyStacklessFunctionDeclarationObject *sfd, PyObject *module, PyModuleDef *module_def);
 
+/*
+
+Macros for the "stackless protocol"
+===================================
+
+How to does Stackless Python decide, if a function may return an unwind-token.
+
+There is one global variable _PyStackless_TRY_STACKLESS which is used
+like an implicit parameter. Since we don't have a real parameter,
+the flag is copied into the local variable "stackless" and cleared.
+This is done by the STACKLESS_GETARG() macro, which should be added to
+the top of the function's declarations.
+The idea is to keep the chances to introduce error to the minimum.
+A function can safely do some tests and return before calling
+anything, since the flag is in a local variable.
+Depending on context, this flag is propagated to other called
+functions. They *must* obey the protocol. To make this sure,
+the STACKLESS_ASSERT() macro has to be called after every such call.
+
+Many internal functions have been patched to support this protocol.
+
+STACKLESS_GETARG()
+
+  Move the _PyStackless_TRY_STACKLESS flag into the local variable "stackless".
+
+STACKLESS_PROMOTE_ALL()
+
+  is used for cases where we know that the called function will take
+  care of our object, and we need no test. For example, PyObject_Call
+  and all other Py{Object,Function,CFunction}_*Call* functions use
+  STACKLESS_PROMOTE_xxx, itself, so we don't need to check further.
+
+STACKLESS_PROMOTE(func)
+
+  If stackless was set and the function's type has set
+  Py_TPFLAGS_HAVE_STACKLESS_CALL, then this flag will be
+  put back into _PyStackless_TRY_STACKLESS, and we expect that the
+  function handles it correctly.
+
+STACKLESS_PROMOTE_FLAG(flag)
+
+  is used for special cases, like PyCFunction objects. PyCFunction_Type
+  says that it supports a stackless call, but the final action depends
+  on the METH_STACKLESS flag in the object to be called. Therefore,
+  PyCFunction_Call uses PROMOTE_FLAG(flags & METH_STACKLESS) to
+  take care of PyCFunctions which don't care about it.
+
+  Another example is the "next" method of iterators. To support this,
+  the wrapperobject's type has the Py_TPFLAGS_HAVE_STACKLESS_CALL
+  flag set, but wrapper_call then examines the wrapper descriptors
+  flags if PyWrapperFlag_STACKLESS is set. "next" has it set.
+  It also checks whether Py_TPFLAGS_HAVE_STACKLESS_CALL is set
+  for the iterator's type.
+
+STACKLESS_ASSERT()
+
+  Make sure that _PyStackless_TRY_STACKLESS was cleared. This debug feature
+  tries to ensure that no unexpected nonrecursive call can happen.
+
+STACKLESS_RETRACT()
+
+  Reset _PyStackless_TRY_STACKLESS. Rarely needed.
+
+*/
+
+#define STACKLESS_GETARG() \
+    int stackless = (STACKLESS__GETARG_ASSERT, \
+                     stackless = _PyStackless_TRY_STACKLESS, \
+                     _PyStackless_TRY_STACKLESS = 0, \
+                     stackless)
+
+#define STACKLESS_PROMOTE(func) \
+    (stackless ? _PyStackless_TRY_STACKLESS = \
+     Py_TYPE(func)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_CALL : 0)
+
+#define STACKLESS_PROMOTE_FLAG(flag) \
+    (stackless ? _PyStackless_TRY_STACKLESS = (flag) : 0)
+
+#define STACKLESS_PROMOTE_METHOD(obj, meth) do { \
+    if ((Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_STACKLESS_EXTENSION) && \
+     Py_TYPE(obj)->tp_as_mapping) \
+        _PyStackless_TRY_STACKLESS = stackless && Py_TYPE(obj)->tp_as_mapping->slpflags.meth; \
+} while (0)
+
+#define STACKLESS_PROMOTE_WRAPPER(descr) \
+    (_PyStackless_TRY_STACKLESS = stackless && (descr)->d_slpmask)
+
+#define STACKLESS_PROMOTE_ALL() ((void)(_PyStackless_TRY_STACKLESS = stackless, NULL))
+
+#define STACKLESS_RETRACT() (_PyStackless_TRY_STACKLESS = 0)
+
+#define STACKLESS_ASSERT() assert(!_PyStackless_TRY_STACKLESS)
+
+#else /* STACKLESS */
+/* turn the stackless flag macros into dummies */
+#define STACKLESS_GETARG() int stackless = 0
+#define STACKLESS_PROMOTE(func) stackless = 0
+#define STACKLESS_PROMOTE_FLAG(flag) stackless = 0
+#define STACKLESS_PROMOTE_METHOD(obj, meth) stackless = 0
+#define STACKLESS_PROMOTE_WRAPPER(descr) stackless = 0
+#define STACKLESS_PROMOTE_ALL() stackless = 0
+#define STACKLESS_RETRACT() assert(1)
+#define STACKLESS_ASSERT() assert(1)
+#endif
+#ifdef STACKLESS
 
 /******************************************************
 
@@ -476,6 +581,7 @@ PyAPI_FUNC(PyObject *) PyStackless_CallMethod_Main(PyObject *o, char *name,
 PyAPI_FUNC(PyObject *) PyStackless_CallCMethod_Main(
                     PyMethodDef *meth, PyObject *self, char *format, ...);
 
+
 #endif /* STACKLESS */
 
 #ifdef __cplusplus