Add new instruction to make generator or coroutine from current frame.

Author: markshannon

Include/opcode.h

@@ -380,6 +380,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
 #     Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
 #     Python 3.11a4 3474 (Add RESUME opcode)
+#     Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
 
 #     Python 3.12 will start with magic number 3500
 
@@ -393,7 +394,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3474).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3476).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/importlib/_bootstrap_external.py

@@ -1819,11 +1819,11 @@ def getgeneratorstate(generator):
     """
     if generator.gi_running:
         return GEN_RUNNING
+    if generator.gi_suspended:
+        return GEN_SUSPENDED
     if generator.gi_frame is None:
         return GEN_CLOSED
-    if generator.gi_frame.f_lasti == -1:
-        return GEN_CREATED
-    return GEN_SUSPENDED
+    return GEN_CREATED
 
 
 def getgeneratorlocals(generator):
@@ -1861,11 +1861,11 @@ def getcoroutinestate(coroutine):
     """
     if coroutine.cr_running:
         return CORO_RUNNING
+    if coroutine.cr_suspended:
+        return CORO_SUSPENDED
     if coroutine.cr_frame is None:
         return CORO_CLOSED
-    if coroutine.cr_frame.f_lasti == -1:
-        return CORO_CREATED
-    return CORO_SUSPENDED
+    return CORO_CREATED
 
 
 def getcoroutinelocals(coroutine):

Lib/inspect.py

@@ -94,6 +94,7 @@ def jabs_op(name, op):
 
 def_op('GET_AWAITABLE', 73)
 def_op('LOAD_ASSERTION_ERROR', 74)
+def_op('RETURN_GENERATOR', 75)
 
 def_op('LIST_TO_TUPLE', 82)
 def_op('RETURN_VALUE', 83)
@@ -152,10 +153,9 @@ def jabs_op(name, op):
 jabs_op('POP_JUMP_IF_NOT_NONE', 128)
 jabs_op('POP_JUMP_IF_NONE', 129)
 def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
-
 def_op('MAKE_FUNCTION', 132)    # Flags
 def_op('BUILD_SLICE', 133)      # Number of items
-
+jabs_op('YIELD_FROM_LOOP', 134) # Target byte offset from beginning of code
 def_op('MAKE_CELL', 135)
 hasfree.append(135)
 def_op('LOAD_CLOSURE', 136)

Lib/opcode.py

@@ -322,6 +322,7 @@ async def runner():
         self.loop.run_until_complete(runner())
 
     def test_task_repr(self):
+        self.maxDiff = None
         self.loop.set_debug(False)
 
         async def notmuch():

Lib/test/test_asyncio/test_tasks.py

@@ -28,7 +28,8 @@ def generator1(self):
 
     def generator2(self):
         try:
-            yield
+            while True:
+                yield
         except KeyboardInterrupt:
             return "PASSED"
         else:
@@ -897,7 +898,7 @@ def b():
 >>> type(i)
 <class 'generator'>
 >>> [s for s in dir(i) if not s.startswith('_')]
-['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_yieldfrom', 'send', 'throw']
+['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_suspended', 'gi_yieldfrom', 'send', 'throw']
 >>> from test.support import HAVE_DOCSTRINGS
 >>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).')
 Implement next(self).

Lib/test/test_generators.py

@@ -242,6 +242,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
                     break;
                 }
                 case JUMP_ABSOLUTE:
+                case YIELD_FROM_LOOP:
                     j = get_arg(code, i);
                     assert(j < len);
                     if (stacks[j] == UNINITIALIZED && j < i) {

Objects/frameobject.c

@@ -87,7 +87,7 @@ _PyGen_Finalize(PyObject *self)
        issue a RuntimeWarning. */
     if (gen->gi_code != NULL &&
         ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE &&
-        ((InterpreterFrame *)gen->gi_iframe)->f_lasti == -1)
+        ((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_CREATED)
     {
         _PyErr_WarnUnawaitedCoroutine((PyObject *)gen);
     }
@@ -156,7 +156,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     PyObject *result;
 
     *presult = NULL;
-    if (frame->f_lasti < 0 && arg && arg != Py_None) {
+    if (frame->f_state == FRAME_CREATED && arg && arg != Py_None) {
         const char *msg = "can't send non-None value to a "
                             "just-started generator";
         if (PyCoro_CheckExact(gen)) {
@@ -753,6 +753,15 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored))
     return PyBool_FromLong(_PyFrame_IsExecuting((InterpreterFrame *)gen->gi_iframe));
 }
 
+static PyObject *
+gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored))
+{
+    if (gen->gi_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_SUSPENDED);
+}
+
 static PyObject *
 _gen_getframe(PyGenObject *gen, const char *const name)
 {
@@ -780,6 +789,7 @@ static PyGetSetDef gen_getsetlist[] = {
      PyDoc_STR("object being iterated by yield from, or None")},
     {"gi_running", (getter)gen_getrunning, NULL, NULL},
     {"gi_frame", (getter)gen_getframe,  NULL, NULL},
+    {"gi_suspended", (getter)gen_getsuspended,  NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -901,7 +911,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func)
 }
 
 static PyObject *
-compute_cr_origin(int origin_depth);
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame);
 
 PyObject *
 _Py_MakeCoro(PyFunctionObject *func)
@@ -935,7 +945,8 @@ _Py_MakeCoro(PyFunctionObject *func)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        assert(_PyEval_GetFrame());
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()->previous);
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
@@ -1097,6 +1108,15 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored))
     return yf;
 }
 
+static PyObject *
+cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored))
+{
+    if (coro->cr_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)coro->cr_iframe)->f_state == FRAME_SUSPENDED);
+}
+
 static PyObject *
 cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored))
 {
@@ -1122,6 +1142,7 @@ static PyGetSetDef coro_getsetlist[] = {
      PyDoc_STR("object being awaited on, or None")},
     {"cr_running", (getter)cr_getrunning, NULL, NULL},
     {"cr_frame", (getter)cr_getframe, NULL, NULL},
+    {"cr_suspended", (getter)cr_getsuspended, NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -1299,9 +1320,9 @@ PyTypeObject _PyCoroWrapper_Type = {
 };
 
 static PyObject *
-compute_cr_origin(int origin_depth)
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame)
 {
-    InterpreterFrame *frame = _PyEval_GetFrame();
+    InterpreterFrame *frame = current_frame;
     /* First count how many frames we have */
     int frame_count = 0;
     for (; frame && frame_count < origin_depth; ++frame_count) {
@@ -1313,7 +1334,10 @@ compute_cr_origin(int origin_depth)
     if (cr_origin == NULL) {
         return NULL;
     }
-    frame = _PyEval_GetFrame();
+    frame = current_frame;
+    if (frame) {
+
+    }
     for (int i = 0; i < frame_count; ++i) {
         PyCodeObject *code = frame->f_code;
         PyObject *frameinfo = Py_BuildValue("OiO",
@@ -1345,7 +1369,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame());
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);