Add option to compile() to accept top-level await features.

In some narrow but rate case; it is useful to be allow to compile code
with top-level await features; for example when writing a REPL.
This attempt to allow that.

Author: Matthias Bussonnier

Doc/library/functions.rst

@@ -257,6 +257,12 @@ are always available.  They are listed here in alphabetical order.
    can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on
    the :class:`~__future__._Feature` instance in the :mod:`__future__` module.
 
+   The optional argument *flags* also control wether the compiled source is
+   allowed to contain top-level ``await``, ``async for`` and ``async with``.
+   When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the the return code
+   object has ``CO_COROUTINE`` set in ``co_code``, and can be interacitvely
+   executed via ``yield from eval(code_object)``.
+
    The argument *optimize* specifies the optimization level of the compiler; the
    default value of ``-1`` selects the optimization level of the interpreter as
    given by :option:`-O` options.  Explicit levels are ``0`` (no optimization;
@@ -290,6 +296,10 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
+   .. versionadded:: 3.8
+      *flags* can now be used to accept source that would contain a top-level
+      ``await``, ``async for`` or ``async with``.
+
 
 .. class:: complex([real[, imag]])
 

Include/compile.h

@@ -23,7 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
 #define PyCF_ONLY_AST 0x0400
 #define PyCF_IGNORE_COOKIE 0x0800
 #define PyCF_TYPE_COMMENTS 0x1000
-#define PyCF_AWAIT 0x2000
+#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
 
 #ifndef Py_LIMITED_API
 typedef struct {

Lib/test/test_builtin.py

@@ -18,6 +18,8 @@
 import unittest
 import warnings
 from contextlib import ExitStack
+from inspect import CO_COROUTINE
+from textwrap import dedent
 from operator import neg
 from test.support import (
     EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink)
@@ -358,6 +360,33 @@ def f(): """doc"""
                 rv = ns['f']()
                 self.assertEqual(rv, tuple(expected))
 
+    def test_compile_top_level_await(self):
+        for mode in ('single', 'exec'):
+            for num,code_sample in enumerate(['''await sleep(0)''',
+                                '''async for i in range(10):
+                                       print(i)''',
+                                '''async with asyncio.Lock() as l:
+                                       pass''']):
+                source = dedent(code_sample)
+                with self.assertRaises(SyntaxError,
+                                  msg='source={!r} mode={!r})'.format(source, mode)):
+                    compile(source, '?' , mode)
+                co = compile(source, '?', mode, flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+                self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE)
+
+    def test_compile_async_generator(self):
+        co = compile(dedent("""async def ticker():
+                for i in range(10):
+                    yield i
+                    await asyncio.sleep(0)"""), '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)
+        glob = {}
+        exec(co, glob)
+
+        from types import AsyncGeneratorType
+
+        self.assertEqual(type(glob['ticker']()), AsyncGeneratorType)
+
+
     def test_delattr(self):
         sys.spam = 1
         delattr(sys, 'spam')

Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst

@@ -0,0 +1 @@
+The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag,  which allow to compile sources that  contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL.

Parser/asdl_c.py

@@ -1000,6 +1000,8 @@ def visitModule(self, mod):
         self.emit("if (!m) return NULL;", 1)
         self.emit("d = PyModule_GetDict(m);", 1)
         self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
+        self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1)
+        self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
         self.emit("return NULL;", 2)
         self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1)

Python/Python-ast.c

@@ -2608,10 +2608,13 @@ static int
 compiler_async_for(struct compiler *c, stmt_ty s)
 {
     basicblock *start, *except, *end;
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
         return compiler_error(c, "'async for' outside async function");
     }
 
+
     start = compiler_new_block(c);
     except = compiler_new_block(c);
     end = compiler_new_block(c);
@@ -4550,10 +4553,10 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos)
     withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos);
 
     assert(s->kind == AsyncWith_kind);
-    if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) {
-        if (!(c->c_flags->cf_flags & 0x2000)) {
-            return compiler_error(c, "'async with' outside async function");
-        }
+    if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+        c->u->u_ste->ste_coroutine = 1;
+    } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){
+        return compiler_error(c, "'async with' outside async function");
     }
 
     block = compiler_new_block(c);
@@ -4761,18 +4764,17 @@ compiler_visit_expr1(struct compiler *c, expr_ty e)
         ADDOP(c, YIELD_FROM);
         break;
     case Await_kind:
-        if (c->u->u_ste->ste_type != FunctionBlock) {
-            if (!(c->c_flags->cf_flags & 0x2000)) {
+        if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){
+            c->u->u_ste->ste_coroutine = 1;
+        } else {
+            if (c->u->u_ste->ste_type != FunctionBlock){
                 return compiler_error(c, "'await' outside function");
             }
 
-            c->u->u_ste->ste_coroutine = 1;
-        }
-
-        if (!(c->c_flags->cf_flags & 0x2000)) {
-            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
-                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION)
+            if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && 
+                    c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){
                 return compiler_error(c, "'await' outside async function");
+            }
         }
 
         VISIT(c, expr, e->v.Await.value);
@@ -5708,7 +5710,7 @@ compute_code_flags(struct compiler *c)
     /* (Only) inherit compilerflags in PyCF_MASK */
     flags |= (c->c_flags->cf_flags & PyCF_MASK);
 
-    if ((c->c_flags->cf_flags & 0x2000) && c->u->u_ste->ste_coroutine) {
+    if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) && ste->ste_coroutine && !ste->ste_generator) {
         flags |= CO_COROUTINE;
     }