bpo-10544: Disallow "yield" in comprehensions and generator expressions.

Author: serhiy-storchaka

Doc/reference/expressions.rst

@@ -186,6 +186,9 @@ each time the innermost block is reached.
 Note that the comprehension is executed in a separate scope, so names assigned
 to in the target list don't "leak" into the enclosing scope.
 
+Yield expressions are not allowed in comprehensions
+except the expression for the outermost iterable.
+
 Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for`
 clause may be used to iterate over a :term:`asynchronous iterator`.
 A comprehension in an :keyword:`async def` function may consist of either a
@@ -326,6 +329,9 @@ range(10) for y in bar(x))``.
 The parentheses can be omitted on calls with only one argument.  See section
 :ref:`calls` for details.
 
+Yield expressions are not allowed in generator expressions
+except the expression for the outermost iterable.
+
 If a generator expression contains either :keyword:`async for`
 clauses or :keyword:`await` expressions it is called an
 :dfn:`asynchronous generator expression`.  An asynchronous generator
@@ -364,6 +370,9 @@ coroutine function to be an asynchronous generator. For example::
     async def agen(): # defines an asynchronous generator function (PEP 525)
         yield 123
 
+Yield expressions are not allowed in comprehensions and generator expressions
+except the expression for the outermost iterable.
+
 Generator functions are described below, while asynchronous generator
 functions are described separately in section
 :ref:`asynchronous-generator-functions`.

Doc/whatsnew/3.7.rst

@@ -673,6 +673,12 @@ Changes in Python behavior
   parentheses can be omitted only on calls.
   (Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.)
 
+* Yield expressions now are not allowed in comprehensions and generator
+  expressions except the expression for the outermost iterable since
+  comprehensions and generator expressions are implemented using
+  an internal function.
+  (Contributed by Serhiy Storchaka in :issue:`10544`.)
+
 
 Changes in the Python API
 -------------------------

Lib/test/test_generators.py

@@ -1834,7 +1834,9 @@ def printsolution(self, x):
 An obscene abuse of a yield expression within a generator expression:
 
 >>> list((yield 21) for i in range(4))
-[21, None, 21, None, 21, None, 21, None]
+Traceback (most recent call last):
+  ...
+SyntaxError: 'yield' inside generator expression
 
 And a more sane, but still weird usage:
 
@@ -2106,10 +2108,6 @@ def printsolution(self, x):
 >>> type(f())
 <class 'generator'>
 
->>> def f(): x=(i for i in (yield) if (yield))
->>> type(f())
-<class 'generator'>
-
 >>> def f(d): d[(yield "a")] = d[(yield "b")] = 27
 >>> data = [1,2]
 >>> g = f(data)

Lib/test/test_grammar.py

@@ -841,6 +841,21 @@ def g(): f((yield from ()), 1)
         # Check annotation refleak on SyntaxError
         check_syntax_error(self, "def g(a:(yield)): pass")
 
+    def test_yield_in_comprehensions(self):
+        # Check yield in comprehensions
+        def g(): [x for x in [(yield 1)]]
+        def g(): [x for x in [(yield from ())]]
+        check_syntax_error(self, "def g(): [(yield x) for x in ()]")
+        check_syntax_error(self, "def g(): [x for x in () if not (yield x)]")
+        check_syntax_error(self, "def g(): [y for x in () for y in [(yield x)]]")
+        check_syntax_error(self, "def g(): {(yield x) for x in ()}")
+        check_syntax_error(self, "def g(): {(yield x): x for x in ()}")
+        check_syntax_error(self, "def g(): {x: (yield x) for x in ()}")
+        check_syntax_error(self, "def g(): ((yield x) for x in ())")
+        check_syntax_error(self, "def g(): [(yield from x) for x in ()]")
+        check_syntax_error(self, "class C: [(yield x) for x in ()]")
+        check_syntax_error(self, "[(yield x) for x in ()]")
+
     def test_raise(self):
         # 'raise' test [',' test]
         try: raise RuntimeError('just testing')

Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst

@@ -0,0 +1,2 @@
+Yield expressions now are disallowed in comprehensions and generator
+expressions except the expression for the outermost iterable.

Python/symtable.c

@@ -1734,7 +1734,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
                               e->lineno, e->col_offset)) {
         return 0;
     }
-    st->st_cur->ste_generator = is_generator;
     if (outermost->is_async) {
         st->st_cur->ste_coroutine = 1;
     }
@@ -1754,6 +1753,17 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
     if (value)
         VISIT(st, expr, value);
     VISIT(st, expr, elt);
+    if (st->st_cur->ste_generator) {
+        PyErr_SetString(PyExc_SyntaxError,
+            (e->kind == ListComp_kind) ? "'yield' inside list comprehension" :
+            (e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
+            (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
+            "'yield' inside generator expression");
+        PyErr_SyntaxLocationObject(st->st_filename,
+                                   st->st_cur->ste_lineno,
+                                   st->st_cur->ste_col_offset);
+    }
+    st->st_cur->ste_generator = is_generator;
     return symtable_exit_block(st, (void *)e);
 }