Skip to content

Commit

Permalink
[2.7] bpo-10544: Deprecate "yield" in comprehensions and generator ex…
Browse files Browse the repository at this point in the history 
…pressions in Py3k mode. (GH-4579) (#4676)
  • Loading branch information
@serhiy-storchaka
serhiy-storchaka authored Dec 2, 2017
1 parent be6b74c commit 65d1887
Show file tree
Hide file tree
Showing 5 changed files with 120 additions and 28 deletions.
4 changes: 2 additions & 2 deletions Lib/test/support/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -847,8 +847,8 @@ def make_bad_fd():
file.close()
unlink(TESTFN)

def check_syntax_error(testcase, statement, lineno=None, offset=None):
with testcase.assertRaises(SyntaxError) as cm:
def check_syntax_error(testcase, statement, errtext='', lineno=None, offset=None):
with testcase.assertRaisesRegexp(SyntaxError, errtext) as cm:
compile(statement, '<test string>', 'exec')
err = cm.exception
if lineno is not None:
Expand Down
12 changes: 3 additions & 9 deletions Lib/test/test_generators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1524,13 +1524,7 @@ def printsolution(self, x):
[None]
An obscene abuse of a yield expression within a generator expression:
>>> list((yield 21) for i in range(4))
[21, None, 21, None, 21, None, 21, None]
And a more sane, but still weird usage:
Yield is allowed only in the outermost iterable in generator expression:
>>> def f(): list(i for i in [(yield 26)])
>>> type(f())
Expand Down Expand Up @@ -1571,7 +1565,7 @@ def printsolution(self, x):
>>> def f(): return lambda x=(yield): 1
Traceback (most recent call last):
...
SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[22]>, line 1)
SyntaxError: 'return' with argument inside generator (<doctest test.test_generators.__test__.coroutine[21]>, line 1)
>>> def f(): x = yield = y
Traceback (most recent call last):
Expand Down Expand Up @@ -1784,7 +1778,7 @@ def printsolution(self, x):
>>> type(f())
<type 'generator'>
>>> def f(): x=(i for i in (yield) if (yield))
>>> def f(): x=(i for i in (yield) if i)
>>> type(f())
<type 'generator'>
Expand Down
40 changes: 40 additions & 0 deletions Lib/test/test_grammar.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,46 @@ def g2(): return 1
def testYield(self):
check_syntax_error(self, "class foo:yield 1")

def test_yield_in_comprehensions(self):
# Check yield in comprehensions
def g(): [x for x in [(yield 1)]]

def check(code, warntext):
with check_py3k_warnings((warntext, DeprecationWarning)):
compile(code, '<test string>', 'exec')
if sys.py3kwarning:
import warnings
with warnings.catch_warnings():
warnings.filterwarnings('error', category=DeprecationWarning)
with self.assertRaises(SyntaxError) as cm:
compile(code, '<test string>', 'exec')
self.assertIn(warntext, str(cm.exception))

check("def g(): [(yield x) for x in ()]",
"'yield' inside list comprehension")
check("def g(): [x for x in () if not (yield x)]",
"'yield' inside list comprehension")
check("def g(): [y for x in () for y in [(yield x)]]",
"'yield' inside list comprehension")
check("def g(): {(yield x) for x in ()}",
"'yield' inside set comprehension")
check("def g(): {(yield x): x for x in ()}",
"'yield' inside dict comprehension")
check("def g(): {x: (yield x) for x in ()}",
"'yield' inside dict comprehension")
check("def g(): ((yield x) for x in ())",
"'yield' inside generator expression")
with check_py3k_warnings(("'yield' inside list comprehension",
DeprecationWarning)):
check_syntax_error(self, "class C: [(yield x) for x in ()]")
check("class C: ((yield x) for x in ())",
"'yield' inside generator expression")
with check_py3k_warnings(("'yield' inside list comprehension",
DeprecationWarning)):
check_syntax_error(self, "[(yield x) for x in ()]")
check("((yield x) for x in ())",
"'yield' inside generator expression")

def testRaise(self):
# 'raise' test [',' test]
try: raise RuntimeError, 'just testing'
Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Yield expressions are now deprecated in comprehensions and generator
expressions when checking Python 3 compatibility. They are still
permitted in the definition of the outermost iterable, as that is
evaluated directly in the enclosing scope.
88 changes: 71 additions & 17 deletions Python/symtable.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,12 +162,14 @@ PyTypeObject PySTEntry_Type = {
};

static int symtable_analyze(struct symtable *st);
static int symtable_warn(struct symtable *st, char *msg, int lineno);
static int symtable_warn(struct symtable *st,
PyObject *warn, const char *msg, int lineno);
static int symtable_enter_block(struct symtable *st, identifier name,
_Py_block_ty block, void *ast, int lineno);
static int symtable_exit_block(struct symtable *st, void *ast);
static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
static int symtable_visit_expr(struct symtable *st, expr_ty s);
static int symtable_visit_listcomp(struct symtable *st, expr_ty e);
static int symtable_visit_genexp(struct symtable *st, expr_ty s);
static int symtable_visit_setcomp(struct symtable *st, expr_ty e);
static int symtable_visit_dictcomp(struct symtable *st, expr_ty e);
Expand Down Expand Up @@ -796,14 +798,18 @@ symtable_analyze(struct symtable *st)


static int
symtable_warn(struct symtable *st, char *msg, int lineno)
symtable_warn(struct symtable *st, PyObject *warn, const char *msg, int lineno)
{
if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, st->st_filename,
lineno, NULL, NULL) < 0) {
if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) {
if (lineno < 0) {
lineno = st->st_cur->ste_lineno;
}
if (PyErr_WarnExplicit(warn, msg, st->st_filename, lineno, NULL, NULL) < 0) {
if (PyErr_ExceptionMatches(warn)) {
/* Replace the warning exception with a SyntaxError
to get a more accurate error report */
PyErr_Clear();
PyErr_SetString(PyExc_SyntaxError, msg);
PyErr_SyntaxLocation(st->st_filename,
st->st_cur->ste_lineno);
PyErr_SyntaxLocation(st->st_filename, lineno);
}
return 0;
}
Expand Down Expand Up @@ -1153,7 +1159,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
PyOS_snprintf(buf, sizeof(buf),
GLOBAL_AFTER_USE,
c_name);
if (!symtable_warn(st, buf, s->lineno))
if (!symtable_warn(st, PyExc_SyntaxWarning, buf, s->lineno))
return 0;
}
if (!symtable_add_def(st, name, DEF_GLOBAL))
Expand Down Expand Up @@ -1221,8 +1227,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
VISIT_SEQ(st, expr, e->v.Set.elts);
break;
case ListComp_kind:
VISIT(st, expr, e->v.ListComp.elt);
VISIT_SEQ(st, comprehension, e->v.ListComp.generators);
if (!symtable_visit_listcomp(st, e))
return 0;
break;
case GeneratorExp_kind:
if (!symtable_visit_genexp(st, e))
Expand Down Expand Up @@ -1420,12 +1426,11 @@ symtable_visit_alias(struct symtable *st, alias_ty a)
return r;
}
else {
if (st->st_cur->ste_type != ModuleBlock) {
int lineno = st->st_cur->ste_lineno;
if (!symtable_warn(st, IMPORT_STAR_WARNING, lineno)) {
Py_DECREF(store_name);
return 0;
}
if (st->st_cur->ste_type != ModuleBlock &&
!symtable_warn(st, PyExc_SyntaxWarning, IMPORT_STAR_WARNING, -1))
{
Py_DECREF(store_name);
return 0;
}
st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR;
Py_DECREF(store_name);
Expand Down Expand Up @@ -1509,7 +1514,10 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
!symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) {
return 0;
}
st->st_cur->ste_generator = is_generator;
/* In order to check for yield expressions under '-3', we clear
the generator flag, and restore it at the end */
is_generator |= st->st_cur->ste_generator;
st->st_cur->ste_generator = 0;
/* Outermost iter is received as an argument */
if (!symtable_implicit_arg(st, 0)) {
symtable_exit_block(st, (void *)e);
Expand All @@ -1527,9 +1535,55 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
if (value)
VISIT_IN_BLOCK(st, expr, value, (void*)e);
VISIT_IN_BLOCK(st, expr, elt, (void*)e);
if (Py_Py3kWarningFlag && st->st_cur->ste_generator) {
const char *msg = (
(e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
(e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
"'yield' inside generator expression");
if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) {
symtable_exit_block(st, (void *)e);
return 0;
}
}
st->st_cur->ste_generator |= is_generator;
return symtable_exit_block(st, (void *)e);
}

static int
symtable_visit_listcomp(struct symtable *st, expr_ty e)
{
asdl_seq *generators = e->v.ListComp.generators;
int i, is_generator;
/* In order to check for yield expressions under '-3', we clear
the generator flag, and restore it at the end */
is_generator = st->st_cur->ste_generator;
st->st_cur->ste_generator = 0;
VISIT(st, expr, e->v.ListComp.elt);
for (i = 0; i < asdl_seq_LEN(generators); i++) {
comprehension_ty lc = (comprehension_ty)asdl_seq_GET(generators, i);
VISIT(st, expr, lc->target);
if (i == 0 && !st->st_cur->ste_generator) {
/* 'yield' in the outermost iterator doesn't cause a warning */
VISIT(st, expr, lc->iter);
is_generator |= st->st_cur->ste_generator;
st->st_cur->ste_generator = 0;
}
else {
VISIT(st, expr, lc->iter);
}
VISIT_SEQ(st, expr, lc->ifs);
}

if (Py_Py3kWarningFlag && st->st_cur->ste_generator) {
const char *msg = "'yield' inside list comprehension";
if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) {
return 0;
}
}
st->st_cur->ste_generator |= is_generator;
return 1;
}

static int
symtable_visit_genexp(struct symtable *st, expr_ty e)
{
Expand Down

0 comments on commit 65d1887

Please sign in to comment.