Skip to content

Commit dc40226

Browse files
JelleZijlstraJelleZijlstra
authored
[3.12] gh-119311: Fix name mangling with PEP 695 generic classes (#119464) (#119644)
* [3.12] gh-119311: Fix name mangling with PEP 695 generic classes (#119464) Fixes #119311. Fixes #119395. (cherry picked from commit a9a74da)
1 parent 008f9dd commit dc40226

File tree

7 files changed

+152
-14
lines changed

7 files changed

+152
-14
lines changed

‎Doc/data/python3.12.abi

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22611,7 +22611,7 @@
2261122611
<var-decl name='recursion_limit' type-id='type-id-8' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='50' column='1'/>
2261222612
</data-member>
2261322613
</class-decl>
22614-
<class-decl name='_symtable_entry' size-in-bits='960' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='53' column='1' id='type-id-1397'>
22614+
<class-decl name='_symtable_entry' size-in-bits='1024' is-struct='yes' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='53' column='1' id='type-id-1397'>
2261522615
<data-member access='public' layout-offset-in-bits='0'>
2261622616
<var-decl name='ob_base' type-id='type-id-345' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='54' column='1'/>
2261722617
</data-member>
@@ -22702,6 +22702,9 @@
2270222702
<data-member access='public' layout-offset-in-bits='896'>
2270322703
<var-decl name='ste_table' type-id='type-id-209' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='89' column='1'/>
2270422704
</data-member>
22705+
<data-member access='public' layout-offset-in-bits='960'>
22706+
<var-decl name='ste_mangled_names' type-id='type-id-2' visibility='default' filepath='./Include/internal/pycore_symtable.h' line='90' column='1'/>
22707+
</data-member>
2270522708
</class-decl>
2270622709
<typedef-decl name='PySTEntryObject' type-id='type-id-1397' filepath='./Include/internal/pycore_symtable.h' line='90' column='1' id='type-id-1398'/>
2270722710
<typedef-decl name='basicblock' type-id='type-id-1386' filepath='Python/compile.c' line='90' column='1' id='type-id-1399'/>

‎Include/internal/pycore_symtable.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ typedef struct _symtable_entry {
8787
int ste_opt_lineno; /* lineno of last exec or import * */
8888
int ste_opt_col_offset; /* offset of last exec or import * */
8989
struct symtable *ste_table;
90+
PyObject *ste_mangled_names; /* set of names for which mangling should be applied */
9091
} PySTEntryObject;
9192

9293
extern PyTypeObject PySTEntry_Type;
@@ -105,6 +106,7 @@ PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
105106

106107
extern void _PySymtable_Free(struct symtable *);
107108

109+
extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
108110
extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
109111

110112
/* Flags for def-use information */

‎Lib/test/test_type_params.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,100 @@ def meth[__U](self, arg: __T, arg2: __U):
760760

761761
self.assertEqual(Foo.Alias.__value__, (T, V))
762762

763+
def test_no_leaky_mangling_in_module(self):
764+
ns = run_code("""
765+
__before = "before"
766+
class X[T]: pass
767+
__after = "after"
768+
""")
769+
self.assertEqual(ns["__before"], "before")
770+
self.assertEqual(ns["__after"], "after")
771+
772+
def test_no_leaky_mangling_in_function(self):
773+
ns = run_code("""
774+
def f():
775+
class X[T]: pass
776+
_X_foo = 2
777+
__foo = 1
778+
assert locals()['__foo'] == 1
779+
return __foo
780+
""")
781+
self.assertEqual(ns["f"](), 1)
782+
783+
def test_no_leaky_mangling_in_class(self):
784+
ns = run_code("""
785+
class Outer:
786+
__before = "before"
787+
class Inner[T]:
788+
__x = "inner"
789+
__after = "after"
790+
""")
791+
Outer = ns["Outer"]
792+
self.assertEqual(Outer._Outer__before, "before")
793+
self.assertEqual(Outer.Inner._Inner__x, "inner")
794+
self.assertEqual(Outer._Outer__after, "after")
795+
796+
def test_no_mangling_in_bases(self):
797+
ns = run_code("""
798+
class __Base:
799+
def __init_subclass__(self, **kwargs):
800+
self.kwargs = kwargs
801+
802+
class Derived[T](__Base, __kwarg=1):
803+
pass
804+
""")
805+
Derived = ns["Derived"]
806+
self.assertEqual(Derived.__bases__, (ns["__Base"], Generic))
807+
self.assertEqual(Derived.kwargs, {"__kwarg": 1})
808+
809+
def test_no_mangling_in_nested_scopes(self):
810+
ns = run_code("""
811+
from test.test_type_params import make_base
812+
813+
class __X:
814+
pass
815+
816+
class Y[T: __X](
817+
make_base(lambda: __X),
818+
# doubly nested scope
819+
make_base(lambda: (lambda: __X)),
820+
# list comprehension
821+
make_base([__X for _ in (1,)]),
822+
# genexp
823+
make_base(__X for _ in (1,)),
824+
):
825+
pass
826+
""")
827+
Y = ns["Y"]
828+
T, = Y.__type_params__
829+
self.assertIs(T.__bound__, ns["__X"])
830+
base0 = Y.__bases__[0]
831+
self.assertIs(base0.__arg__(), ns["__X"])
832+
base1 = Y.__bases__[1]
833+
self.assertIs(base1.__arg__()(), ns["__X"])
834+
base2 = Y.__bases__[2]
835+
self.assertEqual(base2.__arg__, [ns["__X"]])
836+
base3 = Y.__bases__[3]
837+
self.assertEqual(list(base3.__arg__), [ns["__X"]])
838+
839+
def test_type_params_are_mangled(self):
840+
ns = run_code("""
841+
from test.test_type_params import make_base
842+
843+
class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)):
844+
param = __T
845+
""")
846+
Foo = ns["Foo"]
847+
T, U = Foo.__type_params__
848+
self.assertEqual(T.__name__, "__T")
849+
self.assertEqual(U.__name__, "__U")
850+
self.assertIs(U.__bound__, T)
851+
self.assertIs(Foo.param, T)
852+
853+
base1, base2, *_ = Foo.__bases__
854+
self.assertIs(base1.__arg__, T)
855+
self.assertIs(base2.__arg__(), T)
856+
763857

764858
class TypeParamsComplexCallsTest(unittest.TestCase):
765859
def test_defaults(self):

‎Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix bug where names appearing after a generic class are mangled as if they
2+
are in the generic class.

‎Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix bug where names are unexpectedly mangled in the bases of generic
2+
classes.

‎Python/compile.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1071,7 +1071,7 @@ static int
10711071
compiler_addop_name(struct compiler_unit *u, location loc,
10721072
int opcode, PyObject *dict, PyObject *o)
10731073
{
1074-
PyObject *mangled = _Py_Mangle(u->u_private, o);
1074+
PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o);
10751075
if (!mangled) {
10761076
return ERROR;
10771077
}
@@ -1889,7 +1889,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc,
18891889
arg_ty arg = asdl_seq_GET(kwonlyargs, i);
18901890
expr_ty default_ = asdl_seq_GET(kw_defaults, i);
18911891
if (default_) {
1892-
PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
1892+
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg);
18931893
if (!mangled) {
18941894
goto error;
18951895
}
@@ -1946,7 +1946,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
19461946
if (!annotation) {
19471947
return SUCCESS;
19481948
}
1949-
PyObject *mangled = _Py_Mangle(c->u->u_private, id);
1949+
PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id);
19501950
if (!mangled) {
19511951
return ERROR;
19521952
}
@@ -2554,7 +2554,6 @@ compiler_class(struct compiler *c, stmt_ty s)
25542554
asdl_type_param_seq *type_params = s->v.ClassDef.type_params;
25552555
int is_generic = asdl_seq_LEN(type_params) > 0;
25562556
if (is_generic) {
2557-
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
25582557
ADDOP(c, loc, PUSH_NULL);
25592558
PyObject *type_params_name = PyUnicode_FromFormat("<generic parameters of %U>",
25602559
s->v.ClassDef.name);
@@ -2567,6 +2566,7 @@ compiler_class(struct compiler *c, stmt_ty s)
25672566
return ERROR;
25682567
}
25692568
Py_DECREF(type_params_name);
2569+
Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
25702570
RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params));
25712571
_Py_DECLARE_STR(type_params, ".type_params");
25722572
RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store));
@@ -4123,7 +4123,7 @@ compiler_nameop(struct compiler *c, location loc,
41234123
return ERROR;
41244124
}
41254125

4126-
mangled = _Py_Mangle(c->u->u_private, name);
4126+
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name);
41274127
if (!mangled) {
41284128
return ERROR;
41294129
}
@@ -6383,7 +6383,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
63836383
VISIT(c, expr, s->v.AnnAssign.annotation);
63846384
}
63856385
ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
6386-
mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
6386+
mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
63876387
ADDOP_LOAD_CONST_NEW(c, loc, mangled);
63886388
ADDOP(c, loc, STORE_SUBSCR);
63896389
}

‎Python/symtable.c

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
101101
ste->ste_children = NULL;
102102

103103
ste->ste_directives = NULL;
104+
ste->ste_mangled_names = NULL;
104105

105106
ste->ste_type = block;
106107
ste->ste_nested = 0;
@@ -164,6 +165,7 @@ ste_dealloc(PySTEntryObject *ste)
164165
Py_XDECREF(ste->ste_varnames);
165166
Py_XDECREF(ste->ste_children);
166167
Py_XDECREF(ste->ste_directives);
168+
Py_XDECREF(ste->ste_mangled_names);
167169
PyObject_Free(ste);
168170
}
169171

@@ -1231,6 +1233,11 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
12311233
if (prev) {
12321234
ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
12331235
}
1236+
/* No need to inherit ste_mangled_names in classes, where all names
1237+
* are mangled. */
1238+
if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
1239+
ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
1240+
}
12341241
/* The entry is owned by the stack. Borrow it for st_cur. */
12351242
Py_DECREF(ste);
12361243
st->st_cur = ste;
@@ -1256,7 +1263,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
12561263
static long
12571264
symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
12581265
{
1259-
PyObject *mangled = _Py_Mangle(st->st_private, name);
1266+
PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name);
12601267
if (!mangled)
12611268
return 0;
12621269
long ret = _PyST_GetSymbol(ste, mangled);
@@ -1277,8 +1284,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s
12771284
PyObject *o;
12781285
PyObject *dict;
12791286
long val;
1280-
PyObject *mangled = _Py_Mangle(st->st_private, name);
1281-
1287+
PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
12821288

12831289
if (!mangled)
12841290
return 0;
@@ -1367,6 +1373,11 @@ static int
13671373
symtable_add_def(struct symtable *st, PyObject *name, int flag,
13681374
int lineno, int col_offset, int end_lineno, int end_col_offset)
13691375
{
1376+
if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) {
1377+
if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) {
1378+
return 0;
1379+
}
1380+
}
13701381
return symtable_add_def_helper(st, name, flag, st->st_cur,
13711382
lineno, col_offset, end_lineno, end_col_offset);
13721383
}
@@ -1401,7 +1412,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name,
14011412
lineno, col_offset, end_lineno, end_col_offset)) {
14021413
return 0;
14031414
}
1404-
st->st_private = name;
14051415
// This is used for setting the generic base
14061416
_Py_DECLARE_STR(generic_base, ".generic_base");
14071417
if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL,
@@ -1490,7 +1500,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno,
14901500
if (!st->st_cur->ste_directives)
14911501
return 0;
14921502
}
1493-
mangled = _Py_Mangle(st->st_private, name);
1503+
mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
14941504
if (!mangled)
14951505
return 0;
14961506
data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
@@ -1566,13 +1576,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
15661576
VISIT_QUIT(st, 0);
15671577
if (s->v.ClassDef.decorator_list)
15681578
VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1579+
tmp = st->st_private;
15691580
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
15701581
if (!symtable_enter_type_param_block(st, s->v.ClassDef.name,
15711582
(void *)s->v.ClassDef.type_params,
15721583
false, false, s->kind,
15731584
LOCATION(s))) {
15741585
VISIT_QUIT(st, 0);
15751586
}
1587+
st->st_private = s->v.ClassDef.name;
1588+
st->st_cur->ste_mangled_names = PySet_New(NULL);
1589+
if (!st->st_cur->ste_mangled_names) {
1590+
VISIT_QUIT(st, 0);
1591+
}
15761592
VISIT_SEQ(st, type_param, s->v.ClassDef.type_params);
15771593
}
15781594
VISIT_SEQ(st, expr, s->v.ClassDef.bases);
@@ -1581,7 +1597,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
15811597
(void *)s, s->lineno, s->col_offset,
15821598
s->end_lineno, s->end_col_offset))
15831599
VISIT_QUIT(st, 0);
1584-
tmp = st->st_private;
15851600
st->st_private = s->v.ClassDef.name;
15861601
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
15871602
if (!symtable_add_def(st, &_Py_ID(__type_params__),
@@ -1595,13 +1610,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
15951610
}
15961611
}
15971612
VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1598-
st->st_private = tmp;
15991613
if (!symtable_exit_block(st))
16001614
VISIT_QUIT(st, 0);
16011615
if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
16021616
if (!symtable_exit_block(st))
16031617
VISIT_QUIT(st, 0);
16041618
}
1619+
st->st_private = tmp;
16051620
break;
16061621
}
16071622
case TypeAlias_kind: {
@@ -2663,6 +2678,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
26632678
return st;
26642679
}
26652680

2681+
PyObject *
2682+
_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
2683+
{
2684+
/* Special case for type parameter blocks around generic classes:
2685+
* we want to mangle type parameter names (so a type param with a private
2686+
* name can be used inside the class body), but we don't want to mangle
2687+
* any other names that appear within the type parameter scope.
2688+
*/
2689+
if (ste->ste_mangled_names != NULL) {
2690+
int result = PySet_Contains(ste->ste_mangled_names, name);
2691+
if (result < 0) {
2692+
return NULL;
2693+
}
2694+
if (result == 0) {
2695+
return Py_NewRef(name);
2696+
}
2697+
}
2698+
return _Py_Mangle(privateobj, name);
2699+
}
2700+
26662701
PyObject *
26672702
_Py_Mangle(PyObject *privateobj, PyObject *ident)
26682703
{

0 commit comments

Comments
 (0)