Skip to content

Commit 3368f3c

Browse files
eduardo-elizondoDinoV
eduardo-elizondo
authored and
DinoV
committed
bpo-38140: Make dict and weakref offsets opaque for C heap types (#16076)
* Make dict and weakref offsets opaque for C heap types * Add news
1 parent 079931d commit 3368f3c

File tree

8 files changed

+188
-8
lines changed

8 files changed

+188
-8
lines changed

‎Doc/c-api/type.rst

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,9 +187,8 @@ The following functions and structs are used to create
187187
* :c:member:`~PyTypeObject.tp_cache`
188188
* :c:member:`~PyTypeObject.tp_subclasses`
189189
* :c:member:`~PyTypeObject.tp_weaklist`
190+
* :c:member:`~PyTypeObject.tp_vectorcall`
190191
* :c:member:`~PyTypeObject.tp_print`
191-
* :c:member:`~PyTypeObject.tp_weaklistoffset`
192-
* :c:member:`~PyTypeObject.tp_dictoffset`
193192
* :c:member:`~PyBufferProcs.bf_getbuffer`
194193
* :c:member:`~PyBufferProcs.bf_releasebuffer`
195194

‎Lib/test/test_capi.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import threading
1313
import time
1414
import unittest
15+
import weakref
1516
from test import support
1617
from test.support import MISSING_C_DOCSTRINGS
1718
from test.support.script_helper import assert_python_failure, assert_python_ok
@@ -437,6 +438,32 @@ def __del__(self):
437438
# Test that subtype_dealloc decref the newly assigned __class__ only once
438439
self.assertEqual(new_type_refcnt, sys.getrefcount(A))
439440

441+
def test_heaptype_with_dict(self):
442+
inst = _testcapi.HeapCTypeWithDict()
443+
inst.foo = 42
444+
self.assertEqual(inst.foo, 42)
445+
self.assertEqual(inst.dictobj, inst.__dict__)
446+
self.assertEqual(inst.dictobj, {"foo": 42})
447+
448+
inst = _testcapi.HeapCTypeWithDict()
449+
self.assertEqual({}, inst.__dict__)
450+
451+
def test_heaptype_with_negative_dict(self):
452+
inst = _testcapi.HeapCTypeWithNegativeDict()
453+
inst.foo = 42
454+
self.assertEqual(inst.foo, 42)
455+
self.assertEqual(inst.dictobj, inst.__dict__)
456+
self.assertEqual(inst.dictobj, {"foo": 42})
457+
458+
inst = _testcapi.HeapCTypeWithNegativeDict()
459+
self.assertEqual({}, inst.__dict__)
460+
461+
def test_heaptype_with_weakref(self):
462+
inst = _testcapi.HeapCTypeWithWeakref()
463+
ref = weakref.ref(inst)
464+
self.assertEqual(ref(), inst)
465+
self.assertEqual(inst.weakreflist, ref)
466+
440467
def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self):
441468
subclass_instance = _testcapi.HeapCTypeSubclass()
442469
type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass)

‎Misc/NEWS.d/next/C API/2019-09-13-01-24-47.bpo-38140.y59qaO.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Make dict and weakref offsets opaque for C heap types by passing the offsets
2+
through PyMemberDef

‎Modules/_struct.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2024,7 +2024,7 @@ static struct PyMethodDef s_methods[] = {
20242024
};
20252025

20262026
static PyMemberDef s_members[] = {
2027-
{"__weaklistoffset__", T_NONE, offsetof(PyStructObject, weakreflist), READONLY},
2027+
{"__weaklistoffset__", T_PYSSIZET, offsetof(PyStructObject, weakreflist), READONLY},
20282028
{NULL} /* sentinel */
20292029
};
20302030

‎Modules/_testcapimodule.c

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6363,6 +6363,106 @@ static PyType_Spec HeapCTypeSubclassWithFinalizer_spec = {
63636363
HeapCTypeSubclassWithFinalizer_slots
63646364
};
63656365

6366+
typedef struct {
6367+
PyObject_HEAD
6368+
PyObject *dict;
6369+
} HeapCTypeWithDictObject;
6370+
6371+
static void
6372+
heapctypewithdict_dealloc(HeapCTypeWithDictObject* self)
6373+
{
6374+
6375+
PyTypeObject *tp = Py_TYPE(self);
6376+
Py_XDECREF(self->dict);
6377+
PyObject_DEL(self);
6378+
Py_DECREF(tp);
6379+
}
6380+
6381+
static PyGetSetDef heapctypewithdict_getsetlist[] = {
6382+
{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
6383+
{NULL} /* Sentinel */
6384+
};
6385+
6386+
static struct PyMemberDef heapctypewithdict_members[] = {
6387+
{"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
6388+
{"__dictoffset__", T_PYSSIZET, offsetof(HeapCTypeWithDictObject, dict), READONLY},
6389+
{NULL} /* Sentinel */
6390+
};
6391+
6392+
static PyType_Slot HeapCTypeWithDict_slots[] = {
6393+
{Py_tp_members, heapctypewithdict_members},
6394+
{Py_tp_getset, heapctypewithdict_getsetlist},
6395+
{Py_tp_dealloc, heapctypewithdict_dealloc},
6396+
{0, 0},
6397+
};
6398+
6399+
static PyType_Spec HeapCTypeWithDict_spec = {
6400+
"_testcapi.HeapCTypeWithDict",
6401+
sizeof(HeapCTypeWithDictObject),
6402+
0,
6403+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
6404+
HeapCTypeWithDict_slots
6405+
};
6406+
6407+
static struct PyMemberDef heapctypewithnegativedict_members[] = {
6408+
{"dictobj", T_OBJECT, offsetof(HeapCTypeWithDictObject, dict)},
6409+
{"__dictoffset__", T_PYSSIZET, -sizeof(void*), READONLY},
6410+
{NULL} /* Sentinel */
6411+
};
6412+
6413+
static PyType_Slot HeapCTypeWithNegativeDict_slots[] = {
6414+
{Py_tp_members, heapctypewithnegativedict_members},
6415+
{Py_tp_getset, heapctypewithdict_getsetlist},
6416+
{Py_tp_dealloc, heapctypewithdict_dealloc},
6417+
{0, 0},
6418+
};
6419+
6420+
static PyType_Spec HeapCTypeWithNegativeDict_spec = {
6421+
"_testcapi.HeapCTypeWithNegativeDict",
6422+
sizeof(HeapCTypeWithDictObject),
6423+
0,
6424+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
6425+
HeapCTypeWithNegativeDict_slots
6426+
};
6427+
6428+
typedef struct {
6429+
PyObject_HEAD
6430+
PyObject *weakreflist;
6431+
} HeapCTypeWithWeakrefObject;
6432+
6433+
static struct PyMemberDef heapctypewithweakref_members[] = {
6434+
{"weakreflist", T_OBJECT, offsetof(HeapCTypeWithWeakrefObject, weakreflist)},
6435+
{"__weaklistoffset__", T_PYSSIZET,
6436+
offsetof(HeapCTypeWithWeakrefObject, weakreflist), READONLY},
6437+
{NULL} /* Sentinel */
6438+
};
6439+
6440+
static void
6441+
heapctypewithweakref_dealloc(HeapCTypeWithWeakrefObject* self)
6442+
{
6443+
6444+
PyTypeObject *tp = Py_TYPE(self);
6445+
if (self->weakreflist != NULL)
6446+
PyObject_ClearWeakRefs((PyObject *) self);
6447+
Py_XDECREF(self->weakreflist);
6448+
PyObject_DEL(self);
6449+
Py_DECREF(tp);
6450+
}
6451+
6452+
static PyType_Slot HeapCTypeWithWeakref_slots[] = {
6453+
{Py_tp_members, heapctypewithweakref_members},
6454+
{Py_tp_dealloc, heapctypewithweakref_dealloc},
6455+
{0, 0},
6456+
};
6457+
6458+
static PyType_Spec HeapCTypeWithWeakref_spec = {
6459+
"_testcapi.HeapCTypeWithWeakref",
6460+
sizeof(HeapCTypeWithWeakrefObject),
6461+
0,
6462+
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
6463+
HeapCTypeWithWeakref_slots
6464+
};
6465+
63666466
static PyMethodDef meth_instance_methods[] = {
63676467
{"meth_varargs", meth_varargs, METH_VARARGS},
63686468
{"meth_varargs_keywords", (PyCFunction)(void(*)(void))meth_varargs_keywords, METH_VARARGS|METH_KEYWORDS},
@@ -6596,6 +6696,24 @@ PyInit__testcapi(void)
65966696
Py_DECREF(subclass_bases);
65976697
PyModule_AddObject(m, "HeapCTypeSubclass", HeapCTypeSubclass);
65986698

6699+
PyObject *HeapCTypeWithDict = PyType_FromSpec(&HeapCTypeWithDict_spec);
6700+
if (HeapCTypeWithDict == NULL) {
6701+
return NULL;
6702+
}
6703+
PyModule_AddObject(m, "HeapCTypeWithDict", HeapCTypeWithDict);
6704+
6705+
PyObject *HeapCTypeWithNegativeDict = PyType_FromSpec(&HeapCTypeWithNegativeDict_spec);
6706+
if (HeapCTypeWithNegativeDict == NULL) {
6707+
return NULL;
6708+
}
6709+
PyModule_AddObject(m, "HeapCTypeWithNegativeDict", HeapCTypeWithNegativeDict);
6710+
6711+
PyObject *HeapCTypeWithWeakref = PyType_FromSpec(&HeapCTypeWithWeakref_spec);
6712+
if (HeapCTypeWithWeakref == NULL) {
6713+
return NULL;
6714+
}
6715+
PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref);
6716+
65996717
PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass);
66006718
if (subclass_with_finalizer_bases == NULL) {
66016719
return NULL;

‎Objects/typeobject.c

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2853,15 +2853,27 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
28532853
PyTypeObject *type, *base;
28542854

28552855
PyType_Slot *slot;
2856-
Py_ssize_t nmembers;
2856+
Py_ssize_t nmembers, weaklistoffset, dictoffset;
28572857
char *s, *res_start;
28582858

2859-
nmembers = 0;
2859+
nmembers = weaklistoffset = dictoffset = 0;
28602860
for (slot = spec->slots; slot->slot; slot++) {
28612861
if (slot->slot == Py_tp_members) {
28622862
nmembers = 0;
28632863
for (memb = slot->pfunc; memb->name != NULL; memb++) {
28642864
nmembers++;
2865+
if (strcmp(memb->name, "__weaklistoffset__") == 0) {
2866+
// The PyMemberDef must be a Py_ssize_t and readonly
2867+
assert(memb->type == T_PYSSIZET);
2868+
assert(memb->flags == READONLY);
2869+
weaklistoffset = memb->offset;
2870+
}
2871+
if (strcmp(memb->name, "__dictoffset__") == 0) {
2872+
// The PyMemberDef must be a Py_ssize_t and readonly
2873+
assert(memb->type == T_PYSSIZET);
2874+
assert(memb->flags == READONLY);
2875+
dictoffset = memb->offset;
2876+
}
28652877
}
28662878
}
28672879
}
@@ -2990,6 +3002,17 @@ PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)
29903002
res->ht_cached_keys = _PyDict_NewKeysForClass();
29913003
}
29923004

3005+
if (weaklistoffset) {
3006+
type->tp_weaklistoffset = weaklistoffset;
3007+
if (PyDict_DelItemString((PyObject *)type->tp_dict, "__weaklistoffset__") < 0)
3008+
goto fail;
3009+
}
3010+
if (dictoffset) {
3011+
type->tp_dictoffset = dictoffset;
3012+
if (PyDict_DelItemString((PyObject *)type->tp_dict, "__dictoffset__") < 0)
3013+
goto fail;
3014+
}
3015+
29933016
/* Set type.__module__ */
29943017
s = strrchr(spec->name, '.');
29953018
if (s != NULL) {

‎Parser/asdl_c.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,11 @@ def visitModule(self, mod):
724724
return Py_BuildValue("O()", Py_TYPE(self));
725725
}
726726
727+
static PyMemberDef ast_type_members[] = {
728+
{"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
729+
{NULL} /* Sentinel */
730+
};
731+
727732
static PyMethodDef ast_type_methods[] = {
728733
{"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
729734
{NULL}
@@ -740,6 +745,7 @@ def visitModule(self, mod):
740745
{Py_tp_setattro, PyObject_GenericSetAttr},
741746
{Py_tp_traverse, ast_traverse},
742747
{Py_tp_clear, ast_clear},
748+
{Py_tp_members, ast_type_members},
743749
{Py_tp_methods, ast_type_methods},
744750
{Py_tp_getset, ast_type_getsets},
745751
{Py_tp_init, ast_type_init},
@@ -930,7 +936,6 @@ def visitModule(self, mod):
930936
self.emit("if (init_identifiers() < 0) return 0;", 1)
931937
self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
932938
self.emit("if (!state->AST_type) return 0;", 1)
933-
self.emit("((PyTypeObject*)state->AST_type)->tp_dictoffset = offsetof(AST_object, dict);", 1)
934939
self.emit("if (add_ast_fields() < 0) return 0;", 1)
935940
for dfn in mod.dfns:
936941
self.visit(dfn)
@@ -1372,6 +1377,7 @@ def main(srcfile, dump_module=False):
13721377
f.write('\n')
13731378
f.write('#include "Python.h"\n')
13741379
f.write('#include "%s-ast.h"\n' % mod.name)
1380+
f.write('#include "structmember.h"\n')
13751381
f.write('\n')
13761382

13771383
generate_module_def(f, mod)

‎Python/Python-ast.c

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)