Skip to content

Commit 547ee83

Browse files
markshannonmarkshannon
committed
Specialize LOAD_METHOD for instances with dictionaries.
1 parent 09487c1 commit 547ee83

File tree

6 files changed

+142
-79
lines changed

6 files changed

+142
-79
lines changed

‎Include/internal/pycore_object.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,8 @@ static inline PyObject **_PyObject_ManagedDictPointer(PyObject *obj)
227227
return ((PyObject **)obj)-3;
228228
}
229229

230+
#define MANAGED_DICT_OFFSET ((int)((sizeof(PyObject *))*-3))
231+
230232
extern PyObject ** _PyObject_DictPointer(PyObject *);
231233
extern int _PyObject_VisitInstanceAttributes(PyObject *self, visitproc visit, void *arg);
232234
extern void _PyObject_ClearInstanceAttributes(PyObject *self);

‎Include/opcode.h

Lines changed: 35 additions & 34 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Lib/opcode.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,10 +260,11 @@ def jabs_op(name, op):
260260
"LOAD_GLOBAL_MODULE",
261261
"LOAD_GLOBAL_BUILTIN",
262262
"LOAD_METHOD_ADAPTIVE",
263-
"LOAD_METHOD_CACHED",
264263
"LOAD_METHOD_CLASS",
265264
"LOAD_METHOD_MODULE",
266265
"LOAD_METHOD_NO_DICT",
266+
"LOAD_METHOD_WITH_DICT",
267+
"LOAD_METHOD_WITH_VALUES",
267268
"PRECALL_ADAPTIVE",
268269
"PRECALL_BUILTIN_CLASS",
269270
"PRECALL_NO_KW_BUILTIN_O",

‎Python/ceval.c

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4403,15 +4403,15 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
44034403
}
44044404
}
44054405

4406-
TARGET(LOAD_METHOD_CACHED) {
4406+
TARGET(LOAD_METHOD_WITH_VALUES) {
44074407
/* LOAD_METHOD, with cached method object */
44084408
assert(cframe.use_tracing == 0);
44094409
PyObject *self = TOP();
44104410
PyTypeObject *self_cls = Py_TYPE(self);
44114411
SpecializedCacheEntry *caches = GET_CACHE();
44124412
_PyAttrCache *cache1 = &caches[-1].attr;
44134413
_PyObjectCache *cache2 = &caches[-2].obj;
4414-
4414+
assert(cache1->tp_version != 0);
44154415
DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD);
44164416
assert(self_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT);
44174417
PyDictObject *dict = *(PyDictObject**)_PyObject_ManagedDictPointer(self);
@@ -4427,6 +4427,39 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
44274427
NOTRACE_DISPATCH();
44284428
}
44294429

4430+
TARGET(LOAD_METHOD_WITH_DICT) {
4431+
/* LOAD_METHOD, with a dict
4432+
Can be either a managed dict, or a tp_dictoffset offset.*/
4433+
assert(cframe.use_tracing == 0);
4434+
PyObject *self = TOP();
4435+
PyTypeObject *self_cls = Py_TYPE(self);
4436+
SpecializedCacheEntry *caches = GET_CACHE();
4437+
_PyAdaptiveEntry *cache0 = &caches[0].adaptive;
4438+
_PyAttrCache *cache1 = &caches[-1].attr;
4439+
_PyObjectCache *cache2 = &caches[-2].obj;
4440+
4441+
DEOPT_IF(self_cls->tp_version_tag != cache1->tp_version, LOAD_METHOD);
4442+
/* Treat index as a signed 16 bit value */
4443+
int dictoffset = *(int16_t *)&cache0->index;
4444+
PyDictObject **dictptr = (PyDictObject**)(((char *)self)+dictoffset);
4445+
assert(
4446+
((PyObject **)dictptr == _PyObject_ManagedDictPointer(self) && dictoffset < 0)
4447+
||
4448+
(dictoffset == self_cls->tp_dictoffset && dictoffset > 0)
4449+
);
4450+
PyDictObject *dict = *dictptr;
4451+
DEOPT_IF(dict == NULL, LOAD_METHOD);
4452+
DEOPT_IF(dict->ma_keys->dk_version != cache1->dk_version_or_hint, LOAD_METHOD);
4453+
STAT_INC(LOAD_METHOD, hit);
4454+
PyObject *res = cache2->obj;
4455+
assert(res != NULL);
4456+
assert(_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR));
4457+
Py_INCREF(res);
4458+
SET_TOP(res);
4459+
PUSH(self);
4460+
NOTRACE_DISPATCH();
4461+
}
4462+
44304463
TARGET(LOAD_METHOD_NO_DICT) {
44314464
assert(cframe.use_tracing == 0);
44324465
PyObject *self = TOP();

‎Python/opcode_targets.h

Lines changed: 15 additions & 15 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Python/specialize.c

Lines changed: 53 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,6 +1055,13 @@ specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr, PyObject *nam
10551055
}
10561056
}
10571057

1058+
typedef enum {
1059+
MANAGED_VALUES = 1,
1060+
MANAGED_DICT = 2,
1061+
OFFSET_DICT = 3,
1062+
NO_DICT = 4
1063+
} ObjectDictKind;
1064+
10581065
// Please collect stats carefully before and after modifying. A subtle change
10591066
// can cause a significant drop in cache hits. A possible test is
10601067
// python.exe -m test_typing test_re test_dis test_zlib.
@@ -1064,8 +1071,8 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
10641071
_PyAdaptiveEntry *cache0 = &cache->adaptive;
10651072
_PyAttrCache *cache1 = &cache[-1].attr;
10661073
_PyObjectCache *cache2 = &cache[-2].obj;
1067-
10681074
PyTypeObject *owner_cls = Py_TYPE(owner);
1075+
10691076
if (PyModule_CheckExact(owner)) {
10701077
int err = specialize_module_load_attr(owner, instr, name, cache0, cache1,
10711078
LOAD_METHOD, LOAD_METHOD_MODULE);
@@ -1095,13 +1102,39 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
10951102
SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
10961103
goto fail;
10971104
}
1105+
ObjectDictKind dictkind;
1106+
PyDictKeysObject *keys;
10981107
if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
1099-
PyObject **owner_dictptr = _PyObject_ManagedDictPointer(owner);
1100-
if (*owner_dictptr) {
1101-
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_HAS_MANAGED_DICT);
1108+
PyObject *dict = *_PyObject_ManagedDictPointer(owner);
1109+
keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
1110+
if (dict == NULL) {
1111+
dictkind = MANAGED_VALUES;
1112+
}
1113+
else {
1114+
dictkind = MANAGED_DICT;
1115+
}
1116+
}
1117+
else {
1118+
int dictoffset = owner_cls->tp_dictoffset;
1119+
if (dictoffset < 0 || dictoffset > INT16_MAX) {
1120+
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE);
11021121
goto fail;
11031122
}
1104-
PyDictKeysObject *keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
1123+
if (dictoffset == 0) {
1124+
dictkind = NO_DICT;
1125+
keys = NULL;
1126+
}
1127+
else {
1128+
PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
1129+
if (dict == NULL) {
1130+
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_NO_DICT);
1131+
goto fail;
1132+
}
1133+
keys = ((PyDictObject *)dict)->ma_keys;
1134+
dictkind = OFFSET_DICT;
1135+
}
1136+
}
1137+
if (dictkind != NO_DICT) {
11051138
Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
11061139
if (index != DKIX_EMPTY) {
11071140
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_IS_ATTR);
@@ -1113,31 +1146,24 @@ _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name,
11131146
goto fail;
11141147
}
11151148
cache1->dk_version_or_hint = keys_version;
1116-
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_CACHED, _Py_OPARG(*instr));
11171149
}
1118-
else {
1119-
if (owner_cls->tp_dictoffset == 0) {
1150+
switch(dictkind) {
1151+
case NO_DICT:
11201152
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_NO_DICT, _Py_OPARG(*instr));
1121-
}
1122-
else {
1123-
SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_HAS_DICT);
1124-
goto fail;
1125-
}
1153+
break;
1154+
case MANAGED_VALUES:
1155+
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_VALUES, _Py_OPARG(*instr));
1156+
break;
1157+
case MANAGED_DICT:
1158+
cache0->index = (uint16_t)MANAGED_DICT_OFFSET;
1159+
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr));
1160+
break;
1161+
case OFFSET_DICT:
1162+
assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
1163+
cache0->index = (uint16_t)owner_cls->tp_dictoffset;
1164+
*instr = _Py_MAKECODEUNIT(LOAD_METHOD_WITH_DICT, _Py_OPARG(*instr));
1165+
break;
11261166
}
1127-
/* `descr` is borrowed. This is safe for methods (even inherited ones from
1128-
* super classes!) as long as tp_version_tag is validated for two main reasons:
1129-
*
1130-
* 1. The class will always hold a reference to the method so it will
1131-
* usually not be GC-ed. Should it be deleted in Python, e.g.
1132-
* `del obj.meth`, tp_version_tag will be invalidated, because of reason 2.
1133-
*
1134-
* 2. The pre-existing type method cache (MCACHE) uses the same principles
1135-
* of caching a borrowed descriptor. The MCACHE infrastructure does all the
1136-
* heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO
1137-
* modification, on any type object change along said MRO, etc. (see
1138-
* PyType_Modified usages in typeobject.c). The MCACHE has been
1139-
* working since Python 2.6 and it's battle-tested.
1140-
*/
11411167
cache1->tp_version = owner_cls->tp_version_tag;
11421168
cache2->obj = descr;
11431169
// Fall through.

0 commit comments

Comments
 (0)