Skip to content

Commit 5e8c0f0

Browse files
committed
gh-118527: Intern code consts in free-threaded build
We already intern and immortalize most string constants. In the free-threaded build, other constants can be a source of reference count contention because they are shared by all threads running the same code objects.
1 parent 5a1618a commit 5e8c0f0

File tree

9 files changed

+274
-14
lines changed

9 files changed

+274
-14
lines changed

Include/internal/pycore_code.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11+
#include "pycore_lock.h" // PyMutex
12+
1113

1214
// We hide some of the newer PyCodeObject fields behind macros.
1315
// This helps with backporting certain changes to 3.12.
@@ -16,6 +18,14 @@ extern "C" {
1618
#define _PyCode_HAS_INSTRUMENTATION(CODE) \
1719
(CODE->_co_instrumentation_version > 0)
1820

21+
struct _py_code_state {
22+
PyMutex mutex;
23+
// Interned constants from code objects. Used by the free-threaded build.
24+
struct _Py_hashtable_t *constants;
25+
};
26+
27+
extern PyStatus _PyCode_Init(PyInterpreterState *interp);
28+
extern void _PyCode_Fini(PyInterpreterState *interp);
1929

2030
#define CODE_MAX_WATCHERS 8
2131

Include/internal/pycore_interp.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ struct _is {
245245
struct _Py_long_state long_state;
246246
struct _dtoa_state dtoa;
247247
struct _py_func_state func_state;
248+
struct _py_code_state code_state;
248249

249250
struct _Py_dict_state dict_state;
250251
struct _Py_exc_state exc_state;

Include/internal/pycore_setobject.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ PyAPI_DATA(PyObject *) _PySet_Dummy;
3030

3131
PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key);
3232

33+
// Clears the set without acquiring locks. Used by _PyCode_Fini.
34+
extern void _PySet_ClearInternal(PySetObject *so);
35+
3336
#ifdef __cplusplus
3437
}
3538
#endif

Lib/test/test_ctypes/test_internals.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ def test_ints(self):
2828
self.assertEqual(ci._objects, None)
2929

3030
def test_c_char_p(self):
31-
s = b"Hello, World"
31+
s = "Hello, World".encode("ascii")
3232
refcnt = sys.getrefcount(s)
3333
cs = c_char_p(s)
3434
self.assertEqual(refcnt + 1, sys.getrefcount(s))

Lib/test/test_ctypes/test_python_api.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ def test_PyLong_Long(self):
4747

4848
@support.refcount_test
4949
def test_PyObj_FromPtr(self):
50-
s = "abc def ghi jkl"
50+
s = object()
5151
ref = sys.getrefcount(s)
5252
# id(python-object) is the address
5353
pyobj = _ctypes.PyObj_FromPtr(id(s))

Lib/test/test_memoryio.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -801,7 +801,7 @@ def test_sizeof(self):
801801

802802
def _test_cow_mutation(self, mutation):
803803
# Common code for all BytesIO copy-on-write mutation tests.
804-
imm = b' ' * 1024
804+
imm = (' ' * 1024).encode("ascii")
805805
old_rc = sys.getrefcount(imm)
806806
memio = self.ioclass(imm)
807807
self.assertEqual(sys.getrefcount(imm), old_rc + 1)

Objects/codeobject.c

Lines changed: 244 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
#include "pycore_code.h" // _PyCodeConstructor
77
#include "pycore_frame.h" // FRAME_SPECIALS_SIZE
8+
#include "pycore_hashtable.h" // _Py_hashtable_t
9+
#include "pycore_initconfig.h" // _PyStatus_OK()
810
#include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs
911
#include "pycore_object.h" // _PyObject_SetDeferredRefcount
1012
#include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches
@@ -118,6 +120,10 @@ all_name_chars(PyObject *o)
118120
return 1;
119121
}
120122

123+
#ifdef Py_GIL_DISABLED
124+
static PyObject *intern_one_constant(PyObject *op);
125+
#endif
126+
121127
static int
122128
intern_strings(PyObject *tuple)
123129
{
@@ -135,9 +141,11 @@ intern_strings(PyObject *tuple)
135141
return 0;
136142
}
137143

138-
/* Intern selected string constants */
144+
/* Intern constants. In the default build, this interns selected string
145+
constants. In the free-threaded build, this also interns non-string
146+
constants. */
139147
static int
140-
intern_string_constants(PyObject *tuple, int *modified)
148+
intern_constants(PyObject *tuple, int *modified)
141149
{
142150
for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) {
143151
PyObject *v = PyTuple_GET_ITEM(tuple, i);
@@ -154,7 +162,7 @@ intern_string_constants(PyObject *tuple, int *modified)
154162
}
155163
}
156164
else if (PyTuple_CheckExact(v)) {
157-
if (intern_string_constants(v, NULL) < 0) {
165+
if (intern_constants(v, NULL) < 0) {
158166
return -1;
159167
}
160168
}
@@ -165,7 +173,7 @@ intern_string_constants(PyObject *tuple, int *modified)
165173
return -1;
166174
}
167175
int tmp_modified = 0;
168-
if (intern_string_constants(tmp, &tmp_modified) < 0) {
176+
if (intern_constants(tmp, &tmp_modified) < 0) {
169177
Py_DECREF(tmp);
170178
return -1;
171179
}
@@ -184,6 +192,28 @@ intern_string_constants(PyObject *tuple, int *modified)
184192
}
185193
Py_DECREF(tmp);
186194
}
195+
#ifdef Py_GIL_DISABLED
196+
// Intern non-string consants in the free-threaded build, but only if
197+
// we are also immortalizing objects that use deferred reference
198+
// counting.
199+
PyThreadState *tstate = PyThreadState_GET();
200+
if (!_Py_IsImmortal(v) && !PyCode_Check(v) &&
201+
!PyUnicode_CheckExact(v) &&
202+
tstate->interp->gc.immortalize.enable_on_thread_created)
203+
{
204+
PyObject *interned = intern_one_constant(v);
205+
if (interned == NULL) {
206+
return -1;
207+
}
208+
else if (interned != v) {
209+
PyTuple_SET_ITEM(tuple, i, interned);
210+
Py_SETREF(v, interned);
211+
if (modified) {
212+
*modified = 1;
213+
}
214+
}
215+
}
216+
#endif
187217
}
188218
return 0;
189219
}
@@ -542,18 +572,41 @@ remove_column_info(PyObject *locations)
542572
return res;
543573
}
544574

545-
/* The caller is responsible for ensuring that the given data is valid. */
546-
547-
PyCodeObject *
548-
_PyCode_New(struct _PyCodeConstructor *con)
575+
static int
576+
intern_code_constants(struct _PyCodeConstructor *con)
549577
{
578+
#ifdef Py_GIL_DISABLED
579+
PyInterpreterState *interp = _PyInterpreterState_GET();
580+
struct _py_code_state *state = &interp->code_state;
581+
PyMutex_Lock(&state->mutex);
582+
#endif
550583
if (intern_strings(con->names) < 0) {
551-
return NULL;
584+
goto error;
552585
}
553-
if (intern_string_constants(con->consts, NULL) < 0) {
554-
return NULL;
586+
if (intern_constants(con->consts, NULL) < 0) {
587+
goto error;
555588
}
556589
if (intern_strings(con->localsplusnames) < 0) {
590+
goto error;
591+
}
592+
#ifdef Py_GIL_DISABLED
593+
PyMutex_Unlock(&state->mutex);
594+
#endif
595+
return 0;
596+
597+
error:
598+
#ifdef Py_GIL_DISABLED
599+
PyMutex_Unlock(&state->mutex);
600+
#endif
601+
return -1;
602+
}
603+
604+
/* The caller is responsible for ensuring that the given data is valid. */
605+
606+
PyCodeObject *
607+
_PyCode_New(struct _PyCodeConstructor *con)
608+
{
609+
if (intern_code_constants(con) < 0) {
557610
return NULL;
558611
}
559612

@@ -2399,3 +2452,183 @@ _PyCode_ConstantKey(PyObject *op)
23992452
}
24002453
return key;
24012454
}
2455+
2456+
#ifdef Py_GIL_DISABLED
2457+
static PyObject *
2458+
intern_one_constant(PyObject *op)
2459+
{
2460+
PyInterpreterState *interp = _PyInterpreterState_GET();
2461+
_Py_hashtable_t *consts = interp->code_state.constants;
2462+
2463+
assert(!PyUnicode_CheckExact(op)); // strings are interned separately
2464+
2465+
_Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op);
2466+
if (entry == NULL) {
2467+
if (_Py_hashtable_set(consts, op, op) != 0) {
2468+
return NULL;
2469+
}
2470+
2471+
#ifdef Py_REF_DEBUG
2472+
Py_ssize_t refcnt = Py_REFCNT(op);
2473+
if (refcnt != 1) {
2474+
// Adjust the reftotal to account for the fact that we only
2475+
// restore a single reference in _PyCode_Fini.
2476+
_Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1));
2477+
}
2478+
#endif
2479+
2480+
_Py_SetImmortal(op);
2481+
return op;
2482+
}
2483+
2484+
assert(_Py_IsImmortal(entry->value));
2485+
return (PyObject *)entry->value;
2486+
}
2487+
2488+
static int
2489+
compare_constants(const void *key1, const void *key2) {
2490+
PyObject *op1 = (PyObject *)key1;
2491+
PyObject *op2 = (PyObject *)key2;
2492+
if (op1 == op2) {
2493+
return 1;
2494+
}
2495+
if (Py_TYPE(op1) != Py_TYPE(op2)) {
2496+
return 0;
2497+
}
2498+
// We compare container contents by identity because we have already
2499+
// internalized the items.
2500+
if (PyTuple_CheckExact(op1)) {
2501+
Py_ssize_t size = PyTuple_GET_SIZE(op1);
2502+
if (size != PyTuple_GET_SIZE(op2)) {
2503+
return 0;
2504+
}
2505+
for (Py_ssize_t i = 0; i < size; i++) {
2506+
if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) {
2507+
return 0;
2508+
}
2509+
}
2510+
return 1;
2511+
}
2512+
else if (PyFrozenSet_CheckExact(op1)) {
2513+
if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) {
2514+
return 0;
2515+
}
2516+
Py_ssize_t pos1 = 0, pos2 = 0;
2517+
PyObject *obj1, *obj2;
2518+
Py_hash_t hash1, hash2;
2519+
while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) &&
2520+
(_PySet_NextEntry(op2, &pos2, &obj2, &hash2)))
2521+
{
2522+
if (obj1 != obj2) {
2523+
return 0;
2524+
}
2525+
}
2526+
return 1;
2527+
}
2528+
else if (PySlice_Check(op1)) {
2529+
PySliceObject *s1 = (PySliceObject *)op1;
2530+
PySliceObject *s2 = (PySliceObject *)op2;
2531+
return (s1->start == s2->start &&
2532+
s1->stop == s2->stop &&
2533+
s1->step == s2->step);
2534+
}
2535+
else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) {
2536+
return PyObject_RichCompareBool(op1, op2, Py_EQ);
2537+
}
2538+
else if (PyFloat_CheckExact(op1)) {
2539+
// Ensure that, for example, +0.0 and -0.0 are distinct
2540+
double f1 = PyFloat_AS_DOUBLE(op1);
2541+
double f2 = PyFloat_AS_DOUBLE(op2);
2542+
return memcmp(&f1, &f2, sizeof(double)) == 0;
2543+
}
2544+
else if (PyComplex_CheckExact(op1)) {
2545+
Py_complex c1 = ((PyComplexObject *)op1)->cval;
2546+
Py_complex c2 = ((PyComplexObject *)op2)->cval;
2547+
return memcmp(&c1, &c2, sizeof(Py_complex)) == 0;
2548+
}
2549+
_Py_FatalErrorFormat("unexpected type in compare_constants: %s",
2550+
Py_TYPE(op1)->tp_name);
2551+
return 0;
2552+
}
2553+
2554+
static Py_uhash_t
2555+
hash_const(const void *key)
2556+
{
2557+
PyObject *op = (PyObject *)key;
2558+
if (PySlice_Check(op)) {
2559+
PySliceObject *s = (PySliceObject *)op;
2560+
PyObject *data[3] = { s->start, s->stop, s->step };
2561+
return _Py_HashBytes(&data, sizeof(data));
2562+
}
2563+
else if (PyTuple_CheckExact(op)) {
2564+
Py_ssize_t size = PyTuple_GET_SIZE(op);
2565+
PyObject **data = _PyTuple_ITEMS(op);
2566+
return _Py_HashBytes(data, sizeof(PyObject *) * size);
2567+
}
2568+
Py_hash_t h = PyObject_Hash(op);
2569+
if (h == -1) {
2570+
// This should never happen: all the constants we support have
2571+
// infallible hash functions.
2572+
Py_FatalError("code: hash failed");
2573+
}
2574+
return (Py_uhash_t)h;
2575+
}
2576+
2577+
static int
2578+
clear_containers(_Py_hashtable_t *ht, const void *key, const void *value,
2579+
void *user_data)
2580+
{
2581+
// First clear containers to avoid recursive deallocation later on in
2582+
// destroy_key.
2583+
PyObject *op = (PyObject *)key;
2584+
if (PyTuple_CheckExact(op)) {
2585+
for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) {
2586+
Py_CLEAR(_PyTuple_ITEMS(op)[i]);
2587+
}
2588+
}
2589+
else if (PySlice_Check(op)) {
2590+
PySliceObject *slice = (PySliceObject *)op;
2591+
Py_SETREF(slice->start, Py_None);
2592+
Py_SETREF(slice->stop, Py_None);
2593+
Py_SETREF(slice->step, Py_None);
2594+
}
2595+
else if (PyFrozenSet_CheckExact(op)) {
2596+
_PySet_ClearInternal((PySetObject *)op);
2597+
}
2598+
return 0;
2599+
}
2600+
2601+
static void
2602+
destroy_key(void *key)
2603+
{
2604+
_Py_ClearImmortal(key);
2605+
}
2606+
#endif
2607+
2608+
PyStatus
2609+
_PyCode_Init(PyInterpreterState *interp)
2610+
{
2611+
#ifdef Py_GIL_DISABLED
2612+
struct _py_code_state *state = &interp->code_state;
2613+
state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants,
2614+
&destroy_key, NULL, NULL);
2615+
if (state->constants == NULL) {
2616+
return _PyStatus_NO_MEMORY();
2617+
}
2618+
#endif
2619+
return _PyStatus_OK();
2620+
}
2621+
2622+
void
2623+
_PyCode_Fini(PyInterpreterState *interp)
2624+
{
2625+
#ifdef Py_GIL_DISABLED
2626+
// Free interned constants
2627+
struct _py_code_state *state = &interp->code_state;
2628+
if (state->constants) {
2629+
_Py_hashtable_foreach(state->constants, &clear_containers, NULL);
2630+
_Py_hashtable_destroy(state->constants);
2631+
state->constants = NULL;
2632+
}
2633+
#endif
2634+
}

Objects/setobject.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2621,6 +2621,12 @@ PySet_Clear(PyObject *set)
26212621
return 0;
26222622
}
26232623

2624+
void
2625+
_PySet_ClearInternal(PySetObject *so)
2626+
{
2627+
(void)set_clear_internal(so);
2628+
}
2629+
26242630
int
26252631
PySet_Contains(PyObject *anyset, PyObject *key)
26262632
{

0 commit comments

Comments
 (0)