Skip to content

Commit 556e855

Browse files
authored
gh-117376: Make Py_DECREF a macro in ceval.c in free-threaded build (#122975)
`Py_DECREF` and `PyStackRef_CLOSE` are now implemented as macros in the free-threaded build in ceval.c. There are two motivations; * MSVC has problems inlining functions in ceval.c in the PGO build. * We will want to mark escaping calls in order to spill the stack pointer in ceval.c and we will want to do this around `_Py_Dealloc` (or `_Py_MergeZeroLocalRefcount` or `_Py_DecRefShared`), not around the entire `Py_DECREF` or `PyStackRef_CLOSE` call.
1 parent 67f2c84 commit 556e855

File tree

2 files changed

+53
-27
lines changed

2 files changed

+53
-27
lines changed

Include/internal/pycore_stackref.h

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -190,18 +190,15 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
190190
} while (0)
191191

192192
#ifdef Py_GIL_DISABLED
193-
static inline void
194-
PyStackRef_CLOSE(_PyStackRef stackref)
195-
{
196-
if (PyStackRef_IsDeferred(stackref)) {
197-
// No assert for being immortal or deferred here.
198-
// The GC unsets deferred objects right before clearing.
199-
return;
200-
}
201-
Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
202-
}
193+
# define PyStackRef_CLOSE(REF) \
194+
do { \
195+
_PyStackRef _close_tmp = (REF); \
196+
if (!PyStackRef_IsDeferred(_close_tmp)) { \
197+
Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \
198+
} \
199+
} while (0)
203200
#else
204-
# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
201+
# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref))
205202
#endif
206203

207204
#define PyStackRef_XCLOSE(stackref) \
@@ -227,7 +224,7 @@ PyStackRef_DUP(_PyStackRef stackref)
227224
return stackref;
228225
}
229226
#else
230-
# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)));
227+
# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
231228
#endif
232229

233230
static inline void

Python/ceval.c

Lines changed: 44 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,27 @@
5252
# error "ceval.c must be build with Py_BUILD_CORE define for best performance"
5353
#endif
5454

55-
#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED)
55+
#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
5656
// GH-89279: The MSVC compiler does not inline these static inline functions
5757
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
5858
// the limit of PGO, and that limit cannot be configured.
5959
// Define them as macros to make sure that they are always inlined by the
6060
// preprocessor.
61-
// TODO: implement Py_DECREF macro for Py_GIL_DISABLED
61+
62+
#undef Py_IS_TYPE
63+
#define Py_IS_TYPE(ob, type) \
64+
(_PyObject_CAST(ob)->ob_type == (type))
65+
66+
#undef Py_XDECREF
67+
#define Py_XDECREF(arg) \
68+
do { \
69+
PyObject *xop = _PyObject_CAST(arg); \
70+
if (xop != NULL) { \
71+
Py_DECREF(xop); \
72+
} \
73+
} while (0)
74+
75+
#ifndef Py_GIL_DISABLED
6276

6377
#undef Py_DECREF
6478
#define Py_DECREF(arg) \
@@ -74,19 +88,6 @@
7488
} \
7589
} while (0)
7690

77-
#undef Py_XDECREF
78-
#define Py_XDECREF(arg) \
79-
do { \
80-
PyObject *xop = _PyObject_CAST(arg); \
81-
if (xop != NULL) { \
82-
Py_DECREF(xop); \
83-
} \
84-
} while (0)
85-
86-
#undef Py_IS_TYPE
87-
#define Py_IS_TYPE(ob, type) \
88-
(_PyObject_CAST(ob)->ob_type == (type))
89-
9091
#undef _Py_DECREF_SPECIALIZED
9192
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
9293
do { \
@@ -100,6 +101,34 @@
100101
d(op); \
101102
} \
102103
} while (0)
104+
105+
#else // Py_GIL_DISABLED
106+
107+
#undef Py_DECREF
108+
#define Py_DECREF(arg) \
109+
do { \
110+
PyObject *op = _PyObject_CAST(arg); \
111+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
112+
if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
113+
break; \
114+
} \
115+
_Py_DECREF_STAT_INC(); \
116+
if (_Py_IsOwnedByCurrentThread(op)) { \
117+
local--; \
118+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
119+
if (local == 0) { \
120+
_Py_MergeZeroLocalRefcount(op); \
121+
} \
122+
} \
123+
else { \
124+
_Py_DecRefShared(op); \
125+
} \
126+
} while (0)
127+
128+
#undef _Py_DECREF_SPECIALIZED
129+
#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)
130+
131+
#endif
103132
#endif
104133

105134

0 commit comments

Comments
 (0)