Skip to content

Commit e11fc03

Browse files
gh-59956: Clarify Runtime State Status Expectations (gh-101308)
A PyThreadState can be in one of many states in its lifecycle, represented by some status value. Those statuses haven't been particularly clear, so we're addressing that here. Specifically: * made the distinct lifecycle statuses clear on PyThreadState * identified expectations of how various lifecycle-related functions relate to status * noted the various places where those expectations don't match the actual behavior At some point we'll need to address the mismatches. (This change also includes some cleanup.) #59956
1 parent ea23271 commit e11fc03

File tree

6 files changed

+416
-166
lines changed

6 files changed

+416
-166
lines changed

Include/cpython/pystate.h

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,30 @@ struct _ts {
119119
PyThreadState *next;
120120
PyInterpreterState *interp;
121121

122-
int _status;
122+
struct {
123+
/* Has been initialized to a safe state.
124+
125+
In order to be effective, this must be set to 0 during or right
126+
after allocation. */
127+
unsigned int initialized:1;
128+
129+
/* Has been bound to an OS thread. */
130+
unsigned int bound:1;
131+
/* Has been unbound from its OS thread. */
132+
unsigned int unbound:1;
133+
/* Has been bound aa current for the GILState API. */
134+
unsigned int bound_gilstate:1;
135+
/* Currently in use (maybe holds the GIL). */
136+
unsigned int active:1;
137+
138+
/* various stages of finalization */
139+
unsigned int finalizing:1;
140+
unsigned int cleared:1;
141+
unsigned int finalized:1;
142+
143+
/* padding to align to 4 bytes */
144+
unsigned int :24;
145+
} _status;
123146

124147
int py_recursion_remaining;
125148
int py_recursion_limit;
@@ -245,6 +268,8 @@ struct _ts {
245268
// Alias for backward compatibility with Python 3.8
246269
#define _PyInterpreterState_Get PyInterpreterState_Get
247270

271+
/* An alias for the internal _PyThreadState_New(),
272+
kept for stable ABI compatibility. */
248273
PyAPI_FUNC(PyThreadState *) _PyThreadState_Prealloc(PyInterpreterState *);
249274

250275
/* Similar to PyThreadState_Get(), but don't issue a fatal error

Include/internal/pycore_pystate.h

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -120,13 +120,12 @@ static inline PyInterpreterState* _PyInterpreterState_GET(void) {
120120

121121
// PyThreadState functions
122122

123+
PyAPI_FUNC(PyThreadState *) _PyThreadState_New(PyInterpreterState *interp);
123124
PyAPI_FUNC(void) _PyThreadState_Bind(PyThreadState *tstate);
124125
// We keep this around exclusively for stable ABI compatibility.
125126
PyAPI_FUNC(void) _PyThreadState_Init(
126127
PyThreadState *tstate);
127-
PyAPI_FUNC(void) _PyThreadState_DeleteExcept(
128-
_PyRuntimeState *runtime,
129-
PyThreadState *tstate);
128+
PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
130129

131130

132131
static inline void
@@ -139,18 +138,6 @@ _PyThreadState_UpdateTracingState(PyThreadState *tstate)
139138
}
140139

141140

142-
/* PyThreadState status */
143-
144-
#define PyThreadState_UNINITIALIZED 0
145-
/* Has been initialized to a safe state.
146-
147-
In order to be effective, this must be set to 0 during or right
148-
after allocation. */
149-
#define PyThreadState_INITIALIZED 1
150-
#define PyThreadState_BOUND 2
151-
#define PyThreadState_UNBOUND 3
152-
153-
154141
/* Other */
155142

156143
PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(

Modules/_threadmodule.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1161,7 +1161,7 @@ thread_PyThread_start_new_thread(PyObject *self, PyObject *fargs)
11611161
return PyErr_NoMemory();
11621162
}
11631163
boot->interp = _PyInterpreterState_GET();
1164-
boot->tstate = _PyThreadState_Prealloc(boot->interp);
1164+
boot->tstate = _PyThreadState_New(boot->interp);
11651165
if (boot->tstate == NULL) {
11661166
PyMem_Free(boot);
11671167
if (!PyErr_Occurred()) {

Python/ceval_gil.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,7 +624,7 @@ _PyEval_ReInitThreads(PyThreadState *tstate)
624624
}
625625

626626
/* Destroy all threads except the current one */
627-
_PyThreadState_DeleteExcept(runtime, tstate);
627+
_PyThreadState_DeleteExcept(tstate);
628628
return _PyStatus_OK();
629629
}
630630
#endif

Python/pylifecycle.c

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -696,10 +696,11 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
696696
const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT;
697697
init_interp_settings(interp, &config);
698698

699-
PyThreadState *tstate = PyThreadState_New(interp);
699+
PyThreadState *tstate = _PyThreadState_New(interp);
700700
if (tstate == NULL) {
701701
return _PyStatus_ERR("can't make first thread");
702702
}
703+
_PyThreadState_Bind(tstate);
703704
(void) PyThreadState_Swap(tstate);
704705

705706
status = init_interp_create_gil(tstate);
@@ -1821,6 +1822,11 @@ Py_FinalizeEx(void)
18211822

18221823
/* Get current thread state and interpreter pointer */
18231824
PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
1825+
// XXX assert(_Py_IsMainInterpreter(tstate->interp));
1826+
// XXX assert(_Py_IsMainThread());
1827+
1828+
// Block some operations.
1829+
tstate->interp->finalizing = 1;
18241830

18251831
// Wrap up existing "threading"-module-created, non-daemon threads.
18261832
wait_for_thread_shutdown(tstate);
@@ -1867,7 +1873,23 @@ Py_FinalizeEx(void)
18671873
_PyRuntimeState_SetFinalizing() has been called, no other Python thread
18681874
can take the GIL at this point: if they try, they will exit
18691875
immediately. */
1870-
_PyThreadState_DeleteExcept(runtime, tstate);
1876+
_PyThreadState_DeleteExcept(tstate);
1877+
1878+
/* At this point no Python code should be running at all.
1879+
The only thread state left should be the main thread of the main
1880+
interpreter (AKA tstate), in which this code is running right now.
1881+
There may be other OS threads running but none of them will have
1882+
thread states associated with them, nor will be able to create
1883+
new thread states.
1884+
1885+
Thus tstate is the only possible thread state from here on out.
1886+
It may still be used during finalization to run Python code as
1887+
needed or provide runtime state (e.g. sys.modules) but that will
1888+
happen sparingly. Furthermore, the order of finalization aims
1889+
to not need a thread (or interpreter) state as soon as possible.
1890+
*/
1891+
// XXX Make sure we are preventing the creating of any new thread states
1892+
// (or interpreters).
18711893

18721894
/* Flush sys.stdout and sys.stderr */
18731895
if (flush_std_files() < 0) {
@@ -1958,6 +1980,20 @@ Py_FinalizeEx(void)
19581980
}
19591981
#endif /* Py_TRACE_REFS */
19601982

1983+
/* At this point there's almost no other Python code that will run,
1984+
nor interpreter state needed. The only possibility is the
1985+
finalizers of the objects stored on tstate (and tstate->interp),
1986+
which are triggered via finalize_interp_clear().
1987+
1988+
For now we operate as though none of those finalizers actually
1989+
need an operational thread state or interpreter. In reality,
1990+
those finalizers may rely on some part of tstate or
1991+
tstate->interp, and/or may raise exceptions
1992+
or otherwise fail.
1993+
*/
1994+
// XXX Do this sooner during finalization.
1995+
// XXX Ensure finalizer errors are handled properly.
1996+
19611997
finalize_interp_clear(tstate);
19621998
finalize_interp_delete(tstate->interp);
19631999

@@ -2039,12 +2075,13 @@ new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config)
20392075
return _PyStatus_OK();
20402076
}
20412077

2042-
PyThreadState *tstate = PyThreadState_New(interp);
2078+
PyThreadState *tstate = _PyThreadState_New(interp);
20432079
if (tstate == NULL) {
20442080
PyInterpreterState_Delete(interp);
20452081
*tstate_p = NULL;
20462082
return _PyStatus_OK();
20472083
}
2084+
_PyThreadState_Bind(tstate);
20482085

20492086
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
20502087

0 commit comments

Comments
 (0)