pycore_pystate.h no longer redefines PyThreadState_GET() (GH-28921)

Redefining the PyThreadState_GET() macro in pycore_pystate.h is
useless since it doesn't affect files not including it. Either use
_PyThreadState_GET() directly, or don't use pycore_pystate.h internal
C API. For example, the _testcapi extension don't use the internal C
API, but use the public PyThreadState_Get() function instead.

Replace PyThreadState_Get() with _PyThreadState_GET(). The
_PyThreadState_GET() macro is more efficient than PyThreadState_Get()
and PyThreadState_GET() function calls which call fail with a fatal
Python error.

posixmodule.c and _ctypes extension now include <windows.h> before
pycore header files (like pycore_call.h).

_PyTraceback_Add() now uses _PyErr_Fetch()/_PyErr_Restore() instead
of PyErr_Fetch()/PyErr_Restore().

The _decimal and _xxsubinterpreters extensions are now built with the
Py_BUILD_CORE_MODULE macro defined to get access to the internal C
API.

Author: vstinner

Include/internal/pycore_call.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_pystate.h"       // _PyThreadState_GET()
+
 PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend(
     PyThreadState *tstate,
     PyObject *callable,
@@ -36,7 +38,7 @@ _PyObject_CallNoArgsTstate(PyThreadState *tstate, PyObject *func) {
 // Private static inline function variant of public PyObject_CallNoArgs()
 static inline PyObject *
 _PyObject_CallNoArgs(PyObject *func) {
-    PyThreadState *tstate = PyThreadState_Get();
+    PyThreadState *tstate = _PyThreadState_GET();
     return _PyObject_VectorcallTstate(tstate, func, NULL, 0, NULL);
 }
 

Include/internal/pycore_ceval.h

@@ -12,7 +12,8 @@ extern "C" {
 struct pyruntimestate;
 struct _ceval_runtime_state;
 
-#include "pycore_interp.h"   /* PyInterpreterState.eval_frame */
+#include "pycore_interp.h"        // PyInterpreterState.eval_frame
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
 extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
@@ -93,7 +94,7 @@ static inline int _Py_EnterRecursiveCall(PyThreadState *tstate,
 }
 
 static inline int _Py_EnterRecursiveCall_inline(const char *where) {
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     return _Py_EnterRecursiveCall(tstate, where);
 }
 
@@ -104,7 +105,7 @@ static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate)  {
 }
 
 static inline void _Py_LeaveRecursiveCall_inline(void)  {
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     _Py_LeaveRecursiveCall(tstate);
 }
 

Include/internal/pycore_pystate.h

@@ -82,7 +82,7 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime)
 
    The caller must hold the GIL.
 
-   See also PyThreadState_Get() and PyThreadState_GET(). */
+   See also PyThreadState_Get() and _PyThreadState_UncheckedGet(). */
 static inline PyThreadState*
 _PyThreadState_GET(void)
 {
@@ -93,10 +93,6 @@ _PyThreadState_GET(void)
 #endif
 }
 
-/* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */
-#undef PyThreadState_GET
-#define PyThreadState_GET() _PyThreadState_GET()
-
 PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func);
 
 static inline void

Include/pystate.h

@@ -66,18 +66,10 @@ PyAPI_FUNC(void) PyThreadState_Delete(PyThreadState *);
 
    The caller must hold the GIL.
 
-   See also PyThreadState_GET() and _PyThreadState_GET(). */
+   See also _PyThreadState_UncheckedGet() and _PyThreadState_GET(). */
 PyAPI_FUNC(PyThreadState *) PyThreadState_Get(void);
 
-/* Get the current Python thread state.
-
-   Macro using PyThreadState_Get() or _PyThreadState_GET() depending if
-   pycore_pystate.h is included or not (this header redefines the macro).
-
-   If PyThreadState_Get() is used, issue a fatal error if the current thread
-   state is NULL.
-
-   See also PyThreadState_Get() and _PyThreadState_GET(). */
+// Alias to PyThreadState_Get()
 #define PyThreadState_GET() PyThreadState_Get()
 
 PyAPI_FUNC(PyThreadState *) PyThreadState_Swap(PyThreadState *);

Modules/_asynciomodule.c

@@ -1,5 +1,6 @@
 #include "Python.h"
 #include "pycore_pyerrors.h"      // _PyErr_ClearExcState()
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 #include <stddef.h>               // offsetof()
 
 
@@ -225,7 +226,7 @@ get_running_loop(PyObject **loop)
 {
     PyObject *rl;
 
-    PyThreadState *ts = PyThreadState_Get();
+    PyThreadState *ts = _PyThreadState_GET();
     uint64_t ts_id = PyThreadState_GetID(ts);
     if (ts_id == cached_running_holder_tsid && cached_running_holder != NULL) {
         // Fast path, check the cache.
@@ -287,7 +288,7 @@ set_running_loop(PyObject *loop)
 {
     PyObject *ts_dict = NULL;
 
-    PyThreadState *tstate = PyThreadState_Get();
+    PyThreadState *tstate = _PyThreadState_GET();
     if (tstate != NULL) {
         ts_dict = _PyThreadState_GetDict(tstate);  // borrowed
     }

Modules/_ctypes/_ctypes.c

@@ -102,12 +102,16 @@ bytes(cdata)
 #define PY_SSIZE_T_CLEAN
 
 #include "Python.h"
+// windows.h must be included before pycore internal headers
+#ifdef MS_WIN32
+#  include <windows.h>
+#endif
+
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "structmember.h"         // PyMemberDef
 
 #include <ffi.h>
 #ifdef MS_WIN32
-#include <windows.h>
 #include <malloc.h>
 #ifndef IS_INTRESOURCE
 #define IS_INTRESOURCE(x) (((size_t)(x) >> 16) == 0)

Modules/_ctypes/callbacks.c

@@ -1,13 +1,15 @@
 #include "Python.h"
+// windows.h must be included before pycore internal headers
+#ifdef MS_WIN32
+#  include <windows.h>
+#endif
+
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "frameobject.h"
 
 #include <stdbool.h>
 
 #include <ffi.h>
-#ifdef MS_WIN32
-#include <windows.h>
-#endif
 #include "ctypes.h"
 
 /**************************************************************/

Modules/_ctypes/cfield.c

@@ -1,11 +1,13 @@
 #include "Python.h"
+// windows.h must be included before pycore internal headers
+#ifdef MS_WIN32
+#  include <windows.h>
+#endif
+
 #include "pycore_bitutils.h"      // _Py_bswap32()
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 
 #include <ffi.h>
-#ifdef MS_WIN32
-#include <windows.h>
-#endif
 #include "ctypes.h"
 
 

Modules/_ctypes/stgdict.c

@@ -1,9 +1,13 @@
 #include "Python.h"
+// windows.h must be included before pycore internal headers
+#ifdef MS_WIN32
+#  include <windows.h>
+#endif
+
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include <ffi.h>
 #ifdef MS_WIN32
-#include <windows.h>
-#include <malloc.h>
+#  include <malloc.h>
 #endif
 #include "ctypes.h"
 

Modules/_decimal/_decimal.c

@@ -27,6 +27,7 @@
 
 
 #include <Python.h>
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "longintrepr.h"
 #include "complexobject.h"
 #include "mpdecimal.h"
@@ -1511,18 +1512,20 @@ static PyGetSetDef context_getsets [] =
 static PyObject *
 current_context_from_dict(void)
 {
-    PyObject *dict;
-    PyObject *tl_context;
-    PyThreadState *tstate;
+    PyThreadState *tstate = _PyThreadState_GET();
+#ifdef Py_DEBUG
+    // The caller must hold the GIL
+    _Py_EnsureTstateNotNULL(tstate);
+#endif
 
-    dict = PyThreadState_GetDict();
+    PyObject *dict = _PyThreadState_GetDict(tstate);
     if (dict == NULL) {
         PyErr_SetString(PyExc_RuntimeError,
             "cannot get thread state");
         return NULL;
     }
 
-    tl_context = PyDict_GetItemWithError(dict, tls_context_key);
+    PyObject *tl_context = PyDict_GetItemWithError(dict, tls_context_key);
     if (tl_context != NULL) {
         /* We already have a thread local context. */
         CONTEXT_CHECK(tl_context);
@@ -1548,11 +1551,8 @@ current_context_from_dict(void)
 
     /* Cache the context of the current thread, assuming that it
      * will be accessed several times before a thread switch. */
-    tstate = PyThreadState_GET();
-    if (tstate) {
-        cached_context = (PyDecContextObject *)tl_context;
-        cached_context->tstate = tstate;
-    }
+    cached_context = (PyDecContextObject *)tl_context;
+    cached_context->tstate = tstate;
 
     /* Borrowed reference with refcount==1 */
     return tl_context;
@@ -1562,9 +1562,7 @@ current_context_from_dict(void)
 static PyObject *
 current_context(void)
 {
-    PyThreadState *tstate;
-
-    tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     if (cached_context && cached_context->tstate == tstate) {
         return (PyObject *)cached_context;
     }

Modules/_lsprof.c

@@ -1,5 +1,6 @@
 #include "Python.h"
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "rotatingtree.h"
 
 /************************************************************/
@@ -672,7 +673,7 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) {
         return NULL;
     }
@@ -706,7 +707,7 @@ Stop collecting profiling information.\n\
 static PyObject*
 profiler_disable(ProfilerObject *self, PyObject* noarg)
 {
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) {
         return NULL;
     }
@@ -743,7 +744,7 @@ static void
 profiler_dealloc(ProfilerObject *op)
 {
     if (op->flags & POF_ENABLED) {
-        PyThreadState *tstate = PyThreadState_GET();
+        PyThreadState *tstate = _PyThreadState_GET();
         if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) {
             PyErr_WriteUnraisable((PyObject *)op);
         }

Modules/_testinternalcapi.c

@@ -18,7 +18,8 @@
 #include "pycore_hashtable.h"    // _Py_hashtable_new()
 #include "pycore_initconfig.h"   // _Py_GetConfigsAsDict()
 #include "pycore_interp.h"       // _PyInterpreterState_GetConfigCopy()
-#include "pycore_pyerrors.h"      // _Py_UTF8_Edit_Cost()
+#include "pycore_pyerrors.h"     // _Py_UTF8_Edit_Cost()
+#include "pycore_pystate.h"      // _PyThreadState_GET()
 
 
 static PyObject *
@@ -31,7 +32,7 @@ get_configs(PyObject *self, PyObject *Py_UNUSED(args))
 static PyObject*
 get_recursion_depth(PyObject *self, PyObject *Py_UNUSED(args))
 {
-    PyThreadState *tstate = PyThreadState_Get();
+    PyThreadState *tstate = _PyThreadState_GET();
 
     /* subtract one to ignore the frame of the get_recursion_depth() call */
     return PyLong_FromLong(tstate->recursion_depth - 1);

Modules/_threadmodule.c

@@ -1319,7 +1319,7 @@ static PyObject *
 thread__set_sentinel(PyObject *module, PyObject *Py_UNUSED(ignored))
 {
     PyObject *wr;
-    PyThreadState *tstate = PyThreadState_Get();
+    PyThreadState *tstate = _PyThreadState_GET();
     lockobject *lock;
 
     if (tstate->on_delete_data != NULL) {

Modules/_xxsubinterpretersmodule.c

@@ -5,6 +5,7 @@
 #include "Python.h"
 #include "frameobject.h"
 #include "pycore_frame.h"
+#include "pycore_pystate.h"       // _PyThreadState_GET()
 #include "interpreteridobject.h"
 
 
@@ -2017,7 +2018,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
     }
 
     // Create and initialize the new interpreter.
-    PyThreadState *save_tstate = PyThreadState_Get();
+    PyThreadState *save_tstate = _PyThreadState_GET();
     // XXX Possible GILState issues?
     PyThreadState *tstate = _Py_NewInterpreter(isolated);
     PyThreadState_Swap(save_tstate);

Modules/posixmodule.c

@@ -10,38 +10,36 @@
 #define PY_SSIZE_T_CLEAN
 
 #include "Python.h"
-#include "pycore_call.h"          // _PyObject_CallNoArgs()
-#include "pycore_fileutils.h"     // _Py_closerange()
-#include "pycore_moduleobject.h"  // _PyModule_GetState()
+// Include <windows.h> before pycore internal headers. FSCTL_GET_REPARSE_POINT
+// is not exported by <windows.h> if the WIN32_LEAN_AND_MEAN macro is defined,
+// whereas pycore_condvar.h defines the WIN32_LEAN_AND_MEAN macro.
 #ifdef MS_WINDOWS
-   /* include <windows.h> early to avoid conflict with pycore_condvar.h:
-
-        #define WIN32_LEAN_AND_MEAN
-        #include <windows.h>
-
-      FSCTL_GET_REPARSE_POINT is not exported with WIN32_LEAN_AND_MEAN. */
 #  include <windows.h>
 #  include <pathcch.h>
 #endif
 
-#if !defined(EX_OK) && defined(EXIT_SUCCESS)
-#define EX_OK EXIT_SUCCESS
-#endif
-
 #ifdef __VXWORKS__
 #  include "pycore_bitutils.h"    // _Py_popcount32()
 #endif
+#include "pycore_call.h"          // _PyObject_CallNoArgs()
+#include "pycore_fileutils.h"     // _Py_closerange()
+#include "pycore_moduleobject.h"  // _PyModule_GetState()
 #include "pycore_ceval.h"         // _PyEval_ReInitThreads()
 #include "pycore_import.h"        // _PyImport_ReInitLock()
 #include "pycore_initconfig.h"    // _PyStatus_EXCEPTION()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
+
 #include "structmember.h"         // PyMemberDef
 #ifndef MS_WINDOWS
 #  include "posixmodule.h"
 #else
 #  include "winreparse.h"
 #endif
 
+#if !defined(EX_OK) && defined(EXIT_SUCCESS)
+#  define EX_OK EXIT_SUCCESS
+#endif
+
 /* On android API level 21, 'AT_EACCESS' is not declared although
  * HAVE_FACCESSAT is defined. */
 #ifdef __ANDROID__

Objects/typeobject.c

@@ -3332,7 +3332,7 @@ type_vectorcall(PyObject *metatype, PyObject *const *args,
     }
     /* In other (much less common) cases, fall back to
        more flexible calling conventions. */
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames);
 }
 

Python/bltinmodule.c

@@ -213,7 +213,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs,
         goto error;
     }
     PyFrameConstructor *f =  PyFunction_AS_FRAME_CONSTRUCTOR(func);
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     cell = _PyEval_Vector(tstate, f, ns, NULL, 0, NULL);
     if (cell != NULL) {
         if (bases != orig_bases) {

Python/ceval.c

@@ -1064,7 +1064,7 @@ static int unpack_iterable(PyThreadState *, PyObject *, int, int, PyObject **);
 PyObject *
 PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals)
 {
-    PyThreadState *tstate = PyThreadState_GET();
+    PyThreadState *tstate = _PyThreadState_GET();
     if (locals == NULL) {
         locals = globals;
     }