[3.8] bpo-38070: Py_FatalError() logs runtime state (GH-16258)

* bpo-38070: _Py_DumpTraceback() writes <no Python frame> (GH-16244)

When a Python thread has no frame, _Py_DumpTraceback() and
_Py_DumpTracebackThreads() now write "<no Python frame>", rather than
writing nothing.

(cherry picked from commit 8fa3e17)

* bpo-38070: Enhance _PyObject_Dump() (GH-16243)

_PyObject_Dump() now dumps the object address for freed objects and
objects with ob_type=NULL.

(cherry picked from commit b39afb7)

* bpo-38070: Add _PyRuntimeState.preinitializing (GH-16245)

Add _PyRuntimeState.preinitializing field: set to 1 while
Py_PreInitialize() is running.

_PyRuntimeState: rename also pre_initialized field to preinitialized.

(cherry picked from commit d3b9041)

* bpo-38070: Py_FatalError() logs runtime state (GH-16246)


(cherry picked from commit 1ce16fb)

Author: vstinner

Include/internal/pycore_pystate.h

@@ -193,15 +193,20 @@ struct _gilstate_runtime_state {
 /* Full Python runtime state */
 
 typedef struct pyruntimestate {
-    /* Is Python pre-initialized? Set to 1 by Py_PreInitialize() */
-    int pre_initialized;
+    /* Is running Py_PreInitialize()? */
+    int preinitializing;
+
+    /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */
+    int preinitialized;
 
     /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */
     int core_initialized;
 
     /* Is Python fully initialized? Set to 1 by Py_Initialize() */
     int initialized;
 
+    /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize()
+       is called again. */
     PyThreadState *finalizing;
 
     struct pyinterpreters {
@@ -244,7 +249,7 @@ typedef struct pyruntimestate {
 } _PyRuntimeState;
 
 #define _PyRuntimeState_INIT \
-    {.pre_initialized = 0, .core_initialized = 0, .initialized = 0}
+    {.preinitialized = 0, .core_initialized = 0, .initialized = 0}
 /* Note: _PyRuntimeState_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;

Lib/test/test_capi.py

@@ -198,6 +198,7 @@ def test_return_null_without_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned NULL '
                                 br'without setting an error\n'
+                             br'Python runtime state: initialized\n'
                              br'SystemError: <built-in function '
                                  br'return_null_without_error> returned NULL '
                                  br'without setting an error\n'
@@ -225,6 +226,7 @@ def test_return_result_with_error(self):
             self.assertRegex(err.replace(b'\r', b''),
                              br'Fatal Python error: a function returned a '
                                 br'result with an error set\n'
+                             br'Python runtime state: initialized\n'
                              br'ValueError\n'
                              br'\n'
                              br'The above exception was the direct cause '

Lib/test/test_faulthandler.py

@@ -90,7 +90,8 @@ def get_output(self, code, filename=None, fd=None):
 
     def check_error(self, code, line_number, fatal_error, *,
                     filename=None, all_threads=True, other_regex=None,
-                    fd=None, know_current_thread=True):
+                    fd=None, know_current_thread=True,
+                    py_fatal_error=False):
         """
         Check that the fault handler for fatal errors is enabled and check the
         traceback from the child process output.
@@ -110,10 +111,12 @@ def check_error(self, code, line_number, fatal_error, *,
             {header} \(most recent call first\):
               File "<string>", line {lineno} in <module>
             """
-        regex = dedent(regex.format(
+        if py_fatal_error:
+            fatal_error += "\nPython runtime state: initialized"
+        regex = dedent(regex).format(
             lineno=line_number,
             fatal_error=fatal_error,
-            header=header)).strip()
+            header=header).strip()
         if other_regex:
             regex += '|' + other_regex
         output, exitcode = self.get_output(code, filename=filename, fd=fd)
@@ -170,7 +173,8 @@ def test_fatal_error_c_thread(self):
             """,
             3,
             'in new thread',
-            know_current_thread=False)
+            know_current_thread=False,
+            py_fatal_error=True)
 
     def test_sigabrt(self):
         self.check_fatal_error("""
@@ -226,15 +230,17 @@ def test_fatal_error(self):
             faulthandler._fatal_error(b'xyz')
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     def test_fatal_error_without_gil(self):
         self.check_fatal_error("""
             import faulthandler
             faulthandler._fatal_error(b'xyz', True)
             """,
             2,
-            'xyz')
+            'xyz',
+            py_fatal_error=True)
 
     @unittest.skipIf(sys.platform.startswith('openbsd'),
                      "Issue #12868: sigaltstack() doesn't work on "

Objects/object.c

@@ -464,15 +464,15 @@ void
 _PyObject_Dump(PyObject* op)
 {
     if (op == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
         fflush(stderr);
         return;
     }
 
     if (_PyObject_IsFreed(op)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<Freed object>\n");
+        fprintf(stderr, "<object at %p is freed>\n", op);
         return;
     }
 
@@ -2162,18 +2162,19 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
     fflush(stderr);
 
     if (obj == NULL) {
-        fprintf(stderr, "<NULL object>\n");
+        fprintf(stderr, "<object at NULL>\n");
     }
     else if (_PyObject_IsFreed(obj)) {
         /* It seems like the object memory has been freed:
            don't access it to prevent a segmentation fault. */
-        fprintf(stderr, "<object: freed>\n");
+        fprintf(stderr, "<object at %p is freed>\n", obj);
     }
     else if (Py_TYPE(obj) == NULL) {
-        fprintf(stderr, "<object: ob_type=NULL>\n");
+        fprintf(stderr, "<object at %p: ob_type=NULL>\n", obj);
     }
     else if (_PyObject_IsFreed((PyObject *)Py_TYPE(obj))) {
-        fprintf(stderr, "<object: freed type %p>\n", (void *)Py_TYPE(obj));
+        fprintf(stderr, "<object at %p: type at %p is freed>\n",
+                obj, (void *)Py_TYPE(obj));
     }
     else {
         /* Display the traceback where the object has been allocated.

Python/pylifecycle.c

@@ -719,11 +719,15 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* If it's already configured: ignored the new configuration */
         return _PyStatus_OK();
     }
 
+    /* Note: preinitialized remains 1 on error, it is only set to 0
+       at exit on success. */
+    runtime->preinitializing = 1;
+
     PyPreConfig config;
     _PyPreConfig_InitFromPreConfig(&config, src_config);
 
@@ -737,7 +741,8 @@ _Py_PreInitializeFromPyArgv(const PyPreConfig *src_config, const _PyArgv *args)
         return status;
     }
 
-    runtime->pre_initialized = 1;
+    runtime->preinitializing = 0;
+    runtime->preinitialized = 1;
     return _PyStatus_OK();
 }
 
@@ -777,7 +782,7 @@ _Py_PreInitializeFromConfig(const PyConfig *config,
     }
     _PyRuntimeState *runtime = &_PyRuntime;
 
-    if (runtime->pre_initialized) {
+    if (runtime->preinitialized) {
         /* Already initialized: do nothing */
         return _PyStatus_OK();
     }
@@ -1961,13 +1966,14 @@ init_sys_streams(PyInterpreterState *interp)
 
 
 static void
-_Py_FatalError_DumpTracebacks(int fd)
+_Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp,
+                              PyThreadState *tstate)
 {
     fputc('\n', stderr);
     fflush(stderr);
 
     /* display the current Python stack */
-    _Py_DumpTracebackThreads(fd, NULL, NULL);
+    _Py_DumpTracebackThreads(fd, interp, tstate);
 }
 
 /* Print the current exception (if an exception is set) with its traceback,
@@ -2062,10 +2068,39 @@ fatal_output_debug(const char *msg)
 }
 #endif
 
+
+static void
+fatal_error_dump_runtime(FILE *stream, _PyRuntimeState *runtime)
+{
+    fprintf(stream, "Python runtime state: ");
+    if (runtime->finalizing) {
+        fprintf(stream, "finalizing (tstate=%p)", runtime->finalizing);
+    }
+    else if (runtime->initialized) {
+        fprintf(stream, "initialized");
+    }
+    else if (runtime->core_initialized) {
+        fprintf(stream, "core initialized");
+    }
+    else if (runtime->preinitialized) {
+        fprintf(stream, "preinitialized");
+    }
+    else if (runtime->preinitializing) {
+        fprintf(stream, "preinitializing");
+    }
+    else {
+        fprintf(stream, "unknown");
+    }
+    fprintf(stream, "\n");
+    fflush(stream);
+}
+
+
 static void _Py_NO_RETURN
 fatal_error(const char *prefix, const char *msg, int status)
 {
-    const int fd = fileno(stderr);
+    FILE *stream = stderr;
+    const int fd = fileno(stream);
     static int reentrant = 0;
 
     if (reentrant) {
@@ -2075,45 +2110,48 @@ fatal_error(const char *prefix, const char *msg, int status)
     }
     reentrant = 1;
 
-    fprintf(stderr, "Fatal Python error: ");
+    fprintf(stream, "Fatal Python error: ");
     if (prefix) {
-        fputs(prefix, stderr);
-        fputs(": ", stderr);
+        fputs(prefix, stream);
+        fputs(": ", stream);
     }
     if (msg) {
-        fputs(msg, stderr);
+        fputs(msg, stream);
     }
     else {
-        fprintf(stderr, "<message not set>");
+        fprintf(stream, "<message not set>");
     }
-    fputs("\n", stderr);
-    fflush(stderr); /* it helps in Windows debug build */
+    fputs("\n", stream);
+    fflush(stream); /* it helps in Windows debug build */
 
-    /* Check if the current thread has a Python thread state
-       and holds the GIL */
-    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
-    if (tss_tstate != NULL) {
-        PyThreadState *tstate = _PyThreadState_GET();
-        if (tss_tstate != tstate) {
-            /* The Python thread does not hold the GIL */
-            tss_tstate = NULL;
-        }
-    }
-    else {
-        /* Py_FatalError() has been called from a C thread
-           which has no Python thread state. */
+    _PyRuntimeState *runtime = &_PyRuntime;
+    fatal_error_dump_runtime(stream, runtime);
+
+    PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime);
+    PyInterpreterState *interp = NULL;
+    if (tstate != NULL) {
+        interp = tstate->interp;
     }
-    int has_tstate_and_gil = (tss_tstate != NULL);
 
+    /* Check if the current thread has a Python thread state
+       and holds the GIL.
+
+       tss_tstate is NULL if Py_FatalError() is called from a C thread which
+       has no Python thread state.
+
+       tss_tstate != tstate if the current Python thread does not hold the GIL.
+       */
+    PyThreadState *tss_tstate = PyGILState_GetThisThreadState();
+    int has_tstate_and_gil = (tss_tstate != NULL && tss_tstate == tstate);
     if (has_tstate_and_gil) {
         /* If an exception is set, print the exception with its traceback */
         if (!_Py_FatalError_PrintExc(fd)) {
             /* No exception is set, or an exception is set without traceback */
-            _Py_FatalError_DumpTracebacks(fd);
+            _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
         }
     }
     else {
-        _Py_FatalError_DumpTracebacks(fd);
+        _Py_FatalError_DumpTracebacks(fd, interp, tss_tstate);
     }
 
     /* The main purpose of faulthandler is to display the traceback.

Python/traceback.c

@@ -797,12 +797,15 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header)
     PyFrameObject *frame;
     unsigned int depth;
 
-    if (write_header)
+    if (write_header) {
         PUTS(fd, "Stack (most recent call first):\n");
+    }
 
     frame = _PyThreadState_GetFrame(tstate);
-    if (frame == NULL)
+    if (frame == NULL) {
+        PUTS(fd, "<no Python frame>\n");
         return;
+    }
 
     depth = 0;
     while (frame != NULL) {
@@ -870,9 +873,9 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp,
            Python thread state of the current thread.
 
            PyThreadState_Get() doesn't give the state of the thread that caused
-           the fault if the thread released the GIL, and so this function
-           cannot be used. Read the thread specific storage (TSS) instead: call
-           PyGILState_GetThisThreadState(). */
+           the fault if the thread released the GIL, and so
+           _PyThreadState_GET() cannot be used. Read the thread specific
+           storage (TSS) instead: call PyGILState_GetThisThreadState(). */
         current_tstate = PyGILState_GetThisThreadState();
     }