bpo-35713: Split _Py_InitializeCore into subfunctions (GH-11650)

* Split _Py_InitializeCore_impl() into subfunctions: add multiple pycore_init_xxx() functions
* Preliminary sys.stderr is now set earlier to get an usable
  sys.stderr ealier.
* Move code into _Py_Initialize_ReconfigureCore() to be able to call
  it from _Py_InitializeCore().
* Split _PyExc_Init(): create a new _PyBuiltins_AddExceptions()
  function.
* Call _PyExc_Init() earlier in _Py_InitializeCore_impl()
  and new_interpreter() to get working exceptions earlier.
* _Py_ReadyTypes() now returns _PyInitError rather than calling
  Py_FatalError().
* Misc code cleanup

Author: vstinner

Include/internal/pycore_pylifecycle.h

@@ -30,12 +30,13 @@ extern PyObject * _PyBuiltin_Init(void);
 extern _PyInitError _PySys_BeginInit(PyObject **sysmod);
 extern int _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp);
 extern _PyInitError _PyImport_Init(PyInterpreterState *interp);
-extern _PyInitError _PyExc_Init(PyObject * bltinmod);
+extern _PyInitError _PyExc_Init(void);
+extern _PyInitError _PyBuiltins_AddExceptions(PyObject * bltinmod);
 extern _PyInitError _PyImportHooks_Init(void);
 extern int _PyFloat_Init(void);
 extern _PyInitError _Py_HashRandomization_Init(const _PyCoreConfig *);
 
-extern void _Py_ReadyTypes(void);
+extern _PyInitError _Py_ReadyTypes(void);
 
 /* Various internal finalizers */
 

Misc/NEWS.d/next/Core and Builtins/2019-01-22-18-50-21.bpo-35713.bTeUsa.rst

@@ -0,0 +1,2 @@
+Reorganize Python initialization to get working exceptions and sys.stderr
+earlier.

Objects/exceptions.c

@@ -2492,7 +2492,7 @@ SimpleExtendsException(PyExc_Warning, ResourceWarning,
 #endif /* MS_WINDOWS */
 
 _PyInitError
-_PyExc_Init(PyObject *bltinmod)
+_PyExc_Init(void)
 {
 #define PRE_INIT(TYPE) \
     if (!(_PyExc_ ## TYPE.tp_flags & Py_TPFLAGS_READY)) { \
@@ -2502,21 +2502,6 @@ _PyExc_Init(PyObject *bltinmod)
         Py_INCREF(PyExc_ ## TYPE); \
     }
 
-#define POST_INIT(TYPE) \
-    if (PyDict_SetItemString(bdict, # TYPE, PyExc_ ## TYPE)) { \
-        return _Py_INIT_ERR("Module dictionary insertion problem."); \
-    }
-
-#define INIT_ALIAS(NAME, TYPE) \
-    do { \
-        Py_INCREF(PyExc_ ## TYPE); \
-        Py_XDECREF(PyExc_ ## NAME); \
-        PyExc_ ## NAME = PyExc_ ## TYPE; \
-        if (PyDict_SetItemString(bdict, # NAME, PyExc_ ## NAME)) { \
-            return _Py_INIT_ERR("Module dictionary insertion problem."); \
-        } \
-    } while (0)
-
 #define ADD_ERRNO(TYPE, CODE) \
     do { \
         PyObject *_code = PyLong_FromLong(CODE); \
@@ -2526,8 +2511,6 @@ _PyExc_Init(PyObject *bltinmod)
         Py_DECREF(_code); \
     } while (0)
 
-    PyObject *bdict;
-
     PRE_INIT(BaseException);
     PRE_INIT(Exception);
     PRE_INIT(TypeError);
@@ -2596,6 +2579,68 @@ _PyExc_Init(PyObject *bltinmod)
     PRE_INIT(ProcessLookupError);
     PRE_INIT(TimeoutError);
 
+    if (preallocate_memerrors() < 0) {
+        return _Py_INIT_ERR("Could not preallocate MemoryError object");
+    }
+
+    /* Add exceptions to errnomap */
+    if (!errnomap) {
+        errnomap = PyDict_New();
+        if (!errnomap) {
+            return _Py_INIT_ERR("Cannot allocate map from errnos to OSError subclasses");
+        }
+    }
+
+    ADD_ERRNO(BlockingIOError, EAGAIN);
+    ADD_ERRNO(BlockingIOError, EALREADY);
+    ADD_ERRNO(BlockingIOError, EINPROGRESS);
+    ADD_ERRNO(BlockingIOError, EWOULDBLOCK);
+    ADD_ERRNO(BrokenPipeError, EPIPE);
+#ifdef ESHUTDOWN
+    ADD_ERRNO(BrokenPipeError, ESHUTDOWN);
+#endif
+    ADD_ERRNO(ChildProcessError, ECHILD);
+    ADD_ERRNO(ConnectionAbortedError, ECONNABORTED);
+    ADD_ERRNO(ConnectionRefusedError, ECONNREFUSED);
+    ADD_ERRNO(ConnectionResetError, ECONNRESET);
+    ADD_ERRNO(FileExistsError, EEXIST);
+    ADD_ERRNO(FileNotFoundError, ENOENT);
+    ADD_ERRNO(IsADirectoryError, EISDIR);
+    ADD_ERRNO(NotADirectoryError, ENOTDIR);
+    ADD_ERRNO(InterruptedError, EINTR);
+    ADD_ERRNO(PermissionError, EACCES);
+    ADD_ERRNO(PermissionError, EPERM);
+    ADD_ERRNO(ProcessLookupError, ESRCH);
+    ADD_ERRNO(TimeoutError, ETIMEDOUT);
+
+    return _Py_INIT_OK();
+
+#undef PRE_INIT
+#undef ADD_ERRNO
+}
+
+
+/* Add exception types to the builtins module */
+_PyInitError
+_PyBuiltins_AddExceptions(PyObject *bltinmod)
+{
+#define POST_INIT(TYPE) \
+    if (PyDict_SetItemString(bdict, # TYPE, PyExc_ ## TYPE)) { \
+        return _Py_INIT_ERR("Module dictionary insertion problem."); \
+    }
+
+#define INIT_ALIAS(NAME, TYPE) \
+    do { \
+        Py_INCREF(PyExc_ ## TYPE); \
+        Py_XDECREF(PyExc_ ## NAME); \
+        PyExc_ ## NAME = PyExc_ ## TYPE; \
+        if (PyDict_SetItemString(bdict, # NAME, PyExc_ ## NAME)) { \
+            return _Py_INIT_ERR("Module dictionary insertion problem."); \
+        } \
+    } while (0)
+
+    PyObject *bdict;
+
     bdict = PyModule_GetDict(bltinmod);
     if (bdict == NULL) {
         return _Py_INIT_ERR("exceptions bootstrapping error.");
@@ -2656,61 +2701,28 @@ _PyExc_Init(PyObject *bltinmod)
     POST_INIT(BytesWarning);
     POST_INIT(ResourceWarning);
 
-    if (!errnomap) {
-        errnomap = PyDict_New();
-        if (!errnomap) {
-            return _Py_INIT_ERR("Cannot allocate map from errnos to OSError subclasses");
-        }
-    }
-
     /* OSError subclasses */
     POST_INIT(ConnectionError);
 
     POST_INIT(BlockingIOError);
-    ADD_ERRNO(BlockingIOError, EAGAIN);
-    ADD_ERRNO(BlockingIOError, EALREADY);
-    ADD_ERRNO(BlockingIOError, EINPROGRESS);
-    ADD_ERRNO(BlockingIOError, EWOULDBLOCK);
     POST_INIT(BrokenPipeError);
-    ADD_ERRNO(BrokenPipeError, EPIPE);
-#ifdef ESHUTDOWN
-    ADD_ERRNO(BrokenPipeError, ESHUTDOWN);
-#endif
     POST_INIT(ChildProcessError);
-    ADD_ERRNO(ChildProcessError, ECHILD);
     POST_INIT(ConnectionAbortedError);
-    ADD_ERRNO(ConnectionAbortedError, ECONNABORTED);
     POST_INIT(ConnectionRefusedError);
-    ADD_ERRNO(ConnectionRefusedError, ECONNREFUSED);
     POST_INIT(ConnectionResetError);
-    ADD_ERRNO(ConnectionResetError, ECONNRESET);
     POST_INIT(FileExistsError);
-    ADD_ERRNO(FileExistsError, EEXIST);
     POST_INIT(FileNotFoundError);
-    ADD_ERRNO(FileNotFoundError, ENOENT);
     POST_INIT(IsADirectoryError);
-    ADD_ERRNO(IsADirectoryError, EISDIR);
     POST_INIT(NotADirectoryError);
-    ADD_ERRNO(NotADirectoryError, ENOTDIR);
     POST_INIT(InterruptedError);
-    ADD_ERRNO(InterruptedError, EINTR);
     POST_INIT(PermissionError);
-    ADD_ERRNO(PermissionError, EACCES);
-    ADD_ERRNO(PermissionError, EPERM);
     POST_INIT(ProcessLookupError);
-    ADD_ERRNO(ProcessLookupError, ESRCH);
     POST_INIT(TimeoutError);
-    ADD_ERRNO(TimeoutError, ETIMEDOUT);
 
-    if (preallocate_memerrors() < 0) {
-        return _Py_INIT_ERR("Could not preallocate MemoryError object");
-    }
     return _Py_INIT_OK();
 
-#undef PRE_INIT
 #undef POST_INIT
 #undef INIT_ALIAS
-#undef ADD_ERRNO
 }
 
 void

Objects/fileobject.c

@@ -359,6 +359,9 @@ stdprinter_write(PyStdPrinter_Object *self, PyObject *args)
     Py_ssize_t n;
     int err;
 
+    /* The function can clear the current exception */
+    assert(!PyErr_Occurred());
+
     if (self->fd < 0) {
         /* fd might be invalid on Windows
          * I can't raise an exception here. It may lead to an
@@ -367,10 +370,11 @@ stdprinter_write(PyStdPrinter_Object *self, PyObject *args)
         Py_RETURN_NONE;
     }
 
-    if (!PyArg_ParseTuple(args, "U", &unicode))
+    if (!PyArg_ParseTuple(args, "U", &unicode)) {
         return NULL;
+    }
 
-    /* encode Unicode to UTF-8 */
+    /* Encode Unicode to UTF-8/surrogateescape */
     str = PyUnicode_AsUTF8AndSize(unicode, &n);
     if (str == NULL) {
         PyErr_Clear();

Objects/floatobject.c

@@ -1999,8 +1999,9 @@ _PyFloat_Init(void)
 
     /* Init float info */
     if (FloatInfoType.tp_name == NULL) {
-        if (PyStructSequence_InitType2(&FloatInfoType, &floatinfo_desc) < 0)
+        if (PyStructSequence_InitType2(&FloatInfoType, &floatinfo_desc) < 0) {
             return 0;
+        }
     }
     return 1;
 }

Objects/longobject.c

@@ -5635,8 +5635,9 @@ _PyLong_Init(void)
 
     /* initialize int_info */
     if (Int_InfoType.tp_name == NULL) {
-        if (PyStructSequence_InitType2(&Int_InfoType, &int_info_desc) < 0)
+        if (PyStructSequence_InitType2(&Int_InfoType, &int_info_desc) < 0) {
             return 0;
+        }
     }
 
     return 1;