bpo-36142: _PyPreConfig_Read() sets LC_CTYPE (GH-12188)

* _PyPreConfig_Read() now sets temporarily LC_CTYPE to the user
  preferred locale, as _PyPreConfig_Write() will do permanentely.
* Fix _PyCoreConfig_Clear(): clear run_xxx attributes
* _Py_SetArgcArgv() doesn't have to be exported
* _PyCoreConfig_SetGlobalConfig() no longer applies preconfig

Author: vstinner

Include/internal/pycore_coreconfig.h

@@ -32,7 +32,6 @@ PyAPI_FUNC(_PyInitError) _PyArgv_Decode(const _PyArgv *args,
 /* --- Py_GetArgcArgv() helpers ----------------------------------- */
 
 PyAPI_FUNC(void) _Py_ClearArgcArgv(void);
-PyAPI_FUNC(int) _Py_SetArgcArgv(int argc, wchar_t * const *argv);
 
 /* --- _PyPreConfig ----------------------------------------------- */
 

Python/coreconfig.c

@@ -387,7 +387,7 @@ _Py_ClearArgcArgv(void)
 }
 
 
-int
+static int
 _Py_SetArgcArgv(int argc, wchar_t * const *argv)
 {
     int res;
@@ -473,6 +473,9 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
     CLEAR(config->filesystem_errors);
     CLEAR(config->stdio_encoding);
     CLEAR(config->stdio_errors);
+    CLEAR(config->run_command);
+    CLEAR(config->run_module);
+    CLEAR(config->run_filename);
 #undef CLEAR
 #undef CLEAR_WSTRLIST
 }
@@ -677,8 +680,6 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
 void
 _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config)
 {
-    _PyPreConfig_SetGlobalConfig(&config->preconfig);
-
 #define COPY_FLAG(ATTR, VAR) \
         if (config->ATTR != -1) { \
             VAR = config->ATTR; \
@@ -812,6 +813,7 @@ config_init_executable(_PyCoreConfig *config)
     return _Py_INIT_OK();
 }
 
+
 static const wchar_t*
 config_get_xoption(const _PyCoreConfig *config, wchar_t *name)
 {
@@ -897,35 +899,34 @@ config_wstr_to_int(const wchar_t *wstr, int *result)
 static _PyInitError
 config_read_env_vars(_PyCoreConfig *config)
 {
-#define get_env_flag(CONFIG, ATTR, NAME) \
-        _Py_get_env_flag(&(CONFIG)->preconfig, (ATTR), (NAME))
+    _PyPreConfig *preconfig = &config->preconfig;
 
     /* Get environment variables */
-    get_env_flag(config, &config->parser_debug, "PYTHONDEBUG");
-    get_env_flag(config, &config->verbose, "PYTHONVERBOSE");
-    get_env_flag(config, &config->optimization_level, "PYTHONOPTIMIZE");
-    get_env_flag(config, &config->inspect, "PYTHONINSPECT");
+    _Py_get_env_flag(preconfig, &config->parser_debug, "PYTHONDEBUG");
+    _Py_get_env_flag(preconfig, &config->verbose, "PYTHONVERBOSE");
+    _Py_get_env_flag(preconfig, &config->optimization_level, "PYTHONOPTIMIZE");
+    _Py_get_env_flag(preconfig, &config->inspect, "PYTHONINSPECT");
 
     int dont_write_bytecode = 0;
-    get_env_flag(config, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE");
+    _Py_get_env_flag(preconfig, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE");
     if (dont_write_bytecode) {
         config->write_bytecode = 0;
     }
 
     int no_user_site_directory = 0;
-    get_env_flag(config, &no_user_site_directory, "PYTHONNOUSERSITE");
+    _Py_get_env_flag(preconfig, &no_user_site_directory, "PYTHONNOUSERSITE");
     if (no_user_site_directory) {
         config->user_site_directory = 0;
     }
 
     int unbuffered_stdio = 0;
-    get_env_flag(config, &unbuffered_stdio, "PYTHONUNBUFFERED");
+    _Py_get_env_flag(preconfig, &unbuffered_stdio, "PYTHONUNBUFFERED");
     if (unbuffered_stdio) {
         config->buffered_stdio = 0;
     }
 
 #ifdef MS_WINDOWS
-    get_env_flag(config, &config->legacy_windows_stdio,
+    _Py_get_env_flag(preconfig, &config->legacy_windows_stdio,
                  "PYTHONLEGACYWINDOWSSTDIO");
 #endif
 
@@ -952,8 +953,6 @@ config_read_env_vars(_PyCoreConfig *config)
     }
 
     return _Py_INIT_OK();
-
-#undef get_env_flag
 }
 
 
@@ -1333,10 +1332,7 @@ _PyCoreConfig_ReadPreConfig(_PyCoreConfig *config)
 }
 
 
-/* Read the configuration into _PyCoreConfig and initialize the LC_CTYPE
-   locale: enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538).
-
-   Read the configuration from:
+/* Read the configuration into _PyCoreConfig from:
 
    * Environment variables
    * Py_xxx global configuration variables
@@ -1497,8 +1493,6 @@ config_init_stdio(const _PyCoreConfig *config)
 
 /* Write the configuration:
 
-   - coerce the LC_CTYPE locale (PEP 538)
-   - UTF-8 mode (PEP 540)
    - set Py_xxx global configuration variables
    - initialize C standard streams (stdin, stdout, stderr) */
 void
@@ -2110,10 +2104,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
 }
 
 
-/* Read the configuration into _PyCoreConfig and initialize the LC_CTYPE
-   locale: enable UTF-8 mode (PEP 540) and/or coerce the C locale (PEP 538).
-
-   Read the configuration from:
+/* Read the configuration into _PyCoreConfig from:
 
    * Command line arguments
    * Environment variables

Python/preconfig.c

@@ -472,10 +472,50 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 }
 
 
+static _PyInitError
+get_ctype_locale(char **locale_p)
+{
+    const char *loc = setlocale(LC_CTYPE, NULL);
+    if (loc == NULL) {
+        return _Py_INIT_ERR("failed to LC_CTYPE locale");
+    }
+
+    char *copy = _PyMem_RawStrdup(loc);
+    if (copy == NULL) {
+        return _Py_INIT_NO_MEMORY();
+    }
+
+    *locale_p = copy;
+    return _Py_INIT_OK();
+}
+
+
+/* Read the configuration from:
+
+   - environment variables
+   - Py_xxx global configuration variables
+   - the LC_CTYPE locale
+
+   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
 _PyInitError
 _PyPreConfig_Read(_PyPreConfig *config)
 {
-    return preconfig_read(config, NULL);
+    _PyInitError err;
+    char *old_loc;
+
+    err = get_ctype_locale(&old_loc);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    /* Set LC_CTYPE to the user preferred locale */
+    _Py_SetLocaleFromEnv(LC_CTYPE);
+
+    err = preconfig_read(config, NULL);
+
+    setlocale(LC_CTYPE, old_loc);
+
+    return err;
 }
 
 
@@ -604,7 +644,14 @@ preconfig_from_argv(_PyPreConfig *config, const _PyArgv *args)
 }
 
 
-/* Read the preconfiguration. */
+/* Read the configuration from:
+
+   - command line arguments
+   - environment variables
+   - Py_xxx global configuration variables
+   - the LC_CTYPE locale
+
+   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
 _PyInitError
 _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
 {
@@ -624,15 +671,8 @@ _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
     int locale_coerced = 0;
     int loops = 0;
 
-    /* copy LC_CTYPE locale */
-    const char *loc = setlocale(LC_CTYPE, NULL);
-    if (loc == NULL) {
-        err = _Py_INIT_ERR("failed to LC_CTYPE locale");
-        goto done;
-    }
-    init_ctype_locale = _PyMem_RawStrdup(loc);
-    if (init_ctype_locale == NULL) {
-        err = _Py_INIT_NO_MEMORY();
+    err = get_ctype_locale(&init_ctype_locale);
+    if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
@@ -767,15 +807,23 @@ _PyPreConfig_SetAllocator(_PyPreConfig *config)
 }
 
 
-/* Write the pre-configuration.
+/* Write the pre-configuration:
+
+   - set the memory allocators
+   - set Py_xxx global configuration variables
+   - set the LC_CTYPE locale (coerce C locale, PEP 538) and set the UTF-8 mode
+     (PEP 540)
 
    If the memory allocator is changed, config is re-allocated with new
-   allocator. So calling _PyPreConfig_Clear(config) is safe after this call. */
+   allocator. So calling _PyPreConfig_Clear(config) is safe after this call.
+
+   Do nothing if called after Py_Initialize(): ignore the new
+   pre-configuration. */
 _PyInitError
 _PyPreConfig_Write(_PyPreConfig *config)
 {
     if (_PyRuntime.core_initialized) {
-        /* bpo-34008: Calling Py_Main() after Py_Initialize() ignores
+        /* bpo-34008: Calling this functions after Py_Initialize() ignores
            the new configuration. */
         return _Py_INIT_OK();
     }

Python/pylifecycle.c

@@ -716,9 +716,6 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
 static _PyInitError
 pyinit_preconfig(_PyPreConfig *preconfig, const _PyPreConfig *src_preconfig)
 {
-    /* Set LC_CTYPE to the user preferred locale */
-    _Py_SetLocaleFromEnv(LC_CTYPE);
-
     if (_PyPreConfig_Copy(preconfig, src_preconfig) < 0) {
         return _Py_INIT_ERR("failed to copy pre config");
     }
@@ -736,10 +733,6 @@ static _PyInitError
 pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config,
                   PyInterpreterState **interp_p)
 {
-
-    /* Set LC_CTYPE to the user preferred locale */
-    _Py_SetLocaleFromEnv(LC_CTYPE);
-
     if (_PyCoreConfig_Copy(config, src_config) < 0) {
         return _Py_INIT_ERR("failed to copy core config");
     }