Skip to content

Commit 6d1c467

Browse files
vstinnervstinner
authored
bpo-36763: Fix Python preinitialization (GH-13432)
* Add _PyPreConfig.parse_argv * Add _PyCoreConfig._config_init field and _PyCoreConfigInitEnum enum type * Initialization functions: reject preconfig=NULL and config=NULL * Add config parameter to _PyCoreConfig_DecodeLocaleErr(): pass config->argv to _Py_PreInitializeFromPyArgv(), to parse config command line arguments in preinitialization. * Add config parameter to _PyCoreConfig_SetString(). It now preinitializes Python. * _PyCoreConfig_SetPyArgv() now also preinitializes Python for wide argv * Fix _Py_PreInitializeFromCoreConfig(): don't pass args to _Py_PreInitializeFromPyArgv() if config.parse_argv=0. * Use "char * const *" and "wchar_t * const *" types for 'argv' parameters and _PyArgv.argv. * Add unit test on preinitialization from argv. * _PyPreConfig.allocator type becomes int * Add _PyPreConfig_InitFromPreConfig() and _PyPreConfig_InitFromCoreConfig() helper functions
1 parent 6d965b3 commit 6d1c467

File tree

8 files changed

+476
-178
lines changed

8 files changed

+476
-178
lines changed

‎Include/cpython/coreconfig.h

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@ typedef struct {
4444
int _config_version; /* Internal configuration version,
4545
used for ABI compatibility */
4646

47+
/* Parse _Py_PreInitializeFromArgs() arguments?
48+
See _PyCoreConfig.parse_argv */
49+
int parse_argv;
50+
4751
/* If greater than 0, enable isolated mode: sys.path contains
4852
neither the script's directory nor the user's site-packages directory.
4953
@@ -111,8 +115,9 @@ typedef struct {
111115

112116
int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */
113117

114-
/* Memory allocator: PYTHONMALLOC env var */
115-
PyMemAllocatorName allocator;
118+
/* Memory allocator: PYTHONMALLOC env var.
119+
See PyMemAllocatorName for valid values. */
120+
int allocator;
116121
} _PyPreConfig;
117122

118123
PyAPI_FUNC(void) _PyPreConfig_InitPythonConfig(_PyPreConfig *config);
@@ -121,9 +126,16 @@ PyAPI_FUNC(void) _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config);
121126

122127
/* --- _PyCoreConfig ---------------------------------------------- */
123128

129+
typedef enum {
130+
_PyCoreConfig_INIT = 0,
131+
_PyCoreConfig_INIT_PYTHON = 1,
132+
_PyCoreConfig_INIT_ISOLATED = 2
133+
} _PyCoreConfigInitEnum;
134+
124135
typedef struct {
125136
int _config_version; /* Internal configuration version,
126137
used for ABI compatibility */
138+
int _config_init; /* _PyCoreConfigInitEnum value */
127139

128140
int isolated; /* Isolated mode? see _PyPreConfig.isolated */
129141
int use_environment; /* Use environment variables? see _PyPreConfig.use_environment */
@@ -401,19 +413,21 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config);
401413
PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config);
402414
PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
403415
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetString(
416+
_PyCoreConfig *config,
404417
wchar_t **config_str,
405418
const wchar_t *str);
406419
PyAPI_FUNC(_PyInitError) _PyCoreConfig_DecodeLocale(
420+
_PyCoreConfig *config,
407421
wchar_t **config_str,
408422
const char *str);
409423
PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
410424
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetArgv(
411425
_PyCoreConfig *config,
412426
Py_ssize_t argc,
413-
char **argv);
427+
char * const *argv);
414428
PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideArgv(_PyCoreConfig *config,
415429
Py_ssize_t argc,
416-
wchar_t **argv);
430+
wchar_t * const *argv);
417431

418432
#ifdef __cplusplus
419433
}

‎Include/cpython/pylifecycle.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
3535
PyAPI_FUNC(_PyInitError) _Py_InitializeFromArgs(
3636
const _PyCoreConfig *config,
3737
Py_ssize_t argc,
38-
char **argv);
38+
char * const *argv);
3939
PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs(
4040
const _PyCoreConfig *config,
4141
Py_ssize_t argc,
42-
wchar_t **argv);
42+
wchar_t * const *argv);
4343
PyAPI_FUNC(_PyInitError) _Py_InitializeMain(void);
4444

4545
PyAPI_FUNC(int) _Py_RunMain(void);

‎Include/internal/pycore_coreconfig.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ PyAPI_FUNC(int) _PyWstrList_Extend(_PyWstrList *list,
6363
typedef struct {
6464
Py_ssize_t argc;
6565
int use_bytes_argv;
66-
char **bytes_argv;
67-
wchar_t **wchar_argv;
66+
char * const *bytes_argv;
67+
wchar_t * const *wchar_argv;
6868
} _PyArgv;
6969

7070
PyAPI_FUNC(_PyInitError) _PyArgv_AsWstrList(const _PyArgv *args,
@@ -121,6 +121,12 @@ PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline,
121121
/* --- _PyPreConfig ----------------------------------------------- */
122122

123123
PyAPI_FUNC(void) _PyPreConfig_Init(_PyPreConfig *config);
124+
PyAPI_FUNC(void) _PyPreConfig_InitFromCoreConfig(
125+
_PyPreConfig *config,
126+
const _PyCoreConfig *coreconfig);
127+
PyAPI_FUNC(void) _PyPreConfig_InitFromPreConfig(
128+
_PyPreConfig *config,
129+
const _PyPreConfig *config2);
124130
PyAPI_FUNC(void) _PyPreConfig_Copy(_PyPreConfig *config,
125131
const _PyPreConfig *config2);
126132
PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);

‎Lib/test/test_embed.py

Lines changed: 74 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
PYMEM_ALLOCATOR_DEBUG = 2
1818
PYMEM_ALLOCATOR_MALLOC = 3
1919

20+
CONFIG_INIT = 0
21+
CONFIG_INIT_PYTHON = 1
22+
CONFIG_INIT_ISOLATED = 2
23+
2024

2125
class EmbeddingTestsMixin:
2226
def setUp(self):
@@ -280,20 +284,26 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
280284

281285
DEFAULT_PRE_CONFIG = {
282286
'allocator': PYMEM_ALLOCATOR_NOT_SET,
287+
'parse_argv': 0,
283288
'configure_locale': 1,
284289
'coerce_c_locale': 0,
285290
'coerce_c_locale_warn': 0,
286291
'utf8_mode': 0,
287292
}
293+
if MS_WINDOWS:
294+
DEFAULT_PRE_CONFIG.update({
295+
'legacy_windows_fs_encoding': 0,
296+
})
297+
PYTHON_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
298+
parse_argv=1,
299+
)
288300
ISOLATED_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
289301
configure_locale=0,
290302
isolated=1,
291303
use_environment=0,
292304
utf8_mode=0,
293305
dev_mode=0,
294306
)
295-
if MS_WINDOWS:
296-
ISOLATED_PRE_CONFIG['legacy_windows_fs_encoding'] = 0
297307

298308
COPY_PRE_CONFIG = [
299309
'dev_mode',
@@ -302,6 +312,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
302312
]
303313

304314
DEFAULT_CORE_CONFIG = {
315+
'_config_init': CONFIG_INIT,
305316
'isolated': 0,
306317
'use_environment': 1,
307318
'dev_mode': 0,
@@ -365,9 +376,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
365376
'_init_main': 1,
366377
}
367378
if MS_WINDOWS:
368-
DEFAULT_PRE_CONFIG.update({
369-
'legacy_windows_fs_encoding': 0,
370-
})
371379
DEFAULT_CORE_CONFIG.update({
372380
'legacy_windows_stdio': 0,
373381
})
@@ -439,13 +447,14 @@ def main_xoptions(self, xoptions_list):
439447

440448
def get_expected_config(self, expected_preconfig, expected, env, api,
441449
add_path=None):
442-
if api == "python":
450+
if api == CONFIG_INIT_PYTHON:
443451
default_config = self.PYTHON_CORE_CONFIG
444-
elif api == "isolated":
452+
elif api == CONFIG_INIT_ISOLATED:
445453
default_config = self.ISOLATED_CORE_CONFIG
446454
else:
447455
default_config = self.DEFAULT_CORE_CONFIG
448456
expected = dict(default_config, **expected)
457+
expected['_config_init'] = api
449458

450459
code = textwrap.dedent('''
451460
import json
@@ -519,9 +528,7 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
519528
return expected
520529

521530
def check_pre_config(self, config, expected):
522-
pre_config = dict(config['pre_config'])
523-
core_config = dict(config['core_config'])
524-
self.assertEqual(pre_config, expected)
531+
self.assertEqual(config['pre_config'], expected)
525532

526533
def check_core_config(self, config, expected):
527534
core_config = dict(config['core_config'])
@@ -554,7 +561,7 @@ def check_global_config(self, config):
554561
self.assertEqual(config['global_config'], expected)
555562

556563
def check_config(self, testname, expected_config=None, expected_preconfig=None,
557-
add_path=None, stderr=None, api="default"):
564+
add_path=None, stderr=None, api=CONFIG_INIT):
558565
env = dict(os.environ)
559566
# Remove PYTHON* environment variables to get deterministic environment
560567
for key in list(env):
@@ -565,8 +572,10 @@ def check_config(self, testname, expected_config=None, expected_preconfig=None,
565572
env['PYTHONCOERCECLOCALE'] = '0'
566573
env['PYTHONUTF8'] = '0'
567574

568-
if api == "isolated":
575+
if api == CONFIG_INIT_ISOLATED:
569576
default_preconfig = self.ISOLATED_PRE_CONFIG
577+
elif api == CONFIG_INIT_PYTHON:
578+
default_preconfig = self.PYTHON_PRE_CONFIG
570579
else:
571580
default_preconfig = self.DEFAULT_PRE_CONFIG
572581
if expected_preconfig is None:
@@ -719,15 +728,46 @@ def test_init_dev_mode(self):
719728
'dev_mode': 1,
720729
'warnoptions': ['default'],
721730
}
722-
self.check_config("init_dev_mode", config, preconfig, api="python")
731+
self.check_config("init_dev_mode", config, preconfig,
732+
api=CONFIG_INIT_PYTHON)
733+
734+
def test_preinit_parse_argv(self):
735+
# Pre-initialize implicitly using argv: make sure that -X dev
736+
# is used to configure the allocation in preinitialization
737+
preconfig = {
738+
'allocator': PYMEM_ALLOCATOR_DEBUG,
739+
}
740+
config = {
741+
'argv': ['script.py'],
742+
'run_filename': 'script.py',
743+
'dev_mode': 1,
744+
'faulthandler': 1,
745+
'warnoptions': ['default'],
746+
'xoptions': ['dev'],
747+
}
748+
self.check_config("preinit_parse_argv", config, preconfig,
749+
api=CONFIG_INIT_PYTHON)
750+
751+
def test_preinit_dont_parse_argv(self):
752+
# -X dev must be ignored by isolated preconfiguration
753+
preconfig = {
754+
'isolated': 0,
755+
}
756+
config = {
757+
'argv': ["python3", "-E", "-I",
758+
"-X", "dev", "-X", "utf8", "script.py"],
759+
'isolated': 0,
760+
}
761+
self.check_config("preinit_dont_parse_argv", config, preconfig,
762+
api=CONFIG_INIT_ISOLATED)
723763

724764
def test_init_isolated_flag(self):
725765
config = {
726766
'isolated': 1,
727767
'use_environment': 0,
728768
'user_site_directory': 0,
729769
}
730-
self.check_config("init_isolated_flag", config, api="python")
770+
self.check_config("init_isolated_flag", config, api=CONFIG_INIT_PYTHON)
731771

732772
def test_preinit_isolated1(self):
733773
# _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set
@@ -747,25 +787,30 @@ def test_preinit_isolated2(self):
747787
}
748788
self.check_config("preinit_isolated2", config)
749789

790+
def test_preinit_isolated_config(self):
791+
self.check_config("preinit_isolated_config", api=CONFIG_INIT_ISOLATED)
792+
750793
def test_init_isolated_config(self):
751-
self.check_config("init_isolated_config", api="isolated")
794+
self.check_config("init_isolated_config", api=CONFIG_INIT_ISOLATED)
752795

753796
def test_init_python_config(self):
754-
self.check_config("init_python_config", api="python")
797+
self.check_config("init_python_config", api=CONFIG_INIT_PYTHON)
755798

756799
def test_init_dont_configure_locale(self):
757800
# _PyPreConfig.configure_locale=0
758801
preconfig = {
759802
'configure_locale': 0,
760803
}
761-
self.check_config("init_dont_configure_locale", {}, preconfig, api="python")
804+
self.check_config("init_dont_configure_locale", {}, preconfig,
805+
api=CONFIG_INIT_PYTHON)
762806

763807
def test_init_read_set(self):
764808
core_config = {
765809
'program_name': './init_read_set',
766810
'executable': 'my_executable',
767811
}
768-
self.check_config("init_read_set", core_config, api="python",
812+
self.check_config("init_read_set", core_config,
813+
api=CONFIG_INIT_PYTHON,
769814
add_path="init_read_set_path")
770815

771816
def test_init_run_main(self):
@@ -777,7 +822,8 @@ def test_init_run_main(self):
777822
'run_command': code + '\n',
778823
'parse_argv': 1,
779824
}
780-
self.check_config("init_run_main", core_config, api="python")
825+
self.check_config("init_run_main", core_config,
826+
api=CONFIG_INIT_PYTHON)
781827

782828
def test_init_main(self):
783829
code = ('import _testinternalcapi, json; '
@@ -789,7 +835,8 @@ def test_init_main(self):
789835
'parse_argv': 1,
790836
'_init_main': 0,
791837
}
792-
self.check_config("init_main", core_config, api="python",
838+
self.check_config("init_main", core_config,
839+
api=CONFIG_INIT_PYTHON,
793840
stderr="Run Python code before _Py_InitializeMain")
794841

795842
def test_init_parse_argv(self):
@@ -800,15 +847,20 @@ def test_init_parse_argv(self):
800847
'run_command': 'pass\n',
801848
'use_environment': 0,
802849
}
803-
self.check_config("init_parse_argv", core_config, api="python")
850+
self.check_config("init_parse_argv", core_config,
851+
api=CONFIG_INIT_PYTHON)
804852

805853
def test_init_dont_parse_argv(self):
854+
pre_config = {
855+
'parse_argv': 0,
856+
}
806857
core_config = {
807858
'parse_argv': 0,
808859
'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
809860
'program_name': './argv0',
810861
}
811-
self.check_config("init_dont_parse_argv", core_config, api="python")
862+
self.check_config("init_dont_parse_argv", core_config, pre_config,
863+
api=CONFIG_INIT_PYTHON)
812864

813865

814866
if __name__ == "__main__":

0 commit comments

Comments
 (0)