bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)

* Add test_embed.test_init_setpath_config(): test Py_SetPath()
  with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
  Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import  and base_executable
  fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
  calculated base_executable: _PyConfig_InitPathConfig() copies
  executable to base_executable.
* Complete path config documentation.

Author: vstinner

Doc/c-api/init_config.rst

@@ -864,37 +864,46 @@ Path Configuration
 
 :c:type:`PyConfig` contains multiple fields for the path configuration:
 
-* Path configuration input fields:
+* Path configuration inputs:
 
   * :c:member:`PyConfig.home`
   * :c:member:`PyConfig.pathconfig_warnings`
   * :c:member:`PyConfig.program_name`
   * :c:member:`PyConfig.pythonpath_env`
+  * current working directory: to get absolute paths
+  * ``PATH`` environment variable to get the program full path
+    (from :c:member:`PyConfig.program_name`)
+  * ``__PYVENV_LAUNCHER__`` environment variable
+  * (Windows only) Application paths in the registry under
+    "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and
+    HKEY_LOCAL_MACHINE (where X.Y is the Python version).
 
 * Path configuration output fields:
 
+  * :c:member:`PyConfig.base_exec_prefix`
   * :c:member:`PyConfig.base_executable`
+  * :c:member:`PyConfig.base_prefix`
   * :c:member:`PyConfig.exec_prefix`
   * :c:member:`PyConfig.executable`
-  * :c:member:`PyConfig.prefix`
   * :c:member:`PyConfig.module_search_paths_set`,
     :c:member:`PyConfig.module_search_paths`
+  * :c:member:`PyConfig.prefix`
 
-If at least one "output field" is not set, Python computes the path
+If at least one "output field" is not set, Python calculates the path
 configuration to fill unset fields. If
 :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
 :c:member:`~PyConfig.module_search_paths` is overridden and
 :c:member:`~PyConfig.module_search_paths_set` is set to 1.
 
-It is possible to completely ignore the function computing the default
+It is possible to completely ignore the function calculating the default
 path configuration by setting explicitly all path configuration output
 fields listed above. A string is considered as set even if it is non-empty.
 ``module_search_paths`` is considered as set if
 ``module_search_paths_set`` is set to 1. In this case, path
 configuration input fields are ignored as well.
 
 Set :c:member:`~PyConfig.pathconfig_warnings` to 0 to suppress warnings when
-computing the path configuration (Unix only, Windows does not log any warning).
+calculating the path configuration (Unix only, Windows does not log any warning).
 
 If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix`
 fields are not set, they inherit their value from :c:member:`~PyConfig.prefix`

Include/internal/pycore_pathconfig.h

@@ -19,19 +19,26 @@ typedef struct _PyPathConfig {
     wchar_t *program_name;
     /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
     wchar_t *home;
+#ifdef MS_WINDOWS
     /* isolated and site_import are used to set Py_IsolatedFlag and
        Py_NoSiteFlag flags on Windows in read_pth_file(). These fields
        are ignored when their value are equal to -1 (unset). */
     int isolated;
     int site_import;
     /* Set when a venv is detected */
     wchar_t *base_executable;
+#endif
 } _PyPathConfig;
 
-#define _PyPathConfig_INIT \
-    {.module_search_path = NULL, \
-     .isolated = -1, \
-     .site_import = -1}
+#ifdef MS_WINDOWS
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL, \
+       .isolated = -1, \
+       .site_import = -1}
+#else
+#  define _PyPathConfig_INIT \
+      {.module_search_path = NULL}
+#endif
 /* Note: _PyPathConfig_INIT sets other fields to 0/NULL */
 
 PyAPI_DATA(_PyPathConfig) _Py_path_config;

Lib/test/test_embed.py

@@ -635,16 +635,19 @@ def check_global_config(self, configs):
         self.assertEqual(configs['global_config'], expected)
 
     def check_all_configs(self, testname, expected_config=None,
-                     expected_preconfig=None, modify_path_cb=None, stderr=None,
-                     *, api, env=None, ignore_stderr=False, cwd=None):
+                          expected_preconfig=None, modify_path_cb=None,
+                          stderr=None, *, api, preconfig_api=None,
+                          env=None, ignore_stderr=False, cwd=None):
         new_env = remove_python_envvars()
         if env is not None:
             new_env.update(env)
         env = new_env
 
-        if api == API_ISOLATED:
+        if preconfig_api is None:
+            preconfig_api = api
+        if preconfig_api == API_ISOLATED:
             default_preconfig = self.PRE_CONFIG_ISOLATED
-        elif api == API_PYTHON:
+        elif preconfig_api == API_PYTHON:
             default_preconfig = self.PRE_CONFIG_PYTHON
         else:
             default_preconfig = self.PRE_CONFIG_COMPAT
@@ -1002,8 +1005,21 @@ def test_init_dont_parse_argv(self):
         self.check_all_configs("test_init_dont_parse_argv", config, pre_config,
                                api=API_PYTHON)
 
+    def default_program_name(self, config):
+        if MS_WINDOWS:
+            program_name = 'python'
+            executable = self.test_exe
+        else:
+            program_name = 'python3'
+            executable = shutil.which(program_name) or ''
+        config.update({
+            'program_name': program_name,
+            'base_executable': executable,
+            'executable': executable,
+        })
+
     def test_init_setpath(self):
-        # Test Py_SetProgramName() + Py_SetPath()
+        # Test Py_SetPath()
         config = self._get_expected_config()
         paths = config['config']['module_search_paths']
 
@@ -1014,11 +1030,38 @@ def test_init_setpath(self):
             'exec_prefix': '',
             'base_exec_prefix': '',
         }
+        self.default_program_name(config)
         env = {'TESTPATH': os.path.pathsep.join(paths)}
         self.check_all_configs("test_init_setpath", config,
                                api=API_COMPAT, env=env,
                                ignore_stderr=True)
 
+    def test_init_setpath_config(self):
+        # Test Py_SetPath() with PyConfig
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+
+        config = {
+            # set by Py_SetPath()
+            'module_search_paths': paths,
+            'prefix': '',
+            'base_prefix': '',
+            'exec_prefix': '',
+            'base_exec_prefix': '',
+            # overriden by PyConfig
+            'program_name': 'conf_program_name',
+            'base_executable': 'conf_executable',
+            'executable': 'conf_executable',
+        }
+        env = {'TESTPATH': os.path.pathsep.join(paths)}
+        # Py_SetPath() preinitialized Python using the compat API,
+        # so we need preconfig_api=API_COMPAT.
+        self.check_all_configs("test_init_setpath_config", config,
+                               api=API_PYTHON,
+                               preconfig_api=API_COMPAT,
+                               env=env,
+                               ignore_stderr=True)
+
     def module_search_paths(self, prefix=None, exec_prefix=None):
         config = self._get_expected_config()
         if prefix is None:
@@ -1067,8 +1110,7 @@ def tmpdir_with_python(self):
             yield tmpdir
 
     def test_init_setpythonhome(self):
-        # Test Py_SetPythonHome(home) + PYTHONPATH env var
-        # + Py_SetProgramName()
+        # Test Py_SetPythonHome(home) with PYTHONPATH env var
         config = self._get_expected_config()
         paths = config['config']['module_search_paths']
         paths_str = os.path.pathsep.join(paths)
@@ -1095,7 +1137,8 @@ def test_init_setpythonhome(self):
             'base_exec_prefix': exec_prefix,
             'pythonpath_env': paths_str,
         }
-        env = {'TESTHOME': home, 'TESTPATH': paths_str}
+        self.default_program_name(config)
+        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 

PC/getpathp.c

@@ -1099,11 +1099,11 @@ calculate_free(PyCalculatePath *calculate)
    - __PYVENV_LAUNCHER__ environment variable
    - GetModuleFileNameW(NULL): fully qualified path of the executable file of
      the current process
-   - .pth configuration file
+   - ._pth configuration file
    - pyvenv.cfg configuration file
    - Registry key "Software\Python\PythonCore\X.Y\PythonPath"
-     of HKEY_LOCAL_MACHINE and HKEY_CURRENT_USER where X.Y is the Python
-     version (major.minor).
+     of HKEY_CURRENT_USER and HKEY_LOCAL_MACHINE where X.Y is the Python
+     version.
 
    Outputs, 'pathconfig' fields:
 

Programs/_testembed.c

@@ -1425,8 +1425,6 @@ static int test_init_sys_add(void)
 
 static int test_init_setpath(void)
 {
-    Py_SetProgramName(PROGRAM_NAME);
-
     char *env = getenv("TESTPATH");
     if (!env) {
         fprintf(stderr, "missing TESTPATH env var\n");
@@ -1448,23 +1446,35 @@ static int test_init_setpath(void)
 }
 
 
-static int mysetenv(const char *name, const char *value)
+static int test_init_setpath_config(void)
 {
-    size_t len = strlen(name) + 1 + strlen(value) + 1;
-    char *env = PyMem_RawMalloc(len);
-    if (env == NULL) {
-        fprintf(stderr, "out of memory\n");
-        return -1;
+    char *env = getenv("TESTPATH");
+    if (!env) {
+        fprintf(stderr, "missing TESTPATH env var\n");
+        return 1;
     }
-    strcpy(env, name);
-    strcat(env, "=");
-    strcat(env, value);
+    wchar_t *path = Py_DecodeLocale(env, NULL);
+    if (path == NULL) {
+        fprintf(stderr, "failed to decode TESTPATH\n");
+        return 1;
+    }
+    Py_SetPath(path);
+    PyMem_RawFree(path);
+    putenv("TESTPATH=");
 
-    putenv(env);
+    PyStatus status;
+    PyConfig config;
 
-    /* Don't call PyMem_RawFree(env), but leak env memory block:
-       putenv() does not copy the string. */
+    status = PyConfig_InitPythonConfig(&config);
+    if (PyStatus_Exception(status)) {
+        Py_ExitStatusException(status);
+    }
+    config_set_string(&config, &config.program_name, L"conf_program_name");
+    config_set_string(&config, &config.executable, L"conf_executable");
+    init_from_config_clear(&config);
 
+    dump_config();
+    Py_Finalize();
     return 0;
 }
 
@@ -1485,19 +1495,6 @@ static int test_init_setpythonhome(void)
     PyMem_RawFree(home);
     putenv("TESTHOME=");
 
-    char *path = getenv("TESTPATH");
-    if (!path) {
-        fprintf(stderr, "missing TESTPATH env var\n");
-        return 1;
-    }
-
-    if (mysetenv("PYTHONPATH", path) < 0) {
-        return 1;
-    }
-    putenv("TESTPATH=");
-
-    Py_SetProgramName(PROGRAM_NAME);
-
     Py_Initialize();
     dump_config();
     Py_Finalize();
@@ -1642,6 +1639,7 @@ static struct TestCase TestCases[] = {
     {"test_init_main", test_init_main},
     {"test_init_sys_add", test_init_sys_add},
     {"test_init_setpath", test_init_setpath},
+    {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_run_main", test_run_main},
 

Python/pathconfig.c

@@ -58,7 +58,10 @@ pathconfig_clear(_PyPathConfig *config)
     CLEAR(config->module_search_path);
     CLEAR(config->program_name);
     CLEAR(config->home);
+#ifdef MS_WINDOWS
     CLEAR(config->base_executable);
+#endif
+
 #undef CLEAR
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
@@ -83,9 +86,11 @@ pathconfig_copy(_PyPathConfig *config, const _PyPathConfig *config2)
     COPY_ATTR(module_search_path);
     COPY_ATTR(program_name);
     COPY_ATTR(home);
+#ifdef MS_WINDOWS
     config->isolated = config2->isolated;
     config->site_import = config2->site_import;
     COPY_ATTR(base_executable);
+#endif
 
 #undef COPY_ATTR
 
@@ -189,12 +194,14 @@ pathconfig_set_from_config(_PyPathConfig *pathconfig, const PyConfig *config)
             } \
         }
 
-    COPY_CONFIG(base_executable, base_executable);
     COPY_CONFIG(program_full_path, executable);
     COPY_CONFIG(prefix, prefix);
     COPY_CONFIG(exec_prefix, exec_prefix);
     COPY_CONFIG(program_name, program_name);
     COPY_CONFIG(home, home);
+#ifdef MS_WINDOWS
+    COPY_CONFIG(base_executable, base_executable);
+#endif
 
 #undef COPY_CONFIG
 
@@ -330,18 +337,32 @@ config_calculate_pathconfig(PyConfig *config)
             } \
         }
 
+#ifdef MS_WINDOWS
+    if (config->executable != NULL && config->base_executable == NULL) {
+        /* If executable is set explicitly in the configuration,
+           ignore calculated base_executable: _PyConfig_InitPathConfig()
+           will copy executable to base_executable */
+    }
+    else {
+        COPY_ATTR(base_executable, base_executable);
+    }
+#endif
+
     COPY_ATTR(program_full_path, executable);
     COPY_ATTR(prefix, prefix);
     COPY_ATTR(exec_prefix, exec_prefix);
-    COPY_ATTR(base_executable, base_executable);
+
 #undef COPY_ATTR
 
+#ifdef MS_WINDOWS
+    /* If a ._pth file is found: isolated and site_import are overriden */
     if (pathconfig.isolated != -1) {
         config->isolated = pathconfig.isolated;
     }
     if (pathconfig.site_import != -1) {
         config->site_import = pathconfig.site_import;
     }
+#endif
 
     status = _PyStatus_OK();
     goto done;
@@ -360,9 +381,9 @@ _PyConfig_InitPathConfig(PyConfig *config)
 {
     /* Do we need to calculate the path? */
     if (!config->module_search_paths_set
-        || (config->executable == NULL)
-        || (config->prefix == NULL)
-        || (config->exec_prefix == NULL))
+        || config->executable == NULL
+        || config->prefix == NULL
+        || config->exec_prefix == NULL)
     {
         PyStatus status = config_calculate_pathconfig(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -442,7 +463,9 @@ pathconfig_global_init(void)
     assert(_Py_path_config.module_search_path != NULL);
     assert(_Py_path_config.program_name != NULL);
     /* home can be NULL */
+#ifdef MS_WINDOWS
     assert(_Py_path_config.base_executable != NULL);
+#endif
 }