bpo-40854: Allow overriding sys.platlibdir via PYTHONPLATLIBDIR env-var (GH-20605) (GH-20725)

(cherry picked from commit 8f023a2)

Co-authored-by: Sandro Mani <[email protected]>

Author: vstinner

Doc/c-api/init_config.rst

@@ -437,6 +437,14 @@ PyConfig
 
       :data:`sys.base_prefix`.
 
+   .. c:member:: wchar_t* platlibdir
+
+      :data:`sys.platlibdir`: platform library directory name, set at configure time
+      by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR``
+      environment variable.
+
+      .. versionadded:: 3.9
+
    .. c:member:: int buffered_stdio
 
       If equals to 0, enable unbuffered mode, making the stdout and stderr
@@ -885,6 +893,7 @@ Path Configuration
 * Path configuration inputs:
 
   * :c:member:`PyConfig.home`
+  * :c:member:`PyConfig.platlibdir`
   * :c:member:`PyConfig.pathconfig_warnings`
   * :c:member:`PyConfig.program_name`
   * :c:member:`PyConfig.pythonpath_env`

Doc/using/cmdline.rst

@@ -538,6 +538,14 @@ conflict.
    within a Python program as the variable :data:`sys.path`.
 
 
+.. envvar:: PYTHONPLATLIBDIR
+
+   If this is set to a non-empty string, it overrides the :data:`sys.platlibdir`
+   value.
+
+   .. versionadded:: 3.9
+
+
 .. envvar:: PYTHONSTARTUP
 
    If this is the name of a readable file, the Python commands in that file are

Include/cpython/initconfig.h

@@ -388,6 +388,7 @@ typedef struct {
     wchar_t *base_prefix;       /* sys.base_prefix */
     wchar_t *exec_prefix;       /* sys.exec_prefix */
     wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
+    wchar_t *platlibdir;        /* sys.platlibdir */
 
     /* --- Parameter only used by Py_Main() ---------- */
 

Lib/test/test_embed.py

@@ -381,6 +381,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'exec_prefix': GET_DEFAULT_CONFIG,
         'base_exec_prefix': GET_DEFAULT_CONFIG,
         'module_search_paths': GET_DEFAULT_CONFIG,
+        'platlibdir': sys.platlibdir,
 
         'site_import': 1,
         'bytes_warning': 0,
@@ -586,13 +587,14 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
             if value is self.GET_DEFAULT_CONFIG:
                 expected[key] = config[key]
 
-        pythonpath_env = expected['pythonpath_env']
-        if pythonpath_env is not None:
-            paths = pythonpath_env.split(os.path.pathsep)
-            expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
-        if modify_path_cb is not None:
-            expected['module_search_paths'] = expected['module_search_paths'].copy()
-            modify_path_cb(expected['module_search_paths'])
+        if expected['module_search_paths'] is not self.IGNORE_CONFIG:
+            pythonpath_env = expected['pythonpath_env']
+            if pythonpath_env is not None:
+                paths = pythonpath_env.split(os.path.pathsep)
+                expected['module_search_paths'] = [*paths, *expected['module_search_paths']]
+            if modify_path_cb is not None:
+                expected['module_search_paths'] = expected['module_search_paths'].copy()
+                modify_path_cb(expected['module_search_paths'])
 
         for key in self.COPY_PRE_CONFIG:
             if key not in expected_preconfig:
@@ -770,6 +772,8 @@ def test_init_from_config(self):
             'buffered_stdio': 0,
             'user_site_directory': 0,
             'faulthandler': 1,
+            'platlibdir': 'my_platlibdir',
+            'module_search_paths': self.IGNORE_CONFIG,
 
             'check_hash_pycs_mode': 'always',
             'pathconfig_warnings': 0,
@@ -801,6 +805,8 @@ def test_init_compat_env(self):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            'platlibdir': 'env_platlibdir',
+            'module_search_paths': self.IGNORE_CONFIG,
             '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_compat_env", config, preconfig,
@@ -829,6 +835,8 @@ def test_init_python_env(self):
             'user_site_directory': 0,
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
+            'platlibdir': 'env_platlibdir',
+            'module_search_paths': self.IGNORE_CONFIG,
             '_use_peg_parser': 0,
         }
         self.check_all_configs("test_init_python_env", config, preconfig,

Makefile.pre.in

@@ -811,6 +811,11 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydt
 		$(MULTIARCH_CPPFLAGS) \
 		-o $@ $(srcdir)/Python/sysmodule.c
 
+Python/initconfig.o: $(srcdir)/Python/initconfig.c
+	$(CC) -c $(PY_CORE_CFLAGS) \
+		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
+		-o $@ $(srcdir)/Python/initconfig.c
+
 $(IO_OBJS): $(IO_H)
 
 .PHONY: regen-grammar

Misc/NEWS.d/next/Core and Builtins/2020-06-03-13-53-24.bpo-40854.O6vfQU.rst

@@ -0,0 +1 @@
+Allow overriding :data:`sys.platlibdir` via a new :envvar:`PYTHONPLATLIBDIR` environment variable.

Misc/python.man

@@ -413,6 +413,8 @@ inserted in the path in front of $PYTHONPATH.
 The search path can be manipulated from within a Python program as the
 variable
 .IR sys.path .
+.IP PYTHONPLATLIBDIR
+Override sys.platlibdir.
 .IP PYTHONSTARTUP
 If this is the name of a readable file, the Python commands in that
 file are executed before the first prompt is displayed in interactive

Modules/getpath.c

@@ -105,8 +105,8 @@ extern "C" {
 
 
 #if (!defined(PREFIX) || !defined(EXEC_PREFIX) \
-        || !defined(VERSION) || !defined(VPATH) || !defined(PLATLIBDIR))
-#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATLIBDIR macros must be defined"
+        || !defined(VERSION) || !defined(VPATH))
+#error "PREFIX, EXEC_PREFIX, VERSION and VPATH macros must be defined"
 #endif
 
 #ifndef LANDMARK
@@ -128,7 +128,6 @@ typedef struct {
     wchar_t *pythonpath_macro;         /* PYTHONPATH macro */
     wchar_t *prefix_macro;             /* PREFIX macro */
     wchar_t *exec_prefix_macro;        /* EXEC_PREFIX macro */
-    wchar_t *platlibdir_macro;         /* PLATLIBDIR macro */
     wchar_t *vpath_macro;              /* VPATH macro */
 
     wchar_t *lib_python;               /* "lib/pythonX.Y" */
@@ -138,6 +137,7 @@ typedef struct {
 
     int warnings;
     const wchar_t *pythonpath_env;
+    const wchar_t *platlibdir;
 
     wchar_t *argv0_path;
     wchar_t *zip_path;
@@ -811,7 +811,7 @@ calculate_exec_prefix(PyCalculatePath *calculate, _PyPathConfig *pathconfig)
         }
 
         /* <PLATLIBDIR> / "lib-dynload" */
-        wchar_t *lib_dynload = joinpath2(calculate->platlibdir_macro,
+        wchar_t *lib_dynload = joinpath2(calculate->platlibdir,
                                          L"lib-dynload");
         if (lib_dynload == NULL) {
             return _PyStatus_NO_MEMORY();
@@ -1296,8 +1296,8 @@ calculate_zip_path(PyCalculatePath *calculate)
 {
     PyStatus res;
 
-    /* Path: <PLATLIBDIR> / "python00.zip" */
-    wchar_t *path = joinpath2(calculate->platlibdir_macro, L"python00.zip");
+    /* Path: <PLATLIBDIR> / "pythonXY.zip" */
+    wchar_t *path = joinpath2(calculate->platlibdir, L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) L".zip");
     if (path == NULL) {
         return _PyStatus_NO_MEMORY();
     }
@@ -1456,10 +1456,6 @@ calculate_init(PyCalculatePath *calculate, const PyConfig *config)
     if (!calculate->vpath_macro) {
         return DECODE_LOCALE_ERR("VPATH macro", len);
     }
-    calculate->platlibdir_macro = Py_DecodeLocale(PLATLIBDIR, &len);
-    if (!calculate->platlibdir_macro) {
-        return DECODE_LOCALE_ERR("PLATLIBDIR macro", len);
-    }
 
     calculate->lib_python = Py_DecodeLocale(PLATLIBDIR "/python" VERSION, &len);
     if (!calculate->lib_python) {
@@ -1468,6 +1464,7 @@ calculate_init(PyCalculatePath *calculate, const PyConfig *config)
 
     calculate->warnings = config->pathconfig_warnings;
     calculate->pythonpath_env = config->pythonpath_env;
+    calculate->platlibdir = config->platlibdir;
 
     return _PyStatus_OK();
 }
@@ -1480,7 +1477,6 @@ calculate_free(PyCalculatePath *calculate)
     PyMem_RawFree(calculate->prefix_macro);
     PyMem_RawFree(calculate->exec_prefix_macro);
     PyMem_RawFree(calculate->vpath_macro);
-    PyMem_RawFree(calculate->platlibdir_macro);
     PyMem_RawFree(calculate->lib_python);
     PyMem_RawFree(calculate->path_env);
     PyMem_RawFree(calculate->zip_path);

Programs/_testembed.c

@@ -548,6 +548,13 @@ static int test_init_from_config(void)
     /* FIXME: test home */
     /* FIXME: test path config: module_search_path .. dll_path */
 
+    putenv("PYTHONPLATLIBDIR=env_platlibdir");
+    status = PyConfig_SetBytesString(&config, &config.platlibdir, "my_platlibdir");
+    if (PyStatus_Exception(status)) {
+        PyConfig_Clear(&config);
+        Py_ExitStatusException(status);
+    }
+
     putenv("PYTHONVERBOSE=0");
     Py_VerboseFlag = 0;
     config.verbose = 1;
@@ -668,6 +675,7 @@ static void set_most_env_vars(void)
     putenv("PYTHONFAULTHANDLER=1");
     putenv("PYTHONIOENCODING=iso8859-1:replace");
     putenv("PYTHONOLDPARSER=1");
+    putenv("PYTHONPLATLIBDIR=env_platlibdir");
 }
 
 

Python/initconfig.c

@@ -24,6 +24,10 @@
 #  endif
 #endif
 
+#ifndef PLATLIBDIR
+#  error "PLATLIBDIR macro must be defined"
+#endif
+
 
 /* --- Command line options --------------------------------------- */
 
@@ -110,6 +114,7 @@ PYTHONPATH   : '%lc'-separated list of directories prefixed to the\n\
 static const char usage_5[] =
 "PYTHONHOME   : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n"
 "               The default module search path uses %s.\n"
+"PYTHONPLATLIBDIR : override sys.platlibdir.\n"
 "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n"
 "PYTHONUTF8: if set to 1, enable the UTF-8 mode.\n"
 "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n"
@@ -586,6 +591,7 @@ PyConfig_Clear(PyConfig *config)
     CLEAR(config->base_prefix);
     CLEAR(config->exec_prefix);
     CLEAR(config->base_exec_prefix);
+    CLEAR(config->platlibdir);
 
     CLEAR(config->filesystem_encoding);
     CLEAR(config->filesystem_errors);
@@ -822,6 +828,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_WSTR_ATTR(base_prefix);
     COPY_WSTR_ATTR(exec_prefix);
     COPY_WSTR_ATTR(base_exec_prefix);
+    COPY_WSTR_ATTR(platlibdir);
 
     COPY_ATTR(site_import);
     COPY_ATTR(bytes_warning);
@@ -925,6 +932,7 @@ config_as_dict(const PyConfig *config)
     SET_ITEM_WSTR(base_prefix);
     SET_ITEM_WSTR(exec_prefix);
     SET_ITEM_WSTR(base_exec_prefix);
+    SET_ITEM_WSTR(platlibdir);
     SET_ITEM_INT(site_import);
     SET_ITEM_INT(bytes_warning);
     SET_ITEM_INT(inspect);
@@ -1336,6 +1344,14 @@ config_read_env_vars(PyConfig *config)
         }
     }
 
+    if(config->platlibdir == NULL) {
+        status = CONFIG_GET_ENV_DUP(config, &config->platlibdir,
+                                    L"PYTHONPLATLIBDIR", "PYTHONPLATLIBDIR");
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+
     if (config->use_hash_seed < 0) {
         status = config_init_hash_seed(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -1731,6 +1747,14 @@ config_read(PyConfig *config)
         }
     }
 
+    if(config->platlibdir == NULL) {
+        status = CONFIG_SET_BYTES_STR(config, &config->platlibdir, PLATLIBDIR,
+                                      "PLATLIBDIR macro");
+        if (_PyStatus_EXCEPTION(status)) {
+            return status;
+        }
+    }
+
     if (config->_install_importlib) {
         status = _PyConfig_InitPathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
@@ -2560,6 +2584,7 @@ PyConfig_Read(PyConfig *config)
         assert(config->exec_prefix != NULL);
         assert(config->base_exec_prefix != NULL);
     }
+    assert(config->platlibdir != NULL);
     assert(config->filesystem_encoding != NULL);
     assert(config->filesystem_errors != NULL);
     assert(config->stdio_encoding != NULL);
@@ -2710,6 +2735,7 @@ _Py_DumpPathConfig(PyThreadState *tstate)
     DUMP_SYS(_base_executable);
     DUMP_SYS(base_prefix);
     DUMP_SYS(base_exec_prefix);
+    DUMP_SYS(platlibdir);
     DUMP_SYS(executable);
     DUMP_SYS(prefix);
     DUMP_SYS(exec_prefix);

Python/sysmodule.c

@@ -2922,13 +2922,7 @@ _PySys_InitMain(PyThreadState *tstate)
     SET_SYS_FROM_WSTR("base_prefix", config->base_prefix);
     SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix);
     SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix);
-    {
-        PyObject *str = PyUnicode_FromString(PLATLIBDIR);
-        if (str == NULL) {
-            return -1;
-        }
-        SET_SYS_FROM_STRING("platlibdir", str);
-    }
+    SET_SYS_FROM_WSTR("platlibdir", config->platlibdir);
 
     if (config->pycache_prefix != NULL) {
         SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix);