[WIP] bpo-1294959: Better support for systems with /usr/lib64

Doc/whatsnew/3.9.rst

@@ -131,6 +131,15 @@ Build and C API Changes
   way to call a callable Python object without any argument.
   (Contributed by Victor Stinner in :issue:`37194`.)
 
+* Introduce a ``platsubdir`` sysconfig variable that contains the name of the
+  platform-specific "lib" directory. This is used to construct paths that do
+  not differ by prefix, i.e., ``/usr/lib`` vs ``/usr/lib64``. The
+  ``posix_prefix`` install scheme is changed to take this into account.
+
+  By default, this is ``basename $libdir``, but can be overriden by
+  ``./configure --with-custom-platsubdir``.
+  (Contributed by matejcik in :issue:`1294959`.)
+
 
 
 Deprecated

Lib/distutils/command/install.py

@@ -30,7 +30,7 @@
 INSTALL_SCHEMES = {
     'unix_prefix': {
         'purelib': '$base/lib/python$py_version_short/site-packages',
-        'platlib': '$platbase/lib/python$py_version_short/site-packages',
+        'platlib': '$platbase/$platsubdir/python$py_version_short/site-packages',
         'headers': '$base/include/python$py_version_short$abiflags/$dist_name',
         'scripts': '$base/bin',
         'data'   : '$base',
@@ -281,7 +281,7 @@ def finalize_options(self):
         # about needing recursive variable expansion (shudder).
 
         py_version = sys.version.split()[0]
-        (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix')
+        (prefix, exec_prefix, platsubdir) = get_config_vars('prefix', 'exec_prefix', 'platsubdir')
         try:
             abiflags = sys.abiflags
         except AttributeError:
@@ -298,6 +298,7 @@ def finalize_options(self):
                             'sys_exec_prefix': exec_prefix,
                             'exec_prefix': exec_prefix,
                             'abiflags': abiflags,
+                            'platsubdir': platsubdir,
                            }
 
         if HAS_USER_SITE:

Lib/distutils/sysconfig.py

@@ -146,8 +146,9 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             prefix = plat_specific and EXEC_PREFIX or PREFIX
 
     if os.name == "posix":
+        libdir = plat_specific and get_config_var("platsubdir") or "lib"
         libpython = os.path.join(prefix,
-                                 "lib", "python" + get_python_version())
+                                 libdir, "python" + get_python_version())
         if standard_lib:
             return libpython
         else:

Lib/pydoc.py

@@ -66,6 +66,7 @@ class or function within a module or module in a package.  If the
 import platform
 import re
 import sys
+import sysconfig
 import time
 import tokenize
 import urllib.parse
@@ -392,9 +393,7 @@ def fail(self, object, name=None, *args):
 
     docmodule = docclass = docroutine = docother = docproperty = docdata = fail
 
-    def getdocloc(self, object,
-                  basedir=os.path.join(sys.base_exec_prefix, "lib",
-                                       "python%d.%d" %  sys.version_info[:2])):
+    def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')):
         """Return the location of module docs or None"""
 
         try:

Lib/site.py

@@ -335,9 +335,15 @@ def getsitepackages(prefixes=None):
         seen.add(prefix)
 
         if os.sep == '/':
-            sitepackages.append(os.path.join(prefix, "lib",
+            from sysconfig import get_config_var
+            platsubdir = get_config_var("platsubdir")
+            sitepackages.append(os.path.join(prefix, platsubdir,
                                         "python%d.%d" % sys.version_info[:2],
                                         "site-packages"))
+            if platsubdir != "lib":
+                sitepackages.append(os.path.join(prefix, "lib",
+                                            "python%d.%d" % sys.version_info[:2],
+                                            "site-packages"))
         else:
             sitepackages.append(prefix)
             sitepackages.append(os.path.join(prefix, "lib", "site-packages"))

Lib/sysconfig.py

@@ -20,10 +20,10 @@
 
 _INSTALL_SCHEMES = {
     'posix_prefix': {
-        'stdlib': '{installed_base}/lib/python{py_version_short}',
-        'platstdlib': '{platbase}/lib/python{py_version_short}',
+        'stdlib': '{installed_base}/{platsubdir}/python{py_version_short}',
+        'platstdlib': '{platbase}/{platsubdir}/python{py_version_short}',
         'purelib': '{base}/lib/python{py_version_short}/site-packages',
-        'platlib': '{platbase}/lib/python{py_version_short}/site-packages',
+        'platlib': '{platbase}/{platsubdir}/python{py_version_short}/site-packages',
         'include':
             '{installed_base}/include/python{py_version_short}{abiflags}',
         'platinclude':

Lib/test/test_site.py

@@ -266,8 +266,9 @@ def test_getsitepackages(self):
         dirs = site.getsitepackages()
         if os.sep == '/':
             # OS X, Linux, FreeBSD, etc
-            self.assertEqual(len(dirs), 1)
-            wanted = os.path.join('xoxo', 'lib',
+            platsubdir = sysconfig.get_config_var('platsubdir')
+            self.assertTrue(len(dirs) in (1,2))
+            wanted = os.path.join('xoxo', platsubdir,
                                   'python%d.%d' % sys.version_info[:2],
                                   'site-packages')
             self.assertEqual(dirs[0], wanted)

Lib/test/test_sysconfig.py

@@ -243,6 +243,7 @@ def test_user_similar(self):
         # is similar to the global posix_prefix one
         base = get_config_var('base')
         user = get_config_var('userbase')
+        platsubdir = get_config_var("platsubdir")
         # the global scheme mirrors the distinction between prefix and
         # exec-prefix but not the user scheme, so we have to adapt the paths
         # before comparing (issue #9100)
@@ -257,8 +258,19 @@ def test_user_similar(self):
                 # before comparing
                 global_path = global_path.replace(sys.base_prefix, sys.prefix)
                 base = base.replace(sys.base_prefix, sys.prefix)
+
+            if platsubdir != "lib":
+                platbase = os.path.join(base, platsubdir)
+                purebase = os.path.join(base, "lib")
+                userlib = os.path.join(user, "lib")
+                # replace platbase first because usually purebase is a prefix of platbase
+                # /usr/lib is prefix of /usr/lib64 and would get replaced first
+                modified_path = global_path.replace(platbase, userlib, 1).replace(purebase, userlib, 1)
+            else:
+                modified_path = global_path.replace(base, user, 1)
+
             user_path = get_path(name, 'posix_user')
-            self.assertEqual(user_path, global_path.replace(base, user, 1))
+            self.assertEqual(user_path, modified_path)
 
     def test_main(self):
         # just making sure _main() runs and returns things in the stdout

Lib/trace.py

@@ -52,6 +52,7 @@
 import linecache
 import os
 import sys
+import sysconfig
 import token
 import tokenize
 import inspect
@@ -660,9 +661,8 @@ def main():
     opts = parser.parse_args()
 
     if opts.ignore_dir:
-        rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info)
-        _prefix = os.path.join(sys.base_prefix, *rel_path)
-        _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path)
+        _prefix = sysconfig.get_path("stdlib")
+        _exec_prefix = sysconfig.get_path("platstdlib")
 
     def parse_ignore_dir(s):
         s = os.path.expanduser(os.path.expandvars(s))

Makefile.pre.in

@@ -137,13 +137,16 @@ exec_prefix=	@exec_prefix@
 # Install prefix for data files
 datarootdir=    @datarootdir@
 
+# Name of "lib" directory under prefix
+platsubdir=	@platsubdir@
+
 # Expanded directories
 BINDIR=		@bindir@
 LIBDIR=		@libdir@
 MANDIR=		@mandir@
 INCLUDEDIR=	@includedir@
 CONFINCLUDEDIR=	$(exec_prefix)/include
-SCRIPTDIR=	$(prefix)/lib
+SCRIPTDIR=	@libdir@
 ABIFLAGS=	@ABIFLAGS@
 
 # Detailed destination directories
@@ -752,6 +755,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
 		-DEXEC_PREFIX='"$(exec_prefix)"' \
 		-DVERSION='"$(VERSION)"' \
 		-DVPATH='"$(VPATH)"' \
+		-DPLATSUBDIR='"$(platsubdir)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
 Programs/python.o: $(srcdir)/Programs/python.c

Misc/ACKS

@@ -1044,6 +1044,7 @@ Sidney San Martín
 Westley Martínez
 Sébastien Martini
 Roger Masse
+Jan Matějek
 Nick Mathewson
 Simon Mathieu
 Laura Matson

Misc/NEWS.d/next/Build/2019-07-14-14-12-04.bpo-1294959.RGk38C.rst

@@ -0,0 +1,3 @@
+On 64bit Linux systems that use ``$prefix/lib64``, this is now the default
+install location for the standard library. Python will still be able to
+import modules installed in ``$prefix/lib``

Misc/NEWS.d/next/Library/2019-07-14-14-13-05.bpo-1294959.VosXOa.rst

@@ -0,0 +1,7 @@
+Introduce a ``platsubdir`` sysconfig variable that contains the name of the
+platform-specific "lib" directory. This is used to construct paths that do
+not differ by prefix, i.e., ``/usr/lib`` vs ``/usr/lib64``. The
+``posix_prefix`` install scheme is changed to take this into account.
+
+By default, this is ``basename $libdir``, but can be overriden by
+``./configure --with-custom-platsubdir``.

Modules/getpath.c

@@ -55,9 +55,10 @@
  * pybuilddir.txt.  If the landmark is found, we're done.
  *
  * For the remaining steps, the prefix landmark will always be
- * lib/python$VERSION/os.py and the exec_prefix will always be
- * lib/python$VERSION/lib-dynload, where $VERSION is Python's version
- * number as supplied by the Makefile.  Note that this means that no more
+ * $lib/python$VERSION/os.py and the exec_prefix will always be
+ * $lib/python$VERSION/lib-dynload, where $VERSION is Python's version
+ * number and $lib is PLATSUBDIR as supplied by the Makefile. (usually
+ * "lib", "lib32" or "lib64").  Note that this means that no more
  * build directory checking is performed; if the first step did not find
  * the landmarks, the assumption is that python is running from an
  * installed setup.
@@ -86,7 +87,7 @@
  * containing the shared library modules is appended.  The environment
  * variable $PYTHONPATH is inserted in front of it all.  Finally, the
  * prefix and exec_prefix globals are tweaked so they reflect the values
- * expected by other code, by stripping the "lib/python$VERSION/..." stuff
+ * expected by other code, by stripping the "$lib/python$VERSION/..." stuff
  * off.  If either points to the build directory, the globals are reset to
  * the corresponding preprocessor variables (so sys.prefix will reflect the
  * installation location, even though sys.path points into the build
@@ -105,8 +106,8 @@ extern "C" {
 #endif
 
 
-#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH)
-#error "PREFIX, EXEC_PREFIX, VERSION, and VPATH must be constant defined"
+#if !defined(PREFIX) || !defined(EXEC_PREFIX) || !defined(VERSION) || !defined(VPATH) || !defined(PLATSUBDIR)
+#error "PREFIX, EXEC_PREFIX, VERSION, VPATH and PLATSUBDIR must be constant defined"
 #endif
 
 #ifndef LANDMARK
@@ -1145,7 +1146,7 @@ calculate_init(PyCalculatePath *calculate,
     if (!calculate->prefix) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
-    calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);
+    calculate->lib_python = Py_DecodeLocale(PLATSUBDIR "/python" VERSION, &len);
     if (!calculate->lib_python) {
         return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }

configure

@@ -631,6 +631,7 @@ SRCDIRS
 THREADHEADERS
 LIBPL
 PY_ENABLE_SHARED
+platsubdir
 LIBPYTHON
 EXT_SUFFIX
 ALT_SOABI
@@ -841,6 +842,7 @@ with_dtrace
 with_libm
 with_libc
 enable_big_digits
+with_custom_platsubdir
 with_computed_gotos
 with_ensurepip
 with_openssl
@@ -1537,6 +1539,10 @@ Optional Packages:
   --with(out)-dtrace      disable/enable DTrace support
   --with-libm=STRING      math library
   --with-libc=STRING      C library
+  --with-custom-platsubdir=<libdirname>
+                          name of the platform-specific lib subdirectory of
+                          $prefix. This is usually "lib", "lib32" or "lib64".
+                          Defaults to basename($libdir)
   --with(out)-computed-gotos
                           Use computed gotos in evaluation loop (enabled by
                           default on supported compilers)
@@ -15179,11 +15185,34 @@ else
   LIBPYTHON=''
 fi
 
+# platsubdir must be defined before LIBPL definition
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for custom platsubdir" >&5
+$as_echo_n "checking for custom platsubdir... " >&6; }
+
+# Check whether --with-custom-platsubdir was given.
+if test "${with_custom_platsubdir+set}" = set; then :
+  withval=$with_custom_platsubdir;
+else
+  with_custom_platsubdir=yes
+fi
+
+case $with_custom_platsubdir in #(
+  yes) :
+    platsubdir=`basename ${libdir}` ;; #(
+  no) :
+    platsubdir=lib ;; #(
+  *) :
+    platsubdir=$with_custom_platsubdir ;;
+esac
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $platsubdir" >&5
+$as_echo "$platsubdir" >&6; }
+
+
 
 if test x$PLATFORM_TRIPLET = x; then
-  LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}"
+  LIBPL='$(prefix)'"/${platsubdir}/python${VERSION}/config-${LDVERSION}"
 else
-  LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
+  LIBPL='$(prefix)'"/${platsubdir}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
 fi
 
 

configure.ac

@@ -4652,12 +4652,26 @@ else
   LIBPYTHON=''
 fi
 
+# platsubdir must be defined before LIBPL definition
+AC_MSG_CHECKING(for custom platsubdir)
+AC_ARG_WITH(custom-platsubdir,
+    [AS_HELP_STRING([--with-custom-platsubdir=<libdirname>],
+        [name of the platform-specific lib subdirectory of $prefix. This is usually "lib", "lib32" or "lib64". Defaults to basename($libdir)])],
+    [],
+    [with_custom_platsubdir=yes])
+AS_CASE($with_custom_platsubdir,
+    [yes],[platsubdir=`basename ${libdir}`],
+    [no],[platsubdir=lib],
+    [platsubdir=$with_custom_platsubdir])
+AC_MSG_RESULT($platsubdir)
+AC_SUBST(platsubdir)
+
 dnl define LIBPL after ABIFLAGS and LDVERSION is defined.
 AC_SUBST(PY_ENABLE_SHARED)
 if test x$PLATFORM_TRIPLET = x; then
-  LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}"
+  LIBPL='$(prefix)'"/${platsubdir}/python${VERSION}/config-${LDVERSION}"
 else
-  LIBPL='$(prefix)'"/lib/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
+  LIBPL='$(prefix)'"/${platsubdir}/python${VERSION}/config-${LDVERSION}-${PLATFORM_TRIPLET}"
 fi
 AC_SUBST(LIBPL)