[WIP] bpo-42856: Add --with-wheel-pkg-dir=PATH configure option

Doc/whatsnew/3.10.rst

@@ -599,6 +599,18 @@ Build Changes
   don't build nor install test modules.
   (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.)
 
+* Add ``--with-wheel-pkg-dir=PATH`` option to ./configure. If specified, the
+  :mod:`ensurepip` module also looks for wheel packages in this directory, and
+  picks the most recent versions in :mod:`ensurepip._bundled` and the specified
+  directory.
+
+  Some Linux distribution packaging policies recommand against bundling
+  dependencies. For example, Fedora installs wheel packages in the
+  ``/usr/share/python-wheels/`` directory and don't install the
+  ``ensurepip._bundled`` package.
+
+  (Contributed by Victor Stinner in :issue:`42856`.)
+
 
 C API Changes
 =============

Lib/ensurepip/__init__.py

@@ -1,26 +1,95 @@
-import os
+import collections
 import os.path
+import subprocess
 import sys
-import runpy
+import sysconfig
 import tempfile
-import subprocess
-from importlib import resources
-
-from . import _bundled
 
 
 
 __all__ = ["version", "bootstrap"]
+_PACKAGE_NAMES = ('setuptools', 'pip')
+_Package = collections.namedtuple('Package', ('version', 'filename'))
 
+# Directory of system wheel packages. Some Linux distribution packaging
+# policies recommand against bundling dependencies. For example, Fedora
+# installs wheel packages in the /usr/share/python-wheels/ directory and don't
+# install the ensurepip._bundled package.
+_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR')
 
-_SETUPTOOLS_VERSION = "47.1.0"
 
-_PIP_VERSION = "20.2.3"
+def _get_versions(path):
+    versions = {}
+    try:
+        names = os.listdir(path)
+    except OSError:
+        # Ignore: path doesn't exist or permission error
+        names = ()
+    for name in names:
+        # name is like 'pip-20.2.3-py2.py3-none-any.whl'
+        if not name.endswith(".whl"):
+            continue
+        for package in _PACKAGE_NAMES:
+            prefix = package + '-'
+            if not name.startswith(prefix):
+                continue
+            part = name.removeprefix(prefix)
+            break
+        else:
+            continue
+
+        # Extract '20.2.3' from '20.2.3-py2.py3-none-any.whl'
+        version = []
+        part = part.split('-', 1)[0]
+        for part in part.split('.'):
+            try:
+                number = int(part)
+            except ValueError:
+                break
+            version.append(number)
+        if not version:
+            # failed to parse the version: ignore the package
+            continue
+
+        fullname = os.path.join(path, name)
+        versions[package] = _Package(tuple(version), fullname)
+    return versions
+
+
+def _get_package_search_paths():
+    # last item has the highest priority
+    paths = []
+    if _WHEEL_PKG_DIR:
+        paths.append(_WHEEL_PKG_DIR)
+    try:
+        from . import _bundled
+    except ImportError:
+        pass
+    else:
+        paths.extend(reversed(_bundled.__path__))
+    return paths
 
-_PROJECTS = [
-    ("setuptools", _SETUPTOOLS_VERSION, "py3"),
-    ("pip", _PIP_VERSION, "py2.py3"),
-]
+
+def _find_packages(paths):
+    # package => (version, filename): (None, None) if no package is found
+    packages = {name: _Package(None, None) for name in _PACKAGE_NAMES}
+    for path in paths:
+        versions = _get_versions(path)
+        for name, item in versions.items():
+            if (packages[name].version is None
+               or packages[name][0] < item.version):
+                packages[name] = item
+
+    return packages
+
+
+def _get_packages():
+    global _PACKAGES
+    if _PACKAGES is None:
+        paths = _get_package_search_paths()
+        _PACKAGES = _find_packages(paths)
+    return _PACKAGES
+_PACKAGES = None
 
 
 def _run_pip(args, additional_paths=None):
@@ -42,7 +111,10 @@ def version():
     """
     Returns a string specifying the bundled version of pip.
     """
-    return _PIP_VERSION
+    version = _get_packages()['pip'].version
+    if version is None:
+        return None
+    return '.'.join(map(str, version))
 
 def _disable_pip_configuration_settings():
     # We deliberately ignore all pip environment variables
@@ -104,12 +176,14 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
         # Put our bundled wheels into a temporary directory and construct the
         # additional paths that need added to sys.path
         additional_paths = []
-        for project, version, py_tag in _PROJECTS:
-            wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag)
-            whl = resources.read_binary(
-                _bundled,
-                wheel_name,
-            )
+        for name, package in _get_packages().items():
+            if package.filename is None:
+                raise ValueError(f"cannot find {name} wheel package")
+
+            with open(package.filename, "rb") as fp:
+                whl = fp.read()
+
+            wheel_name = os.path.basename(package.filename)
             with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                 fp.write(whl)
 
@@ -126,7 +200,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False,
         if verbosity:
             args += ["-" + "v" * verbosity]
 
-        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
+        return _run_pip([*args, *_PACKAGE_NAMES], additional_paths)
 
 def _uninstall_helper(*, verbosity=0):
     """Helper to support a clean default uninstall process on Windows
@@ -140,10 +214,10 @@ def _uninstall_helper(*, verbosity=0):
         return
 
     # If the pip version doesn't match the bundled one, leave it alone
-    if pip.__version__ != _PIP_VERSION:
-        msg = ("ensurepip will only uninstall a matching version "
-               "({!r} installed, {!r} bundled)")
-        print(msg.format(pip.__version__, _PIP_VERSION), file=sys.stderr)
+    if pip.__version__ != version():
+        print(f"ensurepip will only uninstall a matching version "
+              f"({pip.__version__!r} installed, {version()!r} bundled)",
+              file=sys.stderr)
         return
 
     _disable_pip_configuration_settings()
@@ -153,7 +227,7 @@ def _uninstall_helper(*, verbosity=0):
     if verbosity:
         args += ["-" + "v" * verbosity]
 
-    return _run_pip(args + [p[0] for p in reversed(_PROJECTS)])
+    return _run_pip([*args, reversed(_PACKAGE_NAMES)])
 
 
 def _main(argv=None):

Lib/test/test_ensurepip.py

@@ -5,19 +5,87 @@
 import os.path
 import contextlib
 import sys
+import tempfile
 
 import ensurepip
 import ensurepip._uninstall
 
 
-class TestEnsurePipVersion(unittest.TestCase):
+def clear_caches():
+    ensurepip._PACKAGES = None
+
+
+class TestPackages(unittest.TestCase):
+    def setUp(self):
+        clear_caches()
+
+    def tearDown(self):
+        clear_caches()
+
+    def touch(self, directory, filename):
+        fullname = os.path.join(directory, filename)
+        open(fullname, "wb").close()
+
+    def test_version(self):
+        # Test version()
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self.touch(tmpdir, "pip-20.2.2-py2.py3-none-any.whl")
+            with unittest.mock.patch.object(ensurepip,
+                                            '_get_package_search_paths',
+                                            return_value=[tmpdir]):
+                self.assertEqual(ensurepip.version(), '20.2.2')
+
+    def test_find_packages_no_dir(self):
+        # Test _find_packages() with no directories
+        with tempfile.TemporaryDirectory() as tmpdir:
+            packages = ensurepip._find_packages([tmpdir])
+        self.assertIsNone(packages['pip'].version)
+        self.assertIsNone(packages['setuptools'].version)
+
+    def test_find_packages_one_dir(self):
+        # Test _find_packages() with one directory
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self.touch(tmpdir, "setuptools-49.1.3-py3-none-any.whl")
+            self.touch(tmpdir, "pip-20.2.2-py2.py3-none-any.whl")
+            # not used, make sure that it's ignored
+            self.touch(tmpdir, "wheel-0.34.2-py2.py3-none-any.whl")
+
+            packages = ensurepip._find_packages([tmpdir])
+
+            self.assertEqual(packages['pip'].version, (20, 2, 2))
+            self.assertEqual(packages['setuptools'].version, (49, 1, 3))
+            self.assertNotIn('wheel', packages)
+
+            with unittest.mock.patch.object(ensurepip,
+                                            '_get_package_search_paths',
+                                            return_value=[tmpdir]):
+                self.assertEqual(ensurepip.version(), '20.2.2')
+
+    def test_find_packages_two_dirs(self):
+        # Test _find_packages() with two directories
+        with tempfile.TemporaryDirectory() as tmpdir1:
+            self.touch(tmpdir1, "setuptools-49.1.3-py3-none-any.whl")
+            self.touch(tmpdir1, "pip-20.2.2-py2.py3-none-any.whl")
+            with tempfile.TemporaryDirectory() as tmpdir2:
+                self.touch(tmpdir2, "setuptools-47.1.0-py3-none-any.whl")
+                self.touch(tmpdir2, "pip-20.2.3-py2.py3-none-any.whl")
+
+                packages = ensurepip._find_packages([tmpdir1, tmpdir2])
+
+        self.assertEqual(packages['pip'].version, (20, 2, 3))
+        self.assertEqual(packages['pip'].filename,
+                         os.path.join(tmpdir2, "pip-20.2.3-py2.py3-none-any.whl"))
+        self.assertEqual(packages['setuptools'].version, (49, 1, 3))
+        self.assertEqual(packages['setuptools'].filename,
+                         os.path.join(tmpdir1, "setuptools-49.1.3-py3-none-any.whl"))
+        self.assertNotIn('wheel', packages)
 
-    def test_returns_version(self):
-        self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version())
 
 class EnsurepipMixin:
 
     def setUp(self):
+        clear_caches()
+
         run_pip_patch = unittest.mock.patch("ensurepip._run_pip")
         self.run_pip = run_pip_patch.start()
         self.run_pip.return_value = 0
@@ -32,6 +100,9 @@ def setUp(self):
         patched_os.path = os.path
         self.os_environ = patched_os.environ = os.environ.copy()
 
+    def tearDown(self):
+        clear_caches()
+
 
 class TestBootstrap(EnsurepipMixin, unittest.TestCase):
 
@@ -147,7 +218,7 @@ def test_pip_config_file_disabled(self):
         self.assertEqual(self.os_environ["PIP_CONFIG_FILE"], os.devnull)
 
 @contextlib.contextmanager
-def fake_pip(version=ensurepip._PIP_VERSION):
+def fake_pip(version=ensurepip.version()):
     if version is None:
         pip = None
     else:
@@ -243,16 +314,14 @@ def test_pip_config_file_disabled(self):
 
 # Basic testing of the main functions and their argument parsing
 
-EXPECTED_VERSION_OUTPUT = "pip " + ensurepip._PIP_VERSION
-
 class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase):
 
     def test_bootstrap_version(self):
         with test.support.captured_stdout() as stdout:
             with self.assertRaises(SystemExit):
                 ensurepip._main(["--version"])
         result = stdout.getvalue().strip()
-        self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+        self.assertEqual(result, "pip " + ensurepip.version())
         self.assertFalse(self.run_pip.called)
 
     def test_basic_bootstrapping(self):
@@ -283,7 +352,7 @@ def test_uninstall_version(self):
             with self.assertRaises(SystemExit):
                 ensurepip._uninstall._main(["--version"])
         result = stdout.getvalue().strip()
-        self.assertEqual(result, EXPECTED_VERSION_OUTPUT)
+        self.assertEqual(result, "pip " + ensurepip.version())
         self.assertFalse(self.run_pip.called)
 
     def test_basic_uninstall(self):

Makefile.pre.in

@@ -146,6 +146,8 @@ CONFINCLUDEDIR=	$(exec_prefix)/include
 PLATLIBDIR=	@PLATLIBDIR@
 SCRIPTDIR=	$(prefix)/$(PLATLIBDIR)
 ABIFLAGS=	@ABIFLAGS@
+# Variable used by ensurepip
+WHEEL_PKG_DIR=	@WHEEL_PKG_DIR@
 
 # Detailed destination directories
 BINLIBDEST=	@BINLIBDEST@

Misc/NEWS.d/next/Build/2021-01-07-12-51-38.bpo-42856.n3cMHV.rst

@@ -0,0 +1,9 @@
+Add ``--with-wheel-pkg-dir=PATH`` option to ./configure. If specified, the
+:mod:`ensurepip` module also looks for wheel packages in this directory, and
+picks the most recent versions in :mod:`ensurepip._bundled` and the
+specified directory.
+
+Some Linux distribution packaging policies recommand against bundling
+dependencies. For example, Fedora installs wheel packages in the
+``/usr/share/python-wheels/`` directory and don't install the
+``ensurepip._bundled`` package.