Close #15386: There was a loophole that meant importlib.machinery and imp would sometimes reference an uninitialised copy of importlib._bootstrap

Author: ncoghlan

Lib/importlib/__init__.py

@@ -2,14 +2,21 @@
 __all__ = ['__import__', 'import_module', 'invalidate_caches']
 
 # Bootstrap help #####################################################
-import imp
+
+# Until bootstrapping is complete, DO NOT import any modules that attempt
+# to import importlib._bootstrap (directly or indirectly). Since this
+# partially initialised package would be present in sys.modules, those
+# modules would get an uninitialised copy of the source version, instead
+# of a fully initialised version (either the frozen one or the one
+# initialised below if the frozen one is not available).
+import _imp  # Just the builtin component, NOT the full Python module
 import sys
 
 try:
     import _frozen_importlib as _bootstrap
 except ImportError:
     from . import _bootstrap
-    _bootstrap._setup(sys, imp)
+    _bootstrap._setup(sys, _imp)
 else:
     # importlib._bootstrap is the built-in import, ensure we don't create
     # a second copy of the module.
@@ -22,6 +29,8 @@
 _w_long = _bootstrap._w_long
 _r_long = _bootstrap._r_long
 
+# Fully bootstrapped at this point, import whatever you like, circular
+# dependencies and startup overhead minimisation permitting :)
 
 # Public API #########################################################
 

Lib/pkgutil.py

@@ -2,8 +2,8 @@
 
 import os
 import sys
-import imp
 import importlib
+import imp
 import os.path
 from warnings import warn
 from types import ModuleType

Lib/runpy.py

@@ -12,8 +12,8 @@
 
 import os
 import sys
+import importlib.machinery # importlib first so we can test #15386 via -m
 import imp
-import importlib.machinery
 from pkgutil import read_code, get_loader, get_importer
 
 __all__ = [

Lib/test/regrtest.py

@@ -165,6 +165,9 @@
 option '-uall,-gui'.
 """
 
+# We import importlib *ASAP* in order to test #15386
+import importlib
+
 import builtins
 import faulthandler
 import getopt

Lib/test/test_import.py

@@ -1,8 +1,9 @@
+# We import importlib *ASAP* in order to test #15386
+import importlib
 import builtins
 import imp
 from importlib.test.import_ import test_suite as importlib_import_test_suite
 from importlib.test.import_ import util as importlib_util
-import importlib
 import marshal
 import os
 import platform
@@ -777,6 +778,15 @@ def test_frozen_importlib_is_bootstrap(self):
         self.assertEqual(mod.__package__, 'importlib')
         self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__)
 
+    def test_there_can_be_only_one(self):
+        # Issue #15386 revealed a tricky loophole in the bootstrapping
+        # This test is technically redundant, since the bug caused importing
+        # this test module to crash completely, but it helps prove the point
+        from importlib import machinery
+        mod = sys.modules['_frozen_importlib']
+        self.assertIs(machinery.FileFinder, mod.FileFinder)
+        self.assertIs(imp.new_module, mod.new_module)
+
 
 class ImportTracebackTests(unittest.TestCase):