Skip to content

Commit 5c39e47

Browse files
authored
[3.10] bpo-45703: Invalidate _NamespacePath cache on importlib.invalidate_cache (GH-29384) (GH-30922)
Consider the following directory structure: . └── PATH1 └── namespace └── sub1 └── __init__.py And both PATH1 and PATH2 in sys path: $ PYTHONPATH=PATH1:PATH2 python3.11 >>> import namespace >>> import namespace.sub1 >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> ... While this interpreter still runs, PATH2/namespace/sub2 is created: . ├── PATH1 │ └── namespace │ └── sub1 │ └── __init__.py └── PATH2 └── namespace └── sub2 └── __init__.py The newly created module cannot be imported: >>> ... >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> import namespace.sub2 Traceback (most recent call last): File "<stdin>", line 1, in <module> ModuleNotFoundError: No module named 'namespace.sub2' Calling importlib.invalidate_caches() now newly allows to import it: >>> import importlib >>> importlib.invalidate_caches() >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace']) >>> import namespace.sub2 >>> namespace.__path__ _NamespacePath(['.../PATH1/namespace', '.../PATH2/namespace']) This was not previously possible.
1 parent 89db090 commit 5c39e47

File tree

4 files changed

+954
-899
lines changed

4 files changed

+954
-899
lines changed

Lib/importlib/_bootstrap_external.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1212,10 +1212,15 @@ class _NamespacePath:
12121212
using path_finder. For top-level modules, the parent module's path
12131213
is sys.path."""
12141214

1215+
# When invalidate_caches() is called, this epoch is incremented
1216+
# https://bugs.python.org/issue45703
1217+
_epoch = 0
1218+
12151219
def __init__(self, name, path, path_finder):
12161220
self._name = name
12171221
self._path = path
12181222
self._last_parent_path = tuple(self._get_parent_path())
1223+
self._last_epoch = self._epoch
12191224
self._path_finder = path_finder
12201225

12211226
def _find_parent_path_names(self):
@@ -1235,14 +1240,15 @@ def _get_parent_path(self):
12351240
def _recalculate(self):
12361241
# If the parent's path has changed, recalculate _path
12371242
parent_path = tuple(self._get_parent_path()) # Make a copy
1238-
if parent_path != self._last_parent_path:
1243+
if parent_path != self._last_parent_path or self._epoch != self._last_epoch:
12391244
spec = self._path_finder(self._name, parent_path)
12401245
# Note that no changes are made if a loader is returned, but we
12411246
# do remember the new parent path
12421247
if spec is not None and spec.loader is None:
12431248
if spec.submodule_search_locations:
12441249
self._path = spec.submodule_search_locations
12451250
self._last_parent_path = parent_path # Save the copy
1251+
self._last_epoch = self._epoch
12461252
return self._path
12471253

12481254
def __iter__(self):
@@ -1330,6 +1336,9 @@ def invalidate_caches():
13301336
del sys.path_importer_cache[name]
13311337
elif hasattr(finder, 'invalidate_caches'):
13321338
finder.invalidate_caches()
1339+
# Also invalidate the caches of _NamespacePaths
1340+
# https://bugs.python.org/issue45703
1341+
_NamespacePath._epoch += 1
13331342

13341343
@staticmethod
13351344
def _path_hooks(path):

Lib/test/test_importlib/test_namespace_pkgs.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import importlib
33
import os
44
import sys
5+
import tempfile
56
import unittest
67
import warnings
78

@@ -128,6 +129,40 @@ def test_imports(self):
128129
self.assertEqual(foo.two.attr, 'portion2 foo two')
129130

130131

132+
class SeparatedNamespacePackagesCreatedWhileRunning(NamespacePackageTest):
133+
paths = ['portion1']
134+
135+
def test_invalidate_caches(self):
136+
with tempfile.TemporaryDirectory() as temp_dir:
137+
# we manipulate sys.path before anything is imported to avoid
138+
# accidental cache invalidation when changing it
139+
sys.path.append(temp_dir)
140+
141+
import foo.one
142+
self.assertEqual(foo.one.attr, 'portion1 foo one')
143+
144+
# the module does not exist, so it cannot be imported
145+
with self.assertRaises(ImportError):
146+
import foo.just_created
147+
148+
# util.create_modules() manipulates sys.path
149+
# so we must create the modules manually instead
150+
namespace_path = os.path.join(temp_dir, 'foo')
151+
os.mkdir(namespace_path)
152+
module_path = os.path.join(namespace_path, 'just_created.py')
153+
with open(module_path, 'w', encoding='utf-8') as file:
154+
file.write('attr = "just_created foo"')
155+
156+
# the module is not known, so it cannot be imported yet
157+
with self.assertRaises(ImportError):
158+
import foo.just_created
159+
160+
# but after explicit cache invalidation, it is importable
161+
importlib.invalidate_caches()
162+
import foo.just_created
163+
self.assertEqual(foo.just_created.attr, 'just_created foo')
164+
165+
131166
class SeparatedOverlappingNamespacePackages(NamespacePackageTest):
132167
paths = ['portion1', 'both_portions']
133168

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
When a namespace package is imported before another module from the same
2+
namespace is created/installed in a different :data:`sys.path` location
3+
while the program is running, calling the
4+
:func:`importlib.invalidate_caches` function will now also guarantee the new
5+
module is noticed.

0 commit comments

Comments
 (0)