Skip to content

Commit ee37845

Browse files
authored
bpo-33053: -m now adds *starting* directory to sys.path (GH-6231) (#6236)
Historically, -m added the empty string as sys.path zero, meaning it resolved imports against the current working directory, the same way -c and the interactive prompt do. This changes the sys.path initialisation to add the *starting* working directory as sys.path[0] instead, such that changes to the working directory while the program is running will have no effect on imports when using the -m switch. (cherry picked from commit d5d9e02)
1 parent 5666a55 commit ee37845

File tree

9 files changed

+104
-76
lines changed

9 files changed

+104
-76
lines changed

Doc/library/test.rst

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1332,8 +1332,8 @@ script execution tests.
13321332
.. function:: run_python_until_end(*args, **env_vars)
13331333

13341334
Set up the environment based on *env_vars* for running the interpreter
1335-
in a subprocess. The values can include ``__isolated``, ``__cleavenv``,
1336-
and ``TERM``.
1335+
in a subprocess. The values can include ``__isolated``, ``__cleanenv``,
1336+
``__cwd``, and ``TERM``.
13371337

13381338

13391339
.. function:: assert_python_ok(*args, **env_vars)

Doc/whatsnew/3.7.rst

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,12 @@ Other Language Changes
421421
writable.
422422
(Contributed by Nathaniel J. Smith in :issue:`30579`.)
423423

424+
* When using the :option:`-m` switch, ``sys.path[0]`` is now eagerly expanded
425+
to the full starting directory path, rather than being left as the empty
426+
directory (which allows imports from the *current* working directory at the
427+
time when an import occurs)
428+
(Contributed by Nick Coghlan in :issue:`33053`.)
429+
424430

425431
New Modules
426432
===========
@@ -1138,6 +1144,11 @@ Changes in Python behavior
11381144
parentheses can be omitted only on calls.
11391145
(Contributed by Serhiy Storchaka in :issue:`32012` and :issue:`32023`.)
11401146

1147+
* When using the ``-m`` switch, the starting directory is now added to sys.path,
1148+
rather than the current working directory. Any programs that are found to be
1149+
relying on the previous behaviour will need to be updated to manipulate
1150+
:data:`sys.path` appropriately.
1151+
11411152

11421153
Changes in the Python API
11431154
-------------------------

Lib/test/support/script_helper.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@ def fail(self, cmd_line):
8787
# Executing the interpreter in a subprocess
8888
def run_python_until_end(*args, **env_vars):
8989
env_required = interpreter_requires_environment()
90+
cwd = env_vars.pop('__cwd', None)
9091
if '__isolated' in env_vars:
9192
isolated = env_vars.pop('__isolated')
9293
else:
@@ -125,7 +126,7 @@ def run_python_until_end(*args, **env_vars):
125126
cmd_line.extend(args)
126127
proc = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
127128
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
128-
env=env)
129+
env=env, cwd=cwd)
129130
with proc:
130131
try:
131132
out, err = proc.communicate()

Lib/test/test_bdb.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,13 +524,13 @@ def gen(a, b):
524524
test.id = lambda : None
525525
test.expect_set = list(gen(repeat(()), iter(sl)))
526526
with create_modules(modules):
527-
sys.path.append(os.getcwd())
528527
with TracerRun(test, skip=skip) as tracer:
529528
tracer.runcall(tfunc_import)
530529

531530
@contextmanager
532531
def create_modules(modules):
533532
with test.support.temp_cwd():
533+
sys.path.append(os.getcwd())
534534
try:
535535
for m in modules:
536536
fname = m + '.py'
@@ -542,6 +542,7 @@ def create_modules(modules):
542542
finally:
543543
for m in modules:
544544
test.support.forget(m)
545+
sys.path.pop()
545546

546547
def break_in_func(funcname, fname=__file__, temporary=False, cond=None):
547548
return 'break', (fname, None, temporary, cond, funcname)

Lib/test/test_cmd_line_script.py

Lines changed: 46 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -87,31 +87,11 @@ def _make_test_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename,
8787
importlib.invalidate_caches()
8888
return to_return
8989

90-
# There's no easy way to pass the script directory in to get
91-
# -m to work (avoiding that is the whole point of making
92-
# directories and zipfiles executable!)
93-
# So we fake it for testing purposes with a custom launch script
94-
launch_source = """\
95-
import sys, os.path, runpy
96-
sys.path.insert(0, %s)
97-
runpy._run_module_as_main(%r)
98-
"""
99-
100-
def _make_launch_script(script_dir, script_basename, module_name, path=None):
101-
if path is None:
102-
path = "os.path.dirname(__file__)"
103-
else:
104-
path = repr(path)
105-
source = launch_source % (path, module_name)
106-
to_return = make_script(script_dir, script_basename, source)
107-
importlib.invalidate_caches()
108-
return to_return
109-
11090
class CmdLineTest(unittest.TestCase):
11191
def _check_output(self, script_name, exit_code, data,
11292
expected_file, expected_argv0,
11393
expected_path0, expected_package,
114-
expected_loader):
94+
expected_loader, expected_cwd=None):
11595
if verbose > 1:
11696
print("Output from test script %r:" % script_name)
11797
print(repr(data))
@@ -121,7 +101,9 @@ def _check_output(self, script_name, exit_code, data,
121101
printed_package = '__package__==%r' % expected_package
122102
printed_argv0 = 'sys.argv[0]==%a' % expected_argv0
123103
printed_path0 = 'sys.path[0]==%a' % expected_path0
124-
printed_cwd = 'cwd==%a' % os.getcwd()
104+
if expected_cwd is None:
105+
expected_cwd = os.getcwd()
106+
printed_cwd = 'cwd==%a' % expected_cwd
125107
if verbose > 1:
126108
print('Expected output:')
127109
print(printed_file)
@@ -135,23 +117,33 @@ def _check_output(self, script_name, exit_code, data,
135117
self.assertIn(printed_path0.encode('utf-8'), data)
136118
self.assertIn(printed_cwd.encode('utf-8'), data)
137119

138-
def _check_script(self, script_name, expected_file,
120+
def _check_script(self, script_exec_args, expected_file,
139121
expected_argv0, expected_path0,
140122
expected_package, expected_loader,
141-
*cmd_line_switches):
123+
*cmd_line_switches, cwd=None, **env_vars):
124+
if isinstance(script_exec_args, str):
125+
script_exec_args = [script_exec_args]
142126
run_args = [*support.optim_args_from_interpreter_flags(),
143-
*cmd_line_switches, script_name, *example_args]
144-
rc, out, err = assert_python_ok(*run_args, __isolated=False)
145-
self._check_output(script_name, rc, out + err, expected_file,
127+
*cmd_line_switches, *script_exec_args, *example_args]
128+
rc, out, err = assert_python_ok(
129+
*run_args, __isolated=False, __cwd=cwd, **env_vars
130+
)
131+
self._check_output(script_exec_args, rc, out + err, expected_file,
146132
expected_argv0, expected_path0,
147-
expected_package, expected_loader)
133+
expected_package, expected_loader, cwd)
148134

149-
def _check_import_error(self, script_name, expected_msg,
150-
*cmd_line_switches):
151-
run_args = cmd_line_switches + (script_name,)
152-
rc, out, err = assert_python_failure(*run_args)
135+
def _check_import_error(self, script_exec_args, expected_msg,
136+
*cmd_line_switches, cwd=None, **env_vars):
137+
if isinstance(script_exec_args, str):
138+
script_exec_args = (script_exec_args,)
139+
else:
140+
script_exec_args = tuple(script_exec_args)
141+
run_args = cmd_line_switches + script_exec_args
142+
rc, out, err = assert_python_failure(
143+
*run_args, __isolated=False, __cwd=cwd, **env_vars
144+
)
153145
if verbose > 1:
154-
print('Output from test script %r:' % script_name)
146+
print('Output from test script %r:' % script_exec_args)
155147
print(repr(err))
156148
print('Expected output: %r' % expected_msg)
157149
self.assertIn(expected_msg.encode('utf-8'), err)
@@ -287,35 +279,35 @@ def test_module_in_package(self):
287279
pkg_dir = os.path.join(script_dir, 'test_pkg')
288280
make_pkg(pkg_dir)
289281
script_name = _make_test_script(pkg_dir, 'script')
290-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script')
291-
self._check_script(launch_name, script_name, script_name,
282+
self._check_script(["-m", "test_pkg.script"], script_name, script_name,
292283
script_dir, 'test_pkg',
293-
importlib.machinery.SourceFileLoader)
284+
importlib.machinery.SourceFileLoader,
285+
cwd=script_dir)
294286

295287
def test_module_in_package_in_zipfile(self):
296288
with support.temp_dir() as script_dir:
297289
zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
298-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
299-
self._check_script(launch_name, run_name, run_name,
300-
zip_name, 'test_pkg', zipimport.zipimporter)
290+
self._check_script(["-m", "test_pkg.script"], run_name, run_name,
291+
script_dir, 'test_pkg', zipimport.zipimporter,
292+
PYTHONPATH=zip_name, cwd=script_dir)
301293

302294
def test_module_in_subpackage_in_zipfile(self):
303295
with support.temp_dir() as script_dir:
304296
zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
305-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
306-
self._check_script(launch_name, run_name, run_name,
307-
zip_name, 'test_pkg.test_pkg',
308-
zipimport.zipimporter)
297+
self._check_script(["-m", "test_pkg.test_pkg.script"], run_name, run_name,
298+
script_dir, 'test_pkg.test_pkg',
299+
zipimport.zipimporter,
300+
PYTHONPATH=zip_name, cwd=script_dir)
309301

310302
def test_package(self):
311303
with support.temp_dir() as script_dir:
312304
pkg_dir = os.path.join(script_dir, 'test_pkg')
313305
make_pkg(pkg_dir)
314306
script_name = _make_test_script(pkg_dir, '__main__')
315-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
316-
self._check_script(launch_name, script_name,
307+
self._check_script(["-m", "test_pkg"], script_name,
317308
script_name, script_dir, 'test_pkg',
318-
importlib.machinery.SourceFileLoader)
309+
importlib.machinery.SourceFileLoader,
310+
cwd=script_dir)
319311

320312
def test_package_compiled(self):
321313
with support.temp_dir() as script_dir:
@@ -325,19 +317,18 @@ def test_package_compiled(self):
325317
compiled_name = py_compile.compile(script_name, doraise=True)
326318
os.remove(script_name)
327319
pyc_file = support.make_legacy_pyc(script_name)
328-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
329-
self._check_script(launch_name, pyc_file,
320+
self._check_script(["-m", "test_pkg"], pyc_file,
330321
pyc_file, script_dir, 'test_pkg',
331-
importlib.machinery.SourcelessFileLoader)
322+
importlib.machinery.SourcelessFileLoader,
323+
cwd=script_dir)
332324

333325
def test_package_error(self):
334326
with support.temp_dir() as script_dir:
335327
pkg_dir = os.path.join(script_dir, 'test_pkg')
336328
make_pkg(pkg_dir)
337329
msg = ("'test_pkg' is a package and cannot "
338330
"be directly executed")
339-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
340-
self._check_import_error(launch_name, msg)
331+
self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir)
341332

342333
def test_package_recursion(self):
343334
with support.temp_dir() as script_dir:
@@ -348,8 +339,7 @@ def test_package_recursion(self):
348339
msg = ("Cannot use package as __main__ module; "
349340
"'test_pkg' is a package and cannot "
350341
"be directly executed")
351-
launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
352-
self._check_import_error(launch_name, msg)
342+
self._check_import_error(["-m", "test_pkg"], msg, cwd=script_dir)
353343

354344
def test_issue8202(self):
355345
# Make sure package __init__ modules see "-m" in sys.argv0 while
@@ -365,7 +355,7 @@ def test_issue8202(self):
365355
expected = "init_argv0==%r" % '-m'
366356
self.assertIn(expected.encode('utf-8'), out)
367357
self._check_output(script_name, rc, out,
368-
script_name, script_name, '', 'test_pkg',
358+
script_name, script_name, script_dir, 'test_pkg',
369359
importlib.machinery.SourceFileLoader)
370360

371361
def test_issue8202_dash_c_file_ignored(self):
@@ -394,7 +384,7 @@ def test_issue8202_dash_m_file_ignored(self):
394384
rc, out, err = assert_python_ok('-m', 'other', *example_args,
395385
__isolated=False)
396386
self._check_output(script_name, rc, out,
397-
script_name, script_name, '', '',
387+
script_name, script_name, script_dir, '',
398388
importlib.machinery.SourceFileLoader)
399389

400390
@contextlib.contextmanager
@@ -627,7 +617,7 @@ def test_consistent_sys_path_for_module_execution(self):
627617
# direct execution test cases
628618
p = spawn_python("-sm", "script_pkg.__main__", cwd=work_dir)
629619
out_by_module = kill_python(p).decode().splitlines()
630-
self.assertEqual(out_by_module[0], '')
620+
self.assertEqual(out_by_module[0], work_dir)
631621
self.assertNotIn(script_dir, out_by_module)
632622
# Package execution should give the same output
633623
p = spawn_python("-sm", "script_pkg", cwd=work_dir)

Lib/test/test_doctest.py

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import sys
1010
import importlib
1111
import unittest
12-
12+
import tempfile
1313

1414
# NOTE: There are some additional tests relating to interaction with
1515
# zipimport in the test_zipimport_support test module.
@@ -688,10 +688,16 @@ class TestDocTestFinder(unittest.TestCase):
688688

689689
def test_empty_namespace_package(self):
690690
pkg_name = 'doctest_empty_pkg'
691-
os.mkdir(pkg_name)
692-
mod = importlib.import_module(pkg_name)
693-
assert doctest.DocTestFinder().find(mod) == []
694-
os.rmdir(pkg_name)
691+
with tempfile.TemporaryDirectory() as parent_dir:
692+
pkg_dir = os.path.join(parent_dir, pkg_name)
693+
os.mkdir(pkg_dir)
694+
sys.path.append(parent_dir)
695+
try:
696+
mod = importlib.import_module(pkg_name)
697+
finally:
698+
support.forget(pkg_name)
699+
sys.path.pop()
700+
assert doctest.DocTestFinder().find(mod) == []
695701

696702

697703
def test_DocTestParser(): r"""

Lib/test/test_import/__init__.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -826,8 +826,11 @@ def test_missing_source_legacy(self):
826826
unload(TESTFN)
827827
importlib.invalidate_caches()
828828
m = __import__(TESTFN)
829-
self.assertEqual(m.__file__,
830-
os.path.join(os.curdir, os.path.relpath(pyc_file)))
829+
try:
830+
self.assertEqual(m.__file__,
831+
os.path.join(os.curdir, os.path.relpath(pyc_file)))
832+
finally:
833+
os.remove(pyc_file)
831834

832835
def test___cached__(self):
833836
# Modules now also have an __cached__ that points to the pyc file.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
When using the -m switch, sys.path[0] is now explicitly expanded as the
2+
*starting* working directory, rather than being left as the empty path
3+
(which allows imports from the current working directory at the time of the
4+
import)

Python/pathconfig.c

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "Python.h"
44
#include "osdefs.h"
55
#include "internal/pystate.h"
6+
#include <wchar.h>
67

78
#ifdef __cplusplus
89
extern "C" {
@@ -255,18 +256,15 @@ Py_GetProgramName(void)
255256
return _Py_path_config.program_name;
256257
}
257258

258-
259-
#define _HAVE_SCRIPT_ARGUMENT(argc, argv) \
260-
(argc > 0 && argv0 != NULL && \
261-
wcscmp(argv0, L"-c") != 0 && wcscmp(argv0, L"-m") != 0)
262-
263259
/* Compute argv[0] which will be prepended to sys.argv */
264260
PyObject*
265261
_PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
266262
{
267263
wchar_t *argv0;
268264
wchar_t *p = NULL;
269265
Py_ssize_t n = 0;
266+
int have_script_arg = 0;
267+
int have_module_arg = 0;
270268
#ifdef HAVE_READLINK
271269
wchar_t link[MAXPATHLEN+1];
272270
wchar_t argv0copy[2*MAXPATHLEN+1];
@@ -278,11 +276,25 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
278276
wchar_t fullpath[MAX_PATH];
279277
#endif
280278

281-
282279
argv0 = argv[0];
280+
if (argc > 0 && argv0 != NULL) {
281+
have_module_arg = (wcscmp(argv0, L"-m") == 0);
282+
have_script_arg = !have_module_arg && (wcscmp(argv0, L"-c") != 0);
283+
}
284+
285+
if (have_module_arg) {
286+
#if defined(HAVE_REALPATH) || defined(MS_WINDOWS)
287+
_Py_wgetcwd(fullpath, Py_ARRAY_LENGTH(fullpath));
288+
argv0 = fullpath;
289+
n = wcslen(argv0);
290+
#else
291+
argv0 = L".";
292+
n = 1;
293+
#endif
294+
}
283295

284296
#ifdef HAVE_READLINK
285-
if (_HAVE_SCRIPT_ARGUMENT(argc, argv))
297+
if (have_script_arg)
286298
nr = _Py_wreadlink(argv0, link, MAXPATHLEN);
287299
if (nr > 0) {
288300
/* It's a symlink */
@@ -310,7 +322,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
310322

311323
#if SEP == '\\'
312324
/* Special case for Microsoft filename syntax */
313-
if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) {
325+
if (have_script_arg) {
314326
wchar_t *q;
315327
#if defined(MS_WINDOWS)
316328
/* Replace the first element in argv with the full path. */
@@ -334,7 +346,7 @@ _PyPathConfig_ComputeArgv0(int argc, wchar_t **argv)
334346
}
335347
}
336348
#else /* All other filename syntaxes */
337-
if (_HAVE_SCRIPT_ARGUMENT(argc, argv)) {
349+
if (have_script_arg) {
338350
#if defined(HAVE_REALPATH)
339351
if (_Py_wrealpath(argv0, fullpath, Py_ARRAY_LENGTH(fullpath))) {
340352
argv0 = fullpath;

0 commit comments

Comments
 (0)