Skip to content

Commit f925407

Browse files
serhiy-storchakaserhiy-storchaka
authored
[3.9] bpo-41069: Make TESTFN and the CWD for tests containing non-ascii characters. (GH-21035). (GH-21156)
(cherry picked from commit 700cfa8)
1 parent ad7a667 commit f925407

23 files changed

+108
-76
lines changed

‎Lib/test/libregrtest/main.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,7 @@ def create_temp_dir(self):
596596
test_cwd = 'test_python_worker_{}'.format(pid)
597597
else:
598598
test_cwd = 'test_python_{}'.format(pid)
599+
test_cwd += support.FS_NONASCII
599600
test_cwd = os.path.join(self.tmp_dir, test_cwd)
600601
return test_cwd
601602

‎Lib/test/support/__init__.py

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -720,21 +720,21 @@ def requires_lzma(reason='requires lzma'):
720720
# Filename used for testing
721721
if os.name == 'java':
722722
# Jython disallows @ in module names
723-
TESTFN = '$test'
723+
TESTFN_ASCII = '$test'
724724
else:
725-
TESTFN = '@test'
725+
TESTFN_ASCII = '@test'
726726

727727
# Disambiguate TESTFN for parallel testing, while letting it remain a valid
728728
# module name.
729-
TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
729+
TESTFN_ASCII = "{}_{}_tmp".format(TESTFN_ASCII, os.getpid())
730730

731731
# Define the URL of a dedicated HTTP server for the network tests.
732732
# The URL must use clear-text HTTP: no redirection to encrypted HTTPS.
733733
TEST_HTTP_URL = "http://www.pythontest.net"
734734

735735
# FS_NONASCII: non-ASCII character encodable by os.fsencode(),
736-
# or None if there is no such character.
737-
FS_NONASCII = None
736+
# or an empty string if there is no such character.
737+
FS_NONASCII = ''
738738
for character in (
739739
# First try printable and common characters to have a readable filename.
740740
# For each character, the encoding list are just example of encodings able
@@ -781,7 +781,7 @@ def requires_lzma(reason='requires lzma'):
781781
break
782782

783783
# TESTFN_UNICODE is a non-ascii filename
784-
TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
784+
TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f"
785785
if sys.platform == 'darwin':
786786
# In Mac OS X's VFS API file names are, by definition, canonically
787787
# decomposed Unicode, encoded using UTF-8. See QA1173:
@@ -799,7 +799,7 @@ def requires_lzma(reason='requires lzma'):
799799
if sys.getwindowsversion().platform >= 2:
800800
# Different kinds of characters from various languages to minimize the
801801
# probability that the whole name is encodable to MBCS (issue #9819)
802-
TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80"
802+
TESTFN_UNENCODABLE = TESTFN_ASCII + "-\u5171\u0141\u2661\u0363\uDC80"
803803
try:
804804
TESTFN_UNENCODABLE.encode(TESTFN_ENCODING)
805805
except UnicodeEncodeError:
@@ -816,7 +816,7 @@ def requires_lzma(reason='requires lzma'):
816816
b'\xff'.decode(TESTFN_ENCODING)
817817
except UnicodeDecodeError:
818818
# 0xff will be encoded using the surrogate character u+DCFF
819-
TESTFN_UNENCODABLE = TESTFN \
819+
TESTFN_UNENCODABLE = TESTFN_ASCII \
820820
+ b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape')
821821
else:
822822
# File system encoding (eg. ISO-8859-* encodings) can encode
@@ -850,13 +850,14 @@ def requires_lzma(reason='requires lzma'):
850850
try:
851851
name.decode(TESTFN_ENCODING)
852852
except UnicodeDecodeError:
853-
TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
853+
TESTFN_UNDECODABLE = os.fsencode(TESTFN_ASCII) + name
854854
break
855855

856856
if FS_NONASCII:
857-
TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
857+
TESTFN_NONASCII = TESTFN_ASCII + FS_NONASCII
858858
else:
859859
TESTFN_NONASCII = None
860+
TESTFN = TESTFN_NONASCII or TESTFN_ASCII
860861

861862
# Save the initial cwd
862863
SAVEDCWD = os.getcwd()

‎Lib/test/test_binhex.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@
1313
class BinHexTestCase(unittest.TestCase):
1414

1515
def setUp(self):
16-
self.fname1 = support.TESTFN + "1"
17-
self.fname2 = support.TESTFN + "2"
18-
self.fname3 = support.TESTFN + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
16+
# binhex supports only file names encodable to Latin1
17+
self.fname1 = support.TESTFN_ASCII + "1"
18+
self.fname2 = support.TESTFN_ASCII + "2"
19+
self.fname3 = support.TESTFN_ASCII + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
1920

2021
def tearDown(self):
2122
support.unlink(self.fname1)

‎Lib/test/test_cgitb.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ def test_syshook_no_logdir_default_format(self):
4141
rc, out, err = assert_python_failure(
4242
'-c',
4343
('import cgitb; cgitb.enable(logdir=%s); '
44-
'raise ValueError("Hello World")') % repr(tracedir))
45-
out = out.decode(sys.getfilesystemencoding())
44+
'raise ValueError("Hello World")') % repr(tracedir),
45+
PYTHONIOENCODING='utf-8')
46+
out = out.decode()
4647
self.assertIn("ValueError", out)
4748
self.assertIn("Hello World", out)
4849
self.assertIn("<strong>&lt;module&gt;</strong>", out)
@@ -56,8 +57,9 @@ def test_syshook_no_logdir_text_format(self):
5657
rc, out, err = assert_python_failure(
5758
'-c',
5859
('import cgitb; cgitb.enable(format="text", logdir=%s); '
59-
'raise ValueError("Hello World")') % repr(tracedir))
60-
out = out.decode(sys.getfilesystemencoding())
60+
'raise ValueError("Hello World")') % repr(tracedir),
61+
PYTHONIOENCODING='utf-8')
62+
out = out.decode()
6163
self.assertIn("ValueError", out)
6264
self.assertIn("Hello World", out)
6365
self.assertNotIn('<p>', out)

‎Lib/test/test_compileall.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,13 +456,15 @@ def _get_run_args(self, args):
456456

457457
def assertRunOK(self, *args, **env_vars):
458458
rc, out, err = script_helper.assert_python_ok(
459-
*self._get_run_args(args), **env_vars)
459+
*self._get_run_args(args), **env_vars,
460+
PYTHONIOENCODING='utf-8')
460461
self.assertEqual(b'', err)
461462
return out
462463

463464
def assertRunNotOK(self, *args, **env_vars):
464465
rc, out, err = script_helper.assert_python_failure(
465-
*self._get_run_args(args), **env_vars)
466+
*self._get_run_args(args), **env_vars,
467+
PYTHONIOENCODING='utf-8')
466468
return rc, out, err
467469

468470
def assertCompiled(self, fn):

‎Lib/test/test_embed.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1353,7 +1353,7 @@ def test_audit_run_file(self):
13531353
returncode=1)
13541354

13551355
def test_audit_run_interactivehook(self):
1356-
startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
1356+
startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
13571357
with open(startup, "w", encoding="utf-8") as f:
13581358
print("import sys", file=f)
13591359
print("sys.__interactivehook__ = lambda: None", file=f)
@@ -1366,7 +1366,7 @@ def test_audit_run_interactivehook(self):
13661366
os.unlink(startup)
13671367

13681368
def test_audit_run_startup(self):
1369-
startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
1369+
startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
13701370
with open(startup, "w", encoding="utf-8") as f:
13711371
print("pass", file=f)
13721372
try:

‎Lib/test/test_fstring.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1061,8 +1061,9 @@ def test_filename_in_syntaxerror(self):
10611061
file_path = os.path.join(cwd, 't.py')
10621062
with open(file_path, 'w') as f:
10631063
f.write('f"{a b}"') # This generates a SyntaxError
1064-
_, _, stderr = assert_python_failure(file_path)
1065-
self.assertIn(file_path, stderr.decode('utf-8'))
1064+
_, _, stderr = assert_python_failure(file_path,
1065+
PYTHONIOENCODING='ascii')
1066+
self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr)
10661067

10671068
def test_loop(self):
10681069
for i in range(1000):

‎Lib/test/test_genericpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,7 @@ def test_import(self):
534534
class PathLikeTests(unittest.TestCase):
535535

536536
def setUp(self):
537-
self.file_name = support.TESTFN.lower()
537+
self.file_name = support.TESTFN
538538
self.file_path = FakePath(support.TESTFN)
539539
self.addCleanup(support.unlink, self.file_name)
540540
create_file(self.file_name, b"test_genericpath.PathLikeTests")

‎Lib/test/test_gzip.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -328,8 +328,15 @@ def test_metadata(self):
328328
cmByte = fRead.read(1)
329329
self.assertEqual(cmByte, b'\x08') # deflate
330330

331+
try:
332+
expectedname = self.filename.encode('Latin-1') + b'\x00'
333+
expectedflags = b'\x08' # only the FNAME flag is set
334+
except UnicodeEncodeError:
335+
expectedname = b''
336+
expectedflags = b'\x00'
337+
331338
flagsByte = fRead.read(1)
332-
self.assertEqual(flagsByte, b'\x08') # only the FNAME flag is set
339+
self.assertEqual(flagsByte, expectedflags)
333340

334341
mtimeBytes = fRead.read(4)
335342
self.assertEqual(mtimeBytes, struct.pack('<i', mtime)) # little-endian
@@ -344,9 +351,8 @@ def test_metadata(self):
344351
# RFC 1952 specifies that this is the name of the input file, if any.
345352
# However, the gzip module defaults to storing the name of the output
346353
# file in this field.
347-
expected = self.filename.encode('Latin-1') + b'\x00'
348-
nameBytes = fRead.read(len(expected))
349-
self.assertEqual(nameBytes, expected)
354+
nameBytes = fRead.read(len(expectedname))
355+
self.assertEqual(nameBytes, expectedname)
350356

351357
# Since no other flags were set, the header ends here.
352358
# Rather than process the compressed data, let's seek to the trailer.
@@ -358,6 +364,10 @@ def test_metadata(self):
358364
isizeBytes = fRead.read(4)
359365
self.assertEqual(isizeBytes, struct.pack('<i', len(data1)))
360366

367+
def test_metadata_ascii_name(self):
368+
self.filename = support.TESTFN_ASCII
369+
self.test_metadata()
370+
361371
def test_compresslevel_metadata(self):
362372
# see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
363373
# specifically, discussion of XFL in section 2.3.1

‎Lib/test/test_msilib.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
""" Test suite for the code in msilib """
22
import os
33
import unittest
4-
from test.support import TESTFN, FS_NONASCII, import_module, unlink
4+
from test.support import TESTFN, import_module, unlink
55
msilib = import_module('msilib')
66
import msilib.schema
77

88

99
def init_database():
10-
path = TESTFN + (FS_NONASCII or '') + '.msi'
10+
path = TESTFN + '.msi'
1111
db = msilib.init_database(
1212
path,
1313
msilib.schema,

‎Lib/test/test_ntpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,7 @@ class PathLikeTests(NtpathTestCase):
725725
path = ntpath
726726

727727
def setUp(self):
728-
self.file_name = support.TESTFN.lower()
728+
self.file_name = support.TESTFN
729729
self.file_path = FakePath(support.TESTFN)
730730
self.addCleanup(support.unlink, self.file_name)
731731
with open(self.file_name, 'xb', 0) as file:

‎Lib/test/test_os.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,7 +1173,7 @@ def setUp(self):
11731173
os.makedirs(t2_path)
11741174

11751175
for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
1176-
with open(path, "x") as f:
1176+
with open(path, "x", encoding='utf-8') as f:
11771177
f.write("I'm " + path + " and proud of it. Blame test_os.\n")
11781178

11791179
if support.can_symlink():
@@ -2359,7 +2359,7 @@ def setUp(self):
23592359
file_name = 'FILE%d' % i
23602360
file_path = os.path.join(support.TESTFN, file_name)
23612361
os.makedirs(dir_path)
2362-
with open(file_path, 'w') as f:
2362+
with open(file_path, 'w', encoding='utf-8') as f:
23632363
f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
23642364
self.created_paths.extend([dir_name, file_name])
23652365
self.created_paths.sort()
@@ -3737,7 +3737,7 @@ def test_path_t_converter(self):
37373737
if os.name == 'nt':
37383738
bytes_fspath = bytes_filename = None
37393739
else:
3740-
bytes_filename = support.TESTFN.encode('ascii')
3740+
bytes_filename = os.fsencode(support.TESTFN)
37413741
bytes_fspath = FakePath(bytes_filename)
37423742
fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT)
37433743
self.addCleanup(support.unlink, support.TESTFN)

‎Lib/test/test_pdb.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,6 +1198,7 @@ def _run_pdb(self, pdb_args, commands):
11981198
stdout=subprocess.PIPE,
11991199
stdin=subprocess.PIPE,
12001200
stderr=subprocess.STDOUT,
1201+
env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
12011202
) as proc:
12021203
stdout, stderr = proc.communicate(str.encode(commands))
12031204
stdout = stdout and bytes.decode(stdout)
@@ -1353,10 +1354,11 @@ def start_pdb():
13531354
stdout=subprocess.PIPE,
13541355
stdin=subprocess.PIPE,
13551356
stderr=subprocess.STDOUT,
1357+
env={**os.environ, 'PYTHONIOENCODING': 'utf-8'}
13561358
)
13571359
self.addCleanup(proc.stdout.close)
13581360
stdout, stderr = proc.communicate(b'cont\n')
1359-
self.assertNotIn('Error', stdout.decode(),
1361+
self.assertNotIn(b'Error', stdout,
13601362
"Got an error running test script under PDB")
13611363

13621364
def test_issue36250(self):
@@ -1382,10 +1384,11 @@ def start_pdb():
13821384
stdout=subprocess.PIPE,
13831385
stdin=subprocess.PIPE,
13841386
stderr=subprocess.STDOUT,
1387+
env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
13851388
)
13861389
self.addCleanup(proc.stdout.close)
13871390
stdout, stderr = proc.communicate(b'cont\ncont\n')
1388-
self.assertNotIn('Error', stdout.decode(),
1391+
self.assertNotIn(b'Error', stdout,
13891392
"Got an error running test script under PDB")
13901393

13911394
def test_issue16180(self):
@@ -1425,8 +1428,8 @@ def test_readrc_kwarg(self):
14251428
)
14261429
with proc:
14271430
stdout, stderr = proc.communicate(b'q\n')
1428-
self.assertNotIn("NameError: name 'invalid' is not defined",
1429-
stdout.decode())
1431+
self.assertNotIn(b"NameError: name 'invalid' is not defined",
1432+
stdout)
14301433

14311434
finally:
14321435
if save_home is not None:

‎Lib/test/test_posixpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ class PathLikeTests(unittest.TestCase):
627627
path = posixpath
628628

629629
def setUp(self):
630-
self.file_name = support.TESTFN.lower()
630+
self.file_name = support.TESTFN
631631
self.file_path = FakePath(support.TESTFN)
632632
self.addCleanup(support.unlink, self.file_name)
633633
with open(self.file_name, 'xb', 0) as file:

‎Lib/test/test_tarfile.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2305,7 +2305,8 @@ def test_test_command(self):
23052305
def test_test_command_verbose(self):
23062306
for tar_name in testtarnames:
23072307
for opt in '-v', '--verbose':
2308-
out = self.tarfilecmd(opt, '-t', tar_name)
2308+
out = self.tarfilecmd(opt, '-t', tar_name,
2309+
PYTHONIOENCODING='utf-8')
23092310
self.assertIn(b'is a tar archive.\n', out)
23102311

23112312
def test_test_command_invalid_file(self):
@@ -2376,7 +2377,8 @@ def test_create_command_verbose(self):
23762377
'and-utf8-bom-sig-only.txt')]
23772378
for opt in '-v', '--verbose':
23782379
try:
2379-
out = self.tarfilecmd(opt, '-c', tmpname, *files)
2380+
out = self.tarfilecmd(opt, '-c', tmpname, *files,
2381+
PYTHONIOENCODING='utf-8')
23802382
self.assertIn(b' file created.', out)
23812383
with tarfile.open(tmpname) as tar:
23822384
tar.getmembers()
@@ -2434,7 +2436,8 @@ def test_extract_command_verbose(self):
24342436
for opt in '-v', '--verbose':
24352437
try:
24362438
with support.temp_cwd(tarextdir):
2437-
out = self.tarfilecmd(opt, '-e', tmpname)
2439+
out = self.tarfilecmd(opt, '-e', tmpname,
2440+
PYTHONIOENCODING='utf-8')
24382441
self.assertIn(b' file is extracted.', out)
24392442
finally:
24402443
support.rmtree(tarextdir)

‎Lib/test/test_tools/test_pathfix.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,18 @@ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr='',
3030
with open(filename, 'w', encoding='utf8') as f:
3131
f.write(f'{shebang}\n' + 'print("Hello world")\n')
3232

33+
encoding = sys.getfilesystemencoding()
3334
proc = subprocess.run(
3435
[sys.executable, self.script,
3536
*pathfix_flags, '-n', pathfix_arg],
36-
capture_output=True, text=1)
37+
env={**os.environ, 'PYTHONIOENCODING': encoding},
38+
capture_output=True)
3739

3840
if stdout == '' and proc.returncode == 0:
3941
stdout = f'{filename}: updating\n'
4042
self.assertEqual(proc.returncode, exitcode, proc)
41-
self.assertEqual(proc.stdout, stdout, proc)
42-
self.assertEqual(proc.stderr, stderr, proc)
43+
self.assertEqual(proc.stdout.decode(encoding), stdout.replace('\n', os.linesep), proc)
44+
self.assertEqual(proc.stderr.decode(encoding), stderr.replace('\n', os.linesep), proc)
4345

4446
with open(filename, 'r', encoding='utf8') as f:
4547
output = f.read()

0 commit comments

Comments
 (0)