Skip to content

Commit 700cfa8

Browse files
serhiy-storchakaserhiy-storchaka
authored
bpo-41069: Make TESTFN and the CWD for tests containing non-ascii characters. (GH-21035)
1 parent 8ea6353 commit 700cfa8

24 files changed

+110
-77
lines changed

‎Lib/test/libregrtest/main.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -597,6 +597,7 @@ def create_temp_dir(self):
597597
test_cwd = 'test_python_worker_{}'.format(pid)
598598
else:
599599
test_cwd = 'test_python_{}'.format(pid)
600+
test_cwd += support.FS_NONASCII
600601
test_cwd = os.path.join(self.tmp_dir, test_cwd)
601602
return test_cwd
602603

‎Lib/test/support/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
forget, import_fresh_module, import_module, make_legacy_pyc,
2121
modules_cleanup, modules_setup, unload)
2222
from .os_helper import (
23-
FS_NONASCII, SAVEDCWD, TESTFN, TESTFN_NONASCII,
23+
FS_NONASCII, SAVEDCWD, TESTFN, TESTFN_ASCII, TESTFN_NONASCII,
2424
TESTFN_UNENCODABLE, TESTFN_UNDECODABLE,
2525
TESTFN_UNICODE, can_symlink, can_xattr,
2626
change_cwd, create_empty_file, fd_count,

‎Lib/test/support/os_helper.py

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,16 @@
1313
# Filename used for testing
1414
if os.name == 'java':
1515
# Jython disallows @ in module names
16-
TESTFN = '$test'
16+
TESTFN_ASCII = '$test'
1717
else:
18-
TESTFN = '@test'
18+
TESTFN_ASCII = '@test'
1919

2020
# Disambiguate TESTFN for parallel testing, while letting it remain a valid
2121
# module name.
22-
TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
22+
TESTFN_ASCII = "{}_{}_tmp".format(TESTFN_ASCII, os.getpid())
2323

2424
# TESTFN_UNICODE is a non-ascii filename
25-
TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
25+
TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f"
2626
if sys.platform == 'darwin':
2727
# In Mac OS X's VFS API file names are, by definition, canonically
2828
# decomposed Unicode, encoded using UTF-8. See QA1173:
@@ -39,7 +39,7 @@
3939
if sys.getwindowsversion().platform >= 2:
4040
# Different kinds of characters from various languages to minimize the
4141
# probability that the whole name is encodable to MBCS (issue #9819)
42-
TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80"
42+
TESTFN_UNENCODABLE = TESTFN_ASCII + "-\u5171\u0141\u2661\u0363\uDC80"
4343
try:
4444
TESTFN_UNENCODABLE.encode(sys.getfilesystemencoding())
4545
except UnicodeEncodeError:
@@ -56,16 +56,16 @@
5656
b'\xff'.decode(sys.getfilesystemencoding())
5757
except UnicodeDecodeError:
5858
# 0xff will be encoded using the surrogate character u+DCFF
59-
TESTFN_UNENCODABLE = TESTFN \
59+
TESTFN_UNENCODABLE = TESTFN_ASCII \
6060
+ b'-\xff'.decode(sys.getfilesystemencoding(), 'surrogateescape')
6161
else:
6262
# File system encoding (eg. ISO-8859-* encodings) can encode
6363
# the byte 0xff. Skip some unicode filename tests.
6464
pass
6565

6666
# FS_NONASCII: non-ASCII character encodable by os.fsencode(),
67-
# or None if there is no such character.
68-
FS_NONASCII = None
67+
# or an empty string if there is no such character.
68+
FS_NONASCII = ''
6969
for character in (
7070
# First try printable and common characters to have a readable filename.
7171
# For each character, the encoding list are just example of encodings able
@@ -141,13 +141,14 @@
141141
try:
142142
name.decode(sys.getfilesystemencoding())
143143
except UnicodeDecodeError:
144-
TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
144+
TESTFN_UNDECODABLE = os.fsencode(TESTFN_ASCII) + name
145145
break
146146

147147
if FS_NONASCII:
148-
TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
148+
TESTFN_NONASCII = TESTFN_ASCII + FS_NONASCII
149149
else:
150150
TESTFN_NONASCII = None
151+
TESTFN = TESTFN_NONASCII or TESTFN_ASCII
151152

152153

153154
def make_bad_fd():

‎Lib/test/test_binhex.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@
1313
class BinHexTestCase(unittest.TestCase):
1414

1515
def setUp(self):
16-
self.fname1 = support.TESTFN + "1"
17-
self.fname2 = support.TESTFN + "2"
18-
self.fname3 = support.TESTFN + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
16+
# binhex supports only file names encodable to Latin1
17+
self.fname1 = support.TESTFN_ASCII + "1"
18+
self.fname2 = support.TESTFN_ASCII + "2"
19+
self.fname3 = support.TESTFN_ASCII + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
1920

2021
def tearDown(self):
2122
support.unlink(self.fname1)

‎Lib/test/test_cgitb.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ def test_syshook_no_logdir_default_format(self):
4141
rc, out, err = assert_python_failure(
4242
'-c',
4343
('import cgitb; cgitb.enable(logdir=%s); '
44-
'raise ValueError("Hello World")') % repr(tracedir))
45-
out = out.decode(sys.getfilesystemencoding())
44+
'raise ValueError("Hello World")') % repr(tracedir),
45+
PYTHONIOENCODING='utf-8')
46+
out = out.decode()
4647
self.assertIn("ValueError", out)
4748
self.assertIn("Hello World", out)
4849
self.assertIn("<strong>&lt;module&gt;</strong>", out)
@@ -56,8 +57,9 @@ def test_syshook_no_logdir_text_format(self):
5657
rc, out, err = assert_python_failure(
5758
'-c',
5859
('import cgitb; cgitb.enable(format="text", logdir=%s); '
59-
'raise ValueError("Hello World")') % repr(tracedir))
60-
out = out.decode(sys.getfilesystemencoding())
60+
'raise ValueError("Hello World")') % repr(tracedir),
61+
PYTHONIOENCODING='utf-8')
62+
out = out.decode()
6163
self.assertIn("ValueError", out)
6264
self.assertIn("Hello World", out)
6365
self.assertNotIn('<p>', out)

‎Lib/test/test_compileall.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -456,13 +456,15 @@ def _get_run_args(self, args):
456456

457457
def assertRunOK(self, *args, **env_vars):
458458
rc, out, err = script_helper.assert_python_ok(
459-
*self._get_run_args(args), **env_vars)
459+
*self._get_run_args(args), **env_vars,
460+
PYTHONIOENCODING='utf-8')
460461
self.assertEqual(b'', err)
461462
return out
462463

463464
def assertRunNotOK(self, *args, **env_vars):
464465
rc, out, err = script_helper.assert_python_failure(
465-
*self._get_run_args(args), **env_vars)
466+
*self._get_run_args(args), **env_vars,
467+
PYTHONIOENCODING='utf-8')
466468
return rc, out, err
467469

468470
def assertCompiled(self, fn):

‎Lib/test/test_embed.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,7 +1349,7 @@ def test_audit_run_file(self):
13491349
returncode=1)
13501350

13511351
def test_audit_run_interactivehook(self):
1352-
startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
1352+
startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
13531353
with open(startup, "w", encoding="utf-8") as f:
13541354
print("import sys", file=f)
13551355
print("sys.__interactivehook__ = lambda: None", file=f)
@@ -1362,7 +1362,7 @@ def test_audit_run_interactivehook(self):
13621362
os.unlink(startup)
13631363

13641364
def test_audit_run_startup(self):
1365-
startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
1365+
startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
13661366
with open(startup, "w", encoding="utf-8") as f:
13671367
print("pass", file=f)
13681368
try:

‎Lib/test/test_fstring.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,8 +1055,9 @@ def test_filename_in_syntaxerror(self):
10551055
file_path = os.path.join(cwd, 't.py')
10561056
with open(file_path, 'w') as f:
10571057
f.write('f"{a b}"') # This generates a SyntaxError
1058-
_, _, stderr = assert_python_failure(file_path)
1059-
self.assertIn(file_path, stderr.decode('utf-8'))
1058+
_, _, stderr = assert_python_failure(file_path,
1059+
PYTHONIOENCODING='ascii')
1060+
self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr)
10601061

10611062
def test_loop(self):
10621063
for i in range(1000):

‎Lib/test/test_genericpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -534,7 +534,7 @@ def test_import(self):
534534
class PathLikeTests(unittest.TestCase):
535535

536536
def setUp(self):
537-
self.file_name = support.TESTFN.lower()
537+
self.file_name = support.TESTFN
538538
self.file_path = FakePath(support.TESTFN)
539539
self.addCleanup(support.unlink, self.file_name)
540540
create_file(self.file_name, b"test_genericpath.PathLikeTests")

‎Lib/test/test_gzip.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -328,8 +328,15 @@ def test_metadata(self):
328328
cmByte = fRead.read(1)
329329
self.assertEqual(cmByte, b'\x08') # deflate
330330

331+
try:
332+
expectedname = self.filename.encode('Latin-1') + b'\x00'
333+
expectedflags = b'\x08' # only the FNAME flag is set
334+
except UnicodeEncodeError:
335+
expectedname = b''
336+
expectedflags = b'\x00'
337+
331338
flagsByte = fRead.read(1)
332-
self.assertEqual(flagsByte, b'\x08') # only the FNAME flag is set
339+
self.assertEqual(flagsByte, expectedflags)
333340

334341
mtimeBytes = fRead.read(4)
335342
self.assertEqual(mtimeBytes, struct.pack('<i', mtime)) # little-endian
@@ -344,9 +351,8 @@ def test_metadata(self):
344351
# RFC 1952 specifies that this is the name of the input file, if any.
345352
# However, the gzip module defaults to storing the name of the output
346353
# file in this field.
347-
expected = self.filename.encode('Latin-1') + b'\x00'
348-
nameBytes = fRead.read(len(expected))
349-
self.assertEqual(nameBytes, expected)
354+
nameBytes = fRead.read(len(expectedname))
355+
self.assertEqual(nameBytes, expectedname)
350356

351357
# Since no other flags were set, the header ends here.
352358
# Rather than process the compressed data, let's seek to the trailer.
@@ -358,6 +364,10 @@ def test_metadata(self):
358364
isizeBytes = fRead.read(4)
359365
self.assertEqual(isizeBytes, struct.pack('<i', len(data1)))
360366

367+
def test_metadata_ascii_name(self):
368+
self.filename = support.TESTFN_ASCII
369+
self.test_metadata()
370+
361371
def test_compresslevel_metadata(self):
362372
# see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
363373
# specifically, discussion of XFL in section 2.3.1

‎Lib/test/test_msilib.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
""" Test suite for the code in msilib """
22
import os
33
import unittest
4-
from test.support import TESTFN, FS_NONASCII, import_module, unlink
4+
from test.support import TESTFN, import_module, unlink
55
msilib = import_module('msilib')
66
import msilib.schema
77

88

99
def init_database():
10-
path = TESTFN + (FS_NONASCII or '') + '.msi'
10+
path = TESTFN + '.msi'
1111
db = msilib.init_database(
1212
path,
1313
msilib.schema,

‎Lib/test/test_ntpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,7 @@ class PathLikeTests(NtpathTestCase):
725725
path = ntpath
726726

727727
def setUp(self):
728-
self.file_name = support.TESTFN.lower()
728+
self.file_name = support.TESTFN
729729
self.file_path = FakePath(support.TESTFN)
730730
self.addCleanup(support.unlink, self.file_name)
731731
with open(self.file_name, 'xb', 0) as file:

‎Lib/test/test_os.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1174,7 +1174,7 @@ def setUp(self):
11741174
os.makedirs(t2_path)
11751175

11761176
for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
1177-
with open(path, "x") as f:
1177+
with open(path, "x", encoding='utf-8') as f:
11781178
f.write("I'm " + path + " and proud of it. Blame test_os.\n")
11791179

11801180
if support.can_symlink():
@@ -2360,7 +2360,7 @@ def setUp(self):
23602360
file_name = 'FILE%d' % i
23612361
file_path = os.path.join(support.TESTFN, file_name)
23622362
os.makedirs(dir_path)
2363-
with open(file_path, 'w') as f:
2363+
with open(file_path, 'w', encoding='utf-8') as f:
23642364
f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
23652365
self.created_paths.extend([dir_name, file_name])
23662366
self.created_paths.sort()
@@ -3738,7 +3738,7 @@ def test_path_t_converter(self):
37383738
if os.name == 'nt':
37393739
bytes_fspath = bytes_filename = None
37403740
else:
3741-
bytes_filename = support.TESTFN.encode('ascii')
3741+
bytes_filename = os.fsencode(support.TESTFN)
37423742
bytes_fspath = FakePath(bytes_filename)
37433743
fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT)
37443744
self.addCleanup(support.unlink, support.TESTFN)

‎Lib/test/test_pdb.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1198,6 +1198,7 @@ def _run_pdb(self, pdb_args, commands):
11981198
stdout=subprocess.PIPE,
11991199
stdin=subprocess.PIPE,
12001200
stderr=subprocess.STDOUT,
1201+
env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
12011202
) as proc:
12021203
stdout, stderr = proc.communicate(str.encode(commands))
12031204
stdout = stdout and bytes.decode(stdout)
@@ -1353,10 +1354,11 @@ def start_pdb():
13531354
stdout=subprocess.PIPE,
13541355
stdin=subprocess.PIPE,
13551356
stderr=subprocess.STDOUT,
1357+
env={**os.environ, 'PYTHONIOENCODING': 'utf-8'}
13561358
)
13571359
self.addCleanup(proc.stdout.close)
13581360
stdout, stderr = proc.communicate(b'cont\n')
1359-
self.assertNotIn('Error', stdout.decode(),
1361+
self.assertNotIn(b'Error', stdout,
13601362
"Got an error running test script under PDB")
13611363

13621364
def test_issue36250(self):
@@ -1382,10 +1384,11 @@ def start_pdb():
13821384
stdout=subprocess.PIPE,
13831385
stdin=subprocess.PIPE,
13841386
stderr=subprocess.STDOUT,
1387+
env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
13851388
)
13861389
self.addCleanup(proc.stdout.close)
13871390
stdout, stderr = proc.communicate(b'cont\ncont\n')
1388-
self.assertNotIn('Error', stdout.decode(),
1391+
self.assertNotIn(b'Error', stdout,
13891392
"Got an error running test script under PDB")
13901393

13911394
def test_issue16180(self):
@@ -1425,8 +1428,8 @@ def test_readrc_kwarg(self):
14251428
)
14261429
with proc:
14271430
stdout, stderr = proc.communicate(b'q\n')
1428-
self.assertNotIn("NameError: name 'invalid' is not defined",
1429-
stdout.decode())
1431+
self.assertNotIn(b"NameError: name 'invalid' is not defined",
1432+
stdout)
14301433

14311434
finally:
14321435
if save_home is not None:

‎Lib/test/test_posixpath.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ class PathLikeTests(unittest.TestCase):
627627
path = posixpath
628628

629629
def setUp(self):
630-
self.file_name = support.TESTFN.lower()
630+
self.file_name = support.TESTFN
631631
self.file_path = FakePath(support.TESTFN)
632632
self.addCleanup(support.unlink, self.file_name)
633633
with open(self.file_name, 'xb', 0) as file:

‎Lib/test/test_tarfile.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2305,7 +2305,8 @@ def test_test_command(self):
23052305
def test_test_command_verbose(self):
23062306
for tar_name in testtarnames:
23072307
for opt in '-v', '--verbose':
2308-
out = self.tarfilecmd(opt, '-t', tar_name)
2308+
out = self.tarfilecmd(opt, '-t', tar_name,
2309+
PYTHONIOENCODING='utf-8')
23092310
self.assertIn(b'is a tar archive.\n', out)
23102311

23112312
def test_test_command_invalid_file(self):
@@ -2376,7 +2377,8 @@ def test_create_command_verbose(self):
23762377
'and-utf8-bom-sig-only.txt')]
23772378
for opt in '-v', '--verbose':
23782379
try:
2379-
out = self.tarfilecmd(opt, '-c', tmpname, *files)
2380+
out = self.tarfilecmd(opt, '-c', tmpname, *files,
2381+
PYTHONIOENCODING='utf-8')
23802382
self.assertIn(b' file created.', out)
23812383
with tarfile.open(tmpname) as tar:
23822384
tar.getmembers()
@@ -2434,7 +2436,8 @@ def test_extract_command_verbose(self):
24342436
for opt in '-v', '--verbose':
24352437
try:
24362438
with support.temp_cwd(tarextdir):
2437-
out = self.tarfilecmd(opt, '-e', tmpname)
2439+
out = self.tarfilecmd(opt, '-e', tmpname,
2440+
PYTHONIOENCODING='utf-8')
24382441
self.assertIn(b' file is extracted.', out)
24392442
finally:
24402443
support.rmtree(tarextdir)

‎Lib/test/test_tools/test_pathfix.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,16 +30,18 @@ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr='',
3030
with open(filename, 'w', encoding='utf8') as f:
3131
f.write(f'{shebang}\n' + 'print("Hello world")\n')
3232

33+
encoding = sys.getfilesystemencoding()
3334
proc = subprocess.run(
3435
[sys.executable, self.script,
3536
*pathfix_flags, '-n', pathfix_arg],
36-
capture_output=True, text=1)
37+
env={**os.environ, 'PYTHONIOENCODING': encoding},
38+
capture_output=True)
3739

3840
if stdout == '' and proc.returncode == 0:
3941
stdout = f'{filename}: updating\n'
4042
self.assertEqual(proc.returncode, exitcode, proc)
41-
self.assertEqual(proc.stdout, stdout, proc)
42-
self.assertEqual(proc.stderr, stderr, proc)
43+
self.assertEqual(proc.stdout.decode(encoding), stdout.replace('\n', os.linesep), proc)
44+
self.assertEqual(proc.stderr.decode(encoding), stderr.replace('\n', os.linesep), proc)
4345

4446
with open(filename, 'r', encoding='utf8') as f:
4547
output = f.read()

0 commit comments

Comments
 (0)