Skip to content

Commit a84cb74

Browse files
authored
gh-109276: libregrtest calls random.seed() before each test (#109279)
libregrtest now calls random.seed() before running each test file when -r/--randomize command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process before running all test files and it was not called in worker processes. * Convert some f-strings to regular strings in test_regrtest when f-string is not needed. * Remove unused all_methods variable from test_regrtest. * Add RunTests members are now mandatory.
1 parent 4e77645 commit a84cb74

File tree

5 files changed

+94
-34
lines changed

5 files changed

+94
-34
lines changed

Lib/test/libregrtest/main.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,8 +112,11 @@ def __init__(self, ns: Namespace):
112112
self.junit_filename: StrPath | None = ns.xmlpath
113113
self.memory_limit: str | None = ns.memlimit
114114
self.gc_threshold: int | None = ns.threshold
115-
self.use_resources: list[str] = ns.use_resources
116-
self.python_cmd: list[str] | None = ns.python
115+
self.use_resources: tuple[str] = tuple(ns.use_resources)
116+
if ns.python:
117+
self.python_cmd: tuple[str] = tuple(ns.python)
118+
else:
119+
self.python_cmd = None
117120
self.coverage: bool = ns.trace
118121
self.coverage_dir: StrPath | None = ns.coverdir
119122
self.tmp_dir: StrPath | None = ns.tempdir
@@ -377,8 +380,11 @@ def create_run_tests(self, tests: TestTuple):
377380
return RunTests(
378381
tests,
379382
fail_fast=self.fail_fast,
383+
fail_env_changed=self.fail_env_changed,
380384
match_tests=self.match_tests,
381385
ignore_tests=self.ignore_tests,
386+
match_tests_dict=None,
387+
rerun=None,
382388
forever=self.forever,
383389
pgo=self.pgo,
384390
pgo_extended=self.pgo_extended,
@@ -393,6 +399,9 @@ def create_run_tests(self, tests: TestTuple):
393399
gc_threshold=self.gc_threshold,
394400
use_resources=self.use_resources,
395401
python_cmd=self.python_cmd,
402+
randomize=self.randomize,
403+
random_seed=self.random_seed,
404+
json_fd=None,
396405
)
397406

398407
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

Lib/test/libregrtest/runtests.py

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,29 +16,31 @@ class HuntRefleak:
1616
@dataclasses.dataclass(slots=True, frozen=True)
1717
class RunTests:
1818
tests: TestTuple
19-
fail_fast: bool = False
20-
fail_env_changed: bool = False
21-
match_tests: FilterTuple | None = None
22-
ignore_tests: FilterTuple | None = None
23-
match_tests_dict: FilterDict | None = None
24-
rerun: bool = False
25-
forever: bool = False
26-
pgo: bool = False
27-
pgo_extended: bool = False
28-
output_on_failure: bool = False
29-
timeout: float | None = None
30-
verbose: int = 0
31-
quiet: bool = False
32-
hunt_refleak: HuntRefleak | None = None
33-
test_dir: StrPath | None = None
34-
use_junit: bool = False
35-
memory_limit: str | None = None
36-
gc_threshold: int | None = None
37-
use_resources: list[str] = dataclasses.field(default_factory=list)
38-
python_cmd: list[str] | None = None
19+
fail_fast: bool
20+
fail_env_changed: bool
21+
match_tests: FilterTuple | None
22+
ignore_tests: FilterTuple | None
23+
match_tests_dict: FilterDict | None
24+
rerun: bool
25+
forever: bool
26+
pgo: bool
27+
pgo_extended: bool
28+
output_on_failure: bool
29+
timeout: float | None
30+
verbose: int
31+
quiet: bool
32+
hunt_refleak: HuntRefleak | None
33+
test_dir: StrPath | None
34+
use_junit: bool
35+
memory_limit: str | None
36+
gc_threshold: int | None
37+
use_resources: tuple[str]
38+
python_cmd: tuple[str] | None
39+
randomize: bool
40+
random_seed: int | None
3941
# On Unix, it's a file descriptor.
4042
# On Windows, it's a handle.
41-
json_fd: int | None = None
43+
json_fd: int | None
4244

4345
def copy(self, **override):
4446
state = dataclasses.asdict(self)

Lib/test/libregrtest/setup.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import faulthandler
22
import os
3+
import random
34
import signal
45
import sys
56
import unittest
@@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests):
127128

128129
if runtests.gc_threshold is not None:
129130
gc.set_threshold(runtests.gc_threshold)
131+
132+
if runtests.randomize:
133+
random.seed(runtests.random_seed)

Lib/test/test_regrtest.py

Lines changed: 50 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import locale
1212
import os.path
1313
import platform
14+
import random
1415
import re
1516
import subprocess
1617
import sys
@@ -504,7 +505,7 @@ def list_regex(line_format, tests):
504505
if rerun is not None:
505506
regex = list_regex('%s re-run test%s', [rerun.name])
506507
self.check_line(output, regex)
507-
regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode"
508+
regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
508509
self.check_line(output, regex)
509510
regex = fr"Re-running {rerun.name} in verbose mode"
510511
if rerun.match:
@@ -1019,13 +1020,13 @@ def test_run(self):
10191020
forever=True)
10201021

10211022
@without_optimizer
1022-
def check_leak(self, code, what, *, multiprocessing=False):
1023+
def check_leak(self, code, what, *, run_workers=False):
10231024
test = self.create_test('huntrleaks', code=code)
10241025

10251026
filename = 'reflog.txt'
10261027
self.addCleanup(os_helper.unlink, filename)
10271028
cmd = ['--huntrleaks', '3:3:']
1028-
if multiprocessing:
1029+
if run_workers:
10291030
cmd.append('-j1')
10301031
cmd.append(test)
10311032
output = self.run_tests(*cmd,
@@ -1044,7 +1045,7 @@ def check_leak(self, code, what, *, multiprocessing=False):
10441045
self.assertIn(line2, reflog)
10451046

10461047
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
1047-
def check_huntrleaks(self, *, multiprocessing: bool):
1048+
def check_huntrleaks(self, *, run_workers: bool):
10481049
# test --huntrleaks
10491050
code = textwrap.dedent("""
10501051
import unittest
@@ -1055,13 +1056,13 @@ class RefLeakTest(unittest.TestCase):
10551056
def test_leak(self):
10561057
GLOBAL_LIST.append(object())
10571058
""")
1058-
self.check_leak(code, 'references', multiprocessing=multiprocessing)
1059+
self.check_leak(code, 'references', run_workers=run_workers)
10591060

10601061
def test_huntrleaks(self):
1061-
self.check_huntrleaks(multiprocessing=False)
1062+
self.check_huntrleaks(run_workers=False)
10621063

10631064
def test_huntrleaks_mp(self):
1064-
self.check_huntrleaks(multiprocessing=True)
1065+
self.check_huntrleaks(run_workers=True)
10651066

10661067
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
10671068
def test_huntrleaks_fd_leak(self):
@@ -1139,8 +1140,6 @@ def test_method3(self):
11391140
def test_method4(self):
11401141
pass
11411142
""")
1142-
all_methods = ['test_method1', 'test_method2',
1143-
'test_method3', 'test_method4']
11441143
testname = self.create_test(code=code)
11451144

11461145
# only run a subset
@@ -1762,7 +1761,7 @@ def test_mp_decode_error(self):
17621761
if encoding is None:
17631762
encoding = sys.__stdout__.encoding
17641763
if encoding is None:
1765-
self.skipTest(f"cannot get regrtest worker encoding")
1764+
self.skipTest("cannot get regrtest worker encoding")
17661765

17671766
nonascii = b"byte:\xa0\xa9\xff\n"
17681767
try:
@@ -1789,7 +1788,7 @@ def test_mp_decode_error(self):
17891788
stats=0)
17901789

17911790
def test_doctest(self):
1792-
code = textwrap.dedent(fr'''
1791+
code = textwrap.dedent(r'''
17931792
import doctest
17941793
import sys
17951794
from test import support
@@ -1827,6 +1826,46 @@ def load_tests(loader, tests, pattern):
18271826
randomize=True,
18281827
stats=TestStats(1, 1, 0))
18291828

1829+
def _check_random_seed(self, run_workers: bool):
1830+
# gh-109276: When -r/--randomize is used, random.seed() is called
1831+
# with the same random seed before running each test file.
1832+
code = textwrap.dedent(r'''
1833+
import random
1834+
import unittest
1835+
1836+
class RandomSeedTest(unittest.TestCase):
1837+
def test_randint(self):
1838+
numbers = [random.randint(0, 1000) for _ in range(10)]
1839+
print(f"Random numbers: {numbers}")
1840+
''')
1841+
tests = [self.create_test(name=f'test_random{i}', code=code)
1842+
for i in range(1, 3+1)]
1843+
1844+
random_seed = 856_656_202
1845+
cmd = ["--randomize", f"--randseed={random_seed}"]
1846+
if run_workers:
1847+
# run as many worker processes than the number of tests
1848+
cmd.append(f'-j{len(tests)}')
1849+
cmd.extend(tests)
1850+
output = self.run_tests(*cmd)
1851+
1852+
random.seed(random_seed)
1853+
# Make the assumption that nothing consume entropy between libregrest
1854+
# setup_tests() which calls random.seed() and RandomSeedTest calling
1855+
# random.randint().
1856+
numbers = [random.randint(0, 1000) for _ in range(10)]
1857+
expected = f"Random numbers: {numbers}"
1858+
1859+
regex = r'^Random numbers: .*$'
1860+
matches = re.findall(regex, output, flags=re.MULTILINE)
1861+
self.assertEqual(matches, [expected] * len(tests))
1862+
1863+
def test_random_seed(self):
1864+
self._check_random_seed(run_workers=False)
1865+
1866+
def test_random_seed_workers(self):
1867+
self._check_random_seed(run_workers=True)
1868+
18301869

18311870
class TestUtils(unittest.TestCase):
18321871
def test_format_duration(self):
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
libregrtest now calls :func:`random.seed()` before running each test file
2+
when ``-r/--randomize`` command line option is used. Moreover, it's also
3+
called in worker processes. It should help to make tests more
4+
deterministic. Previously, it was only called once in the main process before
5+
running all test files and it was not called in worker processes. Patch by
6+
Victor Stinner.

0 commit comments

Comments
 (0)