mirror of
https://github.com/python/cpython.git
synced 2026-01-06 15:32:22 +00:00
gh-109276: libregrtest calls random.seed() before each test (#109279)
libregrtest now calls random.seed() before running each test file when -r/--randomize command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process before running all test files and it was not called in worker processes. * Convert some f-strings to regular strings in test_regrtest when f-string is not needed. * Remove unused all_methods variable from test_regrtest. * Add RunTests members are now mandatory.
This commit is contained in:
Victor Stinner
GitHub
parent
4e77645986
commit
a84cb74d42
5 changed files with 94 additions and 34 deletions
|
|
@ -112,8 +112,11 @@ def __init__(self, ns: Namespace):
|
|||
self.junit_filename: StrPath | None = ns.xmlpath
|
||||
self.memory_limit: str | None = ns.memlimit
|
||||
self.gc_threshold: int | None = ns.threshold
|
||||
self.use_resources: list[str] = ns.use_resources
|
||||
self.python_cmd: list[str] | None = ns.python
|
||||
self.use_resources: tuple[str] = tuple(ns.use_resources)
|
||||
if ns.python:
|
||||
self.python_cmd: tuple[str] = tuple(ns.python)
|
||||
else:
|
||||
self.python_cmd = None
|
||||
self.coverage: bool = ns.trace
|
||||
self.coverage_dir: StrPath | None = ns.coverdir
|
||||
self.tmp_dir: StrPath | None = ns.tempdir
|
||||
|
|
@ -377,8 +380,11 @@ def create_run_tests(self, tests: TestTuple):
|
|||
return RunTests(
|
||||
tests,
|
||||
fail_fast=self.fail_fast,
|
||||
fail_env_changed=self.fail_env_changed,
|
||||
match_tests=self.match_tests,
|
||||
ignore_tests=self.ignore_tests,
|
||||
match_tests_dict=None,
|
||||
rerun=None,
|
||||
forever=self.forever,
|
||||
pgo=self.pgo,
|
||||
pgo_extended=self.pgo_extended,
|
||||
|
|
@ -393,6 +399,9 @@ def create_run_tests(self, tests: TestTuple):
|
|||
gc_threshold=self.gc_threshold,
|
||||
use_resources=self.use_resources,
|
||||
python_cmd=self.python_cmd,
|
||||
randomize=self.randomize,
|
||||
random_seed=self.random_seed,
|
||||
json_fd=None,
|
||||
)
|
||||
|
||||
def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:
|
||||
|
|
|
|||
|
|
@ -16,29 +16,31 @@ class HuntRefleak:
|
|||
@dataclasses.dataclass(slots=True, frozen=True)
|
||||
class RunTests:
|
||||
tests: TestTuple
|
||||
fail_fast: bool = False
|
||||
fail_env_changed: bool = False
|
||||
match_tests: FilterTuple | None = None
|
||||
ignore_tests: FilterTuple | None = None
|
||||
match_tests_dict: FilterDict | None = None
|
||||
rerun: bool = False
|
||||
forever: bool = False
|
||||
pgo: bool = False
|
||||
pgo_extended: bool = False
|
||||
output_on_failure: bool = False
|
||||
timeout: float | None = None
|
||||
verbose: int = 0
|
||||
quiet: bool = False
|
||||
hunt_refleak: HuntRefleak | None = None
|
||||
test_dir: StrPath | None = None
|
||||
use_junit: bool = False
|
||||
memory_limit: str | None = None
|
||||
gc_threshold: int | None = None
|
||||
use_resources: list[str] = dataclasses.field(default_factory=list)
|
||||
python_cmd: list[str] | None = None
|
||||
fail_fast: bool
|
||||
fail_env_changed: bool
|
||||
match_tests: FilterTuple | None
|
||||
ignore_tests: FilterTuple | None
|
||||
match_tests_dict: FilterDict | None
|
||||
rerun: bool
|
||||
forever: bool
|
||||
pgo: bool
|
||||
pgo_extended: bool
|
||||
output_on_failure: bool
|
||||
timeout: float | None
|
||||
verbose: int
|
||||
quiet: bool
|
||||
hunt_refleak: HuntRefleak | None
|
||||
test_dir: StrPath | None
|
||||
use_junit: bool
|
||||
memory_limit: str | None
|
||||
gc_threshold: int | None
|
||||
use_resources: tuple[str]
|
||||
python_cmd: tuple[str] | None
|
||||
randomize: bool
|
||||
random_seed: int | None
|
||||
# On Unix, it's a file descriptor.
|
||||
# On Windows, it's a handle.
|
||||
json_fd: int | None = None
|
||||
json_fd: int | None
|
||||
|
||||
def copy(self, **override):
|
||||
state = dataclasses.asdict(self)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import faulthandler
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
import unittest
|
||||
|
|
@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests):
|
|||
|
||||
if runtests.gc_threshold is not None:
|
||||
gc.set_threshold(runtests.gc_threshold)
|
||||
|
||||
if runtests.randomize:
|
||||
random.seed(runtests.random_seed)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
import locale
|
||||
import os.path
|
||||
import platform
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -504,7 +505,7 @@ def list_regex(line_format, tests):
|
|||
if rerun is not None:
|
||||
regex = list_regex('%s re-run test%s', [rerun.name])
|
||||
self.check_line(output, regex)
|
||||
regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode"
|
||||
regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode"
|
||||
self.check_line(output, regex)
|
||||
regex = fr"Re-running {rerun.name} in verbose mode"
|
||||
if rerun.match:
|
||||
|
|
@ -1019,13 +1020,13 @@ def test_run(self):
|
|||
forever=True)
|
||||
|
||||
@without_optimizer
|
||||
def check_leak(self, code, what, *, multiprocessing=False):
|
||||
def check_leak(self, code, what, *, run_workers=False):
|
||||
test = self.create_test('huntrleaks', code=code)
|
||||
|
||||
filename = 'reflog.txt'
|
||||
self.addCleanup(os_helper.unlink, filename)
|
||||
cmd = ['--huntrleaks', '3:3:']
|
||||
if multiprocessing:
|
||||
if run_workers:
|
||||
cmd.append('-j1')
|
||||
cmd.append(test)
|
||||
output = self.run_tests(*cmd,
|
||||
|
|
@ -1044,7 +1045,7 @@ def check_leak(self, code, what, *, multiprocessing=False):
|
|||
self.assertIn(line2, reflog)
|
||||
|
||||
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
|
||||
def check_huntrleaks(self, *, multiprocessing: bool):
|
||||
def check_huntrleaks(self, *, run_workers: bool):
|
||||
# test --huntrleaks
|
||||
code = textwrap.dedent("""
|
||||
import unittest
|
||||
|
|
@ -1055,13 +1056,13 @@ class RefLeakTest(unittest.TestCase):
|
|||
def test_leak(self):
|
||||
GLOBAL_LIST.append(object())
|
||||
""")
|
||||
self.check_leak(code, 'references', multiprocessing=multiprocessing)
|
||||
self.check_leak(code, 'references', run_workers=run_workers)
|
||||
|
||||
def test_huntrleaks(self):
|
||||
self.check_huntrleaks(multiprocessing=False)
|
||||
self.check_huntrleaks(run_workers=False)
|
||||
|
||||
def test_huntrleaks_mp(self):
|
||||
self.check_huntrleaks(multiprocessing=True)
|
||||
self.check_huntrleaks(run_workers=True)
|
||||
|
||||
@unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
|
||||
def test_huntrleaks_fd_leak(self):
|
||||
|
|
@ -1139,8 +1140,6 @@ def test_method3(self):
|
|||
def test_method4(self):
|
||||
pass
|
||||
""")
|
||||
all_methods = ['test_method1', 'test_method2',
|
||||
'test_method3', 'test_method4']
|
||||
testname = self.create_test(code=code)
|
||||
|
||||
# only run a subset
|
||||
|
|
@ -1762,7 +1761,7 @@ def test_mp_decode_error(self):
|
|||
if encoding is None:
|
||||
encoding = sys.__stdout__.encoding
|
||||
if encoding is None:
|
||||
self.skipTest(f"cannot get regrtest worker encoding")
|
||||
self.skipTest("cannot get regrtest worker encoding")
|
||||
|
||||
nonascii = b"byte:\xa0\xa9\xff\n"
|
||||
try:
|
||||
|
|
@ -1789,7 +1788,7 @@ def test_mp_decode_error(self):
|
|||
stats=0)
|
||||
|
||||
def test_doctest(self):
|
||||
code = textwrap.dedent(fr'''
|
||||
code = textwrap.dedent(r'''
|
||||
import doctest
|
||||
import sys
|
||||
from test import support
|
||||
|
|
@ -1827,6 +1826,46 @@ def load_tests(loader, tests, pattern):
|
|||
randomize=True,
|
||||
stats=TestStats(1, 1, 0))
|
||||
|
||||
def _check_random_seed(self, run_workers: bool):
|
||||
# gh-109276: When -r/--randomize is used, random.seed() is called
|
||||
# with the same random seed before running each test file.
|
||||
code = textwrap.dedent(r'''
|
||||
import random
|
||||
import unittest
|
||||
|
||||
class RandomSeedTest(unittest.TestCase):
|
||||
def test_randint(self):
|
||||
numbers = [random.randint(0, 1000) for _ in range(10)]
|
||||
print(f"Random numbers: {numbers}")
|
||||
''')
|
||||
tests = [self.create_test(name=f'test_random{i}', code=code)
|
||||
for i in range(1, 3+1)]
|
||||
|
||||
random_seed = 856_656_202
|
||||
cmd = ["--randomize", f"--randseed={random_seed}"]
|
||||
if run_workers:
|
||||
# run as many worker processes than the number of tests
|
||||
cmd.append(f'-j{len(tests)}')
|
||||
cmd.extend(tests)
|
||||
output = self.run_tests(*cmd)
|
||||
|
||||
random.seed(random_seed)
|
||||
# Make the assumption that nothing consume entropy between libregrest
|
||||
# setup_tests() which calls random.seed() and RandomSeedTest calling
|
||||
# random.randint().
|
||||
numbers = [random.randint(0, 1000) for _ in range(10)]
|
||||
expected = f"Random numbers: {numbers}"
|
||||
|
||||
regex = r'^Random numbers: .*$'
|
||||
matches = re.findall(regex, output, flags=re.MULTILINE)
|
||||
self.assertEqual(matches, [expected] * len(tests))
|
||||
|
||||
def test_random_seed(self):
|
||||
self._check_random_seed(run_workers=False)
|
||||
|
||||
def test_random_seed_workers(self):
|
||||
self._check_random_seed(run_workers=True)
|
||||
|
||||
|
||||
class TestUtils(unittest.TestCase):
|
||||
def test_format_duration(self):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
libregrtest now calls :func:`random.seed()` before running each test file
|
||||
when ``-r/--randomize`` command line option is used. Moreover, it's also
|
||||
called in worker processes. It should help to make tests more
|
||||
deterministic. Previously, it was only called once in the main process before
|
||||
running all test files and it was not called in worker processes. Patch by
|
||||
Victor Stinner.
|
||||
Loading…
Add table
Add a link
Reference in a new issue