Update libregrtest from master (GH-19517)

* bpo-36670: regrtest bug fixes (GH-16537)

* Fix TestWorkerProcess.__repr__(): start_time is only valid
  if _popen is not None.
* Fix _kill(): don't set _killed to True if _popen is None.
* _run_process(): only set _killed to False after calling
  run_test_in_subprocess().

(cherry picked from commit 2ea71a0)

* [3.8] Update libregrtest from master (GH-19516)

* bpo-37531: regrtest now catchs ProcessLookupError (GH-16827)

Fix a warning on a race condition on TestWorkerProcess.kill(): ignore
silently ProcessLookupError rather than logging an useless warning.

(cherry picked from commit a661392)

* bpo-38502: regrtest uses process groups if available (GH-16829)

test.regrtest now uses process groups in the multiprocessing mode
(-jN command line option) if process groups are available: if
os.setsid() and os.killpg() functions are available.

(cherry picked from commit ecb035c)

* bpo-37957: Allow regrtest to receive a file with test (and subtests) to ignore (GH-16989)

When building Python in some uncommon platforms there are some known tests that will fail. Right now, the test suite has the ability to ignore entire tests using the -x option and to receive a filter file using the --matchfile filter. The problem with the --matchfile option is that it receives a file with patterns to accept and when you want to ignore a couple of tests and subtests, is too cumbersome to lists ALL tests that are not the ones that you want to accept and he problem with -x is that is not easy to ignore just a subtests that fail and the whole test needs to be ignored.

For these reasons, add a new option to allow to ignore a list of test and subtests for these situations.

(cherry picked from commit e0cd8aa)

* regrtest: log timeout at startup (GH-19514)

Reduce also worker timeout.

(cherry picked from commit 4cf65a6)

Co-authored-by: Pablo Galindo <[email protected]>
(cherry picked from commit 67b8a1f)

* bpo-36842: Fix reference leak in tests by running out-of-proc (GH-13556)

(cherry picked from commit 9ddc416)

* Backport libregrtest changes from master

Co-authored-by: Steve Dower <[email protected]>

Author: vstinner

Lib/test/libregrtest/__init__.py

@@ -1,5 +1,2 @@
-# We import importlib *ASAP* in order to test #15386
-import importlib
-
 from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES
 from test.libregrtest.main import main

Lib/test/libregrtest/cmdline.py

@@ -207,10 +207,17 @@ def _create_parser():
     group.add_argument('-m', '--match', metavar='PAT',
                        dest='match_tests', action='append',
                        help='match test cases and methods with glob pattern PAT')
+    group.add_argument('-i', '--ignore', metavar='PAT',
+                       dest='ignore_tests', action='append',
+                       help='ignore test cases and methods with glob pattern PAT')
     group.add_argument('--matchfile', metavar='FILENAME',
                        dest='match_filename',
                        help='similar to --match but get patterns from a '
                             'text file, one pattern per line')
+    group.add_argument('--ignorefile', metavar='FILENAME',
+                       dest='ignore_filename',
+                       help='similar to --matchfile but it receives patterns '
+                            'from text file to ignore')
     group.add_argument('-G', '--failfast', action='store_true',
                        help='fail as soon as a test fails (only with -v or -W)')
     group.add_argument('-u', '--use', metavar='RES1,RES2,...',
@@ -315,7 +322,8 @@ def _parse_args(args, **kwargs):
          findleaks=1, use_resources=None, trace=False, coverdir='coverage',
          runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
          random_seed=None, use_mp=None, verbose3=False, forever=False,
-         header=False, failfast=False, match_tests=None, pgo=False)
+         header=False, failfast=False, match_tests=None, ignore_tests=None,
+         pgo=False)
     for k, v in kwargs.items():
         if not hasattr(ns, k):
             raise TypeError('%r is an invalid keyword argument '
@@ -391,6 +399,12 @@ def _parse_args(args, **kwargs):
         with open(ns.match_filename) as fp:
             for line in fp:
                 ns.match_tests.append(line.strip())
+    if ns.ignore_filename:
+        if ns.ignore_tests is None:
+            ns.ignore_tests = []
+        with open(ns.ignore_filename) as fp:
+            for line in fp:
+                ns.ignore_tests.append(line.strip())
     if ns.forever:
         # --forever implies --failfast
         ns.failfast = True

Lib/test/libregrtest/main.py

@@ -282,7 +282,7 @@ def _list_cases(self, suite):
 
     def list_cases(self):
         support.verbose = False
-        support.set_match_tests(self.ns.match_tests)
+        support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests)
 
         for test_name in self.selected:
             abstest = get_abs_module(self.ns, test_name)
@@ -389,7 +389,10 @@ def run_tests_sequential(self):
 
         save_modules = sys.modules.keys()
 
-        self.log("Run tests sequentially")
+        msg = "Run tests sequentially"
+        if self.ns.timeout:
+            msg += " (timeout: %s)" % format_duration(self.ns.timeout)
+        self.log(msg)
 
         previous_test = None
         for test_index, test_name in enumerate(self.tests, 1):

Lib/test/libregrtest/runtest.py

@@ -123,7 +123,7 @@ def _runtest(ns, test_name):
 
     start_time = time.perf_counter()
     try:
-        support.set_match_tests(ns.match_tests)
+        support.set_match_tests(ns.match_tests, ns.ignore_tests)
         support.junit_xml_list = xml_list = [] if ns.xmlpath else None
         if ns.failfast:
             support.failfast = True
@@ -313,9 +313,7 @@ def cleanup_test_droppings(test_name, verbose):
     # since if a test leaves a file open, it cannot be deleted by name (while
     # there's nothing we can do about that here either, we can display the
     # name of the offending test, which is a real help).
-    for name in (support.TESTFN,
-                 "db_home",
-                ):
+    for name in (support.TESTFN,):
         if not os.path.exists(name):
             continue
 

Lib/test/libregrtest/runtest_mp.py

@@ -3,6 +3,7 @@
 import json
 import os
 import queue
+import signal
 import subprocess
 import sys
 import threading
@@ -31,6 +32,8 @@
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
 
+USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
+
 
 def must_stop(result, ns):
     if result.result == INTERRUPTED:
@@ -59,12 +62,16 @@ def run_test_in_subprocess(testname, ns):
     # Running the child from the same working directory as regrtest's original
     # invocation ensures that TEMPDIR for the child is the same when
     # sysconfig.is_python_build() is true. See issue 15300.
+    kw = {}
+    if USE_PROCESS_GROUP:
+        kw['start_new_session'] = True
     return subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             universal_newlines=True,
                             close_fds=(os.name != 'nt'),
-                            cwd=support.SAVEDCWD)
+                            cwd=support.SAVEDCWD,
+                            **kw)
 
 
 def run_tests_worker(ns, test_name):
@@ -127,32 +134,46 @@ def __init__(self, worker_id, runner):
     def __repr__(self):
         info = [f'TestWorkerProcess #{self.worker_id}']
         if self.is_alive():
-            dt = time.monotonic() - self.start_time
-            info.append("running for %s" % format_duration(dt))
+            info.append("running")
         else:
             info.append('stopped')
         test = self.current_test_name
         if test:
             info.append(f'test={test}')
         popen = self._popen
-        if popen:
-            info.append(f'pid={popen.pid}')
+        if popen is not None:
+            dt = time.monotonic() - self.start_time
+            info.extend((f'pid={self._popen.pid}',
+                         f'time={format_duration(dt)}'))
         return '<%s>' % ' '.join(info)
 
     def _kill(self):
+        popen = self._popen
+        if popen is None:
+            return
+
         if self._killed:
             return
         self._killed = True
 
-        popen = self._popen
-        if popen is None:
-            return
+        if USE_PROCESS_GROUP:
+            what = f"{self} process group"
+        else:
+            what = f"{self}"
 
-        print(f"Kill {self}", file=sys.stderr, flush=True)
+        print(f"Kill {what}", file=sys.stderr, flush=True)
         try:
-            popen.kill()
+            if USE_PROCESS_GROUP:
+                os.killpg(popen.pid, signal.SIGKILL)
+            else:
+                popen.kill()
+        except ProcessLookupError:
+            # popen.kill(): the process completed, the TestWorkerProcess thread
+            # read its exit status, but Popen.send_signal() read the returncode
+            # just before Popen.wait() set returncode.
+            pass
         except OSError as exc:
-            print_warning(f"Failed to kill {self}: {exc!r}")
+            print_warning(f"Failed to kill {what}: {exc!r}")
 
     def stop(self):
         # Method called from a different thread to stop this thread
@@ -170,9 +191,10 @@ def _run_process(self, test_name):
 
         self.current_test_name = test_name
         try:
+            popen = run_test_in_subprocess(test_name, self.ns)
+
             self._killed = False
-            self._popen = run_test_in_subprocess(test_name, self.ns)
-            popen = self._popen
+            self._popen = popen
         except:
             self.current_test_name = None
             raise
@@ -330,16 +352,24 @@ def __init__(self, regrtest):
         self.output = queue.Queue()
         self.pending = MultiprocessIterator(self.regrtest.tests)
         if self.ns.timeout is not None:
-            self.worker_timeout = self.ns.timeout * 1.5
+            # Rely on faulthandler to kill a worker process. This timouet is
+            # when faulthandler fails to kill a worker process. Give a maximum
+            # of 5 minutes to faulthandler to kill the worker.
+            self.worker_timeout = min(self.ns.timeout * 1.5,
+                                      self.ns.timeout + 5 * 60)
         else:
             self.worker_timeout = None
         self.workers = None
 
     def start_workers(self):
         self.workers = [TestWorkerProcess(index, self)
                         for index in range(1, self.ns.use_mp + 1)]
-        self.log("Run tests in parallel using %s child processes"
-                 % len(self.workers))
+        msg = f"Run tests in parallel using {len(self.workers)} child processes"
+        if self.ns.timeout:
+            msg += (" (timeout: %s, worker timeout: %s)"
+                    % (format_duration(self.ns.timeout),
+                       format_duration(self.worker_timeout)))
+        self.log(msg)
         for worker in self.workers:
             worker.start()
 

Lib/test/libregrtest/setup.py

@@ -67,29 +67,34 @@ def setup_tests(ns):
     if ns.threshold is not None:
         gc.set_threshold(ns.threshold)
 
+    suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2)
+
+    support.use_resources = ns.use_resources
+
+
+def suppress_msvcrt_asserts(verbose):
     try:
         import msvcrt
     except ImportError:
-        pass
-    else:
-        msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
-                            msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
-                            msvcrt.SEM_NOGPFAULTERRORBOX|
-                            msvcrt.SEM_NOOPENFILEERRORBOX)
-        try:
-            msvcrt.CrtSetReportMode
-        except AttributeError:
-            # release build
-            pass
+        return
+
+    msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
+                        msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
+                        msvcrt.SEM_NOGPFAULTERRORBOX|
+                        msvcrt.SEM_NOOPENFILEERRORBOX)
+    try:
+        msvcrt.CrtSetReportMode
+    except AttributeError:
+        # release build
+        return
+
+    for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
+        if verbose:
+            msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
+            msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
         else:
-            for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
-                if ns.verbose and ns.verbose >= 2:
-                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
-                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
-                else:
-                    msvcrt.CrtSetReportMode(m, 0)
+            msvcrt.CrtSetReportMode(m, 0)
 
-    support.use_resources = ns.use_resources
 
 
 def replace_stdout():

Lib/test/support/__init__.py

@@ -1936,7 +1936,9 @@ def _run_suite(suite):
 
 # By default, don't filter tests
 _match_test_func = None
-_match_test_patterns = None
+
+_accept_test_patterns = None
+_ignore_test_patterns = None
 
 
 def match_test(test):
@@ -1952,18 +1954,45 @@ def _is_full_match_test(pattern):
     # as a full test identifier.
     # Example: 'test.test_os.FileTests.test_access'.
     #
-    # Reject patterns which contain fnmatch patterns: '*', '?', '[...]'
-    # or '[!...]'. For example, reject 'test_access*'.
+    # ignore patterns which contain fnmatch patterns: '*', '?', '[...]'
+    # or '[!...]'. For example, ignore 'test_access*'.
     return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern))
 
 
-def set_match_tests(patterns):
-    global _match_test_func, _match_test_patterns
+def set_match_tests(accept_patterns=None, ignore_patterns=None):
+    global _match_test_func, _accept_test_patterns, _ignore_test_patterns
 
-    if patterns == _match_test_patterns:
-        # No change: no need to recompile patterns.
-        return
 
+    if accept_patterns is None:
+        accept_patterns = ()
+    if ignore_patterns is None:
+        ignore_patterns = ()
+
+    accept_func = ignore_func = None
+
+    if accept_patterns != _accept_test_patterns:
+        accept_patterns, accept_func = _compile_match_function(accept_patterns)
+    if ignore_patterns != _ignore_test_patterns:
+        ignore_patterns, ignore_func = _compile_match_function(ignore_patterns)
+
+    # Create a copy since patterns can be mutable and so modified later
+    _accept_test_patterns = tuple(accept_patterns)
+    _ignore_test_patterns = tuple(ignore_patterns)
+
+    if accept_func is not None or ignore_func is not None:
+        def match_function(test_id):
+            accept = True
+            ignore = False
+            if accept_func:
+                accept = accept_func(test_id)
+            if ignore_func:
+                ignore = ignore_func(test_id)
+            return accept and not ignore
+
+        _match_test_func = match_function
+
+
+def _compile_match_function(patterns):
     if not patterns:
         func = None
         # set_match_tests(None) behaves as set_match_tests(())
@@ -1991,10 +2020,7 @@ def match_test_regex(test_id):
 
         func = match_test_regex
 
-    # Create a copy since patterns can be mutable and so modified later
-    _match_test_patterns = tuple(patterns)
-    _match_test_func = func
-
+    return patterns, func
 
 
 def run_unittest(*classes):