Put filelock around PEP 561 tests (fixes #12615) (#12857)

Fixes #12615.

As discussed in the linked issue, this PR would put a filesystem-based lock around the pip 
install steps of the suite of PEP561 testcases, to address race conditions.

It introduces a test-dependency on tox-dev/py-filelock. I used 3.0.0 as a lower bound, simply 
because that's what the tox version mentioned in tox.ini specifies as its lower bound. 
However from the release history it seems that Python 3.6 support was dropped in 3.4.2, 
hence the second line in the requirements.

I ended up just adding the location of the lock file to .gitignore -- I guess it would be 
possible to try and remove it after the suite has run, e.g. with atexit.register, but I didn't 
want to make this complicated.

Author: erikkemperman

.gitignore

@@ -14,6 +14,7 @@ venv/
 .mypy_cache/
 .incremental_checker_cache.json
 .cache
+test-data/packages/.pip_lock
 dmypy.json
 .dmypy.json
 

mypy/test/config.py

@@ -15,3 +15,12 @@
 # This is *within* the tempfile.TemporaryDirectory that is chroot'ed per testcase.
 # It is also hard-coded in numerous places, so don't change it.
 test_temp_dir = 'tmp'
+
+# The PEP 561 tests do a bunch of pip installs which, even though they operate
+# on distinct temporary virtual environments, run into race conditions on shared
+# file-system state. To make this work reliably in parallel mode, we'll use a
+# FileLock courtesy of the tox-dev/py-filelock package.
+# Ref. https://github.com/python/mypy/issues/12615
+# Ref. mypy/test/testpep561.py
+pip_lock = os.path.join(package_path, '.pip_lock')
+pip_timeout = 60

mypy/test/testpep561.py

@@ -1,4 +1,5 @@
 from contextlib import contextmanager
+import filelock
 import os
 import pytest
 import re
@@ -9,7 +10,7 @@
 from typing import Tuple, List, Generator
 
 import mypy.api
-from mypy.test.config import package_path
+from mypy.test.config import package_path, pip_lock, pip_timeout
 from mypy.util import try_find_python2_interpreter
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.config import test_temp_dir
@@ -77,11 +78,15 @@ def install_package(pkg: str,
         env = {'PIP_BUILD': dir}
         # Inherit environment for Windows
         env.update(os.environ)
-        proc = subprocess.run(install_cmd,
-                              cwd=working_dir,
-                              stdout=PIPE,
-                              stderr=PIPE,
-                              env=env)
+        try:
+            with filelock.FileLock(pip_lock, timeout=pip_timeout):
+                proc = subprocess.run(install_cmd,
+                                      cwd=working_dir,
+                                      stdout=PIPE,
+                                      stderr=PIPE,
+                                      env=env)
+        except filelock.Timeout as err:
+            raise Exception("Failed to acquire {}".format(pip_lock)) from err
     if proc.returncode != 0:
         raise Exception(proc.stdout.decode('utf-8') + proc.stderr.decode('utf-8'))
 

runtests.py

@@ -58,11 +58,15 @@
     'pytest-slow': ['pytest', '-q', '-k', ' or '.join(
         [SAMPLES,
          TYPESHED,
-         PEP561,
          DAEMON,
          MYPYC_EXTERNAL,
          MYPYC_COMMAND_LINE,
          ERROR_STREAM])],
+
+    # Test cases that might take minutes to run
+    'pytest-slower': ['pytest', '-q', '-k', ' or '.join(
+        [PEP561])],
+
     # Test cases to run in typeshed CI
     'typeshed-ci': ['pytest', '-q', '-k', ' or '.join([CMDLINE,
                                                        EVALUATION,

test-requirements.txt

@@ -1,6 +1,8 @@
 -r mypy-requirements.txt
 -r build-requirements.txt
 attrs>=18.0
+filelock>=3.0.0,<3.4.2; python_version<'3.7'
+filelock>=3.0.0; python_version>='3.7'
 flake8==3.9.2
 flake8-bugbear==22.3.20
 flake8-pyi>=20.5