stubtest: speed up tests by a lot (#9621)

I probably should have done this earlier...

Author: hauntsaninja

mypy/stubtest.py

@@ -1022,7 +1022,7 @@ def strip_comments(s: str) -> str:
                 yield entry
 
 
-def test_stubs(args: argparse.Namespace) -> int:
+def test_stubs(args: argparse.Namespace, use_builtins_fixtures: bool = False) -> int:
     """This is stubtest! It's time to test the stubs!"""
     # Load the allowlist. This is a series of strings corresponding to Error.object_desc
     # Values in the dict will store whether we used the allowlist entry or not.
@@ -1049,6 +1049,7 @@ def test_stubs(args: argparse.Namespace) -> int:
     options.incremental = False
     options.custom_typeshed_dir = args.custom_typeshed_dir
     options.config_file = args.mypy_config_file
+    options.use_builtins_fixtures = use_builtins_fixtures
 
     if options.config_file:
         def set_strict_flags() -> None:  # not needed yet

mypy/test/teststubtest.py

@@ -27,11 +27,42 @@ def use_tmp_dir() -> Iterator[None]:
 
 TEST_MODULE_NAME = "test_module"
 
+stubtest_builtins_stub = """
+from typing import Generic, Mapping, Sequence, TypeVar, overload
+
+T = TypeVar('T')
+T_co = TypeVar('T_co', covariant=True)
+KT = TypeVar('KT')
+VT = TypeVar('VT')
+
+class object:
+    def __init__(self) -> None: pass
+class type: ...
+
+class tuple(Sequence[T_co], Generic[T_co]): ...
+class dict(Mapping[KT, VT]): ...
+
+class function: pass
+class ellipsis: pass
+
+class int: ...
+class float: ...
+class bool(int): ...
+class str: ...
+class bytes: ...
+
+def property(f: T) -> T: ...
+def classmethod(f: T) -> T: ...
+def staticmethod(f: T) -> T: ...
+"""
+
 
 def run_stubtest(
     stub: str, runtime: str, options: List[str], config_file: Optional[str] = None,
 ) -> str:
     with use_tmp_dir():
+        with open("builtins.pyi", "w") as f:
+            f.write(stubtest_builtins_stub)
         with open("{}.pyi".format(TEST_MODULE_NAME), "w") as f:
             f.write(stub)
         with open("{}.py".format(TEST_MODULE_NAME), "w") as f:
@@ -47,7 +78,10 @@ def run_stubtest(
 
         output = io.StringIO()
         with contextlib.redirect_stdout(output):
-            test_stubs(parse_options([TEST_MODULE_NAME] + options))
+            test_stubs(
+                parse_options([TEST_MODULE_NAME] + options),
+                use_builtins_fixtures=True
+            )
 
         return output.getvalue()
 
@@ -60,10 +94,10 @@ def __init__(self, stub: str, runtime: str, error: Optional[str]):
 
 
 def collect_cases(fn: Callable[..., Iterator[Case]]) -> Callable[..., None]:
-    """Repeatedly invoking run_stubtest is slow, so use this decorator to combine cases.
+    """run_stubtest used to be slow, so we used this decorator to combine cases.
 
-    We could also manually combine cases, but this allows us to keep the contrasting stub and
-    runtime definitions next to each other.
+    If you're reading this and bored, feel free to refactor this and make it more like
+    other mypy tests.
 
     """
 
@@ -775,12 +809,6 @@ def f(a: int, b: int, *, c: int, d: int = 0, **kwargs: Any) -> None:
             == "def (a, b, *, c, d = ..., **kwargs)"
         )
 
-
-class StubtestIntegration(unittest.TestCase):
-    def test_typeshed(self) -> None:
-        # check we don't crash while checking typeshed
-        test_stubs(parse_options(["--check-typeshed"]))
-
     def test_config_file(self) -> None:
         runtime = "temp = 5\n"
         stub = "from decimal import Decimal\ntemp: Decimal\n"
@@ -795,3 +823,9 @@ def test_config_file(self) -> None:
         )
         output = run_stubtest(stub=stub, runtime=runtime, options=[], config_file=config_file)
         assert output == ""
+
+
+class StubtestIntegration(unittest.TestCase):
+    def test_typeshed(self) -> None:
+        # check we don't crash while checking typeshed
+        test_stubs(parse_options(["--check-typeshed"]))

runtests.py

@@ -28,8 +28,6 @@
 MYPYC_EXTERNAL = 'TestExternal'
 MYPYC_COMMAND_LINE = 'TestCommandLine'
 ERROR_STREAM = 'ErrorStreamSuite'
-STUBTEST = 'StubtestUnit'
-STUBTEST_MISC = 'StubtestMiscUnit'
 STUBTEST_INTEGRATION = 'StubtestIntegration'
 
 
@@ -47,18 +45,15 @@
     MYPYC_EXTERNAL,
     MYPYC_COMMAND_LINE,
     ERROR_STREAM,
-    STUBTEST,
-    STUBTEST_MISC,
     STUBTEST_INTEGRATION,
 ]
 
 
 # These must be enabled by explicitly including 'mypyc-extra' on the command line.
-MYPYC_OPT_IN = [MYPYC_RUN,
-                MYPYC_RUN_MULTI]
+MYPYC_OPT_IN = [MYPYC_RUN, MYPYC_RUN_MULTI]
 
 # These must be enabled by explicitly including 'stubtest' on the command line.
-STUBTEST_OPT_IN = [STUBTEST, STUBTEST_MISC, STUBTEST_INTEGRATION]
+STUBTEST_OPT_IN = [STUBTEST_INTEGRATION]
 
 # We split the pytest run into three parts to improve test
 # parallelization. Each run should have tests that each take a roughly similar