Modify incremental tests to allow initial error (#2069)

This pull request modifies the testing framework to allow us to specify the expected output during both the first and second runs of incremental mode, instead of mandating that the first run typechecks.

This is useful since it lets us write tests that check whether or not incremental recovers correctly in the presence of errors.

Author: Michael0x2a

mypy/test/data.py

@@ -46,9 +46,11 @@ def parse_test_cases(
             i += 1
 
             files = []  # type: List[Tuple[str, str]] # path and contents
+            tcout = []  # type: List[str]  # Regular output errors
+            tcout2 = []  # type: List[str]  # Output errors for incremental, second run
             stale_modules = None  # type: Optional[Set[str]]  # module names
             rechecked_modules = None  # type: Optional[Set[str]]  # module names
-            while i < len(p) and p[i].id not in ['out', 'case']:
+            while i < len(p) and p[i].id != 'case':
                 if p[i].id == 'file':
                     # Record an extra file needed for the test case.
                     files.append((os.path.join(base_path, p[i].arg),
@@ -74,6 +76,16 @@ def parse_test_cases(
                         rechecked_modules = set()
                     else:
                         rechecked_modules = {item.strip() for item in p[i].arg.split(',')}
+                elif p[i].id == 'out' or p[i].id == 'out1':
+                    tcout = p[i].data
+                    if native_sep and os.path.sep == '\\':
+                        tcout = [fix_win_path(line) for line in tcout]
+                    ok = True
+                elif p[i].id == 'out2':
+                    tcout2 = p[i].data
+                    if native_sep and os.path.sep == '\\':
+                        tcout2 = [fix_win_path(line) for line in tcout2]
+                    ok = True
                 else:
                     raise ValueError(
                         'Invalid section header {} in {} at line {}'.format(
@@ -88,21 +100,14 @@ def parse_test_cases(
                 raise ValueError(
                     'Stale modules must be a subset of rechecked modules ({})'.format(path))
 
-            tcout = []  # type: List[str]
-            if i < len(p) and p[i].id == 'out':
-                tcout = p[i].data
-                if native_sep and os.path.sep == '\\':
-                    tcout = [fix_win_path(line) for line in tcout]
-                ok = True
-                i += 1
-            elif optional_out:
+            if optional_out:
                 ok = True
 
             if ok:
                 input = expand_includes(p[i0].data, include_path)
                 expand_errors(input, tcout, 'main')
                 lastline = p[i].line if i < len(p) else p[i - 1].line + 9999
-                tc = DataDrivenTestCase(p[i0].arg, input, tcout, path,
+                tc = DataDrivenTestCase(p[i0].arg, input, tcout, tcout2, path,
                                         p[i0].line, lastline, perform,
                                         files, stale_modules, rechecked_modules)
                 out.append(tc)
@@ -129,11 +134,12 @@ class DataDrivenTestCase(TestCase):
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
-    def __init__(self, name, input, output, file, line, lastline,
+    def __init__(self, name, input, output, output2, file, line, lastline,
                  perform, files, expected_stale_modules, expected_rechecked_modules):
         super().__init__(name)
         self.input = input
         self.output = output
+        self.output2 = output2
         self.lastline = lastline
         self.file = file
         self.line = line

mypy/test/testcheck.py

@@ -114,12 +114,10 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
         options.use_builtins_fixtures = True
         set_show_tb(True)  # Show traceback on crash.
 
-        output = testcase.output
         if incremental:
             options.incremental = True
             if incremental == 1:
                 # In run 1, copy program text to program file.
-                output = []
                 for module_name, program_path, program_text in module_data:
                     if module_name == '__main__':
                         with open(program_path, 'w') as f:
@@ -155,16 +153,25 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
             a = e.messages
         a = normalize_error_messages(a)
 
+        # Make sure error messages match
+        if incremental == 0:
+            msg = 'Invalid type checker output ({}, line {})'
+            output = testcase.output
+        elif incremental == 1:
+            msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
+            output = testcase.output
+        elif incremental == 2:
+            msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
+            output = testcase.output2
+        else:
+            raise AssertionError()
+
         if output != a and self.update_data:
             update_testcase_output(testcase, a)
-
-        assert_string_arrays_equal(
-            output, a,
-            'Invalid type checker output ({}, line {})'.format(
-                testcase.file, testcase.line))
+        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
 
         if incremental and res:
-            if not options.silent_imports:
+            if not options.silent_imports and testcase.output is None:
                 self.verify_cache(module_data, a, res.manager)
             if incremental == 2:
                 self.check_module_equivalence(