Modify incremental tests to allow initial error

mypy/test/data.py

@@ -46,9 +46,11 @@ def parse_test_cases(
             i += 1
 
             files = []  # type: List[Tuple[str, str]] # path and contents
+            tcout = []  # type: List[str]  # Regular output errors
+            tcout2 = []  # type: List[str]  # Output errors for incremental, second run
             stale_modules = None  # type: Optional[Set[str]]  # module names
             rechecked_modules = None  # type: Optional[Set[str]]  # module names
-            while i < len(p) and p[i].id not in ['out', 'case']:
+            while i < len(p) and p[i].id != 'case':
                 if p[i].id == 'file':
                     # Record an extra file needed for the test case.
                     files.append((os.path.join(base_path, p[i].arg),
@@ -74,6 +76,16 @@ def parse_test_cases(
                         rechecked_modules = set()
                     else:
                         rechecked_modules = {item.strip() for item in p[i].arg.split(',')}
+                elif p[i].id == 'out' or p[i].id == 'out1':
+                    tcout = p[i].data
+                    if native_sep and os.path.sep == '\\':
+                        tcout = [fix_win_path(line) for line in tcout]
+                    ok = True
+                elif p[i].id == 'out2':
+                    tcout2 = p[i].data
+                    if native_sep and os.path.sep == '\\':
+                        tcout2 = [fix_win_path(line) for line in tcout2]
+                    ok = True
                 else:
                     raise ValueError(
                         'Invalid section header {} in {} at line {}'.format(
@@ -88,21 +100,14 @@ def parse_test_cases(
                 raise ValueError(
                     'Stale modules must be a subset of rechecked modules ({})'.format(path))
 
-            tcout = []  # type: List[str]
-            if i < len(p) and p[i].id == 'out':
-                tcout = p[i].data
-                if native_sep and os.path.sep == '\\':
-                    tcout = [fix_win_path(line) for line in tcout]
-                ok = True
-                i += 1
-            elif optional_out:
+            if optional_out:
                 ok = True
 
             if ok:
                 input = expand_includes(p[i0].data, include_path)
                 expand_errors(input, tcout, 'main')
                 lastline = p[i].line if i < len(p) else p[i - 1].line + 9999
-                tc = DataDrivenTestCase(p[i0].arg, input, tcout, path,
+                tc = DataDrivenTestCase(p[i0].arg, input, tcout, tcout2, path,
                                         p[i0].line, lastline, perform,
                                         files, stale_modules, rechecked_modules)
                 out.append(tc)
@@ -129,11 +134,12 @@ class DataDrivenTestCase(TestCase):
 
     clean_up = None  # type: List[Tuple[bool, str]]
 
-    def __init__(self, name, input, output, file, line, lastline,
+    def __init__(self, name, input, output, output2, file, line, lastline,
                  perform, files, expected_stale_modules, expected_rechecked_modules):
         super().__init__(name)
         self.input = input
         self.output = output
+        self.output2 = output2
         self.lastline = lastline
         self.file = file
         self.line = line

mypy/test/testcheck.py

@@ -114,12 +114,10 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
         options.use_builtins_fixtures = True
         set_show_tb(True)  # Show traceback on crash.
 
-        output = testcase.output
         if incremental:
             options.incremental = True
             if incremental == 1:
                 # In run 1, copy program text to program file.
-                output = []
                 for module_name, program_path, program_text in module_data:
                     if module_name == '__main__':
                         with open(program_path, 'w') as f:
@@ -155,16 +153,25 @@ def run_case_once(self, testcase: DataDrivenTestCase, incremental=0) -> None:
             a = e.messages
         a = normalize_error_messages(a)
 
+        # Make sure error messages match
+        if incremental == 0:
+            msg = 'Invalid type checker output ({}, line {})'
+            output = testcase.output
+        elif incremental == 1:
+            msg = 'Invalid type checker output in incremental, run 1 ({}, line {})'
+            output = testcase.output
+        elif incremental == 2:
+            msg = 'Invalid type checker output in incremental, run 2 ({}, line {})'
+            output = testcase.output2
+        else:
+            raise AssertionError()
+
         if output != a and self.update_data:
             update_testcase_output(testcase, a)
-
-        assert_string_arrays_equal(
-            output, a,
-            'Invalid type checker output ({}, line {})'.format(
-                testcase.file, testcase.line))
+        assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line))
 
         if incremental and res:
-            if not options.silent_imports:
+            if not options.silent_imports and testcase.output is None:
                 self.verify_cache(module_data, a, res.manager)
             if incremental == 2:
                 self.check_module_equivalence(