Merge pull request #51 from python/master

Sync Fork from Upstream Repo

Author: sthagen

mypy/build.py

@@ -562,6 +562,7 @@ class BuildManager:
                        not only for debugging, but also required for correctness,
                        in particular to check consistency of the fine-grained dependency cache.
       fscache:         A file system cacher
+      ast_cache:       AST cache to speed up mypy daemon
     """
 
     def __init__(self, data_dir: str,
@@ -645,6 +646,14 @@ def __init__(self, data_dir: str,
         self.processed_targets = []  # type: List[str]
         # Missing stub packages encountered.
         self.missing_stub_packages = set()  # type: Set[str]
+        # Cache for mypy ASTs that have completed semantic analysis
+        # pass 1. When multiple files are added to the build in a
+        # single daemon increment, only one of the files gets added
+        # per step and the others are discarded. This gets repeated
+        # until all the files have been added. This means that a
+        # new file can be processed O(n**2) times. This cache
+        # avoids most of this redundant work.
+        self.ast_cache = {}  # type: Dict[str, Tuple[MypyFile, List[ErrorInfo]]]
 
     def dump_stats(self) -> None:
         if self.options.dump_build_stats:
@@ -1994,8 +2003,14 @@ def parse_file(self) -> None:
             return
 
         manager = self.manager
+
+        # Can we reuse a previously parsed AST? This avoids redundant work in daemon.
+        cached = self.id in manager.ast_cache
         modules = manager.modules
-        manager.log("Parsing %s (%s)" % (self.xpath, self.id))
+        if not cached:
+            manager.log("Parsing %s (%s)" % (self.xpath, self.id))
+        else:
+            manager.log("Using cached AST for %s (%s)" % (self.xpath, self.id))
 
         with self.wrap_context():
             source = self.source
@@ -2026,21 +2041,36 @@ def parse_file(self) -> None:
                 self.source_hash = compute_hash(source)
 
             self.parse_inline_configuration(source)
-            self.tree = manager.parse_file(self.id, self.xpath, source,
-                                           self.ignore_all or self.options.ignore_errors,
-                                           self.options)
+            if not cached:
+                self.tree = manager.parse_file(self.id, self.xpath, source,
+                                               self.ignore_all or self.options.ignore_errors,
+                                               self.options)
 
-        modules[self.id] = self.tree
+            else:
+                # Reuse a cached AST
+                self.tree = manager.ast_cache[self.id][0]
+                manager.errors.set_file_ignored_lines(
+                    self.xpath,
+                    self.tree.ignored_lines,
+                    self.ignore_all or self.options.ignore_errors)
+
+        if not cached:
+            # Make a copy of any errors produced during parse time so that
+            # fine-grained mode can repeat them when the module is
+            # reprocessed.
+            self.early_errors = list(manager.errors.error_info_map.get(self.xpath, []))
+        else:
+            self.early_errors = manager.ast_cache[self.id][1]
 
-        # Make a copy of any errors produced during parse time so that
-        # fine-grained mode can repeat them when the module is
-        # reprocessed.
-        self.early_errors = list(manager.errors.error_info_map.get(self.xpath, []))
+        modules[self.id] = self.tree
 
-        self.semantic_analysis_pass1()
+        if not cached:
+            self.semantic_analysis_pass1()
 
         self.check_blockers()
 
+        manager.ast_cache[self.id] = (self.tree, self.early_errors)
+
     def parse_inline_configuration(self, source: str) -> None:
         """Check for inline mypy: options directive and parse them."""
         flags = get_mypy_comments(source)

mypy/dmypy_server.py

@@ -373,7 +373,7 @@ def cmd_recheck(self,
             assert remove is None and update is None
             messages = self.fine_grained_increment_follow_imports(sources)
         res = self.increment_output(messages, sources, is_tty, terminal_width)
-        self.fscache.flush()
+        self.flush_caches()
         self.update_stats(res)
         return res
 
@@ -392,10 +392,15 @@ def check(self, sources: List[BuildSource],
             else:
                 messages = self.fine_grained_increment_follow_imports(sources)
             res = self.increment_output(messages, sources, is_tty, terminal_width)
-        self.fscache.flush()
+        self.flush_caches()
         self.update_stats(res)
         return res
 
+    def flush_caches(self) -> None:
+        self.fscache.flush()
+        if self.fine_grained_manager:
+            self.fine_grained_manager.flush_cache()
+
     def update_stats(self, res: Dict[str, Any]) -> None:
         if self.fine_grained_manager:
             manager = self.fine_grained_manager.manager
@@ -852,7 +857,7 @@ def cmd_suggest(self,
                 out += "\n"
             return {'out': out, 'err': "", 'status': 0}
         finally:
-            self.fscache.flush()
+            self.flush_caches()
 
     def cmd_hang(self) -> Dict[str, object]:
         """Hang for 100 seconds, as a debug hack."""

mypy/server/update.py

@@ -288,6 +288,14 @@ def trigger(self, target: str) -> List[str]:
         self.previous_messages = self.manager.errors.new_messages()[:]
         return self.update(changed_modules, [])
 
+    def flush_cache(self) -> None:
+        """Flush AST cache.
+
+        This needs to be called after each increment, or file changes won't
+        be detected reliably.
+        """
+        self.manager.ast_cache.clear()
+
     def update_one(self,
                    changed_modules: List[Tuple[str, str]],
                    initial_set: Set[str],

mypy/suggestions.py

@@ -640,6 +640,7 @@ def reload(self, state: State, check_errors: bool = False) -> List[str]:
         If check_errors is true, raise an exception if there are errors.
         """
         assert state.path is not None
+        self.fgmanager.flush_cache()
         return self.fgmanager.update([(state.id, state.path)], [])
 
     def ensure_loaded(self, state: State, force: bool = False) -> MypyFile:

mypy/test/testmerge.py

@@ -124,6 +124,7 @@ def build(self, source: str, testcase: DataDrivenTestCase) -> Optional[BuildResu
     def build_increment(self, manager: FineGrainedBuildManager,
                         module_id: str, path: str) -> Tuple[MypyFile,
                                                             Dict[Expression, Type]]:
+        manager.flush_cache()
         manager.update([(module_id, path)], [])
         module = manager.manager.modules[module_id]
         type_map = manager.graph[module_id].type_map()

runtests.py

@@ -1,18 +1,7 @@
 #!/usr/bin/env python3
 import subprocess
 from subprocess import Popen
-from os import system
-from sys import argv, exit, platform, executable, version_info
-
-
-# Use the Python provided to execute the script, or fall back to a sane default
-if version_info >= (3, 5, 0):
-    python_name = executable
-else:
-    if platform == 'win32':
-        python_name = 'py -3'
-    else:
-        python_name = 'python3'
+from sys import argv, exit, executable
 
 # Slow test suites
 CMDLINE = 'PythonCmdline'
@@ -55,30 +44,33 @@
 # time to run.
 cmds = {
     # Self type check
-    'self': python_name + ' -m mypy --config-file mypy_self_check.ini -p mypy',
+    'self': [executable, '-m', 'mypy', '--config-file', 'mypy_self_check.ini', '-p', 'mypy'],
     # Lint
-    'lint': 'flake8 -j0',
+    'lint': ['flake8', '-j0'],
     # Fast test cases only (this is the bulk of the test suite)
-    'pytest-fast': 'pytest -q -k "not (%s)"' % ' or '.join(ALL_NON_FAST),
+    'pytest-fast': ['pytest', '-q', '-k', 'not (%s)' % ' or '.join(ALL_NON_FAST)],
     # Test cases that invoke mypy (with small inputs)
-    'pytest-cmdline': 'pytest -q -k "%s"' % ' or '.join([CMDLINE,
-                                                         EVALUATION,
-                                                         STUBGEN_CMD,
-                                                         STUBGEN_PY]),
+    'pytest-cmdline': ['pytest', '-q', '-k', ' or '.join([CMDLINE,
+                                                          EVALUATION,
+                                                          STUBGEN_CMD,
+                                                          STUBGEN_PY])],
     # Test cases that may take seconds to run each
-    'pytest-slow': 'pytest -q -k "%s"' % ' or '.join(
+    'pytest-slow': ['pytest', '-q', '-k', ' or '.join(
         [SAMPLES,
          TYPESHED,
          PEP561,
          DAEMON,
          MYPYC_EXTERNAL,
          MYPYC_COMMAND_LINE,
-         ERROR_STREAM]),
+         ERROR_STREAM])],
     # Test cases to run in typeshed CI
-    'typeshed-ci': 'pytest -q -k "%s"' % ' or '.join([CMDLINE, EVALUATION, SAMPLES, TYPESHED]),
+    'typeshed-ci': ['pytest', '-q', '-k', ' or '.join([CMDLINE,
+                                                       EVALUATION,
+                                                       SAMPLES,
+                                                       TYPESHED])],
     # Mypyc tests that aren't run by default, since they are slow and rarely
     # fail for commits that don't touch mypyc
-    'mypyc-extra': 'pytest -q -k "%s"' % ' or '.join(MYPYC_OPT_IN),
+    'mypyc-extra': ['pytest', '-q', '-k', ' or '.join(MYPYC_OPT_IN)],
 }
 
 # Stop run immediately if these commands fail
@@ -93,10 +85,10 @@ def run_cmd(name: str) -> int:
     status = 0
     cmd = cmds[name]
     print('run %s: %s' % (name, cmd))
-    res = (system(cmd) & 0x7F00) >> 8
-    if res:
+    proc = subprocess.run(cmd, stderr=subprocess.STDOUT)
+    if proc.returncode:
         print('\nFAILED: %s' % name)
-        status = res
+        status = proc.returncode
         if name in FAST_FAIL:
             exit(status)
     return status
@@ -105,7 +97,6 @@ def run_cmd(name: str) -> int:
 def start_background_cmd(name: str) -> Popen:
     cmd = cmds[name]
     proc = subprocess.Popen(cmd,
-                            shell=True,
                             stderr=subprocess.STDOUT,
                             stdout=subprocess.PIPE)
     return proc