Merge remote-tracking branch 'upstream/master' into remove-myunit

Author: elazarg

README.md

@@ -14,7 +14,7 @@ sure you've found a bug please search our issue trackers for a
 duplicate before filing a new issue:
 
 - [mypy tracker](https://github.com/python/mypy/issues)
-  for mypy isues
+  for mypy issues
 - [typeshed tracker](https://github.com/python/typeshed/issues)
   for issues with specific modules
 - [typing tracker](https://github.com/python/typing/issues)

mypy/build.py

@@ -52,6 +52,7 @@
 from mypy.version import __version__
 from mypy.plugin import Plugin, DefaultPlugin, ChainedPlugin
 from mypy.defaults import PYTHON3_VERSION_MIN
+from mypy.server.deps import get_dependencies
 
 
 PYTHON_EXTENSIONS = ['.pyi', '.py']
@@ -1183,6 +1184,7 @@ def compute_hash(text: str) -> str:
 
 
 def write_cache(id: str, path: str, tree: MypyFile,
+                serialized_fine_grained_deps: Dict[str, List[str]],
                 dependencies: List[str], suppressed: List[str],
                 child_modules: List[str], dep_prios: List[int],
                 old_interface_hash: str, source_hash: str,
@@ -1221,7 +1223,9 @@ def write_cache(id: str, path: str, tree: MypyFile,
     assert os.path.dirname(meta_json) == parent
 
     # Serialize data and analyze interface
-    data = tree.serialize()
+    data = {'tree': tree.serialize(),
+            'fine_grained_deps': serialized_fine_grained_deps,
+            }
     if manager.options.debug_cache:
         data_str = json.dumps(data, indent=2, sort_keys=True)
     else:
@@ -1523,6 +1527,8 @@ class State:
     # Whether the module has an error or any of its dependencies have one.
     transitive_error = False
 
+    fine_grained_deps = None  # type: Dict[str, Set[str]]
+
     # Type checker used for checking this file.  Use type_checker() for
     # access and to construct this on demand.
     _type_checker = None  # type: Optional[TypeChecker]
@@ -1551,6 +1557,7 @@ def __init__(self,
         self.id = id or '__main__'
         self.options = manager.options.clone_for_module(self.id)
         self._type_checker = None
+        self.fine_grained_deps = {}
         if not path and source is None:
             assert id is not None
             file_id = id
@@ -1734,7 +1741,9 @@ def load_tree(self) -> None:
         with open(self.meta.data_json) as f:
             data = json.load(f)
         # TODO: Assert data file wasn't changed.
-        self.tree = MypyFile.deserialize(data)
+        self.tree = MypyFile.deserialize(data['tree'])
+        self.fine_grained_deps = {k: set(v) for k, v in data['fine_grained_deps'].items()}
+
         self.manager.modules[self.id] = self.tree
         self.manager.add_stats(fresh_trees=1)
 
@@ -1977,6 +1986,19 @@ def _patch_indirect_dependencies(self,
             elif dep not in self.suppressed and dep in self.manager.missing_modules:
                 self.suppressed.append(dep)
 
+    def compute_fine_grained_deps(self) -> None:
+        assert self.tree is not None
+        if '/typeshed/' in self.xpath or self.xpath.startswith('typeshed/'):
+            # We don't track changes to typeshed -- the assumption is that they are only changed
+            # as part of mypy updates, which will invalidate everything anyway.
+            #
+            # TODO: Not a reliable test, as we could have a package named typeshed.
+            # TODO: Consider relaxing this -- maybe allow some typeshed changes to be tracked.
+            return
+        self.fine_grained_deps = get_dependencies(target=self.tree,
+                                                  type_map=self.type_map(),
+                                                  python_version=self.options.python_version)
+
     def valid_references(self) -> Set[str]:
         assert self.ancestors is not None
         valid_refs = set(self.dependencies + self.suppressed + self.ancestors)
@@ -2003,6 +2025,7 @@ def write_cache(self) -> None:
         dep_prios = self.dependency_priorities()
         new_interface_hash, self.meta = write_cache(
             self.id, self.path, self.tree,
+            {k: list(v) for k, v in self.fine_grained_deps.items()},
             list(self.dependencies), list(self.suppressed), list(self.child_modules),
             dep_prios, self.interface_hash, self.source_hash, self.ignore_all,
             self.manager)
@@ -2534,6 +2557,8 @@ def process_stale_scc(graph: Graph, scc: List[str], manager: BuildManager) -> No
             graph[id].transitive_error = True
     for id in stale:
         graph[id].finish_passes()
+        if manager.options.cache_fine_grained:
+            graph[id].compute_fine_grained_deps()
         graph[id].generate_unused_ignore_notes()
         manager.flush_errors(manager.errors.file_messages(graph[id].xpath), False)
         graph[id].write_cache()

mypy/dmypy_server.py

@@ -23,6 +23,8 @@
 import mypy.server.update
 from mypy.dmypy_util import STATUS_FILE, receive
 from mypy.gclogger import GcLogger
+from mypy.fscache import FileSystemCache
+from mypy.fswatcher import FileSystemWatcher
 
 
 def daemonize(func: Callable[[], None], log_file: Optional[str] = None) -> int:
@@ -84,13 +86,11 @@ class Server:
     def __init__(self, flags: List[str]) -> None:
         """Initialize the server with the desired mypy flags."""
         self.saved_cache = {}  # type: mypy.build.SavedCache
-        if '--experimental' in flags:
-            self.fine_grained = True
-            self.fine_grained_initialized = False
-            flags.remove('--experimental')
-        else:
-            self.fine_grained = False
-        sources, options = mypy.main.process_options(['-i'] + flags, False)
+        self.fine_grained_initialized = False
+        sources, options = mypy.main.process_options(['-i'] + flags,
+                                                     require_targets=False,
+                                                     server_options=True)
+        self.fine_grained = options.fine_grained_incremental
         if sources:
             sys.exit("dmypy: start/restart does not accept sources")
         if options.report_dirs:
@@ -243,14 +243,11 @@ def check_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str,
             return self.fine_grained_increment(sources)
 
     def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
-        self.file_modified = {}  # type: Dict[str, float]
-        for source in sources:
-            assert source.path
-            try:
-                self.file_modified[source.path] = os.stat(source.path).st_mtime
-            except FileNotFoundError:
-                # Don't crash if passed a non-existent file.
-                pass
+        self.fscache = FileSystemCache(self.options.python_version)
+        self.fswatcher = FileSystemWatcher(self.fscache)
+        self.update_sources(sources)
+        # Stores the initial state of sources as a side effect.
+        self.fswatcher.find_changed()
         try:
             # TODO: alt_lib_path
             result = mypy.build.build(sources=sources,
@@ -270,9 +267,11 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict
         self.previous_messages = messages[:]
         self.fine_grained_initialized = True
         self.previous_sources = sources
+        self.fscache.flush()
         return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
     def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]:
+        self.update_sources(sources)
         changed = self.find_changed(sources)
         if not changed:
             # Nothing changed -- just produce the same result as before.
@@ -282,36 +281,26 @@ def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[
         status = 1 if messages else 0
         self.previous_messages = messages[:]
         self.previous_sources = sources
+        self.fscache.flush()
         return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
 
+    def update_sources(self, sources: List[mypy.build.BuildSource]) -> None:
+        paths = [source.path for source in sources if source.path is not None]
+        self.fswatcher.add_watched_paths(paths)
+
     def find_changed(self, sources: List[mypy.build.BuildSource]) -> List[Tuple[str, str]]:
-        changed = []
-        for source in sources:
-            path = source.path
-            assert path
-            try:
-                mtime = os.stat(path).st_mtime
-            except FileNotFoundError:
-                # A non-existent file was included on the command line.
-                #
-                # TODO: Generate error if file is missing (if not ignoring missing imports)
-                if path in self.file_modified:
-                    changed.append((source.module, path))
-            else:
-                if path not in self.file_modified or self.file_modified[path] != mtime:
-                    self.file_modified[path] = mtime
-                    changed.append((source.module, path))
+        changed_paths = self.fswatcher.find_changed()
+        changed = [(source.module, source.path)
+                   for source in sources
+                   if source.path in changed_paths]
         modules = {source.module for source in sources}
         omitted = [source for source in self.previous_sources if source.module not in modules]
         for source in omitted:
             path = source.path
             assert path
-            # Note that a file could be removed from the list of root sources but still continue
-            # to exist on the file system.
-            if not os.path.isfile(path):
+            # Note that a file could be removed from the list of root sources but have no changes.
+            if path in changed_paths:
                 changed.append((source.module, path))
-                if source.path in self.file_modified:
-                    del self.file_modified[source.path]
         return changed
 
     def cmd_hang(self) -> Dict[str, object]:

mypy/fastparse.py

@@ -960,8 +960,12 @@ def visit_Name(self, n: ast3.Name) -> NameExpr:
 
     # List(expr* elts, expr_context ctx)
     @with_line
-    def visit_List(self, n: ast3.List) -> ListExpr:
-        return ListExpr([self.visit(e) for e in n.elts])
+    def visit_List(self, n: ast3.List) -> Union[ListExpr, TupleExpr]:
+        expr_list = [self.visit(e) for e in n.elts]  # type: List[Expression]
+        if isinstance(n.ctx, ast3.Store):
+            # [x, y] = z and (x, y) = z means exactly the same thing
+            return TupleExpr(expr_list)
+        return ListExpr(expr_list)
 
     # Tuple(expr* elts, expr_context ctx)
     @with_line
@@ -1155,6 +1159,7 @@ def visit_Ellipsis(self, n: ast3.Ellipsis) -> Type:
 
     # List(expr* elts, expr_context ctx)
     def visit_List(self, n: ast3.List) -> Type:
+        assert isinstance(n.ctx, ast3.Load)
         return self.translate_argument_list(n.elts)
 
 

mypy/fastparse2.py

@@ -912,8 +912,12 @@ def visit_Name(self, n: ast27.Name) -> NameExpr:
 
     # List(expr* elts, expr_context ctx)
     @with_line
-    def visit_List(self, n: ast27.List) -> ListExpr:
-        return ListExpr([self.visit(e) for e in n.elts])
+    def visit_List(self, n: ast27.List) -> Union[ListExpr, TupleExpr]:
+        expr_list = [self.visit(e) for e in n.elts]  # type: List[Expression]
+        if isinstance(n.ctx, ast27.Store):
+            # [x, y] = z and (x, y) = z means exactly the same thing
+            return TupleExpr(expr_list)
+        return ListExpr(expr_list)
 
     # Tuple(expr* elts, expr_context ctx)
     @with_line

mypy/fscache.py

@@ -0,0 +1,117 @@
+"""Interface for accessing the file system with automatic caching.
+
+The idea is to cache the results of any file system state reads during
+a single transaction. This has two main benefits:
+
+* This avoids redundant syscalls, as we won't perform the same OS
+  operations multiple times.
+
+* This makes it easier to reason about concurrent FS updates, as different
+  operations targeting the same paths can't report different state during
+  a transaction.
+
+Note that this only deals with reading state, not writing.
+
+Properties maintained by the API:
+
+* The contents of the file are always from the same or later time compared
+  to the reported mtime of the file, even if mtime is queried after reading
+  a file.
+
+* Repeating an operation produces the same result as the first one during
+  a transaction.
+
+* Call flush() to start a new transaction (flush the caches).
+
+The API is a bit limited. It's easy to add new cached operations, however.
+You should perform all file system reads through the API to actually take
+advantage of the benefits.
+"""
+
+import os
+import stat
+from typing import Tuple, Dict, List
+
+from mypy.build import read_with_python_encoding
+from mypy.errors import DecodeError
+
+
+class FileSystemCache:
+    def __init__(self, pyversion: Tuple[int, int]) -> None:
+        self.pyversion = pyversion
+        self.flush()
+
+    def flush(self) -> None:
+        """Start another transaction and empty all caches."""
+        self.stat_cache = {}  # type: Dict[str, os.stat_result]
+        self.stat_error_cache = {}  # type: Dict[str, Exception]
+        self.read_cache = {}  # type: Dict[str, str]
+        self.read_error_cache = {}  # type: Dict[str, Exception]
+        self.hash_cache = {}  # type: Dict[str, str]
+        self.listdir_cache = {}  # type: Dict[str, List[str]]
+        self.listdir_error_cache = {}  # type: Dict[str, Exception]
+
+    def read_with_python_encoding(self, path: str) -> str:
+        if path in self.read_cache:
+            return self.read_cache[path]
+        if path in self.read_error_cache:
+            raise self.read_error_cache[path]
+
+        # Need to stat first so that the contents of file are from no
+        # earlier instant than the mtime reported by self.stat().
+        self.stat(path)
+
+        try:
+            data, md5hash = read_with_python_encoding(path, self.pyversion)
+        except Exception as err:
+            self.read_error_cache[path] = err
+            raise
+        self.read_cache[path] = data
+        self.hash_cache[path] = md5hash
+        return data
+
+    def stat(self, path: str) -> os.stat_result:
+        if path in self.stat_cache:
+            return self.stat_cache[path]
+        if path in self.stat_error_cache:
+            raise self.stat_error_cache[path]
+        try:
+            st = os.stat(path)
+        except Exception as err:
+            self.stat_error_cache[path] = err
+            raise
+        self.stat_cache[path] = st
+        return st
+
+    def listdir(self, path: str) -> List[str]:
+        if path in self.listdir_cache:
+            return self.listdir_cache[path]
+        if path in self.listdir_error_cache:
+            raise self.listdir_error_cache[path]
+        try:
+            results = os.listdir(path)
+        except Exception as err:
+            self.listdir_error_cache[path] = err
+            raise err
+        self.listdir_cache[path] = results
+        return results
+
+    def isfile(self, path: str) -> bool:
+        st = self.stat(path)
+        return stat.S_ISREG(st.st_mode)
+
+    def isdir(self, path: str) -> bool:
+        st = self.stat(path)
+        return stat.S_ISDIR(st.st_mode)
+
+    def exists(self, path: str) -> bool:
+        try:
+            self.stat(path)
+        except FileNotFoundError:
+            return False
+        return True
+
+    def md5(self, path: str) -> str:
+        if path not in self.hash_cache:
+            self.read_with_python_encoding(path)
+        return self.hash_cache[path]