Add implicit module reference data to dependencies

This commit modifies the build process to record all implicit module
references a given file makes when writing cache data. Previously, the
cache data would only save all explicit dependencies (i.e. modules used
within a direct import).

This posed a problem because it turned out it's trivially easy to
construct a series of interlinked files wherein changing the interface
of one module produces or fixes an error in a seemingly unrelated file
(which did NOT import anything from the first module).

This made optimizing incremental mode harder since an interface change
could have unknown effects on the codebase as a whole.

This change modifies how dependencies are recorded so BOTH explicit
(directly imported) and implicit (indirectly reliant) dependences are
recorded for each file.

This means that checking to see if any of our dependents' interfaces
have changed requires simply checking every module in the list of
dependencies (moving one "step") rather then having to potentially crawl
the entire graph.

In order to avoid accidentally perturbing the import graph in
unpredictable ways, all implicit dependences will be given a lower
priority then regular imports (lower then PRI_LOW).

Author: MichaelLeeDBX

mypy/build.py

@@ -29,6 +29,7 @@
                         SymbolTableNode, MODULE_REF)
 from mypy.semanal import FirstPass, SemanticAnalyzer, ThirdPass
 from mypy.checker import TypeChecker
+from mypy.indirection import TypeIndirectionVisitor
 from mypy.errors import Errors, CompileError, DecodeError, report_internal_error
 from mypy import fixup
 from mypy.report import Reports
@@ -305,6 +306,7 @@ def default_lib_path(data_dir: str, pyversion: Tuple[int, int]) -> List[str]:
 PRI_HIGH = 5  # top-level "from X import blah"
 PRI_MED = 10  # top-level "import X"
 PRI_LOW = 20  # either form inside a function
+PRI_INDIRECT = 30  # an indirect dependency
 PRI_ALL = 99  # include all priorities
 
 
@@ -351,6 +353,7 @@ def __init__(self, data_dir: str,
         self.modules = self.semantic_analyzer.modules
         self.semantic_analyzer_pass3 = ThirdPass(self.modules, self.errors)
         self.type_checker = TypeChecker(self.errors, self.modules, options=options)
+        self.indirection_detector = TypeIndirectionVisitor()
         self.missing_modules = set()  # type: Set[str]
         self.stale_modules = set()  # type: Set[str]
         self.rechecked_modules = set()  # type: Set[str]
@@ -1415,11 +1418,35 @@ def type_check(self) -> None:
             return
         with self.wrap_context():
             manager.type_checker.visit_file(self.tree, self.xpath)
+
+            if manager.options.incremental:
+                self._patch_indirect_dependencies(manager.type_checker.module_refs)
+
             if manager.options.dump_inference_stats:
                 dump_type_stats(self.tree, self.xpath, inferred=True,
                                 typemap=manager.type_checker.type_map)
             manager.report_file(self.tree)
 
+    def _patch_indirect_dependencies(self, module_refs: Set[str]) -> None:
+        types = self.manager.type_checker.module_type_map.values()
+        valid = self.valid_references()
+
+        encountered = self.manager.indirection_detector.find_modules(types) | module_refs
+        extra = encountered - valid
+
+        for dep in sorted(extra):
+            self.dependencies.append(dep)
+            self.priorities[dep] = PRI_INDIRECT
+
+    def valid_references(self) -> Set[str]:
+        valid_refs = set(self.dependencies + self.suppressed + self.ancestors)
+        valid_refs .add(self.id)
+
+        if "os" in valid_refs:
+            valid_refs.add("os.path")
+
+        return valid_refs
+
     def write_cache(self) -> None:
         if self.path and self.manager.options.incremental and not self.manager.errors.is_errors():
             dep_prios = [self.priorities.get(dep, PRI_HIGH) for dep in self.dependencies]
@@ -1598,25 +1625,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         else:
             process_stale_scc(graph, scc)
 
-        # TODO: This is a workaround to get around the "chaining imports" problem
-        # with the interface checks.
-        #
-        # That is, if we have a file named `module_a.py` which does:
-        #
-        #     import module_b
-        #     module_b.module_c.foo(3)
-        #
-        # ...and if the type signature of `module_c.foo(...)` were to change,
-        # module_a_ would not be rechecked since the interface of `module_b`
-        # would not be considered changed.
-        #
-        # As a workaround, this check will force a module's interface to be
-        # considered stale if anything it imports has a stale interface,
-        # which ensures these changes are caught and propagated.
-        if len(stale_deps) > 0:
-            for id in scc:
-                graph[id].mark_interface_stale()
-
 
 def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]:
     """Come up with the ideal processing order within an SCC.