Delay processing fresh SCCs until needed (#2110)

Previously, the process_graph function would always process SCCs, fresh
and stale, as they were encountered.

This commit modifies this process so instead, we batch and defer
processing fresh SCCs until we encounter a stale SCCs. If we do not end
up encountering any stale SCCs at all, we throw away the queued fresh
SCCs without having to load any of the data files at all.

Since loading the data file of fresh SCCs does take a small amount of
time, this change will make mypy slightly faster, particularly in the
case where there are very few to no files to recheck, and the
corresponding SCCs occur early in the sorted list of SCCs.

As an example, when running mypy against itself with a fully warm cache,
the time it takes to finish typechecking is now approximately 0.3 to 0.4
seconds -- previously, it was 1.3 to 1.4.

Author: Michael0x2a

mypy/build.py

@@ -1550,6 +1550,9 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
     sccs = sorted_components(graph)
     manager.log("Found %d SCCs; largest has %d nodes" %
                 (len(sccs), max(len(scc) for scc in sccs)))
+
+    fresh_scc_queue = []  # type: List[List[str]]
+
     # We're processing SCCs from leaves (those without further
     # dependencies) to roots (those from which everything else can be
     # reached).
@@ -1627,16 +1630,36 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 fresh_msg += " with stale deps (%s)" % " ".join(sorted(stale_deps))
         else:
             fresh_msg = "stale due to deps (%s)" % " ".join(sorted(stale_deps))
-        if len(scc) == 1:
-            manager.log("Processing SCC singleton (%s) as %s" % (" ".join(scc), fresh_msg))
-        else:
-            manager.log("Processing SCC of size %d (%s) as %s" %
-                        (len(scc), " ".join(scc), fresh_msg))
+
+        scc_str = " ".join(scc)
         if fresh:
-            process_fresh_scc(graph, scc)
+            manager.log("Queuing fresh SCC (%s)" % scc_str)
+            fresh_scc_queue.append(scc)
         else:
+            if len(fresh_scc_queue) > 0:
+                manager.log("Processing the last {} queued SCCs".format(len(fresh_scc_queue)))
+                # Defer processing fresh SCCs until we actually run into a stale SCC
+                # and need the earlier modules to be loaded.
+                #
+                # Note that `process_graph` may end with us not having processed every
+                # single fresh SCC. This is intentional -- we don't need those modules
+                # loaded if there are no more stale SCCs to be rechecked.
+                #
+                # TODO: see if it's possible to determine if we need to process only a
+                # _subset_ of the past SCCs instead of having to process them all.
+                for prev_scc in fresh_scc_queue:
+                    process_fresh_scc(graph, prev_scc)
+                fresh_scc_queue = []
+            size = len(scc)
+            if size == 1:
+                manager.log("Processing SCC singleton (%s) as %s" % (scc_str, fresh_msg))
+            else:
+                manager.log("Processing SCC of size %d (%s) as %s" % (size, scc_str, fresh_msg))
             process_stale_scc(graph, scc)
 
+    sccs_left = len(fresh_scc_queue)
+    manager.log("{} fresh SCCs left in queue (and will remain unprocessed)".format(sccs_left))
+
 
 def order_ascc(graph: Graph, ascc: AbstractSet[str], pri_max: int = PRI_ALL) -> List[str]:
     """Come up with the ideal processing order within an SCC.