[Dependency Scanning] Implement parallel imported module resolution

'ModuleDependencyScanner' maintains a Thread Pool along with a pool of workers
which are capable of executing a filesystem lookup of a named module dependency.
When resolving imports of a given Swift module, each import's resolution
operation can be issued asunchronously.

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -662,6 +662,9 @@ class ModuleDependencyInfo {
   /// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
   bool isSwiftModule() const;
 
+  /// Whether the dependencies are for a textual interface Swift module or a Source Swift module.
+  bool isTextualSwiftModule() const;
+
   /// Whether the dependencies are for a textual Swift module.
   bool isSwiftInterfaceModule() const;
 
@@ -1001,13 +1004,20 @@ class ModuleDependenciesCache {
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
 
 public:
+  /// Whether we have cached dependency information for the given module.
+  bool hasDependency(const ModuleDependencyID &moduleID) const;
   /// Whether we have cached dependency information for the given module.
   bool hasDependency(StringRef moduleName,
                      llvm::Optional<ModuleDependencyKind> kind) const;
+  /// Whether we have cached dependency information for the given module Name.
+  bool hasDependency(StringRef moduleName) const;
 
   SwiftDependencyScanningService &getScanService() {
     return globalScanningService;
   }
+  const SwiftDependencyScanningService &getScanService() const {
+    return globalScanningService;
+  }
   const llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() const {
     return alreadySeenClangModules;
   }
@@ -1035,6 +1045,12 @@ class ModuleDependenciesCache {
   findDependency(StringRef moduleName,
                  llvm::Optional<ModuleDependencyKind> kind) const;
 
+  /// Look for module dependencies for a module with the given name
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  llvm::Optional<const ModuleDependencyInfo *>
+  findDependency(StringRef moduleName) const;
+
   /// Record dependencies for the given module.
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);

include/swift/DependencyScan/ScanDependencies.h

@@ -61,10 +61,6 @@ llvm::ErrorOr<swiftscan_dependency_graph_t>
 performModuleScan(CompilerInstance &instance,
                   ModuleDependenciesCache &cache);
 
-llvm::ErrorOr<swiftscan_dependency_graph_t>
-performParallelModuleScan(CompilerInstance &instance,
-                          ModuleDependenciesCache &cache);
-
 /// Scans the main module of \c instance for all direct module imports
 llvm::ErrorOr<swiftscan_import_set_t>
 performModulePrescan(CompilerInstance &instance,

include/swift/Frontend/FrontendOptions.h

@@ -366,7 +366,8 @@ class FrontendOptions {
   /// Emit remarks indicating use of the serialized module dependency scanning cache.
   bool EmitDependencyScannerCacheRemarks = false;
 
-  /// Whether the dependency scanner invocation should use multiple threads.
+  /// Whether the dependency scanner invocation should resolve imports
+  /// to filesystem modules in parallel.
   bool ParallelDependencyScan = false;
 
   /// When performing an incremental build, ensure that cross-module incremental

include/swift/Serialization/ModuleDependencyScanner.h

@@ -15,53 +15,49 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
+#include "llvm/Support/ThreadPool.h"
 
 namespace swift {
 class DependencyTracker;
 }
 
 namespace swift {
 
+/// A dependency scanning worker which performs filesystem lookup
+/// of a named module dependency.
 class ModuleDependencyScanningWorker {
-private:
-  ModuleDependencyScanningWorker(SwiftDependencyScanningService &globalScanningService,
-                                 const CompilerInvocation &ScanCompilerInvocation,
-                                 const SILOptions &SILOptions,
-                                 ASTContext &ScanASTContext,
-                                 DependencyTracker &DependencyTracker,
-                                 DiagnosticEngine &diags);
+public:
+  ModuleDependencyScanningWorker(
+      SwiftDependencyScanningService &globalScanningService,
+      const CompilerInvocation &ScanCompilerInvocation,
+      const SILOptions &SILOptions, ASTContext &ScanASTContext,
+      DependencyTracker &DependencyTracker, DiagnosticEngine &diags);
 
+private:
   /// Retrieve the module dependencies for the module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *> scanForModuleDependency(
-      StringRef moduleName, ModuleDependenciesCache &cache,
-      bool optionalDependencyLookup = false, bool isTestableImport = false,
-      llvm::Optional<ModuleDependencyID> dependencyOf = llvm::None);
+  ModuleDependencyVector
+  scanFilesystemForModuleDependency(StringRef moduleName,
+                                    const ModuleDependenciesCache &cache,
+                                    bool isTestableImport = false);
 
   /// Retrieve the module dependencies for the Clang module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  scanForClangModuleDependency(StringRef moduleName,
-                               ModuleDependenciesCache &cache);
+  ModuleDependencyVector
+  scanFilesystemForClangModuleDependency(StringRef moduleName,
+                                         const ModuleDependenciesCache &cache);
 
   /// Retrieve the module dependencies for the Swift module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  scanForSwiftModuleDependency(StringRef moduleName,
-                               ModuleDependenciesCache &cache);
-
+  ModuleDependencyVector
+  scanFilesystemForSwiftModuleDependency(StringRef moduleName,
+                                         const ModuleDependenciesCache &cache);
 
-private:
-  DiagnosticEngine &Diagnostics;
-
-  // An AST delegate for interface scanning
+  // An AST delegate for interface scanning.
   std::unique_ptr<InterfaceSubContextDelegateImpl> ScanningASTDelegate;
-  // Clang scanner tool
+  // The Clang scanner tool used by this worker.
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
-
-  // Swift and Clang module loaders acting as the corresponding
-  // scanners.
+  // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
   std::unique_ptr<ClangImporter> clangScannerModuleLoader;
-
-  // Only the parent scanner can use this 
+  // Restrict access to the parent scanner class.
   friend class ModuleDependencyScanner;
 };
 
@@ -72,33 +68,59 @@ class ModuleDependencyScanner {
                           const SILOptions &SILOptions,
                           ASTContext &ScanASTContext,
                           DependencyTracker &DependencyTracker,
-                          DiagnosticEngine &diags);
+                          DiagnosticEngine &diags, bool ParallelScan);
 
   /// Identify the scanner invocation's main module's dependencies
   llvm::ErrorOr<ModuleDependencyInfo> getMainModuleDependencyInfo(
       ModuleDecl *mainModule,
       llvm::Optional<SwiftDependencyTracker> tracker = llvm::None);
 
-  /// Resolve module dependencies of the given module, direct and transitive.
+  /// Resolve module dependencies of the given module, computing a full
+  /// transitive closure dependency graph.
   std::vector<ModuleDependencyID>
-  resolveDependencies(ModuleDependencyID moduleID,
-                      ModuleDependenciesCache &cache);
+  getModuleDependencies(ModuleDependencyID moduleID,
+                        ModuleDependenciesCache &cache);
 
-  /// Retrieve the module dependencies for the Clang module with the given name.
+  /// Query the module dependency info for the Clang module with the given name.
+  /// Explicit by-name lookups are useful for batch mode scanning.
   llvm::Optional<const ModuleDependencyInfo *>
-  scanForClangModuleDependency(StringRef moduleName,
-                               ModuleDependenciesCache &cache);
+  getNamedClangModuleDependencyInfo(StringRef moduleName,
+                                    ModuleDependenciesCache &cache);
 
-  /// Retrieve the module dependencies for the Swift module with the given name.
+  /// Query the module dependency info for the Swift module with the given name.
+  /// Explicit by-name lookups are useful for batch mode scanning.
   llvm::Optional<const ModuleDependencyInfo *>
-  scanForSwiftModuleDependency(StringRef moduleName,
-                               ModuleDependenciesCache &cache);
+  getNamedSwiftModuleDependencyInfo(StringRef moduleName,
+                                    ModuleDependenciesCache &cache);
 
 private:
   /// Resolve the direct dependencies of the given module.
   std::vector<ModuleDependencyID>
-  resolveModuleImports(ModuleDependencyID moduleID,
-                       ModuleDependenciesCache &cache);
+  resolveDirectModuleDependencies(ModuleDependencyID moduleID,
+                                  ModuleDependenciesCache &cache);
+
+  /// Resolve imported module names of a given module to concrete
+  /// modules. If `ParallelScan` is enabled, this operation is multithreaded.
+  void
+  resolveImportDependencies(const ModuleDependencyID &moduleID,
+                            ModuleDependenciesCache &cache,
+                            ModuleDependencyIDSetVector &directDependencies);
+
+  /// If a module has a bridging header, execute a dependency scan
+  /// on it and record the dependencies.
+  void resolveBridgingHeaderDependencies(
+      const ModuleDependencyID &moduleID, ModuleDependenciesCache &cache,
+      std::vector<std::string> &allClangModules,
+      llvm::StringSet<> &alreadyKnownModules,
+      ModuleDependencyIDSetVector &directDependencies);
+
+  /// Resolve all module dependencies comprised of Swift overlays
+  /// of this module's Clang module dependencies.
+  void resolveSwiftOverlayDependencies(
+      const ModuleDependencyID &moduleID,
+      const std::vector<std::string> &clangDependencies,
+      ModuleDependenciesCache &cache,
+      ModuleDependencyIDSetVector &swiftOverlayDependencies);
 
   /// Identify all cross-import overlay modules of the specified
   /// dependency set and apply an action for each.
@@ -107,13 +129,22 @@ class ModuleDependencyScanner {
       ModuleDependenciesCache &cache,
       llvm::function_ref<void(ModuleDependencyID)> action);
 
+  /// Perform an operation utilizing one of the Scanning workers
+  /// available to this scanner.
+  template <typename Function, typename... Args>
+  auto withDependencyScanningWorker(Function &&F, Args &&...ArgList);
+
 private:
   const CompilerInvocation &ScanCompilerInvocation;
   ASTContext &ScanASTContext;
   DiagnosticEngine &Diagnostics;
 
-  // For now, the sole worker doing the scanning
-  ModuleDependencyScanningWorker ScanningWorker;
+  /// The available pool of workers for filesystem module search
+  unsigned NumThreads;
+  std::list<std::unique_ptr<ModuleDependencyScanningWorker>> Workers;
+  llvm::ThreadPool ScanningThreadPool;
+  /// Protect worker access.
+  std::mutex WorkersLock;
 };
 
 /// A module "loader" that looks for .swiftinterface and .swiftmodule files

lib/AST/ModuleDependencies.cpp

@@ -32,6 +32,10 @@ bool ModuleDependencyInfo::isSwiftModule() const {
          isSwiftBinaryModule() || isSwiftPlaceholderModule();
 }
 
+bool ModuleDependencyInfo::isTextualSwiftModule() const {
+  return isSwiftInterfaceModule() || isSwiftSourceModule();
+}
+
 ModuleDependencyKind &operator++(ModuleDependencyKind &e) {
   if (e == ModuleDependencyKind::LastKind) {
     llvm_unreachable(
@@ -656,11 +660,34 @@ ModuleDependenciesCache::findDependency(
   return optionalDep;
 }
 
+llvm::Optional<const ModuleDependencyInfo *>
+ModuleDependenciesCache::findDependency(StringRef moduleName) const {
+  for (auto kind = ModuleDependencyKind::FirstKind;
+       kind != ModuleDependencyKind::LastKind; ++kind) {
+    if (auto found = findDependency(moduleName, kind))
+      return found;
+  }
+  return llvm::None;
+}
+
+bool ModuleDependenciesCache::hasDependency(const ModuleDependencyID &moduleID) const {
+  return hasDependency(moduleID.ModuleName, moduleID.Kind);
+}
+
 bool ModuleDependenciesCache::hasDependency(
     StringRef moduleName, llvm::Optional<ModuleDependencyKind> kind) const {
   return findDependency(moduleName, kind).has_value();
 }
 
+bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const {
+  for (auto kind = ModuleDependencyKind::FirstKind;
+       kind != ModuleDependencyKind::LastKind; ++kind) {
+    if (findDependency(moduleName, kind).has_value())
+      return true;
+  }
+  return false;
+}
+
 void ModuleDependenciesCache::recordDependency(
     StringRef moduleName, ModuleDependencyInfo dependencies) {
   auto dependenciesKind = dependencies.getKind();
@@ -675,7 +702,8 @@ void ModuleDependenciesCache::recordDependency(
 void ModuleDependenciesCache::recordDependencies(
     ModuleDependencyVector moduleDependencies) {
   for (const auto &dep : moduleDependencies) {
-    recordDependency(dep.first.ModuleName, dep.second);
+    if (!hasDependency(dep.first))
+      recordDependency(dep.first.ModuleName, dep.second);
     if (dep.second.getKind() == ModuleDependencyKind::Clang) {
       auto clangModuleDetails = dep.second.getAsClangModule();
       addSeenClangModule(clang::tooling::dependencies::ModuleID{

lib/AST/SearchPathOptions.cpp

@@ -101,7 +101,7 @@ ModuleSearchPathLookup::searchPathsContainingFile(
   llvm::SmallSet<std::pair<ModuleSearchPathKind, unsigned>, 4> ResultIds;
 
   for (auto &Filename : Filenames) {
-    for (auto &Entry : LookupTable[Filename]) {
+    for (auto &Entry : LookupTable.lookup(Filename)) {
       if (ResultIds.insert(std::make_pair(Entry->getKind(), Entry->getIndex()))
               .second) {
         Result.push_back(Entry.get());

lib/DependencyScan/ScanDependencies.cpp

@@ -831,10 +831,8 @@ static void writeJSON(llvm::raw_ostream &out,
                              /*trailingComma=*/false);
     } else if (swiftBinaryDeps) {
       out << "\"swiftPrebuiltExternal\": {\n";
-      bool hasCompiledModulePath =
-          swiftBinaryDeps->compiled_module_path.data &&
-          get_C_string(swiftBinaryDeps->compiled_module_path)[0] != '\0';
-      assert(hasCompiledModulePath &&
+      assert(swiftBinaryDeps->compiled_module_path.data &&
+             get_C_string(swiftBinaryDeps->compiled_module_path)[0] != '\0' &&
              "Expected .swiftmodule for a Binary Swift Module Dependency.");
 
       writeJSONSingleField(out, "compiledModulePath",
@@ -1231,9 +1229,7 @@ static bool diagnoseCycle(CompilerInstance &instance,
   while (!openSet.empty()) {
     auto &lastOpen = openSet.back();
     auto beforeSize = openSet.size();
-
-    auto optionalDepInfo = cache.findDependency(lastOpen);
-    assert(optionalDepInfo.has_value() &&
+    assert(cache.findDependency(lastOpen).has_value() &&
            "Missing dependency info during cycle diagnosis.");
 
     for (const auto &dep : cache.getAllDependencies(lastOpen)) {
@@ -1618,7 +1614,8 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
   auto scanner = ModuleDependencyScanner(
       cache.getScanService(), instance.getInvocation(),
       instance.getSILOptions(), instance.getASTContext(),
-      *instance.getDependencyTracker(), instance.getDiags());
+      *instance.getDependencyTracker(), instance.getDiags(),
+      instance.getInvocation().getFrontendOptions().ParallelDependencyScan);
 
   // Identify imports of the main module and add an entry for it
   // to the dependency graph.
@@ -1638,7 +1635,7 @@ swift::dependencies::performModuleScan(CompilerInstance &instance,
     cache.recordDependency(mainModuleName, std::move(*mainModuleDepInfo));
 
   // Perform the full module scan starting at the main module.
-  auto allModules = scanner.resolveDependencies(mainModuleID, cache);
+  auto allModules = scanner.getModuleDependencies(mainModuleID, cache);
 
 #ifndef NDEBUG
   // Verify that all collected dependencies have had their
@@ -1669,7 +1666,8 @@ swift::dependencies::performModulePrescan(CompilerInstance &instance,
   auto scanner = ModuleDependencyScanner(
       cache.getScanService(), instance.getInvocation(),
       instance.getSILOptions(), instance.getASTContext(),
-      *instance.getDependencyTracker(), instance.getDiags());
+      *instance.getDependencyTracker(), instance.getDiags(),
+      instance.getInvocation().getFrontendOptions().ParallelDependencyScan);
   // Execute import prescan, and write JSON output to the output stream
   auto mainDependencies =
       scanner.getMainModuleDependencyInfo(instance.getMainModule());
@@ -1698,7 +1696,8 @@ swift::dependencies::performBatchModuleScan(
         auto scanner = ModuleDependencyScanner(
             cache.getScanService(), instance.getInvocation(),
             instance.getSILOptions(), instance.getASTContext(),
-            *instance.getDependencyTracker(), instance.getDiags());
+            *instance.getDependencyTracker(), instance.getDiags(),
+            instance.getInvocation().getFrontendOptions().ParallelDependencyScan);
 
         StringRef moduleName = entry.moduleName;
         bool isClang = !entry.isSwift;
@@ -1707,9 +1706,9 @@ swift::dependencies::performBatchModuleScan(
           // Loading the clang module using Clang importer.
           // This action will populate the cache with the main module's
           // dependencies.
-          rootDeps = scanner.scanForClangModuleDependency(moduleName, cache);
+          rootDeps = scanner.getNamedClangModuleDependencyInfo(moduleName, cache);
         } else {
-          rootDeps = scanner.scanForSwiftModuleDependency(moduleName, cache);
+          rootDeps = scanner.getNamedSwiftModuleDependencyInfo(moduleName, cache);
         }
         if (!rootDeps.has_value()) {
           // We cannot find the clang module, abort.
@@ -1722,7 +1721,7 @@ swift::dependencies::performBatchModuleScan(
         ModuleDependencyID moduleID{
             moduleName.str(), isClang ? ModuleDependencyKind::Clang
                                       : ModuleDependencyKind::SwiftInterface};
-        auto allDependencies = scanner.resolveDependencies(moduleID, cache);
+        auto allDependencies = scanner.getModuleDependencies(moduleID, cache);
         batchScanResult.push_back(
             generateFullDependencyGraph(instance, cache, allDependencies));
       });

lib/Frontend/Frontend.cpp

@@ -53,6 +53,7 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/VirtualOutputBackends.h"
+#include "llvm/Support/ThreadPool.h"
 #include <llvm/ADT/StringExtras.h>
 
 using namespace swift;