[Dependency Scanning] Refactor the scanner to resolve unqualified module imports

- Re-running dependency resolution many times
- CacheOnly mode on dependency resolution
- Filesystem search when failed to find a Swift module in the cache, before checking the Clang cache.

Now:
- After discovering a dependency, its imports are "resolved" to kind-qualified module IDs: `SwiftTextual`, `SwiftBinary`, `Clang`, etc.
- Every lookup queries the cache first
- Scanner "loaders" no longer query the cache

Author: artemcm

include/swift/AST/ASTContext.h

@@ -991,16 +991,18 @@ class ASTContext final {
 
   /// Retrieve the module dependencies for the module with the given name.
   ///
-  /// \param isUnderlyingClangModule When true, only look for a Clang module
-  /// with the given name, ignoring any Swift modules.
   Optional<ModuleDependencyInfo> getModuleDependencies(
       StringRef moduleName,
-      bool isUnderlyingClangModule,
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate,
-      bool cacheOnly = false,
       llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>> dependencyOf = None);
 
+  /// Retrieve the module dependencies for the Clang module with the given name.
+  Optional<ModuleDependencyInfo> getClangModuleDependencies(
+      StringRef moduleName,
+      ModuleDependenciesCache &cache,
+      InterfaceSubContextDelegate &delegate);
+
   /// Retrieve the module dependencies for the Swift module with the given name.
   Optional<ModuleDependencyInfo> getSwiftModuleDependencies(
       StringRef moduleName,

include/swift/AST/ModuleDependencies.h

@@ -74,29 +74,44 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
+using ModuleDependencyID = std::pair<std::string, ModuleDependencyKind>;
+using ModuleDependencyIDSetVector =
+    llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
+                    std::set<ModuleDependencyID>>;
+
 struct ModuleDependencyKindHash {
   std::size_t operator()(ModuleDependencyKind k) const {
     using UnderlyingType = std::underlying_type<ModuleDependencyKind>::type;
     return std::hash<UnderlyingType>{}(static_cast<UnderlyingType>(k));
   }
 };
 
+namespace dependencies {
+  std::string createEncodedModuleKindAndName(ModuleDependencyID id);
+}
+
 /// Base class for the variant storage of ModuleDependencyInfo.
 ///
 /// This class is mostly an implementation detail for \c ModuleDependencyInfo.
 class ModuleDependencyInfoStorageBase {
 public:
   const ModuleDependencyKind dependencyKind;
 
-  ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind)
-      : dependencyKind(dependencyKind) { }
+  ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
+                                  bool resolved = false)
+      : dependencyKind(dependencyKind), resolved(resolved) { }
 
   virtual ModuleDependencyInfoStorageBase *clone() const = 0;
 
   virtual ~ModuleDependencyInfoStorageBase();
 
   /// The set of modules on which this module depends.
-  std::vector<std::string> moduleDependencies;
+  std::vector<std::string> moduleImports;
+
+  /// The set of modules on which this module depends, resolved
+  /// to Module IDs, qualified by module kind: Swift, Clang, etc.
+  ModuleDependencyIDSetVector resolvedModuleDependencies;
+  bool resolved;
 };
 
 struct CommonSwiftTextualModuleDependencyDetails {
@@ -400,9 +415,41 @@ class ModuleDependencyInfo {
           compiledModulePath, moduleDocPath, sourceInfoPath));
   }
 
-  /// Retrieve the module-level dependencies.
-  ArrayRef<std::string> getModuleDependencies() const {
-    return storage->moduleDependencies;
+  /// Retrieve the module-level imports.
+  ArrayRef<std::string> getModuleImports() const {
+    return storage->moduleImports;
+  }
+
+  /// Retreive the module-level dependencies.
+  const ModuleDependencyIDSetVector &getModuleDependencies() const {
+    assert(storage->resolved);
+    return storage->resolvedModuleDependencies;
+  }
+
+  /// Resolve a dependency's set of `imports` with qualified Module IDs
+  void resolveDependencies(ModuleDependencyIDSetVector dependencyIDs) {
+#ifndef NDEBUG
+    // Sanity check against the identified imports for this module
+    for (auto importedModuleName : storage->moduleImports) {
+      auto matchingModuleIDIt =
+          std::find_if(dependencyIDs.begin(), dependencyIDs.end(),
+                       [&](ModuleDependencyID dependencyID) {
+                         return dependencyID.first == importedModuleName;
+                       });
+      assert(matchingModuleIDIt != dependencyIDs.end() &&
+             "Imported module not found in resolved module dependency set");
+    }
+#endif
+    assert(!storage->resolved && "Resolving an already-resolved dependency");
+    storage->resolved = true;
+    storage->resolvedModuleDependencies = dependencyIDs;
+  }
+
+  bool isResolved() const {
+    return storage->resolved;
+  }
+  void setIsResolved(bool isResolved) {
+    storage->resolved = isResolved;
   }
 
   /// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
@@ -444,19 +491,22 @@ class ModuleDependencyInfo {
     getAsPlaceholderDependencyModule() const;
 
   /// Add a dependency on the given module, if it was not already in the set.
-  void addModuleDependency(StringRef module,
-                           llvm::StringSet<> *alreadyAddedModules = nullptr);
+  void addModuleImport(StringRef module,
+                       llvm::StringSet<> *alreadyAddedModules = nullptr);
 
   /// Add a dependency on the given module, if it was not already in the set.
-  void addModuleDependency(ImportPath::Module module,
-                           llvm::StringSet<> *alreadyAddedModules = nullptr) {
-    addModuleDependency(module.front().Item.str(), alreadyAddedModules);
+  void addModuleImport(ImportPath::Module module,
+                       llvm::StringSet<> *alreadyAddedModules = nullptr) {
+    addModuleImport(module.front().Item.str(), alreadyAddedModules);
   }
 
-  /// Add all of the module dependencies for the imports in the given source
-  /// file to the set of module dependencies.
-  void addModuleDependencies(const SourceFile &sf,
-                             llvm::StringSet<> &alreadyAddedModules);
+  /// Add all of the module imports in the given source
+  /// file to the set of module imports.
+  void addModuleImport(const SourceFile &sf,
+                       llvm::StringSet<> &alreadyAddedModules);
+  /// Add a kind-qualified module dependency ID to the set of
+  /// module dependencies.
+  void addModuleDependency(ModuleDependencyID dependencyID);
 
   /// Get the bridging header.
   Optional<std::string> getBridgingHeader() const;
@@ -480,8 +530,6 @@ class ModuleDependencyInfo {
   collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName);
 };
 
-using ModuleDependencyID = std::pair<std::string, ModuleDependencyKind>;
-using ModuleDependenciesVector = llvm::SmallVector<ModuleDependencyInfo, 1>;
 using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependencyKind,
@@ -587,8 +635,8 @@ class SwiftDependencyScanningService {
   }
 
   /// Whether we have cached dependency information for the given module.
-  bool hasDependencies(StringRef moduleName,
-                       Optional<ModuleDependencyKind> kind) const;
+  bool hasDependency(StringRef moduleName,
+                     Optional<ModuleDependencyKind> kind) const;
 
   /// Return a pointer to the context-specific cache state of the current
   /// scanning action.
@@ -606,15 +654,15 @@ class SwiftDependencyScanningService {
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
   Optional<ModuleDependencyInfo>
-  findDependencies(StringRef moduleName,
-                   Optional<ModuleDependencyKind> kind) const;
+  findDependency(StringRef moduleName,
+                 Optional<ModuleDependencyKind> kind) const;
 
   /// Record dependencies for the given module.
-  const ModuleDependencyInfo *recordDependencies(StringRef moduleName,
+  const ModuleDependencyInfo *recordDependency(StringRef moduleName,
                                                ModuleDependencyInfo dependencies);
 
   /// Update stored dependencies for the given module.
-  const ModuleDependencyInfo *updateDependencies(ModuleDependencyID moduleID,
+  const ModuleDependencyInfo *updateDependency(ModuleDependencyID moduleID,
                                                ModuleDependencyInfo dependencies);
 
   /// Reference the list of all module dependencies that are not source-based
@@ -658,7 +706,7 @@ class ModuleDependenciesCache {
   /// The Clang dependency scanner tool
   clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
   /// Discovered Clang modules are only cached locally.
-  llvm::StringMap<ModuleDependenciesVector> clangModuleDependencies;
+  llvm::StringMap<ModuleDependencyInfo> clangModuleDependencies;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -669,8 +717,8 @@ class ModuleDependenciesCache {
 
   /// Local cache results lookup, only for modules which were discovered during
   /// the current scanner invocation.
-  bool hasDependenciesLocalOnly(StringRef moduleName,
-                                Optional<ModuleDependencyKind> kind) const;
+  bool hasDependencyLocalOnly(StringRef moduleName,
+                              Optional<ModuleDependencyKind> kind) const;
 
   /// Local cache results lookup, only for modules which were discovered during
   /// the current scanner invocation.
@@ -687,8 +735,8 @@ class ModuleDependenciesCache {
 
 public:
   /// Whether we have cached dependency information for the given module.
-  bool hasDependencies(StringRef moduleName,
-                       Optional<ModuleDependencyKind> kind) const;
+  bool hasDependency(StringRef moduleName,
+                     Optional<ModuleDependencyKind> kind) const;
 
   /// Produce a reference to the Clang scanner tool associated with this cache
   clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
@@ -702,16 +750,21 @@ class ModuleDependenciesCache {
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
   Optional<ModuleDependencyInfo>
-  findDependencies(StringRef moduleName,
-                   Optional<ModuleDependencyKind> kind) const;
+  findDependency(StringRef moduleName,
+                 Optional<ModuleDependencyKind> kind) const;
 
   /// Record dependencies for the given module.
-  void recordDependencies(StringRef moduleName,
-                          ModuleDependencyInfo dependencies);
+  void recordDependency(StringRef moduleName,
+                        ModuleDependencyInfo dependencies);
 
   /// Update stored dependencies for the given module.
-  void updateDependencies(ModuleDependencyID moduleID,
-                          ModuleDependencyInfo dependencies);
+  void updateDependency(ModuleDependencyID moduleID,
+                        ModuleDependencyInfo dependencies);
+
+  /// Resolve a dependency module's set of imports
+  /// to a kind-qualified set of module IDs.
+  void resolveDependencyImports(ModuleDependencyID moduleID,
+                                ModuleDependencyIDSetVector dependencyIDs);
 
   const std::vector<ModuleDependencyID> &getAllSourceModules() const {
     return globalScanningService.getAllSourceModules();

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -36,7 +36,7 @@ using llvm::BCVBR;
 
 /// Every .moddepcache file begins with these 4 bytes, for easy identification.
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 3;
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 4;
 /// Increment this on every change.
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
 
@@ -45,19 +45,21 @@ const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
 using IdentifierIDField = BCVBR<13>;
 using FileIDField = IdentifierIDField;
 using ModuleIDField = IdentifierIDField;
-using ContextHashField = IdentifierIDField;
+using ContextHashIDField = IdentifierIDField;
 
 /// A bit that indicates whether or not a module is a framework
 using IsFrameworkField = BCFixed<1>;
 
 /// Arrays of various identifiers, distinguished for readability
 using IdentifierIDArryField = llvm::BCArray<IdentifierIDField>;
+using ModuleIDArryField = llvm::BCArray<IdentifierIDField>;
 
 /// Identifiers used to refer to the above arrays
 using FileIDArrayIDField = IdentifierIDField;
 using ContextHashIDField = IdentifierIDField;
-using DependencyIDArrayIDField = IdentifierIDField;
+using ImportArrayIDField = IdentifierIDField;
 using FlagIDArrayIDField = IdentifierIDField;
+using DependencyIDArrayIDField = IdentifierIDField;
 
 /// The ID of the top-level block containing the dependency graph
 const unsigned GRAPH_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID;
@@ -115,34 +117,35 @@ using IdentifierArrayLayout =
 // - SwiftPlaceholderModuleDetails
 // - ClangModuleDetails
 using ModuleInfoLayout =
-    BCRecordLayout<MODULE_NODE,             // ID
-                   IdentifierIDField,       // module name
-                   ContextHashIDField,      // 
-                   DependencyIDArrayIDField // directDependencies
+    BCRecordLayout<MODULE_NODE,                  // ID
+                   IdentifierIDField,            // moduleName
+                   ContextHashIDField,           // contextHash
+                   ImportArrayIDField,           // moduleImports
+                   DependencyIDArrayIDField      // resolvedModuleDependencies
                    >;
 
 using SwiftInterfaceModuleDetailsLayout =
     BCRecordLayout<SWIFT_INTERFACE_MODULE_DETAILS_NODE, // ID
-                   FileIDField,        // outputFilePath
-                   FileIDField,        // swiftInterfaceFile
-                   FileIDArrayIDField, // compiledModuleCandidates
-                   FlagIDArrayIDField, // buildCommandLine
-                   FlagIDArrayIDField, // extraPCMArgs
-                   ContextHashField,   // contextHash
-                   IsFrameworkField,   // isFramework
-                   FileIDField,        // bridgingHeaderFile
-                   FileIDArrayIDField, // sourceFiles
-                   FileIDArrayIDField, // bridgingSourceFiles
-                   FileIDArrayIDField   // bridgingModuleDependencies
+                   FileIDField,                         // outputFilePath
+                   FileIDField,                         // swiftInterfaceFile
+                   FileIDArrayIDField,                  // compiledModuleCandidates
+                   FlagIDArrayIDField,                  // buildCommandLine
+                   FlagIDArrayIDField,                  // extraPCMArgs
+                   ContextHashIDField,                  // contextHash
+                   IsFrameworkField,                    // isFramework
+                   FileIDField,                         // bridgingHeaderFile
+                   FileIDArrayIDField,                  // sourceFiles
+                   FileIDArrayIDField,                  // bridgingSourceFiles
+                   FileIDArrayIDField                   // bridgingModuleDependencies
                    >;
 
 using SwiftSourceModuleDetailsLayout =
     BCRecordLayout<SWIFT_SOURCE_MODULE_DETAILS_NODE, // ID
-                   FlagIDArrayIDField, // extraPCMArgs
-                   FileIDField,        // bridgingHeaderFile
-                   FileIDArrayIDField, // sourceFiles
-                   FileIDArrayIDField, // bridgingSourceFiles
-                   FileIDArrayIDField  // bridgingModuleDependencies
+                   FlagIDArrayIDField,               // extraPCMArgs
+                   FileIDField,                      // bridgingHeaderFile
+                   FileIDArrayIDField,               // sourceFiles
+                   FileIDArrayIDField,               // bridgingSourceFiles
+                   FileIDArrayIDField                // bridgingModuleDependencies
                    >;
 
 using SwiftBinaryModuleDetailsLayout =
@@ -157,14 +160,14 @@ using SwiftPlaceholderModuleDetailsLayout =
     BCRecordLayout<SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE, // ID
                    FileIDField,                           // compiledModulePath
                    FileIDField,                           // moduleDocPath
-                   FileIDField // moduleSourceInfoPath
+                   FileIDField                            // moduleSourceInfoPath
                    >;
 
 using ClangModuleDetailsLayout =
     BCRecordLayout<CLANG_MODULE_DETAILS_NODE, // ID
                    FileIDField,               // pcmOutputPath
                    FileIDField,               // moduleMapPath
-                   ContextHashField,          // contextHash
+                   ContextHashIDField,        // contextHash
                    FlagIDArrayIDField,        // commandLine
                    FileIDArrayIDField,        // fileDependencies
                    FlagIDArrayIDField         // capturedPCMArgs