Merge pull request #68252 from artemcm/ParallelScan

[Dependency Scanning] Implement parallel imported module resolution

Author: artemcm

include/swift/AST/ASTContext.h

@@ -92,7 +92,6 @@ namespace swift {
   class LazyIterableDeclContextData;
   class LazyMemberLoader;
   struct MacroDiscriminatorContext;
-  class ModuleDependencyInfo;
   class PatternBindingDecl;
   class PatternBindingInitializer;
   class PluginLoader;
@@ -1045,27 +1044,6 @@ class ASTContext final {
   /// Retrieve the module interface checker associated with this AST context.
   ModuleInterfaceChecker *getModuleInterfaceChecker() const;
 
-  /// Retrieve the module dependencies for the module with the given name.
-  ///
-  llvm::Optional<const ModuleDependencyInfo *> getModuleDependencies(
-      StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate,
-      bool optionalDependencyLookup = false, bool isTestableImport = false,
-      llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>>
-          dependencyOf = llvm::None);
-
-  /// Retrieve the module dependencies for the Clang module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  getClangModuleDependencies(StringRef moduleName,
-                             ModuleDependenciesCache &cache,
-                             InterfaceSubContextDelegate &delegate);
-
-  /// Retrieve the module dependencies for the Swift module with the given name.
-  llvm::Optional<const ModuleDependencyInfo *>
-  getSwiftModuleDependencies(StringRef moduleName,
-                             ModuleDependenciesCache &cache,
-                             InterfaceSubContextDelegate &delegate);
-
   /// Compute the extra implicit framework search paths on Apple platforms:
   /// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/.
   std::vector<std::string> getDarwinImplicitFrameworkSearchPaths() const;

include/swift/AST/ModuleDependencies.h

@@ -39,6 +39,7 @@
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <vector>
 
 namespace swift {
@@ -86,17 +87,38 @@ enum class ModuleDependencyKind : int8_t {
   LastKind = SwiftPlaceholder + 1
 };
 
-using ModuleDependencyID = std::pair<std::string, ModuleDependencyKind>;
-using ModuleDependencyIDSetVector =
-    llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
-                    std::set<ModuleDependencyID>>;
+/// This is used to identify a specific module.
+struct ModuleDependencyID {
+  std::string ModuleName;
+  ModuleDependencyKind Kind;
+  bool operator==(const ModuleDependencyID &Other) const {
+    return std::tie(ModuleName, Kind) ==
+           std::tie(Other.ModuleName, Other.Kind);
+  }
+  bool operator<(const ModuleDependencyID& Other) const {
+    return std::tie(ModuleName, Kind) <
+           std::tie(Other.ModuleName, Other.Kind);
+  }
+};
 
 struct ModuleDependencyKindHash {
   std::size_t operator()(ModuleDependencyKind k) const {
     using UnderlyingType = std::underlying_type<ModuleDependencyKind>::type;
     return std::hash<UnderlyingType>{}(static_cast<UnderlyingType>(k));
   }
 };
+struct ModuleDependencyIDHash {
+  std::size_t operator()(ModuleDependencyID id) const {
+    return llvm::hash_combine(id.ModuleName, id.Kind);
+  }
+};
+
+using ModuleDependencyIDSet =
+    std::unordered_set<ModuleDependencyID,
+                       ModuleDependencyIDHash>;
+using ModuleDependencyIDSetVector =
+    llvm::SetVector<ModuleDependencyID, std::vector<ModuleDependencyID>,
+                    std::set<ModuleDependencyID>>;
 
 namespace dependencies {
   std::string createEncodedModuleKindAndName(ModuleDependencyID id);
@@ -645,6 +667,9 @@ class ModuleDependencyInfo {
   /// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
   bool isSwiftModule() const;
 
+  /// Whether the dependencies are for a textual interface Swift module or a Source Swift module.
+  bool isTextualSwiftModule() const;
+
   /// Whether the dependencies are for a textual Swift module.
   bool isSwiftInterfaceModule() const;
 
@@ -750,6 +775,7 @@ class ModuleDependencyInfo {
   collectCrossImportOverlayNames(ASTContext &ctx, StringRef moduleName) const;
 };
 
+using ModuleDependencyVector = llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>;
 using ModuleNameToDependencyMap = llvm::StringMap<ModuleDependencyInfo>;
 using ModuleDependenciesKindMap =
     std::unordered_map<ModuleDependencyKind,
@@ -861,6 +887,7 @@ class SwiftDependencyScanningService {
   }
 
   bool usingCachingFS() const { return !UseClangIncludeTree && (bool)CacheFS; }
+  llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> getCachingFS() const { return CacheFS; }
 
   llvm::cas::CachingOnDiskFileSystem &getSharedCachingFS() const {
     assert(CacheFS && "Expect CachingOnDiskFileSystem");
@@ -895,9 +922,10 @@ class SwiftDependencyScanningService {
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
   friend class ModuleDependenciesCache;
+  friend class ModuleDependencyScanner;
+  friend class ModuleDependencyScanningWorker;
   friend class ModuleDependenciesCacheDeserializer;
   friend class ModuleDependenciesCacheSerializer;
-  friend class DependencyScanningTool;
 
   /// Configure the current state of the cache to respond to queries
   /// for the specified scanning context hash.
@@ -964,8 +992,6 @@ class ModuleDependenciesCache {
   std::string scannerContextHash;
   /// The location of where the built modules will be output to
   std::string moduleOutputPath;
-  /// The Clang dependency scanner tool
-  clang::tooling::dependencies::DependencyScanningTool clangScanningTool;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -983,38 +1009,60 @@ class ModuleDependenciesCache {
   ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
 
 public:
+  /// Whether we have cached dependency information for the given module.
+  bool hasDependency(const ModuleDependencyID &moduleID) const;
   /// Whether we have cached dependency information for the given module.
   bool hasDependency(StringRef moduleName,
                      llvm::Optional<ModuleDependencyKind> kind) const;
+  /// Whether we have cached dependency information for the given module Name.
+  bool hasDependency(StringRef moduleName) const;
 
-  /// Produce a reference to the Clang scanner tool associated with this cache
-  clang::tooling::dependencies::DependencyScanningTool& getClangScannerTool() {
-    return clangScanningTool;
-  }
   SwiftDependencyScanningService &getScanService() {
     return globalScanningService;
   }
-  llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() {
+  const SwiftDependencyScanningService &getScanService() const {
+    return globalScanningService;
+  }
+  const llvm::DenseSet<clang::tooling::dependencies::ModuleID>& getAlreadySeenClangModules() const {
     return alreadySeenClangModules;
   }
   void addSeenClangModule(clang::tooling::dependencies::ModuleID newModule) {
     alreadySeenClangModules.insert(newModule);
   }
-  std::string getModuleOutputPath() {
+  std::string getModuleOutputPath() const {
     return moduleOutputPath;
   }
 
+  /// Query all dependencies, direct and Swift overlay.
+  std::vector<ModuleDependencyID>
+  getAllDependencies(const ModuleDependencyID &moduleID) const;
+
+  /// Look for module dependencies for a module with the given ID
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  llvm::Optional<const ModuleDependencyInfo *>
+  findDependency(const ModuleDependencyID moduleID) const;
+
   /// Look for module dependencies for a module with the given name
   ///
   /// \returns the cached result, or \c None if there is no cached entry.
   llvm::Optional<const ModuleDependencyInfo *>
   findDependency(StringRef moduleName,
                  llvm::Optional<ModuleDependencyKind> kind) const;
 
+  /// Look for module dependencies for a module with the given name
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  llvm::Optional<const ModuleDependencyInfo *>
+  findDependency(StringRef moduleName) const;
+
   /// Record dependencies for the given module.
   void recordDependency(StringRef moduleName,
                         ModuleDependencyInfo dependencies);
 
+  /// Record dependencies for the given module collection.
+  void recordDependencies(ModuleDependencyVector moduleDependencies);
+
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,
                         ModuleDependencyInfo dependencies);
@@ -1039,12 +1087,8 @@ class ModuleDependenciesCache {
 namespace std {
 template <>
 struct hash<swift::ModuleDependencyID> {
-  using UnderlyingKindType = std::underlying_type<swift::ModuleDependencyKind>::type;
   std::size_t operator()(const swift::ModuleDependencyID &id) const {
-    auto underlyingKindValue = static_cast<UnderlyingKindType>(id.second);
-
-    return (hash<string>()(id.first) ^
-            (hash<UnderlyingKindType>()(underlyingKindValue)));
+    return llvm::hash_combine(id.ModuleName, id.Kind);
   }
 };
 } // namespace std

include/swift/AST/ModuleLoader.h

@@ -28,6 +28,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/Support/VersionTuple.h"
 #include <system_error>
 
@@ -36,10 +37,19 @@ class FileCollectorBase;
 namespace vfs {
 class OutputBackend;
 }
+namespace cas {
+class CachingOnDiskFileSystem;
+}
 }
 
 namespace clang {
 class DependencyCollector;
+namespace tooling {
+namespace dependencies {
+struct ModuleID;
+class DependencyScanningTool;
+}
+}
 }
 
 namespace swift {
@@ -51,6 +61,7 @@ class ClassDecl;
 class FileUnit;
 class ModuleDecl;
 class ModuleDependencyInfo;
+struct ModuleDependencyID;
 class ModuleDependenciesCache;
 class NominalTypeDecl;
 class SourceFile;
@@ -327,8 +338,12 @@ class ModuleLoader {
 
   /// Retrieve the dependencies for the given, named module, or \c None
   /// if no such module exists.
-  virtual llvm::Optional<const ModuleDependencyInfo *>
-  getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
+  virtual llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  getModuleDependencies(StringRef moduleName,
+                        StringRef moduleOutputPath,
+                        llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
+                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
+                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
                         bool isTestableImport = false) = 0;
 };

include/swift/ClangImporter/ClangImporter.h

@@ -75,6 +75,7 @@ class FuncDecl;
 class ImportDecl;
 class IRGenOptions;
 class ModuleDecl;
+struct ModuleDependencyID;
 class NominalTypeDecl;
 class StructDecl;
 class SwiftLookupTable;
@@ -433,32 +434,37 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
-  void recordModuleDependencies(
-      ModuleDependenciesCache &cache,
-      const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies);
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
+      const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
+      StringRef moduleOutputPath);
+
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
+                        llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
+                        const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
+                        clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+                        InterfaceSubContextDelegate &delegate,
+                        bool isTestableImport = false) override;
 
   void recordBridgingHeaderOptions(
       ModuleDependencyInfo &MDI,
       const clang::tooling::dependencies::TranslationUnitDeps &deps);
 
-  llvm::Optional<const ModuleDependencyInfo *>
-  getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
-                        InterfaceSubContextDelegate &delegate,
-                        bool isTestableImport = false) override;
-
   /// Add dependency information for the bridging header.
   ///
-  /// \param moduleName the name of the Swift module whose dependency
+  /// \param moduleID the name of the Swift module whose dependency
   /// information will be augmented with information about the given
   /// bridging header.
   ///
+  /// \param clangScanningTool The clang dependency scanner.
+  ///
   /// \param cache The module dependencies cache to update, with information
   /// about new Clang modules discovered along the way.
   ///
   /// \returns \c true if an error occurred, \c false otherwise
   bool addBridgingHeaderDependencies(
-      StringRef moduleName,
-      ModuleDependencyKind moduleKind,
+      ModuleDependencyID moduleID,
+      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       ModuleDependenciesCache &cache);
   clang::TargetInfo &getModuleAvailabilityTarget() const override;
   clang::ASTContext &getClangASTContext() const override;