Merge pull request #39374 from artemcm/55MultiTargetScan

[5.5][Dependency Scanning] Make dependency scanner cache specific to a given target triple.

Author: artemcm

include/swift-c/DependencyScan/DependencyScan.h

@@ -49,6 +49,8 @@ typedef struct {
 } swiftscan_string_set_t;
 
 typedef enum {
+  // This dependency info encodes two ModuleDependencyKind types:
+  // SwiftInterface and SwiftSource.
   SWIFTSCAN_DEPENDENCY_INFO_SWIFT_TEXTUAL = 0,
   SWIFTSCAN_DEPENDENCY_INFO_SWIFT_BINARY = 1,
   SWIFTSCAN_DEPENDENCY_INFO_SWIFT_PLACEHOLDER = 2,

include/swift/AST/ModuleDependencies.h

@@ -37,7 +37,7 @@ class Identifier;
 /// Which kind of module dependencies we are looking for.
 enum class ModuleDependenciesKind : int8_t {
   FirstKind,
-  SwiftTextual = FirstKind,
+  SwiftInterface = FirstKind,
   SwiftBinary,
   // Placeholder dependencies are a kind of dependencies used only by the
   // dependency scanner. They are swift modules that the scanner will not be
@@ -63,7 +63,8 @@ enum class ModuleDependenciesKind : int8_t {
   // of all targets, individually, have been computed.
   SwiftPlaceholder,
   Clang,
-  LastKind = Clang + 1
+  SwiftSource,
+  LastKind = SwiftSource + 1
 };
 
 struct ModuleDependenciesKindHash {
@@ -97,14 +98,33 @@ class ModuleDependenciesStorageBase {
   std::vector<std::string> moduleDependencies;
 };
 
-/// Describes the dependencies of a Swift module.
+struct CommonSwiftTextualModuleDependencyDetails {
+  CommonSwiftTextualModuleDependencyDetails(ArrayRef<StringRef> extraPCMArgs)
+     : extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()) {}
+
+  /// To build a PCM to be used by this Swift module, we need to append these
+  /// arguments to the generic PCM build arguments reported from the dependency
+  /// graph.
+  const std::vector<std::string> extraPCMArgs;
+
+  /// Bridging header file, if there is one.
+  Optional<std::string> bridgingHeaderFile;
+
+  /// Source files on which the bridging header depends.
+  std::vector<std::string> bridgingSourceFiles;
+
+  /// (Clang) modules on which the bridging header depends.
+  std::vector<std::string> bridgingModuleDependencies;
+};
+
+/// Describes the dependencies of a Swift module described by an Swift interface file.
 ///
 /// This class is mostly an implementation detail for \c ModuleDependencies.
-class SwiftTextualModuleDependenciesStorage :
+class SwiftInterfaceModuleDependenciesStorage :
   public ModuleDependenciesStorageBase {
 public:
-  /// The Swift interface file, if it can be used to generate the module file.
-  const Optional<std::string> swiftInterfaceFile;
+  /// The Swift interface file to be used to generate the module file.
+  const std::string swiftInterfaceFile;
 
   /// Potentially ready-to-use compiled modules for the interface file.
   const std::vector<std::string> compiledModuleCandidates;
@@ -113,50 +133,64 @@ class SwiftTextualModuleDependenciesStorage :
   /// interface.
   const std::vector<std::string> buildCommandLine;
 
-  /// To build a PCM to be used by this Swift module, we need to append these
-  /// arguments to the generic PCM build arguments reported from the dependency
-  /// graph.
-  const std::vector<std::string> extraPCMArgs;
-
   /// The hash value that will be used for the generated module
   const std::string contextHash;
 
   /// A flag that indicates this dependency is a framework
   const bool isFramework;
 
-  /// Bridging header file, if there is one.
-  Optional<std::string> bridgingHeaderFile;
-
-  /// Swift source files that are part of the Swift module, when known.
-  std::vector<std::string> sourceFiles;
+  /// Details common to Swift textual (interface or source) modules
+  CommonSwiftTextualModuleDependencyDetails textualModuleDetails;
 
-  /// Source files on which the bridging header depends.
-  std::vector<std::string> bridgingSourceFiles;
-
-  /// (Clang) modules on which the bridging header depends.
-  std::vector<std::string> bridgingModuleDependencies;
-
-  SwiftTextualModuleDependenciesStorage(
-      const Optional<std::string> &swiftInterfaceFile,
+  SwiftInterfaceModuleDependenciesStorage(
+      const std::string swiftInterfaceFile,
       ArrayRef<std::string> compiledModuleCandidates,
       ArrayRef<StringRef> buildCommandLine,
       ArrayRef<StringRef> extraPCMArgs,
       StringRef contextHash,
       bool isFramework
-  ) : ModuleDependenciesStorageBase(ModuleDependenciesKind::SwiftTextual),
+  ) : ModuleDependenciesStorageBase(ModuleDependenciesKind::SwiftInterface),
       swiftInterfaceFile(swiftInterfaceFile),
       compiledModuleCandidates(compiledModuleCandidates.begin(),
                                compiledModuleCandidates.end()),
       buildCommandLine(buildCommandLine.begin(), buildCommandLine.end()),
-      extraPCMArgs(extraPCMArgs.begin(), extraPCMArgs.end()),
-      contextHash(contextHash), isFramework(isFramework) { }
+      contextHash(contextHash), isFramework(isFramework),
+      textualModuleDetails(extraPCMArgs)
+      {}
 
   ModuleDependenciesStorageBase *clone() const override {
-    return new SwiftTextualModuleDependenciesStorage(*this);
+    return new SwiftInterfaceModuleDependenciesStorage(*this);
   }
 
   static bool classof(const ModuleDependenciesStorageBase *base) {
-    return base->dependencyKind == ModuleDependenciesKind::SwiftTextual;
+    return base->dependencyKind == ModuleDependenciesKind::SwiftInterface;
+  }
+};
+
+/// Describes the dependencies of a Swift module
+///
+/// This class is mostly an implementation detail for \c ModuleDependencies.
+class SwiftSourceModuleDependenciesStorage :
+  public ModuleDependenciesStorageBase {
+public:
+
+  /// Swift source files that are part of the Swift module, when known.
+  std::vector<std::string> sourceFiles;
+
+  /// Details common to Swift textual (interface or source) modules
+  CommonSwiftTextualModuleDependencyDetails textualModuleDetails;
+
+  SwiftSourceModuleDependenciesStorage(
+    ArrayRef<StringRef> extraPCMArgs
+  ) : ModuleDependenciesStorageBase(ModuleDependenciesKind::SwiftSource),
+      textualModuleDetails(extraPCMArgs) {}
+
+  ModuleDependenciesStorageBase *clone() const override {
+    return new SwiftSourceModuleDependenciesStorage(*this);
+  }
+
+  static bool classof(const ModuleDependenciesStorageBase *base) {
+    return base->dependencyKind == ModuleDependenciesKind::SwiftSource;
   }
 };
 
@@ -291,15 +325,15 @@ class ModuleDependencies {
 
   /// Describe the module dependencies for a Swift module that can be
   /// built from a Swift interface file (\c .swiftinterface).
-  static ModuleDependencies forSwiftTextualModule(
-      const Optional<std::string> &swiftInterfaceFile,
+  static ModuleDependencies forSwiftInterfaceModule(
+      std::string swiftInterfaceFile,
       ArrayRef<std::string> compiledCandidates,
       ArrayRef<StringRef> buildCommands,
       ArrayRef<StringRef> extraPCMArgs,
       StringRef contextHash,
       bool isFramework) {
     return ModuleDependencies(
-        std::make_unique<SwiftTextualModuleDependenciesStorage>(
+        std::make_unique<SwiftInterfaceModuleDependenciesStorage>(
           swiftInterfaceFile, compiledCandidates, buildCommands,
           extraPCMArgs, contextHash, isFramework));
   }
@@ -316,11 +350,9 @@ class ModuleDependencies {
   }
 
   /// Describe the main Swift module.
-  static ModuleDependencies forMainSwiftModule(ArrayRef<StringRef> extraPCMArgs) {
+  static ModuleDependencies forSwiftSourceModule(ArrayRef<StringRef> extraPCMArgs) {
     return ModuleDependencies(
-        std::make_unique<SwiftTextualModuleDependenciesStorage>(
-          None, ArrayRef<std::string>(), ArrayRef<StringRef>(),
-          extraPCMArgs, StringRef(), false));
+        std::make_unique<SwiftSourceModuleDependenciesStorage>(extraPCMArgs));
   }
 
   /// Describe the module dependencies for a Clang module that can be
@@ -350,11 +382,14 @@ class ModuleDependencies {
     return storage->moduleDependencies;
   }
 
-  /// Whether the dependencies are for a Swift module: either Textual, Binary, or Placeholder.
+  /// Whether the dependencies are for a Swift module: either Textual, Source, Binary, or Placeholder.
   bool isSwiftModule() const;
 
   /// Whether the dependencies are for a textual Swift module.
-  bool isSwiftTextualModule() const;
+  bool isSwiftInterfaceModule() const;
+
+  /// Whether the dependencies are for a textual Swift module.
+  bool isSwiftSourceModule() const;
 
   /// Whether the dependencies are for a binary Swift module.
   bool isSwiftBinaryModule() const;
@@ -368,8 +403,11 @@ class ModuleDependencies {
   ModuleDependenciesKind getKind() const {
     return storage->dependencyKind;
   }
+  /// Retrieve the dependencies for a Swift textual-interface module.
+  const SwiftInterfaceModuleDependenciesStorage *getAsSwiftInterfaceModule() const;
+
   /// Retrieve the dependencies for a Swift module.
-  const SwiftTextualModuleDependenciesStorage *getAsSwiftTextualModule() const;
+  const SwiftSourceModuleDependenciesStorage *getAsSwiftSourceModule() const;
 
   /// Retrieve the dependencies for a binary Swift module.
   const SwiftBinaryModuleDependencyStorage *getAsSwiftBinaryModule() const;
@@ -420,6 +458,14 @@ class ModuleDependencies {
 
 using ModuleDependencyID = std::pair<std::string, ModuleDependenciesKind>;
 using ModuleDependenciesVector = llvm::SmallVector<ModuleDependencies, 1>;
+using ModuleDependenciesKindMap =
+    std::unordered_map<ModuleDependenciesKind,
+                       llvm::StringMap<ModuleDependenciesVector>,
+                       ModuleDependenciesKindHash>;
+using ModuleDependenciesKindRefMap =
+    std::unordered_map<ModuleDependenciesKind,
+                       llvm::StringMap<const ModuleDependencies *>,
+                       ModuleDependenciesKindHash>;
 
 /// A cache describing the set of module dependencies that has been queried
 /// thus far. This cache records/stores the actual Dependency values and can be
@@ -431,18 +477,35 @@ using ModuleDependenciesVector = llvm::SmallVector<ModuleDependencies, 1>;
 /// ensure that the returned cached dependency was one that can be found in the
 /// current scanning action's filesystem view.
 class GlobalModuleDependenciesCache {
-  /// All cached module dependencies, in the order in which they were
-  /// encountered.
-  std::vector<ModuleDependencyID> AllModules;
-
-  /// Dependencies for modules that have already been computed.
-  /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects,
-  /// which correspond to instances of the same module that may have been found
-  /// in different sets of search paths.
-  std::unordered_map<ModuleDependenciesKind,
-                     llvm::StringMap<ModuleDependenciesVector>,
-                     ModuleDependenciesKindHash>
-      ModuleDependenciesKindMap;
+  /// Global cache contents specific to a target-triple specified on a scanner invocation
+  struct TargetSpecificGlobalCacheState {
+    /// All cached module dependencies, in the order in which they were
+    /// encountered.
+    std::vector<ModuleDependencyID> AllModules;
+
+    /// Dependencies for modules that have already been computed.
+    /// This maps a dependency kind to a map of a module's name to a vector of Dependency objects,
+    /// which correspond to instances of the same module that may have been found
+    /// in different sets of search paths.
+    ModuleDependenciesKindMap ModuleDependenciesMap;
+  };
+
+  /// All cached Swift source module dependencies, in the order in which they were encountered
+  std::vector<ModuleDependencyID> AllSourceModules;
+
+  /// Dependencies for all Swift source-based modules discovered. Each one is the main
+  /// module of a prior invocation of the scanner.
+  llvm::StringMap<ModuleDependencies> SwiftSourceModuleDependenciesMap;
+
+  /// A map from a String representing the target triple of a scanner invocation to the corresponding
+  /// cached dependencies discovered so far when using this triple.
+  llvm::StringMap<std::unique_ptr<TargetSpecificGlobalCacheState>> TargetSpecificCacheMap;
+
+  /// The current target triple cache configuration
+  Optional<std::string> CurrentTriple;
+
+  /// The triples used by scanners using this cache, in the order in which they were used
+  std::vector<std::string> AllTriples;
 
   /// Additional information needed for Clang dependency scanning.
   ClangModuleDependenciesCacheImpl *clangImpl = nullptr;
@@ -464,13 +527,19 @@ class GlobalModuleDependenciesCache {
   getDependenciesMap(ModuleDependenciesKind kind) const;
 
 public:
-  GlobalModuleDependenciesCache();
+  GlobalModuleDependenciesCache() {};
   GlobalModuleDependenciesCache(const GlobalModuleDependenciesCache &) = delete;
   GlobalModuleDependenciesCache &
   operator=(const GlobalModuleDependenciesCache &) = delete;
 
   virtual ~GlobalModuleDependenciesCache() { destroyClangImpl(); }
 
+  void configureForTriple(std::string triple);
+
+  const std::vector<std::string>& getAllTriples() const {
+    return AllTriples;
+  }
+
 private:
   /// Enforce clients not being allowed to query this cache directly, it must be
   /// wrapped in an instance of `ModuleDependenciesCache`.
@@ -500,6 +569,12 @@ class GlobalModuleDependenciesCache {
   Optional<ModuleDependencies>
   findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
 
+  /// Return a pointer to the target-specific cache state of the current triple configuration.
+  TargetSpecificGlobalCacheState* getCurrentCache() const;
+
+  /// Return a pointer to the target-specific cache state of the specified triple configuration.
+  TargetSpecificGlobalCacheState* getCacheForTriple(StringRef triple) const;
+
 public:
   /// Look for module dependencies for a module with the given name.
   /// This method has a deliberately-obtuse name to indicate that it is not to
@@ -510,6 +585,10 @@ class GlobalModuleDependenciesCache {
   findAllDependenciesIrrespectiveOfSearchPaths(
       StringRef moduleName, Optional<ModuleDependenciesKind> kind) const;
 
+  /// Look for source-based module dependency details
+  Optional<ModuleDependencies>
+  findSourceModuleDependency(StringRef moduleName) const;
+
   /// Record dependencies for the given module.
   const ModuleDependencies *recordDependencies(StringRef moduleName,
                                                ModuleDependencies dependencies);
@@ -518,9 +597,16 @@ class GlobalModuleDependenciesCache {
   const ModuleDependencies *updateDependencies(ModuleDependencyID moduleID,
                                                ModuleDependencies dependencies);
 
-  /// Reference the list of all module dependencies.
-  const std::vector<ModuleDependencyID> &getAllModules() const {
-    return AllModules;
+  /// Reference the list of all module dependencies that are not source-based modules
+  /// (i.e. interface dependencies, binary dependencies, clang dependencies).
+  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef triple) const {
+    auto targetSpecificCache = getCacheForTriple(triple);
+    return targetSpecificCache->AllModules;
+  }
+
+  /// Return the list of all source-based modules discovered by this cache
+  const std::vector<ModuleDependencyID> &getAllSourceModules() const {
+    return AllSourceModules;
   }
 };
 
@@ -536,10 +622,7 @@ class ModuleDependenciesCache {
 
   /// References to data in `globalCache` for dependencies accimulated during
   /// the current scanning action.
-  std::unordered_map<ModuleDependenciesKind,
-                     llvm::StringMap<const ModuleDependencies *>,
-                     ModuleDependenciesKindHash>
-      ModuleDependenciesKindMap;
+  ModuleDependenciesKindRefMap ModuleDependenciesMap;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -609,9 +692,8 @@ class ModuleDependenciesCache {
   void updateDependencies(ModuleDependencyID moduleID,
                           ModuleDependencies dependencies);
 
-  /// Reference the list of all module dependencies.
-  const std::vector<ModuleDependencyID> &getAllModules() const {
-    return globalCache.getAllModules();
+  const std::vector<ModuleDependencyID> &getAllSourceModules() const {
+    return globalCache.getAllSourceModules();
   }
 };
 

include/swift/ClangImporter/ClangImporter.h

@@ -409,7 +409,9 @@ class ClangImporter final : public ClangModuleLoader {
   ///
   /// \returns \c true if an error occurred, \c false otherwise
   bool addBridgingHeaderDependencies(
-      StringRef moduleName, ModuleDependenciesCache &cache);
+      StringRef moduleName,
+      ModuleDependenciesKind moduleKind,
+      ModuleDependenciesCache &cache);
 
   clang::TargetInfo &getTargetInfo() const override;
   clang::ASTContext &getClangASTContext() const override;

include/swift/DependencyScan/DependencyScanImpl.h

@@ -37,7 +37,8 @@ struct swiftscan_dependency_info_s {
   /// The format is:
   /// `<module-kind>:<module-name>`
   /// where `module-kind` is one of:
-  /// "swiftTextual"
+  /// "swiftInterface"
+  /// "swiftSource"
   /// "swiftBinary"
   /// "swiftPlaceholder"
   /// "clang""