[Dependency Scanning] Key into a global (shared) dependency scanning cache via context hash

Introduces a concept of a dependency scanning action context hash, which is used to select an instance of a global dependency scanning cache which gets re-used across dependency scanning actions.

Author: artemcm

include/swift/AST/IRGenOptions.h

@@ -550,6 +550,12 @@ class IRGenOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   bool hasMultipleIRGenThreads() const { return !UseSingleModuleLLVMEmission && NumThreads > 1; }
   bool shouldPerformIRGenerationInParallel() const { return !UseSingleModuleLLVMEmission && NumThreads != 0; }
   bool hasMultipleIGMs() const { return hasMultipleIRGenThreads(); }

include/swift/AST/ModuleDependencies.h

@@ -501,8 +501,8 @@ using ModuleDependenciesKindRefMap =
 /// ensure that the returned cached dependency was one that can be found in the
 /// current scanning action's filesystem view.
 class GlobalModuleDependenciesCache {
-  /// Global cache contents specific to a target-triple specified on a scanner invocation
-  struct TargetSpecificGlobalCacheState {
+  /// Global cache contents specific to a specific scanner invocation context
+  struct ContextSpecificGlobalCacheState {
     /// All cached module dependencies, in the order in which they were
     /// encountered.
     std::vector<ModuleDependencyID> AllModules;
@@ -526,13 +526,13 @@ class GlobalModuleDependenciesCache {
 
   /// A map from a String representing the target triple of a scanner invocation to the corresponding
   /// cached dependencies discovered so far when using this triple.
-  llvm::StringMap<std::unique_ptr<TargetSpecificGlobalCacheState>> TargetSpecificCacheMap;
+  llvm::StringMap<std::unique_ptr<ContextSpecificGlobalCacheState>> ContextSpecificCacheMap;
 
-  /// The current target triple cache configuration
-  Optional<std::string> CurrentTriple;
+  /// The current context hash configuration
+  Optional<std::string> CurrentContextHash;
 
-  /// The triples used by scanners using this cache, in the order in which they were used
-  std::vector<std::string> AllTriples;
+  /// The context hashes used by scanners using this cache, in the order in which they were used
+  std::vector<std::string> AllContextHashes;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -548,10 +548,10 @@ class GlobalModuleDependenciesCache {
   operator=(const GlobalModuleDependenciesCache &) = delete;
   virtual ~GlobalModuleDependenciesCache() {}
 
-  void configureForTriple(std::string triple);
+  void configureForContextHash(std::string scanningContextHash);
 
-  const std::vector<std::string>& getAllTriples() const {
-    return AllTriples;
+  const std::vector<std::string>& getAllContextHashes() const {
+    return AllContextHashes;
   }
 
 private:
@@ -570,11 +570,11 @@ class GlobalModuleDependenciesCache {
   Optional<ModuleDependencies>
   findDependencies(StringRef moduleName, ModuleLookupSpecifics details) const;
 
-  /// Return a pointer to the target-specific cache state of the current triple configuration.
-  TargetSpecificGlobalCacheState* getCurrentCache() const;
+  /// Return a pointer to the context-specific cache state of the current scanning action.
+  ContextSpecificGlobalCacheState* getCurrentCache() const;
 
-  /// Return a pointer to the target-specific cache state of the specified triple configuration.
-  TargetSpecificGlobalCacheState* getCacheForTriple(StringRef triple) const;
+  /// Return a pointer to the cache state of the specified context hash.
+  ContextSpecificGlobalCacheState* getCacheForScanningContextHash(StringRef scanningContextHash) const;
 
 public:
   /// Look for module dependencies for a module with the given name.
@@ -600,9 +600,9 @@ class GlobalModuleDependenciesCache {
 
   /// Reference the list of all module dependencies that are not source-based modules
   /// (i.e. interface dependencies, binary dependencies, clang dependencies).
-  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef triple) const {
-    auto targetSpecificCache = getCacheForTriple(triple);
-    return targetSpecificCache->AllModules;
+  const std::vector<ModuleDependencyID> &getAllNonSourceModules(StringRef scanningContextHash) const {
+    auto contextSpecificCache = getCacheForScanningContextHash(scanningContextHash);
+    return contextSpecificCache->AllModules;
   }
 
   /// Return the list of all source-based modules discovered by this cache

include/swift/AST/SILOptions.h

@@ -281,6 +281,12 @@ class SILOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   bool shouldOptimize() const {
     return OptMode > OptimizationMode::NoOptimization;
   }

include/swift/AST/SearchPathOptions.h

@@ -421,6 +421,12 @@ class SearchPathOptions {
                                            RuntimeLibraryImportPaths.end()),
                         DisableModulesValidateSystemDependencies);
   }
+
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return getPCHHashComponents();
+  }
 };
 }
 

include/swift/Basic/DiagnosticOptions.h

@@ -86,6 +86,12 @@ class DiagnosticOptions {
     // Nothing here that contributes anything significant when emitting the PCH.
     return llvm::hash_value(0);
   }
+
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
 };
 
 } // end namespace swift

include/swift/Basic/LangOptions.h

@@ -628,6 +628,12 @@ namespace swift {
       return llvm::hash_combine(Target.str(), OS.str());
     }
 
+    /// Return a hash code of any components from these options that should
+    /// contribute to a Swift Dependency Scanning hash.
+    llvm::hash_code getModuleScanningHashComponents() const {
+      return getPCHHashComponents();
+    }
+
   private:
     llvm::SmallVector<std::pair<PlatformConditionKind, std::string>, 6>
         PlatformConditionValues;
@@ -832,6 +838,12 @@ namespace swift {
                           EnableClangSPI);
     }
 
+    /// Return a hash code of any components from these options that should
+    /// contribute to a Swift Dependency Scanning hash.
+    llvm::hash_code getModuleScanningHashComponents() const {
+      return getPCHHashComponents();
+    }
+
     std::vector<std::string> getRemappedExtraArgs(
         std::function<std::string(StringRef)> pathRemapCallback) const;
   };

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -35,9 +35,8 @@ using llvm::BCRecordLayout;
 using llvm::BCVBR;
 
 /// Every .moddepcache file begins with these 4 bytes, for easy identification.
-const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D',
-                                                                  'C'};
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 2;
+const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 3;
 /// Increment this on every change.
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 0;
 
@@ -56,7 +55,7 @@ using IdentifierIDArryField = llvm::BCArray<IdentifierIDField>;
 
 /// Identifiers used to refer to the above arrays
 using FileIDArrayIDField = IdentifierIDField;
-using TripleIDField = IdentifierIDField;
+using ContextHashIDField = IdentifierIDField;
 using DependencyIDArrayIDField = IdentifierIDField;
 using FlagIDArrayIDField = IdentifierIDField;
 
@@ -118,7 +117,7 @@ using IdentifierArrayLayout =
 using ModuleInfoLayout =
     BCRecordLayout<MODULE_NODE,             // ID
                    IdentifierIDField,       // module name
-                   TripleIDField,           // target triple
+                   ContextHashIDField,      // 
                    DependencyIDArrayIDField // directDependencies
                    >;
 

include/swift/Frontend/Frontend.h

@@ -345,6 +345,11 @@ class CompilerInvocation {
   /// in generating a cached PCH file for the bridging header.
   std::string getPCHHash() const;
 
+  /// Retrieve a module hash string that is suitable for uniquely
+  /// identifying the conditions under which the current module is built,
+  /// from the perspective of a dependency scanning action.
+  std::string getModuleScanningHash() const;
+
   /// Retrieve the stdlib kind to implicitly import.
   ImplicitStdlibKind getImplicitStdlibKind() const {
     if (FrontendOpts.InputMode == FrontendOptions::ParseInputMode::SIL) {

include/swift/Frontend/FrontendOptions.h

@@ -430,6 +430,12 @@ class FrontendOptions {
     return llvm::hash_value(0);
   }
 
+  /// Return a hash code of any components from these options that should
+  /// contribute to a Swift Dependency Scanning hash.
+  llvm::hash_code getModuleScanningHashComponents() const {
+    return llvm::hash_value(0);
+  }
+
   StringRef determineFallbackModuleName() const;
 
   bool isCompilingExactlyOneSwiftFile() const {

lib/AST/ModuleDependencies.cpp

@@ -234,20 +234,20 @@ GlobalModuleDependenciesCache::GlobalModuleDependenciesCache()
                          /* SharedFS */ nullptr,
                          /* ReuseFileManager */ false,
                          /* OptimizeArgs */ false) {}
-GlobalModuleDependenciesCache::TargetSpecificGlobalCacheState *
+GlobalModuleDependenciesCache::ContextSpecificGlobalCacheState *
 GlobalModuleDependenciesCache::getCurrentCache() const {
-  assert(CurrentTriple.has_value() &&
+  assert(CurrentContextHash.has_value() &&
          "Global Module Dependencies Cache not configured with Triple.");
-  return getCacheForTriple(CurrentTriple.value());
+  return getCacheForScanningContextHash(CurrentContextHash.value());
 }
 
-GlobalModuleDependenciesCache::TargetSpecificGlobalCacheState *
-GlobalModuleDependenciesCache::getCacheForTriple(StringRef triple) const {
-  auto targetSpecificCache = TargetSpecificCacheMap.find(triple);
-  assert(targetSpecificCache != TargetSpecificCacheMap.end() &&
-         "Global Module Dependencies Cache not configured with Triple-specific "
+GlobalModuleDependenciesCache::ContextSpecificGlobalCacheState *
+GlobalModuleDependenciesCache::getCacheForScanningContextHash(StringRef scanningContextHash) const {
+  auto contextSpecificCache = ContextSpecificCacheMap.find(scanningContextHash);
+  assert(contextSpecificCache != ContextSpecificCacheMap.end() &&
+         "Global Module Dependencies Cache not configured with context-specific "
          "state.");
-  return targetSpecificCache->getValue().get();
+  return contextSpecificCache->getValue().get();
 }
 
 llvm::StringMap<ModuleDependenciesVector> &
@@ -333,24 +333,24 @@ moduleContainedInImportPathSet(const ModuleDependencies &module,
   return false;
 }
 
-void GlobalModuleDependenciesCache::configureForTriple(std::string triple) {
-  auto knownTriple = TargetSpecificCacheMap.find(triple);
-  if (knownTriple != TargetSpecificCacheMap.end()) {
-    // Set the current triple and leave the rest as-is
-    CurrentTriple = triple;
+void GlobalModuleDependenciesCache::configureForContextHash(std::string scanningContextHash) {
+  auto knownContext = ContextSpecificCacheMap.find(scanningContextHash);
+  if (knownContext != ContextSpecificCacheMap.end()) {
+    // Set the current context and leave the rest as-is
+    CurrentContextHash = scanningContextHash;
   } else {
     // First time scanning with this triple, initialize target-specific state.
-    std::unique_ptr<TargetSpecificGlobalCacheState> targetSpecificCache =
-        std::make_unique<TargetSpecificGlobalCacheState>();
+    std::unique_ptr<ContextSpecificGlobalCacheState> contextSpecificCache =
+        std::make_unique<ContextSpecificGlobalCacheState>();
     for (auto kind = ModuleDependenciesKind::FirstKind;
          kind != ModuleDependenciesKind::LastKind; ++kind) {
-      targetSpecificCache->ModuleDependenciesMap.insert(
+      contextSpecificCache->ModuleDependenciesMap.insert(
           {kind, llvm::StringMap<ModuleDependenciesVector>()});
     }
 
-    TargetSpecificCacheMap.insert({triple, std::move(targetSpecificCache)});
-    CurrentTriple = triple;
-    AllTriples.push_back(triple);
+    ContextSpecificCacheMap.insert({scanningContextHash, std::move(contextSpecificCache)});
+    CurrentContextHash = scanningContextHash;
+    AllContextHashes.push_back(scanningContextHash);
   }
 }
 

lib/DependencyScan/DependencyScanningTool.cpp

@@ -205,8 +205,8 @@ DependencyScanningTool::initScannerForAction(
   auto instanceOrErr = initCompilerInstanceForScan(Command);
   if (instanceOrErr.getError())
     return instanceOrErr;
-  SharedCache->configureForTriple((*instanceOrErr)->getInvocation()
-                                  .getLangOptions().Target.str());
+  SharedCache->configureForContextHash((*instanceOrErr)->getInvocation()
+                                       .getModuleScanningHash());
   return instanceOrErr;
 }