[Dependency Scanning] Add functionality to validate contents of a loaded scanner cache state

Checking each module dependency info if it is up-to-date with respect to when the cache contents were serialized in a prior scan.

- Add a timestamp field to the serialization format for the dependency scanner cache
- Add a flag "-validate-prior-dependency-scan-cache" which, when combined with "-load-dependency-scan-cache" will have the scanner prune dependencies from the deserialized cache which have inputs that are newer than the prior scan itself

With the above in-place, the scan otherwise proceeds as-is, getting cache hits for entries still valid since the prior scan.

Author: artemcm

include/swift/AST/DiagnosticsCommon.def

@@ -212,17 +212,25 @@ ERROR(scanner_arguments_invalid, none,
 ERROR(error_scanner_extra, none,
       "failed inside dependency scanner: '%0'", (StringRef))
 
-WARNING(warn_scanner_deserialize_failed, none,
-        "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
+REMARK(warn_scanner_deserialize_failed, none,
+       "Incremental module scan: Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
 
 REMARK(remark_reuse_cache, none,
-       "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
+       "Incremental module scan: Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
 
 REMARK(remark_scanner_uncached_lookups, none,
        "Module Dependency Scanner queries executed: '%0'.", (unsigned))
 
 REMARK(remark_save_cache, none,
-       "Serializing module scanning dependency cache to: '%0'.", (StringRef))
+       "Incremental module scan: Serializing module scanning dependency cache to: '%0'.", (StringRef))
+
+REMARK(remark_scanner_stale_result_invalidate, none,
+       "Incremental module scan: Dependency info for module '%0' invalidated due to a modified input"
+       " since last scan: '%1'.", (StringRef, StringRef))
+
+REMARK(remark_scanner_invalidate_upstream, none,
+       "Incremental module scan: Dependency info for module '%0' invalidated due to an out-of-date"
+       " dependency.", (StringRef))
 
 //------------------------------------------------------------------------------
 // MARK: custom attribute diagnostics

include/swift/AST/ModuleDependencies.h

@@ -1252,6 +1252,9 @@ class ModuleDependenciesCache {
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,
                         ModuleDependencyInfo dependencyInfo);
+  
+  /// Remove a given dependency info from the cache.
+  void removeDependency(ModuleDependencyID moduleID);
 
   /// Resolve this module's set of directly-imported Swift module
   /// dependencies

include/swift/Basic/Statistics.def

@@ -165,6 +165,9 @@ FRONTEND_STATISTIC(AST, ModuleVisibilityCacheMiss)
 FRONTEND_STATISTIC(AST, ModuleShadowCacheHit)
 FRONTEND_STATISTIC(AST, ModuleShadowCacheMiss)
 
+/// Number of types deserialized.
+FRONTEND_STATISTIC(AST, NumDepScanFilesystemLookups)
+
 /// Number of full function bodies parsed.
 FRONTEND_STATISTIC(Parse, NumFunctionsParsed)
 

include/swift/DependencyScan/ScanDependencies.h

@@ -27,6 +27,10 @@ namespace swift {
 class CompilerInvocation;
 class CompilerInstance;
 class ModuleDependenciesCache;
+struct ModuleDependencyID;
+struct ModuleDependencyIDHash;
+using ModuleDependencyIDSet =
+    std::unordered_set<ModuleDependencyID, ModuleDependencyIDHash>;
 class SwiftDependencyScanningService;
 
 namespace dependencies {
@@ -77,6 +81,36 @@ performBatchModuleScan(CompilerInstance &invocationInstance,
                        CompilerArgInstanceCacheMap *versionedPCMInstanceCache,
                        llvm::StringSaver &saver,
                        const std::vector<BatchScanInput> &BatchInput);
+
+namespace incremental {
+/// For the given module dependency graph captured in the 'cache',
+/// validate whether each dependency node is up-to-date w.r.t. serialization
+/// time-stamp. i.e. if any of the input files of a module dependency are newer
+/// than the serialized dependency graph, it is considered invalidated and must
+/// be re-scanned.
+void validateInterModuleDependenciesCache(
+    const ModuleDependencyID &rootModuleID, ModuleDependenciesCache &cache,
+    const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
+    DiagnosticEngine &diags, bool emitRemarks = false);
+
+/// Perform a postorder DFS to locate modules whose build recipe is out-of-date
+/// with respect to their inputs. Upon encountering such a module, add it to the
+/// set of invalidated modules, along with the path from the root to this
+/// module.
+void outOfDateModuleScan(const ModuleDependencyID &sourceModuleID,
+                         const ModuleDependenciesCache &cache,
+                         const llvm::sys::TimePoint<> &cacheTimeStamp,
+                         llvm::vfs::FileSystem &fs, DiagnosticEngine &diags,
+                         bool emitRemarks, ModuleDependencyIDSet &visited,
+                         ModuleDependencyIDSet &modulesRequiringRescan);
+
+/// Validate whether all inputs of a given module dependency
+/// are older than the cache serialization time.
+bool verifyModuleDependencyUpToDate(
+    const ModuleDependencyID &moduleID, const ModuleDependenciesCache &cache,
+    const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
+    DiagnosticEngine &diags, bool emitRemarks);
+} // end namespace incremental
 } // end namespace dependencies
 } // end namespace swift
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -95,6 +95,7 @@ namespace graph_block {
 enum {
   METADATA = 1,
   MODULE_NODE,
+  TIME_NODE,
   LINK_LIBRARY_NODE,
   LINK_LIBRARY_ARRAY_NODE,
   MACRO_DEPENDENCY_NODE,
@@ -113,13 +114,19 @@ enum {
 
 // Always the first record in the file.
 using MetadataLayout = BCRecordLayout<
-    METADATA,    // ID
-    BCFixed<16>, // Inter-Module Dependency graph format major version
-    BCFixed<16>, // Inter-Module Dependency graph format minor version
-    BCBlob       // Scanner Invocation Context Hash
+    METADATA,       // ID
+    BCFixed<16>,    // Inter-Module Dependency graph format major version
+    BCFixed<16>,    // Inter-Module Dependency graph format minor version
+    BCBlob          // Scanner Invocation Context Hash
     >;
 
-// After the metadata record, we have zero or more identifier records,
+// After the metadata record, emit serialization time-stamp.
+using TimeLayout = BCRecordLayout<
+    TIME_NODE,       // ID
+    BCBlob           // Nanoseconds since epoch as a string
+    >;
+
+// After the time stamp record, we have zero or more identifier records,
 // for each unique string that is referenced in the graph.
 //
 // Identifiers are referenced by their sequence number, starting from 1.
@@ -138,29 +145,31 @@ using IdentifierNodeLayout = BCRecordLayout<IDENTIFIER_NODE, BCBlob>;
 using IdentifierArrayLayout =
     BCRecordLayout<IDENTIFIER_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record for a given link library node containing information
+// required for the build system client to capture a requirement
+// to link a given dependency library.
 using LinkLibraryLayout =
     BCRecordLayout<LINK_LIBRARY_NODE,            // ID
                    IdentifierIDField,            // libraryName
                    IsFrameworkField,             // isFramework
                    IsForceLoadField              // forceLoad
                    >;
-// ACTODO: Comment
 using LinkLibraryArrayLayout =
     BCRecordLayout<LINK_LIBRARY_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record for a Macro module dependency of a given dependency
+// node.
 using MacroDependencyLayout =
     BCRecordLayout<MACRO_DEPENDENCY_NODE,        // ID
                    IdentifierIDField,            // macroModuleName
                    IdentifierIDField,            // libraryPath
                    IdentifierIDField             // executablePath
                    >;
-// ACTODO: Comment
 using MacroDependencyArrayLayout =
     BCRecordLayout<MACRO_DEPENDENCY_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record capturing information about a given 'import' statement
+// captured in a dependency node, including its source location.
 using ImportStatementLayout =
     BCRecordLayout<IMPORT_STATEMENT_NODE,        // ID
                    IdentifierIDField,            // importIdentifier
@@ -169,7 +178,6 @@ using ImportStatementLayout =
                    ColumnNumberField,            // columnNumber
                    IsOptionalImport              // isOptional
                    >;
-// ACTODO: Comment
 using ImportStatementArrayLayout =
     BCRecordLayout<IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;
 using OptionalImportStatementArrayLayout =
@@ -271,12 +279,14 @@ using ClangModuleDetailsLayout =
 /// Tries to read the dependency graph from the given buffer.
 /// Returns \c true if there was an error.
 bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer,
-                                      ModuleDependenciesCache &cache);
+                                      ModuleDependenciesCache &cache,
+                                      llvm::sys::TimePoint<> &serializedCacheTimeStamp);
 
 /// Tries to read the dependency graph from the given path name.
 /// Returns true if there was an error.
 bool readInterModuleDependenciesCache(llvm::StringRef path,
-                                      ModuleDependenciesCache &cache);
+                                      ModuleDependenciesCache &cache,
+                                      llvm::sys::TimePoint<> &serializedCacheTimeStamp);
 
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.

include/swift/Frontend/FrontendOptions.h

@@ -363,6 +363,11 @@ class FrontendOptions {
   /// Load and re-use a prior serialized dependency scanner cache.
   bool ReuseDependencyScannerCache = false;
 
+  /// Upon loading a prior serialized dependency scanner cache, filter out
+  /// dependency module information which is no longer up-to-date with respect
+  /// to input files of every given module.
+  bool ValidatePriorDependencyScannerCache = false;
+
   /// The path at which to either serialize or deserialize the dependency scanner cache.
   std::string SerializedDependencyScannerCachePath;
 

include/swift/Option/FrontendOptions.td

@@ -270,7 +270,10 @@ def serialize_dependency_scan_cache : Flag<["-"], "serialize-dependency-scan-cac
    HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
 
 def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">,
-   HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
+   HelpText<"For performing a dependency scan, deserialize the scanner's internal state from a prior scan.">;
+   
+def validate_prior_dependency_scan_cache : Flag<["-"], "validate-prior-dependency-scan-cache">,
+   HelpText<"For performing a dependency scan with a prior scanner state, validate module dependencies.">;
 
 def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
   HelpText<"The path to output the dependency scanner's internal state.">;

lib/AST/ModuleDependencies.cpp

@@ -807,6 +807,7 @@ ModuleDependenciesCache::findSwiftDependency(StringRef moduleName) const {
 
 const ModuleDependencyInfo &ModuleDependenciesCache::findKnownDependency(
     const ModuleDependencyID &moduleID) const {
+  
   auto dep = findDependency(moduleID);
   assert(dep && "dependency unknown");
   return **dep;
@@ -860,6 +861,11 @@ void ModuleDependenciesCache::updateDependency(
   map.insert_or_assign(moduleID.ModuleName, std::move(dependencyInfo));
 }
 
+void ModuleDependenciesCache::removeDependency(ModuleDependencyID moduleID) {
+  auto &map = getDependenciesMap(moduleID.Kind);
+  map.erase(moduleID.ModuleName);
+}
+
 void
 ModuleDependenciesCache::setImportedSwiftDependencies(ModuleDependencyID moduleID,
                                                       const ArrayRef<ModuleDependencyID> dependencyIDs) {

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -58,6 +58,7 @@ class ModuleDependenciesCacheDeserializer {
   bool readSignature();
   bool enterGraphBlock();
   bool readMetadata(StringRef scannerContextHash);
+  bool readSerializationTime(llvm::sys::TimePoint<> &SerializationTimeStamp);
   bool readGraph(ModuleDependenciesCache &cache);
 
   std::optional<std::string> getIdentifier(unsigned n);
@@ -76,7 +77,8 @@ class ModuleDependenciesCacheDeserializer {
 public:
   ModuleDependenciesCacheDeserializer(llvm::MemoryBufferRef Data)
       : Cursor(Data) {}
-  bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache);
+  bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache,
+                                        llvm::sys::TimePoint<> &serializedCacheTimeStamp);
 };
 
 } // namespace swift
@@ -177,6 +179,35 @@ bool ModuleDependenciesCacheDeserializer::readMetadata(StringRef scannerContextH
   return false;
 }
 
+bool ModuleDependenciesCacheDeserializer::readSerializationTime(llvm::sys::TimePoint<> &SerializationTimeStamp) {
+  using namespace graph_block;
+
+  auto entry = Cursor.advance();
+  if (!entry) {
+    consumeError(entry.takeError());
+    return true;
+  }
+
+  if (entry->Kind != llvm::BitstreamEntry::Record)
+    return true;
+
+  auto recordID = Cursor.readRecord(entry->ID, Scratch, &BlobData);
+  if (!recordID) {
+    consumeError(recordID.takeError());
+    return true;
+  }
+
+  if (*recordID != TIME_NODE)
+    return true;
+  
+  TimeLayout::readRecord(Scratch);
+  std::string serializedTimeStamp = BlobData.str();
+  
+  SerializationTimeStamp =
+    llvm::sys::TimePoint<>(llvm::sys::TimePoint<>::duration(std::stoll(serializedTimeStamp)));
+  return SerializationTimeStamp == llvm::sys::TimePoint<>();
+}
+
 /// Read in the top-level block's graph structure by first reading in
 /// all of the file's identifiers and arrays of identifiers, followed by
 /// consuming individual module info records and registering them into the
@@ -808,7 +839,8 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
 }
 
 bool ModuleDependenciesCacheDeserializer::readInterModuleDependenciesCache(
-    ModuleDependenciesCache &cache) {
+    ModuleDependenciesCache &cache,
+    llvm::sys::TimePoint<> &serializedCacheTimeStamp) {
   using namespace graph_block;
 
   if (readSignature())
@@ -819,6 +851,9 @@ bool ModuleDependenciesCacheDeserializer::readInterModuleDependenciesCache(
 
   if (readMetadata(cache.scannerContextHash))
     return true;
+  
+  if (readSerializationTime(serializedCacheTimeStamp))
+    return true;
 
   if (readGraph(cache))
     return true;
@@ -992,21 +1027,23 @@ ModuleDependenciesCacheDeserializer::getModuleDependencyIDArray(unsigned n) {
 
 bool swift::dependencies::module_dependency_cache_serialization::
     readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer,
-                                     ModuleDependenciesCache &cache) {
+                                     ModuleDependenciesCache &cache,
+                                     llvm::sys::TimePoint<> &serializedCacheTimeStamp) {
   ModuleDependenciesCacheDeserializer deserializer(buffer.getMemBufferRef());
-  return deserializer.readInterModuleDependenciesCache(cache);
+  return deserializer.readInterModuleDependenciesCache(cache, serializedCacheTimeStamp);
 }
 
 bool swift::dependencies::module_dependency_cache_serialization::
     readInterModuleDependenciesCache(StringRef path,
-                                     ModuleDependenciesCache &cache) {
+                                     ModuleDependenciesCache &cache,
+                                     llvm::sys::TimePoint<> &serializedCacheTimeStamp) {
   PrettyStackTraceStringAction stackTrace(
       "loading inter-module dependency graph", path);
   auto buffer = llvm::MemoryBuffer::getFile(path);
   if (!buffer)
     return true;
 
-  return readInterModuleDependenciesCache(*buffer.get(), cache);
+  return readInterModuleDependenciesCache(*buffer.get(), cache, serializedCacheTimeStamp);
 }
 
 // MARK: Serialization
@@ -1126,6 +1163,7 @@ class ModuleDependenciesCacheSerializer {
   void writeBlockInfoBlock();
 
   void writeMetadata(StringRef scanningContextHash);
+  void writeSerializationTime();
   void writeIdentifiers();
   void writeArraysOfIdentifiers();
 
@@ -1189,6 +1227,7 @@ void ModuleDependenciesCacheSerializer::writeBlockInfoBlock() {
 
   BLOCK(GRAPH_BLOCK);
   BLOCK_RECORD(graph_block, METADATA);
+  BLOCK_RECORD(graph_block, TIME_NODE);
   BLOCK_RECORD(graph_block, IDENTIFIER_NODE);
   BLOCK_RECORD(graph_block, IDENTIFIER_ARRAY_NODE);
 
@@ -1221,6 +1260,16 @@ void ModuleDependenciesCacheSerializer::writeMetadata(StringRef scanningContextH
                              scanningContextHash);
 }
 
+void ModuleDependenciesCacheSerializer::writeSerializationTime() {
+  using namespace graph_block;
+  llvm::sys::TimePoint<> now = std::chrono::system_clock::now();
+  auto timeSinceEpoch = now.time_since_epoch().count();
+  std::string serializationData = std::to_string(timeSinceEpoch);
+  TimeLayout::emitRecord(Out, ScratchRecord,
+                         AbbrCodes[TimeLayout::Code],
+                         serializationData);
+}
+
 void ModuleDependenciesCacheSerializer::writeIdentifiers() {
   using namespace graph_block;
   for (auto str : Identifiers) {
@@ -1795,6 +1844,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->sourceInfoPath);
         addIdentifier(swiftBinDeps->moduleCacheKey);
         addIdentifier(swiftBinDeps->headerImport);
+        addIdentifier(swiftBinDeps->definingModuleInterfacePath);
         addIdentifier(swiftBinDeps->userModuleVersion);
         addStringArray(moduleID,
                        ModuleIdentifierArrayKind::HeaderInputModuleDependencies,
@@ -1877,6 +1927,7 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   using namespace graph_block;
 
   registerRecordAbbr<MetadataLayout>();
+  registerRecordAbbr<TimeLayout>();
   registerRecordAbbr<IdentifierNodeLayout>();
   registerRecordAbbr<IdentifierArrayLayout>();
   registerRecordAbbr<LinkLibraryLayout>();
@@ -1899,6 +1950,9 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache(
   // Write the version information
   writeMetadata(cache.scannerContextHash);
 
+  // The current time-stamp
+  writeSerializationTime();
+
   // Write the strings
   writeIdentifiers();