[Dependency Scanning] Add functionality to validate contents of a loaded scanner cache state

Checking each module dependency info if it is up-to-date with respect to when the cache contents were serialized in a prior scan.

- Add a timestamp field to the serialization format for the dependency scanner cache
- Add a flag "-validate-prior-dependency-scan-cache" which, when combined with "-load-dependency-scan-cache" will have the scanner prune dependencies from the deserialized cache which have inputs that are newer than the prior scan itself

With the above in-place, the scan otherwise proceeds as-is, getting cache hits for entries still valid since the prior scan.

Author: artemcm

include/swift/AST/DiagnosticsCommon.def

@@ -212,17 +212,28 @@ ERROR(scanner_arguments_invalid, none,
 ERROR(error_scanner_extra, none,
       "failed inside dependency scanner: '%0'", (StringRef))
 
-WARNING(warn_scanner_deserialize_failed, none,
-        "Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
+REMARK(warn_scanner_deserialize_failed, none,
+       "Incremental module scan: Failed to load module scanning dependency cache from: '%0', re-building scanner cache from scratch.", (StringRef))
 
 REMARK(remark_reuse_cache, none,
-       "Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
+       "Incremental module scan: Re-using serialized module scanning dependency cache from: '%0'.", (StringRef))
 
 REMARK(remark_scanner_uncached_lookups, none,
        "Module Dependency Scanner queries executed: '%0'.", (unsigned))
 
 REMARK(remark_save_cache, none,
-       "Serializing module scanning dependency cache to: '%0'.", (StringRef))
+       "Incremental module scan: Serializing module scanning dependency cache to: '%0'.", (StringRef))
+
+REMARK(remark_scanner_stale_result_invalidate, none,
+       "Incremental module scan: Dependency info for module '%0' invalidated due to a modified input"
+       " since last scan: '%1'.", (StringRef, StringRef))
+
+REMARK(remark_scanner_invalidate_upstream, none,
+       "Incremental module scan: Dependency info for module '%0' invalidated due to an out-of-date"
+       " dependency.", (StringRef))
+
+REMARK(remark_cache_reuse_disabled_with_cas, none,
+       "Incremental module scan: Re-using serialized module scanning dependency cache disabled in Caching build", ())
 
 //------------------------------------------------------------------------------
 // MARK: custom attribute diagnostics

include/swift/AST/ModuleDependencies.h

@@ -1129,6 +1129,9 @@ class ModuleDependenciesCache {
   std::string scannerContextHash;
   /// The location of where the built modules will be output to
   std::string moduleOutputPath;
+  /// The timestamp of the beginning of the scanning query action
+  /// using this cache
+  const llvm::sys::TimePoint<> scanInitializationTime;
 
   /// Retrieve the dependencies map that corresponds to the given dependency
   /// kind.
@@ -1235,6 +1238,9 @@ class ModuleDependenciesCache {
   /// Update stored dependencies for the given module.
   void updateDependency(ModuleDependencyID moduleID,
                         ModuleDependencyInfo dependencyInfo);
+  
+  /// Remove a given dependency info from the cache.
+  void removeDependency(ModuleDependencyID moduleID);
 
   /// Resolve this module's set of directly-imported Swift module
   /// dependencies

include/swift/Basic/Statistics.def

@@ -165,6 +165,9 @@ FRONTEND_STATISTIC(AST, ModuleVisibilityCacheMiss)
 FRONTEND_STATISTIC(AST, ModuleShadowCacheHit)
 FRONTEND_STATISTIC(AST, ModuleShadowCacheMiss)
 
+/// Number of types deserialized.
+FRONTEND_STATISTIC(AST, NumDepScanFilesystemLookups)
+
 /// Number of full function bodies parsed.
 FRONTEND_STATISTIC(Parse, NumFunctionsParsed)
 

include/swift/DependencyScan/ScanDependencies.h

@@ -27,6 +27,10 @@ namespace swift {
 class CompilerInvocation;
 class CompilerInstance;
 class ModuleDependenciesCache;
+struct ModuleDependencyID;
+struct ModuleDependencyIDHash;
+using ModuleDependencyIDSet =
+    std::unordered_set<ModuleDependencyID, ModuleDependencyIDHash>;
 class SwiftDependencyScanningService;
 
 namespace dependencies {
@@ -58,6 +62,35 @@ performModulePrescan(CompilerInstance &instance,
                      DependencyScanDiagnosticCollector *diagnostics,
                      ModuleDependenciesCache &cache);
 
+namespace incremental {
+/// For the given module dependency graph captured in the 'cache',
+/// validate whether each dependency node is up-to-date w.r.t. serialization
+/// time-stamp. i.e. if any of the input files of a module dependency are newer
+/// than the serialized dependency graph, it is considered invalidated and must
+/// be re-scanned.
+void validateInterModuleDependenciesCache(
+    const ModuleDependencyID &rootModuleID, ModuleDependenciesCache &cache,
+    const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
+    DiagnosticEngine &diags, bool emitRemarks = false);
+
+/// Perform a postorder DFS to locate modules whose build recipe is out-of-date
+/// with respect to their inputs. Upon encountering such a module, add it to the
+/// set of invalidated modules, along with the path from the root to this
+/// module.
+void outOfDateModuleScan(const ModuleDependencyID &sourceModuleID,
+                         const ModuleDependenciesCache &cache,
+                         const llvm::sys::TimePoint<> &cacheTimeStamp,
+                         llvm::vfs::FileSystem &fs, DiagnosticEngine &diags,
+                         bool emitRemarks, ModuleDependencyIDSet &visited,
+                         ModuleDependencyIDSet &modulesRequiringRescan);
+
+/// Validate whether all inputs of a given module dependency
+/// are older than the cache serialization time.
+bool verifyModuleDependencyUpToDate(
+    const ModuleDependencyID &moduleID, const ModuleDependenciesCache &cache,
+    const llvm::sys::TimePoint<> &cacheTimeStamp, llvm::vfs::FileSystem &fs,
+    DiagnosticEngine &diags, bool emitRemarks);
+} // end namespace incremental
 } // end namespace dependencies
 } // end namespace swift
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -95,6 +95,7 @@ namespace graph_block {
 enum {
   METADATA = 1,
   MODULE_NODE,
+  TIME_NODE,
   LINK_LIBRARY_NODE,
   LINK_LIBRARY_ARRAY_NODE,
   MACRO_DEPENDENCY_NODE,
@@ -113,13 +114,19 @@ enum {
 
 // Always the first record in the file.
 using MetadataLayout = BCRecordLayout<
-    METADATA,    // ID
-    BCFixed<16>, // Inter-Module Dependency graph format major version
-    BCFixed<16>, // Inter-Module Dependency graph format minor version
-    BCBlob       // Scanner Invocation Context Hash
+    METADATA,       // ID
+    BCFixed<16>,    // Inter-Module Dependency graph format major version
+    BCFixed<16>,    // Inter-Module Dependency graph format minor version
+    BCBlob          // Scanner Invocation Context Hash
     >;
 
-// After the metadata record, we have zero or more identifier records,
+// After the metadata record, emit serialization time-stamp.
+using TimeLayout = BCRecordLayout<
+    TIME_NODE,       // ID
+    BCBlob           // Nanoseconds since epoch as a string
+    >;
+
+// After the time stamp record, we have zero or more identifier records,
 // for each unique string that is referenced in the graph.
 //
 // Identifiers are referenced by their sequence number, starting from 1.
@@ -138,29 +145,31 @@ using IdentifierNodeLayout = BCRecordLayout<IDENTIFIER_NODE, BCBlob>;
 using IdentifierArrayLayout =
     BCRecordLayout<IDENTIFIER_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record for a given link library node containing information
+// required for the build system client to capture a requirement
+// to link a given dependency library.
 using LinkLibraryLayout =
     BCRecordLayout<LINK_LIBRARY_NODE,            // ID
                    IdentifierIDField,            // libraryName
                    IsFrameworkField,             // isFramework
                    IsForceLoadField              // forceLoad
                    >;
-// ACTODO: Comment
 using LinkLibraryArrayLayout =
     BCRecordLayout<LINK_LIBRARY_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record for a Macro module dependency of a given dependency
+// node.
 using MacroDependencyLayout =
     BCRecordLayout<MACRO_DEPENDENCY_NODE,        // ID
                    IdentifierIDField,            // macroModuleName
                    IdentifierIDField,            // libraryPath
                    IdentifierIDField             // executablePath
                    >;
-// ACTODO: Comment
 using MacroDependencyArrayLayout =
     BCRecordLayout<MACRO_DEPENDENCY_ARRAY_NODE, IdentifierIDArryField>;
 
-// ACTODO: Comment
+// A record capturing information about a given 'import' statement
+// captured in a dependency node, including its source location.
 using ImportStatementLayout =
     BCRecordLayout<IMPORT_STATEMENT_NODE,        // ID
                    IdentifierIDField,            // importIdentifier
@@ -169,7 +178,6 @@ using ImportStatementLayout =
                    ColumnNumberField,            // columnNumber
                    IsOptionalImport              // isOptional
                    >;
-// ACTODO: Comment
 using ImportStatementArrayLayout =
     BCRecordLayout<IMPORT_STATEMENT_ARRAY_NODE, IdentifierIDArryField>;
 using OptionalImportStatementArrayLayout =
@@ -268,12 +276,14 @@ using ClangModuleDetailsLayout =
 /// Tries to read the dependency graph from the given buffer.
 /// Returns \c true if there was an error.
 bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer,
-                                      ModuleDependenciesCache &cache);
+                                      ModuleDependenciesCache &cache,
+                                      llvm::sys::TimePoint<> &serializedCacheTimeStamp);
 
 /// Tries to read the dependency graph from the given path name.
 /// Returns true if there was an error.
 bool readInterModuleDependenciesCache(llvm::StringRef path,
-                                      ModuleDependenciesCache &cache);
+                                      ModuleDependenciesCache &cache,
+                                      llvm::sys::TimePoint<> &serializedCacheTimeStamp);
 
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.

include/swift/Frontend/FrontendOptions.h

@@ -363,6 +363,11 @@ class FrontendOptions {
   /// Load and re-use a prior serialized dependency scanner cache.
   bool ReuseDependencyScannerCache = false;
 
+  /// Upon loading a prior serialized dependency scanner cache, filter out
+  /// dependency module information which is no longer up-to-date with respect
+  /// to input files of every given module.
+  bool ValidatePriorDependencyScannerCache = false;
+
   /// The path at which to either serialize or deserialize the dependency scanner cache.
   std::string SerializedDependencyScannerCachePath;
 

include/swift/Option/FrontendOptions.td

@@ -266,7 +266,10 @@ def serialize_dependency_scan_cache : Flag<["-"], "serialize-dependency-scan-cac
    HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
 
 def reuse_dependency_scan_cache : Flag<["-"], "load-dependency-scan-cache">,
-   HelpText<"After performing a dependency scan, serialize the scanner's internal state.">;
+   HelpText<"For performing a dependency scan, deserialize the scanner's internal state from a prior scan.">;
+   
+def validate_prior_dependency_scan_cache : Flag<["-"], "validate-prior-dependency-scan-cache">,
+   HelpText<"For performing a dependency scan with a prior scanner state, validate module dependencies.">;
 
 def dependency_scan_cache_path : Separate<["-"], "dependency-scan-cache-path">,
   HelpText<"The path to output the dependency scanner's internal state.">;

include/swift/Option/Options.td

@@ -1979,6 +1979,11 @@ def driver_explicit_module_build:
   Flags<[HelpHidden, NewDriverOnlyOption]>,
   HelpText<"Prebuild module dependencies to make them explicit">;
 
+def incremental_dependency_scan:
+  Flag<["-"], "incremental-dependency-scan">,
+  Flags<[HelpHidden, NewDriverOnlyOption]>,
+  HelpText<"Re-use/validate prior build dependency scan artifacts">;
+
 def driver_experimental_explicit_module_build:
   Flag<["-"], "experimental-explicit-module-build">,
   Flags<[HelpHidden, NewDriverOnlyOption]>,

lib/AST/ModuleDependencies.cpp

@@ -728,7 +728,8 @@ ModuleDependenciesCache::ModuleDependenciesCache(
     : globalScanningService(globalScanningService),
       mainScanModuleName(mainScanModuleName),
       scannerContextHash(scannerContextHash),
-      moduleOutputPath(moduleOutputPath) {
+      moduleOutputPath(moduleOutputPath),
+      scanInitializationTime(std::chrono::system_clock::now()) {
   for (auto kind = ModuleDependencyKind::FirstKind;
        kind != ModuleDependencyKind::LastKind; ++kind)
     ModuleDependenciesMap.insert({kind, ModuleNameToDependencyMap()});
@@ -807,6 +808,7 @@ ModuleDependenciesCache::findSwiftDependency(StringRef moduleName) const {
 
 const ModuleDependencyInfo &ModuleDependenciesCache::findKnownDependency(
     const ModuleDependencyID &moduleID) const {
+  
   auto dep = findDependency(moduleID);
   assert(dep && "dependency unknown");
   return **dep;
@@ -860,6 +862,11 @@ void ModuleDependenciesCache::updateDependency(
   map.insert_or_assign(moduleID.ModuleName, std::move(dependencyInfo));
 }
 
+void ModuleDependenciesCache::removeDependency(ModuleDependencyID moduleID) {
+  auto &map = getDependenciesMap(moduleID.Kind);
+  map.erase(moduleID.ModuleName);
+}
+
 void
 ModuleDependenciesCache::setImportedSwiftDependencies(ModuleDependencyID moduleID,
                                                       const ArrayRef<ModuleDependencyID> dependencyIDs) {