[Explicit Module Builds] Add libSwiftScan API for scanner cache serialization/deserialization

swiftlang/swift#37723 added API to libSwiftScan to save and restore the dependency scanner's state on the filesystem.
This PR adds this API to the corresponding client code in the driver.

Author: artemcm

Package.resolved

@@ -200,17 +200,20 @@ typedef struct {
   //=== Scanner Functions ---------------------------------------------------===//  
   swiftscan_scanner_t (*swiftscan_scanner_create)(void);
   void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
-
   swiftscan_dependency_graph_t
   (*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
-
   swiftscan_batch_scan_result_t *
   (*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
                                         swiftscan_batch_scan_input_t *,
                                         swiftscan_scan_invocation_t);
-
   swiftscan_import_set_t
   (*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
+
+  //=== Scanner Cache Functions ---------------------------------------------===//
+  void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
+  bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
+  bool (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
+
 } swiftscan_functions_t;
 
 #endif // SWIFT_C_DEPENDENCY_SCAN_H

Sources/CSwiftScan/include/swiftscan_header.h

@@ -73,6 +73,33 @@ public class InterModuleDependencyOracle {
     }
   }
 
+  @_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to serialize scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      swiftScan.serializeScannerCache(to: path)
+    }
+  }
+
+  @_spi(Testing) public func loadScannerCache(from path: AbsolutePath) {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to load scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      swiftScan.loadScannerCache(from: path)
+    }
+  }
+
+  @_spi(Testing) public func resetScannerCache() {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to reset scanner cache with no scanner instance.")
+    }
+    if swiftScan.canLoadStoreScannerCache() {
+      swiftScan.resetScannerCache()
+    }
+  }
+
   private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
 
   /// Queue to sunchronize accesses to the scanner

Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

@@ -93,7 +93,7 @@ extension Driver {
          .privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
          .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
          .pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
-         .indexUnitOutputPath, nil:
+         .indexUnitOutputPath, .modDepCache, nil:
       return false
     }
   }
@@ -444,7 +444,7 @@ extension FileType {
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
          .yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
          .privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -181,6 +181,26 @@ internal final class SwiftScan {
     return resultGraphMap
   }
 
+  @_spi(Testing) public func canLoadStoreScannerCache() -> Bool {
+    return api.swiftscan_scanner_cache_load != nil &&
+           api.swiftscan_scanner_cache_serialize != nil &&
+           api.swiftscan_scanner_cache_reset != nil
+  }
+
+  func serializeScannerCache(to path: AbsolutePath) {
+    api.swiftscan_scanner_cache_serialize(scanner,
+                                          path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func loadScannerCache(from path: AbsolutePath) {
+    api.swiftscan_scanner_cache_load(scanner,
+                                     path.description.cString(using: String.Encoding.utf8))
+  }
+
+  func resetScannerCache() {
+    api.swiftscan_scanner_cache_reset(scanner)
+  }
+
   @_spi(Testing) public func canQuerySupportedArguments() -> Bool {
     return api.swiftscan_compiler_supported_arguments_query != nil &&
            api.swiftscan_string_set_dispose != nil
@@ -226,13 +246,22 @@ private extension swiftscan_functions_t {
       }
       return sym
     }
+    // Supported features/flags query
     self.swiftscan_string_set_dispose =
       try loadOptional("swiftscan_string_set_dispose")
     self.swiftscan_compiler_supported_arguments_query =
       try loadOptional("swiftscan_compiler_supported_arguments_query")
     self.swiftscan_compiler_supported_features_query =
       try loadOptional("swiftscan_compiler_supported_features_query")
 
+    // Dependency scanner serialization/deserialization features
+    self.swiftscan_scanner_cache_serialize =
+      try loadOptional("swiftscan_scanner_cache_serialize")
+    self.swiftscan_scanner_cache_load =
+      try loadOptional("swiftscan_scanner_cache_load")
+    self.swiftscan_scanner_cache_reset =
+      try loadOptional("swiftscan_scanner_cache_reset")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = dlsym(swiftscan, symbol: symbol) else {

Sources/SwiftDriver/SwiftScan/SwiftScan.swift

@@ -15,7 +15,7 @@
 /// The raw values for these enumerations describe the default extension for
 /// the file type.
 public enum FileType: String, Hashable, CaseIterable, Codable {
-  /// Swift source file.
+  /// Swift source file
   case swift
 
   /// (Canonical) SIL source file
@@ -27,34 +27,34 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// AST dump
   case ast
 
-  /// An executable image.
+  /// An executable image
   case image = "out"
 
-  /// An object file.
+  /// An object file
   case object = "o"
 
-  /// A dSYM directory.
+  /// A dSYM directory
   case dSYM
 
-  /// A file containing make-style dependencies.
+  /// A file containing make-style dependencies
   case dependencies = "d"
 
   /// An autolink input file
   case autolink
 
-  /// A compiled Swift module file.
+  /// A compiled Swift module file
   case swiftModule = "swiftmodule"
 
-  /// Swift documentation for a module.
+  /// Swift documentation for a module
   case swiftDocumentation = "swiftdoc"
 
-  /// A textual Swift interface file.
+  /// A textual Swift interface file
   case swiftInterface = "swiftinterface"
 
-  /// An SPI Swift Interface file.
+  /// An SPI Swift Interface file
   case privateSwiftInterface = "private.swiftinterface"
 
-  /// Serialized source information.
+  /// Serialized source information
   case swiftSourceInfoFile = "swiftsourceinfo"
 
   /// Assembler source.
@@ -78,16 +78,19 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Objective-C header
   case objcHeader = "h"
 
-  /// Swift dependencies file.
+  /// Swift dependencies file
   case swiftDeps = "swiftdeps"
 
+  /// Serialized dependency scanner state
+  case modDepCache = "moddepcache"
+
   /// Remapping file
   case remap
 
   /// Imported modules.
   case importedModules = "importedmodules"
 
-  /// Text-based dylib (TBD) file.
+  /// Text-based dylib (TBD) file
   case tbd
 
   /// JSON-based Module Dependency Scanner output
@@ -102,27 +105,27 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// JSON-based binary Swift module artifact description
   case jsonSwiftArtifacts = "artifacts.json"
 
-  /// Module trace file.
+  /// Module trace file
   ///
   /// Module traces are used by Apple's internal build infrastructure. Apple
   /// engineers can see more details on the "Swift module traces" page in the
   /// Swift section of the internal wiki.
   case moduleTrace = "trace.json"
 
-  /// Indexing data directory.
+  /// Indexing data directory
   ///
-  /// The extension isn't real.
+  /// The extension isn't real, rather this FileType specifies a directory path.
   case indexData
 
   /// Output path to record in the indexing data store
   ///
   /// This is only needed for use as a key in the output file map.
   case indexUnitOutputPath
 
-  /// Optimization record.
+  /// Optimization record
   case yamlOptimizationRecord = "opt.yaml"
 
-  /// Bitstream optimization record.
+  /// Bitstream optimization record
   case bitstreamOptimizationRecord = "opt.bitstream"
 
   /// Clang compiler module file
@@ -169,6 +172,9 @@ extension FileType: CustomStringConvertible {
     case .swiftDeps:
       return "swift-dependencies"
 
+    case .modDepCache:
+        return "dependency-scanner-cache"
+
     case .jsonDependencies:
       return "json-dependencies"
 
@@ -218,7 +224,7 @@ extension FileType {
          .swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
          .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
          .clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       return false
     }
   }
@@ -289,6 +295,8 @@ extension FileType {
       return "objc-header"
     case .swiftDeps:
       return "swift-dependencies"
+    case .modDepCache:
+      return "dependency-scanner-cache"
     case .jsonDependencies:
       return "json-dependencies"
     case .jsonTargetInfo:
@@ -327,7 +335,7 @@ extension FileType {
     case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
          .swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
          .pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
-         .indexUnitOutputPath:
+         .indexUnitOutputPath, .modDepCache:
       return false
     }
   }
@@ -341,7 +349,7 @@ extension FileType {
     case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .autolink,
          .swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface,
          .swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
-         .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
+         .importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache,
          .bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
          .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
       return false