Merge pull request #1453 from cachemeifyoucan/eng/PR-swift-driver-path-remap-and-replay

[SwiftDriver] PathRemapping and Cache Replay Support

Author: cachemeifyoucan

README.md

@@ -156,7 +156,7 @@ $ apt-get install libncurses-dev
 be found, e.g.:
 
 ```
-$ swift build -Xcc -Xcc -I/path/to/build/Ninja-Release/swift-.../include -Xcc -I/path/to/build/Ninja-Release/llvm-.../include -Xcc -I/path/to/source/llvm-project/llvm/include --product makeOptions
+$ swift build -Xcc -I/path/to/build/Ninja-Release/swift-.../include -Xcc -I/path/to/build/Ninja-Release/llvm-.../include -Xcc -I/path/to/source/llvm-project/llvm/include --product makeOptions
 ```
 
 Then, run `makeOptions` and redirect the output to overwrite `Options.swift`:

Sources/CSwiftScan/include/swiftscan_header.h

@@ -18,7 +18,7 @@
 #include <stdint.h>
 
 #define SWIFTSCAN_VERSION_MAJOR 0
-#define SWIFTSCAN_VERSION_MINOR 5
+#define SWIFTSCAN_VERSION_MINOR 6
 
 //=== Public Scanner Data Types -------------------------------------------===//
 
@@ -78,17 +78,15 @@ typedef struct swiftscan_scan_invocation_s *swiftscan_scan_invocation_t;
 typedef void *swiftscan_scanner_t;
 
 //=== CAS/Caching Specification -------------------------------------------===//
-typedef struct swiftscan_cas_s *swiftscan_cas_t;
 typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t;
-
-typedef enum {
-  SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
-  SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
-  SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
-  SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3,
-  SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
-  SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
-} swiftscan_output_kind_t;
+typedef struct swiftscan_cas_s *swiftscan_cas_t;
+typedef struct swiftscan_cached_compilation_s *swiftscan_cached_compilation_t;
+typedef struct swiftscan_cached_output_s *swiftscan_cached_output_t;
+typedef struct swiftscan_cache_replay_instance_s
+    *swiftscan_cache_replay_instance_t;
+typedef struct swiftscan_cache_replay_result_s *swiftscan_cache_replay_result_t;
+typedef struct swiftscan_cache_cancellation_token_s
+    *swiftscan_cache_cancellation_token_t;
 
 //=== libSwiftScan Functions ------------------------------------------------===//
 
@@ -281,18 +279,74 @@ typedef struct {
                                                 const char *path);
   void (*swiftscan_cas_options_set_plugin_path)(swiftscan_cas_options_t options,
                                                 const char *path);
-  bool (*swiftscan_cas_options_set_option)(swiftscan_cas_options_t options,
-                                           const char *name, const char *value,
-                                           swiftscan_string_ref_t *error);
+  bool (*swiftscan_cas_options_set_plugin_option)(
+      swiftscan_cas_options_t options, const char *name, const char *value,
+      swiftscan_string_ref_t *error);
   swiftscan_cas_t (*swiftscan_cas_create_from_options)(
       swiftscan_cas_options_t options, swiftscan_string_ref_t *error);
   void (*swiftscan_cas_dispose)(swiftscan_cas_t cas);
   swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas,
                                                 uint8_t *data, unsigned size,
                                                 swiftscan_string_ref_t *error);
-  swiftscan_string_ref_t (*swiftscan_compute_cache_key)(
-      swiftscan_cas_t cas, int argc, const char *argv, const char *input,
-      swiftscan_output_kind_t, swiftscan_string_ref_t *error);
+  swiftscan_string_ref_t (*swiftscan_cache_compute_key)(
+      swiftscan_cas_t cas, int argc, const char **argv, const char *input,
+      swiftscan_string_ref_t *error);
+
+  //=== Scanner Caching Query/Replay Operations -----------------------------===//
+  swiftscan_cached_compilation_t (*swiftscan_cache_query)(
+      swiftscan_cas_t cas, const char *key, bool globally,
+      swiftscan_string_ref_t *error);
+  void (*swiftscan_cache_query_async)(
+      swiftscan_cas_t cas, const char *key, bool globally, void *ctx,
+      void (*callback)(void *ctx, swiftscan_cached_compilation_t,
+                       swiftscan_string_ref_t error),
+      swiftscan_cache_cancellation_token_t *);
+
+
+  unsigned (*swiftscan_cached_compilation_get_num_outputs)(
+      swiftscan_cached_compilation_t);
+  swiftscan_cached_output_t (*swiftscan_cached_compilation_get_output)(
+      swiftscan_cached_compilation_t, unsigned idx);
+  bool (*swiftscan_cached_compilation_is_uncacheable)(
+      swiftscan_cached_compilation_t);
+  void (*swiftscan_cached_compilation_make_global_async)(
+      swiftscan_cached_compilation_t, void *ctx,
+      void (*callback)(void *ctx, swiftscan_string_ref_t error),
+      swiftscan_cache_cancellation_token_t *);
+  void (*swiftscan_cached_compilation_dispose)(swiftscan_cached_compilation_t);
+
+  bool (*swiftscan_cached_output_load)(swiftscan_cached_output_t,
+                                       swiftscan_string_ref_t *error);
+  void (*swiftscan_cached_output_load_async)(
+      swiftscan_cached_output_t, void *ctx,
+      void (*callback)(void *ctx, bool success, swiftscan_string_ref_t error),
+      swiftscan_cache_cancellation_token_t *);
+  bool (*swiftscan_cached_output_is_materialized)(swiftscan_cached_output_t);
+  swiftscan_string_ref_t (*swiftscan_cached_output_get_casid)(
+      swiftscan_cached_output_t);
+  swiftscan_string_ref_t (*swiftscan_cached_output_get_name)(
+      swiftscan_cached_output_t);
+  void (*swiftscan_cached_output_dispose)(swiftscan_cached_output_t);
+
+  void (*swiftscan_cache_action_cancel)(swiftscan_cache_cancellation_token_t);
+  void (*swiftscan_cache_cancellation_token_dispose)(
+      swiftscan_cache_cancellation_token_t);
+
+  swiftscan_cache_replay_instance_t (*swiftscan_cache_replay_instance_create)(
+      int argc, const char **argv, swiftscan_string_ref_t *error);
+  void (*swiftscan_cache_replay_instance_dispose)(
+      swiftscan_cache_replay_instance_t);
+
+  swiftscan_cache_replay_result_t (*swiftscan_cache_replay_compilation)(
+      swiftscan_cache_replay_instance_t, swiftscan_cached_compilation_t,
+      swiftscan_string_ref_t *error);
+
+  swiftscan_string_ref_t (*swiftscan_cache_replay_result_get_stdout)(
+      swiftscan_cache_replay_result_t);
+  swiftscan_string_ref_t (*swiftscan_cache_replay_result_get_stderr)(
+      swiftscan_cache_replay_result_t);
+  void (*swiftscan_cache_replay_result_dispose)(
+      swiftscan_cache_replay_result_t);
 
 } swiftscan_functions_t;
 

Sources/SwiftDriver/CMakeLists.txt

@@ -18,6 +18,7 @@ add_library(SwiftDriver
   SwiftScan/DependencyGraphBuilder.swift
   SwiftScan/Loader.swift
   SwiftScan/SwiftScan.swift
+  SwiftScan/SwiftScanCAS.swift
 
   Driver/CompilerMode.swift
   Driver/DebugInfo.swift

Sources/SwiftDriver/Driver/Driver.swift

@@ -224,8 +224,10 @@ public struct Driver {
   /// Should use file lists for inputs (number of inputs exceeds `fileListThreshold`).
   let shouldUseInputFileList: Bool
 
-  /// VirtualPath for shared all sources file list. `nil` if unused.
-  @_spi(Testing) public let allSourcesFileList: VirtualPath?
+  /// VirtualPath for shared all sources file list. `nil` if unused. This is used as a cache for
+  /// the file list computed during CompileJob creation and only holds valid to be query by tests
+  /// after planning to build.
+  @_spi(Testing) public var allSourcesFileList: VirtualPath? = nil
 
   /// The mode in which the compiler will execute.
   @_spi(Testing) public let compilerMode: CompilerMode
@@ -272,6 +274,43 @@ public struct Driver {
   let enableCaching: Bool
   let useClangIncludeTree: Bool
 
+  /// CAS instance used for compilation.
+  public var cas: SwiftScanCAS? = nil
+
+  /// Is swift caching enabled.
+  lazy var isCachingEnabled: Bool = {
+    return enableCaching && isFeatureSupported(.cache_compile_job)
+  }()
+
+  /// Scanner prefix mapping.
+  let scannerPrefixMap: [AbsolutePath: AbsolutePath]
+  let scannerPrefixMapSDK: AbsolutePath?
+  let scannerPrefixMapToolchain: AbsolutePath?
+  lazy var prefixMapping: [(AbsolutePath, AbsolutePath)] = {
+    var mapping: [(AbsolutePath, AbsolutePath)] = scannerPrefixMap.map {
+      return ($0.key, $0.value)
+    }
+    do {
+      guard isFrontendArgSupported(.scannerPrefixMap) else {
+        return []
+      }
+      if let sdkMapping = scannerPrefixMapSDK,
+         let sdkPath = absoluteSDKPath {
+        mapping.append((sdkPath, sdkMapping))
+      }
+      if let toolchainMapping = scannerPrefixMapToolchain {
+        let toolchainPath = try toolchain.executableDir.parentDirectory // usr
+                                                       .parentDirectory // toolchain
+        mapping.append((toolchainPath, toolchainMapping))
+      }
+      // The mapping needs to be sorted so the mapping is determinisitic.
+      // The sorting order is reversed so /tmp/tmp is preferred over /tmp in remapping.
+      return mapping.sorted { $0.0 > $1.0 }
+    } catch {
+      return mapping.sorted { $0.0 > $1.0 }
+    }
+  }()
+
   /// Code & data for incremental compilation. Nil if not running in incremental mode.
   /// Set during planning because needs the jobs to look at outputs.
   @_spi(Testing) public private(set) var incrementalCompilationState: IncrementalCompilationState? = nil
@@ -385,6 +424,7 @@ public struct Driver {
   @_spi(Testing)
   public enum KnownCompilerFeature: String {
     case emit_abi_descriptor = "emit-abi-descriptor"
+    case cache_compile_job = "cache-compile-job"
   }
 
   lazy var sdkPath: VirtualPath? = {
@@ -597,7 +637,18 @@ public struct Driver {
 
     let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING")
     self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride
-    self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE")
+    self.useClangIncludeTree = !parsedOptions.hasArgument(.noClangIncludeTree) && !env.keys.contains("SWIFT_CACHING_USE_CLANG_CAS_FS")
+    self.scannerPrefixMap = try Self.computeScanningPrefixMapper(&parsedOptions)
+    if let sdkMapping =  parsedOptions.getLastArgument(.scannerPrefixMapSdk)?.asSingle {
+      self.scannerPrefixMapSDK = try AbsolutePath(validating: sdkMapping)
+    } else {
+      self.scannerPrefixMapSDK = nil
+    }
+    if let toolchainMapping = parsedOptions.getLastArgument(.scannerPrefixMapToolchain)?.asSingle {
+      self.scannerPrefixMapToolchain = try AbsolutePath(validating: toolchainMapping)
+    } else {
+      self.scannerPrefixMapToolchain = nil
+    }
 
     // Compute the working directory.
     workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in
@@ -678,13 +729,6 @@ public struct Driver {
 
     self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)
     self.shouldUseInputFileList = inputFiles.count > fileListThreshold
-    if shouldUseInputFileList {
-      let swiftInputs = inputFiles.filter(\.type.isPartOfSwiftCompilation)
-      self.allSourcesFileList = try VirtualPath.createUniqueFilelist(RelativePath(validating: "sources"),
-                                                                     .list(swiftInputs.map(\.file)))
-    } else {
-      self.allSourcesFileList = nil
-    }
 
     self.lto = Self.ltoKind(&parsedOptions, diagnosticsEngine: diagnosticsEngine)
     // Figure out the primary outputs from the driver.
@@ -3524,4 +3568,18 @@ extension Driver {
     }
     return options
   }
+
+  static func computeScanningPrefixMapper(_ parsedOptions: inout ParsedOptions) throws -> [AbsolutePath: AbsolutePath] {
+    var mapping: [AbsolutePath: AbsolutePath] = [:]
+    for opt in parsedOptions.arguments(for: .scannerPrefixMap) {
+      let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
+      if pluginArg.count != 2 {
+        throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle)
+      }
+      let key = try AbsolutePath(validating: String(pluginArg[0]))
+      let value = try AbsolutePath(validating: String(pluginArg[1]))
+      mapping[key] = value
+    }
+    return mapping
+  }
 }

Sources/SwiftDriver/Execution/ArgsResolver.swift

@@ -65,12 +65,16 @@ public final class ArgsResolver {
   public func resolveArgumentList(for job: Job, useResponseFiles: ResponseFileHandling = .heuristic)
   throws -> ([String], usingResponseFile: Bool) {
     let tool = try resolve(.path(job.tool))
-    var arguments = [tool] + (try job.commandLine.map { try resolve($0) })
+    var arguments = [tool] + (try resolveArgumentList(for: job.commandLine))
     let usingResponseFile = try createResponseFileIfNeeded(for: job, resolvedArguments: &arguments,
                                                            useResponseFiles: useResponseFiles)
     return (arguments, usingResponseFile)
   }
 
+  public func resolveArgumentList(for commandLine: [Job.ArgTemplate]) throws -> [String] {
+    return try commandLine.map { try resolve($0) }
+  }
+
   @available(*, deprecated, message: "use resolveArgumentList(for:,useResponseFiles:,quotePaths:)")
   public func resolveArgumentList(for job: Job, forceResponseFiles: Bool,
                                   quotePaths: Bool = false) throws -> [String] {