Skip to content

Commit 03054e2

Browse files
cachemeifyoucancachemeifyoucan
committed
[Caching] Direct cache replay support
Teach swift-driver to replay cached compilation result using libSwiftScan APIs directly.
1 parent b6386b0 commit 03054e2

19 files changed

+711
-135
lines changed

‎Sources/CSwiftScan/include/swiftscan_header.h

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
#include <stdint.h>
1919

2020
#define SWIFTSCAN_VERSION_MAJOR 0
21-
#define SWIFTSCAN_VERSION_MINOR 5
21+
#define SWIFTSCAN_VERSION_MINOR 6
2222

2323
//=== Public Scanner Data Types -------------------------------------------===//
2424

@@ -79,17 +79,40 @@ typedef void *swiftscan_scanner_t;
7979

8080
//=== CAS/Caching Specification -------------------------------------------===//
8181
typedef struct swiftscan_cas_s *swiftscan_cas_t;
82+
typedef struct swiftscan_cas_id_s *swiftscan_cas_id_t;
8283
typedef struct swiftscan_cas_options_s *swiftscan_cas_options_t;
84+
typedef struct swiftscan_cache_cancellation_token_s
85+
*swiftscan_cache_cancellation_token_t;
8386

8487
typedef enum {
8588
SWIFTSCAN_OUTPUT_TYPE_OBJECT = 0,
8689
SWIFTSCAN_OUTPUT_TYPE_SWIFTMODULE = 1,
8790
SWIFTSCAN_OUTPUT_TYPE_SWIFTINTERFACE = 2,
8891
SWIFTSCAN_OUTPUT_TYPE_SWIFTPRIVATEINTERFACE = 3,
8992
SWIFTSCAN_OUTPUT_TYPE_CLANG_MODULE = 4,
90-
SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5
93+
SWIFTSCAN_OUTPUT_TYPE_CLANG_PCH = 5,
94+
SWIFTSCAN_OUTPUT_TYPE_CLANG_HEADER = 6,
95+
SWIFTSCAN_OUTPUT_TYPE_SWIFT_SOURCE_INFO = 7,
96+
SWIFTSCAN_OUTPUT_TYPE_SWIFT_MODULE_DOC = 8,
97+
SWIFTSCAN_OUTPUT_TYPE_DEPENDENCIES = 9,
98+
SWIFTSCAN_OUTPUT_TYPE_SWIFT_DEPS = 10,
99+
SWIFTSCAN_OUTPUT_TYPE_MODULE_TRACE = 11,
100+
SWIFTSCAN_OUTPUT_TYPE_TBD = 12,
101+
SWIFTSCAN_OUTPUT_TYPE_SWIFT_MODULE_SUMMARY = 13,
102+
SWIFTSCAN_OUTPUT_TYPE_SWIFT_ABI_DESCRIPTOR = 14,
103+
SWIFTSCAN_OUTPUT_TYPE_CONST_VALUE = 15,
104+
SWIFTSCAN_OUTPUT_TYPE_MODULE_SEMANTIC_INFO = 16,
105+
SWIFTSCAN_OUTPUT_TYPE_YAML_OPT_RECORD = 17,
106+
SWIFTSCAN_OUTPUT_TYPE_BITSTREAM_OPT_RECORD = 18,
107+
SWIFTSCAN_OUTPUT_TYPE_CACHED_DIAGNOSTICS = 19
91108
} swiftscan_output_kind_t;
92109

110+
typedef enum {
111+
SWIFTSCAN_CACHE_RESULT_SUCCESS = 0,
112+
SWIFTSCAN_CACHE_RESULT_NOT_FOUND = 1,
113+
SWIFTSCAN_CACHE_RESULT_ERROR = 2,
114+
} swiftscan_cache_result_t;
115+
93116
//=== libSwiftScan Functions ------------------------------------------------===//
94117

95118
typedef struct {
@@ -291,8 +314,44 @@ typedef struct {
291314
uint8_t *data, unsigned size,
292315
swiftscan_string_ref_t *error);
293316
swiftscan_string_ref_t (*swiftscan_compute_cache_key)(
294-
swiftscan_cas_t cas, int argc, const char *argv, const char *input,
295-
swiftscan_output_kind_t, swiftscan_string_ref_t *error);
317+
swiftscan_cas_t cas, int argc, const char **argv, const char *input,
318+
swiftscan_output_kind_t kind, swiftscan_string_ref_t *error);
319+
320+
//=== Scanner Caching Query/Replay Operations -----------------------------===//
321+
swiftscan_cas_id_t (*swiftscan_cache_query)(swiftscan_cas_t cas,
322+
const char *key, bool globally,
323+
swiftscan_string_ref_t *error);
324+
void (*swiftscan_cache_query_async)(
325+
swiftscan_cas_t cas, const char *key, bool globally, void *ctx,
326+
void (*callback)(void *ctx, swiftscan_cache_result_t, swiftscan_cas_id_t,
327+
swiftscan_string_ref_t error),
328+
swiftscan_cache_cancellation_token_t *);
329+
void (*swiftscan_cas_id_dispose)(swiftscan_cas_id_t);
330+
swiftscan_cache_result_t (*swiftscan_cache_load_object)(
331+
swiftscan_cas_t cas, swiftscan_cas_id_t, swiftscan_string_ref_t *error);
332+
void (*swiftscan_cache_load_object_async)(
333+
swiftscan_cas_t cas, swiftscan_cas_id_t, void *ctx,
334+
void (*callback)(void *ctx, swiftscan_cache_result_t,
335+
swiftscan_string_ref_t error),
336+
swiftscan_cache_cancellation_token_t *);
337+
void (*swiftscan_cache_make_global_async)(
338+
swiftscan_cas_t cas, const char *key, void *ctx,
339+
void (*callback)(void *ctx, swiftscan_string_ref_t error),
340+
swiftscan_cache_cancellation_token_t *);
341+
void (*swiftscan_cache_action_cancel)(swiftscan_cache_cancellation_token_t);
342+
void (*swiftscan_cache_cancellation_token_dispose)(
343+
swiftscan_cache_cancellation_token_t);
344+
swiftscan_cache_result_t (*swiftscan_cache_replay_compilation)(
345+
swiftscan_cas_t cas, int argc, const char **argv, void *reserved,
346+
swiftscan_string_ref_t *error);
347+
void (*swiftscan_cache_replay_compilation_async)(
348+
swiftscan_cas_t cas, int argc, const char **argv, void *reserved,
349+
void *ctx,
350+
void (*callback)(void *ctx, swiftscan_cache_result_t,
351+
swiftscan_string_ref_t std_out,
352+
swiftscan_string_ref_t std_err,
353+
swiftscan_string_ref_t error),
354+
swiftscan_cache_cancellation_token_t *);
296355

297356
} swiftscan_functions_t;
298357

‎Sources/SwiftDriver/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ add_library(SwiftDriver
1818
SwiftScan/DependencyGraphBuilder.swift
1919
SwiftScan/Loader.swift
2020
SwiftScan/SwiftScan.swift
21+
SwiftScan/SwiftScanCAS.swift
2122

2223
Driver/CompilerMode.swift
2324
Driver/DebugInfo.swift

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,8 @@ public struct Driver {
272272
let enableCaching: Bool
273273
let useClangIncludeTree: Bool
274274

275+
var cas: SwiftScanCAS?
276+
275277
/// Scanner prefix mapping.
276278
let scannerPrefixMap: [AbsolutePath: AbsolutePath]
277279
let scannerPrefixMapSDK: AbsolutePath?
@@ -470,6 +472,14 @@ public struct Driver {
470472
return supportedFrontendFeatures.contains(feature.rawValue)
471473
}
472474

475+
@_spi(Testing)
476+
public func getCAS() throws -> SwiftScanCAS {
477+
guard let cas = self.cas else {
478+
throw DependencyScanningError.casError("CAS is not initialized but requested")
479+
}
480+
return cas
481+
}
482+
473483
@_spi(Testing)
474484
public static func findBlocklists(RelativeTo execDir: AbsolutePath) throws -> [AbsolutePath] {
475485
// Expect to find all blocklists in such dir:
@@ -1592,7 +1602,8 @@ extension Driver {
15921602
buildRecordInfo: buildRecordInfo,
15931603
showJobLifecycle: showJobLifecycle,
15941604
argsResolver: executor.resolver,
1595-
diagnosticEngine: diagnosticEngine)
1605+
diagnosticEngine: diagnosticEngine,
1606+
cas: cas)
15961607
}
15971608

15981609
private mutating func performTheBuild(

‎Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,18 +56,21 @@ import var TSCBasic.stdoutStream
5656
private var nextBatchQuasiPID: Int
5757
private let argsResolver: ArgsResolver
5858
private var batchJobInputQuasiPIDMap = TwoLevelMap<Job, TypedVirtualPath, Int>()
59+
private var cas: SwiftScanCAS?
5960

6061
@_spi(Testing) public init(mode: ToolExecutionDelegate.Mode,
6162
buildRecordInfo: BuildRecordInfo?,
6263
showJobLifecycle: Bool,
6364
argsResolver: ArgsResolver,
64-
diagnosticEngine: DiagnosticsEngine) {
65+
diagnosticEngine: DiagnosticsEngine,
66+
cas: SwiftScanCAS? = nil) {
6567
self.mode = mode
6668
self.buildRecordInfo = buildRecordInfo
6769
self.showJobLifecycle = showJobLifecycle
6870
self.diagnosticEngine = diagnosticEngine
6971
self.argsResolver = argsResolver
7072
self.nextBatchQuasiPID = ToolExecutionDelegate.QUASI_PID_START
73+
self.cas = cas
7174
}
7275

7376
public func jobStarted(job: Job, arguments: [String], pid: Int) {
@@ -95,6 +98,21 @@ import var TSCBasic.stdoutStream
9598

9699
buildRecordInfo?.jobFinished(job: job, result: result)
97100

101+
do {
102+
if let cas = self.cas {
103+
for (spec, key) in job.outputCacheKeys {
104+
if let cacheKey = key {
105+
guard let _ = try cas.queryCacheKey(cacheKey, globally: false) else {
106+
throw DependencyScanningError.casError("cannot find \(spec): \(cacheKey)")
107+
}
108+
}
109+
}
110+
}
111+
} catch {
112+
diagnosticEngine.emit(.error("failured to lookup output from CAS: \(error)"))
113+
}
114+
115+
98116
#if os(Windows)
99117
if case .abnormal = result.exitStatus {
100118
anyJobHadAbnormalExit = true

‎Sources/SwiftDriver/Execution/ArgsResolver.swift

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,16 @@ public final class ArgsResolver {
6565
public func resolveArgumentList(for job: Job, useResponseFiles: ResponseFileHandling = .heuristic)
6666
throws -> ([String], usingResponseFile: Bool) {
6767
let tool = try resolve(.path(job.tool))
68-
var arguments = [tool] + (try job.commandLine.map { try resolve($0) })
68+
var arguments = [tool] + (try resolveArgumentList(for: job.commandLine))
6969
let usingResponseFile = try createResponseFileIfNeeded(for: job, resolvedArguments: &arguments,
7070
useResponseFiles: useResponseFiles)
7171
return (arguments, usingResponseFile)
7272
}
7373

74+
public func resolveArgumentList(for commandLine: [Job.ArgTemplate]) throws -> [String] {
75+
return try commandLine.map { try resolve($0) }
76+
}
77+
7478
@available(*, deprecated, message: "use resolveArgumentList(for:,useResponseFiles:,quotePaths:)")
7579
public func resolveArgumentList(for job: Job, forceResponseFiles: Bool,
7680
quotePaths: Bool = false) throws -> [String] {

‎Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

Lines changed: 39 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
4545
/// Whether we are using the integrated driver via libSwiftDriver shared lib
4646
private let integratedDriver: Bool
4747
private let mainModuleName: String?
48-
private let enableCAS: Bool
48+
private let cas: SwiftScanCAS?
4949
private let swiftScanOracle: InterModuleDependencyOracle
5050

5151
/// Clang PCM names contain a hash of the command-line arguments that were used to build them.
@@ -60,15 +60,15 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
6060
dependencyOracle: InterModuleDependencyOracle,
6161
integratedDriver: Bool = true,
6262
supportsExplicitInterfaceBuild: Bool = false,
63-
enableCAS: Bool = false) throws {
63+
cas: SwiftScanCAS? = nil) throws {
6464
self.dependencyGraph = dependencyGraph
6565
self.toolchain = toolchain
6666
self.swiftScanOracle = dependencyOracle
6767
self.integratedDriver = integratedDriver
6868
self.mainModuleName = dependencyGraph.mainModuleName
6969
self.reachabilityMap = try dependencyGraph.computeTransitiveClosure()
7070
self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild
71-
self.enableCAS = enableCAS
71+
self.cas = cas
7272
}
7373

7474
/// Supports resolving bridging header pch command from swiftScan.
@@ -136,9 +136,6 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
136136
for moduleId in swiftDependencies {
137137
let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId)
138138
var inputs: [TypedVirtualPath] = []
139-
let outputs: [TypedVirtualPath] = [
140-
TypedVirtualPath(file: moduleInfo.modulePath.path, type: .swiftModule)
141-
]
142139
var commandLine: [Job.ArgTemplate] = []
143140
// First, take the command line options provided in the dependency information
144141
let moduleDetails = try dependencyGraph.swiftModuleDetails(of: moduleId)
@@ -155,9 +152,14 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
155152
throw Driver.Error.malformedModuleDependency(moduleId.moduleName,
156153
"no `moduleInterfacePath` object")
157154
}
158-
inputs.append(TypedVirtualPath(file: moduleInterfacePath.path,
159-
type: .swiftInterface))
160155

156+
let inputInterfacePath = TypedVirtualPath(file: moduleInterfacePath.path, type: .swiftInterface)
157+
inputs.append(inputInterfacePath)
158+
let outputModulePath = TypedVirtualPath(file: moduleInfo.modulePath.path, type: .swiftModule)
159+
let outputs = [outputModulePath]
160+
161+
// TODO: Add diagnostics for explicit module build?
162+
let cacheKeys = [Job.OutputSpec(input: inputInterfacePath, output: outputModulePath): moduleDetails.moduleCacheKey]
161163
// Add precompiled module candidates, if present
162164
if let compiledCandidateList = moduleDetails.compiledModuleCandidates {
163165
for compiledCandidate in compiledCandidateList {
@@ -173,7 +175,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
173175
commandLine: commandLine,
174176
inputs: inputs,
175177
primaryInputs: [],
176-
outputs: outputs
178+
outputs: outputs,
179+
outputCacheKeys: cacheKeys
177180
))
178181
}
179182
return jobs
@@ -205,15 +208,17 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
205208
try resolveExplicitModuleDependencies(moduleId: moduleId, inputs: &inputs,
206209
commandLine: &commandLine)
207210

208-
let moduleMapPath = moduleDetails.moduleMapPath.path
209-
let modulePCMPath = moduleInfo.modulePath
210-
outputs.append(TypedVirtualPath(file: modulePCMPath.path, type: .pcm))
211+
let moduleMapPath = TypedVirtualPath(file: moduleDetails.moduleMapPath.path, type: .clangModuleMap)
212+
let modulePCMPath = TypedVirtualPath(file: moduleInfo.modulePath.path, type: .pcm)
213+
outputs.append(modulePCMPath)
211214

212215
// The only required input is the .modulemap for this module.
213216
// Command line options in the dependency scanner output will include the
214217
// required modulemap, so here we must only add it to the list of inputs.
215-
inputs.append(TypedVirtualPath(file: moduleMapPath,
216-
type: .clangModuleMap))
218+
inputs.append(moduleMapPath)
219+
220+
// TODO: Add diagnostics for explicit module build?
221+
let cacheKeys = [Job.OutputSpec(input: moduleMapPath, output: modulePCMPath): moduleDetails.moduleCacheKey]
217222

218223
jobs.append(Job(
219224
moduleName: moduleId.moduleName,
@@ -222,7 +227,8 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
222227
commandLine: commandLine,
223228
inputs: inputs,
224229
primaryInputs: [],
225-
outputs: outputs
230+
outputs: outputs,
231+
outputCacheKeys: cacheKeys
226232
))
227233
}
228234
return jobs
@@ -247,7 +253,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
247253
type: .swiftModule))
248254

249255
let prebuiltHeaderDependencyPaths = dependencyModule.prebuiltHeaderDependencyPaths ?? []
250-
if enableCAS && !prebuiltHeaderDependencyPaths.isEmpty {
256+
if cas != nil && !prebuiltHeaderDependencyPaths.isEmpty {
251257
throw Driver.Error.unsupportedConfigurationForCaching("module \(dependencyModule.moduleName) has prebuilt header dependency")
252258
}
253259

@@ -277,9 +283,9 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
277283
try serializeModuleDependencies(for: moduleId,
278284
swiftDependencyArtifacts: swiftDependencyArtifacts,
279285
clangDependencyArtifacts: clangDependencyArtifacts)
280-
if enableCAS {
286+
if let cas = self.cas {
281287
// When using a CAS, write JSON into CAS and pass the ID on command-line.
282-
let casID = try swiftScanOracle.store(data: dependencyFileContent)
288+
let casID = try cas.store(data: dependencyFileContent)
283289
commandLine.appendFlag("-explicit-swift-module-map-file")
284290
commandLine.appendFlag(casID)
285291
} else {
@@ -445,7 +451,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
445451
return
446452
}
447453

448-
assert(!enableCAS, "Caching build should always return command-line from scanner")
454+
assert(cas == nil, "Caching build should always return command-line from scanner")
449455
// Prohibit the frontend from implicitly building textual modules into binary modules.
450456
commandLine.appendFlags("-disable-implicit-swift-modules",
451457
"-Xcc", "-fno-implicit-modules",
@@ -465,6 +471,20 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
465471
type: .jsonSwiftArtifacts))
466472
}
467473

474+
private func computeOutputCacheKey(commandLine: [Job.ArgTemplate],
475+
cacheSpecs: [Job.OutputSpec]) throws -> [Job.OutputSpec: String?] {
476+
return try cacheSpecs.reduce(into: [:]) { keys, spec in
477+
if let cas = self.cas, spec.output.type.supportCaching {
478+
keys[spec] = try swiftScanOracle.computeCacheKeyForOutput(cas: cas,
479+
kind: spec.output.type,
480+
commandLine: commandLine,
481+
input: spec.input.file.intern())
482+
} else {
483+
keys[spec] = nil
484+
}
485+
}
486+
}
487+
468488
/// Serialize the output file artifacts for a given module in JSON format.
469489
private func serializeModuleDependencies(for moduleId: ModuleDependencyId,
470490
swiftDependencyArtifacts: [SwiftModuleArtifactInfo],

0 commit comments

Comments
 (0)