Skip to content

Commit b8e944f

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Add libSwiftScan API for scanner cache serialization/deserialization
swiftlang/swift#37723 added API to libSwiftScan to save and restore the dependency scanner's state on the filesystem. This PR adds this API to the corresponding client code in the driver.
1 parent 02dd592 commit b8e944f

File tree

7 files changed

+152
-20
lines changed

7 files changed

+152
-20
lines changed

‎Package.resolved

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Sources/CSwiftScan/include/swiftscan_header.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,17 +200,20 @@ typedef struct {
200200
//=== Scanner Functions ---------------------------------------------------===//
201201
swiftscan_scanner_t (*swiftscan_scanner_create)(void);
202202
void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
203-
204203
swiftscan_dependency_graph_t
205204
(*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
206-
207205
swiftscan_batch_scan_result_t *
208206
(*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
209207
swiftscan_batch_scan_input_t *,
210208
swiftscan_scan_invocation_t);
211-
212209
swiftscan_import_set_t
213210
(*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
211+
212+
//=== Scanner Cache Functions ---------------------------------------------===//
213+
void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
214+
bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
215+
void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
216+
214217
} swiftscan_functions_t;
215218

216219
#endif // SWIFT_C_DEPENDENCY_SCAN_H

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,33 @@ public class InterModuleDependencyOracle {
7373
}
7474
}
7575

76+
@_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) {
77+
guard let swiftScan = swiftScanLibInstance else {
78+
fatalError("Attempting to serialize scanner cache with no scanner instance.")
79+
}
80+
if swiftScan.canLoadStoreScannerCache() {
81+
swiftScan.serializeScannerCache(to: path)
82+
}
83+
}
84+
85+
@_spi(Testing) public func loadScannerCache(from path: AbsolutePath) {
86+
guard let swiftScan = swiftScanLibInstance else {
87+
fatalError("Attempting to load scanner cache with no scanner instance.")
88+
}
89+
if swiftScan.canLoadStoreScannerCache() {
90+
swiftScan.loadScannerCache(from: path)
91+
}
92+
}
93+
94+
@_spi(Testing) public func resetScannerCache() {
95+
guard let swiftScan = swiftScanLibInstance else {
96+
fatalError("Attempting to reset scanner cache with no scanner instance.")
97+
}
98+
if swiftScan.canLoadStoreScannerCache() {
99+
swiftScan.resetScannerCache()
100+
}
101+
}
102+
76103
private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
77104

78105
/// Queue to sunchronize accesses to the scanner

‎Sources/SwiftDriver/Jobs/CompileJob.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ extension Driver {
9393
.privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
9494
.remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
9595
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
96-
.indexUnitOutputPath, nil:
96+
.indexUnitOutputPath, .modDepCache, nil:
9797
return false
9898
}
9999
}
@@ -444,7 +444,7 @@ extension FileType {
444444
.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
445445
.yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
446446
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
447-
.indexUnitOutputPath:
447+
.indexUnitOutputPath, .modDepCache:
448448
fatalError("Output type can never be a primary output")
449449
}
450450
}

‎Sources/SwiftDriver/SwiftScan/SwiftScan.swift

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,26 @@ internal final class SwiftScan {
181181
return resultGraphMap
182182
}
183183

184+
@_spi(Testing) public func canLoadStoreScannerCache() -> Bool {
185+
return api.swiftscan_scanner_cache_load != nil &&
186+
api.swiftscan_scanner_cache_serialize != nil &&
187+
api.swiftscan_scanner_cache_reset != nil
188+
}
189+
190+
func serializeScannerCache(to path: AbsolutePath) {
191+
api.swiftscan_scanner_cache_serialize(scanner,
192+
path.description.cString(using: String.Encoding.utf8))
193+
}
194+
195+
func loadScannerCache(from path: AbsolutePath) {
196+
api.swiftscan_scanner_cache_load(scanner,
197+
path.description.cString(using: String.Encoding.utf8))
198+
}
199+
200+
func resetScannerCache() {
201+
api.swiftscan_scanner_cache_reset(scanner)
202+
}
203+
184204
@_spi(Testing) public func canQuerySupportedArguments() -> Bool {
185205
return api.swiftscan_compiler_supported_arguments_query != nil &&
186206
api.swiftscan_string_set_dispose != nil
@@ -226,13 +246,22 @@ private extension swiftscan_functions_t {
226246
}
227247
return sym
228248
}
249+
// Supported features/flags query
229250
self.swiftscan_string_set_dispose =
230251
try loadOptional("swiftscan_string_set_dispose")
231252
self.swiftscan_compiler_supported_arguments_query =
232253
try loadOptional("swiftscan_compiler_supported_arguments_query")
233254
self.swiftscan_compiler_supported_features_query =
234255
try loadOptional("swiftscan_compiler_supported_features_query")
235256

257+
// Dependency scanner serialization/deserialization features
258+
self.swiftscan_scanner_cache_serialize =
259+
try loadOptional("swiftscan_scanner_cache_serialize")
260+
self.swiftscan_scanner_cache_load =
261+
try loadOptional("swiftscan_scanner_cache_load")
262+
self.swiftscan_scanner_cache_reset =
263+
try loadOptional("swiftscan_scanner_cache_reset")
264+
236265
// MARK: Required Methods
237266
func loadRequired<T>(_ symbol: String) throws -> T {
238267
guard let sym: T = dlsym(swiftscan, symbol: symbol) else {

‎Sources/SwiftDriver/Utilities/FileType.swift

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
8181
/// Swift dependencies file.
8282
case swiftDeps = "swiftdeps"
8383

84+
/// Serialized dependency scanner state
85+
case modDepCache = "moddepcache"
86+
8487
/// Remapping file
8588
case remap
8689

@@ -109,20 +112,20 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
109112
/// Swift section of the internal wiki.
110113
case moduleTrace = "trace.json"
111114

112-
/// Indexing data directory.
115+
/// Indexing data directory
113116
///
114-
/// The extension isn't real.
117+
/// The extension isn't real, rather this FileType specifies a directory path.
115118
case indexData
116119

117120
/// Output path to record in the indexing data store
118121
///
119122
/// This is only needed for use as a key in the output file map.
120123
case indexUnitOutputPath
121124

122-
/// Optimization record.
125+
/// Optimization record
123126
case yamlOptimizationRecord = "opt.yaml"
124127

125-
/// Bitstream optimization record.
128+
/// Bitstream optimization record
126129
case bitstreamOptimizationRecord = "opt.bitstream"
127130

128131
/// Clang compiler module file
@@ -169,6 +172,9 @@ extension FileType: CustomStringConvertible {
169172
case .swiftDeps:
170173
return "swift-dependencies"
171174

175+
case .modDepCache:
176+
return "dependency-scanner-cache"
177+
172178
case .jsonDependencies:
173179
return "json-dependencies"
174180

@@ -218,7 +224,7 @@ extension FileType {
218224
.swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
219225
.swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
220226
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
221-
.indexUnitOutputPath:
227+
.indexUnitOutputPath, .modDepCache:
222228
return false
223229
}
224230
}
@@ -289,6 +295,8 @@ extension FileType {
289295
return "objc-header"
290296
case .swiftDeps:
291297
return "swift-dependencies"
298+
case .modDepCache:
299+
return "dependency-scanner-cache"
292300
case .jsonDependencies:
293301
return "json-dependencies"
294302
case .jsonTargetInfo:
@@ -327,7 +335,7 @@ extension FileType {
327335
case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
328336
.swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
329337
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
330-
.indexUnitOutputPath:
338+
.indexUnitOutputPath, .modDepCache:
331339
return false
332340
}
333341
}
@@ -341,7 +349,7 @@ extension FileType {
341349
case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .autolink,
342350
.swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface,
343351
.swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
344-
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
352+
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache,
345353
.bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
346354
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
347355
return false

‎Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -601,24 +601,31 @@ final class ExplicitModuleBuildTests: XCTestCase {
601601
}
602602
}
603603

604-
/// Test the libSwiftScan dependency scanning.
605-
func testDependencyScanning() throws {
604+
private func getDriverArtifactsForScanning() throws -> (stdLibPath: AbsolutePath,
605+
shimsPath: AbsolutePath,
606+
toolchain: Toolchain,
607+
hostTriple: Triple) {
606608
// Just instantiating to get at the toolchain path
607609
let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
608610
"-module-name", "testDependencyScanning",
609611
"test.swift"])
610612
let (stdLibPath, shimsPath) = try getStdlibShimsPaths(driver)
611-
612613
XCTAssertTrue(localFileSystem.exists(stdLibPath),
613614
"expected Swift StdLib at: \(stdLibPath.description)")
614615
XCTAssertTrue(localFileSystem.exists(shimsPath),
615616
"expected Swift Shims at: \(shimsPath.description)")
617+
return (stdLibPath, shimsPath, driver.toolchain, driver.hostTriple)
618+
}
619+
620+
/// Test the libSwiftScan dependency scanning.
621+
func testDependencyScanning() throws {
622+
let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
616623

617624
// The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
618625
// queries.
619626
let dependencyOracle = InterModuleDependencyOracle()
620-
let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain,
621-
hostTriple: driver.hostTriple,
627+
let scanLibPath = try Driver.getScanLibPath(of: toolchain,
628+
hostTriple: hostTriple,
622629
env: ProcessEnv.vars)
623630
guard try dependencyOracle
624631
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -653,8 +660,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
653660
// Module `X` is only imported on Darwin when:
654661
// #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000
655662
let expectedNumberOfDependencies: Int
656-
if driver.targetTriple.isMacOSX,
657-
driver.targetTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
663+
if hostTriple.isMacOSX,
664+
hostTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
658665
expectedNumberOfDependencies = 11
659666
} else {
660667
expectedNumberOfDependencies = 12
@@ -692,6 +699,64 @@ final class ExplicitModuleBuildTests: XCTestCase {
692699
}
693700
}
694701

702+
703+
/// Test the libSwiftScan dependency scanning.
704+
func testDependencyScanReuseCache() throws {
705+
let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
706+
try withTemporaryDirectory { path in
707+
let cacheSavePath = path.appending(component: "saved.moddepcache")
708+
let main = path.appending(component: "testDependencyScanning.swift")
709+
try localFileSystem.writeFileContents(main) {
710+
$0 <<< "import C;"
711+
$0 <<< "import E;"
712+
$0 <<< "import G;"
713+
}
714+
let packageRootPath = URL(fileURLWithPath: #file).pathComponents
715+
.prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
716+
let testInputsPath = packageRootPath + "/TestInputs"
717+
let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
718+
let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
719+
let scannerCommand = ["-scan-dependencies",
720+
"-I", cHeadersPath,
721+
"-I", swiftModuleInterfacesPath,
722+
"-I", stdLibPath.description,
723+
"-I", shimsPath.description,
724+
main.pathString]
725+
726+
let scanLibPath = try Driver.getScanLibPath(of: toolchain,
727+
hostTriple: hostTriple,
728+
env: ProcessEnv.vars)
729+
// Run the first scan and serialize the cache contents.
730+
let firstDependencyOracle = InterModuleDependencyOracle()
731+
guard try firstDependencyOracle
732+
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
733+
swiftScanLibPath: scanLibPath) else {
734+
XCTFail("Dependency scanner library not found")
735+
return
736+
}
737+
738+
let firstScanGraph =
739+
try! firstDependencyOracle.getDependencies(workingDirectory: path,
740+
commandLine: scannerCommand)
741+
firstDependencyOracle.serializeScannerCache(to: cacheSavePath)
742+
743+
// Run the second scan, re-using the serialized cache contents.
744+
let secondDependencyOracle = InterModuleDependencyOracle()
745+
guard try secondDependencyOracle
746+
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
747+
swiftScanLibPath: scanLibPath) else {
748+
XCTFail("Dependency scanner library not found")
749+
return
750+
}
751+
secondDependencyOracle.loadScannerCache(from: cacheSavePath)
752+
let secondScanGraph =
753+
try! secondDependencyOracle.getDependencies(workingDirectory: path,
754+
commandLine: scannerCommand)
755+
756+
XCTAssertTrue(firstScanGraph.modules.count == secondScanGraph.modules.count)
757+
}
758+
}
759+
695760
func testDependencyGraphMerge() throws {
696761
let moduleDependencyGraph1 =
697762
try JSONDecoder().decode(

0 commit comments

Comments
 (0)