Skip to content

Commit 41f1bb3

Browse files
artemcmartemcm
authored
Merge pull request #696 from artemcm/IncrementalExplicitBuilds
[Explicit Module Builds] Add libSwiftScan API for scanner cache serialization/deserialization
2 parents d05dd6e + 9a0976d commit 41f1bb3

File tree

6 files changed

+152
-19
lines changed

6 files changed

+152
-19
lines changed

‎Sources/CSwiftScan/include/swiftscan_header.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,17 +200,20 @@ typedef struct {
200200
//=== Scanner Functions ---------------------------------------------------===//
201201
swiftscan_scanner_t (*swiftscan_scanner_create)(void);
202202
void (*swiftscan_scanner_dispose)(swiftscan_scanner_t);
203-
204203
swiftscan_dependency_graph_t
205204
(*swiftscan_dependency_graph_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
206-
207205
swiftscan_batch_scan_result_t *
208206
(*swiftscan_batch_scan_result_create)(swiftscan_scanner_t,
209207
swiftscan_batch_scan_input_t *,
210208
swiftscan_scan_invocation_t);
211-
212209
swiftscan_import_set_t
213210
(*swiftscan_import_set_create)(swiftscan_scanner_t, swiftscan_scan_invocation_t);
211+
212+
//=== Scanner Cache Functions ---------------------------------------------===//
213+
void (*swiftscan_scanner_cache_serialize)(swiftscan_scanner_t scanner, const char * path);
214+
bool (*swiftscan_scanner_cache_load)(swiftscan_scanner_t scanner, const char * path);
215+
void (*swiftscan_scanner_cache_reset)(swiftscan_scanner_t scanner);
216+
214217
} swiftscan_functions_t;
215218

216219
#endif // SWIFT_C_DEPENDENCY_SCAN_H

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,34 @@ public class InterModuleDependencyOracle {
7373
}
7474
}
7575

76+
@_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) {
77+
guard let swiftScan = swiftScanLibInstance else {
78+
fatalError("Attempting to serialize scanner cache with no scanner instance.")
79+
}
80+
if swiftScan.canLoadStoreScannerCache() {
81+
swiftScan.serializeScannerCache(to: path)
82+
}
83+
}
84+
85+
@_spi(Testing) public func loadScannerCache(from path: AbsolutePath) -> Bool {
86+
guard let swiftScan = swiftScanLibInstance else {
87+
fatalError("Attempting to load scanner cache with no scanner instance.")
88+
}
89+
if swiftScan.canLoadStoreScannerCache() {
90+
return swiftScan.loadScannerCache(from: path)
91+
}
92+
return false
93+
}
94+
95+
@_spi(Testing) public func resetScannerCache() {
96+
guard let swiftScan = swiftScanLibInstance else {
97+
fatalError("Attempting to reset scanner cache with no scanner instance.")
98+
}
99+
if swiftScan.canLoadStoreScannerCache() {
100+
swiftScan.resetScannerCache()
101+
}
102+
}
103+
76104
private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil }
77105

78106
/// Queue to sunchronize accesses to the scanner

‎Sources/SwiftDriver/Jobs/CompileJob.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ extension Driver {
9393
.privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
9494
.remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
9595
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
96-
.indexUnitOutputPath, nil:
96+
.indexUnitOutputPath, .modDepCache, nil:
9797
return false
9898
}
9999
}
@@ -444,7 +444,7 @@ extension FileType {
444444
.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
445445
.yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
446446
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
447-
.indexUnitOutputPath:
447+
.indexUnitOutputPath, .modDepCache:
448448
fatalError("Output type can never be a primary output")
449449
}
450450
}

‎Sources/SwiftDriver/SwiftScan/SwiftScan.swift

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,26 @@ internal final class SwiftScan {
181181
return resultGraphMap
182182
}
183183

184+
@_spi(Testing) public func canLoadStoreScannerCache() -> Bool {
185+
return api.swiftscan_scanner_cache_load != nil &&
186+
api.swiftscan_scanner_cache_serialize != nil &&
187+
api.swiftscan_scanner_cache_reset != nil
188+
}
189+
190+
func serializeScannerCache(to path: AbsolutePath) {
191+
api.swiftscan_scanner_cache_serialize(scanner,
192+
path.description.cString(using: String.Encoding.utf8))
193+
}
194+
195+
func loadScannerCache(from path: AbsolutePath) -> Bool {
196+
return api.swiftscan_scanner_cache_load(scanner,
197+
path.description.cString(using: String.Encoding.utf8))
198+
}
199+
200+
func resetScannerCache() {
201+
api.swiftscan_scanner_cache_reset(scanner)
202+
}
203+
184204
@_spi(Testing) public func canQuerySupportedArguments() -> Bool {
185205
return api.swiftscan_compiler_supported_arguments_query != nil &&
186206
api.swiftscan_string_set_dispose != nil
@@ -226,13 +246,22 @@ private extension swiftscan_functions_t {
226246
}
227247
return sym
228248
}
249+
// Supported features/flags query
229250
self.swiftscan_string_set_dispose =
230251
try loadOptional("swiftscan_string_set_dispose")
231252
self.swiftscan_compiler_supported_arguments_query =
232253
try loadOptional("swiftscan_compiler_supported_arguments_query")
233254
self.swiftscan_compiler_supported_features_query =
234255
try loadOptional("swiftscan_compiler_supported_features_query")
235256

257+
// Dependency scanner serialization/deserialization features
258+
self.swiftscan_scanner_cache_serialize =
259+
try loadOptional("swiftscan_scanner_cache_serialize")
260+
self.swiftscan_scanner_cache_load =
261+
try loadOptional("swiftscan_scanner_cache_load")
262+
self.swiftscan_scanner_cache_reset =
263+
try loadOptional("swiftscan_scanner_cache_reset")
264+
236265
// MARK: Required Methods
237266
func loadRequired<T>(_ symbol: String) throws -> T {
238267
guard let sym: T = dlsym(swiftscan, symbol: symbol) else {

‎Sources/SwiftDriver/Utilities/FileType.swift

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,9 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
8181
/// Swift dependencies file.
8282
case swiftDeps = "swiftdeps"
8383

84+
/// Serialized dependency scanner state
85+
case modDepCache = "moddepcache"
86+
8487
/// Remapping file
8588
case remap
8689

@@ -109,20 +112,20 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
109112
/// Swift section of the internal wiki.
110113
case moduleTrace = "trace.json"
111114

112-
/// Indexing data directory.
115+
/// Indexing data directory
113116
///
114-
/// The extension isn't real.
117+
/// The extension isn't real, rather this FileType specifies a directory path.
115118
case indexData
116119

117120
/// Output path to record in the indexing data store
118121
///
119122
/// This is only needed for use as a key in the output file map.
120123
case indexUnitOutputPath
121124

122-
/// Optimization record.
125+
/// Optimization record
123126
case yamlOptimizationRecord = "opt.yaml"
124127

125-
/// Bitstream optimization record.
128+
/// Bitstream optimization record
126129
case bitstreamOptimizationRecord = "opt.bitstream"
127130

128131
/// Clang compiler module file
@@ -169,6 +172,9 @@ extension FileType: CustomStringConvertible {
169172
case .swiftDeps:
170173
return "swift-dependencies"
171174

175+
case .modDepCache:
176+
return "dependency-scanner-cache"
177+
172178
case .jsonDependencies:
173179
return "json-dependencies"
174180

@@ -218,7 +224,7 @@ extension FileType {
218224
.swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
219225
.swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
220226
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
221-
.indexUnitOutputPath:
227+
.indexUnitOutputPath, .modDepCache:
222228
return false
223229
}
224230
}
@@ -289,6 +295,8 @@ extension FileType {
289295
return "objc-header"
290296
case .swiftDeps:
291297
return "swift-dependencies"
298+
case .modDepCache:
299+
return "dependency-scanner-cache"
292300
case .jsonDependencies:
293301
return "json-dependencies"
294302
case .jsonTargetInfo:
@@ -327,7 +335,7 @@ extension FileType {
327335
case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
328336
.swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
329337
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
330-
.indexUnitOutputPath:
338+
.indexUnitOutputPath, .modDepCache:
331339
return false
332340
}
333341
}
@@ -341,7 +349,7 @@ extension FileType {
341349
case .swift, .sil, .sib, .ast, .image, .dSYM, .dependencies, .autolink,
342350
.swiftModule, .swiftDocumentation, .swiftInterface, .privateSwiftInterface,
343351
.swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
344-
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
352+
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord, .modDepCache,
345353
.bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
346354
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
347355
return false

‎Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

Lines changed: 72 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -642,24 +642,31 @@ final class ExplicitModuleBuildTests: XCTestCase {
642642
}
643643
}
644644

645-
/// Test the libSwiftScan dependency scanning.
646-
func testDependencyScanning() throws {
645+
private func getDriverArtifactsForScanning() throws -> (stdLibPath: AbsolutePath,
646+
shimsPath: AbsolutePath,
647+
toolchain: Toolchain,
648+
hostTriple: Triple) {
647649
// Just instantiating to get at the toolchain path
648650
let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
649651
"-module-name", "testDependencyScanning",
650652
"test.swift"])
651653
let (stdLibPath, shimsPath) = try getStdlibShimsPaths(driver)
652-
653654
XCTAssertTrue(localFileSystem.exists(stdLibPath),
654655
"expected Swift StdLib at: \(stdLibPath.description)")
655656
XCTAssertTrue(localFileSystem.exists(shimsPath),
656657
"expected Swift Shims at: \(shimsPath.description)")
658+
return (stdLibPath, shimsPath, driver.toolchain, driver.hostTriple)
659+
}
660+
661+
/// Test the libSwiftScan dependency scanning.
662+
func testDependencyScanning() throws {
663+
let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
657664

658665
// The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
659666
// queries.
660667
let dependencyOracle = InterModuleDependencyOracle()
661-
let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain,
662-
hostTriple: driver.hostTriple,
668+
let scanLibPath = try Driver.getScanLibPath(of: toolchain,
669+
hostTriple: hostTriple,
663670
env: ProcessEnv.vars)
664671
guard try dependencyOracle
665672
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
@@ -694,8 +701,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
694701
// Module `X` is only imported on Darwin when:
695702
// #if __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 110000
696703
let expectedNumberOfDependencies: Int
697-
if driver.targetTriple.isMacOSX,
698-
driver.targetTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
704+
if hostTriple.isMacOSX,
705+
hostTriple.version(for: .macOS) >= Triple.Version(11, 0, 0) {
699706
expectedNumberOfDependencies = 11
700707
} else {
701708
expectedNumberOfDependencies = 12
@@ -733,6 +740,64 @@ final class ExplicitModuleBuildTests: XCTestCase {
733740
}
734741
}
735742

743+
744+
/// Test the libSwiftScan dependency scanning.
745+
func testDependencyScanReuseCache() throws {
746+
let (stdLibPath, shimsPath, toolchain, hostTriple) = try getDriverArtifactsForScanning()
747+
try withTemporaryDirectory { path in
748+
let cacheSavePath = path.appending(component: "saved.moddepcache")
749+
let main = path.appending(component: "testDependencyScanning.swift")
750+
try localFileSystem.writeFileContents(main) {
751+
$0 <<< "import C;"
752+
$0 <<< "import E;"
753+
$0 <<< "import G;"
754+
}
755+
let packageRootPath = URL(fileURLWithPath: #file).pathComponents
756+
.prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
757+
let testInputsPath = packageRootPath + "/TestInputs"
758+
let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
759+
let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
760+
let scannerCommand = ["-scan-dependencies",
761+
"-I", cHeadersPath,
762+
"-I", swiftModuleInterfacesPath,
763+
"-I", stdLibPath.description,
764+
"-I", shimsPath.description,
765+
main.pathString]
766+
767+
let scanLibPath = try Driver.getScanLibPath(of: toolchain,
768+
hostTriple: hostTriple,
769+
env: ProcessEnv.vars)
770+
// Run the first scan and serialize the cache contents.
771+
let firstDependencyOracle = InterModuleDependencyOracle()
772+
guard try firstDependencyOracle
773+
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
774+
swiftScanLibPath: scanLibPath) else {
775+
XCTFail("Dependency scanner library not found")
776+
return
777+
}
778+
779+
let firstScanGraph =
780+
try! firstDependencyOracle.getDependencies(workingDirectory: path,
781+
commandLine: scannerCommand)
782+
firstDependencyOracle.serializeScannerCache(to: cacheSavePath)
783+
784+
// Run the second scan, re-using the serialized cache contents.
785+
let secondDependencyOracle = InterModuleDependencyOracle()
786+
guard try secondDependencyOracle
787+
.verifyOrCreateScannerInstance(fileSystem: localFileSystem,
788+
swiftScanLibPath: scanLibPath) else {
789+
XCTFail("Dependency scanner library not found")
790+
return
791+
}
792+
XCTAssertFalse(secondDependencyOracle.loadScannerCache(from: cacheSavePath))
793+
let secondScanGraph =
794+
try! secondDependencyOracle.getDependencies(workingDirectory: path,
795+
commandLine: scannerCommand)
796+
797+
XCTAssertTrue(firstScanGraph.modules.count == secondScanGraph.modules.count)
798+
}
799+
}
800+
736801
func testDependencyGraphMerge() throws {
737802
let moduleDependencyGraph1 =
738803
try JSONDecoder().decode(

0 commit comments

Comments
 (0)