Skip to content

Commit a50d48e

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Add libSwiftScan parallel scanning query test.
1 parent 66709b6 commit a50d48e

File tree

6 files changed

+127
-49
lines changed

6 files changed

+127
-49
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -395,25 +395,6 @@ public struct Driver {
395395
return ($0, modTime)
396396
})
397397

398-
// Create an instance of an inter-module dependency oracle, if the driver's
399-
// client did not provide one. The clients are expected to provide an oracle
400-
// when they wish to share module dependency information across targets.
401-
if let dependencyOracle = interModuleDependencyOracle {
402-
self.interModuleDependencyOracle = dependencyOracle
403-
} else {
404-
self.interModuleDependencyOracle =
405-
try InterModuleDependencyOracle(fileSystem: fileSystem,
406-
toolchainPath: AbsolutePath("/Volumes/Data/GHWorkspace2/build/Ninja-RelWithDebInfoAssert/swift-macosx-x86_64"))
407-
408-
// This is a shim for backwards-compatibility with ModuleInfoMap-based API
409-
// used by SwiftPM
410-
if let externalArtifacts = externalBuildArtifacts {
411-
if !externalArtifacts.1.isEmpty {
412-
try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
413-
}
414-
}
415-
}
416-
417398
do {
418399
let outputFileMap: OutputFileMap?
419400
// Initialize an empty output file map, which will be populated when we start creating jobs.
@@ -435,6 +416,29 @@ public struct Driver {
435416
}
436417
}
437418

419+
// Create an instance of an inter-module dependency oracle, if the driver's
420+
// client did not provide one. The clients are expected to provide an oracle
421+
// when they wish to share module dependency information across targets.
422+
if let dependencyOracle = interModuleDependencyOracle {
423+
self.interModuleDependencyOracle = dependencyOracle
424+
} else {
425+
// For now, we expect to find the library in the same toolchain as the compiler itself
426+
let toolchainRootPath: AbsolutePath = try toolchain.getToolPath(.swiftCompiler)
427+
.parentDirectory // bin
428+
.parentDirectory // toolchain root
429+
self.interModuleDependencyOracle =
430+
try InterModuleDependencyOracle(fileSystem: fileSystem,
431+
toolchainPath: toolchainRootPath)
432+
433+
// This is a shim for backwards-compatibility with ModuleInfoMap-based API
434+
// used by SwiftPM
435+
if let externalArtifacts = externalBuildArtifacts {
436+
if !externalArtifacts.1.isEmpty {
437+
try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
438+
}
439+
}
440+
}
441+
438442
self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)
439443
self.shouldUseInputFileList = inputFiles.count > fileListThreshold
440444
if shouldUseInputFileList {

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,24 +42,17 @@ public class InterModuleDependencyOracle {
4242
swiftScanLibInstance = try SwiftScan(dylib: swiftScanLibPath)
4343
}
4444

45-
/// Query the ModuleInfo of a module with a given ID
46-
internal func getExternalModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? {
47-
self.lock.withLock {
48-
return externalModules[moduleId]
49-
}
50-
}
51-
52-
internal func getDependencies(workingDirectory: AbsolutePath,
53-
commandLine: [String]) throws -> InterModuleDependencyGraph {
45+
public func getDependencies(workingDirectory: AbsolutePath,
46+
commandLine: [String]) throws -> InterModuleDependencyGraph {
5447
try self.lock.withLock {
5548
return try swiftScanLibInstance.scanDependencies(workingDirectory: workingDirectory,
5649
invocationCommand: commandLine)
5750
}
5851
}
5952

60-
internal func getBatchDependencies(workingDirectory: AbsolutePath,
61-
commandLine: [String],
62-
batchInfos: [BatchScanModuleInfo])
53+
public func getBatchDependencies(workingDirectory: AbsolutePath,
54+
commandLine: [String],
55+
batchInfos: [BatchScanModuleInfo])
6356
throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
6457
try self.lock.withLock {
6558
return try swiftScanLibInstance.batchScanDependencies(workingDirectory: workingDirectory,
@@ -70,14 +63,22 @@ public class InterModuleDependencyOracle {
7063

7164
internal let lock = Lock()
7265

66+
/// A reference to an instance of the compiler's libSwiftScan shared library
67+
private let swiftScanLibInstance: SwiftScan
68+
69+
// The below API is a legacy implementation of the oracle that is in-place to allow clients to
70+
// transition to the new API. It is to be removed once that transition is complete.
7371
/// The complete set of modules discovered so far, spanning potentially multiple targets,
7472
/// accumulated across builds of multiple targets.
7573
/// TODO: This is currently only used for placeholder resolution. libSwiftScan should allow us to move away
7674
/// from the concept of a placeholder module so we should be able to get rid of this in the future.
7775
internal var externalModules: ModuleInfoMap = [:]
78-
79-
/// A reference to an instance of the compiler's libSwiftScan shared library
80-
private let swiftScanLibInstance: SwiftScan
76+
/// Query the ModuleInfo of a module with a given ID
77+
public func getExternalModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? {
78+
self.lock.withLock {
79+
return externalModules[moduleId]
80+
}
81+
}
8182
}
8283

8384
// This is a shim for backwards-compatibility with existing API used by SwiftPM.

‎Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ import Foundation
6161
/// - Module Name
6262
/// - Extra PCM build arguments (for Clang modules only)
6363
/// - Dependency graph output path
64-
internal enum BatchScanModuleInfo: Encodable {
64+
public enum BatchScanModuleInfo: Encodable {
6565
case swift(BatchScanSwiftModuleInfo)
6666
case clang(BatchScanClangModuleInfo)
6767
}
6868

69-
internal struct BatchScanSwiftModuleInfo: Encodable {
69+
public struct BatchScanSwiftModuleInfo: Encodable {
7070
var swiftModuleName: String
7171
var output: String
7272

@@ -76,7 +76,7 @@ internal struct BatchScanSwiftModuleInfo: Encodable {
7676
}
7777
}
7878

79-
internal struct BatchScanClangModuleInfo: Encodable {
79+
public struct BatchScanClangModuleInfo: Encodable {
8080
var clangModuleName: String
8181
var arguments: String
8282
var output: String
@@ -88,7 +88,7 @@ internal struct BatchScanClangModuleInfo: Encodable {
8888
}
8989
}
9090

91-
internal extension BatchScanModuleInfo {
91+
public extension BatchScanModuleInfo {
9292
func encode(to encoder: Encoder) throws {
9393
var container = encoder.singleValueContainer()
9494
switch self {

‎Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,16 @@ internal extension SwiftScan {
4242
return resultGraph
4343
}
4444

45+
/// From a reference to a binary-format dependency graph collection returned by libSwiftScan batch scan query,
46+
/// corresponding to the specified batch scan input (`BatchScanModuleInfo`), construct instances of
47+
/// `InterModuleDependencyGraph` for each result.
4548
func constructBatchResultGraphs(for batchInfos: [BatchScanModuleInfo],
4649
from batchResultRef: swiftscan_batch_scan_result_t) throws
4750
-> [ModuleDependencyId: [InterModuleDependencyGraph]] {
4851
var resultMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = [:]
4952
let resultGraphRefArray = Array(UnsafeBufferPointer(start: batchResultRef.results,
5053
count: Int(batchResultRef.count)))
54+
// Note, respective indeces of the batch scan input and the returned result must be aligned.
5155
for (index, resultGraphRefOrNull) in resultGraphRefArray.enumerated() {
5256
guard let resultGraphRef = resultGraphRefOrNull else {
5357
throw DependencyScanningError.dependencyScanFailed
@@ -61,6 +65,7 @@ internal extension SwiftScan {
6165
case .clang(let clangModuleBatchScanInfo):
6266
moduleId = .clang(clangModuleBatchScanInfo.clangModuleName)
6367
}
68+
// Update the map with either yet another graph or create an entry for this module
6469
if resultMap[moduleId] != nil {
6570
resultMap[moduleId]!.append(decodedGraph)
6671
} else {
@@ -72,8 +77,7 @@ internal extension SwiftScan {
7277
}
7378

7479
private extension SwiftScan {
75-
76-
/// From a reference to a binary-format module dependency info returned by libSwiftScan,
80+
/// From a reference to a binary-format module dependency module info returned by libSwiftScan,
7781
/// construct an instance of an `ModuleInfo` as used by the driver
7882
func constructModuleInfo(from moduleInfoRef: swiftscan_dependency_info_t)
7983
throws -> (ModuleDependencyId, ModuleInfo) {
@@ -82,6 +86,7 @@ private extension SwiftScan {
8286
try toSwiftString(api.swiftscan_module_info_get_module_name(moduleInfoRef))
8387
let moduleId = try decodeModuleNameAndKind(from: encodedModuleName)
8488

89+
// Decode module path and source file locations
8590
let modulePath = try toSwiftString(api.swiftscan_module_info_get_module_path(moduleInfoRef))
8691
let sourceFiles: [String]?
8792
if let sourceFilesSetRef = api.swiftscan_module_info_get_source_files(moduleInfoRef) {
@@ -90,6 +95,7 @@ private extension SwiftScan {
9095
sourceFiles = nil
9196
}
9297

98+
// Decode all dependencies of this module
9399
let directDependencies: [ModuleDependencyId]?
94100
if let encodedDirectDepsRef = api.swiftscan_module_info_get_direct_dependencies(moduleInfoRef) {
95101
let encodedDirectDependencies = try toSwiftStringArray(encodedDirectDepsRef.pointee)

‎Sources/SwiftDriver/SwiftScan/SwiftScan.swift

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//===--- MultiJobExecutor.swift - Builtin DriverExecutor implementation ---===//
1+
//===------------------------ SwiftScan.swift -----------------------------===//
22
//
33
// This source file is part of the Swift.org open source project
44
//
@@ -10,7 +10,6 @@
1010
//
1111
//===----------------------------------------------------------------------===//
1212

13-
// AC: Consider making this implementation-only
1413
@_implementationOnly import CSwiftScan
1514

1615
import TSCUtility
@@ -65,7 +64,6 @@ internal final class SwiftScan {
6564

6665
init(dylib path: AbsolutePath) throws {
6766
self.path = path
68-
print("OPENINGNG : \(path.description)")
6967
#if os(Windows)
7068
self.dylib = try dlopen(path.pathString, mode: [])
7169
#else

‎Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,11 @@ final class ExplicitModuleBuildTests: XCTestCase {
159159
try JSONDecoder().decode(
160160
InterModuleDependencyGraph.self,
161161
from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
162-
let dependencyOracle = InterModuleDependencyOracle()
162+
let toolchainRootPath: AbsolutePath = try driver.toolchain.getToolPath(.swiftCompiler)
163+
.parentDirectory // bin
164+
.parentDirectory // toolchain root
165+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
166+
toolchainPath: toolchainRootPath)
163167
try dependencyOracle.mergeModules(from: moduleDependencyGraph)
164168
driver.explicitDependencyBuildPlanner =
165169
try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph,
@@ -204,7 +208,22 @@ final class ExplicitModuleBuildTests: XCTestCase {
204208
from: ModuleDependenciesInputs.bPlaceHolderInput.data(using: .utf8)!)
205209
let targetModulePathMap: ExternalTargetModulePathMap =
206210
[ModuleDependencyId.swiftPlaceholder("B"):AbsolutePath("/Somewhere/B.swiftmodule")]
207-
let dependencyOracle = InterModuleDependencyOracle()
211+
212+
let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
213+
processSet: ProcessSet(),
214+
fileSystem: localFileSystem,
215+
env: ProcessEnv.vars)
216+
var toolchain: Toolchain
217+
#if os(macOS)
218+
toolchain = DarwinToolchain(env: ProcessEnv.vars, executor: executor)
219+
#else
220+
toolchain = GenericUnixToolchain(env: ProcessEnv.vars, executor: executor)
221+
#endif
222+
let toolchainRootPath: AbsolutePath = try toolchain.getToolPath(.swiftCompiler)
223+
.parentDirectory // bin
224+
.parentDirectory // toolchain root
225+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
226+
toolchainPath: toolchainRootPath)
208227
try dependencyOracle.mergeModules(from: inputDependencyGraph)
209228

210229
// Construct a module dependency graph that will contain .swiftPlaceholder("B"),
@@ -217,10 +236,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
217236
// Construct the driver with explicit external dependency input
218237
let commandLine = ["swiftc", "-experimental-explicit-module-build",
219238
"test.swift", "-module-name", "A", "-g"]
220-
let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
221-
processSet: ProcessSet(),
222-
fileSystem: localFileSystem,
223-
env: ProcessEnv.vars)
239+
224240
var driver = try Driver(args: commandLine, executor: executor,
225241
externalBuildArtifacts: (targetModulePathMap, [:]),
226242
interModuleDependencyOracle: dependencyOracle)
@@ -486,6 +502,59 @@ final class ExplicitModuleBuildTests: XCTestCase {
486502
#endif
487503
}
488504

505+
/// Test the libSwiftScan dependency scanning.
506+
func testDependencyScanning() throws {
507+
// Just instantiating to get at the toolchain path
508+
let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
509+
"-module-name", "testDependencyScanning",
510+
"test.swift"])
511+
let toolchainRootPath: AbsolutePath = try driver.toolchain.getToolPath(.swiftCompiler)
512+
.parentDirectory // bin
513+
.parentDirectory // toolchain root
514+
let stdLibPath = toolchainRootPath.appending(component: "lib")
515+
.appending(component: "swift")
516+
.appending(component: "macosx")
517+
let shimsPath = toolchainRootPath.appending(component: "lib")
518+
.appending(component: "swift")
519+
.appending(component: "shims")
520+
// The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
521+
// queries.
522+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
523+
toolchainPath: toolchainRootPath)
524+
525+
// Create a simple test case.
526+
try withTemporaryDirectory { path in
527+
let main = path.appending(component: "testDependencyScanning.swift")
528+
try localFileSystem.writeFileContents(main) {
529+
$0 <<< "import C;"
530+
$0 <<< "import E;"
531+
$0 <<< "import G;"
532+
}
533+
let packageRootPath = URL(fileURLWithPath: #file).pathComponents
534+
.prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
535+
let testInputsPath = packageRootPath + "/TestInputs"
536+
let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
537+
let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
538+
let scannerCommand = ["-scan-dependencies",
539+
"-I", cHeadersPath,
540+
"-I", swiftModuleInterfacesPath,
541+
"-I", stdLibPath.description,
542+
"-I", shimsPath.description,
543+
main.pathString]
544+
545+
// Dispatch several iterations in parallel
546+
DispatchQueue.concurrentPerform(iterations: 20) { index in
547+
// Give the main modules different names
548+
let iterationCommand = scannerCommand + ["-module-name",
549+
"testDependencyScanning\(index)"]
550+
let dependencyGraph =
551+
try! dependencyOracle.getDependencies(workingDirectory: path,
552+
commandLine: iterationCommand)
553+
XCTAssertTrue(dependencyGraph.modules.count == 11)
554+
}
555+
}
556+
}
557+
489558
func testDependencyGraphMerge() throws {
490559
let moduleDependencyGraph1 =
491560
try JSONDecoder().decode(

0 commit comments

Comments
 (0)