Skip to content

Commit eb69ee5

Browse files
artemcmartemcm
committed
[Explicit Module Builds] Add libSwiftScan parallel scanning query test.
1 parent 9735178 commit eb69ee5

File tree

6 files changed

+127
-49
lines changed

6 files changed

+127
-49
lines changed

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 23 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -390,25 +390,6 @@ public struct Driver {
390390
return ($0, modTime)
391391
})
392392

393-
// Create an instance of an inter-module dependency oracle, if the driver's
394-
// client did not provide one. The clients are expected to provide an oracle
395-
// when they wish to share module dependency information across targets.
396-
if let dependencyOracle = interModuleDependencyOracle {
397-
self.interModuleDependencyOracle = dependencyOracle
398-
} else {
399-
self.interModuleDependencyOracle =
400-
try InterModuleDependencyOracle(fileSystem: fileSystem,
401-
toolchainPath: AbsolutePath("/Volumes/Data/GHWorkspace2/build/Ninja-RelWithDebInfoAssert/swift-macosx-x86_64"))
402-
403-
// This is a shim for backwards-compatibility with ModuleInfoMap-based API
404-
// used by SwiftPM
405-
if let externalArtifacts = externalBuildArtifacts {
406-
if !externalArtifacts.1.isEmpty {
407-
try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
408-
}
409-
}
410-
}
411-
412393
do {
413394
let outputFileMap: OutputFileMap?
414395
// Initialize an empty output file map, which will be populated when we start creating jobs.
@@ -430,6 +411,29 @@ public struct Driver {
430411
}
431412
}
432413

414+
// Create an instance of an inter-module dependency oracle, if the driver's
415+
// client did not provide one. The clients are expected to provide an oracle
416+
// when they wish to share module dependency information across targets.
417+
if let dependencyOracle = interModuleDependencyOracle {
418+
self.interModuleDependencyOracle = dependencyOracle
419+
} else {
420+
// For now, we expect to find the library in the same toolchain as the compiler itself
421+
let toolchainRootPath: AbsolutePath = try toolchain.getToolPath(.swiftCompiler)
422+
.parentDirectory // bin
423+
.parentDirectory // toolchain root
424+
self.interModuleDependencyOracle =
425+
try InterModuleDependencyOracle(fileSystem: fileSystem,
426+
toolchainPath: toolchainRootPath)
427+
428+
// This is a shim for backwards-compatibility with ModuleInfoMap-based API
429+
// used by SwiftPM
430+
if let externalArtifacts = externalBuildArtifacts {
431+
if !externalArtifacts.1.isEmpty {
432+
try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1)
433+
}
434+
}
435+
}
436+
433437
self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine)
434438
self.shouldUseInputFileList = inputFiles.count > fileListThreshold
435439
if shouldUseInputFileList {

‎Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -42,24 +42,17 @@ public class InterModuleDependencyOracle {
4242
swiftScanLibInstance = try SwiftScan(dylib: swiftScanLibPath)
4343
}
4444

45-
/// Query the ModuleInfo of a module with a given ID
46-
internal func getExternalModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? {
47-
self.lock.withLock {
48-
return externalModules[moduleId]
49-
}
50-
}
51-
52-
internal func getDependencies(workingDirectory: AbsolutePath,
53-
commandLine: [String]) throws -> InterModuleDependencyGraph {
45+
public func getDependencies(workingDirectory: AbsolutePath,
46+
commandLine: [String]) throws -> InterModuleDependencyGraph {
5447
try self.lock.withLock {
5548
return try swiftScanLibInstance.scanDependencies(workingDirectory: workingDirectory,
5649
invocationCommand: commandLine)
5750
}
5851
}
5952

60-
internal func getBatchDependencies(workingDirectory: AbsolutePath,
61-
commandLine: [String],
62-
batchInfos: [BatchScanModuleInfo])
53+
public func getBatchDependencies(workingDirectory: AbsolutePath,
54+
commandLine: [String],
55+
batchInfos: [BatchScanModuleInfo])
6356
throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
6457
try self.lock.withLock {
6558
return try swiftScanLibInstance.batchScanDependencies(workingDirectory: workingDirectory,
@@ -70,14 +63,22 @@ public class InterModuleDependencyOracle {
7063

7164
internal let lock = Lock()
7265

66+
/// A reference to an instance of the compiler's libSwiftScan shared library
67+
private let swiftScanLibInstance: SwiftScan
68+
69+
// The below API is a legacy implementation of the oracle that is in-place to allow clients to
70+
// transition to the new API. It is to be removed once that transition is complete.
7371
/// The complete set of modules discovered so far, spanning potentially multiple targets,
7472
/// accumulated across builds of multiple targets.
7573
/// TODO: This is currently only used for placeholder resolution. libSwiftScan should allow us to move away
7674
/// from the concept of a placeholder module so we should be able to get rid of this in the future.
7775
internal var externalModules: ModuleInfoMap = [:]
78-
79-
/// A reference to an instance of the compiler's libSwiftScan shared library
80-
private let swiftScanLibInstance: SwiftScan
76+
/// Query the ModuleInfo of a module with a given ID
77+
public func getExternalModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? {
78+
self.lock.withLock {
79+
return externalModules[moduleId]
80+
}
81+
}
8182
}
8283

8384
// This is a shim for backwards-compatibility with existing API used by SwiftPM.

‎Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ import Foundation
6161
/// - Module Name
6262
/// - Extra PCM build arguments (for Clang modules only)
6363
/// - Dependency graph output path
64-
internal enum BatchScanModuleInfo: Encodable {
64+
public enum BatchScanModuleInfo: Encodable {
6565
case swift(BatchScanSwiftModuleInfo)
6666
case clang(BatchScanClangModuleInfo)
6767
}
6868

69-
internal struct BatchScanSwiftModuleInfo: Encodable {
69+
public struct BatchScanSwiftModuleInfo: Encodable {
7070
var swiftModuleName: String
7171
var output: String
7272

@@ -76,7 +76,7 @@ internal struct BatchScanSwiftModuleInfo: Encodable {
7676
}
7777
}
7878

79-
internal struct BatchScanClangModuleInfo: Encodable {
79+
public struct BatchScanClangModuleInfo: Encodable {
8080
var clangModuleName: String
8181
var arguments: String
8282
var output: String
@@ -88,7 +88,7 @@ internal struct BatchScanClangModuleInfo: Encodable {
8888
}
8989
}
9090

91-
internal extension BatchScanModuleInfo {
91+
public extension BatchScanModuleInfo {
9292
func encode(to encoder: Encoder) throws {
9393
var container = encoder.singleValueContainer()
9494
switch self {

‎Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,16 @@ internal extension SwiftScan {
4242
return resultGraph
4343
}
4444

45+
/// From a reference to a binary-format dependency graph collection returned by libSwiftScan batch scan query,
46+
/// corresponding to the specified batch scan input (`BatchScanModuleInfo`), construct instances of
47+
/// `InterModuleDependencyGraph` for each result.
4548
func constructBatchResultGraphs(for batchInfos: [BatchScanModuleInfo],
4649
from batchResultRef: swiftscan_batch_scan_result_t) throws
4750
-> [ModuleDependencyId: [InterModuleDependencyGraph]] {
4851
var resultMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = [:]
4952
let resultGraphRefArray = Array(UnsafeBufferPointer(start: batchResultRef.results,
5053
count: Int(batchResultRef.count)))
54+
// Note, respective indeces of the batch scan input and the returned result must be aligned.
5155
for (index, resultGraphRefOrNull) in resultGraphRefArray.enumerated() {
5256
guard let resultGraphRef = resultGraphRefOrNull else {
5357
throw DependencyScanningError.dependencyScanFailed
@@ -61,6 +65,7 @@ internal extension SwiftScan {
6165
case .clang(let clangModuleBatchScanInfo):
6266
moduleId = .clang(clangModuleBatchScanInfo.clangModuleName)
6367
}
68+
// Update the map with either yet another graph or create an entry for this module
6469
if resultMap[moduleId] != nil {
6570
resultMap[moduleId]!.append(decodedGraph)
6671
} else {
@@ -72,8 +77,7 @@ internal extension SwiftScan {
7277
}
7378

7479
private extension SwiftScan {
75-
76-
/// From a reference to a binary-format module dependency info returned by libSwiftScan,
80+
/// From a reference to a binary-format module dependency module info returned by libSwiftScan,
7781
/// construct an instance of an `ModuleInfo` as used by the driver
7882
func constructModuleInfo(from moduleInfoRef: swiftscan_dependency_info_t)
7983
throws -> (ModuleDependencyId, ModuleInfo) {
@@ -82,6 +86,7 @@ private extension SwiftScan {
8286
try toSwiftString(api.swiftscan_module_info_get_module_name(moduleInfoRef))
8387
let moduleId = try decodeModuleNameAndKind(from: encodedModuleName)
8488

89+
// Decode module path and source file locations
8590
let modulePath = try toSwiftString(api.swiftscan_module_info_get_module_path(moduleInfoRef))
8691
let sourceFiles: [String]?
8792
if let sourceFilesSetRef = api.swiftscan_module_info_get_source_files(moduleInfoRef) {
@@ -90,6 +95,7 @@ private extension SwiftScan {
9095
sourceFiles = nil
9196
}
9297

98+
// Decode all dependencies of this module
9399
let directDependencies: [ModuleDependencyId]?
94100
if let encodedDirectDepsRef = api.swiftscan_module_info_get_direct_dependencies(moduleInfoRef) {
95101
let encodedDirectDependencies = try toSwiftStringArray(encodedDirectDepsRef.pointee)

‎Sources/SwiftDriver/SwiftScan/SwiftScan.swift

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//===--- MultiJobExecutor.swift - Builtin DriverExecutor implementation ---===//
1+
//===------------------------ SwiftScan.swift -----------------------------===//
22
//
33
// This source file is part of the Swift.org open source project
44
//
@@ -10,7 +10,6 @@
1010
//
1111
//===----------------------------------------------------------------------===//
1212

13-
// AC: Consider making this implementation-only
1413
@_implementationOnly import CSwiftScan
1514

1615
import TSCUtility
@@ -65,7 +64,6 @@ internal final class SwiftScan {
6564

6665
init(dylib path: AbsolutePath) throws {
6766
self.path = path
68-
print("OPENINGNG : \(path.description)")
6967
#if os(Windows)
7068
self.dylib = try dlopen(path.pathString, mode: [])
7169
#else

‎Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

Lines changed: 75 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,11 @@ final class ExplicitModuleBuildTests: XCTestCase {
159159
try JSONDecoder().decode(
160160
InterModuleDependencyGraph.self,
161161
from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
162-
let dependencyOracle = InterModuleDependencyOracle()
162+
let toolchainRootPath: AbsolutePath = try driver.toolchain.getToolPath(.swiftCompiler)
163+
.parentDirectory // bin
164+
.parentDirectory // toolchain root
165+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
166+
toolchainPath: toolchainRootPath)
163167
try dependencyOracle.mergeModules(from: moduleDependencyGraph)
164168
driver.explicitDependencyBuildPlanner =
165169
try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph,
@@ -204,7 +208,22 @@ final class ExplicitModuleBuildTests: XCTestCase {
204208
from: ModuleDependenciesInputs.bPlaceHolderInput.data(using: .utf8)!)
205209
let targetModulePathMap: ExternalTargetModulePathMap =
206210
[ModuleDependencyId.swiftPlaceholder("B"):AbsolutePath("/Somewhere/B.swiftmodule")]
207-
let dependencyOracle = InterModuleDependencyOracle()
211+
212+
let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
213+
processSet: ProcessSet(),
214+
fileSystem: localFileSystem,
215+
env: ProcessEnv.vars)
216+
var toolchain: Toolchain
217+
#if os(macOS)
218+
toolchain = DarwinToolchain(env: ProcessEnv.vars, executor: executor)
219+
#else
220+
toolchain = GenericUnixToolchain(env: ProcessEnv.vars, executor: executor)
221+
#endif
222+
let toolchainRootPath: AbsolutePath = try toolchain.getToolPath(.swiftCompiler)
223+
.parentDirectory // bin
224+
.parentDirectory // toolchain root
225+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
226+
toolchainPath: toolchainRootPath)
208227
try dependencyOracle.mergeModules(from: inputDependencyGraph)
209228

210229
// Construct a module dependency graph that will contain .swiftPlaceholder("B"),
@@ -217,10 +236,7 @@ final class ExplicitModuleBuildTests: XCTestCase {
217236
// Construct the driver with explicit external dependency input
218237
let commandLine = ["swiftc", "-experimental-explicit-module-build",
219238
"test.swift", "-module-name", "A", "-g"]
220-
let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]),
221-
processSet: ProcessSet(),
222-
fileSystem: localFileSystem,
223-
env: ProcessEnv.vars)
239+
224240
var driver = try Driver(args: commandLine, executor: executor,
225241
externalBuildArtifacts: (targetModulePathMap, [:]),
226242
interModuleDependencyOracle: dependencyOracle)
@@ -407,6 +423,59 @@ final class ExplicitModuleBuildTests: XCTestCase {
407423
#endif
408424
}
409425

426+
/// Test the libSwiftScan dependency scanning.
427+
func testDependencyScanning() throws {
428+
// Just instantiating to get at the toolchain path
429+
let driver = try Driver(args: ["swiftc", "-experimental-explicit-module-build",
430+
"-module-name", "testDependencyScanning",
431+
"test.swift"])
432+
let toolchainRootPath: AbsolutePath = try driver.toolchain.getToolPath(.swiftCompiler)
433+
.parentDirectory // bin
434+
.parentDirectory // toolchain root
435+
let stdLibPath = toolchainRootPath.appending(component: "lib")
436+
.appending(component: "swift")
437+
.appending(component: "macosx")
438+
let shimsPath = toolchainRootPath.appending(component: "lib")
439+
.appending(component: "swift")
440+
.appending(component: "shims")
441+
// The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across
442+
// queries.
443+
let dependencyOracle = try InterModuleDependencyOracle(fileSystem: localFileSystem,
444+
toolchainPath: toolchainRootPath)
445+
446+
// Create a simple test case.
447+
try withTemporaryDirectory { path in
448+
let main = path.appending(component: "testDependencyScanning.swift")
449+
try localFileSystem.writeFileContents(main) {
450+
$0 <<< "import C;"
451+
$0 <<< "import E;"
452+
$0 <<< "import G;"
453+
}
454+
let packageRootPath = URL(fileURLWithPath: #file).pathComponents
455+
.prefix(while: { $0 != "Tests" }).joined(separator: "/").dropFirst()
456+
let testInputsPath = packageRootPath + "/TestInputs"
457+
let cHeadersPath : String = testInputsPath + "/ExplicitModuleBuilds/CHeaders"
458+
let swiftModuleInterfacesPath : String = testInputsPath + "/ExplicitModuleBuilds/Swift"
459+
let scannerCommand = ["-scan-dependencies",
460+
"-I", cHeadersPath,
461+
"-I", swiftModuleInterfacesPath,
462+
"-I", stdLibPath.description,
463+
"-I", shimsPath.description,
464+
main.pathString]
465+
466+
// Dispatch several iterations in parallel
467+
DispatchQueue.concurrentPerform(iterations: 20) { index in
468+
// Give the main modules different names
469+
let iterationCommand = scannerCommand + ["-module-name",
470+
"testDependencyScanning\(index)"]
471+
let dependencyGraph =
472+
try! dependencyOracle.getDependencies(workingDirectory: path,
473+
commandLine: iterationCommand)
474+
XCTAssertTrue(dependencyGraph.modules.count == 11)
475+
}
476+
}
477+
}
478+
410479
func testDependencyGraphMerge() throws {
411480
let moduleDependencyGraph1 =
412481
try JSONDecoder().decode(

0 commit comments

Comments
 (0)