Skip to content

Commit 0d96570

Browse files
artemcmartemcm
authored
Merge pull request #2827 from artemcm/ExplicitPackageBuilds
[Explicit Module Builds] Support explicit builds of packages witn inter-dependent Swift targets
2 parents 77c7a24 + a4e21bd commit 0d96570

File tree

2 files changed

+329
-90
lines changed

2 files changed

+329
-90
lines changed

‎Sources/Build/ManifestBuilder.swift

Lines changed: 219 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public class LLBuildManifestBuilder {
4646
self.plan = plan
4747
}
4848

49+
// MARK:- Generate Manifest
4950
/// Generate manifest at the given path.
5051
public func generateManifest(at path: AbsolutePath) throws {
5152
manifest.createTarget(TargetKind.main.targetName)
@@ -54,14 +55,22 @@ public class LLBuildManifestBuilder {
5455

5556
addPackageStructureCommand()
5657
addBinaryDependencyCommands()
57-
58-
// Create commands for all target descriptions in the plan.
59-
for (_, description) in plan.targetMap {
60-
switch description {
61-
case .swift(let desc):
62-
createSwiftCompileCommand(desc)
63-
case .clang(let desc):
64-
createClangCompileCommand(desc)
58+
if buildParameters.useExplicitModuleBuild {
59+
// Explicit module builds use the integrated driver directly and
60+
// require that every target's build jobs specify its dependencies explicitly to plan
61+
// its build.
62+
// Currently behind:
63+
// --experimental-explicit-module-build
64+
try addTargetsToExplicitBuildManifest()
65+
} else {
66+
// Create commands for all target descriptions in the plan.
67+
for (_, description) in plan.targetMap {
68+
switch description {
69+
case .swift(let desc):
70+
try createSwiftCompileCommand(desc)
71+
case .clang(let desc):
72+
createClangCompileCommand(desc)
73+
}
6574
}
6675
}
6776

@@ -177,7 +186,7 @@ extension LLBuildManifestBuilder {
177186
/// Create a llbuild target for a Swift target description.
178187
private func createSwiftCompileCommand(
179188
_ target: SwiftTargetBuildDescription
180-
) {
189+
) throws {
181190
// Inputs.
182191
let inputs = computeSwiftCompileCmdInputs(target)
183192

@@ -187,7 +196,7 @@ extension LLBuildManifestBuilder {
187196
let cmdOutputs = objectNodes + [moduleNode]
188197

189198
if buildParameters.useIntegratedSwiftDriver {
190-
addSwiftCmdsViaIntegratedDriver(target, inputs: inputs, objectNodes: objectNodes, moduleNode: moduleNode)
199+
try addSwiftCmdsViaIntegratedDriver(target, inputs: inputs, objectNodes: objectNodes, moduleNode: moduleNode)
191200
} else if buildParameters.emitSwiftModuleSeparately {
192201
addSwiftCmdsEmitSwiftModuleSeparately(target, inputs: inputs, objectNodes: objectNodes, moduleNode: moduleNode)
193202
} else {
@@ -203,75 +212,208 @@ extension LLBuildManifestBuilder {
203212
inputs: [Node],
204213
objectNodes: [Node],
205214
moduleNode: Node
206-
) {
207-
do {
208-
// Use the integrated Swift driver to compute the set of frontend
209-
// jobs needed to build this Swift target.
210-
var commandLine = target.emitCommandLine();
211-
commandLine.append("-driver-use-frontend-path")
212-
commandLine.append(buildParameters.toolchain.swiftCompiler.pathString)
213-
if buildParameters.useExplicitModuleBuild {
214-
commandLine.append("-experimental-explicit-module-build")
215+
) throws {
216+
// Use the integrated Swift driver to compute the set of frontend
217+
// jobs needed to build this Swift target.
218+
var commandLine = target.emitCommandLine();
219+
commandLine.append("-driver-use-frontend-path")
220+
commandLine.append(buildParameters.toolchain.swiftCompiler.pathString)
221+
// FIXME: At some point SwiftPM should provide its own executor for
222+
// running jobs/launching processes during planning
223+
let executor = try SwiftDriverExecutor(diagnosticsEngine: plan.diagnostics,
224+
processSet: ProcessSet(),
225+
fileSystem: target.fs,
226+
env: ProcessEnv.vars)
227+
var driver = try Driver(args: commandLine,
228+
diagnosticsEngine: plan.diagnostics,
229+
fileSystem: target.fs,
230+
executor: executor)
231+
let jobs = try driver.planBuild()
232+
try addSwiftDriverJobs(for: target, jobs: jobs, inputs: inputs,
233+
isMainModule: { driver.isExplicitMainModuleJob(job: $0)})
234+
}
235+
236+
private func addSwiftDriverJobs(for targetDescription: SwiftTargetBuildDescription,
237+
jobs: [Job], inputs: [Node],
238+
isMainModule: (Job) -> Bool) throws {
239+
// Add build jobs to the manifest
240+
let resolver = try ArgsResolver(fileSystem: targetDescription.fs)
241+
for job in jobs {
242+
let tool = try resolver.resolve(.path(job.tool))
243+
let commandLine = try job.commandLine.map{ try resolver.resolve($0) }
244+
let arguments = [tool] + commandLine
245+
246+
let jobInputs = job.inputs.map { $0.resolveToNode() }
247+
let jobOutputs = job.outputs.map { $0.resolveToNode() }
248+
249+
// Add target dependencies as inputs to the main module build command.
250+
//
251+
// Jobs for a target's intermediate build artifacts, such as PCMs or
252+
// modules built from a .swiftinterface, do not have a
253+
// dependency on cross-target build products. If multiple targets share
254+
// common intermediate dependency modules, such dependencies can lead
255+
// to cycles in the resulting manifest.
256+
var manifestNodeInputs : [Node] = []
257+
if buildParameters.useExplicitModuleBuild && !isMainModule(job) {
258+
manifestNodeInputs = jobInputs
259+
} else {
260+
manifestNodeInputs = (inputs + jobInputs).uniqued()
215261
}
216-
// FIXME: At some point SwiftPM should provide its own executor for
217-
// running jobs/launching processes during planning
218-
let executor = try SwiftDriverExecutor(diagnosticsEngine: plan.diagnostics,
219-
processSet: ProcessSet(),
220-
fileSystem: target.fs,
221-
env: ProcessEnv.vars)
222-
var driver = try Driver(args: commandLine,
223-
diagnosticsEngine: plan.diagnostics,
224-
fileSystem: target.fs,
225-
executor: executor)
226-
let jobs = try driver.planBuild()
227-
let resolver = try ArgsResolver(fileSystem: target.fs)
228-
229-
for job in jobs {
230-
let tool = try resolver.resolve(.path(job.tool))
231-
let commandLine = try job.commandLine.map{ try resolver.resolve($0) }
232-
let arguments = [tool] + commandLine
233-
234-
let jobInputs = job.inputs.map { $0.resolveToNode() }
235-
let jobOutputs = job.outputs.map { $0.resolveToNode() }
236-
237-
// Add target dependencies as inputs to the main module build command.
238-
//
239-
// Jobs for a target's intermediate build artifacts, such as PCMs or
240-
// modules built from a .swiftinterface, do not have a
241-
// dependency on cross-target build products. If multiple targets share
242-
// common intermediate dependency modules, such dependencies can lead
243-
// to cycles in the resulting manifest.
244-
var manifestNodeInputs : [Node] = []
245-
if buildParameters.useExplicitModuleBuild && !driver.isExplicitMainModuleJob(job: job) {
246-
manifestNodeInputs = jobInputs
247-
} else {
248-
manifestNodeInputs = (inputs + jobInputs).uniqued()
249-
}
250262

251-
let moduleName = target.target.c99name
252-
let description = job.description
253-
if job.kind.isSwiftFrontend {
254-
manifest.addSwiftFrontendCmd(
255-
name: jobOutputs.first!.name,
256-
moduleName: moduleName,
257-
description: description,
258-
inputs: manifestNodeInputs,
259-
outputs: jobOutputs,
260-
args: arguments
261-
)
262-
} else {
263-
manifest.addShellCmd(
264-
name: jobOutputs.first!.name,
265-
description: description,
266-
inputs: manifestNodeInputs,
267-
outputs: jobOutputs,
268-
args: arguments
269-
)
270-
}
271-
}
272-
} catch {
273-
fatalError("\(error)")
274-
}
263+
let moduleName = targetDescription.target.c99name
264+
let description = job.description
265+
if job.kind.isSwiftFrontend {
266+
manifest.addSwiftFrontendCmd(
267+
name: jobOutputs.first!.name,
268+
moduleName: moduleName,
269+
description: description,
270+
inputs: manifestNodeInputs,
271+
outputs: jobOutputs,
272+
args: arguments
273+
)
274+
} else {
275+
manifest.addShellCmd(
276+
name: jobOutputs.first!.name,
277+
description: description,
278+
inputs: manifestNodeInputs,
279+
outputs: jobOutputs,
280+
args: arguments
281+
)
282+
}
283+
}
284+
}
285+
286+
// Building a Swift module in Explicit Module Build mode requires passing all of its module
287+
// dependencies as explicit arguments to the build command. Thus, building a SwiftPM package
288+
// with multiple inter-dependent targets thus requires that each target’s build job must
289+
// have its target dependencies’ modules passed into it as explicit module dependencies.
290+
// Because none of the targets have been built yet, a given target's dependency scanning
291+
// action will not be able to discover its target dependencies' modules. Instead, it is
292+
// SwiftPM's responsibility to communicate to the driver, when planning a given target's
293+
// build, that this target has dependencies that are other targets, along with a list of
294+
// future artifacts of such dependencies (.swiftmodule and .pcm files).
295+
// The driver will then use those artifacts as explicit inputs to its module’s build jobs.
296+
//
297+
// Consider an example SwiftPM package with two targets: target B, and target A, where A
298+
// depends on B:
299+
// SwiftPM will process targets in a topological order and “bubble-up” each target’s
300+
// inter-module dependency graph to its dependees. First, SwiftPM will process B, and be
301+
// able to plan its full build because it does not have any target dependencies. Then the
302+
// driver is tasked with planning a build for A. SwiftPM will pass as input to the driver
303+
// the module dependency graph of its target’s dependencies, in this case, just the
304+
// dependency graph of B. The driver is then responsible for the necessary post-processing
305+
// to merge the dependency graphs and plan the build for A, using artifacts of B as explicit
306+
// inputs.
307+
public func addTargetsToExplicitBuildManifest() throws {
308+
// Sort the product targets in topological order in order to collect and "bubble up"
309+
// their respective dependency graphs to the depending targets.
310+
let nodes: [ResolvedTarget.Dependency] = plan.targetMap.keys.map { ResolvedTarget.Dependency.target($0, conditions: []) }
311+
let allTargets = try! topologicalSort(nodes, successors: { $0.dependencies })
312+
313+
// Collect all targets' dependency graphs
314+
var targetDepGraphMap : [ResolvedTarget: InterModuleDependencyGraph] = [:]
315+
316+
// Create commands for all target descriptions in the plan.
317+
for dependency in allTargets.reversed() {
318+
guard case .target(let target, _) = dependency else {
319+
// TODO
320+
fatalError("Product Dependencies not supported in Explicit Module Build Mode.")
321+
}
322+
guard let description = plan.targetMap[target] else {
323+
fatalError("Expected description for target: \(target)")
324+
}
325+
switch description {
326+
case .swift(let desc):
327+
try createExplicitSwiftTargetCompileCommand(description: desc,
328+
targetDepGraphMap: &targetDepGraphMap)
329+
case .clang(let desc):
330+
createClangCompileCommand(desc)
331+
}
332+
}
333+
}
334+
335+
private func createExplicitSwiftTargetCompileCommand(
336+
description: SwiftTargetBuildDescription,
337+
targetDepGraphMap: inout [ResolvedTarget: InterModuleDependencyGraph]
338+
) throws {
339+
// Inputs.
340+
let inputs = computeSwiftCompileCmdInputs(description)
341+
342+
// Outputs.
343+
let objectNodes = description.objects.map(Node.file)
344+
let moduleNode = Node.file(description.moduleOutputPath)
345+
let cmdOutputs = objectNodes + [moduleNode]
346+
347+
// Commands.
348+
try addExplicitBuildSwiftCmds(description, inputs: inputs,
349+
objectNodes: objectNodes,
350+
moduleNode: moduleNode,
351+
targetDepGraphMap: &targetDepGraphMap)
352+
353+
addTargetCmd(description, cmdOutputs: cmdOutputs)
354+
addModuleWrapCmd(description)
355+
}
356+
357+
private func addExplicitBuildSwiftCmds(
358+
_ targetDescription: SwiftTargetBuildDescription,
359+
inputs: [Node], objectNodes: [Node], moduleNode: Node,
360+
targetDepGraphMap: inout [ResolvedTarget: InterModuleDependencyGraph]
361+
) throws {
362+
// Pass the driver its external dependencies (target dependencies)
363+
let targetDependencyMap = collectTargetDependencyInfos(for: targetDescription.target,
364+
targetDepGraphMap: targetDepGraphMap)
365+
366+
// Compute the set of frontend
367+
// jobs needed to build this Swift target.
368+
var commandLine = targetDescription.emitCommandLine();
369+
commandLine.append("-driver-use-frontend-path")
370+
commandLine.append(buildParameters.toolchain.swiftCompiler.pathString)
371+
commandLine.append("-experimental-explicit-module-build")
372+
// FIXME: At some point SwiftPM should provide its own executor for
373+
// running jobs/launching processes during planning
374+
let executor = try SwiftDriverExecutor(diagnosticsEngine: plan.diagnostics,
375+
processSet: ProcessSet(),
376+
fileSystem: targetDescription.fs,
377+
env: ProcessEnv.vars)
378+
var driver = try Driver(args: commandLine, fileSystem: targetDescription.fs,
379+
executor: executor,
380+
externalModuleDependencies: targetDependencyMap)
381+
382+
let jobs = try driver.planBuild()
383+
384+
// Save the dependency graph of this target to be used by its dependents
385+
guard let dependencyGraph = driver.interModuleDependencyGraph else {
386+
fatalError("Expected module dependency graph for target: \(targetDescription)")
387+
}
388+
targetDepGraphMap[targetDescription.target] = dependencyGraph
389+
try addSwiftDriverJobs(for: targetDescription, jobs: jobs, inputs: inputs,
390+
isMainModule: { driver.isExplicitMainModuleJob(job: $0)})
391+
}
392+
393+
/// Collect a map from all target dependencies of the specified target to the build planning artifacts for said dependency,
394+
/// in the form of a path to a .swiftmodule file and the dependency's InterModuleDependencyGraph.
395+
private func collectTargetDependencyInfos(for target: ResolvedTarget,
396+
targetDepGraphMap: [ResolvedTarget: InterModuleDependencyGraph])
397+
-> SwiftDriver.ExternalDependencyArtifactMap {
398+
var targetDependencyMap : [ModuleDependencyId: (AbsolutePath, InterModuleDependencyGraph)] = [:]
399+
for dependency in target.dependencies {
400+
guard let dependencyTarget = dependency.target else {
401+
fatalError("Expected dependency target: \(dependency.description)")
402+
}
403+
// Only other swift targets will have corresponding dependency maps
404+
guard case .swift(let dependencySwiftTargetDescription) =
405+
plan.targetMap[dependencyTarget] else {
406+
continue
407+
}
408+
guard let dependencyGraph = targetDepGraphMap[dependencyTarget] else {
409+
fatalError("Expected dependency graph for target: \(dependency.description)")
410+
}
411+
let moduleName = dependencyTarget.name
412+
let dependencyModulePath = dependencySwiftTargetDescription.moduleOutputPath
413+
targetDependencyMap[ModuleDependencyId.swiftPlaceholder(moduleName)] =
414+
(dependencyModulePath, dependencyGraph)
415+
}
416+
return targetDependencyMap
275417
}
276418

277419
private func addSwiftCmdsEmitSwiftModuleSeparately(

0 commit comments

Comments
 (0)