Skip to content

Commit bbaf4b9

Browse files
artemcmartemcm
committed
[WIP] Print commands required to explicitly build module dependencies.
This PR resurrects #47 and builds on top of it. Adds new flag -driver-print-module-dependencies-jobs to instruct the driver to scan for dependencies and generate build jobs for each dependent Swift or C module. At present, this WIP does the following: - Invoke the Swift frontend Fast Dependency Scanner to prescan for dependencies. - Construct a Module Dependency Graph from the dependency scanner's JSON output. - Compute the order in which dependent modules should be built. - Generate and print build jbos for each dependent module. Next Steps: Actually have the driver schedule and execute the dependent module build jobs ahead of time, eliminating the need to re-build those modules when it tries to import them.
1 parent 573a322 commit bbaf4b9

File tree

9 files changed

+413
-7
lines changed

9 files changed

+413
-7
lines changed

‎Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
//===--------------- ModuleDependencyBuildGeneration.swift ----------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import TSCBasic
13+
import TSCUtility
14+
import Foundation
15+
16+
extension ModuleDependencyGraph {
17+
/// A topologically-ordered list of dependent modules to be built.
18+
nonmutating func topologicalBuildOrder() -> [ModuleDependencyId] {
19+
return topologicalBuildBatchOrder().flatMap { $0 }
20+
}
21+
22+
/// Use Kahn's algorithm to determine the order in which the dependent
23+
/// modules should be built.
24+
nonmutating func topologicalBuildBatchOrder() -> [[ModuleDependencyId]] {
25+
var orderedModuleBatchList : [[ModuleDependencyId]] = []
26+
// A worklist of modules and their dependencies
27+
var moduleDependenciesMap = modules.mapValues { value in Set(value.directDependencies) }
28+
// Find modules without any dependencies and remove them from the worklist
29+
var currentBatch = Array(moduleDependenciesMap.filter { $0.value.isEmpty }.keys)
30+
var nextBatch : [ModuleDependencyId] = []
31+
currentBatch.forEach { moduleDependenciesMap.removeValue(forKey: $0) }
32+
orderedModuleBatchList.append(currentBatch)
33+
34+
// Proceed until there are no more independent modules to be built
35+
while (!currentBatch.isEmpty) {
36+
let readyModule = currentBatch.removeFirst()
37+
// Remove dependency on `readyModule` from all other modules
38+
for key in moduleDependenciesMap.keys {
39+
var deps = moduleDependenciesMap[key]
40+
deps?.remove(readyModule)
41+
moduleDependenciesMap[key] = deps
42+
}
43+
44+
// Add newly 'freed' modules to the the next batch
45+
for newlyIndependentModule in moduleDependenciesMap.filter({ $0.value.isEmpty }) {
46+
nextBatch.append(newlyIndependentModule.key)
47+
moduleDependenciesMap.removeValue(forKey: newlyIndependentModule.key)
48+
}
49+
50+
// This batch is done, write it to the output and move on to the next one
51+
if (currentBatch.isEmpty && !nextBatch.isEmpty) {
52+
currentBatch = nextBatch
53+
nextBatch.removeAll()
54+
orderedModuleBatchList.append(currentBatch)
55+
}
56+
}
57+
58+
// If there are still modules left to build, the module dependencies
59+
// must be malformed (contain cycles).
60+
assert(moduleDependenciesMap.isEmpty, "Cycles in module dependency graph detected.")
61+
return orderedModuleBatchList
62+
}
63+
}
64+
65+
extension Driver {
66+
/// For the current moduleDependencyGraph, plan the order and generate jobs
67+
/// for explicitly building all dependency modules.
68+
mutating func planExplicitModuleDependenciesCompile() throws -> [Job] {
69+
var jobs: [Job] = []
70+
guard let dependencyGraph = moduleDependencyGraph else { return jobs }
71+
let moduleBuildOrder = dependencyGraph.topologicalBuildOrder()
72+
for id in moduleBuildOrder {
73+
switch id {
74+
case .swift(let moduleName):
75+
if let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleId: id,
76+
moduleName: moduleName) {
77+
jobs.append(swiftModuleBuildJob)
78+
}
79+
case .clang(let moduleName):
80+
if let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleId: id,
81+
moduleName: moduleName) {
82+
jobs.append(clangModuleBuildJob)
83+
}
84+
}
85+
}
86+
return jobs
87+
}
88+
89+
/// For a given swift module dependency, generate a build job
90+
mutating private func genSwiftModuleDependencyBuildJob(moduleId: ModuleDependencyId,
91+
moduleName: String) throws -> Job? {
92+
// FIXIT: Needs more error handling
93+
guard let moduleInfo = moduleDependencyGraph!.modules[moduleId] else {
94+
fatalError("Constructing build job for unknown module: \(moduleName)")
95+
}
96+
guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
97+
fatalError("Malformed Swift Module Dependency: \(moduleName), no details object.")
98+
}
99+
100+
var inputs: [TypedVirtualPath] = []
101+
var outputs: [TypedVirtualPath] = [
102+
TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule)
103+
]
104+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
105+
commandLine.appendFlag("-frontend")
106+
107+
// If source files are given, compile them into a module re-using the
108+
// current compile's options
109+
if (!moduleInfo.sourceFiles.isEmpty) {
110+
commandLine.appendFlags("-emit-module", "-module-name", moduleName)
111+
commandLine.appendFlag("-emit-module-path=" + moduleInfo.modulePath)
112+
commandLine.append(contentsOf: try moduleInfo.sourceFiles.map {
113+
Job.ArgTemplate.path(try VirtualPath(path: $0))})
114+
inputs.append(contentsOf: try moduleInfo.sourceFiles.map {
115+
TypedVirtualPath(file: try VirtualPath(path: $0), type: .swift) })
116+
try addCommonFrontendOptions(commandLine: &commandLine)
117+
try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
118+
} else {
119+
// If no source files are provided, there must be a .swiftinterfaces file
120+
// and a list of command line options specified in the `details` field.
121+
guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else {
122+
fatalError("Malformed Swift Module Dependency: \(moduleName), no moduleInterfacePath object.")
123+
}
124+
inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath),
125+
type: .swiftInterface))
126+
try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
127+
swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) }
128+
}
129+
130+
return Job(
131+
kind: .emitModule,
132+
tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
133+
commandLine: commandLine,
134+
inputs: inputs,
135+
outputs: outputs
136+
)
137+
}
138+
139+
/// For a given clang module dependency, generate a build job
140+
mutating private func genClangModuleDependencyBuildJob(moduleId: ModuleDependencyId,
141+
moduleName: String) throws -> Job? {
142+
// For clang modules, the Fast Dependency Scanner emits a list of source
143+
// files (with a .modulemap among them), and a list of compile command
144+
// options.
145+
// FIXIT: Needs more error handling
146+
guard var moduleInfo = moduleDependencyGraph!.modules[moduleId] else {
147+
fatalError("Constructing build job for unknown module: \(moduleName)")
148+
}
149+
guard case .clang(let clangModuleDetails) = moduleInfo.details else {
150+
fatalError("Malformed Clang Module Dependency: \(moduleName), no details object.")
151+
}
152+
var inputs: [TypedVirtualPath] = []
153+
var outputs: [TypedVirtualPath] = [
154+
TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm)
155+
]
156+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
157+
commandLine.appendFlag("-frontend")
158+
commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
159+
160+
// The .modulemap is currently duplicated in moduleMapPath and in sourceFiles.
161+
// It also must be the very first input argument to the compile command.
162+
if let index = moduleInfo.sourceFiles.firstIndex(of: clangModuleDetails.moduleMapPath) {
163+
moduleInfo.sourceFiles.swapAt(0, index)
164+
}
165+
commandLine.append(contentsOf: try moduleInfo.sourceFiles.map {
166+
Job.ArgTemplate.path(try VirtualPath(path: $0))})
167+
inputs.append(contentsOf: try moduleInfo.sourceFiles.map {
168+
TypedVirtualPath(file: try VirtualPath(path: $0), type: .swift) })
169+
try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
170+
clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) }
171+
172+
return Job(
173+
kind: .generatePCM,
174+
tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
175+
commandLine: commandLine,
176+
inputs: inputs,
177+
outputs: outputs
178+
)
179+
}
180+
}

‎Sources/SwiftDriver/Dependency Scanning/ModuleDependencyGraph.swift

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
//===--------------- ModuleDependencyGraph.swift --------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import Foundation
13+
14+
15+
enum ModuleDependencyId: Hashable {
16+
case swift(String)
17+
case clang(String)
18+
19+
var moduleName: String {
20+
switch self {
21+
case .swift(let name): return name
22+
case .clang(let name): return name
23+
}
24+
}
25+
}
26+
27+
extension ModuleDependencyId: Codable {
28+
enum CodingKeys: CodingKey {
29+
case swift
30+
case clang
31+
}
32+
33+
init(from decoder: Decoder) throws {
34+
let container = try decoder.container(keyedBy: CodingKeys.self)
35+
do {
36+
let moduleName = try container.decode(String.self, forKey: .swift)
37+
self = .swift(moduleName)
38+
} catch {
39+
let moduleName = try container.decode(String.self, forKey: .clang)
40+
self = .clang(moduleName)
41+
}
42+
}
43+
44+
func encode(to encoder: Encoder) throws {
45+
var container = encoder.container(keyedBy: CodingKeys.self)
46+
switch self {
47+
case .swift(let moduleName):
48+
try container.encode(moduleName, forKey: .swift)
49+
case .clang(let moduleName):
50+
try container.encode(moduleName, forKey: .clang)
51+
}
52+
}
53+
}
54+
55+
/// Details specific to Swift modules.
56+
struct SwiftModuleDetails: Codable {
57+
/// The module interface from which this module was built, if any.
58+
var moduleInterfacePath: String?
59+
60+
/// The bridging header, if any.
61+
var bridgingHeaderPath: String?
62+
63+
/// The source files referenced by the bridging header.
64+
var bridgingSourceFiles: [String]? = []
65+
66+
/// Options to the compile command
67+
var commandLine: [String]? = []
68+
}
69+
70+
/// Details specific to Clang modules.
71+
struct ClangModuleDetails: Codable {
72+
/// The path to the module map used to build this module.
73+
var moduleMapPath: String
74+
75+
/// clang-generated context hash
76+
var contextHash: String?
77+
78+
/// Options to the compile command
79+
var commandLine: [String]? = []
80+
}
81+
82+
struct ModuleInfo: Codable {
83+
/// The path for the module.
84+
var modulePath: String
85+
86+
/// The source files used to build this module.
87+
var sourceFiles: [String] = []
88+
89+
/// The set of direct module dependencies of this module.
90+
var directDependencies: [ModuleDependencyId] = []
91+
92+
/// Specific details of a particular kind of module.
93+
var details: Details
94+
95+
/// Specific details of a particular kind of module.
96+
enum Details {
97+
/// Swift modules may be built from a module interface, and may have
98+
/// a bridging header.
99+
case swift(SwiftModuleDetails)
100+
101+
/// Clang modules are built from a module map file.
102+
case clang(ClangModuleDetails)
103+
}
104+
}
105+
106+
extension ModuleInfo.Details: Codable {
107+
enum CodingKeys: CodingKey {
108+
case swift
109+
case clang
110+
}
111+
112+
init(from decoder: Decoder) throws {
113+
let container = try decoder.container(keyedBy: CodingKeys.self)
114+
do {
115+
let details = try container.decode(SwiftModuleDetails.self, forKey: .swift)
116+
self = .swift(details)
117+
} catch {
118+
let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
119+
self = .clang(details)
120+
}
121+
}
122+
123+
func encode(to encoder: Encoder) throws {
124+
var container = encoder.container(keyedBy: CodingKeys.self)
125+
switch self {
126+
case .swift(let details):
127+
try container.encode(details, forKey: .swift)
128+
case .clang(let details):
129+
try container.encode(details, forKey: .clang)
130+
}
131+
}
132+
}
133+
134+
/// Describes the complete set of dependencies for a Swift module, including
135+
/// all of the Swift and C modules and source files it depends on.
136+
struct ModuleDependencyGraph: Codable {
137+
/// The name of the main module.
138+
var mainModuleName: String
139+
140+
/// The complete set of modules discovered
141+
var modules: [ModuleDependencyId: ModuleInfo] = [:]
142+
143+
/// Information about the main module.
144+
var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
145+
}

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,11 @@ public struct Driver {
194194
/// This will force the driver to first emit the module and then run compile jobs.
195195
public var forceEmitModuleInSingleInvocation: Bool = false
196196

197+
/// The module dependency graph, which is populated during the planning phase
198+
/// only when all modules will be prebuilt and treated as explicit by the
199+
/// various compilation jobs.
200+
var moduleDependencyGraph: ModuleDependencyGraph? = nil
201+
197202
/// Handler for emitting diagnostics to stderr.
198203
public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in
199204
let stream = stderrStream

‎Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
//===--------------- ModuleDependencyScanning.swift -----------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import Foundation
13+
import TSCBasic
14+
15+
extension Driver {
16+
/// Precompute the dependencies for a given Swift compilation, producing a
17+
/// complete dependency graph including all Swift and C module files and
18+
/// source files.
19+
// TODO: Instead of directly invoking the frontend,
20+
// treat this as any other `Job`.
21+
mutating func computeModuleDependencyGraph() throws
22+
-> ModuleDependencyGraph? {
23+
// Grab the swift compiler
24+
let resolver = try ArgsResolver()
25+
let compilerPath = VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler))
26+
let tool = try resolver.resolve(.path(compilerPath))
27+
28+
// Aggregate the fast dependency scanner arguments
29+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
30+
commandLine.appendFlag("-frontend")
31+
commandLine.appendFlag("-scan-dependencies")
32+
if parsedOptions.hasArgument(.parseStdlib) {
33+
commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule)
34+
}
35+
try addCommonFrontendOptions(commandLine: &commandLine)
36+
// FIXME: MSVC runtime flags
37+
38+
// Pass on the input files
39+
commandLine.append(contentsOf: inputFiles.map { .path($0.file)})
40+
41+
// Execute dependency scanner and decode its output
42+
let arguments = [tool] + (try commandLine.map { try resolver.resolve($0) })
43+
let scanProcess = try Process.launchProcess(arguments: arguments, env: env)
44+
let result = try scanProcess.waitUntilExit()
45+
guard let outputData = try? Data(result.utf8Output().utf8) else {
46+
return nil
47+
}
48+
return try JSONDecoder().decode(ModuleDependencyGraph.self, from: outputData)
49+
}
50+
}

0 commit comments

Comments
 (0)