Skip to content

Commit 0d10e67

Browse files
artemcmartemcm
authored
Merge pull request #104 from artemcm/ExplicitModuleDependencyJobGen
Print commands required to explicitly build module dependencies.
2 parents 1e4f9ef + d8c6c24 commit 0d10e67

File tree

11 files changed

+625
-7
lines changed

11 files changed

+625
-7
lines changed

‎Sources/SwiftDriver/CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,16 @@
77
# See http://swift.org/CONTRIBUTORS.txt for Swift project authors
88

99
add_library(SwiftDriver
10+
"Dependency Scanning/ModuleDependencyBuildGeneration.swift"
11+
"Dependency Scanning/InterModuleDependencyGraph.swift"
12+
1013
Driver/CompilerMode.swift
1114
Driver/DebugInfo.swift
1215
Driver/Driver.swift
1316
Driver/LinkKind.swift
1417
Driver/OutputFileMap.swift
1518
Driver/ToolExecutionDelegate.swift
19+
Driver/ModuleDependencyScanning.swift
1620

1721
Execution/JobExecutor.swift
1822
Execution/ParsableOutput.swift

‎Sources/SwiftDriver/Dependency Scanning/InterModuleDependencyGraph.swift

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
//===--------------- ModuleDependencyGraph.swift --------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import Foundation
13+
14+
15+
enum ModuleDependencyId: Hashable {
16+
case swift(String)
17+
case clang(String)
18+
19+
var moduleName: String {
20+
switch self {
21+
case .swift(let name): return name
22+
case .clang(let name): return name
23+
}
24+
}
25+
}
26+
27+
extension ModuleDependencyId: Codable {
28+
enum CodingKeys: CodingKey {
29+
case swift
30+
case clang
31+
}
32+
33+
init(from decoder: Decoder) throws {
34+
let container = try decoder.container(keyedBy: CodingKeys.self)
35+
do {
36+
let moduleName = try container.decode(String.self, forKey: .swift)
37+
self = .swift(moduleName)
38+
} catch {
39+
let moduleName = try container.decode(String.self, forKey: .clang)
40+
self = .clang(moduleName)
41+
}
42+
}
43+
44+
func encode(to encoder: Encoder) throws {
45+
var container = encoder.container(keyedBy: CodingKeys.self)
46+
switch self {
47+
case .swift(let moduleName):
48+
try container.encode(moduleName, forKey: .swift)
49+
case .clang(let moduleName):
50+
try container.encode(moduleName, forKey: .clang)
51+
}
52+
}
53+
}
54+
55+
/// Details specific to Swift modules.
56+
struct SwiftModuleDetails: Codable {
57+
/// The module interface from which this module was built, if any.
58+
var moduleInterfacePath: String?
59+
60+
/// The bridging header, if any.
61+
var bridgingHeaderPath: String?
62+
63+
/// The source files referenced by the bridging header.
64+
var bridgingSourceFiles: [String]? = []
65+
66+
/// Options to the compile command
67+
var commandLine: [String]? = []
68+
}
69+
70+
/// Details specific to Clang modules.
71+
struct ClangModuleDetails: Codable {
72+
/// The path to the module map used to build this module.
73+
var moduleMapPath: String
74+
75+
/// clang-generated context hash
76+
var contextHash: String?
77+
78+
/// Options to the compile command
79+
var commandLine: [String]? = []
80+
}
81+
82+
struct ModuleInfo: Codable {
83+
/// The path for the module.
84+
var modulePath: String
85+
86+
/// The source files used to build this module.
87+
var sourceFiles: [String] = []
88+
89+
/// The set of direct module dependencies of this module.
90+
var directDependencies: [ModuleDependencyId] = []
91+
92+
/// Specific details of a particular kind of module.
93+
var details: Details
94+
95+
/// Specific details of a particular kind of module.
96+
enum Details {
97+
/// Swift modules may be built from a module interface, and may have
98+
/// a bridging header.
99+
case swift(SwiftModuleDetails)
100+
101+
/// Clang modules are built from a module map file.
102+
case clang(ClangModuleDetails)
103+
}
104+
}
105+
106+
extension ModuleInfo.Details: Codable {
107+
enum CodingKeys: CodingKey {
108+
case swift
109+
case clang
110+
}
111+
112+
init(from decoder: Decoder) throws {
113+
let container = try decoder.container(keyedBy: CodingKeys.self)
114+
do {
115+
let details = try container.decode(SwiftModuleDetails.self, forKey: .swift)
116+
self = .swift(details)
117+
} catch {
118+
let details = try container.decode(ClangModuleDetails.self, forKey: .clang)
119+
self = .clang(details)
120+
}
121+
}
122+
123+
func encode(to encoder: Encoder) throws {
124+
var container = encoder.container(keyedBy: CodingKeys.self)
125+
switch self {
126+
case .swift(let details):
127+
try container.encode(details, forKey: .swift)
128+
case .clang(let details):
129+
try container.encode(details, forKey: .clang)
130+
}
131+
}
132+
}
133+
134+
/// Describes the complete set of dependencies for a Swift module, including
135+
/// all of the Swift and C modules and source files it depends on.
136+
struct InterModuleDependencyGraph: Codable {
137+
/// The name of the main module.
138+
var mainModuleName: String
139+
140+
/// The complete set of modules discovered
141+
var modules: [ModuleDependencyId: ModuleInfo] = [:]
142+
143+
/// Information about the main module.
144+
var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! }
145+
}

‎Sources/SwiftDriver/Dependency Scanning/ModuleDependencyBuildGeneration.swift

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
//===--------------- ModuleDependencyBuildGeneration.swift ----------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import TSCBasic
13+
import TSCUtility
14+
import Foundation
15+
16+
extension Driver {
17+
/// For the current moduleDependencyGraph, plan the order and generate jobs
18+
/// for explicitly building all dependency modules.
19+
mutating func planExplicitModuleDependenciesCompile(dependencyGraph: InterModuleDependencyGraph)
20+
throws -> [Job] {
21+
var jobs: [Job] = []
22+
for (id, moduleInfo) in dependencyGraph.modules {
23+
// The generation of the main module file will be handled elsewhere in the driver.
24+
if (id.moduleName == dependencyGraph.mainModuleName) {
25+
continue
26+
}
27+
switch id {
28+
case .swift(let moduleName):
29+
let swiftModuleBuildJob = try genSwiftModuleDependencyBuildJob(moduleInfo: moduleInfo,
30+
moduleName: moduleName)
31+
jobs.append(swiftModuleBuildJob)
32+
case .clang(let moduleName):
33+
let clangModuleBuildJob = try genClangModuleDependencyBuildJob(moduleInfo: moduleInfo,
34+
moduleName: moduleName)
35+
jobs.append(clangModuleBuildJob)
36+
37+
}
38+
}
39+
return jobs
40+
}
41+
42+
/// For a given swift module dependency, generate a build job
43+
mutating private func genSwiftModuleDependencyBuildJob(moduleInfo: ModuleInfo,
44+
moduleName: String) throws -> Job {
45+
// FIXIT: Needs more error handling
46+
guard case .swift(let swiftModuleDetails) = moduleInfo.details else {
47+
throw Error.malformedModuleDependency(moduleName, "no `details` object")
48+
}
49+
50+
var inputs: [TypedVirtualPath] = []
51+
var outputs: [TypedVirtualPath] = [
52+
TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .swiftModule)
53+
]
54+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
55+
commandLine.appendFlag("-frontend")
56+
57+
// Build the .swiftinterfaces file using a list of command line options specified in the
58+
// `details` field.
59+
guard let moduleInterfacePath = swiftModuleDetails.moduleInterfacePath else {
60+
throw Error.malformedModuleDependency(moduleName, "no `moduleInterfacePath` object")
61+
}
62+
inputs.append(TypedVirtualPath(file: try VirtualPath(path: moduleInterfacePath),
63+
type: .swiftInterface))
64+
try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
65+
swiftModuleDetails.commandLine?.forEach { commandLine.appendFlag($0) }
66+
67+
return Job(
68+
kind: .emitModule,
69+
tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
70+
commandLine: commandLine,
71+
inputs: inputs,
72+
outputs: outputs
73+
)
74+
}
75+
76+
/// For a given clang module dependency, generate a build job
77+
mutating private func genClangModuleDependencyBuildJob(moduleInfo: ModuleInfo,
78+
moduleName: String) throws -> Job {
79+
// For clang modules, the Fast Dependency Scanner emits a list of source
80+
// files (with a .modulemap among them), and a list of compile command
81+
// options.
82+
// FIXIT: Needs more error handling
83+
guard case .clang(let clangModuleDetails) = moduleInfo.details else {
84+
throw Error.malformedModuleDependency(moduleName, "no `details` object")
85+
}
86+
var inputs: [TypedVirtualPath] = []
87+
var outputs: [TypedVirtualPath] = [
88+
TypedVirtualPath(file: try VirtualPath(path: moduleInfo.modulePath), type: .pcm)
89+
]
90+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
91+
commandLine.appendFlag("-frontend")
92+
commandLine.appendFlags("-emit-pcm", "-module-name", moduleName)
93+
94+
// The only required input is the .modulemap for this module.
95+
commandLine.append(Job.ArgTemplate.path(try VirtualPath(path: clangModuleDetails.moduleMapPath)))
96+
inputs.append(TypedVirtualPath(file: try VirtualPath(path: clangModuleDetails.moduleMapPath),
97+
type: .clangModuleMap))
98+
try addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs)
99+
clangModuleDetails.commandLine?.forEach { commandLine.appendFlags("-Xcc", $0) }
100+
101+
return Job(
102+
kind: .generatePCM,
103+
tool: .absolute(try toolchain.getToolPath(.swiftCompiler)),
104+
commandLine: commandLine,
105+
inputs: inputs,
106+
outputs: outputs
107+
)
108+
}
109+
}

‎Sources/SwiftDriver/Driver/Driver.swift

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public struct Driver {
5050
case subcommandPassedToDriver
5151
case integratedReplRemoved
5252
case conflictingOptions(Option, Option)
53+
case malformedModuleDependency(String, String)
5354

5455
public var description: String {
5556
switch self {
@@ -68,6 +69,8 @@ public struct Driver {
6869
return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead."
6970
case .conflictingOptions(let one, let two):
7071
return "conflicting options '\(one.spelling)' and '\(two.spelling)'"
72+
case .malformedModuleDependency(let moduleName, let errorDescription):
73+
return "Malformed Module Dependency: \(moduleName), \(errorDescription)"
7174
}
7275
}
7376
}

‎Sources/SwiftDriver/Driver/ModuleDependencyScanning.swift

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
//===--------------- ModuleDependencyScanning.swift -----------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
import Foundation
13+
import TSCBasic
14+
15+
extension Driver {
16+
/// Precompute the dependencies for a given Swift compilation, producing a
17+
/// complete dependency graph including all Swift and C module files and
18+
/// source files.
19+
// TODO: Instead of directly invoking the frontend,
20+
// treat this as any other `Job`.
21+
mutating func computeModuleDependencyGraph() throws
22+
-> InterModuleDependencyGraph? {
23+
// Grab the swift compiler
24+
let resolver = try ArgsResolver()
25+
let compilerPath = VirtualPath.absolute(try toolchain.getToolPath(.swiftCompiler))
26+
let tool = try resolver.resolve(.path(compilerPath))
27+
28+
// Aggregate the fast dependency scanner arguments
29+
var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) }
30+
commandLine.appendFlag("-frontend")
31+
commandLine.appendFlag("-scan-dependencies")
32+
if parsedOptions.hasArgument(.parseStdlib) {
33+
commandLine.appendFlag(.disableObjcAttrRequiresFoundationModule)
34+
}
35+
try addCommonFrontendOptions(commandLine: &commandLine)
36+
// FIXME: MSVC runtime flags
37+
38+
// Pass on the input files
39+
commandLine.append(contentsOf: inputFiles.map { .path($0.file)})
40+
41+
// Execute dependency scanner and decode its output
42+
let arguments = [tool] + (try commandLine.map { try resolver.resolve($0) })
43+
let scanProcess = try Process.launchProcess(arguments: arguments, env: env)
44+
let result = try scanProcess.waitUntilExit()
45+
guard let outputData = try? Data(result.utf8Output().utf8) else {
46+
return nil
47+
}
48+
return try JSONDecoder().decode(InterModuleDependencyGraph.self, from: outputData)
49+
}
50+
}

‎Sources/SwiftDriver/Jobs/CompileJob.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ extension Driver {
6969
.swiftDocumentation, .swiftInterface,
7070
.swiftSourceInfoFile, .raw_sib, .llvmBitcode, .diagnostics,
7171
.objcHeader, .swiftDeps, .remap, .importedModules, .tbd, .moduleTrace,
72-
.indexData, .optimizationRecord, .pcm, .pch, nil:
72+
.indexData, .optimizationRecord, .pcm, .pch, .clangModuleMap, nil:
7373
return false
7474
}
7575
}
@@ -253,7 +253,7 @@ extension FileType {
253253

254254
case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
255255
.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
256-
.optimizationRecord, .swiftInterface, .swiftSourceInfoFile:
256+
.optimizationRecord, .swiftInterface, .swiftSourceInfoFile, .clangModuleMap:
257257
fatalError("Output type can never be a primary output")
258258
}
259259
}

‎Sources/SwiftDriver/Jobs/Planning.swift

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,23 @@ extension Driver {
3434
private mutating func planStandardCompile() throws -> [Job] {
3535
var jobs = [Job]()
3636

37+
// If we've been asked to prebuild module dependencies, prescan the source
38+
// files to produce a module dependency graph and turn it into a set
39+
// of jobs required to build all dependencies.
40+
// For the time being, just prints the jobs' compile commands.
41+
if parsedOptions.contains(.driverPrintModuleDependenciesJobs) {
42+
let moduleDependencyGraph = try computeModuleDependencyGraph()
43+
let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles)
44+
if let dependencyGraph = moduleDependencyGraph {
45+
let modulePrebuildJobs =
46+
try planExplicitModuleDependenciesCompile(dependencyGraph: dependencyGraph)
47+
for job in modulePrebuildJobs {
48+
try Self.printJob(job, resolver: try ArgsResolver(),
49+
forceResponseFiles: forceResponseFiles)
50+
}
51+
}
52+
}
53+
3754
// Keep track of the various outputs we care about from the jobs we build.
3855
var linkerInputs: [TypedVirtualPath] = []
3956
var moduleInputs: [TypedVirtualPath] = []

0 commit comments

Comments
 (0)