Skip to content

Commit e7b468a

Browse files
author
Nathan Hawes
authored
Merge pull request #516 from nathawes/add-index-unit-output-path
Add driver support to specify an overriding output path to record in the index data
2 parents c947d59 + 9da73a9 commit e7b468a

File tree

4 files changed

+213
-14
lines changed

4 files changed

+213
-14
lines changed

‎Sources/SwiftDriver/Jobs/CompileJob.swift

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,19 @@ extension Driver {
2626
}
2727
}
2828

29+
mutating func computeIndexUnitOutput(for input: TypedVirtualPath, outputType: FileType, topLevel: Bool) -> TypedVirtualPath? {
30+
if let path = outputFileMap?.existingOutput(inputFile: input.file, outputType: .indexUnitOutputPath) {
31+
return TypedVirtualPath(file: path, type: outputType)
32+
}
33+
if topLevel {
34+
if let baseOutput = parsedOptions.getLastArgument(.indexUnitOutputPath)?.asSingle,
35+
let baseOutputPath = try? VirtualPath(path: baseOutput) {
36+
return TypedVirtualPath(file: baseOutputPath, type: outputType)
37+
}
38+
}
39+
return nil
40+
}
41+
2942
mutating func computePrimaryOutput(for input: TypedVirtualPath, outputType: FileType,
3043
isTopLevel: Bool) -> TypedVirtualPath {
3144
if let path = outputFileMap?.existingOutput(inputFile: input.file, outputType: outputType) {
@@ -79,19 +92,22 @@ extension Driver {
7992
case .swift, .image, .dSYM, .dependencies, .autolink, .swiftDocumentation, .swiftInterface,
8093
.privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
8194
.remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
82-
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, nil:
95+
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
96+
.indexUnitOutputPath, nil:
8397
return false
8498
}
8599
}
86100

87101
/// Add the compiler inputs for a frontend compilation job, and return the
88-
/// corresponding primary set of outputs.
102+
/// corresponding primary set of outputs and, if not identical, the output
103+
/// paths to record in the index data (empty otherwise).
89104
mutating func addCompileInputs(primaryInputs: [TypedVirtualPath],
90105
indexFilePath: TypedVirtualPath?,
91106
inputs: inout [TypedVirtualPath],
92107
inputOutputMap: inout [TypedVirtualPath: TypedVirtualPath],
93108
outputType: FileType?,
94-
commandLine: inout [Job.ArgTemplate]) -> [TypedVirtualPath] {
109+
commandLine: inout [Job.ArgTemplate])
110+
-> ([TypedVirtualPath], [TypedVirtualPath]) {
95111
// Collect the set of input files that are part of the Swift compilation.
96112
let swiftInputFiles: [TypedVirtualPath] = inputFiles.filter { $0.type.isPartOfSwiftCompilation }
97113

@@ -136,6 +152,8 @@ extension Driver {
136152

137153
// Add each of the input files.
138154
var primaryOutputs: [TypedVirtualPath] = []
155+
var primaryIndexUnitOutputs: [TypedVirtualPath] = []
156+
var indexUnitOutputDiffers = false
139157
for input in swiftInputFiles {
140158
inputs.append(input)
141159

@@ -160,6 +178,13 @@ extension Driver {
160178
isTopLevel: isTopLevel)
161179
primaryOutputs.append(output)
162180
inputOutputMap[input] = output
181+
182+
if let indexUnitOut = computeIndexUnitOutput(for: input, outputType: outputType, topLevel: isTopLevel) {
183+
indexUnitOutputDiffers = true
184+
primaryIndexUnitOutputs.append(indexUnitOut)
185+
} else {
186+
primaryIndexUnitOutputs.append(output)
187+
}
163188
}
164189
}
165190

@@ -172,9 +197,22 @@ extension Driver {
172197
isTopLevel: isTopLevel)
173198
primaryOutputs.append(output)
174199
inputOutputMap[input] = output
200+
201+
if let indexUnitOut = computeIndexUnitOutput(for: input, outputType: outputType, topLevel: isTopLevel) {
202+
indexUnitOutputDiffers = true
203+
primaryIndexUnitOutputs.append(indexUnitOut)
204+
} else {
205+
primaryIndexUnitOutputs.append(output)
206+
}
207+
}
208+
209+
if !indexUnitOutputDiffers {
210+
primaryIndexUnitOutputs.removeAll()
211+
} else {
212+
assert(primaryOutputs.count == primaryIndexUnitOutputs.count)
175213
}
176214

177-
return primaryOutputs
215+
return (primaryOutputs, primaryIndexUnitOutputs)
178216
}
179217

180218
/// Form a compile job, which executes the Swift frontend to produce various outputs.
@@ -200,12 +238,13 @@ extension Driver {
200238
indexFilePath = nil
201239
}
202240

203-
let primaryOutputs = addCompileInputs(primaryInputs: primaryInputs,
204-
indexFilePath: indexFilePath,
205-
inputs: &inputs,
206-
inputOutputMap: &inputOutputMap,
207-
outputType: outputType,
208-
commandLine: &commandLine)
241+
let (primaryOutputs, primaryIndexUnitOutputs) =
242+
addCompileInputs(primaryInputs: primaryInputs,
243+
indexFilePath: indexFilePath,
244+
inputs: &inputs,
245+
inputOutputMap: &inputOutputMap,
246+
outputType: outputType,
247+
commandLine: &commandLine)
209248
outputs += primaryOutputs
210249

211250
// FIXME: optimization record arguments are added before supplementary outputs
@@ -265,6 +304,20 @@ extension Driver {
265304
}
266305
}
267306

307+
// Add index unit output paths if needed.
308+
if !primaryIndexUnitOutputs.isEmpty {
309+
if primaryIndexUnitOutputs.count > fileListThreshold {
310+
commandLine.appendFlag(.indexUnitOutputPathFilelist)
311+
let path = RelativePath(createTemporaryFileName(prefix: "index-unit-outputs"))
312+
commandLine.appendPath(.fileList(path, .list(primaryIndexUnitOutputs.map { $0.file })))
313+
} else {
314+
for primaryIndexUnitOutput in primaryIndexUnitOutputs {
315+
commandLine.appendFlag(.indexUnitOutputPath)
316+
commandLine.appendPath(primaryIndexUnitOutput.file)
317+
}
318+
}
319+
}
320+
268321
try commandLine.appendLast(.embedBitcodeMarker, from: &parsedOptions)
269322

270323
// For `-index-file` mode add `-disable-typo-correction`, since the errors
@@ -380,7 +433,8 @@ extension FileType {
380433
case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
381434
.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
382435
.yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
383-
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts:
436+
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
437+
.indexUnitOutputPath:
384438
fatalError("Output type can never be a primary output")
385439
}
386440
}

‎Sources/SwiftDriver/Utilities/FileType.swift

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
114114
/// The extension isn't real.
115115
case indexData
116116

117+
/// Output path to record in the indexing data store
118+
///
119+
/// This is only needed for use as a key in the output file map.
120+
case indexUnitOutputPath
121+
117122
/// Optimization record.
118123
case yamlOptimizationRecord = "opt.yaml"
119124

@@ -185,6 +190,9 @@ extension FileType: CustomStringConvertible {
185190
case .indexData:
186191
return "index-data"
187192

193+
case .indexUnitOutputPath:
194+
return "index-unit-output-path"
195+
188196
case .yamlOptimizationRecord:
189197
return "yaml-opt-record"
190198

@@ -209,7 +217,8 @@ extension FileType {
209217
.swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image,
210218
.swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
211219
.swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
212-
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts:
220+
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
221+
.indexUnitOutputPath:
213222
return false
214223
}
215224
}
@@ -300,6 +309,8 @@ extension FileType {
300309
return "bitstream-opt-record"
301310
case .diagnostics:
302311
return "diagnostics"
312+
case .indexUnitOutputPath:
313+
return "index-unit-output-path"
303314
}
304315
}
305316
}
@@ -315,7 +326,8 @@ extension FileType {
315326
return true
316327
case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
317328
.swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
318-
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord:
329+
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
330+
.indexUnitOutputPath:
319331
return false
320332
}
321333
}
@@ -331,7 +343,7 @@ extension FileType {
331343
.swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
332344
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
333345
.bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
334-
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts:
346+
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
335347
return false
336348
}
337349
}

‎Sources/SwiftOptions/Options.swift

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,8 @@ extension Option {
309309
public static let indexIgnoreSystemModules: Option = Option("-index-ignore-system-modules", .flag, attributes: [.noInteractive], helpText: "Avoid indexing system modules")
310310
public static let indexStorePath: Option = Option("-index-store-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>", helpText: "Store indexing data to <path>")
311311
public static let indexSystemModules: Option = Option("-index-system-modules", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Emit index data for imported serialized swift system modules")
312+
public static let indexUnitOutputPathFilelist: Option = Option("-index-unit-output-path-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Specify index unit output paths in a file rather than on the command line")
313+
public static let indexUnitOutputPath: Option = Option("-index-unit-output-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>", helpText: "Use <path> as the output path in the produced index data.")
312314
public static let interpret: Option = Option("-interpret", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Immediate mode", group: .modes)
313315
public static let I: Option = Option("-I", .joinedOrSeparate, attributes: [.frontend, .argumentIsPath], helpText: "Add directory to the import search path")
314316
public static let i: Option = Option("-i", .flag, group: .modes)
@@ -813,6 +815,8 @@ extension Option {
813815
Option.indexIgnoreSystemModules,
814816
Option.indexStorePath,
815817
Option.indexSystemModules,
818+
Option.indexUnitOutputPathFilelist,
819+
Option.indexUnitOutputPath,
816820
Option.interpret,
817821
Option.I,
818822
Option.i,

‎Tests/SwiftDriverTests/SwiftDriverTests.swift

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -614,6 +614,135 @@ final class SwiftDriverTests: XCTestCase {
614614
}
615615
}
616616

617+
func testIndexUnitOutputPath() throws {
618+
let contents = """
619+
{
620+
"/tmp/main.swift": {
621+
"object": "/tmp/build1/main.o",
622+
"index-unit-output-path": "/tmp/build2/main.o",
623+
},
624+
"/tmp/second.swift": {
625+
"object": "/tmp/build1/second.o",
626+
"index-unit-output-path": "/tmp/build2/second.o",
627+
}
628+
}
629+
"""
630+
631+
func getFileListElements(for filelistOpt: String, job: Job) -> [VirtualPath] {
632+
let optIndex = job.commandLine.firstIndex(of: .flag(filelistOpt))!
633+
let value = job.commandLine[job.commandLine.index(after: optIndex)]
634+
guard case let .path(.fileList(_, valueFileList)) = value else {
635+
XCTFail("Argument wasn't a filelist")
636+
return []
637+
}
638+
guard case let .list(inputs) = valueFileList else {
639+
XCTFail("FileList wasn't List")
640+
return []
641+
}
642+
return inputs
643+
}
644+
645+
try withTemporaryFile { file in
646+
try assertNoDiagnostics { diags in
647+
try localFileSystem.writeFileContents(file.path) { $0 <<< contents }
648+
649+
// 1. Incremental mode (single primary file)
650+
// a) without filelists
651+
var driver = try Driver(args: [
652+
"swiftc", "-c",
653+
"-output-file-map", file.path.pathString,
654+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
655+
])
656+
var jobs = try driver.planBuild()
657+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
658+
XCTAssertTrue(jobs[1].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
659+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
660+
XCTAssertTrue(jobs[1].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))
661+
662+
// b) with filelists
663+
driver = try Driver(args: [
664+
"swiftc", "-c", "-driver-filelist-threshold=0",
665+
"-output-file-map", file.path.pathString,
666+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
667+
])
668+
jobs = try driver.planBuild()
669+
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
670+
[.absolute(.init("/tmp/build1/second.o"))])
671+
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
672+
[.absolute(.init("/tmp/build2/second.o"))])
673+
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[1]),
674+
[.absolute(.init("/tmp/build1/main.o"))])
675+
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[1]),
676+
[.absolute(.init("/tmp/build2/main.o"))])
677+
678+
679+
// 2. Batch mode (two primary files)
680+
// a) without filelists
681+
driver = try Driver(args: [
682+
"swiftc", "-c", "-enable-batch-mode", "-driver-batch-count", "1",
683+
"-output-file-map", file.path.pathString,
684+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
685+
])
686+
jobs = try driver.planBuild()
687+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
688+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
689+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
690+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))
691+
692+
// b) with filelists
693+
driver = try Driver(args: [
694+
"swiftc", "-c", "-driver-filelist-threshold=0",
695+
"-enable-batch-mode", "-driver-batch-count", "1",
696+
"-output-file-map", file.path.pathString,
697+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
698+
])
699+
jobs = try driver.planBuild()
700+
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
701+
[.absolute(.init("/tmp/build1/second.o")), .absolute(.init("/tmp/build1/main.o"))])
702+
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
703+
[.absolute(.init("/tmp/build2/second.o")), .absolute(.init("/tmp/build2/main.o"))])
704+
705+
// 3. Multi-threaded WMO
706+
// a) without filelists
707+
driver = try Driver(args: [
708+
"swiftc", "-c", "-whole-module-optimization", "-num-threads", "2",
709+
"-output-file-map", file.path.pathString,
710+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
711+
])
712+
jobs = try driver.planBuild()
713+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
714+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
715+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
716+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))
717+
718+
// b) with filelists
719+
driver = try Driver(args: [
720+
"swiftc", "-c", "-driver-filelist-threshold=0",
721+
"-whole-module-optimization", "-num-threads", "2",
722+
"-output-file-map", file.path.pathString,
723+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
724+
])
725+
jobs = try driver.planBuild()
726+
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
727+
[.absolute(.init("/tmp/build1/second.o")), .absolute(.init("/tmp/build1/main.o"))])
728+
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
729+
[.absolute(.init("/tmp/build2/second.o")), .absolute(.init("/tmp/build2/main.o"))])
730+
731+
// 4. Index-file (single primary)
732+
driver = try Driver(args: [
733+
"swiftc", "-c", "-enable-batch-mode", "-driver-batch-count", "1",
734+
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift",
735+
"-index-file", "-index-file-path", "/tmp/second.swift",
736+
"-disable-batch-mode", "-o", "/tmp/build1/second.o",
737+
"-index-unit-output-path", "/tmp/build2/second.o"
738+
])
739+
jobs = try driver.planBuild()
740+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
741+
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
742+
}
743+
}
744+
}
745+
617746
func testMergeModuleEmittingDependencies() throws {
618747
var driver1 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Foo", "-emit-dependencies", "-emit-module", "-serialize-diagnostics", "-driver-filelist-threshold=9999"])
619748
let plannedJobs = try driver1.planBuild().removingAutolinkExtractJobs()

0 commit comments

Comments
 (0)