Skip to content

Add driver support to specify an overriding output path to record in the index data #516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 6, 2021
Merged

Add driver support to specify an overriding output path to record in the index data #516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 65 additions & 11 deletions Sources/SwiftDriver/Jobs/CompileJob.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,19 @@ extension Driver {
}
}

mutating func computeIndexUnitOutput(for input: TypedVirtualPath, outputType: FileType, topLevel: Bool) -> TypedVirtualPath? {
if let path = outputFileMap?.existingOutput(inputFile: input.file, outputType: .indexUnitOutputPath) {
return TypedVirtualPath(file: path, type: outputType)
}
if topLevel {
if let baseOutput = parsedOptions.getLastArgument(.indexUnitOutputPath)?.asSingle,
let baseOutputPath = try? VirtualPath(path: baseOutput) {
return TypedVirtualPath(file: baseOutputPath, type: outputType)
}
}
return nil
}

mutating func computePrimaryOutput(for input: TypedVirtualPath, outputType: FileType,
isTopLevel: Bool) -> TypedVirtualPath {
if let path = outputFileMap?.existingOutput(inputFile: input.file, outputType: outputType) {
Expand Down Expand Up @@ -79,19 +92,22 @@ extension Driver {
case .swift, .image, .dSYM, .dependencies, .autolink, .swiftDocumentation, .swiftInterface,
.privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps,
.remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm,
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, nil:
.pch, .clangModuleMap, .jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts,
.indexUnitOutputPath, nil:
return false
}
}

/// Add the compiler inputs for a frontend compilation job, and return the
/// corresponding primary set of outputs.
/// corresponding primary set of outputs and, if not identical, the output
/// paths to record in the index data (empty otherwise).
mutating func addCompileInputs(primaryInputs: [TypedVirtualPath],
indexFilePath: TypedVirtualPath?,
inputs: inout [TypedVirtualPath],
inputOutputMap: inout [TypedVirtualPath: TypedVirtualPath],
outputType: FileType?,
commandLine: inout [Job.ArgTemplate]) -> [TypedVirtualPath] {
commandLine: inout [Job.ArgTemplate])
-> ([TypedVirtualPath], [TypedVirtualPath]) {
// Collect the set of input files that are part of the Swift compilation.
let swiftInputFiles: [TypedVirtualPath] = inputFiles.filter { $0.type.isPartOfSwiftCompilation }

Expand Down Expand Up @@ -136,6 +152,8 @@ extension Driver {

// Add each of the input files.
var primaryOutputs: [TypedVirtualPath] = []
var primaryIndexUnitOutputs: [TypedVirtualPath] = []
var indexUnitOutputDiffers = false
for input in swiftInputFiles {
inputs.append(input)

Expand All @@ -160,6 +178,13 @@ extension Driver {
isTopLevel: isTopLevel)
primaryOutputs.append(output)
inputOutputMap[input] = output

if let indexUnitOut = computeIndexUnitOutput(for: input, outputType: outputType, topLevel: isTopLevel) {
indexUnitOutputDiffers = true
primaryIndexUnitOutputs.append(indexUnitOut)
} else {
primaryIndexUnitOutputs.append(output)
}
}
}

Expand All @@ -172,9 +197,22 @@ extension Driver {
isTopLevel: isTopLevel)
primaryOutputs.append(output)
inputOutputMap[input] = output

if let indexUnitOut = computeIndexUnitOutput(for: input, outputType: outputType, topLevel: isTopLevel) {
indexUnitOutputDiffers = true
primaryIndexUnitOutputs.append(indexUnitOut)
} else {
primaryIndexUnitOutputs.append(output)
}
}

if !indexUnitOutputDiffers {
primaryIndexUnitOutputs.removeAll()
} else {
assert(primaryOutputs.count == primaryIndexUnitOutputs.count)
}

return primaryOutputs
return (primaryOutputs, primaryIndexUnitOutputs)
}

/// Form a compile job, which executes the Swift frontend to produce various outputs.
Expand All @@ -200,12 +238,13 @@ extension Driver {
indexFilePath = nil
}

let primaryOutputs = addCompileInputs(primaryInputs: primaryInputs,
indexFilePath: indexFilePath,
inputs: &inputs,
inputOutputMap: &inputOutputMap,
outputType: outputType,
commandLine: &commandLine)
let (primaryOutputs, primaryIndexUnitOutputs) =
addCompileInputs(primaryInputs: primaryInputs,
indexFilePath: indexFilePath,
inputs: &inputs,
inputOutputMap: &inputOutputMap,
outputType: outputType,
commandLine: &commandLine)
outputs += primaryOutputs

// FIXME: optimization record arguments are added before supplementary outputs
Expand Down Expand Up @@ -265,6 +304,20 @@ extension Driver {
}
}

// Add index unit output paths if needed.
if !primaryIndexUnitOutputs.isEmpty {
if primaryIndexUnitOutputs.count > fileListThreshold {
commandLine.appendFlag(.indexUnitOutputPathFilelist)
let path = RelativePath(createTemporaryFileName(prefix: "index-unit-outputs"))
commandLine.appendPath(.fileList(path, .list(primaryIndexUnitOutputs.map { $0.file })))
} else {
for primaryIndexUnitOutput in primaryIndexUnitOutputs {
commandLine.appendFlag(.indexUnitOutputPath)
commandLine.appendPath(primaryIndexUnitOutput.file)
}
}
}

try commandLine.appendLast(.embedBitcodeMarker, from: &parsedOptions)

// For `-index-file` mode add `-disable-typo-correction`, since the errors
Expand Down Expand Up @@ -380,7 +433,8 @@ extension FileType {
case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
.diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
.yamlOptimizationRecord, .bitstreamOptimizationRecord, .swiftInterface,
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts:
.privateSwiftInterface, .swiftSourceInfoFile, .clangModuleMap, .jsonSwiftArtifacts,
.indexUnitOutputPath:
fatalError("Output type can never be a primary output")
}
}
Expand Down
18 changes: 15 additions & 3 deletions Sources/SwiftDriver/Utilities/FileType.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
/// The extension isn't real.
case indexData

/// Output path to record in the indexing data store
///
/// This is only needed for use as a key in the output file map.
case indexUnitOutputPath
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, I'm now realizing that indexData is the same pattern; but, everything else here is a file type, whereas indexData and indexUnitOutputPath are just directories.
It may be worth refactoring in the future to be a common directory FileType, with an enum associated value for the specific kind of directory. But it's alright in this PR.


/// Optimization record.
case yamlOptimizationRecord = "opt.yaml"

Expand Down Expand Up @@ -185,6 +190,9 @@ extension FileType: CustomStringConvertible {
case .indexData:
return "index-data"

case .indexUnitOutputPath:
return "index-unit-output-path"

case .yamlOptimizationRecord:
return "yaml-opt-record"

Expand All @@ -209,7 +217,8 @@ extension FileType {
.swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image,
.swiftDeps, .moduleTrace, .tbd, .yamlOptimizationRecord, .bitstreamOptimizationRecord,
.swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .jsonDependencies,
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts:
.clangModuleMap, .jsonTargetInfo, .jsonCompilerFeatures, .jsonSwiftArtifacts,
.indexUnitOutputPath:
return false
}
}
Expand Down Expand Up @@ -300,6 +309,8 @@ extension FileType {
return "bitstream-opt-record"
case .diagnostics:
return "diagnostics"
case .indexUnitOutputPath:
return "index-unit-output-path"
}
}
}
Expand All @@ -315,7 +326,8 @@ extension FileType {
return true
case .image, .object, .dSYM, .pch, .sib, .raw_sib, .swiftModule,
.swiftDocumentation, .swiftSourceInfoFile, .llvmBitcode, .diagnostics,
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord:
.pcm, .swiftDeps, .remap, .indexData, .bitstreamOptimizationRecord,
.indexUnitOutputPath:
return false
}
}
Expand All @@ -331,7 +343,7 @@ extension FileType {
.swiftSourceInfoFile, .raw_sil, .raw_sib, .diagnostics, .objcHeader, .swiftDeps, .remap,
.importedModules, .tbd, .moduleTrace, .indexData, .yamlOptimizationRecord,
.bitstreamOptimizationRecord, .pcm, .pch, .jsonDependencies, .clangModuleMap,
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts:
.jsonCompilerFeatures, .jsonTargetInfo, .jsonSwiftArtifacts, .indexUnitOutputPath:
return false
}
}
Expand Down
4 changes: 4 additions & 0 deletions Sources/SwiftOptions/Options.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,8 @@ extension Option {
public static let indexIgnoreSystemModules: Option = Option("-index-ignore-system-modules", .flag, attributes: [.noInteractive], helpText: "Avoid indexing system modules")
public static let indexStorePath: Option = Option("-index-store-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>", helpText: "Store indexing data to <path>")
public static let indexSystemModules: Option = Option("-index-system-modules", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Emit index data for imported serialized swift system modules")
public static let indexUnitOutputPathFilelist: Option = Option("-index-unit-output-path-filelist", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Specify index unit output paths in a file rather than on the command line")
public static let indexUnitOutputPath: Option = Option("-index-unit-output-path", .separate, attributes: [.frontend, .argumentIsPath], metaVar: "<path>", helpText: "Use <path> as the output path in the produced index data.")
public static let interpret: Option = Option("-interpret", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Immediate mode", group: .modes)
public static let I: Option = Option("-I", .joinedOrSeparate, attributes: [.frontend, .argumentIsPath], helpText: "Add directory to the import search path")
public static let i: Option = Option("-i", .flag, group: .modes)
Expand Down Expand Up @@ -813,6 +815,8 @@ extension Option {
Option.indexIgnoreSystemModules,
Option.indexStorePath,
Option.indexSystemModules,
Option.indexUnitOutputPathFilelist,
Option.indexUnitOutputPath,
Option.interpret,
Option.I,
Option.i,
Expand Down
129 changes: 129 additions & 0 deletions Tests/SwiftDriverTests/SwiftDriverTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,135 @@ final class SwiftDriverTests: XCTestCase {
}
}

func testIndexUnitOutputPath() throws {
let contents = """
{
"/tmp/main.swift": {
"object": "/tmp/build1/main.o",
"index-unit-output-path": "/tmp/build2/main.o",
},
"/tmp/second.swift": {
"object": "/tmp/build1/second.o",
"index-unit-output-path": "/tmp/build2/second.o",
}
}
"""

func getFileListElements(for filelistOpt: String, job: Job) -> [VirtualPath] {
let optIndex = job.commandLine.firstIndex(of: .flag(filelistOpt))!
let value = job.commandLine[job.commandLine.index(after: optIndex)]
guard case let .path(.fileList(_, valueFileList)) = value else {
XCTFail("Argument wasn't a filelist")
return []
}
guard case let .list(inputs) = valueFileList else {
XCTFail("FileList wasn't List")
return []
}
return inputs
}

try withTemporaryFile { file in
try assertNoDiagnostics { diags in
try localFileSystem.writeFileContents(file.path) { $0 <<< contents }

// 1. Incremental mode (single primary file)
// a) without filelists
var driver = try Driver(args: [
"swiftc", "-c",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
var jobs = try driver.planBuild()
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
XCTAssertTrue(jobs[1].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
XCTAssertTrue(jobs[1].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))

// b) with filelists
driver = try Driver(args: [
"swiftc", "-c", "-driver-filelist-threshold=0",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
jobs = try driver.planBuild()
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build1/second.o"))])
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build2/second.o"))])
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[1]),
[.absolute(.init("/tmp/build1/main.o"))])
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[1]),
[.absolute(.init("/tmp/build2/main.o"))])


// 2. Batch mode (two primary files)
// a) without filelists
driver = try Driver(args: [
"swiftc", "-c", "-enable-batch-mode", "-driver-batch-count", "1",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
jobs = try driver.planBuild()
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))

// b) with filelists
driver = try Driver(args: [
"swiftc", "-c", "-driver-filelist-threshold=0",
"-enable-batch-mode", "-driver-batch-count", "1",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
jobs = try driver.planBuild()
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build1/second.o")), .absolute(.init("/tmp/build1/main.o"))])
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build2/second.o")), .absolute(.init("/tmp/build2/main.o"))])

// 3. Multi-threaded WMO
// a) without filelists
driver = try Driver(args: [
"swiftc", "-c", "-whole-module-optimization", "-num-threads", "2",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
jobs = try driver.planBuild()
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/main.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/main.o")))]))

// b) with filelists
driver = try Driver(args: [
"swiftc", "-c", "-driver-filelist-threshold=0",
"-whole-module-optimization", "-num-threads", "2",
"-output-file-map", file.path.pathString,
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift"
])
jobs = try driver.planBuild()
XCTAssertEqual(getFileListElements(for: "-output-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build1/second.o")), .absolute(.init("/tmp/build1/main.o"))])
XCTAssertEqual(getFileListElements(for: "-index-unit-output-path-filelist", job: jobs[0]),
[.absolute(.init("/tmp/build2/second.o")), .absolute(.init("/tmp/build2/main.o"))])

// 4. Index-file (single primary)
driver = try Driver(args: [
"swiftc", "-c", "-enable-batch-mode", "-driver-batch-count", "1",
"-module-name", "test", "/tmp/second.swift", "/tmp/main.swift",
"-index-file", "-index-file-path", "/tmp/second.swift",
"-disable-batch-mode", "-o", "/tmp/build1/second.o",
"-index-unit-output-path", "/tmp/build2/second.o"
])
jobs = try driver.planBuild()
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-o", .path(.absolute(.init("/tmp/build1/second.o")))]))
XCTAssertTrue(jobs[0].commandLine.contains(subsequence: ["-index-unit-output-path", .path(.absolute(.init("/tmp/build2/second.o")))]))
}
}
}

func testMergeModuleEmittingDependencies() throws {
var driver1 = try Driver(args: ["swiftc", "foo.swift", "bar.swift", "-module-name", "Foo", "-emit-dependencies", "-emit-module", "-serialize-diagnostics", "-driver-filelist-threshold=9999"])
let plannedJobs = try driver1.planBuild().removingAutolinkExtractJobs()
Expand Down