Skip to content

[5.5, NFC, Incremental] Create a domain-specific type for the inputDependencySourceMap #710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Sources/SwiftDriver/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ add_library(SwiftDriver
Execution/ParsableOutput.swift
Execution/ProcessProtocol.swift

"IncrementalCompilation/ModuleDependencyGraphParts/InputDependencySourceMap.swift"
"IncrementalCompilation/ModuleDependencyGraphParts/Integrator.swift"
"IncrementalCompilation/ModuleDependencyGraphParts/Node.swift"
"IncrementalCompilation/ModuleDependencyGraphParts/NodeFinder.swift"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
else {
return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals()
}
guard graph.populateInputDependencySourceMap() else {
guard graph.populateInputDependencySourceMap(for: .inputsAddedSincePriors) else {
return nil
}
graph.dotFileWriter?.write(graph)
Expand Down Expand Up @@ -266,7 +266,8 @@ extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
{
let graph = ModuleDependencyGraph(self, .buildingWithoutAPrior)
assert(outputFileMap.onlySourceFilesHaveSwiftDeps())
guard graph.populateInputDependencySourceMap() else {

guard graph.populateInputDependencySourceMap(for: .buildingFromSwiftDeps) else {
return nil
}

Expand Down
145 changes: 64 additions & 81 deletions Sources/SwiftDriver/IncrementalCompilation/ModuleDependencyGraph.swift
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,7 @@ import SwiftOptions
@_spi(Testing) public var nodeFinder = NodeFinder()

/// Maps input files (e.g. .swift) to and from the DependencySource object.
///
// FIXME: The map between swiftdeps and swift files is absolutely *not*
// a bijection. In particular, more than one swiftdeps file can be encountered
// in the course of deserializing priors *and* reading the output file map
// *and* re-reading swiftdeps files after frontends complete
// that correspond to the same swift file. These cause two problems:
// - overwrites in this data structure that lose data and
// - cache misses in `getInput(for:)` that cause the incremental build to
// turn over when e.g. entries in the output file map change. This should be
// replaced by a multi-map from swift files to dependency sources,
// and a regular map from dependency sources to swift files -
// since that direction really is one-to-one.
@_spi(Testing) public private(set) var inputDependencySourceMap = BidirectionalMap<TypedVirtualPath, DependencySource>()
@_spi(Testing) public private(set) var inputDependencySourceMap: InputDependencySourceMap

// The set of paths to external dependencies known to be in the graph
public internal(set) var fingerprintedExternalDependencies = Set<FingerprintedExternalDependency>()
Expand Down Expand Up @@ -64,34 +52,18 @@ import SwiftOptions
: nil
self.phase = phase
self.creationPhase = phase
self.inputDependencySourceMap = InputDependencySourceMap(simulateGetInputFailure: info.simulateGetInputFailure)
}

private func addMapEntry(_ input: TypedVirtualPath, _ dependencySource: DependencySource) {
assert(input.type == .swift && dependencySource.typedFile.type == .swiftDeps)
inputDependencySourceMap[input] = dependencySource
}

@_spi(Testing) public func getSource(for input: TypedVirtualPath,
function: String = #function,
file: String = #file,
line: Int = #line) -> DependencySource {
guard let source = inputDependencySourceMap[input] else {
fatalError("\(input.file) not found in map: \(inputDependencySourceMap), \(file):\(line) in \(function)")
}
return source
}

@_spi(Testing) public func getInput(for source: DependencySource) -> TypedVirtualPath? {
guard let input =
info.simulateGetInputFailure ? nil
: inputDependencySourceMap[source]
@_spi(Testing) public func sourceRequired(for input: TypedVirtualPath,
function: String = #function,
file: String = #file,
line: Int = #line) -> DependencySource {
guard let source = inputDependencySourceMap.sourceIfKnown(for: input)
else {
info.diagnosticEngine.emit(warning: "Failed to find source file for '\(source.file.basename)', recovering with a full rebuild. Next build will be incremental.")
info.reporter?.report(
"\(info.simulateGetInputFailure ? "Simulating i" : "I")nput not found in inputDependencySourceMap; created for: \(creationPhase), now: \(phase)")
return nil
fatalError("\(input.file.basename) not found in inputDependencySourceMap, \(file):\(line) in \(function)")
}
return input
return source
}
}

Expand Down Expand Up @@ -161,7 +133,7 @@ extension ModuleDependencyGraph {
return TransitivelyInvalidatedInputSet()
}
return collectInputsRequiringCompilationAfterProcessing(
dependencySource: getSource(for: input))
dependencySource: sourceRequired(for: input))
}
}

Expand All @@ -183,17 +155,16 @@ extension ModuleDependencyGraph {
/// speculatively scheduled in the first wave.
func collectInputsInvalidatedBy(input: TypedVirtualPath
) -> TransitivelyInvalidatedInputArray {
let changedSource = getSource(for: input)
let changedSource = sourceRequired(for: input)
let allDependencySourcesToRecompile =
collectSwiftDepsUsing(dependencySource: changedSource)

return allDependencySourcesToRecompile.compactMap {
dependencySource in
guard dependencySource != changedSource else {return nil}
let dependentSource = inputDependencySourceMap[dependencySource]
info.reporter?.report(
"Found dependent of \(input.file.basename):", dependentSource)
return dependentSource
let inputToRecompile = inputDependencySourceMap.inputIfKnown(for: dependencySource)
info.reporter?.report("Found dependent of \(input.file.basename):", inputToRecompile)
return inputToRecompile
}
}

Expand All @@ -210,7 +181,7 @@ extension ModuleDependencyGraph {
/// Does the graph contain any dependency nodes for a given source-code file?
func containsNodes(forSourceFile file: TypedVirtualPath) -> Bool {
precondition(file.type == .swift)
guard let source = inputDependencySourceMap[file] else {
guard let source = inputDependencySourceMap.sourceIfKnown(for: file) else {
return false
}
return containsNodes(forDependencySource: source)
Expand All @@ -220,17 +191,46 @@ extension ModuleDependencyGraph {
return nodeFinder.findNodes(for: source).map {!$0.isEmpty}
?? false
}

/// Return true on success
func populateInputDependencySourceMap() -> Bool {

/// - Returns: false on error
func populateInputDependencySourceMap(
`for` purpose: InputDependencySourceMap.AdditionPurpose
) -> Bool {
let ofm = info.outputFileMap
let de = info.diagnosticEngine
return info.inputFiles.reduce(true) { okSoFar, input in
ofm.getDependencySource(for: input, diagnosticEngine: de)
.map {source in addMapEntry(input, source); return okSoFar } ?? false
let diags = info.diagnosticEngine
var allFound = true
for input in info.inputFiles {
if let source = ofm.dependencySource(for: input, diagnosticEngine: diags) {
inputDependencySourceMap.addEntry(input, source, for: purpose)
} else {
// Don't break in order to report all failures.
allFound = false
}
}
return allFound
}
}
extension OutputFileMap {
fileprivate func dependencySource(
for sourceFile: TypedVirtualPath,
diagnosticEngine: DiagnosticsEngine
) -> DependencySource? {
assert(sourceFile.type == FileType.swift)
guard let swiftDepsPath = existingOutput(inputFile: sourceFile.fileHandle,
outputType: .swiftDeps)
else {
// The legacy driver fails silently here.
diagnosticEngine.emit(
.remarkDisabled("\(sourceFile.file.basename) has no swiftDeps file")
)
return nil
}
assert(VirtualPath.lookup(swiftDepsPath).extension == FileType.swiftDeps.rawValue)
let typedSwiftDepsFile = TypedVirtualPath(file: swiftDepsPath, type: .swiftDeps)
return DependencySource(typedSwiftDepsFile)
}
}

// MARK: - Scheduling the 2nd wave
extension ModuleDependencyGraph {
/// After `source` has been compiled, figure out what other source files need compiling.
Expand All @@ -240,7 +240,7 @@ extension ModuleDependencyGraph {
func collectInputsRequiringCompilation(byCompiling input: TypedVirtualPath
) -> TransitivelyInvalidatedInputSet? {
precondition(input.type == .swift)
let dependencySource = getSource(for: input)
let dependencySource = sourceRequired(for: input)
return collectInputsRequiringCompilationAfterProcessing(
dependencySource: dependencySource)
}
Expand Down Expand Up @@ -327,8 +327,10 @@ extension ModuleDependencyGraph {
) -> TransitivelyInvalidatedInputSet? {
var invalidatedInputs = TransitivelyInvalidatedInputSet()
for invalidatedSwiftDeps in collectSwiftDepsUsingInvalidated(nodes: directlyInvalidatedNodes) {
guard let invalidatedInput = getInput(for: invalidatedSwiftDeps)
guard let invalidatedInput = inputDependencySourceMap.inputIfKnown(for: invalidatedSwiftDeps)
else {
info.diagnosticEngine.emit(
warning: "Failed to find source file for '\(invalidatedSwiftDeps.file.basename)', recovering with a full rebuild. Next build will be incremental.")
return nil
}
invalidatedInputs.insert(invalidatedInput)
Expand Down Expand Up @@ -405,27 +407,6 @@ extension ModuleDependencyGraph {
}
}

extension OutputFileMap {
fileprivate func getDependencySource(
for sourceFile: TypedVirtualPath,
diagnosticEngine: DiagnosticsEngine
) -> DependencySource? {
assert(sourceFile.type == FileType.swift)
guard let swiftDepsPath = existingOutput(inputFile: sourceFile.fileHandle,
outputType: .swiftDeps)
else {
// The legacy driver fails silently here.
diagnosticEngine.emit(
.remarkDisabled("\(sourceFile.file.basename) has no swiftDeps file")
)
return nil
}
assert(VirtualPath.lookup(swiftDepsPath).extension == FileType.swiftDeps.rawValue)
let typedSwiftDepsFile = TypedVirtualPath(file: swiftDepsPath, type: .swiftDeps)
return DependencySource(typedSwiftDepsFile)
}
}

// MARK: - tracking traced nodes
extension ModuleDependencyGraph {

Expand Down Expand Up @@ -550,10 +531,11 @@ extension ModuleDependencyGraph {
.record(def: dependencyKey, use: self.allNodes[useID])
assert(isNewUse, "Duplicate use def-use arc in graph?")
}
for (input, source) in inputDependencySourceMap {
graph.addMapEntry(input, source)
for (input, dependencySource) in inputDependencySourceMap {
graph.inputDependencySourceMap.addEntry(input,
dependencySource,
for: .readingPriors)
}

return self.graph
}

Expand Down Expand Up @@ -849,7 +831,7 @@ extension ModuleDependencyGraph {
}
}

for (input, dependencySource) in graph.inputDependencySourceMap {
graph.inputDependencySourceMap.enumerateToSerializePriors { input, dependencySource in
self.addIdentifier(input.file.name)
self.addIdentifier(dependencySource.file.name)
}
Expand Down Expand Up @@ -994,7 +976,8 @@ extension ModuleDependencyGraph {
}
}
}
for (input, dependencySource) in graph.inputDependencySourceMap {
graph.inputDependencySourceMap.enumerateToSerializePriors {
input, dependencySource in
serializer.stream.writeRecord(serializer.abbreviations[.mapNode]!) {
$0.append(RecordID.mapNode)
$0.append(serializer.lookupIdentifierCode(for: input.file.name))
Expand Down Expand Up @@ -1112,7 +1095,7 @@ extension Set where Element == ModuleDependencyGraph.Node {
}
}

extension BidirectionalMap where T1 == TypedVirtualPath, T2 == DependencySource {
extension InputDependencySourceMap {
fileprivate func matches(_ other: Self) -> Bool {
self == other
}
Expand All @@ -1130,6 +1113,6 @@ extension ModuleDependencyGraph {
_ mockInput: TypedVirtualPath,
_ mockDependencySource: DependencySource
) {
addMapEntry(mockInput, mockDependencySource)
inputDependencySourceMap.addEntry(mockInput, mockDependencySource, for: .mocking)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
//===------------- InputDependencySourceMap.swift ---------------- --------===//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2021 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
import Foundation
import TSCBasic

@_spi(Testing) public struct InputDependencySourceMap: Equatable {

/// Maps input files (e.g. .swift) to and from the DependencySource object.
///
// FIXME: The map between swiftdeps and swift files is absolutely *not*
// a bijection. In particular, more than one swiftdeps file can be encountered
// in the course of deserializing priors *and* reading the output file map
// *and* re-reading swiftdeps files after frontends complete
// that correspond to the same swift file. These cause two problems:
// - overwrites in this data structure that lose data and
// - cache misses in `getInput(for:)` that cause the incremental build to
// turn over when e.g. entries in the output file map change. This should be
// replaced by a multi-map from swift files to dependency sources,
// and a regular map from dependency sources to swift files -
// since that direction really is one-to-one.

public typealias BiMap = BidirectionalMap<TypedVirtualPath, DependencySource>
@_spi(Testing) public var biMap = BiMap()

private let simulateGetInputFailure: Bool
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this a format part of the API? We know how to create this failure mode in tests for the current implementation, and the new one should structurally avoid the issue altogether.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

simulateGetInputFailure is gone on the main branch. It remains here in the interest of minimal changes to this branch. If you insist, I'm happy with removing it. Whatever the folks in charge of this branch prefer.


init(simulateGetInputFailure: Bool) {
self.simulateGetInputFailure = simulateGetInputFailure
}
}

// MARK: - Accessing
extension InputDependencySourceMap {
@_spi(Testing) public func sourceIfKnown(for input: TypedVirtualPath) -> DependencySource? {
biMap[input]
}

@_spi(Testing) public func inputIfKnown(for source: DependencySource) -> TypedVirtualPath? {
simulateGetInputFailure ? nil : biMap[source]
}

@_spi(Testing) public func enumerateToSerializePriors(
_ eachFn: (TypedVirtualPath, DependencySource) -> Void
) {
biMap.forEach(eachFn)
}
}

Comment on lines +41 to +56
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would very much prefer we not conflate the point of use with the names of these APIs.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please elaborate? I want to be sure I understand the point here. What would you suggest instead?
If you feel that enumerateToSerializePriors is too specific, then we probably disagree about the value of intention-revealing names.

If that is the source of disagreement, here is a nice quote I found that explains what I am trying to accomplish:

If a developer must consider the implementation of a component in order to use it, the value of encapsulation is lost. If someone other than the original developer must infer the purpose of an object or operation based on its implementation, that new developer may infer a purpose that the operation or class fulfills only by chance. If that was not the intent, the code may work for the moment, but the conceptual basis of the design will have been corrupted, and the two developers will be working at cross-purposes.

Therefore,

Name classes and operations to describe their effect and purpose, without reference to the means by which they do what they promise. This relieves the client developer of the need to understand the internals.

// MARK: - Populating
extension InputDependencySourceMap {
public enum AdditionPurpose {
/// For testing:
case mocking
/// When building from a `swiftdeps` file:
case buildingFromSwiftDeps
/// When deserializing the map from a prior build:
case readingPriors
/// Adding an entry for an input added to the build since the priors were stored:
case inputsAddedSincePriors
}
@_spi(Testing) public mutating func addEntry(_ input: TypedVirtualPath,
_ dependencySource: DependencySource,
`for` _ : AdditionPurpose) {
assert(input.type == .swift && dependencySource.typedFile.type == .swiftDeps)
biMap[input] = dependencySource
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ extension ModuleDependencyGraph.Tracer {
path.compactMap { node in
node.dependencySource.map {
source in
graph.inputDependencySourceMap[source].map { input in
graph.inputDependencySourceMap.inputIfKnown(for: source).map {
input in
"\(node.key) in \(input.file.basename)"
}
?? "\(node.key)"
Expand Down
3 changes: 1 addition & 2 deletions Tests/SwiftDriverTests/IncrementalCompilationTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -513,9 +513,8 @@ extension IncrementalCompilationTests {
.remark("Finished Compiling main.swift"),
.remark("Incremental compilation: Fingerprint changed for interface of source file main.swiftdeps in main.swiftdeps"),
.remark("Incremental compilation: Fingerprint changed for implementation of source file main.swiftdeps in main.swiftdeps"),
.remark("Incremental compilation: Traced: interface of source file main.swiftdeps in main.swift -> interface of top-level name 'foo' in main.swift -> implementation of source file other.swiftdeps in other.swift"),
.remark("Incremental compilation: Traced: interface of source file main.swiftdeps -> interface of top-level name 'foo' -> implementation of source file other.swiftdeps"),
.warning("Failed to find source file for '"),
.remark("Simulating input not found in inputDependencySourceMap; created for: updatingFromAPrior, now: updatingAfterCompilation"),
.remark("Incremental compilation: Failed to read some dependencies source; compiling everything {compile: main.o <= main.swift}"),
.remark("Incremental compilation: Queuing because of dependencies discovered later: {compile: other.o <= other.swift}"),
.remark("Incremental compilation: Scheduling invalidated {compile: other.o <= other.swift}"),
Expand Down