-
Notifications
You must be signed in to change notification settings - Fork 10.5k
[3.0] [Driver] Make sure to rebuild dependents when a dirty file fails. #3957
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -456,6 +456,69 @@ int Compilation::performJobsImpl() { | |
llvm::errs() << Output; | ||
} | ||
|
||
// In order to handle both old dependencies that have disappeared and new | ||
// dependencies that have arisen, we need to reload the dependency file. | ||
// Do this whether or not the build succeeded. | ||
SmallVector<const Job *, 16> Dependents; | ||
if (getIncrementalBuildEnabled()) { | ||
const CommandOutput &Output = FinishedCmd->getOutput(); | ||
StringRef DependenciesFile = | ||
Output.getAdditionalOutputForType(types::TY_SwiftDeps); | ||
if (!DependenciesFile.empty() && | ||
(ReturnCode == EXIT_SUCCESS || ReturnCode == EXIT_FAILURE)) { | ||
bool wasCascading = DepGraph.isMarked(FinishedCmd); | ||
|
||
switch (DepGraph.loadFromPath(FinishedCmd, DependenciesFile)) { | ||
case DependencyGraphImpl::LoadResult::HadError: | ||
if (ReturnCode == EXIT_SUCCESS) { | ||
disableIncrementalBuild(); | ||
for (const Job *Cmd : DeferredCommands) | ||
scheduleCommandIfNecessaryAndPossible(Cmd); | ||
DeferredCommands.clear(); | ||
Dependents.clear(); | ||
} // else, let the next build handle it. | ||
break; | ||
case DependencyGraphImpl::LoadResult::UpToDate: | ||
if (!wasCascading) | ||
break; | ||
SWIFT_FALLTHROUGH; | ||
case DependencyGraphImpl::LoadResult::AffectsDownstream: | ||
DepGraph.markTransitive(Dependents, FinishedCmd); | ||
break; | ||
} | ||
} else { | ||
// If there's a crash, assume the worst. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Forgive my ignorance above. I guess, in the case of a crash, I find this comment a little confusing. It mentions the case that there was a crash, but this code may also be executed if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's a good point. The intent is to be conservative for whatever reason, but the comment should probably be updated to reflect that. |
||
switch (FinishedCmd->getCondition()) { | ||
case Job::Condition::NewlyAdded: | ||
// The job won't be treated as newly added next time. Conservatively | ||
// mark it as affecting other jobs, because some of them may have | ||
// completed already. | ||
DepGraph.markTransitive(Dependents, FinishedCmd); | ||
break; | ||
case Job::Condition::Always: | ||
// This applies to non-incremental tasks as well, but any incremental | ||
// task that shows up here has already been marked. | ||
break; | ||
case Job::Condition::RunWithoutCascading: | ||
// If this file changed, it might have been a non-cascading change and | ||
// it might not. Unfortunately, the interface hash has been updated or | ||
// compromised, so we don't actually know anymore; we have to | ||
// conservatively assume the changes could affect other files. | ||
DepGraph.markTransitive(Dependents, FinishedCmd); | ||
break; | ||
case Job::Condition::CheckDependencies: | ||
// If the only reason we're running this is because something else | ||
// changed, then we can trust the dependency graph as to whether it's | ||
// a cascading or non-cascading change. That is, if whatever /caused/ | ||
// the error isn't supposed to affect other files, and whatever | ||
// /fixes/ the error isn't supposed to affect other files, then | ||
// there's no need to recompile any other inputs. If either of those | ||
// are false, we /do/ need to recompile other inputs. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These explanations are awesome -- thanks! (Of course DependencyAnalysis.rst is also super great.) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It was as much demonstrating it to myself as for anyone reading it later. :-) |
||
break; | ||
} | ||
} | ||
} | ||
|
||
if (ReturnCode != EXIT_SUCCESS) { | ||
// The task failed, so return true without performing any further | ||
// dependency analysis. | ||
|
@@ -481,39 +544,10 @@ int Compilation::performJobsImpl() { | |
// might have been blocked. | ||
markFinished(FinishedCmd); | ||
|
||
// In order to handle both old dependencies that have disappeared and new | ||
// dependencies that have arisen, we need to reload the dependency file. | ||
if (getIncrementalBuildEnabled()) { | ||
const CommandOutput &Output = FinishedCmd->getOutput(); | ||
StringRef DependenciesFile = | ||
Output.getAdditionalOutputForType(types::TY_SwiftDeps); | ||
if (!DependenciesFile.empty()) { | ||
SmallVector<const Job *, 16> Dependents; | ||
bool wasCascading = DepGraph.isMarked(FinishedCmd); | ||
|
||
switch (DepGraph.loadFromPath(FinishedCmd, DependenciesFile)) { | ||
case DependencyGraphImpl::LoadResult::HadError: | ||
disableIncrementalBuild(); | ||
for (const Job *Cmd : DeferredCommands) | ||
scheduleCommandIfNecessaryAndPossible(Cmd); | ||
DeferredCommands.clear(); | ||
Dependents.clear(); | ||
break; | ||
case DependencyGraphImpl::LoadResult::UpToDate: | ||
if (!wasCascading) | ||
break; | ||
SWIFT_FALLTHROUGH; | ||
case DependencyGraphImpl::LoadResult::AffectsDownstream: | ||
DepGraph.markTransitive(Dependents, FinishedCmd); | ||
break; | ||
} | ||
|
||
for (const Job *Cmd : Dependents) { | ||
DeferredCommands.erase(Cmd); | ||
noteBuilding(Cmd, "because of dependencies discovered later"); | ||
scheduleCommandIfNecessaryAndPossible(Cmd); | ||
} | ||
} | ||
for (const Job *Cmd : Dependents) { | ||
DeferredCommands.erase(Cmd); | ||
noteBuilding(Cmd, "because of dependencies discovered later"); | ||
scheduleCommandIfNecessaryAndPossible(Cmd); | ||
} | ||
|
||
return TaskFinishedResponse::ContinueExecution; | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1604,36 +1604,40 @@ handleCompileJobCondition(Job *J, CompileJobAction::InputInfo inputInfo, | |
return; | ||
} | ||
|
||
if (!alwaysRebuildDependents) { | ||
// Default all non-newly added files to being rebuilt without cascading. | ||
J->setCondition(Job::Condition::RunWithoutCascading); | ||
} | ||
|
||
bool hasValidModTime = false; | ||
llvm::sys::fs::file_status inputStatus; | ||
if (llvm::sys::fs::status(input, inputStatus)) | ||
return; | ||
|
||
J->setInputModTime(inputStatus.getLastModificationTime()); | ||
if (J->getInputModTime() != inputInfo.previousModTime) | ||
return; | ||
if (!llvm::sys::fs::status(input, inputStatus)) { | ||
J->setInputModTime(inputStatus.getLastModificationTime()); | ||
hasValidModTime = true; | ||
} | ||
|
||
Job::Condition condition; | ||
switch (inputInfo.status) { | ||
case CompileJobAction::InputInfo::UpToDate: | ||
if (!llvm::sys::fs::exists(J->getOutput().getPrimaryOutputFilename())) | ||
if (!hasValidModTime || J->getInputModTime() != inputInfo.previousModTime) { | ||
if (alwaysRebuildDependents || | ||
inputInfo.status == CompileJobAction::InputInfo::NeedsCascadingBuild) { | ||
condition = Job::Condition::Always; | ||
} else { | ||
condition = Job::Condition::RunWithoutCascading; | ||
else | ||
condition = Job::Condition::CheckDependencies; | ||
break; | ||
case CompileJobAction::InputInfo::NeedsCascadingBuild: | ||
condition = Job::Condition::Always; | ||
break; | ||
case CompileJobAction::InputInfo::NeedsNonCascadingBuild: | ||
condition = Job::Condition::RunWithoutCascading; | ||
break; | ||
case CompileJobAction::InputInfo::NewlyAdded: | ||
llvm_unreachable("handled above"); | ||
} | ||
} else { | ||
switch (inputInfo.status) { | ||
case CompileJobAction::InputInfo::UpToDate: | ||
if (!llvm::sys::fs::exists(J->getOutput().getPrimaryOutputFilename())) | ||
condition = Job::Condition::RunWithoutCascading; | ||
else | ||
condition = Job::Condition::CheckDependencies; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit-pick: I only mention it because I've noticed this in a few conditionals in this pull request, but: personally I find it difficult to read
The
Anyway, just my two cents. 💸 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fair enough. |
||
break; | ||
case CompileJobAction::InputInfo::NeedsCascadingBuild: | ||
condition = Job::Condition::Always; | ||
break; | ||
case CompileJobAction::InputInfo::NeedsNonCascadingBuild: | ||
condition = Job::Condition::RunWithoutCascading; | ||
break; | ||
case CompileJobAction::InputInfo::NewlyAdded: | ||
llvm_unreachable("handled above"); | ||
} | ||
} | ||
|
||
J->setCondition(condition); | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
# Dependencies after compilation: | ||
provides-top-level: [a] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
# Dependencies after compilation: | ||
depends-top-level: [a] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
# Dependencies after compilation: | ||
depends-top-level: [!private a] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
{ | ||
"./main.swift": { | ||
"object": "./main.o", | ||
"swift-dependencies": "./main.swiftdeps" | ||
}, | ||
"./crash.swift": { | ||
"object": "./crash.o", | ||
"swift-dependencies": "./crash.swiftdeps" | ||
}, | ||
"./other.swift": { | ||
"object": "./other.o", | ||
"swift-dependencies": "./other.swiftdeps" | ||
}, | ||
"": { | ||
"swift-dependencies": "./main~buildrecord.swiftdeps" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
# Dependencies after compilation: | ||
provides-top-level: [bad] | ||
interface-hash: "after" | ||
garbage: "" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies before compilation: | ||
provides-top-level: [bad] | ||
interface-hash: "before" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies after compilation: | ||
depends-top-level: [bad] | ||
interface-hash: "after" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies before compilation: | ||
depends-top-level: [bad] | ||
interface-hash: "before" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies after compilation: | ||
depends-top-level: [main] | ||
interface-hash: "after" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies before compilation: | ||
depends-top-level: [main] | ||
interface-hash: "before" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies after compilation: | ||
provides-top-level: [main] | ||
interface-hash: "after" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Dependencies before compilation: | ||
provides-top-level: [main] | ||
interface-hash: "before" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
{ | ||
"./main.swift": { | ||
"object": "./main.o", | ||
"swift-dependencies": "./main.swiftdeps" | ||
}, | ||
"./bad.swift": { | ||
"object": "./bad.o", | ||
"swift-dependencies": "./bad.swiftdeps" | ||
}, | ||
"./depends-on-main.swift": { | ||
"object": "./depends-on-main.o", | ||
"swift-dependencies": "./depends-on-main.swiftdeps" | ||
}, | ||
"./depends-on-bad.swift": { | ||
"object": "./depends-on-bad.o", | ||
"swift-dependencies": "./depends-on-bad.swiftdeps" | ||
}, | ||
"": { | ||
"swift-dependencies": "./main~buildrecord.swiftdeps" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,23 +11,38 @@ | |
# | ||
# ---------------------------------------------------------------------------- | ||
# | ||
# Fails if the input file is named "bad.swift"; otherwise dispatches to | ||
# update-dependencies.py. | ||
# Fails if the input file is named "bad.swift" or "crash.swift"; otherwise | ||
# dispatches to update-dependencies.py. "crash.swift" gives an exit code | ||
# other than 1. | ||
# | ||
# ---------------------------------------------------------------------------- | ||
|
||
from __future__ import print_function | ||
|
||
import os | ||
import shutil | ||
import sys | ||
|
||
assert sys.argv[1] == '-frontend' | ||
|
||
primaryFile = sys.argv[sys.argv.index('-primary-file') + 1] | ||
|
||
if os.path.basename(primaryFile) == 'bad.swift': | ||
if (os.path.basename(primaryFile) == 'bad.swift' or | ||
os.path.basename(primaryFile) == 'crash.swift'): | ||
print("Handled", os.path.basename(primaryFile)) | ||
sys.exit(1) | ||
|
||
# Replace the dependencies file with the input file. | ||
try: | ||
depsFile = sys.argv[sys.argv.index( | ||
'-emit-reference-dependencies-path') + 1] | ||
shutil.copyfile(primaryFile, depsFile) | ||
except ValueError: | ||
pass | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit-pick: I think it'd be preferable to use the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That's more overhead than I wanted here. These scripts are trying to mimic the Swift frontend, and I don't want them to be trapped in whatever argparse does if we diverge anywhere. |
||
|
||
if os.path.basename(primaryFile) == 'bad.swift': | ||
sys.exit(1) | ||
else: | ||
sys.exit(129) | ||
|
||
dir = os.path.dirname(os.path.abspath(__file__)) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit-pick: |
||
execfile(os.path.join(dir, "update-dependencies.py")) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I interpret this reference to mean that
ReturnCode
must always be eitherEXIT_SUCCESS
orEXIT_FAILURE
. Is there a case in which it is neither? In other words, is this check necessary?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah, it's a check for crashes. It's not the most direct way to check for abnormal exit, but it ought to work.