Skip to content

Ensure swift jobs with the same command line but different dependenci… #520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 22, 2025
Merged

Ensure swift jobs with the same command line but different dependenci… #520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions Sources/SWBCore/LibSwiftDriver/PlannedBuild.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,6 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
public let outputs: [Path]
/// The command line to execute for this job
public let commandLine: [SWBUtil.ByteString]
/// A signature which uniquely identifies the job.
public let signature: SWBUtil.ByteString
/// Cache keys for the swift-frontend invocation (one key per output producing input)
public let cacheKeys: [String]

Expand All @@ -103,15 +101,10 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
self.cacheKeys = job.outputCacheKeys.reduce(into: [String]()) { result, key in
result.append(key.value)
}.sorted()
let md5 = InsecureHashContext()
for arg in commandLine {
md5.add(bytes: arg)
}
self.signature = md5.signature
}

public func serialize<T>(to serializer: T) where T : Serializer {
serializer.serializeAggregate(10) {
serializer.serializeAggregate(9) {
serializer.serialize(kind)
serializer.serialize(ruleInfoType)
serializer.serialize(moduleName)
Expand All @@ -120,13 +113,12 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
serializer.serialize(outputs)
serializer.serialize(commandLine)
serializer.serialize(descriptionForLifecycle)
serializer.serialize(signature)
serializer.serialize(cacheKeys)
}
}

public init(from deserializer: any Deserializer) throws {
try deserializer.beginAggregate(10)
try deserializer.beginAggregate(9)
try self.kind = deserializer.deserialize()
try self.ruleInfoType = deserializer.deserialize()
try self.moduleName = deserializer.deserialize()
Expand All @@ -135,7 +127,6 @@ public struct SwiftDriverJob: Serializable, CustomDebugStringConvertible {
try self.outputs = deserializer.deserialize()
try self.commandLine = deserializer.deserialize()
try self.descriptionForLifecycle = deserializer.deserialize()
try self.signature = deserializer.deserialize()
try self.cacheKeys = deserializer.deserialize()
}

Expand Down Expand Up @@ -173,20 +164,30 @@ extension LibSwiftDriver {
public let dependencies: [JobKey]
/// Working directory for running this job
public let workingDirectory: Path
/// A signature which uniquely identifies this planned job.
public let signature: SWBUtil.ByteString

internal init(key: JobKey, driverJob: SwiftDriverJob, dependencies: [JobKey], workingDirectory: Path) {
self.key = key
self.driverJob = driverJob
self.dependencies = dependencies
self.workingDirectory = workingDirectory
let md5 = InsecureHashContext()
for arg in driverJob.commandLine {
md5.add(bytes: arg)
}
md5.add(string: workingDirectory.str)
md5.add(number: dependencies.hashValue)
self.signature = md5.signature
}

public func serialize<T>(to serializer: T) where T : Serializer {
serializer.serializeAggregate(4) {
serializer.serializeAggregate(5) {
serializer.serialize(key)
serializer.serialize(driverJob)
serializer.serialize(dependencies)
serializer.serialize(workingDirectory)
serializer.serialize(signature)
}
}

Expand All @@ -196,6 +197,7 @@ extension LibSwiftDriver {
try driverJob = deserializer.deserialize()
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@owenv This missed changing 4 to 5 in deserialization

try dependencies = deserializer.deserialize()
try workingDirectory = deserializer.deserialize()
try signature = deserializer.deserialize()
}

public func addingDependencies(_ newDependencies: [JobKey]) -> PlannedSwiftDriverJob {
Expand Down
6 changes: 3 additions & 3 deletions Sources/SWBTaskExecution/TaskActions/SwiftDriverJobSchedulingTaskAction.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,7 +291,7 @@ open class SwiftDriverJobSchedulingTaskAction: TaskAction {
outputDelegate.previouslyBatchedSubtaskUpToDate(signature: SwiftCompilerSpec.computeRuleInfoAndSignatureForPerFileVirtualBatchSubtask(variant: driverPayload.variant, arch: driverPayload.architecture, path: singleInput).1, target: target)
} else {
// Other jobs are reported as skipped/up-to-date in the usual way.
let taskKey = SwiftDriverJobTaskKey(identifier: driverPayload.uniqueID, variant: driverPayload.variant, arch: driverPayload.architecture, driverJobKey: job.key, driverJobSignature: job.driverJob.signature, isUsingWholeModuleOptimization: driverPayload.isUsingWholeModuleOptimization, compilerLocation: driverPayload.compilerLocation, casOptions: driverPayload.casOptions)
let taskKey = SwiftDriverJobTaskKey(identifier: driverPayload.uniqueID, variant: driverPayload.variant, arch: driverPayload.architecture, driverJobKey: job.key, driverJobSignature: job.signature, isUsingWholeModuleOptimization: driverPayload.isUsingWholeModuleOptimization, compilerLocation: driverPayload.compilerLocation, casOptions: driverPayload.casOptions)
let dynamicTask = DynamicTask(toolIdentifier: SwiftDriverJobTaskAction.toolIdentifier, taskKey: .swiftDriverJob(taskKey), workingDirectory: task.workingDirectory, environment: task.environment, target: task.forTarget, showEnvironment: task.showEnvironment)
let subtask = try spec.buildExecutableTask(dynamicTask: dynamicTask, context: dynamicExecutionDelegate.operationContext)
outputDelegate.subtaskUpToDate(subtask)
Expand All @@ -306,7 +306,7 @@ open class SwiftDriverJobSchedulingTaskAction: TaskAction {
key = .swiftDriverExplicitDependencyJob(SwiftDriverExplicitDependencyJobTaskKey(
arch: driverPayload.architecture,
driverJobKey: plannedJob.key,
driverJobSignature: plannedJob.driverJob.signature,
driverJobSignature: plannedJob.signature,
compilerLocation: driverPayload.compilerLocation,
casOptions: driverPayload.casOptions))
} else {
Expand All @@ -315,7 +315,7 @@ open class SwiftDriverJobSchedulingTaskAction: TaskAction {
variant: driverPayload.variant,
arch: driverPayload.architecture,
driverJobKey: plannedJob.key,
driverJobSignature: plannedJob.driverJob.signature,
driverJobSignature: plannedJob.signature,
isUsingWholeModuleOptimization: driverPayload.isUsingWholeModuleOptimization,
compilerLocation: driverPayload.compilerLocation,
casOptions: driverPayload.casOptions))
Expand Down
6 changes: 3 additions & 3 deletions Sources/SWBTaskExecution/TaskActions/SwiftDriverJobTaskAction.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas
public override func getSignature(_ task: any ExecutableTask, executionDelegate: any TaskExecutionDelegate) -> ByteString {
let md5 = InsecureHashContext()
// We intentionally do not integrate the superclass signature here, because the driver job's signature captures the same information without requiring expensive serialization.
md5.add(bytes: driverJob.driverJob.signature)
md5.add(bytes: driverJob.signature)
task.environment.computeSignature(into: md5)
return md5.signature
}
Expand Down Expand Up @@ -211,7 +211,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas
key = .swiftDriverExplicitDependencyJob(SwiftDriverExplicitDependencyJobTaskKey(
arch: arch,
driverJobKey: plannedJob.key,
driverJobSignature: plannedJob.driverJob.signature,
driverJobSignature: plannedJob.signature,
compilerLocation: compilerLocation,
casOptions: casOptions))
} else {
Expand All @@ -226,7 +226,7 @@ public final class SwiftDriverJobTaskAction: TaskAction, BuildValueValidatingTas
variant: variant,
arch: arch,
driverJobKey: plannedJob.key,
driverJobSignature: plannedJob.driverJob.signature,
driverJobSignature: plannedJob.signature,
isUsingWholeModuleOptimization: isUsingWholeModuleOptimization,
compilerLocation: compilerLocation,
casOptions: casOptions))
Expand Down
143 changes: 143 additions & 0 deletions Tests/SWBBuildSystemTests/SwiftDriverTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5143,4 +5143,147 @@ fileprivate struct SwiftDriverTests: CoreBasedTests {
#expect(cleanContents == incrementalContents)
}
}

@Test(.requireSDKs(.macOS))
func ensureIdenticalCommandLinesWithDifferentDependenciesAreNotDeduplicated() async throws {
try await withTemporaryDirectory { tmpDir in
let testWorkspace = try await TestWorkspace(
"Test",
sourceRoot: tmpDir.join("Test"),
projects: [
TestProject(
"aProject",
groupTree: TestGroup(
"Sources",
children: [
TestFile("Framework1.h"),
TestFile("file_1.c"),
TestFile("Framework2.h"),
TestFile("file_2.c"),
TestFile("file_3.swift"),
]),
buildConfigurations: [TestBuildConfiguration(
"Debug",
buildSettings: [
"PRODUCT_NAME": "$(TARGET_NAME)",
"CLANG_ENABLE_MODULES": "YES",
"SWIFT_ENABLE_EXPLICIT_MODULES": "YES",
"SWIFT_VERSION": swiftVersion,
"DEFINES_MODULE": "YES",
"VALID_ARCHS": "arm64",
"DSTROOT": tmpDir.join("dstroot").str,
"SWIFT_ENABLE_COMPILE_CACHE": "YES",
])],
targets: [
TestStandardTarget(
"Framework1",
type: .framework,
buildPhases: [
TestHeadersBuildPhase([TestBuildFile("Framework1.h", headerVisibility: .public)]),
TestSourcesBuildPhase(["file_1.c"]),
]),
TestStandardTarget(
"Framework2",
type: .framework,
buildPhases: [
TestHeadersBuildPhase([TestBuildFile("Framework2.h", headerVisibility: .public)]),
TestSourcesBuildPhase(["file_2.c"]),
]),
TestStandardTarget(
"Framework3",
type: .framework,
buildPhases: [
TestSourcesBuildPhase(["file_3.swift"]),
],
dependencies: [
"Framework1",
"Framework2"
]),
])])

let tester = try await BuildOperationTester(getCore(), testWorkspace, simulated: false)

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/Framework1.h")) { stream in
stream <<<
"""
void foo(void);
"""
}

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file_1.c")) { stream in
stream <<<
"""
void foo(void) {}
"""
}

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/Framework2.h")) { stream in
stream <<<
"""
void qux(void);
"""
}

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file_2.c")) { stream in
stream <<<
"""
void qux(void) {}
"""
}

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/file_3.swift")) { stream in
stream <<<
"""
import Framework1
import Framework2
public func bar() {
foo()
qux()
}
"""
}

let parameters = BuildParameters(configuration: "Debug", overrides: [:])
let buildRequest = BuildRequest(parameters: parameters, buildTargets: tester.workspace.projects[0].targets.map({ BuildRequest.BuildTargetInfo(parameters: parameters, target: $0) }), continueBuildingAfterErrors: false, useParallelTargets: true, useImplicitDependencies: false, useDryRun: false)

try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
results.checkTasks(.matchRuleType("SwiftExplicitDependencyGeneratePcm")) { tasks in
#expect(tasks.count == 4)
}
results.checkNoDiagnostics()
}

try await tester.checkNullBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true)

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/Framework1.h")) { stream in
stream <<<
"""
void foo(void); introduce an error
"""
}

try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
results.checkTaskExists(.matchRuleType("SwiftDriver"))
results.checkTasks(.matchRuleType("SwiftExplicitDependencyGeneratePcm")) { tasks in
#expect(tasks.count == 1)
}
results.checkedErrors = true
}

try await tester.fs.writeFileContents(testWorkspace.sourceRoot.join("aProject/Framework1.h")) { stream in
stream <<<
"""
void foo(void);
"""
}

try await tester.checkBuild(runDestination: .macOS, buildRequest: buildRequest, persistent: true) { results in
results.checkTaskExists(.matchRuleType("SwiftDriver"))
results.checkTasks(.matchRuleType("SwiftExplicitDependencyGeneratePcm")) { tasks in
#expect(tasks.count == 1)
}
results.checkNoDiagnostics()
}
}
}
}
Loading