Skip to content

Commit b222551

Browse files
neonichuneonichu
committed
Write link file list as a build command
This moves the generation of link file lists into the build system instead of doing it ad-hoc outside of the build. For this, we have a new `WriteAuxiliaryFile` tool and associated command that should be usable for any kind of writing of auxiliary files during the build. Note that this change opted to not touch the existing infrastructure for in-process tools, so any inputs that are needed for the file generation will need to be flattened into a generic array of input nodes. The different types of file generation are keyed off a virtual node at the start of that array.
1 parent 2d072e4 commit b222551

File tree

8 files changed

+199
-44
lines changed

8 files changed

+199
-44
lines changed

‎Sources/Build/BuildDescription/ProductBuildDescription.swift

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -336,20 +336,6 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
336336
return self.stripInvalidArguments(args)
337337
}
338338

339-
/// Writes link filelist to the filesystem.
340-
func writeLinkFilelist(_ fs: FileSystem) throws {
341-
var content = self.objects
342-
.map { $0.pathString.spm_shellEscaped() }
343-
.joined(separator: "\n")
344-
345-
// not sure this is needed, added here for backward compatibility
346-
if !content.isEmpty {
347-
content.append("\n")
348-
}
349-
350-
try fs.writeFileContents(self.linkFileListPath, string: content)
351-
}
352-
353339
/// Returns the build flags from the declared build settings.
354340
private func buildSettingsFlags() -> [String] {
355341
var flags: [String] = []

‎Sources/Build/BuildOperation.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -699,6 +699,7 @@ extension BuildDescription {
699699
let testDiscoveryCommands = llbuild.manifest.getCmdToolMap(kind: TestDiscoveryTool.self)
700700
let testEntryPointCommands = llbuild.manifest.getCmdToolMap(kind: TestEntryPointTool.self)
701701
let copyCommands = llbuild.manifest.getCmdToolMap(kind: CopyTool.self)
702+
let writeCommands = llbuild.manifest.getCmdToolMap(kind: WriteAuxiliaryFile.self)
702703

703704
// Create the build description.
704705
let buildDescription = try BuildDescription(
@@ -708,6 +709,7 @@ extension BuildDescription {
708709
testDiscoveryCommands: testDiscoveryCommands,
709710
testEntryPointCommands: testEntryPointCommands,
710711
copyCommands: copyCommands,
712+
writeCommands: writeCommands,
711713
pluginDescriptions: plan.pluginDescriptions
712714
)
713715
try fileSystem.createDirectory(

‎Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,9 @@ public struct BuildDescription: Codable {
303303
/// The map of copy commands.
304304
let copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool]
305305

306+
/// The map of write commands.
307+
let writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile]
308+
306309
/// A flag that indicates this build should perform a check for whether targets only import
307310
/// their explicitly-declared dependencies
308311
let explicitTargetDependencyImportCheckingMode: BuildParameters.TargetDependencyImportCheckingMode
@@ -329,13 +332,15 @@ public struct BuildDescription: Codable {
329332
testDiscoveryCommands: [BuildManifest.CmdName: LLBuildManifest.TestDiscoveryTool],
330333
testEntryPointCommands: [BuildManifest.CmdName: LLBuildManifest.TestEntryPointTool],
331334
copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool],
335+
writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile],
332336
pluginDescriptions: [PluginDescription]
333337
) throws {
334338
self.swiftCommands = swiftCommands
335339
self.swiftFrontendCommands = swiftFrontendCommands
336340
self.testDiscoveryCommands = testDiscoveryCommands
337341
self.testEntryPointCommands = testEntryPointCommands
338342
self.copyCommands = copyCommands
343+
self.writeCommands = writeCommands
339344
self.explicitTargetDependencyImportCheckingMode = plan.buildParameters
340345
.explicitTargetDependencyImportCheckingMode
341346
self.targetDependencyMap = try plan.targets.reduce(into: [TargetName: [TargetName]]()) {
@@ -465,6 +470,66 @@ public final class BuildExecutionContext {
465470
}
466471
}
467472

473+
final class WriteAuxiliaryFileCommand: CustomLLBuildCommand {
474+
override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
475+
guard let buildDescription = self.context.buildDescription else {
476+
return []
477+
}
478+
guard let tool = buildDescription.copyCommands[command.name] else {
479+
return []
480+
}
481+
482+
do {
483+
let encoder = JSONEncoder.makeWithDefaults()
484+
var hash = Data()
485+
hash += try encoder.encode(tool.inputs)
486+
hash += try encoder.encode(tool.outputs)
487+
return [UInt8](hash)
488+
} catch {
489+
self.context.observabilityScope.emit(error: "getSignature() failed: \(error.interpolationDescription)")
490+
return []
491+
}
492+
}
493+
494+
override func execute(
495+
_ command: SPMLLBuild.Command,
496+
_: SPMLLBuild.BuildSystemCommandInterface
497+
) -> Bool {
498+
do {
499+
guard let buildDescription = self.context.buildDescription else {
500+
throw InternalError("unknown build description")
501+
}
502+
guard let tool = buildDescription.writeCommands[command.name] else {
503+
throw StringError("command \(command.name) not registered")
504+
}
505+
506+
guard let output = tool.outputs.first, output.kind == .file else {
507+
throw StringError("invalid output path")
508+
}
509+
let outputFilePath = try AbsolutePath(validating: output.name)
510+
511+
try self.context.fileSystem.writeFileContents(outputFilePath, string: getFileContents(tool: tool))
512+
return true
513+
} catch {
514+
self.context.observabilityScope.emit(error: "failed to write auxiliary file: \(error.interpolationDescription)")
515+
return false
516+
}
517+
}
518+
519+
func getFileContents(tool: WriteAuxiliaryFile) throws -> String {
520+
guard tool.inputs.first?.kind == .virtual, let generatedFileType = tool.inputs.first?.name.dropFirst().dropLast() else {
521+
throw StringError("invalid inputs")
522+
}
523+
524+
switch generatedFileType {
525+
case WriteAuxiliary.LinkFileList.name:
526+
return try WriteAuxiliary.LinkFileList.getFileContents(inputs: Array(tool.inputs.dropFirst()))
527+
default:
528+
throw InternalError("unhandled generated file type '\(generatedFileType)'")
529+
}
530+
}
531+
}
532+
468533
public protocol PackageStructureDelegate {
469534
func packageStructureChanged() -> Bool
470535
}
@@ -584,6 +649,8 @@ final class BuildOperationBuildSystemDelegateHandler: LLBuildBuildSystemDelegate
584649
return InProcessTool(buildExecutionContext, type: PackageStructureCommand.self)
585650
case CopyTool.name:
586651
return InProcessTool(buildExecutionContext, type: CopyCommand.self)
652+
case WriteAuxiliaryFile.name:
653+
return InProcessTool(buildExecutionContext, type: WriteAuxiliaryFileCommand.self)
587654
default:
588655
return nil
589656
}

‎Sources/Build/BuildPlan.swift

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -682,12 +682,6 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
682682
}
683683
buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
684684

685-
// Write the link filelist file.
686-
//
687-
// FIXME: We should write this as a custom llbuild task once we adopt it
688-
// as a library.
689-
try buildProduct.writeLinkFilelist(fileSystem)
690-
691685
buildProduct.availableTools = dependencies.availableTools
692686
}
693687

‎Sources/Build/LLBuildManifestBuilder.swift

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -989,13 +989,13 @@ extension LLBuildManifestBuilder {
989989
try self.manifest.addShellCmd(
990990
name: cmdName,
991991
description: "Archiving \(buildProduct.binaryPath.prettyPath())",
992-
inputs: buildProduct.objects.map(Node.file),
992+
inputs: (buildProduct.objects + [buildProduct.linkFileListPath]).map(Node.file),
993993
outputs: [.file(buildProduct.binaryPath)],
994994
arguments: try buildProduct.archiveArguments()
995995
)
996996

997997
default:
998-
let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath }
998+
let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath } + [buildProduct.linkFileListPath]
999999

10001000
try self.manifest.addShellCmd(
10011001
name: cmdName,
@@ -1023,6 +1023,8 @@ extension LLBuildManifestBuilder {
10231023
}
10241024
self.addNode(output, toTarget: .test)
10251025
}
1026+
1027+
self.manifest.addWriteLinkFileListCommand(objects: Array(buildProduct.objects), linkFileListPath: buildProduct.linkFileListPath)
10261028
}
10271029
}
10281030

‎Sources/LLBuildManifest/BuildManifest.swift

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,33 @@
1212

1313
import Basics
1414

15+
public enum WriteAuxiliary {
16+
public struct LinkFileList {
17+
public static let name = "link-file-list"
18+
19+
public static func getFileContents(inputs: [Node]) throws -> String {
20+
let objects = inputs.compactMap {
21+
if $0.kind == .file {
22+
return $0.name
23+
} else {
24+
return nil
25+
}
26+
}
27+
28+
var content = objects
29+
.map { $0.spm_shellEscaped() }
30+
.joined(separator: "\n")
31+
32+
// not sure this is needed, added here for backward compatibility
33+
if !content.isEmpty {
34+
content.append("\n")
35+
}
36+
37+
return content
38+
}
39+
}
40+
}
41+
1542
public struct BuildManifest {
1643
public typealias TargetName = String
1744
public typealias CmdName = String
@@ -87,6 +114,16 @@ public struct BuildManifest {
87114
commands[name] = Command(name: name, tool: tool)
88115
}
89116

117+
public mutating func addWriteLinkFileListCommand(
118+
objects: [AbsolutePath],
119+
linkFileListPath: AbsolutePath
120+
) {
121+
let inputs = [.virtual(WriteAuxiliary.LinkFileList.name)] + objects.map { Node.file($0) }
122+
let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: linkFileListPath)
123+
let name = linkFileListPath.pathString
124+
commands[name] = Command(name: name, tool: tool)
125+
}
126+
90127
public mutating func addPkgStructureCmd(
91128
name: String,
92129
inputs: [Node],

‎Sources/LLBuildManifest/Tools.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,26 @@ public struct ShellTool: ToolProtocol {
150150
}
151151
}
152152

153+
public struct WriteAuxiliaryFile: ToolProtocol {
154+
public static let name: String = "write-auxiliary-file"
155+
156+
public let inputs: [Node]
157+
private let outputFilePath: AbsolutePath
158+
159+
public init(inputs: [Node], outputFilePath: AbsolutePath) {
160+
self.inputs = inputs
161+
self.outputFilePath = outputFilePath
162+
}
163+
164+
public var outputs: [Node] {
165+
return [.file(outputFilePath)]
166+
}
167+
168+
public func write(to stream: ManifestToolStream) {
169+
stream["description"] = "Write auxiliary file \(outputFilePath.pathString)"
170+
}
171+
}
172+
153173
public struct ClangTool: ToolProtocol {
154174
public static let name: String = "clang"
155175

‎Tests/BuildTests/BuildPlanTests.swift

Lines changed: 69 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -865,9 +865,11 @@ final class BuildPlanTests: XCTestCase {
865865

866866
let buildPath: AbsolutePath = plan.buildParameters.dataPath.appending(components: "release")
867867

868-
let linkedFileList: String = try fs.readFileContents("/path/to/build/release/exe.product/Objects.LinkFileList")
869-
XCTAssertMatch(linkedFileList, .contains("PkgLib"))
870-
XCTAssertNoMatch(linkedFileList, .contains("ExtLib"))
868+
let result = try BuildPlanResult(plan: plan)
869+
let buildProduct = try result.buildProduct(for: "exe")
870+
let objectDirectoryNames = buildProduct.objects.map { $0.parentDirectory.basename }
871+
XCTAssertTrue(objectDirectoryNames.contains("PkgLib.build"))
872+
XCTAssertFalse(objectDirectoryNames.contains("ExtLib.build"))
871873

872874
let yaml = try fs.tempDirectory.appending(components: UUID().uuidString, "release.yaml")
873875
try fs.createDirectory(yaml.parentDirectory, recursive: true)
@@ -891,9 +893,11 @@ final class BuildPlanTests: XCTestCase {
891893
observabilityScope: observability.topScope
892894
)
893895

894-
let linkedFileList: String = try fs.readFileContents("/path/to/build/debug/exe.product/Objects.LinkFileList")
895-
XCTAssertNoMatch(linkedFileList, .contains("PkgLib"))
896-
XCTAssertNoMatch(linkedFileList, .contains("ExtLib"))
896+
let result = try BuildPlanResult(plan: plan)
897+
let buildProduct = try result.buildProduct(for: "exe")
898+
let objectDirectoryNames = buildProduct.objects.map { $0.parentDirectory.basename }
899+
XCTAssertFalse(objectDirectoryNames.contains("PkgLib.build"))
900+
XCTAssertFalse(objectDirectoryNames.contains("ExtLib.build"))
897901

898902
let yaml = try fs.tempDirectory.appending(components: UUID().uuidString, "debug.yaml")
899903
try fs.createDirectory(yaml.parentDirectory, recursive: true)
@@ -1260,13 +1264,8 @@ final class BuildPlanTests: XCTestCase {
12601264
])
12611265
#endif
12621266

1263-
let linkedFileList: String = try fs.readFileContents(buildPath.appending(components: "exe.product", "Objects.LinkFileList"))
1264-
XCTAssertEqual(linkedFileList, """
1265-
\(buildPath.appending(components: "exe.build", "main.c.o"))
1266-
\(buildPath.appending(components: "extlib.build", "extlib.c.o"))
1267-
\(buildPath.appending(components: "lib.build", "lib.c.o"))
1268-
1269-
""")
1267+
let buildProduct = try XCTUnwrap(result.productMap["exe"])
1268+
XCTAssertEqual(Array(buildProduct.objects), [buildPath.appending(components: "exe.build", "main.c.o"), buildPath.appending(components: "extlib.build", "extlib.c.o"), buildPath.appending(components: "lib.build", "lib.c.o")])
12701269
}
12711270

12721271
func testClangConditionalDependency() throws {
@@ -4053,28 +4052,76 @@ final class BuildPlanTests: XCTestCase {
40534052
XCTAssertMatch(contents, .contains("""
40544053
"C.rary-debug.a":
40554054
tool: shell
4056-
inputs: ["\(buildPath.appending(components: "rary.build", "rary.swift.o").escapedPathString())","\(buildPath.appending(components: "rary.build", "rary.swiftmodule.o").escapedPathString())"]
4055+
inputs: ["\(
4056+
buildPath.appending(components: "rary.build", "rary.swift.o")
4057+
.escapedPathString()
4058+
)","\(
4059+
buildPath.appending(components: "rary.build", "rary.swiftmodule.o")
4060+
.escapedPathString()
4061+
)","\(
4062+
buildPath.appending(components: "rary.product", "Objects.LinkFileList")
4063+
.escapedPathString()
4064+
)"]
40574065
outputs: ["\(buildPath.appending(components: "library.a").escapedPathString())"]
40584066
description: "Archiving \(buildPath.appending(components: "library.a").escapedPathString())"
4059-
args: ["\(result.plan.buildParameters.toolchain.librarianPath.escapedPathString())","/LIB","/OUT:\(buildPath.appending(components: "library.a").escapedPathString())","@\(buildPath.appending(components: "rary.product", "Objects.LinkFileList").escapedPathString())"]
4060-
"""))
4067+
args: ["\(
4068+
result.plan.buildParameters.toolchain.librarianPath
4069+
.escapedPathString()
4070+
)","/LIB","/OUT:\(
4071+
buildPath.appending(components: "library.a")
4072+
.escapedPathString()
4073+
)","@\(
4074+
buildPath.appending(components: "rary.product", "Objects.LinkFileList")
4075+
.escapedPathString()
4076+
)"]
4077+
"""))
40614078
} else if result.plan.buildParameters.triple.isDarwin() {
40624079
XCTAssertMatch(contents, .contains("""
40634080
"C.rary-debug.a":
40644081
tool: shell
4065-
inputs: ["\(buildPath.appending(components: "rary.build", "rary.swift.o").escapedPathString())"]
4082+
inputs: ["\(
4083+
buildPath.appending(components: "rary.build", "rary.swift.o")
4084+
.escapedPathString()
4085+
)","\(
4086+
buildPath.appending(components: "rary.product", "Objects.LinkFileList")
4087+
.escapedPathString()
4088+
)"]
40664089
outputs: ["\(buildPath.appending(components: "library.a").escapedPathString())"]
40674090
description: "Archiving \(buildPath.appending(components: "library.a").escapedPathString())"
4068-
args: ["\(result.plan.buildParameters.toolchain.librarianPath.escapedPathString())","-static","-o","\(buildPath.appending(components: "library.a").escapedPathString())","@\(buildPath.appending(components: "rary.product", "Objects.LinkFileList").escapedPathString())"]
4069-
"""))
4070-
} else { // assume Unix `ar` is the librarian
4091+
args: ["\(
4092+
result.plan.buildParameters.toolchain.librarianPath
4093+
.escapedPathString()
4094+
)","-static","-o","\(
4095+
buildPath.appending(components: "library.a")
4096+
.escapedPathString()
4097+
)","@\(
4098+
buildPath.appending(components: "rary.product", "Objects.LinkFileList")
4099+
.escapedPathString()
4100+
)"]
4101+
"""))
4102+
} else { // assume Unix `ar` is the librarian
40714103
XCTAssertMatch(contents, .contains("""
40724104
"C.rary-debug.a":
40734105
tool: shell
4074-
inputs: ["\(buildPath.appending(components: "rary.build", "rary.swift.o").escapedPathString())","\(buildPath.appending(components: "rary.build", "rary.swiftmodule.o").escapedPathString())"]
4106+
inputs: ["\(
4107+
buildPath.appending(components: "rary.build", "rary.swift.o")
4108+
.escapedPathString()
4109+
)","\(
4110+
buildPath.appending(components: "rary.build", "rary.swiftmodule.o")
4111+
.escapedPathString()
4112+
)","\(
4113+
buildPath.appending(components: "rary.product", "Objects.LinkFileList")
4114+
.escapedPathString()
4115+
)"]
40754116
outputs: ["\(buildPath.appending(components: "library.a").escapedPathString())"]
40764117
description: "Archiving \(buildPath.appending(components: "library.a").escapedPathString())"
4077-
args: ["\(result.plan.buildParameters.toolchain.librarianPath.escapedPathString())","crs","\(buildPath.appending(components: "library.a").escapedPathString())","@\(buildPath.appending(components: "rary.product", "Objects.LinkFileList").escapedPathString())"]
4118+
args: ["\(result.plan.buildParameters.toolchain.librarianPath.escapedPathString())","crs","\(
4119+
buildPath
4120+
.appending(components: "library.a").escapedPathString()
4121+
)","@\(
4122+
buildPath
4123+
.appending(components: "rary.product", "Objects.LinkFileList").escapedPathString()
4124+
)"]
40784125
"""))
40794126
}
40804127
}

0 commit comments

Comments
 (0)