WIP: Write link file list as a build command

Author: neonichu

Sources/Build/BuildDescription/ProductBuildDescription.swift

@@ -336,20 +336,6 @@ public final class ProductBuildDescription: SPMBuildCore.ProductBuildDescription
         return self.stripInvalidArguments(args)
     }
 
-    /// Writes link filelist to the filesystem.
-    func writeLinkFilelist(_ fs: FileSystem) throws {
-        var content = self.objects
-            .map { $0.pathString.spm_shellEscaped() }
-            .joined(separator: "\n")
-
-        // not sure this is needed, added here for backward compatibility
-        if !content.isEmpty {
-            content.append("\n")
-        }
-
-        try fs.writeFileContents(self.linkFileListPath, string: content)
-    }
-
     /// Returns the build flags from the declared build settings.
     private func buildSettingsFlags() -> [String] {
         var flags: [String] = []

Sources/Build/BuildOperation.swift

@@ -699,6 +699,7 @@ extension BuildDescription {
         let testDiscoveryCommands = llbuild.manifest.getCmdToolMap(kind: TestDiscoveryTool.self)
         let testEntryPointCommands = llbuild.manifest.getCmdToolMap(kind: TestEntryPointTool.self)
         let copyCommands = llbuild.manifest.getCmdToolMap(kind: CopyTool.self)
+        let writeCommands = llbuild.manifest.getCmdToolMap(kind: WriteAuxiliaryFile.self)
 
         // Create the build description.
         let buildDescription = try BuildDescription(
@@ -708,6 +709,7 @@ extension BuildDescription {
             testDiscoveryCommands: testDiscoveryCommands,
             testEntryPointCommands: testEntryPointCommands,
             copyCommands: copyCommands,
+            writeCommands: writeCommands,
             pluginDescriptions: plan.pluginDescriptions
         )
         try fileSystem.createDirectory(

Sources/Build/BuildOperationBuildSystemDelegateHandler.swift

@@ -303,6 +303,9 @@ public struct BuildDescription: Codable {
     /// The map of copy commands.
     let copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool]
 
+    /// The map of write commands.
+    let writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile]
+
     /// A flag that indicates this build should perform a check for whether targets only import
     /// their explicitly-declared dependencies
     let explicitTargetDependencyImportCheckingMode: BuildParameters.TargetDependencyImportCheckingMode
@@ -329,13 +332,15 @@ public struct BuildDescription: Codable {
         testDiscoveryCommands: [BuildManifest.CmdName: LLBuildManifest.TestDiscoveryTool],
         testEntryPointCommands: [BuildManifest.CmdName: LLBuildManifest.TestEntryPointTool],
         copyCommands: [BuildManifest.CmdName: LLBuildManifest.CopyTool],
+        writeCommands: [BuildManifest.CmdName: LLBuildManifest.WriteAuxiliaryFile],
         pluginDescriptions: [PluginDescription]
     ) throws {
         self.swiftCommands = swiftCommands
         self.swiftFrontendCommands = swiftFrontendCommands
         self.testDiscoveryCommands = testDiscoveryCommands
         self.testEntryPointCommands = testEntryPointCommands
         self.copyCommands = copyCommands
+        self.writeCommands = writeCommands
         self.explicitTargetDependencyImportCheckingMode = plan.buildParameters
             .explicitTargetDependencyImportCheckingMode
         self.targetDependencyMap = try plan.targets.reduce(into: [TargetName: [TargetName]]()) {
@@ -465,6 +470,65 @@ public final class BuildExecutionContext {
     }
 }
 
+final class WriteLinkFilelistCommand: CustomLLBuildCommand {
+    override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
+        guard let buildDescription = self.context.buildDescription else {
+            return []
+        }
+        guard let tool = buildDescription.copyCommands[command.name] else {
+            return []
+        }
+
+        let encoder = JSONEncoder.makeWithDefaults()
+        var hash = Data()
+        hash += try! encoder.encode(tool.inputs)
+        hash += try! encoder.encode(tool.outputs)
+        return [UInt8](hash)
+    }
+
+    override func execute(
+        _ command: SPMLLBuild.Command,
+        _: SPMLLBuild.BuildSystemCommandInterface
+    ) -> Bool {
+        do {
+            guard let buildDescription = self.context.buildDescription else {
+                throw InternalError("unknown build description")
+            }
+            guard let tool = buildDescription.writeCommands[command.name] else {
+                throw StringError("command \(command.name) not registered")
+            }
+
+            guard let output = tool.outputs.first, output.kind == .file else {
+                throw StringError("invalid output path")
+            }
+            let linkFileListPath = try AbsolutePath(validating: output.name)
+
+            let objects = tool.inputs.compactMap {
+                if $0.kind == .file {
+                    return $0.name
+                } else {
+                    return nil
+                }
+            }
+
+            var content = objects
+                .map { $0.spm_shellEscaped() }
+                .joined(separator: "\n")
+
+            // not sure this is needed, added here for backward compatibility
+            if !content.isEmpty {
+                content.append("\n")
+            }
+
+            try self.context.fileSystem.writeFileContents(linkFileListPath, string: content)
+            return true
+        } catch {
+            self.context.observabilityScope.emit(error: "failed to write link file list: \(error.interpolationDescription)")
+            return false
+        }
+    }
+}
+
 public protocol PackageStructureDelegate {
     func packageStructureChanged() -> Bool
 }
@@ -584,6 +648,8 @@ final class BuildOperationBuildSystemDelegateHandler: LLBuildBuildSystemDelegate
             return InProcessTool(buildExecutionContext, type: PackageStructureCommand.self)
         case CopyTool.name:
             return InProcessTool(buildExecutionContext, type: CopyCommand.self)
+        case WriteAuxiliaryFile.name:
+            return InProcessTool(buildExecutionContext, type: WriteLinkFilelistCommand.self)
         default:
             return nil
         }

Sources/Build/BuildPlan.swift

@@ -682,12 +682,6 @@ public class BuildPlan: SPMBuildCore.BuildPlan {
         }
         buildProduct.libraryBinaryPaths = dependencies.libraryBinaryPaths
 
-        // Write the link filelist file.
-        //
-        // FIXME: We should write this as a custom llbuild task once we adopt it
-        // as a library.
-        try buildProduct.writeLinkFilelist(fileSystem)
-
         buildProduct.availableTools = dependencies.availableTools
     }
 

Sources/Build/LLBuildManifestBuilder.swift

@@ -995,7 +995,7 @@ extension LLBuildManifestBuilder {
             )
 
         default:
-            let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath }
+            let inputs = try buildProduct.objects + buildProduct.dylibs.map{ try $0.binaryPath } + [buildProduct.linkFileListPath]
 
             try self.manifest.addShellCmd(
                 name: cmdName,
@@ -1023,6 +1023,8 @@ extension LLBuildManifestBuilder {
             }
             self.addNode(output, toTarget: .test)
         }
+
+        self.manifest.addWriteAuxiliaryFileCommand(objects: Array(buildProduct.objects), linkFileListPath: buildProduct.linkFileListPath)
     }
 }
 

Sources/LLBuildManifest/BuildManifest.swift

@@ -87,6 +87,15 @@ public struct BuildManifest {
         commands[name] = Command(name: name, tool: tool)
     }
 
+    public mutating func addWriteAuxiliaryFileCommand(
+        objects: [AbsolutePath],
+        linkFileListPath: AbsolutePath
+    ) {
+        let tool = WriteAuxiliaryFile(objects: objects, linkFileListPath: linkFileListPath)
+        let name = linkFileListPath.pathString
+        commands[name] = Command(name: name, tool: tool)
+    }
+
     public mutating func addPkgStructureCmd(
         name: String,
         inputs: [Node],

Sources/LLBuildManifest/Tools.swift

@@ -150,6 +150,30 @@ public struct ShellTool: ToolProtocol {
     }
 }
 
+public struct WriteAuxiliaryFile: ToolProtocol {
+    public static let name: String = "write-link-file-list"
+
+    private let objects: [AbsolutePath]
+    private let linkFileListPath: AbsolutePath
+
+    public init(objects: [AbsolutePath], linkFileListPath: AbsolutePath) {
+        self.objects = objects
+        self.linkFileListPath = linkFileListPath
+    }
+
+    public var inputs: [Node] {
+        return self.objects.map { Node.file($0) }
+    }
+
+    public var outputs: [Node] {
+        return [Node.file(linkFileListPath)]
+    }
+
+    public func write(to stream: ManifestToolStream) {
+        stream["description"] = "yolo"
+    }
+}
+
 public struct ClangTool: ToolProtocol {
     public static let name: String = "clang"