SwiftPM does not build all dependency executables before invoking plugin build command (#5636)

Make the list of input dependencies of an llbuild command that runs a plugin-provided build tool also have dependencies on the paths of all the tools on which the plugin has declared a dependency, not just the one that appears as an executable in the build command.

Also adds a mildly elaborate unit test to make sure this works.

rdar://93679015

Author: abertelrud

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Package.swift

@@ -0,0 +1,34 @@
+// swift-tools-version: 5.7
+import PackageDescription
+
+let package = Package(
+    name: "MyBuildToolPluginDependencies",
+    targets: [
+        // A local tool that uses a build tool plugin.
+        .executableTarget(
+            name: "MyLocalTool",
+            plugins: [
+                "MySourceGenBuildToolPlugin",
+            ]
+        ),
+        // The plugin that generates build tool commands to invoke MySourceGenBuildTool.
+        .plugin(
+            name: "MySourceGenBuildToolPlugin",
+            capability: .buildTool(),
+            dependencies: [
+                "MySourceGenBuildTool",
+            ]
+        ),
+        // A command line tool that generates source files.
+        .executableTarget(
+            name: "MySourceGenBuildTool",
+            dependencies: [
+                "MySourceGenBuildToolLib",
+            ]
+        ),
+        // A library used by MySourceGenBuildTool (not the client).
+        .target(
+            name: "MySourceGenBuildToolLib"
+        ),
+    ]
+)

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Plugins/MySourceGenBuildToolPlugin/plugin.swift

@@ -0,0 +1,38 @@
+import PackagePlugin
+
+@main
+struct MyPlugin: BuildToolPlugin {
+    
+    // Create build commands that don't invoke the MySourceGenBuildTool source generator
+    // tool directly, but instead invoke a system tool that invokes it indirectly.  We
+    // want to test that we still end up with a dependency on not only that tool but also
+    // on the library it depends on, even without including an explicit dependency on it.
+    func createBuildCommands(context: PluginContext, target: Target) throws -> [Command] {
+        print("Hello from the Build Tool Plugin!")
+        guard let target = target as? SourceModuleTarget else { return [] }
+        let inputFiles = target.sourceFiles.filter({ $0.path.extension == "dat" })
+        return try inputFiles.map {
+            let inputFile = $0
+            let inputPath = inputFile.path
+            let outputName = inputPath.stem + ".swift"
+            let outputPath = context.pluginWorkDirectory.appending(outputName)
+            return .buildCommand(
+                displayName:
+                    "Generating \(outputName) from \(inputPath.lastComponent)",
+                executable:
+                    Path("/usr/bin/env"),
+                arguments: [
+                    try context.tool(named: "MySourceGenBuildTool").path,
+                    "\(inputPath)",
+                    "\(outputPath)"
+                ],
+                inputFiles: [
+                    inputPath,
+                ],
+                outputFiles: [
+                    outputPath
+                ]
+            )
+        }
+    }
+}

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Sources/MyLocalTool/foo.dat

@@ -0,0 +1 @@
+let foo = "I am Foo!"

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Sources/MyLocalTool/main.swift

@@ -0,0 +1 @@
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Sources/MySourceGenBuildTool/main.swift

@@ -0,0 +1,18 @@
+import Foundation
+import MySourceGenBuildToolLib
+
+// Sample source generator tool that emits a Swift variable declaration of a string containing the hex representation of the contents of a file as a quoted string.  The variable name is the base name of the input file.  The input file is the first argument and the output file is the second.
+if ProcessInfo.processInfo.arguments.count != 3 {
+    print("usage: MySourceGenBuildTool <input> <output>")
+    exit(1)
+}
+let inputFile = ProcessInfo.processInfo.arguments[1]
+let outputFile = ProcessInfo.processInfo.arguments[2]
+
+let variableName = URL(fileURLWithPath: inputFile).deletingPathExtension().lastPathComponent
+
+let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+let outputString = "public var \(variableName) = \(dataAsHex.quotedForSourceCode)\n"
+let outputData = outputString.data(using: .utf8)
+FileManager.default.createFile(atPath: outputFile, contents: outputData)

Fixtures/Miscellaneous/Plugins/MyBuildToolPluginDependencies/Sources/MySourceGenBuildToolLib/library.swift

@@ -0,0 +1,11 @@
+import Foundation
+
+extension String {
+    
+    public var quotedForSourceCode: String {
+        return "\"" + self
+            .replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "\"", with: "\\\"")
+            + "\""
+    }
+}

Sources/Build/LLBuildManifestBuilder.swift

@@ -625,7 +625,7 @@ extension LLBuildManifestBuilder {
                 manifest.addShellCmd(
                     name: displayName + "-" + ByteString(encodingAsUTF8: uniquedName).sha256Checksum,
                     description: displayName,
-                    inputs: [.file(execPath)] + command.inputFiles.map{ .file($0) },
+                    inputs: command.inputFiles.map{ .file($0) },
                     outputs: command.outputFiles.map{ .file($0) },
                     arguments: commandLine,
                     environment: command.configuration.environment,

Sources/SPMBuildCore/PluginInvocation.swift

@@ -353,6 +353,9 @@ extension PackageGraph {
                         dict[name] = path
                     }
                 })
+                
+                // Determine additional input dependencies for any plugin commands, based on any executables the plugin target depends on.
+                let toolPaths = toolNamesToPaths.values.sorted()
 
                 // Assign a plugin working directory based on the package, target, and plugin.
                 let pluginOutputDir = outputDir.appending(components: package.identity.description, target.name, pluginTarget.name)
@@ -367,13 +370,15 @@ extension PackageGraph {
                 let delegateQueue = DispatchQueue(label: "plugin-invocation")
                 class PluginDelegate: PluginInvocationDelegate {
                     let delegateQueue: DispatchQueue
+                    let toolPaths: [AbsolutePath]
                     var outputData = Data()
                     var diagnostics = [Basics.Diagnostic]()
                     var buildCommands = [BuildToolPluginInvocationResult.BuildCommand]()
                     var prebuildCommands = [BuildToolPluginInvocationResult.PrebuildCommand]()
 
-                    init(delegateQueue: DispatchQueue) {
+                    init(delegateQueue: DispatchQueue, toolPaths: [AbsolutePath]) {
                         self.delegateQueue = delegateQueue
+                        self.toolPaths = toolPaths
                     }
 
                     func pluginEmittedOutput(_ data: Data) {
@@ -395,7 +400,7 @@ extension PackageGraph {
                                 arguments: arguments,
                                 environment: environment,
                                 workingDirectory: workingDirectory),
-                            inputFiles: inputFiles,
+                            inputFiles: toolPaths + inputFiles,
                             outputFiles: outputFiles))
                     }
 
@@ -411,7 +416,7 @@ extension PackageGraph {
                             outputFilesDirectory: outputFilesDirectory))
                     }
                 }
-                let delegate = PluginDelegate(delegateQueue: delegateQueue)
+                let delegate = PluginDelegate(delegateQueue: delegateQueue, toolPaths: toolPaths)
 
                 // Invoke the build tool plugin with the input parameters and the delegate that will collect outputs.
                 let startTime = DispatchTime.now()

Tests/FunctionalTests/PluginTests.swift

@@ -102,6 +102,20 @@ class PluginTests: XCTestCase {
             XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
         }
     }
+    
+    func testBuildToolPluginDependencies() throws {        
+        // Only run the test if the environment in which we're running actually supports Swift concurrency (which the plugin APIs require).
+        try XCTSkipIf(!UserToolchain.default.supportsSwiftConcurrency(), "skipping because test environment doesn't support concurrency")
+        
+        try fixture(name: "Miscellaneous/Plugins") { fixturePath in
+            let (stdout, _) = try executeSwiftBuild(fixturePath.appending(component: "MyBuildToolPluginDependencies"))
+            XCTAssert(stdout.contains("Compiling MySourceGenBuildTool main.swift"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Compiling MyLocalTool foo.swift"), "stdout:\n\(stdout)")
+            XCTAssert(stdout.contains("Build complete!"), "stdout:\n\(stdout)")
+        }
+    }
 
     func testContrivedTestCases() throws {
         // Only run the test if the environment in which we're running actually supports Swift concurrency (which the plugin APIs require).

Tests/SPMBuildCoreTests/PluginInvocationTests.swift

@@ -212,7 +212,7 @@ class PluginInvocationTests: XCTestCase {
         XCTAssertEqual(evalFirstCommand.configuration.arguments, ["-c", "/Foo/Sources/Foo/SomeFile.abc"])
         XCTAssertEqual(evalFirstCommand.configuration.environment, ["X": "Y"])
         XCTAssertEqual(evalFirstCommand.configuration.workingDirectory, AbsolutePath("/Foo/Sources/Foo"))
-        XCTAssertEqual(evalFirstCommand.inputFiles, [])
+        XCTAssertEqual(evalFirstCommand.inputFiles, [builtToolsDir.appending(component: "FooTool")])
         XCTAssertEqual(evalFirstCommand.outputFiles, [])
 
         XCTAssertEqual(evalFirstResult.diagnostics.count, 1)