Run prebuild commands from plugins before the build starts

The generated source file paths are also added to the build plan llbuild task so that the build plan is recomputed if there are any changes to them.

rdar://74614476

Author: abertelrud

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Package.swift

@@ -7,18 +7,18 @@ let package = Package(
         .package(path: "../MySourceGenPlugin")
     ],
     targets: [
-        // A tool that uses an plugin.
+        // A tool that uses a plugin.
         .executableTarget(
             name: "MyTool",
             dependencies: [
-                .product(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .product(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         ),
-        // A unit that uses the plugin.
+        // A unit test that uses the plugin.
         .testTarget(
             name: "MyTests",
             plugins: [
-                .plugin(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .plugin(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/Foo.dat

@@ -1 +1 @@
-Hello Extension Product!
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/main.swift

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Package.swift

@@ -4,42 +4,60 @@ import PackageDescription
 let package = Package(
     name: "MySourceGenPlugin",
     products: [
-        // The product that vends MySourceGenPlugin to client packages.
+        // The product that vends MySourceGenBuildToolPlugin to client packages.
         .plugin(
-            name: "MySourceGenPlugin",
-            targets: ["MySourceGenPlugin"]
+            name: "MySourceGenBuildToolPlugin",
+            targets: ["MySourceGenBuildToolPlugin"]
         ),
+        // The product that vends the MySourceGenBuildTool executable to client packages.
         .executable(
-            name: "MySourceGenTool",
-            targets: ["MySourceGenTool"]
-        )
+            name: "MySourceGenBuildTool",
+            targets: ["MySourceGenBuildTool"]
+        ),
+        // The product that vends MySourceGenPrebuildPlugin to client packages.
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            targets: ["MySourceGenPrebuildPlugin"]
+        ),
     ],
     targets: [
-        // A local tool that uses a plugin.
+        // A local tool that uses a build tool plugin.
         .executableTarget(
             name: "MyLocalTool",
             dependencies: [
-                "MySourceGenPlugin",
+                "MySourceGenBuildToolPlugin",
+            ]
+        ),
+        // A local tool that uses a prebuild plugin.
+        .executableTarget(
+            name: "MyOtherLocalTool",
+            dependencies: [
+                "MySourceGenPrebuildPlugin",
             ]
         ),
-        // The target that implements the plugin and generates commands to invoke MySourceGenTool.
+        // The plugin that generates build tool commands to invoke MySourceGenBuildTool.
         .plugin(
-            name: "MySourceGenPlugin",
+            name: "MySourceGenBuildToolPlugin",
             capability: .buildTool(),
             dependencies: [
-                "MySourceGenTool"
+                "MySourceGenBuildTool",
             ]
         ),
+        // The plugin that generates prebuild commands (currently to invoke a system tool).
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            capability: .prebuild()
+        ),
         // The command line tool that generates source files.
         .executableTarget(
-            name: "MySourceGenTool",
+            name: "MySourceGenBuildTool",
             dependencies: [
-                "MySourceGenToolLib",
+                "MySourceGenBuildToolLib",
             ]
         ),
-        // A library used by MySourceGenTool (not the client).
+        // A library used by MySourceGenBuildTool (not the client).
         .target(
-            name: "MySourceGenToolLib"
+            name: "MySourceGenBuildToolLib"
         ),
         // A runtime library that the client needs to link against.
         .target(
@@ -49,10 +67,11 @@ let package = Package(
         .testTarget(
             name: "MySourceGenPluginTests",
             dependencies: [
-                "MySourceGenRuntimeLib"
+                "MySourceGenRuntimeLib",
             ],
             plugins: [
-                "MySourceGenPlugin"
+                "MySourceGenBuildToolPlugin",
+                "MySourceGenPrebuildPlugin",
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/Foo.dat

@@ -1 +1 @@
-Hello Extension Target!
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/main.swift

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/bar.dat

@@ -0,0 +1 @@
+I am Bar!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/baz.dat

@@ -0,0 +1 @@
+I am Baz!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/main.swift

@@ -0,0 +1,2 @@
+// print("Generated string Bar: '\(bar)'")
+// print("Generated string Baz: '\(baz)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildTool/main.swift

@@ -0,0 +1,18 @@
+import Foundation
+import MySourceGenBuildToolLib
+
+// Sample source generator tool that emits a Swift variable declaration of a string containing the hex representation of the contents of a file as a quoted string.  The variable name is the base name of the input file.  The input file is the first argument and the output file is the second.
+if ProcessInfo.processInfo.arguments.count != 3 {
+    print("usage: MySourceGenBuildTool <input> <output>")
+    exit(1)
+}
+let inputFile = ProcessInfo.processInfo.arguments[1]
+let outputFile = ProcessInfo.processInfo.arguments[2]
+
+let variableName = URL(fileURLWithPath: inputFile).deletingPathExtension().lastPathComponent
+
+let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+let outputString = "public var \(variableName) = \(dataAsHex.quotedForSourceCode)\n"
+let outputData = outputString.data(using: .utf8)
+FileManager.default.createFile(atPath: outputFile, contents: outputData)

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenToolLib/library.swift renamed to Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildToolLib/library.swift

@@ -1,4 +1,6 @@
 import PackagePlugin
+ 
+print("Hello from the Build Tool Plugin!")
 
 for inputPath in targetBuildContext.otherFiles {
     guard inputPath.suffix == ".dat" else { continue }
@@ -8,7 +10,7 @@ for inputPath in targetBuildContext.otherFiles {
         displayName:
             "Generating \(outputName) from \(inputPath.filename)",
         executable:
-            try targetBuildContext.lookupTool(named: "MySourceGenTool"),
+            try targetBuildContext.lookupTool(named: "MySourceGenBuildTool"),
         arguments: [
             "\(inputPath)",
             "\(outputPath)"

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPlugin/plugin.swift renamed to Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildToolPlugin/plugin.swift

@@ -0,0 +1,20 @@
+import PackagePlugin
+ 
+print("Hello from the Prebuild Plugin!")
+
+let outputPaths: [Path] = targetBuildContext.otherFiles.filter{ $0.suffix == ".dat" }.map { path in
+    targetBuildContext.outputDir.appending(path.basename + ".swift")
+}
+
+if !outputPaths.isEmpty {
+    commandConstructor.createCommand(
+        displayName:
+            "Running prebuild command for target \(targetBuildContext.targetName)",
+        executable:
+            Path("/usr/bin/touch"),
+        arguments: 
+            outputPaths.map{ $0.string },
+        derivedSourcePaths:
+            outputPaths
+    )
+}

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPrebuildPlugin/plugin.swift

@@ -163,14 +163,53 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         }
     }
 
+    /// Run any prebuild commands associated with the given list of prebuild results, and return the paths of any created files or directories.
+    private func runPrebuildCommands(for results: [ExtensionEvaluationResult]) throws -> [AbsolutePath] {
+        // Run through all the commands from all the extension usages in the target.
+        var outputPaths: [AbsolutePath] = []
+        for command in results.reduce([], { $0 + $1.commands }) {
+            if case .prebuildCommand(let displayName, let executable, let arguments, _, _, let derivedSourceDirPaths) = command {
+                // Run the prebuild command.
+                // TODO: We need to also respect any environment and working directory configurations.
+                let execPath = AbsolutePath(executable, relativeTo: buildParameters.buildPath)
+                
+                // FIXME: Is this the right way to emit progress here?
+                stdoutStream.write(displayName)
+                
+                // Run the command as a subprocess.
+                // TODO: This should move somewhere else, and we should also be able to run independent prebuild commands concurrently.
+                let command = [execPath.pathString] + arguments
+                let result = try Process.popen(arguments: command)
+                let output = try (result.utf8Output() + result.utf8stderrOutput()).spm_chuzzle() ?? ""
+                if result.exitStatus != .terminated(code: 0) {
+                    // TODO: Make this a proper error.
+                    throw StringError("failed: \(command)\n\n\(output)")
+                }
+                
+                // Keep track of the output paths.
+                outputPaths.append(contentsOf: derivedSourceDirPaths)
+            }
+        }
+        return outputPaths
+    }
     /// Create the build plan and return the build description.
     private func plan() throws -> BuildDescription {
         let graph = try getPackageGraph()
         let pluginInvocationResults = try getPluginInvocationResults(for: graph)
+
+        // Run any prebuild commands.  We get here in the case where there was no build description or it needs to change.
+        var prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]] = [:]
+        for target in graph.reachableTargets {
+            if let results = pluginInvocationResults[target] {
+                prebuildOutputPaths[target] = try runPrebuildCommands(for: results)
+            }
+        }
+
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
             pluginInvocationResults: pluginInvocationResults,
+            prebuildOutputPaths: prebuildOutputPaths,
             diagnostics: diagnostics
         )
         self.buildPlan = plan

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenTool/main.swift

@@ -544,11 +544,15 @@ public final class SwiftTargetBuildDescription {
     /// The results of applying any plugins to this target.
     public let pluginInvocationResults: [PluginInvocationResult]
 
+    /// The output paths of any prebuild commands that were run for this target.
+    public let prebuildOutputPaths: [AbsolutePath]
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         buildParameters: BuildParameters,
         pluginInvocationResults: [PluginInvocationResult] = [],
+        prebuildOutputPaths: [AbsolutePath] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
         fs: FileSystem = localFileSystem
@@ -563,7 +567,14 @@ public final class SwiftTargetBuildDescription {
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
         self.pluginInvocationResults = pluginInvocationResults
+        self.prebuildOutputPaths = prebuildOutputPaths
 
+        // Add any derived source paths from prebuild commands.
+        for absPath in self.prebuildOutputPaths {
+            let relPath = absPath.relative(to: self.derivedSources.root)
+            self.derivedSources.relativePaths.append(relPath)
+        }
+        
         // Add any derived source paths declared by build-tool plugins that were applied to this target.  We do
         // this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
         // where sources are processed, e.g. when determining names of object files, etc.
@@ -1278,6 +1289,9 @@ public class BuildPlan {
     /// The results of invoking any plugins used by targets in this build.
     public let pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]]
 
+    /// The output paths of any prebuild commands run for targets in this build.
+    public let prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]]
+
     /// The filesystem to operate on.
     let fileSystem: FileSystem
 
@@ -1358,12 +1372,14 @@ public class BuildPlan {
         buildParameters: BuildParameters,
         graph: PackageGraph,
         pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]] = [:],
+        prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
     ) throws {
         self.buildParameters = buildParameters
         self.graph = graph
         self.pluginInvocationResults = pluginInvocationResults
+        self.prebuildOutputPaths = prebuildOutputPaths
         self.diagnostics = diagnostics
         self.fileSystem = fileSystem
 
@@ -1389,6 +1405,7 @@ public class BuildPlan {
                     target: target,
                     buildParameters: buildParameters,
                     pluginInvocationResults: pluginInvocationResults[target] ?? [],
+                    prebuildOutputPaths: prebuildOutputPaths[target] ?? [],
                     fs: fileSystem))
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(

Sources/Build/BuildOperation.swift

@@ -108,6 +108,10 @@ extension LLBuildManifestBuilder {
                 .sorted()
                 .map { Node.directoryStructure($0) }
 
+            // Add the output paths of any prebuilds that were run, so that we redo the plan if they change.
+            let nodes = plan.prebuildOutputPaths.values.flatMap{ $0 }.sorted().map { Node.file($0) }
+            inputs.append(contentsOf: nodes)
+            
             // FIXME: Need to handle version-specific manifests.
             inputs.append(file: package.manifest.path)
 

Sources/Build/BuildPlan.swift

@@ -302,9 +302,12 @@ final class PackageToolTests: XCTestCase {
             // Check the contents of the JSON.
             XCTAssertEqual(try XCTUnwrap(json["name"]).string, "MySourceGenPlugin")
             let targetsArray = try XCTUnwrap(json["targets"]?.array)
-            let extensionTarget = try XCTUnwrap(targetsArray.first{ $0["name"]?.string == "MySourceGenPlugin" }?.dictionary)
-            XCTAssertEqual(extensionTarget["module_type"]?.string, "PluginTarget")
-            XCTAssertEqual(extensionTarget["plugin_capability"]?.dictionary?["type"]?.string, "buildTool")
+            let buildToolExtTarget = try XCTUnwrap(targetsArray.first{ $0["name"]?.string == "MySourceGenBuildToolPlugin" }?.dictionary)
+            XCTAssertEqual(buildToolExtTarget["module_type"]?.string, "PluginTarget")
+            XCTAssertEqual(buildToolExtTarget["plugin_capability"]?.dictionary?["type"]?.string, "buildTool")
+            let prebuildExtTarget = try XCTUnwrap(targetsArray.first{ $0["name"]?.string == "MySourceGenPrebuildPlugin" }?.dictionary)
+            XCTAssertEqual(prebuildExtTarget["module_type"]?.string, "PluginTarget")
+            XCTAssertEqual(prebuildExtTarget["plugin_capability"]?.dictionary?["type"]?.string, "prebuild")
         }
     }