Run prebuild commands provided by plugins before the build starts

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Package.swift

@@ -7,18 +7,18 @@ let package = Package(
         .package(path: "../MySourceGenPlugin")
     ],
     targets: [
-        // A tool that uses an plugin.
+        // A tool that uses a plugin.
         .executableTarget(
             name: "MyTool",
             dependencies: [
-                .product(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .product(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         ),
-        // A unit that uses the plugin.
+        // A unit test that uses the plugin.
         .testTarget(
             name: "MyTests",
             plugins: [
-                .plugin(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .plugin(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/foo.dat

@@ -0,0 +1 @@
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/main.swift

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Package.swift

@@ -4,42 +4,60 @@ import PackageDescription
 let package = Package(
     name: "MySourceGenPlugin",
     products: [
-        // The product that vends MySourceGenPlugin to client packages.
+        // The product that vends MySourceGenBuildToolPlugin to client packages.
         .plugin(
-            name: "MySourceGenPlugin",
-            targets: ["MySourceGenPlugin"]
+            name: "MySourceGenBuildToolPlugin",
+            targets: ["MySourceGenBuildToolPlugin"]
         ),
+        // The product that vends the MySourceGenBuildTool executable to client packages.
         .executable(
-            name: "MySourceGenTool",
-            targets: ["MySourceGenTool"]
-        )
+            name: "MySourceGenBuildTool",
+            targets: ["MySourceGenBuildTool"]
+        ),
+        // The product that vends MySourceGenPrebuildPlugin to client packages.
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            targets: ["MySourceGenPrebuildPlugin"]
+        ),
     ],
     targets: [
-        // A local tool that uses a plugin.
+        // A local tool that uses a build tool plugin.
         .executableTarget(
             name: "MyLocalTool",
             dependencies: [
-                "MySourceGenPlugin",
+                "MySourceGenBuildToolPlugin",
+            ]
+        ),
+        // A local tool that uses a prebuild plugin.
+        .executableTarget(
+            name: "MyOtherLocalTool",
+            dependencies: [
+                "MySourceGenPrebuildPlugin",
             ]
         ),
-        // The target that implements the plugin and generates commands to invoke MySourceGenTool.
+        // The plugin that generates build tool commands to invoke MySourceGenBuildTool.
         .plugin(
-            name: "MySourceGenPlugin",
+            name: "MySourceGenBuildToolPlugin",
             capability: .buildTool(),
             dependencies: [
-                "MySourceGenTool"
+                "MySourceGenBuildTool",
             ]
         ),
+        // The plugin that generates prebuild commands (currently to invoke a system tool).
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            capability: .prebuild()
+        ),
         // The command line tool that generates source files.
         .executableTarget(
-            name: "MySourceGenTool",
+            name: "MySourceGenBuildTool",
             dependencies: [
-                "MySourceGenToolLib",
+                "MySourceGenBuildToolLib",
             ]
         ),
-        // A library used by MySourceGenTool (not the client).
+        // A library used by MySourceGenBuildTool (not the client).
         .target(
-            name: "MySourceGenToolLib"
+            name: "MySourceGenBuildToolLib"
         ),
         // A runtime library that the client needs to link against.
         .target(
@@ -49,10 +67,11 @@ let package = Package(
         .testTarget(
             name: "MySourceGenPluginTests",
             dependencies: [
-                "MySourceGenRuntimeLib"
+                "MySourceGenRuntimeLib",
             ],
             plugins: [
-                "MySourceGenPlugin"
+                "MySourceGenBuildToolPlugin",
+                "MySourceGenPrebuildPlugin",
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/foo.dat

@@ -0,0 +1 @@
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/main.swift

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/bar.dat

@@ -0,0 +1 @@
+I am Bar!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/baz.dat

@@ -0,0 +1 @@
+I am Baz!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/main.swift

@@ -0,0 +1,2 @@
+// print("Generated string Bar: '\(bar)'")
+// print("Generated string Baz: '\(baz)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildTool/main.swift

@@ -0,0 +1,18 @@
+import Foundation
+import MySourceGenBuildToolLib
+
+// Sample source generator tool that emits a Swift variable declaration of a string containing the hex representation of the contents of a file as a quoted string.  The variable name is the base name of the input file.  The input file is the first argument and the output file is the second.
+if ProcessInfo.processInfo.arguments.count != 3 {
+    print("usage: MySourceGenBuildTool <input> <output>")
+    exit(1)
+}
+let inputFile = ProcessInfo.processInfo.arguments[1]
+let outputFile = ProcessInfo.processInfo.arguments[2]
+
+let variableName = URL(fileURLWithPath: inputFile).deletingPathExtension().lastPathComponent
+
+let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+let outputString = "public var \(variableName) = \(dataAsHex.quotedForSourceCode)\n"
+let outputData = outputString.data(using: .utf8)
+FileManager.default.createFile(atPath: outputFile, contents: outputData)

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPlugin/plugin.swift → Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildToolPlugin/plugin.swift

@@ -1,4 +1,6 @@
 import PackagePlugin
+
+print("Hello from the Build Tool Plugin!")
 
 for inputPath in targetBuildContext.otherFiles {
     guard inputPath.suffix == ".dat" else { continue }
@@ -8,7 +10,7 @@ for inputPath in targetBuildContext.otherFiles {
         displayName:
             "Generating \(outputName) from \(inputPath.filename)",
         executable:
-            try targetBuildContext.lookupTool(named: "MySourceGenTool"),
+            try targetBuildContext.lookupTool(named: "MySourceGenBuildTool"),
         arguments: [
             "\(inputPath)",
             "\(outputPath)"
@@ -18,9 +20,7 @@ for inputPath in targetBuildContext.otherFiles {
         ],
         outputPaths: [
             outputPath
-        ],
-        derivedSourcePaths: [
-            outputPath
         ]
     )
+    commandConstructor.addGeneratedOutputFile(path: outputPath)
 }

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPrebuildPlugin/plugin.swift

@@ -0,0 +1,20 @@
+import PackagePlugin
+
+print("Hello from the Prebuild Plugin!")
+
+let outputPaths: [Path] = targetBuildContext.otherFiles.filter{ $0.suffix == ".dat" }.map { path in
+    targetBuildContext.outputDir.appending(path.basename + ".swift")
+}
+
+if !outputPaths.isEmpty {
+    commandConstructor.createCommand(
+        displayName:
+            "Running prebuild command for target \(targetBuildContext.targetName)",
+        executable:
+            Path("/usr/bin/touch"),
+        arguments: 
+            outputPaths.map{ $0.string }
+    )
+}
+
+commandConstructor.addPrebuildOutputDirectory(path: targetBuildContext.outputDir)

Sources/Build/BuildOperation.swift

@@ -165,16 +165,28 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// Create the build plan and return the build description.
     private func plan() throws -> BuildDescription {
+        // Load the package graph.
         let graph = try getPackageGraph()
+
+        // Invoke any plugins in the graph, and get the results.
         let pluginInvocationResults = try getPluginInvocationResults(for: graph)
+
+        // Run any prebuild commands provided by plugins. Any failure stops the build.
+        let prebuildCommandResults = try graph.reachableTargets.reduce(into: [:], { partial, target in
+            partial[target] = try pluginInvocationResults[target].map { try runPrebuildCommands(for: $0) }
+        })
+
+        // Create the build plan based, on the graph and any information from plugins.
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
             pluginInvocationResults: pluginInvocationResults,
+            prebuildCommandResults: prebuildCommandResults,
             diagnostics: diagnostics
         )
         self.buildPlan = plan
-
+
+        // Finally create the llbuild manifest from the plan.
         return try BuildDescription.create(with: plan)
     }
 

Sources/Build/BuildPlan.swift

@@ -482,18 +482,23 @@ public final class SwiftTargetBuildDescription {
     ///
     /// These are the source files generated during the build.
     private var derivedSources: Sources
+
+    /// These are the source files derived from plugins.
+    private var pluginDerivedSources: Sources
 
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
         buildParameters.bundlePath(for: target)
     }
 
     /// The list of all source files in the target, including the derived ones.
-    public var sources: [AbsolutePath] { target.sources.paths + derivedSources.paths }
+    public var sources: [AbsolutePath] {
+        target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
+    }
 
     /// The objects in this target.
     public var objects: [AbsolutePath] {
-        let relativePaths = target.sources.relativePaths + derivedSources.relativePaths
+        let relativePaths = target.sources.relativePaths + derivedSources.relativePaths + pluginDerivedSources.relativePaths
         return relativePaths.map{ tempsPath.appending(RelativePath("\($0.pathString).o")) }
     }
 
@@ -544,11 +549,15 @@ public final class SwiftTargetBuildDescription {
     /// The results of applying any plugins to this target.
     public let pluginInvocationResults: [PluginInvocationResult]
 
+    /// The results of running any prebuild commands for this target.
+    public let prebuildCommandResults: [PrebuildCommandResult]
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         buildParameters: BuildParameters,
         pluginInvocationResults: [PluginInvocationResult] = [],
+        prebuildCommandResults: [PrebuildCommandResult] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
         fs: FileSystem = localFileSystem
@@ -562,22 +571,28 @@ public final class SwiftTargetBuildDescription {
         self.fs = fs
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
+        self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
         self.pluginInvocationResults = pluginInvocationResults
-
-        // Add any derived source paths declared by build-tool plugins that were applied to this target.  We do
-        // this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
-        // where sources are processed, e.g. when determining names of object files, etc.
-        for command in pluginInvocationResults.reduce([], { $0 + $1.commands }) {
-            // Prebuild and postbuild commands are handled outside the build system.
-            if case .buildToolCommand(_, _, _, _, _, _, _, let derivedSourcePaths) = command {
-                // TODO: What should we do if we find non-Swift sources here?
-                for absPath in derivedSourcePaths {
-                    let relPath = absPath.relative(to: self.derivedSources.root)
-                    self.derivedSources.relativePaths.append(relPath)
-                }
+        self.prebuildCommandResults = prebuildCommandResults
+
+        // Add any derived source files that were declared in any plugin invocations.
+        for pluginResult in pluginInvocationResults {
+            // TODO: What should we do if we find non-Swift sources here?
+            for absPath in pluginResult.derivedSourceFiles {
+                let relPath = absPath.relative(to: self.pluginDerivedSources.root)
+                self.pluginDerivedSources.relativePaths.append(relPath)
             }
         }
 
+        // Add any derived source files that were discovered from output directories of prebuild commands.
+        for result in self.prebuildCommandResults {
+            // TODO: What should we do if we find non-Swift sources here?
+            for path in result.derivedSourceFiles {
+                let relPath = path.relative(to: self.pluginDerivedSources.root)
+                self.pluginDerivedSources.relativePaths.append(relPath)
+            }
+        }
+
         if shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
         }
@@ -861,7 +876,7 @@ public final class SwiftTargetBuildDescription {
         stream <<< "  },\n"
 
         // Write out the entries for each source file.
-        let sources = target.sources.paths + derivedSources.paths
+        let sources = target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
         for (idx, source) in sources.enumerated() {
             let object = objects[idx]
             let objectDir = object.parentDirectory
@@ -1278,6 +1293,10 @@ public class BuildPlan {
     /// The results of invoking any plugins used by targets in this build.
     public let pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]]
 
+    /// The results of running any prebuild commands for the targets in this build.  This includes any derived
+    /// source files as well as directories to which any changes should cause us to reevaluate the build plan.
+    public let prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]]
+
     /// The filesystem to operate on.
     let fileSystem: FileSystem
 
@@ -1358,12 +1377,14 @@ public class BuildPlan {
         buildParameters: BuildParameters,
         graph: PackageGraph,
         pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]] = [:],
+        prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
     ) throws {
         self.buildParameters = buildParameters
         self.graph = graph
         self.pluginInvocationResults = pluginInvocationResults
+        self.prebuildCommandResults = prebuildCommandResults
         self.diagnostics = diagnostics
         self.fileSystem = fileSystem
 
@@ -1389,6 +1410,7 @@ public class BuildPlan {
                     target: target,
                     buildParameters: buildParameters,
                     pluginInvocationResults: pluginInvocationResults[target] ?? [],
+                    prebuildCommandResults: prebuildCommandResults[target] ?? [],
                     fs: fileSystem))
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(

Sources/Build/ManifestBuilder.swift

@@ -108,6 +108,13 @@ extension LLBuildManifestBuilder {
                 .sorted()
                 .map { Node.directoryStructure($0) }
 
+            // Add the output paths of any prebuilds that were run, so that we redo the plan if they change.
+            var derivedSourceDirPaths: [AbsolutePath] = []
+            for result in plan.prebuildCommandResults.values.flatMap({ $0 }) {
+                derivedSourceDirPaths.append(contentsOf: result.outputDirectories)
+            }
+            inputs.append(contentsOf: derivedSourceDirPaths.sorted().map{ Node.directoryStructure($0) })
+
             // FIXME: Need to handle version-specific manifests.
             inputs.append(file: package.manifest.path)
 
@@ -577,7 +584,7 @@ extension LLBuildManifestBuilder {
 
         // Add any build tool commands created by plugins for the target (prebuild and postbuild commands are handled outside the build).
         for command in target.pluginInvocationResults.reduce([], { $0 + $1.commands }) {
-            if case .buildToolCommand(let displayName, let executable, let arguments, _, _, let inputPaths, let outputPaths, _) = command {
+            if case .buildToolCommand(let displayName, let executable, let arguments, _, _, let inputPaths, let outputPaths) = command {
                 // Create a shell command to invoke the executable.  We include the path of the executable as a dependency.
                 // FIXME: We will need to extend the addShellCmd() function to also take working directory and environment.
                 let execPath = AbsolutePath(executable, relativeTo: buildParameters.buildPath)