Run prebuild commands from plugins before the build starts (#3307)

These commands run before the build starts, and can specify directories whose contents should be considered as inputs to the build planning.  The contents of the directories are added as derived sources to the build, and are expected to be compatible with the kind of target for which they are generating sources (e.g. .swift files for Swift targets).  The diagnostics for this are currently missing, resulting in a compilation failure rather than a better diagnostics, but will be added.  Also, resource files should be treated as resources, etc — this tool will be added.

The directories into which outputs are written are added to the set of directories whose contents should affect the build plan, so that changes to the set of generated source files are taken into account when constructing the build plan.

Author: abertelrud

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Package.swift

@@ -7,18 +7,18 @@ let package = Package(
         .package(path: "../MySourceGenPlugin")
     ],
     targets: [
-        // A tool that uses an plugin.
+        // A tool that uses a plugin.
         .executableTarget(
             name: "MyTool",
             dependencies: [
-                .product(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .product(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         ),
-        // A unit that uses the plugin.
+        // A unit test that uses the plugin.
         .testTarget(
             name: "MyTests",
             plugins: [
-                .plugin(name: "MySourceGenPlugin", package: "MySourceGenPlugin")
+                .plugin(name: "MySourceGenBuildToolPlugin", package: "MySourceGenPlugin")
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/Foo.dat

@@ -0,0 +1 @@
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/foo.dat

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenClient/Sources/MyTool/main.swift

@@ -4,42 +4,60 @@ import PackageDescription
 let package = Package(
     name: "MySourceGenPlugin",
     products: [
-        // The product that vends MySourceGenPlugin to client packages.
+        // The product that vends MySourceGenBuildToolPlugin to client packages.
         .plugin(
-            name: "MySourceGenPlugin",
-            targets: ["MySourceGenPlugin"]
+            name: "MySourceGenBuildToolPlugin",
+            targets: ["MySourceGenBuildToolPlugin"]
         ),
+        // The product that vends the MySourceGenBuildTool executable to client packages.
         .executable(
-            name: "MySourceGenTool",
-            targets: ["MySourceGenTool"]
-        )
+            name: "MySourceGenBuildTool",
+            targets: ["MySourceGenBuildTool"]
+        ),
+        // The product that vends MySourceGenPrebuildPlugin to client packages.
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            targets: ["MySourceGenPrebuildPlugin"]
+        ),
     ],
     targets: [
-        // A local tool that uses a plugin.
+        // A local tool that uses a build tool plugin.
         .executableTarget(
             name: "MyLocalTool",
             dependencies: [
-                "MySourceGenPlugin",
+                "MySourceGenBuildToolPlugin",
+            ]
+        ),
+        // A local tool that uses a prebuild plugin.
+        .executableTarget(
+            name: "MyOtherLocalTool",
+            dependencies: [
+                "MySourceGenPrebuildPlugin",
             ]
         ),
-        // The target that implements the plugin and generates commands to invoke MySourceGenTool.
+        // The plugin that generates build tool commands to invoke MySourceGenBuildTool.
         .plugin(
-            name: "MySourceGenPlugin",
+            name: "MySourceGenBuildToolPlugin",
             capability: .buildTool(),
             dependencies: [
-                "MySourceGenTool"
+                "MySourceGenBuildTool",
             ]
         ),
+        // The plugin that generates prebuild commands (currently to invoke a system tool).
+        .plugin(
+            name: "MySourceGenPrebuildPlugin",
+            capability: .prebuild()
+        ),
         // The command line tool that generates source files.
         .executableTarget(
-            name: "MySourceGenTool",
+            name: "MySourceGenBuildTool",
             dependencies: [
-                "MySourceGenToolLib",
+                "MySourceGenBuildToolLib",
             ]
         ),
-        // A library used by MySourceGenTool (not the client).
+        // A library used by MySourceGenBuildTool (not the client).
         .target(
-            name: "MySourceGenToolLib"
+            name: "MySourceGenBuildToolLib"
         ),
         // A runtime library that the client needs to link against.
         .target(
@@ -49,10 +67,11 @@ let package = Package(
         .testTarget(
             name: "MySourceGenPluginTests",
             dependencies: [
-                "MySourceGenRuntimeLib"
+                "MySourceGenRuntimeLib",
             ],
             plugins: [
-                "MySourceGenPlugin"
+                "MySourceGenBuildToolPlugin",
+                "MySourceGenPrebuildPlugin",
             ]
         )
     ]

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Package.swift

@@ -0,0 +1 @@
+I am Foo!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/Foo.dat

@@ -1 +1 @@
-print("Generated string: '\(generatedString)'")
+print("Generated string Foo: '\(foo)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/foo.dat

@@ -0,0 +1 @@
+I am Bar!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyLocalTool/main.swift

@@ -0,0 +1 @@
+I am Baz!

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/bar.dat

@@ -0,0 +1,2 @@
+// print("Generated string Bar: '\(bar)'")
+// print("Generated string Baz: '\(baz)'")

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/baz.dat

@@ -0,0 +1,18 @@
+import Foundation
+import MySourceGenBuildToolLib
+
+// Sample source generator tool that emits a Swift variable declaration of a string containing the hex representation of the contents of a file as a quoted string.  The variable name is the base name of the input file.  The input file is the first argument and the output file is the second.
+if ProcessInfo.processInfo.arguments.count != 3 {
+    print("usage: MySourceGenBuildTool <input> <output>")
+    exit(1)
+}
+let inputFile = ProcessInfo.processInfo.arguments[1]
+let outputFile = ProcessInfo.processInfo.arguments[2]
+
+let variableName = URL(fileURLWithPath: inputFile).deletingPathExtension().lastPathComponent
+
+let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+let outputString = "public var \(variableName) = \(dataAsHex.quotedForSourceCode)\n"
+let outputData = outputString.data(using: .utf8)
+FileManager.default.createFile(atPath: outputFile, contents: outputData)

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MyOtherLocalTool/main.swift

@@ -1,4 +1,6 @@
 import PackagePlugin
+ 
+print("Hello from the Build Tool Plugin!")
 
 for inputPath in targetBuildContext.otherFiles {
     guard inputPath.suffix == ".dat" else { continue }
@@ -8,7 +10,7 @@ for inputPath in targetBuildContext.otherFiles {
         displayName:
             "Generating \(outputName) from \(inputPath.filename)",
         executable:
-            try targetBuildContext.lookupTool(named: "MySourceGenTool"),
+            try targetBuildContext.lookupTool(named: "MySourceGenBuildTool"),
         arguments: [
             "\(inputPath)",
             "\(outputPath)"
@@ -18,9 +20,7 @@ for inputPath in targetBuildContext.otherFiles {
         ],
         outputPaths: [
             outputPath
-        ],
-        derivedSourcePaths: [
-            outputPath
         ]
     )
+    commandConstructor.addGeneratedOutputFile(path: outputPath)
 }

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildTool/main.swift

@@ -0,0 +1,20 @@
+import PackagePlugin
+ 
+print("Hello from the Prebuild Plugin!")
+
+let outputPaths: [Path] = targetBuildContext.otherFiles.filter{ $0.suffix == ".dat" }.map { path in
+    targetBuildContext.outputDir.appending(path.basename + ".swift")
+}
+
+if !outputPaths.isEmpty {
+    commandConstructor.createCommand(
+        displayName:
+            "Running prebuild command for target \(targetBuildContext.targetName)",
+        executable:
+            Path("/usr/bin/touch"),
+        arguments: 
+            outputPaths.map{ $0.string }
+    )
+}
+
+commandConstructor.addPrebuildOutputDirectory(path: targetBuildContext.outputDir)

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenToolLib/library.swift renamed to Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildToolLib/library.swift

@@ -165,16 +165,28 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// Create the build plan and return the build description.
     private func plan() throws -> BuildDescription {
+        // Load the package graph.
         let graph = try getPackageGraph()
+        
+        // Invoke any plugins in the graph, and get the results.
         let pluginInvocationResults = try getPluginInvocationResults(for: graph)
+
+        // Run any prebuild commands provided by plugins. Any failure stops the build.
+        let prebuildCommandResults = try graph.reachableTargets.reduce(into: [:], { partial, target in
+            partial[target] = try pluginInvocationResults[target].map { try runPrebuildCommands(for: $0) }
+        })
+        
+        // Create the build plan based, on the graph and any information from plugins.
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
             pluginInvocationResults: pluginInvocationResults,
+            prebuildCommandResults: prebuildCommandResults,
             diagnostics: diagnostics
         )
         self.buildPlan = plan
-
+        
+        // Finally create the llbuild manifest from the plan.
         return try BuildDescription.create(with: plan)
     }
 

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPlugin/plugin.swift renamed to Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenBuildToolPlugin/plugin.swift

@@ -482,18 +482,23 @@ public final class SwiftTargetBuildDescription {
     ///
     /// These are the source files generated during the build.
     private var derivedSources: Sources
+    
+    /// These are the source files derived from plugins.
+    private var pluginDerivedSources: Sources
 
     /// Path to the bundle generated for this module (if any).
     var bundlePath: AbsolutePath? {
         buildParameters.bundlePath(for: target)
     }
 
     /// The list of all source files in the target, including the derived ones.
-    public var sources: [AbsolutePath] { target.sources.paths + derivedSources.paths }
+    public var sources: [AbsolutePath] {
+        target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
+    }
 
     /// The objects in this target.
     public var objects: [AbsolutePath] {
-        let relativePaths = target.sources.relativePaths + derivedSources.relativePaths
+        let relativePaths = target.sources.relativePaths + derivedSources.relativePaths + pluginDerivedSources.relativePaths
         return relativePaths.map{ tempsPath.appending(RelativePath("\($0.pathString).o")) }
     }
 
@@ -544,11 +549,15 @@ public final class SwiftTargetBuildDescription {
     /// The results of applying any plugins to this target.
     public let pluginInvocationResults: [PluginInvocationResult]
 
+    /// The results of running any prebuild commands for this target.
+    public let prebuildCommandResults: [PrebuildCommandResult]
+
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         buildParameters: BuildParameters,
         pluginInvocationResults: [PluginInvocationResult] = [],
+        prebuildCommandResults: [PrebuildCommandResult] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
         fs: FileSystem = localFileSystem
@@ -562,22 +571,28 @@ public final class SwiftTargetBuildDescription {
         self.fs = fs
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
+        self.pluginDerivedSources = Sources(paths: [], root: buildParameters.dataPath)
         self.pluginInvocationResults = pluginInvocationResults
-
-        // Add any derived source paths declared by build-tool plugins that were applied to this target.  We do
-        // this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
-        // where sources are processed, e.g. when determining names of object files, etc.
-        for command in pluginInvocationResults.reduce([], { $0 + $1.commands }) {
-            // Prebuild and postbuild commands are handled outside the build system.
-            if case .buildToolCommand(_, _, _, _, _, _, _, let derivedSourcePaths) = command {
-                // TODO: What should we do if we find non-Swift sources here?
-                for absPath in derivedSourcePaths {
-                    let relPath = absPath.relative(to: self.derivedSources.root)
-                    self.derivedSources.relativePaths.append(relPath)
-                }
+        self.prebuildCommandResults = prebuildCommandResults
+
+        // Add any derived source files that were declared in any plugin invocations.
+        for pluginResult in pluginInvocationResults {
+            // TODO: What should we do if we find non-Swift sources here?
+            for absPath in pluginResult.derivedSourceFiles {
+                let relPath = absPath.relative(to: self.pluginDerivedSources.root)
+                self.pluginDerivedSources.relativePaths.append(relPath)
             }
         }
 
+        // Add any derived source files that were discovered from output directories of prebuild commands.
+        for result in self.prebuildCommandResults {
+            // TODO: What should we do if we find non-Swift sources here?
+            for path in result.derivedSourceFiles {
+                let relPath = path.relative(to: self.pluginDerivedSources.root)
+                self.pluginDerivedSources.relativePaths.append(relPath)
+            }
+        }
+        
         if shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
         }
@@ -861,7 +876,7 @@ public final class SwiftTargetBuildDescription {
         stream <<< "  },\n"
 
         // Write out the entries for each source file.
-        let sources = target.sources.paths + derivedSources.paths
+        let sources = target.sources.paths + derivedSources.paths + pluginDerivedSources.paths
         for (idx, source) in sources.enumerated() {
             let object = objects[idx]
             let objectDir = object.parentDirectory
@@ -1278,6 +1293,10 @@ public class BuildPlan {
     /// The results of invoking any plugins used by targets in this build.
     public let pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]]
 
+    /// The results of running any prebuild commands for the targets in this build.  This includes any derived
+    /// source files as well as directories to which any changes should cause us to reevaluate the build plan.
+    public let prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]]
+
     /// The filesystem to operate on.
     let fileSystem: FileSystem
 
@@ -1358,12 +1377,14 @@ public class BuildPlan {
         buildParameters: BuildParameters,
         graph: PackageGraph,
         pluginInvocationResults: [ResolvedTarget: [PluginInvocationResult]] = [:],
+        prebuildCommandResults: [ResolvedTarget: [PrebuildCommandResult]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
     ) throws {
         self.buildParameters = buildParameters
         self.graph = graph
         self.pluginInvocationResults = pluginInvocationResults
+        self.prebuildCommandResults = prebuildCommandResults
         self.diagnostics = diagnostics
         self.fileSystem = fileSystem
 
@@ -1389,6 +1410,7 @@ public class BuildPlan {
                     target: target,
                     buildParameters: buildParameters,
                     pluginInvocationResults: pluginInvocationResults[target] ?? [],
+                    prebuildCommandResults: prebuildCommandResults[target] ?? [],
                     fs: fileSystem))
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(

Fixtures/Miscellaneous/Plugins/MySourceGenPlugin/Sources/MySourceGenPrebuildPlugin/plugin.swift

@@ -108,6 +108,13 @@ extension LLBuildManifestBuilder {
                 .sorted()
                 .map { Node.directoryStructure($0) }
 
+            // Add the output paths of any prebuilds that were run, so that we redo the plan if they change.
+            var derivedSourceDirPaths: [AbsolutePath] = []
+            for result in plan.prebuildCommandResults.values.flatMap({ $0 }) {
+                derivedSourceDirPaths.append(contentsOf: result.outputDirectories)
+            }
+            inputs.append(contentsOf: derivedSourceDirPaths.sorted().map{ Node.directoryStructure($0) })
+            
             // FIXME: Need to handle version-specific manifests.
             inputs.append(file: package.manifest.path)
 
@@ -577,7 +584,7 @@ extension LLBuildManifestBuilder {
 
         // Add any build tool commands created by plugins for the target (prebuild and postbuild commands are handled outside the build).
         for command in target.pluginInvocationResults.reduce([], { $0 + $1.commands }) {
-            if case .buildToolCommand(let displayName, let executable, let arguments, _, _, let inputPaths, let outputPaths, _) = command {
+            if case .buildToolCommand(let displayName, let executable, let arguments, _, _, let inputPaths, let outputPaths) = command {
                 // Create a shell command to invoke the executable.  We include the path of the executable as a dependency.
                 // FIXME: We will need to extend the addShellCmd() function to also take working directory and environment.
                 let execPath = AbsolutePath(executable, relativeTo: buildParameters.buildPath)