Add PackageExtension library and incorporate build tool commands into native build system

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Package.swift

@@ -11,10 +11,18 @@ let package = Package(
         // )
     ],
     targets: [
+        // A local tool that uses an extension.
+        .executableTarget(
+            name: "MyLocalTool",
+            dependencies: [
+                "MySourceGenExt",
+                "MySourceGenTool"
+            ]
+        ),
         // The target that implements the extension and generates commands to invoke MySourceGenTool.
         .extension(
             name: "MySourceGenExt",
-            capability: .prebuild(),
+            capability: .buildTool(),
             dependencies: [
                 "MySourceGenTool"
             ]
@@ -23,24 +31,17 @@ let package = Package(
         .executableTarget(
             name: "MySourceGenTool",
             dependencies: [
-                "MySourceGenToolLib"
+                "MySourceGenToolLib",
             ]
         ),
         // A library used by MySourceGenTool (not the client).
-        .executableTarget(
+        .target(
             name: "MySourceGenToolLib"
         ),
         // A runtime library that the client needs to link against.
         .target(
             name: "MySourceGenRuntimeLib"
         ),
-        // A local tool that uses the extension.
-        .executableTarget(
-            name: "MyLocalTool",
-            dependencies: [
-                "MySourceGenExt"
-            ]
-        ),
         // Unit tests for the extension.
         .testTarget(
             name: "MySourceGenExtTests",

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/Foo.dat

@@ -0,0 +1 @@
+Hello Extension!

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/main.swift

@@ -1 +1 @@
-print("Exec: \\(name)")
+print("Exec: \(data)")

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenExt/extension.swift

@@ -1,4 +1,26 @@
 import PackageExtension
 
-print("Hello MySourceGenExt")
-print(targetBuildContext)
+for inputPath in targetBuildContext.otherFiles {
+    guard inputPath.suffix == ".dat" else { continue }
+    let outputName = inputPath.basename + ".swift"
+    let outputPath = targetBuildContext.outputDir.appending(outputName)
+    commandConstructor.createCommand(
+        displayName:
+            "MySourceGenTooling \(inputPath)",
+        executable:
+            try targetBuildContext.lookupTool(named: "MySourceGenTool"),
+        arguments: [
+            "\(inputPath)",
+            "\(outputPath)"
+        ],
+        inputPaths: [
+            inputPath,
+        ],
+        outputPaths: [
+            outputPath
+        ],
+        derivedSourcePaths: [
+            outputPath
+        ]
+    )
+}

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenTool/main.swift

@@ -1 +1,16 @@
-print("Hello MySourceGenTool")
+import Foundation
+import MySourceGenToolLib
+
+// Sample source generator tool that just emits the hex representation of the contents of a file as a quoted string.  The input file is the first argument and the output file is the second.
+if ProcessInfo.processInfo.arguments.count != 3 {
+    print("usage: MySourceGenTool <input> <output>")
+    exit(1)
+}
+let inputFile = ProcessInfo.processInfo.arguments[1]
+let outputFile = ProcessInfo.processInfo.arguments[2]
+
+let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+let outputString = "public var data = \(dataAsHex.quotedForSourceCode)\n"
+let outputData = outputString.data(using: .utf8)
+FileManager.default.createFile(atPath: outputFile, contents: outputData)

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenToolLib/library.swift

@@ -1,3 +1,11 @@
-public func GetLibraryName() -> String {
-    return "MySourceGenToolLib"
+import Foundation
+
+extension String {
+
+    public var quotedForSourceCode: String {
+        return "\"" + self
+            .replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "\"", with: "\\\"")
+            + "\""
+    }
 }

Package.swift

@@ -91,6 +91,12 @@ let package = Package(
             targets: ["PackageDescription"]
         ),
 
+        .library(
+            name: "PackageExtension",
+            type: .dynamic,
+            targets: ["PackageExtension"]
+        ),
+
         .library(
             name: "PackageCollectionsModel",
             targets: ["PackageCollectionsModel"]
@@ -107,6 +113,9 @@ let package = Package(
             swiftSettings: [
                 .define("PACKAGE_DESCRIPTION_4_2"),
             ]),
+
+        .target(
+            name: "PackageExtension"),
 
         // MARK: SwiftPM specific support libraries
 

Sources/Build/BuildOperation.swift

@@ -30,6 +30,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// The closure for loading the package graph.
     let packageGraphLoader: () throws -> PackageGraph
+
+    /// The closure for evaluating extensions in the package graph.
+    let extensionEvaluator: (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]]
 
     /// The llbuild build delegate reference.
     private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
@@ -60,12 +63,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
+        extensionEvaluator: @escaping (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]],
         diagnostics: DiagnosticsEngine,
         stdoutStream: OutputByteStream
     ) {
         self.buildParameters = buildParameters
         self.cacheBuildManifest = cacheBuildManifest
         self.packageGraphLoader = packageGraphLoader
+        self.extensionEvaluator = extensionEvaluator
         self.diagnostics = diagnostics
         self.stdoutStream = stdoutStream
     }
@@ -75,6 +80,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             try self.packageGraphLoader()
         }
     }
+
+    public func getExtensionEvaluationResults(for graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
+        return try self.extensionEvaluator(graph)
+    }
 
     /// Compute and return the latest build description.
     ///
@@ -157,9 +166,11 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// Create the build plan and return the build description.
     private func plan() throws -> BuildDescription {
         let graph = try getPackageGraph()
+        let extensionEvaluationResults = try getExtensionEvaluationResults(for: graph)
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
+            extensionEvaluationResults: extensionEvaluationResults,
             diagnostics: diagnostics
         )
         self.buildPlan = plan

Sources/Build/BuildPlan.swift

@@ -544,11 +544,15 @@ public final class SwiftTargetBuildDescription {
 
     /// The modulemap file for this target, if any.
     private(set) var moduleMap: AbsolutePath?
+
+    /// The results of having applied any extensions to this target.
+    public let extensionEvaluationResults: [ExtensionEvaluationResult]
 
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         buildParameters: BuildParameters,
+        extensionEvaluationResults: [ExtensionEvaluationResult] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
         fs: FileSystem = localFileSystem
@@ -562,6 +566,21 @@ public final class SwiftTargetBuildDescription {
         self.fs = fs
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
+        self.extensionEvaluationResults = extensionEvaluationResults
+
+        // Add any derived source paths declared by build-tool extensions that were applied to this target.  We do
+        // this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
+        // where sources are processed, e.g. when determining names of object files, etc.
+        for command in extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
+            // Prebuild and postbuild commands are handled outside the build system.
+            if case .buildToolCommand(_, _, _, _, _, _, _, let derivedSourcePaths) = command {
+                // TODO: What should we do if we find non-Swift sources here?
+                for absPath in derivedSourcePaths {
+                    let relPath = absPath.relative(to: self.derivedSources.root)
+                    self.derivedSources.relativePaths.append(relPath)
+                }
+            }
+        }
 
         if shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
@@ -1259,6 +1278,9 @@ public class BuildPlan {
         return AnySequence(productMap.values)
     }
 
+    /// The results of evaluating any extensions used by targets in this build.
+    public let extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]]
+
     /// The filesystem to operate on.
     let fileSystem: FileSystem
 
@@ -1327,11 +1349,13 @@ public class BuildPlan {
     public init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
+        extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
     ) throws {
         self.buildParameters = buildParameters
         self.graph = graph
+        self.extensionEvaluationResults = extensionEvaluationResults
         self.diagnostics = diagnostics
         self.fileSystem = fileSystem
 
@@ -1353,7 +1377,11 @@ public class BuildPlan {
 
             switch target.underlyingTarget {
             case is SwiftTarget:
-                targetMap[target] = try .swift(SwiftTargetBuildDescription(target: target, buildParameters: buildParameters, fs: fileSystem))
+                targetMap[target] = try .swift(SwiftTargetBuildDescription(
+                    target: target,
+                    buildParameters: buildParameters,
+                    extensionEvaluationResults: extensionEvaluationResults[target] ?? [],
+                    fs: fileSystem))
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(
                     target: target,

Sources/Build/ManifestBuilder.swift

@@ -510,6 +510,8 @@ extension LLBuildManifestBuilder {
             if target.underlyingTarget is SystemLibraryTarget { return }
             // Ignore Binary Modules.
             if target.underlyingTarget is BinaryTarget { return }
+            // Ignore Extension Targets.
+            if target.underlyingTarget is ExtensionTarget { return }
 
             // Depend on the binary for executable targets.
             if target.type == .executable {
@@ -573,6 +575,20 @@ extension LLBuildManifestBuilder {
             }
         }
 
+        // Add any build tool commands created by extensions for the target (prebuild and postbuild commands are handled outside the build).
+        for command in target.extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
+            if case .buildToolCommand(let displayName, let execPath, let arguments, _, _, let inputPaths, let outputPaths, _) = command {
+                // Create a shell command to invoke the executable.  We include the path of the executable as a dependency.
+                // FIXME: We will need to extend the addShellCmd() function to also take working directory and environment.
+                manifest.addShellCmd(
+                    name: displayName,
+                    description: displayName,
+                    inputs: [.file(execPath)] + inputPaths.map{ .file($0) },
+                    outputs: outputPaths.map{ .file($0) },
+                    args: [execPath.pathString] + arguments)
+            }
+        }
+
         return inputs
     }
 

Sources/CMakeLists.txt

@@ -14,6 +14,7 @@ add_subdirectory(PackageCollections)
 add_subdirectory(PackageCollectionsModel)
 add_subdirectory(PackageCollectionsSigning)
 add_subdirectory(PackageDescription)
+add_subdirectory(PackageExtension)
 add_subdirectory(PackageGraph)
 add_subdirectory(PackageLoading)
 add_subdirectory(PackageModel)

Sources/Commands/APIDigester.swift

@@ -112,6 +112,7 @@ struct APIDigesterBaselineDumper {
             buildParameters: buildParameters,
             cacheBuildManifest: false,
             packageGraphLoader: { graph },
+            extensionEvaluator: { _ in [:] },
             diagnostics: diags,
             stdoutStream: stdoutStream
         )

Sources/Commands/SwiftRunTool.swift

@@ -118,6 +118,7 @@ public struct SwiftRunTool: SwiftCommand {
                 buildParameters: buildParameters,
                 cacheBuildManifest: false,
                 packageGraphLoader: graphLoader,
+                extensionEvaluator: { _ in [:] },
                 diagnostics: swiftTool.diagnostics,
                 stdoutStream: swiftTool.stdoutStream
             )

Sources/Commands/SwiftTool.swift

@@ -594,6 +594,33 @@ public class SwiftTool {
             throw error
         }
     }
+
+    /// Evaluate extensions for any reachable targets in the graph, and return a mapping from targets to corresponding evaluation results.
+    func evaluateExtensions(graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
+        do {
+            // Configure the inputs to the extension evaluation.
+            // FIXME: These paths are still fairly preliminary.
+            let buildEnvironment = try buildParameters().buildEnvironment
+            let dataDir = try self.getActiveWorkspace().dataPath
+            let extensionsDir = dataDir.appending(component: "extensions")
+            let cacheDir = extensionsDir.appending(component: "cache")
+            let extensionRunner = try DefaultExtensionRunner(cacheDir: cacheDir, manifestResources: self._hostToolchain.get().manifestResources)
+            let outputDir = extensionsDir.appending(component: "outputs")
+            // FIXME: Too many assumptions!
+            let execsDir = dataDir.appending(components: try self._hostToolchain.get().triple.tripleString, buildEnvironment.configuration.dirname)
+            let diagnostics = DiagnosticsEngine()
+
+            // Create the cache directory, if needed.
+            try localFileSystem.createDirectory(cacheDir, recursive: true)
+
+            // Ask the graph to evaluate extensions, and return the result.
+            let result = try graph.evaluateExtensions(buildEnvironment: buildEnvironment, execsDir: execsDir, outputDir: outputDir, extensionRunner: extensionRunner, diagnostics: diagnostics, fileSystem: localFileSystem)
+            return result
+        }
+        catch {
+            throw error
+        }
+    }
 
     /// Returns the user toolchain to compile the actual product.
     func getToolchain() throws -> UserToolchain {
@@ -638,6 +665,7 @@ public class SwiftTool {
             buildParameters: buildParameters(),
             cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
             packageGraphLoader: graphLoader,
+            extensionEvaluator: { _ in [:] },
             diagnostics: diagnostics,
             stdoutStream: self.stdoutStream
         )
@@ -652,15 +680,18 @@ public class SwiftTool {
         switch options.buildSystem {
         case .native:
             let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct) }
+            let extensionEvaluator = { try self.evaluateExtensions(graph: $0) }
             buildSystem = try BuildOperation(
                 buildParameters: buildParameters ?? self.buildParameters(),
                 cacheBuildManifest: self.canUseCachedBuildManifest(),
                 packageGraphLoader: graphLoader,
+                extensionEvaluator: extensionEvaluator,
                 diagnostics: diagnostics,
                 stdoutStream: stdoutStream
             )
         case .xcode:
             let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct, createMultipleTestProducts: true) }
+            // FIXME: Implement the custom build command provider also.
             buildSystem = try XcodeBuildSystem(
                 buildParameters: buildParameters ?? self.buildParameters(),
                 packageGraphLoader: graphLoader,