Add a concrete specialization of ExtensionRunner and use it from the native build system (for build tools only so far; prebuild and postbuild are coming). The default extension runner compiles and runs the extension as an executable, passing it input and receiving output in coordination with the implementation in PackageExtension (this works much the same as PackageDescription).

There are still some stubbed-out areas to be filled in, and there needs to be caching and sandboxing added, but it is complete and works for what is implemented so far.

Author: abertelrud

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Package.swift

@@ -10,11 +10,23 @@ let package = Package(
         //     target: "MySourceGenExt"
         // )
     ],
+    dependencies: [
+        .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "0.3.1")),
+    ],
     targets: [
+        // A local tool that uses an extension.
+        .executableTarget(
+            name: "MyLocalTool",
+            dependencies: [
+                "MySourceGenExt",
+                .product(name: "ArgumentParser", package: "swift-argument-parser"),
+                "MySourceGenTool"
+            ]
+        ),
         // The target that implements the extension and generates commands to invoke MySourceGenTool.
         .extension(
             name: "MySourceGenExt",
-            capability: .prebuild(),
+            capability: .buildTool(),
             dependencies: [
                 "MySourceGenTool"
             ]
@@ -23,24 +35,18 @@ let package = Package(
         .executableTarget(
             name: "MySourceGenTool",
             dependencies: [
-                "MySourceGenToolLib"
+                "MySourceGenToolLib",
+                .product(name: "ArgumentParser", package: "swift-argument-parser")
             ]
         ),
         // A library used by MySourceGenTool (not the client).
-        .executableTarget(
+        .target(
             name: "MySourceGenToolLib"
         ),
         // A runtime library that the client needs to link against.
         .target(
             name: "MySourceGenRuntimeLib"
         ),
-        // A local tool that uses the extension.
-        .executableTarget(
-            name: "MyLocalTool",
-            dependencies: [
-                "MySourceGenExt"
-            ]
-        ),
         // Unit tests for the extension.
         .testTarget(
             name: "MySourceGenExtTests",

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/Foo.dat

@@ -0,0 +1 @@
+Hello Extension!

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/main.swift

@@ -1 +1 @@
-print("Exec: \\(name)")
+print("Exec: \(data)")

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenExt/extension.swift

@@ -1,4 +1,30 @@
 import PackageExtension
 
-print("Hello MySourceGenExt")
-print(targetBuildContext)
+
+for inputPath in targetBuildContext.otherFiles {
+    guard inputPath.hasSuffix(".dat") else { continue }
+    
+    let outputPath = targetBuildContext.outputDir.appending(inputPath.basename + ".swift")
+    print("inputPath:  \(inputPath)")
+    print("outputPath: \(outputPath)")
+
+    commandConstructor.createCommand(
+        displayName:
+            "MySourceGenTooling \(outputPath.string)",
+        executable:
+            try targetBuildContext.lookupTool(named: "MySourceGenTool"),
+        arguments: [
+            inputPath.string,
+            outputPath.string
+        ],
+        inputPaths: [
+            inputPath,
+        ],
+        outputPaths: [
+            outputPath
+        ],
+        derivedSourcePaths: [
+            outputPath
+        ]
+    )
+}

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenTool/main.swift

@@ -1 +1,19 @@
-print("Hello MySourceGenTool")
+import ArgumentParser
+import Foundation
+import MySourceGenToolLib
+
+// Sample source generator tool that just emits the hex representation of the contents of a file as a quoted string.
+struct MySourceGenTool: ParsableCommand {
+    @Argument() var inputFile: String
+    @Argument() var outputFile: String
+    
+    func run() {
+        let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
+        let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
+        let outputString = "public var data = \(dataAsHex.quotedForSourceCode)\n"
+        let outputData = outputString.data(using: .utf8)
+        FileManager.default.createFile(atPath: outputFile, contents: outputData)
+    }
+}
+
+MySourceGenTool.main()

Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenToolLib/library.swift

@@ -1,3 +1,11 @@
-public func GetLibraryName() -> String {
-    return "MySourceGenToolLib"
+import Foundation
+
+extension String {
+    
+    public var quotedForSourceCode: String {
+        return "\"" + self
+            .replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "\"", with: "\\\"")
+            + "\""
+    }
 }

Sources/Build/BuildOperation.swift

@@ -30,6 +30,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
 
     /// The closure for loading the package graph.
     let packageGraphLoader: () throws -> PackageGraph
+    
+    /// The closure for evaluating extensions in the package graph.
+    let extensionEvaluator: (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]]
 
     /// The llbuild build delegate reference.
     private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
@@ -60,12 +63,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
         buildParameters: BuildParameters,
         cacheBuildManifest: Bool,
         packageGraphLoader: @escaping () throws -> PackageGraph,
+        extensionEvaluator: @escaping (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]],
         diagnostics: DiagnosticsEngine,
         stdoutStream: OutputByteStream
     ) {
         self.buildParameters = buildParameters
         self.cacheBuildManifest = cacheBuildManifest
         self.packageGraphLoader = packageGraphLoader
+        self.extensionEvaluator = extensionEvaluator
         self.diagnostics = diagnostics
         self.stdoutStream = stdoutStream
     }
@@ -75,6 +80,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
             try self.packageGraphLoader()
         }
     }
+    
+    public func getExtensionEvaluationResults(for graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
+        return try self.extensionEvaluator(graph)
+    }
 
     /// Compute and return the latest build description.
     ///
@@ -157,9 +166,11 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
     /// Create the build plan and return the build description.
     private func plan() throws -> BuildDescription {
         let graph = try getPackageGraph()
+        let extensionEvaluationResults = try getExtensionEvaluationResults(for: graph)
         let plan = try BuildPlan(
             buildParameters: buildParameters,
             graph: graph,
+            extensionEvaluationResults: extensionEvaluationResults,
             diagnostics: diagnostics
         )
         self.buildPlan = plan

Sources/Build/BuildPlan.swift

@@ -544,11 +544,15 @@ public final class SwiftTargetBuildDescription {
 
     /// The modulemap file for this target, if any.
     private(set) var moduleMap: AbsolutePath?
+    
+    /// The results of having applied any extensions to this target.
+    public private(set) var extensionEvaluationResults: [ExtensionEvaluationResult]
 
     /// Create a new target description with target and build parameters.
     init(
         target: ResolvedTarget,
         buildParameters: BuildParameters,
+        extensionEvaluationResults: [ExtensionEvaluationResult] = [],
         isTestTarget: Bool? = nil,
         testDiscoveryTarget: Bool = false,
         fs: FileSystem = localFileSystem
@@ -562,6 +566,21 @@ public final class SwiftTargetBuildDescription {
         self.fs = fs
         self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
         self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
+        self.extensionEvaluationResults = extensionEvaluationResults
+
+        // Add any derived source paths declared by build-tool extensions that were applied to this target.  We do
+        // this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
+        // where sources are processed, e.g. when determining names of object files, etc.
+        for command in extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
+            // Prebuild and postbuild commands are handled outside the build system.
+            if case .buildToolCommand(_, _, _, _, _, _, _, let derivedSourcePaths) = command {
+                // TODO: What should we do if we find non-Swift sources here?
+                for absPath in derivedSourcePaths {
+                    let relPath = absPath.relative(to: self.derivedSources.root)
+                    self.derivedSources.relativePaths.append(relPath)
+                }
+            }
+        }
 
         if shouldEmitObjCCompatibilityHeader {
             self.moduleMap = try self.generateModuleMap()
@@ -1259,6 +1278,9 @@ public class BuildPlan {
         return AnySequence(productMap.values)
     }
 
+    /// The results of evaluating any extensions used by targets in this build.
+    private(set) var extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]]
+
     /// The filesystem to operate on.
     let fileSystem: FileSystem
 
@@ -1327,11 +1349,13 @@ public class BuildPlan {
     public init(
         buildParameters: BuildParameters,
         graph: PackageGraph,
+        extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]] = [:],
         diagnostics: DiagnosticsEngine,
         fileSystem: FileSystem = localFileSystem
     ) throws {
         self.buildParameters = buildParameters
         self.graph = graph
+        self.extensionEvaluationResults = extensionEvaluationResults
         self.diagnostics = diagnostics
         self.fileSystem = fileSystem
 
@@ -1353,7 +1377,11 @@ public class BuildPlan {
 
             switch target.underlyingTarget {
             case is SwiftTarget:
-                targetMap[target] = try .swift(SwiftTargetBuildDescription(target: target, buildParameters: buildParameters, fs: fileSystem))
+                targetMap[target] = try .swift(SwiftTargetBuildDescription(
+                    target: target,
+                    buildParameters: buildParameters,
+                    extensionEvaluationResults: extensionEvaluationResults[target] ?? [],
+                    fs: fileSystem))
             case is ClangTarget:
                 targetMap[target] = try .clang(ClangTargetBuildDescription(
                     target: target,

Sources/Build/ManifestBuilder.swift

@@ -510,6 +510,8 @@ extension LLBuildManifestBuilder {
             if target.underlyingTarget is SystemLibraryTarget { return }
             // Ignore Binary Modules.
             if target.underlyingTarget is BinaryTarget { return }
+            // Ignore Extension Targets.
+            if target.underlyingTarget is ExtensionTarget { return }
 
             // Depend on the binary for executable targets.
             if target.type == .executable {
@@ -573,6 +575,20 @@ extension LLBuildManifestBuilder {
             }
         }
 
+        // Add any build tool commands created by extensions for the target (prebuild and postbuild commands are handled outside the build).
+        for command in target.extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
+            if case .buildToolCommand(let displayName, let execPath, let arguments, _, _, let inputPaths, let outputPaths, _) = command {
+                // Create a shell command to invoke the executable.  We include the path of the executable as a dependency.
+                // FIXME: We will need to extend the addShellCmd() function to also take working directory and environment.
+                manifest.addShellCmd(
+                    name: displayName,
+                    description: displayName,
+                    inputs: [.file(execPath)] + inputPaths.map{ .file($0) },
+                    outputs: outputPaths.map{ .file($0) },
+                    args: [execPath.pathString] + arguments)
+            }
+        }
+
         return inputs
     }
 

Sources/Commands/APIDigester.swift

@@ -112,6 +112,7 @@ struct APIDigesterBaselineDumper {
             buildParameters: buildParameters,
             cacheBuildManifest: false,
             packageGraphLoader: { graph },
+            extensionEvaluator: { _ in [:] },
             diagnostics: diags,
             stdoutStream: stdoutStream
         )

Sources/Commands/SwiftRunTool.swift

@@ -118,6 +118,7 @@ public struct SwiftRunTool: SwiftCommand {
                 buildParameters: buildParameters,
                 cacheBuildManifest: false,
                 packageGraphLoader: graphLoader,
+                extensionEvaluator: { _ in [:] },
                 diagnostics: swiftTool.diagnostics,
                 stdoutStream: swiftTool.stdoutStream
             )

Sources/Commands/SwiftTool.swift

@@ -594,6 +594,33 @@ public class SwiftTool {
             throw error
         }
     }
+    
+    /// Evaluate extensions for any reachable targets in the graph, and return a mapping from targets to corresponding evaluation results.
+    func evaluateExtensions(graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
+        do {
+            // Configure the inputs to the extension evaluation.
+            // FIXME: These paths are still fairly preliminary.
+            let buildEnvironment = try buildParameters().buildEnvironment
+            let dataDir = try self.getActiveWorkspace().dataPath
+            let extensionsDir = dataDir.appending(component: "extensions")
+            let cacheDir = extensionsDir.appending(component: "cache")
+            let extensionRunner = try DefaultExtensionRunner(cacheDir: cacheDir, manifestResources: self._hostToolchain.get().manifestResources)
+            let outputDir = extensionsDir.appending(component: "outputs")
+            // FIXME: Too many assumptions!
+            let execsDir = dataDir.appending(components: try self._hostToolchain.get().triple.tripleString, buildEnvironment.configuration.dirname)
+            let diagnostics = DiagnosticsEngine()
+            
+            // Create the cache directory, if needed.
+            try localFileSystem.createDirectory(cacheDir, recursive: true)
+
+            // Ask the graph to evaluate extensions, and return the result.
+            let result = try graph.evaluateExtensions(buildEnvironment: buildEnvironment, execsDir: execsDir, outputDir: outputDir, extensionRunner: extensionRunner, diagnostics: diagnostics, fileSystem: localFileSystem)
+            return result
+        }
+        catch {
+            throw error
+        }
+    }
 
     /// Returns the user toolchain to compile the actual product.
     func getToolchain() throws -> UserToolchain {
@@ -638,6 +665,7 @@ public class SwiftTool {
             buildParameters: buildParameters(),
             cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
             packageGraphLoader: graphLoader,
+            extensionEvaluator: { _ in [:] },
             diagnostics: diagnostics,
             stdoutStream: self.stdoutStream
         )
@@ -652,15 +680,18 @@ public class SwiftTool {
         switch options.buildSystem {
         case .native:
             let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct) }
+            let extensionEvaluator = { try self.evaluateExtensions(graph: $0) }
             buildSystem = try BuildOperation(
                 buildParameters: buildParameters ?? self.buildParameters(),
                 cacheBuildManifest: self.canUseCachedBuildManifest(),
                 packageGraphLoader: graphLoader,
+                extensionEvaluator: extensionEvaluator,
                 diagnostics: diagnostics,
                 stdoutStream: stdoutStream
             )
         case .xcode:
             let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct, createMultipleTestProducts: true) }
+            // FIXME: Implement the custom build command provider also.
             buildSystem = try XcodeBuildSystem(
                 buildParameters: buildParameters ?? self.buildParameters(),
                 packageGraphLoader: graphLoader,

Sources/Workspace/CMakeLists.txt

@@ -1,12 +1,13 @@
 # This source file is part of the Swift.org open source project
 #
-# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
 # Licensed under Apache License v2.0 with Runtime Library Exception
 #
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
 add_library(Workspace
+  DefaultExtensionRunner.swift
   Destination.swift
   Diagnostics.swift
   Export.swift