Skip to content

Commit a7815f5

Browse files
abertelrudabertelrud
authored
Add PackageExtension library and incorporate build tool commands into native build system (#3287)
* Extend TargetSourcesBuilder to return other files than sources and resources, and extend the ExtensionEvaluator to pass them on to extensions. * Pass through the tools version of the package that declares the extension, which will control availability annotations in the PackageExtension module. * Add a PackageExtension module and library (analogous to PackageDescription). This contains the APIs that extensions can use. * Add a concrete specialization of ExtensionRunner and use it from the native build system (for build tools only so far; prebuild and postbuild are coming). The default extension runner compiles and runs the extension as an executable, passing it input and receiving output in coordination with the implementation in PackageExtension (this works much the same as PackageDescription). There are still some stubbed-out areas to be filled in, and there needs to be caching and sandboxing added, but it is complete and works for what is implemented so far.
1 parent 5d0e191 commit a7815f5

File tree

32 files changed

+906
-44
lines changed

32 files changed

+906
-44
lines changed

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Package.swift

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,18 @@ let package = Package(
1111
// )
1212
],
1313
targets: [
14+
// A local tool that uses an extension.
15+
.executableTarget(
16+
name: "MyLocalTool",
17+
dependencies: [
18+
"MySourceGenExt",
19+
"MySourceGenTool"
20+
]
21+
),
1422
// The target that implements the extension and generates commands to invoke MySourceGenTool.
1523
.extension(
1624
name: "MySourceGenExt",
17-
capability: .prebuild(),
25+
capability: .buildTool(),
1826
dependencies: [
1927
"MySourceGenTool"
2028
]
@@ -23,24 +31,17 @@ let package = Package(
2331
.executableTarget(
2432
name: "MySourceGenTool",
2533
dependencies: [
26-
"MySourceGenToolLib"
34+
"MySourceGenToolLib",
2735
]
2836
),
2937
// A library used by MySourceGenTool (not the client).
30-
.executableTarget(
38+
.target(
3139
name: "MySourceGenToolLib"
3240
),
3341
// A runtime library that the client needs to link against.
3442
.target(
3543
name: "MySourceGenRuntimeLib"
3644
),
37-
// A local tool that uses the extension.
38-
.executableTarget(
39-
name: "MyLocalTool",
40-
dependencies: [
41-
"MySourceGenExt"
42-
]
43-
),
4445
// Unit tests for the extension.
4546
.testTarget(
4647
name: "MySourceGenExtTests",

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/Foo.dat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello Extension!

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/main.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
print("Exec: \\(name)")
1+
print("Exec: \(data)")

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenExt/extension.swift

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,26 @@
11
import PackageExtension
22

3-
print("Hello MySourceGenExt")
4-
print(targetBuildContext)
3+
for inputPath in targetBuildContext.otherFiles {
4+
guard inputPath.suffix == ".dat" else { continue }
5+
let outputName = inputPath.basename + ".swift"
6+
let outputPath = targetBuildContext.outputDir.appending(outputName)
7+
commandConstructor.createCommand(
8+
displayName:
9+
"MySourceGenTooling \(inputPath)",
10+
executable:
11+
try targetBuildContext.lookupTool(named: "MySourceGenTool"),
12+
arguments: [
13+
"\(inputPath)",
14+
"\(outputPath)"
15+
],
16+
inputPaths: [
17+
inputPath,
18+
],
19+
outputPaths: [
20+
outputPath
21+
],
22+
derivedSourcePaths: [
23+
outputPath
24+
]
25+
)
26+
}

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenTool/main.swift

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,16 @@
1-
print("Hello MySourceGenTool")
1+
import Foundation
2+
import MySourceGenToolLib
3+
4+
// Sample source generator tool that just emits the hex representation of the contents of a file as a quoted string. The input file is the first argument and the output file is the second.
5+
if ProcessInfo.processInfo.arguments.count != 3 {
6+
print("usage: MySourceGenTool <input> <output>")
7+
exit(1)
8+
}
9+
let inputFile = ProcessInfo.processInfo.arguments[1]
10+
let outputFile = ProcessInfo.processInfo.arguments[2]
11+
12+
let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
13+
let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
14+
let outputString = "public var data = \(dataAsHex.quotedForSourceCode)\n"
15+
let outputData = outputString.data(using: .utf8)
16+
FileManager.default.createFile(atPath: outputFile, contents: outputData)

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenToolLib/library.swift

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1-
public func GetLibraryName() -> String {
2-
return "MySourceGenToolLib"
1+
import Foundation
2+
3+
extension String {
4+
5+
public var quotedForSourceCode: String {
6+
return "\"" + self
7+
.replacingOccurrences(of: "\\", with: "\\\\")
8+
.replacingOccurrences(of: "\"", with: "\\\"")
9+
+ "\""
10+
}
311
}

‎Package.swift

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,12 @@ let package = Package(
9191
targets: ["PackageDescription"]
9292
),
9393

94+
.library(
95+
name: "PackageExtension",
96+
type: .dynamic,
97+
targets: ["PackageExtension"]
98+
),
99+
94100
.library(
95101
name: "PackageCollectionsModel",
96102
targets: ["PackageCollectionsModel"]
@@ -107,6 +113,9 @@ let package = Package(
107113
swiftSettings: [
108114
.define("PACKAGE_DESCRIPTION_4_2"),
109115
]),
116+
117+
.target(
118+
name: "PackageExtension"),
110119

111120
// MARK: SwiftPM specific support libraries
112121

‎Sources/Build/BuildOperation.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
3030

3131
/// The closure for loading the package graph.
3232
let packageGraphLoader: () throws -> PackageGraph
33+
34+
/// The closure for evaluating extensions in the package graph.
35+
let extensionEvaluator: (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]]
3336

3437
/// The llbuild build delegate reference.
3538
private var buildSystemDelegate: BuildOperationBuildSystemDelegateHandler?
@@ -60,12 +63,14 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
6063
buildParameters: BuildParameters,
6164
cacheBuildManifest: Bool,
6265
packageGraphLoader: @escaping () throws -> PackageGraph,
66+
extensionEvaluator: @escaping (PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]],
6367
diagnostics: DiagnosticsEngine,
6468
stdoutStream: OutputByteStream
6569
) {
6670
self.buildParameters = buildParameters
6771
self.cacheBuildManifest = cacheBuildManifest
6872
self.packageGraphLoader = packageGraphLoader
73+
self.extensionEvaluator = extensionEvaluator
6974
self.diagnostics = diagnostics
7075
self.stdoutStream = stdoutStream
7176
}
@@ -75,6 +80,10 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
7580
try self.packageGraphLoader()
7681
}
7782
}
83+
84+
public func getExtensionEvaluationResults(for graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
85+
return try self.extensionEvaluator(graph)
86+
}
7887

7988
/// Compute and return the latest build description.
8089
///
@@ -157,9 +166,11 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
157166
/// Create the build plan and return the build description.
158167
private func plan() throws -> BuildDescription {
159168
let graph = try getPackageGraph()
169+
let extensionEvaluationResults = try getExtensionEvaluationResults(for: graph)
160170
let plan = try BuildPlan(
161171
buildParameters: buildParameters,
162172
graph: graph,
173+
extensionEvaluationResults: extensionEvaluationResults,
163174
diagnostics: diagnostics
164175
)
165176
self.buildPlan = plan

‎Sources/Build/BuildPlan.swift

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -544,11 +544,15 @@ public final class SwiftTargetBuildDescription {
544544

545545
/// The modulemap file for this target, if any.
546546
private(set) var moduleMap: AbsolutePath?
547+
548+
/// The results of having applied any extensions to this target.
549+
public let extensionEvaluationResults: [ExtensionEvaluationResult]
547550

548551
/// Create a new target description with target and build parameters.
549552
init(
550553
target: ResolvedTarget,
551554
buildParameters: BuildParameters,
555+
extensionEvaluationResults: [ExtensionEvaluationResult] = [],
552556
isTestTarget: Bool? = nil,
553557
testDiscoveryTarget: Bool = false,
554558
fs: FileSystem = localFileSystem
@@ -562,6 +566,21 @@ public final class SwiftTargetBuildDescription {
562566
self.fs = fs
563567
self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
564568
self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
569+
self.extensionEvaluationResults = extensionEvaluationResults
570+
571+
// Add any derived source paths declared by build-tool extensions that were applied to this target. We do
572+
// this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
573+
// where sources are processed, e.g. when determining names of object files, etc.
574+
for command in extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
575+
// Prebuild and postbuild commands are handled outside the build system.
576+
if case .buildToolCommand(_, _, _, _, _, _, _, let derivedSourcePaths) = command {
577+
// TODO: What should we do if we find non-Swift sources here?
578+
for absPath in derivedSourcePaths {
579+
let relPath = absPath.relative(to: self.derivedSources.root)
580+
self.derivedSources.relativePaths.append(relPath)
581+
}
582+
}
583+
}
565584

566585
if shouldEmitObjCCompatibilityHeader {
567586
self.moduleMap = try self.generateModuleMap()
@@ -1259,6 +1278,9 @@ public class BuildPlan {
12591278
return AnySequence(productMap.values)
12601279
}
12611280

1281+
/// The results of evaluating any extensions used by targets in this build.
1282+
public let extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]]
1283+
12621284
/// The filesystem to operate on.
12631285
let fileSystem: FileSystem
12641286

@@ -1327,11 +1349,13 @@ public class BuildPlan {
13271349
public init(
13281350
buildParameters: BuildParameters,
13291351
graph: PackageGraph,
1352+
extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]] = [:],
13301353
diagnostics: DiagnosticsEngine,
13311354
fileSystem: FileSystem = localFileSystem
13321355
) throws {
13331356
self.buildParameters = buildParameters
13341357
self.graph = graph
1358+
self.extensionEvaluationResults = extensionEvaluationResults
13351359
self.diagnostics = diagnostics
13361360
self.fileSystem = fileSystem
13371361

@@ -1353,7 +1377,11 @@ public class BuildPlan {
13531377

13541378
switch target.underlyingTarget {
13551379
case is SwiftTarget:
1356-
targetMap[target] = try .swift(SwiftTargetBuildDescription(target: target, buildParameters: buildParameters, fs: fileSystem))
1380+
targetMap[target] = try .swift(SwiftTargetBuildDescription(
1381+
target: target,
1382+
buildParameters: buildParameters,
1383+
extensionEvaluationResults: extensionEvaluationResults[target] ?? [],
1384+
fs: fileSystem))
13571385
case is ClangTarget:
13581386
targetMap[target] = try .clang(ClangTargetBuildDescription(
13591387
target: target,

‎Sources/Build/ManifestBuilder.swift

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -510,6 +510,8 @@ extension LLBuildManifestBuilder {
510510
if target.underlyingTarget is SystemLibraryTarget { return }
511511
// Ignore Binary Modules.
512512
if target.underlyingTarget is BinaryTarget { return }
513+
// Ignore Extension Targets.
514+
if target.underlyingTarget is ExtensionTarget { return }
513515

514516
// Depend on the binary for executable targets.
515517
if target.type == .executable {
@@ -573,6 +575,20 @@ extension LLBuildManifestBuilder {
573575
}
574576
}
575577

578+
// Add any build tool commands created by extensions for the target (prebuild and postbuild commands are handled outside the build).
579+
for command in target.extensionEvaluationResults.reduce([], { $0 + $1.commands }) {
580+
if case .buildToolCommand(let displayName, let execPath, let arguments, _, _, let inputPaths, let outputPaths, _) = command {
581+
// Create a shell command to invoke the executable. We include the path of the executable as a dependency.
582+
// FIXME: We will need to extend the addShellCmd() function to also take working directory and environment.
583+
manifest.addShellCmd(
584+
name: displayName,
585+
description: displayName,
586+
inputs: [.file(execPath)] + inputPaths.map{ .file($0) },
587+
outputs: outputPaths.map{ .file($0) },
588+
args: [execPath.pathString] + arguments)
589+
}
590+
}
591+
576592
return inputs
577593
}
578594

‎Sources/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ add_subdirectory(PackageCollections)
1414
add_subdirectory(PackageCollectionsModel)
1515
add_subdirectory(PackageCollectionsSigning)
1616
add_subdirectory(PackageDescription)
17+
add_subdirectory(PackageExtension)
1718
add_subdirectory(PackageGraph)
1819
add_subdirectory(PackageLoading)
1920
add_subdirectory(PackageModel)

‎Sources/Commands/APIDigester.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ struct APIDigesterBaselineDumper {
112112
buildParameters: buildParameters,
113113
cacheBuildManifest: false,
114114
packageGraphLoader: { graph },
115+
extensionEvaluator: { _ in [:] },
115116
diagnostics: diags,
116117
stdoutStream: stdoutStream
117118
)

‎Sources/Commands/SwiftRunTool.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ public struct SwiftRunTool: SwiftCommand {
118118
buildParameters: buildParameters,
119119
cacheBuildManifest: false,
120120
packageGraphLoader: graphLoader,
121+
extensionEvaluator: { _ in [:] },
121122
diagnostics: swiftTool.diagnostics,
122123
stdoutStream: swiftTool.stdoutStream
123124
)

‎Sources/Commands/SwiftTool.swift

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,33 @@ public class SwiftTool {
594594
throw error
595595
}
596596
}
597+
598+
/// Evaluate extensions for any reachable targets in the graph, and return a mapping from targets to corresponding evaluation results.
599+
func evaluateExtensions(graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
600+
do {
601+
// Configure the inputs to the extension evaluation.
602+
// FIXME: These paths are still fairly preliminary.
603+
let buildEnvironment = try buildParameters().buildEnvironment
604+
let dataDir = try self.getActiveWorkspace().dataPath
605+
let extensionsDir = dataDir.appending(component: "extensions")
606+
let cacheDir = extensionsDir.appending(component: "cache")
607+
let extensionRunner = try DefaultExtensionRunner(cacheDir: cacheDir, manifestResources: self._hostToolchain.get().manifestResources)
608+
let outputDir = extensionsDir.appending(component: "outputs")
609+
// FIXME: Too many assumptions!
610+
let execsDir = dataDir.appending(components: try self._hostToolchain.get().triple.tripleString, buildEnvironment.configuration.dirname)
611+
let diagnostics = DiagnosticsEngine()
612+
613+
// Create the cache directory, if needed.
614+
try localFileSystem.createDirectory(cacheDir, recursive: true)
615+
616+
// Ask the graph to evaluate extensions, and return the result.
617+
let result = try graph.evaluateExtensions(buildEnvironment: buildEnvironment, execsDir: execsDir, outputDir: outputDir, extensionRunner: extensionRunner, diagnostics: diagnostics, fileSystem: localFileSystem)
618+
return result
619+
}
620+
catch {
621+
throw error
622+
}
623+
}
597624

598625
/// Returns the user toolchain to compile the actual product.
599626
func getToolchain() throws -> UserToolchain {
@@ -638,6 +665,7 @@ public class SwiftTool {
638665
buildParameters: buildParameters(),
639666
cacheBuildManifest: cacheBuildManifest && self.canUseCachedBuildManifest(),
640667
packageGraphLoader: graphLoader,
668+
extensionEvaluator: { _ in [:] },
641669
diagnostics: diagnostics,
642670
stdoutStream: self.stdoutStream
643671
)
@@ -652,15 +680,18 @@ public class SwiftTool {
652680
switch options.buildSystem {
653681
case .native:
654682
let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct) }
683+
let extensionEvaluator = { try self.evaluateExtensions(graph: $0) }
655684
buildSystem = try BuildOperation(
656685
buildParameters: buildParameters ?? self.buildParameters(),
657686
cacheBuildManifest: self.canUseCachedBuildManifest(),
658687
packageGraphLoader: graphLoader,
688+
extensionEvaluator: extensionEvaluator,
659689
diagnostics: diagnostics,
660690
stdoutStream: stdoutStream
661691
)
662692
case .xcode:
663693
let graphLoader = { try self.loadPackageGraph(explicitProduct: explicitProduct, createMultipleTestProducts: true) }
694+
// FIXME: Implement the custom build command provider also.
664695
buildSystem = try XcodeBuildSystem(
665696
buildParameters: buildParameters ?? self.buildParameters(),
666697
packageGraphLoader: graphLoader,

0 commit comments

Comments
 (0)