Skip to content

Commit 318dca5

Browse files
abertelrudabertelrud
committed
WIP: Support for running extension-provided prebuild commands
1 parent 8e919a0 commit 318dca5

File tree

19 files changed

+219
-91
lines changed

19 files changed

+219
-91
lines changed

‎Fixtures/Miscellaneous/Extensions/MySourceGenClient/Package.swift

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@ let package = Package(
1111
.executableTarget(
1212
name: "MyTool",
1313
dependencies: [
14-
.product(name: "MySourceGenExt", package: "MySourceGenExtension")
14+
.product(name: "MySourceGenBuildToolExt", package: "MySourceGenExtension")
1515
]
1616
),
1717
// A unit that uses the extension.
1818
.testTarget(
1919
name: "MyTests",
2020
dependencies: [
21-
.product(name: "MySourceGenExt", package: "MySourceGenExtension")
21+
.product(name: "MySourceGenBuildToolExt", package: "MySourceGenExtension")
2222
]
2323
)
2424
]

‎Fixtures/Miscellaneous/Extensions/MySourceGenClient/Sources/MyTool/main.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
print("Generated string: '\(generatedString)'")
1+
print("Generated string Foo: '\(foo)'")

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Package.swift

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,53 @@ import PackageDescription
44
let package = Package(
55
name: "MySourceGenExtension",
66
products: [
7-
// The product that vends MySourceGenExt to client packages.
7+
// The product that vends MySourceGenBuildToolExt to client packages.
88
.extension(
9-
name: "MySourceGenExt",
10-
targets: ["MySourceGenExt"]
9+
name: "MySourceGenBuildToolExt",
10+
targets: ["MySourceGenBuildToolExt"]
11+
),
12+
.extension(
13+
name: "MySourceGenPrebuildToolExt",
14+
targets: ["MySourceGenPrebuildToolExt"]
1115
),
1216
.executable(
13-
name: "MySourceGenTool",
14-
targets: ["MySourceGenTool"]
17+
name: "MySourceGenBuildTool",
18+
targets: ["MySourceGenBuildTool"]
1519
)
1620
],
1721
targets: [
18-
// A local tool that uses an extension.
22+
// A local tool that uses a build tool extension.
1923
.executableTarget(
2024
name: "MyLocalTool",
2125
dependencies: [
22-
"MySourceGenExt",
26+
"MySourceGenBuildToolExt",
27+
]
28+
),
29+
// A local tool that uses a prebuild extension.
30+
.executableTarget(
31+
name: "MyOtherLocalTool",
32+
dependencies: [
33+
"MySourceGenPrebuildToolExt",
2334
]
2435
),
25-
// The target that implements the extension and generates commands to invoke MySourceGenTool.
36+
// The target that implements the extension and generates build tool commands to invoke MySourceGenTool.
2637
.extension(
27-
name: "MySourceGenExt",
38+
name: "MySourceGenBuildToolExt",
2839
capability: .buildTool(),
2940
dependencies: [
30-
"MySourceGenTool"
41+
"MySourceGenBuildTool"
42+
]
43+
),
44+
// The target that implements the extension and generates prebuild commands to invoke MySourceGenTool.
45+
.extension(
46+
name: "MySourceGenPrebuildToolExt",
47+
capability: .prebuild(),
48+
dependencies: [
3149
]
3250
),
3351
// The command line tool that generates source files.
3452
.executableTarget(
35-
name: "MySourceGenTool",
53+
name: "MySourceGenBuildTool",
3654
dependencies: [
3755
"MySourceGenToolLib",
3856
]
@@ -49,7 +67,8 @@ let package = Package(
4967
.testTarget(
5068
name: "MySourceGenExtTests",
5169
dependencies: [
52-
"MySourceGenExt",
70+
"MySourceGenBuildToolExt",
71+
"MySourceGenPrebuildToolExt",
5372
"MySourceGenRuntimeLib"
5473
]
5574
)

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyLocalTool/main.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
print("Generated string: '\(generatedString)'")
1+
print("Generated string Foo: '\(foo)'")

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyOtherLocalTool/Bar.dat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello Extension Target!

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyOtherLocalTool/Baz.dat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Hello Extension Target!

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MyOtherLocalTool/main.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// print("Generated string Bar: '\(bar)'")
2+
// print("Generated string Baz: '\(baz)'")

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenBuildTool/main.swift

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import Foundation
2+
import MySourceGenToolLib
3+
4+
// Sample source generator tool that emits a Swift variable declaration of a string containing the hex representation of the contents of a file as a quoted string. The variable name is the base name of the input file. The input file is the first argument and the output file is the second.
5+
if ProcessInfo.processInfo.arguments.count != 3 {
6+
print("usage: MySourceGenBuildTool <input> <output>")
7+
exit(1)
8+
}
9+
let inputFile = ProcessInfo.processInfo.arguments[1]
10+
let outputFile = ProcessInfo.processInfo.arguments[2]
11+
12+
let variableName = URL(fileURLWithPath: inputFile).deletingPathExtension().lastPathComponent
13+
14+
let inputData = FileManager.default.contents(atPath: inputFile) ?? Data()
15+
let dataAsHex = inputData.map { String(format: "%02hhx", $0) }.joined()
16+
let outputString = "public var \(variableName) = \(dataAsHex.quotedForSourceCode)\n"
17+
let outputData = outputString.data(using: .utf8)
18+
FileManager.default.createFile(atPath: outputFile, contents: outputData)

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenExt/extension.swift renamed to ‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenBuildToolExt/extension.swift

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import PackageExtension
22

3+
print("build tool extension")
4+
35
for inputPath in targetBuildContext.otherFiles {
46
guard inputPath.suffix == ".dat" else { continue }
57
let outputName = inputPath.basename + ".swift"
@@ -8,7 +10,7 @@ for inputPath in targetBuildContext.otherFiles {
810
displayName:
911
"Generating \(outputName) from \(inputPath.filename)",
1012
executable:
11-
try targetBuildContext.lookupTool(named: "MySourceGenTool"),
13+
try targetBuildContext.lookupTool(named: "MySourceGenBuildTool"),
1214
arguments: [
1315
"\(inputPath)",
1416
"\(outputPath)"

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenPrebuildToolExt/extension.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import PackageExtension
2+
3+
print("prebuild extension")
4+
5+
let outputPaths: [Path] = targetBuildContext.otherFiles.filter{ $0.suffix == ".dat" }.map { path in
6+
targetBuildContext.outputDir.appending(path.basename + ".swift")
7+
}
8+
9+
if !outputPaths.isEmpty {
10+
commandConstructor.createCommand(
11+
displayName:
12+
"Running prebuild command for target \(targetBuildContext.targetName)",
13+
executable:
14+
Path("/usr/bin/touch"),
15+
arguments:
16+
outputPaths.map{ $0.string },
17+
derivedSourcePaths:
18+
outputPaths
19+
)
20+
}

‎Fixtures/Miscellaneous/Extensions/MySourceGenExtension/Sources/MySourceGenTool/main.swift

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎Sources/Build/BuildOperation.swift

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,15 +162,55 @@ public final class BuildOperation: PackageStructureDelegate, SPMBuildCore.BuildS
162162
throw Diagnostics.fatalError
163163
}
164164
}
165+
166+
/// Run any prebuild commands associated with the given list of prebuild results, and return the paths of any created files or directories.
167+
private func runPrebuildCommands(for results: [ExtensionEvaluationResult]) throws -> [AbsolutePath] {
168+
// Run through all the commands from all the extension usages in the target.
169+
var outputPaths: [AbsolutePath] = []
170+
for command in results.reduce([], { $0 + $1.commands }) {
171+
if case .prebuildCommand(let displayName, let executable, let arguments, _, _, let derivedSourceDirPaths) = command {
172+
// Run the prebuild command.
173+
// TODO: We need to also respect any environment and working directory configurations.
174+
let execPath = AbsolutePath(executable, relativeTo: buildParameters.buildPath)
175+
176+
// FIXME: Is this the right way to emit progress here?
177+
stdoutStream.write(displayName)
178+
179+
// Run the command as a subprocess.
180+
// TODO: This should move somewhere else, and we should also be able to run independent prebuild commands concurrently.
181+
let command = [execPath.pathString] + arguments
182+
let result = try Process.popen(arguments: command)
183+
let output = try (result.utf8Output() + result.utf8stderrOutput()).spm_chuzzle() ?? ""
184+
if result.exitStatus != .terminated(code: 0) {
185+
// TODO: Make this a proper error.
186+
throw StringError("failed: \(command)\n\n\(output)")
187+
}
188+
189+
// Keep track of the output paths.
190+
outputPaths.append(contentsOf: derivedSourceDirPaths)
191+
}
192+
}
193+
return outputPaths
194+
}
165195

166196
/// Create the build plan and return the build description.
167197
private func plan() throws -> BuildDescription {
168198
let graph = try getPackageGraph()
169199
let extensionEvaluationResults = try getExtensionEvaluationResults(for: graph)
200+
201+
// Run any prebuild commands. We get here in the case where there was no build description or it needs to change.
202+
var prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]] = [:]
203+
for target in graph.reachableTargets {
204+
if let results = extensionEvaluationResults[target] {
205+
prebuildOutputPaths[target] = try runPrebuildCommands(for: results)
206+
}
207+
}
208+
170209
let plan = try BuildPlan(
171210
buildParameters: buildParameters,
172211
graph: graph,
173212
extensionEvaluationResults: extensionEvaluationResults,
213+
prebuildOutputPaths: prebuildOutputPaths,
174214
diagnostics: diagnostics
175215
)
176216
self.buildPlan = plan

‎Sources/Build/BuildPlan.swift

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -547,12 +547,16 @@ public final class SwiftTargetBuildDescription {
547547

548548
/// The results of having applied any extensions to this target.
549549
public let extensionEvaluationResults: [ExtensionEvaluationResult]
550+
551+
/// The output paths of any prebuild commands that were run for this target.
552+
public let prebuildOutputPaths: [AbsolutePath]
550553

551554
/// Create a new target description with target and build parameters.
552555
init(
553556
target: ResolvedTarget,
554557
buildParameters: BuildParameters,
555558
extensionEvaluationResults: [ExtensionEvaluationResult] = [],
559+
prebuildOutputPaths: [AbsolutePath] = [],
556560
isTestTarget: Bool? = nil,
557561
testDiscoveryTarget: Bool = false,
558562
fs: FileSystem = localFileSystem
@@ -567,6 +571,13 @@ public final class SwiftTargetBuildDescription {
567571
self.tempsPath = buildParameters.buildPath.appending(component: target.c99name + ".build")
568572
self.derivedSources = Sources(paths: [], root: tempsPath.appending(component: "DerivedSources"))
569573
self.extensionEvaluationResults = extensionEvaluationResults
574+
self.prebuildOutputPaths = prebuildOutputPaths
575+
576+
// Add any derived source paths from prebuild commands.
577+
for absPath in self.prebuildOutputPaths {
578+
let relPath = absPath.relative(to: self.derivedSources.root)
579+
self.derivedSources.relativePaths.append(relPath)
580+
}
570581

571582
// Add any derived source paths declared by build-tool extensions that were applied to this target. We do
572583
// this here and not just in the LLBuildManifestBuilder because we need to include them in any situation
@@ -1285,6 +1296,9 @@ public class BuildPlan {
12851296

12861297
/// The results of evaluating any extensions used by targets in this build.
12871298
public let extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]]
1299+
1300+
/// The output paths of any prebuild commands run for targets in this build.
1301+
public let prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]]
12881302

12891303
/// The filesystem to operate on.
12901304
let fileSystem: FileSystem
@@ -1366,12 +1380,14 @@ public class BuildPlan {
13661380
buildParameters: BuildParameters,
13671381
graph: PackageGraph,
13681382
extensionEvaluationResults: [ResolvedTarget: [ExtensionEvaluationResult]] = [:],
1383+
prebuildOutputPaths: [ResolvedTarget: [AbsolutePath]] = [:],
13691384
diagnostics: DiagnosticsEngine,
13701385
fileSystem: FileSystem = localFileSystem
13711386
) throws {
13721387
self.buildParameters = buildParameters
13731388
self.graph = graph
13741389
self.extensionEvaluationResults = extensionEvaluationResults
1390+
self.prebuildOutputPaths = prebuildOutputPaths
13751391
self.diagnostics = diagnostics
13761392
self.fileSystem = fileSystem
13771393

@@ -1397,6 +1413,7 @@ public class BuildPlan {
13971413
target: target,
13981414
buildParameters: buildParameters,
13991415
extensionEvaluationResults: extensionEvaluationResults[target] ?? [],
1416+
prebuildOutputPaths: prebuildOutputPaths[target] ?? [],
14001417
fs: fileSystem))
14011418
case is ClangTarget:
14021419
targetMap[target] = try .clang(ClangTargetBuildDescription(

‎Sources/Build/ManifestBuilder.swift

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,10 @@ extension LLBuildManifestBuilder {
108108
.sorted()
109109
.map { Node.directoryStructure($0) }
110110

111+
// Add the output paths of any prebuilds that were run, so that we redo the plan if they change.
112+
let nodes = plan.prebuildOutputPaths.values.flatMap{ $0 }.sorted().map { Node.file($0) }
113+
inputs.append(contentsOf: nodes)
114+
111115
// FIXME: Need to handle version-specific manifests.
112116
inputs.append(file: package.manifest.path)
113117

‎Sources/Commands/SwiftTool.swift

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -599,21 +599,35 @@ public class SwiftTool {
599599
func evaluateExtensions(graph: PackageGraph) throws -> [ResolvedTarget: [ExtensionEvaluationResult]] {
600600
do {
601601
// Configure the inputs to the extension evaluation.
602-
// FIXME: These paths are still fairly preliminary.
602+
603+
// The `extensions` directory is inside the workspace's main data directory, and contains all temporary
604+
// files related to all extensions in the workspace.
603605
let buildEnvironment = try buildParameters().buildEnvironment
604606
let dataDir = try self.getActiveWorkspace().dataPath
605-
let extensionsDir = dataDir.appending(component: "extensions")
606-
let cacheDir = extensionsDir.appending(component: "cache")
607+
let extDir = dataDir.appending(component: "extensions")
608+
609+
// The `cache` directory is in the extensions directory and is where the extension runner caches compiled
610+
// extension binaries and any other derived information.
611+
let cacheDir = extDir.appending(component: "cache")
607612
let extensionRunner = try DefaultExtensionRunner(cacheDir: cacheDir, manifestResources: self._hostToolchain.get().manifestResources)
608-
let outputDir = extensionsDir.appending(component: "outputs")
609-
// FIXME: Too many assumptions!
613+
614+
// The `outputs` directory contains subdirectories for each combination of package, target, and extension.
615+
// Each usage of an extension has an output directory that is writable by the extension, where it can write
616+
// additional files, and to which it can configure tools to write their outputs, etc.
617+
let outputDir = extDir.appending(component: "outputs")
618+
619+
// The `tools` directory contains any command line tools (executables) that are available for any commands
620+
// defined by the executable.
621+
// FIXME: At the moment we just pass the built products directory for the host. We will need to extend this
622+
// with a map of the names of tools available to each extension. In particular this would not work with any
623+
// binary targets.
610624
let execsDir = dataDir.appending(components: try self._hostToolchain.get().triple.tripleString, buildEnvironment.configuration.dirname)
611-
let diagnostics = DiagnosticsEngine()
612625

613626
// Create the cache directory, if needed.
614627
try localFileSystem.createDirectory(cacheDir, recursive: true)
615628

616629
// Ask the graph to evaluate extensions, and return the result.
630+
let diagnostics = DiagnosticsEngine()
617631
let result = try graph.evaluateExtensions(buildEnvironment: buildEnvironment, execsDir: execsDir, outputDir: outputDir, extensionRunner: extensionRunner, diagnostics: diagnostics, fileSystem: localFileSystem)
618632
return result
619633
}

‎Sources/PackageExtension/PublicAPI/TargetBuildContext.swift

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,6 @@ public final class TargetBuildContext: Decodable {
6363
/// package extension itself may also write to this directory.
6464
public let outputDir: Path
6565

66-
/// A directory into which the package extension or the tool it invokes can
67-
/// write any caches that speed up its operation or any other intermediate
68-
/// files that shouldn't be further processed.
69-
public let cacheDir: Path
70-
7166
/// Looks up and returns the path of a named command line executable tool.
7267
/// The executable must be either in the toolchain or in the system search
7368
/// path for executables, or be provided by an executable target or binary
@@ -77,12 +72,12 @@ public final class TargetBuildContext: Decodable {
7772
public func lookupTool(named name: String) throws -> Path {
7873
// TODO: Rather than just appending the name, this should instead use
7974
// a mapping of tool names to paths (passed in from the context).
80-
return self.execsDir.appending(name)
75+
return self.toolsDir.appending(name)
8176
}
8277

83-
/// A directory in which the built executables available to the extension
84-
/// will be located.
78+
/// A directory in which any built or provided command line tools will be
79+
/// available to the extension.
8580
// TODO: This should instead be a mapping of tool names to paths (passed
86-
/// in from the context).
87-
private let execsDir: Path
81+
// in from the context).
82+
private let toolsDir: Path
8883
}

0 commit comments

Comments
 (0)