Skip to content

Commit 8e4dd07

Browse files
dschaefer2dschaefer2
authored
Forward cherry pick the prebuilts feature to main. (#8610)
To get the prebuilts feature out in time for 6.1, the bulk of the work was done on the release/6.1 branch. Cherry pick those changes to the main branch to get 6.2 caught up. This is a manual cherry pick so it's going to be a bit hard to follow. I did a diff between main and release/6.1 and copied all the changes I made there.
1 parent 0036a0b commit 8e4dd07

27 files changed

+3305
-947
lines changed

‎Sources/Basics/Concurrency/SendableBox.swift

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,24 @@ import struct Foundation.Date
1616
/// an `async` closure. This type serves as a replacement for `ThreadSafeBox`
1717
/// implemented with Swift Concurrency primitives.
1818
public actor SendableBox<Value: Sendable> {
19-
init(_ value: Value? = nil) {
19+
public init(_ value: Value) {
2020
self.value = value
2121
}
2222

23-
var value: Value?
23+
public var value: Value
24+
25+
public func set(_ value: Value) {
26+
self.value = value
27+
}
2428
}
2529

2630
extension SendableBox where Value == Int {
2731
func increment() {
28-
if let value {
29-
self.value = value + 1
30-
}
32+
self.value = value + 1
3133
}
3234

3335
func decrement() {
34-
if let value {
35-
self.value = value - 1
36-
}
36+
self.value = value - 1
3737
}
3838
}
3939

‎Sources/Basics/FileSystem/FileSystem+Extensions.swift

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -681,3 +681,17 @@ extension FileSystem {
681681
}
682682
}
683683
}
684+
685+
extension FileSystem {
686+
/// Do a deep enumeration, passing each file to block
687+
public func enumerate(directory: AbsolutePath, block: (AbsolutePath) throws -> ()) throws {
688+
for file in try getDirectoryContents(directory) {
689+
let path = directory.appending(file)
690+
if isDirectory(path) {
691+
try enumerate(directory: path, block: block)
692+
} else {
693+
try block(path)
694+
}
695+
}
696+
}
697+
}

‎Sources/Build/BuildPlan/BuildPlan+Test.swift

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import struct PackageGraph.ResolvedProduct
2727
import struct PackageGraph.ResolvedModule
2828

2929
import struct PackageModel.Sources
30+
import enum PackageModel.BuildSettings
3031
import class PackageModel.SwiftModule
3132
import class PackageModel.Module
3233
import struct SPMBuildCore.BuildParameters
@@ -80,17 +81,28 @@ extension BuildPlan {
8081
let discoveryMainFile = discoveryDerivedDir.appending(component: TestDiscoveryTool.mainFileName)
8182

8283
var discoveryPaths: [AbsolutePath] = []
84+
var discoveryBuildSettings: BuildSettings.AssignmentTable = .init()
8385
discoveryPaths.append(discoveryMainFile)
8486
for testTarget in testProduct.modules {
8587
let path = discoveryDerivedDir.appending(components: testTarget.name + ".swift")
8688
discoveryPaths.append(path)
89+
// Add in the include path from the test targets to ensure this module builds
90+
if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] {
91+
for assignment in flags {
92+
let values = assignment.values.filter({ $0.hasPrefix("-I") })
93+
if !values.isEmpty {
94+
discoveryBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS)
95+
}
96+
}
97+
}
8798
}
8899

89100
let discoveryTarget = SwiftModule(
90101
name: discoveryTargetName,
91102
dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) },
92103
packageAccess: true, // test target is allowed access to package decls by default
93-
testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir)
104+
testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir),
105+
buildSettings: discoveryBuildSettings
94106
)
95107
let discoveryResolvedModule = ResolvedModule(
96108
packageIdentity: testProduct.packageIdentity,
@@ -127,13 +139,28 @@ extension BuildPlan {
127139
let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName)
128140
let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir)
129141

142+
var entryPointBuildSettings: BuildSettings.AssignmentTable = .init()
143+
for testTarget in testProduct.modules {
144+
// Add in the include path from the test targets to ensure this module builds
145+
if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] {
146+
for assignment in flags {
147+
let values = assignment.values.filter({ $0.hasPrefix("-I") })
148+
if !values.isEmpty {
149+
entryPointBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS)
150+
}
151+
}
152+
}
153+
}
154+
130155
let entryPointTarget = SwiftModule(
131156
name: testProduct.name,
132157
type: .library,
133158
dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) } + swiftTargetDependencies,
134159
packageAccess: true, // test target is allowed access to package decls
135-
testEntryPointSources: entryPointSources
160+
testEntryPointSources: entryPointSources,
161+
buildSettings: entryPointBuildSettings
136162
)
163+
137164
let entryPointResolvedTarget = ResolvedModule(
138165
packageIdentity: testProduct.packageIdentity,
139166
underlying: entryPointTarget,
@@ -249,7 +276,8 @@ private extension PackageModel.SwiftModule {
249276
type: PackageModel.Module.Kind? = nil,
250277
dependencies: [PackageModel.Module.Dependency],
251278
packageAccess: Bool,
252-
testEntryPointSources sources: Sources
279+
testEntryPointSources sources: Sources,
280+
buildSettings: BuildSettings.AssignmentTable = .init()
253281
) {
254282
self.init(
255283
name: name,
@@ -258,6 +286,7 @@ private extension PackageModel.SwiftModule {
258286
sources: sources,
259287
dependencies: dependencies,
260288
packageAccess: packageAccess,
289+
buildSettings: buildSettings,
261290
usesUnsafeFlags: false
262291
)
263292
}

‎Sources/CoreCommands/Options.swift

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,19 @@ public struct CachingOptions: ParsableArguments {
199199
inversion: .prefixedEnableDisable,
200200
help: "Whether to use prebuilt swift-syntax libraries for macros.")
201201
public var usePrebuilts: Bool = false
202+
203+
/// Hidden option to override the prebuilts download location for testing
204+
@Option(
205+
name: .customLong("experimental-prebuilts-download-url"),
206+
help: .hidden
207+
)
208+
public var prebuiltsDownloadURL: String?
209+
210+
@Option(
211+
name: .customLong("experimental-prebuilts-root-cert"),
212+
help: .hidden
213+
)
214+
public var prebuiltsRootCertPath: String?
202215
}
203216

204217
public struct LoggingOptions: ParsableArguments {

‎Sources/CoreCommands/SwiftCommandState.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,8 @@ public final class SwiftCommandState {
508508
},
509509
manifestImportRestrictions: .none,
510510
usePrebuilts: self.options.caching.usePrebuilts,
511+
prebuiltsDownloadURL: options.caching.prebuiltsDownloadURL,
512+
prebuiltsRootCertPath: options.caching.prebuiltsRootCertPath,
511513
pruneDependencies: self.options.resolver.pruneDependencies,
512514
traitConfiguration: traitConfiguration
513515
),

‎Sources/PackageGraph/ModulesGraph+Loading.swift

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,12 @@ extension ModulesGraph {
223223
)
224224

225225
let rootPackages = resolvedPackages.filter { root.manifests.values.contains($0.manifest) }
226-
checkAllDependenciesAreUsed(packages: resolvedPackages, rootPackages, observabilityScope: observabilityScope)
226+
checkAllDependenciesAreUsed(
227+
packages: resolvedPackages,
228+
rootPackages,
229+
prebuilts: prebuilts,
230+
observabilityScope: observabilityScope
231+
)
227232

228233
return try ModulesGraph(
229234
rootPackages: rootPackages,
@@ -238,6 +243,7 @@ extension ModulesGraph {
238243
private func checkAllDependenciesAreUsed(
239244
packages: IdentifiableSet<ResolvedPackage>,
240245
_ rootPackages: [ResolvedPackage],
246+
prebuilts: [PackageIdentity: [String: PrebuiltLibrary]],
241247
observabilityScope: ObservabilityScope
242248
) {
243249
for package in rootPackages {
@@ -315,9 +321,10 @@ private func checkAllDependenciesAreUsed(
315321
let usedByPackage = productDependencies.contains { $0.name == product.name }
316322
// We check if any of the products of this dependency is guarded by a trait.
317323
let traitGuarded = traitGuardedProductDependencies.contains(product.name)
324+
// Consider prebuilts as used
325+
let prebuilt = prebuilts[dependency.identity]?.keys.contains(product.name) ?? false
318326

319-
// If the product is either used directly or guarded by a trait we consider it as used
320-
return usedByPackage || traitGuarded
327+
return usedByPackage || traitGuarded || prebuilt
321328
}
322329

323330
if !dependencyIsUsed && !observabilityScope.errorsReportedInAnyScope {
@@ -733,6 +740,24 @@ private func createResolvedPackages(
733740

734741
// Establish product dependencies.
735742
for case .product(let productRef, let conditions) in moduleBuilder.module.dependencies {
743+
if let package = productRef.package, prebuilts[.plain(package)]?[productRef.name] != nil {
744+
// See if we're using a prebuilt instead
745+
if moduleBuilder.module.type == .macro {
746+
continue
747+
} else if moduleBuilder.module.type == .test {
748+
// use prebuilt if this is a test that depends a macro target
749+
// these are guaranteed built for host
750+
if moduleBuilder.module.dependencies.contains(where: { dep in
751+
guard let module = dep.module else {
752+
return false
753+
}
754+
return module.type == .macro
755+
}) {
756+
continue
757+
}
758+
}
759+
}
760+
736761
// Find the product in this package's dependency products.
737762
// Look it up by ID if module aliasing is used, otherwise by name.
738763
let product = lookupByProductIDs ? productDependencyMap[productRef.identity] :

‎Sources/PackageLoading/PackageBuilder.swift

Lines changed: 50 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -281,8 +281,8 @@ public struct BinaryArtifact {
281281

282282
/// A structure representing a prebuilt library to be used instead of a source dependency
283283
public struct PrebuiltLibrary {
284-
/// The package reference.
285-
public let packageRef: PackageReference
284+
/// The package identity.
285+
public let identity: PackageIdentity
286286

287287
/// The name of the binary target the artifact corresponds to.
288288
public let libraryName: String
@@ -296,8 +296,8 @@ public struct PrebuiltLibrary {
296296
/// The C modules that need their includes directory added to the include path
297297
public let cModules: [String]
298298

299-
public init(packageRef: PackageReference, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
300-
self.packageRef = packageRef
299+
public init(identity: PackageIdentity, libraryName: String, path: AbsolutePath, products: [String], cModules: [String]) {
300+
self.identity = identity
301301
self.libraryName = libraryName
302302
self.path = path
303303
self.products = products
@@ -1297,31 +1297,34 @@ public final class PackageBuilder {
12971297
table.add(assignment, for: .SWIFT_ACTIVE_COMPILATION_CONDITIONS)
12981298
}
12991299

1300-
// Add in flags for prebuilts
1301-
let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
1302-
guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
1303-
let package = package,
1304-
let prebuilt = prebuilts[.plain(package)]?[name]
1305-
else {
1306-
return
1307-
}
1300+
// Add in flags for prebuilts if the target is a macro or a macro test.
1301+
// Currently we only support prebuilts for macros.
1302+
if target.type == .macro || target.isMacroTest(in: manifest) {
1303+
let prebuiltLibraries: [String: PrebuiltLibrary] = target.dependencies.reduce(into: .init()) {
1304+
guard case let .product(name: name, package: package, moduleAliases: _, condition: _) = $1,
1305+
let package = package,
1306+
let prebuilt = prebuilts[.plain(package)]?[name]
1307+
else {
1308+
return
1309+
}
13081310

1309-
$0[prebuilt.libraryName] = prebuilt
1310-
}
1311+
$0[prebuilt.libraryName] = prebuilt
1312+
}
13111313

1312-
for prebuilt in prebuiltLibraries.values {
1313-
let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
1314-
var ldFlagsAssignment = BuildSettings.Assignment()
1315-
ldFlagsAssignment.values = [lib]
1316-
table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
1314+
for prebuilt in prebuiltLibraries.values {
1315+
let lib = prebuilt.path.appending(components: ["lib", "lib\(prebuilt.libraryName).a"]).pathString
1316+
var ldFlagsAssignment = BuildSettings.Assignment()
1317+
ldFlagsAssignment.values = [lib]
1318+
table.add(ldFlagsAssignment, for: .OTHER_LDFLAGS)
13171319

1318-
var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
1319-
for cModule in prebuilt.cModules {
1320-
includeDirs.append(prebuilt.path.appending(components: "include", cModule))
1320+
var includeDirs: [AbsolutePath] = [prebuilt.path.appending(component: "Modules")]
1321+
for cModule in prebuilt.cModules {
1322+
includeDirs.append(prebuilt.path.appending(components: "include", cModule))
1323+
}
1324+
var includeAssignment = BuildSettings.Assignment()
1325+
includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
1326+
table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
13211327
}
1322-
var includeAssignment = BuildSettings.Assignment()
1323-
includeAssignment.values = includeDirs.map({ "-I\($0.pathString)" })
1324-
table.add(includeAssignment, for: .OTHER_SWIFT_FLAGS)
13251328
}
13261329

13271330
return table
@@ -1908,4 +1911,26 @@ extension TargetDescription {
19081911
fileprivate var usesUnsafeFlags: Bool {
19091912
settings.filter(\.kind.isUnsafeFlags).isEmpty == false
19101913
}
1914+
1915+
fileprivate func isMacroTest(in manifest: Manifest) -> Bool {
1916+
guard self.type == .test else { return false }
1917+
1918+
return self.dependencies.contains(where: {
1919+
let name: String
1920+
switch $0 {
1921+
case .byName(name: let n, condition: _):
1922+
name = n
1923+
case .target(name: let n, condition: _):
1924+
name = n
1925+
default:
1926+
return false
1927+
}
1928+
1929+
guard let target = manifest.targetMap[name] else {
1930+
return false
1931+
}
1932+
1933+
return target.type == .macro
1934+
})
1935+
}
19111936
}

‎Sources/PackageModel/Module/SwiftModule.swift

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ public final class SwiftModule: Module {
2828
[defaultTestEntryPointName, "LinuxMain.swift"]
2929
}
3030

31-
public init(name: String, dependencies: [Module.Dependency], packageAccess: Bool, testDiscoverySrc: Sources) {
31+
public init(
32+
name: String,
33+
dependencies: [Module.Dependency],
34+
packageAccess: Bool,
35+
testDiscoverySrc: Sources,
36+
buildSettings: BuildSettings.AssignmentTable = .init()) {
3237
self.declaredSwiftVersions = []
3338

3439
super.init(
@@ -38,7 +43,7 @@ public final class SwiftModule: Module {
3843
sources: testDiscoverySrc,
3944
dependencies: dependencies,
4045
packageAccess: packageAccess,
41-
buildSettings: .init(),
46+
buildSettings: buildSettings,
4247
buildSettingsDescription: [],
4348
pluginUsages: [],
4449
usesUnsafeFlags: false

‎Sources/Workspace/CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ add_library(Workspace
1414
ManagedArtifact.swift
1515
ManagedDependency.swift
1616
ManagedPrebuilt.swift
17+
ManifestSigning/Base64URL.swift
18+
ManifestSigning/CertificatePolicy.swift
19+
ManifestSigning/embedded_resources.swift
20+
ManifestSigning/ManifestSigning.swift
21+
ManifestSigning/Signature.swift
22+
ManifestSigning/Utilities.swift
23+
ManifestSigning/X509Extensions.swift
1724
PackageContainer/FileSystemPackageContainer.swift
1825
PackageContainer/RegistryPackageContainer.swift
1926
PackageContainer/SourceControlPackageContainer.swift

0 commit comments

Comments
 (0)