Skip to content

Enable building prebuilts on linux hosts without docker. #8269

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 1, 2025
Merged

Enable building prebuilts on linux hosts without docker. #8269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3fc97da
Enable building prebuilts on linux hosts without docker.
dschaefer2 Jan 31, 2025
52d23d8
Add support for Fedora.
dschaefer2 Jan 31, 2025
602209c
Add support for Debian 12.
dschaefer2 Jan 31, 2025
d1d9481
Fix tests broken by hostPlatform change.
dschaefer2 Feb 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
279 changes: 161 additions & 118 deletions Sources/Workspace/Workspace+Prebuilts.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,20 @@ extension Workspace {
case macos_x86_64
case windows_aarch64
case windows_x86_64
// noble is currently missing
case ubuntu_noble_aarch64
case ubuntu_noble_x86_64
case ubuntu_jammy_aarch64
case ubuntu_jammy_x86_64
case ubuntu_focal_aarch64
case ubuntu_focal_x86_64
// bookworm is currently missing
// fedora39 is currently missing
case fedora_39_aarch64
case fedora_39_x86_64
case amazonlinux2_aarch64
case amazonlinux2_x86_64
case rhel_ubi9_aarch64
case rhel_ubi9_x86_64
case debian_12_aarch64
case debian_12_x86_64

public enum Arch: String {
case x86_64
Expand All @@ -108,35 +111,6 @@ extension Workspace {
case windows
case linux
}

public var arch: Arch {
switch self {
case .macos_aarch64, .windows_aarch64,
.ubuntu_jammy_aarch64, .ubuntu_focal_aarch64,
.amazonlinux2_aarch64,
.rhel_ubi9_aarch64:
return .aarch64
case .macos_x86_64, .windows_x86_64,
.ubuntu_jammy_x86_64, .ubuntu_focal_x86_64,
.amazonlinux2_x86_64,
.rhel_ubi9_x86_64:
return .x86_64
}
}

public var os: OS {
switch self {
case .macos_aarch64, .macos_x86_64:
return .macos
case .windows_aarch64, .windows_x86_64:
return .windows
case .ubuntu_jammy_aarch64, .ubuntu_jammy_x86_64,
.ubuntu_focal_aarch64, .ubuntu_focal_x86_64,
.amazonlinux2_aarch64, .amazonlinux2_x86_64,
.rhel_ubi9_aarch64, .rhel_ubi9_x86_64:
return .linux
}
}
}
}

Expand All @@ -145,15 +119,18 @@ extension Workspace {
let httpClient: HTTPClient?
let archiver: Archiver?
let useCache: Bool?
let hostPlatform: PrebuiltsManifest.Platform?

public init(
httpClient: HTTPClient? = .none,
archiver: Archiver? = .none,
useCache: Bool? = .none
useCache: Bool? = .none,
hostPlatform: PrebuiltsManifest.Platform? = nil
) {
self.httpClient = httpClient
self.archiver = archiver
self.useCache = useCache
self.hostPlatform = hostPlatform
}
}

Expand All @@ -170,9 +147,11 @@ extension Workspace {
private let delegate: Delegate?
private let hashAlgorithm: HashAlgorithm = SHA256()
private let prebuiltsDownloadURL: URL
let hostPlatform: PrebuiltsManifest.Platform

init(
fileSystem: FileSystem,
hostPlatform: PrebuiltsManifest.Platform,
authorizationProvider: AuthorizationProvider?,
scratchPath: AbsolutePath,
cachePath: AbsolutePath?,
Expand All @@ -182,6 +161,7 @@ extension Workspace {
prebuiltsDownloadURL: String?
) {
self.fileSystem = fileSystem
self.hostPlatform = hostPlatform
self.authorizationProvider = authorizationProvider
self.httpClient = customHTTPClient ?? HTTPClient()
self.archiver = customArchiver ?? ZipArchiver(fileSystem: fileSystem)
Expand Down Expand Up @@ -515,7 +495,7 @@ extension Workspace {
addedOrUpdatedPackages: [PackageReference],
observabilityScope: ObservabilityScope
) async throws {
guard let prebuiltsManager = self.prebuiltsManager else {
guard let prebuiltsManager else {
// Disabled
return
}
Expand All @@ -536,14 +516,10 @@ extension Workspace {
continue
}

let hostPlatform = hostPrebuiltsPlatform
let hostPlatform = prebuiltsManager.hostPlatform

for library in prebuiltManifest.libraries {
for artifact in library.artifacts {
guard artifact.platform == hostPlatform else {
continue
}

for artifact in library.artifacts where artifact.platform == hostPlatform {
if let path = try await prebuiltsManager
.downloadPrebuilt(
package: prebuilt,
Expand Down Expand Up @@ -577,100 +553,167 @@ extension Workspace {

try self.state.save()
}
}

extension Workspace.PrebuiltsManifest.Platform {
public var arch: Arch {
switch self {
case .macos_aarch64, .windows_aarch64,
.ubuntu_noble_aarch64, .ubuntu_jammy_aarch64, .ubuntu_focal_aarch64,
.fedora_39_aarch64,
.amazonlinux2_aarch64,
.rhel_ubi9_aarch64,
.debian_12_aarch64:
return .aarch64
case .macos_x86_64, .windows_x86_64,
.ubuntu_noble_x86_64, .ubuntu_jammy_x86_64, .ubuntu_focal_x86_64,
.fedora_39_x86_64,
.amazonlinux2_x86_64,
.rhel_ubi9_x86_64,
.debian_12_x86_64:
return .x86_64
}
}

public var os: OS {
switch self {
case .macos_aarch64, .macos_x86_64:
return .macos
case .windows_aarch64, .windows_x86_64:
return .windows
case .ubuntu_noble_aarch64, .ubuntu_noble_x86_64,
.ubuntu_jammy_aarch64, .ubuntu_jammy_x86_64,
.ubuntu_focal_aarch64, .ubuntu_focal_x86_64,
.fedora_39_aarch64, .fedora_39_x86_64,
.amazonlinux2_aarch64, .amazonlinux2_x86_64,
.rhel_ubi9_aarch64, .rhel_ubi9_x86_64,
.debian_12_aarch64, .debian_12_x86_64:
return .linux
}
}

var hostPrebuiltsPlatform: PrebuiltsManifest.Platform? {
if self.hostToolchain.targetTriple.isDarwin() {
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .macos_aarch64
case .x86_64:
return .macos_x86_64
/// Determine host platform based on compilation target
public static var hostPlatform: Self? {
let arch: Arch?
#if arch(arm64)
arch = .aarch64
#elseif arch(x86_64)
arch = .x86_64
#endif
guard let arch else {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dschaefer2, this recent change broke building SwiftPM 6.1 for all 32-bit platforms, including Android armv7 on my Android CI, which I just had to patch to get building again. Your change didn't affect trunk, only because this pull hasn't been merged there.

I don't expect you to add any functionality to download these prebuilt libraries for armv7, but it would be good to add a change like mine to simply keep SwiftPM building on armv7, both because I distribute an Android armv7 toolchain myself and there is a recent contributor who wants and is working to distribute a linux armv7 toolchain in the future.

If you'd like me to just submit a pull with my linked patch, let me know and I will.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry about that. I'm about to push another one so I'll make sure all of these have an #else condition.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to be sorry, 32-bit toolchains are a tiny niche in Swift now. Just ping me on any fix and I will easily cross-compile it for armv7 and let you know if it works.

return nil
}

#if os(macOS)
switch arch {
case .aarch64:
return .macos_aarch64
case .x86_64:
return .macos_x86_64
}
#elseif os(Windows)
switch arch {
case .aarch64:
return .windows_aarch64
case .x86_64:
return .windows_x86_64
}
#elseif os(Linux)
// Load up the os-release file into a dictionary
guard let osData = try? String(contentsOfFile: "/etc/os-release", encoding: .utf8)
else {
return nil
}
let osLines = osData.split(separator: "\n")
let osDict = osLines.reduce(into: [Substring: String]()) {
(dict, line) in
let parts = line.split(separator: "=", maxSplits: 2)
dict[parts[0]] = parts[1...].joined(separator: "=").trimmingCharacters(in: ["\""])
}

switch osDict["ID"] {
case "ubuntu":
switch osDict["VERSION_CODENAME"] {
case "noble":
switch arch {
case .aarch64:
return .ubuntu_noble_aarch64
case .x86_64:
return .ubuntu_noble_x86_64
}
case "jammy":
switch arch {
case .aarch64:
return .ubuntu_jammy_aarch64
case .x86_64:
return .ubuntu_jammy_x86_64
}
case "focal":
switch arch {
case .aarch64:
return .ubuntu_focal_aarch64
case .x86_64:
return .ubuntu_focal_x86_64
}
default:
return nil
}
} else if self.hostToolchain.targetTriple.isWindows() {
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .windows_aarch64
case .x86_64:
return .windows_x86_64
case "fedora":
switch osDict["VERSION_ID"] {
case "39", "41":
switch arch {
case .aarch64:
return .fedora_39_aarch64
case .x86_64:
return .fedora_39_x86_64
}
default:
return nil
}
} else if self.hostToolchain.targetTriple.isLinux() {
// Load up the os-release file into a dictionary
guard let osData = try? String(contentsOfFile: "/etc/os-release", encoding: .utf8)
else {
case "amzn":
switch osDict["VERSION_ID"] {
case "2":
switch arch {
case .aarch64:
return .amazonlinux2_aarch64
case .x86_64:
return .amazonlinux2_x86_64
}
default:
return nil
}
let osLines = osData.split(separator: "\n")
let osDict = osLines.reduce(into: [Substring: String]()) {
(dict, line) in
let parts = line.split(separator: "=", maxSplits: 2)
dict[parts[0]] = parts[1...].joined(separator: "=").trimmingCharacters(in: ["\""])
case "rhel":
guard let version = osDict["VERSION_ID"] else {
return nil
}

switch osDict["ID"] {
case "ubuntu":
switch osDict["VERSION_CODENAME"] {
case "jammy":
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .ubuntu_jammy_aarch64
case .x86_64:
return .ubuntu_jammy_x86_64
default:
return nil
}
case "focal":
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .ubuntu_focal_aarch64
case .x86_64:
return .ubuntu_focal_x86_64
default:
return nil
}
default:
return nil
switch version.split(separator: ".")[0] {
case "9":
switch arch {
case .aarch64:
return .rhel_ubi9_aarch64
case .x86_64:
return .rhel_ubi9_x86_64
}
case "amzn":
switch osDict["VERSION_ID"] {
case "2":
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .amazonlinux2_aarch64
case .x86_64:
return .amazonlinux2_x86_64
default:
return nil
}
default:
return nil
}
case "rhel":
guard let version = osDict["VERSION_ID"] else {
return nil
}
switch version.split(separator: ".")[0] {
case "9":
switch self.hostToolchain.targetTriple.arch {
case .aarch64:
return .rhel_ubi9_aarch64
case .x86_64:
return .rhel_ubi9_x86_64
default:
return nil
}
default:
return nil
default:
return nil
}
case "debian":
switch osDict["VERSION_ID"] {
case "12":
switch arch {
case .aarch64:
return .debian_12_aarch64
case .x86_64:
return .debian_12_x86_64
}
default:
return nil
}
} else {
default:
return nil
}
#else
return nil
#endif
}

}
Loading