Skip to content

Commit b0b5b2a

Browse files
gottesmmgottesmm
authored
Merge pull request #76292 from gottesmm/pr-16c5ea0901cf1d5a352b7e8dd0c3f9b6f644bed3
[utils] Add a small swift tool called swift_snapshot_tool for working with swift.org snapshots.
2 parents 702474f + efc87d9 commit b0b5b2a

File tree

9 files changed

+410
-0
lines changed

9 files changed

+410
-0
lines changed

‎utils/swift_snapshot_tool/Package.swift

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// swift-tools-version: 5.10.0
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
6+
let package = Package(
7+
name: "swift_snapshot_tool",
8+
platforms: [.macOS(.v12)],
9+
products: [
10+
// Products define the executables and libraries a package produces, making them visible to other packages.
11+
.executable(
12+
name: "swift_snapshot_tool",
13+
targets: ["swift_snapshot_tool"]),
14+
],
15+
dependencies: [
16+
// other dependencies
17+
.package(url: "https://github.com/apple/swift-argument-parser", from: "1.3.0"),
18+
],
19+
targets: [
20+
// Targets are the basic building blocks of a package, defining a module or a test suite.
21+
// Targets can depend on other targets in this package and products from dependencies.
22+
.executableTarget(
23+
name: "swift_snapshot_tool",
24+
dependencies: [.product(name: "ArgumentParser", package: "swift-argument-parser")]),
25+
.testTarget(
26+
name: "swift_snapshot_toolTests",
27+
dependencies: ["swift_snapshot_tool"]
28+
),
29+
]
30+
)

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/bisect_toolchains.swift

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
import ArgumentParser
2+
import Foundation
3+
4+
struct BisectToolchains: AsyncParsableCommand {
5+
static let configuration = CommandConfiguration(
6+
commandName: "bisect",
7+
discussion: """
8+
Bisects on exit status of attached script. Passes in name of swift as the
9+
environment variabless SWIFTC and SWIFT_FRONTEND
10+
""")
11+
12+
@Flag var platform: Platform = .osx
13+
14+
@Flag(help: "The specific branch of toolchains we should download")
15+
var branch: Branch = .development
16+
17+
@Option(
18+
help: """
19+
The directory where toolchains should be downloaded to.
20+
""")
21+
var workspace: String = "/tmp/workspace"
22+
23+
@Option(
24+
help: """
25+
The script that should be run. The environment variable
26+
SWIFT_EXEC is used by the script to know where swift-frontend is
27+
""")
28+
var script: String
29+
30+
@Option(help: "Oldest tag. Expected to pass")
31+
var goodTag: String
32+
33+
@Option(help: "Newest tag. Expected to fail. If not set, use newest snapshot")
34+
var badTag: String?
35+
36+
@Flag(help: "Invert the test so that we assume the newest succeeds")
37+
var invert = false
38+
39+
mutating func run() async throws {
40+
if !FileManager.default.fileExists(atPath: workspace) {
41+
do {
42+
log("[INFO] Creating workspace: \(workspace)")
43+
try FileManager.default.createDirectory(
44+
atPath: workspace,
45+
withIntermediateDirectories: true, attributes: nil)
46+
} catch {
47+
log(error.localizedDescription)
48+
}
49+
}
50+
51+
// Load our tags from swift's github repo
52+
let tags = try! await getTagsFromSwiftRepo(branch: branch)
53+
54+
guard let goodTagIndex = tags.firstIndex(where: { $0.name == self.goodTag }) else {
55+
log("Failed to find tag: \(self.goodTag)")
56+
fatalError()
57+
}
58+
59+
let badTagIndex: Array<Tag>.Index
60+
if let badTag = self.badTag {
61+
guard let n = tags.firstIndex(where: { $0.name == badTag }) else {
62+
log("Failed to find tag: \(badTag)")
63+
fatalError()
64+
}
65+
badTagIndex = n
66+
} else {
67+
badTagIndex = 0
68+
}
69+
70+
let totalTags = goodTagIndex - badTagIndex
71+
if totalTags < 0 {
72+
log("Good tag is newer than bad tag... good tag expected to be older than bad tag")
73+
fatalError()
74+
}
75+
76+
log("[INFO] Testing \(totalTags) toolchains")
77+
78+
var startIndex = goodTagIndex
79+
var endIndex = badTagIndex
80+
while startIndex != endIndex && startIndex != (endIndex - 1) {
81+
let mid = (startIndex + endIndex) / 2
82+
log(
83+
"[INFO] Visiting Mid: \(mid) with (Start, End) = (\(startIndex),\(endIndex)). Tag: \(tags[mid])"
84+
)
85+
let result = try! await downloadToolchainAndRunTest(
86+
platform: platform, tag: tags[mid], branch: branch, workspace: workspace, script: script)
87+
88+
var success = result == 0
89+
if self.invert {
90+
success = !success
91+
}
92+
93+
if success {
94+
log("[INFO] PASSES! Setting start to mid!")
95+
startIndex = mid
96+
} else {
97+
log("[INFO] FAILS! Setting end to mid")
98+
endIndex = mid
99+
}
100+
}
101+
}
102+
}

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/download_toolchain.swift

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import Foundation
2+
3+
private let SWIFT_BASE_URL = "https://swift.org/builds"
4+
5+
private func fileExists(_ path: String) -> Bool {
6+
FileManager.default.fileExists(atPath: path)
7+
}
8+
9+
enum DownloadFileError: Error {
10+
case failedToGetURL
11+
}
12+
13+
private func downloadFile(url: URL, localPath: URL, verboseDownload: Bool = true) async throws {
14+
let config = URLSessionConfiguration.`default`
15+
let session = URLSession(configuration: config)
16+
return try await withCheckedThrowingContinuation { continuation in
17+
let task = session.downloadTask(with: url) {
18+
urlOrNil, responseOrNil, errorOrNil in
19+
// check for and handle errors:
20+
// * errorOrNil should be nil
21+
// * responseOrNil should be an HTTPURLResponse with statusCode in 200..<299
22+
23+
guard let fileURL = urlOrNil else {
24+
continuation.resume(throwing: DownloadFileError.failedToGetURL)
25+
return
26+
}
27+
28+
do {
29+
// Remove it if it is already there. Swallow the error if it is not
30+
// there.
31+
try FileManager.default.removeItem(at: localPath)
32+
} catch {}
33+
34+
do {
35+
// Then move it... not swalling any errors.
36+
try FileManager.default.moveItem(at: fileURL, to: localPath)
37+
} catch let e {
38+
continuation.resume(throwing: e)
39+
return
40+
}
41+
42+
continuation.resume()
43+
}
44+
45+
task.resume()
46+
while task.state != URLSessionTask.State.completed {
47+
sleep(1)
48+
if verboseDownload {
49+
let percent = String(
50+
format: "%.2f",
51+
Float(task.countOfBytesReceived) / Float(task.countOfBytesExpectedToReceive) * 100)
52+
log("\(percent)%. Bytes \(task.countOfBytesReceived)/\(task.countOfBytesExpectedToReceive)")
53+
}
54+
}
55+
}
56+
}
57+
58+
private func shell(_ command: String, environment: [String: String] = [:]) -> (
59+
stdout: String, stderr: String, exitCode: Int
60+
) {
61+
let task = Process()
62+
let stdout = Pipe()
63+
let stderr = Pipe()
64+
65+
task.standardOutput = stdout
66+
task.standardError = stderr
67+
task.arguments = ["-c", command]
68+
if !environment.isEmpty {
69+
if let e = task.environment {
70+
print("Task Env: \(e)")
71+
print("Passed in Env: \(environment)")
72+
task.environment = e.merging(
73+
environment,
74+
uniquingKeysWith: {
75+
(current, _) in current
76+
})
77+
} else {
78+
task.environment = environment
79+
}
80+
}
81+
82+
task.launchPath = "/bin/zsh"
83+
task.standardInput = nil
84+
task.launch()
85+
task.waitUntilExit()
86+
87+
let stderrData = String(
88+
decoding: stderr.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)
89+
let stdoutData = String(
90+
decoding: stdout.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)
91+
if task.terminationStatus != 0 {
92+
log("Command Failed: \(command)")
93+
log("StdErr:\n\(stderrData)\n")
94+
log("StdOut:\n\(stdoutData)\n")
95+
fatalError()
96+
}
97+
98+
return (stdout: stdoutData, stderr: stderrData, Int(task.terminationStatus))
99+
}
100+
101+
func downloadToolchainAndRunTest(
102+
platform: Platform, tag: Tag, branch: Branch,
103+
workspace: String, script: String,
104+
verboseDownload: Bool = true
105+
) async throws -> Int {
106+
let fileType = platform.fileType
107+
let toolchainType = platform.toolchainType
108+
109+
let realBranch = branch != .development ? "swift-\(branch)-branch" : branch.rawValue
110+
let toolchainDir = "\(workspace)/\(tag.name)-\(platform)"
111+
let downloadPath = URL(fileURLWithPath: "\(workspace)/\(tag.name)-\(platform).\(fileType)")
112+
if !fileExists(toolchainDir) {
113+
let downloadURL = URL(
114+
string:
115+
"\(SWIFT_BASE_URL)/\(realBranch)/\(toolchainType)/\(tag.name)/\(tag.name)-\(platform).\(fileType)"
116+
)!
117+
log("[INFO] Starting Download: \(downloadURL) -> \(downloadPath)")
118+
try await downloadFile(url: downloadURL, localPath: downloadPath)
119+
log("[INFO] Finished Download: \(downloadURL) -> \(downloadPath)")
120+
}
121+
122+
switch platform {
123+
case .osx:
124+
if !fileExists(toolchainDir) {
125+
log("[INFO] Installing: \(downloadPath)")
126+
_ = shell("pkgutil --expand \(downloadPath.path) \(toolchainDir)")
127+
let payloadPath = "\(toolchainDir)/\(tag.name)-osx-package.pkg/Payload"
128+
_ = shell("tar -xf \(payloadPath) -C \(toolchainDir)")
129+
let swiftcPath = "\(toolchainDir)/usr/bin/swiftc"
130+
let swiftFrontendPath = "\(toolchainDir)/usr/bin/swift-frontend"
131+
log(shell("\(swiftcPath) --version").stdout)
132+
let exitCode = shell(
133+
"\(script)", environment: ["SWIFTC": swiftcPath, "SWIFT_FRONTEND": swiftFrontendPath]
134+
).exitCode
135+
log("[INFO] Exit code: \(exitCode). Tag: \(tag.name). Script: \(script)")
136+
return exitCode
137+
}
138+
return 0
139+
case .ubuntu1404, .ubuntu1604, .ubuntu1804:
140+
fatalError("Unsupported platform: \(platform)")
141+
}
142+
}

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/get_tags.swift

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import Foundation
2+
3+
struct CommitInfo: Decodable {
4+
let sha: String
5+
let type: String
6+
let url: String
7+
}
8+
9+
struct Tag: Decodable {
10+
let ref: String
11+
let nodeId: String
12+
let object: CommitInfo
13+
let url: String
14+
var name: Substring {
15+
ref.dropFirst(10)
16+
}
17+
}
18+
19+
extension Tag: CustomDebugStringConvertible {
20+
var debugDescription: String {
21+
String(name)
22+
}
23+
}
24+
25+
func getTagsFromSwiftRepo(branch: Branch) async throws -> [Tag] {
26+
let GITHUB_BASE_URL = "https://api.github.com"
27+
let GITHUB_TAG_LIST_URL = URL(string: "\(GITHUB_BASE_URL)/repos/apple/swift/git/refs/tags")!
28+
29+
let decoder = JSONDecoder()
30+
decoder.keyDecodingStrategy = .convertFromSnakeCase
31+
32+
log("[INFO] Starting to download snapshot information from github.")
33+
async let data = URLSession.shared.data(from: GITHUB_TAG_LIST_URL).0
34+
let allTags = try! decoder.decode([Tag].self, from: await data)
35+
log("[INFO] Finished downloading snapshot information from github.")
36+
37+
let snapshotTagPrefix = "swift-\(branch.rawValue.uppercased())"
38+
39+
// Then filter the tags to just include the specific snapshot prefix.
40+
var filteredTags = allTags.filter {
41+
$0.name.starts(with: snapshotTagPrefix)
42+
}
43+
filteredTags.sort { $0.ref < $1.ref }
44+
filteredTags.reverse()
45+
46+
return filteredTags
47+
}

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/github_repository.swift

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import ArgumentParser
2+
3+
enum Platform: String, EnumerableFlag {
4+
case osx
5+
case ubuntu1404
6+
case ubuntu1604
7+
case ubuntu1804
8+
9+
var fileType: String {
10+
switch self {
11+
case .osx:
12+
return "pkg"
13+
case .ubuntu1404,
14+
.ubuntu1604,
15+
.ubuntu1804:
16+
return "tar.gz"
17+
}
18+
}
19+
20+
var toolchainType: String {
21+
switch self {
22+
case .osx:
23+
return "xcode"
24+
case .ubuntu1404,
25+
.ubuntu1604,
26+
.ubuntu1804:
27+
return self.rawValue
28+
}
29+
}
30+
}
31+
32+
enum Branch: String, EnumerableFlag {
33+
case development
34+
case release50 = "5.0"
35+
case release60 = "6.0"
36+
}

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/list_snapshots.swift

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import ArgumentParser
2+
3+
struct ListSnapshots: AsyncParsableCommand {
4+
static let configuration = CommandConfiguration(
5+
commandName: "list")
6+
7+
@Flag var platform: Platform = .osx
8+
9+
@Flag(help: "The specific branch of toolchains we should download")
10+
var branch: Branch = .development
11+
12+
mutating func run() async throws {
13+
// Load our tags from swift's github repo
14+
let tags = try! await getTagsFromSwiftRepo(branch: branch)
15+
for t in tags.enumerated() {
16+
print(t.0, t.1)
17+
}
18+
}
19+
}

‎utils/swift_snapshot_tool/Sources/swift_snapshot_tool/logging.swift

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import Foundation
2+
3+
func log(_ msg: String) {
4+
let msgWithSpace = "\(msg)\n"
5+
msgWithSpace.data(using: .utf8)
6+
.map(FileHandle.standardError.write)
7+
}

0 commit comments

Comments
 (0)