[swift_snapshot_tool] Some small fixes and add a run_toolchain command for running a command against a specific toolchain.

run_toolchain is mainly useful for diagnosing issues with ones script without
needing to fully bisect... but it also can just be used independently.

Author: gottesmm

utils/swift_snapshot_tool/Sources/swift_snapshot_tool/bisect_toolchains.swift

@@ -27,15 +27,42 @@ struct BisectToolchains: AsyncParsableCommand {
       """)
   var script: String
 
-  @Option(help: "Oldest tag. Expected to pass")
-  var goodTag: String
+  @Option(help: "Oldest Date. Expected to Pass. We use the first snapshot produced before the given date")
+  var oldDate: String
+
+  var oldDateAsDate: Date {
+    let d = DateFormatter()
+    d.dateFormat = "yyyy-MM-dd"
+    guard let result = d.date(from: oldDate) else {
+      log("Improperly formatted date: \(oldDate)! Expected format: yyyy_MM_dd.")
+      fatalError()
+    }
+    return result
+  }
 
-  @Option(help: "Newest tag. Expected to fail. If not set, use newest snapshot")
-  var badTag: String?
+  @Option(help: """
+    Newest Date. Expected to fail. If not set, use newest snapshot. We use the
+    first snapshot after new date
+    """)
+  var newDate: String?
+
+  var newDateAsDate: Date? {
+    guard let newDate = self.newDate else { return nil }
+    let d = DateFormatter()
+    d.dateFormat = "yyyy-MM-dd"
+    guard let result = d.date(from: newDate) else {
+      log("Improperly formatted date: \(newDate)! Expected format: yyyy_MM_dd.")
+      fatalError()
+    }
+    return result
+  }
 
   @Flag(help: "Invert the test so that we assume the newest succeeds")
   var invert = false
 
+  @Argument(help: "Extra constant arguments to pass to the test")
+  var extraArgs: [String] = []
+
   mutating func run() async throws {
     if !FileManager.default.fileExists(atPath: workspace) {
       do {
@@ -49,20 +76,25 @@ struct BisectToolchains: AsyncParsableCommand {
     }
 
     // Load our tags from swift's github repo
-    let tags = try! await getTagsFromSwiftRepo(branch: branch)
-
-    guard let goodTagIndex = tags.firstIndex(where: { $0.name == self.goodTag }) else {
-      log("Failed to find tag: \(self.goodTag)")
+    let tags = try! await getTagsFromSwiftRepo(branch: branch, dryRun: true)
+
+    // Newest is first. So 0 maps to the newest tag. We do this so someone can
+    // just say 50 toolchains ago. To get a few weeks worth. This is easier than
+    // writing dates a lot.
+    let oldDateAsDate = self.oldDateAsDate
+    guard let goodTagIndex = tags.firstIndex(where: { $0.tag.date(branch: self.branch) < oldDateAsDate }) else {
+      log("Failed to find tag with date: \(oldDateAsDate)")
       fatalError()
     }
 
-    let badTagIndex: Array<Tag>.Index
-    if let badTag = self.badTag {
-      guard let n = tags.firstIndex(where: { $0.name == badTag }) else {
-        log("Failed to find tag: \(badTag)")
+    let badTagIndex: Int
+    if let newDateAsDate = self.newDateAsDate {
+      let b = tags.firstIndex(where: { $0.tag.date(branch: self.branch) < newDateAsDate })
+      guard let b else {
+        log("Failed to find tag newer than date: \(newDateAsDate)")
         fatalError()
       }
-      badTagIndex = n
+      badTagIndex = b
     } else {
       badTagIndex = 0
     }
@@ -73,30 +105,73 @@ struct BisectToolchains: AsyncParsableCommand {
       fatalError()
     }
 
-    log("[INFO] Testing \(totalTags) toolchains")
-
     var startIndex = goodTagIndex
     var endIndex = badTagIndex
-    while startIndex != endIndex && startIndex != (endIndex - 1) {
+
+    // First check if the newest toolchain succeeds. We assume this in our bisection.
+    do {
+      log("Testing that Oldest Tag Succeeds: \(tags[startIndex].tag))")
+      let result = try! await downloadToolchainAndRunTest(
+        platform: platform, tag: tags[startIndex].tag, branch: branch, workspace: workspace, script: script,
+        extraArgs: extraArgs)
+      var success = result == 0
+      if self.invert {
+        success = !success
+      }
+      if !success {
+        log("[INFO] Oldest snapshot fails?! We assume that the oldest snapshot is known good!")
+      } else {
+        log("[INFO] Oldest snapshot passes test. Snapshot: \(tags[startIndex])")
+      }
+    }
+
+    do {
+      log("Testing that Newest Tag Fails: \(tags[endIndex].tag))")
+      let result = try! await downloadToolchainAndRunTest(
+        platform: platform, tag: tags[endIndex].tag, branch: branch, workspace: workspace, script: script,
+        extraArgs: extraArgs)
+      var success = result != 0
+      if self.invert {
+        success = !success
+      }
+      if !success {
+        log("[INFO] Newest snapshot succeceds?! We assume that the newest snapshot is known bad!")
+      } else {
+        log("[INFO] Newest snapshot passes test. Snapshot: \(tags[endIndex])")
+      }
+    }
+
+    log("[INFO] Testing \(totalTags) toolchains")
+    while startIndex != endIndex && startIndex != endIndex {
       let mid = (startIndex + endIndex) / 2
+
+      let midValue = tags[mid].tag
       log(
-        "[INFO] Visiting Mid: \(mid) with (Start, End) = (\(startIndex),\(endIndex)). Tag: \(tags[mid])"
+        "[INFO] Visiting Mid: \(mid) with (Start, End) = (\(startIndex),\(endIndex)). Tag: \(midValue)"
       )
       let result = try! await downloadToolchainAndRunTest(
-        platform: platform, tag: tags[mid], branch: branch, workspace: workspace, script: script)
+        platform: platform, tag: midValue, branch: branch, workspace: workspace, script: script,
+        extraArgs: extraArgs)
 
       var success = result == 0
       if self.invert {
         success = !success
       }
 
+      let midIsEndIndex = mid == endIndex
+
       if success {
         log("[INFO] PASSES! Setting start to mid!")
         startIndex = mid
       } else {
         log("[INFO] FAILS! Setting end to mid")
         endIndex = mid
       }
+
+      if midIsEndIndex {
+        log("Last successful value: \(tags[mid+1])")
+        break
+      }
     }
   }
 }

utils/swift_snapshot_tool/Sources/swift_snapshot_tool/download_toolchain.swift

@@ -55,7 +55,9 @@ private func downloadFile(url: URL, localPath: URL, verboseDownload: Bool = true
   }
 }
 
-private func shell(_ command: String, environment: [String: String] = [:]) -> (
+private func shell(_ command: String, environment: [String: String] = [:],
+                   mustSucceed: Bool = true,verbose: Bool = false,
+                   extraArgs: [String] = []) -> (
   stdout: String, stderr: String, exitCode: Int
 ) {
   let task = Process()
@@ -64,11 +66,15 @@ private func shell(_ command: String, environment: [String: String] = [:]) -> (
 
   task.standardOutput = stdout
   task.standardError = stderr
-  task.arguments = ["-c", command]
+  var newCommand = command
+  if extraArgs.count != 0 {
+    newCommand = newCommand.appending(" ").appending(extraArgs.joined(separator: " "))
+  }
+  task.arguments = ["-c", newCommand]
   if !environment.isEmpty {
     if let e = task.environment {
-      print("Task Env: \(e)")
-      print("Passed in Env: \(environment)")
+      log("Task Env: \(e)")
+      log("Passed in Env: \(environment)")
       task.environment = e.merging(
         environment,
         uniquingKeysWith: {
@@ -78,7 +84,9 @@ private func shell(_ command: String, environment: [String: String] = [:]) -> (
       task.environment = environment
     }
   }
-
+  if verbose {
+    log("Command: \(command)\n")
+  }
   task.launchPath = "/bin/zsh"
   task.standardInput = nil
   task.launch()
@@ -88,20 +96,22 @@ private func shell(_ command: String, environment: [String: String] = [:]) -> (
     decoding: stderr.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)
   let stdoutData = String(
     decoding: stdout.fileHandleForReading.readDataToEndOfFile(), as: UTF8.self)
-  if task.terminationStatus != 0 {
-    log("Command Failed: \(command)")
+  if verbose {
     log("StdErr:\n\(stderrData)\n")
     log("StdOut:\n\(stdoutData)\n")
+  }
+  if mustSucceed && task.terminationStatus != 0 {
+    log("Command Failed!")
     fatalError()
   }
-
   return (stdout: stdoutData, stderr: stderrData, Int(task.terminationStatus))
 }
 
 func downloadToolchainAndRunTest(
   platform: Platform, tag: Tag, branch: Branch,
   workspace: String, script: String,
-  verboseDownload: Bool = true
+  extraArgs: [String],
+  verbose: Bool = false
 ) async throws -> Int {
   let fileType = platform.fileType
   let toolchainType = platform.toolchainType
@@ -117,6 +127,8 @@ func downloadToolchainAndRunTest(
     log("[INFO] Starting Download: \(downloadURL) -> \(downloadPath)")
     try await downloadFile(url: downloadURL, localPath: downloadPath)
     log("[INFO] Finished Download: \(downloadURL) -> \(downloadPath)")
+  } else {
+    log("[INFO] File exists! No need to download! Path: \(downloadPath)")
   }
 
   switch platform {
@@ -126,16 +138,21 @@ func downloadToolchainAndRunTest(
       _ = shell("pkgutil --expand \(downloadPath.path) \(toolchainDir)")
       let payloadPath = "\(toolchainDir)/\(tag.name)-osx-package.pkg/Payload"
       _ = shell("tar -xf \(payloadPath) -C \(toolchainDir)")
-      let swiftcPath = "\(toolchainDir)/usr/bin/swiftc"
-      let swiftFrontendPath = "\(toolchainDir)/usr/bin/swift-frontend"
-      log(shell("\(swiftcPath) --version").stdout)
-      let exitCode = shell(
-        "\(script)", environment: ["SWIFTC": swiftcPath, "SWIFT_FRONTEND": swiftFrontendPath]
-      ).exitCode
-      log("[INFO] Exit code: \(exitCode). Tag: \(tag.name). Script: \(script)")
-      return exitCode
+    } else {
+      log("[INFO] No need to install: \(downloadPath)")
     }
-    return 0
+
+    let swiftcPath = "\(toolchainDir)/usr/bin/swiftc"
+    let swiftFrontendPath = "\(toolchainDir)/usr/bin/swift-frontend"
+    log(shell("\(swiftcPath) --version").stdout)
+    let exitCode = shell(
+      "\(script)", environment: ["SWIFTC": swiftcPath, "SWIFT_FRONTEND": swiftFrontendPath],
+      mustSucceed: false,
+      verbose: verbose,
+      extraArgs: extraArgs
+    ).exitCode
+    log("[INFO] Exit code: \(exitCode). Tag: \(tag.name). Script: \(script)")
+    return exitCode
   case .ubuntu1404, .ubuntu1604, .ubuntu1804:
     fatalError("Unsupported platform: \(platform)")
   }

utils/swift_snapshot_tool/Sources/swift_snapshot_tool/get_tags.swift

@@ -14,33 +14,69 @@ struct Tag: Decodable {
   var name: Substring {
     ref.dropFirst(10)
   }
+
+  func dateString(_ branch: Branch) -> Substring {
+    // FIXME: If we ever actually use interesting a-b builds, we should capture this information
+    // would be better to do it sooner than later.
+    return name.dropFirst("swift-".count + branch.rawValue.count + "-SNAPSHOT-".count).dropLast(2)
+  }
+
+  func date(branch: Branch) -> Date {
+    // TODO: I think that d might be a class... if so, we really want to memoize
+    // this.
+    let d = DateFormatter()
+    d.dateFormat = "yyyy-MM-dd"
+    return d.date(from: String(dateString(branch)))!
+  }
 }
 
+
 extension Tag: CustomDebugStringConvertible {
   var debugDescription: String {
     String(name)
   }
 }
 
-func getTagsFromSwiftRepo(branch: Branch) async throws -> [Tag] {
-  let GITHUB_BASE_URL = "https://api.github.com"
-  let GITHUB_TAG_LIST_URL = URL(string: "\(GITHUB_BASE_URL)/repos/apple/swift/git/refs/tags")!
+/// A pair of a branch and a tag
+struct BranchTag {
+  var tag: Tag
+  var branch: Branch
+}
+
+extension BranchTag: CustomDebugStringConvertible {
+  var debugDescription: String {
+    tag.debugDescription
+  }
+}
+
+func getTagsFromSwiftRepo(branch: Branch, dryRun: Bool = false) async throws -> [BranchTag] {
+  let github_tag_list_url: URL
+  if !dryRun {
+    github_tag_list_url = URL(string: "https://api.github.com/repos/apple/swift/git/refs/tags")!
+  } else {
+    github_tag_list_url = URL(string: "file:///Users/gottesmm/triage/github_data")!
+  }
 
   let decoder = JSONDecoder()
   decoder.keyDecodingStrategy = .convertFromSnakeCase
 
   log("[INFO] Starting to download snapshot information from github.")
-  async let data = URLSession.shared.data(from: GITHUB_TAG_LIST_URL).0
+  async let data = URLSession.shared.data(from: github_tag_list_url).0
   let allTags = try! decoder.decode([Tag].self, from: await data)
   log("[INFO] Finished downloading snapshot information from github.")
 
   let snapshotTagPrefix = "swift-\(branch.rawValue.uppercased())"
 
-  // Then filter the tags to just include the specific snapshot prefix.
-  var filteredTags = allTags.filter {
+  // Then filter the tags to just include the specific snapshot branch
+  // prefix. Add the branch to an aggregate BranchTag.
+  var filteredTags: [BranchTag] = allTags.filter {
     $0.name.starts(with: snapshotTagPrefix)
+  }.map {
+    BranchTag(tag: $0, branch: branch)
   }
-  filteredTags.sort { $0.ref < $1.ref }
+
+  // Then sort so that the newest branch prefix 
+  filteredTags.sort { $0.tag.ref < $1.tag.ref }
   filteredTags.reverse()
 
   return filteredTags