Merge branch 'master' into where-are-the-tools

Author: JhonnyBillM

README.md

@@ -136,7 +136,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Parseable output, as used by SwiftPM
   * [x] Response files
   * [ ] Input and primary input file lists
-  * [ ] Complete `OutputFileMap` implementation to handle all file types uniformly
+  * [x] Complete `OutputFileMap` implementation to handle all file types uniformly
 * Testing
   * [ ] Build stuff with SwiftPM or Xcode or your favorite build system, using `swift-driver`. Were the results identical? What changed?
   * [x] Shim in `swift-driver` so it can run the Swift repository's [driver test suite](https://github.com/apple/swift/tree/master/test/Driver).

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -100,24 +100,48 @@ fileprivate struct OutputFileMapJSON: Codable {
   /// The data associated with an input file.
   /// \c fileprivate so that the \c store method above can see it
   fileprivate struct Entry: Codable {
-    enum CodingKeys: String, CodingKey {
-      case dependencies
-      case object
-      case swiftmodule
-      case swiftinterface
-      case swiftDependencies = "swift-dependencies"
-      case diagnostics
+
+    private struct CodingKeys: CodingKey {
+
+      let fileType: FileType
+
+      init(fileType: FileType) {
+        self.fileType = fileType
+      }
+
+      init?(stringValue: String) {
+        guard let fileType = FileType(name: stringValue) else { return nil }
+        self.fileType = fileType
+      }
+
+      var stringValue: String { fileType.name }
+      var intValue: Int? { nil }
+      init?(intValue: Int) { nil }
     }
 
-    let dependencies: String?
-    let object: String?
-    let swiftmodule: String?
-    let swiftinterface: String?
-    let swiftDependencies: String?
-    let diagnostics: String?
+    let paths: [FileType: String]
+
+    fileprivate init(paths: [FileType: String]) {
+      self.paths = paths
+    }
+
+    init(from decoder: Decoder) throws {
+      let container = try decoder.container(keyedBy: CodingKeys.self)
+
+      paths = try Dictionary(uniqueKeysWithValues:
+        container.allKeys.map { key in (key.fileType, try container.decode(String.self, forKey: key)) }
+      )
+    }
+
+    func encode(to encoder: Encoder) throws {
+
+      var container = encoder.container(keyedBy: CodingKeys.self)
+
+      try paths.forEach { fileType, path in try container.encode(path, forKey: CodingKeys(fileType: fileType)) }
+    }
   }
 
-  /// The parsed entires
+  /// The parsed entries
   /// \c fileprivate so that the \c store method above can see it
   fileprivate let entries: [String: Entry]
 
@@ -132,23 +156,9 @@ fileprivate struct OutputFileMapJSON: Codable {
 
   /// Converts into virtual path entries.
   func toVirtualOutputFileMap() throws -> [VirtualPath : [FileType : VirtualPath]] {
-    var result: [VirtualPath : [FileType : VirtualPath]] = [:]
-
-    for (input, entry) in entries {
-      let input = try VirtualPath(path: input)
-
-      var map: [FileType: String] = [:]
-      map[.dependencies] = entry.dependencies
-      map[.object] = entry.object
-      map[.swiftModule] = entry.swiftmodule
-      map[.swiftInterface] = entry.swiftinterface
-      map[.swiftDeps] = entry.swiftDependencies
-      map[.diagnostics] = entry.diagnostics
-
-      result[input] = try map.mapValues(VirtualPath.init(path:))
-    }
-
-    return result
+    Dictionary(uniqueKeysWithValues: try entries.map { input, entry in
+      (try VirtualPath(path: input), try entry.paths.mapValues(VirtualPath.init(path:)))
+    })
   }
 
   /// Converts from virtual path entries
@@ -161,13 +171,7 @@ fileprivate struct OutputFileMapJSON: Codable {
       return (fixedIfMaster, convert(outputs: entry.value))
     }
     func convert(outputs: [FileType: VirtualPath]) -> Entry {
-      Entry(
-        dependencies: outputs[.dependencies]?.name,
-        object: outputs[.object]?.name,
-        swiftmodule: outputs[.swiftModule]?.name,
-        swiftinterface: outputs[.swiftInterface]?.name,
-        swiftDependencies: outputs[.swiftDeps]?.name,
-        diagnostics: outputs[.diagnostics]?.name)
+      Entry(paths: outputs.mapValues({ $0.name }))
     }
     return Self(entries: Dictionary(uniqueKeysWithValues: entries.map(convert(entry:))))
   }

Sources/SwiftDriver/Jobs/CompileJob.swift

@@ -349,7 +349,7 @@ extension FileType {
 
     case .swift, .dSYM, .autolink, .dependencies, .swiftDocumentation, .pcm,
          .diagnostics, .objcHeader, .image, .swiftDeps, .moduleTrace, .tbd,
-         .optimizationRecord,.swiftInterface:
+         .optimizationRecord, .swiftInterface, .swiftSourceInfoFile:
       fatalError("Output type can never be a primary output")
     }
   }

Sources/SwiftDriver/Toolchains/DarwinToolchain.swift

@@ -11,10 +11,6 @@
 //===----------------------------------------------------------------------===//
 import TSCBasic
 
-fileprivate func envVarName(forExecutable toolName: String) -> String {
-  return "SWIFT_DRIVER_\(toolName.uppercased())_EXEC"
-}
-
 /// Toolchain for Darwin-based platforms, such as macOS and iOS.
 ///
 /// FIXME: This class is not thread-safe.
@@ -119,31 +115,4 @@ public final class DarwinToolchain: Toolchain {
     \(isShared ? "_dynamic.dylib" : ".a")
     """
   }
-  
-  /// Looks for the executable in the `SWIFT_DRIVER_TOOLNAME_EXEC` environment variable, if found nothing,
-  /// looks in the executable path; finally, fallback to xcrunFind.
-  /// - Parameter exec: executable to look for [i.e. `swift`].
-  func lookup(executable: String) throws -> AbsolutePath {
-    if let overrideString = env[envVarName(forExecutable: executable)] {
-      return try AbsolutePath(validating: overrideString)
-    } else if let path = lookupExecutablePath(filename: executable, searchPaths: [executableDir]) {
-      return path
-    }
-    return try xcrunFind(executable: executable)
-  }
-  
-  private func xcrunFind(executable: String) throws -> AbsolutePath {
-    let xcrun = "xcrun"
-    guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {
-      // This is a hack so our tests work on linux. We need a better way for looking up tools in general.
-      // TODO: remove this hack and emit an error once we have a generalized Toolchain routine for finding Tools.
-      return AbsolutePath("/usr/bin/" + executable)
-    }
-    
-    let path = try Process.checkNonZeroExit(
-      arguments: [xcrun, "-sdk", "macosx", "--find", executable],
-      environment: env
-    ).spm_chomp()
-    return AbsolutePath(path)
-  }
 }

Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift

@@ -13,35 +13,10 @@ import TSCBasic
 
 /// Toolchain for Unix-like systems.
 public final class GenericUnixToolchain: Toolchain {
-  enum Error: Swift.Error {
-    case unableToFind(tool: String)
-  }
-  
   public let env: [String: String]
 
-  private let searchPaths: [AbsolutePath]
-
   public init(env: [String: String]) {
     self.env = env
-    self.searchPaths = getEnvSearchPaths(pathString: env["PATH"], currentWorkingDirectory: localFileSystem.currentWorkingDirectory)
-  }
-
-  /// Looks in the `executablePath`, if found nothing, looks in the environment search paths.
-  /// - Parameter executable: executable to look for [i.e. `swift`].
-  private func lookup(executable: String) throws -> AbsolutePath {
-    if let path = lookupExecutablePath(filename: executable, searchPaths: [executableDir]) {
-      return path
-    } else if let path = lookupExecutablePath(filename: executable, searchPaths: searchPaths) {
-      return path
-    }
-
-    // If we happen to be on a macOS host, some tools might not be in our
-    // PATH, so we'll just use xcrun to find them too.
-    #if os(macOS)
-    return try DarwinToolchain(env: self.env).lookup(executable: executable)
-    #else
-    throw Error.unableToFind(tool: executable)
-    #endif
   }
 
   public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String {

Sources/SwiftDriver/Toolchains/Toolchain.swift

@@ -28,6 +28,8 @@ public protocol Toolchain {
 
   var env: [String: String] { get }
 
+  var searchPaths: [AbsolutePath] { get }
+  
   /// Retrieve the absolute path to a particular tool.
   func getToolPath(_ tool: Tool) throws -> AbsolutePath
 
@@ -87,4 +89,50 @@ extension Toolchain {
     }
     return AbsolutePath(path).parentDirectory
   }
+  
+  /// Looks for `SWIFT_DRIVER_TOOLNAME_EXEC` in the `env` property.
+  /// - Returns: Environment variable value, if any.
+  func envVar(forExecutable toolName: String) -> String? {
+    return env[envVarName(for: toolName)]
+  }
+  
+  /// - Returns: String in the form of: `SWIFT_DRIVER_TOOLNAME_EXEC`
+  private func envVarName(for toolName: String) -> String {
+    return "SWIFT_DRIVER_\(toolName.uppercased())_EXEC"
+  }
+  
+  /// Looks for the executable in the `SWIFT_DRIVER_TOOLNAME_EXEC` environment variable, if found nothing,
+  /// looks in the `executableDir`, `xcrunFind` or in the `searchPaths`.
+  /// - Parameter executable: executable to look for [i.e. `swift`].
+  func lookup(executable: String) throws -> AbsolutePath {
+    if let overrideString = envVar(forExecutable: executable) {
+      return try AbsolutePath(validating: overrideString)
+    } else if let path = lookupExecutablePath(filename: executable, searchPaths: [executableDir]) {
+      return path
+    } else if let path = try? xcrunFind(executable: executable) {
+      return path
+    } else if let path = lookupExecutablePath(filename: executable, searchPaths: searchPaths) {
+      return path
+    } else {
+      // This is a hack so our tests work on linux.
+      return AbsolutePath("/usr/bin/" + executable)
+    }
+  }
+  
+  private func xcrunFind(executable: String) throws -> AbsolutePath {
+    let xcrun = "xcrun"
+    guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else {
+      throw ToolchainError.unableToFind(tool: xcrun)
+    }
+    
+    let path = try Process.checkNonZeroExit(
+      arguments: [xcrun, "-sdk", "macosx", "--find", executable],
+      environment: env
+    ).spm_chomp()
+    return AbsolutePath(path)
+  }
+}
+
+fileprivate enum ToolchainError: Swift.Error {
+  case unableToFind(tool: String)
 }

Sources/SwiftDriver/Utilities/FileType.swift

@@ -12,7 +12,7 @@
 
 /// Describes the kinds of files that the driver understands.
 ///
-/// The raw values for these enumerations describe the default extension for]
+/// The raw values for these enumerations describe the default extension for
 /// the file type.
 public enum FileType: String, Hashable, CaseIterable, Codable {
   /// Swift source file.
@@ -45,12 +45,15 @@ public enum FileType: String, Hashable, CaseIterable, Codable {
   /// A compiled Swift module file.
   case swiftModule = "swiftmodule"
 
-  /// Swift documentation for amodule.
+  /// Swift documentation for a module.
   case swiftDocumentation = "swiftdoc"
 
   /// A textual Swift interface file.
   case swiftInterface = "swiftinterface"
 
+  /// Serialized source information.
+  case swiftSourceInfoFile = "swiftsourceinfo"
+
   /// Assembler source.
   case assembly = "s"
 
@@ -110,7 +113,7 @@ extension FileType: CustomStringConvertible {
   public var description: String {
     switch self {
     case .swift, .sil, .sib, .image, .object, .dSYM, .dependencies, .autolink,
-         .swiftModule, .swiftDocumentation, .swiftInterface, .assembly,
+         .swiftModule, .swiftDocumentation, .swiftInterface, .swiftSourceInfoFile, .assembly,
          .remap, .tbd, .pcm, .pch:
       return rawValue
 
@@ -163,8 +166,83 @@ extension FileType {
     case .object, .pch, .ast, .llvmIR, .llvmBitcode, .assembly, .swiftModule,
          .importedModules, .indexData, .remap, .dSYM, .autolink, .dependencies,
          .swiftDocumentation, .pcm, .diagnostics, .objcHeader, .image,
-         .swiftDeps, .moduleTrace, .tbd, .optimizationRecord, .swiftInterface:
+         .swiftDeps, .moduleTrace, .tbd, .optimizationRecord, .swiftInterface, .swiftSourceInfoFile:
       return false
     }
   }
 }
+
+extension FileType {
+
+  private static let typesByName = Dictionary(uniqueKeysWithValues: FileType.allCases.map { ($0.name, $0) })
+
+  init?(name: String) {
+    guard let type = Self.typesByName[name] else { return nil }
+
+    self = type
+  }
+
+  /// The NAME values as specified in FileTypes.def
+  var name: String {
+    switch self {
+    case .swift:
+      return "swift"
+    case .sil:
+      return "sil"
+    case .sib:
+      return "sib"
+    case .image:
+      return "image"
+    case .object:
+      return "object"
+    case .dSYM:
+      return "dSYM"
+    case .dependencies:
+      return "dependencies"
+    case .autolink:
+      return "autolink"
+    case .swiftModule:
+      return "swiftmodule"
+    case .swiftDocumentation:
+      return "swiftdoc"
+    case .swiftInterface:
+      return "swiftinterface"
+    case .swiftSourceInfoFile:
+      return "swiftsourceinfo"
+    case .assembly:
+      return "assembly"
+    case .remap:
+      return "remap"
+    case .tbd:
+      return "tbd"
+    case .pcm:
+      return "pcm"
+    case .pch:
+      return "pch"
+    case .ast:
+      return "ast-dump"
+    case .raw_sil:
+      return "raw-sil"
+    case .raw_sib:
+      return "raw-sib"
+    case .llvmIR:
+      return "llvm-ir"
+    case .llvmBitcode:
+      return "llvm-bc"
+    case .objcHeader:
+      return "objc-header"
+    case .swiftDeps:
+      return "swift-dependencies"
+    case .importedModules:
+      return "imported-modules"
+    case .moduleTrace:
+      return "module-trace"
+    case .indexData:
+      return "index-data"
+    case .optimizationRecord:
+      return "opt-record"
+    case .diagnostics:
+      return "diagnostics"
+    }
+  }
+}