Merge pull request #3 from apple/master

Merge upstream

Author: sjavora

README.md

@@ -97,7 +97,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi
   * [x] Teach the `DarwinToolchain` to also handle iOS, tvOS, watchOS
   * [x] Fill out the `GenericUnixToolchain` toolchain to get it working
   * [ ] Implement a `WindowsToolchain`
-  * [ ] Implement proper tokenization for response files
+  * [x] Implement proper tokenization for response files
 * Compilation modes
   * [x] Batch mode
   * [ ] Whole-module-optimization mode

Sources/SwiftDriver/Driver/Driver.swift

@@ -297,26 +297,70 @@ public struct Driver {
   }
 }
 
-// Response files.
+// MARK: - Response files.
 extension Driver {
-  /// Tokenize a single line in a response file
-  private static func tokenizeResponseFileLine(_ line: String) -> String {
-    // FIXME: This is wrong. We need to do proper shell escaping.
-    return line.replacingOccurrences(of: "\\ ", with: " ")
+  /// Tokenize a single line in a response file.
+  ///
+  /// This method supports response files with:
+  /// 1. Double slash comments at the beginning of a line.
+  /// 2. Backslash escaping.
+  /// 3. Space character (U+0020 SPACE).
+  ///
+  /// - Returns: One line String ready to be used in the shell, if any.
+  ///
+  /// - Complexity: O(*n*), where *n* is the length of the line.
+  private static func tokenizeResponseFileLine<S: StringProtocol>(_ line: S) -> String? {
+    if line.isEmpty { return nil }
+    
+    // Support double dash comments only if they start at the beginning of a line.
+    if line.hasPrefix("//") { return nil }
+    
+    var result: String = ""
+    /// Indicates if we just parsed an escaping backslash.
+    var isEscaping = false
+    
+    for char in line {
+      if char.isNewline { return result }
+
+      // Backslash escapes to the next character.
+      if char == #"\"#, !isEscaping {
+        isEscaping = true
+        continue
+      } else if isEscaping {
+        // Disable escaping and keep parsing.
+        isEscaping = false
+      }
+      
+      // Ignore spacing characters, except by the space character.
+      if char.isWhitespace && char != " " { continue }
+      
+      result.append(char)
+    }
+    return result.isEmpty ? nil : result
   }
 
+  /// Tokenize each line of the response file, omitting empty lines.
+  ///
+  /// - Parameter content: response file's content to be tokenized.
+  private static func tokenizeResponseFile(_ content: String) -> [String] {
+    return content
+      .split(separator: "\n")
+      .compactMap { tokenizeResponseFileLine($0) }
+  }
+  
+  /// Recursively expands the response files.
+  /// - Parameter visitedResponseFiles: Set containing visited response files to detect recursive parsing.
   private static func expandResponseFiles(
     _ args: [String],
     diagnosticsEngine: DiagnosticsEngine,
     visitedResponseFiles: inout Set<AbsolutePath>
   ) throws -> [String] {
-    // FIXME: This is very very prelimary. Need to look at how Swift compiler expands response file.
-
     var result: [String] = []
 
     // Go through each arg and add arguments from response files.
     for arg in args {
       if arg.first == "@", let responseFile = try? AbsolutePath(validating: String(arg.dropFirst())) {
+        // Guard against infinite parsing loop.
         guard visitedResponseFiles.insert(responseFile).inserted else {
           diagnosticsEngine.emit(.warn_recursive_response_file(responseFile))
           continue
@@ -326,7 +370,7 @@ extension Driver {
         }
 
         let contents = try localFileSystem.readFileContents(responseFile).cString
-        let lines = contents.split(separator: "\n", omittingEmptySubsequences: true).map { tokenizeResponseFileLine(String($0)) }
+        let lines = tokenizeResponseFile(contents)
         result.append(contentsOf: try expandResponseFiles(lines, diagnosticsEngine: diagnosticsEngine, visitedResponseFiles: &visitedResponseFiles))
       } else {
         result.append(arg)
@@ -707,7 +751,7 @@ extension Driver {
 }
 
 extension Diagnostic.Message {
-  public static func error_i_mode(_ driverKind: DriverKind) -> Diagnostic.Message {
+  static func error_i_mode(_ driverKind: DriverKind) -> Diagnostic.Message {
     .error(
       """
       the flag '-i' is no longer required and has been removed; \
@@ -731,17 +775,14 @@ extension Driver {
 
     // Make sure we have a non-negative integer value.
     guard let numThreads = Int(numThreadsArg.asSingle), numThreads >= 0 else {
-      diagnosticsEngine.emit(Diagnostic.Message.error_invalid_arg_value(arg: .numThreads, value: numThreadsArg.asSingle))
+      diagnosticsEngine.emit(.error_invalid_arg_value(arg: .numThreads, value: numThreadsArg.asSingle))
       return 0
     }
 
-    #if false
-    // FIXME: Check for batch mode.
-    if false {
+    if case .batchCompile = compilerMode {
       diagnosticsEngine.emit(.warning_cannot_multithread_batch_mode)
       return 0
     }
-    #endif
 
     return numThreads
   }
@@ -1032,7 +1073,7 @@ extension Driver {
       }
     }
 
-    if !moduleName.isSwiftIdentifier {
+    if !moduleName.sd_isSwiftIdentifier {
       fallbackOrDiagnose(.error_bad_module_name(moduleName: moduleName, explicitModuleName: parsedOptions.contains(.moduleName)))
     } else if moduleName == "Swift" && !parsedOptions.contains(.parseStdlib) {
       fallbackOrDiagnose(.error_stdlib_module_name(moduleName: moduleName, explicitModuleName: parsedOptions.contains(.moduleName)))

Sources/SwiftDriver/Toolchains/DarwinToolchain.swift

@@ -15,10 +15,12 @@ fileprivate func envVarName(forExecutable toolName: String) -> String {
   return "SWIFT_DRIVER_\(toolName.uppercased())_EXEC"
 }
 
-// FIXME: This should be in DarwinToolchain, but GenericUnixToolchain is
-// currently using it too for some reason.
-extension Toolchain {
-  /// Utility function to lookup an executable using xcrun.
+/// Toolchain for Darwin-based platforms, such as macOS and iOS.
+///
+/// FIXME: This class is not thread-safe.
+public final class DarwinToolchain: Toolchain {
+  public let env: [String: String]
+
   func xcrunFind(exec: String) throws -> AbsolutePath {
     if let overrideString = env[envVarName(forExecutable: exec)] {
       return try AbsolutePath(validating: overrideString)
@@ -35,14 +37,7 @@ extension Toolchain {
     return AbsolutePath("/usr/bin/" + exec)
   #endif
   }
-}
 
-/// Toolchain for Darwin-based platforms, such as macOS and iOS.
-///
-/// FIXME: This class is not thread-safe.
-public final class DarwinToolchain: Toolchain {
-  public let env: [String: String]
-  
   /// Retrieve the absolute path for a given tool.
   public func getToolPath(_ tool: Tool) throws -> AbsolutePath {
     switch tool {

Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift

@@ -34,7 +34,7 @@ public final class GenericUnixToolchain: Toolchain {
     // If we happen to be on a macOS host, some tools might not be in our
     // PATH, so we'll just use xcrun to find them too.
     #if os(macOS)
-    return try xcrunFind(exec: exec)
+    return try DarwinToolchain(env: self.env).xcrunFind(exec: exec)
     #else
     throw Error.unableToFind(tool: exec)
     #endif

Sources/SwiftDriver/Utilities/StringAdditions.swift

@@ -11,11 +11,15 @@
 //===----------------------------------------------------------------------===//
 extension String {
   /// Whether this string is a Swift identifier.
-  var isSwiftIdentifier: Bool {
-    if isEmpty { return false }
+  public var sd_isSwiftIdentifier: Bool {
+    guard let start = unicodeScalars.first else {
+      return false
+    }
+
+    let continuation = unicodeScalars.dropFirst()
 
-    // FIXME: This is a hack. Check the actual identifier grammar here.
-    return spm_mangledToC99ExtendedIdentifier() == self
+    return start.isValidSwiftIdentifierStart &&
+           continuation.allSatisfy { $0.isValidSwiftIdentifierContinuation }
   }
 }
 
@@ -29,3 +33,103 @@ extension DefaultStringInterpolation {
     appendInterpolation(value)
   }
 }
+
+extension Unicode.Scalar {
+
+  var isValidSwiftIdentifierStart: Bool {
+    guard isValidSwiftIdentifierContinuation else { return false }
+
+    if isASCIIDigit || self == "$" {
+      return false
+    }
+
+    // N1518: Recommendations for extended identifier characters for C and C++
+    // Proposed Annex X.2: Ranges of characters disallowed initially
+    if (0x0300...0x036F).contains(value) ||
+       (0x1DC0...0x1DFF).contains(value) ||
+       (0x20D0...0x20FF).contains(value) ||
+       (0xFE20...0xFE2F).contains(value) {
+      return false
+    }
+
+    return true
+  }
+
+  var isValidSwiftIdentifierContinuation: Bool {
+    if isASCII {
+      return isCIdentifierBody(allowDollar: true)
+    }
+
+    // N1518: Recommendations for extended identifier characters for C and C++
+    // Proposed Annex X.1: Ranges of characters allowed
+    return value == 0x00A8 ||
+           value == 0x00AA ||
+           value == 0x00AD ||
+           value == 0x00AF ||
+
+           (0x00B2...0x00B5).contains(value) ||
+           (0x00B7...0x00BA).contains(value) ||
+           (0x00BC...0x00BE).contains(value) ||
+           (0x00C0...0x00D6).contains(value) ||
+           (0x00D8...0x00F6).contains(value) ||
+           (0x00F8...0x00FF).contains(value) ||
+
+           (0x0100...0x167F).contains(value) ||
+           (0x1681...0x180D).contains(value) ||
+           (0x180F...0x1FFF).contains(value) ||
+
+           (0x200B...0x200D).contains(value) ||
+           (0x202A...0x202E).contains(value) ||
+           (0x203F...0x2040).contains(value) ||
+           value == 0x2054                   ||
+           (0x2060...0x206F).contains(value) ||
+
+           (0x2070...0x218F).contains(value) ||
+           (0x2460...0x24FF).contains(value) ||
+           (0x2776...0x2793).contains(value) ||
+           (0x2C00...0x2DFF).contains(value) ||
+           (0x2E80...0x2FFF).contains(value) ||
+
+           (0x3004...0x3007).contains(value) ||
+           (0x3021...0x302F).contains(value) ||
+           (0x3031...0x303F).contains(value) ||
+
+           (0x3040...0xD7FF).contains(value) ||
+
+           (0xF900...0xFD3D).contains(value) ||
+           (0xFD40...0xFDCF).contains(value) ||
+           (0xFDF0...0xFE44).contains(value) ||
+           (0xFE47...0xFFF8).contains(value) ||
+
+           (0x10000...0x1FFFD).contains(value) ||
+           (0x20000...0x2FFFD).contains(value) ||
+           (0x30000...0x3FFFD).contains(value) ||
+           (0x40000...0x4FFFD).contains(value) ||
+           (0x50000...0x5FFFD).contains(value) ||
+           (0x60000...0x6FFFD).contains(value) ||
+           (0x70000...0x7FFFD).contains(value) ||
+           (0x80000...0x8FFFD).contains(value) ||
+           (0x90000...0x9FFFD).contains(value) ||
+           (0xA0000...0xAFFFD).contains(value) ||
+           (0xB0000...0xBFFFD).contains(value) ||
+           (0xC0000...0xCFFFD).contains(value) ||
+           (0xD0000...0xDFFFD).contains(value) ||
+           (0xE0000...0xEFFFD).contains(value)
+  }
+
+  /// `true` if this character is an ASCII digit: [0-9]
+  var isASCIIDigit: Bool { (0x30...0x39).contains(value) }
+
+  /// `true` if this is a body character of a C identifier,
+  /// which is [a-zA-Z0-9_].
+  func isCIdentifierBody(allowDollar: Bool = false) -> Bool {
+    if (0x41...0x5A).contains(value) ||
+       (0x61...0x7A).contains(value) ||
+       isASCIIDigit                  ||
+       self == "_" {
+      return true
+    } else {
+      return allowDollar && self == "$"
+    }
+  }
+}

Sources/SwiftDriver/Utilities/VirtualPath.swift

@@ -40,23 +40,7 @@ public enum VirtualPath: Hashable {
   }
 
   /// The name of the path for presentation purposes.
-  ///
-  /// FIXME: Maybe this should be debugDescription or description
-  public var name: String {
-    switch self {
-    case .relative(let path):
-      return path.pathString
-
-    case .absolute(let path):
-      return path.pathString
-
-    case .standardInput, .standardOutput:
-      return "-"
-
-    case .temporary(let path):
-      return path.pathString
-    }
-  }
+  public var name: String { description }
 
   /// The extension of this path, for relative or absolute paths.
   public var `extension`: String? {
@@ -147,6 +131,24 @@ extension VirtualPath: Codable {
   }
 }
 
+extension VirtualPath: CustomStringConvertible {
+  public var description: String {
+    switch self {
+    case .relative(let path):
+      return path.pathString
+
+    case .absolute(let path):
+      return path.pathString
+
+    case .standardInput, .standardOutput:
+      return "-"
+
+    case .temporary(let path):
+      return path.pathString
+    }
+  }
+}
+
 extension VirtualPath {
   /// Replace the extension of the given path with a new one based on the
   /// specified file type.