Merge pull request #9381 from apple/direct-transcoding

Author: swift-ci

stdlib/public/core/ASCII.swift

@@ -0,0 +1,88 @@
+//===--- ASCII.swift ------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+extension _Unicode {
+  @_fixed_layout
+  public enum ASCII {}
+}
+
+extension _Unicode.ASCII : UnicodeEncoding {
+  public typealias CodeUnit = UInt8
+  public typealias EncodedScalar = CollectionOfOne<CodeUnit>
+
+  public static var encodedReplacementCharacter : EncodedScalar {
+    return EncodedScalar(0x1a) // U+001A SUBSTITUTE; best we can do for ASCII
+  }
+
+  @inline(__always)
+  @_inlineable
+  public static func _isScalar(_ x: CodeUnit) -> Bool {
+    return true
+  }
+
+  @inline(__always)
+  @_inlineable
+  public static func decode(_ source: EncodedScalar) -> UnicodeScalar {
+    return UnicodeScalar(_unchecked: UInt32(
+        source.first._unsafelyUnwrappedUnchecked))
+  }
+  
+  @inline(__always)
+  @_inlineable
+  public static func encodeIfRepresentable(
+    _ source: UnicodeScalar
+  ) -> EncodedScalar? {
+    guard source.value < (1&<<7) else { return nil }
+    return EncodedScalar(UInt8(extendingOrTruncating: source.value))
+  }
+
+  @inline(__always)
+  public static func transcodeIfRepresentable<FromEncoding : UnicodeEncoding>(
+    _ content: FromEncoding.EncodedScalar, from _: FromEncoding.Type
+  ) -> EncodedScalar? {
+    if _fastPath(FromEncoding.self == UTF16.self) {
+      let c = unsafeBitCast(content, to: UTF16.EncodedScalar.self)
+      guard (c._storage & 0xFF80 == 0) else { return nil }
+      return EncodedScalar(CodeUnit(c._storage & 0x7f))
+    }
+    else if _fastPath(FromEncoding.self == UTF8.self) {
+      let c = unsafeBitCast(content, to: UTF8.EncodedScalar.self)
+      guard (c._storage & 0x80 == 0) else { return nil }
+      return EncodedScalar(CodeUnit(c._storage & 0x7f))
+    }
+    return encodeIfRepresentable(FromEncoding.decode(content))
+  }
+
+  public struct Parser {
+    public init() { }
+  }
+  
+  public typealias ForwardParser = Parser
+  public typealias ReverseParser = Parser
+}
+
+extension _Unicode.ASCII.Parser : UnicodeParser {
+  public typealias Encoding = _Unicode.ASCII
+
+  /// Parses a single Unicode scalar value from `input`.
+  public mutating func parseScalar<I : IteratorProtocol>(
+    from input: inout I
+  ) -> _Unicode.ParseResult<Encoding.EncodedScalar>
+  where I.Element == Encoding.CodeUnit {
+    let n = input.next()
+    if _fastPath(n != nil), let x = n {
+      guard _fastPath(Int8(extendingOrTruncating: x) >= 0)
+      else { return .invalid(length: 1) }
+      return .valid(_Unicode.ASCII.EncodedScalar(x))
+    }
+    return .emptyInput
+  }
+}

stdlib/public/core/CMakeLists.txt

@@ -25,6 +25,7 @@ set(SWIFTLIB_ESSENTIAL
   ArrayCast.swift
   Arrays.swift.gyb
   ArrayType.swift
+  ASCII.swift
   Assert.swift
   AssertCommon.swift
   BidirectionalCollection.swift

stdlib/public/core/CString.swift

@@ -46,8 +46,8 @@ extension String {
   public init(cString: UnsafePointer<CChar>) {
     let len = UTF8._nullCodeUnitOffset(in: cString)
     let (result, _) = cString.withMemoryRebound(to: UInt8.self, capacity: len) {
-      _decodeCString($0, as: UTF8.self, length: len,
-        repairingInvalidCodeUnits: true)!
+      _decodeCString(
+        $0, as: UTF8.self, length: len, repairingInvalidCodeUnits: true)!
     }
     self = result
   }
@@ -58,8 +58,8 @@ extension String {
   /// This is identical to init(cString: UnsafePointer<CChar> but operates on an
   /// unsigned sequence of bytes.
   public init(cString: UnsafePointer<UInt8>) {
-    self = String.decodeCString(cString, as: UTF8.self,
-      repairingInvalidCodeUnits: true)!.result
+    self = String.decodeCString(
+      cString, as: UTF8.self, repairingInvalidCodeUnits: true)!.result
   }
 
   /// Creates a new string by copying and validating the null-terminated UTF-8
@@ -145,7 +145,7 @@ extension String {
   ///   ill-formed sequence is detected, this method returns `nil`.
   ///
   /// - SeeAlso: `UnicodeCodec`
-  public static func decodeCString<Encoding : UnicodeCodec>(
+  public static func decodeCString<Encoding : UnicodeEncoding>(
     _ cString: UnsafePointer<Encoding.CodeUnit>?,
     as encoding: Encoding.Type,
     repairingInvalidCodeUnits isRepairing: Bool = true)
@@ -154,8 +154,11 @@ extension String {
     guard let cString = cString else {
       return nil
     }
-    let len = encoding._nullCodeUnitOffset(in: cString)
-    return _decodeCString(cString, as: encoding, length: len,
+    var end = cString
+    while end.pointee != 0 { end += 1 }
+    let len = end - cString
+    return _decodeCString(
+      cString, as: encoding, length: len,
       repairingInvalidCodeUnits: isRepairing)
   }
 
@@ -180,7 +183,7 @@ public func _persistCString(_ p: UnsafePointer<CChar>?) -> [CChar]? {
 /// the given pointer using the specified encoding.
 ///
 /// This internal helper takes the string length as an argument.
-func _decodeCString<Encoding : UnicodeCodec>(
+func _decodeCString<Encoding : UnicodeEncoding>(
   _ cString: UnsafePointer<Encoding.CodeUnit>,
   as encoding: Encoding.Type, length: Int,
   repairingInvalidCodeUnits isRepairing: Bool = true)

stdlib/public/core/GroupInfo.json

@@ -4,6 +4,7 @@
     "AssertCommon.swift"
   ],
   "String": [
+    "ASCII.swift",
     "CString.swift",
     "Character.swift",
     "StaticString.swift",

stdlib/public/core/String.swift

@@ -344,23 +344,22 @@ public struct String {
 
 extension String {
   public // @testable
-  static func _fromWellFormedCodeUnitSequence<Encoding, Input>(
+  static func _fromWellFormedCodeUnitSequence<
+    Encoding : UnicodeEncoding, Input : Collection
+  >(
     _ encoding: Encoding.Type, input: Input
   ) -> String
-    where
-    Encoding: UnicodeCodec,
-    Input: Collection,
-    Input.Iterator.Element == Encoding.CodeUnit {
+    where  Input.Iterator.Element == Encoding.CodeUnit {
     return String._fromCodeUnitSequence(encoding, input: input)!
   }
 
   public // @testable
-  static func _fromCodeUnitSequence<Encoding, Input>(
+  static func _fromCodeUnitSequence<
+    Encoding : UnicodeEncoding, Input : Collection
+  >(
     _ encoding: Encoding.Type, input: Input
   ) -> String?
     where
-    Encoding: UnicodeCodec,
-    Input: Collection,
     Input.Iterator.Element == Encoding.CodeUnit {
     let (stringBufferOptional, _) =
         _StringBuffer.fromCodeUnits(input, encoding: encoding,
@@ -369,12 +368,12 @@ extension String {
   }
 
   public // @testable
-  static func _fromCodeUnitSequenceWithRepair<Encoding, Input>(
+  static func _fromCodeUnitSequenceWithRepair<
+    Encoding : UnicodeEncoding, Input : Collection
+  >(
     _ encoding: Encoding.Type, input: Input
   ) -> (String, hadError: Bool)
     where
-    Encoding: UnicodeCodec,
-    Input: Collection,
     Input.Iterator.Element == Encoding.CodeUnit {
 
     let (stringBuffer, hadError) =
@@ -485,23 +484,21 @@ extension String {
   /// Returns the number of code units occupied by this string
   /// in the given encoding.
   func _encodedLength<
-    Encoding: UnicodeCodec
+    Encoding: UnicodeEncoding
   >(_ encoding: Encoding.Type) -> Int {
     var codeUnitCount = 0
     self._encode(encoding, into: { _ in codeUnitCount += 1 })
     return codeUnitCount
   }
 
-  // FIXME: this function does not handle the case when a wrapped NSString
+  // FIXME: this function may not handle the case when a wrapped NSString
   // contains unpaired surrogates.  Fix this before exposing this function as a
   // public API.  But it is unclear if it is valid to have such an NSString in
   // the first place.  If it is not, we should not be crashing in an obscure
   // way -- add a test for that.
   // Related: <rdar://problem/17340917> Please document how NSString interacts
   // with unpaired surrogates
-  func _encode<
-    Encoding: UnicodeCodec
-  >(
+  func _encode<Encoding: UnicodeEncoding>(
     _ encoding: Encoding.Type,
     into processCodeUnit: (Encoding.CodeUnit) -> Void
   ) {

stdlib/public/core/StringBuffer.swift

@@ -97,7 +97,7 @@ public struct _StringBuffer {
   ) -> (_StringBuffer?, hadError: Bool)
     where
     Input : Collection, // Sequence?
-    Encoding : UnicodeCodec,
+    Encoding : UnicodeEncoding,
     Input.Iterator.Element == Encoding.CodeUnit {
     // Determine how many UTF-16 code units we'll need
     let inputStream = input.makeIterator()

stdlib/public/core/StringCore.swift

@@ -336,32 +336,60 @@ public struct _StringCore {
     }
   }
 
+  var _unmanagedASCII : UnsafeBufferPointer<_Unicode.ASCII.CodeUnit>? {
+    @inline(__always)
+    get {
+      guard _fastPath(_baseAddress != nil && elementWidth == 1) else {
+        return nil
+      }
+      return UnsafeBufferPointer(
+        start: _baseAddress!.assumingMemoryBound(
+          to: _Unicode.ASCII.CodeUnit.self),
+        count: count
+      )
+    }
+  }
+  
+  var _unmanagedUTF16 : UnsafeBufferPointer<UTF16.CodeUnit>? {
+    @inline(__always)
+    get {
+      guard _fastPath(_baseAddress != nil && elementWidth != 1) else {
+        return nil
+      }
+      return UnsafeBufferPointer(
+        start: _baseAddress!.assumingMemoryBound(to: UTF16.CodeUnit.self),
+        count: count
+      )
+    }
+  }
+  
   /// Write the string, in the given encoding, to output.
-  func encode<Encoding: UnicodeCodec>(
+  func encode<Encoding: UnicodeEncoding>(
     _ encoding: Encoding.Type,
     into processCodeUnit: (Encoding.CodeUnit) -> Void)
   {
-    if _fastPath(_baseAddress != nil) {
-      if _fastPath(elementWidth == 1) {
-        for x in UnsafeBufferPointer(
-          start: _baseAddress!.assumingMemoryBound(to: UTF8.CodeUnit.self),
-          count: count
-        ) {
-          Encoding.encode(UnicodeScalar(x), into: processCodeUnit)
+    defer { _fixLifetime(self) }
+    if let bytes = _unmanagedASCII {
+      if encoding == _Unicode.ASCII.self
+      || encoding == _Unicode.UTF8.self
+      || encoding == _Unicode.UTF16.self
+      || encoding == _Unicode.UTF32.self {
+        bytes.forEach {
+          processCodeUnit(Encoding.CodeUnit(extendingOrTruncating: $0))
         }
       }
       else {
-        let hadError = transcode(
-          UnsafeBufferPointer(
-            start: _baseAddress!.assumingMemoryBound(to: UTF16.CodeUnit.self),
-            count: count
-          ).makeIterator(),
-          from: UTF16.self,
-          to: encoding,
-          stoppingOnError: true,
-          into: processCodeUnit
-        )
-        _sanityCheck(!hadError, "Swift.String with native storage should not have unpaired surrogates")
+        // TODO: be sure tests exercise this code path.
+        for b in bytes {
+          Encoding.encode(
+            UnicodeScalar(_unchecked: UInt32(b))).forEach(processCodeUnit)
+        }
+      }
+    }
+    else if let content = _unmanagedUTF16 {
+    var i = content.makeIterator()
+      _Unicode.UTF16.ForwardParser.parse(&i) {
+        Encoding.transcode($0, from: UTF16.self).forEach(processCodeUnit)
       }
     }
     else if hasCocoaBuffer {

stdlib/public/core/UTF16.swift

@@ -37,7 +37,9 @@ extension _Unicode.UTF16 : UnicodeEncoding {
     return UnicodeScalar(_unchecked: value)
   }
 
-  public static func encode(_ source: UnicodeScalar) -> EncodedScalar? {
+  public static func encodeIfRepresentable(
+    _ source: UnicodeScalar
+  ) -> EncodedScalar? {
     let x = source.value
     if _fastPath(x < (1 << 16)) {
       return EncodedScalar(_storage: x, _bitCount: 16)
@@ -50,7 +52,7 @@ extension _Unicode.UTF16 : UnicodeEncoding {
   }
 
   @inline(__always)
-  public static func transcode<FromEncoding : UnicodeEncoding>(
+  public static func transcodeIfRepresentable<FromEncoding : UnicodeEncoding>(
     _ content: FromEncoding.EncodedScalar, from _: FromEncoding.Type
   ) -> EncodedScalar? {
     if _fastPath(FromEncoding.self == UTF8.self) {
@@ -84,12 +86,12 @@ extension _Unicode.UTF16 : UnicodeEncoding {
       s &>>= 8
       r |= s & 0b0__11_1111
       r &= (1 &<< 21) - 1
-      return encode(UnicodeScalar(_unchecked: r))
+      return encodeIfRepresentable(UnicodeScalar(_unchecked: r))
     }
     else if _fastPath(FromEncoding.self == UTF16.self) {
       return unsafeBitCast(content, to: UTF16.EncodedScalar.self)
     }
-    return encode(FromEncoding.decode(content))
+    return encodeIfRepresentable(FromEncoding.decode(content))
   }
 
   public struct ForwardParser {

stdlib/public/core/UTF32.swift

@@ -34,7 +34,9 @@ extension _Unicode.UTF32 : UnicodeEncoding {
   }
 
   @inline(__always)
-  public static func encode(_ source: UnicodeScalar) -> EncodedScalar? {
+  public static func encodeIfRepresentable(
+    _ source: UnicodeScalar
+  ) -> EncodedScalar? {
     return EncodedScalar(source.value)
   }
 

stdlib/public/core/UTF8.swift

@@ -58,7 +58,9 @@ extension _Unicode.UTF8 : UnicodeEncoding {
 
   @inline(__always)
   @_inlineable
-  public static func encode(_ source: UnicodeScalar) -> EncodedScalar? {
+  public static func encodeIfRepresentable(
+    _ source: UnicodeScalar
+  ) -> EncodedScalar? {
     var c = source.value
     if _fastPath(c < (1&<<7)) {
       return EncodedScalar(_storage: c, _bitCount: 8)
@@ -86,7 +88,7 @@ extension _Unicode.UTF8 : UnicodeEncoding {
   }
 
   @inline(__always)
-  public static func transcode<FromEncoding : UnicodeEncoding>(
+  public static func transcodeIfRepresentable<FromEncoding : UnicodeEncoding>(
     _ content: FromEncoding.EncodedScalar, from _: FromEncoding.Type
   ) -> EncodedScalar? {
     if _fastPath(FromEncoding.self == UTF16.self) {
@@ -116,7 +118,7 @@ extension _Unicode.UTF8 : UnicodeEncoding {
     else if _fastPath(FromEncoding.self == UTF8.self) {
       return unsafeBitCast(content, to: UTF8.EncodedScalar.self)
     }
-    return encode(FromEncoding.decode(content))
+    return encodeIfRepresentable(FromEncoding.decode(content))
   }
 
   @_fixed_layout