Merge pull request #9188 from apple/new-unicode-decoders

swift-ci · web-flow · commit edd13eca60a9 · 2017-05-03T11:38:25.000-07:00
diff --git a/stdlib/public/core/CMakeLists.txt b/stdlib/public/core/CMakeLists.txt
@@ -132,7 +132,6 @@ set(SWIFTLIB_ESSENTIAL
   SwiftNativeNSArray.swift
   UIntBuffer.swift
   UnavailableStringAPIs.swift.gyb
-  Unicode.swift
   UnicodeEncoding.swift
   UnicodeParser.swift
   UnicodeScalar.swift
@@ -146,6 +145,7 @@ set(SWIFTLIB_ESSENTIAL
   UTFEncoding.swift
   UTF8.swift
   UTF16.swift
+  Unicode.swift # ORDER DEPENDENCY: must follow new unicode support
   WriteBackMutableSlice.swift
   )
 
diff --git a/stdlib/public/core/Collection.swift b/stdlib/public/core/Collection.swift
@@ -379,6 +379,7 @@ public struct IndexingIterator<
 > : IteratorProtocol, Sequence {
 
   @_inlineable
+  @inline(__always)
   /// Creates an iterator over the given collection.
   public /// @testable
   init(_elements: Elements) {
@@ -387,6 +388,7 @@ public struct IndexingIterator<
   }
 
   @_inlineable
+  @inline(__always)
   /// Creates an iterator over the given collection.
   public /// @testable
   init(_elements: Elements, _position: Elements.Index) {
@@ -419,12 +421,14 @@ public struct IndexingIterator<
   /// - Returns: The next element in the underlying sequence if a next element
   ///   exists; otherwise, `nil`.
   @_inlineable
+  @inline(__always)
   public mutating func next() -> Elements._Element? {
     if _position == _elements.endIndex { return nil }
     let element = _elements[_position]
     _elements.formIndex(after: &_position)
     return element
   }
+  
   @_versioned
   internal let _elements: Elements
   @_versioned
@@ -1326,12 +1330,15 @@ extension Collection {
   ///     // Prints "10"
   @_inlineable
   public var first: Iterator.Element? {
-    // NB: Accessing `startIndex` may not be O(1) for some lazy collections,
-    // so instead of testing `isEmpty` and then returning the first element,
-    // we'll just rely on the fact that the iterator always yields the
-    // first element first.
-    var i = makeIterator()
-    return i.next()
+    @inline(__always)
+    get {
+      // NB: Accessing `startIndex` may not be O(1) for some lazy collections,
+      // so instead of testing `isEmpty` and then returning the first element,
+      // we'll just rely on the fact that the iterator always yields the
+      // first element first.
+      var i = makeIterator()
+      return i.next()
+    }
   }
   
   // TODO: swift-3-indexing-model - uncomment and replace above ready (or should we still use the iterator one?)
diff --git a/stdlib/public/core/IntegerParsing.swift.gyb b/stdlib/public/core/IntegerParsing.swift.gyb
@@ -28,6 +28,7 @@ word_bits = int(CMAKE_SIZEOF_VOID_P) * 8
 /// - Note: If `text` begins with `"+"` or `"-"`, even if the rest of
 ///   the characters are `"0"`, the result is `nil`.
   // FIXME(integers): support a more general BinaryInteger protocol
+@inline(__always)
 internal func _parseUnsignedAsciiAsUInt64(
   _ u16: String.UTF16View, _ radix: Int, _ maximum: UInt64
 ) -> UInt64? {
@@ -70,6 +71,7 @@ internal func _parseUnsignedAsciiAsUInt64(
 /// - Note: For text matching the regular expression "-0+", the result
 ///   is `0`, not `nil`.
   // FIXME(integers): support a more general BinaryInteger protocol
+@inline(__always)
 internal func _parseAsciiAsUInt64(
   _ utf16: String.UTF16View, _ radix: Int, _ maximum: UInt64
 ) -> UInt64? {
@@ -92,6 +94,7 @@ internal func _parseAsciiAsUInt64(
 /// - Note: For text matching the regular expression "-0+", the result
 ///   is `0`, not `nil`.
   // FIXME(integers): support a more general BinaryInteger protocol
+@inline(__always)
 internal func _parseAsciiAsInt64(
   _ utf16: String.UTF16View, _ radix: Int, _ maximum: Int64
 ) -> Int64? {
@@ -109,6 +112,7 @@ internal func _parseAsciiAsInt64(
 }
 
 /// Strip an optional single leading ASCII plus/minus sign from `utf16`.
+@inline(__always)
 private func _parseOptionalAsciiSign(
   _ utf16: String.UTF16View
 ) -> (digitsUTF16: String.UTF16View, isMinus: Bool) {
diff --git a/stdlib/public/core/String.swift b/stdlib/public/core/String.swift
@@ -14,7 +14,7 @@ import SwiftShims
 
 public protocol StringProtocol
   : RangeReplaceableCollection, BidirectionalCollection,
-  CustomStringConvertible, CustomDebugStringConvertible,
+  CustomDebugStringConvertible,
   CustomReflectable, CustomPlaygroundQuickLookable,
   TextOutputStream, TextOutputStreamable,
   LosslessStringConvertible, ExpressibleByStringLiteral,
diff --git a/stdlib/public/core/StringUTF8.swift b/stdlib/public/core/StringUTF8.swift
@@ -32,6 +32,7 @@ extension _StringCore {
   /// and the second element contains the encoded UTF-8 starting in its
   /// low byte.  Any unused high bytes in the result will be set to
   /// 0xFF.
+  @inline(__always)
   func _encodeSomeUTF8(from i: Int) -> (Int, _UTF8Chunk) {
     _sanityCheck(i <= count)
 
diff --git a/stdlib/public/core/UTF8.swift b/stdlib/public/core/UTF8.swift
@@ -9,17 +9,29 @@
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
+extension _Unicode {
+  @_fixed_layout
+  public enum UTF8 {
+  case _swift3Buffer(_Unicode.UTF8.ForwardParser)
+  }
+}
+
 extension _Unicode.UTF8 : UnicodeEncoding {
+  public typealias CodeUnit = UInt8
   public typealias EncodedScalar = _UIntBuffer<UInt32, UInt8>
 
   public static var encodedReplacementCharacter : EncodedScalar {
     return EncodedScalar(_storage: 0xbdbfef, _bitCount: 24)
   }
 
+  @inline(__always)
+  @_inlineable
   public static func _isScalar(_ x: CodeUnit) -> Bool {
     return x & 0x80 == 0
   }
 
+  @inline(__always)
+  @_inlineable
   public static func decode(_ source: EncodedScalar) -> UnicodeScalar {
     let bits = source._storage
     switch source._bitCount {
@@ -44,50 +56,56 @@ extension _Unicode.UTF8 : UnicodeEncoding {
     }
   }
   
+  @inline(__always)
+  @_inlineable
   public static func encode(_ source: UnicodeScalar) -> EncodedScalar {
-    let x = source.value
-    if _fastPath(x < (1 << 7)) {
-      return EncodedScalar(_storage: x, _bitCount: 8)
-    }
-    else if _fastPath(x < (1 << 11)) {
-      var r = x &>> 6
-      r |= (x & 0b11_1111) &<< 8
-      r |= 0b1000_0000__1100_0000
-      return EncodedScalar(_storage: r, _bitCount: 2*8)
-    }
-    else if _fastPath(x < (1 << 16)) {
-      var r = x &>> 12
-      r |= (x & 0b1111__1100_0000) &<< 2
-      r |= (x & 0b11_1111) &<< 16
-      r |= 0b1000_0000__1000_0000__1110_0000
-      return EncodedScalar(_storage:  r, _bitCount: 3*8)
-    }
-    else {
-      var r = x &>> 18
-      r |= (x & 0b11__1111_0000__0000_0000) &>> 4
-      r |= (x & 0b1111__1100_0000) &<< 10
-      r |= (x & 0b11_1111) << 24
-      r |= 0b1000_0000__1000_0000__1000_0000__1111_0000
-      return EncodedScalar(_storage: r, _bitCount: 4*8)
-    }
+    var c = source.value
+    if _fastPath(c < (1&<<7)) {
+      return EncodedScalar(_storage: c, _bitCount: 8)
+    }
+    var o = c & 0b0__0011_1111
+    c &>>= 6
+    o &<<= 8
+    if _fastPath(c < (1&<<5)) {
+      return EncodedScalar(
+        _storage: o | c | 0b0__1000_0000__1100_0000, _bitCount: 16)
+    }
+    o |= c & 0b0__0011_1111
+    c &>>= 6
+    o &<<= 8
+    if _fastPath(c < (1&<<4)) {
+      return EncodedScalar(
+        _storage: o | c | 0b0__1000_0000__1000_0000__1110_0000, _bitCount: 24)
+    }
+    o |= c & 0b0__0011_1111
+    c &>>= 6
+    o &<<= 8
+    return EncodedScalar(
+      _storage: o | c | 0b0__1000_0000__1000_0000__1000_0000__1111_0000,
+      _bitCount: 32)
   }
   
   public struct ForwardParser {
     public typealias _Buffer = _UIntBuffer<UInt32, UInt8>
+    @inline(__always)
+    @_inlineable
     public init() { _buffer = _Buffer() }
     public var _buffer: _Buffer
   }
   
   public struct ReverseParser {
     public typealias _Buffer = _UIntBuffer<UInt32, UInt8>
+    @inline(__always)
+    @_inlineable
     public init() { _buffer = _Buffer() }
     public var _buffer: _Buffer
   }
 }
 
 extension UTF8.ReverseParser : UnicodeParser, _UTFParser {
   public typealias Encoding = _Unicode.UTF8
-
+  @inline(__always)
+  @_inlineable
   public func _parseMultipleCodeUnits() -> (isValid: Bool, bitCount: UInt8) {
     _sanityCheck(_buffer._storage & 0x80 != 0) // this case handled elsewhere
     if _buffer._storage                & 0b0__1110_0000__1100_0000
@@ -122,6 +140,7 @@ extension UTF8.ReverseParser : UnicodeParser, _UTFParser {
   /// Returns the length of the invalid sequence that ends with the LSB of
   /// buffer.
   @inline(never)
+  @_versioned
   func _invalidLength() -> UInt8 {
     if _buffer._storage                 & 0b0__1111_0000__1100_0000
                                        == 0b0__1110_0000__1000_0000 {
@@ -150,6 +169,8 @@ extension UTF8.ReverseParser : UnicodeParser, _UTFParser {
     return 1
   }
   
+  @inline(__always)
+  @_inlineable
   public func _bufferedScalar(bitCount: UInt8) -> Encoding.EncodedScalar {
     return Encoding.EncodedScalar(
       _storage: _buffer._storage.byteSwapped &>> (32 - bitCount),
@@ -160,7 +181,9 @@ extension UTF8.ReverseParser : UnicodeParser, _UTFParser {
 
 extension _Unicode.UTF8.ForwardParser : UnicodeParser, _UTFParser {
   public typealias Encoding = _Unicode.UTF8
-  
+
+  @inline(__always)
+  @_inlineable
   public func _parseMultipleCodeUnits() -> (isValid: Bool, bitCount: UInt8) {
     _sanityCheck(_buffer._storage & 0x80 != 0) // this case handled elsewhere
     
@@ -195,6 +218,7 @@ extension _Unicode.UTF8.ForwardParser : UnicodeParser, _UTFParser {
   /// Returns the length of the invalid sequence that starts with the LSB of
   /// buffer.
   @inline(never)
+  @_versioned
   func _invalidLength() -> UInt8 {
     if _buffer._storage               & 0b0__1100_0000__1111_0000
                                      == 0b0__1000_0000__1110_0000 {
diff --git a/stdlib/public/core/UTFEncoding.swift b/stdlib/public/core/UTFEncoding.swift
@@ -27,7 +27,8 @@ public protocol _UTFParser {
 
 extension _UTFParser
 where Encoding.EncodedScalar == _UIntBuffer<UInt32, Encoding.CodeUnit> {
-  
+
+  @inline(__always)
   public mutating func parseScalar<I : IteratorProtocol>(
     from input: inout I
   ) -> _Unicode.ParseResult<Encoding.EncodedScalar>
diff --git a/stdlib/public/core/Unicode.swift b/stdlib/public/core/Unicode.swift
diff --git a/stdlib/public/core/UnicodeParser.swift b/stdlib/public/core/UnicodeParser.swift
