[stdlib] Move new Unicode decoders into the stdlib

stdlib/public/core/CMakeLists.txt

@@ -130,8 +130,11 @@ set(SWIFTLIB_ESSENTIAL
   StringUTF8.swift
   Substring.swift.gyb
   SwiftNativeNSArray.swift
+  UIntBuffer.swift
   UnavailableStringAPIs.swift.gyb
   Unicode.swift
+  UnicodeEncoding.swift
+  UnicodeParser.swift
   UnicodeScalar.swift
   UnicodeTrie.swift.gyb
   Unmanaged.swift
@@ -140,6 +143,9 @@ set(SWIFTLIB_ESSENTIAL
   UnsafeRawBufferPointer.swift.gyb
   UnsafePointer.swift.gyb
   UnsafeRawPointer.swift.gyb
+  UTFEncoding.swift
+  UTF8.swift
+  UTF16.swift
   WriteBackMutableSlice.swift
   )
 

stdlib/public/core/GroupInfo.json

@@ -23,9 +23,14 @@
     "StringUnicodeScalarView.swift",
     "Substring.swift",
     "Unicode.swift",
+    "UnicodeEncoding.swift",
+    "UnicodeParser.swift",
     "UnicodeScalar.swift",
     "UnicodeTrie.swift",
-    "UnavailableStringAPIs.swift"
+    "UnavailableStringAPIs.swift",
+    "UTFEncoding.swift",
+    "UTF8.swift",
+    "UTF16.swift"
   ],
   "Bool": [
     "Bool.swift"
@@ -52,6 +57,7 @@
     "Existential.swift",
     "WriteBackMutableSlice.swift",
     "UnfoldSequence.swift",
+    "UIntBuffer.swift",
     {
       "Type-erased": [
         "ExistentialCollection.swift"

stdlib/public/core/MutableCollection.swift

@@ -27,13 +27,6 @@ public protocol _MutableIndexable : _Indexable {
   // `Iterator` type from a minimal collection, but it is also used in
   // exposed places like as a constraint on `IndexingIterator`.
 
-  /// A type that represents a valid position in the collection.
-  ///
-  /// Valid indices consist of the position of every element and a
-  /// "past the end" position that's not valid for use as a subscript.
-  // TODO: swift-3-indexing-model - Index only needs to be comparable or must be comparable..?
-  associatedtype Index
-
   /// The position of the first element in a nonempty collection.
   ///
   /// If the collection is empty, `startIndex` is equal to `endIndex`.
@@ -56,16 +49,6 @@ public protocol _MutableIndexable : _Indexable {
   /// If the collection is empty, `endIndex` is equal to `startIndex`.
   var endIndex: Index { get }
 
-  // The declaration of _Element and subscript here is a trick used to
-  // break a cyclic conformance/deduction that Swift can't handle.  We
-  // need something other than a Collection.Iterator.Element that can
-  // be used as IndexingIterator<T>'s Element.  Here we arrange for
-  // the Collection itself to have an Element type that's deducible from
-  // its subscript.  Ideally we'd like to constrain this Element to be the same
-  // as Collection.Iterator.Element (see below), but we have no way of
-  // expressing it today.
-  associatedtype _Element
-
   /// Accesses the element at the specified position.
   ///
   /// For example, you can replace an element of an array by using its
@@ -86,10 +69,6 @@ public protocol _MutableIndexable : _Indexable {
   ///   `endIndex` property.
   subscript(position: Index) -> _Element { get set }
 
-  /// A collection that represents a contiguous subrange of the collection's
-  /// elements.
-  associatedtype SubSequence
-
   /// Accesses a contiguous subrange of the collection's elements.
   ///
   /// The accessed slice uses the same indices for the same elements as the

stdlib/public/core/Reverse.swift

@@ -34,6 +34,36 @@ extension MutableCollection where Self : BidirectionalCollection {
   }
 }
 
+/// An iterator that can be much faster than the iterator of a reversed slice.
+// TODO: See about using this in more places
+@_fixed_layout
+public struct _ReverseIndexingIterator<
+  Elements : BidirectionalCollection
+> : IteratorProtocol, Sequence {
+
+  @_inlineable
+  @inline(__always)
+  /// Creates an iterator over the given collection.
+  public /// @testable
+  init(_elements: Elements, _position: Elements.Index) {
+    self._elements = _elements
+    self._position = _position
+  }
+
+  @_inlineable
+  @inline(__always)
+  public mutating func next() -> Elements._Element? {
+    guard _fastPath(_position != _elements.startIndex) else { return nil }
+    _position = _elements.index(before: _position)
+    return _elements[_position]
+  }
+
+  @_versioned
+  internal let _elements: Elements
+  @_versioned
+  internal var _position: Elements.Index
+}
+
 // FIXME(ABI)#59 (Conditional Conformance): we should have just one type,
 // `ReversedCollection`, that has conditional conformances to
 // `RandomAccessCollection`, and possibly `MutableCollection` and

stdlib/public/core/Stride.swift.gyb

@@ -27,9 +27,6 @@ public protocol ${Self} : ${Conformance} {
 %   if Self == '_Strideable':
   /// A type that represents the distance between two values of `Self`.
   associatedtype Stride : SignedNumeric, Comparable
-%   else:
-  /// A type that represents the distance between two values of `Self`.
-  associatedtype Stride
 %   end
 
   /// Returns a stride `x` such that `self.advanced(by: x)` approximates

stdlib/public/core/UIntBuffer.swift

@@ -0,0 +1,206 @@
+//===--- UIntBuffer.swift - Bounded Collection of Unisigned Integer -------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  Stores a smaller unsigned integer type inside a larger one, with a limit of
+//  255 elements.
+//
+//===----------------------------------------------------------------------===//
+@_fixed_layout
+public struct _UIntBuffer<
+  Storage: UnsignedInteger & FixedWidthInteger, 
+  Element: UnsignedInteger & FixedWidthInteger
+> {
+  public var _storage: Storage
+  public var _bitCount: UInt8
+
+  @inline(__always)
+  public init(_storage: Storage, _bitCount: UInt8) {
+    self._storage = _storage
+    self._bitCount = _bitCount
+  }
+
+  @inline(__always)
+  public init(containing e: Element) {
+    _storage = Storage(extendingOrTruncating: e)
+    _bitCount = UInt8(extendingOrTruncating: Element.bitWidth)
+  }
+}
+
+extension _UIntBuffer : Sequence {
+  public typealias SubSequence = RangeReplaceableRandomAccessSlice<_UIntBuffer>
+
+  @_fixed_layout
+  public struct Iterator : IteratorProtocol, Sequence {
+    @inline(__always)
+    public init(_ x: _UIntBuffer) { _impl = x }
+
+    @inline(__always)
+    public mutating func next() -> Element? {
+      if _impl._bitCount == 0 { return nil }
+      defer {
+        _impl._storage = _impl._storage &>> Element.bitWidth
+        _impl._bitCount = _impl._bitCount &- _impl._elementWidth
+      }
+      return Element(extendingOrTruncating: _impl._storage)
+    }
+    public
+    var _impl: _UIntBuffer
+  }
+
+  @inline(__always)
+  public func makeIterator() -> Iterator {
+    return Iterator(self)
+  }
+}
+
+extension _UIntBuffer : Collection {
+  public typealias _Element = Element
+
+  public struct Index : Comparable {
+    @_versioned
+    var bitOffset: UInt8
+
+    @_versioned
+    init(bitOffset: UInt8) { self.bitOffset = bitOffset }
+
+    public static func == (lhs: Index, rhs: Index) -> Bool {
+      return lhs.bitOffset == rhs.bitOffset
+    }
+    public static func < (lhs: Index, rhs: Index) -> Bool {
+      return lhs.bitOffset < rhs.bitOffset
+    }
+  }
+
+  public var startIndex : Index {
+    @inline(__always)
+    get { return Index(bitOffset: 0) }
+  }
+
+  public var endIndex : Index {
+    @inline(__always)
+    get { return Index(bitOffset: _bitCount) }
+  }
+
+  @inline(__always)
+  public func index(after i: Index) -> Index {
+    return Index(bitOffset: i.bitOffset &+ _elementWidth)
+  }
+
+  @_versioned
+  internal var _elementWidth : UInt8 {
+    return UInt8(extendingOrTruncating: Element.bitWidth)
+  }
+
+  public subscript(i: Index) -> Element {
+    @inline(__always)
+    get {
+      return Element(extendingOrTruncating: _storage &>> i.bitOffset)
+    }
+  }
+}
+
+extension _UIntBuffer : BidirectionalCollection {
+  @inline(__always)
+  public func index(before i: Index) -> Index {
+    return Index(bitOffset: i.bitOffset &- _elementWidth)
+  }
+}
+
+extension _UIntBuffer : RandomAccessCollection {
+  public typealias Indices = DefaultRandomAccessIndices<_UIntBuffer>
+  public typealias IndexDistance = Int
+
+  @inline(__always)
+  public func index(_ i: Index, offsetBy n: IndexDistance) -> Index {
+    let x = IndexDistance(i.bitOffset) &+ n &* Element.bitWidth
+    return Index(bitOffset: UInt8(extendingOrTruncating: x))
+  }
+
+  @inline(__always)
+  public func distance(from i: Index, to j: Index) -> IndexDistance {
+    return (Int(j.bitOffset) &- Int(i.bitOffset)) / Element.bitWidth
+  }
+}
+
+extension FixedWidthInteger {
+  @inline(__always)
+  @_versioned
+  func _fullShiftLeft<N: FixedWidthInteger>(_ n: N) -> Self {
+    return (self &<< ((n &+ 1) &>> 1)) &<< (n &>> 1)
+  }
+  @inline(__always)
+  @_versioned
+  func _fullShiftRight<N: FixedWidthInteger>(_ n: N) -> Self {
+    return (self &>> ((n &+ 1) &>> 1)) &>> (n &>> 1)
+  }
+  @inline(__always)
+  @_versioned
+  static func _lowBits<N: FixedWidthInteger>(_ n: N) -> Self {
+    return ~((~0 as Self)._fullShiftLeft(n))
+  }
+}
+
+extension Range {
+  @inline(__always)
+  @_versioned
+  func _contains_(_ other: Range) -> Bool {
+    return other.clamped(to: self) == other
+  }
+}
+
+extension _UIntBuffer : RangeReplaceableCollection {
+  @inline(__always)
+  public init() {
+    _storage = 0
+    _bitCount = 0
+  }
+
+  public var capacity: Int {
+    return Storage.bitWidth / Element.bitWidth
+  }
+
+  @inline(__always)
+  public mutating func append(_ newElement: Element) {
+    _debugPrecondition(count + 1 <= capacity)
+    _storage |= Storage(newElement) &<< _bitCount
+    _bitCount = _bitCount &+ _elementWidth
+  }
+
+  @inline(__always)
+  public mutating func replaceSubrange<C: Collection>(
+    _ target: Range<Index>, with replacement: C
+  ) where C._Element == Element {
+    _debugPrecondition(
+      (0..<_bitCount)._contains_(
+        target.lowerBound.bitOffset..<target.upperBound.bitOffset))
+
+    let replacement1 = _UIntBuffer(replacement)
+
+    let targetCount = distance(
+      from: target.lowerBound, to: target.upperBound)
+    let growth = replacement1.count &- targetCount
+    _debugPrecondition(count + growth <= capacity)
+
+    let headCount = distance(from: startIndex, to: target.lowerBound)
+    let tailOffset = distance(from: startIndex, to: target.upperBound)
+
+    let w = Element.bitWidth
+    let headBits = _storage & ._lowBits(headCount &* w)
+    let tailBits = _storage._fullShiftRight(tailOffset &* w)
+
+    _storage = headBits
+    _storage |= replacement1._storage &<< (headCount &* w)
+    _storage |= tailBits &<< ((tailOffset &+ growth) &* w)
+    _bitCount = UInt8(
+      extendingOrTruncating: IndexDistance(_bitCount) &+ growth &* w)
+  }
+}