Merge pull request #9554 from apple/stringprotocol-4

Implement SE-0163

Author: tkremenek

stdlib/private/StdlibUnicodeUnittest/Collation.swift

@@ -13,7 +13,7 @@
 import StdlibUnittest
 
 struct CollationTableEntry {
-  let scalars: [UnicodeScalar]
+  let scalars: [Unicode.Scalar]
   let collationElements: [UInt64]
   let comment: String
 
@@ -22,7 +22,7 @@ struct CollationTableEntry {
     _ collationElements: [UInt64],
     _ comment: String
   ) {
-    self.scalars = scalars.map { UnicodeScalar($0)! }
+    self.scalars = scalars.map { Unicode.Scalar($0)! }
     self.collationElements = collationElements
     self.comment = comment
   }
@@ -212,9 +212,9 @@ extension HashableArray : ExpressibleByArrayLiteral {
   }
 }
 
-let ducetExtract: [HashableArray<UnicodeScalar> : CollationTableEntry] = {
+let ducetExtract: [HashableArray<Unicode.Scalar> : CollationTableEntry] = {
   () in
-  var result: [HashableArray<UnicodeScalar> : CollationTableEntry] = [:]
+  var result: [HashableArray<Unicode.Scalar> : CollationTableEntry] = [:]
   for entry in ducetExtractData {
     result[HashableArray(entry.scalars)] = entry
   }
@@ -232,7 +232,7 @@ extension String {
   internal var _collationElements: [UInt64] {
     var result: [UInt64] = []
     for us in self.unicodeScalars {
-      let scalars: HashableArray<UnicodeScalar> = [us]
+      let scalars: HashableArray<Unicode.Scalar> = [us]
       let collationElements = ducetExtract[scalars]!.collationElements
       if collationElements[0] != 0 {
         result += collationElements

stdlib/private/StdlibUnicodeUnittest/StdlibUnicodeUnittest.swift

@@ -27,8 +27,8 @@ public struct UTFTest {
   public let string: String
   public let utf8: [UInt8]
   public let utf16: [UInt16]
-  public let unicodeScalars: [UnicodeScalar]
-  public let unicodeScalarsRepairedTail: [UnicodeScalar]
+  public let unicodeScalars: [Unicode.Scalar]
+  public let unicodeScalarsRepairedTail: [Unicode.Scalar]
   public let flags: Flags
   public let loc: SourceLoc
 
@@ -52,9 +52,9 @@ public struct UTFTest {
     self.string = string
     self.utf8 = utf8
     self.utf16 = utf16
-    self.unicodeScalars = scalars.map { UnicodeScalar($0)! }
+    self.unicodeScalars = scalars.map { Unicode.Scalar($0)! }
     self.unicodeScalarsRepairedTail =
-      scalarsRepairedTail.map { UnicodeScalar($0)! }
+      scalarsRepairedTail.map { Unicode.Scalar($0)! }
     self.flags = flags
     self.loc = SourceLoc(file, line, comment: "test data")
   }

stdlib/private/StdlibUnittest/StdlibUnittest.swift.gyb

@@ -589,7 +589,7 @@ func _stdlib_getline() -> String? {
       }
       return String._fromWellFormedCodeUnitSequence(UTF8.self, input: result)
     }
-    if c == CInt(UnicodeScalar("\n").value) {
+    if c == CInt(Unicode.Scalar("\n").value) {
       return String._fromWellFormedCodeUnitSequence(UTF8.self, input: result)
     }
     result.append(UInt8(c))

stdlib/private/SwiftPrivate/IO.swift

@@ -25,7 +25,7 @@ public struct _FDInputStream {
 
   public mutating func getline() -> String? {
     if let newlineIndex =
-      _buffer[0..<_bufferUsed].index(of: UInt8(UnicodeScalar("\n").value)) {
+      _buffer[0..<_bufferUsed].index(of: UInt8(Unicode.Scalar("\n").value)) {
       let result = String._fromWellFormedCodeUnitSequence(
         UTF8.self, input: _buffer[0..<newlineIndex])
       _buffer.removeSubrange(0...newlineIndex)

stdlib/public/SDK/Foundation/CharacterSet.swift

@@ -14,13 +14,13 @@
 import CoreFoundation
 import _SwiftCoreFoundationOverlayShims
 
-private func _utfRangeToCFRange(_ inRange : Range<UnicodeScalar>) -> CFRange {
+private func _utfRangeToCFRange(_ inRange : Range<Unicode.Scalar>) -> CFRange {
     return CFRange(
         location: Int(inRange.lowerBound.value),
         length: Int(inRange.upperBound.value - inRange.lowerBound.value))
 }
 
-private func _utfRangeToCFRange(_ inRange : ClosedRange<UnicodeScalar>) -> CFRange {
+private func _utfRangeToCFRange(_ inRange : ClosedRange<Unicode.Scalar>) -> CFRange {
     return CFRange(
         location: Int(inRange.lowerBound.value),
         length: Int(inRange.upperBound.value - inRange.lowerBound.value + 1))
@@ -104,7 +104,7 @@ fileprivate final class _CharacterSetStorage : Hashable {
 
     // MARK: Mutable functions
 
-    fileprivate func insert(charactersIn range: Range<UnicodeScalar>) {
+    fileprivate func insert(charactersIn range: Range<Unicode.Scalar>) {
         switch _backing {
         case .immutable(let cs):
             let r = CFCharacterSetCreateMutableCopy(nil, cs)!
@@ -115,7 +115,7 @@ fileprivate final class _CharacterSetStorage : Hashable {
         }
     }
 
-    fileprivate func insert(charactersIn range: ClosedRange<UnicodeScalar>) {
+    fileprivate func insert(charactersIn range: ClosedRange<Unicode.Scalar>) {
         switch _backing {
         case .immutable(let cs):
             let r = CFCharacterSetCreateMutableCopy(nil, cs)!
@@ -126,7 +126,7 @@ fileprivate final class _CharacterSetStorage : Hashable {
         }
     }
 
-    fileprivate func remove(charactersIn range: Range<UnicodeScalar>) {
+    fileprivate func remove(charactersIn range: Range<Unicode.Scalar>) {
         switch _backing {
         case .immutable(let cs):
             let r = CFCharacterSetCreateMutableCopy(nil, cs)!
@@ -137,7 +137,7 @@ fileprivate final class _CharacterSetStorage : Hashable {
         }
     }
 
-    fileprivate func remove(charactersIn range: ClosedRange<UnicodeScalar>) {
+    fileprivate func remove(charactersIn range: ClosedRange<Unicode.Scalar>) {
         switch _backing {
         case .immutable(let cs):
             let r = CFCharacterSetCreateMutableCopy(nil, cs)!
@@ -186,28 +186,28 @@ fileprivate final class _CharacterSetStorage : Hashable {
     // MARK: SetAlgebraType
 
     @discardableResult
-    fileprivate func insert(_ character: UnicodeScalar) -> (inserted: Bool, memberAfterInsert: UnicodeScalar) {
-        insert(charactersIn: character..<UnicodeScalar(character.value + 1)!)
+    fileprivate func insert(_ character: Unicode.Scalar) -> (inserted: Bool, memberAfterInsert: Unicode.Scalar) {
+        insert(charactersIn: character..<Unicode.Scalar(character.value + 1)!)
         // TODO: This should probably return the truth, but figuring it out requires two calls into NSCharacterSet
         return (true, character)
     }
 
     @discardableResult
-    fileprivate func update(with character: UnicodeScalar) -> UnicodeScalar? {
+    fileprivate func update(with character: Unicode.Scalar) -> Unicode.Scalar? {
         insert(character)
         // TODO: This should probably return the truth, but figuring it out requires two calls into NSCharacterSet
         return character
     }
 
     @discardableResult
-    fileprivate func remove(_ character: UnicodeScalar) -> UnicodeScalar? {
+    fileprivate func remove(_ character: Unicode.Scalar) -> Unicode.Scalar? {
         // TODO: Add method to CFCharacterSet to do this in one call
-        let result : UnicodeScalar? = contains(character) ? character : nil
-        remove(charactersIn: character..<UnicodeScalar(character.value + 1)!)
+        let result : Unicode.Scalar? = contains(character) ? character : nil
+        remove(charactersIn: character..<Unicode.Scalar(character.value + 1)!)
         return result
     }
 
-    fileprivate func contains(_ member: UnicodeScalar) -> Bool {
+    fileprivate func contains(_ member: Unicode.Scalar) -> Bool {
         switch _backing {
         case .immutable(let cs):
             return CFCharacterSetIsLongCharacterMember(cs, member.value)
@@ -374,15 +374,15 @@ public struct CharacterSet : ReferenceConvertible, Equatable, Hashable, SetAlgeb
 
     /// Initialize with a range of integers.
     ///
-    /// It is the caller's responsibility to ensure that the values represent valid `UnicodeScalar` values, if that is what is desired.
-    public init(charactersIn range: Range<UnicodeScalar>) {
+    /// It is the caller's responsibility to ensure that the values represent valid `Unicode.Scalar` values, if that is what is desired.
+    public init(charactersIn range: Range<Unicode.Scalar>) {
         _storage = _CharacterSetStorage(immutableReference: CFCharacterSetCreateWithCharactersInRange(nil, _utfRangeToCFRange(range)))
     }
 
     /// Initialize with a closed range of integers.
     ///
-    /// It is the caller's responsibility to ensure that the values represent valid `UnicodeScalar` values, if that is what is desired.
-    public init(charactersIn range: ClosedRange<UnicodeScalar>) {
+    /// It is the caller's responsibility to ensure that the values represent valid `Unicode.Scalar` values, if that is what is desired.
+    public init(charactersIn range: ClosedRange<Unicode.Scalar>) {
         _storage = _CharacterSetStorage(immutableReference: CFCharacterSetCreateWithCharactersInRange(nil, _utfRangeToCFRange(range)))
     }
 
@@ -569,8 +569,8 @@ public struct CharacterSet : ReferenceConvertible, Equatable, Hashable, SetAlgeb
 
     /// Insert a range of integer values in the `CharacterSet`.
     ///
-    /// It is the caller's responsibility to ensure that the values represent valid `UnicodeScalar` values, if that is what is desired.
-    public mutating func insert(charactersIn range: Range<UnicodeScalar>) {
+    /// It is the caller's responsibility to ensure that the values represent valid `Unicode.Scalar` values, if that is what is desired.
+    public mutating func insert(charactersIn range: Range<Unicode.Scalar>) {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
@@ -579,24 +579,24 @@ public struct CharacterSet : ReferenceConvertible, Equatable, Hashable, SetAlgeb
 
     /// Insert a closed range of integer values in the `CharacterSet`.
     ///
-    /// It is the caller's responsibility to ensure that the values represent valid `UnicodeScalar` values, if that is what is desired.
-    public mutating func insert(charactersIn range: ClosedRange<UnicodeScalar>) {
+    /// It is the caller's responsibility to ensure that the values represent valid `Unicode.Scalar` values, if that is what is desired.
+    public mutating func insert(charactersIn range: ClosedRange<Unicode.Scalar>) {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
         _storage.insert(charactersIn: range)
     }
 
     /// Remove a range of integer values from the `CharacterSet`.
-    public mutating func remove(charactersIn range: Range<UnicodeScalar>) {
+    public mutating func remove(charactersIn range: Range<Unicode.Scalar>) {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
         _storage.remove(charactersIn: range)
     }
 
     /// Remove a closed range of integer values from the `CharacterSet`.
-    public mutating func remove(charactersIn range: ClosedRange<UnicodeScalar>) {
+    public mutating func remove(charactersIn range: ClosedRange<Unicode.Scalar>) {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
@@ -631,42 +631,42 @@ public struct CharacterSet : ReferenceConvertible, Equatable, Hashable, SetAlgeb
     // MARK: -
     // MARK: SetAlgebraType
 
-    /// Insert a `UnicodeScalar` representation of a character into the `CharacterSet`.
+    /// Insert a `Unicode.Scalar` representation of a character into the `CharacterSet`.
     ///
-    /// `UnicodeScalar` values are available on `Swift.String.UnicodeScalarView`.
+    /// `Unicode.Scalar` values are available on `Swift.String.UnicodeScalarView`.
     @discardableResult
-    public mutating func insert(_ character: UnicodeScalar) -> (inserted: Bool, memberAfterInsert: UnicodeScalar) {
+    public mutating func insert(_ character: Unicode.Scalar) -> (inserted: Bool, memberAfterInsert: Unicode.Scalar) {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
         return _storage.insert(character)
     }
 
-    /// Insert a `UnicodeScalar` representation of a character into the `CharacterSet`.
+    /// Insert a `Unicode.Scalar` representation of a character into the `CharacterSet`.
     ///
-    /// `UnicodeScalar` values are available on `Swift.String.UnicodeScalarView`.
+    /// `Unicode.Scalar` values are available on `Swift.String.UnicodeScalarView`.
     @discardableResult
-    public mutating func update(with character: UnicodeScalar) -> UnicodeScalar? {
+    public mutating func update(with character: Unicode.Scalar) -> Unicode.Scalar? {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
         return _storage.update(with: character)
     }
 
 
-    /// Remove a `UnicodeScalar` representation of a character from the `CharacterSet`.
+    /// Remove a `Unicode.Scalar` representation of a character from the `CharacterSet`.
     ///
-    /// `UnicodeScalar` values are available on `Swift.String.UnicodeScalarView`.
+    /// `Unicode.Scalar` values are available on `Swift.String.UnicodeScalarView`.
     @discardableResult
-    public mutating func remove(_ character: UnicodeScalar) -> UnicodeScalar? {
+    public mutating func remove(_ character: Unicode.Scalar) -> Unicode.Scalar? {
         if !isKnownUniquelyReferenced(&_storage) {
             _storage = _storage.mutableCopy()
         }
         return _storage.remove(character)
     }
 
-    /// Test for membership of a particular `UnicodeScalar` in the `CharacterSet`.
-    public func contains(_ member: UnicodeScalar) -> Bool {
+    /// Test for membership of a particular `Unicode.Scalar` in the `CharacterSet`.
+    public func contains(_ member: Unicode.Scalar) -> Bool {
         return _storage.contains(member)
     }
 

stdlib/public/SwiftOnoneSupport/SwiftOnoneSupport.swift

@@ -147,7 +147,7 @@ struct _Prespecialize {
     // Force pre-specialization of arrays with elements of different
     // character and unicode scalar types.
     _createArrayUser("a" as Character)
-    _createArrayUser("a" as UnicodeScalar)
+    _createArrayUser("a" as Unicode.Scalar)
     _createArrayUserWithoutSorting("a".utf8)
     _createArrayUserWithoutSorting("a".utf16)
     _createArrayUserWithoutSorting("a".unicodeScalars)

stdlib/public/core/ASCII.swift

@@ -0,0 +1,88 @@
+//===--- ASCII.swift ------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+extension Unicode {
+  @_fixed_layout
+  public enum ASCII {}
+}
+
+extension Unicode.ASCII : Unicode.Encoding {
+  public typealias CodeUnit = UInt8
+  public typealias EncodedScalar = CollectionOfOne<CodeUnit>
+
+  public static var encodedReplacementCharacter : EncodedScalar {
+    return EncodedScalar(0x1a) // U+001A SUBSTITUTE; best we can do for ASCII
+  }
+
+  @inline(__always)
+  @_inlineable
+  public static func _isScalar(_ x: CodeUnit) -> Bool {
+    return true
+  }
+
+  @inline(__always)
+  @_inlineable
+  public static func decode(_ source: EncodedScalar) -> Unicode.Scalar {
+    return Unicode.Scalar(_unchecked: UInt32(
+        source.first._unsafelyUnwrappedUnchecked))
+  }
+  
+  @inline(__always)
+  @_inlineable
+  public static func encode(
+    _ source: Unicode.Scalar
+  ) -> EncodedScalar? {
+    guard source.value < (1&<<7) else { return nil }
+    return EncodedScalar(UInt8(extendingOrTruncating: source.value))
+  }
+
+  @inline(__always)
+  public static func transcode<FromEncoding : Unicode.Encoding>(
+    _ content: FromEncoding.EncodedScalar, from _: FromEncoding.Type
+  ) -> EncodedScalar? {
+    if _fastPath(FromEncoding.self == UTF16.self) {
+      let c = unsafeBitCast(content, to: UTF16.EncodedScalar.self)
+      guard (c._storage & 0xFF80 == 0) else { return nil }
+      return EncodedScalar(CodeUnit(c._storage & 0x7f))
+    }
+    else if _fastPath(FromEncoding.self == UTF8.self) {
+      let c = unsafeBitCast(content, to: UTF8.EncodedScalar.self)
+      guard (c._storage & 0x80 == 0) else { return nil }
+      return EncodedScalar(CodeUnit(c._storage & 0x7f))
+    }
+    return encode(FromEncoding.decode(content))
+  }
+
+  public struct Parser {
+    public init() { }
+  }
+  
+  public typealias ForwardParser = Parser
+  public typealias ReverseParser = Parser
+}
+
+extension Unicode.ASCII.Parser : Unicode.Parser {
+  public typealias Encoding = Unicode.ASCII
+
+  /// Parses a single Unicode scalar value from `input`.
+  public mutating func parseScalar<I : IteratorProtocol>(
+    from input: inout I
+  ) -> Unicode.ParseResult<Encoding.EncodedScalar>
+  where I.Element == Encoding.CodeUnit {
+    let n = input.next()
+    if _fastPath(n != nil), let x = n {
+      guard _fastPath(Int8(extendingOrTruncating: x) >= 0)
+      else { return .error(length: 1) }
+      return .valid(Unicode.ASCII.EncodedScalar(x))
+    }
+    return .emptyInput
+  }
+}