[String.Index] Deprecate encodedOffset var/init

String.Index has an encodedOffset-based initializer and computed
property that exists for serialization purposes. It was documented as
UTF-16 in the SE proposal introducing it, which was String's
underlying encoding at the time, but the dream of String even then was
to abstract away whatever encoding happend to be used.

Serialization needs an explicit encoding for serialized indices to
make sense: the offsets need to align with the view. With String
utilizing UTF-8 encoding for native contents in Swift 5, serialization
isn't necessarily the most efficient in UTF-16.

Furthermore, the majority of usage of encodedOffset in the wild is
buggy and operates under the assumption that a UTF-16 code unit was a
Swift Character, which isn't even valid if the String is known to be
all-ASCII (because CR-LF).

This change introduces a set of new initializers and methods with an
explicit encoding (through the type of the provided view) for
serialization. It also adds an init and method for String's default
view, which most code in the wild should migrate to if it eliminates
their underlying bugs.

Author: milseman

stdlib/public/Darwin/Foundation/URLComponents.swift

@@ -192,16 +192,8 @@ public struct URLComponents : ReferenceConvertible, Hashable, Equatable, _Mutabl
 
     @available(macOS 10.11, iOS 9.0, *)
     private func _toStringRange(_ r : NSRange) -> Range<String.Index>? {
-        guard r.location != NSNotFound else { return nil }
-        
-        let utf16Start = String.UTF16View.Index(encodedOffset: r.location)
-        let utf16End = String.UTF16View.Index(encodedOffset: r.location + r.length)
-
         guard let s = self.string else { return nil }
-        guard let start = String.Index(utf16Start, within: s) else { return nil }
-        guard let end = String.Index(utf16End, within: s) else { return nil }
-        
-        return start..<end
+        return Range(r, in: s)
     }
 
     /// Returns the character range of the scheme in the string returned by `var string`.

stdlib/public/Darwin/NaturalLanguage/NLTagger.swift

@@ -18,15 +18,15 @@ extension NLTagger {
   @nonobjc
   public func tokenRange(at index: String.Index, unit: NLTokenUnit) -> Range<String.Index> {
     let str = self.string ?? ""
-    let characterIndex = index.encodedOffset
+    let characterIndex = index.offset(within: str.utf16)
     let nsrange = self.__tokenRange(at: characterIndex, unit: unit)
     return Range(nsrange, in: str)!
   }
 
   @nonobjc
   public func tag(at index: String.Index, unit: NLTokenUnit, scheme: NLTagScheme) -> (NLTag?, Range<String.Index>) {
     let str = self.string ?? ""
-    let characterIndex = index.encodedOffset
+    let characterIndex = index.offset(within: str.utf16)
     let rangePointer = NSRangePointer.allocate(capacity: 1)
     rangePointer.initialize(to: NSMakeRange(0, 0))
     let tag = self.__tag(at: characterIndex, unit: unit, scheme: scheme, tokenRange: rangePointer)

stdlib/public/Darwin/NaturalLanguage/NLTokenizer.swift

@@ -18,7 +18,7 @@ extension NLTokenizer {
   @nonobjc
   public func tokenRange(at index: String.Index) -> Range<String.Index> {
     let str = self.string ?? ""
-    let characterIndex = index.encodedOffset
+    let characterIndex = index.offset(within: str.utf16)
     let nsrange = self.__tokenRange(at:characterIndex)
     return Range(nsrange, in: str)!
   }

stdlib/public/core/String.swift

@@ -925,8 +925,8 @@ internal func _fastWithNormalizedCodeUnitsImpl(
   var icuInputBuffer = icuInputBuffer
   var icuOutputBuffer = icuOutputBuffer
 
-  var index = String.Index(encodedOffset: 0)
-  let cachedEndIndex = String.Index(encodedOffset: sourceBuffer.count)
+  var index = String.Index(_encodedOffset: 0)
+  let cachedEndIndex = String.Index(_encodedOffset: sourceBuffer.count)
 
   var hasBufferOwnership = false
 

stdlib/public/core/StringBreadcrumbs.swift

@@ -79,8 +79,8 @@ extension _StringBreadcrumbs {
   internal func getBreadcrumb(
     forIndex idx: String.Index
   ) -> (lowerBound: String.Index, offset: Int) {
-    var lowerBound = idx.encodedOffset / 3 / stride
-    var upperBound = Swift.min(1 + (idx.encodedOffset / stride), crumbs.count)
+    var lowerBound = idx._encodedOffset / 3 / stride
+    var upperBound = Swift.min(1 + (idx._encodedOffset / stride), crumbs.count)
     _internalInvariant(crumbs[lowerBound] <= idx)
     _internalInvariant(upperBound == crumbs.count || crumbs[upperBound] >= idx)
 

stdlib/public/core/StringCharacterView.swift

@@ -66,9 +66,9 @@ extension String: BidirectionalCollection {
 
     // TODO: known-ASCII fast path, single-scalar-grapheme fast path, etc.
     let stride = _characterStride(startingAt: i)
-    let nextOffset = i.encodedOffset &+ stride
+    let nextOffset = i._encodedOffset &+ stride
     let nextStride = _characterStride(
-      startingAt: Index(encodedOffset: nextOffset))
+      startingAt: Index(_encodedOffset: nextOffset))
 
     return Index(
       encodedOffset: nextOffset, characterStride: nextStride)
@@ -84,7 +84,7 @@ extension String: BidirectionalCollection {
 
     // TODO: known-ASCII fast path, single-scalar-grapheme fast path, etc.
     let stride = _characterStride(endingAt: i)
-    let priorOffset = i.encodedOffset &- stride
+    let priorOffset = i._encodedOffset &- stride
     return Index(encodedOffset: priorOffset, characterStride: stride)
   }
   /// Returns an index that is the specified distance from the given index.
@@ -198,7 +198,7 @@ extension String: BidirectionalCollection {
       let i = _guts.scalarAlign(i)
       let distance = _characterStride(startingAt: i)
       return _guts.errorCorrectedCharacter(
-        startingAt: i.encodedOffset, endingAt: i.encodedOffset &+ distance)
+        startingAt: i._encodedOffset, endingAt: i._encodedOffset &+ distance)
     }
   }
 
@@ -209,14 +209,14 @@ extension String: BidirectionalCollection {
 
     if i == endIndex { return 0 }
 
-    return _guts._opaqueCharacterStride(startingAt: i.encodedOffset)
+    return _guts._opaqueCharacterStride(startingAt: i._encodedOffset)
   }
 
   @inlinable @inline(__always)
   internal func _characterStride(endingAt i: Index) -> Int {
     if i == startIndex { return 0 }
 
-    return _guts._opaqueCharacterStride(endingAt: i.encodedOffset)
+    return _guts._opaqueCharacterStride(endingAt: i._encodedOffset)
   }
 }
 

stdlib/public/core/StringComparison.swift

@@ -328,10 +328,10 @@ extension _StringGutsSlice {
     if _fastPath(self.isFastUTF8 && other.isFastUTF8) {
       return self.withFastUTF8 { leftUTF8 in
         other.withFastUTF8 { rightUTF8 in
-          let leftStartIndex = String.Index(encodedOffset: 0)
-          let rightStartIndex = String.Index(encodedOffset: 0)
-          let leftEndIndex = String.Index(encodedOffset: leftUTF8.count)
-          let rightEndIndex = String.Index(encodedOffset: rightUTF8.count)
+          let leftStartIndex = String.Index(_encodedOffset: 0)
+          let rightStartIndex = String.Index(_encodedOffset: 0)
+          let leftEndIndex = String.Index(_encodedOffset: leftUTF8.count)
+          let rightEndIndex = String.Index(_encodedOffset: rightUTF8.count)
           return _normalizedCompareImpl(
             left_outputBuffer: _castOutputBuffer(&left_output),
             left_icuInputBuffer: _castOutputBuffer(&left_icuInput),

stdlib/public/core/StringGraphemeBreaking.swift

@@ -156,7 +156,7 @@ extension _StringGuts {
   internal func isOnGraphemeClusterBoundary(_ i: String.Index) -> Bool {
     guard i.transcodedOffset == 0 else { return false }
 
-    let offset = i.encodedOffset
+    let offset = i._encodedOffset
     if offset == 0 || offset == self.count { return true }
 
     guard isOnUnicodeScalarBoundary(i) else { return false }
@@ -197,7 +197,7 @@ extension _StringGuts {
     let count = _object.largeCount
     let cocoa = _object.cocoaObject
 
-    let startIdx = String.Index(encodedOffset: i)
+    let startIdx = String.Index(_encodedOffset: i)
     let (sc1, len) = foreignErrorCorrectedScalar(startingAt: startIdx)
     if i &+ len == count {
       // Last scalar is last grapheme
@@ -263,7 +263,7 @@ extension _StringGuts {
     let count = _object.largeCount
     let cocoa = _object.cocoaObject
 
-    let endIdx = String.Index(encodedOffset: i)
+    let endIdx = String.Index(_encodedOffset: i)
     let (sc2, len) = foreignErrorCorrectedScalar(endingAt: endIdx)
     if i &- len == 0 {
       // First scalar is first grapheme

stdlib/public/core/StringGuts.swift

@@ -274,11 +274,11 @@ extension _StringGuts {
 
   @inlinable
   internal var startIndex: String.Index {
-    @inline(__always) get { return Index(encodedOffset: 0) }
+    @inline(__always) get { return Index(_encodedOffset: 0) }
   }
   @inlinable
   internal var endIndex: String.Index {
-    @inline(__always) get { return Index(encodedOffset: self.count) }
+    @inline(__always) get { return Index(_encodedOffset: self.count) }
   }
 }
 

stdlib/public/core/StringGutsRangeReplaceable.swift

@@ -218,8 +218,8 @@ extension _StringGuts {
   }
 
   internal mutating func remove(from lower: Index, to upper: Index) {
-    let lowerOffset = lower.encodedOffset
-    let upperOffset = upper.encodedOffset
+    let lowerOffset = lower._encodedOffset
+    let upperOffset = upper._encodedOffset
     _internalInvariant(lower.transcodedOffset == 0 && upper.transcodedOffset == 0)
     _internalInvariant(lowerOffset <= upperOffset && upperOffset <= self.count)
 
@@ -279,16 +279,16 @@ extension _StringGuts {
     isASCII: Bool
   ) {
     let neededCapacity =
-      bounds.lowerBound.encodedOffset
-      + codeUnits.count + (self.count - bounds.upperBound.encodedOffset)
+      bounds.lowerBound._encodedOffset
+      + codeUnits.count + (self.count - bounds.upperBound._encodedOffset)
     reserveCapacity(neededCapacity)
 
     _internalInvariant(bounds.lowerBound.transcodedOffset == 0)
     _internalInvariant(bounds.upperBound.transcodedOffset == 0)
 
     _object.nativeStorage.replace(
-      from: bounds.lowerBound.encodedOffset,
-      to: bounds.upperBound.encodedOffset,
+      from: bounds.lowerBound._encodedOffset,
+      to: bounds.upperBound._encodedOffset,
       with: codeUnits)
     self = _StringGuts(_object.nativeStorage)
   }
@@ -300,16 +300,16 @@ extension _StringGuts {
     let replCount = codeUnits.count
 
     let neededCapacity =
-      bounds.lowerBound.encodedOffset
-      + replCount + (self.count - bounds.upperBound.encodedOffset)
+      bounds.lowerBound._encodedOffset
+      + replCount + (self.count - bounds.upperBound._encodedOffset)
     reserveCapacity(neededCapacity)
 
     _internalInvariant(bounds.lowerBound.transcodedOffset == 0)
     _internalInvariant(bounds.upperBound.transcodedOffset == 0)
 
     _object.nativeStorage.replace(
-      from: bounds.lowerBound.encodedOffset,
-      to: bounds.upperBound.encodedOffset,
+      from: bounds.lowerBound._encodedOffset,
+      to: bounds.upperBound._encodedOffset,
       with: codeUnits,
       replacementCount: replCount)
     self = _StringGuts(_object.nativeStorage)

stdlib/public/core/StringGutsSlice.swift

@@ -74,8 +74,8 @@ internal struct _StringGutsSlice {
   @inlinable
   internal var range: Range<String.Index> {
     @inline(__always) get {
-      return String.Index(encodedOffset: _offsetRange.lowerBound)
-         ..< String.Index(encodedOffset: _offsetRange.upperBound)
+      return String.Index(_encodedOffset: _offsetRange.lowerBound)
+         ..< String.Index(_encodedOffset: _offsetRange.upperBound)
     }
   }