[stdlib] Make String.Index(_:within:) initializers more permissive

In Swift 5.6 and below, (broken) code that acquired indices from a
UTF-16-encoded string bridged from Cocoa and kept using them after a
`makeContiguousUTF8` call (or other mutation) may have appeared to be
working correctly as long as the string was ASCII.

Since swiftlang#41417, the
`String(_:within:)` initializers recognize miscoded indices and reject
them by returning nil. This is technically correct, but it
unfortunately may be a binary compatibility issue, as these used to
return non-nil in previous versions.

Mitigate this issue by accepting UTF-16 indices on a UTF-8 string,
transcoding their offset as needed. (Attempting to use an UTF-8 index
on a UTF-16 string is still rejected — we do not implicitly convert
strings in that direction.)

rdar://89369680

Author: lorentey

stdlib/public/core/StringGuts.swift

@@ -333,23 +333,56 @@ extension _StringGuts {
     i._hasMatchingEncoding(isUTF8: isUTF8)
   }
 
-  /// Return an index whose encoding can be assumed to match that of `self`.
+  /// Return an index whose encoding can be assumed to match that of `self`,
+  /// trapping if `i` has an incompatible encoding.
+  ///
+  /// If `i` is UTF-8 encoded, but `self` is an UTF-16 string, then trap.
+  ///
+  /// If `i` is UTF-16 encoded, but `self` is an UTF-8 string, then transcode
+  /// `i`'s offset to UTF-8 and return the resulting index. This allows the use
+  /// of indices from a bridged Cocoa string after the string has been converted
+  /// to a native Swift string. (Such indices are technically still considered
+  /// invalid, but we allow this specific case to keep compatibility with
+  /// existing code that assumes otherwise.)
   ///
   /// Detecting an encoding mismatch isn't always possible -- older binaries did
   /// not set the flags that this method relies on. However, false positives
   /// cannot happen: if this method detects a mismatch, then it is guaranteed to
   /// be a real one.
   @_alwaysEmitIntoClient
   @inline(__always)
-  internal func ensureMatchingEncoding(_ i: String.Index) -> String.Index {
+  internal func ensureMatchingEncoding(_ i: Index) -> Index {
     if _fastPath(hasMatchingEncoding(i)) { return i }
+    if let i = _slowEnsureMatchingEncoding(i) { return i }
+    _preconditionFailure("Invalid string index")
+  }
+
+  /// Return an index that corresponds to the same position as `i`, but whose
+  /// encoding can be assumed to match that of `self`, returning `nil` if `i`
+  /// has incompatible encoding.
+  ///
+  /// If `i` is UTF-8 encoded, but `self` is an UTF-16 string, then return nil.
+  ///
+  /// If `i` is UTF-16 encoded, but `self` is an UTF-8 string, then transcode
+  /// `i`'s offset to UTF-8 and return the resulting index. This allows the use
+  /// of indices from a bridged Cocoa string after the string has been converted
+  /// to a native Swift string. (Such indices are technically still considered
+  /// invalid, but we allow this specific case to keep compatibility with
+  /// existing code that assumes otherwise.)
+  ///
+  /// Detecting an encoding mismatch isn't always possible -- older binaries did
+  /// not set the flags that this method relies on. However, false positives
+  /// cannot happen: if this method detects a mismatch, then it is guaranteed to
+  /// be a real one.
+  internal func ensureMatchingEncodingNoTrap(_ i: Index) -> Index? {
+    if hasMatchingEncoding(i) { return i }
     return _slowEnsureMatchingEncoding(i)
   }
 
   @_alwaysEmitIntoClient
   @inline(never)
   @_effects(releasenone)
-  internal func _slowEnsureMatchingEncoding(_ i: String.Index) -> String.Index {
+  internal func _slowEnsureMatchingEncoding(_ i: Index) -> Index? {
     guard isUTF8 else {
       // Attempt to use an UTF-8 index on a UTF-16 string. Strings don't usually
       // get converted to UTF-16 storage, so it seems okay to trap in this case
@@ -367,7 +400,7 @@ extension _StringGuts {
       //
       // This trap can never trigger on OSes that have stdlibs <= 5.6, because
       // those versions never set the `isKnownUTF16` flag in `_StringObject`.
-      _preconditionFailure("Invalid string index")
+      return nil
     }
     // Attempt to use an UTF-16 index on a UTF-8 string.
     //
@@ -383,10 +416,14 @@ extension _StringGuts {
     // FIXME: Consider emitting a runtime warning here.
     // FIXME: Consider performing a linked-on-or-after check & trapping if the
     // client executable was built on some particular future Swift release.
-    let utf16 = String(self).utf16
-    let base = utf16.index(utf16.startIndex, offsetBy: i._encodedOffset)
-    if i.transcodedOffset == 0 { return base }
-    return base.encoded(offsetBy: i.transcodedOffset)._knownUTF8
+    let utf16 = String.UTF16View(self)
+    var r = utf16.index(utf16.startIndex, offsetBy: i._encodedOffset)
+    if i.transcodedOffset != 0 {
+      r = r.encoded(offsetBy: i.transcodedOffset)
+    } else {
+      r = r._copyingAlignment(from: i)
+    }
+    return r._knownUTF8
   }
 }
 

stdlib/public/core/StringIndex.swift

@@ -365,6 +365,14 @@ extension String.Index {
   }
 }
 
+extension String.Index {
+  @_alwaysEmitIntoClient // Swift 5.7
+  internal func _copyingAlignment(from index: Self) -> Self {
+    let mask = Self.__scalarAlignmentBit | Self.__characterAlignmentBit
+    return Self((_rawBits & ~mask) | (index._rawBits & mask))
+  }
+}
+
 // ### Index Encoding
 //
 // Swift 5.7 introduced bookkeeping to keep track of the Unicode encoding

stdlib/public/core/StringIndexConversions.swift

@@ -50,8 +50,17 @@ extension String.Index {
   ///     of `target`.
   ///   - target: The string referenced by the resulting index.
   public init?(_ sourcePosition: String.Index, within target: String) {
-    guard target._isValidIndex(sourcePosition) else { return nil }
-    self = sourcePosition._characterAligned
+    // As a special exception, we allow `sourcePosition` to be an UTF-16 index
+    // when `self` is a UTF-8 string, to preserve compatibility with (broken)
+    // code that keeps using indices from a bridged string after converting the
+    // string to a native representation. Such indices are invalid, but
+    // returning nil here can break code that appeared to work fine for ASCII
+    // strings in Swift releases prior to 5.7.
+    guard
+      let i = target._guts.ensureMatchingEncodingNoTrap(sourcePosition),
+      target._isValidIndex(i)
+    else { return nil }
+    self = i._characterAligned
   }
 
   /// Creates an index in the given string that corresponds exactly to the
@@ -101,8 +110,17 @@ extension String.Index {
       return
     }
     if let str = target as? Substring {
-      guard str._isValidIndex(sourcePosition) else { return nil }
-      self = sourcePosition
+      // As a special exception, we allow `sourcePosition` to be an UTF-16 index
+      // when `self` is a UTF-8 string, to preserve compatibility with (broken)
+      // code that keeps using indices from a bridged string after converting
+      // the string to a native representation. Such indices are invalid, but
+      // returning nil here can break code that appeared to work fine for ASCII
+      // strings in Swift releases prior to 5.7.
+      guard
+        let i = str._wholeGuts.ensureMatchingEncodingNoTrap(sourcePosition),
+        str._isValidIndex(i)
+      else { return nil }
+      self = i
       return
     }
     self.init(sourcePosition, within: String(target))

stdlib/public/core/StringUTF16View.swift

@@ -402,8 +402,17 @@ extension String.UTF16View.Index {
   public init?(
     _ idx: String.Index, within target: String.UTF16View
   ) {
-    guard target._guts.hasMatchingEncoding(idx) else { return nil }
-    guard idx._encodedOffset <= target._guts.count else { return nil }
+    // As a special exception, we allow `idx` to be an UTF-16 index when `self`
+    // is a UTF-8 string, to preserve compatibility with (broken) code that
+    // keeps using indices from a bridged string after converting the string to
+    // a native representation. Such indices are invalid, but returning nil here
+    // can break code that appeared to work fine for ASCII strings in Swift
+    // releases prior to 5.7.
+    guard
+      let idx = target._guts.ensureMatchingEncodingNoTrap(idx),
+      idx._encodedOffset <= target._guts.count
+    else { return nil }
+
     if _slowPath(target._guts.isForeign) {
       guard idx._foreignIsWithin(target) else { return nil }
     } else {

stdlib/public/core/StringUTF8View.swift

@@ -359,8 +359,17 @@ extension String.UTF8View.Index {
   public init?(_ idx: String.Index, within target: String.UTF8View) {
     // Note: This method used to be inlinable until Swift 5.7.
 
-    guard target._guts.hasMatchingEncoding(idx) else { return nil }
-    guard idx._encodedOffset <= target._guts.count else { return nil }
+    // As a special exception, we allow `idx` to be an UTF-16 index when `self`
+    // is a UTF-8 string, to preserve compatibility with (broken) code that
+    // keeps using indices from a bridged string after converting the string to
+    // a native representation. Such indices are invalid, but returning nil here
+    // can break code that appeared to work fine for ASCII strings in Swift
+    // releases prior to 5.7.
+    guard
+      let idx = target._guts.ensureMatchingEncodingNoTrap(idx),
+      idx._encodedOffset <= target._guts.count
+    else { return nil }
+
     if _slowPath(target._guts.isForeign) {
       guard idx._foreignIsWithin(target) else { return nil }
     } else {

stdlib/public/core/StringUnicodeScalarView.swift

@@ -429,13 +429,13 @@ extension String.UnicodeScalarIndex {
     within unicodeScalars: String.UnicodeScalarView
   ) {
     guard
-      unicodeScalars._guts.hasMatchingEncoding(sourcePosition),
-      sourcePosition._encodedOffset <= unicodeScalars._guts.count,
-      unicodeScalars._guts.isOnUnicodeScalarBoundary(sourcePosition)
+      let i = unicodeScalars._guts.ensureMatchingEncodingNoTrap(sourcePosition),
+      i._encodedOffset <= unicodeScalars._guts.count,
+      unicodeScalars._guts.isOnUnicodeScalarBoundary(i)
     else {
       return nil
     }
-    self = sourcePosition
+    self = i
   }
 
   /// Returns the position in the given string that corresponds exactly to this