Hide WordView for now (also separate Index type)

add bidirectional conformance

Fix tests

Author: Azoy

stdlib/public/core/StringIndex.swift

@@ -17,7 +17,7 @@ import SwiftShims
 String's Index has the following layout:
 
  ┌──────────┬────────────────╥────────────────┬───────╥───────┐
- │ b63:b16  │      b15:b14   ║     b13:b8     │ b7:b5 ║ b4:b0 │
+ │ b63:b16  │      b15:b14   ║     b13:b8     │ b7:b4 ║ b3:b0 │
  ├──────────┼────────────────╫────────────────┼───────╫───────┤
  │ position │ transc. offset ║ grapheme cache │ rsvd  ║ flags │
  └──────────┴────────────────╨────────────────┴───────╨───────┘
@@ -43,7 +43,7 @@ isn't frozen.
   looking back at scalars preceding the index. (Substrings that don't start on a
   `Character` boundary heavily rely on this.)
 
-- reserved: 3 unused bits available for future flags etc. The meaning of each
+- reserved: 4 unused bits available for future flags etc. The meaning of each
   bit may change between stdlib versions. These must be set to zero if
   constructing an index in inlinable code.
 
@@ -70,11 +70,6 @@ isn't frozen.
     If set, the position is known to be expressed in UTF-16 code units.
     (Introduced in Swift 5.7)
 
-  * b4: `_isWordAligned`
-
-    If set, the index is known to be on a Unicode word boundary.
-    (Introduced in Swift 5.7)
-
 Before Swift 5.7, bits b1, b2 and b3 used to be part of the resilient slice. See
 the notes on Character Alignment and Index Encoding below to see how this works.
 
@@ -266,9 +261,6 @@ extension String.Index {
   @_alwaysEmitIntoClient @inline(__always) // Swift 5.7
   internal static var __utf16Bit: UInt64 { 0x8 }
 
-  @_alwaysEmitIntoClient @inline(__always) // Swift 5.7
-  internal static var __wordAlignmentBit: UInt64 { 0x10 }
-
   @_alwaysEmitIntoClient @inline(__always) // Swift 5.7
   internal static func __encodingBit(utf16: Bool) -> UInt64 {
     let utf16 = Int8(Builtin.zext_Int1_Int8(utf16._value))
@@ -373,35 +365,11 @@ extension String.Index {
   }
 }
 
-// ### Word Alignment
-//
-// Enter some pretty cool information about Unicode words
-extension String.Index {
-  @_alwaysEmitIntoClient // Swift 5.7
-  @inline(__always)
-  internal var _isWordAligned: Bool {
-    _rawBits & Self.__wordAlignmentBit != 0
-  }
-
-  @_alwaysEmitIntoClient // Swift 5.7
-  @inline(__always)
-  internal var _wordAligned: String.Index {
-    let r = _rawBits
-      | Self.__wordAlignmentBit
-      | Self.__characterAlignmentBit
-      | Self.__scalarAlignmentBit
-    let idx = Self(r)
-    idx._invariantCheck()
-    return idx
-  }
-}
-
 extension String.Index {
   @_alwaysEmitIntoClient // Swift 5.7
   internal func _copyingAlignment(from index: Self) -> Self {
     let mask = Self.__scalarAlignmentBit
       | Self.__characterAlignmentBit
-      | Self.__wordAlignmentBit
     return Self((_rawBits & ~mask) | (index._rawBits & mask))
   }
 }

stdlib/public/core/StringIndexValidation.swift

@@ -21,11 +21,6 @@ extension _StringGuts {
   internal func isFastCharacterIndex(_ i: String.Index) -> Bool {
     hasMatchingEncoding(i) && i._isCharacterAligned
   }
-
-  @_alwaysEmitIntoClient @inline(__always)
-  internal func isFastWordIndex(_ i: String.Index) -> Bool {
-    hasMatchingEncoding(i) && i._isWordAligned
-  }
 }
 
 // Subscalar index validation (UTF-8 & UTF-16 views)
@@ -305,47 +300,6 @@ extension _StringGuts {
       scalarAlign(validateInclusiveSubscalarIndex(i, in: bounds)),
       in: bounds)
   }
-
-  internal func validateCharacterRange(
-    _ range: Range<String.Index>
-  ) -> Range<String.Index> {
-    if
-      isFastCharacterIndex(range.lowerBound),
-      isFastCharacterIndex(range.upperBound)
-    {
-      _precondition(range.upperBound._encodedOffset <= count,
-        "String index range is out of bounds")
-      return range
-    }
-
-    let r = validateSubscalarRange(range)
-    let l = roundDownToNearestCharacter(scalarAlign(r.lowerBound))
-    let u = roundDownToNearestCharacter(scalarAlign(r.upperBound))
-    return Range(_uncheckedBounds: (l, u))
-  }
-
-  internal func validateCharacterRange(
-    _ range: Range<String.Index>,
-    in bounds: Range<String.Index>
-  ) -> Range<String.Index> {
-    _internalInvariant(bounds.upperBound <= endIndex)
-
-    if
-      isFastCharacterIndex(range.lowerBound),
-      isFastCharacterIndex(range.upperBound)
-    {
-      _precondition(
-        range.lowerBound >= bounds.lowerBound
-        && range.upperBound <= bounds.upperBound,
-        "String index range is out of bounds")
-      return range
-    }
-
-    let r = validateSubscalarRange(range, in: bounds)
-    let l = roundDownToNearestCharacter(scalarAlign(r.lowerBound), in: bounds)
-    let u = roundDownToNearestCharacter(scalarAlign(r.upperBound), in: bounds)
-    return Range(_uncheckedBounds: (l, u))
-  }
 }
 
 // Temporary additions to deal with binary compatibility issues with existing
@@ -448,24 +402,24 @@ extension _StringGuts {
 // Word index validation (String)
 extension _StringGuts {
   @available(SwiftStdlib 5.7, *)
-  internal func validateWordIndex(_ i: String.Index) -> String.Index {
-    if isFastWordIndex(i) {
-      _precondition(i._encodedOffset < count, "String index is out of bounds")
-      return i
-    }
+  internal func validateWordIndex(
+    _ i: String._WordView.Index
+  ) -> String._WordView.Index {
+    let i = String.Index(_encodedOffset: i._encodedOffset)
 
-    return roundDownToNearestWord(scalarAlign(validateSubscalarIndex(i)))
+    return roundDownToNearestWord(
+      String._WordView.Index(scalarAlign(validateSubscalarIndex(i)))
+    )
   }
 
   @available(SwiftStdlib 5.7, *)
-  internal func validateInclusiveWordIndex(_ i: String.Index) -> String.Index {
-    if isFastWordIndex(i) {
-      _precondition(i._encodedOffset < count, "String index is out of bounds")
-      return i
-    }
+  internal func validateInclusiveWordIndex(
+    _ i: String._WordView.Index
+  ) -> String._WordView.Index {
+    let i = String.Index(_encodedOffset: i._encodedOffset)
 
-    return roundDownToNearestCharacter(
-      scalarAlign(validateInclusiveSubscalarIndex(i))
+    return roundDownToNearestWord(
+      String._WordView.Index(scalarAlign(validateInclusiveSubscalarIndex(i)))
     )
   }
 }

stdlib/public/core/StringWordBreaking.swift

@@ -11,42 +11,119 @@
 
 extension _StringGuts {
   @inline(__always)
-  @available(SwiftStdlib 5.7, *)
-  internal func roundDownToNearestWord(_ i: String.Index) -> String.Index {
-    _internalInvariant(i._isScalarAligned)
-    _internalInvariant(hasMatchingEncoding(i))
+  internal func roundDownToNearestWord(
+    _ i: String._WordView.Index
+  ) -> String._WordView.Index {
     _internalInvariant(i._encodedOffset <= count)
 
-    if _fastPath(i._isWordAligned) {
-      return i
-    }
-
     let offset = i._encodedOffset
 
     if offset == 0 || offset == count {
-      return i._wordAligned
+      return i
     }
 
     return _slowRoundDownToNearestWord(i)
   }
 
   @inline(never)
-  @available(SwiftStdlib 5.7, *)
-  internal func _slowRoundDownToNearestWord(_ i: String.Index) -> String.Index {
-    let words = String._WordView(self)
-
+  internal func _slowRoundDownToNearestWord(
+    _ i: String._WordView.Index
+  ) -> String._WordView.Index {
     let offset = i._encodedOffset
-    let start = offset &- words._uncheckedIndex(before: i)._encodedOffset
-    let startIndex = String.Index(_encodedOffset: start)._wordAligned
-    let stride = words._uncheckedIndex(after: startIndex)._encodedOffset
+    let start = _opaquePreviousWordIndex(endingAt: offset)
+    let stride = _opaqueNextWordIndex(startingAt: start) &- start
     _internalInvariant(offset <= start + stride, "Word breaking inconsistency")
 
     if offset >= start + stride {
-      return i._wordAligned
+      return i
     }
 
-    let r = String.Index(_encodedOffset: start)._wordAligned
-    return markEncoding(r)
+    return String._WordView.Index(_encodedOffset: start)
+  }
+
+  @inline(never)
+  @_effects(releasenone)
+  internal func _opaqueNextWordIndex(startingAt i: Int) -> Int {
+    if _slowPath(isForeign) {
+      return _foreignOpaqueNextWordIndex(startingAt: i)
+    }
+
+    return withFastUTF8 { utf8 in
+      nextWordBoundary(startingAt: i) {
+        _internalInvariant($0 >= 0)
+
+        guard $0 < utf8.count else {
+          return nil
+        }
+
+        let (scalar, len) = _decodeScalar(utf8, startingAt: $0)
+        return (scalar, $0 &+ len)
+      }
+    }
+  }
+
+  internal func _foreignOpaqueNextWordIndex(startingAt i: Int) -> Int {
+#if _runtime(_ObjC)
+    return nextWordBoundary(startingAt: i) {
+      _internalInvariant($0 >= 0)
+
+      guard $0 < count else {
+        return nil
+      }
+
+      let scalars = String.UnicodeScalarView(self)
+      let idx = String.Index(_encodedOffset: $0)
+
+      let scalar = scalars[idx]
+      let nextIndex = scalars.index(after: idx)
+
+      return (scalar, nextIndex._encodedOffset)
+    }
+#else
+    fatalError("No foreign strings on this platform in this version of Swift.")
+#endif
+  }
+
+  internal func _opaquePreviousWordIndex(endingAt i: Int) -> Int {
+    if _slowPath(isForeign) {
+      return _foreignOpaquePreviousWordIndex(endingAt: i)
+    }
+
+    return withFastUTF8 { utf8 in
+      previousWordBoundary(endingAt: i) {
+        _internalInvariant($0 <= count)
+
+        guard $0 > 0 else {
+          return nil
+        }
+
+        let (scalar, len) = _decodeScalar(utf8, endingAt: $0)
+        return (scalar, $0 &- len)
+      }
+    }
+  }
+
+  @inline(never)
+  internal func _foreignOpaquePreviousWordIndex(endingAt i: Int) -> Int {
+    #if _runtime(_ObjC)
+    return previousWordBoundary(endingAt: i) {
+      _internalInvariant($0 <= count)
+
+      guard $0 > 0 else {
+        return nil
+      }
+
+      let scalars = String.UnicodeScalarView(self)
+      let idx = String.Index(_encodedOffset: $0)
+
+      let previousIndex = scalars.index(before: idx)
+      let scalar = scalars[previousIndex]
+
+      return (scalar, previousIndex._encodedOffset)
+    }
+#else
+    fatalError("No foreign strings on this platform in this version of Swift.")
+#endif
   }
 }
 
@@ -77,10 +154,9 @@ internal struct _WordBreakingState {
   var shouldBreakRI = false
 }
 
-@available(SwiftStdlib 5.7, *)
-extension String._WordView {
+extension _StringGuts {
   // Returns the stride of the next word at the previous boundary offset.
-  internal func nextBoundary(
+  internal func nextWordBoundary(
     startingAt index: Int,
     nextScalar: (Int) -> (scalar: Unicode.Scalar, end: Int)?
   ) -> Int {
@@ -109,7 +185,7 @@ extension String._WordView {
   }
 
   // Returns the stride of the previous word at the current boundary offset.
-  internal func previousBoundary(
+  internal func previousWordBoundary(
     endingAt index: Int,
     previousScalar: (Int) -> (scalar: Unicode.Scalar, start: Int)?
   ) -> Int {
@@ -145,8 +221,7 @@ extension String._WordView {
   }
 }
 
-@available(SwiftStdlib 5.7, *)
-extension String._WordView {
+extension _StringGuts {
   // The "algorithm" that determines whether or not we should break between
   // certain word break properties.
   //
@@ -345,8 +420,7 @@ extension String._WordView {
   }
 }
 
-@available(SwiftStdlib 5.7, *)
-extension String._WordView {
+extension _StringGuts {
   // The "algorithm" that determines whether or not we should break between
   // certain word break properties.
   //