WIP: simple forwards exhaustive test working

Author: milseman

stdlib/public/core/StringCharacterView.swift

@@ -189,15 +189,15 @@ extension String: BidirectionalCollection {
   public subscript(i: Index) -> Character {
     _boundsCheck(i)
 
-    print(Array(self.utf16).map { String($0, radix: 16, uppercase: true) })
-    print(String(i._rawBits, radix: 16, uppercase: true))
-    print(String(i._encodedOffset))
+    // print(Array(self.utf16).map { String($0, radix: 16, uppercase: true) })
+    // print(String(i._rawBits, radix: 16, uppercase: true))
+    // print(String(i._encodedOffset))
 
     let i = _guts.scalarAlign(i)
-    print(String(i._rawBits, radix: 16, uppercase: true))
-    print(String(i._encodedOffset))
+    // print(String(i._rawBits, radix: 16, uppercase: true))
+    // print(String(i._encodedOffset))
     let distance = _characterStride(startingAt: i)
-    print(distance)
+    // print(distance)
 
     return _guts.errorCorrectedCharacter(
       startingAt: i._encodedOffset, endingAt: i._encodedOffset &+ distance)
@@ -207,7 +207,7 @@ extension String: BidirectionalCollection {
   internal func _characterStride(startingAt i: Index) -> Int {
     // Fast check if it's already been measured, otherwise check resiliently
     if let d = i.characterStride {
-      print("cached stride: \(d)")
+      // print("cached stride: \(d)")
       return d
     }
 

stdlib/public/core/StringGraphemeBreaking.swift

@@ -86,19 +86,18 @@ private func _hasGraphemeBreakBetween(
 private func _measureCharacterStrideICU(
   of utf8: UnsafeBufferPointer<UInt8>, startingAt i: Int
 ) -> Int {
-  // FIXME: need slice
+  // ICU will gives us a different result if we feed in the whole buffer, so
+  // slice it appropriately.
+  let utf8Slice = UnsafeBufferPointer(rebasing: utf8[i...])
+  let iterator = _ThreadLocalStorage.getUBreakIterator(utf8Slice)
+  let offset = __swift_stdlib_ubrk_following(iterator, 0)
 
-  let iterator = _ThreadLocalStorage.getUBreakIterator(utf8)
-  let offset = __swift_stdlib_ubrk_following(
-    iterator, Int32(truncatingIfNeeded: i))
   // ubrk_following returns -1 (UBRK_DONE) when it hits the end of the buffer.
-  if _fastPath(offset != -1) {
-    // The offset into our buffer is the distance.
-    _internalInvariant(offset > i, "zero-sized grapheme?")
-    return Int(truncatingIfNeeded: offset) &- i
-  }
-  _internalInvariant(utf8.count > i)
-  return utf8.count &- i
+  guard _fastPath(offset != -1) else { return utf8Slice.count }
+
+  // The offset into our buffer is the distance.
+  _internalInvariant(offset > 0, "zero-sized grapheme?")
+  return Int(truncatingIfNeeded: offset)
 }
 
 @inline(never) // slow-path

test/stdlib/StringIndex.swift

@@ -5,22 +5,22 @@ import StdlibUnittest
 
 var StringIndexTests = TestSuite("StringIndexTests")
 
-enum SimpleString: String {
-  case smallASCII = "abcdefg"
-  case smallUnicode = "abéÏ𓀀"
-  case largeASCII = "012345678901234567890"
-  case largeUnicode = "abéÏ012345678901234567890𓀀"
-  case emoji = "😀😃🤢🤮👩🏿‍🎤🧛🏻‍♂️🧛🏻‍♂️👩‍👩‍👦‍👦"
-}
+// enum SimpleString: String {
+//   case smallASCII = "abcdefg"
+//   case smallUnicode = "abéÏ𓀀"
+//   case largeASCII = "012345678901234567890"
+//   case largeUnicode = "abéÏ012345678901234567890𓀀"
+//   case emoji = "😀😃🤢🤮👩🏿‍🎤🧛🏻‍♂️🧛🏻‍♂️👩‍👩‍👦‍👦"
+// }
 
-let simpleStrings: [String] = [
-    SimpleString.smallASCII.rawValue,
-    SimpleString.smallUnicode.rawValue,
-    SimpleString.largeASCII.rawValue,
-    SimpleString.largeUnicode.rawValue,
-    SimpleString.emoji.rawValue,
-    "",
-]
+// let simpleStrings: [String] = [
+//     SimpleString.smallASCII.rawValue,
+//     SimpleString.smallUnicode.rawValue,
+//     SimpleString.largeASCII.rawValue,
+//     SimpleString.largeUnicode.rawValue,
+//     SimpleString.emoji.rawValue,
+//     "",
+// ]
 
 // StringIndexTests.test("basic sanity checks") {
 //   for s in simpleStrings {
@@ -248,65 +248,137 @@ import Foundation
 //   }
 // }
 
-StringIndexTests.test("Misaligned") {
-  func doIt(_ str: String) {
-    let characterIndices = Array(str.indices)
-    let scalarIndices = Array(str.unicodeScalars.indices) + [str.endIndex]
-    let utf8Indices = Array(str.utf8.indices)
-    let utf16Indices = Array(str.utf16.indices)
-
-    var lastScalarI = 0
-    for i in 1..<utf8Indices.count {
-      let idx = utf8Indices[i]
-
-      // Skip aligned indices
-      guard idx < scalarIndices[lastScalarI + 1] else {
-        assert(idx == scalarIndices[lastScalarI + 1])
-        lastScalarI += 1
-        continue
-      }
-      expectTrue(UTF8.isContinuation(str.utf8[idx]))
+// StringIndexTests.test("Misaligned") {
+//   func doIt(_ str: String) {
+//     let characterIndices = Array(str.indices)
+//     let scalarIndices = Array(str.unicodeScalars.indices) + [str.endIndex]
+//     let utf8Indices = Array(str.utf8.indices)
+//     let utf16Indices = Array(str.utf16.indices)
+
+//     var lastScalarI = 0
+//     for i in 1..<utf8Indices.count {
+//       let idx = utf8Indices[i]
+
+//       // Skip aligned indices
+//       guard idx < scalarIndices[lastScalarI + 1] else {
+//         assert(idx == scalarIndices[lastScalarI + 1])
+//         lastScalarI += 1
+//         continue
+//       }
+//       expectTrue(UTF8.isContinuation(str.utf8[idx]))
 
-      let lastScalarIdx = scalarIndices[lastScalarI]
+//       let lastScalarIdx = scalarIndices[lastScalarI]
 
-      dump(idx)
-      dump(lastScalarIdx)
-      dump(scalarIndices[lastScalarI + 1])
+//       dump(idx)
+//       dump(lastScalarIdx)
+//       dump(scalarIndices[lastScalarI + 1])
 
-      // Check aligning-down
-      expectEqual(str[lastScalarIdx], str[idx])
-      expectEqual(str.utf16[lastScalarIdx], str.utf16[idx])
-      expectEqual(str.unicodeScalars[lastScalarIdx], str.unicodeScalars[idx])
+//       // Check aligning-down
+//       expectEqual(str[lastScalarIdx], str[idx])
+//       expectEqual(str.utf16[lastScalarIdx], str.utf16[idx])
+//       expectEqual(str.unicodeScalars[lastScalarIdx], str.unicodeScalars[idx])
+
+//       // Check distance
+//       let (start, end) = (str.startIndex, str.endIndex)
+//       if characterIndices.contains(lastScalarIdx) {
+//         expectEqual(0, str.distance(from: lastScalarIdx, to: idx))
+//         expectEqual(str[..<idx].count, str.distance(from: start, to: idx))
+//         expectEqual(str[idx...].count, str.distance(from: idx, to: end))
+//       }
+//       expectEqual(
+//         0, str.unicodeScalars.distance(from: lastScalarIdx, to: idx))
+//       expectEqual(
+//         str.unicodeScalars[..<idx].count,
+//         str.unicodeScalars.distance(from: start, to: idx))
+//       expectEqual(
+//         str.unicodeScalars[idx...].count,
+//         str.unicodeScalars.distance(from: idx, to: end))
+
+//       expectEqual(0, str.utf16.distance(from: lastScalarIdx, to: idx))
+//       expectEqual(
+//         str.utf16[..<idx].count, str.utf16.distance(from: start, to: idx))
+//       expectEqual(
+//         str.utf16[idx...].count, str.utf16.distance(from: idx, to: end))
+//     }
+//   }
+
+//   let nsstring: NSString = "aодиde\u{301}日🧟‍♀️"
+//   doIt(nsstring as String)
+
+//   let string = "aодиde\u{301}日🧟‍♀️"
+//   doIt(string)
+// }
+
+StringIndexTests.test("Index interchange") {
+  // Exhaustively test aspects of string index interchange
+  func testInterchange(
+    _ str: String,
+    stackTrace: SourceLocStack = SourceLocStack(),
+    showFrame: Bool = true,
+    file: String = #file,
+    line: UInt = #line
+  ) {
+    let stackTrace = stackTrace.pushIf(showFrame, file: file, line: line)
+    func expect(
+      _ condition: @autoclosure () -> Bool,
+      _ message: String = "",
+      file: String = #file,
+      line: UInt = #line
+    ) {
+      expectTrue(condition(), message,
+        stackTrace: stackTrace, showFrame: showFrame,
+        file: file, line: line)
+    }
 
-      // Check distance
-      let (start, end) = (str.startIndex, str.endIndex)
-      if characterIndices.contains(lastScalarIdx) {
-        expectEqual(0, str.distance(from: lastScalarIdx, to: idx))
-        expectEqual(str[..<idx].count, str.distance(from: start, to: idx))
-        expectEqual(str[idx...].count, str.distance(from: idx, to: end))
+    var curCharIdx = str.startIndex
+    var curScalarIdx = str.startIndex
+    var curUTF8Idx = str.startIndex
+    var curUTF16Idx = str.startIndex
+
+    while curCharIdx < str.endIndex {
+      let curChar = str[curCharIdx]
+      expect(curChar == str[curScalarIdx])
+      expect(curChar == str[curUTF8Idx])
+      expect(curChar == str[curUTF16Idx])
+
+      // Advance the character index once and have the scalar index catch up
+      str.formIndex(after: &curCharIdx)
+
+      while curScalarIdx < curCharIdx {
+        let curScalar = str.unicodeScalars[curScalarIdx]
+
+        print(curScalar)
+        print(curScalarIdx.encodedOffset)
+
+        let curSubChar = str[curScalarIdx]
+
+        // Advance the scalar index once and have the code unit indices catch up
+        str.unicodeScalars.formIndex(after: &curScalarIdx)
+
+        while curUTF8Idx < curScalarIdx {
+          expect(curScalar == str.unicodeScalars[curUTF8Idx])
+          // expect(curSubChar == str[curUTF8Idx])
+          expect(!UTF16.isTrailSurrogate(str.utf16[curUTF8Idx]))
+          str.utf8.formIndex(after: &curUTF8Idx)
+        }
+        expect(curUTF8Idx == curScalarIdx)
+
+        while curUTF16Idx < curScalarIdx {
+          expect(curScalar == str.unicodeScalars[curUTF16Idx])
+          // expect(curSubChar == str[curUTF16Idx])
+          expect(!UTF8.isContinuation(str.utf8[curUTF16Idx]))
+          str.utf16.formIndex(after: &curUTF16Idx)
+        }
+        expect(curUTF16Idx == curScalarIdx)
       }
-      expectEqual(
-        0, str.unicodeScalars.distance(from: lastScalarIdx, to: idx))
-      expectEqual(
-        str.unicodeScalars[..<idx].count,
-        str.unicodeScalars.distance(from: start, to: idx))
-      expectEqual(
-        str.unicodeScalars[idx...].count,
-        str.unicodeScalars.distance(from: idx, to: end))
-
-      expectEqual(0, str.utf16.distance(from: lastScalarIdx, to: idx))
-      expectEqual(
-        str.utf16[..<idx].count, str.utf16.distance(from: start, to: idx))
-      expectEqual(
-        str.utf16[idx...].count, str.utf16.distance(from: idx, to: end))
     }
   }
 
-  let nsstring: NSString = "aодиde\u{301}日🧟‍♀️"
-  doIt(nsstring as String)
+  testInterchange("abc\r\ndefg")
+  testInterchange(("abc\r\ndefg" as NSString) as String)
 
-  let string = "aодиde\u{301}日🧟‍♀️"
-  doIt(string)
+  testInterchange("ab\r\ncдиde\u{301}日🧟‍♀️")
+  testInterchange(("ab\r\ncдиde\u{301}日🧟‍♀️" as NSString) as String)
 }
 
 #endif // _runtime(_ObjC)