testing iterators

Author: milseman

stdlib/public/core/UTF8Span.swift

@@ -199,6 +199,8 @@ extension UTF8Span {
   ) throws(E) -> Result {
     try body(unsafeBaseAddress._ubp(0..<count))
   }
+
+  // TODO: withSpan or similar?
 }
 
 // MARK: Internals

stdlib/public/core/UTF8SpanInternalHelpers.swift

@@ -111,6 +111,8 @@ extension UnsafeRawPointer {
     _ i: Int,
     limitedBy end: Int
   ) -> Bool {
+    print(i)
+    print(end)
     _internalInvariant(i >= 0 && i <= end)
     if i == 0 || i == end {
       return true

stdlib/public/core/UTF8SpanIterators.swift

@@ -1,5 +1,10 @@
 @available(SwiftStdlib 6.1, *)
 extension UTF8Span {
+
+  public func _makeScalarIterator() -> ScalarIterator {
+    .init(self)
+  }
+
   /// Iterate the `Unicode.Scalar`s  contents of a `UTF8Span`.
   ///
   /// **TODO**: Examples
@@ -12,6 +17,7 @@ extension UTF8Span {
     /// **TODO**: private(set)?
     public var currentCodeUnitOffset: Int
 
+    // TODO: underscored init?
     public init(_ codeUnits: UTF8Span) {
       self.codeUnits = codeUnits
       self.currentCodeUnitOffset = 0
@@ -141,6 +147,11 @@ extension UTF8Span {
 
 @available(SwiftStdlib 6.1, *)
 extension UTF8Span {
+
+  public func _makeCharacterIterator() -> CharacterIterator {
+    .init(self)
+  }
+
   /// Iterate the `Character` contents of a `UTF8Span`.
   ///
   /// **TODO**: Examples
@@ -177,6 +188,7 @@ extension UTF8Span {
     /// **TODO**: private(set)?
     public var currentCodeUnitOffset: Int
 
+    // TODO: underscored init?
     public init(_ span: UTF8Span) {
       self.codeUnits = span
       self.currentCodeUnitOffset = 0
@@ -212,7 +224,7 @@ extension UTF8Span {
       _internalInvariant(codeUnits.isScalarAligned(currentCodeUnitOffset))
       let (result, newPos) = codeUnits.unsafeBaseAddress._decodeCharacter(
         endingAt: currentCodeUnitOffset,
-        limitedBy: 0)
+        limitedBy: codeUnits.count)
       self.currentCodeUnitOffset = newPos
       return result
     }
@@ -274,11 +286,16 @@ extension UTF8Span {
       fatalError()
     }
   }
+
 }
 
 @available(SwiftStdlib 6.1, *)
 extension UTF8Span {
 
+  public func _makeGraphemeBreakIterator() -> GraphemeBreakIterator {
+    .init(self)
+  }
+
   /// **QUESTION**: There are many ways we could expose this functionality and
   ///   I'd like some help here. As written, the caller would be looking at
   ///   `currentCodeUnitOffset` and `state` while `next()` / `previous
@@ -292,12 +309,14 @@ extension UTF8Span {
     public var currentCodeUnitOffset: Int
     public var state: Unicode.GraphemeBreakingState
 
+    // TODO: underscored init?
     public init(_ span: UTF8Span) {
       self.codeUnits = span
       self.currentCodeUnitOffset = 0
       self.state = .init()
     }
 
+    // TODO: underscored init?
     public init(_ span: UTF8Span, using state: Unicode.GraphemeBreakingState) {
       self.codeUnits = span
       self.currentCodeUnitOffset = 0

test/stdlib/UTF8SpanIteratorTests.swift

@@ -0,0 +1,124 @@
+// RUN: %target-run-stdlib-swift(-enable-experimental-feature Span) -enable-experimental-feature Span %S/Inputs/
+
+// REQUIRES: executable_test
+
+import Swift
+import StdlibUnittest
+
+var suite = TestSuite("UTF8SpanIterator")
+defer { runAllTests() }
+
+@available(SwiftStdlib 6.1, *)
+extension Array {
+  func withSpan<R>(_ f: (Span<Element>) throws -> R) rethrows -> R {
+    try self.withUnsafeBufferPointer {
+      try f(Span(_unsafeElements: $0))
+    }
+  }
+}
+
+@available(SwiftStdlib 6.1, *)
+extension UTF8Span {
+  func withSpan<R>(_ f: (Span<UInt8>) throws -> R) rethrows -> R {
+    try self.withUnsafeBufferPointer {
+      try f(Span(_unsafeElements: $0))
+    }
+  }
+}
+
+// 
+@available(SwiftStdlib 6.1, *)
+struct ContentEquivalenceTestCase {
+  var str: String
+  var loc: SourceLocStack
+
+  func withUTF8Span<R>(_ f: (UTF8Span) throws -> R) rethrows -> R {
+    try Array(str.utf8).withSpan { span in 
+      try f(try! UTF8Span(_validating: span))
+    }
+  }
+
+  func testBytes() {
+    let otherBytes = Array((str+"abc").utf8)
+
+    withUTF8Span { utf8Span in
+      utf8Span.withUnsafeBufferPointer {
+        expectEqualSequence(str.utf8, $0, stackTrace: loc)
+      }
+    }
+
+    // NOTE: There's a slight jarring due to not having the same 
+    // iterators for code units
+  }
+
+  func testScalars() {
+    withUTF8Span { utf8Span in 
+      // Test forwards
+      var utf8SpanIter = utf8Span._makeScalarIterator()
+      var stringIter = str.unicodeScalars.makeIterator()
+      while let scalar = utf8SpanIter.next() {
+        expectEqual(scalar, stringIter.next(), stackTrace: loc)
+      }
+      expectNil(stringIter.next(), stackTrace: loc)
+
+      // Test backwards
+      var stringRevIter = str.unicodeScalars.reversed().makeIterator()
+      while let scalar = utf8SpanIter.previous() {
+        expectEqual(scalar, stringRevIter.next(), stackTrace: loc)
+      }
+      expectNil(stringRevIter.next(), stackTrace: loc)
+
+    }
+  }
+
+  func testCharacters() {
+    withUTF8Span { utf8Span in 
+      // Test forwards
+      var utf8SpanIter = utf8Span._makeCharacterIterator()
+      var stringIter = str.makeIterator()
+      while let char = utf8SpanIter.next() {
+        expectEqual(char, stringIter.next(), stackTrace: loc)
+      }
+      expectNil(stringIter.next(), stackTrace: loc)
+
+      // Test backwards
+      var stringRevIter = str.reversed().makeIterator()
+      while let char = utf8SpanIter.previous() {
+        expectEqual(char, stringRevIter.next(), stackTrace: loc)
+      }
+      expectNil(stringRevIter.next(), stackTrace: loc)
+    }
+  }
+
+  func run() {
+    testBytes()
+    testScalars()
+    testCharacters()
+  }
+
+}
+
+if #available(SwiftStdlib 6.1, *) {
+  suite.test("UTF8Span/iterators") {
+    func test(
+      _ s: String,
+      file: String = #file,
+      line: UInt = #line
+    ) {
+      print("testing: \(s)")
+      let t = ContentEquivalenceTestCase(
+        str: s, loc: .init(SourceLoc(file, line)))
+      t.run()
+    }
+
+    test("abc")
+    test("abcde\u{301}")
+    test("abcde\u{301}")
+    test("abéÏ𓀀")
+    test("012345678901234567890")
+    test("abéÏ012345678901234567890𓀀")
+    test("😀😃🤢🤮👩🏿‍🎤🧛🏻‍♂️🧛🏻‍♂️👩‍👩‍👦‍👦")
+    test("")
+  }
+}
+