Implement String.WordView

Author: Azoy

stdlib/private/StdlibUnicodeUnittest/CMakeLists.txt

@@ -7,6 +7,7 @@ add_swift_target_library(swiftStdlibUnicodeUnittest ${SWIFT_STDLIB_LIBRARY_BUILD
   Collation.swift
   UnicodeScalarProperties.swift
   GraphemeBreaking.swift
+  WordBreaking.swift
 
   SWIFT_MODULE_DEPENDS StdlibUnittest
   SWIFT_MODULE_DEPENDS_LINUX Glibc

stdlib/private/StdlibUnicodeUnittest/WordBreaking.swift

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+// Normalization tests are currently only avaible on Darwin, awaiting a sensible
+// file API...
+#if _runtime(_ObjC)
+import Foundation
+
+func parseWordBreakTests(
+  _ data: String,
+  into result: inout [(String, [String])]
+) {
+  for line in data.split(separator: "\n") {
+    // Only look at actual tests
+    guard line.hasPrefix("÷") else {
+      continue
+    }
+
+    let info = line.split(separator: "#")
+    let components = info[0].split(separator: " ")
+    print(components)
+
+    var string = ""
+    var words: [String] = [""]
+
+    for i in components.indices {
+      guard i != 0 else {
+        continue
+      }
+
+      // If we're an odd index, this is a scalar.
+      if !i.isMultiple(of: 2) {
+        let scalar = Unicode.Scalar(UInt32(components[i], radix: 16)!)!
+
+        string.unicodeScalars.append(scalar)
+        words[words.count - 1].unicodeScalars.append(scalar)
+      } else {
+        // Otherwise, it is a word breaking operator.
+
+        // If this is a break, record the +1 count. Otherwise it is × which is
+        // not a break.
+        if components[i] == "÷" {
+          words.append("")
+        }
+      }
+    }
+
+    words.removeLast()
+
+    result.append((string, words))
+  }
+}
+
+public let wordBreakTests: [(String, [String])] = {
+  var result: [(String, [String])] = []
+
+  let testFile = readInputFile("WordBreakTest.txt")
+
+  parseWordBreakTests(testFile, into: &result)
+
+  return result
+}()
+#endif

stdlib/public/SwiftShims/UnicodeData.h

@@ -64,6 +64,13 @@ __swift_uint8_t _swift_stdlib_getGraphemeBreakProperty(__swift_uint32_t scalar);
 SWIFT_RUNTIME_STDLIB_INTERNAL
 __swift_bool _swift_stdlib_isLinkingConsonant(__swift_uint32_t scalar);
 
+//===----------------------------------------------------------------------===//
+// Word Breaking
+//===----------------------------------------------------------------------===//
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_uint8_t _swift_stdlib_getWordBreakProperty(__swift_uint32_t scalar);
+
 //===----------------------------------------------------------------------===//
 // Unicode.Scalar.Properties
 //===----------------------------------------------------------------------===//

stdlib/public/core/CMakeLists.txt

@@ -170,6 +170,8 @@ set(SWIFTLIB_ESSENTIAL
   StringUTF16View.swift
   StringUTF8View.swift
   StringUTF8Validation.swift
+  StringWordBreaking.swift
+  StringWordView.swift
   Substring.swift
   SwiftNativeNSArray.swift
   TemporaryAllocation.swift
@@ -178,7 +180,7 @@ set(SWIFTLIB_ESSENTIAL
   UnavailableStringAPIs.swift
   UnicodeData.swift
   UnicodeEncoding.swift
-  UnicodeGraphemeBreakProperty.swift
+  UnicodeBreakProperty.swift
   UnicodeHelpers.swift
   UnicodeParser.swift
   UnicodeScalarProperties.swift

stdlib/public/core/GroupInfo.json

@@ -41,11 +41,13 @@
     "StringUTF8View.swift",
     "StringUTF8Validation.swift",
     "StringUnicodeScalarView.swift",
+    "StringWordBreaking.swift",
+    "StringWordView.swift",
     "Substring.swift",
     "Unicode.swift",
     "UnicodeData.swift",
     "UnicodeEncoding.swift",
-    "UnicodeGraphemeBreakProperty.swift",
+    "UnicodeBreakProperty.swift",
     "UnicodeHelpers.swift",
     "UnicodeParser.swift",
     "UnicodeScalar.swift",

stdlib/public/core/StringIndexValidation.swift

@@ -342,3 +342,20 @@ extension _StringGuts {
     return Range(_uncheckedBounds: (l, u))
   }
 }
+
+// Word index validation (String)
+extension _StringGuts {
+  @available(SwiftStdlib 5.7, *)
+  internal func validateWordIndex(_ i: String.Index) -> String.Index {
+    // TODO: Maybe fast word index bit?
+    return roundDownToNearestWord(scalarAlign(validateSubscalarIndex(i)))
+  }
+
+  @available(SwiftStdlib 5.7, *)
+  internal func validateInclusiveWordIndex(_ i: String.Index) -> String.Index {
+    // TODO: Maybe fast word index bit?
+    return roundDownToNearestCharacter(
+      scalarAlign(validateInclusiveSubscalarIndex(i))
+    )
+  }
+}