Implement native scalar mappings

Author: Azoy

stdlib/public/SwiftShims/UnicodeData.h

@@ -74,7 +74,14 @@ SWIFT_RUNTIME_STDLIB_INTERNAL
 double _swift_stdlib_getNumericValue(__swift_uint32_t scalar);
 
 SWIFT_RUNTIME_STDLIB_INTERNAL
-const char *_swift_stdlib_getNameAlias(__swift_uint32_t);
+const char *_swift_stdlib_getNameAlias(__swift_uint32_t scalar);
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_int32_t _swift_stdlib_getMapping(__swift_uint32_t scalar,
+                                         __swift_uint8_t mapping);
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+const __swift_uint32_t *_swift_stdlib_getSpecialMapping(__swift_uint32_t scalar);
 
 #ifdef __cplusplus
 } // extern "C"

stdlib/public/SwiftShims/UnicodeShims.h

@@ -29,16 +29,6 @@
 extern "C" {
 #endif
 
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t _swift_stdlib_unicode_strToUpper(
-  __swift_uint16_t *Destination, __swift_int32_t DestinationCapacity,
-  const __swift_uint16_t *Source, __swift_int32_t SourceLength);
-
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t _swift_stdlib_unicode_strToLower(
-  __swift_uint16_t *Destination, __swift_int32_t DestinationCapacity,
-  const __swift_uint16_t *Source, __swift_int32_t SourceLength);
-
 typedef enum __swift_stdlib_UProperty {
   __swift_stdlib_UCHAR_GENERAL_CATEGORY = 0x1005,
   __swift_stdlib_UCHAR_NUMERIC_TYPE = 0x1009,
@@ -296,25 +286,6 @@ __swift_int32_t __swift_stdlib_u_charName(
     char *_Nullable buffer, __swift_int32_t bufferLength,
     __swift_stdlib_UErrorCode *pErrorCode);
 
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t __swift_stdlib_u_strToLower(
-    __swift_stdlib_UChar *dest, __swift_int32_t destCapacity,
-    const __swift_stdlib_UChar *src, __swift_int32_t srcLength,
-    const char *_Nullable locale, __swift_stdlib_UErrorCode *pErrorCode);
-
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t __swift_stdlib_u_strToTitle(
-    __swift_stdlib_UChar *dest, __swift_int32_t destCapacity,
-    const __swift_stdlib_UChar *src, __swift_int32_t srcLength,
-    __swift_stdlib_UBreakIterator *_Nullable titleIter,
-    const char *_Nullable locale, __swift_stdlib_UErrorCode *pErrorCode);
-
-SWIFT_RUNTIME_STDLIB_API
-__swift_int32_t __swift_stdlib_u_strToUpper(
-    __swift_stdlib_UChar *dest, __swift_int32_t destCapacity,
-    const __swift_stdlib_UChar *src, __swift_int32_t srcLength,
-    const char *_Nullable locale, __swift_stdlib_UErrorCode *pErrorCode);
-
 
 #ifdef __cplusplus
 } // extern "C"

stdlib/public/core/String.swift

@@ -849,41 +849,13 @@ extension String {
       }
     }
 
-    // TODO(String performance): Try out incremental case-conversion rather than
-    // make UTF-16 array beforehand
-    let codeUnits = Array(self.utf16).withUnsafeBufferPointer {
-      (uChars: UnsafeBufferPointer<UInt16>) -> Array<UInt16> in
-      var length: Int = 0
-      let result = Array<UInt16>(unsafeUninitializedCapacity: uChars.count) {
-        buffer, initializedCount in
-        var error = __swift_stdlib_U_ZERO_ERROR
-        length = Int(truncatingIfNeeded:
-          __swift_stdlib_u_strToLower(
-            buffer.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(buffer.count),
-            uChars.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(uChars.count),
-            "",
-            &error))
-        initializedCount = min(length, uChars.count)
-      }
-      if length > uChars.count {
-        var error = __swift_stdlib_U_ZERO_ERROR
-        return Array<UInt16>(unsafeUninitializedCapacity: length) {
-          buffer, initializedCount in
-          __swift_stdlib_u_strToLower(
-            buffer.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(buffer.count),
-            uChars.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(uChars.count),
-            "",
-            &error)
-          initializedCount = length
-        }
-      }
-      return result
+    var result = ""
+
+    for scalar in unicodeScalars {
+      result += scalar.properties.lowercaseMapping
     }
-    return codeUnits.withUnsafeBufferPointer { String._uncheckedFromUTF16($0) }
+
+    return result
   }
 
   /// Returns an uppercase version of the string.
@@ -910,41 +882,13 @@ extension String {
       }
     }
 
-    // TODO(String performance): Try out incremental case-conversion rather than
-    // make UTF-16 array beforehand
-    let codeUnits = Array(self.utf16).withUnsafeBufferPointer {
-      (uChars: UnsafeBufferPointer<UInt16>) -> Array<UInt16> in
-      var length: Int = 0
-      let result = Array<UInt16>(unsafeUninitializedCapacity: uChars.count) {
-        buffer, initializedCount in
-        var err = __swift_stdlib_U_ZERO_ERROR
-        length = Int(truncatingIfNeeded:
-          __swift_stdlib_u_strToUpper(
-            buffer.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(buffer.count),
-            uChars.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(uChars.count),
-            "",
-            &err))
-        initializedCount = min(length, uChars.count)
-      }
-      if length > uChars.count {
-        var err = __swift_stdlib_U_ZERO_ERROR
-        return Array<UInt16>(unsafeUninitializedCapacity: length) {
-          buffer, initializedCount in
-          __swift_stdlib_u_strToUpper(
-            buffer.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(buffer.count),
-            uChars.baseAddress._unsafelyUnwrappedUnchecked,
-            Int32(uChars.count),
-            "",
-            &err)
-          initializedCount = length
-        }
-      }
-      return result
+    var result = ""
+
+    for scalar in unicodeScalars {
+      result += scalar.properties.uppercaseMapping
     }
-    return codeUnits.withUnsafeBufferPointer { String._uncheckedFromUTF16($0) }
+
+    return result
   }
 
   /// Creates an instance from the description of a given

stdlib/public/core/UnicodeScalarProperties.swift

@@ -748,69 +748,68 @@ extension Unicode.Scalar.Properties {
 
 /// Case mapping properties.
 extension Unicode.Scalar.Properties {
-  // The type of ICU case conversion functions.
-  internal typealias _U_StrToX = (
-    /* dest */ UnsafeMutablePointer<__swift_stdlib_UChar>,
-    /* destCapacity */ Int32,
-    /* src */ UnsafePointer<__swift_stdlib_UChar>,
-    /* srcLength */ Int32,
-    /* locale */ UnsafePointer<Int8>,
-    /* pErrorCode */ UnsafeMutablePointer<__swift_stdlib_UErrorCode>
-  ) -> Int32
-
-  /// Applies the given ICU string mapping to the scalar.
-  ///
-  /// This function attempts first to write the mapping into a stack-based
-  /// UTF-16 buffer capable of holding 16 code units, which should be enough for
-  /// all current case mappings. In the event more space is needed, it will be
-  /// allocated on the heap.
-  internal func _applyMapping(_ u_strTo: _U_StrToX) -> String {
-    // Allocate 16 code units on the stack.
-    var fixedArray = _FixedArray16<UInt16>(allZeros: ())
-    let count: Int = fixedArray.withUnsafeMutableBufferPointer { buf in
-      return _scalar.withUTF16CodeUnits { utf16 in
-        var err = __swift_stdlib_U_ZERO_ERROR
-        let correctSize = u_strTo(
-          buf.baseAddress._unsafelyUnwrappedUnchecked,
-          Int32(buf.count),
-          utf16.baseAddress._unsafelyUnwrappedUnchecked,
-          Int32(utf16.count),
-          "",
-          &err)
-        guard err.isSuccess else {
-          fatalError("Unexpected error case-converting Unicode scalar.")
+  fileprivate enum _CaseMapping: UInt8 {
+    case uppercase
+    case lowercase
+    case titlecase
+  }
+
+  fileprivate func _getMapping(_ mapping: _CaseMapping) -> String {
+    // First, check if our scalar has a special mapping where it's mapped to
+    // more than 1 scalar.
+    let specialMappingPtr = _swift_stdlib_getSpecialMapping(_scalar.value)
+
+    if let specialMapping = specialMappingPtr {
+      func readSpecialMapping(_ ptr: UnsafePointer<UInt32>) -> String {
+        let count = Int(ptr.pointee)
+
+        if count == 0 {
+          return "\(_scalar)"
         }
-        return Int(correctSize)
-      }
-    }
-    if _fastPath(count <= 16) {
-      fixedArray.count = count
-      return fixedArray.withUnsafeBufferPointer {
-        String._uncheckedFromUTF16($0)
-      }
-    }
-    // Allocate `count` code units on the heap.
-    let array = Array<UInt16>(unsafeUninitializedCapacity: count) {
-      buf, initializedCount in
-      _scalar.withUTF16CodeUnits { utf16 in
-        var err = __swift_stdlib_U_ZERO_ERROR
-        let correctSize = u_strTo(
-          buf.baseAddress._unsafelyUnwrappedUnchecked,
-          Int32(buf.count),
-          utf16.baseAddress._unsafelyUnwrappedUnchecked,
-          Int32(utf16.count),
-          "",
-          &err)
-        guard err.isSuccess else {
-          fatalError("Unexpected error case-converting Unicode scalar.")
+
+        var result = ""
+
+        for i in 0 ..< count {
+          result += "\(Unicode.Scalar(_unchecked: ptr[1 + i]))"
         }
-        _internalInvariant(count == correctSize, "inconsistent ICU behavior")
-        initializedCount = count
+
+        return result
+      }
+
+      switch mapping {
+      case .uppercase:
+        return readSpecialMapping(specialMapping)
+
+      case .lowercase:
+        let upperCount = Int(specialMapping.pointee)
+
+        return readSpecialMapping(specialMapping + upperCount + 1)
+
+      case .titlecase:
+        let upperCount = Int(specialMapping.pointee)
+        let lowerPtr = specialMapping + upperCount + 1
+        let lowerCount = Int(lowerPtr.pointee)
+
+        return readSpecialMapping(lowerPtr + lowerCount + 1)
       }
     }
-    return array.withUnsafeBufferPointer {
-      String._uncheckedFromUTF16($0)
+
+    // If we did not have a special mapping, check if we have a direct scalar
+    // to scalar mapping.
+    let mappingDistance = _swift_stdlib_getMapping(
+      _scalar.value,
+      mapping.rawValue
+    )
+
+    if mappingDistance != 0 {
+      let scalar = Unicode.Scalar(
+        _unchecked: UInt32(Int(_scalar.value) &+ Int(mappingDistance))
+      )
+      return "\(scalar)"
     }
+
+    // We did not have any mapping. Return the scalar as is.
+    return "\(_scalar)"
   }
 
   /// The lowercase mapping of the scalar.
@@ -824,7 +823,7 @@ extension Unicode.Scalar.Properties {
   /// This property corresponds to the "Lowercase_Mapping" property in the
   /// [Unicode Standard](http://www.unicode.org/versions/latest/).
   public var lowercaseMapping: String {
-    return _applyMapping(__swift_stdlib_u_strToLower)
+    _getMapping(.lowercase)
   }
 
   /// The titlecase mapping of the scalar.
@@ -838,9 +837,7 @@ extension Unicode.Scalar.Properties {
   /// This property corresponds to the "Titlecase_Mapping" property in the
   /// [Unicode Standard](http://www.unicode.org/versions/latest/).
   public var titlecaseMapping: String {
-    return _applyMapping { ptr, cap, src, len, locale, err in
-      return __swift_stdlib_u_strToTitle(ptr, cap, src, len, nil, locale, err)
-    }
+    _getMapping(.titlecase)
   }
 
   /// The uppercase mapping of the scalar.
@@ -854,7 +851,7 @@ extension Unicode.Scalar.Properties {
   /// This property corresponds to the "Uppercase_Mapping" property in the
   /// [Unicode Standard](http://www.unicode.org/versions/latest/).
   public var uppercaseMapping: String {
-    return _applyMapping(__swift_stdlib_u_strToUpper)
+    _getMapping(.uppercase)
   }
 }
 

stdlib/public/stubs/UnicodeScalarProps.cpp

@@ -13,6 +13,7 @@
 #include "UnicodeScalarProps.h"
 #include "../SwiftShims/UnicodeData.h"
 #include <limits>
+#include <iostream>
 
 SWIFT_RUNTIME_STDLIB_INTERNAL
 __swift_uint64_t _swift_stdlib_getBinaryProperties(__swift_uint32_t scalar) {
@@ -80,11 +81,11 @@ __swift_uint8_t _swift_stdlib_getNumericType(__swift_uint32_t scalar) {
 
     auto entry = _swift_stdlib_numeric_type[idx];
 
-    auto lowerBoundScalar = (entry << 21) >> 21;
+    auto lowerBoundScalar = (entry << 11) >> 11;
     auto rangeCount = (entry << 3) >> 24;
     auto upperBoundScalar = lowerBoundScalar + rangeCount;
 
-    auto numericType = (__swift_uint8_t) entry >> 29;
+    auto numericType = (__swift_uint8_t)(entry >> 29);
 
     if (scalar >= lowerBoundScalar && scalar <= upperBoundScalar) {
       return numericType;
@@ -130,3 +131,61 @@ const char *_swift_stdlib_getNameAlias(__swift_uint32_t scalar) {
 
   return _swift_stdlib_nameAlias_data[dataIdx];
 }
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+__swift_int32_t _swift_stdlib_getMapping(__swift_uint32_t scalar,
+                                         __swift_uint8_t mapping) {
+  auto dataIdx = _swift_stdlib_getScalarBitArrayIdx(scalar,
+                                                    _swift_stdlib_mappings,
+                                                  _swift_stdlib_mappings_ranks);
+
+  if (dataIdx == std::numeric_limits<__swift_intptr_t>::max()) {
+    return 0;
+  }
+
+  auto mappings = _swift_stdlib_mappings_data_indices[dataIdx];
+
+  __swift_uint8_t mappingIdx;
+
+  switch (mapping) {
+    // Uppercase
+    case 0:
+      mappingIdx = mappings & 0xFF;
+      break;
+
+    // Lowercase
+    case 1:
+      mappingIdx = (mappings & 0xFF00) >> 8;
+      break;
+
+    // Titlecase
+    case 2:
+      mappingIdx = (mappings & 0xFF0000) >> 16;
+      break;
+
+    // Unknown mapping
+    default:
+      return 0;
+  }
+
+  if (mappingIdx == 0xFF) {
+    return 0;
+  }
+
+  return _swift_stdlib_mappings_data[mappingIdx];
+}
+
+SWIFT_RUNTIME_STDLIB_INTERNAL
+const __swift_uint32_t *_swift_stdlib_getSpecialMapping(__swift_uint32_t scalar) {
+  auto dataIdx = _swift_stdlib_getScalarBitArrayIdx(scalar,
+                                                 _swift_stdlib_special_mappings,
+                                          _swift_stdlib_special_mappings_ranks);
+
+  if (dataIdx == std::numeric_limits<__swift_intptr_t>::max()) {
+    return nullptr;
+  }
+
+  auto index = _swift_stdlib_special_mappings_data_indices[dataIdx];
+
+  return _swift_stdlib_special_mappings_data + index;
+}