[String] Use a UTF-8 representation for native strings

benchmark/single-source/StringComparison.swift.gyb

@@ -30,32 +30,58 @@ extension String {
   }
 }
 
-% Names = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo", "longSharedPrefix"]
+% AllWorkloads = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo", "longSharedPrefix"]
+% ComparisonWorkloads = AllWorkloads
+% HashingWorkloads = ["ascii", "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal", "emoji", "abnormal", "zalgo", "longSharedPrefix"]
 
-public let StringComparison = [
-% for Name in Names:
+// TODO(UTF8 post-merge): Disable longSharedPrefix hashing benchmark, which is
+// enabled here for 1-to-1 comparison vs master
+
+// TODO(UTF8 post-merge): Enable NormalizedIteratorWorkloads for ["ascii",
+// "latin1", "fastPrenormal", "slowerPrenormal", "nonBMPSlowestPrenormal",
+// "emoji", "abnormal", "zalgo"]
+
+% NormalizedIteratorWorkloads = []
+
+public let StringComparison: [BenchmarkInfo] = [
+% for Name in ComparisonWorkloads:
   BenchmarkInfo(
     name: "StringComparison_${Name}",
     runFunction: run_StringComparison_${Name},
     tags: [.validation, .api, .String],
-    setUpFunction: { blackHole(Workload_${Name}) }),
-% end # Names
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # ComparisonWorkloads
 ]
 
-public let StringHashing = [
-% for Name in Names:
+public let StringHashing: [BenchmarkInfo] = [
+% for Name in HashingWorkloads:
   BenchmarkInfo(
     name: "StringHashing_${Name}",
     runFunction: run_StringHashing_${Name},
     tags: [.validation, .api, .String],
-    setUpFunction: { blackHole(Workload_${Name}) }),
-% end # Names
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # HashingWorkloads
 ]
 
-% for Name in Names:
+public let NormalizedIterator: [BenchmarkInfo] = [
+% for Name in NormalizedIteratorWorkloads:
+  BenchmarkInfo(
+    name: "NormalizedIterator_${Name}",
+    runFunction: run_NormalizedIterator_${Name},
+    tags: [.validation, .String],
+    setUpFunction: { blackHole(Workload_${Name}) }
+  ),
+% end # NormalizedIteratorWorkloads
+]
 
+% for Name in AllWorkloads:
 var Workload_${Name}: Workload! = Workload.${Name}
 
+% end # AllWorkloads
+
+%for Name in ComparisonWorkloads:
 @inline(never)
 public func run_StringComparison_${Name}(_ N: Int) {
   let workload: Workload = Workload_${Name}
@@ -70,6 +96,9 @@ public func run_StringComparison_${Name}(_ N: Int) {
   }
 }
 
+% end # ComparisonWorkloads
+
+%for Name in HashingWorkloads:
 @inline(never)
 public func run_StringHashing_${Name}(_ N: Int) {
   let workload: Workload = Workload.${Name}
@@ -81,8 +110,25 @@ public func run_StringHashing_${Name}(_ N: Int) {
     }
   }
 }
-
-% end # Names
+
+% end # HashingWorkloads
+
+%for Name in NormalizedIteratorWorkloads:
+@inline(never)
+public func run_NormalizedIterator_${Name}(_ N: Int) {
+  let workload: Workload = Workload.${Name}
+  let tripCount = workload.tripCount
+  let payload = workload.payload
+  for _ in 1...tripCount*N {
+    for str in payload {
+      str._withNFCCodeUnits { cu in
+        blackHole(cu)
+      }
+    }
+  }
+}
+
+% end # NormalizedIteratorWorkloads
 
 struct Workload {
   static let N = 100

benchmark/utils/main.swift

@@ -266,6 +266,7 @@ registerBenchmark(NSErrorTest)
 registerBenchmark(NSStringConversion)
 registerBenchmark(NibbleSort)
 registerBenchmark(NopDeinit)
+registerBenchmark(NormalizedIterator)
 registerBenchmark(ObjectAllocation)
 #if os(macOS) || os(iOS) || os(watchOS) || os(tvOS)
 registerBenchmark(ObjectiveCBridging)

include/swift/AST/DiagnosticsSema.def

@@ -2825,8 +2825,6 @@ ERROR(builtin_unicode_scalar_literal_broken_proto,none,
 ERROR(unicode_scalar_literal_broken_proto,none,
       "protocol 'ExpressibleByUnicodeScalarLiteral' is broken", ())
 
-ERROR(builtin_utf16_extended_grapheme_cluster_literal_broken_proto,none,
-      "protocol '_ExpressibleByBuiltinUTF16ExtendedGraphemeClusterLiteral' is broken", ())
 ERROR(builtin_extended_grapheme_cluster_literal_broken_proto,none,
       "protocol '_ExpressibleByBuiltinExtendedGraphemeClusterLiteral' is broken", ())
 ERROR(extended_grapheme_cluster_literal_broken_proto,none,

include/swift/AST/KnownIdentifiers.def

@@ -161,7 +161,6 @@ IDENTIFIER_(builtinUnicodeScalarLiteral)
 IDENTIFIER(unicodeScalarLiteral)
 
 IDENTIFIER(stringLiteral)
-IDENTIFIER_(builtinUTF16StringLiteral)
 IDENTIFIER_(builtinStringLiteral)
 IDENTIFIER(StringLiteralType)
 IDENTIFIER(stringInterpolation)

include/swift/AST/KnownProtocols.def

@@ -88,12 +88,10 @@ EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByImageLiteral)
 EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByFileReferenceLiteral)
 
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinBooleanLiteral)
-BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinUTF16ExtendedGraphemeClusterLiteral)
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinExtendedGraphemeClusterLiteral)
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinFloatLiteral)
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinIntegerLiteral)
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinStringLiteral)
-BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinUTF16StringLiteral)
 BUILTIN_EXPRESSIBLE_BY_LITERAL_PROTOCOL_(ExpressibleByBuiltinUnicodeScalarLiteral)
 
 #undef EXPRESSIBLE_BY_LITERAL_PROTOCOL

lib/IRGen/GenMeta.cpp

@@ -4040,12 +4040,10 @@ SpecialProtocol irgen::getSpecialProtocolID(ProtocolDecl *P) {
   case KnownProtocolKind::ExpressibleByImageLiteral:
   case KnownProtocolKind::ExpressibleByFileReferenceLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinBooleanLiteral:
-  case KnownProtocolKind::ExpressibleByBuiltinUTF16ExtendedGraphemeClusterLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinExtendedGraphemeClusterLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinFloatLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinIntegerLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinStringLiteral:
-  case KnownProtocolKind::ExpressibleByBuiltinUTF16StringLiteral:
   case KnownProtocolKind::ExpressibleByBuiltinUnicodeScalarLiteral:
   case KnownProtocolKind::OptionSet:
   case KnownProtocolKind::BridgedNSError:

lib/SILOptimizer/Utils/Local.cpp

@@ -732,16 +732,10 @@ bool StringConcatenationOptimizer::extractStringConcatOperands() {
   auto AILeftOperandsNum = AILeft->getNumOperands();
   auto AIRightOperandsNum = AIRight->getNumOperands();
 
-  // makeUTF16 should have following parameters:
-  // (start: RawPointer, utf16CodeUnitCount: Word)
   // makeUTF8 should have following parameters:
   // (start: RawPointer, utf8CodeUnitCount: Word, isASCII: Int1)
-  if (!((FRILeftFun->hasSemanticsAttr("string.makeUTF16") &&
-         AILeftOperandsNum == 4) ||
-        (FRILeftFun->hasSemanticsAttr("string.makeUTF8") &&
+  if (!((FRILeftFun->hasSemanticsAttr("string.makeUTF8") &&
          AILeftOperandsNum == 5) ||
-        (FRIRightFun->hasSemanticsAttr("string.makeUTF16") &&
-         AIRightOperandsNum == 4) ||
         (FRIRightFun->hasSemanticsAttr("string.makeUTF8") &&
          AIRightOperandsNum == 5)))
     return false;

lib/Sema/CSApply.cpp

@@ -2094,55 +2094,24 @@ namespace {
       Diag<> brokenBuiltinProtocolDiag;
 
       if (isStringLiteral) {
-        // If the string contains only ASCII, force a UTF8 representation
-        bool forceASCII = stringLiteral != nullptr;
-        if (forceASCII) {
-          for (auto c: stringLiteral->getValue()) {
-            if (c & (1 << 7)) {
-              forceASCII = false;
-              break;
-            }
-          }
-        }
-
         literalType = tc.Context.Id_StringLiteralType;
 
         literalFuncName = DeclName(tc.Context, DeclBaseName::createConstructor(),
                                    { tc.Context.Id_stringLiteral });
 
-        // If the string contains non-ASCII and the type can handle
-        // UTF-16 string literals, prefer them.
         builtinProtocol = tc.getProtocol(
             expr->getLoc(),
-            KnownProtocolKind::ExpressibleByBuiltinUTF16StringLiteral);
-
-        if (!forceASCII && (tc.conformsToProtocol(
-                                      type, builtinProtocol, cs.DC,
-                                      ConformanceCheckFlags::InExpression))) {
-          builtinLiteralFuncName =
-              DeclName(tc.Context, DeclBaseName::createConstructor(),
-                       {tc.Context.Id_builtinUTF16StringLiteral,
-                        tc.Context.getIdentifier("utf16CodeUnitCount")});
-
-          if (stringLiteral)
-            stringLiteral->setEncoding(StringLiteralExpr::UTF16);
-          else
-            magicLiteral->setStringEncoding(StringLiteralExpr::UTF16);
-        } else {
-          // Otherwise, fall back to UTF-8.
-          builtinProtocol = tc.getProtocol(
-              expr->getLoc(),
-              KnownProtocolKind::ExpressibleByBuiltinStringLiteral);
-          builtinLiteralFuncName 
-            = DeclName(tc.Context, DeclBaseName::createConstructor(),
-                       { tc.Context.Id_builtinStringLiteral,
-                         tc.Context.getIdentifier("utf8CodeUnitCount"),
-                         tc.Context.getIdentifier("isASCII") });
-          if (stringLiteral)
-            stringLiteral->setEncoding(StringLiteralExpr::UTF8);
-          else
-            magicLiteral->setStringEncoding(StringLiteralExpr::UTF8);
-        }
+            KnownProtocolKind::ExpressibleByBuiltinStringLiteral);
+        builtinLiteralFuncName
+          = DeclName(tc.Context, DeclBaseName::createConstructor(),
+                     { tc.Context.Id_builtinStringLiteral,
+                       tc.Context.getIdentifier("utf8CodeUnitCount"),
+                       tc.Context.getIdentifier("isASCII") });
+        if (stringLiteral)
+          stringLiteral->setEncoding(StringLiteralExpr::UTF8);
+        else
+          magicLiteral->setStringEncoding(StringLiteralExpr::UTF8);
+
         brokenProtocolDiag = diag::string_literal_broken_proto;
         brokenBuiltinProtocolDiag = diag::builtin_string_literal_broken_proto;
       } else if (isGraphemeClusterLiteral) {
@@ -2163,26 +2132,6 @@ namespace {
             diag::extended_grapheme_cluster_literal_broken_proto;
         brokenBuiltinProtocolDiag =
             diag::builtin_extended_grapheme_cluster_literal_broken_proto;
-
-        auto *builtinUTF16ExtendedGraphemeClusterProtocol = tc.getProtocol(
-            expr->getLoc(),
-            KnownProtocolKind::ExpressibleByBuiltinUTF16ExtendedGraphemeClusterLiteral);
-        if (tc.conformsToProtocol(type,
-                                  builtinUTF16ExtendedGraphemeClusterProtocol,
-                                  cs.DC, ConformanceCheckFlags::InExpression)) {
-          builtinLiteralFuncName
-            = DeclName(tc.Context, DeclBaseName::createConstructor(),
-                       { tc.Context.Id_builtinExtendedGraphemeClusterLiteral,
-                         tc.Context.getIdentifier("utf16CodeUnitCount") });
-
-          builtinProtocol = builtinUTF16ExtendedGraphemeClusterProtocol;
-          brokenBuiltinProtocolDiag =
-            diag::builtin_utf16_extended_grapheme_cluster_literal_broken_proto;
-          if (stringLiteral)
-            stringLiteral->setEncoding(StringLiteralExpr::UTF16);
-          else
-            magicLiteral->setStringEncoding(StringLiteralExpr::UTF16);
-        }
       } else {
         // Otherwise, we should have just one Unicode scalar.
         literalType = tc.Context.Id_UnicodeScalarLiteralType;

stdlib/private/StdlibUnicodeUnittest/StdlibUnicodeUnittest.swift

@@ -12,6 +12,108 @@
 
 import StdlibUnittest
 
+extension String {
+  func parseUTF8CodeUnits() -> [UInt8] {
+    var utf8 = [UInt8]()
+    let units = self.split(separator: " ")
+    let scalars = units.compactMap { string -> Unicode.Scalar? in
+      let i = Int(string, radix: 16)!
+      return Unicode.Scalar(i)
+
+    }
+
+    for scalar in scalars {
+      utf8 += String(scalar).utf8
+    }
+    return utf8
+  }
+
+  func parseUTF16CodeUnits() -> [UInt16] {
+    var utf16 = [UInt16]()
+    let units = self.split(separator: " ")
+    let scalars = units.compactMap { string -> Unicode.Scalar? in
+      let i = Int(string, radix: 16)!
+      return Unicode.Scalar(i)
+    }
+
+    for scalar in scalars {
+      utf16 += scalar.utf16
+    }
+    return utf16
+  }
+}
+
+public struct NormalizationTest {
+  public let loc: SourceLoc
+  public let sourceUTF16: [UInt16]
+  public let source: [UInt8]
+  public let NFC: [UInt8]
+  public let NFD: [UInt8]
+  public let NFKC: [UInt8]
+  public let NFKD: [UInt8]
+
+  init(
+    loc: SourceLoc,
+    source: String,
+    NFC: String,
+    NFD: String,
+    NFKC: String,
+    NFKD: String
+  ) {
+    self.loc = loc
+    self.sourceUTF16 = source.parseUTF16CodeUnits()
+    self.source = source.parseUTF8CodeUnits()
+    self.NFC = NFC.parseUTF8CodeUnits()
+    self.NFD = NFD.parseUTF8CodeUnits()
+    self.NFKC = NFKC.parseUTF8CodeUnits()
+    self.NFKD = NFKD.parseUTF8CodeUnits()
+  }
+}
+
+// Normalization tests are currently only avaible on Darwin, awaiting a sensible
+// file API...
+#if _runtime(_ObjC)
+import Foundation
+public let normalizationTests: [NormalizationTest] = {
+  var tests = [NormalizationTest]()
+
+  let file = CommandLine.arguments[2]
+  let fileURL = URL(fileURLWithPath: file)
+
+  let fileContents = try! String(contentsOf: fileURL) + "" // go faster
+
+  var lineNumber: UInt = 0
+  for line in fileContents.split(separator: "\n") {
+    lineNumber += 1
+    guard line.hasPrefix("#") == false else {
+      continue
+    }
+
+    let content = line.split(separator: "#").first!
+
+    guard !content.isEmpty else {
+      continue
+    }
+    guard !content.hasPrefix("@") else {
+      continue
+    }
+
+    let columns = content.split(separator: ";").filter { $0 != " " }.map(String.init)
+    let test = NormalizationTest(
+      loc: SourceLoc(file, lineNumber),
+      source: columns[0],
+      NFC: columns[1],
+      NFD: columns[2],
+      NFKC: columns[3],
+      NFKD: columns[4])
+
+    tests.append(test)
+  }
+
+  return tests
+}()
+#endif
+
 public struct UTFTest {
   public struct Flags : OptionSet {
     public let rawValue: Int

stdlib/private/StdlibUnittest/StdlibCoreExtras.swift

@@ -28,7 +28,7 @@ import Foundation
 //
 
 func findSubstring(_ haystack: Substring, _ needle: String) -> String.Index? {
-  return findSubstring(String(haystack._ephemeralContent), needle)
+  return findSubstring(haystack._ephemeralString, needle)
 }
 
 func findSubstring(_ string: String, _ substring: String) -> String.Index? {