Make sure we convert the source to a native UTF-8 string before parsing

It appears like the compiler is now optimizing the `self += ""` away in
release builds. Furthermore, we didn't catch the issue since the
assertion checking for a native UTF-8 string also gets optimized away in
release builds.

Fix the issue in two ways:
1. Make the assertion a precondition that also fails in release builds
2. Generate the native UTF-8 string through an iterator that indexes
   into the foreign string. This is a much more complex transformation
   and harder to optimize away. It also should be a big performance hit
   since the Swift standard library also loops over the foreign string
   to generate a native one.

Author: ahoppen

Sources/SwiftSyntax/SyntaxParser.swift

@@ -127,7 +127,7 @@ public enum SyntaxParser {
     _ filenameForDiagnostics: String,
     _ diagnosticEngine: DiagnosticEngine?
   ) -> RawSyntax {
-    assert(source.isNativeUTF8)
+    precondition(source.isNativeUTF8)
     let c_parser = swiftparse_parser_create()
     defer {
       swiftparse_parser_dispose(c_parser)

Sources/SwiftSyntax/Utils.swift

@@ -80,7 +80,11 @@ extension String {
 
   mutating func makeNativeUTF8IfNeeded() {
     if !isNativeUTF8 {
-      self += ""
+      self = String(IteratorSequence(self.makeIterator()))
+      // Assert that we have actually generated a native UTF-8 string.
+      // We use precondition instead of assert since assert gets optimized away in release builds.
+      // We do, however, want to check that the optimzier does not optimize the conversion to a UTF-8 native string away in release builds.
+      precondition(isNativeUTF8)
     }
   }
 

Tests/SwiftSyntaxTest/SyntaxTests.swift

@@ -144,4 +144,11 @@ public class SyntaxTests: XCTestCase {
     XCTAssertEqual(Syntax(integerExpr), Syntax(fromProtocol: integerExpr as SyntaxProtocol))
     XCTAssertEqual(Syntax(integerExpr), Syntax(fromProtocol: integerExpr as ExprSyntaxProtocol))
   }
+
+  public func testRunParserOnForeignString() {
+    // Store the source code in a foreign non-UTF-8 string.
+    // If SwiftSyntax fails to convert it to a native UTF-8 string, internal assertions should fail.
+    let sourceNsString = "var 🎉 = 2" as NSString
+    _ = try? SyntaxParser.parse(source: sourceNsString as String)
+  }
 }