Diagnose invalid escape sequences in string literals

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -1601,7 +1601,7 @@ extension Lexer.Cursor {
       }
       return cv
 
-    case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\n"), UInt8(ascii: "\r"):
+    case UInt8(ascii: "\n"), UInt8(ascii: "\r"):
       if isMultilineString && self.maybeConsumeNewlineEscape() {
         return UInt32(UInt8(ascii: "\n"))
       }
@@ -1780,12 +1780,15 @@ extension Lexer.Cursor {
         .fixItInsert(Lexer::getSourceLoc(CurPtr), "\n")
     }
 */
+    var error: (LexerError.Kind, Lexer.Cursor)? = nil
+
     while true {
       switch self.peek() {
       case UInt8(ascii: "\\"):
         if self.isAtStringInterpolationAnchor(delimiterLength: delimiterLength) {
           return Lexer.Result(
             .stringSegment,
+            error: error,
             stateTransition: .push(newState: .inStringInterpolationStart(stringLiteralKind: stringLiteralKind))
           )
         } else if self.isAtEscapedNewline(delimiterLength: delimiterLength) {
@@ -1802,16 +1805,17 @@ extension Lexer.Cursor {
           if character == UInt8(ascii: "\r") {
             _ = self.advance(matching: "\n")
           }
-          return Lexer.Result(.stringSegment)
+          return Lexer.Result(.stringSegment, error: error)
         } else {
           // Single line literals cannot span multiple lines.
           // Terminate the string here and go back to normal lexing (instead of `afterStringLiteral`)
           // since we aren't looking for the closing quote anymore.
-          return Lexer.Result(.stringSegment, stateTransition: .pop)
+          return Lexer.Result(.stringSegment, error: error, stateTransition: .pop)
         }
       case nil:
         return Lexer.Result(
           .stringSegment,
+          error: error,
           stateTransition: .replace(newState: .afterStringLiteral(isRawString: delimiterLength > 0))
         )
       default:
@@ -1829,14 +1833,15 @@ extension Lexer.Cursor {
         if escapedCharacter == "\n" || escapedCharacter == "\r" {
           // Make sure each line starts a new string segment so the parser can
           // validate the multi-line string literal's indentation.
-          return Lexer.Result(.stringSegment)
+          return Lexer.Result(.stringSegment, error: error)
         }
       case .error:
-        // TODO: Diagnose error
+        error = (.invalidEscapeSequenceInStringLiteral, self)
         self = clone
       case .endOfString:
         return Lexer.Result(
           .stringSegment,
+          error: error,
           stateTransition: .replace(newState: .afterStringLiteral(isRawString: delimiterLength > 0))
         )
       }

Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift

@@ -40,6 +40,7 @@ public extension LexerError {
 public enum StaticLexerError: String, DiagnosticMessage {
   case expectedBinaryExponentInHexFloatLiteral = "hexadecimal floating point literal must end with an exponent"
   case expectedDigitInFloatLiteral = "expected a digit in floating point exponent"
+  case invalidEscapeSequenceInStringLiteral = "invalid escape sequence in literal"
   case lexerErrorOffsetOverflow = "the lexer dicovered an error in this token but was not able to represent its offset due to overflow; please split the token"
 
   public var message: String { self.rawValue }
@@ -119,10 +120,8 @@ public extension SwiftSyntax.LexerError {
       return InvalidDigitInIntegerLiteral(kind: .binary(scalarAtErrorOffset))
     case .invalidDecimalDigitInIntegerLiteral:
       return InvalidDigitInIntegerLiteral(kind: .decimal(scalarAtErrorOffset))
-    case .invalidFloatingPointCharacter:
-      fatalError()
-    case .invalidFloatingPointDigit:
-      fatalError()
+    case .invalidEscapeSequenceInStringLiteral:
+      return StaticLexerError.invalidEscapeSequenceInStringLiteral
     case .invalidFloatingPointExponentCharacter:
       return InvalidFloatingPointExponentDigit(kind: .character(scalarAtErrorOffset))
     case .invalidFloatingPointExponentDigit:

Sources/SwiftSyntax/LexerError.swift

@@ -22,8 +22,7 @@ public struct LexerError: Hashable {
     case insufficientIndentationInMultilineStringLiteral
     case invalidBinaryDigitInIntegerLiteral
     case invalidDecimalDigitInIntegerLiteral
-    case invalidFloatingPointCharacter
-    case invalidFloatingPointDigit
+    case invalidEscapeSequenceInStringLiteral
     case invalidFloatingPointExponentCharacter
     case invalidFloatingPointExponentDigit
     case invalidHexDigitInIntegerLiteral

Tests/SwiftParserTest/ExpressionTests.swift

@@ -611,9 +611,10 @@ final class ExpressionTests: XCTestCase {
     )
 
     AssertParse(
-      ###""\1️⃣"###,
+      ###""1️⃣\2️⃣"###,
       diagnostics: [
-        DiagnosticSpec(message: #"expected '"' to end string literal"#)
+        DiagnosticSpec(locationMarker: "1️⃣", message: "invalid escape sequence in literal"),
+        DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#),
       ]
     )
   }
@@ -699,9 +700,12 @@ final class ExpressionTests: XCTestCase {
   func testPoundsInStringInterpolationWhereNotNecessary() {
     AssertParse(
       ##"""
-      "\#(1)"
+      "1️⃣\#(1)"
       """##,
-      substructure: Syntax(StringSegmentSyntax(content: .stringSegment(##"\#(1)"##)))
+      substructure: Syntax(StringSegmentSyntax(content: .stringSegment(##"\#(1)"##))),
+      diagnostics: [
+        DiagnosticSpec(message: "invalid escape sequence in literal")
+      ]
     )
   }
 

Tests/SwiftParserTest/LexerTests.swift

@@ -130,12 +130,11 @@ public class LexerTests: XCTestCase {
 
     AssertLexemes(
       #"""
-      "\u{12341234}"
+      "1️⃣\u{12341234}"
       """#,
       lexemes: [
-        // FIXME: We should diagnose invalid unicode characters in string literals
         LexemeSpec(.stringQuote, text: #"""#),
-        LexemeSpec(.stringSegment, text: #"\u{12341234}"#),
+        LexemeSpec(.stringSegment, text: #"\u{12341234}"#, error: "invalid escape sequence in literal"),
         LexemeSpec(.stringQuote, text: #"""#),
       ]
     )

Tests/SwiftParserTest/translated/MultilineErrorsTests.swift

@@ -476,13 +476,13 @@ final class MultilineErrorsTests: XCTestCase {
   func testMultilineErrors20() {
     AssertParseWithAllNewlineEndings(
       ##"""
-      _ = 1️⃣"""
-        line one \ non-whitespace
+      _ = """
+        line one 1️⃣\ non-whitespace
         line two
         """
       """##,
       diagnostics: [
-        // TODO: Old parser expected error on line 2: invalid escape sequence in literal
+        DiagnosticSpec(message: "invalid escape sequence in literal")
       ]
     )
   }
@@ -583,10 +583,11 @@ final class MultilineErrorsTests: XCTestCase {
     AssertParseWithAllNewlineEndings(
       ##"""
       _ = """
-        foo\1️⃣
+        foo1️⃣\2️⃣
       """##,
       diagnostics: [
-        DiagnosticSpec(message: #"expected '"""' to end string literal"#)
+        DiagnosticSpec(locationMarker: "1️⃣", message: "invalid escape sequence in literal"),
+        DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"""' to end string literal"#),
       ]
     )
   }
@@ -600,7 +601,6 @@ final class MultilineErrorsTests: XCTestCase {
       """#,
       diagnostics: [
         DiagnosticSpec(message: "escaped newline at the last line of a multi-line string literal is not allowed")
-        // TODO: Old parser expected error on line 2: escaped newline at the last line is not allowed, Fix-It replacements: 1 - 2 = ''
       ],
       fixedSource: #"""
         _ = """

Tests/SwiftParserTest/translated/StringLiteralEofTests.swift

@@ -52,11 +52,11 @@ final class StringLiteralEofTests: XCTestCase {
   func testStringLiteralEof3() {
     AssertParse(
       ##"""
-      _ = "foo \1️⃣
+      _ = "foo 1️⃣\2️⃣
       """##,
       diagnostics: [
-        // TODO: Old parser expected error on line 1: invalid escape sequence in literal
-        DiagnosticSpec(message: #"expected '"' to end string literal"#)
+        DiagnosticSpec(locationMarker: "1️⃣", message: "invalid escape sequence in literal"),
+        DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#),
       ]
     )
   }
@@ -65,12 +65,11 @@ final class StringLiteralEofTests: XCTestCase {
     AssertParse(
       ##"""
       // NOTE: DO NOT add a newline at EOF.
-      _ = "foo \1️⃣
+      _ = "foo 1️⃣\2️⃣
       """##,
       diagnostics: [
-        // TODO: Old parser expected error on line 2: unterminated string literal
-        // TODO: Old parser expected error on line 2: invalid escape sequence in literal
-        DiagnosticSpec(message: #"expected '"' to end string literal"#)
+        DiagnosticSpec(locationMarker: "1️⃣", message: "invalid escape sequence in literal"),
+        DiagnosticSpec(locationMarker: "2️⃣", message: #"expected '"' to end string literal"#),
       ]
     )
   }