When a token has both a lexer warning and a lexer error, report the error

Otherwise, we would be surpressing lexer errors by warnings, accepting source code that should be invalid.

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -309,7 +309,7 @@ extension Lexer.Cursor {
     if let leadingTriviaMode = self.currentState.leadingTriviaLexingMode(cursor: self) {
       let triviaResult = self.lexTrivia(mode: leadingTriviaMode)
       newlineInLeadingTrivia = triviaResult.newlinePresence
-      diagnostic = diagnostic ?? triviaResult.error?.tokenDiagnostic(tokenStart: cursor)
+      diagnostic = TokenDiagnostic(combining: diagnostic, triviaResult.error?.tokenDiagnostic(tokenStart: cursor))
     } else {
       newlineInLeadingTrivia = .absent
     }
@@ -345,7 +345,7 @@ extension Lexer.Cursor {
     let trailingTriviaStart = self
     if let trailingTriviaMode = result.trailingTriviaLexingMode ?? currentState.trailingTriviaLexingMode(cursor: self) {
       let triviaResult = self.lexTrivia(mode: trailingTriviaMode)
-      diagnostic = diagnostic ?? triviaResult.error?.tokenDiagnostic(tokenStart: cursor)
+      diagnostic = TokenDiagnostic(combining: diagnostic, triviaResult.error?.tokenDiagnostic(tokenStart: cursor))
     }
 
     if self.currentState.shouldPopStateWhenReachingNewlineInTrailingTrivia && self.is(at: "\r", "\n") {
@@ -358,7 +358,7 @@ extension Lexer.Cursor {
     }
 
     self.previousTokenKind = result.tokenKind.base
-    diagnostic = diagnostic ?? result.error?.tokenDiagnostic(tokenStart: cursor)
+    diagnostic = TokenDiagnostic(combining: diagnostic, result.error?.tokenDiagnostic(tokenStart: cursor))
 
     return .init(
       tokenKind: result.tokenKind,

Sources/SwiftParser/StringLiterals.swift

@@ -55,7 +55,7 @@ fileprivate class StringLiteralExpressionIndentationChecker {
     if hasSufficientIndentation {
       return nil
     }
-    if token.tokenView.tokenDiagnostic != nil {
+    if let tokenDiagnostic = token.tokenView.tokenDiagnostic, tokenDiagnostic.severity == .error {
       // Token already has a lexer error, ignore the indentation error until that
       // error is fixed
       return nil

Sources/SwiftSyntax/TokenDiagnostic.swift

@@ -14,9 +14,9 @@
 /// surrounding structure, this defines the type of the error.
 /// `byteOffset` specifies at which offset the error occurred.
 public struct TokenDiagnostic: Hashable {
-  public enum Severity {
-    case error
+  public enum Severity: Comparable {
     case warning
+    case error
   }
 
   public enum Kind {
@@ -72,6 +72,26 @@ public struct TokenDiagnostic: Hashable {
     }
   }
 
+  /// Picks the more severe error of `lhs` and `rhs`, preferring `lhs` if they
+  /// have the same severity.
+  public init?(combining lhs: TokenDiagnostic?, _ rhs: TokenDiagnostic?) {
+    switch (lhs, rhs) {
+    case (let lhs?, let rhs?):
+      if rhs.severity > lhs.severity {
+        // Prefer the rhs if it is more severe, otherwise continue using lhs.
+        self = rhs
+      } else {
+        self = lhs
+      }
+    case (let lhs?, nil):
+      self = lhs
+    case (nil, let rhs?):
+      self = rhs
+    case (nil, nil):
+      return nil
+    }
+  }
+
   public var severity: Severity {
     switch kind {
     case .expectedBinaryExponentInHexFloatLiteral: return .error

Tests/SwiftParserTest/LexerTests.swift

@@ -1238,4 +1238,15 @@ public class LexerTests: XCTestCase {
       ]
     )
   }
+
+  func testLexerErrorOverridesLexerWarning() {
+    // Make sure we output the error about the malformed hex literal instead of
+    // the warning about the non-breaking whitespace.
+    AssertLexemes(
+      "\u{a0}0x1️⃣r",
+      lexemes: [
+        LexemeSpec(.integerLiteral, leading: "\u{a0}", text: "0xr", diagnostic: "'r' is not a valid hexadecimal digit (0-9, A-F) in integer literal")
+      ]
+    )
+  }
 }