Only emit warnings for nul characters and non-breaking space in the source file

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -2161,6 +2161,9 @@ extension Lexer.Cursor {
       // we have in the parser. Classify the entire curly string as an identifier
       // for best recovery.
       return .lexemeContents(Lexer.Result(.identifier, error: (.unicodeCurlyQuote, position: start)))
+    } else if codepoint.value == 0 {  // Nul character
+      self = tmp
+      return .trivia(error: (kind: .nulCharacter, position: start))
     }
 
     // TODO: Try map confusables to ASCII characters

Sources/SwiftParserDiagnostics/LexerDiagnosticMessages.swift

@@ -49,8 +49,6 @@ public enum StaticLexerError: String, DiagnosticMessage {
   case invalidNumberOfHexDigitsInUnicodeEscape = #"\u{...} escape sequence expects between 1 and 8 hex digits"#
   case invalidUtf8 = "invalid UTF-8 found in source file"
   case lexerErrorOffsetOverflow = "the lexer dicovered an error in this token but was not able to represent its offset due to overflow; please split the token"
-  case nonBreakingSpace = "non-breaking space (U+00A0) used instead of regular space"
-  case nulCharacter = "nul character embedded in middle of file"
   case sourceConflictMarker = "source control conflict marker in source file"
   case unexpectedBlockCommentEnd = "unexpected end of block comment"
   case unicodeCurlyQuote = #"unicode curly quote found; use '"' instead"#
@@ -65,6 +63,20 @@ public enum StaticLexerError: String, DiagnosticMessage {
   public var severity: DiagnosticSeverity { .error }
 }
 
+/// Please order the cases in this enum alphabetically by case name.
+public enum StaticLexerWarning: String, DiagnosticMessage {
+  case nonBreakingSpace = "non-breaking space (U+00A0) used instead of regular space"
+  case nulCharacter = "nul character embedded in middle of file"
+
+  public var message: String { self.rawValue }
+
+  public var diagnosticID: MessageID {
+    MessageID(domain: diagnosticDomain, id: "\(type(of: self)).\(self)")
+  }
+
+  public var severity: DiagnosticSeverity { .warning }
+}
+
 public struct InvalidFloatingPointExponentDigit: LexerError {
   public enum Kind {
     case digit(Unicode.Scalar)
@@ -142,8 +154,8 @@ public extension SwiftSyntax.LexerError {
     case .invalidOctalDigitInIntegerLiteral: return InvalidDigitInIntegerLiteral(kind: .octal(scalarAtErrorOffset))
     case .invalidUtf8: return StaticLexerError.invalidUtf8
     case .lexerErrorOffsetOverflow: return StaticLexerError.lexerErrorOffsetOverflow
-    case .nonBreakingSpace: return StaticLexerError.nonBreakingSpace
-    case .nulCharacter: return StaticLexerError.nulCharacter
+    case .nonBreakingSpace: return StaticLexerWarning.nonBreakingSpace
+    case .nulCharacter: return StaticLexerWarning.nulCharacter
     case .sourceConflictMarker: return StaticLexerError.sourceConflictMarker
     case .unexpectedBlockCommentEnd: return StaticLexerError.unexpectedBlockCommentEnd
     case .unicodeCurlyQuote: return StaticLexerError.unicodeCurlyQuote

Sources/SwiftParserDiagnostics/ParseDiagnosticsGenerator.swift

@@ -30,6 +30,19 @@ fileprivate extension TokenSyntax {
   }
 }
 
+fileprivate extension DiagnosticSeverity {
+  func matches(_ lexerErorSeverity: SwiftSyntax.LexerError.Severity) -> Bool {
+    switch (self, lexerErorSeverity) {
+    case (.error, .error):
+      return true
+    case (.warning, .warning):
+      return true
+    default:
+      return false
+    }
+  }
+}
+
 public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
   private var diagnostics: [Diagnostic] = []
 
@@ -101,7 +114,7 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
   /// Whether the node should be skipped for diagnostic emission.
   /// Every visit method must check this at the beginning.
   func shouldSkip<T: SyntaxProtocol>(_ node: T) -> Bool {
-    if !node.hasError {
+    if !node.hasError && !node.hasWarning {
       return true
     }
     return handledNodes.contains(node.id)
@@ -347,10 +360,12 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
       handleMissingToken(token)
     } else {
       if let lexerError = token.lexerError {
+        let message = lexerError.diagnosticMessage(in: token)
+        assert(message.severity.matches(lexerError.severity))
         self.addDiagnostic(
           token,
           position: token.position.advanced(by: Int(lexerError.byteOffset)),
-          lexerError.diagnosticMessage(in: token),
+          message,
           fixIts: lexerError.fixIts(in: token)
         )
       }

Sources/SwiftSyntax/LexerError.swift

@@ -14,6 +14,11 @@
 /// `lexerErrorOffset` in the token will specify at which offset the error
 /// occurred.
 public struct LexerError: Hashable {
+  public enum Severity {
+    case error
+    case warning
+  }
+
   public enum Kind {
     // Please order these alphabetically
 
@@ -66,4 +71,33 @@ public struct LexerError: Hashable {
       self.byteOffset = UInt16(byteOffset)
     }
   }
+
+  public var severity: Severity {
+    switch kind {
+    case .expectedBinaryExponentInHexFloatLiteral: return .error
+    case .expectedClosingBraceInUnicodeEscape: return .error
+    case .expectedDigitInFloatLiteral: return .error
+    case .expectedHexCodeInUnicodeEscape: return .error
+    case .expectedHexDigitInHexLiteral: return .error
+    case .insufficientIndentationInMultilineStringLiteral: return .error
+    case .invalidBinaryDigitInIntegerLiteral: return .error
+    case .invalidCharacter: return .error
+    case .invalidDecimalDigitInIntegerLiteral: return .error
+    case .invalidEscapeSequenceInStringLiteral: return .error
+    case .invalidFloatingPointExponentCharacter: return .error
+    case .invalidFloatingPointExponentDigit: return .error
+    case .invalidHexDigitInIntegerLiteral: return .error
+    case .invalidIdentifierStartCharacter: return .error
+    case .invalidNumberOfHexDigitsInUnicodeEscape: return .error
+    case .invalidOctalDigitInIntegerLiteral: return .error
+    case .invalidUtf8: return .error
+    case .lexerErrorOffsetOverflow: return .error
+    case .nonBreakingSpace: return .warning
+    case .nulCharacter: return .warning
+    case .sourceConflictMarker: return .error
+    case .unexpectedBlockCommentEnd: return .error
+    case .unicodeCurlyQuote: return .error
+    case .unprintableAsciiCharacter: return .error
+    }
+  }
 }

Sources/SwiftSyntax/Raw/RawSyntax.swift

@@ -23,10 +23,16 @@ fileprivate extension SyntaxKind {
 struct RecursiveRawSyntaxFlags: OptionSet {
   let rawValue: UInt8
 
-  /// Whether the tree contained by this layout has any missing or unexpected nodes.
+  /// Whether the tree contained by this layout has any
+  ///  - missing nodes or
+  ///  - unexpected nodes or
+  ///  - tokens with a `LexerError` of severity `error`
   static let hasError = RecursiveRawSyntaxFlags(rawValue: 1 << 0)
-  static let hasSequenceExpr = RecursiveRawSyntaxFlags(rawValue: 1 << 1)
-  static let hasMaximumNestingLevelOverflow = RecursiveRawSyntaxFlags(rawValue: 1 << 2)
+  /// Whether the tree contained by this layout has any tokens with a `LexerError`
+  /// of severity `warning`.
+  static let hasWarning = RecursiveRawSyntaxFlags(rawValue: 1 << 1)
+  static let hasSequenceExpr = RecursiveRawSyntaxFlags(rawValue: 1 << 2)
+  static let hasMaximumNestingLevelOverflow = RecursiveRawSyntaxFlags(rawValue: 1 << 3)
 }
 
 /// Node data for RawSyntax tree. Tagged union plus common data.
@@ -227,9 +233,17 @@ extension RawSyntax {
     switch view {
     case .token(let tokenView):
       var recursiveFlags: RecursiveRawSyntaxFlags = []
-      if tokenView.lexerError != nil || tokenView.presence == .missing {
+      if tokenView.presence == .missing {
         recursiveFlags.insert(.hasError)
       }
+      switch tokenView.lexerError?.severity {
+      case .error:
+        recursiveFlags.insert(.hasError)
+      case .warning:
+        recursiveFlags.insert(.hasWarning)
+      case nil:
+        break
+      }
       return recursiveFlags
     case .layout(let layoutView):
       return layoutView.recursiveFlags

Sources/SwiftSyntax/Syntax.swift

@@ -283,11 +283,20 @@ public extension SyntaxProtocol {
     return raw.kind.isSyntaxCollection
   }
 
-  /// Whether this tree contains a missing token or unexpected node.
+  /// Whether the tree contained by this layout has any
+  ///  - missing nodes or
+  ///  - unexpected nodes or
+  ///  - tokens with a `LexerError` of severity `error`
   var hasError: Bool {
     return raw.recursiveFlags.contains(.hasError)
   }
 
+  /// Whether the tree contained by this layout has any tokens with a `LexerError`
+  /// of severity `warning`.
+  var hasWarning: Bool {
+    return raw.recursiveFlags.contains(.hasWarning)
+  }
+
   /// Whether this tree contains a missing token or unexpected node.
   var hasSequenceExpr: Bool {
     return raw.recursiveFlags.contains(.hasSequenceExpr)

Tests/SwiftParserTest/Assertions.swift

@@ -234,6 +234,8 @@ struct DiagnosticSpec {
   let id: MessageID?
   /// If not `nil`, assert that the diagnostic has the given message.
   let message: String?
+  /// Assert that the diagnostic has the given severity.
+  let severity: DiagnosticSeverity
   /// If not `nil`, assert that the highlighted range has this content.
   let highlight: String?
   /// If not `nil`, assert that the diagnostic contains notes with these messages.
@@ -250,6 +252,7 @@ struct DiagnosticSpec {
     locationMarker: String = "1️⃣",
     id: MessageID? = nil,
     message: String?,
+    severity: DiagnosticSeverity = .error,
     highlight: String? = nil,
     notes: [NoteSpec]? = nil,
     fixIts: [String]? = nil,
@@ -259,6 +262,7 @@ struct DiagnosticSpec {
     self.locationMarker = locationMarker
     self.id = id
     self.message = message
+    self.severity = severity
     self.highlight = highlight
     self.notes = notes
     self.fixIts = fixIts
@@ -395,6 +399,7 @@ func AssertDiagnostic<T: SyntaxProtocol>(
   if let message = spec.message {
     AssertStringsEqualWithDiff(diag.message, message, file: file, line: line)
   }
+  XCTAssertEqual(spec.severity, diag.diagMessage.severity, file: file, line: line)
   if diag.message.contains("\n") {
     XCTFail(
       """

Tests/SwiftParserTest/ExpressionTests.swift

@@ -1158,7 +1158,7 @@ final class ExpressionTests: XCTestCase {
     AssertParse(
       "a 1️⃣\u{a0}+ 2",
       diagnostics: [
-        DiagnosticSpec(message: "non-breaking space (U+00A0) used instead of regular space", fixIts: ["replace non-breaking space by ' '"])
+        DiagnosticSpec(message: "non-breaking space (U+00A0) used instead of regular space", severity: .warning, fixIts: ["replace non-breaking space by ' '"])
       ],
       fixedSource: "a  + 2"
     )
@@ -1183,6 +1183,15 @@ final class ExpressionTests: XCTestCase {
       """#
     )
   }
+
+  func testNulCharacterInSourceFile() {
+    AssertParse(
+      "let a = 1️⃣\u{0}1",
+      diagnostics: [
+        DiagnosticSpec(message: "nul character embedded in middle of file", severity: .warning)
+      ]
+    )
+  }
 }
 
 final class MemberExprTests: XCTestCase {