Add an AssertLexemes method to simplify LexerTests

Author: ahoppen

Sources/SwiftParser/Lexer.swift

@@ -24,7 +24,7 @@ public struct Lexer {
   /// represents a fully identified, meaningful part of the input text that
   /// will can be consumed by a ``Parser``.
   public struct Lexeme: CustomDebugStringConvertible {
-    public struct Flags: OptionSet {
+    public struct Flags: OptionSet, CustomDebugStringConvertible {
       public var rawValue: UInt8
 
       public init(rawValue: UInt8) {
@@ -34,6 +34,20 @@ public struct Lexer {
       public static let isAtStartOfLine = Flags(rawValue: 1 << 0)
       public static let isMultilineStringLiteral = Flags(rawValue: 1 << 1)
       public static let isErroneous = Flags(rawValue: 1 << 2)
+
+      public var debugDescription: String {
+        var descriptionComponents: [String] = []
+        if self.contains(.isAtStartOfLine) {
+          descriptionComponents.append("isAtStartOfLine")
+        }
+        if self.contains(.isMultilineStringLiteral) {
+          descriptionComponents.append("isMultilineStringLiteral")
+        }
+        if self.contains(.isErroneous) {
+          descriptionComponents.append("isErroneous")
+        }
+        return "[\(descriptionComponents.joined(separator: ", "))]"
+      }
     }
 
     @_spi(RawSyntax)

Tests/SwiftParserTest/Assertions.swift

@@ -19,6 +19,36 @@ import _SwiftSyntaxTestSupport
 
 // MARK: Lexing Assertions
 
+struct LexemeSpec {
+  let tokenKind: RawTokenKind
+  let leadingTrivia: SyntaxText
+  let tokenText: SyntaxText
+  let trailingTrivia: SyntaxText
+  let flags: Lexer.Lexeme.Flags
+
+  /// The file and line at which this `LexemeSpec` was created, so that assertion failures can be reported at its location.
+  let file: StaticString
+  let line: UInt
+
+  init(
+    _ tokenKind: RawTokenKind,
+    leading: SyntaxText = "",
+    text: SyntaxText,
+    trailing: SyntaxText = "",
+    flags: Lexer.Lexeme.Flags = [],
+    file: StaticString = #file,
+    line: UInt = #line
+  ) {
+    self.tokenKind = tokenKind
+    self.leadingTrivia = leading
+    self.tokenText = text
+    self.trailingTrivia = trailing
+    self.flags = flags
+    self.file = file
+    self.line = line
+  }
+}
+
 /// Asserts that two lexical streams are structurally equal, including their trivia and any
 /// text.
 ///
@@ -29,45 +59,85 @@ import _SwiftSyntaxTestSupport
 ///     which this function was called.
 ///   - line: The line number on which failure occurred. Defaults to the line number on which this
 ///     function was called.
-func AssertEqualTokens(_ actual: [Lexer.Lexeme], _ expected: [Lexer.Lexeme], file: StaticString = #file, line: UInt = #line) {
+private func AssertTokens(_ actual: [Lexer.Lexeme], _ expected: [LexemeSpec], file: StaticString = #file, line: UInt = #line) {
   guard actual.count == expected.count else {
-    return XCTFail("Number of tokens does not match! \(actual.count) != \(expected.count)", file: file, line: line)
+    return XCTFail(
+      "Expected \(expected.count) tokens but got \(actual.count)",
+      file: file,
+      line: line
+    )
   }
 
-  for (idx, (l, r)) in zip(actual, expected).enumerated() {
-    guard l.tokenKind == r.tokenKind else {
-      return XCTFail("Token at index \(idx) does not match! \(l.tokenKind) != \(r.tokenKind)", file: file, line: line)
+  for (actualLexeme, expectedLexeme) in zip(actual, expected) {
+    if actualLexeme.tokenKind != expectedLexeme.tokenKind {
+      XCTFail(
+        "Expected token kind \(expectedLexeme.tokenKind) but got \(actualLexeme.tokenKind)",
+        file: expectedLexeme.file,
+        line: expectedLexeme.line
+      )
     }
 
-    guard l.leadingTriviaText == r.leadingTriviaText else {
-      return FailStringsEqualWithDiff(
-        String(syntaxText: l.leadingTriviaText),
-        String(syntaxText: r.leadingTriviaText),
-        "Token at index \(idx) does not have matching leading trivia",
-        file: file,
-        line: line
+    if actualLexeme.leadingTriviaText != expectedLexeme.leadingTrivia {
+      FailStringsEqualWithDiff(
+        String(syntaxText: actualLexeme.leadingTriviaText),
+        String(syntaxText: expectedLexeme.leadingTrivia),
+        "Leading trivia does not match",
+        file: expectedLexeme.file,
+        line: expectedLexeme.line
       )
     }
 
-    guard l.tokenText.debugDescription == r.tokenText.debugDescription else {
-      return FailStringsEqualWithDiff(
-        l.tokenText.debugDescription,
-        r.tokenText.debugDescription,
-        "Text at index \(idx) does not have matching text",
-        file: file,
-        line: line
+    if actualLexeme.tokenText.debugDescription != expectedLexeme.tokenText.debugDescription {
+      FailStringsEqualWithDiff(
+        actualLexeme.tokenText.debugDescription,
+        expectedLexeme.tokenText.debugDescription,
+        "Token text does not match",
+        file: expectedLexeme.file,
+        line: expectedLexeme.line
       )
     }
 
-    guard l.trailingTriviaText == r.trailingTriviaText else {
-      return FailStringsEqualWithDiff(
-        String(syntaxText: l.trailingTriviaText),
-        String(syntaxText: r.trailingTriviaText),
-        "Token at index \(idx) does not have matching trailing trivia",
-        file: file,
-        line: line
+    if actualLexeme.trailingTriviaText != expectedLexeme.trailingTrivia {
+      FailStringsEqualWithDiff(
+        String(syntaxText: actualLexeme.trailingTriviaText),
+        String(syntaxText: expectedLexeme.trailingTrivia),
+        "Trailing trivia does not match",
+        file: expectedLexeme.file,
+        line: expectedLexeme.line
       )
     }
+
+    if actualLexeme.flags != expectedLexeme.flags {
+      XCTFail(
+        "Expected flags \(expectedLexeme.flags.debugDescription) but got \(actualLexeme.flags.debugDescription)",
+        file: expectedLexeme.file,
+        line: expectedLexeme.line
+      )
+    }
+  }
+}
+
+func AssertLexemes(
+  _ source: String,
+  lexemes expectedLexemes: [LexemeSpec],
+  file: StaticString = #file,
+  line: UInt = #line
+) {
+  var expectedLexemes = expectedLexemes
+  if expectedLexemes.last?.tokenKind != .eof {
+    expectedLexemes.append(LexemeSpec(.eof, text: ""))
+  }
+  var source = source
+  source.withUTF8 { buf in
+    var lexemes = [Lexer.Lexeme]()
+    for token in Lexer.tokenize(buf, from: 0) {
+      lexemes.append(token)
+
+      guard token.tokenKind != .eof else {
+        break
+      }
+    }
+    AssertTokens(lexemes, expectedLexemes, file: file, line: line)
   }
 }
 
@@ -271,7 +341,9 @@ func AssertDiagnostic<T: SyntaxProtocol>(
       """
       Diagnostic message should only span a single line. Message was:
       \(diag.message)
-      """
+      """,
+      file: file,
+      line: line
     )
   }
   if let highlight = spec.highlight {