Fix multi-line string literal errors with line endings other than \n

Author: ahoppen

Sources/SwiftParser/Lexer/Cursor.swift

@@ -1773,7 +1773,10 @@ extension Lexer.Cursor {
         if stringLiteralKind == .multiLine {
           // Make sure each line starts a new string segment so the parser can
           // validate the multi-line string literal's indentation.
-          _ = self.advance()
+          let charcter = self.advance()
+          if charcter == UInt8(ascii: "\r") {
+            _ = self.advance(matching: "\n")
+          }
           return Lexer.Result(.stringSegment)
         } else {
           // Single line literals cannot span multiple lines.

Sources/SwiftParser/StringLiterals.swift

@@ -47,7 +47,7 @@ fileprivate class StringLiteralExpressionIndentationChecker {
       return nil
     }
     let hasSufficientIndentation = token.tokenView.leadingTrivia { leadingTrivia in
-      let indentationStartIndex = leadingTrivia.lastIndex(where: { $0 == UInt8(ascii: "\n") || $0 == UInt8(ascii: "\n") })?.advanced(by: 1) ?? leadingTrivia.startIndex
+      let indentationStartIndex = leadingTrivia.lastIndex(where: { $0 == UInt8(ascii: "\n") || $0 == UInt8(ascii: "\r") })?.advanced(by: 1) ?? leadingTrivia.startIndex
       return SyntaxText(rebasing: leadingTrivia[indentationStartIndex...]).hasPrefix(expectedIndentation)
     }
     if hasSufficientIndentation {
@@ -86,6 +86,36 @@ fileprivate class StringLiteralExpressionIndentationChecker {
   }
 }
 
+fileprivate extension SyntaxText {
+  /// If the text ends with any newline character, return the trivia for that
+  /// newline, otherwise `nil`.
+  var newlineSuffix: RawTriviaPiece? {
+    if hasSuffix("\r\n") {
+      return .carriageReturnLineFeeds(1)
+    } else if hasSuffix("\n") {
+      return .newlines(1)
+    } else if hasSuffix("\r") {
+      return .carriageReturns(1)
+    } else {
+      return nil
+    }
+  }
+
+  /// If the text is a single newline character, return the trivia piece that
+  /// represents it, otherwise `nil`.
+  var triviaPieceIfNewline: RawTriviaPiece? {
+    if self == "\r\n" {
+      return .carriageReturnLineFeeds(1)
+    } else if self == "\n" {
+      return .newlines(1)
+    } else if self == "\r" {
+      return .carriageReturns(1)
+    } else {
+      return nil
+    }
+  }
+}
+
 extension Parser {
   /// Consumes a raw string delimiter that has the same number of `#` as `openDelimiter`.
   private mutating func parseStringDelimiter(openDelimiter: RawTokenSyntax?) -> (unexpectedBeforeCheckedDelimiter: RawUnexpectedNodesSyntax?, checkedDelimiter: RawTokenSyntax?) {
@@ -129,7 +159,6 @@ extension Parser {
     }
   }
 
-  // FIXME: Handle \r and \r\n if needed in here
   private func postProcessMultilineStringLiteral(
     openQuote: RawTokenSyntax,
     segments allSegments: [RawStringLiteralSegmentsSyntax.Element],
@@ -181,10 +210,10 @@ extension Parser {
     let closeDelimiterOnNewLine: Bool
     switch middleSegments.last {
     case .stringSegment(let lastMiddleSegment):
-      if lastMiddleSegment.content.tokenText.hasSuffix("\n") {
+      if let newlineSuffix = lastMiddleSegment.content.tokenText.newlineSuffix {
         // The newline at the end of the last line in the string literal is not part of the represented string.
         // Mark it as trivia.
-        var content = lastMiddleSegment.content.reclassifyAsTrailingTrivia([.newlines(1)], arena: self.arena)
+        var content = lastMiddleSegment.content.reclassifyAsTrailingTrivia([newlineSuffix], arena: self.arena)
         var unexpectedBeforeContent: RawTokenSyntax?
         if content.tokenText.hasSuffix("\\") {
           // The newline on the last line must not be escaped
@@ -213,7 +242,11 @@ extension Parser {
     case .expressionSegment:
       closeDelimiterOnNewLine = false
     case nil:
-      closeDelimiterOnNewLine = firstSegment?.content.tokenText.hasSuffix("\n") ?? false
+      if let firstSegment = firstSegment {
+        closeDelimiterOnNewLine = firstSegment.content.tokenText.newlineSuffix != nil
+      } else {
+        closeDelimiterOnNewLine = false
+      }
     }
 
     if !closeDelimiterOnNewLine {
@@ -256,8 +289,8 @@ extension Parser {
     // iterating over is on a new line.
     var isSegmentOnNewLine: Bool
 
-    if firstSegment?.content.tokenText == "\n" {
-      openQuote = openQuote.extendingTrailingTrivia(by: [.newlines(1)], arena: self.arena)
+    if let newlineTrivia = firstSegment?.content.tokenText.triviaPieceIfNewline {
+      openQuote = openQuote.extendingTrailingTrivia(by: [newlineTrivia], arena: self.arena)
       isSegmentOnNewLine = true
     } else {
       if let firstSegment = firstSegment {
@@ -307,15 +340,25 @@ extension Parser {
           }
         }
 
-        isSegmentOnNewLine = segment.content.tokenText.hasSuffix("\n")
+        isSegmentOnNewLine = segment.content.tokenText.newlineSuffix != nil
 
         // If the segment has a `\` in front of its trailing newline, that newline
         // is not part of the reprsented string and should be trivia.
 
-        if segment.content.tokenText.hasSuffix("\\\n") {
+        let backslashNewlineSuffix: [RawTriviaPiece]?
+        if segment.content.tokenText.hasSuffix("\\\r\n") {
+          backslashNewlineSuffix = [.backslashs(1), .carriageReturnLineFeeds(1)]
+        } else if segment.content.tokenText.hasSuffix("\\\n") {
+          backslashNewlineSuffix = [.backslashs(1), .newlines(1)]
+        } else if segment.content.tokenText.hasSuffix("\\\r") {
+          backslashNewlineSuffix = [.backslashs(1), .carriageReturns(1)]
+        } else {
+          backslashNewlineSuffix = nil
+        }
+        if let backslashNewlineSuffix = backslashNewlineSuffix {
           segment = RawStringSegmentSyntax(
             segment.unexpectedBeforeContent,
-            content: segment.content.reclassifyAsTrailingTrivia([.backslashs(1), .newlines(1)], arena: self.arena),
+            content: segment.content.reclassifyAsTrailingTrivia(backslashNewlineSuffix, arena: self.arena),
             segment.unexpectedAfterContent,
             arena: self.arena
           )

Tests/SwiftParserTest/Assertions.swift

@@ -440,16 +440,32 @@ func AssertDiagnostic<T: SyntaxProtocol>(
   }
 }
 
+public struct AssertParseOptions: OptionSet {
+  public var rawValue: UInt8
+
+  public init(rawValue: UInt8) {
+    self.rawValue = rawValue
+  }
+
+  /// Trivia mismatches in the substructure should cause failures
+  public static let substructureCheckTrivia = AssertParseOptions(rawValue: 1 << 0)
+
+  /// Replace all `\r` and `\r\n` in the fixed source by `\n`.
+  /// Useful to match source code that contains other line endings to expected
+  /// fixedfixed source that has `\n` line endings.
+  public static let normalizeNewlinesInFixedSource = AssertParseOptions(rawValue: 1 << 1)
+}
+
 /// Same as `AssertParse` overload with a `(String) -> S` `parse`,
 /// parsing the resulting `String` as a `SourceFileSyntax`.
 func AssertParse(
   _ markedSource: String,
   substructure expectedSubstructure: Syntax? = nil,
   substructureAfterMarker: String = "START",
-  substructureCheckTrivia: Bool = false,
   diagnostics expectedDiagnostics: [DiagnosticSpec] = [],
   applyFixIts: [String]? = nil,
   fixedSource expectedFixedSource: String? = nil,
+  options: AssertParseOptions = [],
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -458,10 +474,10 @@ func AssertParse(
     { SourceFileSyntax.parse(from: &$0) },
     substructure: expectedSubstructure,
     substructureAfterMarker: substructureAfterMarker,
-    substructureCheckTrivia: substructureCheckTrivia,
     diagnostics: expectedDiagnostics,
     applyFixIts: applyFixIts,
     fixedSource: expectedFixedSource,
+    options: options,
     file: file,
     line: line
   )
@@ -475,10 +491,10 @@ func AssertParse<S: SyntaxProtocol>(
   _ parse: (inout Parser) -> S,
   substructure expectedSubstructure: Syntax? = nil,
   substructureAfterMarker: String = "START",
-  substructureCheckTrivia: Bool = false,
   diagnostics expectedDiagnostics: [DiagnosticSpec] = [],
   applyFixIts: [String]? = nil,
   fixedSource expectedFixedSource: String? = nil,
+  options: AssertParseOptions = [],
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -490,10 +506,10 @@ func AssertParse<S: SyntaxProtocol>(
     },
     substructure: expectedSubstructure,
     substructureAfterMarker: substructureAfterMarker,
-    substructureCheckTrivia: substructureCheckTrivia,
     diagnostics: expectedDiagnostics,
     applyFixIts: applyFixIts,
     fixedSource: expectedFixedSource,
+    options: options,
     file: file,
     line: line
   )
@@ -522,10 +538,10 @@ func AssertParse<S: SyntaxProtocol>(
   _ parse: (String) -> S,
   substructure expectedSubstructure: Syntax? = nil,
   substructureAfterMarker: String = "START",
-  substructureCheckTrivia: Bool = false,
   diagnostics expectedDiagnostics: [DiagnosticSpec] = [],
   applyFixIts: [String]? = nil,
   fixedSource expectedFixedSource: String? = nil,
+  options: AssertParseOptions = [],
   file: StaticString = #file,
   line: UInt = #line
 ) {
@@ -553,7 +569,7 @@ func AssertParse<S: SyntaxProtocol>(
   if let expectedSubstructure = expectedSubstructure {
     let subtreeMatcher = SubtreeMatcher(Syntax(tree), markers: markerLocations)
     do {
-      try subtreeMatcher.assertSameStructure(afterMarker: substructureAfterMarker, Syntax(expectedSubstructure), includeTrivia: substructureCheckTrivia, file: file, line: line)
+      try subtreeMatcher.assertSameStructure(afterMarker: substructureAfterMarker, Syntax(expectedSubstructure), includeTrivia: options.contains(.substructureCheckTrivia), file: file, line: line)
     } catch {
       XCTFail("Matching for a subtree failed with error: \(error)", file: file, line: line)
     }
@@ -579,8 +595,15 @@ func AssertParse<S: SyntaxProtocol>(
   // Applying Fix-Its
   if let expectedFixedSource = expectedFixedSource {
     let fixedTree = FixItApplier.applyFixes(in: diags, withMessages: applyFixIts, to: tree)
+    var fixedTreeDescription = fixedTree.description
+    if options.contains(.normalizeNewlinesInFixedSource) {
+      fixedTreeDescription =
+        fixedTreeDescription
+        .replacingOccurrences(of: "\r\n", with: "\n")
+        .replacingOccurrences(of: "\r", with: "\n")
+    }
     AssertStringsEqualWithDiff(
-      fixedTree.description.trimmingTrailingWhitespace(),
+      fixedTreeDescription.trimmingTrailingWhitespace(),
       expectedFixedSource.trimmingTrailingWhitespace(),
       file: file,
       line: line

Tests/SwiftParserTest/ExpressionTests.swift

@@ -924,7 +924,7 @@ final class ExpressionTests: XCTestCase {
           closeQuote: .multilineStringQuoteToken(leadingTrivia: .spaces(2))
         )
       ),
-      substructureCheckTrivia: true
+      options: [.substructureCheckTrivia]
     )
 
     AssertParse(
@@ -944,7 +944,7 @@ final class ExpressionTests: XCTestCase {
           closeQuote: .multilineStringQuoteToken(leadingTrivia: .spaces(2))
         )
       ),
-      substructureCheckTrivia: true
+      options: [.substructureCheckTrivia]
     )
 
     AssertParse(
@@ -969,10 +969,10 @@ final class ExpressionTests: XCTestCase {
           closeQuote: .multilineStringQuoteToken(leadingTrivia: .spaces(2))
         )
       ),
-      substructureCheckTrivia: true,
       diagnostics: [
         DiagnosticSpec(message: "escaped newline at the last line of a multi-line string literal is not allowed")
-      ]
+      ],
+      options: [.substructureCheckTrivia]
     )
   }
 }

Tests/SwiftParserTest/translated/HashbangLibraryTests.swift

@@ -52,7 +52,7 @@ final class HashbangLibraryTests: XCTestCase {
           eofToken: .eof()
         )
       ),
-      substructureCheckTrivia: true
+      options: [.substructureCheckTrivia]
     )
   }
 }