An escaped backslash at the end of a line in a mulit-line string literal does not escape the newline

Author: ahoppen

CodeGeneration/Sources/SyntaxSupport/gyb_generated/Trivia.swift

@@ -142,6 +142,14 @@ public let TRIVIAS: [Trivia] = [
          swiftCharacters: [
           Character("\\")
         ]),
+  Trivia(name: "Pound",
+         comment: #"A '#' that is at the end of a line in a multi-line string literal to escape the newline."#,
+         characters: [
+           Character("#")
+         ],
+         swiftCharacters: [
+          Character("#")
+        ]),
   Trivia(name: "UnexpectedText",
          comment: #"Any skipped unexpected text."#),
   Trivia(name: "Shebang",

Sources/SwiftParser/StringLiterals.swift

@@ -136,24 +136,32 @@ extension Parser {
   /// not part of the represented string. If the last line has its newline
   /// escaped by a trailing `\`, mark that string segment as unexpected and
   /// generate a missing segment that doesn't have a trailing `\`.
-  private func reclassifyNewlineOfLastSegmentAsTrivia(firstSegment: RawStringSegmentSyntax?, middleSegments: inout [RawStringLiteralSegmentsSyntax.Element]) -> Bool {
+  private func reclassifyNewlineOfLastSegmentAsTrivia(rawStringDelimitersToken: RawTokenSyntax?, firstSegment: RawStringSegmentSyntax?, middleSegments: inout [RawStringLiteralSegmentsSyntax.Element]) -> Bool {
     switch middleSegments.last {
     case .stringSegment(let lastMiddleSegment):
       if let newlineSuffix = lastMiddleSegment.content.tokenText.newlineSuffix {
         // The newline at the end of the last line in the string literal is not part of the represented string.
         // Mark it as trivia.
         var content = lastMiddleSegment.content.reclassifyAsTrailingTrivia([newlineSuffix], arena: self.arena)
         var unexpectedBeforeContent: RawTokenSyntax?
-        if content.tokenText.hasSuffix("\\") {
-          // The newline on the last line must not be escaped
-          unexpectedBeforeContent = content
-          content = RawTokenSyntax(
-            missing: .stringSegment,
-            text: SyntaxText(rebasing: content.tokenText[0..<content.tokenText.count - 1]),
-            leadingTriviaPieces: content.leadingTriviaPieces,
-            trailingTriviaPieces: content.trailingTriviaPieces,
-            arena: self.arena
-          )
+        if content.tokenText.hasNonEscapedBackslashSuffix(rawStringDelimiters: rawStringDelimitersToken?.tokenText ?? "", newline: "") {
+          // The newline on the last line must not be escaped in non-raw string literals.
+
+          if let rawStringDelimitersToken = rawStringDelimitersToken {
+            // ... except in raw string literals where the C++ parser accepts the
+            // last line to be escaped. To match the C++ parser's behavior, we also
+            // need to allow escaped newline in raw string literals.
+            content = content.reclassifyAsTrailingTrivia([.backslashes(1), .pounds(rawStringDelimitersToken.tokenText.count)], arena: self.arena)
+          } else {
+            unexpectedBeforeContent = content
+            content = RawTokenSyntax(
+              missing: .stringSegment,
+              text: SyntaxText(rebasing: content.tokenText[0..<content.tokenText.count - 1]),
+              leadingTriviaPieces: content.leadingTriviaPieces,
+              trailingTriviaPieces: content.trailingTriviaPieces,
+              arena: self.arena
+            )
+          }
         }
 
         middleSegments[middleSegments.count - 1] = .stringSegment(
@@ -185,6 +193,7 @@ extension Parser {
   /// `.insufficientIndentationInMultilineStringLiteral` lexer error will be
   /// attached to the string segment token.
   private func postProcessIndentationAndEscapedNewlineOfMiddleSegments(
+    rawStringDelimitersToken: RawTokenSyntax?,
     middleSegments: inout [RawStringLiteralSegmentsSyntax.Element],
     isFirstSegmentOnNewLine: Bool,
     indentation: SyntaxText,
@@ -231,16 +240,19 @@ extension Parser {
 
         isSegmentOnNewLine = segment.content.tokenText.newlineSuffix != nil
 
+        let rawDelimiters = rawStringDelimitersToken?.tokenText ?? ""
+        assert(rawDelimiters.allSatisfy({ $0 == UInt8(ascii: "#") }))
+
         // If the segment has a `\` in front of its trailing newline, that newline
         // is not part of the represented string and should be trivia.
 
         let backslashNewlineSuffix: [RawTriviaPiece]?
-        if segment.content.tokenText.hasSuffix("\\\r\n") {
-          backslashNewlineSuffix = [.backslashes(1), .carriageReturnLineFeeds(1)]
-        } else if segment.content.tokenText.hasSuffix("\\\n") {
-          backslashNewlineSuffix = [.backslashes(1), .newlines(1)]
-        } else if segment.content.tokenText.hasSuffix("\\\r") {
-          backslashNewlineSuffix = [.backslashes(1), .carriageReturns(1)]
+        if segment.content.tokenText.hasNonEscapedBackslashSuffix(rawStringDelimiters: rawDelimiters, newline: "\r\n") {
+          backslashNewlineSuffix = [.backslashes(1), .pounds(rawDelimiters.count), .carriageReturnLineFeeds(1)].filter { $0.byteLength > 0 }
+        } else if segment.content.tokenText.hasNonEscapedBackslashSuffix(rawStringDelimiters: rawDelimiters, newline: "\n") {
+          backslashNewlineSuffix = [.backslashes(1), .pounds(rawDelimiters.count), .newlines(1)].filter { $0.byteLength > 0 }
+        } else if segment.content.tokenText.hasNonEscapedBackslashSuffix(rawStringDelimiters: rawDelimiters, newline: "\r") {
+          backslashNewlineSuffix = [.backslashes(1), .pounds(rawDelimiters.count), .carriageReturns(1)].filter { $0.byteLength > 0 }
         } else {
           backslashNewlineSuffix = nil
         }
@@ -281,6 +293,7 @@ extension Parser {
   ///    escaped by a trailing `\`, mark that string segment as unexpected and
   ///    generate a missing segment that doesn't have a trailing `\`.
   private func postProcessMultilineStringLiteral(
+    rawStringDelimitersToken: RawTokenSyntax?,
     openQuote: RawTokenSyntax,
     segments allSegments: [RawStringLiteralSegmentsSyntax.Element],
     closeQuote: RawTokenSyntax
@@ -331,6 +344,7 @@ extension Parser {
     // Check that the close quote is on new line
 
     let closeDelimiterOnNewLine = reclassifyNewlineOfLastSegmentAsTrivia(
+      rawStringDelimitersToken: rawStringDelimitersToken,
       firstSegment: firstSegment,
       middleSegments: &middleSegments
     )
@@ -391,6 +405,7 @@ extension Parser {
     // Check indentation of segments and escaped newlines at end of segment
 
     postProcessIndentationAndEscapedNewlineOfMiddleSegments(
+      rawStringDelimitersToken: rawStringDelimitersToken,
       middleSegments: &middleSegments,
       isFirstSegmentOnNewLine: isFirstSegmentOnNewLine,
       indentation: indentation,
@@ -525,7 +540,7 @@ extension Parser {
     let (unexpectedBeforeCloseDelimiter, closeDelimiter) = self.parseStringDelimiter(openDelimiter: openDelimiter)
 
     if openQuote.tokenKind == .multilineStringQuote, !openQuote.isMissing, !closeQuote.isMissing {
-      let postProcessed = postProcessMultilineStringLiteral(openQuote: openQuote, segments: segments, closeQuote: closeQuote)
+      let postProcessed = postProcessMultilineStringLiteral(rawStringDelimitersToken: openDelimiter, openQuote: openQuote, segments: segments, closeQuote: closeQuote)
       return RawStringLiteralExprSyntax(
         openDelimiter: openDelimiter,
         RawUnexpectedNodesSyntax(combining: unexpectedBeforeOpenQuote, postProcessed.unexpectedBeforeOpenQuote, arena: self.arena),
@@ -552,3 +567,20 @@ extension Parser {
     }
   }
 }
+
+// MARK: - Utilities
+
+fileprivate extension SyntaxText {
+  private func hasSuffix(_ other: String) -> Bool {
+    var other = other
+    return other.withSyntaxText { self.hasSuffix($0) }
+  }
+
+  /// Returns `true` if this string end with `\<rawStringDelimiters><newline>`
+  /// but the backslash is not escaped, i.e. doesn't end with
+  /// `\\<rawStringDelimiters><newline>`
+  func hasNonEscapedBackslashSuffix(rawStringDelimiters: SyntaxText, newline: SyntaxText) -> Bool {
+    return self.hasSuffix("\\\(rawStringDelimiters)\(newline)")
+      && !self.hasSuffix("\\\\\(rawStringDelimiters)\(newline)")
+  }
+}

Sources/SwiftParser/TriviaParser.swift

@@ -89,6 +89,9 @@ public struct TriviaParser {
           pieces.append(.shebang(start.text(upTo: cursor)))
           continue
         }
+        cursor.advance(while: { $0 == "#" })
+        pieces.append(.pounds(start.distance(to: cursor)))
+        continue
 
       case UInt8(ascii: "<"), UInt8(ascii: ">"):
         // SCM conflict markers.

Sources/SwiftParserDiagnostics/DiagnosticExtensions.swift

@@ -176,6 +176,8 @@ extension Trivia {
         return Array(repeating: TriviaPiece.newlines(1), count: count)
       case .backslashes(let count):
         return Array(repeating: TriviaPiece.backslashes(1), count: count)
+      case .pounds(let count):
+        return Array(repeating: TriviaPiece.pounds(1), count: count)
       case .carriageReturns(let count):
         return Array(repeating: TriviaPiece.carriageReturns(1), count: count)
       case .carriageReturnLineFeeds(let count):

Sources/SwiftSyntax/SourceLocation.swift

@@ -475,7 +475,8 @@ fileprivate extension RawTriviaPiece {
       let .tabs(count),
       let .verticalTabs(count),
       let .formfeeds(count),
-      let .backslashes(count):
+      let .backslashes(count),
+      let .pounds(count):
       lineLength += SourceLength(utf8Length: count)
     case let .newlines(count),
       let .carriageReturns(count):

Sources/SwiftSyntax/generated/Trivia.swift

@@ -65,6 +65,9 @@ public enum TriviaPiece {
   /// A backslash that is at the end of a line in a multi-line string literal to escape the newline.
   case backslashes(Int)
 
+  /// A '#' that is at the end of a line in a multi-line string literal to escape the newline.
+  case pounds(Int)
+  
   /// Any skipped unexpected text.
   case unexpectedText(String)
 
@@ -107,6 +110,8 @@ extension TriviaPiece: TextOutputStreamable {
       target.write(text)
     case let .backslashes(count): 
       printRepeated(#"\"#, count: count)
+    case let .pounds(count): 
+      printRepeated("#", count: count)
     case let .unexpectedText(text): 
       target.write(text)
     case let .shebang(text): 
@@ -143,6 +148,8 @@ extension TriviaPiece: CustomDebugStringConvertible {
       return "docBlockComment(\(name.debugDescription))"
     case .backslashes(let data): 
       return "backslashes(\(data))"
+    case .pounds(let data): 
+      return "pounds(\(data))"
     case .unexpectedText(let name): 
       return "unexpectedText(\(name.debugDescription))"
     case .shebang(let name): 
@@ -303,6 +310,16 @@ public struct Trivia {
     return .backslashes(1)
   }
 
+  /// Returns a piece of trivia for some number of "#" characters.
+  public static func pounds(_ count: Int) -> Trivia {
+    return [.pounds(count)]
+  }
+  
+  /// Gets a piece of trivia for "#" characters.
+  public static var pound: Trivia {
+    return .pounds(1)
+  }
+  
   /// Returns a piece of trivia for UnexpectedText.
   public static func unexpectedText(_ text: String) -> Trivia {
     return [.unexpectedText(text)]
@@ -418,6 +435,8 @@ extension TriviaPiece {
       return SourceLength(of: text)
     case let .backslashes(count): 
       return SourceLength(utf8Length: count)
+    case let .pounds(count): 
+      return SourceLength(utf8Length: count)
     case let .unexpectedText(text): 
       return SourceLength(of: text)
     case let .shebang(text): 
@@ -456,6 +475,8 @@ public enum RawTriviaPiece: Equatable {
 
   case backslashes(Int)
 
+  case pounds(Int)
+  
   case unexpectedText(SyntaxText)
 
   case shebang(SyntaxText)
@@ -486,6 +507,8 @@ public enum RawTriviaPiece: Equatable {
       return .docBlockComment(arena.intern(text))
     case let .backslashes(count): 
       return .backslashes(count)
+    case let .pounds(count): 
+      return .pounds(count)
     case let .unexpectedText(text): 
       return .unexpectedText(arena.intern(text))
     case let .shebang(text): 
@@ -533,6 +556,8 @@ extension TriviaPiece {
       self = .docBlockComment(String(syntaxText: text))
     case let .backslashes(count): 
       self = .backslashes(count)
+    case let .pounds(count): 
+      self = .pounds(count)
     case let .unexpectedText(text): 
       self = .unexpectedText(String(syntaxText: text))
     case let .shebang(text): 
@@ -568,6 +593,8 @@ extension RawTriviaPiece {
       return text.count
     case let .backslashes(count): 
       return count
+    case let .pounds(count): 
+      return count
     case let .unexpectedText(text): 
       return text.count
     case let .shebang(text): 
@@ -601,6 +628,8 @@ extension RawTriviaPiece {
       return text
     case .backslashes(_): 
       return nil
+    case .pounds(_): 
+      return nil
     case .unexpectedText(let text): 
       return text
     case .shebang(let text): 

Tests/SwiftParserTest/TriviaParserTests.swift

@@ -62,7 +62,8 @@ final class TriviaParserTests: XCTestCase {
         position: .trailing
       ),
       [
-        .unexpectedText("#!/bin/env"),
+        .pounds(1),
+        .unexpectedText("!/bin/env"),
         .spaces(1),
         .unexpectedText("swift"),
       ]