Implement recovery in eat

At the moment this doesn’t change anything because all places that call `eat` have previous checks that make sure the parser is currently positioned at the expected token.

In the future, this will allow us to start parsing functions that `eat` a token at a different token and this function is responsible for eating all intermediate tokens until the expected one.

Author: ahoppen

Sources/SwiftParser/Attributes.swift

@@ -182,30 +182,34 @@ extension Parser {
     }
 
 
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     let ident = self.consumeIdentifierOrRethrows()
+    let unexpectedBeforeLeftParen: RawUnexpectedNodesSyntax?
     let leftParen: RawTokenSyntax?
     let arg: RawSyntax?
     let unexpectedBeforeRightParen: RawUnexpectedNodesSyntax?
     let rightParen: RawTokenSyntax?
     if self.at(.leftParen) {
       var args = [RawTokenSyntax]()
-      leftParen = self.eat(.leftParen)
+      (unexpectedBeforeLeftParen, leftParen) = self.eat(.leftParen)
       var loopProgress = LoopProgressCondition()
       while !self.at(.eof) && !self.at(.rightParen) && loopProgress.evaluate(currentToken) {
         args.append(self.consumeAnyToken())
       }
       arg = RawSyntax(RawTokenListSyntax(elements: args, arena: self.arena))
       (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     } else {
+      unexpectedBeforeLeftParen = nil
       leftParen = nil
       arg = nil
       unexpectedBeforeRightParen = nil
       rightParen = nil
     }
     return RawSyntax(RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: ident,
+      unexpectedBeforeLeftParen,
       leftParen: leftParen,
       argument: arg,
       unexpectedBeforeRightParen,
@@ -215,7 +219,7 @@ extension Parser {
   }
 
   mutating func parseCustomAttribute() -> RawCustomAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     let attrName = self.parseType()
 
     // Custom attributes are stricter than normal attributes about their
@@ -226,11 +230,14 @@ extension Parser {
         leftParen: nil, argumentList: nil, rightParen: nil,
         arena: self.arena)
     }
-    let leftParen = self.eat(.leftParen)
+    let (unexpectedBeforeLeftParen, leftParen) = self.eat(.leftParen)
     let arguments = self.parseArgumentListElements()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawCustomAttributeSyntax(
-      atSignToken: atSign, attributeName: attrName,
+      unexpectedBeforeAtSign,
+      atSignToken: atSign,
+      attributeName: attrName,
+      unexpectedBeforeLeftParen,
       leftParen: leftParen,
       argumentList: RawTupleExprElementListSyntax(elements: arguments, arena: self.arena),
       unexpectedBeforeRightParen,
@@ -241,7 +248,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseAvailabilityAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "available")
     let available = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
@@ -259,6 +266,7 @@ extension Parser {
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
 
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: available,
       unexpectedBeforeLeftParen,
@@ -273,7 +281,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseDifferentiableAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "differentiable")
     let differentiable = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
@@ -282,6 +290,7 @@ extension Parser {
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
 
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: differentiable,
       unexpectedBeforeLeftParen,
@@ -355,7 +364,7 @@ extension Parser {
       )
     }
 
-    let leftParen = self.eat(.leftParen)
+    let (unexpectedBeforeLeftParen, leftParen) = self.eat(.leftParen)
     var elements = [RawDifferentiabilityParamSyntax]()
     var loopProgress = LoopProgressCondition()
     while !self.at(.eof) && !self.at(.rightParen) && loopProgress.evaluate(currentToken) {
@@ -368,6 +377,7 @@ extension Parser {
 
     let parameters = RawDifferentiabilityParamListSyntax(elements: elements, arena: self.arena)
     let list = RawDifferentiabilityParamsSyntax(
+      unexpectedBeforeLeftParen,
       leftParen: leftParen,
       diffParams: parameters,
       unexpectedBeforeRightParen,
@@ -396,10 +406,14 @@ extension Parser {
       return RawDifferentiabilityParamSyntax(
         parameter: RawSyntax(token), trailingComma: comma, arena: self.arena)
     case .selfKeyword:
-      let token = self.eat(.selfKeyword)
+      let (unexpectedBeforeToken, token) = self.eat(.selfKeyword)
       let comma = self.consume(if: .comma)
       return RawDifferentiabilityParamSyntax(
-        parameter: RawSyntax(token), trailingComma: comma, arena: self.arena)
+        unexpectedBeforeToken,
+        parameter: RawSyntax(token),
+        trailingComma: comma,
+        arena: self.arena
+      )
     default:
       return nil
     }
@@ -408,27 +422,32 @@ extension Parser {
 
 extension Parser {
   mutating func parseObjectiveCAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "objc")
     let objc = self.consumeAnyToken()
 
+    let unexpectedBeforeLeftParen: RawUnexpectedNodesSyntax?
     let leftParen: RawTokenSyntax?
     let argument: RawObjCSelectorSyntax?
     let unexpectedBeforeRightParen: RawUnexpectedNodesSyntax?
     let rightParen: RawTokenSyntax?
     if self.at(.leftParen) {
-      leftParen = self.eat(.leftParen)
+      (unexpectedBeforeLeftParen, leftParen) = self.eat(.leftParen)
       argument = self.parseObjectiveCSelector()
       (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     } else {
+      unexpectedBeforeLeftParen = nil
       leftParen = nil
       argument = nil
       unexpectedBeforeRightParen = nil
       rightParen = nil
     }
 
     return RawAttributeSyntax(
-      atSignToken: atSign, attributeName: objc,
+      unexpectedBeforeAtSign,
+      atSignToken: atSign,
+      attributeName: objc,
+      unexpectedBeforeLeftParen,
       leftParen: leftParen,
       argument: argument.map(RawSyntax.init),
       unexpectedBeforeRightParen,
@@ -443,9 +462,13 @@ extension Parser {
     while !self.at(.eof) && !self.at(.rightParen) && loopProgress.evaluate(currentToken) {
       // Empty selector piece.
       if self.at(.colon) {
-        let colon = self.eat(.colon)
+        let (unexpectedBeforeColon, colon) = self.eat(.colon)
         elements.append(RawObjCSelectorPieceSyntax(
-          name: nil, colon: colon, arena: self.arena))
+          name: nil,
+          unexpectedBeforeColon,
+          colon: colon,
+          arena: self.arena
+        ))
         continue
       }
 
@@ -475,13 +498,14 @@ extension Parser {
 
 extension Parser {
   mutating func parseSpecializeAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "_specialize")
     let specializeToken = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
     let argument = self.parseSpecializeAttributeSpecList()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: specializeToken,
       unexpectedBeforeLeftParen,
@@ -659,7 +683,7 @@ extension Parser {
 
 extension Parser {
   mutating func parsePrivateImportAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "_private")
     let privateToken = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
@@ -668,6 +692,7 @@ extension Parser {
     let filename = self.consumeAnyToken()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: privateToken,
       unexpectedBeforeLeftParen,
@@ -688,7 +713,7 @@ extension Parser {
 
 extension Parser {
   mutating func parseDynamicReplacementAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "_dynamicReplacement")
     let dynamicReplacementToken = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
@@ -705,6 +730,7 @@ extension Parser {
     let method = RawDeclNameSyntax(declBaseName: RawSyntax(base), declNameArguments: args, arena: self.arena)
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: dynamicReplacementToken,
       unexpectedBeforeLeftParen,
@@ -725,13 +751,14 @@ extension Parser {
 
 extension Parser {
   mutating func parseSPIAttribute() -> RawAttributeSyntax {
-    let atSign = self.eat(.atSign)
+    let (unexpectedBeforeAtSign, atSign) = self.eat(.atSign)
     assert(self.currentToken.tokenText == "_spi")
     let spiToken = self.consumeAnyToken()
     let (unexpectedBeforeLeftParen, leftParen) = self.expect(.leftParen)
     let label = self.consumeAnyToken()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawAttributeSyntax(
+      unexpectedBeforeAtSign,
       atSignToken: atSign,
       attributeName: spiToken,
       unexpectedBeforeLeftParen,

Sources/SwiftParser/Availability.swift

@@ -132,10 +132,15 @@ extension Parser {
         case .deprecated:
           let argumentLabel = self.consumeAnyToken()
           if self.at(.colon) {
-            let colon = self.eat(.colon)
+            let (unexpectedBeforeColon, colon) = self.eat(.colon)
             let version = self.parseVersionTuple()
             entry = RawSyntax(RawAvailabilityLabeledArgumentSyntax(
-              label: argumentLabel, colon: colon, value: RawSyntax(version), arena: self.arena))
+              label: argumentLabel,
+              unexpectedBeforeColon,
+              colon: colon,
+              value: RawSyntax(version),
+              arena: self.arena
+            ))
           } else {
             entry = RawSyntax(argumentLabel)
           }