Refactor canRecoverTo to return a RecoveryConsumptionHandle

Author: ahoppen

Sources/SwiftParser/Lookahead.swift

@@ -83,7 +83,7 @@ extension Parser.Lookahead {
       return
     }
     var lookahead = self.lookahead()
-    if lookahead.canRecoverTo([kind]) {
+    if lookahead.canRecoverTo([kind]) != nil {
       for _ in 0..<lookahead.tokensConsumed {
         self.consumeAnyToken()
       }

Sources/SwiftParser/Parser.swift

@@ -209,9 +209,12 @@ extension Parser {
   /// Checks if it can reach a token of the given `kind` by skipping unexpected
   /// tokens that have lower ``TokenPrecedence`` than expected token.
   @_spi(RawSyntax)
-  public func canRecoverTo(_ kind: RawTokenKind) -> Bool {
+  public func canRecoverTo(_ kind: RawTokenKind) -> RecoveryConsumptionHandle? {
     if self.at(kind) {
-      return true
+      return RecoveryConsumptionHandle(
+        unexpectedTokens: 0,
+        tokenConsumptionHandle: TokenConsumptionHandle(tokenKind: kind)
+      )
     }
     var lookahead = self.lookahead()
     return lookahead.canRecoverTo([kind])
@@ -224,9 +227,15 @@ extension Parser {
   public func canRecoverToContextualKeyword(
     _ name: SyntaxText,
     precedence: TokenPrecedence = TokenPrecedence(.contextualKeyword)
-  ) -> Bool {
+  ) -> RecoveryConsumptionHandle? {
     if self.atContextualKeyword(name) {
-      return true
+      return RecoveryConsumptionHandle(
+        unexpectedTokens: 0,
+        tokenConsumptionHandle: TokenConsumptionHandle(
+          tokenKind: self.currentToken.tokenKind,
+          remappedKind: .contextualKeyword
+        )
+      )
     }
     var lookahead = self.lookahead()
     return lookahead.canRecoverTo([], contextualKeywords: [name])
@@ -238,9 +247,12 @@ extension Parser {
   @_spi(RawSyntax)
   public func canRecoverTo(
     any kinds: [RawTokenKind]
-  ) -> Bool {
+  ) -> RecoveryConsumptionHandle? {
     if self.at(any: kinds) {
-      return true
+      return RecoveryConsumptionHandle(
+        unexpectedTokens: 0,
+        tokenConsumptionHandle: TokenConsumptionHandle(tokenKind: self.currentToken.tokenKind)
+      )
     }
     var lookahead = self.lookahead()
     return lookahead.canRecoverTo(kinds)
@@ -279,20 +291,16 @@ extension Parser {
   /// Implements the paradigm shared across all `expect` methods.
   private mutating func expectImpl(
     consume: (inout Parser) -> RawTokenSyntax?,
-    canRecoverTo: (inout Lookahead) -> Bool,
+    canRecoverTo: (inout Lookahead) -> RecoveryConsumptionHandle?,
     makeMissing: (inout Parser) -> RawTokenSyntax
   ) -> (unexpected: RawUnexpectedNodesSyntax?, token: RawTokenSyntax) {
     if let tok = consume(&self) {
       return (nil, tok)
     }
     var lookahead = self.lookahead()
-    if canRecoverTo(&lookahead) {
-      var unexpectedTokens = [RawSyntax]()
-      for _ in 0..<lookahead.tokensConsumed {
-        unexpectedTokens.append(RawSyntax(self.consumeAnyToken()))
-      }
-      let token = consume(&self)!
-      return (RawUnexpectedNodesSyntax(elements: unexpectedTokens, arena: self.arena), token)
+    if let handle = canRecoverTo(&lookahead) {
+      let (unexpectedTokens, token) = self.eat(handle)
+      return (unexpectedTokens, token)
     }
     return (nil, makeMissing(&self))
   }

Sources/SwiftParser/Recovery.swift

@@ -18,7 +18,8 @@
 /// After calling `consume(ifAnyFrom:)` we know which token we are positioned
 /// at based on that function's return type. This handle allows consuming that
 /// token.
-struct RecoveryConsumptionHandle {
+@_spi(RawSyntax)
+public struct RecoveryConsumptionHandle {
   var unexpectedTokens: Int
   var tokenConsumptionHandle: TokenConsumptionHandle
 }
@@ -36,7 +37,9 @@ extension Parser.Lookahead {
     _ kinds: [RawTokenKind],
     contextualKeywords: [SyntaxText] = [],
     recoveryPrecedence: TokenPrecedence? = nil
-  ) -> Bool {
+  ) -> RecoveryConsumptionHandle? {
+    let initialTokensConsumed = self.tokensConsumed
+
     var precedences = kinds.map(TokenPrecedence.init)
     if !contextualKeywords.isEmpty {
       precedences += [TokenPrecedence(.identifier), TokenPrecedence(.contextualKeyword)]
@@ -49,22 +52,28 @@ extension Parser.Lookahead {
         break
       }
       if self.at(any: kinds, contextualKeywords: contextualKeywords) {
-        return true
+        return RecoveryConsumptionHandle(
+          unexpectedTokens: self.tokensConsumed - initialTokensConsumed,
+          tokenConsumptionHandle: TokenConsumptionHandle(
+            tokenKind: self.currentToken.tokenKind,
+            remappedKind: self.at(any: [], contextualKeywords: contextualKeywords) ? .contextualKeyword : nil
+          )
+        )
       }
       let currentTokenPrecedence = TokenPrecedence(self.currentToken.tokenKind)
       if currentTokenPrecedence >= recoveryPrecedence {
         break
       }
       self.consumeAnyToken()
       if let closingDelimiter = currentTokenPrecedence.closingTokenKind {
-        guard self.canRecoverTo([closingDelimiter]) else {
+        guard self.canRecoverTo([closingDelimiter]) != nil else {
           break
         }
         self.eat(closingDelimiter)
       }
     }
 
-    return false
+    return nil
   }
 
   /// Checks if we can reach a token in `subset` by skipping tokens that have
@@ -76,6 +85,8 @@ extension Parser.Lookahead {
     anyIn subset: Subset.Type,
     recoveryPrecedence: TokenPrecedence? = nil
   ) -> (Subset, RecoveryConsumptionHandle)? {
+    let initialTokensConsumed = self.tokensConsumed
+
     assert(!subset.allCases.isEmpty, "Subset must have at least one case")
     let recoveryPrecedence = recoveryPrecedence ?? subset.allCases.map({
       if let precedence = $0.precedence {
@@ -84,8 +95,6 @@ extension Parser.Lookahead {
         return TokenPrecedence($0.rawTokenKind)
       }
     }).min()!
-    let initialTokensConsumed = self.tokensConsumed
-    assert(!subset.allCases.isEmpty)
     while !self.at(.eof) {
       if !recoveryPrecedence.shouldSkipOverNewlines,
           self.currentToken.isAtStartOfLine {
@@ -103,7 +112,7 @@ extension Parser.Lookahead {
       }
       self.consumeAnyToken()
       if let closingDelimiter = currentTokenPrecedence.closingTokenKind {
-        guard self.canRecoverTo([closingDelimiter]) else {
+        guard self.canRecoverTo([closingDelimiter]) != nil else {
           break
         }
         self.eat(closingDelimiter)