Return nil from parsing functions that didn’t consume any tokens

Sources/SwiftParser/Attributes.swift

@@ -217,15 +217,20 @@ extension Parser {
     // argument lists "immediately" following the attribute name.
     guard self.at(.leftParen) && !self.currentToken.isAtStartOfLine && self.lookahead().isCustomAttributeArgument() else {
       return RawCustomAttributeSyntax(
-        atSignToken: atSign, attributeName: attrName,
-        leftParen: nil, argumentList: nil, rightParen: nil,
-        arena: self.arena)
+        atSignToken: atSign,
+        attributeName: attrName.orMissing(arena: self.arena),
+        leftParen: nil,
+        argumentList: nil,
+        rightParen: nil,
+        arena: self.arena
+      )
     }
     let leftParen = self.eat(.leftParen)
     let arguments = self.parseArgumentListElements()
     let (unexpectedBeforeRightParen, rightParen) = self.expect(.rightParen)
     return RawCustomAttributeSyntax(
-      atSignToken: atSign, attributeName: attrName,
+      atSignToken: atSign,
+      attributeName: attrName.orMissing(arena: self.arena),
       leftParen: leftParen,
       argumentList: RawTupleExprElementListSyntax(elements: arguments, arena: self.arena),
       unexpectedBeforeRightParen,
@@ -340,7 +345,7 @@ extension Parser {
     guard self.at(.leftParen) else {
       // If no opening '(' for parameter list, parse a single parameter.
       let param = self.parseDifferentiabilityParameter().map(RawSyntax.init(_:))
-                  ?? RawSyntax(RawTokenListSyntax(elements: self.recover(), arena: self.arena))
+                  ?? RawSyntax(RawMissingSyntax(arena: self.arena))
       return RawDifferentiabilityParamsClauseSyntax(
         wrtLabel: wrt,
         unexpectedBeforeColon,

Sources/SwiftParser/Declarations.swift

@@ -48,23 +48,11 @@ extension Parser {
   ///
   ///     declarations → declaration declarations?
   @_spi(RawSyntax)
-  public mutating func parseDeclaration() -> RawDeclSyntax {
+  public mutating func parseDeclaration() -> RawDeclSyntax? {
     if self.at(.poundIfKeyword) {
       return RawDeclSyntax(self.parsePoundIfDirective { parser in
-        var parsedDecl = parser.parseDeclaration()
-        if parsedDecl.is(RawMissingDeclSyntax.self) {
-          // Try to recover from a bogus decl.
-          var tokenList = [RawTokenSyntax]()
-          while !parser.at(.eof) && !parser.at(.poundElseKeyword) &&
-                  !parser.at(.poundElseifKeyword) && !parser.at(.poundEndifKeyword) {
-            let tokens = parser.recover()
-            guard !tokens.isEmpty else {
-              break
-            }
-            tokenList.append(contentsOf: tokens)
-          }
-          let unexpected = RawUnexpectedNodesSyntax(elements: tokenList.map(RawSyntax.init), arena: parser.arena)
-          parsedDecl = RawDeclSyntax(RawMissingDeclSyntax(unexpected, attributes: nil, modifiers: nil, arena: parser.arena))
+        guard let parsedDecl = parser.parseDeclaration() else {
+          return nil
         }
         let semicolon = parser.consume(if: .semicolon)
         return RawMemberDeclListItemSyntax(
@@ -118,10 +106,14 @@ extension Parser {
     case _ where self.currentToken.isContextualKeyword("actor"):
       return RawDeclSyntax(self.parseActorDeclaration(attrs))
     default:
-      return RawDeclSyntax(RawMissingDeclSyntax(
-        attributes: attrs.attributes,
-        modifiers: attrs.modifiers,
-        arena: self.arena))
+      if attrs.attributes == nil && attrs.modifiers == nil {
+        return nil
+      } else {
+        return RawDeclSyntax(RawMissingDeclSyntax(
+          attributes: attrs.attributes,
+          modifiers: attrs.modifiers,
+          arena: self.arena))
+      }
     }
   }
 }
@@ -215,7 +207,7 @@ extension Parser {
     return RawExtensionDeclSyntax(
       attributes: attrs.attributes, modifiers: attrs.modifiers,
       extensionKeyword: extensionKeyword,
-      extendedType: type,
+      extendedType: type.orMissing(arena: self.arena),
       inheritanceClause: inheritance,
       genericWhereClause: whereClause,
       members: members,
@@ -317,8 +309,7 @@ extension Parser {
     do {
       var keepGoing: RawTokenSyntax? = nil
       repeat {
-        let firstType = self.parseType()
-        guard !firstType.is(RawMissingTypeSyntax.self) else {
+        guard let firstType = self.parseType() else {
           keepGoing = self.consume(if: .comma)
           elements.append(RawGenericRequirementSyntax(
             body: RawSyntax(RawMissingSyntax(arena: self.arena)),
@@ -328,7 +319,7 @@ extension Parser {
           continue
         }
 
-        let requirement: RawSyntax
+        let requirement: RawSyntax?
         if self.at(.colon) {
           // A conformance-requirement.
           let colon = self.eat(.colon)
@@ -382,7 +373,7 @@ extension Parser {
             requirement = RawSyntax(RawConformanceRequirementSyntax(
               leftTypeIdentifier: firstType,
               colon: colon,
-              rightTypeIdentifier: secondType,
+              rightTypeIdentifier: secondType.orMissing(arena: self.arena),
               arena: self.arena))
           }
         } else if (self.currentToken.isAnyOperator && self.currentToken.tokenText == "==") || self.at(.equal) {
@@ -391,20 +382,25 @@ extension Parser {
           requirement = RawSyntax(RawSameTypeRequirementSyntax(
             leftTypeIdentifier: firstType,
             equalityToken: equal,
-            rightTypeIdentifier: secondType,
+            rightTypeIdentifier: secondType.orMissing(arena: self.arena),
             arena: self.arena))
         } else {
-          requirement = RawSyntax(RawSameTypeRequirementSyntax(
-            leftTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
-            equalityToken: RawTokenSyntax(missing: .equal, arena: self.arena),
-            rightTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
-            arena: self.arena
-          ))
+          requirement = nil
         }
 
         keepGoing = self.consume(if: .comma)
-        elements.append(RawGenericRequirementSyntax(
-          body: requirement, trailingComma: keepGoing, arena: self.arena))
+        if requirement != nil || keepGoing != nil {
+          elements.append(RawGenericRequirementSyntax(
+            body: requirement ?? RawSyntax(RawSameTypeRequirementSyntax(
+              leftTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
+              equalityToken: RawTokenSyntax(missing: .equal, arena: self.arena),
+              rightTypeIdentifier: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
+              arena: self.arena
+            )),
+            trailingComma: keepGoing,
+            arena: self.arena
+          ))
+        }
       } while keepGoing != nil
     }
 
@@ -422,13 +418,12 @@ extension Parser {
     let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)
     do {
       while !self.at(.eof) && !self.at(.rightBrace) {
-        let decl = self.parseDeclaration()
+        guard let decl = self.parseDeclaration() else {
+          break
+        }
         let semi = self.consume(if: .semicolon)
         elements.append(RawMemberDeclListItemSyntax(
           decl: decl, semicolon: semi, arena: self.arena))
-        if decl.is(RawMissingDeclSyntax.self) {
-          break
-        }
       }
     }
     let (unexpectedBeforeRBrace, rbrace) = self.expect(.rightBrace)
@@ -535,7 +530,7 @@ extension Parser {
     do {
       var keepGoing: RawTokenSyntax? = nil
       repeat {
-        let type: RawTypeSyntax
+        let type: RawTypeSyntax?
         if self.at(.classKeyword) {
           let classKeyword = self.eat(.classKeyword)
           type = RawTypeSyntax(RawClassRestrictionTypeSyntax(
@@ -546,8 +541,14 @@ extension Parser {
         }
 
         keepGoing = self.consume(if: .comma)
-        elements.append(RawInheritedTypeSyntax(
-          typeName: type, trailingComma: keepGoing, arena: self.arena))
+
+        if type != nil || keepGoing != nil {
+          elements.append(RawInheritedTypeSyntax(
+            typeName: type.orMissing(arena: self.arena),
+            trailingComma: keepGoing,
+            arena: self.arena
+          ))
+        }
       } while keepGoing != nil
     }
     return RawTypeInheritanceClauseSyntax(
@@ -667,7 +668,11 @@ extension Parser {
         if self.at(.equal) {
           let eq = self.eat(.equal)
           let value = self.parseExpression()
-          rawValue = RawInitializerClauseSyntax(equal: eq, value: value, arena: self.arena)
+          rawValue = RawInitializerClauseSyntax(
+            equal: eq,
+            value: value.orMissing(arena: self.arena),
+            arena: self.arena
+          )
         } else {
           rawValue = nil
         }
@@ -902,7 +907,8 @@ extension Parser {
       let equal = self.eat(.equal)
       let type = self.parseType()
       defaultType = RawTypeInitializerClauseSyntax(
-        equal: equal, value: type,
+        equal: equal,
+        value: type.orMissing(arena: self.arena),
         arena: self.arena)
     } else {
       defaultType = nil
@@ -1173,7 +1179,7 @@ extension Parser {
     let returnClause = RawReturnClauseSyntax(
       arrow: arrow,
       unexpectedBeforeReturnType,
-      returnType: result,
+      returnType: result.orMissing(arena: self.arena),
       arena: self.arena)
     return (returnClause, misplacedThrowsKeyword)
   }
@@ -1283,15 +1289,11 @@ extension Parser {
 
     let indices = self.parseParameterClause()
 
-    let result: RawReturnClauseSyntax
+    let result: RawReturnClauseSyntax?
     if self.at(.arrow) {
       result = self.parseFunctionReturnClause().returnClause
     } else {
-      result = RawReturnClauseSyntax(
-        arrow: RawTokenSyntax(missing: .arrow, arena: self.arena),
-        returnType: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
-        arena: self.arena
-      )
+      result = nil
     }
 
     // Parse a 'where' clause if present.
@@ -1315,7 +1317,11 @@ extension Parser {
       subscriptKeyword: subscriptKeyword,
       genericParameterClause: genericParameterClause,
       indices: indices,
-      result: result,
+      result: result ?? RawReturnClauseSyntax(
+        arrow: RawTokenSyntax(missing: .arrow, arena: self.arena),
+        returnType: RawTypeSyntax(RawMissingTypeSyntax(arena: self.arena)),
+        arena: self.arena
+      ),
       genericWhereClause: genericWhereClause,
       accessor: accessor,
       arena: self.arena)
@@ -1355,7 +1361,8 @@ extension Parser {
           let equal = self.eat(.equal)
           let value = self.parseExpression()
           initializer = RawInitializerClauseSyntax(
-            equal: equal, value: value,
+            equal: equal,
+            value: value.orMissing(arena: self.arena),
             arena: self.arena)
         } else {
           initializer = nil
@@ -1504,22 +1511,6 @@ extension Parser {
           // There can only be an implicit getter if no other accessors were
           // seen before this one.
           guard elements.isEmpty else {
-            // Recover until the matching right brace. It's a little
-            // presumptuous of us to assume everything between here and there
-            // is an accessor, but we cannot stick unexpected anywhere for the
-            // moment...
-            while !self.at(.eof) && !self.at(.rightBrace) {
-              for token in self.recover() {
-                elements.append(RawAccessorDeclSyntax(
-                  attributes: nil, modifier: nil,
-                  accessorKind: token,
-                  parameter: nil,
-                  asyncKeyword: nil, throwsKeyword: nil,
-                  body: nil,
-                  arena: self.arena))
-              }
-            }
-
             let (unexpectedBeforeRBrace, rbrace) = self.expect(.rightBrace)
             return RawSyntax(RawAccessorBlockSyntax(
               leftBrace: lbrace,
@@ -1531,7 +1522,11 @@ extension Parser {
 
           var body = [RawCodeBlockItemSyntax]()
           while !self.at(.eof) && !self.at(.rightBrace) {
-            body.append(self.parseCodeBlockItem())
+            if let newItem = self.parseCodeBlockItem() {
+              body.append(newItem)
+            } else {
+              break
+            }
           }
           let (unexpectedBeforeRBrace, rbrace) = self.expect(.rightBrace)
           return RawSyntax(RawCodeBlockSyntax(
@@ -1626,7 +1621,7 @@ extension Parser {
     let initializer = RawTypeInitializerClauseSyntax(
       unexpectedBeforeEqual,
       equal: equal,
-      value: value,
+      value: value.orMissing(arena: self.arena),
       arena: self.arena
     )
 
@@ -1719,7 +1714,7 @@ extension Parser {
     let (unexpectedBeforeLBrace, lbrace) = self.expect(.leftBrace)
     var elements = [RawSyntax]()
     do {
-      while !self.at(.eof) && !self.at(.rightBrace) {
+      LOOP: while !self.at(.eof) && !self.at(.rightBrace) {
         switch self.currentToken.tokenText {
         case "associativity":
           let associativity = self.consumeIdentifier()
@@ -1772,15 +1767,7 @@ extension Parser {
             otherNames: RawPrecedenceGroupNameListSyntax(elements: names, arena: self.arena),
             arena: self.arena)))
         default:
-          var tokenList = [RawTokenSyntax]()
-          while !self.at(.eof) && !self.at(.rightBrace) {
-            let tokens = self.recover()
-            guard !tokens.isEmpty else {
-              break
-            }
-            tokenList.append(contentsOf: tokens)
-          }
-          elements.append(RawSyntax(RawTokenListSyntax(elements: tokenList, arena: self.arena)))
+          break LOOP
         }
       }
     }

Sources/SwiftParser/Diagnostics/ParseDiagnosticsGenerator.swift

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 import SwiftDiagnostics
-import SwiftSyntax
+@_spi(RawSyntax) import SwiftSyntax
 
 extension UnexpectedNodesSyntax {
   func tokens(satisfying isIncluded: (TokenSyntax) -> Bool) -> [TokenSyntax] {
@@ -58,6 +58,9 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
   ) -> [Diagnostic] {
     let diagProducer = ParseDiagnosticsGenerator()
     diagProducer.walk(tree)
+    diagProducer.diagnostics.sort {
+      return $0.node.id.indexInTree < $1.node.id.indexInTree
+    }
     return diagProducer.diagnostics
   }
 
@@ -172,6 +175,17 @@ public class ParseDiagnosticsGenerator: SyntaxAnyVisitor {
     return .visitChildren
   }
 
+  public override func visit(_ node: SourceFileSyntax) -> SyntaxVisitorContinueKind {
+    if shouldSkip(node) {
+      return .skipChildren
+    }
+    if let extraneous = node.unexpectedBetweenStatementsAndEOFToken, !extraneous.isEmpty {
+      addDiagnostic(extraneous, ExtaneousCodeAtTopLevel(extraneousCode: extraneous))
+      markNodesAsHandled(extraneous.id)
+    }
+    return .visitChildren
+  }
+
   public override func visit(_ node: UnresolvedTernaryExprSyntax) -> SyntaxVisitorContinueKind {
     if shouldSkip(node) {
       return .skipChildren