Add *Syntax.parse methods for the nodes that we support parsing

Package.swift

@@ -112,10 +112,11 @@ let package = Package(
       dependencies: ["SwiftDiagnostics", "SwiftSyntax"],
       exclude: [
         "CMakeLists.txt",
-        "README.md",
-        "TypeAttribute.swift.gyb",
         "DeclarationModifier.swift.gyb",
         "DeclarationAttribute.swift.gyb",
+        "Parser+Entry.swift.gyb",
+        "README.md",
+        "TypeAttribute.swift.gyb",
       ]
     ),
     .target(

Sources/SwiftCompilerSupport/ConsistencyCheck.swift

@@ -11,9 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 import SwiftOperators
-@_spi(Testing) @_spi(RawSyntax) import SwiftParser
+import SwiftParser
 import SwiftParserDiagnostics
-@_spi(RawSyntax) import SwiftSyntax
+import SwiftSyntax
 
 extension Syntax {
   /// Whether this syntax node is or is enclosed within a #if.
@@ -51,11 +51,11 @@ public func _parserConsistencyCheck(
   var parser = Parser(buffer)
   return withExtendedLifetime(parser) { () -> CInt in
     // Parse the source file
-    let rawSourceFile = parser.parseSourceFile()
+    let sourceFile = SourceFileSyntax.parse(from: &parser)
 
     // Round-trip test.
     if flags & 0x01 != 0 {
-      if rawSourceFile.raw.syntaxTextBytes != [UInt8](buffer) {
+      if sourceFile.syntaxTextBytes != [UInt8](buffer) {
         print(
           "\(String(cString: filename)): error: file failed to round-trip")
         return 1
@@ -66,7 +66,6 @@ public func _parserConsistencyCheck(
     if flags & 0x02 != 0 {
       var anyDiags = false
 
-      let sourceFile = Syntax(raw: rawSourceFile.raw).as(SourceFileSyntax.self)!
       let diags = ParseDiagnosticsGenerator.diagnostics(
         for: sourceFile)
       for diag in diags {

Sources/SwiftParser/CMakeLists.txt

@@ -34,6 +34,7 @@ add_library(SwiftParser STATIC
 
   gyb_generated/DeclarationAttribute.swift
   gyb_generated/DeclarationModifier.swift
+  gyb_generated/Parser+Entry.swift
   gyb_generated/TypeAttribute.swift)
 
 target_link_libraries(SwiftParser PUBLIC

Sources/SwiftParser/Parser+Entry.swift.gyb

@@ -0,0 +1,76 @@
+%{
+  from gyb_syntax_support import *
+  # -*- mode: Swift -*-
+  # Ignore the following admonition it applies to the resulting .swift file only
+}%
+//// Automatically Generated From Entry.swift.gyb.
+//// Do Not Edit Directly!
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_spi(RawSyntax) import SwiftSyntax
+
+extension Parser {
+  /// Parse the source code in the given string as Swift source file. See
+  /// `Parser.init` for more details.
+  public static func parse(
+    source: String,
+    parseTransition: IncrementalParseTransition? = nil
+  ) -> SourceFileSyntax {
+    var parser = Parser(source)
+    return SourceFileSyntax.parse(from: &parser)
+  }
+
+  /// Parse the source code in the given string as Swift source file. See
+  /// `Parser.init` for more details.
+  public static func parse(
+    source: UnsafeBufferPointer<UInt8>,
+    maximumNestingLevel: Int? = nil,
+    parseTransition: IncrementalParseTransition? = nil
+  ) -> SourceFileSyntax {
+    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
+    return SourceFileSyntax.parse(from: &parser)
+  }
+}
+
+public protocol SyntaxParseable: SyntaxProtocol {
+  static func parse(from parser: inout Parser) -> Self
+}
+
+% for node in SYNTAX_NODES:
+%   if node.parser_function:
+extension ${node.name}: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.${node.parser_function}()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+%   end
+% end
+fileprivate extension Parser {
+  mutating func parseRemainder<R: RawSyntaxNodeProtocol>(into: R) -> R {
+    guard !into.raw.kind.isSyntaxCollection, let layout = into.raw.layoutView else {
+      assertionFailure("Only support parsing of non-collection layout nodes")
+      return into
+    }
+
+    let remainingTokens = self.consumeRemainingTokens()
+    if remainingTokens.isEmpty {
+      return into
+    }
+
+    let unexpected = RawUnexpectedNodesSyntax(elements: remainingTokens, arena: self.arena)
+    let withUnexpected = layout.replacingChild(at: layout.children.count - 1, with: unexpected.raw, arena: self.arena)
+    return R.init(withUnexpected)!
+  }
+}

Sources/SwiftParser/Parser.swift

@@ -12,39 +12,6 @@
 
 @_spi(RawSyntax) import SwiftSyntax
 
-extension Parser {
-  /// Parse the source code in the given string as Swift source file.
-  public static func parse(
-    source: String,
-    parseTransition: IncrementalParseTransition? = nil
-  ) -> SourceFileSyntax {
-    var source = source
-    source.makeContiguousUTF8()
-    return source.withUTF8 { buffer in
-      return parse(source: buffer, parseTransition: parseTransition)
-    }
-  }
-
-  /// Parse the source code in the given string as Swift source file.
-  /// If `maximumNestingLevel` is set, the parser will stop if a nesting level
-  /// that is greater than this value is reached to avoid overflowing the stack.
-  /// The nesting level is increased whenever a bracketed expression like `(`
-  /// or `{` is stared.
-  public static func parse(
-    source: UnsafeBufferPointer<UInt8>,
-    maximumNestingLevel: Int? = nil,
-    parseTransition: IncrementalParseTransition? = nil
-  ) -> SourceFileSyntax {
-    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
-    // Extended lifetime is required because `SyntaxArena` in the parser must
-    // be alive until `Syntax(raw:)` retains the arena.
-    return withExtendedLifetime(parser) {
-      let rawSourceFile =  parser.parseSourceFile()
-      return rawSourceFile.syntax
-    }
-  }
-}
-
 /// A parser for the Swift programming language.
 ///
 /// `Parser` implements a recursive descent parser that produces a SwiftSyntax
@@ -142,17 +109,37 @@ public struct Parser: TokenConsumer {
   public static let defaultMaximumNestingLevel = 256
   #endif
 
-  /// Initializes a Parser from the given input buffer.
-  ///
-  /// The lexer will copy any string data it needs from the resulting buffer
-  /// so it is the caller's responsibility to free it.
+  /// Initializes a `Parser` from the given string.
+  public init(_ input: String, maximumNestingLevel: Int? = nil) {
+    self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
+
+    self.arena = ParsingSyntaxArena(
+      parseTriviaFunction: TriviaParser.parseTrivia(_:position:))
+
+    var input = input
+    input.makeContiguousUTF8()
+    let interned = input.withUTF8 { [arena] buffer in
+      return arena.internSourceBuffer(buffer)
+    }
+
+    self.lexemes = Lexer.tokenize(interned)
+    self.currentToken = self.lexemes.advance()
+  }
+
+  /// Initializes a `Parser` from the given input buffer.
   ///
   /// - Parameters
   ///   - input: An input buffer containing Swift source text.
+  ///   - maximumNestingLevel: To avoid overflowing the stack, the parser will
+  ///                          stop if a nesting level greater than this value
+  ///                          is reached. The nesting level is increased
+  ///                          whenever a bracketed expression like `(` or `{`
+  ///                          is started. `defaultMaximumNestingLevel` is used
+  ///                          if this is `nil`.
   ///   - arena: Arena the parsing syntax are made into. If it's `nil`, a new
   ///            arena is created automatically, and `input` copied into the
-  ///            arena. If non-`nil`, `input` must be the registered source
-  ///            buffer of `arena` or a slice of the source buffer.
+  ///            arena. If non-`nil`, `input` must be within its registered
+  ///            source buffer or allocator.
   public init(_ input: UnsafeBufferPointer<UInt8>, maximumNestingLevel: Int? = nil, arena: ParsingSyntaxArena? = nil) {
     self.maximumNestingLevel = maximumNestingLevel ?? Self.defaultMaximumNestingLevel
 
@@ -166,6 +153,7 @@ public struct Parser: TokenConsumer {
       parseTriviaFunction: TriviaParser.parseTrivia(_:position:))
      sourceBuffer = self.arena.internSourceBuffer(input)
     }
+
     self.lexemes = Lexer.tokenize(sourceBuffer)
     self.currentToken = self.lexemes.advance()
   }
@@ -419,7 +407,7 @@ extension Parser {
   /// This should be set if keywords aren't strong recovery marker at this
   /// position, e.g. because the parser expects a punctuator next.
   @_spi(RawSyntax)
-  public mutating func expectIdentifier(allowIdentifierLikeKeywords: Bool = true, keywordRecovery: Bool = false) -> (RawUnexpectedNodesSyntax?, Token) {
+  public mutating func expectIdentifier(allowIdentifierLikeKeywords: Bool = true, keywordRecovery: Bool = false) -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
     if allowIdentifierLikeKeywords {
       if let (_, handle) = self.canRecoverTo(anyIn: IdentifierTokens.self) {
         return self.eat(handle)
@@ -454,7 +442,7 @@ extension Parser {
   }
 
   @_spi(RawSyntax)
-  public mutating func expectIdentifierOrRethrows() -> (RawUnexpectedNodesSyntax?, Token) {
+  public mutating func expectIdentifierOrRethrows() -> (RawUnexpectedNodesSyntax?, RawTokenSyntax) {
     if let (_, handle) = self.canRecoverTo(anyIn: IdentifierOrRethrowsTokens.self) {
       return self.eat(handle)
     }

Sources/SwiftParser/gyb_generated/Parser+Entry.swift

@@ -0,0 +1,130 @@
+//// Automatically Generated From Entry.swift.gyb.
+//// Do Not Edit Directly!
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@_spi(RawSyntax) import SwiftSyntax
+
+extension Parser {
+  /// Parse the source code in the given string as Swift source file. See
+  /// `Parser.init` for more details.
+  public static func parse(
+    source: String,
+    parseTransition: IncrementalParseTransition? = nil
+  ) -> SourceFileSyntax {
+    var parser = Parser(source)
+    return SourceFileSyntax.parse(from: &parser)
+  }
+
+  /// Parse the source code in the given string as Swift source file. See
+  /// `Parser.init` for more details.
+  public static func parse(
+    source: UnsafeBufferPointer<UInt8>,
+    maximumNestingLevel: Int? = nil,
+    parseTransition: IncrementalParseTransition? = nil
+  ) -> SourceFileSyntax {
+    var parser = Parser(source, maximumNestingLevel: maximumNestingLevel)
+    return SourceFileSyntax.parse(from: &parser)
+  }
+}
+
+public protocol SyntaxParseable: SyntaxProtocol {
+  static func parse(from parser: inout Parser) -> Self
+}
+
+extension DeclSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseDeclaration()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension ExprSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseExpression()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension StmtSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseStatement()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension TypeSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseType()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension PatternSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parsePattern()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension MemberDeclBlockSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseMemberDeclList()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension SourceFileSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseSourceFile()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension SwitchCaseSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseSwitchCase()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension CatchClauseSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseCatchClause()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+extension GenericParameterClauseSyntax: SyntaxParseable {
+  public static func parse(from parser: inout Parser) -> Self {
+    let node = parser.parseGenericParameters()
+    return parser.parseRemainder(into: node).syntax
+  }
+}
+
+fileprivate extension Parser {
+  mutating func parseRemainder<R: RawSyntaxNodeProtocol>(into: R) -> R {
+    guard !into.raw.kind.isSyntaxCollection, let layout = into.raw.layoutView else {
+      assertionFailure("Only support parsing of non-collection layout nodes")
+      return into
+    }
+
+    let remainingTokens = self.consumeRemainingTokens()
+    if remainingTokens.isEmpty {
+      return into
+    }
+
+    let unexpected = RawUnexpectedNodesSyntax(elements: remainingTokens, arena: self.arena)
+    let withUnexpected = layout.replacingChild(at: layout.children.count - 1, with: unexpected.raw, arena: self.arena)
+    return R.init(withUnexpected)!
+  }
+}

Sources/SwiftSyntax/Raw/RawSyntaxNodeProtocol.swift

@@ -236,10 +236,13 @@ public protocol RawSyntaxToSyntax: RawSyntaxNodeProtocol {
   associatedtype SyntaxType: SyntaxProtocol
 }
 
+// TODO: Is it worth having this?
+// The only place that should be using it is the parser, which only converts
+// to `Syntax` in a few places anyway - we could just do the case there.
 @_spi(RawSyntax)
 public extension RawSyntaxToSyntax {
   /// Realizes a `Syntax` node for this `RawSyntax` node.
   var syntax: SyntaxType {
-    return Syntax(raw: raw).as(SyntaxType.self)!
+    return Syntax(raw: raw).cast(SyntaxType.self)
   }
 }