[syntax-parse] Migrate parsing of literals and magic identifiers

include/swift/Parse/ASTGen.h

@@ -0,0 +1,75 @@
+//===--- ASTGen.h ---------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_ASTGEN_H
+#define SWIFT_PARSE_ASTGEN_H
+
+#include "swift/AST/ASTContext.h"
+#include "swift/AST/Expr.h"
+#include "swift/Syntax/SyntaxNodes.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace swift {
+
+using namespace swift::syntax;
+
+class ComponentIdentTypeRepr;
+class TupleTypeRepr;
+
+/// Generates AST nodes from Syntax nodes.
+class ASTGen {
+  ASTContext &Context;
+public:
+  explicit ASTGen(ASTContext &Context) : Context(Context) {}
+
+  //===--------------------------------------------------------------------===//
+  // MARK: - Expressions
+public:
+  Expr *generate(const BooleanLiteralExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const FloatLiteralExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const IntegerLiteralExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const NilLiteralExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundColumnExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundDsohandleExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundFileExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundFileIDExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundFilePathExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundLineExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const PoundFunctionExprSyntax &Expr, const SourceLoc &Loc);
+  Expr *generate(const UnknownExprSyntax &Expr, const SourceLoc &Loc);
+
+private:
+  /// Map magic literal tokens such as #file to their MagicIdentifierLiteralExpr
+  /// kind.
+  MagicIdentifierLiteralExpr::Kind getMagicIdentifierLiteralKind(tok Kind);
+
+  Expr *
+  generateMagicIdentifierLiteralExpr(const TokenSyntax &PoundToken,
+                                     const SourceLoc &Loc);
+
+  //===--------------------------------------------------------------------===//
+  // MARK: - Other
+public:
+  /// Copy a numeric literal value into AST-owned memory, stripping underscores
+  /// so the semantic part of the value can be parsed by APInt/APFloat parsers.
+  static StringRef copyAndStripUnderscores(StringRef Orig, ASTContext &Context);
+
+private:
+  StringRef copyAndStripUnderscores(StringRef Orig);
+
+  static SourceLoc advanceLocBegin(const SourceLoc &Loc, const Syntax &Node);
+  static SourceLoc advanceLocEnd(const SourceLoc &Loc, const TokenSyntax &Token);
+  static SourceLoc advanceLocAfter(const SourceLoc &Loc, const Syntax &Node);
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_ASTGEN_H

include/swift/Parse/HiddenLibSyntaxAction.h

@@ -0,0 +1,119 @@
+//===--- HiddenLibSyntaxAction.h ------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 202- Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+#define SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+
+#include "swift/Parse/SyntaxParseActions.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+#include "llvm/Support/Allocator.h"
+
+namespace swift {
+namespace syntax {
+class RawSyntax;
+}
+
+// TODO: (syntax-parse) remove when possible
+/// Dispatches all syntax actions to both an explicit action and an implicit
+/// (hidden) action that generates a libSyntax tree. It thereby guarantees that
+/// the libSyntax tree is always generated. The returned opaque nodes contain
+/// both the node created by the explicit action and the implicit action.
+/// \c getLibSyntaxNodeFor and \c getExplicitNodeFor can be used to retrieve
+/// the opaque nodes of the underlying actions.
+class HiddenLibSyntaxAction : public SyntaxParseActions {
+
+  /// The type of the opaque nodes returned by the \c HiddenLibSyntaxAction.
+  /// Contains a pointer to the explicit action's opaque node and the opaque
+  /// libSyntax node.
+  struct HiddenNode {
+    OpaqueSyntaxNode ExplicitActionNode;
+    OpaqueSyntaxNode LibSyntaxNode;
+
+    HiddenNode(OpaqueSyntaxNode explicitActionNode,
+               OpaqueSyntaxNode libSyntaxNode)
+        : ExplicitActionNode(explicitActionNode), LibSyntaxNode(libSyntaxNode) {
+    }
+  };
+
+  /// The explicit action. 
+  std::shared_ptr<SyntaxParseActions> ExplicitAction;
+
+  /// The implicit action that generates the libSyntax tree. Is never \c 
+  /// nullptr.
+  std::shared_ptr<SyntaxTreeCreator> LibSyntaxAction;
+
+  /// An allocator using which the \c HiddenNodes are created.
+  llvm::SpecificBumpPtrAllocator<HiddenNode> NodeAllocator;
+
+  /// Create a hidden node that contains the given explicit and libSyntax node.
+  OpaqueSyntaxNode makeHiddenNode(OpaqueSyntaxNode explicitActionNode,
+                                  OpaqueSyntaxNode libSyntaxNode);
+
+public:
+  /// Create a new HiddenLibSyntaxAction. \c LibSyntaxAction must not be \c
+  /// null.
+  HiddenLibSyntaxAction(
+      const std::shared_ptr<SyntaxParseActions> &explicitAction,
+      const std::shared_ptr<SyntaxTreeCreator> &libSyntaxAction)
+      : ExplicitAction(explicitAction),
+        LibSyntaxAction(libSyntaxAction) {
+    assert(libSyntaxAction != nullptr);
+  };
+
+  OpaqueSyntaxNode recordToken(tok tokenKind,
+                               ArrayRef<ParsedTriviaPiece> leadingTrivia,
+                               ArrayRef<ParsedTriviaPiece> trailingTrivia,
+                               CharSourceRange range) override;
+
+  OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;
+
+  OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
+                                   ArrayRef<OpaqueSyntaxNode> elements,
+                                   CharSourceRange range) override;
+
+  /// Realize the syntax root using the implicit \c LibSyntaxAction.
+  Optional<syntax::SourceFileSyntax>
+  realizeSyntaxRoot(OpaqueSyntaxNode root, const SourceFile &SF) override;
+
+  void discardRecordedNode(OpaqueSyntaxNode node) override;
+
+  /// Look a node at the given lexer offset up using the \c ExplicitAction.
+  ///
+  /// IMPORTANT: The \c LibSyntaxAction will not be called during the node
+  /// lookup and the LibSyntaxNode part of the resulting \c HiddenNode will be a
+  /// \c nullptr.
+  ///
+  /// Discussion: We only perform the lookup because otherwise we could get a
+  /// lookup mismatch between the two actions. In that case, it is not clear,
+  /// what kind of node and lexer offset we should return. For now, just
+  /// performing the lookup in the explicit action seems to be sufficient and
+  /// since \c HiddenLibSyntax action is designed to only be temporary, we
+  /// should be able to live with it.
+  std::pair<size_t, OpaqueSyntaxNode>
+  lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) override;
+
+  /// Returns the libSyntax node from the specified node that has been created
+  /// by this action.
+  syntax::RawSyntax *getLibSyntaxNodeFor(OpaqueSyntaxNode node);
+
+  /// Returns the node created by explicit syntax action from the specified
+  /// node that has been created by this action.
+  OpaqueSyntaxNode getExplicitNodeFor(OpaqueSyntaxNode node);
+
+  /// Returns the underlying libSyntax \c SyntaxTreeCreator.
+  std::shared_ptr<SyntaxTreeCreator> getLibSyntaxAction() {
+    return LibSyntaxAction;
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H

include/swift/Parse/LibSyntaxGenerator.h

@@ -0,0 +1,82 @@
+//===----------- LibSyntaxGenerator.h -------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+#define SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+
+#include "swift/Parse/HiddenLibSyntaxAction.h"
+#include "swift/Parse/ParsedRawSyntaxNode.h"
+#include "swift/Parse/ParsedRawSyntaxRecorder.h"
+#include "swift/Parse/SyntaxParsingCache.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+
+namespace swift {
+// TODO: (swift-parse) remove when possible
+/// Generates libSyntax nodes either by looking them up using
+/// HiddenLibSyntaxAction (based on provided OpaqueSyntaxNode) or by recording
+/// them with ParsedRawSyntaxRecorder.
+class LibSyntaxGenerator {
+  std::shared_ptr<HiddenLibSyntaxAction> Actions;
+  ParsedRawSyntaxRecorder Recorder;
+
+public:
+  explicit LibSyntaxGenerator(std::shared_ptr<HiddenLibSyntaxAction> spActions)
+      : Actions(std::move(spActions)), Recorder(Actions->getLibSyntaxAction()) {
+  }
+
+  /// Create a \c TokenSyntax from the raw data and record it in the
+  /// \c HiddenLibSyntaxAction's \c LibSyntaxAction.
+  TokenSyntax createToken(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredToken());
+
+    auto Kind = Node.getTokenKind();
+    auto Range = Node.getDeferredTokenRangeWithTrivia();
+    auto LeadingTriviaPieces = Node.getDeferredLeadingTriviaPieces();
+    auto TrailingTriviaPieces = Node.getDeferredTrailingTriviaPieces();
+
+    auto Recorded = Recorder.recordToken(Kind, Range, LeadingTriviaPieces,
+                                         TrailingTriviaPieces);
+    auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
+    return make<TokenSyntax>(Raw);
+  }
+
+  /// Create a \c SyntaxNode from the raw data and record it in the
+  /// \c HiddenLibSyntaxAction's \c LibSyntaxAction.
+  template <typename SyntaxNode>
+  SyntaxNode createNode(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredLayout());
+    auto Kind = Node.getKind();
+    auto Children = Node.getDeferredChildren();
+
+    auto Recorded = Recorder.recordRawSyntax(Kind, Children);
+    RC<RawSyntax> Raw{static_cast<RawSyntax *>(Recorded.takeOpaqueNode())};
+    Raw->Release(); // -1 since it's transfer of ownership.
+    return make<SyntaxNode>(Raw);
+  }
+
+  /// Return the libSyntax node stored in the given opaque \c Node.
+  /// Assumes that \c Node is of type \c HiddenNode.
+  TokenSyntax getLibSyntaxTokenFor(OpaqueSyntaxNode Node) {
+    return getLibSyntaxNodeFor<TokenSyntax>(Node);
+  }
+
+  /// Return the libSyntax node stored in the given opaque \c Node.
+  /// Assumes that \c Node is of type \c HiddenNode.
+  template <typename SyntaxNode>
+  SyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode Node) {
+    return make<SyntaxNode>(Actions->getLibSyntaxNodeFor(Node));
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_LIBSYNTAXGENERATOR_H

include/swift/Parse/ParsedSyntaxRecorder.h.gyb

@@ -65,11 +65,29 @@ public:
 
   static Parsed${node.name} makeBlank${node.syntax_kind}(SourceLoc loc,
       SyntaxParsingContext &SPCtx);
+%   elif node.is_unknown():
+private:
+  static Parsed${node.name} record${node.syntax_kind}(
+      MutableArrayRef<ParsedRawSyntaxNode> elts,
+      ParsedRawSyntaxRecorder &rec);
+  static Parsed${node.name} defer${node.syntax_kind}(
+      MutableArrayRef<ParsedRawSyntaxNode> layout,
+      SyntaxParsingContext &SPCtx);
+public:
+  static Parsed${node.name} make${node.syntax_kind}(
+      MutableArrayRef<ParsedSyntax> elts,
+      SyntaxParsingContext &SPCtx);
 %   end
 % end
 
 #pragma mark - Convenience APIs
 
+public:
+  static ParsedTokenSyntax makeToken(const Token &Tok,
+                                     const ParsedTrivia &LeadingTrivia,
+                                     const ParsedTrivia &TrailingTrivia,
+                                     SyntaxParsingContext &SPCtx);
+
   /// Records an unlabelled TupleTypeElementSyntax with the provided type and
   /// optional trailing comma.
   static ParsedTupleTypeElementSyntax

include/swift/Parse/Parser.h

@@ -26,10 +26,12 @@
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
+#include "swift/Parse/ASTGen.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
 #include "swift/Parse/PersistentParserState.h"
 #include "swift/Parse/Token.h"
+#include "swift/Parse/ParsedSyntaxNodes.h"
 #include "swift/Parse/ParserPosition.h"
 #include "swift/Parse/ParserResult.h"
 #include "swift/Parse/SyntaxParsingContext.h"
@@ -405,17 +407,20 @@ class Parser {
   /// Has \c AvailabilityMacros been computed?
   bool AvailabilityMacrosComputed = false;
 
+  /// The AST generator that generates AST nodes from libSyntax nodes.
+  ASTGen ASTGenerator;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
          SILParserStateBase *SIL, PersistentParserState *PersistentState,
-         std::shared_ptr<SyntaxParseActions> SPActions = nullptr);
+         std::shared_ptr<HiddenLibSyntaxAction> SPActions = nullptr);
   Parser(unsigned BufferID, SourceFile &SF, SILParserStateBase *SIL,
          PersistentParserState *PersistentState = nullptr,
-         std::shared_ptr<SyntaxParseActions> SPActions = nullptr);
+         std::shared_ptr<HiddenLibSyntaxAction> SPActions = nullptr);
   Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
          SILParserStateBase *SIL = nullptr,
          PersistentParserState *PersistentState = nullptr,
-         std::shared_ptr<SyntaxParseActions> SPActions = nullptr);
+         std::shared_ptr<HiddenLibSyntaxAction> SPActions = nullptr);
   ~Parser();
 
   /// Returns true if the buffer being parsed is allowed to contain SIL.
@@ -892,6 +897,16 @@ class Parser {
   }
   ParserResult<BraceStmt> parseBraceItemList(Diag<> ID);
 
+  //===--------------------------------------------------------------------===//
+  // MARK: - Primitive Parsing using libSyntax
+
+  /// Consume a token and return the corresponding \c ParsedTokenSyntax.
+  ParsedTokenSyntax consumeTokenSyntax();
+  ParsedTokenSyntax consumeTokenSyntax(tok K) {
+    assert(Tok.is(K) && "Consuming wrong token kind");
+    return consumeTokenSyntax();
+  }
+
   //===--------------------------------------------------------------------===//
   // Decl Parsing
 
@@ -1616,6 +1631,21 @@ class Parser {
 
   void validateCollectionElement(ParserResult<Expr> element);
 
+  //===--------------------------------------------------------------------===//
+  // MARK: - Expression parsing using libSyntax
+
+  // TODO: (syntax-parse) remove when possible
+  /// Parse the upcoming expression of type \c SyntaxNode through libSyntax,
+  /// i.e. parse it into a libSyntax node and generate the \c Expr node from the
+  /// libSyntax node using \c ASTGen.
+  template <typename SyntaxNode> ParserResult<Expr> parseExprAST();
+
+  // TODO: (syntax-parse) create new result type for ParsedSyntax
+  // TODO: (syntax-parse) turn into proper non-templated methods later
+  /// Parse the upcoming expression of type \c SyntaxNode into a libSyntax node
+  /// and return it.
+  template <typename SyntaxNode> ParsedExprSyntax parseExprSyntax();
+
   //===--------------------------------------------------------------------===//
   // Statement Parsing