Revert "Revert "Merge pull request swiftlang#26403 from rintaro/gsoc-2019-part1""

This reverts commit 8a03e08.

Author: rintaro

include/swift/Parse/ASTGen.h

@@ -0,0 +1,71 @@
+//===--- ASTGen.h ---------------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_ASTGEN_H
+#define SWIFT_PARSE_ASTGEN_H
+
+#include "swift/AST/ASTContext.h"
+#include "swift/AST/Expr.h"
+#include "swift/Syntax/SyntaxNodes.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace swift {
+class ComponentIdentTypeRepr;
+class TupleTypeRepr;
+
+/// Generates AST nodes from Syntax nodes.
+class ASTGen {
+  ASTContext &Context;
+  // A stack of source locations of syntax constructs. Allows us to get the
+  // SourceLoc necessary to create AST nodes for nodes in not-yet-complete
+  // Syntax tree. The topmost item should always correspond to the token/node
+  // that has been parsed/transformed most recently.
+  // todo [gsoc]: remove when possible
+  llvm::SmallVector<SourceLoc, 16> LocStack;
+
+public:
+  explicit ASTGen(ASTContext &Context) : Context(Context) {}
+
+  IntegerLiteralExpr *generate(syntax::IntegerLiteralExprSyntax &Expr);
+  FloatLiteralExpr *generate(syntax::FloatLiteralExprSyntax &Expr);
+  NilLiteralExpr *generate(syntax::NilLiteralExprSyntax &Expr);
+  BooleanLiteralExpr *generate(syntax::BooleanLiteralExprSyntax &Expr);
+  MagicIdentifierLiteralExpr *generate(syntax::PoundFileExprSyntax &Expr);
+  MagicIdentifierLiteralExpr *generate(syntax::PoundLineExprSyntax &Expr);
+  MagicIdentifierLiteralExpr *generate(syntax::PoundColumnExprSyntax &Expr);
+  MagicIdentifierLiteralExpr *generate(syntax::PoundFunctionExprSyntax &Expr);
+  MagicIdentifierLiteralExpr *generate(syntax::PoundDsohandleExprSyntax &Expr);
+  Expr *generate(syntax::UnknownExprSyntax &Expr);
+
+  /// Stores source location necessary for AST creation.
+  void pushLoc(SourceLoc Loc);
+
+  /// Copy a numeric literal value into AST-owned memory, stripping underscores
+  /// so the semantic part of the value can be parsed by APInt/APFloat parsers.
+  static StringRef copyAndStripUnderscores(StringRef Orig, ASTContext &Context);
+
+private:
+  StringRef copyAndStripUnderscores(StringRef Orig);
+
+  SourceLoc topLoc();
+
+  MagicIdentifierLiteralExpr *
+  generateMagicIdentifierLiteralExpr(const syntax::TokenSyntax &PoundToken);
+
+  /// Map magic literal tokens such as #file to their MagicIdentifierLiteralExpr
+  /// kind.
+  static MagicIdentifierLiteralExpr::Kind
+  getMagicIdentifierLiteralKind(tok Kind);
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_ASTGEN_H

include/swift/Parse/HiddenLibSyntaxAction.h

@@ -0,0 +1,96 @@
+//===--- HiddenLibSyntaxAction.h ------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+#define SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H
+
+#include "swift/Parse/SyntaxParseActions.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+#include "llvm/Support/Allocator.h"
+
+namespace swift {
+namespace syntax {
+class RawSyntax;
+}
+
+/// Holds an explicitly provided action and uses it to handle all function
+/// calls. Also hides an implicit SyntaxTreeCreator and ensures libSyntax nodes
+/// are always created. Provides an interface to map results of the explicitly
+/// provided action to the hidden libSyntax action.
+// todo [gsoc]: remove when possible
+class HiddenLibSyntaxAction : public SyntaxParseActions {
+
+  struct Node {
+    OpaqueSyntaxNode ExplicitActionNode;
+    OpaqueSyntaxNode LibSyntaxNode;
+
+    Node(OpaqueSyntaxNode ExplicitActionNode, OpaqueSyntaxNode LibSyntaxNode)
+        : ExplicitActionNode(ExplicitActionNode), LibSyntaxNode(LibSyntaxNode) {
+    }
+  };
+
+  std::shared_ptr<SyntaxParseActions> ExplicitAction;
+  std::shared_ptr<SyntaxTreeCreator> LibSyntaxAction;
+  llvm::SpecificBumpPtrAllocator<Node> NodeAllocator;
+
+  bool areBothLibSyntax() {
+    return ExplicitAction->getOpaqueKind() == OpaqueSyntaxNodeKind::LibSyntax;
+  }
+
+  OpaqueSyntaxNode makeHiddenNode(OpaqueSyntaxNode explicitActionNode,
+                                  OpaqueSyntaxNode libSyntaxNode);
+
+public:
+  HiddenLibSyntaxAction(
+      const std::shared_ptr<SyntaxParseActions> &SPActions,
+      const std::shared_ptr<SyntaxTreeCreator> &LibSyntaxAction)
+      : ExplicitAction(SPActions != nullptr ? SPActions : LibSyntaxAction),
+        LibSyntaxAction(LibSyntaxAction){};
+
+  OpaqueSyntaxNode recordToken(tok tokenKind,
+                               ArrayRef<ParsedTriviaPiece> leadingTrivia,
+                               ArrayRef<ParsedTriviaPiece> trailingTrivia,
+                               CharSourceRange range) override;
+
+  OpaqueSyntaxNode recordMissingToken(tok tokenKind, SourceLoc loc) override;
+
+  OpaqueSyntaxNode recordRawSyntax(syntax::SyntaxKind kind,
+                                   ArrayRef<OpaqueSyntaxNode> elements,
+                                   CharSourceRange range) override;
+
+  std::pair<size_t, OpaqueSyntaxNode>
+  lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) override;
+
+  OpaqueSyntaxNodeKind getOpaqueKind() override {
+    return ExplicitAction->getOpaqueKind();
+  }
+
+  /// Returns the libSyntax node from the specified node that has been created
+  /// by this action.
+  syntax::RawSyntax *getLibSyntaxNodeFor(OpaqueSyntaxNode node);
+
+  /// Returns the node created by explicit syntax action from the specified
+  /// node that has been created by this action.
+  OpaqueSyntaxNode getExplicitNodeFor(OpaqueSyntaxNode node);
+
+  bool isReleaseNeeded() {
+    return ExplicitAction == LibSyntaxAction || !areBothLibSyntax();
+  }
+
+  /// Returns the underlying libSyntax SyntaxTreeCreator.
+  std::shared_ptr<SyntaxTreeCreator> getLibSyntaxAction() {
+    return LibSyntaxAction;
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_HIDDENLIBSYNTAXACTION_H

include/swift/Parse/LibSyntaxGenerator.h

@@ -0,0 +1,80 @@
+//===----------- LibSyntaxGenerator.h -------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+#define SWIFT_PARSE_LIBSYNTAXGENERATOR_H
+
+#include "swift/Parse/HiddenLibSyntaxAction.h"
+#include "swift/Parse/ParsedRawSyntaxNode.h"
+#include "swift/Parse/ParsedRawSyntaxRecorder.h"
+#include "swift/Parse/SyntaxParsingCache.h"
+#include "swift/Syntax/RawSyntax.h"
+#include "swift/SyntaxParse/SyntaxTreeCreator.h"
+
+namespace swift {
+/// Generates libSyntax nodes either by looking them up using
+/// HiddenLibSyntaxAction (based on provided OpaqueSyntaxNode) or by recording
+/// them with ParsedRawSyntaxRecorder.
+// todo [gsoc]: remove when possible
+class LibSyntaxGenerator {
+  std::shared_ptr<HiddenLibSyntaxAction> Actions;
+  ParsedRawSyntaxRecorder Recorder;
+
+public:
+  explicit LibSyntaxGenerator(std::shared_ptr<HiddenLibSyntaxAction> TwoActions)
+      : Actions(std::move(TwoActions)),
+        Recorder(Actions->getLibSyntaxAction()) {}
+
+  TokenSyntax createToken(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredToken());
+
+    auto Kind = Node.getTokenKind();
+    auto Range = Node.getDeferredTokenRangeWithTrivia();
+    auto LeadingTriviaPieces = Node.getDeferredLeadingTriviaPieces();
+    auto TrailingTriviaPieces = Node.getDeferredTrailingTriviaPieces();
+
+    auto Recorded = Recorder.recordToken(Kind, Range, LeadingTriviaPieces,
+                                         TrailingTriviaPieces);
+    auto Raw = static_cast<RawSyntax *>(Recorded.getOpaqueNode());
+    return make<TokenSyntax>(Raw);
+  }
+
+  template <typename SyntaxNode>
+  SyntaxNode createNode(ParsedRawSyntaxNode Node) {
+    assert(Node.isDeferredLayout());
+    auto Kind = Node.getKind();
+    auto Children = Node.getDeferredChildren();
+
+    auto Recorded = Recorder.recordRawSyntax(Kind, Children);
+    RC<RawSyntax> Raw {static_cast<RawSyntax *>(Recorded.getOpaqueNode()) };
+    Raw->Release(); // -1 since it's transfer of ownership.
+    return make<SyntaxNode>(Raw);
+  }
+
+  TokenSyntax getLibSyntaxTokenFor(OpaqueSyntaxNode Node) {
+    return getLibSyntaxNodeFor<TokenSyntax>(Node);
+  }
+
+  template <typename SyntaxNode>
+  SyntaxNode getLibSyntaxNodeFor(OpaqueSyntaxNode Node) {
+    return make<SyntaxNode>(Actions->getLibSyntaxNodeFor(Node));
+  }
+
+  OpaqueSyntaxNode finalizeNode(OpaqueSyntaxNode Node) {
+    if (Actions->isReleaseNeeded())
+      Actions->getLibSyntaxNodeFor(Node)->Release();
+    return Actions->getExplicitNodeFor(Node);
+  }
+};
+} // namespace swift
+
+#endif // SWIFT_PARSE_LIBSYNTAXGENERATOR_H

include/swift/Parse/ParsedRawSyntaxNode.h

@@ -156,6 +156,19 @@ class ParsedRawSyntaxNode {
 
   // Deferred Token Data =====================================================//
 
+  CharSourceRange getDeferredTokenRangeWithTrivia() const {
+    assert(DK == DataKind::DeferredToken);
+    auto leadTriviaPieces = getDeferredLeadingTriviaPieces();
+    auto trailTriviaPieces = getDeferredTrailingTriviaPieces();
+
+    auto leadTriviaLen = ParsedTriviaPiece::getTotalLength(leadTriviaPieces);
+    auto trailTriviaLen = ParsedTriviaPiece::getTotalLength(trailTriviaPieces);
+
+    SourceLoc begin = DeferredToken.TokLoc.getAdvancedLoc(-leadTriviaLen);
+    unsigned len = leadTriviaLen + DeferredToken.TokLength + trailTriviaLen;
+
+    return CharSourceRange{begin, len};
+  }
   CharSourceRange getDeferredTokenRange() const {
     assert(DK == DataKind::DeferredToken);
     return CharSourceRange{DeferredToken.TokLoc, DeferredToken.TokLength};

include/swift/Parse/ParsedSyntaxRecorder.h.gyb

@@ -66,11 +66,31 @@ public:
 
   static Parsed${node.name} makeBlank${node.syntax_kind}(SourceLoc loc,
       SyntaxParsingContext &SPCtx);
+%   elif node.is_unknown():
+private:
+  static Parsed${node.name} record${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      ParsedRawSyntaxRecorder &rec);
+
+public:
+  static Parsed${node.name} defer${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      SyntaxParsingContext &SPCtx);
+
+  static Parsed${node.name} make${node.syntax_kind}(
+      ArrayRef<ParsedSyntax> elts,
+      SyntaxParsingContext &SPCtx);
 %   end
 % end
 
 #pragma mark - Convenience APIs
 
+public:
+  static ParsedTokenSyntax makeToken(const Token &Tok,
+                                     const ParsedTrivia &LeadingTrivia,
+                                     const ParsedTrivia &TrailingTrivia,
+                                     SyntaxParsingContext &SPCtx);
+
   /// Records an unlabelled TupleTypeElementSyntax with the provided type and
   /// optional trailing comma.
   static ParsedTupleTypeElementSyntax

include/swift/Parse/Parser.h

@@ -25,10 +25,12 @@
 #include "swift/AST/Pattern.h"
 #include "swift/AST/Stmt.h"
 #include "swift/Basic/OptionSet.h"
+#include "swift/Parse/ASTGen.h"
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/LocalContext.h"
 #include "swift/Parse/PersistentParserState.h"
 #include "swift/Parse/Token.h"
+#include "swift/Parse/ParsedSyntaxNodes.h"
 #include "swift/Parse/ParserPosition.h"
 #include "swift/Parse/ParserResult.h"
 #include "swift/Parse/SyntaxParserResult.h"
@@ -369,6 +371,9 @@ class Parser {
   /// Current syntax parsing context where call backs should be directed to.
   SyntaxParsingContext *SyntaxContext;
 
+  /// The AST generator.
+  ASTGen Generator;
+
 public:
   Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
          SILParserTUStateBase *SIL,
@@ -390,7 +395,7 @@ class Parser {
 
   /// Calling this function to finalize libSyntax tree creation without destroying
   /// the parser instance.
-  ParsedRawSyntaxNode finalizeSyntaxTree() {
+  OpaqueSyntaxNode finalizeSyntaxTree() {
     assert(Tok.is(tok::eof) && "not done parsing yet");
     return SyntaxContext->finalizeRoot();
   }
@@ -511,6 +516,12 @@ class Parser {
     assert(Tok.is(K) && "Consuming wrong token kind");
     return consumeToken();
   }
+  /// Consume a token without providing it to the SyntaxParsingContext.
+  ParsedTokenSyntax consumeTokenSyntax();
+  ParsedTokenSyntax consumeTokenSyntax(tok K) {
+    assert(Tok.is(K) && "Consuming wrong token kind");
+    return consumeTokenSyntax();
+  }
 
   SourceLoc consumeIdentifier(Identifier *Result = nullptr,
                               bool allowDollarIdentifier = false) {
@@ -1336,6 +1347,15 @@ class Parser {
   ParserResult<Expr> parseExprSuper();
   ParserResult<Expr> parseExprStringLiteral();
 
+  // todo [gsoc]: create new result type for ParsedSyntax
+  // todo [gsoc]: turn into proper non-templated methods later
+  template <typename SyntaxNode>
+  ParsedExprSyntax parseExprSyntax();
+
+  // todo [gsoc]: remove when possible
+  template <typename SyntaxNode>
+  ParserResult<Expr> parseExprAST();
+
   StringRef copyAndStripUnderscores(StringRef text);
 
   ParserStatus parseStringSegments(SmallVectorImpl<Lexer::StringSegment> &Segments,

include/swift/Parse/SyntaxParseActions.h

@@ -33,6 +33,12 @@ namespace syntax {
 
 typedef void *OpaqueSyntaxNode;
 
+// todo [gsoc]: remove when possible
+enum class OpaqueSyntaxNodeKind {
+  SwiftSyntax,
+  LibSyntax,
+};
+
 class SyntaxParseActions {
   virtual void _anchor();
 
@@ -60,6 +66,9 @@ class SyntaxParseActions {
   lookupNode(size_t lexerOffset, syntax::SyntaxKind kind) {
     return std::make_pair(0, nullptr);
   }
+
+  /// Returns what kind of OpaqueSyntaxNode is created by `recordXXX` methods.
+  virtual OpaqueSyntaxNodeKind getOpaqueKind() = 0;
 };
 
 } // end namespace swift