libSyntax: Ensure round-trip printing when we build syntax tree from parser incrementally. (#12709)

Author: nkcsgexi

include/swift/AST/Module.h

@@ -83,7 +83,7 @@ namespace syntax {
   class SourceFileSyntax;
   class SyntaxParsingContext;
   class SyntaxParsingContextRoot;
-  struct RawTokenInfo;
+  struct RawSyntaxInfo;
 }
 
 /// Discriminator for file-units.
@@ -1092,7 +1092,7 @@ class SourceFile final : public FileUnit {
   Optional<std::vector<Token>> AllCorrectedTokens;
 
   /// All of the raw token syntax nodes in the underlying source.
-  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
+  std::vector<syntax::RawSyntaxInfo> AllRawTokenSyntax;
 
   SourceFileSyntaxInfo &SyntaxInfo;
 

include/swift/Parse/Lexer.h

@@ -246,7 +246,7 @@ class Lexer {
   }
 
   /// Lex a full token including leading and trailing trivia.
-  syntax::RawTokenInfo fullLex();
+  syntax::RawSyntaxInfo fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

include/swift/Parse/Parser.h

@@ -1429,7 +1429,7 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
 void populateTokenSyntaxMap(const LangOptions &LangOpts,
                             const SourceManager &SM,
                             unsigned BufferID,
-                            std::vector<syntax::RawTokenInfo> &Result);
+                            std::vector<syntax::RawSyntaxInfo> &Result);
 } // end namespace swift
 
 #endif

include/swift/Syntax/SyntaxParsingContext.h

@@ -23,12 +23,30 @@
 
 namespace swift {
   class SourceFile;
+  class Token;
 
 namespace syntax {
 
-struct RawTokenInfo {
-  SourceLoc Loc;
-  RC<RawTokenSyntax> Token;
+/// The handler for parser to generate libSyntax entities.
+struct RawSyntaxInfo {
+  /// Start location of this syntax node.
+  SourceLoc StartLoc;
+
+  /// The number of tokens belong to the syntax node.
+  unsigned TokCount;
+
+  /// The raw node.
+  RC<RawSyntax> RawNode;
+  RawSyntaxInfo(SourceLoc StartLoc, RC<RawSyntax> RawNode):
+    RawSyntaxInfo(StartLoc, 1, RawNode) {}
+  RawSyntaxInfo(SourceLoc StartLoc, unsigned TokCount, RC<RawSyntax> RawNode);
+  template <typename SyntaxNode>
+  SyntaxNode makeSyntax() const { return make<SyntaxNode>(RawNode); }
+
+  template <typename RawSyntaxNode>
+  RC<RawSyntaxNode> getRaw() const {
+    return RC<RawSyntaxNode>(cast<RawSyntaxNode>(RawNode));
+  }
 };
 
 enum class SyntaxParsingContextKind: uint8_t {
@@ -40,7 +58,7 @@ enum class SyntaxParsingContextKind: uint8_t {
 /// create syntax nodes.
 class SyntaxParsingContext {
 protected:
-  SyntaxParsingContext(bool Enabled);
+  SyntaxParsingContext(SourceFile &SF, unsigned BufferID);
   SyntaxParsingContext(SyntaxParsingContext &Another);
 public:
   struct ContextInfo;
@@ -66,12 +84,10 @@ class SyntaxParsingContext {
 // of all other entity-specific contexts. This is the context Parser
 // has when the parser instance is firstly created.
 class SyntaxParsingContextRoot: public SyntaxParsingContext {
+  SourceFile &File;
 public:
-  struct GlobalInfo;
-
-  // Contains global information of the source file under parsing.
-  GlobalInfo &GlobalData;
-  SyntaxParsingContextRoot(SourceFile &SF, unsigned BufferID);
+  SyntaxParsingContextRoot(SourceFile &File, unsigned BufferID):
+    SyntaxParsingContext(File, BufferID), File(File) {}
   ~SyntaxParsingContextRoot();
   void addTokenSyntax(SourceLoc Loc) override {};
   void makeNode(SyntaxKind Kind) override {};
@@ -80,21 +96,23 @@ class SyntaxParsingContextRoot: public SyntaxParsingContext {
   };
 };
 
+enum class SyntaxContextKind: uint8_t{
+  Expr,
+  Decl,
+};
+
 // The base class for contexts that are created from a parent context.
 // The stack instance will set the context holder when the context
 // is firstly created and reset the context holder to the parent when
 // it's destructed.
 class SyntaxParsingContextChild: public SyntaxParsingContext {
   SyntaxParsingContext *Parent;
   SyntaxParsingContext *&ContextHolder;
-  const SyntaxKind FinalKind;
+  const SyntaxContextKind Kind;
+  Token &Tok;
 public:
   SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
-                            SyntaxKind FinalKind):
-    SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
-    ContextHolder(ContextHolder), FinalKind(FinalKind) {
-      ContextHolder = this;
-  }
+                            SyntaxContextKind Kind, Token &Tok);
   ~SyntaxParsingContextChild();
   void makeNode(SyntaxKind Kind) override;
   void addTokenSyntax(SourceLoc Loc) override;

lib/Parse/Lexer.cpp

@@ -737,7 +737,7 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-syntax::RawTokenInfo Lexer::fullLex() {
+syntax::RawSyntaxInfo Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.insert(TrailingTrivia.begin(),

lib/Parse/ParseDecl.cpp

@@ -37,6 +37,7 @@
 #include <algorithm>
 
 using namespace swift;
+using namespace syntax;
 
 namespace {
   /// A RAII object for deciding whether this DeclKind needs special
@@ -2146,7 +2147,8 @@ void Parser::delayParseFromBeginningToHere(ParserPosition BeginParserPosition,
 ParserResult<Decl>
 Parser::parseDecl(ParseDeclOptions Flags,
                   llvm::function_ref<void(Decl*)> Handler) {
-
+  SyntaxParsingContextChild DeclParsingContext(SyntaxContext,
+                                               SyntaxContextKind::Decl, Tok);
   if (Tok.is(tok::pound_if)) {
     auto IfConfigResult = parseIfConfig(
       [&](SmallVectorImpl<ASTNode> &Decls, bool IsActive) {

lib/Parse/ParseExpr.cpp

@@ -40,7 +40,8 @@ using namespace swift::syntax;
 /// \param isExprBasic Whether we're only parsing an expr-basic.
 ParserResult<Expr> Parser::parseExprImpl(Diag<> Message, bool isExprBasic) {
   // Start a context for creating expression syntax.
-  SyntaxParsingContextChild ExprParsingContext(SyntaxContext, SyntaxKind::Expr);
+  SyntaxParsingContextChild ExprParsingContext(SyntaxContext,
+                                               SyntaxContextKind::Expr, Tok);
 
   // If we are parsing a refutable pattern, check to see if this is the start
   // of a let/var/is pattern.  If so, parse it to an UnresolvedPatternExpr and
@@ -1804,7 +1805,8 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
   // Create a syntax node for string literal.
   SyntaxContext->addTokenSyntax(Tok.getLoc());
   SyntaxContext->makeNode(SyntaxKind::StringLiteralExpr);
-  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::Expr);
+  SyntaxParsingContextChild LocalContext(SyntaxContext,
+                                         SyntaxContextKind::Expr, Tok);
 
   // FIXME: Avoid creating syntax nodes for string interpolation.
   LocalContext.disable();

lib/Parse/Parser.cpp

@@ -268,7 +268,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
                         syntax::AbsolutePosition>> Tokens;
   syntax::AbsolutePosition RunningPos;
   do {
-    auto ThisToken = L.fullLex().Token;
+    auto ThisToken = L.fullLex().getRaw<syntax::RawTokenSyntax>();
     auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
     Tokens.push_back({ThisToken, ThisTokenPos});
   } while (Tokens.back().first->isNot(tok::eof));
@@ -279,15 +279,15 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
 void swift::populateTokenSyntaxMap(const LangOptions &LangOpts,
                                    const SourceManager &SM,
                                    unsigned BufferID,
-                                   std::vector<syntax::RawTokenInfo> &Result) {
+                                   std::vector<syntax::RawSyntaxInfo> &Result) {
   if (!Result.empty())
     return;
   Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
           CommentRetentionMode::AttachToNextToken,
           TriviaRetentionMode::WithTrivia);
   do {
     Result.emplace_back(L.fullLex());
-    if (Result.back().Token->is(tok::eof))
+    if (Result.back().getRaw<syntax::RawTokenSyntax>()->is(tok::eof))
       return;
   } while (true);
 }