Re-apply libSyntax patches after fixing ASAN issue (#12730)

* Re-apply "libSyntax: Ensure round-trip printing when we build syntax tree from parser incrementally. (#12709)"

* Re-apply "libSyntax: Root parsing context should hold a reference to the current token in the parser, NFC."

* Re-apply "libSyntax: avoid copying token text when lexing token syntax nodes, NFC. (#12723)"

* Actually fix the container-overflow issue.

Author: nkcsgexi

include/swift/AST/Module.h

@@ -83,7 +83,7 @@ namespace syntax {
   class SourceFileSyntax;
   class SyntaxParsingContext;
   class SyntaxParsingContextRoot;
-  struct RawTokenInfo;
+  struct RawSyntaxInfo;
 }
 
 /// Discriminator for file-units.
@@ -1092,7 +1092,7 @@ class SourceFile final : public FileUnit {
   Optional<std::vector<Token>> AllCorrectedTokens;
 
   /// All of the raw token syntax nodes in the underlying source.
-  std::vector<syntax::RawTokenInfo> AllRawTokenSyntax;
+  std::vector<syntax::RawSyntaxInfo> AllRawTokenSyntax;
 
   SourceFileSyntaxInfo &SyntaxInfo;
 

include/swift/Parse/Lexer.h

@@ -30,7 +30,7 @@ namespace swift {
 
 namespace syntax {
   struct RawTokenSyntax;
-  struct RawTokenInfo;
+  struct RawSyntaxInfo;
 }
 
   /// Given a pointer to the starting byte of a UTF8 character, validate it and
@@ -246,7 +246,7 @@ class Lexer {
   }
 
   /// Lex a full token including leading and trailing trivia.
-  syntax::RawTokenInfo fullLex();
+  syntax::RawSyntaxInfo fullLex();
 
   bool isKeepingComments() const {
     return RetainComments == CommentRetentionMode::ReturnAsTokens;

include/swift/Parse/Parser.h

@@ -56,7 +56,7 @@ namespace swift {
   namespace syntax {
     class AbsolutePosition;
     class SyntaxParsingContext;
-    struct RawTokenInfo;
+    struct RawSyntaxInfo;
     struct RawTokenSyntax;
   }// end of syntax namespace
 
@@ -1434,7 +1434,7 @@ tokenizeWithTrivia(const LangOptions &LangOpts,
 void populateTokenSyntaxMap(const LangOptions &LangOpts,
                             const SourceManager &SM,
                             unsigned BufferID,
-                            std::vector<syntax::RawTokenInfo> &Result);
+                            std::vector<syntax::RawSyntaxInfo> &Result);
 } // end namespace swift
 
 #endif

include/swift/Syntax/SyntaxParsingContext.h

@@ -13,17 +13,39 @@
 #ifndef SWIFT_SYNTAX_PARSING_CONTEXT_H
 #define SWIFT_SYNTAX_PARSING_CONTEXT_H
 
+ #include "swift/Syntax/Syntax.h"
+
 namespace swift {
   class SourceLoc;
   class SourceFile;
+  class Token;
 
 namespace syntax {
   struct RawTokenSyntax;
+  struct RawSyntax;
   enum class SyntaxKind;
 
-struct RawTokenInfo {
-  SourceLoc Loc;
-  RC<RawTokenSyntax> Token;
+/// The handler for parser to generate libSyntax entities.
+struct RawSyntaxInfo {
+  /// Start location of this syntax node.
+  SourceLoc StartLoc;
+
+  /// The number of tokens belong to the syntax node.
+  unsigned TokCount;
+
+  /// The raw node.
+  RC<RawSyntax> RawNode;
+  RawSyntaxInfo(SourceLoc StartLoc, RC<RawSyntax> RawNode):
+    RawSyntaxInfo(StartLoc, 1, RawNode) {}
+  RawSyntaxInfo(SourceLoc StartLoc, unsigned TokCount, RC<RawSyntax> RawNode);
+
+  template <typename SyntaxNode>
+  SyntaxNode makeSyntax() const { return make<SyntaxNode>(RawNode); }
+
+  template <typename RawSyntaxNode>
+  RC<RawSyntaxNode> getRaw() const {
+    return RC<RawSyntaxNode>(cast<RawSyntaxNode>(RawNode));
+  }
 };
 
 enum class SyntaxParsingContextKind: uint8_t {
@@ -35,11 +57,12 @@ enum class SyntaxParsingContextKind: uint8_t {
 /// create syntax nodes.
 class SyntaxParsingContext {
 protected:
-  SyntaxParsingContext(bool Enabled);
+  SyntaxParsingContext(SourceFile &SF, unsigned BufferID, Token &Tok);
   SyntaxParsingContext(SyntaxParsingContext &Another);
 public:
   struct ContextInfo;
   ContextInfo &ContextData;
+  const Token &Tok;
 
   // Add a token syntax at the given source location to the context; this
   // token node can be used to build more complex syntax nodes in later call
@@ -61,12 +84,10 @@ class SyntaxParsingContext {
 // of all other entity-specific contexts. This is the context Parser
 // has when the parser instance is firstly created.
 class SyntaxParsingContextRoot: public SyntaxParsingContext {
+  SourceFile &File;
 public:
-  struct GlobalInfo;
-
-  // Contains global information of the source file under parsing.
-  GlobalInfo &GlobalData;
-  SyntaxParsingContextRoot(SourceFile &SF, unsigned BufferID);
+  SyntaxParsingContextRoot(SourceFile &File, unsigned BufferID, Token &Tok):
+    SyntaxParsingContext(File, BufferID, Tok), File(File) {}
   ~SyntaxParsingContextRoot();
   void addTokenSyntax(SourceLoc Loc) override {};
   void makeNode(SyntaxKind Kind) override {};
@@ -75,21 +96,22 @@ class SyntaxParsingContextRoot: public SyntaxParsingContext {
   };
 };
 
+enum class SyntaxContextKind: uint8_t{
+  Expr,
+  Decl,
+};
+
 // The base class for contexts that are created from a parent context.
 // The stack instance will set the context holder when the context
 // is firstly created and reset the context holder to the parent when
 // it's destructed.
 class SyntaxParsingContextChild: public SyntaxParsingContext {
   SyntaxParsingContext *Parent;
   SyntaxParsingContext *&ContextHolder;
-  const SyntaxKind FinalKind;
+  const SyntaxContextKind Kind;
 public:
   SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
-                            SyntaxKind FinalKind):
-    SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
-    ContextHolder(ContextHolder), FinalKind(FinalKind) {
-      ContextHolder = this;
-  }
+                            SyntaxContextKind Kind);
   ~SyntaxParsingContextChild();
   void makeNode(SyntaxKind Kind) override;
   void addTokenSyntax(SourceLoc Loc) override;

lib/Parse/Lexer.cpp

@@ -738,15 +738,15 @@ static bool rangeContainsPlaceholderEnd(const char *CurPtr,
   return false;
 }
 
-syntax::RawTokenInfo Lexer::fullLex() {
+syntax::RawSyntaxInfo Lexer::fullLex() {
   if (NextToken.isEscapedIdentifier()) {
     LeadingTrivia.push_back(syntax::TriviaPiece::backtick());
     TrailingTrivia.insert(TrailingTrivia.begin(),
                           syntax::TriviaPiece::backtick());
   }
   auto Loc = NextToken.getLoc();
   auto Result = syntax::RawTokenSyntax::make(NextToken.getKind(),
-                                        OwnedString(NextToken.getText()).copy(),
+                                        OwnedString(NextToken.getText()),
                                         syntax::SourcePresence::Present,
                                         {LeadingTrivia}, {TrailingTrivia});
   LeadingTrivia.clear();

lib/Parse/ParseDecl.cpp

@@ -18,6 +18,9 @@
 #include "swift/Parse/CodeCompletionCallbacks.h"
 #include "swift/Parse/DelayedParsingCallbacks.h"
 #include "swift/Parse/ParseSILSupport.h"
+#include "swift/Syntax/SyntaxFactory.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "swift/Subsystems.h"
 #include "swift/AST/Attr.h"
 #include "swift/AST/DebuggerClient.h"
@@ -37,6 +40,7 @@
 #include <algorithm>
 
 using namespace swift;
+using namespace syntax;
 
 namespace {
   /// A RAII object for deciding whether this DeclKind needs special
@@ -2146,7 +2150,8 @@ void Parser::delayParseFromBeginningToHere(ParserPosition BeginParserPosition,
 ParserResult<Decl>
 Parser::parseDecl(ParseDeclOptions Flags,
                   llvm::function_ref<void(Decl*)> Handler) {
-
+  SyntaxParsingContextChild DeclParsingContext(SyntaxContext,
+                                               SyntaxContextKind::Decl);
   if (Tok.is(tok::pound_if)) {
     auto IfConfigResult = parseIfConfig(
       [&](SmallVectorImpl<ASTNode> &Decls, bool IsActive) {

lib/Parse/ParseExpr.cpp

@@ -40,7 +40,8 @@ using namespace swift::syntax;
 /// \param isExprBasic Whether we're only parsing an expr-basic.
 ParserResult<Expr> Parser::parseExprImpl(Diag<> Message, bool isExprBasic) {
   // Start a context for creating expression syntax.
-  SyntaxParsingContextChild ExprParsingContext(SyntaxContext, SyntaxKind::Expr);
+  SyntaxParsingContextChild ExprParsingContext(SyntaxContext,
+                                               SyntaxContextKind::Expr);
 
   // If we are parsing a refutable pattern, check to see if this is the start
   // of a let/var/is pattern.  If so, parse it to an UnresolvedPatternExpr and
@@ -1804,7 +1805,8 @@ ParserResult<Expr> Parser::parseExprStringLiteral() {
   // Create a syntax node for string literal.
   SyntaxContext->addTokenSyntax(Tok.getLoc());
   SyntaxContext->makeNode(SyntaxKind::StringLiteralExpr);
-  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::Expr);
+  SyntaxParsingContextChild LocalContext(SyntaxContext,
+                                         SyntaxContextKind::Expr);
 
   // FIXME: Avoid creating syntax nodes for string interpolation.
   LocalContext.disable();

lib/Parse/Parser.cpp

@@ -269,7 +269,7 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
                         syntax::AbsolutePosition>> Tokens;
   syntax::AbsolutePosition RunningPos;
   do {
-    auto ThisToken = L.fullLex().Token;
+    auto ThisToken = L.fullLex().getRaw<syntax::RawTokenSyntax>();
     auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
     Tokens.push_back({ThisToken, ThisTokenPos});
   } while (Tokens.back().first->isNot(tok::eof));
@@ -280,15 +280,15 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts,
 void swift::populateTokenSyntaxMap(const LangOptions &LangOpts,
                                    const SourceManager &SM,
                                    unsigned BufferID,
-                                   std::vector<syntax::RawTokenInfo> &Result) {
+                                   std::vector<syntax::RawSyntaxInfo> &Result) {
   if (!Result.empty())
     return;
   Lexer L(LangOpts, SM, BufferID, /*Diags=*/nullptr, /*InSILMode=*/false,
           CommentRetentionMode::AttachToNextToken,
           TriviaRetentionMode::WithTrivia);
   do {
     Result.emplace_back(L.fullLex());
-    if (Result.back().Token->is(tok::eof))
+    if (Result.back().getRaw<syntax::RawTokenSyntax>()->is(tok::eof))
       return;
   } while (true);
 }
@@ -435,7 +435,7 @@ Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
     TokReceiver(SF.shouldKeepTokens() ?
                 new TokenRecorder(SF) :
                 new ConsumeTokenReceiver()),
-    SyntaxContext(new syntax::SyntaxParsingContextRoot(SF, L->getBufferID())) {
+    SyntaxContext(new syntax::SyntaxParsingContextRoot(SF, L->getBufferID(), Tok)) {
 
   State = PersistentState;
   if (!State) {