libSyntax: parse codeblock syntax node.

Author: nkcsgexi

include/swift/Syntax/SyntaxParsingContext.h

@@ -13,7 +13,8 @@
 #ifndef SWIFT_SYNTAX_PARSING_CONTEXT_H
 #define SWIFT_SYNTAX_PARSING_CONTEXT_H
 
- #include "swift/Syntax/Syntax.h"
+#include "swift/Basic/SourceLoc.h"
+#include "swift/Syntax/Syntax.h"
 
 namespace swift {
   class SourceLoc;
@@ -25,19 +26,28 @@ namespace syntax {
   struct RawSyntax;
   enum class SyntaxKind;
 
+enum class SyntaxContextKind: uint8_t{
+  Expr,
+  Decl,
+  Stmt,
+};
+
 /// The handler for parser to generate libSyntax entities.
 struct RawSyntaxInfo {
   /// Start location of this syntax node.
   SourceLoc StartLoc;
 
-  /// The number of tokens belong to the syntax node.
-  unsigned TokCount;
+  /// The end location of this syntax node.
+  SourceLoc EndLoc;
 
   /// The raw node.
   RC<RawSyntax> RawNode;
+  RawSyntaxInfo(RC<RawSyntax> RawNode): RawSyntaxInfo(SourceLoc(), RawNode) {}
   RawSyntaxInfo(SourceLoc StartLoc, RC<RawSyntax> RawNode):
-    RawSyntaxInfo(StartLoc, 1, RawNode) {}
-  RawSyntaxInfo(SourceLoc StartLoc, unsigned TokCount, RC<RawSyntax> RawNode);
+    RawSyntaxInfo(StartLoc, StartLoc, RawNode) {}
+  RawSyntaxInfo(SourceLoc StartLoc, SourceLoc EndLoc, RC<RawSyntax> RawNode);
+
+  bool isImplicit() const { return StartLoc.isInvalid(); }
 
   template <typename SyntaxNode>
   SyntaxNode makeSyntax() const { return make<SyntaxNode>(RawNode); }
@@ -46,6 +56,7 @@ struct RawSyntaxInfo {
   RC<RawSyntaxNode> getRaw() const {
     return RC<RawSyntaxNode>(cast<RawSyntaxNode>(RawNode));
   }
+  void brigeWithContext(SyntaxContextKind Kind);
 };
 
 enum class SyntaxParsingContextKind: uint8_t {
@@ -96,22 +107,28 @@ class SyntaxParsingContextRoot: public SyntaxParsingContext {
   };
 };
 
-enum class SyntaxContextKind: uint8_t{
-  Expr,
-  Decl,
-};
-
 // The base class for contexts that are created from a parent context.
 // The stack instance will set the context holder when the context
 // is firstly created and reset the context holder to the parent when
 // it's destructed.
 class SyntaxParsingContextChild: public SyntaxParsingContext {
   SyntaxParsingContext *Parent;
   SyntaxParsingContext *&ContextHolder;
-  const SyntaxContextKind Kind;
+  Optional<SyntaxContextKind> Kind;
+  Optional<SyntaxKind> KnownSyntax;
+  void makeNodeWhole(SyntaxKind Kind);
+  SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
+                            Optional<SyntaxContextKind> Kind,
+                            Optional<SyntaxKind> KnownSyntax);
 public:
   SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
-                            SyntaxContextKind Kind);
+    SyntaxContextKind Kind): SyntaxParsingContextChild(ContextHolder,
+                                                       Kind, None) {}
+
+  SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
+    SyntaxKind KnownSyntax): SyntaxParsingContextChild(ContextHolder,
+                             None, KnownSyntax) {};
+
   ~SyntaxParsingContextChild();
   void makeNode(SyntaxKind Kind) override;
   void addTokenSyntax(SourceLoc Loc) override;

lib/Parse/ParseStmt.cpp

@@ -22,12 +22,15 @@
 #include "swift/Parse/Lexer.h"
 #include "swift/Parse/CodeCompletionCallbacks.h"
 #include "swift/Subsystems.h"
+#include "swift/Syntax/TokenSyntax.h"
+#include "swift/Syntax/SyntaxParsingContext.h"
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/SaveAndRestore.h"
 
 using namespace swift;
+using namespace swift::syntax;
 
 /// isStartOfStmt - Return true if the current token starts a statement.
 ///
@@ -223,7 +226,7 @@ void Parser::consumeTopLevelDecl(ParserPosition BeginParserPosition,
 ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
                                      BraceItemListKind Kind,
                                      BraceItemListKind ConditionalBlockKind) {
-  
+  SyntaxParsingContextChild StmtListContext(SyntaxContext, SyntaxKind::StmtList);
   bool IsTopLevel = (Kind == BraceItemListKind::TopLevelCode) ||
                     (Kind == BraceItemListKind::TopLevelLibrary);
   bool isActiveConditionalBlock =
@@ -259,6 +262,7 @@ ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
          Tok.isNot(tok::kw_sil_default_witness_table) &&
          (isConditionalBlock ||
           !isTerminatorForBraceItemListKind(Kind, Entries))) {
+    SyntaxParsingContextChild StmtContext(SyntaxContext, SyntaxContextKind::Stmt);
     if (Kind == BraceItemListKind::TopLevelLibrary &&
         skipExtraTopLevelRBraces())
       continue;
@@ -489,7 +493,7 @@ static ParserResult<Stmt> recoverFromInvalidCase(Parser &P) {
 }
 
 ParserResult<Stmt> Parser::parseStmt() {
-
+  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxContextKind::Stmt);
   // Note that we're parsing a statement.
   StructureMarkerRAII ParsingStmt(*this, Tok.getLoc(),
                                   StructureMarkerKind::Statement);
@@ -586,15 +590,19 @@ ParserResult<BraceStmt> Parser::parseBraceItemList(Diag<> ID) {
     if (Tok.isNot(tok::l_brace))
       return nullptr;
   }
+  SyntaxParsingContextChild LocalContext(SyntaxContext, SyntaxKind::CodeBlock);
   SourceLoc LBLoc = consumeToken(tok::l_brace);
+  LocalContext.addTokenSyntax(LBLoc);
 
   SmallVector<ASTNode, 16> Entries;
   SourceLoc RBLoc;
 
   ParserStatus Status = parseBraceItems(Entries, BraceItemListKind::Brace,
                                         BraceItemListKind::Brace);
-  parseMatchingToken(tok::r_brace, RBLoc,
-                     diag::expected_rbrace_in_brace_stmt, LBLoc);
+  if (!parseMatchingToken(tok::r_brace, RBLoc,
+                          diag::expected_rbrace_in_brace_stmt, LBLoc)) {
+    LocalContext.addTokenSyntax(RBLoc);
+  }
 
   return makeParserResult(Status,
                           BraceStmt::create(Context, LBLoc, Entries, RBLoc));

lib/Syntax/RawSyntax.cpp

@@ -29,6 +29,7 @@ static bool isTrivialSyntaxKind(SyntaxKind Kind) {
   case SyntaxKind::SourceFile:
   case SyntaxKind::TopLevelCodeDecl:
   case SyntaxKind::ExpressionStmt:
+  case SyntaxKind::DeclarationStmt:
     return true;
   default:
     return false;

lib/Syntax/SyntaxParsingContext.cpp

@@ -43,15 +43,11 @@ static ArrayRef<Syntax> getSyntaxNodes(ArrayRef<RawSyntaxInfo> RawNodes,
   return Scratch;
 }
 
-static unsigned countTokens(ArrayRef<RawSyntaxInfo> AllNodes) {
-  return std::accumulate(AllNodes.begin(), AllNodes.end(), 0,
-    [](unsigned Sum, const RawSyntaxInfo &Info) { return Sum + Info.TokCount; });
-}
 } // End of anonymous namespace
 
-RawSyntaxInfo::RawSyntaxInfo(SourceLoc StartLoc, unsigned TokCount,
-  RC<RawSyntax> RawNode): StartLoc(StartLoc), TokCount(TokCount), RawNode(RawNode) {
-    assert(StartLoc.isValid());
+RawSyntaxInfo::RawSyntaxInfo(SourceLoc StartLoc, SourceLoc EndLoc,
+    RC<RawSyntax> RawNode): StartLoc(StartLoc), EndLoc(EndLoc), RawNode(RawNode) {
+  assert(StartLoc.isValid() == EndLoc.isValid());
 }
 
 struct SyntaxParsingContext::ContextInfo {
@@ -66,7 +62,7 @@ struct SyntaxParsingContext::ContextInfo {
 
   ArrayRef<RawSyntaxInfo>::const_iterator findTokenAt(SourceLoc Loc) {
     for (auto It = Tokens.begin(); It != Tokens.end(); It ++) {
-      assert(It->TokCount == 1);
+      assert(It->StartLoc == It->EndLoc);
       if (It->StartLoc == Loc)
         return It;
     }
@@ -98,8 +94,9 @@ struct SyntaxParsingContext::ContextInfo {
   ArrayRef<RawSyntaxInfo> getPendingSyntax() const { return PendingSyntax; };
 
   void addPendingSyntax(RawSyntaxInfo Info) {
-    assert(PendingSyntax.empty() || PendingSyntax.back().StartLoc.
-           getOpaquePointerValue() < Info.StartLoc.getOpaquePointerValue());
+    assert(Info.isImplicit() || PendingSyntax.empty() ||
+           PendingSyntax.back().StartLoc.getOpaquePointerValue() <
+             Info.StartLoc.getOpaquePointerValue());
     PendingSyntax.push_back(Info);
   }
 
@@ -139,10 +136,15 @@ SyntaxParsingContext::ContextInfo::collectAllSyntax() {
       // If no remaining syntax nodes, add the token.
       Results.emplace_back(Tok);
       It ++;
+    } else if (CurSyntax->isImplicit()) {
+      // Skip implicit syntax node.
+      CurSyntax ++;
     } else if (CurSyntax->StartLoc == Tok.StartLoc) {
       // Prefer syntax nodes to tokens.
       Results.emplace_back(*CurSyntax);
-      It += CurSyntax->TokCount;
+      while(It->EndLoc != CurSyntax->EndLoc) It++;
+      assert(It < Tokens.end() && It->EndLoc == CurSyntax->EndLoc);
+      It ++;
       CurSyntax ++;
     } else {
       // We have to add token in this case since the next syntax node has not
@@ -178,7 +180,7 @@ SyntaxParsingContext::ContextInfo::createFromBack(SyntaxKind Kind, unsigned N) {
     Result.emplace(makeUnknownSyntax(SyntaxFactory::getUnknownKind(Kind),
                                      SyntaxParts));
   }
-  RawSyntaxInfo NewSyntaxNode(Parts.front().StartLoc, countTokens(Parts),
+  RawSyntaxInfo NewSyntaxNode(Parts.front().StartLoc, Parts.back().EndLoc,
                               Result->getRaw());
   // Remove the building bricks and re-append the result.
   for (unsigned I = 0; I < N; I ++)
@@ -209,26 +211,9 @@ SyntaxParsingContextRoot::~SyntaxParsingContextRoot() {
     }
   }
   for (auto Info: ContextData.getPendingSyntax()) {
-    std::vector<StmtSyntax> AllStmts;
-    auto S = Info.makeSyntax<Syntax>();
-    if (S.isDecl()) {
-      AllStmts.push_back(SyntaxFactory::makeDeclarationStmt(
-        S.getAs<DeclSyntax>().getValue(), None));
-    } else if (S.isExpr()) {
-      AllStmts.push_back(SyntaxFactory::makeExpressionStmt(
-        S.getAs<ExprSyntax>().getValue(), None));
-    } else if (S.isStmt()) {
-      AllStmts.push_back(S.getAs<StmtSyntax>().getValue());
-    } else {
-      // If this is a standalone token, we create an unknown expression wrapper
-      // for it.
-      AllStmts.push_back(SyntaxFactory::makeExpressionStmt(
-        *makeUnknownSyntax(SyntaxKind::UnknownExpr,
-                           { *S.getAs<TokenSyntax>() }).getAs<ExprSyntax>(),
-                                                           None));
-    }
+    assert(Info.RawNode->Kind == SyntaxKind::StmtList);
     AllTopLevel.push_back(SyntaxFactory::makeTopLevelCodeDecl(
-      SyntaxFactory::makeStmtList(AllStmts)));
+      Info.makeSyntax<StmtListSyntax>()));
   }
 
   File.setSyntaxRoot(
@@ -248,14 +233,16 @@ SyntaxParsingContextRoot &SyntaxParsingContextChild::getRoot() {
 
 SyntaxParsingContextChild::
 SyntaxParsingContextChild(SyntaxParsingContext *&ContextHolder,
-                          SyntaxContextKind Kind):
+      Optional<SyntaxContextKind> Kind, Optional<SyntaxKind> KnownSyntax):
     SyntaxParsingContext(*ContextHolder), Parent(ContextHolder),
-    ContextHolder(ContextHolder), Kind(Kind) {
+    ContextHolder(ContextHolder), Kind(Kind), KnownSyntax(KnownSyntax) {
+  assert(Kind.hasValue() != KnownSyntax.hasValue());
   ContextHolder = this;
   if (ContextData.Enabled)
     ContextData.setContextStart(Tok.getLoc());
 }
 
+
 void SyntaxParsingContextChild::addTokenSyntax(SourceLoc Loc) {
   if (ContextData.Enabled)
     ContextData.promoteTokenAt(Loc);
@@ -281,6 +268,48 @@ void SyntaxParsingContextChild::makeNode(SyntaxKind Kind) {
   }
 
   default:
+    llvm_unreachable("Unrecognized node kind.");
+  }
+}
+
+void SyntaxParsingContextChild::makeNodeWhole(SyntaxKind Kind) {
+  assert(ContextData.Enabled);
+  switch (Kind) {
+  case SyntaxKind::CodeBlock: {
+    ContextData.createFromBack(Kind);
+    break;
+  }
+  case SyntaxKind::StmtList: {
+    if (ContextData.getPendingSyntax().empty()) {
+      // Create an empty statement list if no statement is in the context.
+      ContextData.addPendingSyntax({SyntaxFactory::makeBlankStmtList().getRaw()});
+    } else {
+      ContextData.createFromBack(Kind);
+    }
+    break;
+  }
+  default:
+    llvm_unreachable("Unrecognized node kind.");
+  }
+}
+
+void RawSyntaxInfo::brigeWithContext(SyntaxContextKind Kind) {
+  switch (Kind) {
+  case SyntaxContextKind::Stmt: {
+    if (RawNode->isDecl()) {
+      // Wrap a declaration with a declaration statement
+      RawNode = SyntaxFactory::createSyntax(SyntaxKind::DeclarationStmt,
+        { makeSyntax<Syntax>() })->getRaw();
+    } else if (RawNode->isExpr()) {
+      // Wrap an expression with an expression statement
+      RawNode = SyntaxFactory::createSyntax(SyntaxKind::ExpressionStmt,
+        { makeSyntax<Syntax>() })->getRaw();
+    }
+    assert(RawNode->isStmt());
+    break;
+  }
+  case SyntaxContextKind::Decl:
+  case SyntaxContextKind::Expr:
     break;
   }
 }
@@ -295,32 +324,51 @@ SyntaxParsingContextChild::~SyntaxParsingContextChild() {
 
   // Set the end of the context.
   ContextData.setContextEnd(Tok.getLoc());
+  if (KnownSyntax) {
+    // If the entire context should be created to a known syntax kind, create
+    // all pending syntax nodes into that node.
+    makeNodeWhole(*KnownSyntax);
+    assert(ContextData.getPendingSyntax().size() == 1);
+    auto AllNodes = ContextData.collectAllSyntax();
+    assert(AllNodes.size() == 1);
+    Parent->ContextData.addPendingSyntax(AllNodes.front());
+    return;
+  }
   auto AllNodes = ContextData.collectAllSyntax();
-  assert(countTokens(AllNodes) == ContextData.allTokens().size());
   RC<RawSyntax> FinalResult;
   if (AllNodes.empty())
     return;
 
+  // Make sure we used all tokens.
+  assert(AllNodes.front().StartLoc == ContextData.allTokens().front().StartLoc);
+  assert(AllNodes.back().EndLoc == ContextData.allTokens().back().StartLoc);
+
   if (AllNodes.size() == 1) {
-    // FIXME: Check kind
-    Parent->ContextData.addPendingSyntax(AllNodes.front());
+    // If we have only one syntax node remaining, we are done.
+    auto Result = AllNodes.front();
+    // Bridge the syntax node to the expected context kind.
+    Result.brigeWithContext(*Kind);
+    Parent->ContextData.addPendingSyntax(Result);
     return;
   }
 
   llvm::SmallVector<Syntax, 8> Scratch;
   auto SyntaxNodes = getSyntaxNodes(AllNodes, Scratch);
   SourceLoc Start = AllNodes.front().StartLoc;
-  unsigned TokCount = countTokens(AllNodes);
+  SourceLoc End = AllNodes.back().EndLoc;
   SyntaxKind UnknownKind;
-  switch (Kind) {
+  switch (*Kind) {
     case SyntaxContextKind::Expr:
       UnknownKind = SyntaxKind::UnknownExpr;
       break;
     case SyntaxContextKind::Decl:
       UnknownKind = SyntaxKind::UnknownDecl;
       break;
+    case SyntaxContextKind::Stmt:
+      UnknownKind = SyntaxKind::UnknownStmt;
+      break;
   }
   // Create an unknown node and give it to the parent context.
-  Parent->ContextData.addPendingSyntax({Start, TokCount,
+  Parent->ContextData.addPendingSyntax({Start, End,
     makeUnknownSyntax(UnknownKind, SyntaxNodes).getRaw()});
 }