[Syntax] Introduce CodeBlockItem (#14458)

CodeBlockItem represents Decl, Stmt or Expr that optionally followed by
semi-colon.
SourceFile syntax holds a list of CodeBlockItem.

Author: rintaro

lib/Parse/ParseDecl.cpp

@@ -3905,7 +3905,8 @@ bool Parser::parseGetSetImpl(ParseDeclOptions Flags,
   if (Flags.contains(PD_InProtocol) || isInSILMode()) {
     if (Tok.is(tok::r_brace)) {
       // Give syntax node an empty statement list.
-      SyntaxParsingContext StmtListContext(SyntaxContext, SyntaxKind::StmtList);
+      SyntaxParsingContext StmtListContext(SyntaxContext,
+                                           SyntaxKind::CodeBlockItemList);
     }
     while (Tok.isNot(tok::r_brace)) {
       if (Tok.is(tok::eof))
@@ -3988,7 +3989,8 @@ bool Parser::parseGetSetImpl(ParseDeclOptions Flags,
   // an implicit fallthrough off the end.
   if (Tok.is(tok::r_brace)) {
     // Give syntax node an empty statement list.
-    SyntaxParsingContext StmtListContext(SyntaxContext, SyntaxKind::StmtList);
+    SyntaxParsingContext StmtListContext(SyntaxContext,
+                                         SyntaxKind::CodeBlockItemList);
     diagnose(Tok, diag::computed_property_no_accessors);
     return true;
   }

lib/Parse/ParseStmt.cpp

@@ -89,6 +89,7 @@ bool Parser::isStartOfStmt() {
 
 ParserStatus Parser::parseExprOrStmt(ASTNode &Result) {
   if (Tok.is(tok::semi)) {
+    SyntaxParsingContext ErrorCtxt(SyntaxContext, SyntaxContextKind::Stmt);
     diagnose(Tok, diag::illegal_semi_stmt)
       .fixItRemove(SourceRange(Tok.getLoc()));
     consumeToken();
@@ -112,6 +113,7 @@ ParserStatus Parser::parseExprOrStmt(ASTNode &Result) {
   if (Tok.is(tok::code_complete)) {
     if (CodeCompletion)
       CodeCompletion->completeStmtOrExpr();
+    SyntaxParsingContext ErrorCtxt(SyntaxContext, SyntaxContextKind::Stmt);
     consumeToken(tok::code_complete);
     return makeParserCodeCompletionStatus();
   }
@@ -228,7 +230,8 @@ void Parser::consumeTopLevelDecl(ParserPosition BeginParserPosition,
 ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
                                      BraceItemListKind Kind,
                                      BraceItemListKind ConditionalBlockKind) {
-  SyntaxParsingContext StmtListContext(SyntaxContext, SyntaxKind::StmtList);
+  SyntaxParsingContext ItemListContext(SyntaxContext,
+                                       SyntaxKind::CodeBlockItemList);
 
   bool IsTopLevel = (Kind == BraceItemListKind::TopLevelCode) ||
                     (Kind == BraceItemListKind::TopLevelLibrary);
@@ -264,9 +267,10 @@ ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
          (isConditionalBlock ||
           !isTerminatorForBraceItemListKind(Kind, Entries))) {
 
-    SyntaxParsingContext StmtContext(SyntaxContext, SyntaxContextKind::Stmt);
+    SyntaxParsingContext NodeContext(SyntaxContext, SyntaxKind::CodeBlockItem);
 
     if (Tok.is(tok::r_brace)) {
+      SyntaxParsingContext ErrContext(SyntaxContext, SyntaxContextKind::Stmt);
       assert(IsTopLevel);
       diagnose(Tok, diag::extra_rbrace)
         .fixItRemove(Tok.getLoc());
@@ -275,8 +279,11 @@ ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
     }
 
     // Eat invalid tokens instead of allowing them to produce downstream errors.
-    if (consumeIf(tok::unknown))
+    if (Tok.is(tok::unknown)) {
+      SyntaxParsingContext ErrContext(SyntaxContext, SyntaxContextKind::Stmt);
+      consumeToken();
       continue;
+    }
 
     bool NeedParseErrorRecovery = false;
     ASTNode Result;
@@ -411,6 +418,11 @@ ParserStatus Parser::parseBraceItems(SmallVectorImpl<ASTNode> &Entries,
     }
 
     if (NeedParseErrorRecovery) {
+      SyntaxContext->createNodeInPlace(SyntaxKind::CodeBlockItem);
+      SyntaxContext->setTransparent();
+
+      SyntaxParsingContext ItemCtxt(SyntaxContext, SyntaxKind::CodeBlockItem);
+      SyntaxParsingContext StmtCtxt(SyntaxContext, SyntaxContextKind::Stmt);
       // If we had a parse error, skip to the start of the next stmt, decl or
       // '{'.
       //

lib/Parse/SyntaxParsingContext.cpp

@@ -68,17 +68,10 @@ RC<RawSyntax> SyntaxParsingContext::bridgeAs(SyntaxContextKind Kind,
   if (Parts.size() == 1) {
     auto RawNode = Parts.front();
     switch (Kind) {
-    case SyntaxContextKind::Stmt: {
-      if (RawNode->isStmt())
-        return RawNode;
-      else if (RawNode->isDecl())
-        return createSyntaxAs(SyntaxKind::DeclarationStmt, Parts);
-      else if (RawNode->isExpr())
-        return createSyntaxAs(SyntaxKind::ExpressionStmt, Parts);
-      else
+    case SyntaxContextKind::Stmt:
+      if (!RawNode->isStmt())
         return makeUnknownSyntax(SyntaxKind::UnknownStmt, Parts);
       break;
-    }
     case SyntaxContextKind::Decl:
       if (!RawNode->isDecl())
         return makeUnknownSyntax(SyntaxKind::UnknownDecl, Parts);
@@ -186,6 +179,7 @@ void SyntaxParsingContext::createNodeInPlace(SyntaxKind Kind) {
     createNodeInPlace(Kind, Pair.first);
     break;
   }
+  case SyntaxKind::CodeBlockItem:
   case SyntaxKind::IdentifierExpr:
   case SyntaxKind::SpecializeExpr:
   case SyntaxKind::MemberAccessExpr:
@@ -255,8 +249,8 @@ void finalizeSourceFile(RootContextData &RootData,
 
   if (SF.hasSyntaxRoot()) {
     auto SourceRaw = SF.getSyntaxRoot().getRaw();
-    auto Decls = SourceRaw->getChild(SourceFileSyntax::Cursor::TopLevelDecls)
-                     ->getLayout();
+    auto Decls =
+        SourceRaw->getChild(SourceFileSyntax::Cursor::Items)->getLayout();
     std::copy(Decls.begin(), Decls.end(), std::back_inserter(AllTopLevel));
     EOFToken = SourceRaw->getChild(SourceFileSyntax::Cursor::EOFToken);
   }
@@ -267,12 +261,13 @@ void finalizeSourceFile(RootContextData &RootData,
   }
 
   for (auto RawNode : Parts) {
-    if (RawNode->getKind() != SyntaxKind::StmtList)
+    if (RawNode->getKind() != SyntaxKind::CodeBlockItemList)
       // FIXME: Skip toplevel garbage nodes for now. we shouldn't emit them in
       // the first place.
       continue;
-    AllTopLevel.push_back(
-        SyntaxFactory::createRaw(SyntaxKind::TopLevelCodeDecl, {RawNode}));
+
+    auto Items = RawNode->getLayout();
+    std::copy(Items.begin(), Items.end(), std::back_inserter(AllTopLevel));
   }
 
   if (!EOFToken)
@@ -281,8 +276,10 @@ void finalizeSourceFile(RootContextData &RootData,
   auto newRaw = SyntaxFactory::createRaw(
       SyntaxKind::SourceFile,
       {
-          SyntaxFactory::createRaw(SyntaxKind::DeclList, AllTopLevel), EOFToken,
+          SyntaxFactory::createRaw(SyntaxKind::CodeBlockItemList, AllTopLevel),
+          EOFToken,
       });
+  assert(newRaw);
   SF.setSyntaxRoot(make<SourceFileSyntax>(newRaw));
 
   if (SF.getASTContext().LangOpts.VerifySyntaxTree) {

lib/Syntax/RawSyntax.cpp

@@ -30,7 +30,7 @@ static bool isTrivialSyntaxKind(SyntaxKind Kind) {
     return true;
   switch(Kind) {
   case SyntaxKind::SourceFile:
-  case SyntaxKind::TopLevelCodeDecl:
+  case SyntaxKind::CodeBlockItem:
   case SyntaxKind::ExpressionStmt:
   case SyntaxKind::DeclarationStmt:
     return true;
@@ -179,6 +179,8 @@ RawSyntax::accumulateAbsolutePosition(AbsolutePosition &Pos) const {
       Trailer.accumulateAbsolutePosition(Pos);
   } else {
     for (auto &Child : getLayout()) {
+      if (!Child)
+        continue;
       auto Result = Child->accumulateAbsolutePosition(Pos);
       if (!Ret && Result)
         Ret = Result;
@@ -254,6 +256,8 @@ void RawSyntax::dump(llvm::raw_ostream &OS, unsigned Indent) const {
     }
   } else {
     for (auto &Child : getLayout()) {
+      if (!Child)
+        continue;
       OS << "\n";
       Child->dump(OS, Indent + 1);
     }

test/SwiftSyntax/SyntaxChildren.swift

@@ -28,25 +28,21 @@ var SyntaxChildrenAPI = TestSuite("SyntaxChildrenAPI")
 SyntaxChildrenAPI.test("IterateWithAllPresent") {
   let returnStmt = SyntaxFactory.makeReturnStmt(
     returnKeyword: SyntaxFactory.makeReturnKeyword(),
-    expression: SyntaxFactory.makeBlankUnknownExpr(),
-    semicolon: SyntaxFactory.makeSemicolonToken())
+    expression: SyntaxFactory.makeBlankUnknownExpr())
 
   var iterator = returnStmt.children.makeIterator()
   expectNext(&iterator) { ($0 as? TokenSyntax)?.tokenKind == .returnKeyword }
   expectNext(&iterator) { $0 is ExprSyntax }
-  expectNext(&iterator) { ($0 as? TokenSyntax)?.tokenKind == .semicolon }
   expectNextIsNil(&iterator)
 }
 
 SyntaxChildrenAPI.test("IterateWithSomeMissing") {
   let returnStmt = SyntaxFactory.makeReturnStmt(
     returnKeyword: SyntaxFactory.makeReturnKeyword(),
-    expression: nil,
-    semicolon: SyntaxFactory.makeSemicolonToken())
+    expression: nil)
 
   var iterator = returnStmt.children.makeIterator()
   expectNext(&iterator) { ($0 as? TokenSyntax)?.tokenKind == .returnKeyword }
-  expectNext(&iterator) { ($0 as? TokenSyntax)?.tokenKind == .semicolon }
   expectNextIsNil(&iterator)
 }