Parser: allow on-demand member decl parsing for other nominal types.

include/swift/AST/ASTContext.h

@@ -791,6 +791,10 @@ class ASTContext final {
   /// \param IDC The context whose member decls should be lazily parsed.
   void parseMembers(IterableDeclContext *IDC);
 
+  /// Use the lazy parsers associated with the context to check whether the decl
+  /// context has been parsed.
+  bool hasUnparsedMembers(const IterableDeclContext *IDC) const;
+
   /// Get the lazy function data for the given generic context.
   ///
   /// \param lazyLoader If non-null, the lazy loader to use when creating the

include/swift/AST/LazyResolver.h

@@ -101,6 +101,12 @@ class LazyMemberParser {
   ///
   /// The implementation should add the members to IDC.
   virtual void parseMembers(IterableDeclContext *IDC) = 0;
+
+  /// Return whether the iterable decl context needs parsing.
+  virtual bool hasUnparsedMembers(const IterableDeclContext *IDC) = 0;
+
+  /// Parse all delayed decl list members.
+  virtual void parseAllDelayedDeclLists() = 0;
 };
 
 /// Context data for generic contexts.

include/swift/Frontend/Frontend.h

@@ -552,7 +552,8 @@ class CompilerInstance {
 
   /// Parses the input file but does no type-checking or module imports.
   /// Note that this only supports parsing an invocation with a single file.
-  void performParseOnly(bool EvaluateConditionals = false);
+  void performParseOnly(bool EvaluateConditionals = false,
+                        bool ParseDelayedBodyOnEnd = false);
 
   /// Parses and performs name binding on all input files.
   ///

include/swift/Parse/Parser.h

@@ -346,6 +346,9 @@ class Parser {
   SyntaxParsingContext *SyntaxContext;
 
 public:
+  Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
+         SILParserTUStateBase *SIL,
+         PersistentParserState *PersistentState);
   Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
          PersistentParserState *PersistentState = nullptr);
   Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,

include/swift/Parse/PersistentParserState.h

@@ -174,10 +174,12 @@ class PersistentParserState: public LazyMemberParser {
   std::unique_ptr<DelayedDeclListState>
     takeDelayedDeclListState(IterableDeclContext *IDC);
 
-  bool hasDelayedDeclList(IterableDeclContext *IDC) {
+  bool hasUnparsedMembers(const IterableDeclContext *IDC) override {
     return DelayedDeclListStates.find(IDC) != DelayedDeclListStates.end();
   }
 
+  void parseAllDelayedDeclLists() override;
+
   TopLevelContext &getTopLevelContext() {
     return TopLevelCode;
   }

lib/AST/ASTContext.cpp

@@ -573,6 +573,7 @@ void ASTContext::addLazyParser(LazyMemberParser *lazyParser) {
 
 void ASTContext::removeLazyParser(LazyMemberParser *lazyParser) {
   auto removed = getImpl().lazyParsers.erase(lazyParser);
+  (void)removed;
   assert(removed && "Removing an non-existing lazy parser.");
 }
 
@@ -1902,9 +1903,16 @@ LazyContextData *ASTContext::getOrCreateLazyContextData(
   return contextData;
 }
 
+bool ASTContext::hasUnparsedMembers(const IterableDeclContext *IDC) const {
+  auto parsers = getImpl().lazyParsers;
+  return std::any_of(parsers.begin(), parsers.end(),
+    [IDC](LazyMemberParser *p) { return p->hasUnparsedMembers(IDC); });
+}
+
 void ASTContext::parseMembers(IterableDeclContext *IDC) {
   for (auto *p: getImpl().lazyParsers) {
-    p->parseMembers(IDC);
+    if (p->hasUnparsedMembers(IDC))
+      p->parseMembers(IDC);
   }
 }
 

lib/AST/ASTVerifier.cpp

@@ -709,6 +709,8 @@ class Verifier : public ASTWalker {
       pushScope(fn);                                            \
       if (fn->hasLazyMembers())                                 \
         return false;                                           \
+      if (fn->getASTContext().hasUnparsedMembers(fn))           \
+        return false;                                           \
       return shouldVerify(cast<ASTNodeBase<NODE*>::BaseTy>(fn));\
     }                                                           \
     void cleanup(NODE *fn) {                                    \

lib/Frontend/Frontend.cpp

@@ -925,7 +925,8 @@ SourceFile *CompilerInstance::createSourceFileForMainModule(
   return inputFile;
 }
 
-void CompilerInstance::performParseOnly(bool EvaluateConditionals) {
+void CompilerInstance::performParseOnly(bool EvaluateConditionals,
+                                        bool ParseDelayedBodyOnEnd) {
   const InputFileKind Kind = Invocation.getInputKind();
   ModuleDecl *const MainModule = getMainModule();
   Context->LoadedModules[MainModule->getName()] = MainModule;
@@ -947,6 +948,10 @@ void CompilerInstance::performParseOnly(bool EvaluateConditionals) {
   }
 
   PersistentParserState PersistentState(getASTContext());
+  SWIFT_DEFER {
+    if (ParseDelayedBodyOnEnd)
+      PersistentState.parseAllDelayedDeclLists();
+  };
   PersistentState.PerformConditionEvaluation = EvaluateConditionals;
   // Parse all the library files.
   for (auto BufferID : InputSourceCodeBufferIDs) {

lib/FrontendTool/FrontendTool.cpp

@@ -902,8 +902,12 @@ static bool performCompile(CompilerInstance &Instance,
     return compileLLVMIR(Invocation, Instance, Stats);
 
   if (FrontendOptions::shouldActionOnlyParse(Action)) {
+    bool ParseDelayedDeclListsOnEnd =
+      Action == FrontendOptions::ActionType::DumpParse ||
+      Invocation.getDiagnosticOptions().VerifyMode != DiagnosticOptions::NoVerify;
     Instance.performParseOnly(/*EvaluateConditionals*/
-                    Action == FrontendOptions::ActionType::EmitImportedModules);
+                    Action == FrontendOptions::ActionType::EmitImportedModules,
+                              ParseDelayedDeclListsOnEnd);
   } else if (Action == FrontendOptions::ActionType::ResolveImports) {
     Instance.performParseAndResolveImportsOnly();
   } else {

lib/Parse/ParseDecl.cpp

@@ -181,18 +181,21 @@ namespace {
 } // end anonymous namespace
 
 void PersistentParserState::parseMembers(IterableDeclContext *IDC) {
-  if (!hasDelayedDeclList(IDC))
-    return;
   SourceFile &SF = *IDC->getDecl()->getDeclContext()->getParentSourceFile();
   assert(!SF.hasInterfaceHash() &&
     "Cannot delay parsing if we care about the interface hash.");
+  assert(SF.Kind != SourceFileKind::SIL && "cannot delay parsing SIL");
   unsigned BufferID = *SF.getBufferID();
+
   // MarkedPos is not useful for delayed parsing because we know where we should
   // jump the parser to. However, we should recover the MarkedPos here in case
   // the PersistentParserState will be used to continuously parse the rest of
   // the file linearly.
   llvm::SaveAndRestore<ParserPosition> Pos(MarkedPos, ParserPosition());
-  Parser TheParser(BufferID, SF, nullptr, this);
+
+  // Lexer diaganostics have been emitted during skipping, so we disable lexer's
+  // diagnostic engine here.
+  Parser TheParser(BufferID, SF, /*No Lexer Diags*/nullptr, nullptr, this);
   // Disable libSyntax creation in the delayed parsing.
   TheParser.SyntaxContext->disable();
   TheParser.parseDeclListDelayed(IDC);
@@ -2292,7 +2295,8 @@ static unsigned skipUntilMatchingRBrace(Parser &P, bool &HasPoundDirective,
   SyntaxParsingContext BodyContext(SyntaxContext, SyntaxKind::TokenList);
   unsigned OpenBraces = 1;
   while (OpenBraces != 0 && P.Tok.isNot(tok::eof)) {
-    HasPoundDirective |= P.Tok.isAny(tok::pound_sourceLocation, tok::pound_line);
+    HasPoundDirective |= P.Tok.isAny(tok::pound_sourceLocation, tok::pound_line,
+      tok::pound_if, tok::pound_else, tok::pound_endif, tok::pound_elseif);
     if (P.consumeIf(tok::l_brace)) {
       OpenBraces++;
       continue;
@@ -2902,6 +2906,15 @@ void Parser::parseDeclListDelayed(IterableDeclContext *IDC) {
                   ParseDeclOptions(DelayedState->Flags),
                   [&] (Decl *D) { ext->addMember(D); });
     ext->setBraces({LBLoc, RBLoc});
+  } else if (auto *cd = dyn_cast<ClassDecl>(D)) {
+    auto handler = [&] (Decl *D) {
+      cd->addMember(D);
+      if (isa<DestructorDecl>(D))
+        cd->setHasDestructor();
+    };
+    parseDeclList(cd->getBraces().Start, RBLoc, Id,
+                  ParseDeclOptions(DelayedState->Flags), handler);
+    cd->setBraces({LBLoc, RBLoc});
   } else {
     auto *ntd = cast<NominalTypeDecl>(D);
     parseDeclList(ntd->getBraces().Start, RBLoc, Id,
@@ -3349,6 +3362,10 @@ bool Parser::parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc,
 }
 
 bool Parser::canDelayMemberDeclParsing() {
+  // There's no fundamental reasons that SIL cannnot be lasily parsed. We need
+  // to keep SILParserTUStateBase persistent to make it happen.
+  if (isInSILMode())
+    return false;
   // Calculating interface hash requires tokens consumed in the original order.
   if (SF.hasInterfaceHash())
     return false;
@@ -5831,6 +5848,7 @@ ParserResult<StructDecl> Parser::parseDeclStruct(ParseDeclOptions Flags,
   // Make the entities of the struct as a code block.
   SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock);
   SourceLoc LBLoc, RBLoc;
+  SourceLoc PosBeforeLB = Tok.getLoc();
   if (parseToken(tok::l_brace, LBLoc, diag::expected_lbrace_struct)) {
     LBLoc = PreviousLoc;
     RBLoc = LBLoc;
@@ -5839,9 +5857,20 @@ ParserResult<StructDecl> Parser::parseDeclStruct(ParseDeclOptions Flags,
     // Parse the body.
     Scope S(this, ScopeKind::StructBody);
     ParseDeclOptions Options(PD_HasContainerType | PD_InStruct);
-    if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_struct,
-                      Options, [&](Decl *D) {SD->addMember(D);}))
-      Status.setIsParseError();
+    if (canDelayMemberDeclParsing()) {
+      if (Tok.is(tok::r_brace)) {
+        RBLoc = consumeToken();
+      } else {
+        RBLoc = Tok.getLoc();
+        Status.setIsParseError();
+      }
+      State->delayDeclList(SD, Options.toRaw(), CurDeclContext, { LBLoc, RBLoc },
+                           PosBeforeLB);
+    } else {
+      if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_struct,
+                        Options, [&](Decl *D) {SD->addMember(D);}))
+        Status.setIsParseError();
+    }
   }
 
   SD->setBraces({LBLoc, RBLoc});
@@ -5942,6 +5971,7 @@ ParserResult<ClassDecl> Parser::parseDeclClass(ParseDeclOptions Flags,
 
   SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock);
   SourceLoc LBLoc, RBLoc;
+  auto PosBeforeLB = Tok.getLoc();
   if (parseToken(tok::l_brace, LBLoc, diag::expected_lbrace_class)) {
     LBLoc = PreviousLoc;
     RBLoc = LBLoc;
@@ -5951,14 +5981,25 @@ ParserResult<ClassDecl> Parser::parseDeclClass(ParseDeclOptions Flags,
     Scope S(this, ScopeKind::ClassBody);
     ParseDeclOptions Options(PD_HasContainerType | PD_AllowDestructor |
                              PD_InClass);
-    auto Handler = [&] (Decl *D) {
-      CD->addMember(D);
-      if (isa<DestructorDecl>(D))
-        CD->setHasDestructor();
-    };
-    if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_class,
-                      Options, Handler))
-      Status.setIsParseError();
+    if (canDelayMemberDeclParsing()) {
+      if (Tok.is(tok::r_brace)) {
+        RBLoc = consumeToken();
+      } else {
+        RBLoc = Tok.getLoc();
+        Status.setIsParseError();
+      }
+      State->delayDeclList(CD, Options.toRaw(), CurDeclContext, { LBLoc, RBLoc },
+                           PosBeforeLB);
+    } else {
+      auto Handler = [&] (Decl *D) {
+        CD->addMember(D);
+        if (isa<DestructorDecl>(D))
+          CD->setHasDestructor();
+      };
+      if (parseDeclList(LBLoc, RBLoc, diag::expected_rbrace_class,
+                        Options, Handler))
+        Status.setIsParseError();
+    }
   }
 
   CD->setBraces({LBLoc, RBLoc});
@@ -6040,6 +6081,7 @@ parseDeclProtocol(ParseDeclOptions Flags, DeclAttributes &Attributes) {
     SyntaxParsingContext BlockContext(SyntaxContext, SyntaxKind::MemberDeclBlock);
     SourceLoc LBraceLoc;
     SourceLoc RBraceLoc;
+    SourceLoc PosBeforeLB = Tok.getLoc();
     if (parseToken(tok::l_brace, LBraceLoc, diag::expected_lbrace_protocol)) {
       LBraceLoc = PreviousLoc;
       RBraceLoc = LBraceLoc;
@@ -6049,9 +6091,21 @@ parseDeclProtocol(ParseDeclOptions Flags, DeclAttributes &Attributes) {
       ParseDeclOptions Options(PD_HasContainerType |
                                PD_DisallowInit |
                                PD_InProtocol);
-      if (parseDeclList(LBraceLoc, RBraceLoc, diag::expected_rbrace_protocol,
-                        Options, [&](Decl *D) {Proto->addMember(D);}))
-        Status.setIsParseError();
+      if (canDelayMemberDeclParsing()) {
+        if (Tok.is(tok::r_brace)) {
+          RBraceLoc = consumeToken();
+        } else {
+          RBraceLoc = Tok.getLoc();
+          Status.setIsParseError();
+        }
+        State->delayDeclList(Proto, Options.toRaw(), CurDeclContext,
+                             { LBraceLoc, RBraceLoc },
+                             PosBeforeLB);
+      } else {
+        if (parseDeclList(LBraceLoc, RBraceLoc, diag::expected_rbrace_protocol,
+                          Options, [&](Decl *D) {Proto->addMember(D);}))
+          Status.setIsParseError();
+      }
     }
 
     // Install the protocol elements.

lib/Parse/Parser.cpp

@@ -331,12 +331,18 @@ swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
 // Setup and Helper Methods
 //===----------------------------------------------------------------------===//
 
+
 Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
                PersistentParserState *PersistentState)
+    : Parser(BufferID, SF, &SF.getASTContext().Diags, SIL, PersistentState) {}
+
+Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
+               SILParserTUStateBase *SIL,
+               PersistentParserState *PersistentState)
     : Parser(
           std::unique_ptr<Lexer>(new Lexer(
               SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr,
-              BufferID, &SF.getASTContext().Diags,
+              BufferID, LexerDiags,
               /*InSILMode=*/SIL != nullptr,
               SF.Kind == SourceFileKind::Main
                   ? HashbangMode::Allowed

lib/Parse/PersistentParserState.cpp

@@ -81,6 +81,16 @@ void PersistentParserState::delayDeclList(IterableDeclContext* D,
     ParentContext, BodyRange, PreviousLoc, ScopeInfo.saveCurrentScope());
 }
 
+void PersistentParserState::parseAllDelayedDeclLists() {
+  std::vector<IterableDeclContext*> AllDelayed;
+  for (auto &P: DelayedDeclListStates) {
+    AllDelayed.push_back(P.first);
+  }
+  for (auto *D: AllDelayed) {
+    parseMembers(D);
+  }
+}
+
 void PersistentParserState::delayTopLevel(TopLevelCodeDecl *TLCD,
                                           SourceRange BodyRange,
                                           SourceLoc PreviousLoc) {

tools/swift-ide-test/swift-ide-test.cpp

@@ -1476,7 +1476,7 @@ static int doInputCompletenessTest(StringRef SourceFilename) {
   llvm::raw_ostream &OS = llvm::outs();
   OS << SourceFilename << ": ";
   if (isSourceInputComplete(std::move(FileBufOrErr.get()),
-                            SourceFileKind::Main).IsComplete) {
+                            SourceFileKind::REPL).IsComplete) {
     OS << "IS_COMPLETE\n";
   } else {
     OS << "IS_INCOMPLETE\n";

utils/scale-test

@@ -109,6 +109,8 @@ def run_once_with_primary(args, ast, rng, primary_idx):
         mode = "-c"
         if args.typecheck:
             mode = "-typecheck"
+        if args.parse:
+            mode = "-parse"
 
         focus = ["-primary-file", primary]
         if args.whole_module_optimization:
@@ -716,6 +718,9 @@ def main():
         '--exponential-threshold', type=float,
         default=1.2,
         help='minimum base for exponential fit to consider "bad scaling"')
+    parser.add_argument(
+        '-parse', '--parse', action='store_true',
+        default=False, help='only run compiler with -parse')
     parser.add_argument(
         '-typecheck', '--typecheck', action='store_true',
         default=False, help='only run compiler with -typecheck')

validation-test/compiler_scale/nominal_bodies.gyb

@@ -0,0 +1,9 @@
+// RUN: %scale-test --sum-multi --parse --begin 5 --end 16 --step 5 --select NumIterableDeclContextParsed %s
+// REQUIRES: OS=macosx
+// REQUIRES: asserts
+
+struct S${N} {}
+class C${N} {}
+enum E${N} {}
+extension C${N} {}
+protocol P${N} {}