Merge pull request #26925 from DougGregor/parse-members-request

[Parser] Introduce a request for parsing type and extension members.

Author: DougGregor

include/swift/AST/ASTContext.h

@@ -826,7 +826,7 @@ class ASTContext final {
   /// of the given decl context.
   ///
   /// \param IDC The context whose member decls should be lazily parsed.
-  void parseMembers(IterableDeclContext *IDC);
+  std::vector<Decl *> parseMembers(IterableDeclContext *IDC);
 
   /// Get the lazy function data for the given generic context.
   ///

include/swift/AST/ASTTypeIDZone.def

@@ -33,3 +33,4 @@ SWIFT_TYPEID(AncestryFlags)
 SWIFT_TYPEID_NAMED(GenericSignature *, GenericSignature)
 SWIFT_TYPEID_NAMED(GenericTypeParamType *, GenericTypeParamType)
 SWIFT_TYPEID(Requirement)
+SWIFT_TYPEID_NAMED(IterableDeclContext *, IterableDeclContext)

include/swift/AST/ASTTypeIDs.h

@@ -17,19 +17,26 @@
 #ifndef SWIFT_AST_ASTTYPEIDS_H
 #define SWIFT_AST_ASTTYPEIDS_H
 
+#include "swift/Basic/LLVM.h"
 #include "swift/Basic/TypeID.h"
 namespace swift {
 
 class CustomAttr;
+class Decl;
 class GenericSignature;
 class GenericTypeParamType;
+class IterableDeclContext;
 class NominalTypeDecl;
+class OperatorDecl;
 struct PropertyWrapperBackingPropertyInfo;
 struct PropertyWrapperTypeInfo;
 enum class CtorInitializerKind;
 struct PropertyWrapperMutability;
+class ProtocolDecl;
 class Requirement;
+enum class ResilienceExpansion : unsigned;
 class Type;
+class ValueDecl;
 class VarDecl;
 class TypeAliasDecl;
 class Type;

include/swift/AST/Decl.h

@@ -7248,6 +7248,17 @@ void simple_display(llvm::raw_ostream &out, const Decl *decl);
 /// Display ValueDecl subclasses.
 void simple_display(llvm::raw_ostream &out, const ValueDecl *decl);
 
+/// Display ExtensionDecls.
+inline void simple_display(llvm::raw_ostream &out, const ExtensionDecl *decl) {
+  simple_display(out, static_cast<const Decl *>(decl));
+}
+
+/// Display NominalTypeDecls.
+inline void simple_display(llvm::raw_ostream &out,
+                           const NominalTypeDecl *decl) {
+  simple_display(out, static_cast<const Decl *>(decl));
+}
+
 /// Extract the source location from the given declaration.
 SourceLoc extractNearestSourceLoc(const Decl *decl);
 
@@ -7261,6 +7272,11 @@ inline SourceLoc extractNearestSourceLoc(const GenericTypeDecl *type) {
   return extractNearestSourceLoc(static_cast<const Decl *>(type));
 }
 
+/// Extract the source location from the given declaration.
+inline SourceLoc extractNearestSourceLoc(const NominalTypeDecl *type) {
+  return extractNearestSourceLoc(static_cast<const Decl *>(type));
+}
+
 /// Extract the source location from the given declaration.
 inline SourceLoc extractNearestSourceLoc(const AbstractFunctionDecl *func) {
   return extractNearestSourceLoc(static_cast<const Decl *>(func));

include/swift/AST/DeclContext.h

@@ -699,8 +699,8 @@ class IterableDeclContext {
   /// time, but I think it's a better trade to just keep a count here.
   unsigned MemberCount : 29;
 
-  /// Whether parsing the members of this context has been delayed.
-  unsigned HasUnparsedMembers : 1;
+  /// Whether we have already added the parsed members into the context.
+  unsigned AddedParsedMembers : 1;
 
   /// Whether delayed parsing detected a possible operator definition
   /// while skipping the body of this context.
@@ -722,8 +722,8 @@ class IterableDeclContext {
   IterableDeclContext(IterableDeclContextKind kind)
     : LastDeclAndKind(nullptr, kind) {
     MemberCount = 0;
+    AddedParsedMembers = 0;
     HasOperatorDeclarations = 0;
-    HasUnparsedMembers = 0;
     HasNestedClassDeclarations = 0;
   }
 
@@ -732,13 +732,7 @@ class IterableDeclContext {
     return LastDeclAndKind.getInt();
   }
 
-  bool hasUnparsedMembers() const {
-    return HasUnparsedMembers;
-  }
-
-  void setHasUnparsedMembers() {
-    HasUnparsedMembers = 1;
-  }
+  bool hasUnparsedMembers() const;
 
   bool maybeHasOperatorDeclarations() const {
     return HasOperatorDeclarations;
@@ -839,9 +833,14 @@ void simple_display(llvm::raw_ostream &out, const ParamT *dc) {
     out << "(null)";
 }
 
+void simple_display(llvm::raw_ostream &out, const IterableDeclContext *idc);
+
 /// Extract the source location from the given declaration context.
 SourceLoc extractNearestSourceLoc(const DeclContext *dc);
 
+/// Extract the source location from the given declaration context.
+SourceLoc extractNearestSourceLoc(const IterableDeclContext *idc);
+
 } // end namespace swift
 
 namespace llvm {

include/swift/AST/LazyResolver.h

@@ -90,10 +90,8 @@ class LazyMemberParser {
 public:
   virtual ~LazyMemberParser() = default;
 
-  /// Populates a given decl context \p IDC with all of its members.
-  ///
-  /// The implementation should add the members to IDC.
-  virtual void parseMembers(IterableDeclContext *IDC) = 0;
+  /// Retrieves the parsed members for the given decl context \p IDC.
+  virtual std::vector<Decl *> parseMembers(IterableDeclContext *IDC) = 0;
 
   /// Return whether the iterable decl context needs parsing.
   virtual bool hasUnparsedMembers(const IterableDeclContext *IDC) = 0;

include/swift/AST/ParseRequests.h

@@ -0,0 +1,70 @@
+//===--- ParseRequests.h - Parsing Requests ---------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2019 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines parsing requests.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SWIFT_PARSE_REQUESTS_H
+#define SWIFT_PARSE_REQUESTS_H
+
+#include "swift/AST/ASTTypeIDs.h"
+#include "swift/AST/SimpleRequest.h"
+
+namespace swift {
+
+/// Report that a request of the given kind is being evaluated, so it
+/// can be recorded by the stats reporter.
+template<typename Request>
+void reportEvaluatedRequest(UnifiedStatsReporter &stats,
+                            const Request &request);
+
+/// Parse the members of a nominal type declaration or extension.
+class ParseMembersRequest :
+    public SimpleRequest<ParseMembersRequest,
+                         ArrayRef<Decl *>(IterableDeclContext *),
+                         CacheKind::Cached>
+{
+public:
+  using SimpleRequest::SimpleRequest;
+
+private:
+  friend SimpleRequest;
+
+  // Evaluation.
+  ArrayRef<Decl *> evaluate(Evaluator &evaluator,
+                            IterableDeclContext *idc) const;
+
+public:
+  // Caching
+  bool isCached() const { return true; }
+};
+
+/// The zone number for the parser.
+#define SWIFT_TYPEID_ZONE Parse
+#define SWIFT_TYPEID_HEADER "swift/AST/ParseTypeIDZone.def"
+#include "swift/Basic/DefineTypeIDZone.h"
+#undef SWIFT_TYPEID_ZONE
+#undef SWIFT_TYPEID_HEADER
+
+// Set up reporting of evaluated requests.
+#define SWIFT_REQUEST(Zone, RequestType)                         \
+template<>                                                       \
+inline void reportEvaluatedRequest(UnifiedStatsReporter &stats,  \
+                            const RequestType &request) {        \
+  ++stats.getFrontendCounters().RequestType;                     \
+}
+#include "swift/AST/ParseTypeIDZone.def"
+#undef SWIFT_REQUEST
+
+} // end namespace swift
+
+#endif // SWIFT_PARSE_REQUESTS_H

include/swift/AST/ParseTypeIDZone.def

@@ -0,0 +1,17 @@
+//===--- ParseTypeIDZone.def ------------------------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+//  This definition file describes the requests in the parser's zone.
+//
+//===----------------------------------------------------------------------===//
+
+SWIFT_REQUEST(Parse, ParseMembersRequest)

include/swift/Basic/Statistics.def

@@ -195,6 +195,10 @@ FRONTEND_STATISTIC(Parse, NumFunctionsParsed)
 /// Number of full braced decl list parsed.
 FRONTEND_STATISTIC(Parse, NumIterableDeclContextParsed)
 
+#define SWIFT_REQUEST(ZONE, NAME) FRONTEND_STATISTIC(Parse, NAME)
+#include "swift/AST/ParseTypeIDZone.def"
+#undef SWIFT_REQUEST
+
 /// Number of conformances that were deserialized by this frontend job.
 FRONTEND_STATISTIC(Sema, NumConformancesDeserialized)
 

include/swift/Basic/TypeID.h

@@ -35,6 +35,7 @@ enum class Zone : uint8_t {
   IDE                     = 137,
   IDETypeChecking         = 97,
   NameLookup              = 9,
+  Parse                   = 8,
   TypeChecker             = 10,
   // N.B. This is not a formal zone and exists solely to support the unit tests.
   ArithmeticEvaluator     = 255,

include/swift/Parse/Parser.h

@@ -971,7 +971,7 @@ class Parser {
 
   void parseDeclDelayed();
 
-  void parseDeclListDelayed(IterableDeclContext *IDC);
+  std::vector<Decl *> parseDeclListDelayed(IterableDeclContext *IDC);
 
   bool parseMemberDeclList(SourceLoc LBLoc, SourceLoc &RBLoc,
                            SourceLoc PosBeforeLB,
@@ -1080,9 +1080,10 @@ class Parser {
   ParserStatus parseDeclItem(bool &PreviousHadSemi,
                              Parser::ParseDeclOptions Options,
                              llvm::function_ref<void(Decl*)> handler);
-  bool parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc,
-                     Diag<> ErrorDiag, ParseDeclOptions Options,
-                     IterableDeclContext *IDC);
+  std::vector<Decl *> parseDeclList(SourceLoc LBLoc, SourceLoc &RBLoc,
+                                    Diag<> ErrorDiag, ParseDeclOptions Options,
+                                    IterableDeclContext *IDC,
+                                    bool &hadError);
   ParserResult<ExtensionDecl> parseDeclExtension(ParseDeclOptions Flags,
                                                  DeclAttributes &Attributes);
   ParserResult<EnumDecl> parseDeclEnum(ParseDeclOptions Flags,

include/swift/Parse/PersistentParserState.h

@@ -153,7 +153,7 @@ class PersistentParserState: public LazyMemberParser {
   void delayTopLevel(TopLevelCodeDecl *TLCD, SourceRange BodyRange,
                      SourceLoc PreviousLoc);
 
-  void parseMembers(IterableDeclContext *IDC) override;
+  std::vector<Decl *> parseMembers(IterableDeclContext *IDC) override;
 
   bool hasDelayedDecl() {
     return CodeCompletionDelayedDeclState.get() != nullptr;

include/swift/Subsystems.h

@@ -389,6 +389,13 @@ namespace swift {
   /// The ASTContext will automatically call these upon construction.
   void registerNameLookupRequestFunctions(Evaluator &evaluator);
 
+  /// Register Parse-level request functions with the evaluator.
+  ///
+  /// Clients that form an ASTContext and will perform any parsing queries
+  /// using Parse-level logic should call these functions after forming the
+  /// ASTContext.
+  void registerParseRequestFunctions(Evaluator &evaluator);
+
   /// Register Sema-level request functions with the evaluator.
   ///
   /// Clients that form an ASTContext and will perform any semantic queries

lib/AST/ASTContext.cpp

@@ -2015,12 +2015,14 @@ LazyContextData *ASTContext::getOrCreateLazyContextData(
   return entry;
 }
 
-void ASTContext::parseMembers(IterableDeclContext *IDC) {
+std::vector<Decl *> ASTContext::parseMembers(IterableDeclContext *IDC) {
   assert(IDC->hasUnparsedMembers());
   for (auto *p: getImpl().lazyParsers) {
     if (p->hasUnparsedMembers(IDC))
-      p->parseMembers(IDC);
+      return p->parseMembers(IDC);
   }
+
+  return { };
 }
 
 LazyIterableDeclContextData *ASTContext::getOrCreateLazyIterableContextData(

lib/AST/DeclContext.cpp

@@ -19,6 +19,7 @@
 #include "swift/AST/Initializer.h"
 #include "swift/AST/LazyResolver.h"
 #include "swift/AST/Module.h"
+#include "swift/AST/ParseRequests.h"
 #include "swift/AST/Types.h"
 #include "swift/AST/TypeCheckRequests.h"
 #include "swift/Basic/SourceManager.h"
@@ -804,15 +805,39 @@ void IterableDeclContext::setMemberLoader(LazyMemberLoader *loader,
     ++s->getFrontendCounters().NumUnloadedLazyIterableDeclContexts;
   }
 }
+bool IterableDeclContext::hasUnparsedMembers() const {
+  if (AddedParsedMembers)
+    return false;
+
+  if (!getDecl()->getDeclContext()->getParentSourceFile()) {
+    // There will never be any parsed members to add, so set the flag to say
+    // we are done so we can short-circuit next time.
+    const_cast<IterableDeclContext *>(this)->AddedParsedMembers = 1;
+    return false;
+  }
+
+  return true;
+}
 
 void IterableDeclContext::loadAllMembers() const {
   ASTContext &ctx = getASTContext();
 
-  // Lazily parse members.
-  if (HasUnparsedMembers) {
-    auto *IDC = const_cast<IterableDeclContext *>(this);
-    ctx.parseMembers(IDC);
-    IDC->HasUnparsedMembers = 0;
+  // For contexts within a source file, get the list of parsed members.
+  if (getDecl()->getDeclContext()->getParentSourceFile()) {
+    // Retrieve the parsed members. Even if we've already added the parsed
+    // members to this context, this call is important for recording the
+    // dependency edge.
+    auto mutableThis = const_cast<IterableDeclContext *>(this);
+    auto members = evaluateOrDefault(
+        ctx.evaluator, ParseMembersRequest{mutableThis}, ArrayRef<Decl*>());
+
+    // If we haven't already done so, add these members to this context.
+    if (!AddedParsedMembers) {
+      mutableThis->AddedParsedMembers = 1;
+      for (auto member : members) {
+        mutableThis->addMember(member);
+      }
+    }
   }
 
   if (!hasLazyMembers())
@@ -1072,3 +1097,12 @@ static void verify_DeclContext_is_start_of_node() {
 #include "swift/AST/ExprNodes.def"
 }
 #endif
+
+void swift::simple_display(llvm::raw_ostream &out,
+                           const IterableDeclContext *idc) {
+  simple_display(out, idc->getDecl());
+}
+
+SourceLoc swift::extractNearestSourceLoc(const IterableDeclContext *idc) {
+  return extractNearestSourceLoc(idc->getDecl());
+}

lib/AST/NameLookup.cpp

@@ -1225,7 +1225,7 @@ TinyPtrVector<ValueDecl *> NominalTypeDecl::lookupDirect(
       // We still have to parse any unparsed extensions.
       if (!ignoreNewExtensions) {
         for (auto *e : getExtensions()) {
-          if (e->hasUnparsedMembers())
+          if (e->getParentSourceFile() != nullptr)
             e->loadAllMembers();
         }
       }

lib/Frontend/Frontend.cpp

@@ -186,6 +186,7 @@ bool CompilerInstance::setUpASTContextIfNeeded() {
   Context.reset(ASTContext::get(Invocation.getLangOptions(),
                                 Invocation.getSearchPathOptions(), SourceMgr,
                                 Diagnostics));
+  registerParseRequestFunctions(Context->evaluator);
   registerTypeCheckerRequestFunctions(Context->evaluator);
 
   // Migrator, indexing and typo correction need some IDE requests.

lib/Parse/CMakeLists.txt

@@ -18,6 +18,7 @@ add_swift_host_library(swiftParse STATIC
   ParseIfConfig.cpp
   ParsePattern.cpp
   Parser.cpp
+  ParseRequests.cpp
   ParseStmt.cpp
   ParseType.cpp
   PersistentParserState.cpp