Merge pull request #12669 from graydon/named-lazy-member-loading-3

[WIP] Named lazy member loading 3

Author: graydon

include/swift/AST/DeclContext.h

@@ -26,6 +26,7 @@
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/SourceLoc.h"
 #include "swift/Basic/STLExtras.h"
+#include "llvm/ADT/PointerEmbeddedInt.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PointerUnion.h"
 
@@ -67,6 +68,10 @@ namespace swift {
   class SerializedTopLevelCodeDecl;
   class StructDecl;
 
+namespace serialization {
+using DeclID = llvm::PointerEmbeddedInt<unsigned, 31>;
+}
+
 enum class DeclContextKind : uint8_t {
   AbstractClosureExpr,
   Initializer,
@@ -604,6 +609,10 @@ class IterableDeclContext {
   mutable llvm::PointerIntPair<Decl *, 2, IterableDeclContextKind> 
     LastDeclAndKind;
 
+  // The DeclID this IDC was deserialized from, if any. Used for named lazy
+  // member loading, as a key when doing lookup in this IDC.
+  serialization::DeclID SerialID;
+
   template<class A, class B, class C>
   friend struct ::llvm::cast_convert_val;
 
@@ -631,14 +640,33 @@ class IterableDeclContext {
   /// Check whether there are lazily-loaded members.
   bool hasLazyMembers() const {
     return FirstDeclAndLazyMembers.getInt();
-  }  
+  }
 
   /// Setup the loader for lazily-loaded members.
   void setMemberLoader(LazyMemberLoader *loader, uint64_t contextData);
 
   /// Load all of the members of this context.
   void loadAllMembers() const;
 
+  /// Determine whether this was deserialized (and thus SerialID is
+  /// valid).
+  bool wasDeserialized() const;
+
+  /// Return 'this' as a \c Decl.
+  const Decl *getDecl() const;
+
+  /// Get the DeclID this Decl was deserialized from.
+  serialization::DeclID getDeclID() const {
+    assert(wasDeserialized());
+    return SerialID;
+  }
+
+  /// Set the DeclID this Decl was deserialized from.
+  void setDeclID(serialization::DeclID d) {
+    assert(wasDeserialized());
+    SerialID = d;
+  }
+
   // Some Decls are IterableDeclContexts, but not all.
   static bool classof(const Decl *D);
 

include/swift/AST/LazyResolver.h

@@ -26,6 +26,7 @@ namespace swift {
 class AssociatedTypeDecl;
 class Decl;
 class DeclContext;
+class IterableDeclContext;
 class ExtensionDecl;
 class Identifier;
 class NominalTypeDecl;
@@ -224,13 +225,14 @@ class alignas(void*) LazyMemberLoader {
   virtual void
   loadAllMembers(Decl *D, uint64_t contextData) = 0;
 
-  /// Populates a vector with all members of \p D that have DeclName
+  /// Populates a vector with all members of \p IDC that have DeclName
   /// matching \p N.
   ///
   /// Returns None if an error occurred \em or named member-lookup
   /// was otherwise unsupported in this implementation or Decl.
   virtual Optional<TinyPtrVector<ValueDecl *>>
-  loadNamedMembers(const Decl *D, DeclName N, uint64_t contextData) = 0;
+  loadNamedMembers(const IterableDeclContext *IDC, DeclName N,
+                   uint64_t contextData) = 0;
 
   /// Populates the given vector with all conformances for \p D.
   ///

include/swift/Serialization/ModuleFile.h

@@ -66,6 +66,7 @@ class ModuleFile
 
   llvm::BitstreamCursor SILCursor;
   llvm::BitstreamCursor SILIndexCursor;
+  llvm::BitstreamCursor DeclMemberTablesCursor;
 
   /// The name of the module.
   StringRef Name;
@@ -335,14 +336,26 @@ class ModuleFile
   using SerializedNestedTypeDeclsTable =
       llvm::OnDiskIterableChainedHashTable<NestedTypeDeclsTableInfo>;
 
+  class DeclMemberNamesTableInfo;
+  using SerializedDeclMemberNamesTable =
+      llvm::OnDiskIterableChainedHashTable<DeclMemberNamesTableInfo>;
+
+  class DeclMembersTableInfo;
+  using SerializedDeclMembersTable =
+      llvm::OnDiskIterableChainedHashTable<DeclMembersTableInfo>;
+
   std::unique_ptr<SerializedDeclTable> TopLevelDecls;
   std::unique_ptr<SerializedDeclTable> OperatorDecls;
   std::unique_ptr<SerializedDeclTable> PrecedenceGroupDecls;
-  std::unique_ptr<SerializedDeclTable> ClassMembersByName;
+  std::unique_ptr<SerializedDeclTable> ClassMembersForDynamicLookup;
   std::unique_ptr<SerializedDeclTable> OperatorMethodDecls;
   std::unique_ptr<SerializedExtensionTable> ExtensionDecls;
   std::unique_ptr<SerializedLocalDeclTable> LocalTypeDecls;
   std::unique_ptr<SerializedNestedTypeDeclsTable> NestedTypeDecls;
+  std::unique_ptr<SerializedDeclMemberNamesTable> DeclMemberNames;
+
+  llvm::DenseMap<uint32_t,
+           std::unique_ptr<SerializedDeclMembersTable>> DeclMembersTables;
 
   class ObjCMethodTableInfo;
   using SerializedObjCMethodTable =
@@ -477,6 +490,21 @@ class ModuleFile
   std::unique_ptr<SerializedNestedTypeDeclsTable>
   readNestedTypeDeclsTable(ArrayRef<uint64_t> fields, StringRef blobData);
 
+  /// Read an on-disk local decl-name hash table stored in
+  /// index_block::DeclMemberNamesLayout format.
+  std::unique_ptr<SerializedDeclMemberNamesTable>
+  readDeclMemberNamesTable(ArrayRef<uint64_t> fields, StringRef blobData);
+
+  /// Read an on-disk local decl-members hash table stored in
+  /// index_block::DeclMembersLayout format.
+  std::unique_ptr<SerializedDeclMembersTable>
+  readDeclMembersTable(ArrayRef<uint64_t> fields, StringRef blobData);
+
+  /// Main logic of getDeclChecked.
+  llvm::Expected<Decl *>
+  getDeclCheckedImpl(serialization::DeclID DID,
+                     Optional<DeclContext *> ForcedContext = None);
+
   /// Reads the index block, which contains global tables.
   ///
   /// Returns false if there was an error.
@@ -702,7 +730,7 @@ class ModuleFile
 
   virtual
   Optional<TinyPtrVector<ValueDecl *>>
-  loadNamedMembers(const Decl *D, DeclName N,
+  loadNamedMembers(const IterableDeclContext *IDC, DeclName N,
                    uint64_t contextData) override;
 
   virtual void

include/swift/Serialization/ModuleFormat.h

@@ -54,9 +54,8 @@ const uint16_t VERSION_MAJOR = 0;
 /// in source control, you should also update the comment to briefly
 /// describe what change you made. The content of this comment isn't important;
 /// it just ensures a conflict if two people change the module format.
-const uint16_t VERSION_MINOR = 377; // Last change: SILFunctionType witness_method conformances
+const uint16_t VERSION_MINOR = 378; // Last change: DECL_MEMBER_NAMES
 
-using DeclID = PointerEmbeddedInt<unsigned, 31>;
 using DeclIDField = BCFixed<31>;
 
 // TypeID must be the same as DeclID because it is stored in the same way.
@@ -464,7 +463,12 @@ enum BlockID {
   /// The comment block, which contains documentation comments.
   ///
   /// \sa comment_block
-  COMMENT_BLOCK_ID
+  COMMENT_BLOCK_ID,
+
+  /// The declaration member-tables index block, a sub-blocb of the index block.
+  ///
+  /// \sa decl_member_tables_block
+  DECL_MEMBER_TABLES_BLOCK_ID
 };
 
 /// The record types within the control block.
@@ -1499,7 +1503,7 @@ namespace index_block {
     TOP_LEVEL_DECLS,
     OPERATORS,
     EXTENSIONS,
-    CLASS_MEMBERS,
+    CLASS_MEMBERS_FOR_DYNAMIC_LOOKUP,
     OPERATOR_METHODS,
 
     /// The Objective-C method index, which contains a mapping from
@@ -1517,8 +1521,9 @@ namespace index_block {
 
     PRECEDENCE_GROUPS,
     NESTED_TYPE_DECLS,
+    DECL_MEMBER_NAMES,
 
-    LastRecordKind = PRECEDENCE_GROUPS,
+    LastRecordKind = DECL_MEMBER_NAMES,
   };
 
   constexpr const unsigned RecordIDFieldWidth = 5;
@@ -1560,12 +1565,32 @@ namespace index_block {
     BCBlob  // map from identifier strings to decl kinds / decl IDs
   >;
 
+  using DeclMemberNamesLayout = BCRecordLayout<
+    DECL_MEMBER_NAMES, // record ID
+    BCVBR<16>,  // table offset within the blob (see below)
+    BCBlob  // map from member DeclBaseNames to offsets of DECL_MEMBERS records
+  >;
+
   using EntryPointLayout = BCRecordLayout<
     ENTRY_POINT,
     DeclIDField  // the ID of the main class; 0 if there was a main source file
   >;
 }
 
+/// \sa DECL_MEMBER_TABLES_BLOCK_ID
+namespace decl_member_tables_block {
+  enum RecordKind {
+    DECL_MEMBERS = 1,
+  };
+
+  using DeclMembersLayout = BCRecordLayout<
+    DECL_MEMBERS, // record ID
+    BCVBR<16>,  // table offset within the blob (see below)
+    BCBlob  // maps from DeclIDs to DeclID vectors
+  >;
+
+}
+
 /// \sa COMMENT_BLOCK_ID
 namespace comment_block {
   enum RecordKind {

include/swift/Serialization/SerializationOptions.h

@@ -37,6 +37,7 @@ namespace swift {
 
     bool AutolinkForceLoad = false;
     bool EnableNestedTypeLookupTable = false;
+    bool EnableDeclMemberNamesTable = false;
     bool SerializeAllSIL = false;
     bool SerializeOptionsForDebugging = false;
     bool IsSIB = false;

lib/AST/DeclContext.cpp

@@ -867,18 +867,24 @@ unsigned DeclContext::printContext(raw_ostream &OS, unsigned indent) const {
   return Depth + 1;
 }
 
-ASTContext &IterableDeclContext::getASTContext() const {
+const Decl *
+IterableDeclContext::getDecl() const {
   switch (getIterableContextKind()) {
   case IterableDeclContextKind::NominalTypeDecl:
-    return cast<NominalTypeDecl>(this)->getASTContext();
+    return cast<NominalTypeDecl>(this);
+    break;
 
   case IterableDeclContextKind::ExtensionDecl:
-    return cast<ExtensionDecl>(this)->getASTContext();
+    return cast<ExtensionDecl>(this);
+    break;
   }
-
   llvm_unreachable("Unhandled IterableDeclContextKind in switch.");
 }
 
+ASTContext &IterableDeclContext::getASTContext() const {
+  return getDecl()->getASTContext();
+}
+
 DeclRange IterableDeclContext::getMembers() const {
   loadAllMembers();
 
@@ -963,17 +969,7 @@ void IterableDeclContext::loadAllMembers() const {
     /*lazyLoader=*/nullptr);
   FirstDeclAndLazyMembers.setInt(false);
 
-  const Decl *container = nullptr;
-  switch (getIterableContextKind()) {
-  case IterableDeclContextKind::NominalTypeDecl:
-    container = cast<NominalTypeDecl>(this);
-    break;
-
-  case IterableDeclContextKind::ExtensionDecl:
-    container = cast<ExtensionDecl>(this);
-    break;
-  }
-
+  const Decl *container = getDecl();
   contextInfo->loader->loadAllMembers(const_cast<Decl *>(container),
                                       contextInfo->memberData);
 
@@ -983,6 +979,14 @@ void IterableDeclContext::loadAllMembers() const {
     s->getFrontendCounters().NumUnloadedLazyIterableDeclContexts--;
 }
 
+bool IterableDeclContext::wasDeserialized() const {
+  const DeclContext *DC = cast<DeclContext>(getDecl());
+  if (auto F = dyn_cast<FileUnit>(DC->getModuleScopeContext())) {
+    return F->getKind() == FileUnitKind::SerializedAST;
+  }
+  return false;
+}
+
 bool IterableDeclContext::classof(const Decl *D) {
   switch (D->getKind()) {
 #define DECL(ID, PARENT)              case DeclKind::ID: return false;