Merge pull request #12429 from graydon/named-lazy-member-loading

Named lazy member loading 1/N

Author: graydon

include/swift/AST/LazyResolver.h

@@ -218,12 +218,20 @@ class alignas(void*) LazyMemberLoader {
 public:
   virtual ~LazyMemberLoader() = default;
 
-  /// Populates the given vector with all member decls for \p D.
+  /// Populates a given decl \p D with all of its members.
   ///
   /// The implementation should add the members to D.
   virtual void
   loadAllMembers(Decl *D, uint64_t contextData) = 0;
 
+  /// Populates a vector with all members of \p D that have DeclName
+  /// matching \p N.
+  ///
+  /// Returns None if an error occurred \em or named member-lookup
+  /// was otherwise unsupported in this implementation or Decl.
+  virtual Optional<TinyPtrVector<ValueDecl *>>
+  loadNamedMembers(const Decl *D, DeclName N, uint64_t contextData) = 0;
+
   /// Populates the given vector with all conformances for \p D.
   ///
   /// The implementation should \em not call setConformances on \p D.

include/swift/Basic/LangOptions.h

@@ -153,6 +153,9 @@ namespace swift {
     /// \brief Enable the iterative type checker.
     bool IterativeTypeChecker = false;
 
+    /// \brief Enable named lazy member loading.
+    bool NamedLazyMemberLoading = false;
+
     /// Debug the generic signatures computed by the generic signature builder.
     bool DebugGenericSignatures = false;
 

include/swift/Basic/Statistics.def

@@ -161,6 +161,12 @@ FRONTEND_STATISTIC(Sema, NumLazyIterableDeclContexts)
 /// Number of direct member-name lookups performed on nominal types.
 FRONTEND_STATISTIC(Sema, NominalTypeLookupDirectCount)
 
+/// Number of member-name lookups that avoided loading all members.
+FRONTEND_STATISTIC(Sema, NamedLazyMemberLoadSuccessCount)
+
+/// Number of member-name lookups that wound up loading all members.
+FRONTEND_STATISTIC(Sema, NamedLazyMemberLoadFailureCount)
+
 /// Number of types deserialized.
 FRONTEND_STATISTIC(Sema, NumTypesDeserialized)
 

include/swift/Option/FrontendOptions.td

@@ -156,6 +156,9 @@ def propagate_constraints : Flag<["-"], "propagate-constraints">,
 def iterative_type_checker : Flag<["-"], "iterative-type-checker">,
   HelpText<"Enable the iterative type checker">;
 
+def enable_named_lazy_member_loading : Flag<["-"], "enable-named-lazy-member-loading">,
+  HelpText<"Enable per-name lazy member loading">;
+
 def debug_generic_signatures : Flag<["-"], "debug-generic-signatures">,
   HelpText<"Debug generic signatures">;
 

include/swift/Serialization/ModuleFile.h

@@ -700,6 +700,11 @@ class ModuleFile
   virtual void loadAllMembers(Decl *D,
                               uint64_t contextData) override;
 
+  virtual
+  Optional<TinyPtrVector<ValueDecl *>>
+  loadNamedMembers(const Decl *D, DeclName N,
+                   uint64_t contextData) override;
+
   virtual void
   loadAllConformances(const Decl *D, uint64_t contextData,
                     SmallVectorImpl<ProtocolConformance*> &Conforms) override;

lib/AST/GenericSignatureBuilder.cpp

@@ -3371,6 +3371,10 @@ ConstraintResult GenericSignatureBuilder::expandConformanceRequirement(
     }
   }
 
+  // Remaining logic is not relevant in ObjC protocol cases.
+  if (proto->isObjC())
+    return ConstraintResult::Resolved;
+
   // Collect all of the inherited associated types and typealiases in the
   // inherited protocols (recursively).
   llvm::MapVector<DeclName, TinyPtrVector<TypeDecl *>> inheritedTypeDecls;

lib/AST/NameLookup.cpp

@@ -29,6 +29,10 @@
 #include "swift/Basic/STLExtras.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+#define DEBUG_TYPE "namelookup"
 
 using namespace swift;
 
@@ -1195,12 +1199,49 @@ TinyPtrVector<ValueDecl *> NominalTypeDecl::lookupDirect(
                                                   DeclName name,
                                                   bool ignoreNewExtensions) {
   RecursiveSharedTimer::Guard guard;
-  if (auto s = getASTContext().Stats) {
+  ASTContext &ctx = getASTContext();
+  if (auto s = ctx.Stats) {
     ++s->getFrontendCounters().NominalTypeLookupDirectCount;
     guard = s->getFrontendRecursiveSharedTimers()
                 .NominalTypeDecl__lookupDirect.getGuard();
   }
 
+  DEBUG(llvm::dbgs() << getNameStr() << ".lookupDirect(" << name << ")"
+        << ", lookupTable.getInt()=" << LookupTable.getInt()
+        << ", hasLazyMembers()=" << hasLazyMembers()
+        << "\n");
+
+  bool hasExtensionsToConsider = false;
+  if (!ignoreNewExtensions) {
+    auto E = getExtensions();
+    hasExtensionsToConsider = E.begin() != E.end();
+  }
+
+  if (!hasExtensionsToConsider &&
+      ctx.LangOpts.NamedLazyMemberLoading &&
+      !LookupTable.getInt() &&
+      hasLazyMembers()) {
+    // The lookup table (containing all loaded members) has not yet been built;
+    // we *might* be able to avoid loading all members, just focus on the ones
+    // matching the name we were asked for.
+    TinyPtrVector<ValueDecl *> results;
+    auto contextInfo =
+        ctx.getOrCreateLazyIterableContextData(this,
+                                               /*lazyLoader=*/nullptr);
+    if (auto results =
+        contextInfo->loader->loadNamedMembers(this, name,
+                                              contextInfo->memberData)) {
+      if (auto s = ctx.Stats) {
+        ++s->getFrontendCounters().NamedLazyMemberLoadSuccessCount;
+      }
+      return *results;
+    } else {
+      if (auto s = ctx.Stats) {
+        ++s->getFrontendCounters().NamedLazyMemberLoadFailureCount;
+      }
+    }
+  }
+
   (void)getMembers();
 
   // Make sure we have the complete list of members (in this nominal and in all

lib/ClangImporter/ClangImporter.cpp

@@ -3274,6 +3274,64 @@ void ClangImporter::Implementation::lookupAllObjCMembers(
   }
 }
 
+Optional<TinyPtrVector<ValueDecl *>>
+ClangImporter::Implementation::loadNamedMembers(
+    const Decl *D, DeclName N, uint64_t contextData) {
+
+  auto *DC = cast<DeclContext>(D);
+
+  auto *CD = D->getClangDecl();
+  assert(CD && "loadNamedMembers on a Decl without a clangDecl");
+
+  // There are 3 cases:
+  //
+  //  - The decl is from a bridging header, CMO is Some(nullptr)
+  //    which denotes the __ObjC Swift module and its associated
+  //    BridgingHeaderLookupTable.
+  //
+  //  - The decl is from a clang module, CMO is Some(M) for non-null
+  //    M and we can use the table for that module.
+  //
+  //  - The decl is a forward declaration, CMO is None, which should
+  //    never be the case if we got here (someone is asking for members).
+  //
+  // findLookupTable, below, handles the first two cases; we assert on the
+  // third.
+
+  auto CMO = getClangSubmoduleForDecl(CD);
+  assert(CMO && "loadNamedMembers on a forward-declared Decl");
+
+  auto table = findLookupTable(*CMO);
+  assert(table && "clang module without lookup table");
+
+  clang::ASTContext &clangCtx = getClangASTContext();
+
+  TinyPtrVector<ValueDecl *> Members;
+  if (auto *CPD = dyn_cast<clang::ObjCProtocolDecl>(CD)) {
+    for (auto entry : table->lookup(SerializedSwiftName(N.getBaseName()), CPD)) {
+      if (!entry.is<clang::NamedDecl *>()) continue;
+      auto member = entry.get<clang::NamedDecl *>();
+      if (!isVisibleClangEntry(clangCtx, member)) continue;
+      SmallVector<Decl*, 4> tmp;
+      insertMembersAndAlternates(member, tmp);
+      for (auto *TD : tmp) {
+        if (auto *V = dyn_cast<ValueDecl>(TD)) {
+          // Skip ValueDecls if they import into different DeclContexts
+          // or under different names than the one we asked about.
+          if (V->getDeclContext() == DC &&
+              V->getFullName().matchesRef(N)) {
+            Members.push_back(V);
+          }
+        }
+      }
+    }
+    return Members;
+  }
+
+  return None;
+}
+
+
 EffectiveClangContext ClangImporter::Implementation::getEffectiveClangContext(
     const NominalTypeDecl *nominal) {
   // If we have a Clang declaration, look at it to determine the

lib/ClangImporter/ImporterImpl.h

@@ -1122,6 +1122,9 @@ class LLVM_LIBRARY_VISIBILITY ClangImporter::Implementation
   virtual void
   loadAllMembers(Decl *D, uint64_t unused) override;
 
+  virtual Optional<TinyPtrVector<ValueDecl *>>
+  loadNamedMembers(const Decl *D, DeclName N, uint64_t contextData) override;
+
 private:
   void
   loadAllMembersOfObjcContainer(Decl *D,

lib/Frontend/CompilerInvocation.cpp

@@ -845,6 +845,7 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   Opts.DebugConstraintSolver |= Args.hasArg(OPT_debug_constraints);
   Opts.EnableConstraintPropagation |= Args.hasArg(OPT_propagate_constraints);
   Opts.IterativeTypeChecker |= Args.hasArg(OPT_iterative_type_checker);
+  Opts.NamedLazyMemberLoading |= Args.hasArg(OPT_enable_named_lazy_member_loading);
   Opts.DebugGenericSignatures |= Args.hasArg(OPT_debug_generic_signatures);
 
   Opts.DebuggerSupport |= Args.hasArg(OPT_debugger_support);

lib/Sema/TypeCheckNameLookup.cpp

@@ -145,8 +145,14 @@ namespace {
       // pull out the superclass instead, and use that below.
       if (foundInType->isExistentialType()) {
         auto layout = foundInType->getExistentialLayout();
-        if (layout.superclass)
+        if (layout.superclass) {
           conformingType = layout.superclass;
+        } else {
+          // Non-subclass existential: don't need to look for further
+          // conformance or witness.
+          addResult(found);
+          return;
+        }
       }
 
       // Dig out the protocol conformance.

lib/Serialization/Deserialization.cpp

@@ -4678,6 +4678,13 @@ void ModuleFile::loadAllMembers(Decl *container, uint64_t contextData) {
   }
 }
 
+Optional<TinyPtrVector<ValueDecl *>>
+ModuleFile::loadNamedMembers(const Decl *D, DeclName N,
+                             uint64_t contextData) {
+  // Not presently supported.
+  return None;
+}
+
 void
 ModuleFile::loadAllConformances(const Decl *D, uint64_t contextData,
                           SmallVectorImpl<ProtocolConformance*> &conformances) {

test/NameBinding/Inputs/NamedLazyMembers/NamedLazyMembers.h

@@ -0,0 +1,34 @@
+#import <Foundation/Foundation.h>
+
+@protocol Doer
+
+- (void)doSomeWork;
+- (void)doSomeWorkWithSpeed:(int)s;
+- (void)doSomeWorkWithSpeed:(int)s thoroughness:(int)t
+  NS_SWIFT_NAME(doVeryImportantWork(speed:thoroughness:));
+- (void)doSomeWorkWithSpeed:(int)s alacrity:(int)a
+  NS_SWIFT_NAME(doSomeWorkWithSpeed(speed:levelOfAlacrity:));
+
+// These we are generally trying to not-import, via laziness.
+- (void)goForWalk;
+- (void)takeNap;
+- (void)eatMeal;
+- (void)tidyHome;
+- (void)callFamily;
+- (void)singSong;
+- (void)readBook;
+- (void)attendLecture;
+- (void)writeLetter;
+
+@end
+
+
+// Don't conform to the protocol; that loads all protocol members.
+@interface SimpleDoer
+
+// These are names we're hoping don't interfere with Doer, above.
++ (SimpleDoer*)Doer;
++ (SimpleDoer*)DoerOfNoWork;
+
+@end
+

test/NameBinding/Inputs/NamedLazyMembers/module.modulemap

@@ -0,0 +1,4 @@
+module NamedLazyMembers {
+  header "NamedLazyMembers.h"
+  export *
+}

test/NameBinding/named_lazy_member_loading_objc_protocol.swift

@@ -0,0 +1,26 @@
+// REQUIRES: objc_interop
+// REQUIRES: OS=macosx
+// RUN: rm -rf %t && mkdir -p %t/stats-pre && mkdir -p %t/stats-post
+//
+// Prime module cache
+// RUN: %target-swift-frontend -typecheck -I %S/Inputs/NamedLazyMembers -typecheck %s
+//
+// Check that without lazy named member loading, we're importing too much.
+// RUN: %target-swift-frontend -typecheck -I %S/Inputs/NamedLazyMembers -typecheck -stats-output-dir %t/stats-pre %s
+// RUN: %utils/process-stats-dir.py --evaluate 'NumTotalClangImportedEntities > 20' %t/stats-pre
+//
+// Check that with lazy named member loading, we're importing less.
+// RUN: %target-swift-frontend -typecheck -I %S/Inputs/NamedLazyMembers -enable-named-lazy-member-loading -stats-output-dir %t/stats-post %s
+// RUN: %utils/process-stats-dir.py --evaluate 'NumTotalClangImportedEntities < 15' %t/stats-post
+
+import NamedLazyMembers
+
+public func foo(d: Doer) {
+  d.doSomeWork()
+  d.doSomeWork(withSpeed:10)
+  d.doVeryImportantWork(speed:10, thoroughness:12)
+  d.doSomeWorkWithSpeed(speed:10, levelOfAlacrity:12)
+
+  let _ = SimpleDoer()
+  let _ = SimpleDoer.ofNoWork()
+}

validation-test/compiler_crashers/28855-swift-genericsignaturebuilder-addrequirement-swift-requirementrepr-const-swift-g.swift renamed to validation-test/compiler_crashers_fixed/28855-swift-genericsignaturebuilder-addrequirement-swift-requirementrepr-const-swift-g.swift

@@ -5,5 +5,5 @@
 // See https://swift.org/LICENSE.txt for license information
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 
-// RUN: not --crash %target-swift-frontend %s -emit-ir
+// RUN: not %target-swift-frontend %s -emit-ir
 @objc protocol P{typealias e where f=a