Merge pull request #31837 from CodaFi/record-scratch

Naive Dependency Replay

Author: CodaFi

include/swift/AST/AnyRequest.h

@@ -66,6 +66,9 @@ struct AnyRequestVTable {
     static SourceLoc getNearestLoc(const void *ptr) {
       return static_cast<const Request *>(ptr)->getNearestLoc();
     }
+    static bool isCached(const void *ptr) {
+      return static_cast<const Request *>(ptr)->isCached();
+    }
   };
 
   const uint64_t typeID;
@@ -78,8 +81,29 @@ struct AnyRequestVTable {
   const std::function<void(const void *, DiagnosticEngine &)> diagnoseCycle;
   const std::function<void(const void *, DiagnosticEngine &)> noteCycleStep;
   const std::function<SourceLoc(const void *)> getNearestLoc;
+  const std::function<bool(const void *)> isCached;
 
-  template <typename Request>
+  template <typename Request,
+            typename std::enable_if<Request::isEverCached>::type * = nullptr>
+  static const AnyRequestVTable *get() {
+    static const AnyRequestVTable vtable = {
+        TypeID<Request>::value,
+        sizeof(Request),
+        &Impl<Request>::copy,
+        &Impl<Request>::getHash,
+        &Impl<Request>::deleter,
+        &Impl<Request>::isEqual,
+        &Impl<Request>::simpleDisplay,
+        &Impl<Request>::diagnoseCycle,
+        &Impl<Request>::noteCycleStep,
+        &Impl<Request>::getNearestLoc,
+        &Impl<Request>::isCached,
+    };
+    return &vtable;
+  }
+
+  template <typename Request,
+            typename std::enable_if<!Request::isEverCached>::type * = nullptr>
   static const AnyRequestVTable *get() {
     static const AnyRequestVTable vtable = {
         TypeID<Request>::value,
@@ -92,6 +116,7 @@ struct AnyRequestVTable {
         &Impl<Request>::diagnoseCycle,
         &Impl<Request>::noteCycleStep,
         &Impl<Request>::getNearestLoc,
+        [](auto){ return false; },
     };
     return &vtable;
   }
@@ -194,6 +219,10 @@ class AnyRequestBase {
     return getVTable()->getNearestLoc(getRawStorage());
   }
 
+  bool isCached() const {
+    return getVTable()->isCached(getRawStorage());
+  }
+
   /// Compare two instances for equality.
   friend bool operator==(const AnyRequestBase<Derived> &lhs,
                          const AnyRequestBase<Derived> &rhs) {

include/swift/AST/Evaluator.h

@@ -225,7 +225,7 @@ class Evaluator {
   /// so all clients must cope with cycles.
   llvm::DenseMap<AnyRequest, std::vector<AnyRequest>> dependencies;
 
-  evaluator::DependencyCollector collector;
+  evaluator::DependencyRecorder recorder;
 
   /// Retrieve the request function for the given zone and request IDs.
   AbstractRequestFunction *getAbstractRequestFunction(uint8_t zoneID,
@@ -268,8 +268,8 @@ class Evaluator {
            typename std::enable_if<Request::isEverCached>::type * = nullptr>
   llvm::Expected<typename Request::OutputType>
   operator()(const Request &request) {
-    evaluator::DependencyCollector::StackRAII<Request> incDeps{collector,
-                                                               request};
+    evaluator::DependencyRecorder::StackRAII<Request> incDeps{recorder,
+                                                              request};
     // The request can be cached, but check a predicate to determine
     // whether this particular instance is cached. This allows more
     // fine-grained control over which instances get cache.
@@ -285,8 +285,8 @@ class Evaluator {
            typename std::enable_if<!Request::isEverCached>::type * = nullptr>
   llvm::Expected<typename Request::OutputType>
   operator()(const Request &request) {
-    evaluator::DependencyCollector::StackRAII<Request> incDeps{collector,
-                                                               request};
+    evaluator::DependencyRecorder::StackRAII<Request> incDeps{recorder,
+                                                              request};
     return getResultUncached(request);
   }
 
@@ -431,19 +431,22 @@ class Evaluator {
   }
 
 private:
-  // Report the result of evaluating a request that is not a dependency sink -
-  // which is to say do nothing.
-  template <typename Request,
-            typename std::enable_if<!Request::isDependencySink>::type * = nullptr>
+  // Report the result of evaluating a request that is not a dependency sink.
+  template <typename Request, typename std::enable_if<
+                                  !Request::isDependencySink>::type * = nullptr>
   void reportEvaluatedResult(const Request &r,
-                             const typename Request::OutputType &o) {}
+                             const typename Request::OutputType &o) {
+    recorder.replay(ActiveRequest(r));
+  }
 
   // Report the result of evaluating a request that is a dependency sink.
   template <typename Request,
             typename std::enable_if<Request::isDependencySink>::type * = nullptr>
   void reportEvaluatedResult(const Request &r,
                              const typename Request::OutputType &o) {
-    r.writeDependencySink(collector, o);
+    return recorder.record(activeRequests, [&r, &o](auto &c) {
+      return r.writeDependencySink(c, o);
+    });
   }
 
 public:

include/swift/AST/EvaluatorDependencies.h

@@ -18,6 +18,7 @@
 #ifndef SWIFT_AST_EVALUATOR_DEPENDENCIES_H
 #define SWIFT_AST_EVALUATOR_DEPENDENCIES_H
 
+#include "swift/AST/AnyRequest.h"
 #include "swift/AST/AttrKind.h"
 #include "swift/AST/SourceFile.h"
 #include "llvm/ADT/PointerIntPair.h"
@@ -106,30 +107,78 @@ inline DependencyScope getScopeForAccessLevel(AccessLevel l) {
 // of individual contexts.
 using DependencySource = llvm::PointerIntPair<SourceFile *, 1, DependencyScope>;
 
-/// A \c DependencyCollector is an aggregator of named references discovered in a
-/// particular \c DependencyScope during the course of request evaluation.
+struct DependencyRecorder;
+
 struct DependencyCollector {
-private:
-  /// A stack of dependency sources in the order they were evaluated.
-  llvm::SmallVector<evaluator::DependencySource, 8> dependencySources;
+  struct Reference {
+  public:
+    enum class Kind {
+      Empty,
+      Tombstone,
+      UsedMember,
+      PotentialMember,
+      TopLevel,
+      Dynamic,
+    } kind;
 
-public:
-  enum class Mode {
-    // Enables the current "status quo" behavior of the dependency collector.
-    //
-    // By default, the dependency collector moves to register dependencies in
-    // the referenced name trackers at the top of the active dependency stack.
-    StatusQuo,
-    // Enables an experimental mode to only register private dependencies.
-    //
-    // This mode restricts the dependency collector to ignore changes of
-    // scope. This has practical effect of charging all unqualified lookups to
-    // the primary file being acted upon instead of to the destination file.
-    ExperimentalPrivateDependencies,
+    NominalTypeDecl *subject;
+    DeclBaseName name;
+
+  private:
+    Reference(Kind kind, NominalTypeDecl *subject, DeclBaseName name)
+        : kind(kind), subject(subject), name(name) {}
+
+  public:
+    static Reference empty() {
+      return {Kind::Empty, llvm::DenseMapInfo<NominalTypeDecl *>::getEmptyKey(),
+              llvm::DenseMapInfo<DeclBaseName>::getEmptyKey()};
+    }
+
+    static Reference tombstone() {
+      return {Kind::Empty,
+              llvm::DenseMapInfo<NominalTypeDecl *>::getTombstoneKey(),
+              llvm::DenseMapInfo<DeclBaseName>::getTombstoneKey()};
+    }
+
+  public:
+    static Reference usedMember(NominalTypeDecl *subject, DeclBaseName name) {
+      return {Kind::UsedMember, subject, name};
+    }
+
+    static Reference potentialMember(NominalTypeDecl *subject) {
+      return {Kind::PotentialMember, subject, DeclBaseName()};
+    }
+
+    static Reference topLevel(DeclBaseName name) {
+      return {Kind::TopLevel, nullptr, name};
+    }
+
+    static Reference dynamic(DeclBaseName name) {
+      return {Kind::Dynamic, nullptr, name};
+    }
+
+  public:
+    struct Info {
+      static inline Reference getEmptyKey() { return Reference::empty(); }
+      static inline Reference getTombstoneKey() {
+        return Reference::tombstone();
+      }
+      static inline unsigned getHashValue(const Reference &Val) {
+        return llvm::hash_combine(Val.kind, Val.subject,
+                                  Val.name.getAsOpaquePointer());
+      }
+      static bool isEqual(const Reference &LHS, const Reference &RHS) {
+        return LHS.kind == RHS.kind && LHS.subject == RHS.subject &&
+               LHS.name == RHS.name;
+      }
+    };
   };
-  Mode mode;
 
-  explicit DependencyCollector(Mode mode) : mode{mode} {};
+  DependencyRecorder &parent;
+  llvm::DenseSet<Reference, Reference::Info> scratch;
+
+public:
+  explicit DependencyCollector(DependencyRecorder &parent) : parent(parent) {}
 
 public:
   /// Registers a named reference from the current dependency scope to a member
@@ -168,6 +217,50 @@ struct DependencyCollector {
   /// a name that is found by \c AnyObject lookup.
   void addDynamicLookupName(DeclBaseName name);
 
+public:
+  const DependencyRecorder &getRecorder() const { return parent; }
+  bool empty() const { return scratch.empty(); }
+};
+
+/// A \c DependencyCollector is an aggregator of named references discovered in a
+/// particular \c DependencyScope during the course of request evaluation.
+struct DependencyRecorder {
+  friend DependencyCollector;
+private:
+  /// A stack of dependency sources in the order they were evaluated.
+  llvm::SmallVector<evaluator::DependencySource, 8> dependencySources;
+
+public:
+  enum class Mode {
+    // Enables the current "status quo" behavior of the dependency collector.
+    //
+    // By default, the dependency collector moves to register dependencies in
+    // the referenced name trackers at the top of the active dependency stack.
+    StatusQuo,
+    // Enables an experimental mode to only register private dependencies.
+    //
+    // This mode restricts the dependency collector to ignore changes of
+    // scope. This has practical effect of charging all unqualified lookups to
+    // the primary file being acted upon instead of to the destination file.
+    ExperimentalPrivateDependencies,
+  };
+  Mode mode;
+  llvm::DenseMap<AnyRequest, llvm::DenseSet<DependencyCollector::Reference,
+                                            DependencyCollector::Reference::Info>>
+      requestReferences;
+  bool isRecording;
+
+  explicit DependencyRecorder(Mode mode)
+      : mode{mode}, requestReferences{}, isRecording{false} {};
+
+private:
+  void realize(const DependencyCollector::Reference &ref);
+
+public:
+  void replay(const swift::ActiveRequest &req);
+  void record(const llvm::SetVector<swift::ActiveRequest> &stack,
+              llvm::function_ref<void(DependencyCollector &)> rec);
+
 public:
   /// Returns the scope of the current active scope.
   ///
@@ -197,14 +290,14 @@ struct DependencyCollector {
   /// dependency source stack. It is specialized to be zero-cost for
   /// requests that are not dependency sources.
   template <typename Request, typename = detail::void_t<>> struct StackRAII {
-    StackRAII(DependencyCollector &DC, const Request &Req) {}
+    StackRAII(DependencyRecorder &DR, const Request &Req) {}
   };
 
   template <typename Request>
   struct StackRAII<Request,
                    typename std::enable_if<Request::isDependencySource>::type> {
-    NullablePtr<DependencyCollector> Coll;
-    StackRAII(DependencyCollector &coll, const Request &Req) {
+    NullablePtr<DependencyRecorder> Coll;
+    StackRAII(DependencyRecorder &coll, const Request &Req) {
       auto Source = Req.readDependencySource(coll);
       // If there is no source to introduce, bail. This can occur if
       // a request originates in the context of a module.

include/swift/AST/IRGenRequests.h

@@ -200,7 +200,7 @@ class IRGenSourceFileRequest
 public:
   // Incremental dependencies.
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 class IRGenWholeModuleRequest

include/swift/AST/NameLookupRequests.h

@@ -189,7 +189,7 @@ class InheritedProtocolsRequest
 public:
   // Incremental dependencies
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &e) const;
+  readDependencySource(const evaluator::DependencyRecorder &e) const;
   void writeDependencySink(evaluator::DependencyCollector &tracker,
                            ArrayRef<ProtocolDecl *> result) const;
 };
@@ -330,7 +330,7 @@ class GetDestructorRequest
 public:
   // Incremental dependencies.
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 class GenericParamListRequest :
@@ -434,7 +434,7 @@ class UnqualifiedLookupRequest
 public:
   // Incremental dependencies
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
   void writeDependencySink(evaluator::DependencyCollector &tracker,
                            LookupResult res) const;
 };
@@ -506,7 +506,7 @@ class ModuleQualifiedLookupRequest
 public:
   // Incremental dependencies
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
   void writeDependencySink(evaluator::DependencyCollector &tracker,
                            QualifiedLookupResult lookupResult) const;
 };
@@ -533,7 +533,7 @@ class QualifiedLookupRequest
 public:
   // Incremental dependencies.
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 /// The input type for a direct lookup request.

include/swift/AST/ParseRequests.h

@@ -103,7 +103,7 @@ class ParseSourceFileRequest
 
 public:
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 void simple_display(llvm::raw_ostream &out,
@@ -125,7 +125,7 @@ class CodeCompletionSecondPassRequest
 
 public:
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 /// The zone number for the parser.

include/swift/AST/SILGenRequests.h

@@ -104,7 +104,7 @@ class SILGenerationRequest
 public:
   // Incremental dependencies.
   evaluator::DependencySource
-  readDependencySource(const evaluator::DependencyCollector &) const;
+  readDependencySource(const evaluator::DependencyRecorder &) const;
 };
 
 /// Parses a .sil file into a SILModule.

include/swift/AST/SimpleRequest.h

@@ -232,7 +232,7 @@ SourceLoc extractNearestSourceLoc(const std::tuple<First, Rest...> &value) {
 /// the 3 caching kinds defined above.
 /// \code
 ///   evaluator::DependencySource
-///   readDependencySource(const evaluator::DependencyCollector &) const;
+///   readDependencySource(const evaluator::DependencyRecorder &) const;
 /// \endcode
 ///
 /// Requests that define dependency sinks should instead override