Teach the Evaluator about Incremental Dependencies

CodaFi · CodaFi · commit 3f8f3a89cb1e · 2020-03-31T15:40:04.000-07:00
Formalize DependencyScope, DependencySource, and the incremental dependency stack.

Also specialize SimpleRequest to formalize dependency sources and dependency sinks. This allows the evaluator's internal entrypoints to specalize away the incremental dependency tracking infrastructure if a request is not actually dependency-relevant.
diff --git a/include/swift/AST/Evaluator.h b/include/swift/AST/Evaluator.h
@@ -19,6 +19,7 @@
 #define SWIFT_AST_EVALUATOR_H
 
 #include "swift/AST/AnyRequest.h"
+#include "swift/AST/EvaluatorDependencies.h"
 #include "swift/Basic/AnyValue.h"
 #include "swift/Basic/Debug.h"
 #include "swift/Basic/Defer.h"
@@ -47,6 +48,12 @@ class DiagnosticEngine;
 class Evaluator;
 class UnifiedStatsReporter;
 
+namespace detail {
+// Remove this when the compiler bumps to C++17.
+template <typename...>
+using void_t = void;
+}
+
 /// An "abstract" request function pointer, which is the storage type
 /// used for each of the
 using AbstractRequestFunction = void(void);
@@ -224,6 +231,38 @@ class Evaluator {
   /// so all clients must cope with cycles.
   llvm::DenseMap<AnyRequest, std::vector<AnyRequest>> dependencies;
 
+  /// A stack of dependency sources in the order they were evaluated.
+  llvm::SmallVector<evaluator::DependencySource, 8> dependencySources;
+
+  /// An RAII type that manages manipulating the evaluator's
+  /// dependency source stack. It is specialized to be zero-cost for
+  /// requests that are not dependency sources.
+  template <typename Request, typename = detail::void_t<>>
+  struct IncrementalDependencyStackRAII {
+    IncrementalDependencyStackRAII(Evaluator &E, const Request &Req) {}
+  };
+
+  template <typename Request>
+  struct IncrementalDependencyStackRAII<
+      Request, typename std::enable_if<Request::isDependencySource>::type> {
+    NullablePtr<Evaluator> Eval;
+    IncrementalDependencyStackRAII(Evaluator &E, const Request &Req) {
+      auto Source = Req.readDependencySource(E);
+      // If there is no source to introduce, bail. This can occur if
+      // a request originates in the context of a module.
+      if (!Source.getPointer()) {
+        return;
+      }
+      E.dependencySources.emplace_back(Source);
+      Eval = &E;
+    }
+
+    ~IncrementalDependencyStackRAII() {
+      if (Eval.isNonNull())
+        Eval.get()->dependencySources.pop_back();
+    }
+  };
+
   /// Retrieve the request function for the given zone and request IDs.
   AbstractRequestFunction *getAbstractRequestFunction(uint8_t zoneID,
                                                       uint8_t requestID) const;
@@ -264,6 +303,7 @@ class Evaluator {
            typename std::enable_if<Request::isEverCached>::type * = nullptr>
   llvm::Expected<typename Request::OutputType>
   operator()(const Request &request) {
+    IncrementalDependencyStackRAII<Request> incDeps{*this, request};
     // The request can be cached, but check a predicate to determine
     // whether this particular instance is cached. This allows more
     // fine-grained control over which instances get cache.
@@ -279,6 +319,7 @@ class Evaluator {
            typename std::enable_if<!Request::isEverCached>::type * = nullptr>
   llvm::Expected<typename Request::OutputType>
   operator()(const Request &request) {
+    IncrementalDependencyStackRAII<Request> incDeps{*this, request};
     return getResultUncached(request);
   }
 
@@ -366,7 +407,9 @@ class Evaluator {
     FrontendStatsTracer statsTracer = make_tracer(stats, request);
     if (stats) reportEvaluatedRequest(*stats, request);
 
-    return getRequestFunction<Request>()(request, *this);
+    auto &&r = getRequestFunction<Request>()(request, *this);
+    reportEvaluatedResult<Request>(request, r);
+    return std::move(r);
   }
 
   /// Get the result of a request, consulting an external cache
@@ -377,8 +420,10 @@ class Evaluator {
   llvm::Expected<typename Request::OutputType>
   getResultCached(const Request &request) {
     // If there is a cached result, return it.
-    if (auto cached = request.getCachedResult())
+    if (auto cached = request.getCachedResult()) {
+      reportEvaluatedResult<Request>(request, *cached);
       return *cached;
+    }
 
     // Compute the result.
     auto result = getResultUncached(request);
@@ -403,7 +448,9 @@ class Evaluator {
     // If we already have an entry for this request in the cache, return it.
     auto known = cache.find_as(request);
     if (known != cache.end()) {
-      return known->second.template castTo<typename Request::OutputType>();
+      auto r = known->second.template castTo<typename Request::OutputType>();
+      reportEvaluatedResult<Request>(request, r);
+      return r;
     }
 
     // Compute the result.
@@ -416,6 +463,53 @@ class Evaluator {
     return result;
   }
 
+private:
+  template <typename Request,
+            typename std::enable_if<!Request::isDependencySink>::type * = nullptr>
+  void reportEvaluatedResult(const Request &r,
+                             const typename Request::OutputType &o) const {}
+
+  template <typename Request,
+            typename std::enable_if<Request::isDependencySink>::type * = nullptr>
+  void reportEvaluatedResult(const Request &r,
+                             const typename Request::OutputType &o) {
+    r.writeDependencySink(*this, o);
+  }
+
+public:
+  /// Returns \c true if the scope of the current active source cascades.
+  ///
+  /// If there is no active scope, the result always cascades.
+  bool isActiveSourceCascading() const {
+    return getActiveSourceScope() == evaluator::DependencyScope::Cascading;
+  }
+
+  /// Returns the scope of the current active scope.
+  ///
+  /// If there is no active scope, the result always cascades.
+  evaluator::DependencyScope getActiveSourceScope() const {
+    if (dependencySources.empty()) {
+      return evaluator::DependencyScope::Cascading;
+    }
+    return dependencySources.back().getInt();
+  }
+
+  /// Returns the active dependency's source file, or \c nullptr if no
+  /// dependency source is active.
+  SourceFile *getActiveDependencySource() const {
+    if (dependencySources.empty())
+      return nullptr;
+    return dependencySources.back().getPointer();
+  }
+
+  /// If there is an active dependency source, returns its
+  /// \c ReferencedNameTracker. Else, returns \c nullptr.
+  ReferencedNameTracker *getActiveDependencyTracker() const {
+    if (auto *source = getActiveDependencySource())
+      return source->getRequestBasedReferencedNameTracker();
+    return nullptr;
+  }
+
 public:
   /// Print the dependencies of the given request as a tree.
   ///
diff --git a/include/swift/AST/EvaluatorDependencies.h b/include/swift/AST/EvaluatorDependencies.h
@@ -0,0 +1,88 @@
+//===--- EvaluatorDependencies.h - Auto-Incremental Dependencies -*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines data structures to support the request evaluator's
+// automatic incremental dependency tracking functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_EVALUATOR_DEPENDENCIES_H
+#define SWIFT_AST_EVALUATOR_DEPENDENCIES_H
+
+#include "swift/AST/AttrKind.h"
+#include "swift/AST/SourceFile.h"
+#include "llvm/ADT/PointerIntPair.h"
+
+namespace swift {
+
+namespace evaluator {
+
+/// The "scope" of a dependency edge tracked by the evaluator.
+///
+/// Dependency scopes come in two flavors: cascading and private. A private
+/// edge captures dependencies discovered in contexts that are not visible to
+/// to other files. For example, a conformance to a private protocol, or the use
+/// of any names inside of a function body. A cascading edge, by contrast,
+/// captures dependencies discovered in the remaining visible contexts. These
+/// are types with at least \c internal visibility, names defined or used
+/// outside of function bodies with at least \c internal visibility, etc. A
+/// dependency that has cascading scope is so-named because upon traversing the
+/// edge, a reader such as the driver should continue transitively evaluating
+/// further dependency edges.
+///
+/// A cascading edge is always conservatively correct to produce, but it comes
+/// at the cost of increased resources spent (and possibly even wasted!) during
+/// incremental compilation. A private edge, by contrast, is more efficient for
+/// incremental compilation but it is harder to safely use.
+///
+/// To ensure that these edges are registered consistently with the correct
+/// scopes, requests that act as the source of dependency edges are required
+/// to specify a \c DependencyScope under which all evaluated sub-requests will
+/// register their dependency edges. In this way, \c DependencyScope values
+/// form a stack-like structure and are pushed and popped by the evaluator
+/// during the course of request evaluation.
+///
+/// When determining the kind of scope a request should use, always err on the
+/// side of a cascading scope unless there is absolute proof any discovered
+/// dependencies will be private. Inner requests may also defensively choose to
+/// flip the dependency scope from private to cascading in the name of safety.
+enum class DependencyScope : bool {
+  Private = false,
+  Cascading = true,
+};
+
+/// Returns a \c DependencyScope appropriate for the given (formal) access level.
+inline DependencyScope getScopeForAccessLevel(AccessLevel l) {
+  switch (l) {
+  case AccessLevel::Private:
+  case AccessLevel::FilePrivate:
+    return DependencyScope::Private;
+  case AccessLevel::Internal:
+  case AccessLevel::Public:
+  case AccessLevel::Open:
+    return DependencyScope::Cascading;
+  }
+}
+
+// A \c DependencySource is a currently defined to be a parent source file and
+// an associated dependency scope.
+//
+// The \c SourceFile instance is an artifact of the current dependency system,
+// and should be scrapped if possible. It currently encodes the idea that
+// edges in the incremental dependency graph invalidate entire files instead
+// of individual contexts.
+using DependencySource = llvm::PointerIntPair<SourceFile *, 1, DependencyScope>;
+} // end namespace evaluator
+
+} // end namespace swift
+
+#endif // SWIFT_AST_EVALUATOR_DEPENDENCIES_H
diff --git a/include/swift/AST/SimpleRequest.h b/include/swift/AST/SimpleRequest.h
@@ -34,15 +34,33 @@ class Evaluator;
 /// Describes how the result for a particular request will be cached.
 enum class CacheKind {
   /// The result for a particular request should never be cached.
-  Uncached,
+  Uncached = 1 << 0,
   /// The result for a particular request should be cached within the
   /// evaluator itself.
-  Cached,
+  Cached = 1 << 1,
   /// The result of a particular request will be cached via some separate
   /// mechanism, such as a mutable data structure.
-  SeparatelyCached,
+  SeparatelyCached = 1 << 2,
+  /// This request introduces the source component of a source-sink
+  /// incremental dependency pair and defines a new dependency scope.
+  ///
+  /// This bit is optional.  High-level requests
+  /// (e.g. \c TypeCheckSourceFileRequest) will require it.
+  DependencySource = 1 << 3,
+  /// This request introduces the sink component of a source-sink
+  /// incremental dependency pair and is a consumer of the current
+  /// dependency scope.
+  ///
+  /// This bit is optional. Name lookup requests
+  /// (e.g. \c DirectLookupRequest) will require it.
+  DependencySink = 1 << 4,
 };
 
+static constexpr inline CacheKind operator|(CacheKind lhs, CacheKind rhs) {
+  return CacheKind(static_cast<std::underlying_type<CacheKind>::type>(lhs) |
+                   static_cast<std::underlying_type<CacheKind>::type>(rhs));
+}
+
 /// -------------------------------------------------------------------------
 /// Extracting the source location "nearest" a request.
 /// -------------------------------------------------------------------------
@@ -123,6 +141,29 @@ namespace detail {
   }
 }
 
+namespace detail {
+constexpr bool cacheContains(CacheKind kind, CacheKind needle) {
+  using cache_t = std::underlying_type<CacheKind>::type;
+  return (static_cast<cache_t>(kind) & static_cast<cache_t>(needle))
+      == static_cast<cache_t>(needle);
+}
+constexpr bool isEverCached(CacheKind kind) {
+  return !cacheContains(kind, CacheKind::Uncached);
+}
+
+constexpr bool hasExternalCache(CacheKind kind) {
+  return cacheContains(kind, CacheKind::SeparatelyCached);
+}
+
+constexpr bool isDependencySource(CacheKind kind) {
+  return cacheContains(kind, CacheKind::DependencySource);
+}
+
+constexpr bool isDependencySink(CacheKind kind) {
+  return cacheContains(kind, CacheKind::DependencySink);
+}
+} // end namespace detail
+
 /// Extract the first, nearest source location from a tuple.
 template<typename First, typename ...Rest,
          typename = typename std::enable_if<
@@ -177,6 +218,24 @@ SourceLoc extractNearestSourceLoc(const std::tuple<First, Rest...> &value) {
 ///   Optional<Output> getCachedResult() const;
 ///   void cacheResult(Output value) const;
 /// \endcode
+///
+/// Incremental dependency tracking occurs automatically during
+/// request evaluation. To support that system, high-level requests that define
+/// dependency sources should override \c readDependencySource()
+/// and specify \c CacheKind::DependencySource in addition to one of
+/// the 3 caching kinds defined above.
+/// \code
+///   evaluator::DependencySource readDependencySource(Evaluator &) const;
+/// \endcode
+///
+/// Requests that define dependency sinks should instead override
+/// \c writeDependencySink() and use the given evaluator and request
+/// result to write an edge into the dependency tracker. In addition,
+/// \c CacheKind::DependencySource should be specified along with
+/// one of the 3 caching kinds defined above.
+/// \code
+///   void writeDependencySink(Evaluator &, Output value) const;
+/// \endcode
 template<typename Derived, typename Signature, CacheKind Caching>
 class SimpleRequest;
 
@@ -205,8 +264,13 @@ class SimpleRequest<Derived, Output(Inputs...), Caching> {
   const std::tuple<Inputs...> &getStorage() const { return storage; }
 
 public:
-  static const bool isEverCached = (Caching != CacheKind::Uncached);
-  static const bool hasExternalCache = (Caching == CacheKind::SeparatelyCached);
+  constexpr static bool isEverCached = detail::isEverCached(Caching);
+  constexpr static bool hasExternalCache = detail::hasExternalCache(Caching);
+
+public:
+  constexpr static bool isDependencySource = detail::isDependencySource(Caching);
+  constexpr static bool isDependencySink = detail::isDependencySink(Caching);
+  constexpr static bool reportsDependencyReads = isDependencySource || isDependencySink;
 
   using OutputType = Output;
   
diff --git a/include/swift/AST/SourceFile.h b/include/swift/AST/SourceFile.h
@@ -142,6 +142,7 @@ class SourceFile final : public FileUnit {
 
   /// If non-null, used to track name lookups that happen within this file.
   Optional<ReferencedNameTracker> ReferencedNames;
+  Optional<ReferencedNameTracker> RequestReferencedNames;
 
   /// The class in this file marked \@NS/UIApplicationMain.
   ClassDecl *MainClass = nullptr;
@@ -472,6 +473,13 @@ class SourceFile final : public FileUnit {
     return ReferencedNames ? ReferencedNames.getPointer() : nullptr;
   }
 
+  ReferencedNameTracker *getRequestBasedReferencedNameTracker() {
+    return RequestReferencedNames ? RequestReferencedNames.getPointer() : nullptr;
+  }
+  const ReferencedNameTracker *getRequestBasedReferencedNameTracker() const {
+    return RequestReferencedNames ? RequestReferencedNames.getPointer() : nullptr;
+  }
+
   void createReferencedNameTracker();
 
   /// The buffer ID for the file that was imported, or None if there
