Merge branch 'master' into remove-reflectionlegacy

Author: mikeash

docs/DriverInternals.rst

@@ -94,14 +94,13 @@ in the current build. This is covered by checking if the input has been
 modified since the last build; if it hasn't, we only need to recompile if
 something it depends on has changed.
 
-
-Execute: Running the Jobs in a Compilation using a TaskQueue
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Schedule: Ordering and skipping jobs by dependency analysis
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A Compilation's goal is to make sure every Job in its list of Jobs is handled.
 If a Job needs to be run, the Compilation attempts to *schedule* it. If the
 Job's dependencies have all been completed (or determined to be skippable), it
-is added to the TaskQueue; otherwise it is marked as *blocked.*
+is scheduled for execution; otherwise it is marked as *blocked.*
 
 To support Jobs compiling individual Swift files, which may or may not need to
 be run, the Compilation keeps track of a DependencyGraph. (If file A depends on
@@ -111,6 +110,22 @@ that were directly blocked on it, and check to see if any other Jobs now need
 to run based on the DependencyGraph. See the section on :doc:`DependencyAnalysis`
 for more information.
 
+Batch: Optionally combine similar jobs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Driver has an experimental "batch mode" that examines the set of scheduled
+jobs just prior to execution, looking for jobs that are identical to one another
+aside from the primary input file they are compiling in a module. If it finds
+such a set, it may replace the set with a single BatchJob, before handing it off
+to the TaskQueue; this helps minimize the overall number of frontend processes
+that run (and thus do potentially redundant work).
+
+Once any batching has taken place, the set of scheduled jobs (batched or
+otherwise) is transferred to the TaskQueue for execution.
+
+Execute: Running the Jobs in a Compilation using a TaskQueue
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
 The Compilation's TaskQueue controls the low-level aspects of managing
 subprocesses. Multiple Jobs may execute simultaneously, but communication with
 the parent process (the driver) is handled on a single thread. The level of

include/swift/Basic/Statistic.h

@@ -15,8 +15,7 @@
 
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Statistic.h"
-#include "swift/AST/Identifier.h"
-#include "swift/Basic/SourceLoc.h"
+#include "swift/Basic/LLVM.h"
 #include "swift/Basic/Timer.h"
 
 #define SWIFT_FUNC_STAT                                                 \
@@ -56,35 +55,29 @@ namespace clang {
 namespace swift {
 
 class Decl;
+class ProtocolConformance;
 class Expr;
 class SILFunction;
+class FrontendStatsTracer;
+class SourceManager;
 
 class UnifiedStatsReporter {
 
 public:
   struct AlwaysOnDriverCounters
   {
-#define DRIVER_STATISTIC(ID) size_t ID;
+#define DRIVER_STATISTIC(ID) int64_t ID;
 #include "Statistics.def"
 #undef DRIVER_STATISTIC
   };
 
   struct AlwaysOnFrontendCounters
   {
-#define FRONTEND_STATISTIC(NAME, ID) size_t ID;
+#define FRONTEND_STATISTIC(NAME, ID) int64_t ID;
 #include "Statistics.def"
 #undef FRONTEND_STATISTIC
   };
 
-  struct AlwaysOnFrontendRecursiveSharedTimers {
-    AlwaysOnFrontendRecursiveSharedTimers();
-#define FRONTEND_RECURSIVE_SHARED_TIMER(ID) RecursiveSharedTimer ID;
-#include "Statistics.def"
-#undef FRONTEND_RECURSIVE_SHARED_TIMER
-
-    int dummyInstanceVariableToGetConstructorToParse;
-  };
-
   // To trace an entity, you have to provide a TraceFormatter for it. This is a
   // separate type since we do not have retroactive conformances in C++, and it
   // is a type that takes void* arguments since we do not have existentials
@@ -98,63 +91,67 @@ class UnifiedStatsReporter {
     virtual ~TraceFormatter();
   };
 
-  struct FrontendStatsTracer
-  {
-    UnifiedStatsReporter *Reporter;
-    llvm::TimeRecord SavedTime;
-    StringRef EventName;
-    const void *Entity;
-    const TraceFormatter *Formatter;
-    FrontendStatsTracer(StringRef EventName,
-                        const void *Entity,
-                        const TraceFormatter *Formatter,
-                        UnifiedStatsReporter *Reporter);
-    FrontendStatsTracer();
-    FrontendStatsTracer(FrontendStatsTracer&& other);
-    FrontendStatsTracer& operator=(FrontendStatsTracer&&);
-    ~FrontendStatsTracer();
-    FrontendStatsTracer(const FrontendStatsTracer&) = delete;
-    FrontendStatsTracer& operator=(const FrontendStatsTracer&) = delete;
-  };
-
   struct FrontendStatsEvent
   {
     uint64_t TimeUSec;
     uint64_t LiveUSec;
     bool IsEntry;
     StringRef EventName;
     StringRef CounterName;
-    size_t CounterDelta;
-    size_t CounterValue;
+    int64_t CounterDelta;
+    int64_t CounterValue;
     const void *Entity;
     const TraceFormatter *Formatter;
   };
 
+  // We only write fine-grained trace entries when the user passed
+  // -trace-stats-events, but we recycle the same FrontendStatsTracers to give
+  // us some free recursion-save phase timings whenever -trace-stats-dir is
+  // active at all. Reduces redundant machinery.
+  class RecursionSafeTimers;
+
+  // We also keep a few banks of optional hierarchical profilers for times and
+  // statistics, activated with -profile-stats-events and
+  // -profile-stats-entities, which are part way between the detail level of the
+  // aggregate statistic JSON files and the fine-grained CSV traces. Naturally
+  // these are written in yet a different file format: the input format for
+  // flamegraphs.
+  struct StatsProfilers;
+
 private:
   bool currentProcessExitStatusSet;
   int currentProcessExitStatus;
   SmallString<128> StatsFilename;
   SmallString<128> TraceFilename;
+  SmallString<128> ProfileDirname;
   llvm::TimeRecord StartedTime;
+
+  // This is unique_ptr because NamedRegionTimer is non-copy-constructable.
   std::unique_ptr<llvm::NamedRegionTimer> Timer;
+
   SourceManager *SourceMgr;
   clang::SourceManager *ClangSourceMgr;
-  std::unique_ptr<AlwaysOnDriverCounters> DriverCounters;
-  std::unique_ptr<AlwaysOnFrontendCounters> FrontendCounters;
-  std::unique_ptr<AlwaysOnFrontendCounters> LastTracedFrontendCounters;
-  std::vector<FrontendStatsEvent> FrontendStatsEvents;
-  std::unique_ptr<AlwaysOnFrontendRecursiveSharedTimers>
-      FrontendRecursiveSharedTimers;
+  Optional<AlwaysOnDriverCounters> DriverCounters;
+  Optional<AlwaysOnFrontendCounters> FrontendCounters;
+  Optional<AlwaysOnFrontendCounters> LastTracedFrontendCounters;
+  Optional<std::vector<FrontendStatsEvent>> FrontendStatsEvents;
+
+  // These are unique_ptr so we can use incomplete types here.
+  std::unique_ptr<RecursionSafeTimers> RecursiveTimers;
+  std::unique_ptr<StatsProfilers> EventProfilers;
+  std::unique_ptr<StatsProfilers> EntityProfilers;
 
   void publishAlwaysOnStatsToLLVM();
-  void printAlwaysOnStatsAndTimers(llvm::raw_ostream &OS);
+  void printAlwaysOnStatsAndTimers(raw_ostream &OS);
 
   UnifiedStatsReporter(StringRef ProgramName,
                        StringRef AuxName,
                        StringRef Directory,
                        SourceManager *SM,
                        clang::SourceManager *CSM,
-                       bool TraceEvents);
+                       bool TraceEvents,
+                       bool ProfileEvents,
+                       bool ProfileEntities);
 public:
   UnifiedStatsReporter(StringRef ProgramName,
                        StringRef ModuleName,
@@ -165,23 +162,83 @@ class UnifiedStatsReporter {
                        StringRef Directory,
                        SourceManager *SM=nullptr,
                        clang::SourceManager *CSM=nullptr,
-                       bool TraceEvents=false);
+                       bool TraceEvents=false,
+                       bool ProfileEvents=false,
+                       bool ProfileEntities=false);
   ~UnifiedStatsReporter();
 
   AlwaysOnDriverCounters &getDriverCounters();
   AlwaysOnFrontendCounters &getFrontendCounters();
-  AlwaysOnFrontendRecursiveSharedTimers &getFrontendRecursiveSharedTimers();
   void noteCurrentProcessExitStatus(int);
-  // We declare 4 explicit overloads here, but the _definitions_ live in the
-  // upper-level files (in libswiftAST or libswiftSIL) that provide the types
-  // being traced. If you want to trace those types, it's assumed you're linking
-  // with the object files that define the tracer.
-  FrontendStatsTracer getStatsTracer(StringRef EventName, const Decl *D);
-  FrontendStatsTracer getStatsTracer(StringRef EventName, const clang::Decl*D);
-  FrontendStatsTracer getStatsTracer(StringRef EventName, const Expr *E);
-  FrontendStatsTracer getStatsTracer(StringRef EventName, const SILFunction *F);
   void saveAnyFrontendStatsEvents(FrontendStatsTracer const &T, bool IsEntry);
 };
 
+// This is a non-nested type just to make it less work to write at call sites.
+class FrontendStatsTracer
+{
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,
+                      StringRef EventName,
+                      const void *Entity,
+                      const UnifiedStatsReporter::TraceFormatter *Formatter);
+
+  // In the general case we do not know how to format an entity for tracing.
+  template<typename T> static
+  const UnifiedStatsReporter::TraceFormatter *getTraceFormatter() {
+    return nullptr;
+  }
+
+public:
+  UnifiedStatsReporter *Reporter;
+  llvm::TimeRecord SavedTime;
+  StringRef EventName;
+  const void *Entity;
+  const UnifiedStatsReporter::TraceFormatter *Formatter;
+  FrontendStatsTracer();
+  FrontendStatsTracer(FrontendStatsTracer&& other);
+  FrontendStatsTracer& operator=(FrontendStatsTracer&&);
+  ~FrontendStatsTracer();
+  FrontendStatsTracer(const FrontendStatsTracer&) = delete;
+  FrontendStatsTracer& operator=(const FrontendStatsTracer&) = delete;
+
+  /// These are the convenience constructors you want to be calling throughout
+  /// the compiler: they select an appropriate trace formatter for the provided
+  /// entity type, and produce a tracer that's either active or inert depending
+  /// on whether the provided \p Reporter is null (nullptr means "tracing is
+  /// disabled").
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName);
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName,
+                      const Decl *D);
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName,
+                      const ProtocolConformance *P);
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName,
+                      const clang::Decl *D);
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName,
+                      const Expr *E);
+  FrontendStatsTracer(UnifiedStatsReporter *Reporter,  StringRef EventName,
+                      const SILFunction *F);
+};
+
+// In particular cases, we do know how to format traced entities: we declare
+// explicit specializations of getTraceFormatter() here, matching the overloaded
+// constructors of FrontendStatsTracer above, where the _definitions_ live in
+// the upper-level files (in libswiftAST or libswiftSIL), and provide tracing
+// for those entity types. If you want to trace those types, it's assumed you're
+// linking with the object files that define the tracer.
+
+template<> const UnifiedStatsReporter::TraceFormatter*
+FrontendStatsTracer::getTraceFormatter<const Decl *>();
+
+template<> const UnifiedStatsReporter::TraceFormatter*
+FrontendStatsTracer::getTraceFormatter<const ProtocolConformance *>();
+
+template<> const UnifiedStatsReporter::TraceFormatter*
+FrontendStatsTracer::getTraceFormatter<const clang::Decl *>();
+
+template<> const UnifiedStatsReporter::TraceFormatter*
+FrontendStatsTracer::getTraceFormatter<const Expr *>();
+
+template<> const UnifiedStatsReporter::TraceFormatter*
+FrontendStatsTracer::getTraceFormatter<const SILFunction *>();
+
 }
 #endif // SWIFT_BASIC_STATISTIC_H

include/swift/Basic/Statistics.def

@@ -19,9 +19,6 @@
 //   - Subsystem is a token to be stringified as a name prefix
 //   - Id is an identifier suitable for use in C++
 //
-// FRONTEND_RECURSIVE_SHARED_TIMER(Id)
-//   - Id is an identifier suitable for use in C++
-//
 //===----------------------------------------------------------------------===//
 
 /// Driver statistics are collected for driver processes
@@ -219,13 +216,3 @@ FRONTEND_STATISTIC(IRModule, NumIRInsts)
 /// the .o file you find on disk after the frontend exits.
 FRONTEND_STATISTIC(LLVM, NumLLVMBytesOutput)
 #endif
-
-/// Frontend timers for recursive routines
-#ifdef FRONTEND_RECURSIVE_SHARED_TIMER
-
-/// Time spent in  NominalTypeDecl::lookupDirect.
-FRONTEND_RECURSIVE_SHARED_TIMER(NominalTypeDecl__lookupDirect)
-
-/// Time spent in ClangImporter::Implementation::loadAllMembers.
-FRONTEND_RECURSIVE_SHARED_TIMER(ClangImporter__Implementation__loadAllMembers)
-#endif

include/swift/Basic/Timer.h

@@ -45,66 +45,6 @@ namespace swift {
       CompilationTimersEnabled = State::Enabled;
     }
   };
-
-  /// A SharedTimer for recursive routines.
-  /// void example() {
-  ///  RecursiveSharedTimer::Guard guard; // MUST BE AT TOP SCOPE of function to
-  ///  work right! if (auto s = getASTContext().Stats) {
-  ///    guard =
-  ///    ctx.Stats->getFrontendRecursiveSharedTimers().NominalTypeDecl__lookupDirect.getGuard();
-  //  }
-  ///   ...
-  /// }
-
-  class RecursiveSharedTimer {
-  private:
-    int recursionCount = 0;
-    const StringRef name;
-    llvm::Optional<SharedTimer> timer;
-
-    void enterRecursiveFunction() {
-      assert(recursionCount >= 0  &&  "too many exits");
-      if (recursionCount++ == 0)
-        timer.emplace(name);
-    }
-    void exitRecursiveFunction() {
-      assert(recursionCount > 0  &&  "too many exits");
-      if (--recursionCount == 0)
-        timer.reset();
-    }
-
-  public:
-    RecursiveSharedTimer(StringRef name) : name(name) {}
-
-    struct Guard {
-      RecursiveSharedTimer *recursiveTimerOrNull;
-
-      Guard(RecursiveSharedTimer *rst) : recursiveTimerOrNull(rst) {
-        if (recursiveTimerOrNull)
-          recursiveTimerOrNull->enterRecursiveFunction();
-      }
-      ~Guard() {
-        if (recursiveTimerOrNull)
-          recursiveTimerOrNull->exitRecursiveFunction();
-      }
-
-      // All this stuff is to do an RAII object that be moved.
-      Guard() : recursiveTimerOrNull(nullptr) {}
-      Guard(Guard &&other) {
-        recursiveTimerOrNull = other.recursiveTimerOrNull;
-        other.recursiveTimerOrNull = nullptr;
-      }
-      Guard &operator=(Guard &&other) {
-        recursiveTimerOrNull = other.recursiveTimerOrNull;
-        other.recursiveTimerOrNull = nullptr;
-        return *this;
-      }
-      Guard(const Guard &) = delete;
-      Guard &operator=(const Guard &) = delete;
-    };
-
-    Guard getGuard() { return Guard(this); }
-  };
 } // end namespace swift
 
 #endif // SWIFT_BASIC_TIMER_H

include/swift/Frontend/FrontendOptions.h

@@ -164,6 +164,13 @@ class FrontendOptions {
   /// Trace changes to stats to files in StatsOutputDir.
   bool TraceStats = false;
 
+  /// Profile changes to stats to files in StatsOutputDir.
+  bool ProfileEvents = false;
+
+  /// Profile changes to stats to files in StatsOutputDir, grouped by source
+  /// entity.
+  bool ProfileEntities = false;
+
   /// If true, serialization encodes an extra lookup table for use in module-
   /// merging when emitting partial modules (the per-file modules in a non-WMO
   /// build).