[Stats] Add -stats-output-dir, wire UnifiedStatsReporter into place.

Author: graydon

include/swift/Driver/Compilation.h

@@ -21,6 +21,7 @@
 #include "swift/Driver/Util.h"
 #include "swift/Basic/ArrayRefView.h"
 #include "swift/Basic/LLVM.h"
+#include "swift/Basic/Statistic.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Chrono.h"
@@ -134,6 +135,9 @@ class Compilation {
   /// execute.
   bool ShowDriverTimeCompilation;
 
+  /// When non-null, record various high-level counters to this.
+  std::unique_ptr<UnifiedStatsReporter> Stats;
+
   /// When true, dumps information about why files are being scheduled to be
   /// rebuilt.
   bool ShowIncrementalBuildDecisions = false;
@@ -156,7 +160,8 @@ class Compilation {
               bool EnableIncrementalBuild = false,
               bool SkipTaskExecution = false,
               bool SaveTemps = false,
-              bool ShowDriverTimeCompilation = false);
+              bool ShowDriverTimeCompilation = false,
+              std::unique_ptr<UnifiedStatsReporter> Stats = nullptr);
   ~Compilation();
 
   ArrayRefView<std::unique_ptr<const Job>, const Job *, Compilation::unwrap>

include/swift/Driver/DependencyGraph.h

@@ -33,6 +33,8 @@ namespace llvm {
 
 namespace swift {
 
+class UnifiedStatsReporter;
+
 /// The non-templated implementation of DependencyGraph.
 ///
 /// \see DependencyGraph
@@ -65,12 +67,14 @@ class DependencyGraphImpl {
   class MarkTracerImpl {
     class Entry;
     llvm::DenseMap<const void *, SmallVector<Entry, 4>> Table;
+    UnifiedStatsReporter *Stats;
 
     friend class DependencyGraphImpl;
   protected:
-    MarkTracerImpl();
+    explicit MarkTracerImpl(UnifiedStatsReporter *Stats);
     ~MarkTracerImpl();
-
+    void countStatsForNodeMarking(const OptionSet<DependencyKind> &Kind,
+                                  bool Cascading) const;
     void printPath(raw_ostream &out, const void *item,
                    llvm::function_ref<void(const void *)> printItem) const;
   };
@@ -223,7 +227,8 @@ class DependencyGraph : public DependencyGraphImpl {
   /// This is intended to be a debugging aid.
   class MarkTracer : public MarkTracerImpl {
   public:
-    MarkTracer() = default;
+    explicit MarkTracer(UnifiedStatsReporter *Stats)
+      : MarkTracerImpl(Stats) {}
 
     /// Dump the path that led to \p node.
     void printPath(raw_ostream &out, T node,

include/swift/Frontend/FrontendOptions.h

@@ -192,6 +192,9 @@ class FrontendOptions {
   /// \sa swift::SharedTimer
   bool DebugTimeCompilation = false;
 
+  /// The path to which we should output statistics files.
+  std::string StatsOutputDir;
+
   /// Indicates whether function body parsing should be delayed
   /// until the end of all files.
   bool DelayedFunctionBodyParsing = false;

include/swift/Option/Options.td

@@ -191,6 +191,9 @@ def save_temps : Flag<["-"], "save-temps">,
 def driver_time_compilation : Flag<["-"], "driver-time-compilation">,
   Flags<[NoInteractiveOption,DoesNotAffectIncrementalBuild]>,
   HelpText<"Prints the total time it took to execute all compilation tasks">;
+def stats_output_dir: Separate<["-"], "stats-output-dir">,
+  Flags<[FrontendOption, HelpHidden]>,
+  HelpText<"Directory to write unified compilation-statistics files to">;
 
 def emit_dependencies : Flag<["-"], "emit-dependencies">,
   Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild]>,

lib/Driver/Compilation.cpp

@@ -15,6 +15,7 @@
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsDriver.h"
 #include "swift/Basic/Program.h"
+#include "swift/Basic/Statistic.h"
 #include "swift/Basic/TaskQueue.h"
 #include "swift/Basic/Version.h"
 #include "swift/Basic/type_traits.h"
@@ -92,7 +93,8 @@ Compilation::Compilation(DiagnosticEngine &Diags, OutputLevel Level,
                          bool EnableIncrementalBuild,
                          bool SkipTaskExecution,
                          bool SaveTemps,
-                         bool ShowDriverTimeCompilation)
+                         bool ShowDriverTimeCompilation,
+                         std::unique_ptr<UnifiedStatsReporter> StatsReporter)
   : Diags(Diags), Level(Level), RawInputArgs(std::move(InputArgs)),
     TranslatedArgs(std::move(TranslatedArgs)), 
     InputFilesWithTypes(std::move(InputsWithTypes)), ArgsHash(ArgsHash),
@@ -101,7 +103,8 @@ Compilation::Compilation(DiagnosticEngine &Diags, OutputLevel Level,
     SkipTaskExecution(SkipTaskExecution),
     EnableIncrementalBuild(EnableIncrementalBuild),
     SaveTemps(SaveTemps),
-    ShowDriverTimeCompilation(ShowDriverTimeCompilation) {
+    ShowDriverTimeCompilation(ShowDriverTimeCompilation),
+    Stats(std::move(StatsReporter)) {
 };
 
 static bool writeFilelistIfNecessary(const Job *job, DiagnosticEngine &diags);
@@ -229,7 +232,13 @@ namespace driver {
                      << ": " << LogJob(Cmd) << "\n";
       }
       FinishedCommands.insert(Cmd);
-
+      if (Comp.Stats) {
+          auto &D = Comp.Stats->getDriverCounters();
+          if (Skipped)
+            D.NumDriverJobsSkipped++;
+          else
+            D.NumDriverJobsRun++;
+      }
       auto BlockedIter = BlockingCommands.find(Cmd);
       if (BlockedIter != BlockingCommands.end()) {
         auto AllBlocked = std::move(BlockedIter->second);
@@ -466,12 +475,14 @@ namespace driver {
     }
 
   public:
-    PerformJobsState(Compilation &Comp) : Comp(Comp) {
+    PerformJobsState(Compilation &Comp)
+      : Comp(Comp),
+        ActualIncrementalTracer(Comp.Stats.get()) {
       if (Comp.SkipTaskExecution)
         TQ.reset(new DummyTaskQueue(Comp.NumberOfParallelCommands));
       else
         TQ.reset(new TaskQueue(Comp.NumberOfParallelCommands));
-      if (Comp.ShowIncrementalBuildDecisions)
+      if (Comp.ShowIncrementalBuildDecisions || Comp.Stats)
         IncrementalTracer = &ActualIncrementalTracer;
     }
 

lib/Driver/DependencyGraph.cpp

@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "swift/Basic/Statistic.h"
 #include "swift/Driver/DependencyGraph.h"
 #include "swift/Demangling/Demangle.h"
 #include "llvm/ADT/SmallString.h"
@@ -41,7 +42,8 @@ class DependencyGraphImpl::MarkTracerImpl::Entry {
   DependencyMaskTy KindMask;
 };
 
-DependencyGraphImpl::MarkTracerImpl::MarkTracerImpl() = default;
+DependencyGraphImpl::MarkTracerImpl::MarkTracerImpl(UnifiedStatsReporter *Stats)
+  : Stats(Stats) {}
 DependencyGraphImpl::MarkTracerImpl::~MarkTracerImpl() = default;
 
 using LoadResult = DependencyGraphImpl::LoadResult;
@@ -350,6 +352,7 @@ DependencyGraphImpl::markTransitive(SmallVectorImpl<const void *> &visited,
 
         MutableArrayRef<MarkTracerImpl::Entry> newReason;
         if (tracer) {
+          tracer->countStatsForNodeMarking(intersectingKinds, isCascading);
           newReason = {scratchAlloc.Allocate(reason.size()+1), reason.size()+1};
           std::uninitialized_copy(reason.begin(), reason.end(),
                                   newReason.begin());
@@ -393,6 +396,38 @@ DependencyGraphImpl::markTransitive(SmallVectorImpl<const void *> &visited,
   }
 }
 
+void DependencyGraphImpl::MarkTracerImpl::countStatsForNodeMarking(
+  const OptionSet<DependencyKind> &Kind, bool IsCascading) const {
+
+  if (!Stats)
+    return;
+
+  auto &D = Stats->getDriverCounters();
+  if (IsCascading) {
+    if (Kind & DependencyKind::TopLevelName)
+      D.DriverDepCascadingTopLevel++;
+    if (Kind & DependencyKind::DynamicLookupName)
+      D.DriverDepCascadingDynamic++;
+    if (Kind & DependencyKind::NominalType)
+      D.DriverDepCascadingNominal++;
+    if (Kind & DependencyKind::NominalTypeMember)
+      D.DriverDepCascadingMember++;
+    if (Kind & DependencyKind::NominalTypeMember)
+      D.DriverDepCascadingExternal++;
+  } else {
+    if (Kind & DependencyKind::TopLevelName)
+      D.DriverDepTopLevel++;
+    if (Kind & DependencyKind::DynamicLookupName)
+      D.DriverDepDynamic++;
+    if (Kind & DependencyKind::NominalType)
+      D.DriverDepNominal++;
+    if (Kind & DependencyKind::NominalTypeMember)
+      D.DriverDepMember++;
+    if (Kind & DependencyKind::NominalTypeMember)
+      D.DriverDepExternal++;
+  }
+}
+
 void DependencyGraphImpl::MarkTracerImpl::printPath(
     raw_ostream &out,
     const void *item,

lib/Driver/Driver.cpp

@@ -25,6 +25,7 @@
 #include "swift/Basic/TaskQueue.h"
 #include "swift/Basic/Version.h"
 #include "swift/Basic/Range.h"
+#include "swift/Basic/Statistic.h"
 #include "swift/Driver/Action.h"
 #include "swift/Driver/Compilation.h"
 #include "swift/Driver/Job.h"
@@ -554,6 +555,14 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
   if (Diags.hadAnyError())
     return nullptr;
 
+  std::unique_ptr<UnifiedStatsReporter> StatsReporter;
+  if (const Arg *A =
+      ArgList->getLastArgNoClaim(options::OPT_stats_output_dir)) {
+    StatsReporter = llvm::make_unique<UnifiedStatsReporter>("swift-driver",
+                                                            OI.ModuleName,
+                                                            A->getValue());
+  }
+
   assert(OI.CompilerOutputType != types::ID::TY_INVALID &&
          "buildOutputInfo() must set a valid output type!");
 
@@ -655,7 +664,8 @@ std::unique_ptr<Compilation> Driver::buildCompilation(
                                                  Incremental,
                                                  DriverSkipExecution,
                                                  SaveTemps,
-                                                 ShowDriverTimeCompilation));
+                                                 ShowDriverTimeCompilation,
+                                                 std::move(StatsReporter)));
 
   buildJobs(Actions, OI, OFM.get(), *TC, *C);
 

lib/Driver/ToolChains.cpp

@@ -147,6 +147,7 @@ static void addCommonFrontendArgs(const ToolChain &TC,
   inputArgs.AddLastArg(arguments, options::OPT_sanitize_coverage_EQ);
   inputArgs.AddLastArg(arguments, options::OPT_swift_version);
   inputArgs.AddLastArg(arguments, options::OPT_enforce_exclusivity_EQ);
+  inputArgs.AddLastArg(arguments, options::OPT_stats_output_dir);
 
   // Pass on any build config options
   inputArgs.AddAllArgs(arguments, options::OPT_D);

lib/Frontend/CompilerInvocation.cpp

@@ -187,6 +187,9 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.DebugTimeExpressionTypeChecking |=
     Args.hasArg(OPT_debug_time_expression_type_checking);
   Opts.DebugTimeCompilation |= Args.hasArg(OPT_debug_time_compilation);
+  if (const Arg *A = Args.getLastArg(OPT_stats_output_dir)) {
+    Opts.StatsOutputDir = A->getValue();
+  }
 
   Opts.ValidateTBDAgainstIR |= Args.hasArg(OPT_validate_tbd_against_ir);
 

lib/FrontendTool/FrontendTool.cpp

@@ -38,6 +38,7 @@
 #include "swift/Basic/FileSystem.h"
 #include "swift/Basic/LLVMContext.h"
 #include "swift/Basic/SourceManager.h"
+#include "swift/Basic/Statistic.h"
 #include "swift/Basic/Timer.h"
 #include "swift/Frontend/DiagnosticVerifier.h"
 #include "swift/Frontend/Frontend.h"
@@ -325,6 +326,28 @@ static void debugFailWithCrash() {
   LLVM_BUILTIN_TRAP;
 }
 
+static void countStatsPostSILGen(UnifiedStatsReporter &Stats,
+                                 const SILModule& Module) {
+  auto &C = Stats.getFrontendCounters();
+  // FIXME: calculate these in constant time, via the dense maps.
+  C.NumSILGenFunctions = Module.getFunctionList().size();
+  C.NumSILGenVtables = Module.getVTableList().size();
+  C.NumSILGenWitnessTables = Module.getWitnessTableList().size();
+  C.NumSILGenDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
+  C.NumSILGenGlobalVariables = Module.getSILGlobalList().size();
+}
+
+static void countStatsPostSILOpt(UnifiedStatsReporter &Stats,
+                                 const SILModule& Module) {
+  auto &C = Stats.getFrontendCounters();
+  // FIXME: calculate these in constant time, via the dense maps.
+  C.NumSILOptFunctions = Module.getFunctionList().size();
+  C.NumSILOptVtables = Module.getVTableList().size();
+  C.NumSILOptWitnessTables = Module.getWitnessTableList().size();
+  C.NumSILOptDefaultWitnessTables = Module.getDefaultWitnessTableList().size();
+  C.NumSILOptGlobalVariables = Module.getSILGlobalList().size();
+}
+
 /// Performs the compile requested by the user.
 /// \param Instance Will be reset after performIRGeneration when the verifier
 ///                 mode is NoVerify and there were no errors.
@@ -333,7 +356,8 @@ static bool performCompile(std::unique_ptr<CompilerInstance> &Instance,
                            CompilerInvocation &Invocation,
                            ArrayRef<const char *> Args,
                            int &ReturnValue,
-                           FrontendObserver *observer) {
+                           FrontendObserver *observer,
+                           UnifiedStatsReporter *Stats) {
   FrontendOptions opts = Invocation.getFrontendOptions();
   FrontendOptions::ActionType Action = opts.RequestedAction;
 
@@ -551,6 +575,9 @@ static bool performCompile(std::unique_ptr<CompilerInstance> &Instance,
   if (observer) {
     observer->performedSILGeneration(*SM);
   }
+  if (Stats) {
+    countStatsPostSILGen(*Stats, *SM);
+  }
 
   // We've been told to emit SIL after SILGen, so write it now.
   if (Action == FrontendOptions::EmitSILGen) {
@@ -624,6 +651,9 @@ static bool performCompile(std::unique_ptr<CompilerInstance> &Instance,
   if (observer) {
     observer->performedSILOptimization(*SM);
   }
+  if (Stats) {
+    countStatsPostSILOpt(*Stats, *SM);
+  }
 
   {
     SharedTimer timer("SIL verification, post-optimization");
@@ -973,6 +1003,23 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     llvm::EnableStatistics();
   }
 
+  const std::string &StatsOutputDir =
+      Invocation.getFrontendOptions().StatsOutputDir;
+  std::unique_ptr<UnifiedStatsReporter> StatsReporter;
+  if (!StatsOutputDir.empty()) {
+    auto &opts = Invocation.getFrontendOptions();
+    std::string TargetName = opts.ModuleName;
+    if (opts.PrimaryInput.hasValue() &&
+        opts.PrimaryInput.getValue().isFilename()) {
+      auto Index = opts.PrimaryInput.getValue().Index;
+      TargetName += ".";
+      TargetName += llvm::sys::path::filename(opts.InputFilenames[Index]);
+    }
+    StatsReporter = llvm::make_unique<UnifiedStatsReporter>("swift-frontend",
+                                                            TargetName,
+                                                            StatsOutputDir);
+  }
+
   const DiagnosticOptions &diagOpts = Invocation.getDiagnosticOptions();
   if (diagOpts.VerifyMode != DiagnosticOptions::NoVerify) {
     enableDiagnosticVerifier(Instance->getSourceMgr());
@@ -995,7 +1042,8 @@ int swift::performFrontend(ArrayRef<const char *> Args,
 
   int ReturnValue = 0;
   bool HadError =
-    performCompile(Instance, Invocation, Args, ReturnValue, observer);
+    performCompile(Instance, Invocation, Args, ReturnValue, observer,
+                   StatsReporter.get());
 
   if (!HadError) {
     Mangle::printManglingStats();