Merge pull request #11988 from graydon/unified-stats-reporter-fixes-for-swift-4.0-branch

graydon · web-flow · commit 00e34e4b1e0e · 2017-09-18T16:44:02.000-07:00
Unified stats reporter fixes for swift 4.0 branch
diff --git a/include/swift/Basic/Statistic.h b/include/swift/Basic/Statistic.h
@@ -74,6 +74,8 @@ class UnifiedStatsReporter {
   struct AlwaysOnFrontendCounters
   {
     size_t NumSourceBuffers;
+    size_t NumSourceLines;
+    size_t NumSourceLinesPerSecond;
     size_t NumLinkLibraries;
     size_t NumLoadedModules;
     size_t NumImportedExternalDefinitions;
@@ -132,6 +134,7 @@ class UnifiedStatsReporter {
 
 private:
   SmallString<128> Filename;
+  llvm::TimeRecord StartedTime;
   std::unique_ptr<llvm::NamedRegionTimer> Timer;
 
   std::unique_ptr<AlwaysOnDriverCounters> DriverCounters;
diff --git a/lib/Basic/Statistic.cpp b/lib/Basic/Statistic.cpp
@@ -124,6 +124,7 @@ UnifiedStatsReporter::UnifiedStatsReporter(StringRef ProgramName,
                                            StringRef AuxName,
                                            StringRef Directory)
   : Filename(Directory),
+    StartedTime(llvm::TimeRecord::getCurrentTime()),
     Timer(make_unique<NamedRegionTimer>(AuxName,
                                         "Building Target",
                                         ProgramName, "Running Program"))
@@ -161,6 +162,8 @@ UnifiedStatsReporter::publishAlwaysOnStatsToLLVM() {
     auto &C = getFrontendCounters();
 
     PUBLISH_STAT(C, "AST", NumSourceBuffers);
+    PUBLISH_STAT(C, "AST", NumSourceLines);
+    PUBLISH_STAT(C, "AST", NumSourceLinesPerSecond);
     PUBLISH_STAT(C, "AST", NumLinkLibraries);
     PUBLISH_STAT(C, "AST", NumLoadedModules);
     PUBLISH_STAT(C, "AST", NumImportedExternalDefinitions);
@@ -251,6 +254,8 @@ UnifiedStatsReporter::printAlwaysOnStatsAndTimers(raw_ostream &OS) {
     auto &C = getFrontendCounters();
 
     PRINT_STAT(OS, delim, C, "AST", NumSourceBuffers);
+    PRINT_STAT(OS, delim, C, "AST", NumSourceLines);
+    PRINT_STAT(OS, delim, C, "AST", NumSourceLinesPerSecond);
     PRINT_STAT(OS, delim, C, "AST", NumLinkLibraries);
     PRINT_STAT(OS, delim, C, "AST", NumLoadedModules);
     PRINT_STAT(OS, delim, C, "AST", NumImportedExternalDefinitions);
@@ -339,11 +344,24 @@ UnifiedStatsReporter::~UnifiedStatsReporter()
   // we're repurposing a bit here.
   Timer.reset();
 
+  // We currently do this by manual TimeRecord keeping because LLVM has decided
+  // not to allow access to the Timers inside NamedRegionTimers.
+  auto ElapsedTime = llvm::TimeRecord::getCurrentTime();
+  ElapsedTime -= StartedTime;
+
   if (DriverCounters) {
     auto &C = getDriverCounters();
     C.ChildrenMaxRSS = getChildrenMaxResidentSetSize();
   }
 
+  if (FrontendCounters) {
+    auto &C = getFrontendCounters();
+    // Convenience calculation for crude top-level "absolute speed".
+    if (C.NumSourceLines != 0 && ElapsedTime.getProcessTime() != 0.0)
+      C.NumSourceLinesPerSecond = (size_t) (((double)C.NumSourceLines) /
+                                            ElapsedTime.getProcessTime());
+  }
+
   std::error_code EC;
   raw_fd_ostream ostream(Filename, EC, fs::F_Append | fs::F_Text);
   if (EC)
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -65,7 +65,6 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Option/OptTable.h"
@@ -373,10 +372,32 @@ static bool emitIndexData(SourceFile *PrimarySourceFile,
       const CompilerInvocation &Invocation,
       CompilerInstance &Instance);
 
+static void countStatsOfSourceFile(UnifiedStatsReporter &Stats,
+                                   CompilerInstance &Instance,
+                                   SourceFile *SF) {
+  auto &C = Stats.getFrontendCounters();
+  auto &SM = Instance.getSourceMgr();
+  C.NumDecls += SF->Decls.size();
+  C.NumLocalTypeDecls += SF->LocalTypeDecls.size();
+  C.NumObjCMethods += SF->ObjCMethods.size();
+  C.NumInfixOperators += SF->InfixOperators.size();
+  C.NumPostfixOperators += SF->PostfixOperators.size();
+  C.NumPrefixOperators += SF->PrefixOperators.size();
+  C.NumPrecedenceGroups += SF->PrecedenceGroups.size();
+  C.NumUsedConformances += SF->getUsedConformances().size();
+
+  auto bufID = SF->getBufferID();
+  if (bufID.hasValue()) {
+    C.NumSourceLines +=
+      SM.getEntireTextForBuffer(bufID.getValue()).count('\n');
+  }
+}
+
 static void countStatsPostSema(UnifiedStatsReporter &Stats,
                                CompilerInstance& Instance) {
   auto &C = Stats.getFrontendCounters();
-  C.NumSourceBuffers = Instance.getSourceMgr().getLLVMSourceMgr().getNumBuffers();
+  auto &SM = Instance.getSourceMgr();
+  C.NumSourceBuffers = SM.getLLVMSourceMgr().getNumBuffers();
   C.NumLinkLibraries = Instance.getLinkLibraries().size();
 
   auto const &AST = Instance.getASTContext();
@@ -395,32 +416,13 @@ static void countStatsPostSema(UnifiedStatsReporter &Stats,
   }
 
   if (auto *SF = Instance.getPrimarySourceFile()) {
-    C.NumDecls = SF->Decls.size();
-    C.NumLocalTypeDecls = SF->LocalTypeDecls.size();
-    C.NumObjCMethods = SF->ObjCMethods.size();
-    C.NumInfixOperators = SF->InfixOperators.size();
-    C.NumPostfixOperators = SF->PostfixOperators.size();
-    C.NumPrefixOperators = SF->PrefixOperators.size();
-    C.NumPrecedenceGroups = SF->PrecedenceGroups.size();
-    C.NumUsedConformances = SF->getUsedConformances().size();
-  }
-}
-
-static void countStatsPostIRGen(UnifiedStatsReporter &Stats,
-                                const llvm::Module& Module) {
-  auto &C = Stats.getFrontendCounters();
-  // FIXME: calculate these in constant time if possible.
-  C.NumIRGlobals = Module.getGlobalList().size();
-  C.NumIRFunctions = Module.getFunctionList().size();
-  C.NumIRAliases = Module.getAliasList().size();
-  C.NumIRIFuncs = Module.getIFuncList().size();
-  C.NumIRNamedMetaData = Module.getNamedMDList().size();
-  C.NumIRValueSymbols = Module.getValueSymbolTable().size();
-  C.NumIRComdatSymbols = Module.getComdatSymbolTable().size();
-  for (auto const &Func : Module) {
-    for (auto const &BB : Func) {
-      C.NumIRBasicBlocks++;
-      C.NumIRInsts += BB.size();
+    countStatsOfSourceFile(Stats, Instance, SF);
+  } else if (auto *M = Instance.getMainModule()) {
+    // No primary source file, but a main module; this is WMO-mode
+    for (auto *F : M->getFiles()) {
+      if (auto *SF = dyn_cast<SourceFile>(F)) {
+        countStatsOfSourceFile(Stats, Instance, SF);
+      }
     }
   }
 }
@@ -961,10 +963,6 @@ static bool performCompile(CompilerInstance &Instance,
     return HadError;
   }
 
-  if (Stats) {
-    countStatsPostIRGen(*Stats, *IRModule);
-  }
-
   bool allSymbols = false;
   switch (opts.ValidateTBDAgainstIR) {
   case FrontendOptions::TBDValidationMode::None:
diff --git a/lib/IRGen/IRGen.cpp b/lib/IRGen/IRGen.cpp
@@ -44,6 +44,7 @@
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Linker/Linker.h"
 #include "llvm/MC/SubtargetFeature.h"
@@ -353,6 +354,25 @@ static bool needsRecompile(StringRef OutputFilename, ArrayRef<uint8_t> HashData,
   return true;
 }
 
+static void countStatsPostIRGen(UnifiedStatsReporter &Stats,
+                                const llvm::Module& Module) {
+  auto &C = Stats.getFrontendCounters();
+  // FIXME: calculate these in constant time if possible.
+  C.NumIRGlobals += Module.getGlobalList().size();
+  C.NumIRFunctions += Module.getFunctionList().size();
+  C.NumIRAliases += Module.getAliasList().size();
+  C.NumIRIFuncs += Module.getIFuncList().size();
+  C.NumIRNamedMetaData += Module.getNamedMDList().size();
+  C.NumIRValueSymbols += Module.getValueSymbolTable().size();
+  C.NumIRComdatSymbols += Module.getComdatSymbolTable().size();
+  for (auto const &Func : Module) {
+    for (auto const &BB : Func) {
+      C.NumIRBasicBlocks++;
+      C.NumIRInsts += BB.size();
+    }
+  }
+}
+
 /// Run the LLVM passes. In multi-threaded compilation this will be done for
 /// multiple LLVM modules in parallel.
 bool swift::performLLVM(IRGenOptions &Opts, DiagnosticEngine *Diags,
@@ -464,6 +484,14 @@ bool swift::performLLVM(IRGenOptions &Opts, DiagnosticEngine *Diags,
   }
   }
 
+  if (Stats) {
+    if (DiagMutex)
+      DiagMutex->lock();
+    countStatsPostIRGen(*Stats, *Module);
+    if (DiagMutex)
+      DiagMutex->unlock();
+  }
+
   EmitPasses.run(*Module);
 
   if (Stats && RawOS.hasValue()) {
@@ -785,7 +813,7 @@ static std::unique_ptr<llvm::Module> performIRGeneration(IRGenOptions &Opts,
     if (performLLVM(Opts, &IGM.Context.Diags, nullptr, IGM.ModuleHash,
                     IGM.getModule(), IGM.TargetMachine.get(),
                     IGM.Context.LangOpts.EffectiveLanguageVersion,
-                    IGM.OutputFilename))
+                    IGM.OutputFilename, IGM.Context.Stats))
       return nullptr;
   }
 
@@ -805,7 +833,7 @@ static void ThreadEntryPoint(IRGenerator *irgen,
     performLLVM(irgen->Opts, &IGM->Context.Diags, DiagMutex, IGM->ModuleHash,
                 IGM->getModule(), IGM->TargetMachine.get(),
                 IGM->Context.LangOpts.EffectiveLanguageVersion,
-                IGM->OutputFilename);
+                IGM->OutputFilename, IGM->Context.Stats);
     if (IGM->Context.Diags.hadAnyError())
       return;
   }
diff --git a/test/Misc/stats_dir.swift b/test/Misc/stats_dir.swift
@@ -2,6 +2,10 @@
 // RUN: %target-swift-frontend -c -o %t/out.o -stats-output-dir %t %s
 // RUN: %utils/process-stats-dir.py --set-csv-baseline %t/frontend.csv %t
 // RUN: %FileCheck -input-file %t/frontend.csv %s
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: %target-swift-frontend -c -wmo -num-threads 4 -o %t/out.o -stats-output-dir %t %s
+// RUN: %utils/process-stats-dir.py --set-csv-baseline %t/frontend.csv %t
+// RUN: %FileCheck -input-file %t/frontend.csv %s
 // RUN: echo '9000000000	"LLVM.NumLLVMBytesOutput"	1' >>%t/frontend.csv
 // RUN: not %utils/process-stats-dir.py --compare-to-csv-baseline %t/frontend.csv %t
 
@@ -11,7 +15,9 @@
 // RUN: %FileCheck -input-file %t/driver.csv %s
 // RUN: %utils/process-stats-dir.py --compare-to-csv-baseline %t/driver.csv %t
 
-// CHECK: LLVM.NumLLVMBytesOutput
+// CHECK: {{"AST.NumSourceLines"	[1-9][0-9]*$}}
+// CHECK: {{"IRModule.NumIRFunctions"	[1-9][0-9]*$}}
+// CHECK: {{"LLVM.NumLLVMBytesOutput"	[1-9][0-9]*$}}
 
 public func foo() {
     print("hello")
