Add an assortment of new "always-on" metrics.

include/swift/AST/ASTContext.h

@@ -103,6 +103,7 @@ namespace swift {
   class SILBoxType;
   class TypeAliasDecl;
   class VarDecl;
+  class UnifiedStatsReporter;
 
   enum class KnownProtocolKind : uint8_t;
 
@@ -241,6 +242,9 @@ class ASTContext {
   /// Cache of remapped types (useful for diagnostics).
   llvm::StringMap<Type> RemappedTypes;
 
+  /// Optional table of counters to report, nullptr when not collecting.
+  UnifiedStatsReporter *Stats = nullptr;
+
 private:
   /// \brief The current generation number, which reflects the number of
   /// times that external modules have been loaded.

include/swift/Basic/Statistic.h

@@ -71,6 +71,38 @@ class UnifiedStatsReporter {
 
   struct AlwaysOnFrontendCounters
   {
+    size_t NumSourceBuffers;
+    size_t NumLinkLibraries;
+    size_t NumLoadedModules;
+    size_t NumImportedExternalDefinitions;
+    size_t NumTotalClangImportedEntities;
+    size_t NumASTBytesAllocated;
+    size_t NumDependencies;
+    size_t NumReferencedTopLevelNames;
+    size_t NumReferencedDynamicNames;
+    size_t NumReferencedMemberNames;
+    size_t NumDecls;
+    size_t NumLocalTypeDecls;
+    size_t NumObjCMethods;
+    size_t NumInfixOperators;
+    size_t NumPostfixOperators;
+    size_t NumPrefixOperators;
+    size_t NumPrecedenceGroups;
+    size_t NumUsedConformances;
+
+    size_t NumConformancesDeserialized;
+    size_t NumConstraintScopes;
+    size_t NumDeclsDeserialized;
+    size_t NumDeclsValidated;
+    size_t NumFunctionsTypechecked;
+    size_t NumGenericSignatureBuilders;
+    size_t NumLazyGenericEnvironments;
+    size_t NumLazyGenericEnvironmentsLoaded;
+    size_t NumLazyIterableDeclContexts;
+    size_t NumTypesDeserialized;
+    size_t NumTypesValidated;
+    size_t NumUnloadedLazyIterableDeclContexts;
+
     size_t NumSILGenFunctions;
     size_t NumSILGenVtables;
     size_t NumSILGenWitnessTables;
@@ -82,6 +114,16 @@ class UnifiedStatsReporter {
     size_t NumSILOptWitnessTables;
     size_t NumSILOptDefaultWitnessTables;
     size_t NumSILOptGlobalVariables;
+
+    size_t NumIRGlobals;
+    size_t NumIRFunctions;
+    size_t NumIRAliases;
+    size_t NumIRIFuncs;
+    size_t NumIRNamedMetaData;
+    size_t NumIRValueSymbols;
+    size_t NumIRComdatSymbols;
+    size_t NumIRBasicBlocks;
+    size_t NumIRInsts;
   };
 
 private:

lib/AST/Decl.cpp

@@ -43,6 +43,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "swift/Basic/Range.h"
 #include "swift/Basic/StringExtras.h"
+#include "swift/Basic/Statistic.h"
 
 #include "clang/Basic/CharInfo.h"
 #include "clang/AST/Attr.h"
@@ -672,6 +673,9 @@ GenericContext::getLazyGenericEnvironmentSlow() const {
 
   const_cast<GenericContext *>(this)->setGenericEnvironment(genericEnv);
   ++NumLazyGenericEnvironmentsLoaded;
+  // FIXME: (transitional) increment the redundant "always-on" counter.
+  if (getASTContext().Stats)
+    getASTContext().Stats->getFrontendCounters().NumLazyGenericEnvironmentsLoaded++;
   return genericEnv;
 }
 
@@ -686,6 +690,10 @@ void GenericContext::setLazyGenericEnvironment(LazyMemberLoader *lazyLoader,
   contextData->genericEnvData = genericEnvData;
 
   ++NumLazyGenericEnvironments;
+  // FIXME: (transitional) increment the redundant "always-on" counter.
+  if (getASTContext().Stats)
+    getASTContext().Stats->getFrontendCounters().NumLazyGenericEnvironments++;
+
 }
 
 ImportDecl *ImportDecl::create(ASTContext &Ctx, DeclContext *DC,

lib/AST/DeclContext.cpp

@@ -20,6 +20,7 @@
 #include "swift/AST/Module.h"
 #include "swift/AST/Types.h"
 #include "swift/Basic/SourceManager.h"
+#include "swift/Basic/Statistic.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/raw_ostream.h"
@@ -888,6 +889,11 @@ void IterableDeclContext::setMemberLoader(LazyMemberLoader *loader,
 
   ++NumLazyIterableDeclContexts;
   ++NumUnloadedLazyIterableDeclContexts;
+  // FIXME: (transitional) increment the redundant "always-on" counter.
+  if (ctx.Stats) {
+    ++ctx.Stats->getFrontendCounters().NumLazyIterableDeclContexts;
+    ++ctx.Stats->getFrontendCounters().NumUnloadedLazyIterableDeclContexts;
+  }
 }
 
 void IterableDeclContext::loadAllMembers() const {
@@ -915,6 +921,9 @@ void IterableDeclContext::loadAllMembers() const {
                                       contextInfo->memberData);
 
   --NumUnloadedLazyIterableDeclContexts;
+  // FIXME: (transitional) decrement the redundant "always-on" counter.
+  if (ctx.Stats)
+    ctx.Stats->getFrontendCounters().NumUnloadedLazyIterableDeclContexts--;
 }
 
 bool IterableDeclContext::classof(const Decl *D) {

lib/AST/GenericSignatureBuilder.cpp

@@ -29,6 +29,7 @@
 #include "swift/AST/TypeRepr.h"
 #include "swift/AST/TypeWalker.h"
 #include "swift/Basic/Defer.h"
+#include "swift/Basic/Statistic.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
@@ -2237,6 +2238,8 @@ GenericSignatureBuilder::GenericSignatureBuilder(
                                std::function<GenericFunction> lookupConformance)
   : Context(ctx), Diags(Context.Diags), Impl(new Implementation) {
   Impl->LookupConformance = std::move(lookupConformance);
+  if (Context.Stats)
+    Context.Stats->getFrontendCounters().NumGenericSignatureBuilders++;
 }
 
 GenericSignatureBuilder::GenericSignatureBuilder(GenericSignatureBuilder &&) = default;

lib/Basic/Statistic.cpp

@@ -74,6 +74,39 @@ void
 UnifiedStatsReporter::publishAlwaysOnStatsToLLVM() {
   if (FrontendCounters) {
     auto &C = getFrontendCounters();
+
+    PUBLISH_STAT(C, "AST", NumSourceBuffers);
+    PUBLISH_STAT(C, "AST", NumLinkLibraries);
+    PUBLISH_STAT(C, "AST", NumLoadedModules);
+    PUBLISH_STAT(C, "AST", NumImportedExternalDefinitions);
+    PUBLISH_STAT(C, "AST", NumTotalClangImportedEntities);
+    PUBLISH_STAT(C, "AST", NumASTBytesAllocated);
+    PUBLISH_STAT(C, "AST", NumDependencies);
+    PUBLISH_STAT(C, "AST", NumReferencedTopLevelNames);
+    PUBLISH_STAT(C, "AST", NumReferencedDynamicNames);
+    PUBLISH_STAT(C, "AST", NumReferencedMemberNames);
+    PUBLISH_STAT(C, "AST", NumDecls);
+    PUBLISH_STAT(C, "AST", NumLocalTypeDecls);
+    PUBLISH_STAT(C, "AST", NumObjCMethods);
+    PUBLISH_STAT(C, "AST", NumInfixOperators);
+    PUBLISH_STAT(C, "AST", NumPostfixOperators);
+    PUBLISH_STAT(C, "AST", NumPrefixOperators);
+    PUBLISH_STAT(C, "AST", NumPrecedenceGroups);
+    PUBLISH_STAT(C, "AST", NumUsedConformances);
+
+    PUBLISH_STAT(C, "Sema", NumConformancesDeserialized);
+    PUBLISH_STAT(C, "Sema", NumConstraintScopes);
+    PUBLISH_STAT(C, "Sema", NumDeclsDeserialized);
+    PUBLISH_STAT(C, "Sema", NumDeclsValidated);
+    PUBLISH_STAT(C, "Sema", NumFunctionsTypechecked);
+    PUBLISH_STAT(C, "Sema", NumGenericSignatureBuilders);
+    PUBLISH_STAT(C, "Sema", NumLazyGenericEnvironments);
+    PUBLISH_STAT(C, "Sema", NumLazyGenericEnvironmentsLoaded);
+    PUBLISH_STAT(C, "Sema", NumLazyIterableDeclContexts);
+    PUBLISH_STAT(C, "Sema", NumTypesDeserialized);
+    PUBLISH_STAT(C, "Sema", NumTypesValidated);
+    PUBLISH_STAT(C, "Sema", NumUnloadedLazyIterableDeclContexts);
+
     PUBLISH_STAT(C, "SILModule", NumSILGenFunctions);
     PUBLISH_STAT(C, "SILModule", NumSILGenVtables);
     PUBLISH_STAT(C, "SILModule", NumSILGenWitnessTables);
@@ -85,6 +118,16 @@ UnifiedStatsReporter::publishAlwaysOnStatsToLLVM() {
     PUBLISH_STAT(C, "SILModule", NumSILOptWitnessTables);
     PUBLISH_STAT(C, "SILModule", NumSILOptDefaultWitnessTables);
     PUBLISH_STAT(C, "SILModule", NumSILOptGlobalVariables);
+
+    PUBLISH_STAT(C, "IRModule", NumIRGlobals);
+    PUBLISH_STAT(C, "IRModule", NumIRFunctions);
+    PUBLISH_STAT(C, "IRModule", NumIRAliases);
+    PUBLISH_STAT(C, "IRModule", NumIRIFuncs);
+    PUBLISH_STAT(C, "IRModule", NumIRNamedMetaData);
+    PUBLISH_STAT(C, "IRModule", NumIRValueSymbols);
+    PUBLISH_STAT(C, "IRModule", NumIRComdatSymbols);
+    PUBLISH_STAT(C, "IRModule", NumIRBasicBlocks);
+    PUBLISH_STAT(C, "IRModule", NumIRInsts);
   }
   if (DriverCounters) {
     auto &C = getDriverCounters();
@@ -118,6 +161,39 @@ UnifiedStatsReporter::printAlwaysOnStatsAndTimers(raw_ostream &OS) {
   const char *delim = "";
   if (FrontendCounters) {
     auto &C = getFrontendCounters();
+
+    PRINT_STAT(OS, delim, C, "AST", NumSourceBuffers);
+    PRINT_STAT(OS, delim, C, "AST", NumLinkLibraries);
+    PRINT_STAT(OS, delim, C, "AST", NumLoadedModules);
+    PRINT_STAT(OS, delim, C, "AST", NumImportedExternalDefinitions);
+    PRINT_STAT(OS, delim, C, "AST", NumTotalClangImportedEntities);
+    PRINT_STAT(OS, delim, C, "AST", NumASTBytesAllocated);
+    PRINT_STAT(OS, delim, C, "AST", NumDependencies);
+    PRINT_STAT(OS, delim, C, "AST", NumReferencedTopLevelNames);
+    PRINT_STAT(OS, delim, C, "AST", NumReferencedDynamicNames);
+    PRINT_STAT(OS, delim, C, "AST", NumReferencedMemberNames);
+    PRINT_STAT(OS, delim, C, "AST", NumDecls);
+    PRINT_STAT(OS, delim, C, "AST", NumLocalTypeDecls);
+    PRINT_STAT(OS, delim, C, "AST", NumObjCMethods);
+    PRINT_STAT(OS, delim, C, "AST", NumInfixOperators);
+    PRINT_STAT(OS, delim, C, "AST", NumPostfixOperators);
+    PRINT_STAT(OS, delim, C, "AST", NumPrefixOperators);
+    PRINT_STAT(OS, delim, C, "AST", NumPrecedenceGroups);
+    PRINT_STAT(OS, delim, C, "AST", NumUsedConformances);
+
+    PRINT_STAT(OS, delim, C, "Sema", NumConformancesDeserialized);
+    PRINT_STAT(OS, delim, C, "Sema", NumConstraintScopes);
+    PRINT_STAT(OS, delim, C, "Sema", NumDeclsDeserialized);
+    PRINT_STAT(OS, delim, C, "Sema", NumDeclsValidated);
+    PRINT_STAT(OS, delim, C, "Sema", NumFunctionsTypechecked);
+    PRINT_STAT(OS, delim, C, "Sema", NumGenericSignatureBuilders);
+    PRINT_STAT(OS, delim, C, "Sema", NumLazyGenericEnvironments);
+    PRINT_STAT(OS, delim, C, "Sema", NumLazyGenericEnvironmentsLoaded);
+    PRINT_STAT(OS, delim, C, "Sema", NumLazyIterableDeclContexts);
+    PRINT_STAT(OS, delim, C, "Sema", NumTypesDeserialized);
+    PRINT_STAT(OS, delim, C, "Sema", NumTypesValidated);
+    PRINT_STAT(OS, delim, C, "Sema", NumUnloadedLazyIterableDeclContexts);
+
     PRINT_STAT(OS, delim, C, "SILModule", NumSILGenFunctions);
     PRINT_STAT(OS, delim, C, "SILModule", NumSILGenVtables);
     PRINT_STAT(OS, delim, C, "SILModule", NumSILGenWitnessTables);
@@ -129,6 +205,16 @@ UnifiedStatsReporter::printAlwaysOnStatsAndTimers(raw_ostream &OS) {
     PRINT_STAT(OS, delim, C, "SILModule", NumSILOptWitnessTables);
     PRINT_STAT(OS, delim, C, "SILModule", NumSILOptDefaultWitnessTables);
     PRINT_STAT(OS, delim, C, "SILModule", NumSILOptGlobalVariables);
+
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRGlobals);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRFunctions);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRAliases);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRIFuncs);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRNamedMetaData);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRValueSymbols);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRComdatSymbols);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRBasicBlocks);
+    PRINT_STAT(OS, delim, C, "IRModule", NumIRInsts);
   }
   if (DriverCounters) {
     auto &C = getDriverCounters();

lib/ClangImporter/ImportDecl.cpp

@@ -42,6 +42,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
 #include "clang/Basic/CharInfo.h"
+#include "swift/Basic/Statistic.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Lookup.h"
 
@@ -7051,6 +7052,9 @@ ClangImporter::Implementation::importDeclImpl(const clang::NamedDecl *ClangDecl,
 void ClangImporter::Implementation::startedImportingEntity() {
   ++NumCurrentImportingEntities;
   ++NumTotalImportedEntities;
+  // FIXME: (transitional) increment the redundant "always-on" counter.
+  if (SwiftContext.Stats)
+    SwiftContext.Stats->getFrontendCounters().NumTotalClangImportedEntities++;
 }
 
 void ClangImporter::Implementation::finishedImportingEntity() {

lib/FrontendTool/FrontendTool.cpp

@@ -63,6 +63,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Option/OptTable.h"
@@ -347,6 +348,58 @@ static void debugFailWithCrash() {
   LLVM_BUILTIN_TRAP;
 }
 
+static void countStatsPostSema(UnifiedStatsReporter &Stats,
+                               CompilerInstance& Instance) {
+  auto &C = Stats.getFrontendCounters();
+  C.NumSourceBuffers = Instance.getSourceMgr().getLLVMSourceMgr().getNumBuffers();
+  C.NumLinkLibraries = Instance.getLinkLibraries().size();
+
+  auto const &AST = Instance.getASTContext();
+  C.NumLoadedModules = AST.LoadedModules.size();
+  C.NumImportedExternalDefinitions = AST.ExternalDefinitions.size();
+  C.NumASTBytesAllocated = AST.getAllocator().getBytesAllocated();
+
+  if (auto *D = Instance.getDependencyTracker()) {
+    C.NumDependencies = D->getDependencies().size();
+  }
+
+  if (auto *R = Instance.getReferencedNameTracker()) {
+    C.NumReferencedTopLevelNames = R->getTopLevelNames().size();
+    C.NumReferencedDynamicNames = R->getDynamicLookupNames().size();
+    C.NumReferencedMemberNames = R->getUsedMembers().size();
+  }
+
+  if (auto *SF = Instance.getPrimarySourceFile()) {
+    C.NumDecls = SF->Decls.size();
+    C.NumLocalTypeDecls = SF->LocalTypeDecls.size();
+    C.NumObjCMethods = SF->ObjCMethods.size();
+    C.NumInfixOperators = SF->InfixOperators.size();
+    C.NumPostfixOperators = SF->PostfixOperators.size();
+    C.NumPrefixOperators = SF->PrefixOperators.size();
+    C.NumPrecedenceGroups = SF->PrecedenceGroups.size();
+    C.NumUsedConformances = SF->getUsedConformances().size();
+  }
+}
+
+static void countStatsPostIRGen(UnifiedStatsReporter &Stats,
+                                const llvm::Module& Module) {
+  auto &C = Stats.getFrontendCounters();
+  // FIXME: calculate these in constant time if possible.
+  C.NumIRGlobals = Module.getGlobalList().size();
+  C.NumIRFunctions = Module.getFunctionList().size();
+  C.NumIRAliases = Module.getAliasList().size();
+  C.NumIRIFuncs = Module.getIFuncList().size();
+  C.NumIRNamedMetaData = Module.getNamedMDList().size();
+  C.NumIRValueSymbols = Module.getValueSymbolTable().size();
+  C.NumIRComdatSymbols = Module.getComdatSymbolTable().size();
+  for (auto const &Func : Module) {
+    for (auto const &BB : Func) {
+      C.NumIRBasicBlocks++;
+      C.NumIRInsts += BB.size();
+    }
+  }
+}
+
 static void countStatsPostSILGen(UnifiedStatsReporter &Stats,
                                  const SILModule& Module) {
   auto &C = Stats.getFrontendCounters();
@@ -452,6 +505,10 @@ static bool performCompile(std::unique_ptr<CompilerInstance> &Instance,
     observer->performedSemanticAnalysis(*Instance);
   }
 
+  if (Stats) {
+    countStatsPostSema(*Stats, *Instance);
+  }
+
   FrontendOptions::DebugCrashMode CrashMode = opts.CrashMode;
   if (CrashMode == FrontendOptions::DebugCrashMode::AssertAfterParse)
     debugFailWithAssertion();
@@ -829,6 +886,10 @@ static bool performCompile(std::unique_ptr<CompilerInstance> &Instance,
     return HadError;
   }
 
+  if (Stats) {
+    countStatsPostIRGen(*Stats, *IRModule);
+  }
+
   if (opts.ValidateTBDAgainstIR) {
     auto hasMultipleIRGenThreads = Invocation.getSILOptions().NumThreads > 1;
     bool error;
@@ -1115,6 +1176,13 @@ int swift::performFrontend(ArrayRef<const char *> Args,
     return 1;
   }
 
+  if (StatsReporter) {
+    // Install stats-reporter somewhere visible for subsystems that
+    // need to bump counters as they work, rather than measure
+    // accumulated work on completion (mostly: TypeChecker).
+    Instance->getASTContext().Stats = StatsReporter.get();
+  }
+
   // The compiler instance has been configured; notify our observer.
   if (observer) {
     observer->configuredCompiler(*Instance);

lib/Sema/ConstraintSystem.cpp

@@ -43,6 +43,9 @@ ConstraintSystem::~ConstraintSystem() {
 void ConstraintSystem::incrementScopeCounter() {
   SWIFT_FUNC_STAT;
   CountScopes++;
+  // FIXME: (transitional) increment the redundant "always-on" counter.
+  if (TC.Context.Stats)
+    TC.Context.Stats->getFrontendCounters().NumConstraintScopes++;
 }
 
 bool ConstraintSystem::hasFreeTypeVariables() {