[MemProf] Add basic summary section support

llvm/include/llvm/Analysis/MemoryProfileInfo.h

@@ -21,11 +21,6 @@
 namespace llvm {
 namespace memprof {
 
-/// Return the allocation type for a given set of memory profile values.
-LLVM_ABI AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
-                                     uint64_t AllocCount,
-                                     uint64_t TotalLifetime);
-
 /// Build callstack metadata from the provided list of call stack ids. Returns
 /// the resulting metadata node.
 LLVM_ABI MDNode *buildCallstackMetadata(ArrayRef<uint64_t> CallStack,

llvm/include/llvm/ProfileData/IndexedMemProfData.h

@@ -24,6 +24,7 @@
 
 namespace llvm {
 namespace memprof {
+class MemProfSummary;
 struct IndexedMemProfData {
   // A map to hold memprof data per function. The lower 64 bits obtained from
   // the md5 hash of the function name is used to index into the map.
@@ -89,7 +90,7 @@ struct IndexedMemProfData {
 Error writeMemProf(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData);
-
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum);
 } // namespace llvm
 #endif

llvm/include/llvm/ProfileData/InstrProfReader.h

@@ -22,6 +22,7 @@
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/ProfileData/InstrProfCorrelator.h"
 #include "llvm/ProfileData/MemProf.h"
+#include "llvm/ProfileData/MemProfSummary.h"
 #include "llvm/ProfileData/MemProfYAML.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -690,6 +691,8 @@ class IndexedMemProfReader {
   /// The MemProf version.
   memprof::IndexedVersion Version =
       static_cast<memprof::IndexedVersion>(memprof::MinimumSupportedVersion);
+  /// MemProf summary (if available, version >= 4).
+  std::unique_ptr<memprof::MemProfSummary> MemProfSum;
   /// MemProf profile schema (if available).
   memprof::MemProfSchema Schema;
   /// MemProf record profile data on-disk indexed via llvm::md5(FunctionName).
@@ -725,6 +728,8 @@ class IndexedMemProfReader {
 
   // Return the entire MemProf profile.
   memprof::AllMemProfData getAllMemProfData() const;
+
+  memprof::MemProfSummary *getSummary() const { return MemProfSum.get(); }
 };
 
 /// Reader for the indexed binary instrprof format.
@@ -887,6 +892,11 @@ class IndexedInstrProfReader : public InstrProfReader {
     }
   }
 
+  /// Return the MemProf summary. Will be null if unavailable (version < 4).
+  memprof::MemProfSummary *getMemProfSummary() const {
+    return MemProfReader.getSummary();
+  }
+
   Error readBinaryIds(std::vector<llvm::object::BuildID> &BinaryIds) override;
   Error printBinaryIds(raw_ostream &OS) override;
 };

llvm/include/llvm/ProfileData/InstrProfWriter.h

@@ -22,6 +22,7 @@
 #include "llvm/ProfileData/DataAccessProf.h"
 #include "llvm/ProfileData/IndexedMemProfData.h"
 #include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProfSummaryBuilder.h"
 #include "llvm/Support/Error.h"
 #include <cstdint>
 #include <memory>
@@ -84,6 +85,10 @@ class InstrProfWriter {
 
   std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData;
 
+  // MemProf summary builder to which records are added as MemProf data is added
+  // to the writer.
+  memprof::MemProfSummaryBuilder MemProfSumBuilder;
+
 public:
   // For memprof testing, random hotness can be assigned to the contexts if
   // MemprofGenerateRandomHotness is enabled. The random seed can be either

llvm/include/llvm/ProfileData/MemProfSummary.h

@@ -0,0 +1,70 @@
+//===- MemProfSummary.h - MemProf summary support ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains MemProf summary support and related interfaces.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_MEMPROFSUMMARY_H
+#define LLVM_PROFILEDATA_MEMPROFSUMMARY_H
+
+#include "llvm/IR/ModuleSummaryIndex.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ProfileData/MemProf.h"
+
+namespace llvm {
+namespace memprof {
+
+/// Return the allocation type for a given set of memory profile values.
+AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity,
+                            uint64_t AllocCount, uint64_t TotalLifetime);
+
+/// Helper to generate a single hash id for a given callstack, used for emitting
+/// matching statistics and useful for uniquing such statistics across modules.
+/// Also used to dedup contexts when computing the summary.
+uint64_t computeFullStackId(ArrayRef<Frame> CallStack);
+
+class MemProfSummary {
+private:
+  /// The number of summary fields below, which is used to enable some forwards
+  /// and backwards compatibility for the summary when serialized in the indexed
+  /// MemProf format. As long as no existing summary fields are removed or
+  /// reordered, and new summary fields are added after existing summary fields,
+  /// the MemProf indexed profile version does not need to be bumped to
+  /// accommodate new summary fields.
+  static constexpr unsigned NumSummaryFields = 6;
+
+  const uint64_t NumContexts, NumColdContexts, NumHotContexts;
+  const uint64_t MaxColdTotalSize, MaxWarmTotalSize, MaxHotTotalSize;
+
+public:
+  MemProfSummary(uint64_t NumContexts, uint64_t NumColdContexts,
+                 uint64_t NumHotContexts, uint64_t MaxColdTotalSize,
+                 uint64_t MaxWarmTotalSize, uint64_t MaxHotTotalSize)
+      : NumContexts(NumContexts), NumColdContexts(NumColdContexts),
+        NumHotContexts(NumHotContexts), MaxColdTotalSize(MaxColdTotalSize),
+        MaxWarmTotalSize(MaxWarmTotalSize), MaxHotTotalSize(MaxHotTotalSize) {}
+
+  static constexpr unsigned getNumSummaryFields() { return NumSummaryFields; }
+  uint64_t getNumContexts() const { return NumContexts; }
+  uint64_t getNumColdContexts() const { return NumColdContexts; }
+  uint64_t getNumHotContexts() const { return NumHotContexts; }
+  uint64_t getMaxColdTotalSize() const { return MaxColdTotalSize; }
+  uint64_t getMaxWarmTotalSize() const { return MaxWarmTotalSize; }
+  uint64_t getMaxHotTotalSize() const { return MaxHotTotalSize; }
+  void printSummaryYaml(raw_ostream &OS) const;
+  /// Write to indexed MemProf profile.
+  void write(ProfOStream &OS) const;
+  /// Read from indexed MemProf profile.
+  static std::unique_ptr<MemProfSummary> deserialize(const unsigned char *&);
+};
+
+} // namespace memprof
+} // namespace llvm
+
+#endif // LLVM_PROFILEDATA_MEMPROFSUMMARY_H

llvm/include/llvm/ProfileData/MemProfSummaryBuilder.h

@@ -0,0 +1,51 @@
+//===- MemProfSummaryBuilder.h - MemProf summary building -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains MemProf summary builder.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H
+#define LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H
+
+#include "llvm/ProfileData/MemProf.h"
+#include "llvm/ProfileData/MemProfSummary.h"
+
+namespace llvm {
+namespace memprof {
+
+class MemProfSummaryBuilder {
+private:
+  // The set of full context IDs that we've recorded so far. This is needed to
+  // dedup the MIBs, which are duplicated between functions containing inline
+  // instances of the same allocations.
+  DenseSet<uint64_t> Contexts;
+
+  void addRecord(uint64_t, const PortableMemInfoBlock &);
+
+protected:
+  uint64_t MaxColdTotalSize = 0;
+  uint64_t MaxWarmTotalSize = 0;
+  uint64_t MaxHotTotalSize = 0;
+  uint64_t NumContexts = 0;
+  uint64_t NumColdContexts = 0;
+  uint64_t NumHotContexts = 0;
+
+public:
+  MemProfSummaryBuilder() = default;
+  ~MemProfSummaryBuilder() = default;
+
+  void addRecord(const IndexedMemProfRecord &);
+  void addRecord(const MemProfRecord &);
+  std::unique_ptr<MemProfSummary> getSummary();
+};
+
+} // namespace memprof
+} // namespace llvm
+
+#endif // LLVM_PROFILEDATA_MEMPROFSUMMARYBUILDER_H

llvm/lib/Analysis/MemoryProfileInfo.cpp

@@ -21,34 +21,6 @@ using namespace llvm::memprof;
 
 #define DEBUG_TYPE "memory-profile-info"
 
-// Upper bound on lifetime access density (accesses per byte per lifetime sec)
-// for marking an allocation cold.
-LLVM_ABI cl::opt<float> MemProfLifetimeAccessDensityColdThreshold(
-    "memprof-lifetime-access-density-cold-threshold", cl::init(0.05),
-    cl::Hidden,
-    cl::desc("The threshold the lifetime access density (accesses per byte per "
-             "lifetime sec) must be under to consider an allocation cold"));
-
-// Lower bound on lifetime to mark an allocation cold (in addition to accesses
-// per byte per sec above). This is to avoid pessimizing short lived objects.
-LLVM_ABI cl::opt<unsigned> MemProfAveLifetimeColdThreshold(
-    "memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden,
-    cl::desc("The average lifetime (s) for an allocation to be considered "
-             "cold"));
-
-// Lower bound on average lifetime accesses density (total life time access
-// density / alloc count) for marking an allocation hot.
-LLVM_ABI cl::opt<unsigned> MemProfMinAveLifetimeAccessDensityHotThreshold(
-    "memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000),
-    cl::Hidden,
-    cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an "
-             "allocation to be considered hot"));
-
-LLVM_ABI cl::opt<bool>
-    MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden,
-                       cl::desc("Enable use of hot hints (only supported for "
-                                "unambigously hot allocations)"));
-
 cl::opt<bool> MemProfReportHintedSizes(
     "memprof-report-hinted-sizes", cl::init(false), cl::Hidden,
     cl::desc("Report total allocation sizes of hinted allocations"));
@@ -73,28 +45,6 @@ cl::opt<unsigned> MinCallsiteColdBytePercent(
     cl::desc("Min percent of cold bytes at a callsite to discard non-cold "
              "contexts"));
 
-AllocationType llvm::memprof::getAllocType(uint64_t TotalLifetimeAccessDensity,
-                                           uint64_t AllocCount,
-                                           uint64_t TotalLifetime) {
-  // The access densities are multiplied by 100 to hold 2 decimal places of
-  // precision, so need to divide by 100.
-  if (((float)TotalLifetimeAccessDensity) / AllocCount / 100 <
-          MemProfLifetimeAccessDensityColdThreshold
-      // Lifetime is expected to be in ms, so convert the threshold to ms.
-      && ((float)TotalLifetime) / AllocCount >=
-             MemProfAveLifetimeColdThreshold * 1000)
-    return AllocationType::Cold;
-
-  // The access densities are multiplied by 100 to hold 2 decimal places of
-  // precision, so need to divide by 100.
-  if (MemProfUseHotHints &&
-      ((float)TotalLifetimeAccessDensity) / AllocCount / 100 >
-          MemProfMinAveLifetimeAccessDensityHotThreshold)
-    return AllocationType::Hot;
-
-  return AllocationType::NotCold;
-}
-
 MDNode *llvm::memprof::buildCallstackMetadata(ArrayRef<uint64_t> CallStack,
                                               LLVMContext &Ctx) {
   SmallVector<Metadata *, 8> StackVals;

llvm/lib/ProfileData/CMakeLists.txt

@@ -10,6 +10,8 @@ add_llvm_component_library(LLVMProfileData
   MemProf.cpp
   MemProfReader.cpp
   MemProfRadixTree.cpp
+  MemProfSummary.cpp
+  MemProfSummaryBuilder.cpp
   PGOCtxProfReader.cpp
   PGOCtxProfWriter.cpp
   ProfileSummaryBuilder.cpp

llvm/lib/ProfileData/IndexedMemProfData.cpp

@@ -15,6 +15,7 @@
 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/ProfileData/MemProf.h"
 #include "llvm/ProfileData/MemProfRadixTree.h"
+#include "llvm/ProfileData/MemProfSummary.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/OnDiskHashTable.h"
 
@@ -220,7 +221,8 @@ static Error writeMemProfRadixTreeBased(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion Version, bool MemProfFullSchema,
     std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData =
-        nullptr) {
+        nullptr,
+    memprof::MemProfSummary *MemProfSum = nullptr) {
   assert((Version == memprof::Version3 || Version == memprof::Version4) &&
          "Unsupported version for radix tree format");
 
@@ -229,9 +231,12 @@ static Error writeMemProfRadixTreeBased(
   OS.write(0ULL); // Reserve space for the memprof call stack payload offset.
   OS.write(0ULL); // Reserve space for the memprof record payload offset.
   OS.write(0ULL); // Reserve space for the memprof record table offset.
-  if (Version >= memprof::Version4)
+  if (Version >= memprof::Version4) {
     OS.write(0ULL); // Reserve space for the data access profile offset.
 
+    MemProfSum->write(OS);
+  }
+
   auto Schema = memprof::getHotColdSchema();
   if (MemProfFullSchema)
     Schema = memprof::getFullSchema();
@@ -297,25 +302,27 @@ static Error writeMemProfV3(ProfOStream &OS,
 static Error writeMemProfV4(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData) {
-  return writeMemProfRadixTreeBased(OS, MemProfData, memprof::Version4,
-                                    MemProfFullSchema,
-                                    std::move(DataAccessProfileData));
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum) {
+  return writeMemProfRadixTreeBased(
+      OS, MemProfData, memprof::Version4, MemProfFullSchema,
+      std::move(DataAccessProfileData), MemProfSum);
 }
 
 // Write out the MemProf data in a requested version.
 Error writeMemProf(
     ProfOStream &OS, memprof::IndexedMemProfData &MemProfData,
     memprof::IndexedVersion MemProfVersionRequested, bool MemProfFullSchema,
-    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData) {
+    std::unique_ptr<memprof::DataAccessProfData> DataAccessProfileData,
+    memprof::MemProfSummary *MemProfSum) {
   switch (MemProfVersionRequested) {
   case memprof::Version2:
     return writeMemProfV2(OS, MemProfData, MemProfFullSchema);
   case memprof::Version3:
     return writeMemProfV3(OS, MemProfData, MemProfFullSchema);
   case memprof::Version4:
     return writeMemProfV4(OS, MemProfData, MemProfFullSchema,
-                          std::move(DataAccessProfileData));
+                          std::move(DataAccessProfileData), MemProfSum);
   }
 
   return make_error<InstrProfError>(
@@ -395,9 +402,11 @@ Error IndexedMemProfReader::deserializeRadixTreeBased(
       support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
 
   uint64_t DataAccessProfOffset = 0;
-  if (Version == memprof::Version4)
+  if (Version >= memprof::Version4) {
     DataAccessProfOffset =
         support::endian::readNext<uint64_t, llvm::endianness::little>(Ptr);
+    MemProfSum = memprof::MemProfSummary::deserialize(Ptr);
+  }
 
   // Read the schema.
   auto SchemaOr = memprof::readMemProfSchema(Ptr);