[Memprof] Adds the option to collect AccessCountHistograms for memprof.

Adds compile time flag  -mllvm -memprof-histogram and runtime flag histogram=true|false to turn Histogram
collection on and off. The -memprof-histogram flag relies on -memprof-use-callbacks=true to work.

Updates shadow mapping logic in histogram mode from having one 8 byte counter for 64 bytes, to 1 byte
for 8 bytes, capped at 255. Only supports this granularity as of now.

Updates the RawMemprofReader and serializing MemoryInfoBlocks to binary format, including changing
to a new version of the raw binary format from version 3 to version 4.

Updates creating MemoryInfoBlocks with and without Histograms. When two MemoryInfoBlocks are merged,
AccessCounts are summed up and the shorter Histogram is removed.

Adds a memprof_histogram test case.

Initial commit for adding AccessCountHistograms up until RawProfile for memprof

Author: mattweingarten

compiler-rt/include/profile/MIBEntryDef.inc

@@ -51,3 +51,5 @@ MIBEntryDef(MaxAccessDensity = 22, MaxAccessDensity, uint32_t)
 MIBEntryDef(TotalLifetimeAccessDensity = 23, TotalLifetimeAccessDensity, uint64_t)
 MIBEntryDef(MinLifetimeAccessDensity = 24, MinLifetimeAccessDensity, uint32_t)
 MIBEntryDef(MaxLifetimeAccessDensity = 25, MaxLifetimeAccessDensity, uint32_t)
+MIBEntryDef(AccessHistogramSize = 26, AccessHistogramSize, uint32_t)
+MIBEntryDef(AccessHistogram = 27, AccessHistogram, uintptr_t)

compiler-rt/include/profile/MemProfData.inc

@@ -22,24 +22,27 @@
 #include <string.h>
 
 #ifdef _MSC_VER
-#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop))
+#define PACKED(...) __pragma(pack(push, 1)) __VA_ARGS__ __pragma(pack(pop))
 #else
 #define PACKED(...) __VA_ARGS__ __attribute__((__packed__))
 #endif
 
-// A 64-bit magic number to uniquely identify the raw binary memprof profile file.
-#define MEMPROF_RAW_MAGIC_64                                                                        \
-  ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 |          \
-   (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129)
+// A 64-bit magic number to uniquely identify the raw binary memprof profile
+// file.
+#define MEMPROF_RAW_MAGIC_64                                                   \
+  ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 |           \
+   (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | (uint64_t)'f' << 16 |           \
+   (uint64_t)'r' << 8 | (uint64_t)129)
 
 // The version number of the raw binary format.
-#define MEMPROF_RAW_VERSION 3ULL
+#define MEMPROF_RAW_VERSION 4ULL
 
 #define MEMPROF_BUILDID_MAX_SIZE 32ULL
 
 namespace llvm {
 namespace memprof {
-// A struct describing the header used for the raw binary memprof profile format.
+// A struct describing the header used for the raw binary memprof profile
+// format.
 PACKED(struct Header {
   uint64_t Magic;
   uint64_t Version;
@@ -62,15 +65,15 @@ PACKED(struct SegmentEntry {
   SegmentEntry(uint64_t S, uint64_t E, uint64_t O)
       : Start(S), End(E), Offset(O), BuildIdSize(0) {}
 
-  SegmentEntry(const SegmentEntry& S) {
+  SegmentEntry(const SegmentEntry &S) {
     Start = S.Start;
     End = S.End;
     Offset = S.Offset;
     BuildIdSize = S.BuildIdSize;
     memcpy(BuildId, S.BuildId, S.BuildIdSize);
   }
 
-  SegmentEntry& operator=(const SegmentEntry& S) {
+  SegmentEntry &operator=(const SegmentEntry &S) {
     Start = S.Start;
     End = S.End;
     Offset = S.Offset;
@@ -79,7 +82,7 @@ PACKED(struct SegmentEntry {
     return *this;
   }
 
-  bool operator==(const SegmentEntry& S) const {
+  bool operator==(const SegmentEntry &S) const {
     return Start == S.Start && End == S.End && Offset == S.Offset &&
            BuildIdSize == S.BuildIdSize &&
            memcmp(BuildId, S.BuildId, S.BuildIdSize) == 0;
@@ -90,111 +93,143 @@ PACKED(struct SegmentEntry {
 // MemProfData.inc since it would mean we are embedding a directive (the
 // #include for MIBEntryDef) into the macros which is undefined behaviour.
 #ifdef _MSC_VER
-__pragma(pack(push,1))
+__pragma(pack(push, 1))
 #endif
 
-// A struct representing the heap allocation characteristics of a particular
-// runtime context. This struct is shared between the compiler-rt runtime and
-// the raw profile reader. The indexed format uses a separate, self-describing
-// backwards compatible format.
-struct MemInfoBlock{
+    // A struct representing the heap allocation characteristics of a particular
+    // runtime context. This struct is shared between the compiler-rt runtime
+    // and the raw profile reader. The indexed format uses a separate,
+    // self-describing backwards compatible format.
+    struct MemInfoBlock {
 
 #define MIBEntryDef(NameTag, Name, Type) Type Name;
 #include "MIBEntryDef.inc"
 #undef MIBEntryDef
 
-bool operator==(const MemInfoBlock& Other) const {
-  bool IsEqual = true;
-#define MIBEntryDef(NameTag, Name, Type) \
+  bool operator==(const MemInfoBlock &Other) const {
+    bool IsEqual = true;
+#define MIBEntryDef(NameTag, Name, Type)                                       \
   IsEqual = (IsEqual && Name == Other.Name);
 #include "MIBEntryDef.inc"
 #undef MIBEntryDef
-  return IsEqual;
-}
+    return IsEqual;
+  }
 
-MemInfoBlock() {
+  MemInfoBlock() {
 #define MIBEntryDef(NameTag, Name, Type) Name = Type();
 #include "MIBEntryDef.inc"
 #undef MIBEntryDef
-}
-
-MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs,
-             uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu)
-    : MemInfoBlock() {
-  AllocCount = 1U;
-  TotalAccessCount = AccessCount;
-  MinAccessCount = AccessCount;
-  MaxAccessCount = AccessCount;
-  TotalSize = Size;
-  MinSize = Size;
-  MaxSize = Size;
-  AllocTimestamp = AllocTs;
-  DeallocTimestamp = DeallocTs;
-  TotalLifetime = DeallocTimestamp - AllocTimestamp;
-  MinLifetime = TotalLifetime;
-  MaxLifetime = TotalLifetime;
-  // Access density is accesses per byte. Multiply by 100 to include the
-  // fractional part.
-  TotalAccessDensity = AccessCount * 100 / Size;
-  MinAccessDensity = TotalAccessDensity;
-  MaxAccessDensity = TotalAccessDensity;
-  // Lifetime access density is the access density per second of lifetime.
-  // Multiply by 1000 to convert denominator lifetime to seconds (using a
-  // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first
-  // to reduce truncations to 0.
-  TotalLifetimeAccessDensity =
-      TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1);
-  MinLifetimeAccessDensity = TotalLifetimeAccessDensity;
-  MaxLifetimeAccessDensity = TotalLifetimeAccessDensity;
-  AllocCpuId = AllocCpu;
-  DeallocCpuId = DeallocCpu;
-  NumMigratedCpu = AllocCpuId != DeallocCpuId;
-}
-
-void Merge(const MemInfoBlock &newMIB) {
-  AllocCount += newMIB.AllocCount;
-
-  TotalAccessCount += newMIB.TotalAccessCount;
-  MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount;
-  MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount;
-
-  TotalSize += newMIB.TotalSize;
-  MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize;
-  MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize;
-
-  TotalLifetime += newMIB.TotalLifetime;
-  MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime;
-  MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime;
-
-  TotalAccessDensity += newMIB.TotalAccessDensity;
-  MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity
-                         ? newMIB.MinAccessDensity
-                         : MinAccessDensity;
-  MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity
-                         ? newMIB.MaxAccessDensity
-                         : MaxAccessDensity;
-
-  TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity;
-  MinLifetimeAccessDensity =
-      newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity
-          ? newMIB.MinLifetimeAccessDensity
-          : MinLifetimeAccessDensity;
-  MaxLifetimeAccessDensity =
-      newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity
-          ? newMIB.MaxLifetimeAccessDensity
-          : MaxLifetimeAccessDensity;
-
-  // We know newMIB was deallocated later, so just need to check if it was
-  // allocated before last one deallocated.
-  NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp;
-  AllocTimestamp = newMIB.AllocTimestamp;
-  DeallocTimestamp = newMIB.DeallocTimestamp;
-
-  NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId;
-  NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId;
-  AllocCpuId = newMIB.AllocCpuId;
-  DeallocCpuId = newMIB.DeallocCpuId;
-}
+  }
+
+  MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs,
+               uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu,
+               uintptr_t Histogram, uint32_t HistogramSize)
+      : MemInfoBlock() {
+    AllocCount = 1U;
+    TotalAccessCount = AccessCount;
+    MinAccessCount = AccessCount;
+    MaxAccessCount = AccessCount;
+    TotalSize = Size;
+    MinSize = Size;
+    MaxSize = Size;
+    AllocTimestamp = AllocTs;
+    DeallocTimestamp = DeallocTs;
+    TotalLifetime = DeallocTimestamp - AllocTimestamp;
+    MinLifetime = TotalLifetime;
+    MaxLifetime = TotalLifetime;
+    // Access density is accesses per byte. Multiply by 100 to include the
+    // fractional part.
+    TotalAccessDensity = AccessCount * 100 / Size;
+    MinAccessDensity = TotalAccessDensity;
+    MaxAccessDensity = TotalAccessDensity;
+    // Lifetime access density is the access density per second of lifetime.
+    // Multiply by 1000 to convert denominator lifetime to seconds (using a
+    // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first
+    // to reduce truncations to 0.
+    TotalLifetimeAccessDensity =
+        TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1);
+    MinLifetimeAccessDensity = TotalLifetimeAccessDensity;
+    MaxLifetimeAccessDensity = TotalLifetimeAccessDensity;
+    AllocCpuId = AllocCpu;
+    DeallocCpuId = DeallocCpu;
+    NumMigratedCpu = AllocCpuId != DeallocCpuId;
+    // For now we assume HistogramSize is the same as user requested size
+    AccessHistogramSize = HistogramSize;
+    AccessHistogram = Histogram;
+  }
+
+  // Merge cannot free the AccessHistogram pointer, since we need to free either
+  // with InternalFree or free depending on where the allocation is made
+  // (runtime or profdata tool). The merge function expects the Histogram
+  // pointer with the smaller size to be freed.
+  void Merge(const MemInfoBlock &newMIB) {
+    AllocCount += newMIB.AllocCount;
+
+    TotalAccessCount += newMIB.TotalAccessCount;
+    MinAccessCount = newMIB.MinAccessCount < MinAccessCount
+                         ? newMIB.MinAccessCount
+                         : MinAccessCount;
+    MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount
+                         ? newMIB.MaxAccessCount
+                         : MaxAccessCount;
+
+    TotalSize += newMIB.TotalSize;
+    MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize;
+    MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize;
+
+    TotalLifetime += newMIB.TotalLifetime;
+    MinLifetime =
+        newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime;
+    MaxLifetime =
+        newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime;
+
+    TotalAccessDensity += newMIB.TotalAccessDensity;
+    MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity
+                           ? newMIB.MinAccessDensity
+                           : MinAccessDensity;
+    MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity
+                           ? newMIB.MaxAccessDensity
+                           : MaxAccessDensity;
+
+    TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity;
+    MinLifetimeAccessDensity =
+        newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity
+            ? newMIB.MinLifetimeAccessDensity
+            : MinLifetimeAccessDensity;
+    MaxLifetimeAccessDensity =
+        newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity
+            ? newMIB.MaxLifetimeAccessDensity
+            : MaxLifetimeAccessDensity;
+
+    // We know newMIB was deallocated later, so just need to check if it was
+    // allocated before last one deallocated.
+    NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp;
+    AllocTimestamp = newMIB.AllocTimestamp;
+    DeallocTimestamp = newMIB.DeallocTimestamp;
+
+    NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId;
+    NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId;
+    AllocCpuId = newMIB.AllocCpuId;
+    DeallocCpuId = newMIB.DeallocCpuId;
+
+    // For merging histograms, we always keep the longer histogram, and add
+    // values of shorter histogram to larger one.
+    uintptr_t ShorterHistogram;
+    uint32_t ShorterHistogramSize;
+    if (newMIB.AccessHistogramSize > AccessHistogramSize) {
+      ShorterHistogram = AccessHistogram;
+      ShorterHistogramSize = AccessHistogramSize;
+      // Swap histogram of current to larger histogram
+      AccessHistogram = newMIB.AccessHistogram;
+      AccessHistogramSize = newMIB.AccessHistogramSize;
+    } else {
+      ShorterHistogram = newMIB.AccessHistogram;
+      ShorterHistogramSize = newMIB.AccessHistogramSize;
+    }
+    for (size_t i = 0; i < ShorterHistogramSize; ++i) {
+      ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i];
+    }
+  }
 
 #ifdef _MSC_VER
 } __pragma(pack(pop));
@@ -205,4 +240,4 @@ void Merge(const MemInfoBlock &newMIB) {
 } // namespace memprof
 } // namespace llvm
 
-#endif
+#endif