Skip to content

Commit ebd1d70

Browse files
kazutakahiratakazutakahirata
committed
[memprof] Add CallStackRadixTreeBuilder
Call stacks are a huge portion of the MemProf profile, taking up 70+% of the profile file size. This patch implements a radix tree to compress call stacks, which are known to have long common prefixes. Specifically, CallStackRadixTreeBuilder, introduced in this patch, takes call stacks in the MemProf profile, sorts them in the dictionary order to maximize the common prefix between adjacent call stacks, and then encodes a radix tree into a single array that is ready for serialization. The resulting radix array is essentially a concatenation of call stack arrays, each encoded with its length followed by the payload, except that these arrays contain "instructions" like "skip 7 elements forward" to borrow common prefixes from other call stacks. This patch does not integrate with the MemProf serialization/deserialization infrastructure yet. Once integrated, the radix tree is expected to roughly halve the file size of the MemProf profile.
1 parent ad884d9 commit ebd1d70

File tree

3 files changed

+362
-0
lines changed

3 files changed

+362
-0
lines changed

‎llvm/include/llvm/ProfileData/MemProf.h

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,106 @@ struct IndexedMemProfData {
905905
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> CallStackData;
906906
};
907907

908+
// Construct a radix tree of call stacks.
909+
//
910+
// A set of call stacks might look like:
911+
//
912+
// CallStackId 1: f1 -> f2 -> f3
913+
// CallStackId 2: f1 -> f2 -> f4 -> f5
914+
// CallStackId 3: f1 -> f2 -> f4 -> f6
915+
// CallStackId 4: f7 -> f8 -> f9
916+
//
917+
// where each fn refers to a stack frame.
918+
//
919+
// Since we expect a lot of common prefixes, we can compress the call stacks
920+
// into a radix tree like:
921+
//
922+
// CallStackId 1: f1 -> f2 -> f3
923+
// |
924+
// CallStackId 2: +---> f4 -> f5
925+
// |
926+
// CallStackId 3: +---> f6
927+
//
928+
// CallStackId 4: f7 -> f8 -> f9
929+
//
930+
// Now, we are interested in retrieving call stacks for a given CallStackId, so
931+
// we just need a pointer from a given call stack to its parent. For example,
932+
// CallStackId 2 would point to CallStackId 1 as a parent.
933+
//
934+
// We serialize the radix tree above into a single array along with the length
935+
// of each call stack and pointers to the parent call stacks.
936+
//
937+
// Index: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
938+
// Array: L3 f9 f8 f7 L4 f6 J3 L4 f5 f4 J3 L3 f3 f2 f1
939+
// ^ ^ ^ ^
940+
// | | | |
941+
// CallStackId 4: 0 --+ | | |
942+
// CallStackId 3: 4 --------------+ | |
943+
// CallStackId 2: 7 -----------------------+ |
944+
// CallStackId 1: 11 -----------------------------------+
945+
//
946+
// - LN indicates the length of a call stack, encoded as ordinary integer N.
947+
//
948+
// - JN indicates a pointer to the parent, encoded as -N.
949+
//
950+
// For example, if we are decoding CallStackId 2, we start a forward traversal
951+
// at Index 7, noting the call stack length of 4 and obtaining f5 and f4. When
952+
// we see J3 at Index 10, we resume a forward traversal at Index 13 = 10 + 3,
953+
// picking up f2 and f1. We are done after collecting 4 frames as indicated at
954+
// the beginning of the traversal.
955+
//
956+
// On-disk IndexedMemProfRecord will refer to call stacks by their indexes into
957+
// the radix tree array, so we do not explicitly encode mappings like:
958+
// "CallStackId 1 -> 11".
959+
class CallStackRadixTreeBuilder {
960+
// The radix tree array.
961+
std::vector<uint32_t> RadixArray;
962+
963+
// Mapping from CallStackIds to indexes into RadixArray.
964+
llvm::DenseMap<CallStackId, uint32_t> CallStackPos;
965+
966+
// The indexes within RadixArray of the last call stack's frames encoded
967+
// satisfying:
968+
//
969+
// RadixArray[Indexes[I]] == (*Prev)[Prev->size() - I - 1]
970+
//
971+
// where Prev is one of the parameters to build.
972+
std::vector<uint32_t> Indexes;
973+
974+
using CSIdPair = std::pair<CallStackId, llvm::SmallVector<FrameId> *>;
975+
976+
// Returns the sorted list of call stacks.
977+
std::vector<CSIdPair>
978+
sortCallStacks(llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
979+
&MemProfCallStackData);
980+
981+
// Encode a call stack into RadixArray. Return the starting index within
982+
// RadixArray.
983+
uint32_t
984+
encodeCallStack(const llvm::SmallVector<FrameId> *CallStack,
985+
const llvm::SmallVector<FrameId> *Prev,
986+
const llvm::DenseMap<FrameId, uint32_t> &MemProfFrameIndexes);
987+
988+
public:
989+
CallStackRadixTreeBuilder() = default;
990+
991+
void build(llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
992+
&MemProfCallStackData,
993+
const llvm::DenseMap<FrameId, uint32_t> &MemProfFrameIndexes);
994+
995+
const std::vector<uint32_t> &getRadixArray() const { return RadixArray; }
996+
997+
const llvm::DenseMap<CallStackId, uint32_t> &getCallStackPos() const {
998+
return CallStackPos;
999+
}
1000+
};
1001+
1002+
llvm::DenseMap<CallStackId, uint32_t>
1003+
writeCallStackRadixTree(raw_ostream &OS,
1004+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
1005+
&MemProfCallStackData,
1006+
llvm::DenseMap<FrameId, uint32_t> &MemProfFrameIndexes);
1007+
9081008
// Verify that each CallStackId is computed with hashCallStack. This function
9091009
// is intended to help transition from CallStack to CSId in
9101010
// IndexedAllocationInfo.

‎llvm/lib/ProfileData/MemProf.cpp

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,161 @@ CallStackId hashCallStack(ArrayRef<FrameId> CS) {
410410
return CSId;
411411
}
412412

413+
// Returns the sorted list of call stacks.
414+
std::vector<CallStackRadixTreeBuilder::CSIdPair>
415+
CallStackRadixTreeBuilder::sortCallStacks(
416+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
417+
&MemProfCallStackData) {
418+
// Create a list of call stacks to be sorted.
419+
std::vector<CSIdPair> CallStacks;
420+
CallStacks.reserve(MemProfCallStackData.size());
421+
for (auto &[CSId, CallStack] : MemProfCallStackData) {
422+
CallStacks.emplace_back(CSId, &CallStack);
423+
}
424+
425+
// Sort the list of call stacks in the dictionary order to maximize the length
426+
// of the common prefix between two adjacent call stacks.
427+
auto LessThan = [&](const CSIdPair &L, const CSIdPair &R) {
428+
// Call stacks are stored from leaf to root. Perform comparisons from the
429+
// root.
430+
return std::lexicographical_compare(
431+
L.second->rbegin(), L.second->rend(), R.second->rbegin(),
432+
R.second->rend(), [&](FrameId F1, FrameId F2) { return F1 < F2; });
433+
};
434+
llvm::sort(CallStacks, LessThan);
435+
436+
return CallStacks;
437+
}
438+
439+
// Encode a call stack into RadixArray. Return the starting index within
440+
// RadixArray. For each call stack we encode, we emit two or three components
441+
// into RadixArray. If a given call stack doesn't have a common prefix relative
442+
// to the previous one, we emit:
443+
//
444+
// - the frames in the given call stack in the reverse order
445+
//
446+
// - the length of the given call stack
447+
//
448+
// If a given call stack has a non-empty common prefix relative to the previous
449+
// one, we emit:
450+
//
451+
// - the relative location of the common prefix, encoded as a negative number.
452+
//
453+
// - a portion of the given call stack that's beyond the common prefix
454+
//
455+
// - the length of the given call stack, including the length of the common
456+
// prefix.
457+
//
458+
// To quickly determine the location of the common prefix within RadixArray,
459+
// Indexes caches the indexes of the previous call stack's frames within
460+
// RadixArray.
461+
uint32_t CallStackRadixTreeBuilder::encodeCallStack(
462+
const llvm::SmallVector<FrameId> *CallStack,
463+
const llvm::SmallVector<FrameId> *Prev,
464+
const llvm::DenseMap<FrameId, uint32_t> &MemProfFrameIndexes) {
465+
// Compute the length of the common prefix between Prev and CallStack.
466+
uint32_t CommonLen = 0;
467+
if (Prev) {
468+
auto Pos = std::mismatch(Prev->rbegin(), Prev->rend(), CallStack->rbegin(),
469+
CallStack->rend());
470+
CommonLen = std::distance(CallStack->rbegin(), Pos.second);
471+
}
472+
473+
// Drop the portion beyond CommonLen.
474+
assert(CommonLen <= Indexes.size());
475+
Indexes.resize(CommonLen);
476+
477+
// Append a pointer to the parent.
478+
if (CommonLen) {
479+
uint32_t CurrentIndex = RadixArray.size();
480+
uint32_t ParentIndex = Indexes.back();
481+
// The offset to the parent must be negative because we are pointing to an
482+
// element we've already added to RadixArray.
483+
assert(ParentIndex < CurrentIndex);
484+
RadixArray.push_back(ParentIndex - CurrentIndex);
485+
}
486+
487+
// Copy the part of the call stack beyond the common prefix to RadixArray.
488+
assert(CommonLen <= CallStack->size());
489+
for (FrameId F : llvm::drop_begin(llvm::reverse(*CallStack), CommonLen)) {
490+
// Remember the index of F in RadixArray.
491+
Indexes.push_back(RadixArray.size());
492+
RadixArray.push_back(MemProfFrameIndexes.find(F)->second);
493+
}
494+
assert(CallStack->size() == Indexes.size());
495+
496+
// End with the call stack length.
497+
RadixArray.push_back(CallStack->size());
498+
499+
// Return the index within RadixArray where we can start reconstructing a
500+
// given call stack from.
501+
return RadixArray.size() - 1;
502+
}
503+
504+
// Build a radix tree array.
505+
void CallStackRadixTreeBuilder::build(
506+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>>
507+
&MemProfCallStackData,
508+
const llvm::DenseMap<FrameId, uint32_t> &MemProfFrameIndexes) {
509+
std::vector<CSIdPair> CallStacks = sortCallStacks(MemProfCallStackData);
510+
assert(CallStacks.size() == MemProfCallStackData.size());
511+
512+
// Reserve some reasonable amount of storage.
513+
RadixArray.clear();
514+
RadixArray.reserve(CallStacks.size() * 8);
515+
516+
// Indexes will grow as long as the longest call stack.
517+
Indexes.clear();
518+
Indexes.reserve(512);
519+
520+
// Compute the radix array. We encode one call stack at a time, computing the
521+
// longest prefix that's shared with the previous call stack we encode. For
522+
// each call stack we encode, we remember a mapping from CallStackId to its
523+
// position within RadixArray.
524+
//
525+
// As an optimization, we encode from the last call stack in CallStacks to
526+
// reduce the number of times we follow pointers to the parents. Consider the
527+
// list of call stacks that has been sorted in the dictionary order:
528+
//
529+
// Call Stack 1: F1
530+
// Call Stack 2: F1 -> F2
531+
// Call Stack 3: F1 -> F2 -> F3
532+
//
533+
// If we traversed CallStacks in the forward order, we would end up with a
534+
// radix tree like:
535+
//
536+
// Call Stack 1: F1
537+
// |
538+
// Call Stack 2: +---> F2
539+
// |
540+
// Call Stack 3: +---> F3
541+
//
542+
// Notice that each call stack jumps to the previous one. However, if we
543+
// traverse CallStacks in the reverse order, then Call Stack 3 has the
544+
// complete call stack encoded without any pointers. Call Stack 1 and 2 point
545+
// to appropriate prefixes of Call Stack 3.
546+
llvm::SmallVector<FrameId> *Prev = nullptr;
547+
for (const auto &[CSId, CallStack] : llvm::reverse(CallStacks)) {
548+
uint32_t Pos = encodeCallStack(CallStack, Prev, MemProfFrameIndexes);
549+
CallStackPos.insert({CSId, Pos});
550+
Prev = CallStack;
551+
}
552+
assert(CallStackPos.size() == MemProfCallStackData.size());
553+
554+
if (RadixArray.size() >= 2) {
555+
// Reverse the radix array in place. We do so mostly for intuitive
556+
// deserialization where we would read the length field and then the call
557+
// stack frames proper just like any other array deserialization, except
558+
// that we have occasional jumps to take advantage of prefixes.
559+
for (size_t I = 0, J = RadixArray.size() - 1; I < J; ++I, --J)
560+
std::swap(RadixArray[I], RadixArray[J]);
561+
562+
// "Reverse" the indexes stored in CallStackPos.
563+
for (auto &[K, V] : CallStackPos)
564+
V = RadixArray.size() - 1 - V;
565+
}
566+
}
567+
413568
void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record) {
414569
for (const auto &AS : Record.AllocSites) {
415570
assert(AS.CSId == hashCallStack(AS.CallStack));

‎llvm/unittests/ProfileData/MemProfTest.cpp

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -662,4 +662,111 @@ TEST(MemProf, MissingFrameId) {
662662
ASSERT_TRUE(FrameIdConv.LastUnmappedId.has_value());
663663
EXPECT_EQ(*FrameIdConv.LastUnmappedId, 3U);
664664
}
665+
666+
// Verify CallStackRadixTreeBuilder can handle empty inputs.
667+
TEST(MemProf, RadixTreeBuilderEmpty) {
668+
llvm::DenseMap<FrameId, uint32_t> MemProfFrameIndexes;
669+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
670+
llvm::memprof::CallStackRadixTreeBuilder Builder;
671+
Builder.build(MemProfCallStackData, MemProfFrameIndexes);
672+
ASSERT_THAT(Builder.getRadixArray(), testing::IsEmpty());
673+
const auto &Mappings = Builder.getCallStackPos();
674+
ASSERT_THAT(Mappings, testing::IsEmpty());
675+
}
676+
677+
// Verify CallStackRadixTreeBuilder can handle one trivial call stack.
678+
TEST(MemProf, RadixTreeBuilderOne) {
679+
llvm::DenseMap<FrameId, uint32_t> MemProfFrameIndexes = {
680+
{11, 1}, {12, 2}, {13, 3}};
681+
llvm::SmallVector<llvm::memprof::FrameId> CS1 = {13, 12, 11};
682+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
683+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
684+
llvm::memprof::CallStackRadixTreeBuilder Builder;
685+
Builder.build(MemProfCallStackData, MemProfFrameIndexes);
686+
EXPECT_THAT(Builder.getRadixArray(), testing::ElementsAreArray({
687+
3U, // Size of CS1,
688+
3U, // MemProfFrameIndexes[13]
689+
2U, // MemProfFrameIndexes[12]
690+
1U // MemProfFrameIndexes[11]
691+
}));
692+
const auto &Mappings = Builder.getCallStackPos();
693+
ASSERT_THAT(Mappings, SizeIs(1));
694+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
695+
llvm::memprof::hashCallStack(CS1), 0U)));
696+
}
697+
698+
// Verify CallStackRadixTreeBuilder can form a link between two call stacks.
699+
TEST(MemProf, RadixTreeBuilderTwo) {
700+
llvm::DenseMap<FrameId, uint32_t> MemProfFrameIndexes = {
701+
{11, 1}, {12, 2}, {13, 3}};
702+
llvm::SmallVector<llvm::memprof::FrameId> CS1 = {12, 11};
703+
llvm::SmallVector<llvm::memprof::FrameId> CS2 = {13, 12, 11};
704+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
705+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
706+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
707+
llvm::memprof::CallStackRadixTreeBuilder Builder;
708+
Builder.build(MemProfCallStackData, MemProfFrameIndexes);
709+
EXPECT_THAT(Builder.getRadixArray(),
710+
testing::ElementsAreArray({
711+
2U, // Size of CS1
712+
static_cast<uint32_t>(-3), // Jump 3 steps
713+
3U, // Size of CS2
714+
3U, // MemProfFrameIndexes[13]
715+
2U, // MemProfFrameIndexes[12]
716+
1U // MemProfFrameIndexes[11]
717+
}));
718+
const auto &Mappings = Builder.getCallStackPos();
719+
ASSERT_THAT(Mappings, SizeIs(2));
720+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
721+
llvm::memprof::hashCallStack(CS1), 0U)));
722+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
723+
llvm::memprof::hashCallStack(CS2), 2U)));
724+
}
725+
726+
// Verify CallStackRadixTreeBuilder can form a jump to a prefix that itself has
727+
// another jump to another prefix.
728+
TEST(MemProf, RadixTreeBuilderSuccessiveJumps) {
729+
llvm::DenseMap<FrameId, uint32_t> MemProfFrameIndexes = {
730+
{11, 1}, {12, 2}, {13, 3}, {14, 4}, {15, 5}, {16, 6}, {17, 7}, {18, 8},
731+
};
732+
llvm::SmallVector<llvm::memprof::FrameId> CS1 = {14, 13, 12, 11};
733+
llvm::SmallVector<llvm::memprof::FrameId> CS2 = {15, 13, 12, 11};
734+
llvm::SmallVector<llvm::memprof::FrameId> CS3 = {17, 16, 12, 11};
735+
llvm::SmallVector<llvm::memprof::FrameId> CS4 = {18, 16, 12, 11};
736+
llvm::MapVector<CallStackId, llvm::SmallVector<FrameId>> MemProfCallStackData;
737+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS1), CS1});
738+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS2), CS2});
739+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS3), CS3});
740+
MemProfCallStackData.insert({llvm::memprof::hashCallStack(CS4), CS4});
741+
llvm::memprof::CallStackRadixTreeBuilder Builder;
742+
Builder.build(MemProfCallStackData, MemProfFrameIndexes);
743+
EXPECT_THAT(Builder.getRadixArray(),
744+
testing::ElementsAreArray({
745+
4U, // Size of CS1
746+
4U, // MemProfFrameIndexes[14]
747+
static_cast<uint32_t>(-3), // Jump 3 steps
748+
4U, // Size of CS2
749+
5U, // MemProfFrameIndexes[15]
750+
3U, // MemProfFrameIndexes[13]
751+
static_cast<uint32_t>(-7), // Jump 7 steps
752+
4U, // Size of CS3
753+
7U, // MemProfFrameIndexes[17]
754+
static_cast<uint32_t>(-3), // Jump 3 steps
755+
4U, // Size of CS4
756+
8U, // MemProfFrameIndexes[18]
757+
6U, // MemProfFrameIndexes[16]
758+
2U, // MemProfFrameIndexes[12]
759+
1U // MemProfFrameIndexes[11]
760+
}));
761+
const auto &Mappings = Builder.getCallStackPos();
762+
ASSERT_THAT(Mappings, SizeIs(4));
763+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
764+
llvm::memprof::hashCallStack(CS1), 0U)));
765+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
766+
llvm::memprof::hashCallStack(CS2), 3U)));
767+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
768+
llvm::memprof::hashCallStack(CS3), 7U)));
769+
EXPECT_THAT(Mappings, testing::Contains(testing::Pair(
770+
llvm::memprof::hashCallStack(CS4), 10U)));
771+
}
665772
} // namespace

0 commit comments

Comments
 (0)