Skip to content

Commit c9a0177

Browse files
xur-llvmxur-llvm
committed
[PGO] Differentiate Clang instrumentation and IR level instrumentation profiles
This patch uses one bit in profile version to differentiate Clang instrumentation and IR level instrumentation profiles. PGOInstrumenation generates a COMDAT variable __llvm_profile_raw_version so that the compiler runtime can set the right profile kind. PGOInstrumenation now checks this bit to make sure it's an IR level instrumentation profile. Differential Revision: http://reviews.llvm.org/D15540 git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@260146 91177308-0d34-0410-b5e6-96231b3b80d8
1 parent ab75a17 commit c9a0177

25 files changed

+147
-13
lines changed

‎include/llvm/ProfileData/InstrProfData.inc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -705,9 +705,11 @@ serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record,
705705
* version for other variants of profile. We set the lowest bit of the upper 8
706706
* bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton
707707
* generated profile, and 0 if this is a Clang FE generated profile.
708-
*/
708+
*/
709709
#define VARIANT_MASKS_ALL 0xff00000000000000ULL
710710
#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL)
711+
#define VARIANT_MASK_IR_PROF (0x1ULL << 56)
712+
#define IR_LEVEL_PROF_VERSION_VAR __llvm_profile_raw_version
711713

712714
/* Runtime section names and name strings. */
713715
#define INSTR_PROF_DATA_SECT_NAME __llvm_prf_data

‎include/llvm/ProfileData/InstrProfReader.h

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ class InstrProfReader {
6464
/// Iterator over profile data.
6565
InstrProfIterator begin() { return InstrProfIterator(this); }
6666
InstrProfIterator end() { return InstrProfIterator(); }
67+
virtual bool isIRLevelProfile() const = 0;
6768

6869
/// Return the PGO symtab. There are three different readers:
6970
/// Raw, Text, and Indexed profile readers. The first two types
@@ -118,18 +119,22 @@ class TextInstrProfReader : public InstrProfReader {
118119
std::unique_ptr<MemoryBuffer> DataBuffer;
119120
/// Iterator over the profile data.
120121
line_iterator Line;
122+
bool IsIRLevelProfile;
121123

122124
TextInstrProfReader(const TextInstrProfReader &) = delete;
123125
TextInstrProfReader &operator=(const TextInstrProfReader &) = delete;
124126
std::error_code readValueProfileData(InstrProfRecord &Record);
125127

126128
public:
127129
TextInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer_)
128-
: DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {}
130+
: DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#'),
131+
IsIRLevelProfile(false) {}
129132

130133
/// Return true if the given buffer is in text instrprof format.
131134
static bool hasFormat(const MemoryBuffer &Buffer);
132135

136+
bool isIRLevelProfile() const override { return IsIRLevelProfile; }
137+
133138
/// Read the header.
134139
std::error_code readHeader() override;
135140
/// Read a single record.
@@ -154,6 +159,10 @@ class RawInstrProfReader : public InstrProfReader {
154159
/// The profile data file contents.
155160
std::unique_ptr<MemoryBuffer> DataBuffer;
156161
bool ShouldSwapBytes;
162+
// The value of the version field of the raw profile data header. The lower 56
163+
// bits specifies the format version and the most significant 8 bits specify
164+
// the variant types of the profile.
165+
uint64_t Version;
157166
uint64_t CountersDelta;
158167
uint64_t NamesDelta;
159168
const RawInstrProf::ProfileData<IntPtrT> *Data;
@@ -177,6 +186,9 @@ class RawInstrProfReader : public InstrProfReader {
177186
static bool hasFormat(const MemoryBuffer &DataBuffer);
178187
std::error_code readHeader() override;
179188
std::error_code readNextRecord(InstrProfRecord &Record) override;
189+
bool isIRLevelProfile() const override {
190+
return (Version & VARIANT_MASK_IR_PROF) != 0;
191+
}
180192

181193
InstrProfSymtab &getSymtab() override {
182194
assert(Symtab.get());
@@ -292,6 +304,7 @@ struct InstrProfReaderIndexBase {
292304
virtual void setValueProfDataEndianness(support::endianness Endianness) = 0;
293305
virtual ~InstrProfReaderIndexBase() {}
294306
virtual uint64_t getVersion() const = 0;
307+
virtual bool isIRLevelProfile() const = 0;
295308
virtual void populateSymtab(InstrProfSymtab &) = 0;
296309
};
297310

@@ -323,7 +336,10 @@ class InstrProfReaderIndex : public InstrProfReaderIndexBase {
323336
HashTable->getInfoObj().setValueProfDataEndianness(Endianness);
324337
}
325338
~InstrProfReaderIndex() override {}
326-
uint64_t getVersion() const override { return FormatVersion; }
339+
uint64_t getVersion() const override { return GET_VERSION(FormatVersion); }
340+
bool isIRLevelProfile() const override {
341+
return (FormatVersion & VARIANT_MASK_IR_PROF) != 0;
342+
}
327343
void populateSymtab(InstrProfSymtab &Symtab) override {
328344
Symtab.create(HashTable->keys());
329345
}
@@ -348,7 +364,9 @@ class IndexedInstrProfReader : public InstrProfReader {
348364
const unsigned char *Cur);
349365

350366
public:
367+
/// Return the profile version.
351368
uint64_t getVersion() const { return Index->getVersion(); }
369+
bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); }
352370
IndexedInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer)
353371
: DataBuffer(std::move(DataBuffer)), Index(nullptr) {}
354372

‎include/llvm/ProfileData/InstrProfWriter.h

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,12 @@ class InstrProfRecordWriterTrait;
3030
class InstrProfWriter {
3131
public:
3232
typedef SmallDenseMap<uint64_t, InstrProfRecord, 1> ProfilingData;
33+
enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel };
3334

3435
private:
3536
bool Sparse;
3637
StringMap<ProfilingData> FunctionData;
38+
ProfKind ProfileKind;
3739
// Use raw pointer here for the incomplete type object.
3840
InstrProfRecordWriterTrait *InfoObj;
3941

@@ -55,6 +57,16 @@ class InstrProfWriter {
5557
/// Write the profile, returning the raw data. For testing.
5658
std::unique_ptr<MemoryBuffer> writeBuffer();
5759

60+
/// Set the ProfileKind. Report error if mixing FE and IR level profiles.
61+
std::error_code setIsIRLevelProfile(bool IsIRLevel) {
62+
if (ProfileKind == PF_Unknown) {
63+
ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE;
64+
return instrprof_error::success;
65+
}
66+
return (IsIRLevel == (ProfileKind == PF_IRLevel)) ?
67+
instrprof_error::success : instrprof_error::unsupported_version;
68+
}
69+
5870
// Internal interface for testing purpose only.
5971
void setValueProfDataEndianness(support::endianness Endianness);
6072
void setOutputSparse(bool Sparse);

‎lib/ProfileData/InstrProfReader.cpp

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,8 +109,26 @@ bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) {
109109
[](char c) { return ::isprint(c) || ::isspace(c); });
110110
}
111111

112+
// Read the profile variant flag from the header: ":FE" means this is a FE
113+
// generated profile. ":IR" means this is an IR level profile. Other strings
114+
// with a leading ':' will be reported an error format.
112115
std::error_code TextInstrProfReader::readHeader() {
113116
Symtab.reset(new InstrProfSymtab());
117+
bool IsIRInstr = false;
118+
if (!Line->startswith(":")) {
119+
IsIRLevelProfile = false;
120+
return success();
121+
}
122+
StringRef Str = (Line)->substr(1);
123+
if (Str.equals_lower("ir"))
124+
IsIRInstr = true;
125+
else if (Str.equals_lower("fe"))
126+
IsIRInstr = false;
127+
else
128+
return instrprof_error::bad_header;
129+
130+
++Line;
131+
IsIRLevelProfile = IsIRInstr;
114132
return success();
115133
}
116134

@@ -293,7 +311,8 @@ void RawInstrProfReader<IntPtrT>::createSymtab(InstrProfSymtab &Symtab) {
293311
template <class IntPtrT>
294312
std::error_code
295313
RawInstrProfReader<IntPtrT>::readHeader(const RawInstrProf::Header &Header) {
296-
if (swap(Header.Version) != RawInstrProf::Version)
314+
Version = swap(Header.Version);
315+
if (GET_VERSION(Version) != RawInstrProf::Version)
297316
return error(instrprof_error::unsupported_version);
298317

299318
CountersDelta = swap(Header.CountersDelta);
@@ -470,10 +489,10 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
470489
return data_type();
471490
uint64_t Hash = endian::readNext<uint64_t, little, unaligned>(D);
472491

473-
// Initialize number of counters for FormatVersion == 1.
492+
// Initialize number of counters for GET_VERSION(FormatVersion) == 1.
474493
uint64_t CountsSize = N / sizeof(uint64_t) - 1;
475494
// If format version is different then read the number of counters.
476-
if (FormatVersion != IndexedInstrProf::ProfVersion::Version1) {
495+
if (GET_VERSION(FormatVersion) != IndexedInstrProf::ProfVersion::Version1) {
477496
if (D + sizeof(uint64_t) > End)
478497
return data_type();
479498
CountsSize = endian::readNext<uint64_t, little, unaligned>(D);
@@ -490,7 +509,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D,
490509
DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer));
491510

492511
// Read value profiling data.
493-
if (FormatVersion > IndexedInstrProf::ProfVersion::Version2 &&
512+
if (GET_VERSION(FormatVersion) > IndexedInstrProf::ProfVersion::Version2 &&
494513
!readValueProfilingData(D, End)) {
495514
DataBuffer.clear();
496515
return data_type();
@@ -603,7 +622,8 @@ std::error_code IndexedInstrProfReader::readHeader() {
603622

604623
// Read the version.
605624
uint64_t FormatVersion = endian::byte_swap<uint64_t, little>(Header->Version);
606-
if (FormatVersion > IndexedInstrProf::ProfVersion::CurrentVersion)
625+
if (GET_VERSION(FormatVersion) >
626+
IndexedInstrProf::ProfVersion::CurrentVersion)
607627
return error(instrprof_error::unsupported_version);
608628

609629
Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur);

‎lib/ProfileData/InstrProfWriter.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ class InstrProfRecordWriterTrait {
142142
}
143143

144144
InstrProfWriter::InstrProfWriter(bool Sparse)
145-
: Sparse(Sparse), FunctionData(),
145+
: Sparse(Sparse), FunctionData(), ProfileKind(PF_Unknown),
146146
InfoObj(new InstrProfRecordWriterTrait()) {}
147147

148148
InstrProfWriter::~InstrProfWriter() { delete InfoObj; }
@@ -230,6 +230,8 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
230230
IndexedInstrProf::Header Header;
231231
Header.Magic = IndexedInstrProf::Magic;
232232
Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
233+
if (ProfileKind == PF_IRLevel)
234+
Header.Version |= VARIANT_MASK_IR_PROF;
233235
Header.Unused = 0;
234236
Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
235237
Header.HashOffset = 0;
@@ -336,6 +338,8 @@ void InstrProfWriter::writeRecordInText(const InstrProfRecord &Func,
336338
}
337339

338340
void InstrProfWriter::writeText(raw_fd_ostream &OS) {
341+
if (ProfileKind == PF_IRLevel)
342+
OS << "# IR level Instrumentation Flag\n:ir\n";
339343
InstrProfSymtab Symtab;
340344
for (const auto &I : FunctionData)
341345
if (shouldEncodeData(I.getValue()))

‎lib/Transforms/Instrumentation/PGOInstrumentation.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,7 +713,22 @@ void PGOUseFunc::setBranchWeights() {
713713
}
714714
} // end anonymous namespace
715715

716+
// Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime
717+
// aware this is an ir_level profile so it can set the version flag.
718+
static void createIRLevelProfileFlagVariable(Module &M) {
719+
Type *IntTy64 = Type::getInt64Ty(M.getContext());
720+
uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION | VARIANT_MASK_IR_PROF);
721+
auto IRLevelVersionVariable =
722+
new GlobalVariable(M, IntTy64, true, GlobalVariable::ExternalLinkage,
723+
Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
724+
INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR));
725+
IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
726+
IRLevelVersionVariable->setComdat(
727+
M.getOrInsertComdat(StringRef(INSTR_PROF_QUOTE(IR_LEVEL_PROF_VERSION_VAR))));
728+
}
729+
716730
bool PGOInstrumentationGen::runOnModule(Module &M) {
731+
createIRLevelProfileFlagVariable(M);
717732
for (auto &F : M) {
718733
if (F.isDeclaration())
719734
continue;
@@ -751,6 +766,13 @@ bool PGOInstrumentationUse::runOnModule(Module &M) {
751766
"Cannot get PGOReader"));
752767
return false;
753768
}
769+
// TODO: might need to change the warning once the clang option is finalized.
770+
if (!PGOReader->isIRLevelProfile()) {
771+
Ctx.diagnose(DiagnosticInfoPGOProfile(
772+
ProfileFileName.data(), "Not an IR level instrumentation profile"));
773+
return false;
774+
}
775+
754776

755777
for (auto &F : M) {
756778
if (F.isDeclaration())

‎test/Transforms/PGOProfile/Inputs/branch1.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_br_1
24
25571299074
35
2

‎test/Transforms/PGOProfile/Inputs/branch2.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_br_2
24
29667547796
35
2

‎test/Transforms/PGOProfile/Inputs/criticaledge.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_criticalEdge
24
82323253069
35
8

‎test/Transforms/PGOProfile/Inputs/diag.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
foo
24
12884999999
35
1

‎test/Transforms/PGOProfile/Inputs/diag_FE.proftext

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
foo
2+
12884999999
3+
1
4+
1
5+

‎test/Transforms/PGOProfile/Inputs/landingpad.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
foo
24
59130013419
35
4

‎test/Transforms/PGOProfile/Inputs/loop1.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_simple_for
24
34137660316
35
2

‎test/Transforms/PGOProfile/Inputs/loop2.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_nested_for
24
53929068288
35
3

‎test/Transforms/PGOProfile/Inputs/switch.proftext

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
# :ir is the flag to indicate this is IR level profile.
2+
:ir
13
test_switch
24
46200943743
35
4

‎test/Transforms/PGOProfile/branch1.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_br_1 = private constant [9 x i8] c"test_br_1"
810

911
define i32 @test_br_1(i32 %i) {

‎test/Transforms/PGOProfile/branch2.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_br_2 = private constant [9 x i8] c"test_br_2"
810

911
define i32 @test_br_2(i32 %i) {

‎test/Transforms/PGOProfile/criticaledge.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_criticalEdge = private constant [17 x i8] c"test_criticalEdge"
810
; GEN: @__profn__stdin__bar = private constant [11 x i8] c"<stdin>:bar"
911

‎test/Transforms/PGOProfile/diag_FE_profile.ll

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
; RUN: llvm-profdata merge %S/Inputs/diag_FE.proftext -o %t.profdata
2+
; RUN: not opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S 2>&1 | FileCheck %s
3+
4+
; CHECK: Not an IR level instrumentation profile
5+
6+
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
7+
target triple = "x86_64-unknown-linux-gnu"
8+
9+
define i32 @foo() {
10+
entry:
11+
ret i32 0
12+
}

‎test/Transforms/PGOProfile/landingpad.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ target triple = "x86_64-unknown-linux-gnu"
66

77
@val = global i32 0, align 4
88
@_ZTIi = external constant i8*
9+
; GEN: $__llvm_profile_raw_version = comdat any
10+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
911
; GEN: @__profn_bar = private constant [3 x i8] c"bar"
1012
; GEN: @__profn_foo = private constant [3 x i8] c"foo"
1113

‎test/Transforms/PGOProfile/loop1.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_simple_for = private constant [15 x i8] c"test_simple_for"
810

911
define i32 @test_simple_for(i32 %n) {

‎test/Transforms/PGOProfile/loop2.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_nested_for = private constant [15 x i8] c"test_nested_for"
810

911
define i32 @test_nested_for(i32 %r, i32 %s) {

‎test/Transforms/PGOProfile/single_bb.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
33
target triple = "x86_64-unknown-linux-gnu"
44

5+
; GEN: $__llvm_profile_raw_version = comdat any
6+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
57
; GEN: @__profn_single_bb = private constant [9 x i8] c"single_bb"
68

79
define i32 @single_bb() {

‎test/Transforms/PGOProfile/switch.ll

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
55
target triple = "x86_64-unknown-linux-gnu"
66

7+
; GEN: $__llvm_profile_raw_version = comdat any
8+
; GEN: @__llvm_profile_raw_version = constant i64 72057594037927939, comdat
79
; GEN: @__profn_test_switch = private constant [11 x i8] c"test_switch"
810

911
define void @test_switch(i32 %i) {

0 commit comments

Comments
 (0)