Skip to content

[Profile] Refactor profile correlation. #70712

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions clang/lib/CodeGen/BackendUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Passes/PassPlugin.h"
#include "llvm/Passes/StandardInstrumentations.h"
#include "llvm/ProfileData/InstrProfCorrelator.h"
#include "llvm/Support/BuryPointer.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/MemoryBuffer.h"
Expand All @@ -55,6 +56,7 @@
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/SubtargetFeature.h"
#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/HipStdPar/HipStdPar.h"
#include "llvm/Transforms/IPO/EmbedBitcodePass.h"
#include "llvm/Transforms/IPO/LowerTypeTests.h"
#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h"
Expand All @@ -78,7 +80,6 @@
#include "llvm/Transforms/Scalar/EarlyCSE.h"
#include "llvm/Transforms/Scalar/GVN.h"
#include "llvm/Transforms/Scalar/JumpThreading.h"
#include "llvm/Transforms/HipStdPar/HipStdPar.h"
#include "llvm/Transforms/Utils/Debugify.h"
#include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
#include "llvm/Transforms/Utils/ModuleUtils.h"
Expand All @@ -98,13 +99,18 @@ extern cl::opt<bool> PrintPipelinePasses;
static cl::opt<bool> ClSanitizeOnOptimizerEarlyEP(
"sanitizer-early-opt-ep", cl::Optional,
cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false));
}

extern cl::opt<bool> DebugInfoCorrelate;
extern cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate;
} // namespace llvm

namespace {

// Default filename used for profile generation.
std::string getDefaultProfileGenName() {
return DebugInfoCorrelate ? "default_%m.proflite" : "default_%m.profraw";
return DebugInfoCorrelate || ProfileCorrelate != InstrProfCorrelator::NONE
? "default_%m.proflite"
: "default_%m.profraw";
}

class EmitAssemblyHelper {
Expand Down Expand Up @@ -197,7 +203,7 @@ class EmitAssemblyHelper {
void EmitAssembly(BackendAction Action,
std::unique_ptr<raw_pwrite_stream> OS);
};
}
} // namespace

static SanitizerCoverageOptions
getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) {
Expand Down
4 changes: 4 additions & 0 deletions compiler-rt/lib/profile/InstrProfiling.c
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,7 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) {
}
lprofSetProfileDumped(0);
}

inline int hasCorrelation() {
return (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
}
6 changes: 6 additions & 0 deletions compiler-rt/lib/profile/InstrProfiling.h
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,9 @@ uint64_t __llvm_profile_get_magic(void);
/*! \brief Get the version of the file format. */
uint64_t __llvm_profile_get_version(void);

/*! \brief If the binary is compiled with profile correlation. */
int hasCorrelation();

/*! \brief Get the number of entries in the profile data section. */
uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
const __llvm_profile_data *End);
Expand All @@ -282,6 +285,9 @@ uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End);
uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
const char *End);

/*! \brief Get the size of the profile name section in bytes. */
uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End);

/* ! \brief Given the sizes of the data and counter information, return the
* number of padding bytes before and after the counters, and after the names,
* in the raw profile.
Expand Down
11 changes: 11 additions & 0 deletions compiler-rt/lib/profile/InstrProfilingBuffer.c
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ uint64_t __llvm_profile_get_size_for_buffer(void) {
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
const __llvm_profile_data *End) {
if (hasCorrelation())
return 0;
intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
sizeof(__llvm_profile_data);
Expand All @@ -64,6 +66,8 @@ uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
const __llvm_profile_data *End) {
if (hasCorrelation())
return 0;
return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
}

Expand Down Expand Up @@ -92,6 +96,13 @@ uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
return (End - Begin);
}

COMPILER_RT_VISIBILITY
uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
if (hasCorrelation())
return 0;
return End - Begin;
}

/// Calculate the number of padding bytes needed to add to \p Offset in order
/// for (\p Offset + Padding) to be page-aligned.
static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
Expand Down
11 changes: 6 additions & 5 deletions compiler-rt/lib/profile/InstrProfilingMerge.c
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,9 @@ int __llvm_profile_check_compatibility(const char *ProfileData,
Header->NumBitmapBytes !=
__llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
__llvm_profile_end_bitmap()) ||
Header->NamesSize != (uint64_t)(__llvm_profile_end_names() -
__llvm_profile_begin_names()) ||
Header->NamesSize !=
__llvm_profile_get_name_size(__llvm_profile_begin_names(),
__llvm_profile_end_names()) ||
Header->ValueKindLast != IPVK_Last)
return 1;

Expand Down Expand Up @@ -138,9 +139,9 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData,
if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
return 1;

// Merge counters by iterating the entire counter section when debug info
// correlation is enabled.
if (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) {
// Merge counters by iterating the entire counter section when correlation is
// enabled.
if (hasCorrelation()) {
for (SrcCounter = SrcCountersStart,
DstCounter = __llvm_profile_begin_counters();
SrcCounter < SrcCountersEnd;) {
Expand Down
21 changes: 10 additions & 11 deletions compiler-rt/lib/profile/InstrProfilingWriter.c
Original file line number Diff line number Diff line change
Expand Up @@ -262,21 +262,19 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
const char *BitmapBegin, const char *BitmapEnd,
VPDataReaderType *VPDataReader, const char *NamesBegin,
const char *NamesEnd, int SkipNameDataWrite) {
int DebugInfoCorrelate =
(__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
int ProfileCorrelation = hasCorrelation();

/* Calculate size of sections. */
const uint64_t DataSectionSize =
DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd);
const uint64_t NumData =
DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd);
__llvm_profile_get_data_size(DataBegin, DataEnd);
const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd);
const uint64_t CountersSectionSize =
__llvm_profile_get_counters_size(CountersBegin, CountersEnd);
const uint64_t NumCounters =
__llvm_profile_get_num_counters(CountersBegin, CountersEnd);
const uint64_t NumBitmapBytes =
__llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);

/* Create the header. */
__llvm_profile_header Header;
Expand Down Expand Up @@ -304,7 +302,7 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
#endif

/* The data and names sections are omitted in lightweight mode. */
if (DebugInfoCorrelate) {
if (ProfileCorrelation) {
Header.CountersDelta = 0;
Header.NamesDelta = 0;
}
Expand All @@ -320,21 +318,22 @@ lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,

/* Write the profile data. */
ProfDataIOVec IOVecData[] = {
{DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
{ProfileCorrelation ? NULL : DataBegin, sizeof(uint8_t), DataSectionSize,
0},
{NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
{CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
{BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
{(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
{(SkipNameDataWrite || ProfileCorrelation) ? NULL : NamesBegin,
sizeof(uint8_t), NamesSize, 0},
{NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
return -1;

/* Value profiling is not yet supported in continuous mode. */
if (__llvm_profile_is_continuous_mode_enabled())
/* Value profiling is not yet supported in continuous mode and profile
* correlation mode. */
if (__llvm_profile_is_continuous_mode_enabled() || ProfileCorrelation)
return 0;

return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Value profiling is currently not supported in lightweight mode.
// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t
// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t.dSYM %t.proflite

Expand All @@ -9,7 +9,7 @@

// RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata)

// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov
// RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov.dSYM %t.cov.proflite

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Disable full debug info and verify that we get warnings during merging

// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t -gline-tables-only -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t
// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=2 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,LIMIT --implicit-check-not=warning
// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite --max-debug-info-correlation-warnings=0 2>&1 >/dev/null | FileCheck %s --check-prefixes=CHECK,NOLIMIT --implicit-check-not=warning
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,19 @@
// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.normal
// RUN: llvm-profdata merge -o %t.normal.profdata %t.profraw

// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t.d4 -g -gdwarf-4 -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.d4.proflite %run %t.d4
// RUN: llvm-profdata merge -o %t.d4.profdata --debug-info=%t.d4 %t.d4.proflite

// RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.d4.profdata)

// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.proflite %run %t
// RUN: llvm-profdata merge -o %t.profdata --debug-info=%t %t.proflite

// RUN: diff <(llvm-profdata show --all-functions --counts %t.normal.profdata) <(llvm-profdata show --all-functions --counts %t.profdata)

// RUN: %clang_pgogen -o %t.cov -g -mllvm --debug-info-correlate -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: %clang_pgogen -o %t.cov -g -mllvm --profile-correlate=debug-info -mllvm -pgo-function-entry-coverage -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
// RUN: env LLVM_PROFILE_FILE=%t.cov.proflite %run %t.cov
// RUN: llvm-profdata merge -o %t.cov.profdata --debug-info=%t.cov %t.cov.proflite

Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
// RUN: %clang_pgogen -o %t -g -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
// RUN: %clang_pgogen -o %t -g -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s
// RUN: llvm-profdata show --debug-info=%t --detailed-summary --show-prof-sym-list | FileCheck %s
// RUN: llvm-profdata show --debug-info=%t --show-format=yaml | FileCheck %s --match-full-lines --check-prefix YAML

// RUN: %clang_pgogen -o %t.no.dbg -mllvm --debug-info-correlate -mllvm --disable-vp=true %s
// RUN: %clang_pgogen -o %t.no.dbg -mllvm --profile-correlate=debug-info -mllvm --disable-vp=true %s
// RUN: not llvm-profdata show --debug-info=%t.no.dbg 2>&1 | FileCheck %s --check-prefix NO-DBG
// NO-DBG: unable to correlate profile: could not find any profile metadata in debug info
// NO-DBG: unable to correlate profile: could not find any profile data metadata in correlated file

// YAML: Probes:
// YAML: - Function Name: a
Expand Down
4 changes: 2 additions & 2 deletions llvm/docs/CommandGuide/llvm-profdata.rst
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ OPTIONS
.. option:: --debug-info=<path>

Specify the executable or ``.dSYM`` that contains debug info for the raw profile.
When ``-debug-info-correlate`` was used for instrumentation, use this option
When ``-profile-correlate=debug-info`` was used for instrumentation, use this option
to correlate the raw profile.

.. option:: --temporal-profile-trace-reservoir-size
Expand Down Expand Up @@ -346,7 +346,7 @@ OPTIONS
.. option:: --debug-info=<path>

Specify the executable or ``.dSYM`` that contains debug info for the raw profile.
When ``-debug-info-correlate`` was used for instrumentation, use this option
When ``-profile-correlate=debug-info`` was used for instrumentation, use this option
to show the correlated functions from the raw profile.

.. option:: --covered
Expand Down
13 changes: 10 additions & 3 deletions llvm/include/llvm/ProfileData/InstrProfCorrelator.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,11 @@ class ObjectFile;
/// to their functions.
class InstrProfCorrelator {
public:
/// Indicate which kind correlator to use.
enum ProfCorrelatorKind { NONE, DEBUG_INFO };

static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(StringRef DebugInfoFilename);
get(StringRef Filename, ProfCorrelatorKind FileKind);

/// Construct a ProfileData vector used to correlate raw instrumentation data
/// to their functions.
Expand Down Expand Up @@ -104,7 +107,7 @@ class InstrProfCorrelator {

private:
static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
get(std::unique_ptr<MemoryBuffer> Buffer);
get(std::unique_ptr<MemoryBuffer> Buffer, ProfCorrelatorKind FileKind);

const InstrProfCorrelatorKind Kind;
};
Expand All @@ -128,7 +131,7 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {

static llvm::Expected<std::unique_ptr<InstrProfCorrelatorImpl<IntPtrT>>>
get(std::unique_ptr<InstrProfCorrelator::Context> Ctx,
const object::ObjectFile &Obj);
const object::ObjectFile &Obj, ProfCorrelatorKind FileKind);

protected:
std::vector<RawInstrProf::ProfileData<IntPtrT>> Data;
Expand All @@ -138,6 +141,8 @@ class InstrProfCorrelatorImpl : public InstrProfCorrelator {
int MaxWarnings,
InstrProfCorrelator::CorrelationData *Data = nullptr) = 0;

virtual Error correlateProfileNameImpl() = 0;

Error dumpYaml(int MaxWarnings, raw_ostream &OS) override;

void addProbe(StringRef FunctionName, uint64_t CFGHash, IntPtrT CounterOffset,
Expand Down Expand Up @@ -205,6 +210,8 @@ class DwarfInstrProfCorrelator : public InstrProfCorrelatorImpl<IntPtrT> {
void correlateProfileDataImpl(
int MaxWarnings,
InstrProfCorrelator::CorrelationData *Data = nullptr) override;

Error correlateProfileNameImpl() override;
};

} // end namespace llvm
Expand Down
2 changes: 2 additions & 0 deletions llvm/include/llvm/ProfileData/InstrProfReader.h
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,8 @@ class RawInstrProfReader : public InstrProfReader {
return (Version & VARIANT_MASK_DBG_CORRELATE) != 0;
}

bool useCorrelate() const { return useDebugInfoCorrelate(); }

bool hasSingleByteCoverage() const override {
return (Version & VARIANT_MASK_BYTE_COVERAGE) != 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@

namespace llvm {

extern cl::opt<bool> DebugInfoCorrelate;

class Function;
class Instruction;
class Module;
Expand Down
19 changes: 19 additions & 0 deletions llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
#include "llvm/MC/MCValue.h"
#include "llvm/MC/SectionKind.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfCorrelator.h"
#include "llvm/Support/Base64.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
Expand All @@ -71,6 +72,24 @@
using namespace llvm;
using namespace dwarf;

namespace llvm {
// Deprecated. Use -profile-correlate=debug-info.
cl::opt<bool> DebugInfoCorrelate(
"debug-info-correlate",
cl::desc("Use debug info to correlate profiles (Deprecated). Use "
"-profile-correlate=debug-info instead."),
cl::init(false));

cl::opt<InstrProfCorrelator::ProfCorrelatorKind> ProfileCorrelate(
"profile-correlate",
cl::desc("Use debug info or binary file to correlate profiles."),
cl::init(InstrProfCorrelator::NONE),
cl::values(clEnumValN(InstrProfCorrelator::NONE, "",
"No profile correlation"),
clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info",
"Use debug info to correlate")));
} // namespace llvm

static cl::opt<bool> JumpTableInFunctionSection(
"jumptable-in-function-section", cl::Hidden, cl::init(false),
cl::desc("Putting Jump Table in function section"));
Expand Down
Loading