Skip to content

Commit 7e256dc

Browse files
mtrofinmtrofin
committed
[ThinLTO] Allow importing based on a workload definition
An example of a "workload definition" would be "the transitive closure of functions actually called to satisfy a RPC request", i.e. a (typically significantly) smaller subset of the transitive closure (static + possible indirect call targets) of callees. This means this workload definition is a type of flat dynamic profile. Producing one is not in scope - it can be produced offline from traces, or from sample-based profiles, etc. This patch adds awareness to ThinLTO of such a concept. A workload is defined as a root and a list of functions. All function references are by-name (more readable than GUIDs). In the case of aliases, the expectation is the list contains all the alternative names. The workload definitions are presented to the linker as a json file, containing a dictionary. The keys are the roots, the values are the list of functions. The import list for a module defining a root will be the functions listed for it in the profile. Using names this way assumes unique names for internal functions, i.e. clang's `-funique-internal-linkage-names`. Note that the behavior affects the entire module where a root is defined (i.e. different workloads best be defined in different modules), and does not affect modules that don't define roots.
1 parent 7ec4f60 commit 7e256dc

File tree

5 files changed

+342
-10
lines changed

5 files changed

+342
-10
lines changed

‎llvm/lib/Transforms/IPO/FunctionImport.cpp

Lines changed: 223 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "llvm/Support/Error.h"
3838
#include "llvm/Support/ErrorHandling.h"
3939
#include "llvm/Support/FileSystem.h"
40+
#include "llvm/Support/JSON.h"
4041
#include "llvm/Support/SourceMgr.h"
4142
#include "llvm/Support/raw_ostream.h"
4243
#include "llvm/Transforms/IPO/Internalize.h"
@@ -138,6 +139,9 @@ static cl::opt<bool>
138139
ImportAllIndex("import-all-index",
139140
cl::desc("Import all external functions in index."));
140141

142+
static cl::opt<std::string> WorkloadDefinitions("thinlto-workload-def",
143+
cl::Hidden);
144+
141145
// Load lazily a module from \p FileName in \p Context.
142146
static std::unique_ptr<Module> loadFile(const std::string &FileName,
143147
LLVMContext &Context) {
@@ -369,29 +373,238 @@ class GlobalsImporter final {
369373
}
370374
};
371375

376+
static const char *getFailureName(FunctionImporter::ImportFailureReason Reason);
377+
372378
/// Determine the list of imports and exports for each module.
373-
class ModuleImportsManager final {
379+
class ModuleImportsManager {
380+
protected:
374381
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
375382
IsPrevailing;
376383
const ModuleSummaryIndex &Index;
377384
DenseMap<StringRef, FunctionImporter::ExportSetTy> *const ExportLists;
378385

379-
public:
380386
ModuleImportsManager(
381387
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
382388
IsPrevailing,
383389
const ModuleSummaryIndex &Index,
384390
DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists = nullptr)
385391
: IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}
386392

393+
public:
394+
virtual ~ModuleImportsManager() = default;
395+
387396
/// Given the list of globals defined in a module, compute the list of imports
388397
/// as well as the list of "exports", i.e. the list of symbols referenced from
389398
/// another module (that may require promotion).
390-
void computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
391-
StringRef ModName,
392-
FunctionImporter::ImportMapTy &ImportList);
399+
virtual void
400+
computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
401+
StringRef ModName,
402+
FunctionImporter::ImportMapTy &ImportList);
403+
404+
static std::unique_ptr<ModuleImportsManager>
405+
create(function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
406+
IsPrevailing,
407+
const ModuleSummaryIndex &Index,
408+
DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists =
409+
nullptr);
393410
};
394411

412+
class WorkloadImportsManager : public ModuleImportsManager {
413+
// Keep a module name -> defined value infos association. We use it to
414+
// determine if a module's import list should be done by the base
415+
// ModuleImportsManager or by us.
416+
StringMap<DenseSet<ValueInfo>> Workloads;
417+
418+
void
419+
computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,
420+
StringRef ModName,
421+
FunctionImporter::ImportMapTy &ImportList) override {
422+
auto SetIter = Workloads.find(ModName);
423+
if (SetIter == Workloads.end()) {
424+
LLVM_DEBUG(dbgs() << "[Workload] " << ModName
425+
<< " does not contain the root of any context.\n");
426+
return ModuleImportsManager::computeImportForModule(DefinedGVSummaries,
427+
ModName, ImportList);
428+
}
429+
LLVM_DEBUG(dbgs() << "[Workload] " << ModName
430+
<< " contains the root(s) of context(s).\n");
431+
432+
GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,
433+
ExportLists);
434+
auto &ValueInfos = SetIter->second;
435+
SmallVector<EdgeInfo, 128> GlobWorklist;
436+
for (auto &VI : llvm::make_early_inc_range(ValueInfos)) {
437+
auto Candidates =
438+
qualifyCalleeCandidates(Index, VI.getSummaryList(), ModName);
439+
440+
const GlobalValueSummary *GVS = nullptr;
441+
FunctionImporter::ImportFailureReason LastReason =
442+
FunctionImporter::ImportFailureReason::None;
443+
for (const auto &Candidate : Candidates) {
444+
LastReason = Candidate.first;
445+
if (Candidate.first == FunctionImporter::ImportFailureReason::None) {
446+
const bool Prevailing = IsPrevailing(VI.getGUID(), Candidate.second);
447+
if (Prevailing || !GVS) {
448+
if (!GVS && !Prevailing)
449+
LLVM_DEBUG(dbgs()
450+
<< "[Workload] Considering " << VI.name() << " from "
451+
<< Candidate.second->modulePath() << " with linkage "
452+
<< Candidate.second->linkage()
453+
<< " although it's not prevailing, but it's the "
454+
"first available candidate.\n");
455+
GVS = Candidate.second;
456+
if (Prevailing) {
457+
LLVM_DEBUG(dbgs()
458+
<< "[Workload] Considering " << VI.name() << " from "
459+
<< GVS->modulePath() << " with linkage "
460+
<< GVS->linkage() << " because it's prevailing.\n");
461+
break;
462+
}
463+
} else {
464+
LLVM_DEBUG(dbgs() << "[Workload] Skipping " << VI.name() << " from "
465+
<< Candidate.second->modulePath()
466+
<< " with linkage " << Candidate.second->linkage()
467+
<< " because it's not prevailing\n");
468+
}
469+
}
470+
}
471+
if (!GVS) {
472+
LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
473+
<< " because can't select Callee. Guid is: "
474+
<< Function::getGUID(VI.name())
475+
<< ". The reason was: " << getFailureName(LastReason)
476+
<< "\n");
477+
continue;
478+
}
479+
const auto *CFS = cast<FunctionSummary>(GVS->getBaseObject());
480+
auto ExportingModule = CFS->modulePath();
481+
if (ExportingModule == ModName) {
482+
LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
483+
<< " because its defining module is the same as the "
484+
"current module\n");
485+
continue;
486+
}
487+
if (!shouldImport(DefinedGVSummaries, VI.getGUID(), CFS)) {
488+
LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()
489+
<< " because we have a local copy.\n");
490+
continue;
491+
}
492+
493+
LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "
494+
<< ExportingModule << " : "
495+
<< Function::getGUID(VI.name()) << "\n");
496+
ImportList[ExportingModule].insert(VI.getGUID());
497+
GVI.onImportingSummary(*GVS);
498+
if (ExportLists)
499+
(*ExportLists)[ExportingModule].insert(VI);
500+
}
501+
LLVM_DEBUG(dbgs() << "[Workload] Done\n");
502+
}
503+
504+
bool shouldImport(const GVSummaryMapTy &DefinedGVSummaries,
505+
Function::GUID Guid, const GlobalValueSummary *Candidate) {
506+
auto DefinedSummary = DefinedGVSummaries.find(Guid);
507+
if (DefinedSummary == DefinedGVSummaries.end())
508+
return true;
509+
510+
// See shouldImportGlobal for the justificaton of the isInterposableLinkage.
511+
if (!IsPrevailing(Guid, DefinedSummary->second) &&
512+
GlobalValue::isInterposableLinkage(DefinedSummary->second->linkage()) &&
513+
IsPrevailing(Guid, Candidate)) {
514+
LLVM_DEBUG(dbgs() << "[Workload] " << Guid
515+
<< ": local non-prevailing in module. Importing from "
516+
<< Candidate->modulePath() << "\n");
517+
return true;
518+
}
519+
LLVM_DEBUG(dbgs() << "[Workload] " << Guid
520+
<< ": ignored! Target already in destination module.\n");
521+
return false;
522+
}
523+
524+
public:
525+
WorkloadImportsManager(
526+
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
527+
IsPrevailing,
528+
const ModuleSummaryIndex &Index,
529+
DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists)
530+
: ModuleImportsManager(IsPrevailing, Index, ExportLists) {
531+
StringMap<ValueInfo> CtxGuidToValueInfo;
532+
for (auto &I : Index) {
533+
ValueInfo VI(Index.haveGVs(), &I);
534+
CtxGuidToValueInfo[VI.name()] = VI;
535+
}
536+
std::error_code EC;
537+
auto BufferOrErr = MemoryBuffer::getFileOrSTDIN(WorkloadDefinitions);
538+
if (std::error_code EC = BufferOrErr.getError()) {
539+
report_fatal_error("Failed to open context file");
540+
return;
541+
}
542+
auto Buffer = std::move(BufferOrErr.get());
543+
std::map<std::string, std::vector<std::string>> WorkloadDefs;
544+
json::Path::Root NullRoot;
545+
auto Parsed = json::parse(Buffer->getBuffer());
546+
if (!Parsed)
547+
report_fatal_error(Parsed.takeError());
548+
if (!json::fromJSON(*Parsed, WorkloadDefs, NullRoot))
549+
report_fatal_error("Invalid thinlto contextual profile format.");
550+
for (const auto &Workload : WorkloadDefs) {
551+
const auto &Root = Workload.first;
552+
LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");
553+
const auto &AllCallees = Workload.second;
554+
auto RootIt = CtxGuidToValueInfo.find(Root);
555+
if (RootIt == CtxGuidToValueInfo.end()) {
556+
LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
557+
<< " not found in this linkage unit.\n");
558+
continue;
559+
}
560+
auto RootVI = RootIt->second;
561+
if (RootVI.getSummaryList().size() != 1) {
562+
LLVM_DEBUG(dbgs() << "[Workload] Root " << Root
563+
<< " should have exactly one summary, but has "
564+
<< RootVI.getSummaryList().size() << ". Skipping.\n");
565+
continue;
566+
}
567+
StringRef RootDefiningModule =
568+
RootVI.getSummaryList().front()->modulePath();
569+
LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root
570+
<< " is : " << RootDefiningModule << "\n");
571+
auto &Set = Workloads[RootDefiningModule];
572+
for (const auto &Callee : AllCallees) {
573+
LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");
574+
auto ElemIt = CtxGuidToValueInfo.find(Callee);
575+
if (ElemIt == CtxGuidToValueInfo.end()) {
576+
LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");
577+
continue;
578+
}
579+
Set.insert(ElemIt->second);
580+
}
581+
LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << " we have "
582+
<< Set.size() << " distinct callees.\n");
583+
LLVM_DEBUG( //
584+
for (const auto &VI
585+
: Set) {
586+
dbgs() << "[Workload] Root: " << Root
587+
<< " Would include: " << VI.getGUID() << "\n";
588+
});
589+
}
590+
}
591+
};
592+
593+
std::unique_ptr<ModuleImportsManager> ModuleImportsManager::create(
594+
function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
595+
IsPrevailing,
596+
const ModuleSummaryIndex &Index,
597+
DenseMap<StringRef, FunctionImporter::ExportSetTy> *ExportLists) {
598+
if (WorkloadDefinitions.empty()) {
599+
LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");
600+
return std::unique_ptr<ModuleImportsManager>(
601+
new ModuleImportsManager(IsPrevailing, Index, ExportLists));
602+
}
603+
LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");
604+
return std::make_unique<WorkloadImportsManager>(IsPrevailing, Index,
605+
ExportLists);
606+
}
607+
395608
static const char *
396609
getFailureName(FunctionImporter::ImportFailureReason Reason) {
397610
switch (Reason) {
@@ -732,14 +945,14 @@ void llvm::ComputeCrossModuleImport(
732945
isPrevailing,
733946
DenseMap<StringRef, FunctionImporter::ImportMapTy> &ImportLists,
734947
DenseMap<StringRef, FunctionImporter::ExportSetTy> &ExportLists) {
735-
ModuleImportsManager MIS(isPrevailing, Index, &ExportLists);
948+
auto MIS = ModuleImportsManager::create(isPrevailing, Index, &ExportLists);
736949
// For each module that has function defined, compute the import/export lists.
737950
for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
738951
auto &ImportList = ImportLists[DefinedGVSummaries.first];
739952
LLVM_DEBUG(dbgs() << "Computing import for Module '"
740953
<< DefinedGVSummaries.first << "'\n");
741-
MIS.computeImportForModule(DefinedGVSummaries.second,
742-
DefinedGVSummaries.first, ImportList);
954+
MIS->computeImportForModule(DefinedGVSummaries.second,
955+
DefinedGVSummaries.first, ImportList);
743956
}
744957

745958
// When computing imports we only added the variables and functions being
@@ -855,8 +1068,8 @@ static void ComputeCrossModuleImportForModuleForTest(
8551068

8561069
// Compute the import list for this module.
8571070
LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
858-
ModuleImportsManager MIS(isPrevailing, Index);
859-
MIS.computeImportForModule(FunctionSummaryMap, ModulePath, ImportList);
1071+
auto MIS = ModuleImportsManager::create(isPrevailing, Index);
1072+
MIS->computeImportForModule(FunctionSummaryMap, ModulePath, ImportList);
8601073

8611074
#ifndef NDEBUG
8621075
dumpImportListForModule(Index, ModulePath, ImportList);

‎llvm/test/ThinLTO/X86/Inputs/workload1.ll

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
2+
target triple = "x86_64-pc-linux-gnu"
3+
4+
declare void @m1_variant()
5+
6+
define dso_local void @m1_f1() {
7+
call void @m1_f2()
8+
call void @noninterposable_f()
9+
ret void
10+
}
11+
12+
define internal void @m1_f2() {
13+
call void @interposable_f()
14+
ret void
15+
}
16+
17+
define linkonce void @interposable_f() {
18+
call void @m1_variant()
19+
ret void
20+
}
21+
22+
define linkonce_odr void @noninterposable_f() {
23+
call void @m1_variant()
24+
ret void
25+
}

‎llvm/test/ThinLTO/X86/Inputs/workload2.ll

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
2+
target triple = "x86_64-pc-linux-gnu"
3+
4+
declare void @m2_variant()
5+
6+
define dso_local void @m2_f1() {
7+
call void @interposable_f()
8+
call void @noninterposable_f()
9+
ret void
10+
}
11+
12+
@m2_f1_alias = alias void (...), ptr @m2_f1
13+
14+
define linkonce_odr void @interposable_f() {
15+
call void @m2_variant()
16+
ret void
17+
}
18+
19+
define linkonce_odr void @noninterposable_f() {
20+
call void @m2_variant()
21+
ret void
22+
}

‎llvm/test/ThinLTO/X86/Inputs/workload3.ll

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
2+
target triple = "x86_64-pc-linux-gnu"
3+
4+
declare void @m1_f1()
5+
6+
define dso_local void @m3_f1() {
7+
call void @m1_f1()
8+
ret void
9+
}

0 commit comments

Comments
 (0)