Skip to content

Commit 3733463

Browse files
xur-llvmxur-llvm
committed
[IR][PGO] Add hot func attribute and use hot/cold attribute in func section
Clang FE currently has hot/cold function attribute. But we only have cold function attribute in LLVM IR. This patch adds support of hot function attribute to LLVM IR. This attribute will be used in setting function section prefix/suffix. Currently .hot and .unlikely suffix only are added in PGO (Sample PGO) compilation (through isFunctionHotInCallGraph and isFunctionColdInCallGraph). This patch changes the behavior. The new behavior is: (1) If the user annotates a function as hot or isFunctionHotInCallGraph is true, this function will be marked as hot. Otherwise, (2) If the user annotates a function as cold or isFunctionColdInCallGraph is true, this function will be marked as cold. The changes are: (1) user annotated function attribute will used in setting function section prefix/suffix. (2) hot attribute overwrites profile count based hotness. (3) profile count based hotness overwrite user annotated cold attribute. The intention for these changes is to provide the user a way to mark certain function as hot in cases where training input is hard to cover all the hot functions. Differential Revision: https://reviews.llvm.org/D92493
1 parent ee2cb90 commit 3733463

File tree

16 files changed

+187
-3
lines changed

16 files changed

+187
-3
lines changed

‎clang/lib/CodeGen/CGCall.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1944,6 +1944,8 @@ void CodeGenModule::ConstructAttributeList(
19441944
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
19451945
if (TargetDecl->hasAttr<ColdAttr>())
19461946
FuncAttrs.addAttribute(llvm::Attribute::Cold);
1947+
if (TargetDecl->hasAttr<HotAttr>())
1948+
FuncAttrs.addAttribute(llvm::Attribute::Hot);
19471949
if (TargetDecl->hasAttr<NoDuplicateAttr>())
19481950
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
19491951
if (TargetDecl->hasAttr<ConvergentAttr>())

‎clang/lib/CodeGen/CodeGenModule.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1744,7 +1744,8 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
17441744
B.addAttribute(llvm::Attribute::OptimizeForSize);
17451745
B.addAttribute(llvm::Attribute::Cold);
17461746
}
1747-
1747+
if (D->hasAttr<HotAttr>())
1748+
B.addAttribute(llvm::Attribute::Hot);
17481749
if (D->hasAttr<MinSizeAttr>())
17491750
B.addAttribute(llvm::Attribute::MinSize);
17501751
}

‎clang/test/CodeGen/attributes.c

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,13 @@ void t72() { t71(); }
6363
// CHECK: call void @t71() [[COLDSITE:#[0-9]+]]
6464
// CHECK: declare void @t71() [[COLDDECL:#[0-9]+]]
6565

66+
// CHECK: define void @t82() [[HOTDEF:#[0-9]+]] {
67+
void t81(void) __attribute__((hot));
68+
void t82() __attribute__((hot));
69+
void t82() { t81(); }
70+
// CHECK: call void @t81() [[HOTSITE:#[0-9]+]]
71+
// CHECK: declare void @t81() [[HOTDECL:#[0-9]+]]
72+
6673
// CHECK: define void @t10() [[NUW]] section "xSECT" {
6774
void t10(void) __attribute__((section("xSECT")));
6875
void t10(void) {}
@@ -72,6 +79,9 @@ void __attribute__((section("xSECT"))) t11(void) {}
7279
// CHECK: define i32 @t19() [[NUW]] {
7380
extern int t19(void) __attribute__((weak_import));
7481
int t19(void) {
82+
// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
83+
// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
84+
// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
7585
return 10;
7686
}
7787

@@ -111,6 +121,9 @@ void t24(f_t f1) {
111121
// CHECK: attributes [[NR]] = { noinline noreturn nounwind{{.*}} }
112122
// CHECK: attributes [[COLDDEF]] = { cold {{.*}}}
113123
// CHECK: attributes [[COLDDECL]] = { cold {{.*}}}
124+
// CHECK: attributes [[HOTDEF]] = { hot {{.*}}}
125+
// CHECK: attributes [[HOTDECL]] = { hot {{.*}}}
114126
// CHECK: attributes [[NOCF_CHECK_FUNC]] = { nocf_check {{.*}}}
115127
// CHECK: attributes [[COLDSITE]] = { cold {{.*}}}
128+
// CHECK: attributes [[HOTSITE]] = { hot {{.*}}}
116129
// CHECK: attributes [[NOCF_CHECK_CALL]] = { nocf_check }

‎llvm/docs/LangRef.rst

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,15 @@ example:
14961496
can prove that the function does not execute any convergent operations.
14971497
Similarly, the optimizer may remove ``convergent`` on calls/invokes when it
14981498
can prove that the call/invoke cannot call a convergent function.
1499+
``hot``
1500+
This attribute indicates that this function is a hot spot of the program
1501+
execution. The function will be optimized more aggressively and will be
1502+
placed into special subsection of the text section to improving locality.
1503+
1504+
When profile feedback is enabled, this attribute has the precedence over
1505+
the profile information. By marking a function ``hot``, users can work
1506+
around the cases where the training input does not have good coverage
1507+
on all the hot functions.
14991508
``inaccessiblememonly``
15001509
This attribute indicates that the function may only access memory that
15011510
is not accessible by the module being compiled. This is a weaker form

‎llvm/include/llvm/Bitcode/LLVMBitCodes.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -654,6 +654,7 @@ enum AttributeKindCodes {
654654
ATTR_KIND_BYREF = 69,
655655
ATTR_KIND_MUSTPROGRESS = 70,
656656
ATTR_KIND_NO_CALLBACK = 71,
657+
ATTR_KIND_HOT = 72,
657658
};
658659

659660
enum ComdatSelectionKindCodes {

‎llvm/include/llvm/IR/Attributes.td

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,9 @@ def Cold : EnumAttr<"cold">;
6363
/// Can only be moved to control-equivalent blocks.
6464
def Convergent : EnumAttr<"convergent">;
6565

66+
/// Marks function as being in a hot path and frequently called.
67+
def Hot: EnumAttr<"hot">;
68+
6669
/// Pointer is known to be dereferenceable.
6770
def Dereferenceable : IntAttr<"dereferenceable">;
6871

‎llvm/lib/AsmParser/LLParser.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,6 +1340,7 @@ bool LLParser::parseFnAttributeValuePairs(AttrBuilder &B,
13401340
case lltok::kw_argmemonly: B.addAttribute(Attribute::ArgMemOnly); break;
13411341
case lltok::kw_builtin: B.addAttribute(Attribute::Builtin); break;
13421342
case lltok::kw_cold: B.addAttribute(Attribute::Cold); break;
1343+
case lltok::kw_hot: B.addAttribute(Attribute::Hot); break;
13431344
case lltok::kw_convergent: B.addAttribute(Attribute::Convergent); break;
13441345
case lltok::kw_inaccessiblememonly:
13451346
B.addAttribute(Attribute::InaccessibleMemOnly); break;

‎llvm/lib/Bitcode/Reader/BitcodeReader.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1539,6 +1539,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
15391539
return Attribute::ByRef;
15401540
case bitc::ATTR_KIND_MUSTPROGRESS:
15411541
return Attribute::MustProgress;
1542+
case bitc::ATTR_KIND_HOT:
1543+
return Attribute::Hot;
15421544
}
15431545
}
15441546

‎llvm/lib/Bitcode/Writer/BitcodeWriter.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
626626
return bitc::ATTR_KIND_IN_ALLOCA;
627627
case Attribute::Cold:
628628
return bitc::ATTR_KIND_COLD;
629+
case Attribute::Hot:
630+
return bitc::ATTR_KIND_HOT;
629631
case Attribute::InaccessibleMemOnly:
630632
return bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY;
631633
case Attribute::InaccessibleMemOrArgMemOnly:

‎llvm/lib/CodeGen/CodeGenPrepare.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -472,9 +472,17 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
472472
PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
473473
OptSize = F.hasOptSize();
474474
if (ProfileGuidedSectionPrefix) {
475-
if (PSI->isFunctionHotInCallGraph(&F, *BFI))
475+
// The hot attribute overwrites profile count based hotness while profile
476+
// counts based hotness overwrite the cold attribute.
477+
// This is a conservative behabvior.
478+
if (F.hasFnAttribute(Attribute::Hot) ||
479+
PSI->isFunctionHotInCallGraph(&F, *BFI))
476480
F.setSectionPrefix("hot");
477-
else if (PSI->isFunctionColdInCallGraph(&F, *BFI))
481+
// If PSI shows this function is not hot, we will placed the function
482+
// into unlikely section if (1) PSI shows this is a cold function, or
483+
// (2) the function has a attribute of cold.
484+
else if (PSI->isFunctionColdInCallGraph(&F, *BFI) ||
485+
F.hasFnAttribute(Attribute::Cold))
478486
F.setSectionPrefix("unlikely");
479487
else if (ProfileUnknownInSpecialSection && PSI->hasPartialSampleProfile() &&
480488
PSI->isFunctionHotnessUnknown(F))

‎llvm/lib/IR/Attributes.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const {
449449
return "zeroext";
450450
if (hasAttribute(Attribute::Cold))
451451
return "cold";
452+
if (hasAttribute(Attribute::Hot))
453+
return "hot";
452454
if (hasAttribute(Attribute::ImmArg))
453455
return "immarg";
454456
if (hasAttribute(Attribute::NoUndef))

‎llvm/lib/IR/Verifier.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1626,6 +1626,7 @@ static bool isFuncOnlyAttr(Attribute::AttrKind Kind) {
16261626
case Attribute::Builtin:
16271627
case Attribute::NoBuiltin:
16281628
case Attribute::Cold:
1629+
case Attribute::Hot:
16291630
case Attribute::OptForFuzzing:
16301631
case Attribute::OptimizeNone:
16311632
case Attribute::JumpTable:

‎llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1921,6 +1921,17 @@ static bool annotateAllFunctions(
19211921
<< "\n");
19221922
}
19231923
for (auto &F : ColdFunctions) {
1924+
// Only set when there is no Attribute::Hot set by the user. For Hot
1925+
// attribute, user's annotation has the precedence over the profile.
1926+
if (F->hasFnAttribute(Attribute::Hot)) {
1927+
auto &Ctx = M.getContext();
1928+
std::string Msg = std::string("Function ") + F->getName().str() +
1929+
std::string(" is annotated as a hot function but"
1930+
" the profile is cold");
1931+
Ctx.diagnose(
1932+
DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));
1933+
continue;
1934+
}
19241935
F->addFnAttr(Attribute::Cold);
19251936
LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()
19261937
<< "\n");

‎llvm/test/Bitcode/attributes.ll

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,18 @@ define void @f69() nocallback
410410
ret void
411411
}
412412

413+
; CHECK: define void @f70() #43
414+
define void @f70() cold
415+
{
416+
ret void
417+
}
418+
419+
; CHECK: define void @f71() #44
420+
define void @f71() hot
421+
{
422+
ret void
423+
}
424+
413425
; CHECK: attributes #0 = { noreturn }
414426
; CHECK: attributes #1 = { nounwind }
415427
; CHECK: attributes #2 = { readnone }
@@ -453,4 +465,6 @@ define void @f69() nocallback
453465
; CHECK: attributes #40 = { null_pointer_is_valid }
454466
; CHECK: attributes #41 = { mustprogress }
455467
; CHECK: attributes #42 = { nocallback }
468+
; CHECK: attributes #43 = { cold }
469+
; CHECK: attributes #44 = { hot }
456470
; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }

‎llvm/test/CodeGen/X86/hot-unlikely-section-prefix.ll

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
; Test hot or unlikely section postfix based on profile and user annotation.
2+
; RUN: llc < %s | FileCheck %s
3+
target triple = "x86_64-unknown-linux-gnu"
4+
5+
; Function Attrs: inlinehint norecurse nounwind readnone uwtable
6+
define dso_local i32 @hot1() #0 !prof !31 {
7+
entry:
8+
ret i32 1
9+
}
10+
; CHECK: .section .text.hot.,"ax",@progbits
11+
; CHECK: .globl hot1
12+
13+
; Function Attrs: cold norecurse nounwind readnone uwtable
14+
define dso_local i32 @cold1() #1 !prof !32 {
15+
entry:
16+
ret i32 1
17+
}
18+
; CHECK: .section .text.unlikely.,"ax",@progbits
19+
; CHECK: .globl cold1
20+
21+
; Function Attrs: cold inlinehint noinline norecurse nounwind optsize readnone uwtable
22+
define dso_local i32 @hot2() #2 !prof !31 {
23+
entry:
24+
ret i32 1
25+
}
26+
; CHECK: .section .text.hot.,"ax",@progbits
27+
; CHECK: .globl hot2
28+
29+
define dso_local i32 @normal() {
30+
entry:
31+
ret i32 1
32+
}
33+
; CHECK: text
34+
; CHECK: .globl normal
35+
36+
; Function Attrs: hot noinline norecurse nounwind readnone uwtable
37+
define dso_local i32 @hot3() #3 !prof !32 {
38+
entry:
39+
ret i32 1
40+
}
41+
; CHECK: .section .text.hot.,"ax",@progbits
42+
; CHECK: .globl hot3
43+
44+
; Function Attrs: cold noinline norecurse nounwind optsize readnone uwtable
45+
define dso_local i32 @cold2() #4 {
46+
entry:
47+
ret i32 1
48+
}
49+
; CHECK: .section .text.unlikely.,"ax",@progbits
50+
; CHECK: .globl cold2
51+
52+
; Function Attrs: hot noinline norecurse nounwind readnone uwtable
53+
define dso_local i32 @hot4() #3 {
54+
entry:
55+
ret i32 1
56+
}
57+
; CHECK: .section .text.hot.,"ax",@progbits
58+
; CHECK: .globl hot4
59+
60+
attributes #0 = { inlinehint norecurse nounwind readnone uwtable }
61+
attributes #1 = { cold norecurse nounwind readnone uwtable }
62+
attributes #2 = { cold inlinehint noinline norecurse nounwind optsize readnone uwtable }
63+
attributes #3 = { hot noinline norecurse nounwind readnone uwtable }
64+
attributes #4 = { cold noinline norecurse nounwind optsize readnone uwtable }
65+
66+
!llvm.module.flags = !{!0, !1}
67+
!llvm.ident = !{!30}
68+
69+
!0 = !{i32 1, !"wchar_size", i32 4}
70+
!1 = !{i32 1, !"ProfileSummary", !2}
71+
!2 = !{!3, !4, !5, !6, !7, !8, !9, !10, !11, !12}
72+
!3 = !{!"ProfileFormat", !"InstrProf"}
73+
!4 = !{!"TotalCount", i64 402020}
74+
!5 = !{!"MaxCount", i64 200000}
75+
!6 = !{!"MaxInternalCount", i64 2000}
76+
!7 = !{!"MaxFunctionCount", i64 200000}
77+
!8 = !{!"NumCounts", i64 7}
78+
!9 = !{!"NumFunctions", i64 5}
79+
!10 = !{!"IsPartialProfile", i64 0}
80+
!11 = !{!"PartialProfileRatio", double 0.000000e+00}
81+
!12 = !{!"DetailedSummary", !13}
82+
!13 = !{!14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29}
83+
!14 = !{i32 10000, i64 200000, i32 1}
84+
!15 = !{i32 100000, i64 200000, i32 1}
85+
!16 = !{i32 200000, i64 200000, i32 1}
86+
!17 = !{i32 300000, i64 200000, i32 1}
87+
!18 = !{i32 400000, i64 200000, i32 1}
88+
!19 = !{i32 500000, i64 100000, i32 3}
89+
!20 = !{i32 600000, i64 100000, i32 3}
90+
!21 = !{i32 700000, i64 100000, i32 3}
91+
!22 = !{i32 800000, i64 100000, i32 3}
92+
!23 = !{i32 900000, i64 100000, i32 3}
93+
!24 = !{i32 950000, i64 100000, i32 3}
94+
!25 = !{i32 990000, i64 100000, i32 3}
95+
!26 = !{i32 999000, i64 2000, i32 4}
96+
!27 = !{i32 999900, i64 2000, i32 4}
97+
!28 = !{i32 999990, i64 10, i32 6}
98+
!29 = !{i32 999999, i64 10, i32 6}
99+
!30 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git 53c5fdd59a5cf7fbb4dcb7a7e84c9c4a40d32a84)"}
100+
!31 = !{!"function_entry_count", i64 100000}
101+
!32 = !{!"function_entry_count", i64 10}

‎llvm/test/MC/AsmParser/function_hot_attr.ll

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
; Test hot function attribute
2+
; RUN: llc < %s | FileCheck %s
3+
target triple = "x86_64-unknown-linux-gnu"
4+
5+
; Function Attrs: hot noinline norecurse nounwind readnone uwtable
6+
define dso_local i32 @hot4() #0 {
7+
entry:
8+
ret i32 1
9+
}
10+
; CHECK: .section .text.hot.,"ax",@progbits
11+
; CHECK: .globl hot4
12+
13+
attributes #0 = { hot noinline norecurse nounwind readnone uwtable }

0 commit comments

Comments
 (0)