Skip to content

Commit 3f96f7b

Browse files
arpilipearpilipe
committed
Fix getInlineCost with ComputeFullInlineCost enabled
Fix a bug when getInlineCost incorrectly returns a cost/threshold pair instead of an explicit never inline. Reviewed By: mtrofin Differential Revision: https://reviews.llvm.org/D111687
1 parent 42ad7e1 commit 3f96f7b

File tree

3 files changed

+149
-9
lines changed

3 files changed

+149
-9
lines changed

‎llvm/lib/Analysis/InlineCost.cpp

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,9 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
545545
// sense that it's not weighted by profile counts at all.
546546
int ColdSize = 0;
547547

548+
// Whether inlining is decided by cost-threshold analysis.
549+
bool DecidedByCostThreshold = false;
550+
548551
// Whether inlining is decided by cost-benefit analysis.
549552
bool DecidedByCostBenefit = false;
550553

@@ -914,14 +917,24 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
914917
return InlineResult::failure("Cost over threshold.");
915918
}
916919

917-
if (IgnoreThreshold || Cost < std::max(1, Threshold))
920+
if (IgnoreThreshold)
918921
return InlineResult::success();
919-
return InlineResult::failure("Cost over threshold.");
922+
923+
DecidedByCostThreshold = true;
924+
return Cost < std::max(1, Threshold)
925+
? InlineResult::success()
926+
: InlineResult::failure("Cost over threshold.");
920927
}
928+
921929
bool shouldStop() override {
930+
if (IgnoreThreshold || ComputeFullInlineCost)
931+
return false;
922932
// Bail out the moment we cross the threshold. This means we'll under-count
923933
// the cost, but only when undercounting doesn't matter.
924-
return !IgnoreThreshold && Cost >= Threshold && !ComputeFullInlineCost;
934+
if (Cost < Threshold)
935+
return false;
936+
DecidedByCostThreshold = true;
937+
return true;
925938
}
926939

927940
void onLoadEliminationOpportunity() override {
@@ -1013,6 +1026,7 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
10131026
int getCost() const { return Cost; }
10141027
Optional<CostBenefitPair> getCostBenefitPair() { return CostBenefit; }
10151028
bool wasDecidedByCostBenefit() const { return DecidedByCostBenefit; }
1029+
bool wasDecidedByCostThreshold() const { return DecidedByCostThreshold; }
10161030
};
10171031

10181032
class InlineCostFeaturesAnalyzer final : public CallAnalyzer {
@@ -2934,13 +2948,13 @@ InlineCost llvm::getInlineCost(
29342948
return InlineCost::getNever("cost over benefit", CA.getCostBenefitPair());
29352949
}
29362950

2937-
// Check if there was a reason to force inlining or no inlining.
2938-
if (!ShouldInline.isSuccess() && CA.getCost() < CA.getThreshold())
2939-
return InlineCost::getNever(ShouldInline.getFailureReason());
2940-
if (ShouldInline.isSuccess() && CA.getCost() >= CA.getThreshold())
2941-
return InlineCost::getAlways("empty function");
2951+
if (CA.wasDecidedByCostThreshold())
2952+
return InlineCost::get(CA.getCost(), CA.getThreshold());
29422953

2943-
return llvm::InlineCost::get(CA.getCost(), CA.getThreshold());
2954+
// No details on how the decision was made, simply return always or never.
2955+
return ShouldInline.isSuccess()
2956+
? InlineCost::getAlways("empty function")
2957+
: InlineCost::getNever(ShouldInline.getFailureReason());
29442958
}
29452959

29462960
InlineResult llvm::isInlineViable(Function &F) {

‎llvm/test/Transforms/Inline/noduplicate.ll

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
; RUN: opt < %s -passes=inline -pass-remarks-missed=inline -inline-cost-full -S 2>&1 | FileCheck %s
2+
; RUN: opt < %s -passes=inline -pass-remarks-missed=inline -S 2>&1 | FileCheck %s
3+
4+
declare void @foo()
5+
declare void @bar()
6+
7+
define void @callee() {
8+
entry:
9+
call void @foo() noduplicate
10+
; Just to inflate the cost
11+
call void @bar() "call-inline-cost"="1000"
12+
ret void
13+
}
14+
15+
define void @caller() {
16+
; CHECK: 'callee' not inlined into 'caller' because it should never be inlined (cost=never): noduplicate
17+
; CHECK: define void @caller()
18+
; CHECK-NEXT: call void @callee()
19+
; CHECK-NEXT: ret void
20+
call void @callee()
21+
ret void
22+
}

‎llvm/test/Transforms/SampleProfile/inline-noduplicate.ll

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline.prof -S | FileCheck %s
2+
3+
; Similar to inline.ll test, but the callee contains a noduplicate instruction.
4+
5+
@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
6+
7+
declare void @foo()
8+
declare void @bar()
9+
10+
; Function Attrs: nounwind uwtable
11+
define i32 @_Z3sumii(i32 %x, i32 %y) #0 !dbg !4 {
12+
entry:
13+
call void @foo() noduplicate
14+
; Just to inflate the cost
15+
call void @bar() "call-inline-cost"="1000"
16+
%x.addr = alloca i32, align 4
17+
%y.addr = alloca i32, align 4
18+
store i32 %x, i32* %x.addr, align 4
19+
store i32 %y, i32* %y.addr, align 4
20+
%0 = load i32, i32* %x.addr, align 4, !dbg !11
21+
%1 = load i32, i32* %y.addr, align 4, !dbg !11
22+
%add = add nsw i32 %0, %1, !dbg !11
23+
ret i32 %add, !dbg !11
24+
}
25+
26+
; Function Attrs: uwtable
27+
define i32 @main() #0 !dbg !7 {
28+
entry:
29+
%retval = alloca i32, align 4
30+
%s = alloca i32, align 4
31+
%i = alloca i32, align 4
32+
store i32 0, i32* %retval
33+
store i32 0, i32* %i, align 4, !dbg !12
34+
br label %while.cond, !dbg !13
35+
36+
while.cond: ; preds = %if.end, %entry
37+
%0 = load i32, i32* %i, align 4, !dbg !14
38+
%inc = add nsw i32 %0, 1, !dbg !14
39+
store i32 %inc, i32* %i, align 4, !dbg !14
40+
%cmp = icmp slt i32 %0, 400000000, !dbg !14
41+
br i1 %cmp, label %while.body, label %while.end, !dbg !14
42+
43+
while.body: ; preds = %while.cond
44+
%1 = load i32, i32* %i, align 4, !dbg !16
45+
%cmp1 = icmp ne i32 %1, 100, !dbg !16
46+
br i1 %cmp1, label %if.then, label %if.else, !dbg !16
47+
48+
49+
if.then: ; preds = %while.body
50+
%2 = load i32, i32* %i, align 4, !dbg !18
51+
%3 = load i32, i32* %s, align 4, !dbg !18
52+
%call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
53+
; _Z3sumii should not be inlined because of the noduplicate call to foo.
54+
; CHECK: call i32 @_Z3sumii
55+
; CHECK-NOT: call void @foo
56+
store i32 %call, i32* %s, align 4, !dbg !18
57+
br label %if.end, !dbg !18
58+
59+
if.else: ; preds = %while.body
60+
store i32 30, i32* %s, align 4, !dbg !20
61+
br label %if.end
62+
63+
if.end: ; preds = %if.else, %if.then
64+
br label %while.cond, !dbg !22
65+
66+
while.end: ; preds = %while.cond
67+
%4 = load i32, i32* %s, align 4, !dbg !24
68+
%call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
69+
ret i32 0, !dbg !25
70+
}
71+
72+
declare i32 @printf(i8*, ...) #2
73+
74+
attributes #0 = { "use-sample-profile" }
75+
76+
!llvm.dbg.cu = !{!0}
77+
!llvm.module.flags = !{!8, !9}
78+
!llvm.ident = !{!10}
79+
80+
!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
81+
!1 = !DIFile(filename: "calls.cc", directory: ".")
82+
!2 = !{}
83+
!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
84+
!5 = !DIFile(filename: "calls.cc", directory: ".")
85+
!6 = !DISubroutineType(types: !2)
86+
!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
87+
!8 = !{i32 2, !"Dwarf Version", i32 4}
88+
!9 = !{i32 1, !"Debug Info Version", i32 3}
89+
!10 = !{!"clang version 3.5 "}
90+
!11 = !DILocation(line: 4, scope: !4)
91+
!12 = !DILocation(line: 8, scope: !7)
92+
!13 = !DILocation(line: 9, scope: !7)
93+
!14 = !DILocation(line: 9, scope: !15)
94+
!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
95+
!16 = !DILocation(line: 10, scope: !17)
96+
!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
97+
!18 = !DILocation(line: 10, scope: !19)
98+
!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
99+
!20 = !DILocation(line: 10, scope: !21)
100+
!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
101+
!22 = !DILocation(line: 10, scope: !23)
102+
!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
103+
!24 = !DILocation(line: 11, scope: !7)
104+
!25 = !DILocation(line: 12, scope: !7)

0 commit comments

Comments
 (0)