Skip to content

Commit 26ecf97

Browse files
david-armdavid-arm
authored
[LoopVectorize] Further improve cost model for early exit loops (#126235)
Following on from #125058, this patch takes into account the work done in the vector early exit block when assessing the profitability of vectorising the loop. I have renamed areRuntimeChecksProfitable to isOutsideLoopWorkProfitable and we now pass in the early exit costs. As part of this, I have added the ExtractFirstActive opcode to VPInstruction::computeCost. It's worth pointing out that when we assess profitability of the loop we calculate a minimum trip count and compare that against the *maximum* trip count. However, since the loop has an early exit the runtime trip count can still end up being less than the minimum. Alternatively, we may never take the early exit at all at runtime and so we have the opposite problem of over-estimating the cost of the loop. The loop vectoriser cannot simultaneously take two contradictory positions and so I feel the only sensible thing to do is be conservative and assume the loop will be more expensive than loops without early exits. We may find in future that we need to adjust the cost according to the probability of taking the early exit. This will become even more important once we support multiple early exits. However, we have to start somewhere and we can always revisit this later.
1 parent 63635c1 commit 26ecf97

File tree

4 files changed

+155
-14
lines changed

4 files changed

+155
-14
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 56 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10171,19 +10171,56 @@ static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
1017110171
}
1017210172
}
1017310173

10174-
static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
10175-
VectorizationFactor &VF, Loop *L,
10176-
PredicatedScalarEvolution &PSE,
10177-
ScalarEpilogueLowering SEL,
10178-
std::optional<unsigned> VScale) {
10179-
InstructionCost CheckCost = Checks.getCost();
10180-
if (!CheckCost.isValid())
10174+
/// For loops with uncountable early exits, find the cost of doing work when
10175+
/// exiting the loop early, such as calculating the final exit values of
10176+
/// variables used outside the loop.
10177+
/// TODO: This is currently overly pessimistic because the loop may not take
10178+
/// the early exit, but better to keep this conservative for now. In future,
10179+
/// it might be possible to relax this by using branch probabilities.
10180+
static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
10181+
VPlan &Plan, ElementCount VF) {
10182+
InstructionCost Cost = 0;
10183+
for (auto *ExitVPBB : Plan.getExitBlocks()) {
10184+
for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
10185+
// If the predecessor is not the middle.block, then it must be the
10186+
// vector.early.exit block, which may contain work to calculate the exit
10187+
// values of variables used outside the loop.
10188+
if (PredVPBB != Plan.getMiddleBlock()) {
10189+
LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
10190+
<< PredVPBB->getName() << ":\n");
10191+
Cost += PredVPBB->cost(VF, CostCtx);
10192+
}
10193+
}
10194+
}
10195+
return Cost;
10196+
}
10197+
10198+
/// This function determines whether or not it's still profitable to vectorize
10199+
/// the loop given the extra work we have to do outside of the loop:
10200+
/// 1. Perform the runtime checks before entering the loop to ensure it's safe
10201+
/// to vectorize.
10202+
/// 2. In the case of loops with uncountable early exits, we may have to do
10203+
/// extra work when exiting the loop early, such as calculating the final
10204+
/// exit values of variables used outside the loop.
10205+
static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
10206+
VectorizationFactor &VF, Loop *L,
10207+
PredicatedScalarEvolution &PSE,
10208+
VPCostContext &CostCtx, VPlan &Plan,
10209+
ScalarEpilogueLowering SEL,
10210+
std::optional<unsigned> VScale) {
10211+
InstructionCost TotalCost = Checks.getCost();
10212+
if (!TotalCost.isValid())
1018110213
return false;
1018210214

10215+
// Add on the cost of any work required in the vector early exit block, if
10216+
// one exists.
10217+
TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF.Width);
10218+
1018310219
// When interleaving only scalar and vector cost will be equal, which in turn
1018410220
// would lead to a divide by 0. Fall back to hard threshold.
1018510221
if (VF.Width.isScalar()) {
10186-
if (CheckCost > VectorizeMemoryCheckThreshold) {
10222+
// TODO: Should we rename VectorizeMemoryCheckThreshold?
10223+
if (TotalCost > VectorizeMemoryCheckThreshold) {
1018710224
LLVM_DEBUG(
1018810225
dbgs()
1018910226
<< "LV: Interleaving only is not profitable due to runtime checks\n");
@@ -10209,7 +10246,9 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
1020910246
// The total cost of the vector loop is
1021010247
// RtC + VecC * (TC / VF) + EpiC
1021110248
// where
10212-
// * RtC is the cost of the generated runtime checks
10249+
// * RtC is the cost of the generated runtime checks plus the cost of
10250+
// performing any additional work in the vector.early.exit block for loops
10251+
// with uncountable early exits.
1021310252
// * VecC is the cost of a single vector iteration.
1021410253
// * TC is the actual trip count of the loop
1021510254
// * VF is the vectorization factor
@@ -10227,7 +10266,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
1022710266
// the computations are performed on doubles, not integers and the result
1022810267
// is rounded up, hence we get an upper estimate of the TC.
1022910268
unsigned IntVF = getEstimatedRuntimeVF(VF.Width, VScale);
10230-
uint64_t RtC = *CheckCost.getValue();
10269+
uint64_t RtC = *TotalCost.getValue();
1023110270
uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
1023210271
uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
1023310272

@@ -10555,8 +10594,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
1055510594
// iteration count is low. However, setting the epilogue policy to
1055610595
// `CM_ScalarEpilogueNotAllowedLowTripLoop` prevents vectorizing loops
1055710596
// with runtime checks. It's more effective to let
10558-
// `areRuntimeChecksProfitable` determine if vectorization is beneficial
10559-
// for the loop.
10597+
// `isOutsideLoopWorkProfitable` determine if vectorization is
10598+
// beneficial for the loop.
1056010599
if (SEL != CM_ScalarEpilogueNotNeededUsePredicate)
1056110600
SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
1056210601
} else {
@@ -10654,9 +10693,12 @@ bool LoopVectorizePass::processLoop(Loop *L) {
1065410693
// Check if it is profitable to vectorize with runtime checks.
1065510694
bool ForceVectorization =
1065610695
Hints.getForce() == LoopVectorizeHints::FK_Enabled;
10696+
VPCostContext CostCtx(CM.TTI, *CM.TLI, CM.Legal->getWidestInductionType(),
10697+
CM, CM.CostKind);
1065710698
if (!ForceVectorization &&
10658-
!areRuntimeChecksProfitable(Checks, VF, L, PSE, SEL,
10659-
CM.getVScaleForTuning())) {
10699+
!isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx,
10700+
LVP.getPlanFor(VF.Width), SEL,
10701+
CM.getVScaleForTuning())) {
1066010702
ORE->emit([&]() {
1066110703
return OptimizationRemarkAnalysisAliasing(
1066210704
DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,18 @@ InstructionCost VPInstruction::computeCost(ElementCount VF,
742742
return Ctx.TTI.getArithmeticReductionCost(
743743
Instruction::Or, cast<VectorType>(VecTy), std::nullopt, Ctx.CostKind);
744744
}
745+
case VPInstruction::ExtractFirstActive: {
746+
// Calculate the cost of determining the lane index.
747+
auto *PredTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(1)), VF);
748+
IntrinsicCostAttributes Attrs(Intrinsic::experimental_cttz_elts,
749+
Type::getInt64Ty(Ctx.LLVMCtx),
750+
{PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
751+
InstructionCost Cost = Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
752+
// Add on the cost of extracting the element.
753+
auto *VecTy = toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF);
754+
return Cost + Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
755+
Ctx.CostKind);
756+
}
745757
default:
746758
// TODO: Compute cost other VPInstructions once the legacy cost model has
747759
// been retired.

‎llvm/test/Transforms/LoopVectorize/AArch64/early_exit_costs.ll

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
2+
; REQUIRES: asserts
3+
; RUN: opt -S < %s -p loop-vectorize -enable-early-exit-vectorization -disable-output \
4+
; RUN: -debug-only=loop-vectorize 2>&1 | FileCheck %s --check-prefixes=CHECK
5+
6+
target triple = "aarch64-unknown-linux-gnu"
7+
8+
declare void @init_mem(ptr, i64);
9+
10+
define i64 @same_exit_block_pre_inc_use1_sve() #1 {
11+
; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_sve'
12+
; CHECK: LV: Selecting VF: vscale x 16
13+
; CHECK: Calculating cost of work in exit block vector.early.exit
14+
; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
15+
; CHECK-NEXT: Cost of 6 for VF vscale x 16: EMIT vp<{{.*}}> = extract-first-active
16+
; CHECK: LV: Minimum required TC for runtime checks to be profitable:32
17+
entry:
18+
%p1 = alloca [1024 x i8]
19+
%p2 = alloca [1024 x i8]
20+
call void @init_mem(ptr %p1, i64 1024)
21+
call void @init_mem(ptr %p2, i64 1024)
22+
br label %loop
23+
24+
loop:
25+
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
26+
%index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
27+
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
28+
%ld1 = load i8, ptr %arrayidx, align 1
29+
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
30+
%ld2 = load i8, ptr %arrayidx1, align 1
31+
%cmp3 = icmp eq i8 %ld1, %ld2
32+
br i1 %cmp3, label %loop.inc, label %loop.end
33+
34+
loop.inc:
35+
%index.next = add i64 %index, 1
36+
%index2.next = add i64 %index2, 2
37+
%exitcond = icmp ne i64 %index.next, 67
38+
br i1 %exitcond, label %loop, label %loop.end
39+
40+
loop.end:
41+
%val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
42+
%val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
43+
%retval = add i64 %val1, %val2
44+
ret i64 %retval
45+
}
46+
47+
define i64 @same_exit_block_pre_inc_use1_nosve() {
48+
; CHECK-LABEL: LV: Checking a loop in 'same_exit_block_pre_inc_use1_nosve'
49+
; CHECK: LV: Selecting VF: 16
50+
; CHECK: Calculating cost of work in exit block vector.early.exit
51+
; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
52+
; CHECK-NEXT: Cost of 50 for VF 16: EMIT vp<{{.*}}> = extract-first-active
53+
; CHECK: LV: Minimum required TC for runtime checks to be profitable:176
54+
; CHECK-NEXT: LV: Vectorization is not beneficial: expected trip count < minimum profitable VF (64 < 176)
55+
; CHECK-NEXT: LV: Too many memory checks needed.
56+
entry:
57+
%p1 = alloca [1024 x i8]
58+
%p2 = alloca [1024 x i8]
59+
call void @init_mem(ptr %p1, i64 1024)
60+
call void @init_mem(ptr %p2, i64 1024)
61+
br label %loop
62+
63+
loop:
64+
%index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
65+
%index2 = phi i64 [ %index2.next, %loop.inc ], [ 15, %entry ]
66+
%arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index
67+
%ld1 = load i8, ptr %arrayidx, align 1
68+
%arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index
69+
%ld2 = load i8, ptr %arrayidx1, align 1
70+
%cmp3 = icmp eq i8 %ld1, %ld2
71+
br i1 %cmp3, label %loop.inc, label %loop.end
72+
73+
loop.inc:
74+
%index.next = add i64 %index, 1
75+
%index2.next = add i64 %index2, 2
76+
%exitcond = icmp ne i64 %index.next, 67
77+
br i1 %exitcond, label %loop, label %loop.end
78+
79+
loop.end:
80+
%val1 = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
81+
%val2 = phi i64 [ %index2, %loop ], [ 98, %loop.inc ]
82+
%retval = add i64 %val1, %val2
83+
ret i64 %retval
84+
}
85+
86+
attributes #1 = { "target-features"="+sve" vscale_range(1,16) }

‎llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ define i64 @loop_contains_safe_div() #1 {
274274
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
275275
; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
276276
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4
277+
; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP12]])
277278
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
278279
; CHECK: vector.ph:
279280
; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64()

0 commit comments

Comments
 (0)