Skip to content

Commit 2c8836c

Browse files
committed
[LV] Don't consider predicated insts as invariant unconditionally in CM.
Predicated instructions cannot hoisted trivially, so don't treat them as uniform value in the cost model. This fixes a difference between legacy and VPlan-based cost model. Fixes #110295.
1 parent 51e0a99 commit 2c8836c

File tree

2 files changed

+64
-2
lines changed

2 files changed

+64
-2
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6539,8 +6539,16 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
65396539
Op2 = cast<SCEVConstant>(PSE.getSCEV(Op2))->getValue();
65406540
}
65416541
auto Op2Info = TTI.getOperandInfo(Op2);
6542-
if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
6543-
Legal->isInvariant(Op2))
6542+
auto IsInvariant = [this](Value *Op) {
6543+
if (!Legal->isInvariant(Op))
6544+
return false;
6545+
// Consider Op2 invariant, if it is not a predicated instruction in the
6546+
// loop. In that case, it is not trivially hoistable.
6547+
return !isa<Instruction>(Op) ||
6548+
!TheLoop->contains(cast<Instruction>(Op)) ||
6549+
!isPredicatedInst(cast<Instruction>(Op));
6550+
};
6551+
if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue && IsInvariant(Op2))
65446552
Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
65456553

65466554
SmallVector<const Value *, 4> Operands(I->operand_values());
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -p loop-vectorize -S %s | FileCheck %s
3+
4+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
5+
target triple = "x86_64-unknown-linux-gnu"
6+
7+
; Test case for https://github.com/llvm/llvm-project/issues/110295.
8+
define void @predicated_urem_shl_cost(ptr %A, i32 %x, i1 %c) {
9+
; CHECK-LABEL: define void @predicated_urem_shl_cost(
10+
; CHECK-SAME: ptr [[A:%.*]], i32 [[X:%.*]], i1 [[C:%.*]]) {
11+
; CHECK-NEXT: [[ENTRY:.*]]:
12+
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
13+
; CHECK: [[LOOP_HEADER]]:
14+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
15+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[IV]]
16+
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4
17+
; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[LOOP_LATCH]]
18+
; CHECK: [[THEN]]:
19+
; CHECK-NEXT: [[REM:%.*]] = urem i32 2, [[X]]
20+
; CHECK-NEXT: [[SHL:%.*]] = shl i32 [[L]], [[REM]]
21+
; CHECK-NEXT: br label %[[LOOP_LATCH]]
22+
; CHECK: [[LOOP_LATCH]]:
23+
; CHECK-NEXT: [[P:%.*]] = phi i32 [ 0, %[[LOOP_HEADER]] ], [ [[SHL]], %[[THEN]] ]
24+
; CHECK-NEXT: store i32 [[P]], ptr [[GEP]], align 4
25+
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
26+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 0
27+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]]
28+
; CHECK: [[EXIT]]:
29+
; CHECK-NEXT: ret void
30+
;
31+
entry:
32+
br label %loop.header
33+
34+
loop.header:
35+
%iv = phi i32 [ 1, %entry ], [ %iv.next, %loop.latch ]
36+
%gep = getelementptr inbounds i32, ptr %A, i32 %iv
37+
%l = load i32, ptr %gep
38+
br i1 %c, label %then, label %loop.latch
39+
40+
then:
41+
%rem = urem i32 2, %x
42+
%shl = shl i32 %l, %rem
43+
br label %loop.latch
44+
45+
loop.latch:
46+
%p = phi i32 [ 0, %loop.header ], [ %shl, %then ]
47+
store i32 %p, ptr %gep
48+
%iv.next = add i32 %iv, 1
49+
%ec = icmp eq i32 %iv, 0
50+
br i1 %ec, label %exit, label %loop.header
51+
52+
exit:
53+
ret void
54+
}

0 commit comments

Comments
 (0)