Skip to content

Commit fec71fc

Browse files
hazzlimhazzlim
committed
[LV] Optimize VPWidenIntOrFpInductionRecipe for known TC
Optimize the IR generated for a VPWidenIntOrFpInductionRecipe to use the narrowest type necessary, when the trip-count of a loop is known to be constant and the only use of the recipe is the condition used by the vector loop's backedge branch.
1 parent 2570c86 commit fec71fc

13 files changed

+4515
-154
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1805,6 +1805,9 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe {
18051805
VPSlotTracker &SlotTracker) const override;
18061806
#endif
18071807

1808+
/// Update the step value of the recipe.
1809+
void setStepValue(VPValue *V) { setOperand(1, V); }
1810+
18081811
VPValue *getVFValue() { return getOperand(2); }
18091812
const VPValue *getVFValue() const { return getOperand(2); }
18101813

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 90 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "VPlanPatternMatch.h"
2121
#include "VPlanUtils.h"
2222
#include "VPlanVerifier.h"
23+
#include "llvm/ADT/APInt.h"
2324
#include "llvm/ADT/PostOrderIterator.h"
2425
#include "llvm/ADT/STLExtras.h"
2526
#include "llvm/ADT/SetVector.h"
@@ -29,6 +30,8 @@
2930
#include "llvm/Analysis/VectorUtils.h"
3031
#include "llvm/IR/Intrinsics.h"
3132
#include "llvm/IR/PatternMatch.h"
33+
#include "llvm/Support/Casting.h"
34+
#include "llvm/Support/TypeSize.h"
3235

3336
using namespace llvm;
3437

@@ -996,11 +999,74 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
996999
}
9971000
}
9981001

999-
void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
1000-
unsigned BestUF,
1001-
PredicatedScalarEvolution &PSE) {
1002-
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
1003-
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
1002+
/// Optimize the width of vector induction variables in \p Plan based on a known
1003+
/// constant Trip Count, \p BestVF and \p BestUF.
1004+
static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
1005+
ElementCount BestVF,
1006+
unsigned BestUF) {
1007+
// Only proceed if we have not completely removed the vector region.
1008+
if (!Plan.getVectorLoopRegion())
1009+
return false;
1010+
1011+
auto *TC = dyn_cast_if_present<ConstantInt>(
1012+
Plan.getTripCount()->getUnderlyingValue());
1013+
if (!TC || !BestVF.isFixed())
1014+
return false;
1015+
1016+
// Calculate the widest type required for known TC, VF and UF.
1017+
auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
1018+
auto AlignedTC =
1019+
Align * APIntOps::RoundingUDiv(TC, APInt(TC.getBitWidth(), Align),
1020+
APInt::Rounding::UP);
1021+
auto MaxVal = AlignedTC - 1;
1022+
return std::max<unsigned>(PowerOf2Ceil(MaxVal.getActiveBits()), 8);
1023+
};
1024+
unsigned NewBitWidth =
1025+
ComputeBitWidth(TC->getValue(), BestVF.getKnownMinValue() * BestUF);
1026+
1027+
LLVMContext &Ctx = Plan.getCanonicalIV()->getScalarType()->getContext();
1028+
auto *NewIVTy = IntegerType::get(Ctx, NewBitWidth);
1029+
1030+
bool MadeChange = false;
1031+
1032+
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
1033+
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
1034+
auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
1035+
if (!WideIV || !WideIV->isCanonical() ||
1036+
WideIV->hasMoreThanOneUniqueUser() ||
1037+
NewIVTy == WideIV->getScalarType())
1038+
continue;
1039+
1040+
// Currently only handle cases where the single user is a header-mask
1041+
// comparison with the backedge-taken-count.
1042+
using namespace VPlanPatternMatch;
1043+
if (!match(*WideIV->user_begin(),
1044+
m_Binary<Instruction::ICmp>(
1045+
m_Specific(WideIV),
1046+
m_Specific(Plan.getOrCreateBackedgeTakenCount()))))
1047+
continue;
1048+
1049+
// Update IV operands and comparison bound to use new narrower type.
1050+
auto *NewStart = Plan.getOrAddLiveIn(ConstantInt::get(NewIVTy, 0));
1051+
WideIV->setStartValue(NewStart);
1052+
auto *NewStep = Plan.getOrAddLiveIn(ConstantInt::get(NewIVTy, 1));
1053+
WideIV->setStepValue(NewStep);
1054+
1055+
auto *NewBTC = new VPWidenCastRecipe(
1056+
Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy);
1057+
Plan.getVectorPreheader()->appendRecipe(NewBTC);
1058+
auto *Cmp = dyn_cast<VPInstruction>(*WideIV->user_begin());
1059+
Cmp->setOperand(1, NewBTC);
1060+
1061+
MadeChange = true;
1062+
}
1063+
1064+
return MadeChange;
1065+
}
1066+
1067+
bool VPlanTransforms::simplifyBranchConditionForVFAndUF(
1068+
VPlan &Plan, ElementCount BestVF, unsigned BestUF,
1069+
PredicatedScalarEvolution &PSE) {
10041070
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
10051071
VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
10061072
auto *Term = &ExitingVPBB->back();
@@ -1013,7 +1079,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
10131079
if (!match(Term, m_BranchOnCount(m_VPValue(), m_VPValue())) &&
10141080
!match(Term,
10151081
m_BranchOnCond(m_Not(m_ActiveLaneMask(m_VPValue(), m_VPValue())))))
1016-
return;
1082+
return false;
10171083

10181084
ScalarEvolution &SE = *PSE.getSE();
10191085
const SCEV *TripCount =
@@ -1024,7 +1090,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
10241090
const SCEV *C = SE.getElementCount(TripCount->getType(), NumElements);
10251091
if (TripCount->isZero() ||
10261092
!SE.isKnownPredicate(CmpInst::ICMP_ULE, TripCount, C))
1027-
return;
1093+
return false;
10281094

10291095
// The vector loop region only executes once. If possible, completely remove
10301096
// the region, otherwise replace the terminator controlling the latch with
@@ -1063,8 +1129,23 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
10631129

10641130
Term->eraseFromParent();
10651131

1066-
Plan.setVF(BestVF);
1067-
Plan.setUF(BestUF);
1132+
return true;
1133+
}
1134+
1135+
void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
1136+
unsigned BestUF,
1137+
PredicatedScalarEvolution &PSE) {
1138+
assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
1139+
assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
1140+
1141+
bool MadeChange =
1142+
simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
1143+
MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
1144+
1145+
if (MadeChange) {
1146+
Plan.setVF(BestVF);
1147+
Plan.setUF(BestUF);
1148+
}
10681149
// TODO: Further simplifications are possible
10691150
// 1. Replace inductions with constants.
10701151
// 2. Replace vector loop region with VPBasicBlock.

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,13 @@ struct VPlanTransforms {
9797
unsigned BestUF,
9898
PredicatedScalarEvolution &PSE);
9999

100+
/// Try to simplify the branch condition of \p Plan. This may restrict the
101+
/// resulting plan to \p BestVF and \p BestUF.
102+
static bool simplifyBranchConditionForVFAndUF(VPlan &Plan,
103+
ElementCount BestVF,
104+
unsigned BestUF,
105+
PredicatedScalarEvolution &PSE);
106+
100107
/// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe
101108
/// optimizations, dead recipe removal, replicate region optimizations and
102109
/// block merging.

‎llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -389,8 +389,8 @@ define void @latch_branch_cost(ptr %dst) {
389389
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
390390
; PRED: vector.body:
391391
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
392-
; PRED-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
393-
; PRED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 99)
392+
; PRED-NEXT: [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
393+
; PRED-NEXT: [[TMP0:%.*]] = icmp ule <8 x i8> [[VEC_IND]], splat (i8 99)
394394
; PRED-NEXT: [[TMP1:%.*]] = extractelement <8 x i1> [[TMP0]], i32 0
395395
; PRED-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
396396
; PRED: pred.store.if:
@@ -456,7 +456,7 @@ define void @latch_branch_cost(ptr %dst) {
456456
; PRED-NEXT: br label [[PRED_STORE_CONTINUE6]]
457457
; PRED: pred.store.continue14:
458458
; PRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
459-
; PRED-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
459+
; PRED-NEXT: [[VEC_IND_NEXT]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
460460
; PRED-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], 104
461461
; PRED-NEXT: br i1 [[TMP25]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
462462
; PRED: middle.block:
@@ -899,9 +899,9 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
899899
; DEFAULT-NEXT: br label [[VECTOR_BODY:%.*]]
900900
; DEFAULT: vector.body:
901901
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE14:%.*]] ]
902-
; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]
902+
; DEFAULT-NEXT: [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]
903903
; DEFAULT-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i8
904-
; DEFAULT-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 6)
904+
; DEFAULT-NEXT: [[TMP1:%.*]] = icmp ule <8 x i8> [[VEC_IND]], splat (i8 6)
905905
; DEFAULT-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
906906
; DEFAULT-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
907907
; DEFAULT: pred.store.if:
@@ -974,7 +974,7 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
974974
; DEFAULT-NEXT: store i8 [[TMP33]], ptr [[TMP32]], align 1
975975
; DEFAULT-NEXT: br label [[PRED_STORE_CONTINUE14]]
976976
; DEFAULT: pred.store.continue14:
977-
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
977+
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
978978
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
979979
; DEFAULT-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
980980
; DEFAULT: middle.block:
@@ -1001,9 +1001,9 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
10011001
; PRED-NEXT: br label [[VECTOR_BODY:%.*]]
10021002
; PRED: vector.body:
10031003
; PRED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE14:%.*]] ]
1004-
; PRED-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]
1004+
; PRED-NEXT: [[VEC_IND:%.*]] = phi <8 x i8> [ <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE14]] ]
10051005
; PRED-NEXT: [[TMP0:%.*]] = trunc i64 [[INDEX]] to i8
1006-
; PRED-NEXT: [[TMP1:%.*]] = icmp ule <8 x i64> [[VEC_IND]], splat (i64 6)
1006+
; PRED-NEXT: [[TMP1:%.*]] = icmp ule <8 x i8> [[VEC_IND]], splat (i8 6)
10071007
; PRED-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
10081008
; PRED-NEXT: br i1 [[TMP2]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
10091009
; PRED: pred.store.if:
@@ -1076,7 +1076,7 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
10761076
; PRED-NEXT: store i8 [[TMP33]], ptr [[TMP32]], align 1
10771077
; PRED-NEXT: br label [[PRED_STORE_CONTINUE14]]
10781078
; PRED: pred.store.continue14:
1079-
; PRED-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
1079+
; PRED-NEXT: [[VEC_IND_NEXT]] = add <8 x i8> [[VEC_IND]], splat (i8 8)
10801080
; PRED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
10811081
; PRED-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
10821082
; PRED: middle.block:

‎llvm/test/Transforms/LoopVectorize/SystemZ/predicated-first-order-recurrence.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ define void @func_21() {
1919
; CHECK: vector.body:
2020
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
2121
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <2 x i32> [ <i32 poison, i32 0>, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[PRED_STORE_CONTINUE4]] ]
22-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
22+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
2323
; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0
2424
; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1
25-
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i64> [[VEC_IND]], splat (i64 4)
25+
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 4)
2626
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
2727
; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
2828
; CHECK: pred.load.if:
@@ -59,7 +59,7 @@ define void @func_21() {
5959
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE4]]
6060
; CHECK: pred.store.continue4:
6161
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
62-
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
62+
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 2)
6363
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 6
6464
; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
6565
; CHECK: middle.block:

‎llvm/test/Transforms/LoopVectorize/X86/consecutive-ptr-uniforms.ll

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ attributes #0 = { "target-cpu"="knl" }
8686
; FORCE-NEXT: br label [[VECTOR_BODY:%.*]]
8787
; FORCE: vector.body:
8888
; FORCE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE4:%.*]] ]
89-
; FORCE-NEXT: [[VEC_IND:%.*]] = phi <2 x i32> [ <i32 0, i32 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
90-
; FORCE-NEXT: [[TMP2:%.*]] = icmp ule <2 x i32> [[VEC_IND]], splat (i32 2)
89+
; FORCE-NEXT: [[VEC_IND:%.*]] = phi <2 x i8> [ <i8 0, i8 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE4]] ]
90+
; FORCE-NEXT: [[TMP2:%.*]] = icmp ule <2 x i8> [[VEC_IND]], splat (i8 2)
9191
; FORCE-NEXT: [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
9292
; FORCE-NEXT: br i1 [[TMP3]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
9393
; FORCE: pred.store.if:
@@ -103,7 +103,7 @@ attributes #0 = { "target-cpu"="knl" }
103103
; FORCE-NEXT: br label [[PRED_STORE_CONTINUE4]]
104104
; FORCE: pred.store.continue2:
105105
; FORCE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
106-
; FORCE-NEXT: [[VEC_IND_NEXT]] = add <2 x i32> [[VEC_IND]], splat (i32 2)
106+
; FORCE-NEXT: [[VEC_IND_NEXT]] = add <2 x i8> [[VEC_IND]], splat (i8 2)
107107
; FORCE-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], 4
108108
; FORCE-NEXT: br i1 [[TMP15]], label {{%.*}}, label [[VECTOR_BODY]]
109109
;

‎llvm/test/Transforms/LoopVectorize/pr45679-fold-tail-by-masking.ll

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ define void @pr45679(ptr %A) optsize {
1818
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
1919
; CHECK: vector.body:
2020
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
21-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
22-
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 13)
21+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
22+
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 13)
2323
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
2424
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
2525
; CHECK: pred.store.if:
@@ -53,7 +53,7 @@ define void @pr45679(ptr %A) optsize {
5353
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]
5454
; CHECK: pred.store.continue6:
5555
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
56-
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
56+
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
5757
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], 16
5858
; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
5959
; CHECK: middle.block:
@@ -213,8 +213,8 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
213213
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
214214
; CHECK: vector.body:
215215
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE6:%.*]] ]
216-
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
217-
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i64> [[VEC_IND]], splat (i64 13)
216+
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE6]] ]
217+
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <4 x i8> [[VEC_IND]], splat (i8 13)
218218
; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i1> [[TMP0]], i32 0
219219
; CHECK-NEXT: br i1 [[TMP1]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
220220
; CHECK: pred.store.if:
@@ -252,7 +252,7 @@ define void @load_variant(ptr noalias %a, ptr noalias %b) {
252252
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE6]]
253253
; CHECK: pred.store.continue6:
254254
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
255-
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
255+
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], splat (i8 4)
256256
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
257257
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
258258
; CHECK: middle.block:

0 commit comments

Comments
 (0)