Skip to content

Commit 7a87bf4

Browse files
SamTebbs33SamTebbs33
committed
Use VPExpandSCEVRecipe
1 parent a1b0669 commit 7a87bf4

File tree

4 files changed

+29
-39
lines changed

4 files changed

+29
-39
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ class LoopVectorizationPlanner {
363363
/// loop iteration.
364364
std::optional<VectorizationFactor>
365365
plan(ElementCount UserVF, unsigned UserIC,
366-
std::optional<ArrayRef<PointerDiffInfo>> DiffChecks, std::function<Value*(const SCEV *)> Expander, bool &HasAliasMask);
366+
std::optional<ArrayRef<PointerDiffInfo>> DiffChecks, bool &HasAliasMask);
367367

368368
/// Use the VPlan-native path to plan how to best vectorize, return the best
369369
/// VF and its cost.
@@ -440,18 +440,17 @@ class LoopVectorizationPlanner {
440440
/// setting HasAliasMask to true in the case that an alias mask is generated
441441
/// and the vector loop should be entered even if the pointers alias across a
442442
/// loop iteration.
443-
VPlanPtr
444-
tryToBuildVPlanWithVPRecipes(VFRange &Range,
445-
SmallVector<PointerDiffInfoValues> RTChecks,
446-
bool &HasAliasMask);
443+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range,
444+
ArrayRef<PointerDiffInfo> RTChecks,
445+
bool &HasAliasMask);
447446

448447
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
449448
/// according to the information gathered by Legal when it checked if it is
450449
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
451450
/// RTChecks contains a list of pointer pairs that an alias mask should be
452451
/// generated for.
453452
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF,
454-
SmallVector<PointerDiffInfoValues> RTChecks,
453+
ArrayRef<PointerDiffInfo> RTChecks,
455454
bool &HasAliasMask);
456455

457456
// Adjust the recipes for reductions. For in-loop reductions the chain of

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1936,10 +1936,6 @@ class GeneratedRTChecks {
19361936
OuterLoop = L->getParentLoop();
19371937
}
19381938

1939-
Value *expandCodeForMemCheck(const SCEV *Scev, Instruction *Loc) {
1940-
return MemCheckExp.expandCodeFor(Scev, Scev->getType(), Loc);
1941-
}
1942-
19431939
InstructionCost getCost() {
19441940
if (SCEVCheckBlock || MemCheckBlock)
19451941
LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
@@ -6905,10 +6901,9 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
69056901
return VectorizationFactor::Disabled();
69066902
}
69076903

6908-
std::optional<VectorizationFactor>
6909-
LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
6910-
std::optional<ArrayRef<PointerDiffInfo>> RTChecks,
6911-
std::function<Value*(const SCEV*)> Expander, bool &HasAliasMask) {
6904+
std::optional<VectorizationFactor> LoopVectorizationPlanner::plan(
6905+
ElementCount UserVF, unsigned UserIC,
6906+
std::optional<ArrayRef<PointerDiffInfo>> RTChecks, bool &HasAliasMask) {
69126907
assert(OrigLoop->isInnermost() && "Inner loop expected.");
69136908
CM.collectValuesToIgnore();
69146909
CM.collectElementTypesForWidening();
@@ -6919,15 +6914,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69196914

69206915
// VPlan needs the aliasing pointers as Values and not SCEVs, so expand them
69216916
// here and put them into a list.
6922-
SmallVector<PointerDiffInfoValues> DiffChecksValues;
6923-
if (RTChecks.has_value()
6924-
&& useActiveLaneMask(CM.getTailFoldingStyle(true))) {
6925-
for (auto Check : *RTChecks) {
6926-
Value *Sink = Expander(Check.SinkStart);
6927-
Value *Src = Expander(Check.SrcStart);
6928-
DiffChecksValues.push_back(PointerDiffInfoValues(Src, Sink));
6929-
}
6930-
}
6917+
ArrayRef<PointerDiffInfo> DiffChecks;
6918+
if (RTChecks.has_value() && useActiveLaneMask(CM.getTailFoldingStyle(true)))
6919+
DiffChecks = *RTChecks;
69316920

69326921
// Invalidate interleave groups if all blocks of loop will be predicated.
69336922
if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
@@ -6957,7 +6946,7 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69576946
CM.collectInLoopReductions();
69586947
if (CM.selectUserVectorizationFactor(UserVF)) {
69596948
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
6960-
buildVPlansWithVPRecipes(UserVF, UserVF, DiffChecksValues, HasAliasMask);
6949+
buildVPlansWithVPRecipes(UserVF, UserVF, DiffChecks, HasAliasMask);
69616950
if (!hasPlanWithVF(UserVF)) {
69626951
LLVM_DEBUG(dbgs() << "LV: No VPlan could be built for " << UserVF
69636952
<< ".\n");
@@ -6992,9 +6981,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69926981
}
69936982

69946983
buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF,
6995-
DiffChecksValues, HasAliasMask);
6984+
DiffChecks, HasAliasMask);
69966985
buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF,
6997-
DiffChecksValues, HasAliasMask);
6986+
DiffChecks, HasAliasMask);
69986987

69996988
LLVM_DEBUG(printPlans(dbgs()));
70006989
if (VPlans.empty())
@@ -8387,8 +8376,8 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
83878376
}
83888377

83898378
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
8390-
ElementCount MinVF, ElementCount MaxVF,
8391-
SmallVector<PointerDiffInfoValues> RTChecks, bool &HasAliasMask) {
8379+
ElementCount MinVF, ElementCount MaxVF, ArrayRef<PointerDiffInfo> RTChecks,
8380+
bool &HasAliasMask) {
83928381
assert(OrigLoop->isInnermost() && "Inner loop expected.");
83938382

83948383
auto MaxVFTimes2 = MaxVF * 2;
@@ -8534,8 +8523,7 @@ static void addLiveOutsForFirstOrderRecurrences(VPlan &Plan) {
85348523
}
85358524

85368525
VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8537-
VFRange &Range, SmallVector<PointerDiffInfoValues> RTChecks,
8538-
bool &HasAliasMask) {
8526+
VFRange &Range, ArrayRef<PointerDiffInfo> RTChecks, bool &HasAliasMask) {
85398527

85408528
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
85418529

@@ -8584,8 +8572,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
85848572
VPBuilder Builder(VecPreheader);
85858573
for (auto C : RTChecks) {
85868574
HasAliasMask = true;
8587-
VPValue *Sink = Plan->getOrAddLiveIn(C.Sink);
8588-
VPValue *Src = Plan->getOrAddLiveIn(C.Src);
8575+
VPValue *Sink = vputils::getOrCreateVPValueForSCEVExpr(*Plan, C.SinkStart,
8576+
*PSE.getSE());
8577+
VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(*Plan, C.SrcStart,
8578+
*PSE.getSE());
85898579
VPValue *M =
85908580
Builder.createNaryOp(VPInstruction::AliasLaneMask, {Sink, Src}, DL,
85918581
"active.lane.mask.alias");
@@ -9921,11 +9911,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
99219911
AddBranchWeights);
99229912

99239913
// Plan how to best vectorize, return the best VF and its cost.
9924-
auto Expand = [&Checks, &L](const SCEV *S) {
9925-
return Checks.expandCodeForMemCheck(S, L->getLoopPreheader()->getTerminator());
9926-
};
99279914
std::optional<VectorizationFactor> MaybeVF =
9928-
LVP.plan(UserVF, UserIC, LVL.getLAI()->getRuntimePointerChecking()->getDiffChecks(), Expand, Checks.HasAliasMask);
9915+
LVP.plan(UserVF, UserIC,
9916+
LVL.getLAI()->getRuntimePointerChecking()->getDiffChecks(),
9917+
Checks.HasAliasMask);
99299918
if (Checks.HasAliasMask)
99309919
LoopsAliasMasked++;
99319920

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,14 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
146146
; PRED-NEXT: entry:
147147
; PRED-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64
148148
; PRED-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
149+
; PRED-NEXT: [[SRC3:%.*]] = ptrtoint ptr [[SRC]] to i64
150+
; PRED-NEXT: [[DST2:%.*]] = ptrtoint ptr [[DST]] to i64
149151
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
150152
; PRED-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
151153
; PRED: vector.memcheck:
152154
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
153155
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 8
154-
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
156+
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST2]], [[SRC3]]
155157
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
156158
; PRED-NEXT: br label [[VECTOR_PH:%.*]]
157159
; PRED: vector.ph:

‎llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ define void @pred_loop(ptr %off, ptr %data, ptr %dst, i32 %n) #0 {
99
; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %i.09 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
1010
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add = add nuw nsw i32 %i.09, 1
1111
; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction: %arrayidx = getelementptr inbounds i32, ptr %data, i32 %add
12-
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %1 = load i32, ptr %arrayidx, align 4
13-
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add1 = add nsw i32 %1, 5
12+
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %0 = load i32, ptr %arrayidx, align 4
13+
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add1 = add nsw i32 %0, 5
1414
; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction: %arrayidx2 = getelementptr inbounds i32, ptr %dst, i32 %i.09
1515
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %add1, ptr %arrayidx2, align 4
1616
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %exitcond.not = icmp eq i32 %add, %n

0 commit comments

Comments
 (0)