Skip to content

Commit 67caa7d

Browse files
fhahnfhahn
committed
[VPlan] Manage noalias/alias_scope metadata in VPlan.
Use VPIRMetadata added in llvm#135272 to also manage no-alias metadata added by versioning. Note that this means we have to build the no-alias metadata up-front once. If it is not used, it will be discarded automatically. !fixup move applyMetadata; !fixup address comments, thanks !fixup address remaining comments, thanks !fixup address latest comments, thanks
1 parent 1b60b83 commit 67caa7d

File tree

10 files changed

+101
-101
lines changed

10 files changed

+101
-101
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class LoopVectorizationLegality;
3636
class LoopVectorizationCostModel;
3737
class PredicatedScalarEvolution;
3838
class LoopVectorizeHints;
39+
class LoopVersioning;
3940
class OptimizationRemarkEmitter;
4041
class TargetTransformInfo;
4142
class TargetLibraryInfo;
@@ -518,7 +519,7 @@ class LoopVectorizationPlanner {
518519
/// returned VPlan is valid for. If no VPlan can be built for the input range,
519520
/// set the largest included VF to the maximum VF for which no plan could be
520521
/// built.
521-
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range);
522+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range, LoopVersioning *LVer);
522523

523524
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
524525
/// according to the information gathered by Legal when it checked if it is

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2337,6 +2337,7 @@ static void scalarizeInstruction(const Instruction *Instr,
23372337
}
23382338

23392339
RepRecipe->applyFlags(*Cloned);
2340+
RepRecipe->applyMetadata(*Cloned);
23402341

23412342
if (auto DL = RepRecipe->getDebugLoc())
23422343
State.setDebugLocFrom(DL);
@@ -2350,7 +2351,6 @@ static void scalarizeInstruction(const Instruction *Instr,
23502351
InputLane = VPLane::getFirstLane();
23512352
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
23522353
}
2353-
State.addNewMetadata(Cloned, Instr);
23542354

23552355
// Place the cloned scalar in the new loop.
23562356
State.Builder.Insert(Cloned);
@@ -7895,24 +7895,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
78957895
if (VectorizingEpilogue)
78967896
VPlanTransforms::removeDeadRecipes(BestVPlan);
78977897

7898-
// Only use noalias metadata when using memory checks guaranteeing no overlap
7899-
// across all iterations.
7900-
const LoopAccessInfo *LAI = Legal->getLAI();
7901-
std::unique_ptr<LoopVersioning> LVer = nullptr;
7902-
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
7903-
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
7904-
7905-
// We currently don't use LoopVersioning for the actual loop cloning but we
7906-
// still use it to add the noalias metadata.
7907-
// TODO: Find a better way to re-use LoopVersioning functionality to add
7908-
// metadata.
7909-
LVer = std::make_unique<LoopVersioning>(
7910-
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
7911-
PSE.getSE());
7912-
State.LVer = &*LVer;
7913-
State.LVer->prepareNoAliasMetadata();
7914-
}
7915-
79167898
ILV.printDebugTracesAtStart();
79177899

79187900
//===------------------------------------------------===//
@@ -8503,13 +8485,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
85038485
Builder.insert(VectorPtr);
85048486
Ptr = VectorPtr;
85058487
}
8488+
auto Metadata = getRecipeMetadata(I);
85068489
if (LoadInst *Load = dyn_cast<LoadInst>(I))
85078490
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
8508-
I->getDebugLoc());
8491+
Metadata, I->getDebugLoc());
85098492

85108493
StoreInst *Store = cast<StoreInst>(I);
85118494
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
8512-
Reverse, I->getDebugLoc());
8495+
Reverse, Metadata, I->getDebugLoc());
85138496
}
85148497

85158498
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8882,7 +8865,8 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
88828865
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
88838866
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
88848867
"Should not predicate a uniform recipe");
8885-
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask);
8868+
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
8869+
getRecipeMetadata(I));
88868870
return Recipe;
88878871
}
88888872

@@ -9003,6 +8987,20 @@ bool VPRecipeBuilder::getScaledReductions(
90038987
return false;
90048988
}
90058989

8990+
VPIRMetadata VPRecipeBuilder::getRecipeMetadata(Instruction *I) const {
8991+
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
8992+
::getMetadataToPropagate(I, Metadata);
8993+
if (!LVer || !isa<LoadInst, StoreInst>(I))
8994+
return {};
8995+
8996+
const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
8997+
if (AliasScopeMD)
8998+
Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
8999+
if (NoAliasMD)
9000+
Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
9001+
return {Metadata};
9002+
}
9003+
90069004
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
90079005
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
90089006
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9129,10 +9127,20 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
91299127
ElementCount MaxVF) {
91309128
assert(OrigLoop->isInnermost() && "Inner loop expected.");
91319129

9130+
const LoopAccessInfo *LAI = Legal->getLAI();
9131+
LoopVersioning LVer(*LAI, LAI->getRuntimePointerChecking()->getChecks(),
9132+
OrigLoop, LI, DT, PSE.getSE());
9133+
if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
9134+
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
9135+
// Only use noalias metadata when using memory checks guaranteeing no
9136+
// overlap across all iterations.
9137+
LVer.prepareNoAliasMetadata();
9138+
}
9139+
91329140
auto MaxVFTimes2 = MaxVF * 2;
91339141
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
91349142
VFRange SubRange = {VF, MaxVFTimes2};
9135-
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
9143+
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, &LVer)) {
91369144
bool HasScalarVF = Plan->hasScalarVFOnly();
91379145
// Now optimize the initial VPlan.
91389146
if (!HasScalarVF)
@@ -9397,7 +9405,8 @@ static void addExitUsersForFirstOrderRecurrences(
93979405
}
93989406

93999407
VPlanPtr
9400-
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
9408+
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
9409+
LoopVersioning *LVer) {
94019410

94029411
using namespace llvm::VPlanPatternMatch;
94039412
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -9453,7 +9462,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
94539462
}
94549463

94559464
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9456-
Builder);
9465+
Builder, LVer);
94579466

94589467
// ---------------------------------------------------------------------------
94599468
// Pre-construction: record ingredients whose recipes we'll need to further
@@ -9559,8 +9568,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
95599568
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
95609569
// Only create recipe for the final invariant store of the reduction.
95619570
if (Legal->isInvariantStoreOfReduction(SI)) {
9562-
auto *Recipe =
9563-
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */);
9571+
auto *Recipe = new VPReplicateRecipe(
9572+
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
9573+
RecipeBuilder.getRecipeMetadata(SI));
95649574
Recipe->insertBefore(*MiddleVPBB, MBIP);
95659575
}
95669576
R.eraseFromParent();

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ class VPRecipeBuilder {
9090
/// A mapping of partial reduction exit instructions to their scaling factor.
9191
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
9292

93+
/// Loop versioning instance for getting noalias metadata guaranteed by
94+
/// runtime checks.
95+
LoopVersioning *LVer;
96+
9397
/// Check if \p I can be widened at the start of \p Range and possibly
9498
/// decrease the range such that the returned value holds for the entire \p
9599
/// Range. The function should not be called for memory instructions or calls.
@@ -155,9 +159,10 @@ class VPRecipeBuilder {
155159
const TargetTransformInfo *TTI,
156160
LoopVectorizationLegality *Legal,
157161
LoopVectorizationCostModel &CM,
158-
PredicatedScalarEvolution &PSE, VPBuilder &Builder)
162+
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
163+
LoopVersioning *LVer = nullptr)
159164
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
160-
CM(CM), PSE(PSE), Builder(Builder) {}
165+
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
161166

162167
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
163168
auto It = ScaledReductionMap.find(ExitInst);
@@ -233,6 +238,10 @@ class VPRecipeBuilder {
233238
}
234239
return Plan.getOrAddLiveIn(V);
235240
}
241+
242+
/// Returns the metatadata that can be preserved from the original instruction
243+
/// \p I, including noalias metadata guaranteed by runtime checks.
244+
VPIRMetadata getRecipeMetadata(Instruction *I) const;
236245
};
237246
} // end namespace llvm
238247

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,9 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
219219
DominatorTree *DT, AssumptionCache *AC,
220220
IRBuilderBase &Builder, VPlan *Plan,
221221
Loop *CurrentParentLoop, Type *CanonicalIVTy)
222-
: TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan),
223-
CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
224-
TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
222+
: TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
223+
CurrentParentLoop(CurrentParentLoop), LVer(nullptr), TypeAnalysis(CanonicalIVTy),
224+
VPDT(*Plan) {}
225225

226226
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
227227
if (Def->isLiveIn())
@@ -355,14 +355,6 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
355355
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
356356
}
357357

358-
void VPTransformState::addNewMetadata(Instruction *To,
359-
const Instruction *Orig) {
360-
// If the loop was versioned with memchecks, add the corresponding no-alias
361-
// metadata.
362-
if (LVer && isa<LoadInst, StoreInst>(Orig))
363-
LVer->annotateInstWithNoAlias(To, Orig);
364-
}
365-
366358
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
367359
const DILocation *DIL = DL;
368360
// When a FSDiscriminator is enabled, we don't need to add the multiply

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1200,10 +1200,14 @@ class VPIRMetadata {
12001200
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
12011201

12021202
protected:
1203-
VPIRMetadata() {}
12041203
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
12051204

12061205
public:
1206+
VPIRMetadata() {}
1207+
VPIRMetadata(ArrayRef<std::pair<unsigned, MDNode *>> Metadata)
1208+
: Metadata(Metadata) {}
1209+
VPIRMetadata(const VPIRMetadata &Other) : Metadata(Other.Metadata) {}
1210+
12071211
/// Add all metadata to \p I.
12081212
void applyMetadata(Instruction &I) const;
12091213
};
@@ -2467,7 +2471,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
24672471
/// copies of the original scalar type, one per lane, instead of producing a
24682472
/// single copy of widened type for all lanes. If the instruction is known to be
24692473
/// uniform only one copy, per lane zero, will be generated.
2470-
class VPReplicateRecipe : public VPRecipeWithIRFlags {
2474+
class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
24712475
/// Indicator if only a single replica per lane is needed.
24722476
bool IsUniform;
24732477

@@ -2476,9 +2480,10 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
24762480

24772481
public:
24782482
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
2479-
bool IsUniform, VPValue *Mask = nullptr)
2483+
bool IsUniform, VPValue *Mask = nullptr,
2484+
VPIRMetadata Metadata = {})
24802485
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2481-
IsUniform(IsUniform), IsPredicated(Mask) {
2486+
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
24822487
if (Mask)
24832488
addOperand(Mask);
24842489
}
@@ -2488,7 +2493,7 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
24882493
VPReplicateRecipe *clone() override {
24892494
auto *Copy =
24902495
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2491-
isPredicated() ? getMask() : nullptr);
2496+
isPredicated() ? getMask() : nullptr, *this);
24922497
Copy->transferFlags(*this);
24932498
return Copy;
24942499
}
@@ -2648,8 +2653,9 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
26482653

26492654
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
26502655
std::initializer_list<VPValue *> Operands,
2651-
bool Consecutive, bool Reverse, DebugLoc DL)
2652-
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
2656+
bool Consecutive, bool Reverse,
2657+
const VPIRMetadata &Metadata, DebugLoc DL)
2658+
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
26532659
Consecutive(Consecutive), Reverse(Reverse) {
26542660
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
26552661
}
@@ -2707,16 +2713,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
27072713
/// optional mask.
27082714
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
27092715
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
2710-
bool Consecutive, bool Reverse, DebugLoc DL)
2716+
bool Consecutive, bool Reverse,
2717+
const VPIRMetadata &Metadata, DebugLoc DL)
27112718
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
2712-
Reverse, DL),
2719+
Reverse, Metadata, DL),
27132720
VPValue(this, &Load) {
27142721
setMask(Mask);
27152722
}
27162723

27172724
VPWidenLoadRecipe *clone() override {
27182725
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
2719-
getMask(), Consecutive, Reverse,
2726+
getMask(), Consecutive, Reverse, *this,
27202727
getDebugLoc());
27212728
}
27222729

@@ -2748,7 +2755,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27482755
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
27492756
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
27502757
{L.getAddr(), &EVL}, L.isConsecutive(),
2751-
L.isReverse(), L.getDebugLoc()),
2758+
L.isReverse(), L, L.getDebugLoc()),
27522759
VPValue(this, &getIngredient()) {
27532760
setMask(Mask);
27542761
}
@@ -2785,16 +2792,17 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27852792
/// to store to and an optional mask.
27862793
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
27872794
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
2788-
VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
2795+
VPValue *Mask, bool Consecutive, bool Reverse,
2796+
const VPIRMetadata &Metadata, DebugLoc DL)
27892797
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
2790-
Consecutive, Reverse, DL) {
2798+
Consecutive, Reverse, Metadata, DL) {
27912799
setMask(Mask);
27922800
}
27932801

27942802
VPWidenStoreRecipe *clone() override {
27952803
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
27962804
getStoredValue(), getMask(), Consecutive,
2797-
Reverse, getDebugLoc());
2805+
Reverse, *this, getDebugLoc());
27982806
}
27992807

28002808
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2828,7 +2836,8 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
28282836
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
28292837
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
28302838
{S.getAddr(), S.getStoredValue(), &EVL},
2831-
S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
2839+
S.isConsecutive(), S.isReverse(), S,
2840+
S.getDebugLoc()) {
28322841
setMask(Mask);
28332842
}
28342843

‎llvm/lib/Transforms/Vectorize/VPlanHelpers.h

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ class VPBasicBlock;
3939
class VPRegionBlock;
4040
class VPlan;
4141
class Value;
42-
class LoopVersioning;
4342

4443
/// Returns a calculation for the total number of elements for a given \p VF.
4544
/// For fixed width vectors this value is a constant, whereas for scalable
@@ -284,13 +283,6 @@ struct VPTransformState {
284283
Iter->second[CacheIdx] = V;
285284
}
286285

287-
/// Add additional metadata to \p To that was not present on \p Orig.
288-
///
289-
/// Currently this is used to add the noalias annotations based on the
290-
/// inserted memchecks. Use this for instructions that are *cloned* into the
291-
/// vector loop.
292-
void addNewMetadata(Instruction *To, const Instruction *Orig);
293-
294286
/// Set the debug location in the builder using the debug location \p DL.
295287
void setDebugLocFrom(DebugLoc DL);
296288

@@ -343,13 +335,6 @@ struct VPTransformState {
343335
/// The parent loop object for the current scope, or nullptr.
344336
Loop *CurrentParentLoop = nullptr;
345337

346-
/// LoopVersioning. It's only set up (non-null) if memchecks were
347-
/// used.
348-
///
349-
/// This is currently only used to add no-alias metadata based on the
350-
/// memchecks. The actually versioning is performed manually.
351-
LoopVersioning *LVer = nullptr;
352-
353338
/// VPlan-based type analysis.
354339
VPTypeAnalysis TypeAnalysis;
355340

0 commit comments

Comments
 (0)