Skip to content

Commit 499becf

Browse files
fhahnfhahn
committed
[VPlan] Manage noalias/alias_scope metadata in VPlan.
Use VPIRMetadata added in llvm#135272 to also manage no-alias metadata added by versioning. Note that this means we have to build the no-alias metadata up-front once. If it is not used, it will be discarded automatically.
1 parent 1b60b83 commit 499becf

File tree

10 files changed

+107
-103
lines changed

10 files changed

+107
-103
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class LoopVectorizationLegality;
3636
class LoopVectorizationCostModel;
3737
class PredicatedScalarEvolution;
3838
class LoopVectorizeHints;
39+
class LoopVersioning;
3940
class OptimizationRemarkEmitter;
4041
class TargetTransformInfo;
4142
class TargetLibraryInfo;
@@ -518,7 +519,7 @@ class LoopVectorizationPlanner {
518519
/// returned VPlan is valid for. If no VPlan can be built for the input range,
519520
/// set the largest included VF to the maximum VF for which no plan could be
520521
/// built.
521-
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range);
522+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range, LoopVersioning *LVer);
522523

523524
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
524525
/// according to the information gathered by Legal when it checked if it is

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 39 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2350,7 +2350,7 @@ static void scalarizeInstruction(const Instruction *Instr,
23502350
InputLane = VPLane::getFirstLane();
23512351
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
23522352
}
2353-
State.addNewMetadata(Cloned, Instr);
2353+
RepRecipe->applyMetadata(*Cloned);
23542354

23552355
// Place the cloned scalar in the new loop.
23562356
State.Builder.Insert(Cloned);
@@ -7895,24 +7895,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
78957895
if (VectorizingEpilogue)
78967896
VPlanTransforms::removeDeadRecipes(BestVPlan);
78977897

7898-
// Only use noalias metadata when using memory checks guaranteeing no overlap
7899-
// across all iterations.
7900-
const LoopAccessInfo *LAI = Legal->getLAI();
7901-
std::unique_ptr<LoopVersioning> LVer = nullptr;
7902-
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
7903-
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
7904-
7905-
// We currently don't use LoopVersioning for the actual loop cloning but we
7906-
// still use it to add the noalias metadata.
7907-
// TODO: Find a better way to re-use LoopVersioning functionality to add
7908-
// metadata.
7909-
LVer = std::make_unique<LoopVersioning>(
7910-
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
7911-
PSE.getSE());
7912-
State.LVer = &*LVer;
7913-
State.LVer->prepareNoAliasMetadata();
7914-
}
7915-
79167898
ILV.printDebugTracesAtStart();
79177899

79187900
//===------------------------------------------------===//
@@ -8503,13 +8485,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
85038485
Builder.insert(VectorPtr);
85048486
Ptr = VectorPtr;
85058487
}
8488+
auto Metadata = getMetadataToPropagate(I);
85068489
if (LoadInst *Load = dyn_cast<LoadInst>(I))
85078490
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
8508-
I->getDebugLoc());
8491+
Metadata, I->getDebugLoc());
85098492

85108493
StoreInst *Store = cast<StoreInst>(I);
85118494
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
8512-
Reverse, I->getDebugLoc());
8495+
Reverse, Metadata, I->getDebugLoc());
85138496
}
85148497

85158498
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8882,7 +8865,7 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
88828865
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
88838866
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
88848867
"Should not predicate a uniform recipe");
8885-
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask);
8868+
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask, getMetadataToPropagate(I));
88868869
return Recipe;
88878870
}
88888871

@@ -9003,6 +8986,20 @@ bool VPRecipeBuilder::getScaledReductions(
90038986
return false;
90048987
}
90058988

8989+
SmallVector<std::pair<unsigned, MDNode *>>
8990+
VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
8991+
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
8992+
::getMetadataToPropagate(I, Metadata);
8993+
if (LVer && isa<LoadInst, StoreInst>(I)) {
8994+
const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
8995+
if (AliasScopeMD)
8996+
Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
8997+
if (NoAliasMD)
8998+
Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
8999+
}
9000+
return Metadata;
9001+
}
9002+
90069003
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
90079004
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
90089005
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9129,10 +9126,22 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
91299126
ElementCount MaxVF) {
91309127
assert(OrigLoop->isInnermost() && "Inner loop expected.");
91319128

9129+
// Only use noalias metadata when using memory checks guaranteeing no overlap
9130+
// across all iterations.
9131+
const LoopAccessInfo *LAI = Legal->getLAI();
9132+
std::unique_ptr<LoopVersioning> LVer = nullptr;
9133+
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
9134+
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
9135+
LVer = std::make_unique<LoopVersioning>(
9136+
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
9137+
PSE.getSE());
9138+
LVer->prepareNoAliasMetadata();
9139+
}
9140+
91329141
auto MaxVFTimes2 = MaxVF * 2;
91339142
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
91349143
VFRange SubRange = {VF, MaxVFTimes2};
9135-
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
9144+
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, LVer.get())) {
91369145
bool HasScalarVF = Plan->hasScalarVFOnly();
91379146
// Now optimize the initial VPlan.
91389147
if (!HasScalarVF)
@@ -9397,7 +9406,8 @@ static void addExitUsersForFirstOrderRecurrences(
93979406
}
93989407

93999408
VPlanPtr
9400-
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
9409+
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
9410+
LoopVersioning *LVer) {
94019411

94029412
using namespace llvm::VPlanPatternMatch;
94039413
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -9453,7 +9463,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
94539463
}
94549464

94559465
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9456-
Builder);
9466+
Builder, LVer);
94579467

94589468
// ---------------------------------------------------------------------------
94599469
// Pre-construction: record ingredients whose recipes we'll need to further
@@ -9559,8 +9569,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
95599569
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
95609570
// Only create recipe for the final invariant store of the reduction.
95619571
if (Legal->isInvariantStoreOfReduction(SI)) {
9562-
auto *Recipe =
9563-
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */);
9572+
auto *Recipe = new VPReplicateRecipe(
9573+
SI, R.operands(), true /* IsUniform */, /*Mask*/ nullptr,
9574+
RecipeBuilder.getMetadataToPropagate(SI));
95649575
Recipe->insertBefore(*MiddleVPBB, MBIP);
95659576
}
95669577
R.eraseFromParent();
@@ -9742,7 +9753,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
97429753
// Collect mapping of IR header phis to header phi recipes, to be used in
97439754
// addScalarResumePhis.
97449755
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9745-
Builder);
9756+
Builder, nullptr);
97469757
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
97479758
if (isa<VPCanonicalIVPHIRecipe>(&R))
97489759
continue;

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ class VPRecipeBuilder {
9090
/// A mapping of partial reduction exit instructions to their scaling factor.
9191
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
9292

93+
/// Loop versioning instance for getting noalias metadata guaranteed by
94+
/// runtime checks.
95+
LoopVersioning *LVer;
96+
9397
/// Check if \p I can be widened at the start of \p Range and possibly
9498
/// decrease the range such that the returned value holds for the entire \p
9599
/// Range. The function should not be called for memory instructions or calls.
@@ -155,9 +159,10 @@ class VPRecipeBuilder {
155159
const TargetTransformInfo *TTI,
156160
LoopVectorizationLegality *Legal,
157161
LoopVectorizationCostModel &CM,
158-
PredicatedScalarEvolution &PSE, VPBuilder &Builder)
162+
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
163+
LoopVersioning *LVer)
159164
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
160-
CM(CM), PSE(PSE), Builder(Builder) {}
165+
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
161166

162167
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
163168
auto It = ScaledReductionMap.find(ExitInst);
@@ -233,6 +238,11 @@ class VPRecipeBuilder {
233238
}
234239
return Plan.getOrAddLiveIn(V);
235240
}
241+
242+
/// Returns the metatadata that can be preserved from the original instruction
243+
/// \p I, including noalias metadata guaranteed by runtime checks.
244+
SmallVector<std::pair<unsigned, MDNode *>>
245+
getMetadataToPropagate(Instruction *I) const;
236246
};
237247
} // end namespace llvm
238248

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -219,9 +219,9 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
219219
DominatorTree *DT, AssumptionCache *AC,
220220
IRBuilderBase &Builder, VPlan *Plan,
221221
Loop *CurrentParentLoop, Type *CanonicalIVTy)
222-
: TTI(TTI), VF(VF), CFG(DT), LI(LI), AC(AC), Builder(Builder), Plan(Plan),
223-
CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
224-
TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
222+
: TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
223+
CurrentParentLoop(CurrentParentLoop), LVer(nullptr), TypeAnalysis(CanonicalIVTy),
224+
VPDT(*Plan) {}
225225

226226
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
227227
if (Def->isLiveIn())
@@ -355,14 +355,6 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
355355
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
356356
}
357357

358-
void VPTransformState::addNewMetadata(Instruction *To,
359-
const Instruction *Orig) {
360-
// If the loop was versioned with memchecks, add the corresponding no-alias
361-
// metadata.
362-
if (LVer && isa<LoadInst, StoreInst>(Orig))
363-
LVer->annotateInstWithNoAlias(To, Orig);
364-
}
365-
366358
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
367359
const DILocation *DIL = DL;
368360
// When a FSDiscriminator is enabled, we don't need to add the multiply

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1194,6 +1194,8 @@ struct VPIRPhi : public VPIRInstruction {
11941194
#endif
11951195
};
11961196

1197+
using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
1198+
11971199
/// Helper to manage IR metadata for recipes. It filters out metadata that
11981200
/// cannot be propagated.
11991201
class VPIRMetadata {
@@ -1202,10 +1204,14 @@ class VPIRMetadata {
12021204
protected:
12031205
VPIRMetadata() {}
12041206
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
1207+
VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
12051208

12061209
public:
12071210
/// Add all metadata to \p I.
12081211
void applyMetadata(Instruction &I) const;
1212+
1213+
/// Return the IR metadata.
1214+
MDArrayRef getMetadata() const { return Metadata; }
12091215
};
12101216

12111217
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2467,7 +2473,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
24672473
/// copies of the original scalar type, one per lane, instead of producing a
24682474
/// single copy of widened type for all lanes. If the instruction is known to be
24692475
/// uniform only one copy, per lane zero, will be generated.
2470-
class VPReplicateRecipe : public VPRecipeWithIRFlags {
2476+
class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
24712477
/// Indicator if only a single replica per lane is needed.
24722478
bool IsUniform;
24732479

@@ -2476,19 +2482,19 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
24762482

24772483
public:
24782484
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
2479-
bool IsUniform, VPValue *Mask = nullptr)
2485+
bool IsUniform, VPValue *Mask = nullptr, ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
24802486
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2481-
IsUniform(IsUniform), IsPredicated(Mask) {
2487+
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
24822488
if (Mask)
24832489
addOperand(Mask);
24842490
}
24852491

24862492
~VPReplicateRecipe() override = default;
24872493

24882494
VPReplicateRecipe *clone() override {
2489-
auto *Copy =
2490-
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2491-
isPredicated() ? getMask() : nullptr);
2495+
auto *Copy = new VPReplicateRecipe(
2496+
getUnderlyingInstr(), operands(), IsUniform,
2497+
isPredicated() ? getMask() : nullptr, getMetadata());
24922498
Copy->transferFlags(*this);
24932499
return Copy;
24942500
}
@@ -2648,8 +2654,9 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
26482654

26492655
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
26502656
std::initializer_list<VPValue *> Operands,
2651-
bool Consecutive, bool Reverse, DebugLoc DL)
2652-
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
2657+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2658+
DebugLoc DL)
2659+
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
26532660
Consecutive(Consecutive), Reverse(Reverse) {
26542661
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
26552662
}
@@ -2707,16 +2714,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
27072714
/// optional mask.
27082715
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
27092716
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
2710-
bool Consecutive, bool Reverse, DebugLoc DL)
2717+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2718+
DebugLoc DL)
27112719
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
2712-
Reverse, DL),
2720+
Reverse, Metadata, DL),
27132721
VPValue(this, &Load) {
27142722
setMask(Mask);
27152723
}
27162724

27172725
VPWidenLoadRecipe *clone() override {
27182726
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
2719-
getMask(), Consecutive, Reverse,
2727+
getMask(), Consecutive, Reverse, getMetadata(),
27202728
getDebugLoc());
27212729
}
27222730

@@ -2748,7 +2756,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27482756
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
27492757
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
27502758
{L.getAddr(), &EVL}, L.isConsecutive(),
2751-
L.isReverse(), L.getDebugLoc()),
2759+
L.isReverse(), L.getMetadata(), L.getDebugLoc()),
27522760
VPValue(this, &getIngredient()) {
27532761
setMask(Mask);
27542762
}
@@ -2785,16 +2793,17 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27852793
/// to store to and an optional mask.
27862794
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
27872795
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
2788-
VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
2796+
VPValue *Mask, bool Consecutive, bool Reverse,
2797+
MDArrayRef Metadata, DebugLoc DL)
27892798
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
2790-
Consecutive, Reverse, DL) {
2799+
Consecutive, Reverse, Metadata, DL) {
27912800
setMask(Mask);
27922801
}
27932802

27942803
VPWidenStoreRecipe *clone() override {
27952804
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
27962805
getStoredValue(), getMask(), Consecutive,
2797-
Reverse, getDebugLoc());
2806+
Reverse, getMetadata(), getDebugLoc());
27982807
}
27992808

28002809
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2828,7 +2837,8 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
28282837
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
28292838
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
28302839
{S.getAddr(), S.getStoredValue(), &EVL},
2831-
S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
2840+
S.isConsecutive(), S.isReverse(), S.getMetadata(),
2841+
S.getDebugLoc()) {
28322842
setMask(Mask);
28332843
}
28342844

‎llvm/lib/Transforms/Vectorize/VPlanHelpers.h

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ class VPBasicBlock;
3939
class VPRegionBlock;
4040
class VPlan;
4141
class Value;
42-
class LoopVersioning;
4342

4443
/// Returns a calculation for the total number of elements for a given \p VF.
4544
/// For fixed width vectors this value is a constant, whereas for scalable
@@ -284,13 +283,6 @@ struct VPTransformState {
284283
Iter->second[CacheIdx] = V;
285284
}
286285

287-
/// Add additional metadata to \p To that was not present on \p Orig.
288-
///
289-
/// Currently this is used to add the noalias annotations based on the
290-
/// inserted memchecks. Use this for instructions that are *cloned* into the
291-
/// vector loop.
292-
void addNewMetadata(Instruction *To, const Instruction *Orig);
293-
294286
/// Set the debug location in the builder using the debug location \p DL.
295287
void setDebugLocFrom(DebugLoc DL);
296288

@@ -343,13 +335,6 @@ struct VPTransformState {
343335
/// The parent loop object for the current scope, or nullptr.
344336
Loop *CurrentParentLoop = nullptr;
345337

346-
/// LoopVersioning. It's only set up (non-null) if memchecks were
347-
/// used.
348-
///
349-
/// This is currently only used to add no-alias metadata based on the
350-
/// memchecks. The actually versioning is performed manually.
351-
LoopVersioning *LVer = nullptr;
352-
353338
/// VPlan-based type analysis.
354339
VPTypeAnalysis TypeAnalysis;
355340

0 commit comments

Comments
 (0)