Skip to content

Commit 5252ec1

Browse files
fhahnfhahn
committed
[VPlan] Manage noalias/alias_scope metadata in VPlan.
Use VPIRMetadata added in llvm#135272 to also manage no-alias metadata added by versioning. Note that this means we have to build the no-alias metadata up-front once. If it is not used, it will be discarded automatically.
1 parent 5f704f9 commit 5252ec1

File tree

10 files changed

+109
-103
lines changed

10 files changed

+109
-103
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ class LoopVectorizationLegality;
3636
class LoopVectorizationCostModel;
3737
class PredicatedScalarEvolution;
3838
class LoopVectorizeHints;
39+
class LoopVersioning;
3940
class OptimizationRemarkEmitter;
4041
class TargetTransformInfo;
4142
class TargetLibraryInfo;
@@ -515,7 +516,7 @@ class LoopVectorizationPlanner {
515516
/// returned VPlan is valid for. If no VPlan can be built for the input range,
516517
/// set the largest included VF to the maximum VF for which no plan could be
517518
/// built.
518-
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range);
519+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range, LoopVersioning *LVer);
519520

520521
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
521522
/// according to the information gathered by Legal when it checked if it is

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 41 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2357,7 +2357,7 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
23572357
InputLane = VPLane::getFirstLane();
23582358
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
23592359
}
2360-
State.addNewMetadata(Cloned, Instr);
2360+
RepRecipe->applyMetadata(*Cloned);
23612361

23622362
// Place the cloned scalar in the new loop.
23632363
State.Builder.Insert(Cloned);
@@ -7893,24 +7893,6 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
78937893
if (VectorizingEpilogue)
78947894
VPlanTransforms::removeDeadRecipes(BestVPlan);
78957895

7896-
// Only use noalias metadata when using memory checks guaranteeing no overlap
7897-
// across all iterations.
7898-
const LoopAccessInfo *LAI = Legal->getLAI();
7899-
std::unique_ptr<LoopVersioning> LVer = nullptr;
7900-
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
7901-
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
7902-
7903-
// We currently don't use LoopVersioning for the actual loop cloning but we
7904-
// still use it to add the noalias metadata.
7905-
// TODO: Find a better way to re-use LoopVersioning functionality to add
7906-
// metadata.
7907-
LVer = std::make_unique<LoopVersioning>(
7908-
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
7909-
PSE.getSE());
7910-
State.LVer = &*LVer;
7911-
State.LVer->prepareNoAliasMetadata();
7912-
}
7913-
79147896
ILV.printDebugTracesAtStart();
79157897

79167898
//===------------------------------------------------===//
@@ -8501,13 +8483,14 @@ VPRecipeBuilder::tryToWidenMemory(Instruction *I, ArrayRef<VPValue *> Operands,
85018483
Builder.insert(VectorPtr);
85028484
Ptr = VectorPtr;
85038485
}
8486+
auto Metadata = getMetadataToPropagate(I);
85048487
if (LoadInst *Load = dyn_cast<LoadInst>(I))
85058488
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
8506-
I->getDebugLoc());
8489+
Metadata, I->getDebugLoc());
85078490

85088491
StoreInst *Store = cast<StoreInst>(I);
85098492
return new VPWidenStoreRecipe(*Store, Ptr, Operands[0], Mask, Consecutive,
8510-
Reverse, I->getDebugLoc());
8493+
Reverse, Metadata, I->getDebugLoc());
85118494
}
85128495

85138496
/// Creates a VPWidenIntOrFpInductionRecpipe for \p Phi. If needed, it will also
@@ -8882,8 +8865,9 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
88828865
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
88838866
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
88848867
"Should not predicate a uniform recipe");
8885-
auto *Recipe = new VPReplicateRecipe(
8886-
I, make_range(Operands.begin(), Operands.end()), IsUniform, BlockInMask);
8868+
auto *Recipe =
8869+
new VPReplicateRecipe(I, make_range(Operands.begin(), Operands.end()),
8870+
IsUniform, BlockInMask, getMetadataToPropagate(I));
88878871
return Recipe;
88888872
}
88898873

@@ -9004,6 +8988,20 @@ bool VPRecipeBuilder::getScaledReductions(
90048988
return false;
90058989
}
90068990

8991+
SmallVector<std::pair<unsigned, MDNode *>>
8992+
VPRecipeBuilder::getMetadataToPropagate(Instruction *I) const {
8993+
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
8994+
::getMetadataToPropagate(I, Metadata);
8995+
if (LVer && isa<LoadInst, StoreInst>(I)) {
8996+
const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(I);
8997+
if (AliasScopeMD)
8998+
Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
8999+
if (NoAliasMD)
9000+
Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
9001+
}
9002+
return Metadata;
9003+
}
9004+
90079005
VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(
90089006
Instruction *Instr, ArrayRef<VPValue *> Operands, VFRange &Range) {
90099007
// First, check for specific widening recipes that deal with inductions, Phi
@@ -9131,10 +9129,22 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
91319129
ElementCount MaxVF) {
91329130
assert(OrigLoop->isInnermost() && "Inner loop expected.");
91339131

9132+
// Only use noalias metadata when using memory checks guaranteeing no overlap
9133+
// across all iterations.
9134+
const LoopAccessInfo *LAI = Legal->getLAI();
9135+
std::unique_ptr<LoopVersioning> LVer = nullptr;
9136+
if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
9137+
!LAI->getRuntimePointerChecking()->getDiffChecks()) {
9138+
LVer = std::make_unique<LoopVersioning>(
9139+
*LAI, LAI->getRuntimePointerChecking()->getChecks(), OrigLoop, LI, DT,
9140+
PSE.getSE());
9141+
LVer->prepareNoAliasMetadata();
9142+
}
9143+
91349144
auto MaxVFTimes2 = MaxVF * 2;
91359145
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
91369146
VFRange SubRange = {VF, MaxVFTimes2};
9137-
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) {
9147+
if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange, LVer.get())) {
91389148
bool HasScalarVF = Plan->hasScalarVFOnly();
91399149
// Now optimize the initial VPlan.
91409150
if (!HasScalarVF)
@@ -9424,7 +9434,8 @@ static void addExitUsersForFirstOrderRecurrences(
94249434
}
94259435

94269436
VPlanPtr
9427-
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
9437+
LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range,
9438+
LoopVersioning *LVer) {
94289439

94299440
using namespace llvm::VPlanPatternMatch;
94309441
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
@@ -9470,7 +9481,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
94709481
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), HasNUW, DL);
94719482

94729483
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9473-
Builder);
9484+
Builder, LVer);
94749485

94759486
// ---------------------------------------------------------------------------
94769487
// Pre-construction: record ingredients whose recipes we'll need to further
@@ -9576,8 +9587,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
95769587
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
95779588
// Only create recipe for the final invariant store of the reduction.
95789589
if (Legal->isInvariantStoreOfReduction(SI)) {
9579-
auto *Recipe =
9580-
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */);
9590+
auto *Recipe = new VPReplicateRecipe(
9591+
SI, R.operands(), true /* IsUniform */, /*Mask*/ nullptr,
9592+
RecipeBuilder.getMetadataToPropagate(SI));
95819593
Recipe->insertBefore(*MiddleVPBB, MBIP);
95829594
}
95839595
R.eraseFromParent();
@@ -9763,7 +9775,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
97639775
// Collect mapping of IR header phis to header phi recipes, to be used in
97649776
// addScalarResumePhis.
97659777
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
9766-
Builder);
9778+
Builder, nullptr);
97679779
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
97689780
if (isa<VPCanonicalIVPHIRecipe>(&R))
97699781
continue;

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ class VPRecipeBuilder {
9090
/// A mapping of partial reduction exit instructions to their scaling factor.
9191
DenseMap<const Instruction *, unsigned> ScaledReductionMap;
9292

93+
/// Loop versioning instance for getting noalias metadata guaranteed by
94+
/// runtime checks.
95+
LoopVersioning *LVer;
96+
9397
/// Check if \p I can be widened at the start of \p Range and possibly
9498
/// decrease the range such that the returned value holds for the entire \p
9599
/// Range. The function should not be called for memory instructions or calls.
@@ -155,9 +159,10 @@ class VPRecipeBuilder {
155159
const TargetTransformInfo *TTI,
156160
LoopVectorizationLegality *Legal,
157161
LoopVectorizationCostModel &CM,
158-
PredicatedScalarEvolution &PSE, VPBuilder &Builder)
162+
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
163+
LoopVersioning *LVer)
159164
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
160-
CM(CM), PSE(PSE), Builder(Builder) {}
165+
CM(CM), PSE(PSE), Builder(Builder), LVer(LVer) {}
161166

162167
std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
163168
auto It = ScaledReductionMap.find(ExitInst);
@@ -233,6 +238,11 @@ class VPRecipeBuilder {
233238
}
234239
return Plan.getOrAddLiveIn(V);
235240
}
241+
242+
/// Returns the metatadata that can be preserved from the original instruction
243+
/// \p I, including noalias metadata guaranteed by runtime checks.
244+
SmallVector<std::pair<unsigned, MDNode *>>
245+
getMetadataToPropagate(Instruction *I) const;
236246
};
237247
} // end namespace llvm
238248

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -220,8 +220,8 @@ VPTransformState::VPTransformState(const TargetTransformInfo *TTI,
220220
InnerLoopVectorizer *ILV, VPlan *Plan,
221221
Loop *CurrentParentLoop, Type *CanonicalIVTy)
222222
: TTI(TTI), VF(VF), CFG(DT), LI(LI), Builder(Builder), ILV(ILV), Plan(Plan),
223-
CurrentParentLoop(CurrentParentLoop), LVer(nullptr),
224-
TypeAnalysis(CanonicalIVTy), VPDT(*Plan) {}
223+
CurrentParentLoop(CurrentParentLoop), TypeAnalysis(CanonicalIVTy),
224+
VPDT(*Plan) {}
225225

226226
Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
227227
if (Def->isLiveIn())
@@ -355,14 +355,6 @@ BasicBlock *VPTransformState::CFGState::getPreheaderBBFor(VPRecipeBase *R) {
355355
return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
356356
}
357357

358-
void VPTransformState::addNewMetadata(Instruction *To,
359-
const Instruction *Orig) {
360-
// If the loop was versioned with memchecks, add the corresponding no-alias
361-
// metadata.
362-
if (LVer && isa<LoadInst, StoreInst>(Orig))
363-
LVer->annotateInstWithNoAlias(To, Orig);
364-
}
365-
366358
void VPTransformState::setDebugLocFrom(DebugLoc DL) {
367359
const DILocation *DIL = DL;
368360
// When a FSDiscriminator is enabled, we don't need to add the multiply

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1194,6 +1194,8 @@ struct VPIRPhi : public VPIRInstruction {
11941194
#endif
11951195
};
11961196

1197+
using MDArrayRef = ArrayRef<std::pair<unsigned, MDNode *>>;
1198+
11971199
/// Helper to manage IR metadata for recipes. It filters out metadata that
11981200
/// cannot be propagated.
11991201
class VPIRMetadata {
@@ -1202,10 +1204,14 @@ class VPIRMetadata {
12021204
protected:
12031205
VPIRMetadata() {}
12041206
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
1207+
VPIRMetadata(MDArrayRef Metadata) : Metadata(Metadata) {}
12051208

12061209
public:
12071210
/// Add all metadata to \p I.
12081211
void applyMetadata(Instruction &I) const;
1212+
1213+
/// Return the IR metadata.
1214+
MDArrayRef getMetadata() const { return Metadata; }
12091215
};
12101216

12111217
/// VPWidenRecipe is a recipe for producing a widened instruction using the
@@ -2466,7 +2472,7 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
24662472
/// copies of the original scalar type, one per lane, instead of producing a
24672473
/// single copy of widened type for all lanes. If the instruction is known to be
24682474
/// uniform only one copy, per lane zero, will be generated.
2469-
class VPReplicateRecipe : public VPRecipeWithIRFlags {
2475+
class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
24702476
/// Indicator if only a single replica per lane is needed.
24712477
bool IsUniform;
24722478

@@ -2476,19 +2482,20 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags {
24762482
public:
24772483
template <typename IterT>
24782484
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
2479-
bool IsUniform, VPValue *Mask = nullptr)
2485+
bool IsUniform, VPValue *Mask = nullptr,
2486+
ArrayRef<std::pair<unsigned, MDNode *>> Metadata = {})
24802487
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2481-
IsUniform(IsUniform), IsPredicated(Mask) {
2488+
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
24822489
if (Mask)
24832490
addOperand(Mask);
24842491
}
24852492

24862493
~VPReplicateRecipe() override = default;
24872494

24882495
VPReplicateRecipe *clone() override {
2489-
auto *Copy =
2490-
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2491-
isPredicated() ? getMask() : nullptr);
2496+
auto *Copy = new VPReplicateRecipe(
2497+
getUnderlyingInstr(), operands(), IsUniform,
2498+
isPredicated() ? getMask() : nullptr, getMetadata());
24922499
Copy->transferFlags(*this);
24932500
return Copy;
24942501
}
@@ -2648,8 +2655,9 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
26482655

26492656
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
26502657
std::initializer_list<VPValue *> Operands,
2651-
bool Consecutive, bool Reverse, DebugLoc DL)
2652-
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(I), Ingredient(I),
2658+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2659+
DebugLoc DL)
2660+
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
26532661
Consecutive(Consecutive), Reverse(Reverse) {
26542662
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
26552663
}
@@ -2707,16 +2715,17 @@ class VPWidenMemoryRecipe : public VPRecipeBase, public VPIRMetadata {
27072715
/// optional mask.
27082716
struct VPWidenLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
27092717
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
2710-
bool Consecutive, bool Reverse, DebugLoc DL)
2718+
bool Consecutive, bool Reverse, MDArrayRef Metadata,
2719+
DebugLoc DL)
27112720
: VPWidenMemoryRecipe(VPDef::VPWidenLoadSC, Load, {Addr}, Consecutive,
2712-
Reverse, DL),
2721+
Reverse, Metadata, DL),
27132722
VPValue(this, &Load) {
27142723
setMask(Mask);
27152724
}
27162725

27172726
VPWidenLoadRecipe *clone() override {
27182727
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
2719-
getMask(), Consecutive, Reverse,
2728+
getMask(), Consecutive, Reverse, getMetadata(),
27202729
getDebugLoc());
27212730
}
27222731

@@ -2748,7 +2757,7 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27482757
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue &EVL, VPValue *Mask)
27492758
: VPWidenMemoryRecipe(VPDef::VPWidenLoadEVLSC, L.getIngredient(),
27502759
{L.getAddr(), &EVL}, L.isConsecutive(),
2751-
L.isReverse(), L.getDebugLoc()),
2760+
L.isReverse(), L.getMetadata(), L.getDebugLoc()),
27522761
VPValue(this, &getIngredient()) {
27532762
setMask(Mask);
27542763
}
@@ -2785,16 +2794,17 @@ struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe, public VPValue {
27852794
/// to store to and an optional mask.
27862795
struct VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
27872796
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
2788-
VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
2797+
VPValue *Mask, bool Consecutive, bool Reverse,
2798+
MDArrayRef Metadata, DebugLoc DL)
27892799
: VPWidenMemoryRecipe(VPDef::VPWidenStoreSC, Store, {Addr, StoredVal},
2790-
Consecutive, Reverse, DL) {
2800+
Consecutive, Reverse, Metadata, DL) {
27912801
setMask(Mask);
27922802
}
27932803

27942804
VPWidenStoreRecipe *clone() override {
27952805
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
27962806
getStoredValue(), getMask(), Consecutive,
2797-
Reverse, getDebugLoc());
2807+
Reverse, getMetadata(), getDebugLoc());
27982808
}
27992809

28002810
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC);
@@ -2828,7 +2838,8 @@ struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
28282838
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue &EVL, VPValue *Mask)
28292839
: VPWidenMemoryRecipe(VPDef::VPWidenStoreEVLSC, S.getIngredient(),
28302840
{S.getAddr(), S.getStoredValue(), &EVL},
2831-
S.isConsecutive(), S.isReverse(), S.getDebugLoc()) {
2841+
S.isConsecutive(), S.isReverse(), S.getMetadata(),
2842+
S.getDebugLoc()) {
28322843
setMask(Mask);
28332844
}
28342845

‎llvm/lib/Transforms/Vectorize/VPlanHelpers.h

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ class VPBasicBlock;
3838
class VPRegionBlock;
3939
class VPlan;
4040
class Value;
41-
class LoopVersioning;
4241

4342
/// Returns a calculation for the total number of elements for a given \p VF.
4443
/// For fixed width vectors this value is a constant, whereas for scalable
@@ -283,13 +282,6 @@ struct VPTransformState {
283282
Iter->second[CacheIdx] = V;
284283
}
285284

286-
/// Add additional metadata to \p To that was not present on \p Orig.
287-
///
288-
/// Currently this is used to add the noalias annotations based on the
289-
/// inserted memchecks. Use this for instructions that are *cloned* into the
290-
/// vector loop.
291-
void addNewMetadata(Instruction *To, const Instruction *Orig);
292-
293285
/// Set the debug location in the builder using the debug location \p DL.
294286
void setDebugLocFrom(DebugLoc DL);
295287

@@ -341,13 +333,6 @@ struct VPTransformState {
341333
/// The parent loop object for the current scope, or nullptr.
342334
Loop *CurrentParentLoop = nullptr;
343335

344-
/// LoopVersioning. It's only set up (non-null) if memchecks were
345-
/// used.
346-
///
347-
/// This is currently only used to add no-alias metadata based on the
348-
/// memchecks. The actually versioning is performed manually.
349-
LoopVersioning *LVer = nullptr;
350-
351336
/// VPlan-based type analysis.
352337
VPTypeAnalysis TypeAnalysis;
353338

0 commit comments

Comments
 (0)