[SLP]Initial support for interleaved loads #112042
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Created using spr 1.3.5
llvmbot
added
backend:RISC-V
vectorizers
llvm:analysis
|
@llvm/pr-subscribers-vectorizers @llvm/pr-subscribers-llvm-analysis Author: Alexey Bataev (alexey-bataev) ChangesAdds initial support for interleaved loads, which allows Vectorizes extra code for RISCV Patch is 80.76 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/112042.diff 7 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 64dc9aacd5c57b..0459941fe05cdc 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -803,6 +803,12 @@ class TargetTransformInfo {
/// Return true if the target supports strided load.
bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
+ /// Return true is the target supports interleaved access for the given vector
+ /// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
+ /// address space \p AddrSpace.
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) const;
+
// Return true if the target supports masked vector histograms.
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const;
@@ -1934,6 +1940,10 @@ class TargetTransformInfo::Concept {
virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0;
+ virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment,
+ unsigned AddrSpace) = 0;
+
virtual bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) = 0;
virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
@@ -2456,6 +2466,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override {
return Impl.isLegalStridedLoadStore(DataType, Alignment);
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment,
+ unsigned AddrSpace) override {
+ return Impl.isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace);
+ }
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) override {
return Impl.isLegalMaskedVectorHistogram(AddrType, DataType);
}
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1c4fcb57755ecf..dbdfb4d8cdfa32 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -332,6 +332,11 @@ class TargetTransformInfoImplBase {
return false;
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) {
+ return false;
+ }
+
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 8ab8a53b753112..1bad3314677038 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -521,6 +521,13 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+bool TargetTransformInfo::isLegalInterleavedAccessType(
+ VectorType *VTy, unsigned Factor, Align Alignment,
+ unsigned AddrSpace) const {
+ return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,
+ AddrSpace);
+}
+
bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
Type *DataType) const {
return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 3f50bd86b9b3b6..13d28e4db49cd9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -295,6 +295,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) {
+ return TLI->isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace,
+ DL);
+ }
+
bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment);
bool isVScaleKnownToBeAPowerOfTwo() const {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5c164075e83259..1e8939988037d6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2922,7 +2922,7 @@ class BoUpSLP {
/// This is the recursive part of buildTree.
void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
- const EdgeInfo &EI);
+ const EdgeInfo &EI, unsigned InterleaveFactor = 0);
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
/// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -3226,7 +3226,15 @@ class BoUpSLP {
Instruction *MainOp = nullptr;
Instruction *AltOp = nullptr;
+ /// Interleaving factor for interleaved loads Vectorize nodes.
+ unsigned InterleaveFactor = 0;
+
public:
+ /// Returns interleave factor for interleave nodes.
+ unsigned getInterleaveFactor() const { return InterleaveFactor; }
+ /// Sets interleaving factor for the interleaving nodes.
+ void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
+
/// Set this bundle's \p OpIdx'th operand to \p OpVL.
void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
if (Operands.size() < OpIdx + 1)
@@ -3390,7 +3398,12 @@ class BoUpSLP {
dbgs() << "State: ";
switch (State) {
case Vectorize:
- dbgs() << "Vectorize\n";
+ if (InterleaveFactor > 0) {
+ dbgs() << "Vectorize with interleave factor " << InterleaveFactor
+ << "\n";
+ } else {
+ dbgs() << "Vectorize\n";
+ }
break;
case ScatterVectorize:
dbgs() << "ScatterVectorize\n";
@@ -3460,11 +3473,15 @@ class BoUpSLP {
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
- ArrayRef<unsigned> ReorderIndices = {}) {
+ ArrayRef<unsigned> ReorderIndices = {},
+ unsigned InterleaveFactor = 0) {
TreeEntry::EntryState EntryState =
Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
- return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
- ReuseShuffleIndices, ReorderIndices);
+ TreeEntry *E = newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
+ ReuseShuffleIndices, ReorderIndices);
+ if (E && InterleaveFactor > 0)
+ E->setInterleave(InterleaveFactor);
+ return E;
}
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
@@ -6932,11 +6949,16 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
// distance between scalar loads in these nodes.
unsigned MaxVF = Slice.size();
unsigned UserMaxVF = 0;
+ unsigned InterleaveFactor = 0;
if (MaxVF == 2) {
UserMaxVF = MaxVF;
} else {
+ // Found distance between segments of the interleaved loads.
+ std::optional<unsigned> InterleavedLoadsDistance = 0;
+ unsigned Order = 0;
std::optional<unsigned> CommonVF = 0;
DenseMap<const TreeEntry *, unsigned> EntryToPosition;
+ SmallPtrSet<const TreeEntry *, 8> DeinterleavedNodes;
for (auto [Idx, V] : enumerate(Slice)) {
for (const TreeEntry *E : ValueToGatherNodes.at(V)) {
UserMaxVF = std::max<unsigned>(UserMaxVF, E->Scalars.size());
@@ -6951,12 +6973,60 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
if (*CommonVF != E->Scalars.size())
CommonVF.reset();
}
+ // Check if the load is the part of the interleaved load.
+ if (Pos != Idx && InterleavedLoadsDistance) {
+ if (!DeinterleavedNodes.contains(E) &&
+ any_of(E->Scalars, [&, Slice = Slice](Value *V) {
+ if (isa<Constant>(V))
+ return false;
+ if (getTreeEntry(V))
+ return true;
+ const auto &Nodes = ValueToGatherNodes.at(V);
+ return (Nodes.size() != 1 || !Nodes.contains(E)) &&
+ !is_contained(Slice, V);
+ })) {
+ InterleavedLoadsDistance.reset();
+ continue;
+ }
+ DeinterleavedNodes.insert(E);
+ if (*InterleavedLoadsDistance == 0) {
+ InterleavedLoadsDistance = Idx - Pos;
+ continue;
+ }
+ if ((Idx - Pos) % *InterleavedLoadsDistance != 0 ||
+ (Idx - Pos) / *InterleavedLoadsDistance < Order)
+ InterleavedLoadsDistance.reset();
+ Order = (Idx - Pos) / InterleavedLoadsDistance.value_or(1);
+ }
+ }
+ }
+ DeinterleavedNodes.clear();
+ // Check if the large load represents interleaved load operation.
+ if (InterleavedLoadsDistance.value_or(0) > 1 &&
+ CommonVF.value_or(0) != 0) {
+ InterleaveFactor = bit_ceil(*InterleavedLoadsDistance);
+ unsigned VF = *CommonVF;
+ OrdersType Order;
+ SmallVector<Value *> PointerOps;
+ // Segmented load detected - vectorize at maximum vector factor.
+ if (TTI->isLegalInterleavedAccessType(
+ getWidenedType(Slice.front()->getType(), VF),
+ InterleaveFactor,
+ cast<LoadInst>(Slice.front())->getAlign(),
+ cast<LoadInst>(Slice.front())
+ ->getPointerAddressSpace()) &&
+ canVectorizeLoads(Slice, Slice.front(), Order,
+ PointerOps) == LoadsState::Vectorize) {
+ UserMaxVF = InterleaveFactor * VF;
+ } else {
+ InterleaveFactor = 0;
}
}
// Cannot represent the loads as consecutive vectorizable nodes -
// just exit.
unsigned ConsecutiveNodesSize = 0;
if (!LoadEntriesToVectorize.empty() &&
+ InterleaveFactor == 0 &&
any_of(zip(LoadEntriesToVectorize, LoadSetsToVectorize),
[&, Slice = Slice](const auto &P) {
const auto *It = find_if(Slice, [&](Value *V) {
@@ -6976,7 +7046,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
continue;
// Try to build long masked gather loads.
UserMaxVF = bit_ceil(UserMaxVF);
- if (any_of(seq<unsigned>(Slice.size() / UserMaxVF),
+ if (InterleaveFactor == 0 &&
+ any_of(seq<unsigned>(Slice.size() / UserMaxVF),
[&, Slice = Slice](unsigned Idx) {
OrdersType Order;
SmallVector<Value *> PointerOps;
@@ -7008,9 +7079,15 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
}))
continue;
unsigned Sz = VectorizableTree.size();
- buildTree_rec(SubSlice, 0, EdgeInfo());
+ buildTree_rec(SubSlice, 0, EdgeInfo(), InterleaveFactor);
if (Sz == VectorizableTree.size()) {
IsVectorized = false;
+ // Try non-interleaved vectorization with smaller vector
+ // factor.
+ if (InterleaveFactor > 0) {
+ VF = 2 * (MaxVF / InterleaveFactor);
+ InterleaveFactor = 0;
+ }
continue;
}
}
@@ -7374,6 +7451,11 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
}
return TreeEntry::ScatterVectorize;
case LoadsState::StridedVectorize:
+ if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
+ // Delay slow vectorized nodes for better vectorization attempts.
+ LoadEntriesToVectorize.insert(VectorizableTree.size());
+ return TreeEntry::NeedToGather;
+ }
return TreeEntry::StridedVectorize;
case LoadsState::Gather:
#ifndef NDEBUG
@@ -7707,7 +7789,8 @@ class PHIHandler {
} // namespace
void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx) {
+ const EdgeInfo &UserTreeIdx,
+ unsigned InterleaveFactor) {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
@@ -8185,7 +8268,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
switch (State) {
case TreeEntry::Vectorize:
TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices, CurrentOrder);
+ ReuseShuffleIndices, CurrentOrder, InterleaveFactor);
if (CurrentOrder.empty())
LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
else
@@ -9895,6 +9978,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Idx = EMask[Idx];
}
CommonVF = E->Scalars.size();
+ } else if (std::optional<unsigned> Factor = E->getInterleaveFactor();
+ Factor && E->Scalars.size() != Mask.size() &&
+ ShuffleVectorInst::isDeInterleaveMaskOfFactor(CommonMask,
+ *Factor)) {
+ // Deinterleaved nodes are free.
+ std::iota(CommonMask.begin(), CommonMask.end(), 0);
}
ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF);
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
@@ -10968,10 +11057,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto *LI0 = cast<LoadInst>(VL0);
auto GetVectorCost = [&](InstructionCost CommonCost) {
InstructionCost VecLdCost;
- if (E->State == TreeEntry::Vectorize) {
+ if (E->State == TreeEntry::Vectorize && !E->getInterleaveFactor()) {
VecLdCost = TTI->getMemoryOpCost(
Instruction::Load, VecTy, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
+ } else if (std::optional<unsigned> Factor = E->getInterleaveFactor();
+ E->State == TreeEntry::Vectorize && Factor.value_or(0) > 0) {
+ VecLdCost = TTI->getInterleavedMemoryOpCost(
+ Instruction::Load, VecTy, *Factor, std::nullopt, LI0->getAlign(),
+ LI0->getPointerAddressSpace(), CostKind);
} else if (E->State == TreeEntry::StridedVectorize) {
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
@@ -11397,6 +11491,11 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
}))
return false;
+ if (VectorizableTree.back()->isGather() &&
+ VectorizableTree.back()->isAltShuffle() &&
+ VectorizableTree.back()->getVectorFactor() > 2)
+ return false;
+
assert(VectorizableTree.empty()
? ExternalUses.empty()
: true && "We shouldn't have any external users");
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 443f17a9c09e7a..8c6b92b65ae050 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -11,9 +11,6 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; CHECK-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; CHECK-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
; CHECK-NEXT: [[CONV33:%.*]] = zext i8 [[TMP10]] to i32
@@ -24,59 +21,42 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; CHECK-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
; CHECK-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; CHECK-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
-; CHECK-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
-; CHECK-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
+; CHECK-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i8> poison, i8 [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i8> [[TMP6]], i8 [[TMP5]], i32 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i8> [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
-; CHECK-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
-; CHECK-NEXT: [[TMP18:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX32_2]], i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT: [[TMP20:%.*]] = zext <2 x i8> [[TMP18]] to <2 x i16>
-; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX34_1]], i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT: [[TMP13:%.*]] = zext <2 x i8> [[TMP12]] to <2 x i16>
-; CHECK-NEXT: [[TMP28:%.*]] = sub <2 x i16> [[TMP20]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[TMP28]], i32 1
-; CHECK-NEXT: [[TMP16:%.*]] = sext i16 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[TMP28]], i32 0
-; CHECK-NEXT: [[CONV33_1:%.*]] = sext i16 [[TMP17]] to i32
+; CHECK-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP5]] to i32
; CHECK-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; CHECK-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; CHECK-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
; CHECK-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
-; CHECK-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
-; C...
[truncated]
|
|
@llvm/pr-subscribers-llvm-transforms Author: Alexey Bataev (alexey-bataev) ChangesAdds initial support for interleaved loads, which allows Vectorizes extra code for RISCV Patch is 80.76 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/112042.diff 7 Files Affected:
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 64dc9aacd5c57b..0459941fe05cdc 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -803,6 +803,12 @@ class TargetTransformInfo {
/// Return true if the target supports strided load.
bool isLegalStridedLoadStore(Type *DataType, Align Alignment) const;
+ /// Return true is the target supports interleaved access for the given vector
+ /// type \p VTy, interleave factor \p Factor, alignment \p Alignment and
+ /// address space \p AddrSpace.
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) const;
+
// Return true if the target supports masked vector histograms.
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const;
@@ -1934,6 +1940,10 @@ class TargetTransformInfo::Concept {
virtual bool isLegalMaskedCompressStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedExpandLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalStridedLoadStore(Type *DataType, Align Alignment) = 0;
+ virtual bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment,
+ unsigned AddrSpace) = 0;
+
virtual bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) = 0;
virtual bool isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
unsigned Opcode1,
@@ -2456,6 +2466,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
bool isLegalStridedLoadStore(Type *DataType, Align Alignment) override {
return Impl.isLegalStridedLoadStore(DataType, Alignment);
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment,
+ unsigned AddrSpace) override {
+ return Impl.isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace);
+ }
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) override {
return Impl.isLegalMaskedVectorHistogram(AddrType, DataType);
}
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 1c4fcb57755ecf..dbdfb4d8cdfa32 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -332,6 +332,11 @@ class TargetTransformInfoImplBase {
return false;
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) {
+ return false;
+ }
+
bool isLegalMaskedVectorHistogram(Type *AddrType, Type *DataType) const {
return false;
}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 8ab8a53b753112..1bad3314677038 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -521,6 +521,13 @@ bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
}
+bool TargetTransformInfo::isLegalInterleavedAccessType(
+ VectorType *VTy, unsigned Factor, Align Alignment,
+ unsigned AddrSpace) const {
+ return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,
+ AddrSpace);
+}
+
bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
Type *DataType) const {
return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 3f50bd86b9b3b6..13d28e4db49cd9 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -295,6 +295,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
return TLI->isLegalStridedLoadStore(DataTypeVT, Alignment);
}
+ bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor,
+ Align Alignment, unsigned AddrSpace) {
+ return TLI->isLegalInterleavedAccessType(VTy, Factor, Alignment, AddrSpace,
+ DL);
+ }
+
bool isLegalMaskedCompressStore(Type *DataTy, Align Alignment);
bool isVScaleKnownToBeAPowerOfTwo() const {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5c164075e83259..1e8939988037d6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2922,7 +2922,7 @@ class BoUpSLP {
/// This is the recursive part of buildTree.
void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
- const EdgeInfo &EI);
+ const EdgeInfo &EI, unsigned InterleaveFactor = 0);
/// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
/// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -3226,7 +3226,15 @@ class BoUpSLP {
Instruction *MainOp = nullptr;
Instruction *AltOp = nullptr;
+ /// Interleaving factor for interleaved loads Vectorize nodes.
+ unsigned InterleaveFactor = 0;
+
public:
+ /// Returns interleave factor for interleave nodes.
+ unsigned getInterleaveFactor() const { return InterleaveFactor; }
+ /// Sets interleaving factor for the interleaving nodes.
+ void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
+
/// Set this bundle's \p OpIdx'th operand to \p OpVL.
void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
if (Operands.size() < OpIdx + 1)
@@ -3390,7 +3398,12 @@ class BoUpSLP {
dbgs() << "State: ";
switch (State) {
case Vectorize:
- dbgs() << "Vectorize\n";
+ if (InterleaveFactor > 0) {
+ dbgs() << "Vectorize with interleave factor " << InterleaveFactor
+ << "\n";
+ } else {
+ dbgs() << "Vectorize\n";
+ }
break;
case ScatterVectorize:
dbgs() << "ScatterVectorize\n";
@@ -3460,11 +3473,15 @@ class BoUpSLP {
const InstructionsState &S,
const EdgeInfo &UserTreeIdx,
ArrayRef<int> ReuseShuffleIndices = {},
- ArrayRef<unsigned> ReorderIndices = {}) {
+ ArrayRef<unsigned> ReorderIndices = {},
+ unsigned InterleaveFactor = 0) {
TreeEntry::EntryState EntryState =
Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
- return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
- ReuseShuffleIndices, ReorderIndices);
+ TreeEntry *E = newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
+ ReuseShuffleIndices, ReorderIndices);
+ if (E && InterleaveFactor > 0)
+ E->setInterleave(InterleaveFactor);
+ return E;
}
TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
@@ -6932,11 +6949,16 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
// distance between scalar loads in these nodes.
unsigned MaxVF = Slice.size();
unsigned UserMaxVF = 0;
+ unsigned InterleaveFactor = 0;
if (MaxVF == 2) {
UserMaxVF = MaxVF;
} else {
+ // Found distance between segments of the interleaved loads.
+ std::optional<unsigned> InterleavedLoadsDistance = 0;
+ unsigned Order = 0;
std::optional<unsigned> CommonVF = 0;
DenseMap<const TreeEntry *, unsigned> EntryToPosition;
+ SmallPtrSet<const TreeEntry *, 8> DeinterleavedNodes;
for (auto [Idx, V] : enumerate(Slice)) {
for (const TreeEntry *E : ValueToGatherNodes.at(V)) {
UserMaxVF = std::max<unsigned>(UserMaxVF, E->Scalars.size());
@@ -6951,12 +6973,60 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
if (*CommonVF != E->Scalars.size())
CommonVF.reset();
}
+ // Check if the load is the part of the interleaved load.
+ if (Pos != Idx && InterleavedLoadsDistance) {
+ if (!DeinterleavedNodes.contains(E) &&
+ any_of(E->Scalars, [&, Slice = Slice](Value *V) {
+ if (isa<Constant>(V))
+ return false;
+ if (getTreeEntry(V))
+ return true;
+ const auto &Nodes = ValueToGatherNodes.at(V);
+ return (Nodes.size() != 1 || !Nodes.contains(E)) &&
+ !is_contained(Slice, V);
+ })) {
+ InterleavedLoadsDistance.reset();
+ continue;
+ }
+ DeinterleavedNodes.insert(E);
+ if (*InterleavedLoadsDistance == 0) {
+ InterleavedLoadsDistance = Idx - Pos;
+ continue;
+ }
+ if ((Idx - Pos) % *InterleavedLoadsDistance != 0 ||
+ (Idx - Pos) / *InterleavedLoadsDistance < Order)
+ InterleavedLoadsDistance.reset();
+ Order = (Idx - Pos) / InterleavedLoadsDistance.value_or(1);
+ }
+ }
+ }
+ DeinterleavedNodes.clear();
+ // Check if the large load represents interleaved load operation.
+ if (InterleavedLoadsDistance.value_or(0) > 1 &&
+ CommonVF.value_or(0) != 0) {
+ InterleaveFactor = bit_ceil(*InterleavedLoadsDistance);
+ unsigned VF = *CommonVF;
+ OrdersType Order;
+ SmallVector<Value *> PointerOps;
+ // Segmented load detected - vectorize at maximum vector factor.
+ if (TTI->isLegalInterleavedAccessType(
+ getWidenedType(Slice.front()->getType(), VF),
+ InterleaveFactor,
+ cast<LoadInst>(Slice.front())->getAlign(),
+ cast<LoadInst>(Slice.front())
+ ->getPointerAddressSpace()) &&
+ canVectorizeLoads(Slice, Slice.front(), Order,
+ PointerOps) == LoadsState::Vectorize) {
+ UserMaxVF = InterleaveFactor * VF;
+ } else {
+ InterleaveFactor = 0;
}
}
// Cannot represent the loads as consecutive vectorizable nodes -
// just exit.
unsigned ConsecutiveNodesSize = 0;
if (!LoadEntriesToVectorize.empty() &&
+ InterleaveFactor == 0 &&
any_of(zip(LoadEntriesToVectorize, LoadSetsToVectorize),
[&, Slice = Slice](const auto &P) {
const auto *It = find_if(Slice, [&](Value *V) {
@@ -6976,7 +7046,8 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
continue;
// Try to build long masked gather loads.
UserMaxVF = bit_ceil(UserMaxVF);
- if (any_of(seq<unsigned>(Slice.size() / UserMaxVF),
+ if (InterleaveFactor == 0 &&
+ any_of(seq<unsigned>(Slice.size() / UserMaxVF),
[&, Slice = Slice](unsigned Idx) {
OrdersType Order;
SmallVector<Value *> PointerOps;
@@ -7008,9 +7079,15 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
}))
continue;
unsigned Sz = VectorizableTree.size();
- buildTree_rec(SubSlice, 0, EdgeInfo());
+ buildTree_rec(SubSlice, 0, EdgeInfo(), InterleaveFactor);
if (Sz == VectorizableTree.size()) {
IsVectorized = false;
+ // Try non-interleaved vectorization with smaller vector
+ // factor.
+ if (InterleaveFactor > 0) {
+ VF = 2 * (MaxVF / InterleaveFactor);
+ InterleaveFactor = 0;
+ }
continue;
}
}
@@ -7374,6 +7451,11 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
}
return TreeEntry::ScatterVectorize;
case LoadsState::StridedVectorize:
+ if (!IsGraphTransformMode && VectorizableTree.size() > 1) {
+ // Delay slow vectorized nodes for better vectorization attempts.
+ LoadEntriesToVectorize.insert(VectorizableTree.size());
+ return TreeEntry::NeedToGather;
+ }
return TreeEntry::StridedVectorize;
case LoadsState::Gather:
#ifndef NDEBUG
@@ -7707,7 +7789,8 @@ class PHIHandler {
} // namespace
void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
- const EdgeInfo &UserTreeIdx) {
+ const EdgeInfo &UserTreeIdx,
+ unsigned InterleaveFactor) {
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
SmallVector<int> ReuseShuffleIndices;
@@ -8185,7 +8268,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
switch (State) {
case TreeEntry::Vectorize:
TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
- ReuseShuffleIndices, CurrentOrder);
+ ReuseShuffleIndices, CurrentOrder, InterleaveFactor);
if (CurrentOrder.empty())
LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
else
@@ -9895,6 +9978,12 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
Idx = EMask[Idx];
}
CommonVF = E->Scalars.size();
+ } else if (std::optional<unsigned> Factor = E->getInterleaveFactor();
+ Factor && E->Scalars.size() != Mask.size() &&
+ ShuffleVectorInst::isDeInterleaveMaskOfFactor(CommonMask,
+ *Factor)) {
+ // Deinterleaved nodes are free.
+ std::iota(CommonMask.begin(), CommonMask.end(), 0);
}
ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF);
V1 = Constant::getNullValue(getWidenedType(ScalarTy, CommonVF));
@@ -10968,10 +11057,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
auto *LI0 = cast<LoadInst>(VL0);
auto GetVectorCost = [&](InstructionCost CommonCost) {
InstructionCost VecLdCost;
- if (E->State == TreeEntry::Vectorize) {
+ if (E->State == TreeEntry::Vectorize && !E->getInterleaveFactor()) {
VecLdCost = TTI->getMemoryOpCost(
Instruction::Load, VecTy, LI0->getAlign(),
LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
+ } else if (std::optional<unsigned> Factor = E->getInterleaveFactor();
+ E->State == TreeEntry::Vectorize && Factor.value_or(0) > 0) {
+ VecLdCost = TTI->getInterleavedMemoryOpCost(
+ Instruction::Load, VecTy, *Factor, std::nullopt, LI0->getAlign(),
+ LI0->getPointerAddressSpace(), CostKind);
} else if (E->State == TreeEntry::StridedVectorize) {
Align CommonAlignment =
computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
@@ -11397,6 +11491,11 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
}))
return false;
+ if (VectorizableTree.back()->isGather() &&
+ VectorizableTree.back()->isAltShuffle() &&
+ VectorizableTree.back()->getVectorFactor() > 2)
+ return false;
+
assert(VectorizableTree.empty()
? ExternalUses.empty()
: true && "We shouldn't have any external users");
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
index 443f17a9c09e7a..8c6b92b65ae050 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/complex-loads.ll
@@ -11,9 +11,6 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[PIX1]], i64 4
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[PIX2]], i64 4
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr i8, ptr [[PIX1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX22:%.*]] = getelementptr i8, ptr [[PIX2]], i64 1
-; CHECK-NEXT: [[ARRAYIDX25:%.*]] = getelementptr i8, ptr [[PIX1]], i64 5
-; CHECK-NEXT: [[ARRAYIDX27:%.*]] = getelementptr i8, ptr [[PIX2]], i64 5
; CHECK-NEXT: [[ARRAYIDX32:%.*]] = getelementptr i8, ptr [[PIX1]], i64 3
; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX32]], align 1
; CHECK-NEXT: [[CONV33:%.*]] = zext i8 [[TMP10]] to i32
@@ -24,59 +21,42 @@ define i32 @test(ptr %pix1, ptr %pix2, i64 %idx.ext, i64 %idx.ext63, ptr %add.pt
; CHECK-NEXT: [[ARRAYIDX3_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 4
; CHECK-NEXT: [[ARRAYIDX5_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 4
; CHECK-NEXT: [[ARRAYIDX8_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 1
-; CHECK-NEXT: [[ARRAYIDX32_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX32_1]], align 1
-; CHECK-NEXT: [[ARRAYIDX25_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 6
-; CHECK-NEXT: [[TMP4:%.*]] = load i8, ptr [[ARRAYIDX25_1]], align 1
-; CHECK-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 6
+; CHECK-NEXT: [[ARRAYIDX27_1:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
; CHECK-NEXT: [[TMP5:%.*]] = load i8, ptr [[ARRAYIDX27_1]], align 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i8> poison, i8 [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i8> [[TMP6]], i8 [[TMP5]], i32 1
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i8> [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = zext i8 [[TMP8]] to i32
-; CHECK-NEXT: [[ARRAYIDX32_2:%.*]] = getelementptr i8, ptr [[ADD_PTR3]], i64 3
-; CHECK-NEXT: [[ARRAYIDX34_1:%.*]] = getelementptr i8, ptr [[ADD_PTR644]], i64 3
-; CHECK-NEXT: [[TMP18:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX32_2]], i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT: [[TMP20:%.*]] = zext <2 x i8> [[TMP18]] to <2 x i16>
-; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i8> @llvm.experimental.vp.strided.load.v2i8.p0.i64(ptr align 1 [[ARRAYIDX34_1]], i64 4, <2 x i1> <i1 true, i1 true>, i32 2)
-; CHECK-NEXT: [[TMP13:%.*]] = zext <2 x i8> [[TMP12]] to <2 x i16>
-; CHECK-NEXT: [[TMP28:%.*]] = sub <2 x i16> [[TMP20]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x i16> [[TMP28]], i32 1
-; CHECK-NEXT: [[TMP16:%.*]] = sext i16 [[TMP15]] to i32
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i16> [[TMP28]], i32 0
-; CHECK-NEXT: [[CONV33_1:%.*]] = sext i16 [[TMP17]] to i32
+; CHECK-NEXT: [[CONV33_1:%.*]] = zext i8 [[TMP5]] to i32
; CHECK-NEXT: [[ADD_PTR_1:%.*]] = getelementptr i8, ptr [[ADD_PTR]], i64 [[IDX_EXT]]
; CHECK-NEXT: [[ADD_PTR64_1:%.*]] = getelementptr i8, ptr [[ADD_PTR64]], i64 [[IDX_EXT63]]
; CHECK-NEXT: [[ARRAYIDX3_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 4
; CHECK-NEXT: [[ARRAYIDX5_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 4
; CHECK-NEXT: [[ARRAYIDX8_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX10_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 1
-; CHECK-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr i8, ptr [[ADD_PTR_1]], i64 5
-; CHECK-NEXT: [[ARRAYIDX15_2:%.*]] = getelementptr i8, ptr [[ADD_PTR64_1]], i64 5
-; C...
[truncated]
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
Created using spr 1.3.5
Created using spr 1.3.5
preames
approved these changes
Oct 11, 2024
| VecLdCost = TTI->getMemoryOpCost( | ||
| Instruction::Load, VecTy, LI0->getAlign(), | ||
| LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo()); | ||
| } else if (std::optional<unsigned> Factor = E->getInterleaveFactor(); |
There was a problem hiding this comment.
I think this would be easy to read as:
if (E->State == TreeEntry::Vectorize) {
if (std::optional Factor = E->getInterleaveFactor()) { do interleave cost }
else { do normal costing; }
}
Created using spr 1.3.5
alexey-bataev
deleted the
users/alexey-bataev/spr/slpinitial-support-for-interleaved-loads
branch
DanielCChen
pushed a commit
to DanielCChen/llvm-project
that referenced
this pull request
Oct 16, 2024
Adds initial support for interleaved loads, which allows emission of segmented loads for RISCV RVV. Vectorizes extra code for RISCV CFP2006/447.dealII, CFP2006/453.povray, CFP2017rate/510.parest_r, CFP2017rate/511.povray_r, CFP2017rate/526.blender_r, CFP2017rate/538.imagick_r, CINT2006/403.gcc, CINT2006/473.astar, CINT2017rate/502.gcc_r, CINT2017rate/525.x264_r Reviewers: RKSimon, preames Reviewed By: preames Pull Request: llvm#112042
RonDahan101
reviewed
Nov 6, 2024
Comment on lines
+11504
to
11511
| if (VectorizableTree.back()->isGather() && | ||
| VectorizableTree.back()->isAltShuffle() && | ||
| VectorizableTree.back()->getVectorFactor() > 2) | ||
| return false; | ||
|
|
||
| assert(VectorizableTree.empty() | ||
| ? ExternalUses.empty() | ||
| : true && "We shouldn't have any external users"); |
There was a problem hiding this comment.
@alexey-bataev @preames
Sorry for the post-review
something here seems unsafe to me.
in line 11509 (inside the assert) we recognize the VectorizableTree could be empty. but in the 'if' statement before it, you do VectorizableTree.back() which in case VectorizableTree is empty will crash.
There was a problem hiding this comment.
Will add the extra check
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Adds initial support for interleaved loads, which allows
emission of segmented loads for RISCV RVV.
Vectorizes extra code for RISCV
CFP2006/447.dealII, CFP2006/453.povray,
CFP2017rate/510.parest_r, CFP2017rate/511.povray_r,
CFP2017rate/526.blender_r, CFP2017rate/538.imagick_r, CINT2006/403.gcc,
CINT2006/473.astar, CINT2017rate/502.gcc_r, CINT2017rate/525.x264_r