@@ -18212,13 +18212,22 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
18212
18212
if ((VF > MinVF && ActualVF <= VF / 2) || (VF == MinVF && ActualVF < 2))
18213
18213
break;
18214
18214
18215
- ArrayRef<Value *> Ops = VL.slice(I, ActualVF);
18216
- // Check that a previous iteration of this loop did not delete the Value.
18217
- if (llvm::any_of(Ops, [&R](Value *V) {
18218
- auto *I = dyn_cast<Instruction>(V);
18219
- return I && R.isDeleted(I);
18220
- }))
18221
- continue;
18215
+ SmallVector<Value *> Ops(ActualVF, nullptr);
18216
+ unsigned Idx = 0;
18217
+ for (Value *V : VL.drop_front(I)) {
18218
+ // Check that a previous iteration of this loop did not delete the
18219
+ // Value.
18220
+ if (auto *Inst = dyn_cast<Instruction>(V);
18221
+ !Inst || !R.isDeleted(Inst)) {
18222
+ Ops[Idx] = V;
18223
+ ++Idx;
18224
+ if (Idx == ActualVF)
18225
+ break;
18226
+ }
18227
+ }
18228
+ // Not enough vectorizable instructions - exit.
18229
+ if (Idx != ActualVF)
18230
+ break;
18222
18231
18223
18232
LLVM_DEBUG(dbgs() << "SLP: Analyzing " << ActualVF << " operations "
18224
18233
<< "\n");
@@ -18286,7 +18295,8 @@ bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
18286
18295
// Vectorize in current basic block only.
18287
18296
auto *Op0 = dyn_cast<Instruction>(I->getOperand(0));
18288
18297
auto *Op1 = dyn_cast<Instruction>(I->getOperand(1));
18289
- if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P)
18298
+ if (!Op0 || !Op1 || Op0->getParent() != P || Op1->getParent() != P ||
18299
+ R.isDeleted(Op0) || R.isDeleted(Op1))
18290
18300
return false;
18291
18301
18292
18302
// First collect all possible candidates
@@ -18299,18 +18309,18 @@ bool SLPVectorizerPass::tryToVectorize(Instruction *I, BoUpSLP &R) {
18299
18309
if (A && B && B->hasOneUse()) {
18300
18310
auto *B0 = dyn_cast<BinaryOperator>(B->getOperand(0));
18301
18311
auto *B1 = dyn_cast<BinaryOperator>(B->getOperand(1));
18302
- if (B0 && B0->getParent() == P)
18312
+ if (B0 && B0->getParent() == P && !R.isDeleted(B0) )
18303
18313
Candidates.emplace_back(A, B0);
18304
- if (B1 && B1->getParent() == P)
18314
+ if (B1 && B1->getParent() == P && !R.isDeleted(B1) )
18305
18315
Candidates.emplace_back(A, B1);
18306
18316
}
18307
18317
// Try to skip A.
18308
18318
if (B && A && A->hasOneUse()) {
18309
18319
auto *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
18310
18320
auto *A1 = dyn_cast<BinaryOperator>(A->getOperand(1));
18311
- if (A0 && A0->getParent() == P)
18321
+ if (A0 && A0->getParent() == P && !R.isDeleted(A0) )
18312
18322
Candidates.emplace_back(A0, B);
18313
- if (A1 && A1->getParent() == P)
18323
+ if (A1 && A1->getParent() == P && !R.isDeleted(A1) )
18314
18324
Candidates.emplace_back(A1, B);
18315
18325
}
18316
18326
@@ -19769,16 +19779,16 @@ static void findBuildAggregate_rec(Instruction *LastInsertInst,
19769
19779
TargetTransformInfo *TTI,
19770
19780
SmallVectorImpl<Value *> &BuildVectorOpds,
19771
19781
SmallVectorImpl<Value *> &InsertElts,
19772
- unsigned OperandOffset) {
19782
+ unsigned OperandOffset, const BoUpSLP &R ) {
19773
19783
do {
19774
19784
Value *InsertedOperand = LastInsertInst->getOperand(1);
19775
19785
std::optional<unsigned> OperandIndex =
19776
19786
getElementIndex(LastInsertInst, OperandOffset);
19777
- if (!OperandIndex)
19787
+ if (!OperandIndex || R.isDeleted(LastInsertInst) )
19778
19788
return;
19779
19789
if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {
19780
19790
findBuildAggregate_rec(cast<Instruction>(InsertedOperand), TTI,
19781
- BuildVectorOpds, InsertElts, *OperandIndex);
19791
+ BuildVectorOpds, InsertElts, *OperandIndex, R );
19782
19792
19783
19793
} else {
19784
19794
BuildVectorOpds[*OperandIndex] = InsertedOperand;
@@ -19807,7 +19817,8 @@ static void findBuildAggregate_rec(Instruction *LastInsertInst,
19807
19817
static bool findBuildAggregate(Instruction *LastInsertInst,
19808
19818
TargetTransformInfo *TTI,
19809
19819
SmallVectorImpl<Value *> &BuildVectorOpds,
19810
- SmallVectorImpl<Value *> &InsertElts) {
19820
+ SmallVectorImpl<Value *> &InsertElts,
19821
+ const BoUpSLP &R) {
19811
19822
19812
19823
assert((isa<InsertElementInst>(LastInsertInst) ||
19813
19824
isa<InsertValueInst>(LastInsertInst)) &&
@@ -19822,7 +19833,8 @@ static bool findBuildAggregate(Instruction *LastInsertInst,
19822
19833
BuildVectorOpds.resize(*AggregateSize);
19823
19834
InsertElts.resize(*AggregateSize);
19824
19835
19825
- findBuildAggregate_rec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0);
19836
+ findBuildAggregate_rec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0,
19837
+ R);
19826
19838
llvm::erase(BuildVectorOpds, nullptr);
19827
19839
llvm::erase(InsertElts, nullptr);
19828
19840
if (BuildVectorOpds.size() >= 2)
@@ -20068,7 +20080,7 @@ bool SLPVectorizerPass::vectorizeInsertValueInst(InsertValueInst *IVI,
20068
20080
20069
20081
SmallVector<Value *, 16> BuildVectorOpds;
20070
20082
SmallVector<Value *, 16> BuildVectorInsts;
20071
- if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, BuildVectorInsts))
20083
+ if (!findBuildAggregate(IVI, TTI, BuildVectorOpds, BuildVectorInsts, R ))
20072
20084
return false;
20073
20085
20074
20086
if (MaxVFOnly && BuildVectorOpds.size() == 2) {
@@ -20090,7 +20102,7 @@ bool SLPVectorizerPass::vectorizeInsertElementInst(InsertElementInst *IEI,
20090
20102
SmallVector<Value *, 16> BuildVectorInsts;
20091
20103
SmallVector<Value *, 16> BuildVectorOpds;
20092
20104
SmallVector<int> Mask;
20093
- if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts) ||
20105
+ if (!findBuildAggregate(IEI, TTI, BuildVectorOpds, BuildVectorInsts, R ) ||
20094
20106
(llvm::all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
20095
20107
isFixedVectorShuffle(BuildVectorOpds, Mask)))
20096
20108
return false;
0 commit comments