@@ -12380,7 +12380,7 @@ bool SLPVectorizerPass::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
12380
12380
}
12381
12381
12382
12382
bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
12383
- bool LimitForRegisterSize ) {
12383
+ bool MaxVFOnly ) {
12384
12384
if (VL.size() < 2)
12385
12385
return false;
12386
12386
@@ -12442,29 +12442,25 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
12442
12442
if (TTI->getNumberOfParts(VecTy) == VF)
12443
12443
continue;
12444
12444
for (unsigned I = NextInst; I < MaxInst; ++I) {
12445
- unsigned OpsWidth = 0 ;
12445
+ unsigned ActualVF = std::min(MaxInst - I, VF) ;
12446
12446
12447
- if (I + VF > MaxInst)
12448
- OpsWidth = MaxInst - I;
12449
- else
12450
- OpsWidth = VF;
12451
-
12452
- if (!isPowerOf2_32(OpsWidth))
12447
+ if (!isPowerOf2_32(ActualVF))
12453
12448
continue;
12454
12449
12455
- if ((LimitForRegisterSize && OpsWidth < MaxVF) ||
12456
- (VF > MinVF && OpsWidth <= VF / 2) || (VF == MinVF && OpsWidth < 2))
12450
+ if (MaxVFOnly && ActualVF < MaxVF)
12451
+ break;
12452
+ if ((VF > MinVF && ActualVF <= VF / 2) || (VF == MinVF && ActualVF < 2))
12457
12453
break;
12458
12454
12459
- ArrayRef<Value *> Ops = VL.slice(I, OpsWidth );
12455
+ ArrayRef<Value *> Ops = VL.slice(I, ActualVF );
12460
12456
// Check that a previous iteration of this loop did not delete the Value.
12461
12457
if (llvm::any_of(Ops, [&R](Value *V) {
12462
12458
auto *I = dyn_cast<Instruction>(V);
12463
12459
return I && R.isDeleted(I);
12464
12460
}))
12465
12461
continue;
12466
12462
12467
- LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
12463
+ LLVM_DEBUG(dbgs() << "SLP: Analyzing " << ActualVF << " operations "
12468
12464
<< "\n");
12469
12465
12470
12466
R.buildTree(Ops);
@@ -12482,7 +12478,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
12482
12478
MinCost = std::min(MinCost, Cost);
12483
12479
12484
12480
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
12485
- << " for VF=" << OpsWidth << "\n");
12481
+ << " for VF=" << ActualVF << "\n");
12486
12482
if (Cost < -SLPCostThreshold) {
12487
12483
LLVM_DEBUG(dbgs() << "SLP: Vectorizing list at cost:" << Cost << ".\n");
12488
12484
R.getORE()->emit(OptimizationRemark(SV_NAME, "VectorizedList",
@@ -14277,7 +14273,7 @@ static bool tryToVectorizeSequence(
14277
14273
SmallVectorImpl<T *> &Incoming, function_ref<bool(T *, T *)> Comparator,
14278
14274
function_ref<bool(T *, T *)> AreCompatible,
14279
14275
function_ref<bool(ArrayRef<T *>, bool)> TryToVectorizeHelper,
14280
- bool LimitForRegisterSize , BoUpSLP &R) {
14276
+ bool MaxVFOnly , BoUpSLP &R) {
14281
14277
bool Changed = false;
14282
14278
// Sort by type, parent, operands.
14283
14279
stable_sort(Incoming, Comparator);
@@ -14305,7 +14301,7 @@ static bool tryToVectorizeSequence(
14305
14301
// same/alternate ops only, this may result in some extra final
14306
14302
// vectorization.
14307
14303
if (NumElts > 1 &&
14308
- TryToVectorizeHelper(ArrayRef(IncIt, NumElts), LimitForRegisterSize )) {
14304
+ TryToVectorizeHelper(ArrayRef(IncIt, NumElts), MaxVFOnly )) {
14309
14305
// Success start over because instructions might have been changed.
14310
14306
Changed = true;
14311
14307
} else {
@@ -14324,20 +14320,19 @@ static bool tryToVectorizeSequence(
14324
14320
// Final attempt to vectorize instructions with the same types.
14325
14321
if (Candidates.size() > 1 &&
14326
14322
(SameTypeIt == E || (*SameTypeIt)->getType() != (*IncIt)->getType())) {
14327
- if (TryToVectorizeHelper(Candidates, /*LimitForRegisterSize =*/false)) {
14323
+ if (TryToVectorizeHelper(Candidates, /*MaxVFOnly =*/false)) {
14328
14324
// Success start over because instructions might have been changed.
14329
14325
Changed = true;
14330
- } else if (LimitForRegisterSize ) {
14326
+ } else if (MaxVFOnly ) {
14331
14327
// Try to vectorize using small vectors.
14332
14328
for (auto *It = Candidates.begin(), *End = Candidates.end();
14333
14329
It != End;) {
14334
14330
auto *SameTypeIt = It;
14335
14331
while (SameTypeIt != End && AreCompatible(*SameTypeIt, *It))
14336
14332
++SameTypeIt;
14337
14333
unsigned NumElts = (SameTypeIt - It);
14338
- if (NumElts > 1 &&
14339
- TryToVectorizeHelper(ArrayRef(It, NumElts),
14340
- /*LimitForRegisterSize=*/false))
14334
+ if (NumElts > 1 && TryToVectorizeHelper(ArrayRef(It, NumElts),
14335
+ /*MaxVFOnly=*/false))
14341
14336
Changed = true;
14342
14337
It = SameTypeIt;
14343
14338
}
@@ -14438,7 +14433,7 @@ bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts,
14438
14433
SmallVector<Value *> Vals(CmpInsts.begin(), CmpInsts.end());
14439
14434
Changed |= tryToVectorizeSequence<Value>(
14440
14435
Vals, CompareSorter, AreCompatibleCompares,
14441
- [this, &R](ArrayRef<Value *> Candidates, bool LimitForRegisterSize ) {
14436
+ [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly ) {
14442
14437
// Exclude possible reductions from other blocks.
14443
14438
bool ArePossiblyReducedInOtherBlock = any_of(Candidates, [](Value *V) {
14444
14439
return any_of(V->users(), [V](User *U) {
@@ -14449,9 +14444,9 @@ bool SLPVectorizerPass::vectorizeCmpInsts(ArrayRef<CmpInst *> CmpInsts,
14449
14444
});
14450
14445
if (ArePossiblyReducedInOtherBlock)
14451
14446
return false;
14452
- return tryToVectorizeList(Candidates, R, LimitForRegisterSize );
14447
+ return tryToVectorizeList(Candidates, R, MaxVFOnly );
14453
14448
},
14454
- /*LimitForRegisterSize =*/true, R);
14449
+ /*MaxVFOnly =*/true, R);
14455
14450
return Changed;
14456
14451
}
14457
14452
@@ -14635,10 +14630,10 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
14635
14630
14636
14631
HaveVectorizedPhiNodes = tryToVectorizeSequence<Value>(
14637
14632
Incoming, PHICompare, AreCompatiblePHIs,
14638
- [this, &R](ArrayRef<Value *> Candidates, bool LimitForRegisterSize ) {
14639
- return tryToVectorizeList(Candidates, R, LimitForRegisterSize );
14633
+ [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly ) {
14634
+ return tryToVectorizeList(Candidates, R, MaxVFOnly );
14640
14635
},
14641
- /*LimitForRegisterSize =*/true, R);
14636
+ /*MaxVFOnly =*/true, R);
14642
14637
Changed |= HaveVectorizedPhiNodes;
14643
14638
VisitedInstrs.insert(Incoming.begin(), Incoming.end());
14644
14639
} while (HaveVectorizedPhiNodes);
@@ -14931,7 +14926,7 @@ bool SLPVectorizerPass::vectorizeStoreChains(BoUpSLP &R) {
14931
14926
[this, &R](ArrayRef<StoreInst *> Candidates, bool) {
14932
14927
return vectorizeStores(Candidates, R);
14933
14928
},
14934
- /*LimitForRegisterSize =*/false, R);
14929
+ /*MaxVFOnly =*/false, R);
14935
14930
}
14936
14931
return Changed;
14937
14932
}
0 commit comments