@@ -2448,7 +2448,7 @@ class BoUpSLP {
2448
2448
}
2449
2449
// TODO: Check if we can remove a check for non-power-2 number of
2450
2450
// scalars after full support of non-power-2 vectorization.
2451
- return UniqueValues.size() != 2 && isPowerOf2_32 (UniqueValues.size());
2451
+ return UniqueValues.size() != 2 && has_single_bit (UniqueValues.size());
2452
2452
};
2453
2453
2454
2454
// If the initial strategy fails for any of the operand indexes, then we
@@ -3240,7 +3240,7 @@ class BoUpSLP {
3240
3240
3241
3241
/// Return true if this is a non-power-of-2 node.
3242
3242
bool isNonPowOf2Vec() const {
3243
- bool IsNonPowerOf2 = !isPowerOf2_32 (Scalars.size());
3243
+ bool IsNonPowerOf2 = !has_single_bit (Scalars.size());
3244
3244
assert((!IsNonPowerOf2 || ReuseShuffleIndices.empty()) &&
3245
3245
"Reshuffling not supported with non-power-of-2 vectors yet.");
3246
3246
return IsNonPowerOf2;
@@ -4696,7 +4696,7 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
4696
4696
// Check the order of pointer operands or that all pointers are the same.
4697
4697
bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, Order);
4698
4698
// FIXME: Reordering isn't implemented for non-power-of-2 nodes yet.
4699
- if (!Order.empty() && !isPowerOf2_32 (VL.size())) {
4699
+ if (!Order.empty() && !has_single_bit (VL.size())) {
4700
4700
assert(VectorizeNonPowerOf2 && "non-power-of-2 number of loads only "
4701
4701
"supported with VectorizeNonPowerOf2");
4702
4702
return LoadsState::Gather;
@@ -4746,13 +4746,14 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
4746
4746
return !getTreeEntry(U) && !MustGather.contains(U);
4747
4747
});
4748
4748
});
4749
- if (IsPossibleStrided && (IsAnyPointerUsedOutGraph ||
4750
- ((Sz > MinProfitableStridedLoads ||
4751
- (static_cast<unsigned>(std::abs(*Diff)) <=
4752
- MaxProfitableLoadStride * Sz &&
4753
- isPowerOf2_32(std::abs(*Diff)))) &&
4754
- static_cast<unsigned>(std::abs(*Diff)) > Sz) ||
4755
- *Diff == -(static_cast<int>(Sz) - 1))) {
4749
+ const unsigned AbsoluteDiff = std::abs(*Diff);
4750
+ if (IsPossibleStrided &&
4751
+ (IsAnyPointerUsedOutGraph ||
4752
+ ((Sz > MinProfitableStridedLoads ||
4753
+ (AbsoluteDiff <= MaxProfitableLoadStride * Sz &&
4754
+ has_single_bit(AbsoluteDiff))) &&
4755
+ AbsoluteDiff > Sz) ||
4756
+ *Diff == -(static_cast<int>(Sz) - 1))) {
4756
4757
int Stride = *Diff / static_cast<int>(Sz - 1);
4757
4758
if (*Diff == Stride * static_cast<int>(Sz - 1)) {
4758
4759
Align Alignment =
@@ -6495,7 +6496,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
6495
6496
case Instruction::ExtractElement: {
6496
6497
bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
6497
6498
// FIXME: Vectorizing is not supported yet for non-power-of-2 ops.
6498
- if (!isPowerOf2_32 (VL.size()))
6499
+ if (!has_single_bit (VL.size()))
6499
6500
return TreeEntry::NeedToGather;
6500
6501
if (Reuse || !CurrentOrder.empty())
6501
6502
return TreeEntry::Vectorize;
@@ -10907,7 +10908,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
10907
10908
// Keep original scalar if number of externally used instructions in
10908
10909
// the same entry is not power of 2. It may help to do some extra
10909
10910
// vectorization for now.
10910
- KeepScalar = ScalarUsesCount <= 1 || !isPowerOf2_32 (ScalarUsesCount);
10911
+ KeepScalar = ScalarUsesCount <= 1 || !has_single_bit (ScalarUsesCount);
10911
10912
}
10912
10913
if (KeepScalar) {
10913
10914
ExternalUsesAsOriginalScalar.insert(EU.Scalar);
@@ -16353,7 +16354,7 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
16353
16354
const unsigned Sz = R.getVectorElementSize(Chain[0]);
16354
16355
unsigned VF = Chain.size();
16355
16356
16356
- if (!isPowerOf2_32 (Sz) || !isPowerOf2_32 (VF) || VF < 2 || VF < MinVF) {
16357
+ if (!has_single_bit (Sz) || !has_single_bit (VF) || VF < 2 || VF < MinVF) {
16357
16358
// Check if vectorizing with a non-power-of-2 VF should be considered. At
16358
16359
// the moment, only consider cases where VF + 1 is a power-of-2, i.e. almost
16359
16360
// all vector lanes are used.
@@ -16372,8 +16373,8 @@ SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
16372
16373
if (all_of(ValOps, IsaPred<Instruction>) && ValOps.size() > 1) {
16373
16374
DenseSet<Value *> Stores(Chain.begin(), Chain.end());
16374
16375
bool IsPowerOf2 =
16375
- isPowerOf2_32 (ValOps.size()) ||
16376
- (VectorizeNonPowerOf2 && isPowerOf2_32 (ValOps.size() + 1));
16376
+ has_single_bit (ValOps.size()) ||
16377
+ (VectorizeNonPowerOf2 && has_single_bit (ValOps.size() + 1));
16377
16378
if ((!IsPowerOf2 && S.getOpcode() && S.getOpcode() != Instruction::Load &&
16378
16379
(!S.MainOp->isSafeToRemove() ||
16379
16380
any_of(ValOps.getArrayRef(),
@@ -16540,7 +16541,7 @@ bool SLPVectorizerPass::vectorizeStores(
16540
16541
// consider cases where VF + 1 is a power-of-2, i.e. almost all vector
16541
16542
// lanes are used.
16542
16543
unsigned CandVF = Operands.size();
16543
- if (isPowerOf2_32 (CandVF + 1) && CandVF <= MaxRegVF)
16544
+ if (has_single_bit (CandVF + 1) && CandVF <= MaxRegVF)
16544
16545
NonPowerOf2VF = CandVF;
16545
16546
}
16546
16547
@@ -16946,7 +16947,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
16946
16947
for (unsigned I = NextInst; I < MaxInst; ++I) {
16947
16948
unsigned ActualVF = std::min(MaxInst - I, VF);
16948
16949
16949
- if (!isPowerOf2_32 (ActualVF))
16950
+ if (!has_single_bit (ActualVF))
16950
16951
continue;
16951
16952
16952
16953
if (MaxVFOnly && ActualVF < MaxVF)
@@ -18292,7 +18293,7 @@ class HorizontalReduction {
18292
18293
Value *emitReduction(Value *VectorizedValue, IRBuilderBase &Builder,
18293
18294
unsigned ReduxWidth, const TargetTransformInfo *TTI) {
18294
18295
assert(VectorizedValue && "Need to have a vectorized tree node");
18295
- assert(isPowerOf2_32 (ReduxWidth) &&
18296
+ assert(has_single_bit (ReduxWidth) &&
18296
18297
"We only handle power-of-two reductions for now");
18297
18298
assert(RdxKind != RecurKind::FMulAdd &&
18298
18299
"A call to the llvm.fmuladd intrinsic is not handled yet");
0 commit comments