Skip to content

Commit 2c387ce

Browse files
artagnonartagnon
committed
IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work when it already checks RecurKind::IAnyOf. Affect a minor adjustment to the code to facilitate skipping the RecurKind::FAnyOf check, and strip the check. The patch has the side-effect of rewriting some flaky code, which would match an ICmp having the reduction phi as an operand with IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is also matched), it would have to rely on failing to match an FCmp with FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully vectorizing an IAnyOf pattern with the incorrect debug output. There is a test for this already in select-cmp.ll: select_i32_from_icmp_same_inputs.
1 parent ace87ec commit 2c387ce

File tree

5 files changed

+25
-37
lines changed

5 files changed

+25
-37
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,8 @@ enum class RecurKind {
4848
FMinimum, ///< FP min with llvm.minimum semantics
4949
FMaximum, ///< FP max with llvm.maximum semantics
5050
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
51-
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
52-
///< loop invariant, and both x and y are integer type.
53-
FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
54-
///< loop invariant, and both x and y are integer type.
51+
AnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
52+
///< loop invariant.
5553
IFindLastIV, ///< FindLast reduction with select(icmp(),x,y) where one of
5654
///< (x,y) is increasing loop induction, and both x and y are
5755
///< integer type.
@@ -173,7 +171,7 @@ class RecurrenceDescriptor {
173171
static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
174172

175173
/// Returns the opcode corresponding to the RecurrenceKind.
176-
static unsigned getOpcode(RecurKind Kind);
174+
static unsigned getOpcode(RecurKind Kind, Type *Ty);
177175

178176
/// Returns true if Phi is a reduction of type Kind and adds it to the
179177
/// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are
@@ -209,7 +207,9 @@ class RecurrenceDescriptor {
209207

210208
RecurKind getRecurrenceKind() const { return Kind; }
211209

212-
unsigned getOpcode() const { return getOpcode(getRecurrenceKind()); }
210+
unsigned getOpcode() const {
211+
return getOpcode(getRecurrenceKind(), getRecurrenceType());
212+
}
213213

214214
FastMathFlags getFastMathFlags() const { return FMF; }
215215

@@ -250,7 +250,7 @@ class RecurrenceDescriptor {
250250
/// Returns true if the recurrence kind is of the form
251251
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
252252
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
253-
return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
253+
return Kind == RecurKind::AnyOf;
254254
}
255255

256256
/// Returns true if the recurrence kind is of the form

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 12 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
4949
case RecurKind::SMin:
5050
case RecurKind::UMax:
5151
case RecurKind::UMin:
52-
case RecurKind::IAnyOf:
53-
case RecurKind::FAnyOf:
52+
case RecurKind::AnyOf:
5453
case RecurKind::IFindLastIV:
5554
case RecurKind::FFindLastIV:
5655
return true;
@@ -417,11 +416,11 @@ bool RecurrenceDescriptor::AddReductionVar(
417416
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
418417
return false;
419418

420-
if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
421-
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
419+
if ((isIntMinMaxRecurrenceKind(Kind) &&
420+
(isa<ICmpInst>(Cur) || IsASelect)) ||
421+
(isFPMinMaxRecurrenceKind(Kind) && (isa<FCmpInst>(Cur) || IsASelect)))
422422
++NumCmpSelectPatternInst;
423-
if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
424-
(isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
423+
if (isAnyOfRecurrenceKind(Kind) && IsASelect)
425424
++NumCmpSelectPatternInst;
426425

427426
// Check whether we found a reduction operator.
@@ -656,8 +655,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
656655
if (!Loop->isLoopInvariant(NonPhi))
657656
return InstDesc(false, I);
658657

659-
return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
660-
: RecurKind::FAnyOf);
658+
return InstDesc(I, RecurKind::AnyOf);
661659
}
662660

663661
// We are looking for loops that do something like this:
@@ -979,10 +977,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
979977
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
980978
return true;
981979
}
982-
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
980+
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
983981
SE)) {
984-
LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
985-
<< *Phi << "\n");
982+
LLVM_DEBUG(dbgs() << "Found an conditional select reduction PHI." << *Phi
983+
<< "\n");
986984
return true;
987985
}
988986
if (AddReductionVar(Phi, RecurKind::IFindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1014,12 +1012,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10141012
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
10151013
return true;
10161014
}
1017-
if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
1018-
SE)) {
1019-
LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
1020-
<< " PHI." << *Phi << "\n");
1021-
return true;
1022-
}
10231015
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
10241016
SE)) {
10251017
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1127,7 +1119,7 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
11271119
return true;
11281120
}
11291121

1130-
unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
1122+
unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind, Type *Ty) {
11311123
switch (Kind) {
11321124
case RecurKind::Add:
11331125
return Instruction::Add;
@@ -1148,16 +1140,16 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11481140
case RecurKind::SMin:
11491141
case RecurKind::UMax:
11501142
case RecurKind::UMin:
1151-
case RecurKind::IAnyOf:
11521143
case RecurKind::IFindLastIV:
11531144
return Instruction::ICmp;
11541145
case RecurKind::FMax:
11551146
case RecurKind::FMin:
11561147
case RecurKind::FMaximum:
11571148
case RecurKind::FMinimum:
1158-
case RecurKind::FAnyOf:
11591149
case RecurKind::FFindLastIV:
11601150
return Instruction::FCmp;
1151+
case RecurKind::AnyOf:
1152+
return Ty->isIntegerTy() ? Instruction::ICmp : Instruction::FCmp;
11611153
default:
11621154
llvm_unreachable("Unknown recurrence operation");
11631155
}

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4318,8 +4318,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
43184318
case RecurKind::FMin:
43194319
case RecurKind::FMax:
43204320
case RecurKind::FMulAdd:
4321-
case RecurKind::IAnyOf:
4322-
case RecurKind::FAnyOf:
4321+
case RecurKind::AnyOf:
43234322
return true;
43244323
default:
43254324
return false;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,8 +348,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
348348
case RecurKind::FMin:
349349
case RecurKind::FMax:
350350
case RecurKind::FMulAdd:
351-
case RecurKind::IAnyOf:
352-
case RecurKind::FAnyOf:
351+
case RecurKind::AnyOf:
353352
return true;
354353
default:
355354
return false;

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19182,7 +19182,7 @@ class HorizontalReduction {
1918219182
/// Creates reduction operation with the current opcode.
1918319183
static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
1918419184
Value *RHS, const Twine &Name, bool UseSelect) {
19185-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19185+
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind, LHS->getType());
1918619186
switch (Kind) {
1918719187
case RecurKind::Or:
1918819188
if (UseSelect &&
@@ -20325,7 +20325,7 @@ class HorizontalReduction {
2032520325
case RecurKind::Xor:
2032620326
case RecurKind::FAdd:
2032720327
case RecurKind::FMul: {
20328-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
20328+
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind, ScalarTy);
2032920329
if (!AllConsts) {
2033020330
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
2033120331
assert(SLPReVec && "FixedVectorType is not expected.");
@@ -20454,8 +20454,7 @@ class HorizontalReduction {
2045420454
case RecurKind::Mul:
2045520455
case RecurKind::FMul:
2045620456
case RecurKind::FMulAdd:
20457-
case RecurKind::IAnyOf:
20458-
case RecurKind::FAnyOf:
20457+
case RecurKind::AnyOf:
2045920458
case RecurKind::IFindLastIV:
2046020459
case RecurKind::FFindLastIV:
2046120460
case RecurKind::None:
@@ -20553,8 +20552,7 @@ class HorizontalReduction {
2055320552
case RecurKind::Mul:
2055420553
case RecurKind::FMul:
2055520554
case RecurKind::FMulAdd:
20556-
case RecurKind::IAnyOf:
20557-
case RecurKind::FAnyOf:
20555+
case RecurKind::AnyOf:
2055820556
case RecurKind::IFindLastIV:
2055920557
case RecurKind::FFindLastIV:
2056020558
case RecurKind::None:

0 commit comments

Comments
 (0)