Skip to content

Commit c9b44e9

Browse files
artagnonMel-Chen
artagnon
and
Mel-Chen
committed
IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work when it already checks RecurKind::IAnyOf. Affect a minor adjustment to the code to facilitate skipping the RecurKind::FAnyOf check, and strip the check. The patch has the side-effect of rewriting some flaky code, which would match an ICmp having the reduction phi as an operand with IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is also matched), it would have to rely on failing to match an FCmp with FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully vectorizing an IAnyOf pattern with the incorrect debug output. There is a test for this already in select-cmp.ll: select_i32_from_icmp_same_inputs. Co-authored-by: Mel Chen <[email protected]>
1 parent 4fa2c62 commit c9b44e9

File tree

5 files changed

+35
-51
lines changed

5 files changed

+35
-51
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -31,29 +31,27 @@ class StoreInst;
3131

3232
/// These are the kinds of recurrences that we support.
3333
enum class RecurKind {
34-
None, ///< Not a recurrence.
35-
Add, ///< Sum of integers.
36-
Mul, ///< Product of integers.
37-
Or, ///< Bitwise or logical OR of integers.
38-
And, ///< Bitwise or logical AND of integers.
39-
Xor, ///< Bitwise or logical XOR of integers.
40-
SMin, ///< Signed integer min implemented in terms of select(cmp()).
41-
SMax, ///< Signed integer max implemented in terms of select(cmp()).
42-
UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
43-
UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
44-
FAdd, ///< Sum of floats.
45-
FMul, ///< Product of floats.
46-
FMin, ///< FP min implemented in terms of select(cmp()).
47-
FMax, ///< FP max implemented in terms of select(cmp()).
48-
FMinimum, ///< FP min with llvm.minimum semantics
49-
FMaximum, ///< FP max with llvm.maximum semantics
34+
None, ///< Not a recurrence.
35+
Add, ///< Sum of integers.
36+
Mul, ///< Product of integers.
37+
Or, ///< Bitwise or logical OR of integers.
38+
And, ///< Bitwise or logical AND of integers.
39+
Xor, ///< Bitwise or logical XOR of integers.
40+
SMin, ///< Signed integer min implemented in terms of select(cmp()).
41+
SMax, ///< Signed integer max implemented in terms of select(cmp()).
42+
UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
43+
UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
44+
FAdd, ///< Sum of floats.
45+
FMul, ///< Product of floats.
46+
FMin, ///< FP min implemented in terms of select(cmp()).
47+
FMax, ///< FP max implemented in terms of select(cmp()).
48+
FMinimum, ///< FP min with llvm.minimum semantics
49+
FMaximum, ///< FP max with llvm.maximum semantics
5050
FMinimumNum, ///< FP min with llvm.minimumnum semantics
5151
FMaximumNum, ///< FP max with llvm.maximumnum semantics
52-
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
53-
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
54-
///< loop invariant, and both x and y are integer type.
55-
FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
56-
///< loop invariant, and both x and y are integer type.
52+
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
53+
AnyOf, ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
54+
///< loop invariant.
5755
IFindLastIV, ///< FindLast reduction with select(icmp(),x,y) where one of
5856
///< (x,y) is increasing loop induction, and both x and y are
5957
///< integer type.
@@ -253,7 +251,7 @@ class RecurrenceDescriptor {
253251
/// Returns true if the recurrence kind is of the form
254252
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
255253
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
256-
return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
254+
return Kind == RecurKind::AnyOf;
257255
}
258256

259257
/// Returns true if the recurrence kind is of the form

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
4949
case RecurKind::SMin:
5050
case RecurKind::UMax:
5151
case RecurKind::UMin:
52-
case RecurKind::IAnyOf:
53-
case RecurKind::FAnyOf:
52+
case RecurKind::AnyOf:
5453
case RecurKind::IFindLastIV:
5554
case RecurKind::FFindLastIV:
5655
return true;
@@ -417,11 +416,11 @@ bool RecurrenceDescriptor::AddReductionVar(
417416
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
418417
return false;
419418

420-
if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
421-
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
419+
if (isIntMinMaxRecurrenceKind(Kind) && (isa<ICmpInst>(Cur) || IsASelect))
422420
++NumCmpSelectPatternInst;
423-
if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
424-
(isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
421+
if (isFPMinMaxRecurrenceKind(Kind) && (isa<FCmpInst>(Cur) || IsASelect))
422+
++NumCmpSelectPatternInst;
423+
if (isAnyOfRecurrenceKind(Kind) && IsASelect)
425424
++NumCmpSelectPatternInst;
426425

427426
// Check whether we found a reduction operator.
@@ -656,8 +655,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
656655
if (!Loop->isLoopInvariant(NonPhi))
657656
return InstDesc(false, I);
658657

659-
return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
660-
: RecurKind::FAnyOf);
658+
return InstDesc(I, RecurKind::AnyOf);
661659
}
662660

663661
// We are looking for loops that do something like this:
@@ -987,10 +985,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
987985
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
988986
return true;
989987
}
990-
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
988+
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
991989
SE)) {
992-
LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
993-
<< *Phi << "\n");
990+
LLVM_DEBUG(dbgs() << "Found a conditional select reduction PHI." << *Phi
991+
<< "\n");
994992
return true;
995993
}
996994
if (AddReductionVar(Phi, RecurKind::IFindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1022,12 +1020,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10221020
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
10231021
return true;
10241022
}
1025-
if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
1026-
SE)) {
1027-
LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
1028-
<< " PHI." << *Phi << "\n");
1029-
return true;
1030-
}
10311023
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
10321024
SE)) {
10331025
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1154,8 +1146,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11541146
return Instruction::Add;
11551147
case RecurKind::Mul:
11561148
return Instruction::Mul;
1157-
case RecurKind::IAnyOf:
1158-
case RecurKind::FAnyOf:
1149+
case RecurKind::AnyOf:
11591150
case RecurKind::Or:
11601151
return Instruction::Or;
11611152
case RecurKind::And:

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5072,8 +5072,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
50725072
case RecurKind::FMin:
50735073
case RecurKind::FMax:
50745074
case RecurKind::FMulAdd:
5075-
case RecurKind::IAnyOf:
5076-
case RecurKind::FAnyOf:
5075+
case RecurKind::AnyOf:
50775076
return true;
50785077
default:
50795078
return false;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,11 +372,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
372372
case RecurKind::SMax:
373373
case RecurKind::UMin:
374374
case RecurKind::UMax:
375-
case RecurKind::IAnyOf:
376375
case RecurKind::FMin:
377376
case RecurKind::FMax:
378377
return true;
379-
case RecurKind::FAnyOf:
378+
case RecurKind::AnyOf:
380379
case RecurKind::FAdd:
381380
case RecurKind::FMulAdd:
382381
// We can't promote f16/bf16 fadd reductions and scalable vectors can't be

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23069,8 +23069,7 @@ class HorizontalReduction {
2306923069
case RecurKind::Mul:
2307023070
case RecurKind::FMul:
2307123071
case RecurKind::FMulAdd:
23072-
case RecurKind::IAnyOf:
23073-
case RecurKind::FAnyOf:
23072+
case RecurKind::AnyOf:
2307423073
case RecurKind::IFindLastIV:
2307523074
case RecurKind::FFindLastIV:
2307623075
case RecurKind::FMaximumNum:
@@ -23205,8 +23204,7 @@ class HorizontalReduction {
2320523204
case RecurKind::Mul:
2320623205
case RecurKind::FMul:
2320723206
case RecurKind::FMulAdd:
23208-
case RecurKind::IAnyOf:
23209-
case RecurKind::FAnyOf:
23207+
case RecurKind::AnyOf:
2321023208
case RecurKind::IFindLastIV:
2321123209
case RecurKind::FFindLastIV:
2321223210
case RecurKind::FMaximumNum:
@@ -23306,8 +23304,7 @@ class HorizontalReduction {
2330623304
case RecurKind::Mul:
2330723305
case RecurKind::FMul:
2330823306
case RecurKind::FMulAdd:
23309-
case RecurKind::IAnyOf:
23310-
case RecurKind::FAnyOf:
23307+
case RecurKind::AnyOf:
2331123308
case RecurKind::IFindLastIV:
2331223309
case RecurKind::FFindLastIV:
2331323310
case RecurKind::FMaximumNum:

0 commit comments

Comments
 (0)