Skip to content

Commit 89e1f95

Browse files
artagnonMel-Chen
artagnon
and
Mel-Chen
committed
IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work when it already checks RecurKind::IAnyOf. Affect a minor adjustment to the code to facilitate skipping the RecurKind::FAnyOf check, and strip the check. The patch has the side-effect of rewriting some flaky code, which would match an ICmp having the reduction phi as an operand with IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is also matched), it would have to rely on failing to match an FCmp with FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully vectorizing an IAnyOf pattern with the incorrect debug output. There is a test for this already in select-cmp.ll: select_i32_from_icmp_same_inputs. Co-authored-by: Mel Chen <[email protected]>
1 parent 30a9d9d commit 89e1f95

File tree

5 files changed

+20
-34
lines changed

5 files changed

+20
-34
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class StoreInst;
3131

3232
/// These are the kinds of recurrences that we support.
3333
enum class RecurKind {
34+
// clang-format off
3435
None, ///< Not a recurrence.
3536
Add, ///< Sum of integers.
3637
Mul, ///< Product of integers.
@@ -50,17 +51,16 @@ enum class RecurKind {
5051
FMinimumNum, ///< FP min with llvm.minimumnum semantics
5152
FMaximumNum, ///< FP max with llvm.maximumnum semantics
5253
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
53-
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
54-
///< loop invariant, and both x and y are integer type.
55-
FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
54+
AnyOf, ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
5655
///< loop invariant, and both x and y are integer type.
5756
IFindLastIV, ///< FindLast reduction with select(icmp(),x,y) where one of
5857
///< (x,y) is increasing loop induction, and both x and y are
5958
///< integer type.
6059
FFindLastIV ///< FindLast reduction with select(fcmp(),x,y) where one of (x,y)
6160
///< is increasing loop induction, and both x and y are integer
6261
///< type.
63-
// TODO: Any_of and FindLast reduction need not be restricted to integer type
62+
// clang-format on
63+
// TODO: AnyOf and FindLast reduction need not be restricted to integer type
6464
// only.
6565
};
6666

@@ -253,7 +253,7 @@ class RecurrenceDescriptor {
253253
/// Returns true if the recurrence kind is of the form
254254
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
255255
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
256-
return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
256+
return Kind == RecurKind::AnyOf;
257257
}
258258

259259
/// Returns true if the recurrence kind is of the form

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
4949
case RecurKind::SMin:
5050
case RecurKind::UMax:
5151
case RecurKind::UMin:
52-
case RecurKind::IAnyOf:
53-
case RecurKind::FAnyOf:
52+
case RecurKind::AnyOf:
5453
case RecurKind::IFindLastIV:
5554
case RecurKind::FFindLastIV:
5655
return true;
@@ -417,11 +416,11 @@ bool RecurrenceDescriptor::AddReductionVar(
417416
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
418417
return false;
419418

420-
if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
421-
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
419+
if (isIntMinMaxRecurrenceKind(Kind) && (isa<ICmpInst>(Cur) || IsASelect))
422420
++NumCmpSelectPatternInst;
423-
if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
424-
(isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
421+
if (isFPMinMaxRecurrenceKind(Kind) && (isa<FCmpInst>(Cur) || IsASelect))
422+
++NumCmpSelectPatternInst;
423+
if (isAnyOfRecurrenceKind(Kind) && IsASelect)
425424
++NumCmpSelectPatternInst;
426425

427426
// Check whether we found a reduction operator.
@@ -654,8 +653,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
654653
if (!Loop->isLoopInvariant(NonPhi))
655654
return InstDesc(false, I);
656655

657-
return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
658-
: RecurKind::FAnyOf);
656+
return InstDesc(I, RecurKind::AnyOf);
659657
}
660658

661659
// We are looking for loops that do something like this:
@@ -983,10 +981,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
983981
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
984982
return true;
985983
}
986-
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
984+
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
987985
SE)) {
988-
LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
989-
<< *Phi << "\n");
986+
LLVM_DEBUG(dbgs() << "Found a conditional select reduction PHI." << *Phi
987+
<< "\n");
990988
return true;
991989
}
992990
if (AddReductionVar(Phi, RecurKind::IFindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1018,12 +1016,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10181016
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
10191017
return true;
10201018
}
1021-
if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
1022-
SE)) {
1023-
LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
1024-
<< " PHI." << *Phi << "\n");
1025-
return true;
1026-
}
10271019
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
10281020
SE)) {
10291021
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1150,8 +1142,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11501142
return Instruction::Add;
11511143
case RecurKind::Mul:
11521144
return Instruction::Mul;
1153-
case RecurKind::IAnyOf:
1154-
case RecurKind::FAnyOf:
1145+
case RecurKind::AnyOf:
11551146
case RecurKind::Or:
11561147
return Instruction::Or;
11571148
case RecurKind::And:

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5072,8 +5072,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
50725072
case RecurKind::FMin:
50735073
case RecurKind::FMax:
50745074
case RecurKind::FMulAdd:
5075-
case RecurKind::IAnyOf:
5076-
case RecurKind::FAnyOf:
5075+
case RecurKind::AnyOf:
50775076
return true;
50785077
default:
50795078
return false;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,11 +372,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
372372
case RecurKind::SMax:
373373
case RecurKind::UMin:
374374
case RecurKind::UMax:
375-
case RecurKind::IAnyOf:
376375
case RecurKind::FMin:
377376
case RecurKind::FMax:
378377
return true;
379-
case RecurKind::FAnyOf:
378+
case RecurKind::AnyOf:
380379
case RecurKind::FAdd:
381380
case RecurKind::FMulAdd:
382381
// We can't promote f16/bf16 fadd reductions and scalable vectors can't be

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23089,8 +23089,7 @@ class HorizontalReduction {
2308923089
case RecurKind::Mul:
2309023090
case RecurKind::FMul:
2309123091
case RecurKind::FMulAdd:
23092-
case RecurKind::IAnyOf:
23093-
case RecurKind::FAnyOf:
23092+
case RecurKind::AnyOf:
2309423093
case RecurKind::IFindLastIV:
2309523094
case RecurKind::FFindLastIV:
2309623095
case RecurKind::FMaximumNum:
@@ -23225,8 +23224,7 @@ class HorizontalReduction {
2322523224
case RecurKind::Mul:
2322623225
case RecurKind::FMul:
2322723226
case RecurKind::FMulAdd:
23228-
case RecurKind::IAnyOf:
23229-
case RecurKind::FAnyOf:
23227+
case RecurKind::AnyOf:
2323023228
case RecurKind::IFindLastIV:
2323123229
case RecurKind::FFindLastIV:
2323223230
case RecurKind::FMaximumNum:
@@ -23326,8 +23324,7 @@ class HorizontalReduction {
2332623324
case RecurKind::Mul:
2332723325
case RecurKind::FMul:
2332823326
case RecurKind::FMulAdd:
23329-
case RecurKind::IAnyOf:
23330-
case RecurKind::FAnyOf:
23327+
case RecurKind::AnyOf:
2333123328
case RecurKind::IFindLastIV:
2333223329
case RecurKind::FFindLastIV:
2333323330
case RecurKind::FMaximumNum:

0 commit comments

Comments
 (0)