Skip to content

Commit 7ddb7dd

Browse files
artagnonMel-Chen
artagnon
and
Mel-Chen
committed
IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work when it already checks RecurKind::IAnyOf. Affect a minor adjustment to the code to facilitate skipping the RecurKind::FAnyOf check, and strip the check. The patch has the side-effect of rewriting some flaky code, which would match an ICmp having the reduction phi as an operand with IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is also matched), it would have to rely on failing to match an FCmp with FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully vectorizing an IAnyOf pattern with the incorrect debug output. There is a test for this already in select-cmp.ll: select_i32_from_icmp_same_inputs. Co-authored-by: Mel Chen <[email protected]>
1 parent 62cae9c commit 7ddb7dd

File tree

5 files changed

+17
-33
lines changed

5 files changed

+17
-33
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,7 @@ enum class RecurKind {
5151
FMinimumNum, ///< FP min with llvm.minimumnum semantics
5252
FMaximumNum, ///< FP max with llvm.maximumnum semantics
5353
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
54-
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
55-
///< loop invariant, and both x and y are integer type.
56-
FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
54+
AnyOf, ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
5755
///< loop invariant, and both x and y are integer type.
5856
FindLastIV, ///< FindLast reduction with select(cmp(),x,y) where one of
5957
///< (x,y) is increasing loop induction, and both x and y are
@@ -252,7 +250,7 @@ class RecurrenceDescriptor {
252250
/// Returns true if the recurrence kind is of the form
253251
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
254252
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
255-
return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
253+
return Kind == RecurKind::AnyOf;
256254
}
257255

258256
/// Returns true if the recurrence kind is of the form

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
4949
case RecurKind::SMin:
5050
case RecurKind::UMax:
5151
case RecurKind::UMin:
52-
case RecurKind::IAnyOf:
53-
case RecurKind::FAnyOf:
52+
case RecurKind::AnyOf:
5453
case RecurKind::FindLastIV:
5554
return true;
5655
}
@@ -416,11 +415,11 @@ bool RecurrenceDescriptor::AddReductionVar(
416415
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
417416
return false;
418417

419-
if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
420-
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
418+
if (isIntMinMaxRecurrenceKind(Kind) && (isa<ICmpInst>(Cur) || IsASelect))
421419
++NumCmpSelectPatternInst;
422-
if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
423-
(isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
420+
if (isFPMinMaxRecurrenceKind(Kind) && (isa<FCmpInst>(Cur) || IsASelect))
421+
++NumCmpSelectPatternInst;
422+
if (isAnyOfRecurrenceKind(Kind) && IsASelect)
424423
++NumCmpSelectPatternInst;
425424

426425
// Check whether we found a reduction operator.
@@ -653,8 +652,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
653652
if (!Loop->isLoopInvariant(NonPhi))
654653
return InstDesc(false, I);
655654

656-
return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
657-
: RecurKind::FAnyOf);
655+
return InstDesc(I, RecurKind::AnyOf);
658656
}
659657

660658
// We are looking for loops that do something like this:
@@ -981,10 +979,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
981979
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
982980
return true;
983981
}
984-
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
982+
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
985983
SE)) {
986-
LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
987-
<< *Phi << "\n");
984+
LLVM_DEBUG(dbgs() << "Found a conditional select reduction PHI." << *Phi
985+
<< "\n");
988986
return true;
989987
}
990988
if (AddReductionVar(Phi, RecurKind::FindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1012,12 +1010,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10121010
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
10131011
return true;
10141012
}
1015-
if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
1016-
SE)) {
1017-
LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
1018-
<< " PHI." << *Phi << "\n");
1019-
return true;
1020-
}
10211013
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
10221014
SE)) {
10231015
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1144,8 +1136,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11441136
return Instruction::Add;
11451137
case RecurKind::Mul:
11461138
return Instruction::Mul;
1147-
case RecurKind::IAnyOf:
1148-
case RecurKind::FAnyOf:
1139+
case RecurKind::AnyOf:
11491140
case RecurKind::FindLastIV:
11501141
case RecurKind::Or:
11511142
return Instruction::Or;

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5072,8 +5072,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
50725072
case RecurKind::FMin:
50735073
case RecurKind::FMax:
50745074
case RecurKind::FMulAdd:
5075-
case RecurKind::IAnyOf:
5076-
case RecurKind::FAnyOf:
5075+
case RecurKind::AnyOf:
50775076
return true;
50785077
default:
50795078
return false;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -372,11 +372,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
372372
case RecurKind::SMax:
373373
case RecurKind::UMin:
374374
case RecurKind::UMax:
375-
case RecurKind::IAnyOf:
376375
case RecurKind::FMin:
377376
case RecurKind::FMax:
378377
return true;
379-
case RecurKind::FAnyOf:
378+
case RecurKind::AnyOf:
380379
case RecurKind::FAdd:
381380
case RecurKind::FMulAdd:
382381
// We can't promote f16/bf16 fadd reductions and scalable vectors can't be

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23089,8 +23089,7 @@ class HorizontalReduction {
2308923089
case RecurKind::Mul:
2309023090
case RecurKind::FMul:
2309123091
case RecurKind::FMulAdd:
23092-
case RecurKind::IAnyOf:
23093-
case RecurKind::FAnyOf:
23092+
case RecurKind::AnyOf:
2309423093
case RecurKind::FindLastIV:
2309523094
case RecurKind::FMaximumNum:
2309623095
case RecurKind::FMinimumNum:
@@ -23224,8 +23223,7 @@ class HorizontalReduction {
2322423223
case RecurKind::Mul:
2322523224
case RecurKind::FMul:
2322623225
case RecurKind::FMulAdd:
23227-
case RecurKind::IAnyOf:
23228-
case RecurKind::FAnyOf:
23226+
case RecurKind::AnyOf:
2322923227
case RecurKind::FindLastIV:
2323023228
case RecurKind::FMaximumNum:
2323123229
case RecurKind::FMinimumNum:
@@ -23324,8 +23322,7 @@ class HorizontalReduction {
2332423322
case RecurKind::Mul:
2332523323
case RecurKind::FMul:
2332623324
case RecurKind::FMulAdd:
23327-
case RecurKind::IAnyOf:
23328-
case RecurKind::FAnyOf:
23325+
case RecurKind::AnyOf:
2332923326
case RecurKind::FindLastIV:
2333023327
case RecurKind::FMaximumNum:
2333123328
case RecurKind::FMinimumNum:

0 commit comments

Comments
 (0)