Skip to content

Commit af86167

Browse files
artagnonartagnon
committed
IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work when it already checks RecurKind::IAnyOf. Affect a minor adjustment to the code to facilitate skipping the RecurKind::FAnyOf check, and strip the check. The patch has the side-effect of rewriting some flaky code, which would match an ICmp having the reduction phi as an operand with IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is also matched), it would have to rely on failing to match an FCmp with FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully vectorizing an IAnyOf pattern with the incorrect debug output. There is a test for this already in select-cmp.ll: select_i32_from_icmp_same_inputs.
1 parent 7e01a32 commit af86167

File tree

5 files changed

+29
-39
lines changed

5 files changed

+29
-39
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,8 @@ enum class RecurKind {
4848
FMinimum, ///< FP min with llvm.minimum semantics
4949
FMaximum, ///< FP max with llvm.maximum semantics
5050
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
51-
IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
52-
///< loop invariant, and both x and y are integer type.
53-
FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
54-
///< loop invariant, and both x and y are integer type.
51+
AnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
52+
///< loop invariant.
5553
IFindLastIV, ///< FindLast reduction with select(icmp(),x,y) where one of
5654
///< (x,y) is increasing loop induction, and both x and y are
5755
///< integer type.
@@ -173,7 +171,7 @@ class RecurrenceDescriptor {
173171
static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
174172

175173
/// Returns the opcode corresponding to the RecurrenceKind.
176-
static unsigned getOpcode(RecurKind Kind);
174+
static unsigned getOpcode(RecurKind Kind, Type *Ty);
177175

178176
/// Returns true if Phi is a reduction of type Kind and adds it to the
179177
/// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are
@@ -209,7 +207,9 @@ class RecurrenceDescriptor {
209207

210208
RecurKind getRecurrenceKind() const { return Kind; }
211209

212-
unsigned getOpcode() const { return getOpcode(getRecurrenceKind()); }
210+
unsigned getOpcode() const {
211+
return getOpcode(getRecurrenceKind(), getRecurrenceType());
212+
}
213213

214214
FastMathFlags getFastMathFlags() const { return FMF; }
215215

@@ -250,7 +250,7 @@ class RecurrenceDescriptor {
250250
/// Returns true if the recurrence kind is of the form
251251
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
252252
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
253-
return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
253+
return Kind == RecurKind::AnyOf;
254254
}
255255

256256
/// Returns true if the recurrence kind is of the form

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
4949
case RecurKind::SMin:
5050
case RecurKind::UMax:
5151
case RecurKind::UMin:
52-
case RecurKind::IAnyOf:
53-
case RecurKind::FAnyOf:
52+
case RecurKind::AnyOf:
5453
case RecurKind::IFindLastIV:
5554
case RecurKind::FFindLastIV:
5655
return true;
@@ -417,11 +416,10 @@ bool RecurrenceDescriptor::AddReductionVar(
417416
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
418417
return false;
419418

420-
if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
421-
(isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
419+
if (isMinMaxRecurrenceKind(Kind) &&
420+
(Cur->getOpcode() == getOpcode(Kind, Cur->getType()) || IsASelect))
422421
++NumCmpSelectPatternInst;
423-
if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
424-
(isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
422+
if (isAnyOfRecurrenceKind(Kind) && IsASelect)
425423
++NumCmpSelectPatternInst;
426424

427425
// Check whether we found a reduction operator.
@@ -656,8 +654,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
656654
if (!Loop->isLoopInvariant(NonPhi))
657655
return InstDesc(false, I);
658656

659-
return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
660-
: RecurKind::FAnyOf);
657+
return InstDesc(I, RecurKind::AnyOf);
661658
}
662659

663660
// We are looking for loops that do something like this:
@@ -979,10 +976,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
979976
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
980977
return true;
981978
}
982-
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
979+
if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
983980
SE)) {
984-
LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
985-
<< *Phi << "\n");
981+
LLVM_DEBUG(dbgs() << "Found an conditional select reduction PHI." << *Phi
982+
<< "\n");
986983
return true;
987984
}
988985
if (AddReductionVar(Phi, RecurKind::IFindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1014,12 +1011,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
10141011
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
10151012
return true;
10161013
}
1017-
if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
1018-
SE)) {
1019-
LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
1020-
<< " PHI." << *Phi << "\n");
1021-
return true;
1022-
}
10231014
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
10241015
SE)) {
10251016
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1127,7 +1118,7 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
11271118
return true;
11281119
}
11291120

1130-
unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
1121+
unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind, Type *Ty) {
11311122
switch (Kind) {
11321123
case RecurKind::Add:
11331124
return Instruction::Add;
@@ -1148,16 +1139,16 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
11481139
case RecurKind::SMin:
11491140
case RecurKind::UMax:
11501141
case RecurKind::UMin:
1151-
case RecurKind::IAnyOf:
11521142
case RecurKind::IFindLastIV:
11531143
return Instruction::ICmp;
11541144
case RecurKind::FMax:
11551145
case RecurKind::FMin:
11561146
case RecurKind::FMaximum:
11571147
case RecurKind::FMinimum:
1158-
case RecurKind::FAnyOf:
11591148
case RecurKind::FFindLastIV:
11601149
return Instruction::FCmp;
1150+
case RecurKind::AnyOf:
1151+
return Ty->isIntegerTy() ? Instruction::ICmp : Instruction::FCmp;
11611152
default:
11621153
llvm_unreachable("Unknown recurrence operation");
11631154
}

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4404,8 +4404,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
44044404
case RecurKind::FMin:
44054405
case RecurKind::FMax:
44064406
case RecurKind::FMulAdd:
4407-
case RecurKind::IAnyOf:
4408-
case RecurKind::FAnyOf:
4407+
case RecurKind::AnyOf:
44094408
return true;
44104409
default:
44114410
return false;

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -348,8 +348,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
348348
case RecurKind::FMin:
349349
case RecurKind::FMax:
350350
case RecurKind::FMulAdd:
351-
case RecurKind::IAnyOf:
352-
case RecurKind::FAnyOf:
351+
case RecurKind::AnyOf:
353352
return true;
354353
default:
355354
return false;

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19410,15 +19410,17 @@ class HorizontalReduction {
1941019410
if (UseSelect &&
1941119411
LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
1941219412
return Builder.CreateSelect(LHS, Builder.getTrue(), RHS, Name);
19413-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19413+
unsigned RdxOpcode =
19414+
RecurrenceDescriptor::getOpcode(Kind, LHS->getType());
1941419415
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
1941519416
Name);
1941619417
}
1941719418
case RecurKind::And: {
1941819419
if (UseSelect &&
1941919420
LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
1942019421
return Builder.CreateSelect(LHS, RHS, Builder.getFalse(), Name);
19421-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19422+
unsigned RdxOpcode =
19423+
RecurrenceDescriptor::getOpcode(Kind, LHS->getType());
1942219424
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
1942319425
Name);
1942419426
}
@@ -19427,7 +19429,8 @@ class HorizontalReduction {
1942719429
case RecurKind::Xor:
1942819430
case RecurKind::FAdd:
1942919431
case RecurKind::FMul: {
19430-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19432+
unsigned RdxOpcode =
19433+
RecurrenceDescriptor::getOpcode(Kind, LHS->getType());
1943119434
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
1943219435
Name);
1943319436
}
@@ -20550,7 +20553,7 @@ class HorizontalReduction {
2055020553
case RecurKind::Xor:
2055120554
case RecurKind::FAdd:
2055220555
case RecurKind::FMul: {
20553-
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
20556+
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind, ScalarTy);
2055420557
if (!AllConsts) {
2055520558
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
2055620559
assert(SLPReVec && "FixedVectorType is not expected.");
@@ -20679,8 +20682,7 @@ class HorizontalReduction {
2067920682
case RecurKind::Mul:
2068020683
case RecurKind::FMul:
2068120684
case RecurKind::FMulAdd:
20682-
case RecurKind::IAnyOf:
20683-
case RecurKind::FAnyOf:
20685+
case RecurKind::AnyOf:
2068420686
case RecurKind::IFindLastIV:
2068520687
case RecurKind::FFindLastIV:
2068620688
case RecurKind::None:
@@ -20778,8 +20780,7 @@ class HorizontalReduction {
2077820780
case RecurKind::Mul:
2077920781
case RecurKind::FMul:
2078020782
case RecurKind::FMulAdd:
20781-
case RecurKind::IAnyOf:
20782-
case RecurKind::FAnyOf:
20783+
case RecurKind::AnyOf:
2078320784
case RecurKind::IFindLastIV:
2078420785
case RecurKind::FFindLastIV:
2078520786
case RecurKind::None:

0 commit comments

Comments
 (0)