Skip to content

Commit 2ed2a3a

Browse files
RKSimonRKSimon
committed
[Transforms][Utils] Add helpers to map between Reduction IntrinsicID and Arithmetic Instruction Opcode and MinMax IntrinsicID / RecurKind
Noticed on #81852
1 parent 0ea64ad commit 2ed2a3a

File tree

4 files changed

+100
-90
lines changed

4 files changed

+100
-90
lines changed

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h

Lines changed: 8 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
#include "llvm/Support/MathExtras.h"
4949
#include "llvm/Target/TargetMachine.h"
5050
#include "llvm/Target/TargetOptions.h"
51+
#include "llvm/Transforms/Utils/LoopUtils.h"
5152
#include <algorithm>
5253
#include <cassert>
5354
#include <cstdint>
@@ -2013,50 +2014,27 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
20132014
CostKind);
20142015
}
20152016
case Intrinsic::vector_reduce_add:
2016-
return thisT()->getArithmeticReductionCost(Instruction::Add, VecOpTy,
2017-
std::nullopt, CostKind);
20182017
case Intrinsic::vector_reduce_mul:
2019-
return thisT()->getArithmeticReductionCost(Instruction::Mul, VecOpTy,
2020-
std::nullopt, CostKind);
20212018
case Intrinsic::vector_reduce_and:
2022-
return thisT()->getArithmeticReductionCost(Instruction::And, VecOpTy,
2023-
std::nullopt, CostKind);
20242019
case Intrinsic::vector_reduce_or:
2025-
return thisT()->getArithmeticReductionCost(Instruction::Or, VecOpTy,
2026-
std::nullopt, CostKind);
20272020
case Intrinsic::vector_reduce_xor:
2028-
return thisT()->getArithmeticReductionCost(Instruction::Xor, VecOpTy,
2029-
std::nullopt, CostKind);
2021+
return thisT()->getArithmeticReductionCost(
2022+
getArithmeticReductionInstruction(IID), VecOpTy, std::nullopt,
2023+
CostKind);
20302024
case Intrinsic::vector_reduce_fadd:
2031-
return thisT()->getArithmeticReductionCost(Instruction::FAdd, VecOpTy,
2032-
FMF, CostKind);
20332025
case Intrinsic::vector_reduce_fmul:
2034-
return thisT()->getArithmeticReductionCost(Instruction::FMul, VecOpTy,
2035-
FMF, CostKind);
2026+
return thisT()->getArithmeticReductionCost(
2027+
getArithmeticReductionInstruction(IID), VecOpTy, FMF, CostKind);
20362028
case Intrinsic::vector_reduce_smax:
2037-
return thisT()->getMinMaxReductionCost(Intrinsic::smax, VecOpTy,
2038-
ICA.getFlags(), CostKind);
20392029
case Intrinsic::vector_reduce_smin:
2040-
return thisT()->getMinMaxReductionCost(Intrinsic::smin, VecOpTy,
2041-
ICA.getFlags(), CostKind);
20422030
case Intrinsic::vector_reduce_umax:
2043-
return thisT()->getMinMaxReductionCost(Intrinsic::umax, VecOpTy,
2044-
ICA.getFlags(), CostKind);
20452031
case Intrinsic::vector_reduce_umin:
2046-
return thisT()->getMinMaxReductionCost(Intrinsic::umin, VecOpTy,
2047-
ICA.getFlags(), CostKind);
20482032
case Intrinsic::vector_reduce_fmax:
2049-
return thisT()->getMinMaxReductionCost(Intrinsic::maxnum, VecOpTy,
2050-
ICA.getFlags(), CostKind);
20512033
case Intrinsic::vector_reduce_fmin:
2052-
return thisT()->getMinMaxReductionCost(Intrinsic::minnum, VecOpTy,
2053-
ICA.getFlags(), CostKind);
20542034
case Intrinsic::vector_reduce_fmaximum:
2055-
return thisT()->getMinMaxReductionCost(Intrinsic::maximum, VecOpTy,
2056-
ICA.getFlags(), CostKind);
20572035
case Intrinsic::vector_reduce_fminimum:
2058-
return thisT()->getMinMaxReductionCost(Intrinsic::minimum, VecOpTy,
2059-
ICA.getFlags(), CostKind);
2036+
return thisT()->getMinMaxReductionCost(getMinMaxReductionIntrinsicOp(IID),
2037+
VecOpTy, ICA.getFlags(), CostKind);
20602038
case Intrinsic::abs: {
20612039
// abs(X) = select(icmp(X,0),X,sub(0,X))
20622040
Type *CondTy = RetTy->getWithNewBitWidth(1);

‎llvm/include/llvm/Transforms/Utils/LoopUtils.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,9 +357,18 @@ bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
357357
SinkAndHoistLICMFlags &LICMFlags,
358358
OptimizationRemarkEmitter *ORE = nullptr);
359359

360+
/// Returns the arithmetic instruction opcode used when expanding a reduction.
361+
unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID);
362+
363+
/// Returns the min/max intrinsic used when expanding a min/max reduction.
364+
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID);
365+
360366
/// Returns the min/max intrinsic used when expanding a min/max reduction.
361367
Intrinsic::ID getMinMaxReductionIntrinsicOp(RecurKind RK);
362368

369+
/// Returns the recurence kind used when expanding a min/max reduction.
370+
RecurKind getMinMaxReductionRecurKind(Intrinsic::ID RdxID);
371+
363372
/// Returns the comparison predicate used when expanding a min/max reduction.
364373
CmpInst::Predicate getMinMaxReductionPredicate(RecurKind RK);
365374

‎llvm/lib/CodeGen/ExpandReductions.cpp

Lines changed: 12 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -26,54 +26,6 @@ using namespace llvm;
2626

2727
namespace {
2828

29-
unsigned getOpcode(Intrinsic::ID ID) {
30-
switch (ID) {
31-
case Intrinsic::vector_reduce_fadd:
32-
return Instruction::FAdd;
33-
case Intrinsic::vector_reduce_fmul:
34-
return Instruction::FMul;
35-
case Intrinsic::vector_reduce_add:
36-
return Instruction::Add;
37-
case Intrinsic::vector_reduce_mul:
38-
return Instruction::Mul;
39-
case Intrinsic::vector_reduce_and:
40-
return Instruction::And;
41-
case Intrinsic::vector_reduce_or:
42-
return Instruction::Or;
43-
case Intrinsic::vector_reduce_xor:
44-
return Instruction::Xor;
45-
case Intrinsic::vector_reduce_smax:
46-
case Intrinsic::vector_reduce_smin:
47-
case Intrinsic::vector_reduce_umax:
48-
case Intrinsic::vector_reduce_umin:
49-
return Instruction::ICmp;
50-
case Intrinsic::vector_reduce_fmax:
51-
case Intrinsic::vector_reduce_fmin:
52-
return Instruction::FCmp;
53-
default:
54-
llvm_unreachable("Unexpected ID");
55-
}
56-
}
57-
58-
RecurKind getRK(Intrinsic::ID ID) {
59-
switch (ID) {
60-
case Intrinsic::vector_reduce_smax:
61-
return RecurKind::SMax;
62-
case Intrinsic::vector_reduce_smin:
63-
return RecurKind::SMin;
64-
case Intrinsic::vector_reduce_umax:
65-
return RecurKind::UMax;
66-
case Intrinsic::vector_reduce_umin:
67-
return RecurKind::UMin;
68-
case Intrinsic::vector_reduce_fmax:
69-
return RecurKind::FMax;
70-
case Intrinsic::vector_reduce_fmin:
71-
return RecurKind::FMin;
72-
default:
73-
return RecurKind::None;
74-
}
75-
}
76-
7729
bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
7830
bool Changed = false;
7931
SmallVector<IntrinsicInst *, 4> Worklist;
@@ -106,7 +58,7 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
10658
FastMathFlags FMF =
10759
isa<FPMathOperator>(II) ? II->getFastMathFlags() : FastMathFlags{};
10860
Intrinsic::ID ID = II->getIntrinsicID();
109-
RecurKind RK = getRK(ID);
61+
RecurKind RK = getMinMaxReductionRecurKind(ID);
11062

11163
Value *Rdx = nullptr;
11264
IRBuilder<> Builder(II);
@@ -120,16 +72,16 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
12072
// and it can't be handled by generating a shuffle sequence.
12173
Value *Acc = II->getArgOperand(0);
12274
Value *Vec = II->getArgOperand(1);
75+
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
12376
if (!FMF.allowReassoc())
124-
Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), RK);
77+
Rdx = getOrderedReduction(Builder, Acc, Vec, RdxOpcode, RK);
12578
else {
12679
if (!isPowerOf2_32(
12780
cast<FixedVectorType>(Vec->getType())->getNumElements()))
12881
continue;
129-
130-
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
131-
Rdx = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(ID),
132-
Acc, Rdx, "bin.rdx");
82+
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RK);
83+
Rdx = Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, Acc, Rdx,
84+
"bin.rdx");
13385
}
13486
break;
13587
}
@@ -159,8 +111,8 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
159111
}
160112
break;
161113
}
162-
163-
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
114+
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
115+
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RK);
164116
break;
165117
}
166118
case Intrinsic::vector_reduce_add:
@@ -174,8 +126,8 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
174126
if (!isPowerOf2_32(
175127
cast<FixedVectorType>(Vec->getType())->getNumElements()))
176128
continue;
177-
178-
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
129+
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
130+
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RK);
179131
break;
180132
}
181133
case Intrinsic::vector_reduce_fmax:
@@ -187,8 +139,8 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) {
187139
cast<FixedVectorType>(Vec->getType())->getNumElements()) ||
188140
!FMF.noNaNs())
189141
continue;
190-
191-
Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), RK);
142+
unsigned RdxOpcode = getArithmeticReductionInstruction(ID);
143+
Rdx = getShuffleReduction(Builder, Vec, RdxOpcode, RK);
192144
break;
193145
}
194146
}

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -917,6 +917,58 @@ bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop,
917917
return true;
918918
}
919919

920+
unsigned llvm::getArithmeticReductionInstruction(Intrinsic::ID RdxID) {
921+
switch (RdxID) {
922+
case Intrinsic::vector_reduce_fadd:
923+
return Instruction::FAdd;
924+
case Intrinsic::vector_reduce_fmul:
925+
return Instruction::FMul;
926+
case Intrinsic::vector_reduce_add:
927+
return Instruction::Add;
928+
case Intrinsic::vector_reduce_mul:
929+
return Instruction::Mul;
930+
case Intrinsic::vector_reduce_and:
931+
return Instruction::And;
932+
case Intrinsic::vector_reduce_or:
933+
return Instruction::Or;
934+
case Intrinsic::vector_reduce_xor:
935+
return Instruction::Xor;
936+
case Intrinsic::vector_reduce_smax:
937+
case Intrinsic::vector_reduce_smin:
938+
case Intrinsic::vector_reduce_umax:
939+
case Intrinsic::vector_reduce_umin:
940+
return Instruction::ICmp;
941+
case Intrinsic::vector_reduce_fmax:
942+
case Intrinsic::vector_reduce_fmin:
943+
return Instruction::FCmp;
944+
default:
945+
llvm_unreachable("Unexpected ID");
946+
}
947+
}
948+
949+
Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID) {
950+
switch (RdxID) {
951+
default:
952+
llvm_unreachable("Unknown min/max recurrence kind");
953+
case Intrinsic::vector_reduce_umin:
954+
return Intrinsic::umin;
955+
case Intrinsic::vector_reduce_umax:
956+
return Intrinsic::umax;
957+
case Intrinsic::vector_reduce_smin:
958+
return Intrinsic::smin;
959+
case Intrinsic::vector_reduce_smax:
960+
return Intrinsic::smax;
961+
case Intrinsic::vector_reduce_fmin:
962+
return Intrinsic::minnum;
963+
case Intrinsic::vector_reduce_fmax:
964+
return Intrinsic::maxnum;
965+
case Intrinsic::vector_reduce_fminimum:
966+
return Intrinsic::minimum;
967+
case Intrinsic::vector_reduce_fmaximum:
968+
return Intrinsic::maximum;
969+
}
970+
}
971+
920972
Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
921973
switch (RK) {
922974
default:
@@ -940,6 +992,25 @@ Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(RecurKind RK) {
940992
}
941993
}
942994

995+
RecurKind llvm::getMinMaxReductionRecurKind(Intrinsic::ID RdxID) {
996+
switch (RdxID) {
997+
case Intrinsic::vector_reduce_smax:
998+
return RecurKind::SMax;
999+
case Intrinsic::vector_reduce_smin:
1000+
return RecurKind::SMin;
1001+
case Intrinsic::vector_reduce_umax:
1002+
return RecurKind::UMax;
1003+
case Intrinsic::vector_reduce_umin:
1004+
return RecurKind::UMin;
1005+
case Intrinsic::vector_reduce_fmax:
1006+
return RecurKind::FMax;
1007+
case Intrinsic::vector_reduce_fmin:
1008+
return RecurKind::FMin;
1009+
default:
1010+
return RecurKind::None;
1011+
}
1012+
}
1013+
9431014
CmpInst::Predicate llvm::getMinMaxReductionPredicate(RecurKind RK) {
9441015
switch (RK) {
9451016
default:

0 commit comments

Comments
 (0)