Skip to content

Commit 9a976f8

Browse files
author
Mikhail Gudim
committed
Improve cost model
1 parent a569a5f commit 9a976f8

File tree

9 files changed

+167
-76
lines changed

9 files changed

+167
-76
lines changed

‎llvm/include/llvm/Transforms/Utils/LoopUtils.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,8 @@ constexpr Intrinsic::ID getReductionIntrinsicID(RecurKind RK);
365365

366366
/// Returns the arithmetic instruction opcode used when expanding a reduction.
367367
unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID);
368+
/// Returns the reduction intrinsic id corresponding to the binary operation.
369+
Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc);
368370

369371
/// Returns the min/max intrinsic used when expanding a min/max reduction.
370372
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID);

‎llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1528,9 +1528,6 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
15281528
if (Instruction *X = foldVectorBinop(I))
15291529
return X;
15301530

1531-
if (Instruction *X = foldBinopOfReductions(I))
1532-
return replaceInstUsesWith(I, X);
1533-
15341531
if (Instruction *Phi = foldBinopWithPhiOperands(I))
15351532
return Phi;
15361533

@@ -2390,8 +2387,19 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
23902387
}
23912388
}
23922389

2393-
if (Instruction *X = foldBinopOfReductions(I))
2394-
return replaceInstUsesWith(I, X);
2390+
auto m_AddRdx = [](Value *&Vec) {
2391+
return m_OneUse(m_Intrinsic<Intrinsic::vector_reduce_add>(m_Value(Vec)));
2392+
};
2393+
Value *V0, *V1;
2394+
if (match(Op0, m_AddRdx(V0)) && match(Op1, m_AddRdx(V1)) &&
2395+
V0->getType() == V1->getType()) {
2396+
// Difference of sums is sum of differences:
2397+
// add_rdx(V0) - add_rdx(V1) --> add_rdx(V0 - V1)
2398+
Value *Sub = Builder.CreateSub(V0, V1);
2399+
Value *Rdx = Builder.CreateIntrinsic(Intrinsic::vector_reduce_add,
2400+
{Sub->getType()}, {Sub});
2401+
return replaceInstUsesWith(I, Rdx);
2402+
}
23952403

23962404
if (Constant *C = dyn_cast<Constant>(Op0)) {
23972405
Value *X;

‎llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2380,9 +2380,6 @@ Instruction *InstCombinerImpl::visitAnd(BinaryOperator &I) {
23802380
if (Instruction *X = foldVectorBinop(I))
23812381
return X;
23822382

2383-
if (Instruction *X = foldBinopOfReductions(I))
2384-
return replaceInstUsesWith(I, X);
2385-
23862383
if (Instruction *Phi = foldBinopWithPhiOperands(I))
23872384
return Phi;
23882385

@@ -3563,9 +3560,6 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
35633560
if (Instruction *X = foldVectorBinop(I))
35643561
return X;
35653562

3566-
if (Instruction *X = foldBinopOfReductions(I))
3567-
return replaceInstUsesWith(I, X);
3568-
35693563
if (Instruction *Phi = foldBinopWithPhiOperands(I))
35703564
return Phi;
35713565

@@ -4677,9 +4671,6 @@ Instruction *InstCombinerImpl::visitXor(BinaryOperator &I) {
46774671
if (Instruction *X = foldVectorBinop(I))
46784672
return X;
46794673

4680-
if (Instruction *X = foldBinopOfReductions(I))
4681-
return replaceInstUsesWith(I, X);
4682-
46834674
if (Instruction *Phi = foldBinopWithPhiOperands(I))
46844675
return Phi;
46854676

‎llvm/lib/Transforms/InstCombine/InstCombineInternal.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -594,7 +594,6 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
594594

595595
/// Canonicalize the position of binops relative to shufflevector.
596596
Instruction *foldVectorBinop(BinaryOperator &Inst);
597-
Instruction *foldBinopOfReductions(BinaryOperator &Inst);
598597
Instruction *foldVectorSelect(SelectInst &Sel);
599598
Instruction *foldSelectShuffle(ShuffleVectorInst &Shuf);
600599

‎llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,9 +199,6 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
199199
if (Instruction *X = foldVectorBinop(I))
200200
return X;
201201

202-
if (Instruction *X = foldBinopOfReductions(I))
203-
return replaceInstUsesWith(I, X);
204-
205202
if (Instruction *Phi = foldBinopWithPhiOperands(I))
206203
return Phi;
207204

‎llvm/lib/Transforms/InstCombine/InstructionCombining.cpp

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -2317,63 +2317,6 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
23172317
return nullptr;
23182318
}
23192319

2320-
static Intrinsic::ID getReductionForBinop(Instruction::BinaryOps Opc) {
2321-
switch (Opc) {
2322-
default:
2323-
break;
2324-
case Instruction::Add:
2325-
return Intrinsic::vector_reduce_add;
2326-
case Instruction::Mul:
2327-
return Intrinsic::vector_reduce_mul;
2328-
case Instruction::And:
2329-
return Intrinsic::vector_reduce_and;
2330-
case Instruction::Or:
2331-
return Intrinsic::vector_reduce_or;
2332-
case Instruction::Xor:
2333-
return Intrinsic::vector_reduce_xor;
2334-
}
2335-
return Intrinsic::not_intrinsic;
2336-
}
2337-
2338-
Instruction *InstCombinerImpl::foldBinopOfReductions(BinaryOperator &Inst) {
2339-
Instruction::BinaryOps BinOpOpc = Inst.getOpcode();
2340-
Intrinsic::ID ReductionIID = getReductionForBinop(BinOpOpc);
2341-
if (BinOpOpc == Instruction::Sub)
2342-
ReductionIID = Intrinsic::vector_reduce_add;
2343-
if (ReductionIID == Intrinsic::not_intrinsic)
2344-
return nullptr;
2345-
2346-
auto checkIntrinsicAndGetItsArgument = [](Value *V,
2347-
Intrinsic::ID IID) -> Value * {
2348-
IntrinsicInst *II = dyn_cast<IntrinsicInst>(V);
2349-
if (!II)
2350-
return nullptr;
2351-
if (II->getIntrinsicID() == IID && II->hasOneUse())
2352-
return II->getArgOperand(0);
2353-
return nullptr;
2354-
};
2355-
2356-
Value *V0 = checkIntrinsicAndGetItsArgument(Inst.getOperand(0), ReductionIID);
2357-
if (!V0)
2358-
return nullptr;
2359-
Value *V1 = checkIntrinsicAndGetItsArgument(Inst.getOperand(1), ReductionIID);
2360-
if (!V1)
2361-
return nullptr;
2362-
2363-
Type *VTy = V0->getType();
2364-
if (V1->getType() != VTy)
2365-
return nullptr;
2366-
2367-
Value *VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
2368-
2369-
if (PossiblyDisjointInst *PDInst = dyn_cast<PossiblyDisjointInst>(&Inst))
2370-
if (auto *PDVectorBO = dyn_cast<PossiblyDisjointInst>(VectorBO))
2371-
PDVectorBO->setIsDisjoint(PDInst->isDisjoint());
2372-
2373-
Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
2374-
return Rdx;
2375-
}
2376-
23772320
/// Try to narrow the width of a binop if at least 1 operand is an extend of
23782321
/// of a value. This requires a potentially expensive known bits check to make
23792322
/// sure the narrow op does not overflow.

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -957,6 +957,7 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
957957
}
958958
}
959959

960+
// This is the inverse to getReductionForBinop
960961
unsigned llvm::getArithmeticReductionInstruction(Intrinsic::ID RdxID) {
961962
switch (RdxID) {
962963
case Intrinsic::vector_reduce_fadd:
@@ -986,6 +987,25 @@ unsigned llvm::getArithmeticReductionInstruction(Intrinsic::ID RdxID) {
986987
}
987988
}
988989

990+
// This is the inverse to getArithmeticReductionInstruction
991+
Intrinsic::ID llvm::getReductionForBinop(Instruction::BinaryOps Opc) {
992+
switch (Opc) {
993+
default:
994+
break;
995+
case Instruction::Add:
996+
return Intrinsic::vector_reduce_add;
997+
case Instruction::Mul:
998+
return Intrinsic::vector_reduce_mul;
999+
case Instruction::And:
1000+
return Intrinsic::vector_reduce_and;
1001+
case Instruction::Or:
1002+
return Intrinsic::vector_reduce_or;
1003+
case Instruction::Xor:
1004+
return Intrinsic::vector_reduce_xor;
1005+
}
1006+
return Intrinsic::not_intrinsic;
1007+
}
1008+
9891009
Intrinsic::ID llvm::getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID) {
9901010
switch (RdxID) {
9911011
default:

‎llvm/lib/Transforms/Vectorize/VectorCombine.cpp

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ class VectorCombine {
113113
bool scalarizeBinopOrCmp(Instruction &I);
114114
bool scalarizeVPIntrinsic(Instruction &I);
115115
bool foldExtractedCmps(Instruction &I);
116+
bool foldBinopOfReductions(Instruction &I);
116117
bool foldSingleElementStore(Instruction &I);
117118
bool scalarizeLoadExtract(Instruction &I);
118119
bool foldConcatOfBoolMasks(Instruction &I);
@@ -1182,6 +1183,135 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) {
11821183
return true;
11831184
}
11841185

1186+
static void analyzeCostOfVecReduction(const IntrinsicInst &II,
1187+
TTI::TargetCostKind CostKind,
1188+
const TargetTransformInfo &TTI,
1189+
InstructionCost &CostBeforeReduction,
1190+
InstructionCost &CostAfterReduction) {
1191+
using namespace llvm::PatternMatch;
1192+
Instruction *Op0, *Op1;
1193+
Instruction *RedOp = dyn_cast<Instruction>(II.getOperand(0));
1194+
VectorType *VecRedTy = cast<VectorType>(II.getOperand(0)->getType());
1195+
unsigned ReductionOpc =
1196+
getArithmeticReductionInstruction(II.getIntrinsicID());
1197+
if (RedOp && match(RedOp, m_ZExtOrSExt(m_Value()))) {
1198+
bool IsUnsigned = isa<ZExtInst>(RedOp);
1199+
VectorType *ExtType =
1200+
cast<VectorType>(RedOp->getOperand(0)->getType());
1201+
1202+
CostBeforeReduction =
1203+
TTI.getCastInstrCost(RedOp->getOpcode(), VecRedTy, ExtType,
1204+
TTI::CastContextHint::None, CostKind, RedOp);
1205+
CostAfterReduction =
1206+
TTI.getExtendedReductionCost(ReductionOpc, IsUnsigned, II.getType(),
1207+
ExtType, FastMathFlags(), CostKind);
1208+
return;
1209+
}
1210+
if (RedOp && II.getIntrinsicID() == Intrinsic::vector_reduce_add &&
1211+
match(RedOp,
1212+
m_ZExtOrSExt(m_Mul(m_Instruction(Op0), m_Instruction(Op1)))) &&
1213+
match(Op0, m_ZExtOrSExt(m_Value())) &&
1214+
Op0->getOpcode() == Op1->getOpcode() &&
1215+
Op0->getOperand(0)->getType() == Op1->getOperand(0)->getType() &&
1216+
(Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
1217+
// Matched reduce.add(ext(mul(ext(A), ext(B)))
1218+
bool IsUnsigned = isa<ZExtInst>(Op0);
1219+
VectorType *ExtType =
1220+
cast<VectorType>(Op0->getOperand(0)->getType());
1221+
VectorType *MulType = VectorType::get(Op0->getType(), VecRedTy);
1222+
1223+
InstructionCost ExtCost =
1224+
TTI.getCastInstrCost(Op0->getOpcode(), MulType, ExtType,
1225+
TTI::CastContextHint::None, CostKind, Op0);
1226+
InstructionCost MulCost =
1227+
TTI.getArithmeticInstrCost(Instruction::Mul, MulType, CostKind);
1228+
InstructionCost Ext2Cost =
1229+
TTI.getCastInstrCost(RedOp->getOpcode(), VecRedTy, MulType,
1230+
TTI::CastContextHint::None, CostKind, RedOp);
1231+
1232+
CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
1233+
CostAfterReduction =
1234+
TTI.getMulAccReductionCost(IsUnsigned, II.getType(), ExtType, CostKind);
1235+
return;
1236+
}
1237+
CostAfterReduction = TTI.getArithmeticReductionCost(ReductionOpc, VecRedTy,
1238+
std::nullopt, CostKind);
1239+
return;
1240+
}
1241+
1242+
bool VectorCombine::foldBinopOfReductions(Instruction &I) {
1243+
Instruction::BinaryOps BinOpOpc = cast<BinaryOperator>(&I)->getOpcode();
1244+
Intrinsic::ID ReductionIID = getReductionForBinop(BinOpOpc);
1245+
if (BinOpOpc == Instruction::Sub)
1246+
ReductionIID = Intrinsic::vector_reduce_add;
1247+
if (ReductionIID == Intrinsic::not_intrinsic)
1248+
return false;
1249+
1250+
auto checkIntrinsicAndGetItsArgument = [](Value *V,
1251+
Intrinsic::ID IID) -> Value * {
1252+
IntrinsicInst *II = dyn_cast<IntrinsicInst>(V);
1253+
if (!II)
1254+
return nullptr;
1255+
if (II->getIntrinsicID() == IID && II->hasOneUse())
1256+
return II->getArgOperand(0);
1257+
return nullptr;
1258+
};
1259+
1260+
Value *V0 = checkIntrinsicAndGetItsArgument(I.getOperand(0), ReductionIID);
1261+
if (!V0)
1262+
return false;
1263+
Value *V1 = checkIntrinsicAndGetItsArgument(I.getOperand(1), ReductionIID);
1264+
if (!V1)
1265+
return false;
1266+
1267+
VectorType *VTy = cast<VectorType>(V0->getType());
1268+
if (V1->getType() != VTy)
1269+
return false;
1270+
const IntrinsicInst &II0 = *cast<IntrinsicInst>(I.getOperand(0));
1271+
const IntrinsicInst &II1 = *cast<IntrinsicInst>(I.getOperand(1));
1272+
unsigned ReductionOpc =
1273+
getArithmeticReductionInstruction(II0.getIntrinsicID());
1274+
1275+
InstructionCost OldCost = 0;
1276+
InstructionCost NewCost = 0;
1277+
InstructionCost CostOfRedOperand0 = 0;
1278+
InstructionCost CostOfRed0 = 0;
1279+
InstructionCost CostOfRedOperand1 = 0;
1280+
InstructionCost CostOfRed1 = 0;
1281+
analyzeCostOfVecReduction(II0, CostKind, TTI, CostOfRedOperand0, CostOfRed0);
1282+
analyzeCostOfVecReduction(II1, CostKind, TTI, CostOfRedOperand1, CostOfRed1);
1283+
OldCost = CostOfRed0 + CostOfRed1 + TTI.getInstructionCost(&I, CostKind);
1284+
NewCost =
1285+
CostOfRedOperand0 + CostOfRedOperand1 +
1286+
TTI.getArithmeticInstrCost(BinOpOpc, VTy, CostKind) +
1287+
TTI.getArithmeticReductionCost(ReductionOpc, VTy, std::nullopt, CostKind);
1288+
// TODO: remove this
1289+
LLVM_DEBUG(
1290+
dbgs() << "CostOfRed0: " << CostOfRed0 << "\n";
1291+
dbgs() << "CostOfRed0: " << CostOfRed1 << "\n";
1292+
dbgs() << "Cost of scalar op: " << TTI.getInstructionCost(&I, CostKind) << "\n";
1293+
dbgs() << "CostOfRedOperand0: " << CostOfRedOperand0 << "\n";
1294+
dbgs() << "CostOfRedOperand1: " << CostOfRedOperand1 << "\n";
1295+
dbgs() << "Cost of vector op: " << TTI.getArithmeticInstrCost(BinOpOpc, VTy, CostKind) << "\n";
1296+
dbgs() << "Cost of final reduction: "<< TTI.getArithmeticReductionCost(ReductionOpc, VTy, std::nullopt, CostKind) << "\n";
1297+
dbgs() << "OldCost: " << OldCost << ", NewCost: " << NewCost << "\n";
1298+
);
1299+
if (NewCost >= OldCost || !NewCost.isValid())
1300+
return false;
1301+
1302+
LLVM_DEBUG(dbgs() << "Found two mergeable reductions: " << I
1303+
<< "\n OldCost: " << OldCost << " vs NewCost: " << NewCost
1304+
<< "\n");
1305+
Value *VectorBO = Builder.CreateBinOp(BinOpOpc, V0, V1);
1306+
if (PossiblyDisjointInst *PDInst = dyn_cast<PossiblyDisjointInst>(&I))
1307+
if (auto *PDVectorBO = dyn_cast<PossiblyDisjointInst>(VectorBO))
1308+
PDVectorBO->setIsDisjoint(PDInst->isDisjoint());
1309+
1310+
Instruction *Rdx = Builder.CreateIntrinsic(ReductionIID, {VTy}, {VectorBO});
1311+
replaceValue(I, *Rdx);
1312+
return true;
1313+
}
1314+
11851315
// Check if memory loc modified between two instrs in the same BB
11861316
static bool isMemModifiedBetween(BasicBlock::iterator Begin,
11871317
BasicBlock::iterator End,
@@ -3241,6 +3371,7 @@ bool VectorCombine::run() {
32413371
if (Instruction::isBinaryOp(Opcode)) {
32423372
MadeChange |= foldExtractExtract(I);
32433373
MadeChange |= foldExtractedCmps(I);
3374+
MadeChange |= foldBinopOfReductions(I);
32443375
}
32453376
break;
32463377
}

‎llvm/test/Transforms/VectorCombine/fold-binop-of-reductions.ll

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2-
; RUN: opt < %s -passes=instcombine -S | FileCheck %s
2+
; RUN: opt < %s -passes=vector-combine -S | FileCheck %s
33

44
define i32 @add_of_reduce_add(<16 x i32> %v0, <16 x i32> %v1) {
55
; CHECK-LABEL: define i32 @add_of_reduce_add(

0 commit comments

Comments
 (0)