Skip to content

Commit 5096501

Browse files
alexey-bataevalexey-bataev
committed
[SLP][TTI][X86]Add addsub pattern cost estimation. (#76461)
SLP/TTI do not know about the cost estimation for addsub pattern, supported by X86. Previously the support for pattern detection was added (seeTTI::isLegalAltInstr), but the cost still did not estimated properly.
1 parent 7a1a476 commit 5096501

File tree

8 files changed

+109
-30
lines changed

8 files changed

+109
-30
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1243,6 +1243,18 @@ class TargetTransformInfo {
12431243
ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
12441244
const Instruction *CxtI = nullptr) const;
12451245

1246+
/// Returns the cost estimation for alternating opcode pattern that can be
1247+
/// lowered to a single instruction on the target. In X86 this is for the
1248+
/// addsub instruction which corrsponds to a Shuffle + Fadd + FSub pattern in
1249+
/// IR. This function expects two opcodes: \p Opcode1 and \p Opcode2 being
1250+
/// selected by \p OpcodeMask. The mask contains one bit per lane and is a `0`
1251+
/// when \p Opcode0 is selected and `1` when Opcode1 is selected.
1252+
/// \p VecTy is the vector type of the instruction to be generated.
1253+
InstructionCost getAltInstrCost(
1254+
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
1255+
const SmallBitVector &OpcodeMask,
1256+
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;
1257+
12461258
/// \return The cost of a shuffle instruction of kind Kind and of type Tp.
12471259
/// The exact mask may be passed as Mask, or else the array will be empty.
12481260
/// The index and subtype parameters are used by the subvector insertion and
@@ -1944,6 +1956,10 @@ class TargetTransformInfo::Concept {
19441956
unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
19451957
OperandValueInfo Opd1Info, OperandValueInfo Opd2Info,
19461958
ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr) = 0;
1959+
virtual InstructionCost getAltInstrCost(
1960+
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
1961+
const SmallBitVector &OpcodeMask,
1962+
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const = 0;
19471963

19481964
virtual InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
19491965
ArrayRef<int> Mask,
@@ -2555,6 +2571,12 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
25552571
return Impl.getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info, Opd2Info,
25562572
Args, CxtI);
25572573
}
2574+
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
2575+
unsigned Opcode1,
2576+
const SmallBitVector &OpcodeMask,
2577+
TTI::TargetCostKind CostKind) const override {
2578+
return Impl.getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
2579+
}
25582580

25592581
InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
25602582
ArrayRef<int> Mask,

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,13 @@ class TargetTransformInfoImplBase {
554554
return 1;
555555
}
556556

557+
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
558+
unsigned Opcode1,
559+
const SmallBitVector &OpcodeMask,
560+
TTI::TargetCostKind CostKind) const {
561+
return InstructionCost::getInvalid();
562+
}
563+
557564
InstructionCost
558565
getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask,
559566
TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,

‎llvm/lib/Analysis/TargetTransformInfo.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,15 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
862862
return Cost;
863863
}
864864

865+
InstructionCost TargetTransformInfo::getAltInstrCost(
866+
VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
867+
const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
868+
InstructionCost Cost =
869+
TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
870+
assert(Cost >= 0 && "TTI should not produce negative costs!");
871+
return Cost;
872+
}
873+
865874
InstructionCost TargetTransformInfo::getShuffleCost(
866875
ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask,
867876
TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,

‎llvm/lib/Target/X86/X86TargetTransformInfo.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1459,6 +1459,15 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
14591459
Args, CxtI);
14601460
}
14611461

1462+
InstructionCost
1463+
X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
1464+
unsigned Opcode1, const SmallBitVector &OpcodeMask,
1465+
TTI::TargetCostKind CostKind) const {
1466+
if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask))
1467+
return TTI::TCC_Basic;
1468+
return InstructionCost::getInvalid();
1469+
}
1470+
14621471
InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
14631472
VectorType *BaseTp,
14641473
ArrayRef<int> Mask,

‎llvm/lib/Target/X86/X86TargetTransformInfo.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,11 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
140140
TTI::OperandValueInfo Op2Info = {TTI::OK_AnyValue, TTI::OP_None},
141141
ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
142142
const Instruction *CxtI = nullptr);
143+
InstructionCost getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
144+
unsigned Opcode1,
145+
const SmallBitVector &OpcodeMask,
146+
TTI::TargetCostKind CostKind) const;
147+
143148
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
144149
ArrayRef<int> Mask,
145150
TTI::TargetCostKind CostKind, int Index,

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8384,6 +8384,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
83848384
(void)E;
83858385
return TTI->getInstructionCost(VI, CostKind);
83868386
};
8387+
// FIXME: Workaround for syntax error reported by MSVC buildbots.
8388+
TargetTransformInfo &TTIRef = *TTI;
83878389
// Need to clear CommonCost since the final shuffle cost is included into
83888390
// vector cost.
83898391
auto GetVectorCost = [&](InstructionCost) {
@@ -8398,14 +8400,15 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
83988400
// No need to add new vector costs here since we're going to reuse
83998401
// same main/alternate vector ops, just do different shuffling.
84008402
} else if (Instruction::isBinaryOp(E->getOpcode())) {
8401-
VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind);
8403+
VecCost =
8404+
TTIRef.getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind);
84028405
VecCost +=
8403-
TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy, CostKind);
8406+
TTIRef.getArithmeticInstrCost(E->getAltOpcode(), VecTy, CostKind);
84048407
} else if (auto *CI0 = dyn_cast<CmpInst>(VL0)) {
84058408
auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), VL.size());
8406-
VecCost = TTI->getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy,
8407-
CI0->getPredicate(), CostKind, VL0);
8408-
VecCost += TTI->getCmpSelInstrCost(
8409+
VecCost = TTIRef.getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy,
8410+
CI0->getPredicate(), CostKind, VL0);
8411+
VecCost += TTIRef.getCmpSelInstrCost(
84098412
E->getOpcode(), VecTy, MaskTy,
84108413
cast<CmpInst>(E->getAltOp())->getPredicate(), CostKind,
84118414
E->getAltOp());
@@ -8414,10 +8417,11 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
84148417
Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType();
84158418
auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size());
84168419
auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size());
8417-
VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty,
8418-
TTI::CastContextHint::None, CostKind);
8419-
VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
8420-
TTI::CastContextHint::None, CostKind);
8420+
VecCost = TTIRef.getCastInstrCost(E->getOpcode(), VecTy, Src0Ty,
8421+
TTI::CastContextHint::None, CostKind);
8422+
VecCost +=
8423+
TTIRef.getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
8424+
TTI::CastContextHint::None, CostKind);
84218425
}
84228426
SmallVector<int> Mask;
84238427
E->buildAltOpShuffleMask(
@@ -8426,8 +8430,27 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
84268430
return I->getOpcode() == E->getAltOpcode();
84278431
},
84288432
Mask);
8429-
VecCost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
8430-
FinalVecTy, Mask);
8433+
VecCost += TTIRef.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc,
8434+
FinalVecTy, Mask);
8435+
// Patterns like [fadd,fsub] can be combined into a single instruction
8436+
// in x86. Reordering them into [fsub,fadd] blocks this pattern. So we
8437+
// need to take into account their order when looking for the most used
8438+
// order.
8439+
unsigned Opcode0 = E->getOpcode();
8440+
unsigned Opcode1 = E->getAltOpcode();
8441+
// The opcode mask selects between the two opcodes.
8442+
SmallBitVector OpcodeMask(E->Scalars.size(), false);
8443+
for (unsigned Lane : seq<unsigned>(0, E->Scalars.size()))
8444+
if (cast<Instruction>(E->Scalars[Lane])->getOpcode() == Opcode1)
8445+
OpcodeMask.set(Lane);
8446+
// If this pattern is supported by the target then we consider the
8447+
// order.
8448+
if (TTIRef.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask)) {
8449+
InstructionCost AltVecCost = TTIRef.getAltInstrCost(
8450+
VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
8451+
return AltVecCost < VecCost ? AltVecCost : VecCost;
8452+
}
8453+
// TODO: Check the reverse order too.
84318454
return VecCost;
84328455
};
84338456
return GetCostDiff(GetScalarCost, GetVectorCost);

‎llvm/test/Transforms/SLPVectorizer/X86/supernode.ll

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -103,21 +103,23 @@ define void @test_supernode_addsub_alt(ptr %Aarray, ptr %Barray, ptr %Carray, pt
103103
; ENABLED-LABEL: @test_supernode_addsub_alt(
104104
; ENABLED-NEXT: entry:
105105
; ENABLED-NEXT: [[IDXA1:%.*]] = getelementptr inbounds double, ptr [[AARRAY:%.*]], i64 1
106-
; ENABLED-NEXT: [[IDXB1:%.*]] = getelementptr inbounds double, ptr [[BARRAY:%.*]], i64 1
107106
; ENABLED-NEXT: [[IDXC1:%.*]] = getelementptr inbounds double, ptr [[CARRAY:%.*]], i64 1
108-
; ENABLED-NEXT: [[IDXS1:%.*]] = getelementptr inbounds double, ptr [[SARRAY:%.*]], i64 1
109107
; ENABLED-NEXT: [[A0:%.*]] = load double, ptr [[AARRAY]], align 8
110108
; ENABLED-NEXT: [[A1:%.*]] = load double, ptr [[IDXA1]], align 8
111-
; ENABLED-NEXT: [[B0:%.*]] = load double, ptr [[BARRAY]], align 8
112-
; ENABLED-NEXT: [[B1:%.*]] = load double, ptr [[IDXB1]], align 8
113109
; ENABLED-NEXT: [[C0:%.*]] = load double, ptr [[CARRAY]], align 8
114110
; ENABLED-NEXT: [[C1:%.*]] = load double, ptr [[IDXC1]], align 8
115-
; ENABLED-NEXT: [[SUBA0B0:%.*]] = fsub fast double [[A0]], [[B0]]
116-
; ENABLED-NEXT: [[ADDB1C1:%.*]] = fadd fast double [[B1]], [[C1]]
117-
; ENABLED-NEXT: [[SUB0:%.*]] = fsub fast double [[SUBA0B0]], [[C0]]
118-
; ENABLED-NEXT: [[ADD1:%.*]] = fadd fast double [[ADDB1C1]], [[A1]]
119-
; ENABLED-NEXT: store double [[SUB0]], ptr [[SARRAY]], align 8
120-
; ENABLED-NEXT: store double [[ADD1]], ptr [[IDXS1]], align 8
111+
; ENABLED-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[BARRAY:%.*]], align 8
112+
; ENABLED-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[A0]], i32 0
113+
; ENABLED-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[C1]], i32 1
114+
; ENABLED-NEXT: [[TMP3:%.*]] = fsub fast <2 x double> [[TMP2]], [[TMP0]]
115+
; ENABLED-NEXT: [[TMP4:%.*]] = fadd fast <2 x double> [[TMP2]], [[TMP0]]
116+
; ENABLED-NEXT: [[TMP5:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP4]], <2 x i32> <i32 0, i32 3>
117+
; ENABLED-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[C0]], i32 0
118+
; ENABLED-NEXT: [[TMP7:%.*]] = insertelement <2 x double> [[TMP6]], double [[A1]], i32 1
119+
; ENABLED-NEXT: [[TMP8:%.*]] = fsub fast <2 x double> [[TMP5]], [[TMP7]]
120+
; ENABLED-NEXT: [[TMP9:%.*]] = fadd fast <2 x double> [[TMP5]], [[TMP7]]
121+
; ENABLED-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP9]], <2 x i32> <i32 0, i32 3>
122+
; ENABLED-NEXT: store <2 x double> [[TMP10]], ptr [[SARRAY:%.*]], align 8
121123
; ENABLED-NEXT: ret void
122124
;
123125
entry:

‎llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,24 @@ define void @foo() {
1212
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[TMP0]], float [[CONV]], i32 1
1313
; CHECK-NEXT: br label [[BB2:%.*]]
1414
; CHECK: bb2:
15-
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x float> [ [[TMP1]], [[BB1]] ], [ [[TMP10:%.*]], [[BB3:%.*]] ]
15+
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x float> [ [[TMP1]], [[BB1]] ], [ [[TMP14:%.*]], [[BB3:%.*]] ]
1616
; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr undef, align 8
1717
; CHECK-NEXT: br i1 undef, label [[BB3]], label [[BB4:%.*]]
1818
; CHECK: bb4:
1919
; CHECK-NEXT: [[TMP4:%.*]] = fpext <4 x float> [[TMP2]] to <4 x double>
2020
; CHECK-NEXT: [[CONV2:%.*]] = uitofp i16 undef to double
21-
; CHECK-NEXT: [[ADD1:%.*]] = fadd double [[TMP3]], [[CONV2]]
22-
; CHECK-NEXT: [[SUB1:%.*]] = fsub double undef, undef
23-
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> <double poison, double poison, double undef, double undef>, double [[SUB1]], i32 0
24-
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[ADD1]], i32 1
25-
; CHECK-NEXT: [[TMP7:%.*]] = fcmp ogt <4 x double> [[TMP6]], [[TMP4]]
26-
; CHECK-NEXT: [[TMP8:%.*]] = fptrunc <4 x double> [[TMP6]] to <4 x float>
27-
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP7]], <4 x float> [[TMP2]], <4 x float> [[TMP8]]
21+
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> <double undef, double poison>, double [[TMP3]], i32 1
22+
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> <double undef, double poison>, double [[CONV2]], i32 1
23+
; CHECK-NEXT: [[TMP7:%.*]] = fsub <2 x double> [[TMP5]], [[TMP6]]
24+
; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x double> [[TMP5]], [[TMP6]]
25+
; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP8]], <2 x i32> <i32 0, i32 3>
26+
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
27+
; CHECK-NEXT: [[TMP11:%.*]] = fcmp ogt <4 x double> [[TMP10]], [[TMP4]]
28+
; CHECK-NEXT: [[TMP12:%.*]] = fptrunc <4 x double> [[TMP10]] to <4 x float>
29+
; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[TMP2]], <4 x float> [[TMP12]]
2830
; CHECK-NEXT: br label [[BB3]]
2931
; CHECK: bb3:
30-
; CHECK-NEXT: [[TMP10]] = phi <4 x float> [ [[TMP9]], [[BB4]] ], [ [[TMP2]], [[BB2]] ]
32+
; CHECK-NEXT: [[TMP14]] = phi <4 x float> [ [[TMP13]], [[BB4]] ], [ [[TMP2]], [[BB2]] ]
3133
; CHECK-NEXT: br label [[BB2]]
3234
;
3335
entry:

0 commit comments

Comments
 (0)