Skip to content

Commit a551272

Browse files
committed
[CostModel][X86] getGSVectorCost - add cost kind support
Don't just assume gather/scatter non-throughput costs are 1 - latency and sizelatency (#uops) costs will be high, and codesize (#instructions) needs to account splitting.
1 parent 11a6799 commit a551272

File tree

4 files changed

+167
-166
lines changed

4 files changed

+167
-166
lines changed

llvm/lib/Target/X86/X86TargetTransformInfo.cpp

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5828,14 +5828,17 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
58285828
Alignment, AddressSpace);
58295829
}
58305830

5831+
// If we didn't split, this will be a single gather/scatter instruction.
5832+
if (CostKind == TTI::TCK_CodeSize)
5833+
return 1;
5834+
58315835
// The gather / scatter cost is given by Intel architects. It is a rough
58325836
// number since we are looking at one instruction in a time.
5833-
const int GSOverhead = (Opcode == Instruction::Load)
5834-
? getGatherOverhead()
5835-
: getScatterOverhead();
5837+
const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
5838+
: getScatterOverhead();
58365839
return GSOverhead + VF * getMemoryOpCost(Opcode, SrcVTy->getScalarType(),
58375840
MaybeAlign(Alignment), AddressSpace,
5838-
TTI::TCK_RecipThroughput);
5841+
CostKind);
58395842
}
58405843

58415844
/// Return the cost of full scalarization of gather / scatter operation.
@@ -5892,19 +5895,17 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost(
58925895
unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask,
58935896
Align Alignment, TTI::TargetCostKind CostKind,
58945897
const Instruction *I = nullptr) {
5895-
if (CostKind != TTI::TCK_RecipThroughput) {
5896-
if ((Opcode == Instruction::Load &&
5897-
isLegalMaskedGather(SrcVTy, Align(Alignment)) &&
5898-
!forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
5899-
Align(Alignment))) ||
5900-
(Opcode == Instruction::Store &&
5901-
isLegalMaskedScatter(SrcVTy, Align(Alignment)) &&
5902-
!forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
5903-
Align(Alignment))))
5904-
return 1;
5898+
if (CostKind != TTI::TCK_RecipThroughput &&
5899+
((Opcode == Instruction::Load &&
5900+
(!isLegalMaskedGather(SrcVTy, Align(Alignment)) ||
5901+
forceScalarizeMaskedGather(cast<VectorType>(SrcVTy),
5902+
Align(Alignment)))) ||
5903+
(Opcode == Instruction::Store &&
5904+
(!isLegalMaskedScatter(SrcVTy, Align(Alignment)) ||
5905+
forceScalarizeMaskedScatter(cast<VectorType>(SrcVTy),
5906+
Align(Alignment))))))
59055907
return BaseT::getGatherScatterOpCost(Opcode, SrcVTy, Ptr, VariableMask,
59065908
Alignment, CostKind, I);
5907-
}
59085909

59095910
assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
59105911
PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());

0 commit comments

Comments
 (0)