Skip to content

Commit fe3f8ad

Browse files
RKSimonRKSimon
committed
[X86] getIntrinsicInstrCost - begin generalizing BSWAP load/store-folding handling.
Move load/store folding 'free costs' inside the adjustTableCost helper so we can some additional intrinsics in the future. The plan is to do something similar for other costs callbacks as well (getArithmeticInstrCost etc.).
1 parent d1a4f0c commit fe3f8ad

File tree

1 file changed

+56
-73
lines changed

1 file changed

+56
-73
lines changed

‎llvm/lib/Target/X86/X86TargetTransformInfo.cpp

Lines changed: 56 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -4262,6 +4262,37 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
42624262
}
42634263

42644264
if (ISD != ISD::DELETED_NODE) {
4265+
auto adjustTableCost = [&](int ISD, unsigned Cost,
4266+
std::pair<InstructionCost, MVT> LT,
4267+
FastMathFlags FMF) -> InstructionCost {
4268+
InstructionCost LegalizationCost = LT.first;
4269+
MVT MTy = LT.second;
4270+
4271+
// If there are no NANs to deal with, then these are reduced to a
4272+
// single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
4273+
// assume is used in the non-fast case.
4274+
if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
4275+
if (FMF.noNaNs())
4276+
return LegalizationCost * 1;
4277+
}
4278+
4279+
// For cases where some ops can be folded into a load/store, assume free.
4280+
if (MTy.isScalarInteger()) {
4281+
if (ISD == ISD::BSWAP && ST->hasMOVBE() && ST->hasFastMOVBE()) {
4282+
if (const Instruction *II = ICA.getInst()) {
4283+
if (II->hasOneUse() && isa<StoreInst>(II->user_back()))
4284+
return TTI::TCC_Free;
4285+
if (auto *LI = dyn_cast<LoadInst>(II->getOperand(0))) {
4286+
if (LI->hasOneUse())
4287+
return TTI::TCC_Free;
4288+
}
4289+
}
4290+
}
4291+
}
4292+
4293+
return LegalizationCost * (int)Cost;
4294+
};
4295+
42654296
// Legalize the type.
42664297
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(OpTy);
42674298
MVT MTy = LT.second;
@@ -4280,180 +4311,132 @@ X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
42804311
if (ISD == ISD::FSQRT && CostKind == TTI::TCK_CodeSize)
42814312
return LT.first;
42824313

4283-
auto adjustTableCost = [](int ISD, unsigned Cost,
4284-
InstructionCost LegalizationCost,
4285-
FastMathFlags FMF) {
4286-
// If there are no NANs to deal with, then these are reduced to a
4287-
// single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
4288-
// assume is used in the non-fast case.
4289-
if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
4290-
if (FMF.noNaNs())
4291-
return LegalizationCost * 1;
4292-
}
4293-
return LegalizationCost * (int)Cost;
4294-
};
4295-
42964314
if (ST->useGLMDivSqrtCosts())
42974315
if (const auto *Entry = CostTableLookup(GLMCostTbl, ISD, MTy))
42984316
if (auto KindCost = Entry->Cost[CostKind])
4299-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4300-
ICA.getFlags());
4317+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43014318

43024319
if (ST->useSLMArithCosts())
43034320
if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
43044321
if (auto KindCost = Entry->Cost[CostKind])
4305-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4306-
ICA.getFlags());
4322+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43074323

43084324
if (ST->hasVBMI2())
43094325
if (const auto *Entry = CostTableLookup(AVX512VBMI2CostTbl, ISD, MTy))
43104326
if (auto KindCost = Entry->Cost[CostKind])
4311-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4312-
ICA.getFlags());
4327+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43134328

43144329
if (ST->hasBITALG())
43154330
if (const auto *Entry = CostTableLookup(AVX512BITALGCostTbl, ISD, MTy))
43164331
if (auto KindCost = Entry->Cost[CostKind])
4317-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4318-
ICA.getFlags());
4332+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43194333

43204334
if (ST->hasVPOPCNTDQ())
43214335
if (const auto *Entry = CostTableLookup(AVX512VPOPCNTDQCostTbl, ISD, MTy))
43224336
if (auto KindCost = Entry->Cost[CostKind])
4323-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4324-
ICA.getFlags());
4337+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43254338

43264339
if (ST->hasGFNI())
43274340
if (const auto *Entry = CostTableLookup(GFNICostTbl, ISD, MTy))
43284341
if (auto KindCost = Entry->Cost[CostKind])
4329-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4330-
ICA.getFlags());
4342+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43314343

43324344
if (ST->hasCDI())
43334345
if (const auto *Entry = CostTableLookup(AVX512CDCostTbl, ISD, MTy))
43344346
if (auto KindCost = Entry->Cost[CostKind])
4335-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4336-
ICA.getFlags());
4347+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43374348

43384349
if (ST->hasBWI())
43394350
if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
43404351
if (auto KindCost = Entry->Cost[CostKind])
4341-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4342-
ICA.getFlags());
4352+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43434353

43444354
if (ST->hasAVX512())
43454355
if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
43464356
if (auto KindCost = Entry->Cost[CostKind])
4347-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4348-
ICA.getFlags());
4357+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43494358

43504359
if (ST->hasXOP())
43514360
if (const auto *Entry = CostTableLookup(XOPCostTbl, ISD, MTy))
43524361
if (auto KindCost = Entry->Cost[CostKind])
4353-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4354-
ICA.getFlags());
4362+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43554363

43564364
if (ST->hasAVX2())
43574365
if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
43584366
if (auto KindCost = Entry->Cost[CostKind])
4359-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4360-
ICA.getFlags());
4367+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43614368

43624369
if (ST->hasAVX())
43634370
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
43644371
if (auto KindCost = Entry->Cost[CostKind])
4365-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4366-
ICA.getFlags());
4372+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43674373

43684374
if (ST->hasSSE42())
43694375
if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
43704376
if (auto KindCost = Entry->Cost[CostKind])
4371-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4372-
ICA.getFlags());
4377+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43734378

43744379
if (ST->hasSSE41())
43754380
if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
43764381
if (auto KindCost = Entry->Cost[CostKind])
4377-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4378-
ICA.getFlags());
4382+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43794383

43804384
if (ST->hasSSSE3())
43814385
if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy))
43824386
if (auto KindCost = Entry->Cost[CostKind])
4383-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4384-
ICA.getFlags());
4387+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43854388

43864389
if (ST->hasSSE2())
43874390
if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
43884391
if (auto KindCost = Entry->Cost[CostKind])
4389-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4390-
ICA.getFlags());
4392+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43914393

43924394
if (ST->hasSSE1())
43934395
if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
43944396
if (auto KindCost = Entry->Cost[CostKind])
4395-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4396-
ICA.getFlags());
4397+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
43974398

43984399
if (ST->hasBMI()) {
43994400
if (ST->is64Bit())
44004401
if (const auto *Entry = CostTableLookup(BMI64CostTbl, ISD, MTy))
44014402
if (auto KindCost = Entry->Cost[CostKind])
4402-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4403-
ICA.getFlags());
4403+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44044404

44054405
if (const auto *Entry = CostTableLookup(BMI32CostTbl, ISD, MTy))
44064406
if (auto KindCost = Entry->Cost[CostKind])
4407-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4408-
ICA.getFlags());
4407+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44094408
}
44104409

44114410
if (ST->hasLZCNT()) {
44124411
if (ST->is64Bit())
44134412
if (const auto *Entry = CostTableLookup(LZCNT64CostTbl, ISD, MTy))
44144413
if (auto KindCost = Entry->Cost[CostKind])
4415-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4416-
ICA.getFlags());
4414+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44174415

44184416
if (const auto *Entry = CostTableLookup(LZCNT32CostTbl, ISD, MTy))
44194417
if (auto KindCost = Entry->Cost[CostKind])
4420-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4421-
ICA.getFlags());
4418+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44224419
}
44234420

44244421
if (ST->hasPOPCNT()) {
44254422
if (ST->is64Bit())
44264423
if (const auto *Entry = CostTableLookup(POPCNT64CostTbl, ISD, MTy))
44274424
if (auto KindCost = Entry->Cost[CostKind])
4428-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4429-
ICA.getFlags());
4425+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44304426

44314427
if (const auto *Entry = CostTableLookup(POPCNT32CostTbl, ISD, MTy))
44324428
if (auto KindCost = Entry->Cost[CostKind])
4433-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4434-
ICA.getFlags());
4435-
}
4436-
4437-
if (ISD == ISD::BSWAP && ST->hasMOVBE() && ST->hasFastMOVBE()) {
4438-
if (const Instruction *II = ICA.getInst()) {
4439-
if (II->hasOneUse() && isa<StoreInst>(II->user_back()))
4440-
return TTI::TCC_Free;
4441-
if (auto *LI = dyn_cast<LoadInst>(II->getOperand(0))) {
4442-
if (LI->hasOneUse())
4443-
return TTI::TCC_Free;
4444-
}
4445-
}
4429+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44464430
}
44474431

44484432
if (ST->is64Bit())
44494433
if (const auto *Entry = CostTableLookup(X64CostTbl, ISD, MTy))
44504434
if (auto KindCost = Entry->Cost[CostKind])
4451-
return adjustTableCost(Entry->ISD, *KindCost, LT.first,
4452-
ICA.getFlags());
4435+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44534436

44544437
if (const auto *Entry = CostTableLookup(X86CostTbl, ISD, MTy))
44554438
if (auto KindCost = Entry->Cost[CostKind])
4456-
return adjustTableCost(Entry->ISD, *KindCost, LT.first, ICA.getFlags());
4439+
return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
44574440
}
44584441

44594442
return BaseT::getIntrinsicInstrCost(ICA, CostKind);

0 commit comments

Comments
 (0)