Skip to content

Commit 554859c

Browse files
krzysz00krzysz00
authored
[TTI] Make isLegalMasked{Load,Store} take an address space (#134006)
In order to facilitate targets that only support masked loads/stores on certain address spaces (AMDGPU will support them in an upcoming patch, but only for address space 7), add an AddressSpace parameter to isLegalMaskedLoad and isLegalMaskedStore
1 parent bb8a7a7 commit 554859c

14 files changed

+82
-48
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -791,9 +791,11 @@ class TargetTransformInfo {
791791
ScalarEvolution *SE) const;
792792

793793
/// Return true if the target supports masked store.
794-
bool isLegalMaskedStore(Type *DataType, Align Alignment) const;
794+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
795+
unsigned AddressSpace) const;
795796
/// Return true if the target supports masked load.
796-
bool isLegalMaskedLoad(Type *DataType, Align Alignment) const;
797+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
798+
unsigned AddressSpace) const;
797799

798800
/// Return true if the target supports nontemporal store.
799801
bool isLegalNTStore(Type *DataType, Align Alignment) const;
@@ -2015,8 +2017,10 @@ class TargetTransformInfo::Concept {
20152017
TargetLibraryInfo *LibInfo) = 0;
20162018
virtual AddressingModeKind
20172019
getPreferredAddressingMode(const Loop *L, ScalarEvolution *SE) const = 0;
2018-
virtual bool isLegalMaskedStore(Type *DataType, Align Alignment) = 0;
2019-
virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment) = 0;
2020+
virtual bool isLegalMaskedStore(Type *DataType, Align Alignment,
2021+
unsigned AddressSpace) = 0;
2022+
virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment,
2023+
unsigned AddressSpace) = 0;
20202024
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
20212025
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
20222026
virtual bool isLegalBroadcastLoad(Type *ElementTy,
@@ -2562,11 +2566,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
25622566
ScalarEvolution *SE) const override {
25632567
return Impl.getPreferredAddressingMode(L, SE);
25642568
}
2565-
bool isLegalMaskedStore(Type *DataType, Align Alignment) override {
2566-
return Impl.isLegalMaskedStore(DataType, Alignment);
2569+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
2570+
unsigned AddressSpace) override {
2571+
return Impl.isLegalMaskedStore(DataType, Alignment, AddressSpace);
25672572
}
2568-
bool isLegalMaskedLoad(Type *DataType, Align Alignment) override {
2569-
return Impl.isLegalMaskedLoad(DataType, Alignment);
2573+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
2574+
unsigned AddressSpace) override {
2575+
return Impl.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
25702576
}
25712577
bool isLegalNTStore(Type *DataType, Align Alignment) override {
25722578
return Impl.isLegalNTStore(DataType, Alignment);

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,11 +276,13 @@ class TargetTransformInfoImplBase {
276276
return TTI::AMK_None;
277277
}
278278

279-
bool isLegalMaskedStore(Type *DataType, Align Alignment) const {
279+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
280+
unsigned AddressSpace) const {
280281
return false;
281282
}
282283

283-
bool isLegalMaskedLoad(Type *DataType, Align Alignment) const {
284+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
285+
unsigned AddressSpace) const {
284286
return false;
285287
}
286288

‎llvm/lib/Analysis/TargetTransformInfo.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -462,14 +462,14 @@ TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
462462
return TTIImpl->getPreferredAddressingMode(L, SE);
463463
}
464464

465-
bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
466-
Align Alignment) const {
467-
return TTIImpl->isLegalMaskedStore(DataType, Alignment);
465+
bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, Align Alignment,
466+
unsigned AddressSpace) const {
467+
return TTIImpl->isLegalMaskedStore(DataType, Alignment, AddressSpace);
468468
}
469469

470-
bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
471-
Align Alignment) const {
472-
return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
470+
bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, Align Alignment,
471+
unsigned AddressSpace) const {
472+
return TTIImpl->isLegalMaskedLoad(DataType, Alignment, AddressSpace);
473473
}
474474

475475
bool TargetTransformInfo::isLegalNTStore(Type *DataType,

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -290,11 +290,13 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
290290
return isElementTypeLegalForScalableVector(DataType->getScalarType());
291291
}
292292

293-
bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
293+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
294+
unsigned /*AddressSpace*/) {
294295
return isLegalMaskedLoadStore(DataType, Alignment);
295296
}
296297

297-
bool isLegalMaskedStore(Type *DataType, Align Alignment) {
298+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
299+
unsigned /*AddressSpace*/) {
298300
return isLegalMaskedLoadStore(DataType, Alignment);
299301
}
300302

‎llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,8 @@ bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) {
11221122
return false;
11231123
}
11241124

1125-
bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
1125+
bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
1126+
unsigned /*AddressSpace*/) {
11261127
if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
11271128
return false;
11281129

@@ -1595,9 +1596,11 @@ ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
15951596
unsigned AddressSpace,
15961597
TTI::TargetCostKind CostKind) {
15971598
if (ST->hasMVEIntegerOps()) {
1598-
if (Opcode == Instruction::Load && isLegalMaskedLoad(Src, Alignment))
1599+
if (Opcode == Instruction::Load &&
1600+
isLegalMaskedLoad(Src, Alignment, AddressSpace))
15991601
return ST->getMVEVectorCostFactor(CostKind);
1600-
if (Opcode == Instruction::Store && isLegalMaskedStore(Src, Alignment))
1602+
if (Opcode == Instruction::Store &&
1603+
isLegalMaskedStore(Src, Alignment, AddressSpace))
16011604
return ST->getMVEVectorCostFactor(CostKind);
16021605
}
16031606
if (!isa<FixedVectorType>(Src))

‎llvm/lib/Target/ARM/ARMTargetTransformInfo.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
184184

185185
bool isProfitableLSRChainElement(Instruction *I);
186186

187-
bool isLegalMaskedLoad(Type *DataTy, Align Alignment);
187+
bool isLegalMaskedLoad(Type *DataTy, Align Alignment, unsigned AddressSpace);
188188

189-
bool isLegalMaskedStore(Type *DataTy, Align Alignment) {
190-
return isLegalMaskedLoad(DataTy, Alignment);
189+
bool isLegalMaskedStore(Type *DataTy, Align Alignment,
190+
unsigned AddressSpace) {
191+
return isLegalMaskedLoad(DataTy, Alignment, AddressSpace);
191192
}
192193

193194
bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {

‎llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -340,13 +340,15 @@ InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
340340
return 1;
341341
}
342342

343-
bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/) {
343+
bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/,
344+
unsigned /*AddressSpace*/) {
344345
// This function is called from scalarize-masked-mem-intrin, which runs
345346
// in pre-isel. Use ST directly instead of calling isHVXVectorType.
346347
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
347348
}
348349

349-
bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/) {
350+
bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/,
351+
unsigned /*AddressSpace*/) {
350352
// This function is called from scalarize-masked-mem-intrin, which runs
351353
// in pre-isel. Use ST directly instead of calling isHVXVectorType.
352354
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);

‎llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -157,8 +157,10 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
157157
return 1;
158158
}
159159

160-
bool isLegalMaskedStore(Type *DataType, Align Alignment);
161-
bool isLegalMaskedLoad(Type *DataType, Align Alignment);
160+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
161+
unsigned AddressSpace);
162+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
163+
unsigned AddressSpace);
162164

163165
/// @}
164166

‎llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,10 +262,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
262262
return TLI->isLegalElementTypeForRVV(ElemType);
263263
}
264264

265-
bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
265+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
266+
unsigned /*AddressSpace*/) {
266267
return isLegalMaskedLoadStore(DataType, Alignment);
267268
}
268-
bool isLegalMaskedStore(Type *DataType, Align Alignment) {
269+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
270+
unsigned /*AddressSpace*/) {
269271
return isLegalMaskedLoadStore(DataType, Alignment);
270272
}
271273

‎llvm/lib/Target/VE/VETargetTransformInfo.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,10 +133,12 @@ class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
133133
}
134134

135135
// Load & Store {
136-
bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) {
136+
bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment,
137+
unsigned /*AddressSpace*/) {
137138
return isVectorLaneType(*getLaneType(DataType));
138139
}
139-
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) {
140+
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment,
141+
unsigned /*AddressSpace*/) {
140142
return isVectorLaneType(*getLaneType(DataType));
141143
}
142144
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {

‎llvm/lib/Target/X86/X86TargetTransformInfo.cpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5368,8 +5368,8 @@ X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
53685368
unsigned NumElem = SrcVTy->getNumElements();
53695369
auto *MaskTy =
53705370
FixedVectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem);
5371-
if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment)) ||
5372-
(IsStore && !isLegalMaskedStore(SrcVTy, Alignment))) {
5371+
if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment, AddressSpace)) ||
5372+
(IsStore && !isLegalMaskedStore(SrcVTy, Alignment, AddressSpace))) {
53735373
// Scalarization
53745374
APInt DemandedElts = APInt::getAllOnes(NumElem);
53755375
InstructionCost MaskSplitCost = getScalarizationOverhead(
@@ -6253,7 +6253,8 @@ static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
62536253
((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
62546254
}
62556255

6256-
bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
6256+
bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
6257+
unsigned AddressSpace) {
62576258
Type *ScalarTy = DataTy->getScalarType();
62586259

62596260
// The backend can't handle a single element vector w/o CFCMOV.
@@ -6265,7 +6266,8 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
62656266
return isLegalMaskedLoadStore(ScalarTy, ST);
62666267
}
62676268

6268-
bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment) {
6269+
bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
6270+
unsigned AddressSpace) {
62696271
Type *ScalarTy = DataTy->getScalarType();
62706272

62716273
// The backend can't handle a single element vector w/o CFCMOV.

‎llvm/lib/Target/X86/X86TargetTransformInfo.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,10 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
262262
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
263263
const TargetTransformInfo::LSRCost &C2);
264264
bool canMacroFuseCmp();
265-
bool isLegalMaskedLoad(Type *DataType, Align Alignment);
266-
bool isLegalMaskedStore(Type *DataType, Align Alignment);
265+
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
266+
unsigned AddressSpace);
267+
bool isLegalMaskedStore(Type *DataType, Align Alignment,
268+
unsigned AddressSpace);
267269
bool isLegalNTLoad(Type *DataType, Align Alignment);
268270
bool isLegalNTStore(Type *DataType, Align Alignment);
269271
bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const;

‎llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,14 +1098,18 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
10981098
// Scalarize unsupported vector masked load
10991099
if (TTI.isLegalMaskedLoad(
11001100
CI->getType(),
1101-
cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
1101+
cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue(),
1102+
cast<PointerType>(CI->getArgOperand(0)->getType())
1103+
->getAddressSpace()))
11021104
return false;
11031105
scalarizeMaskedLoad(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
11041106
return true;
11051107
case Intrinsic::masked_store:
11061108
if (TTI.isLegalMaskedStore(
11071109
CI->getArgOperand(0)->getType(),
1108-
cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
1110+
cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue(),
1111+
cast<PointerType>(CI->getArgOperand(1)->getType())
1112+
->getAddressSpace()))
11091113
return false;
11101114
scalarizeMaskedStore(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
11111115
return true;

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1255,16 +1255,18 @@ class LoopVectorizationCostModel {
12551255

12561256
/// Returns true if the target machine supports masked store operation
12571257
/// for the given \p DataType and kind of access to \p Ptr.
1258-
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
1258+
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
1259+
unsigned AddressSpace) const {
12591260
return Legal->isConsecutivePtr(DataType, Ptr) &&
1260-
TTI.isLegalMaskedStore(DataType, Alignment);
1261+
TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace);
12611262
}
12621263

12631264
/// Returns true if the target machine supports masked load operation
12641265
/// for the given \p DataType and kind of access to \p Ptr.
1265-
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
1266+
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
1267+
unsigned AddressSpace) const {
12661268
return Legal->isConsecutivePtr(DataType, Ptr) &&
1267-
TTI.isLegalMaskedLoad(DataType, Alignment);
1269+
TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
12681270
}
12691271

12701272
/// Returns true if the target machine can represent \p V as a masked gather
@@ -3220,13 +3222,14 @@ bool LoopVectorizationCostModel::isScalarWithPredication(
32203222
case Instruction::Store: {
32213223
auto *Ptr = getLoadStorePointerOperand(I);
32223224
auto *Ty = getLoadStoreType(I);
3225+
unsigned AS = getLoadStoreAddressSpace(I);
32233226
Type *VTy = Ty;
32243227
if (VF.isVector())
32253228
VTy = VectorType::get(Ty, VF);
32263229
const Align Alignment = getLoadStoreAlignment(I);
3227-
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
3230+
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
32283231
TTI.isLegalMaskedGather(VTy, Alignment))
3229-
: !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
3232+
: !(isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
32303233
TTI.isLegalMaskedScatter(VTy, Alignment));
32313234
}
32323235
case Instruction::UDiv:
@@ -3427,8 +3430,9 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
34273430

34283431
auto *Ty = getLoadStoreType(I);
34293432
const Align Alignment = getLoadStoreAlignment(I);
3430-
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
3431-
: TTI.isLegalMaskedStore(Ty, Alignment);
3433+
unsigned AS = getLoadStoreAddressSpace(I);
3434+
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
3435+
: TTI.isLegalMaskedStore(Ty, Alignment, AS);
34323436
}
34333437

34343438
bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(

0 commit comments

Comments
 (0)