Skip to content

[TTI] Make isLegalMasked{Load,Store} take an address space #134006

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 2, 2025
Merged

[TTI] Make isLegalMasked{Load,Store} take an address space #134006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
378d0ee
[TTI] Make isLegalMasked{Load,Store} take an address space
krzysz00 Apr 1, 2025
7f6ba4c
Oh, whoops, forgot about RISCV
krzysz00 Apr 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -791,9 +791,11 @@ class TargetTransformInfo {
ScalarEvolution *SE) const;

/// Return true if the target supports masked store.
bool isLegalMaskedStore(Type *DataType, Align Alignment) const;
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace) const;
/// Return true if the target supports masked load.
bool isLegalMaskedLoad(Type *DataType, Align Alignment) const;
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) const;

/// Return true if the target supports nontemporal store.
bool isLegalNTStore(Type *DataType, Align Alignment) const;
Expand Down Expand Up @@ -2015,8 +2017,10 @@ class TargetTransformInfo::Concept {
TargetLibraryInfo *LibInfo) = 0;
virtual AddressingModeKind
getPreferredAddressingMode(const Loop *L, ScalarEvolution *SE) const = 0;
virtual bool isLegalMaskedStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace) = 0;
virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) = 0;
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalBroadcastLoad(Type *ElementTy,
Expand Down Expand Up @@ -2562,11 +2566,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
ScalarEvolution *SE) const override {
return Impl.getPreferredAddressingMode(L, SE);
}
bool isLegalMaskedStore(Type *DataType, Align Alignment) override {
return Impl.isLegalMaskedStore(DataType, Alignment);
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace) override {
return Impl.isLegalMaskedStore(DataType, Alignment, AddressSpace);
}
bool isLegalMaskedLoad(Type *DataType, Align Alignment) override {
return Impl.isLegalMaskedLoad(DataType, Alignment);
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) override {
return Impl.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
}
bool isLegalNTStore(Type *DataType, Align Alignment) override {
return Impl.isLegalNTStore(DataType, Alignment);
Expand Down
6 changes: 4 additions & 2 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,11 +276,13 @@ class TargetTransformInfoImplBase {
return TTI::AMK_None;
}

bool isLegalMaskedStore(Type *DataType, Align Alignment) const {
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace) const {
return false;
}

bool isLegalMaskedLoad(Type *DataType, Align Alignment) const {
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) const {
return false;
}

Expand Down
12 changes: 6 additions & 6 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,14 +462,14 @@ TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
return TTIImpl->getPreferredAddressingMode(L, SE);
}

bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
Align Alignment) const {
return TTIImpl->isLegalMaskedStore(DataType, Alignment);
bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace) const {
return TTIImpl->isLegalMaskedStore(DataType, Alignment, AddressSpace);
}

bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
Align Alignment) const {
return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace) const {
return TTIImpl->isLegalMaskedLoad(DataType, Alignment, AddressSpace);
}

bool TargetTransformInfo::isLegalNTStore(Type *DataType,
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,11 +290,13 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
return isElementTypeLegalForScalableVector(DataType->getScalarType());
}

bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned /*AddressSpace*/) {
return isLegalMaskedLoadStore(DataType, Alignment);
}

bool isLegalMaskedStore(Type *DataType, Align Alignment) {
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned /*AddressSpace*/) {
return isLegalMaskedLoadStore(DataType, Alignment);
}

Expand Down
9 changes: 6 additions & 3 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1122,7 +1122,8 @@ bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) {
return false;
}

bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
unsigned /*AddressSpace*/) {
if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
return false;

Expand Down Expand Up @@ -1595,9 +1596,11 @@ ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind) {
if (ST->hasMVEIntegerOps()) {
if (Opcode == Instruction::Load && isLegalMaskedLoad(Src, Alignment))
if (Opcode == Instruction::Load &&
isLegalMaskedLoad(Src, Alignment, AddressSpace))
return ST->getMVEVectorCostFactor(CostKind);
if (Opcode == Instruction::Store && isLegalMaskedStore(Src, Alignment))
if (Opcode == Instruction::Store &&
isLegalMaskedStore(Src, Alignment, AddressSpace))
return ST->getMVEVectorCostFactor(CostKind);
}
if (!isa<FixedVectorType>(Src))
Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,10 +184,11 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {

bool isProfitableLSRChainElement(Instruction *I);

bool isLegalMaskedLoad(Type *DataTy, Align Alignment);
bool isLegalMaskedLoad(Type *DataTy, Align Alignment, unsigned AddressSpace);

bool isLegalMaskedStore(Type *DataTy, Align Alignment) {
return isLegalMaskedLoad(DataTy, Alignment);
bool isLegalMaskedStore(Type *DataTy, Align Alignment,
unsigned AddressSpace) {
return isLegalMaskedLoad(DataTy, Alignment, AddressSpace);
}

bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment) {
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,13 +340,15 @@ InstructionCost HexagonTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
return 1;
}

bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/) {
bool HexagonTTIImpl::isLegalMaskedStore(Type *DataType, Align /*Alignment*/,
unsigned /*AddressSpace*/) {
// This function is called from scalarize-masked-mem-intrin, which runs
// in pre-isel. Use ST directly instead of calling isHVXVectorType.
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
}

bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/) {
bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/,
unsigned /*AddressSpace*/) {
// This function is called from scalarize-masked-mem-intrin, which runs
// in pre-isel. Use ST directly instead of calling isHVXVectorType.
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,10 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
return 1;
}

bool isLegalMaskedStore(Type *DataType, Align Alignment);
bool isLegalMaskedLoad(Type *DataType, Align Alignment);
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace);
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace);

/// @}

Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,10 +262,12 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
return TLI->isLegalElementTypeForRVV(ElemType);
}

bool isLegalMaskedLoad(Type *DataType, Align Alignment) {
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned /*AddressSpace*/) {
return isLegalMaskedLoadStore(DataType, Alignment);
}
bool isLegalMaskedStore(Type *DataType, Align Alignment) {
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned /*AddressSpace*/) {
return isLegalMaskedLoadStore(DataType, Alignment);
}

Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/VE/VETargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,12 @@ class VETTIImpl : public BasicTTIImplBase<VETTIImpl> {
}

// Load & Store {
bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment) {
bool isLegalMaskedLoad(Type *DataType, MaybeAlign Alignment,
unsigned /*AddressSpace*/) {
return isVectorLaneType(*getLaneType(DataType));
}
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment) {
bool isLegalMaskedStore(Type *DataType, MaybeAlign Alignment,
unsigned /*AddressSpace*/) {
return isVectorLaneType(*getLaneType(DataType));
}
bool isLegalMaskedGather(Type *DataType, MaybeAlign Alignment) {
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5368,8 +5368,8 @@ X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment,
unsigned NumElem = SrcVTy->getNumElements();
auto *MaskTy =
FixedVectorType::get(Type::getInt8Ty(SrcVTy->getContext()), NumElem);
if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment)) ||
(IsStore && !isLegalMaskedStore(SrcVTy, Alignment))) {
if ((IsLoad && !isLegalMaskedLoad(SrcVTy, Alignment, AddressSpace)) ||
(IsStore && !isLegalMaskedStore(SrcVTy, Alignment, AddressSpace))) {
// Scalarization
APInt DemandedElts = APInt::getAllOnes(NumElem);
InstructionCost MaskSplitCost = getScalarizationOverhead(
Expand Down Expand Up @@ -6253,7 +6253,8 @@ static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
}

bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
unsigned AddressSpace) {
Type *ScalarTy = DataTy->getScalarType();

// The backend can't handle a single element vector w/o CFCMOV.
Expand All @@ -6265,7 +6266,8 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) {
return isLegalMaskedLoadStore(ScalarTy, ST);
}

bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment) {
bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
unsigned AddressSpace) {
Type *ScalarTy = DataTy->getScalarType();

// The backend can't handle a single element vector w/o CFCMOV.
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/X86/X86TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,8 +262,10 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
bool isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
const TargetTransformInfo::LSRCost &C2);
bool canMacroFuseCmp();
bool isLegalMaskedLoad(Type *DataType, Align Alignment);
bool isLegalMaskedStore(Type *DataType, Align Alignment);
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
unsigned AddressSpace);
bool isLegalMaskedStore(Type *DataType, Align Alignment,
unsigned AddressSpace);
bool isLegalNTLoad(Type *DataType, Align Alignment);
bool isLegalNTStore(Type *DataType, Align Alignment);
bool isLegalBroadcastLoad(Type *ElementTy, ElementCount NumElements) const;
Expand Down
8 changes: 6 additions & 2 deletions llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1098,14 +1098,18 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
// Scalarize unsupported vector masked load
if (TTI.isLegalMaskedLoad(
CI->getType(),
cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue(),
cast<PointerType>(CI->getArgOperand(0)->getType())
->getAddressSpace()))
return false;
scalarizeMaskedLoad(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
return true;
case Intrinsic::masked_store:
if (TTI.isLegalMaskedStore(
CI->getArgOperand(0)->getType(),
cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue(),
cast<PointerType>(CI->getArgOperand(1)->getType())
->getAddressSpace()))
return false;
scalarizeMaskedStore(DL, HasBranchDivergence, CI, DTU, ModifiedDT);
return true;
Expand Down
20 changes: 12 additions & 8 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,16 +1255,18 @@ class LoopVectorizationCostModel {

/// Returns true if the target machine supports masked store operation
/// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment) const {
bool isLegalMaskedStore(Type *DataType, Value *Ptr, Align Alignment,
unsigned AddressSpace) const {
return Legal->isConsecutivePtr(DataType, Ptr) &&
TTI.isLegalMaskedStore(DataType, Alignment);
TTI.isLegalMaskedStore(DataType, Alignment, AddressSpace);
}

/// Returns true if the target machine supports masked load operation
/// for the given \p DataType and kind of access to \p Ptr.
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment) const {
bool isLegalMaskedLoad(Type *DataType, Value *Ptr, Align Alignment,
unsigned AddressSpace) const {
return Legal->isConsecutivePtr(DataType, Ptr) &&
TTI.isLegalMaskedLoad(DataType, Alignment);
TTI.isLegalMaskedLoad(DataType, Alignment, AddressSpace);
}

/// Returns true if the target machine can represent \p V as a masked gather
Expand Down Expand Up @@ -3220,13 +3222,14 @@ bool LoopVectorizationCostModel::isScalarWithPredication(
case Instruction::Store: {
auto *Ptr = getLoadStorePointerOperand(I);
auto *Ty = getLoadStoreType(I);
unsigned AS = getLoadStoreAddressSpace(I);
Type *VTy = Ty;
if (VF.isVector())
VTy = VectorType::get(Ty, VF);
const Align Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment, AS) ||
TTI.isLegalMaskedGather(VTy, Alignment))
: !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
: !(isLegalMaskedStore(Ty, Ptr, Alignment, AS) ||
TTI.isLegalMaskedScatter(VTy, Alignment));
}
case Instruction::UDiv:
Expand Down Expand Up @@ -3427,8 +3430,9 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(

auto *Ty = getLoadStoreType(I);
const Align Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
: TTI.isLegalMaskedStore(Ty, Alignment);
unsigned AS = getLoadStoreAddressSpace(I);
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment, AS)
: TTI.isLegalMaskedStore(Ty, Alignment, AS);
}

bool LoopVectorizationCostModel::memoryInstructionCanBeWidened(
Expand Down
Loading