Skip to content

AMDGPU: Directly handle all atomicrmw cases in SIISelLowering #102439

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 8, 2024
Merged

AMDGPU: Directly handle all atomicrmw cases in SIISelLowering #102439

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 0 additions & 28 deletions llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5999,34 +5999,6 @@ bool AMDGPUTargetLowering::isReassocProfitable(MachineRegisterInfo &MRI,
return MRI.hasOneNonDBGUse(N0); // FIXME: handle regbanks
}

TargetLowering::AtomicExpansionKind
AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
switch (RMW->getOperation()) {
case AtomicRMWInst::Nand:
case AtomicRMWInst::FAdd:
case AtomicRMWInst::FSub:
case AtomicRMWInst::FMax:
case AtomicRMWInst::FMin:
return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Xchg: {
const DataLayout &DL = RMW->getFunction()->getDataLayout();
unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
if (ValSize == 32 || ValSize == 64)
return AtomicExpansionKind::None;
return AtomicExpansionKind::CmpXChg;
}
default: {
if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
unsigned Size = IntTy->getBitWidth();
if (Size == 32 || Size == 64)
return AtomicExpansionKind::None;
}

return AtomicExpansionKind::CmpXChg;
}
}
}

/// Whether it is profitable to sink the operands of an
/// Instruction I to the basic block of I.
/// This helps using several modifiers (like abs and neg) more often.
Expand Down
2 changes: 0 additions & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,8 +388,6 @@ class AMDGPUTargetLowering : public TargetLowering {
return MVT::i32;
}

AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;

bool shouldSinkOperands(Instruction *I,
SmallVectorImpl<Use *> &Ops) const override;
};
Expand Down
23 changes: 21 additions & 2 deletions llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2175,14 +2175,33 @@ SDNode *R600TargetLowering::PostISelFolding(MachineSDNode *Node,
TargetLowering::AtomicExpansionKind
R600TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
switch (RMW->getOperation()) {
case AtomicRMWInst::Nand:
case AtomicRMWInst::FAdd:
case AtomicRMWInst::FSub:
case AtomicRMWInst::FMax:
case AtomicRMWInst::FMin:
return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
// FIXME: Cayman at least appears to have instructions for this, but the
// instruction defintions appear to be missing.
return AtomicExpansionKind::CmpXChg;
case AtomicRMWInst::Xchg: {
const DataLayout &DL = RMW->getFunction()->getDataLayout();
unsigned ValSize = DL.getTypeSizeInBits(RMW->getType());
if (ValSize == 32 || ValSize == 64)
return AtomicExpansionKind::None;
return AtomicExpansionKind::CmpXChg;
}
default:
break;
if (auto *IntTy = dyn_cast<IntegerType>(RMW->getType())) {
unsigned Size = IntTy->getBitWidth();
if (Size == 32 || Size == 64)
return AtomicExpansionKind::None;
}

return AtomicExpansionKind::CmpXChg;
}

return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
llvm_unreachable("covered atomicrmw op switch");
}
69 changes: 64 additions & 5 deletions llvm/lib/Target/AMDGPU/SIISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16114,6 +16114,39 @@ static bool isBFloat2(Type *Ty) {
return VT && VT->getNumElements() == 2 && VT->getElementType()->isBFloatTy();
}

/// \return true if atomicrmw integer ops work for the type.
static bool isAtomicRMWLegalIntTy(Type *Ty) {
if (auto *IT = dyn_cast<IntegerType>(Ty)) {
unsigned BW = IT->getBitWidth();
return BW == 32 || BW == 64;
}

return false;
}

/// \return true if this atomicrmw xchg type can be selected.
static bool isAtomicRMWLegalXChgTy(const AtomicRMWInst *RMW) {
Type *Ty = RMW->getType();
if (isAtomicRMWLegalIntTy(Ty))
return true;

if (PointerType *PT = dyn_cast<PointerType>(Ty)) {
const DataLayout &DL = RMW->getFunction()->getParent()->getDataLayout();
unsigned BW = DL.getPointerSizeInBits(PT->getAddressSpace());
return BW == 32 || BW == 64;
}

if (Ty->isFloatTy() || Ty->isDoubleTy())
return true;

if (FixedVectorType *VT = dyn_cast<FixedVectorType>(Ty)) {
return VT->getNumElements() == 2 &&
VT->getElementType()->getPrimitiveSizeInBits() == 16;
}

return false;
}

/// \returns true if it's valid to emit a native instruction for \p RMW, based
/// on the properties of the target memory.
static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
Expand Down Expand Up @@ -16142,6 +16175,14 @@ static bool globalMemoryFPAtomicIsLegal(const GCNSubtarget &Subtarget,
.getValueAsBool();
}

/// \return Action to perform on AtomicRMWInsts for integer operations.
static TargetLowering::AtomicExpansionKind
atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) {
return isAtomicRMWLegalIntTy(RMW->getType())
? TargetLowering::AtomicExpansionKind::None
: TargetLowering::AtomicExpansionKind::CmpXChg;
}

TargetLowering::AtomicExpansionKind
SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
unsigned AS = RMW->getPointerAddressSpace();
Expand All @@ -16161,7 +16202,22 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
SSID == SyncScope::System ||
SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");

switch (RMW->getOperation()) {
auto Op = RMW->getOperation();
switch (Op) {
case AtomicRMWInst::Xchg: {
// PCIe supports add and xchg for system atomics.
return isAtomicRMWLegalXChgTy(RMW)
? TargetLowering::AtomicExpansionKind::None
: TargetLowering::AtomicExpansionKind::CmpXChg;

// PCIe supports add and xchg for system atomics.
return atomicSupportedIfLegalIntType(RMW);
}
case AtomicRMWInst::Add:
case AtomicRMWInst::And:
case AtomicRMWInst::UIncWrap:
case AtomicRMWInst::UDecWrap:
return atomicSupportedIfLegalIntType(RMW);
case AtomicRMWInst::Sub:
case AtomicRMWInst::Or:
case AtomicRMWInst::Xor: {
Expand All @@ -16173,7 +16229,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
return AtomicExpansionKind::Expand;
}

break;
return atomicSupportedIfLegalIntType(RMW);
}
case AtomicRMWInst::FAdd: {
Type *Ty = RMW->getType();
Expand Down Expand Up @@ -16335,13 +16391,16 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
if (HasSystemScope)
return AtomicExpansionKind::CmpXChg;
}
break;

return atomicSupportedIfLegalIntType(RMW);
}
case AtomicRMWInst::Nand:
case AtomicRMWInst::FSub:
default:
break;
return AtomicExpansionKind::CmpXChg;
}

return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
llvm_unreachable("covered atomicrmw op switch");
}

TargetLowering::AtomicExpansionKind
Expand Down
Loading