Skip to content

[SelectionDAG] Pass LoadExtType when ATOMIC_LOAD is created. #136653

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 22, 2025
Merged

[SelectionDAG] Pass LoadExtType when ATOMIC_LOAD is created. #136653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0a12703
[SelectionDAG] Pass LoadExtType when ATOMIC_LOAD is created.
topperc Apr 22, 2025
1781b03
fixup! clang-format
topperc Apr 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions llvm/include/llvm/CodeGen/SelectionDAG.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1325,16 +1325,16 @@ class SelectionDAG {
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
SDValue Ptr, SDValue Val, MachineMemOperand *MMO);

/// Gets a node for an atomic op, produces result and chain and
/// takes 1 operand.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);

/// Gets a node for an atomic op, produces result and chain and takes N
/// operands.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTList, ArrayRef<SDValue> Ops,
MachineMemOperand *MMO);
MachineMemOperand *MMO,
ISD::LoadExtType ExtType = ISD::NON_EXTLOAD);

SDValue getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT MemVT,
EVT VT, SDValue Chain, SDValue Ptr,
MachineMemOperand *MMO);

/// Creates a MemIntrinsicNode that may produce a
/// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
Expand Down
12 changes: 5 additions & 7 deletions llvm/include/llvm/CodeGen/SelectionDAGNodes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1546,15 +1546,13 @@ class MemSDNode : public SDNode {
/// This is an SDNode representing atomic operations.
class AtomicSDNode : public MemSDNode {
public:
AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
EVT MemVT, MachineMemOperand *MMO)
: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
AtomicSDNode(unsigned Order, const DebugLoc &dl, unsigned Opc, SDVTList VTL,
EVT MemVT, MachineMemOperand *MMO, ISD::LoadExtType ETy)
: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||
MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
}

void setExtensionType(ISD::LoadExtType ETy) {
assert(getOpcode() == ISD::ATOMIC_LOAD && "Only used for atomic loads.");
assert((Opc == ISD::ATOMIC_LOAD || ETy == ISD::NON_EXTLOAD) &&
"Only atomic load uses ExtTy");
LoadSDNodeBits.ExtTy = ETy;
}

Expand Down
5 changes: 2 additions & 3 deletions llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13778,10 +13778,9 @@ static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG,

EVT OrigVT = ALoad->getValueType(0);
assert(OrigVT.getSizeInBits() < VT.getSizeInBits() && "VT should be wider.");
auto *NewALoad = cast<AtomicSDNode>(DAG.getAtomic(
ISD::ATOMIC_LOAD, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(),
auto *NewALoad = cast<AtomicSDNode>(DAG.getAtomicLoad(
ExtLoadType, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(),
ALoad->getBasePtr(), ALoad->getMemOperand()));
NewALoad->setExtensionType(ExtLoadType);
DAG.ReplaceAllUsesOfValueWith(
SDValue(ALoad, 0),
DAG.getNode(ISD::TRUNCATE, SDLoc(ALoad), OrigVT, SDValue(NewALoad, 0)));
Expand Down
39 changes: 18 additions & 21 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,30 +381,27 @@ SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {

SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(), ResVT,
N->getChain(), N->getBasePtr(),
N->getMemOperand());
if (N->getOpcode() == ISD::ATOMIC_LOAD) {
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
if (ETy == ISD::NON_EXTLOAD) {
switch (TLI.getExtendForAtomicOps()) {
case ISD::SIGN_EXTEND:
ETy = ISD::SEXTLOAD;
break;
case ISD::ZERO_EXTEND:
ETy = ISD::ZEXTLOAD;
break;
case ISD::ANY_EXTEND:
ETy = ISD::EXTLOAD;
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
ISD::LoadExtType ExtType = N->getExtensionType();
if (ExtType == ISD::NON_EXTLOAD) {
switch (TLI.getExtendForAtomicOps()) {
case ISD::SIGN_EXTEND:
ExtType = ISD::SEXTLOAD;
break;
case ISD::ZERO_EXTEND:
ExtType = ISD::ZEXTLOAD;
break;
case ISD::ANY_EXTEND:
ExtType = ISD::EXTLOAD;
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
cast<AtomicSDNode>(Res)->setExtensionType(ETy);
}

SDValue Res =
DAG.getAtomicLoad(ExtType, SDLoc(N), N->getMemoryVT(), ResVT,
N->getChain(), N->getBasePtr(), N->getMemOperand());

// Legalize the chain result - switch anything that used the old chain to
// use the new one.
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
Expand Down
19 changes: 9 additions & 10 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8992,12 +8992,13 @@ SDValue SelectionDAG::getAtomicMemset(SDValue Chain, const SDLoc &dl,

SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
SDVTList VTList, ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) {
MachineMemOperand *MMO,
ISD::LoadExtType ExtType) {
FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops);
ID.AddInteger(MemVT.getRawBits());
ID.AddInteger(getSyntheticNodeSubclassData<AtomicSDNode>(
Opcode, dl.getIROrder(), VTList, MemVT, MMO));
dl.getIROrder(), Opcode, VTList, MemVT, MMO, ExtType));
Copy link
Collaborator Author

@topperc topperc Apr 22, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to move Opcode later so that the variadic template version of getSyntheticNodeSubclassData would work. Previously we used the other signature that hardcodes that arguments as Opcode, Order, VTList, MemVT, and MMO. The extra ExtType made the hardcoded signature not match.

Changing the operand order seemed better than adding another signature with a hardcoded argument list.

ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
ID.AddInteger(MMO->getFlags());
void* IP = nullptr;
Expand All @@ -9006,8 +9007,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
return SDValue(E, 0);
}

auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, MemVT, MMO);
auto *N = newSDNode<AtomicSDNode>(dl.getIROrder(), dl.getDebugLoc(), Opcode,
VTList, MemVT, MMO, ExtType);
createOperands(N, Ops);

CSEMap.InsertNode(N, IP);
Expand Down Expand Up @@ -9053,14 +9054,12 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
}

SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
EVT VT, SDValue Chain, SDValue Ptr,
MachineMemOperand *MMO) {
assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");

SDValue SelectionDAG::getAtomicLoad(ISD::LoadExtType ExtType, const SDLoc &dl,
EVT MemVT, EVT VT, SDValue Chain,
SDValue Ptr, MachineMemOperand *MMO) {
SDVTList VTs = getVTList(VT, MVT::Other);
SDValue Ops[] = {Chain, Ptr};
return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
return getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, VTs, Ops, MMO, ExtType);
}

/// getMergeValues - Create a MERGE_VALUES node from the given operands.
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5157,8 +5157,8 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);

SDValue Ptr = getValue(I.getPointerOperand());
SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain,
Ptr, MMO);
SDValue L =
DAG.getAtomicLoad(ISD::NON_EXTLOAD, dl, MemVT, MemVT, InChain, Ptr, MMO);

SDValue OutChain = L.getValue(1);
if (MemVT != VT)
Expand Down
7 changes: 3 additions & 4 deletions llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6922,10 +6922,9 @@ SDValue SystemZTargetLowering::lowerLoadF16(SDValue Op,
SDValue NewLd;
if (auto *AtomicLd = dyn_cast<AtomicSDNode>(Op.getNode())) {
assert(EVT(RegVT) == AtomicLd->getMemoryVT() && "Unhandled f16 load");
NewLd = DAG.getAtomic(ISD::ATOMIC_LOAD, DL, MVT::i16, MVT::i64,
AtomicLd->getChain(), AtomicLd->getBasePtr(),
AtomicLd->getMemOperand());
cast<AtomicSDNode>(NewLd)->setExtensionType(ISD::EXTLOAD);
NewLd = DAG.getAtomicLoad(ISD::EXTLOAD, DL, MVT::i16, MVT::i64,
AtomicLd->getChain(), AtomicLd->getBasePtr(),
AtomicLd->getMemOperand());
} else {
LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
assert(EVT(RegVT) == Ld->getMemoryVT() && "Unhandled f16 load");
Expand Down
Loading