Skip to content

Commit 8c9332a

Browse files
JonPsson1chencha3
JonPsson1
authored and
chencha3
committed
[SystemZ] Don't lower ATOMIC_LOAD/STORE to LOAD/STORE (llvm#75879)
- Instead of lowering float/double ISD::ATOMIC_LOAD / ISD::ATOMIC_STORE nodes to regular LOAD/STORE nodes, make them legal and select those nodes properly instead. This avoids exposing them to the DAGCombiner. - AtomicExpand pass no longer casts float/double atomic load/stores to integer (FP128 is still casted).
1 parent 1bc02b3 commit 8c9332a

22 files changed

+1293
-318
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,7 @@ BEGIN_TWO_BYTE_PACK()
558558

559559
class LoadSDNodeBitfields {
560560
friend class LoadSDNode;
561+
friend class AtomicSDNode;
561562
friend class VPLoadSDNode;
562563
friend class VPStridedLoadSDNode;
563564
friend class MaskedLoadSDNode;
@@ -1475,6 +1476,16 @@ class AtomicSDNode : public MemSDNode {
14751476
MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
14761477
}
14771478

1479+
void setExtensionType(ISD::LoadExtType ETy) {
1480+
assert(getOpcode() == ISD::ATOMIC_LOAD && "Only used for atomic loads.");
1481+
LoadSDNodeBits.ExtTy = ETy;
1482+
}
1483+
1484+
ISD::LoadExtType getExtensionType() const {
1485+
assert(getOpcode() == ISD::ATOMIC_LOAD && "Only used for atomic loads.");
1486+
return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
1487+
}
1488+
14781489
const SDValue &getBasePtr() const {
14791490
return getOpcode() == ISD::ATOMIC_STORE ? getOperand(2) : getOperand(1);
14801491
}

‎llvm/include/llvm/Target/TargetSelectionDAG.td

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ def SDTAtomicStore : SDTypeProfile<0, 2, [
318318
SDTCisInt<0>, SDTCisPtrTy<1>
319319
]>;
320320
def SDTAtomicLoad : SDTypeProfile<1, 1, [
321-
SDTCisInt<0>, SDTCisPtrTy<1>
321+
SDTCisPtrTy<1>
322322
]>;
323323

324324
class SDCallSeqStart<list<SDTypeConstraint> constraints> :

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,26 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
349349
N->getMemoryVT(), ResVT,
350350
N->getChain(), N->getBasePtr(),
351351
N->getMemOperand());
352+
if (N->getOpcode() == ISD::ATOMIC_LOAD) {
353+
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
354+
if (ETy == ISD::NON_EXTLOAD) {
355+
switch (TLI.getExtendForAtomicOps()) {
356+
case ISD::SIGN_EXTEND:
357+
ETy = ISD::SEXTLOAD;
358+
break;
359+
case ISD::ZERO_EXTEND:
360+
ETy = ISD::ZEXTLOAD;
361+
break;
362+
case ISD::ANY_EXTEND:
363+
ETy = ISD::EXTLOAD;
364+
break;
365+
default:
366+
llvm_unreachable("Invalid atomic op extension");
367+
}
368+
}
369+
cast<AtomicSDNode>(Res)->setExtensionType(ETy);
370+
}
371+
352372
// Legalize the chain result - switch anything that used the old chain to
353373
// use the new one.
354374
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4070,6 +4070,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
40704070
if (Op.getResNo() == 0) {
40714071
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
40724072
Known.Zero.setBitsFrom(MemBits);
4073+
else if (Op->getOpcode() == ISD::ATOMIC_LOAD &&
4074+
cast<AtomicSDNode>(Op)->getExtensionType() == ISD::ZEXTLOAD)
4075+
Known.Zero.setBitsFrom(MemBits);
40734076
}
40744077
break;
40754078
}
@@ -4875,6 +4878,13 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
48754878
return VTBits - Tmp + 1;
48764879
if (TLI->getExtendForAtomicOps() == ISD::ZERO_EXTEND)
48774880
return VTBits - Tmp;
4881+
if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
4882+
ISD::LoadExtType ETy = cast<AtomicSDNode>(Op)->getExtensionType();
4883+
if (ETy == ISD::SEXTLOAD)
4884+
return VTBits - Tmp + 1;
4885+
if (ETy == ISD::ZEXTLOAD)
4886+
return VTBits - Tmp;
4887+
}
48784888
}
48794889
break;
48804890
}

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -841,6 +841,18 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
841841
} else if (const MemSDNode *M = dyn_cast<MemSDNode>(this)) {
842842
OS << "<";
843843
printMemOperand(OS, *M->getMemOperand(), G);
844+
if (auto *A = dyn_cast<AtomicSDNode>(M))
845+
if (A->getOpcode() == ISD::ATOMIC_LOAD) {
846+
bool doExt = true;
847+
switch (A->getExtensionType()) {
848+
default: doExt = false; break;
849+
case ISD::EXTLOAD: OS << ", anyext"; break;
850+
case ISD::SEXTLOAD: OS << ", sext"; break;
851+
case ISD::ZEXTLOAD: OS << ", zext"; break;
852+
}
853+
if (doExt)
854+
OS << " from " << A->getMemoryVT();
855+
}
844856
OS << ">";
845857
} else if (const BlockAddressSDNode *BA =
846858
dyn_cast<BlockAddressSDNode>(this)) {

‎llvm/lib/Target/PowerPC/PPCInstrInfo.td

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5045,12 +5045,12 @@ defm : TrapExtendedMnemonic<"lng", 6>;
50455045
defm : TrapExtendedMnemonic<"u", 31>;
50465046

50475047
// Atomic loads
5048-
def : Pat<(atomic_load_8 DForm:$src), (LBZ memri:$src)>;
5049-
def : Pat<(atomic_load_16 DForm:$src), (LHZ memri:$src)>;
5050-
def : Pat<(atomic_load_32 DForm:$src), (LWZ memri:$src)>;
5051-
def : Pat<(atomic_load_8 XForm:$src), (LBZX memrr:$src)>;
5052-
def : Pat<(atomic_load_16 XForm:$src), (LHZX memrr:$src)>;
5053-
def : Pat<(atomic_load_32 XForm:$src), (LWZX memrr:$src)>;
5048+
def : Pat<(i32 (atomic_load_8 DForm:$src)), (LBZ memri:$src)>;
5049+
def : Pat<(i32 (atomic_load_16 DForm:$src)), (LHZ memri:$src)>;
5050+
def : Pat<(i32 (atomic_load_32 DForm:$src)), (LWZ memri:$src)>;
5051+
def : Pat<(i32 (atomic_load_8 XForm:$src)), (LBZX memrr:$src)>;
5052+
def : Pat<(i32 (atomic_load_16 XForm:$src)), (LHZX memrr:$src)>;
5053+
def : Pat<(i32 (atomic_load_32 XForm:$src)), (LWZX memrr:$src)>;
50545054

50555055
// Atomic stores
50565056
def : Pat<(atomic_store_8 i32:$val, DForm:$ptr), (STB gprc:$val, memri:$ptr)>;

‎llvm/lib/Target/PowerPC/PPCInstrP10.td

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1289,13 +1289,13 @@ let Predicates = [PCRelativeMemops] in {
12891289
(PSTXVpc $XS, $ga, 0)>;
12901290

12911291
// Atomic Load
1292-
def : Pat<(atomic_load_8 (PPCmatpcreladdr PCRelForm:$ga)),
1292+
def : Pat<(i32 (atomic_load_8 (PPCmatpcreladdr PCRelForm:$ga))),
12931293
(PLBZpc $ga, 0)>;
1294-
def : Pat<(atomic_load_16 (PPCmatpcreladdr PCRelForm:$ga)),
1294+
def : Pat<(i32 (atomic_load_16 (PPCmatpcreladdr PCRelForm:$ga))),
12951295
(PLHZpc $ga, 0)>;
1296-
def : Pat<(atomic_load_32 (PPCmatpcreladdr PCRelForm:$ga)),
1296+
def : Pat<(i32 (atomic_load_32 (PPCmatpcreladdr PCRelForm:$ga))),
12971297
(PLWZpc $ga, 0)>;
1298-
def : Pat<(atomic_load_64 (PPCmatpcreladdr PCRelForm:$ga)),
1298+
def : Pat<(i64 (atomic_load_64 (PPCmatpcreladdr PCRelForm:$ga))),
12991299
(PLDpc $ga, 0)>;
13001300

13011301
// Atomic Store
@@ -2347,10 +2347,10 @@ let Predicates = [PrefixInstrs] in {
23472347
def : Pat<(store f64:$FRS, PDForm:$dst), (PSTFD $FRS, memri34:$dst)>;
23482348

23492349
// Atomic Load
2350-
def : Pat<(atomic_load_8 PDForm:$src), (PLBZ memri34:$src)>;
2351-
def : Pat<(atomic_load_16 PDForm:$src), (PLHZ memri34:$src)>;
2352-
def : Pat<(atomic_load_32 PDForm:$src), (PLWZ memri34:$src)>;
2353-
def : Pat<(atomic_load_64 PDForm:$src), (PLD memri34:$src)>;
2350+
def : Pat<(i32 (atomic_load_8 PDForm:$src)), (PLBZ memri34:$src)>;
2351+
def : Pat<(i32 (atomic_load_16 PDForm:$src)), (PLHZ memri34:$src)>;
2352+
def : Pat<(i32 (atomic_load_32 PDForm:$src)), (PLWZ memri34:$src)>;
2353+
def : Pat<(i64 (atomic_load_64 PDForm:$src)), (PLD memri34:$src)>;
23542354

23552355
// Atomic Store
23562356
def : Pat<(atomic_store_8 i32:$RS, PDForm:$dst), (PSTB $RS, memri34:$dst)>;

‎llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,9 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
344344
// requirements for a PC-relative access.
345345
bool storeLoadIsAligned(SDNode *N) const;
346346

347+
// Return the load extension type of a load or atomic load.
348+
ISD::LoadExtType getLoadExtType(SDNode *N) const;
349+
347350
// Try to expand a boolean SELECT_CCMASK using an IPM sequence.
348351
SDValue expandSelectBoolean(SDNode *Node);
349352

@@ -1507,15 +1510,17 @@ bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N,
15071510

15081511
bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
15091512

1510-
auto *MemAccess = cast<LSBaseSDNode>(N);
1513+
auto *MemAccess = cast<MemSDNode>(N);
1514+
auto *LdSt = dyn_cast<LSBaseSDNode>(MemAccess);
15111515
TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize();
15121516
SDValue BasePtr = MemAccess->getBasePtr();
15131517
MachineMemOperand *MMO = MemAccess->getMemOperand();
15141518
assert(MMO && "Expected a memory operand.");
15151519

15161520
// The memory access must have a proper alignment and no index register.
1521+
// Only load and store nodes have the offset operand (atomic loads do not).
15171522
if (MemAccess->getAlign().value() < StoreSize ||
1518-
!MemAccess->getOffset().isUndef())
1523+
(LdSt && !LdSt->getOffset().isUndef()))
15191524
return false;
15201525

15211526
// The MMO must not have an unaligned offset.
@@ -1545,6 +1550,17 @@ bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
15451550
return true;
15461551
}
15471552

1553+
ISD::LoadExtType SystemZDAGToDAGISel::getLoadExtType(SDNode *N) const {
1554+
ISD::LoadExtType ETy;
1555+
if (auto *L = dyn_cast<LoadSDNode>(N))
1556+
ETy = L->getExtensionType();
1557+
else if (auto *AL = dyn_cast<AtomicSDNode>(N))
1558+
ETy = AL->getExtensionType();
1559+
else
1560+
llvm_unreachable("Unkown load node type.");
1561+
return ETy;
1562+
}
1563+
15481564
void SystemZDAGToDAGISel::Select(SDNode *Node) {
15491565
// If we have a custom node, we already have selected!
15501566
if (Node->isMachineOpcode()) {
@@ -1742,6 +1758,26 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) {
17421758
}
17431759
break;
17441760
}
1761+
1762+
case ISD::ATOMIC_STORE: {
1763+
auto *AtomOp = cast<AtomicSDNode>(Node);
1764+
// Replace the atomic_store with a regular store and select it. This is
1765+
// ok since we know all store instructions <= 8 bytes are atomic, and the
1766+
// 16 byte case is already handled during lowering.
1767+
StoreSDNode *St = cast<StoreSDNode>(CurDAG->getTruncStore(
1768+
AtomOp->getChain(), SDLoc(AtomOp), AtomOp->getVal(),
1769+
AtomOp->getBasePtr(), AtomOp->getMemoryVT(), AtomOp->getMemOperand()));
1770+
assert(St->getMemOperand()->isAtomic() && "Broken MMO.");
1771+
SDNode *Chain = St;
1772+
// We have to enforce sequential consistency by performing a
1773+
// serialization operation after the store.
1774+
if (AtomOp->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
1775+
Chain = CurDAG->getMachineNode(SystemZ::Serialize, SDLoc(AtomOp),
1776+
MVT::Other, SDValue(Chain, 0));
1777+
ReplaceNode(Node, Chain);
1778+
SelectCode(St);
1779+
return;
1780+
}
17451781
}
17461782

17471783
SelectCode(Node);

‎llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

Lines changed: 62 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -194,11 +194,6 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
194194
setOperationAction(ISD::UADDO_CARRY, VT, Custom);
195195
setOperationAction(ISD::USUBO_CARRY, VT, Custom);
196196

197-
// Lower ATOMIC_LOAD and ATOMIC_STORE into normal volatile loads and
198-
// stores, putting a serialization instruction after the stores.
199-
setOperationAction(ISD::ATOMIC_LOAD, VT, Custom);
200-
setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
201-
202197
// Lower ATOMIC_LOAD_SUB into ATOMIC_LOAD_ADD if LAA and LAAG are
203198
// available, or if the operand is constant.
204199
setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
@@ -920,6 +915,22 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const
920915
return false;
921916
}
922917

918+
TargetLowering::AtomicExpansionKind
919+
SystemZTargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
920+
// Lower fp128 the same way as i128.
921+
if (LI->getType()->isFP128Ty())
922+
return AtomicExpansionKind::CastToInteger;
923+
return AtomicExpansionKind::None;
924+
}
925+
926+
TargetLowering::AtomicExpansionKind
927+
SystemZTargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const {
928+
// Lower fp128 the same way as i128.
929+
if (SI->getValueOperand()->getType()->isFP128Ty())
930+
return AtomicExpansionKind::CastToInteger;
931+
return AtomicExpansionKind::None;
932+
}
933+
923934
TargetLowering::AtomicExpansionKind
924935
SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
925936
// Don't expand subword operations as they require special treatment.
@@ -4503,40 +4514,14 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
45034514
return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
45044515
}
45054516

4506-
// Op is an atomic load. Lower it into a normal volatile load.
4507-
SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
4508-
SelectionDAG &DAG) const {
4509-
auto *Node = cast<AtomicSDNode>(Op.getNode());
4510-
if (Node->getMemoryVT() == MVT::i128) {
4511-
// Use same code to handle both legal and non-legal i128 types.
4512-
SmallVector<SDValue, 2> Results;
4513-
LowerOperationWrapper(Node, Results, DAG);
4514-
return DAG.getMergeValues(Results, SDLoc(Op));
4515-
}
4516-
return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
4517-
Node->getChain(), Node->getBasePtr(),
4518-
Node->getMemoryVT(), Node->getMemOperand());
4519-
}
4520-
4521-
// Op is an atomic store. Lower it into a normal volatile store.
4522-
SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
4523-
SelectionDAG &DAG) const {
4517+
SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
4518+
SelectionDAG &DAG) const {
45244519
auto *Node = cast<AtomicSDNode>(Op.getNode());
4525-
if (Node->getMemoryVT() == MVT::i128) {
4526-
// Use same code to handle both legal and non-legal i128 types.
4527-
SmallVector<SDValue, 1> Results;
4528-
LowerOperationWrapper(Node, Results, DAG);
4529-
return DAG.getMergeValues(Results, SDLoc(Op));
4530-
}
4531-
SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
4532-
Node->getBasePtr(), Node->getMemoryVT(),
4533-
Node->getMemOperand());
4534-
// We have to enforce sequential consistency by performing a
4535-
// serialization operation after the store.
4536-
if (Node->getSuccessOrdering() == AtomicOrdering::SequentiallyConsistent)
4537-
Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
4538-
MVT::Other, Chain), 0);
4539-
return Chain;
4520+
assert(Node->getMemoryVT() == MVT::i128 && "Only custom lowering i128.");
4521+
// Use same code to handle both legal and non-legal i128 types.
4522+
SmallVector<SDValue, 2> Results;
4523+
LowerOperationWrapper(Node, Results, DAG);
4524+
return DAG.getMergeValues(Results, SDLoc(Op));
45404525
}
45414526

45424527
// Prepare for a Compare And Swap for a subword operation. This needs to be
@@ -5662,6 +5647,9 @@ static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
56625647
bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
56635648
if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())
56645649
return true;
5650+
if (auto *AL = dyn_cast<AtomicSDNode>(Op))
5651+
if (AL->getOpcode() == ISD::ATOMIC_LOAD)
5652+
return true;
56655653
if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
56665654
return true;
56675655
return false;
@@ -6138,9 +6126,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
61386126
case ISD::ATOMIC_SWAP:
61396127
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
61406128
case ISD::ATOMIC_STORE:
6141-
return lowerATOMIC_STORE(Op, DAG);
61426129
case ISD::ATOMIC_LOAD:
6143-
return lowerATOMIC_LOAD(Op, DAG);
6130+
return lowerATOMIC_LDST_I128(Op, DAG);
61446131
case ISD::ATOMIC_LOAD_ADD:
61456132
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
61466133
case ISD::ATOMIC_LOAD_SUB:
@@ -6587,6 +6574,27 @@ SDValue SystemZTargetLowering::combineTruncateExtract(
65876574
return SDValue();
65886575
}
65896576

6577+
// Replace ALoad with a new ATOMIC_LOAD with a result that is extended to VT
6578+
// per ETy.
6579+
static SDValue extendAtomicLoad(AtomicSDNode *ALoad, EVT VT, SelectionDAG &DAG,
6580+
ISD::LoadExtType ETy) {
6581+
if (VT.getSizeInBits() > 64)
6582+
return SDValue();
6583+
EVT OrigVT = ALoad->getValueType(0);
6584+
assert(OrigVT.getSizeInBits() < VT.getSizeInBits() && "VT should be wider.");
6585+
EVT MemoryVT = ALoad->getMemoryVT();
6586+
auto *NewALoad = dyn_cast<AtomicSDNode>(DAG.getAtomic(
6587+
ISD::ATOMIC_LOAD, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(),
6588+
ALoad->getBasePtr(), ALoad->getMemOperand()));
6589+
NewALoad->setExtensionType(ETy);
6590+
DAG.ReplaceAllUsesOfValueWith(
6591+
SDValue(ALoad, 0),
6592+
DAG.getNode(ISD::TRUNCATE, SDLoc(ALoad), OrigVT, SDValue(NewALoad, 0)));
6593+
// Update the chain uses.
6594+
DAG.ReplaceAllUsesOfValueWith(SDValue(ALoad, 1), SDValue(NewALoad, 1));
6595+
return SDValue(NewALoad, 0);
6596+
}
6597+
65906598
SDValue SystemZTargetLowering::combineZERO_EXTEND(
65916599
SDNode *N, DAGCombinerInfo &DCI) const {
65926600
// Convert (zext (select_ccmask C1, C2)) into (select_ccmask C1', C2')
@@ -6611,6 +6619,13 @@ SDValue SystemZTargetLowering::combineZERO_EXTEND(
66116619
return NewSelect;
66126620
}
66136621
}
6622+
6623+
// Fold into ATOMIC_LOAD unless it is already sign extending.
6624+
if (auto *ALoad = dyn_cast<AtomicSDNode>(N0))
6625+
if (ALoad->getOpcode() == ISD::ATOMIC_LOAD &&
6626+
ALoad->getExtensionType() != ISD::SEXTLOAD)
6627+
return extendAtomicLoad(ALoad, VT, DAG, ISD::ZEXTLOAD);
6628+
66146629
return SDValue();
66156630
}
66166631

@@ -6662,6 +6677,13 @@ SDValue SystemZTargetLowering::combineSIGN_EXTEND(
66626677
}
66636678
}
66646679
}
6680+
6681+
// Fold into ATOMIC_LOAD unless it is already zero extending.
6682+
if (auto *ALoad = dyn_cast<AtomicSDNode>(N0))
6683+
if (ALoad->getOpcode() == ISD::ATOMIC_LOAD &&
6684+
ALoad->getExtensionType() != ISD::ZEXTLOAD)
6685+
return extendAtomicLoad(ALoad, VT, DAG, ISD::SEXTLOAD);
6686+
66656687
return SDValue();
66666688
}
66676689

0 commit comments

Comments
 (0)