Skip to content

Commit 4d0e6f7

Browse files
JonPsson1JonPsson1
committed
Updates after review
1 parent ddba819 commit 4d0e6f7

File tree

7 files changed

+63
-30
lines changed

7 files changed

+63
-30
lines changed

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -343,12 +343,19 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
343343
if (N->getOpcode() == ISD::ATOMIC_LOAD) {
344344
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
345345
if (ETy == ISD::NON_EXTLOAD) {
346-
if (TLI.getExtendForAtomicOps() == ISD::SIGN_EXTEND)
346+
switch (TLI.getExtendForAtomicOps()) {
347+
case ISD::SIGN_EXTEND:
347348
ETy = ISD::SEXTLOAD;
348-
else if (TLI.getExtendForAtomicOps() == ISD::ZERO_EXTEND)
349+
break;
350+
case ISD::ZERO_EXTEND:
349351
ETy = ISD::ZEXTLOAD;
350-
else
352+
break;
353+
case ISD::ANY_EXTEND:
351354
ETy = ISD::EXTLOAD;
355+
break;
356+
default:
357+
llvm_unreachable("Invalid atomic op extension");
358+
}
352359
}
353360
cast<AtomicSDNode>(Res)->setExtensionType(ETy);
354361
}

‎llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1516,13 +1516,10 @@ bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const {
15161516
MachineMemOperand *MMO = MemAccess->getMemOperand();
15171517
assert(MMO && "Expected a memory operand.");
15181518

1519-
// These instructions are not atomic.
1520-
if (MMO->isAtomic())
1521-
return false;
1522-
15231519
// The memory access must have a proper alignment and no index register.
1520+
// ATOMIC_LOADs do not have the offset operand.
15241521
if (MemAccess->getAlign().value() < StoreSize ||
1525-
!MemAccess->getOffset().isUndef())
1522+
(!MMO->isAtomic() && !MemAccess->getOffset().isUndef()))
15261523
return false;
15271524

15281525
// The MMO must not have an unaligned offset.

‎llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4515,7 +4515,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
45154515
return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
45164516
}
45174517

4518-
SDValue SystemZTargetLowering::lowerATOMIC_I128_LDST(SDValue Op,
4518+
SDValue SystemZTargetLowering::lowerATOMIC_LDST_I128(SDValue Op,
45194519
SelectionDAG &DAG) const {
45204520
auto *Node = cast<AtomicSDNode>(Op.getNode());
45214521
assert(Node->getMemoryVT() == MVT::i128 && "Only custom lowering i128.");
@@ -5645,12 +5645,11 @@ static SDValue tryBuildVectorShuffle(SelectionDAG &DAG,
56455645
return GS.getNode(DAG, SDLoc(BVN));
56465646
}
56475647

5648-
bool SystemZTargetLowering::isVectorElementLoad(SDValue Op, EVT VecVT) const {
5648+
bool SystemZTargetLowering::isVectorElementLoad(SDValue Op) const {
56495649
if (Op.getOpcode() == ISD::LOAD && cast<LoadSDNode>(Op)->isUnindexed())
56505650
return true;
56515651
if (auto *AL = dyn_cast<AtomicSDNode>(Op))
5652-
if (AL->getOpcode() == ISD::ATOMIC_LOAD && SDValue(AL, 0).hasOneUse() &&
5653-
AL->getMemoryVT() == VecVT.getScalarType())
5652+
if (AL->getOpcode() == ISD::ATOMIC_LOAD)
56545653
return true;
56555654
if (Subtarget.hasVectorEnhancements2() && Op.getOpcode() == SystemZISD::LRV)
56565655
return true;
@@ -5689,13 +5688,13 @@ SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
56895688
// we would need 2 instructions to replicate it: VLVGP followed by VREPx.
56905689
// This is only a win if the single defined element is used more than once.
56915690
// In other cases we're better off using a single VLVGx.
5692-
if (Single.getNode() && (Count > 1 || isVectorElementLoad(Single, VT)))
5691+
if (Single.getNode() && (Count > 1 || isVectorElementLoad(Single)))
56935692
return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Single);
56945693

56955694
// If all elements are loads, use VLREP/VLEs (below).
56965695
bool AllLoads = true;
56975696
for (auto Elem : Elems)
5698-
if (!isVectorElementLoad(Elem, VT)) {
5697+
if (!isVectorElementLoad(Elem)) {
56995698
AllLoads = false;
57005699
break;
57015700
}
@@ -5767,7 +5766,7 @@ SystemZTargetLowering::buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
57675766
std::map<const SDNode*, unsigned> UseCounts;
57685767
SDNode *LoadMaxUses = nullptr;
57695768
for (unsigned I = 0; I < NumElements; ++I)
5770-
if (isVectorElementLoad(Elems[I], VT)) {
5769+
if (isVectorElementLoad(Elems[I])) {
57715770
SDNode *Ld = Elems[I].getNode();
57725771
UseCounts[Ld]++;
57735772
if (LoadMaxUses == nullptr || UseCounts[LoadMaxUses] < UseCounts[Ld])
@@ -6129,7 +6128,7 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
61296128
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_SWAPW);
61306129
case ISD::ATOMIC_STORE:
61316130
case ISD::ATOMIC_LOAD:
6132-
return lowerATOMIC_I128_LDST(Op, DAG);
6131+
return lowerATOMIC_LDST_I128(Op, DAG);
61336132
case ISD::ATOMIC_LOAD_ADD:
61346133
return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_ADD);
61356134
case ISD::ATOMIC_LOAD_SUB:

‎llvm/lib/Target/SystemZ/SystemZISelLowering.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -694,7 +694,7 @@ class SystemZTargetLowering : public TargetLowering {
694694
SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
695695
SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
696696
SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
697-
SDValue lowerATOMIC_I128_LDST(SDValue Op, SelectionDAG &DAG) const;
697+
SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
698698
SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
699699
unsigned Opcode) const;
700700
SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
@@ -704,7 +704,7 @@ class SystemZTargetLowering : public TargetLowering {
704704
SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
705705
SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
706706
SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
707-
bool isVectorElementLoad(SDValue Op, EVT VecVT) const;
707+
bool isVectorElementLoad(SDValue Op) const;
708708
SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
709709
SmallVectorImpl<SDValue> &Elems) const;
710710
SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;

‎llvm/lib/Target/SystemZ/SystemZInstrFP.td

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -495,17 +495,17 @@ let Uses = [FPC], mayRaiseFPException = 1 in {
495495
def MAEBR : TernaryRRD<"maebr", 0xB30E, z_any_fma, FP32, FP32>;
496496
def MADBR : TernaryRRD<"madbr", 0xB31E, z_any_fma, FP64, FP64>;
497497

498-
defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32, nonatomic_ld, 4>;
499-
defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64, nonatomic_ld, 8>;
498+
defm MAEB : TernaryRXFAndPseudo<"maeb", 0xED0E, z_any_fma, FP32, FP32, load, 4>;
499+
defm MADB : TernaryRXFAndPseudo<"madb", 0xED1E, z_any_fma, FP64, FP64, load, 8>;
500500
}
501501

502502
// Fused multiply-subtract.
503503
let Uses = [FPC], mayRaiseFPException = 1 in {
504504
def MSEBR : TernaryRRD<"msebr", 0xB30F, z_any_fms, FP32, FP32>;
505505
def MSDBR : TernaryRRD<"msdbr", 0xB31F, z_any_fms, FP64, FP64>;
506506

507-
defm MSEB : TernaryRXFAndPseudo<"mseb", 0xED0F, z_any_fms, FP32, FP32, nonatomic_ld, 4>;
508-
defm MSDB : TernaryRXFAndPseudo<"msdb", 0xED1F, z_any_fms, FP64, FP64, nonatomic_ld, 8>;
507+
defm MSEB : TernaryRXFAndPseudo<"mseb", 0xED0F, z_any_fms, FP32, FP32, load, 4>;
508+
defm MSDB : TernaryRXFAndPseudo<"msdb", 0xED1F, z_any_fms, FP64, FP64, load, 8>;
509509
}
510510

511511
// Division.

‎llvm/lib/Target/SystemZ/SystemZOperators.td

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -607,10 +607,6 @@ def nonvolatile_anyextloadi8 : NonvolatileLoad<anyextloadi8>;
607607
def nonvolatile_anyextloadi16 : NonvolatileLoad<anyextloadi16>;
608608
def nonvolatile_anyextloadi32 : NonvolatileLoad<anyextloadi32>;
609609

610-
def nonatomic_ld : PatFrag<(ops node:$ptr), (load node:$ptr), [{
611-
return !cast<LoadSDNode>(N)->isAtomic();
612-
}]>;
613-
614610
// Non-volatile stores.
615611
class NonvolatileStore<SDPatternOperator store>
616612
: PatFrag<(ops node:$src, node:$addr), (store node:$src, node:$addr), [{

‎llvm/test/CodeGen/SystemZ/atomic-memofolds.ll

Lines changed: 38 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -170,12 +170,11 @@ define i64 @f14(i64 %a, ptr %src) {
170170
ret i64 %sub
171171
}
172172

173-
; Check that maeb (reg/mem) is *not* used for an atomic load.
174173
define float @f15(float %f1, ptr %ptr, float %acc) {
175174
; CHECK-LABEL: f15:
176175
; CHECK: # %bb.0:
177-
; CHECK-NEXT: lde %f1, 0(%r2)
178-
; CHECK-NEXT: wfmasb %f0, %f0, %f1, %f2
176+
; CHECK-NEXT: maeb %f2, %f0, 0(%r2)
177+
; CHECK-NEXT: ldr %f0, %f2
179178
; CHECK-NEXT: br %r14
180179
%f2 = load atomic float, ptr %ptr seq_cst, align 4
181180
%res = call float @llvm.fma.f32 (float %f1, float %f2, float %acc)
@@ -387,6 +386,39 @@ define void @f25_b(ptr %src, ptr %dst) {
387386
ret void
388387
}
389388

389+
; Do *not* use vlrep for an extending load.
390+
define <4 x i32> @f25_c(ptr %ptr) {
391+
; CHECK-LABEL: f25_c:
392+
; CHECK: # %bb.0:
393+
; CHECK-NEXT: lb %r0, 0(%r2)
394+
; CHECK-NEXT: vlvgp %v0, %r0, %r0
395+
; CHECK-NEXT: vrepf %v24, %v0, 1
396+
; CHECK-NEXT: br %r14
397+
%L = load atomic i8, ptr %ptr seq_cst, align 4
398+
%S = sext i8 %L to i32
399+
%val = insertelement <4 x i32> undef, i32 %S, i32 0
400+
%ret = shufflevector <4 x i32> %val, <4 x i32> undef,
401+
<4 x i32> zeroinitializer
402+
ret <4 x i32> %ret
403+
}
404+
405+
; Do *not* use vlrep if there is another scalar use.
406+
define <4 x i32> @f25_d(ptr %ptr, ptr %dst) {
407+
; CHECK-LABEL: f25_d:
408+
; CHECK: # %bb.0:
409+
; CHECK-NEXT: l %r0, 0(%r2)
410+
; CHECK-NEXT: vlvgp %v0, %r0, %r0
411+
; CHECK-NEXT: vrepf %v24, %v0, 1
412+
; CHECK-NEXT: st %r0, 0(%r3)
413+
; CHECK-NEXT: br %r14
414+
%L = load atomic i32, ptr %ptr seq_cst, align 4
415+
store i32 %L, ptr %dst, align 4
416+
%val = insertelement <4 x i32> undef, i32 %L, i32 0
417+
%ret = shufflevector <4 x i32> %val, <4 x i32> undef,
418+
<4 x i32> zeroinitializer
419+
ret <4 x i32> %ret
420+
}
421+
390422
define void @f26(ptr %src, ptr %dst) {
391423
; CHECK-LABEL: f26:
392424
; CHECK: # %bb.0:
@@ -412,6 +444,8 @@ define void @f26_b(ptr %src, ptr %dst) {
412444
ret void
413445
}
414446

447+
448+
415449
; Vector Load logical element and zero.
416450
define <16 x i8> @f27(ptr %ptr) {
417451
; CHECK-LABEL: f27:
@@ -607,7 +641,7 @@ define void @f43(ptr %ptr) {
607641
define void @f44(ptr %ptr) {
608642
; CHECK-LABEL: f44:
609643
; CHECK: # %bb.0:
610-
; CHECK-NEXT: larl %r1, .LCPI48_0
644+
; CHECK-NEXT: larl %r1, .LCPI50_0
611645
; CHECK-NEXT: ld %f0, 0(%r1)
612646
; CHECK-NEXT: std %f0, 0(%r2)
613647
; CHECK-NEXT: bcr 14, %r0

0 commit comments

Comments
 (0)