Skip to content

Commit 7334b3d

Browse files
JonPssonJonPsson
committed
[SystemZ] Reimplement the i8/i16 compare-and-swap logic.
Even though the implementation in emitAtomicCmpSwapW() was correct, it made Valgrind report an error. Instead of using a RISBG on CmpVal, an LL[CH]R can be made on the OldVal, and the problem is avoided. Review: Ulrich Weigand Differential Revision: https://reviews.llvm.org/D97604
1 parent 1c2935a commit 7334b3d

File tree

5 files changed

+53
-50
lines changed

5 files changed

+53
-50
lines changed

‎llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4056,7 +4056,10 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op,
40564056
SDValue Success = emitSETCC(DAG, DL, AtomicOp.getValue(1),
40574057
SystemZ::CCMASK_ICMP, SystemZ::CCMASK_CMP_EQ);
40584058

4059-
DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), AtomicOp.getValue(0));
4059+
// emitAtomicCmpSwapW() will zero extend the result (original value).
4060+
SDValue OrigVal = DAG.getNode(ISD::AssertZext, DL, WideVT, AtomicOp.getValue(0),
4061+
DAG.getValueType(NarrowVT));
4062+
DAG.ReplaceAllUsesOfValueWith(Op.getValue(0), OrigVal);
40604063
DAG.ReplaceAllUsesOfValueWith(Op.getValue(1), Success);
40614064
DAG.ReplaceAllUsesOfValueWith(Op.getValue(2), AtomicOp.getValue(2));
40624065
return SDValue();
@@ -7578,7 +7581,6 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax(
75787581
MachineBasicBlock *
75797582
SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
75807583
MachineBasicBlock *MBB) const {
7581-
75827584
MachineFunction &MF = *MBB->getParent();
75837585
const SystemZInstrInfo *TII =
75847586
static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
@@ -7588,7 +7590,7 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
75887590
Register Dest = MI.getOperand(0).getReg();
75897591
MachineOperand Base = earlyUseOperand(MI.getOperand(1));
75907592
int64_t Disp = MI.getOperand(2).getImm();
7591-
Register OrigCmpVal = MI.getOperand(3).getReg();
7593+
Register CmpVal = MI.getOperand(3).getReg();
75927594
Register OrigSwapVal = MI.getOperand(4).getReg();
75937595
Register BitShift = MI.getOperand(5).getReg();
75947596
Register NegBitShift = MI.getOperand(6).getReg();
@@ -7597,19 +7599,19 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
75977599

75987600
const TargetRegisterClass *RC = &SystemZ::GR32BitRegClass;
75997601

7600-
// Get the right opcodes for the displacement.
7602+
// Get the right opcodes for the displacement and zero-extension.
76017603
unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp);
76027604
unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp);
7605+
unsigned ZExtOpcode = BitSize == 8 ? SystemZ::LLCR : SystemZ::LLHR;
76037606
assert(LOpcode && CSOpcode && "Displacement out of range");
76047607

76057608
// Create virtual registers for temporary results.
76067609
Register OrigOldVal = MRI.createVirtualRegister(RC);
76077610
Register OldVal = MRI.createVirtualRegister(RC);
7608-
Register CmpVal = MRI.createVirtualRegister(RC);
76097611
Register SwapVal = MRI.createVirtualRegister(RC);
76107612
Register StoreVal = MRI.createVirtualRegister(RC);
7613+
Register OldValRot = MRI.createVirtualRegister(RC);
76117614
Register RetryOldVal = MRI.createVirtualRegister(RC);
7612-
Register RetryCmpVal = MRI.createVirtualRegister(RC);
76137615
Register RetrySwapVal = MRI.createVirtualRegister(RC);
76147616

76157617
// Insert 2 basic blocks for the loop.
@@ -7631,52 +7633,45 @@ SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI,
76317633

76327634
// LoopMBB:
76337635
// %OldVal = phi [ %OrigOldVal, EntryBB ], [ %RetryOldVal, SetMBB ]
7634-
// %CmpVal = phi [ %OrigCmpVal, EntryBB ], [ %RetryCmpVal, SetMBB ]
76357636
// %SwapVal = phi [ %OrigSwapVal, EntryBB ], [ %RetrySwapVal, SetMBB ]
7636-
// %Dest = RLL %OldVal, BitSize(%BitShift)
7637+
// %OldValRot = RLL %OldVal, BitSize(%BitShift)
76377638
// ^^ The low BitSize bits contain the field
76387639
// of interest.
7639-
// %RetryCmpVal = RISBG32 %CmpVal, %Dest, 32, 63-BitSize, 0
7640+
// %RetrySwapVal = RISBG32 %SwapVal, %OldValRot, 32, 63-BitSize, 0
76407641
// ^^ Replace the upper 32-BitSize bits of the
7641-
// comparison value with those that we loaded,
7642-
// so that we can use a full word comparison.
7643-
// CR %Dest, %RetryCmpVal
7642+
// swap value with those that we loaded and rotated.
7643+
// %Dest = LL[CH] %OldValRot
7644+
// CR %Dest, %CmpVal
76447645
// JNE DoneMBB
76457646
// # Fall through to SetMBB
76467647
MBB = LoopMBB;
76477648
BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal)
76487649
.addReg(OrigOldVal).addMBB(StartMBB)
76497650
.addReg(RetryOldVal).addMBB(SetMBB);
7650-
BuildMI(MBB, DL, TII->get(SystemZ::PHI), CmpVal)
7651-
.addReg(OrigCmpVal).addMBB(StartMBB)
7652-
.addReg(RetryCmpVal).addMBB(SetMBB);
76537651
BuildMI(MBB, DL, TII->get(SystemZ::PHI), SwapVal)
76547652
.addReg(OrigSwapVal).addMBB(StartMBB)
76557653
.addReg(RetrySwapVal).addMBB(SetMBB);
7656-
BuildMI(MBB, DL, TII->get(SystemZ::RLL), Dest)
7654+
BuildMI(MBB, DL, TII->get(SystemZ::RLL), OldValRot)
76577655
.addReg(OldVal).addReg(BitShift).addImm(BitSize);
7658-
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetryCmpVal)
7659-
.addReg(CmpVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
7656+
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
7657+
.addReg(SwapVal).addReg(OldValRot).addImm(32).addImm(63 - BitSize).addImm(0);
7658+
BuildMI(MBB, DL, TII->get(ZExtOpcode), Dest)
7659+
.addReg(OldValRot);
76607660
BuildMI(MBB, DL, TII->get(SystemZ::CR))
7661-
.addReg(Dest).addReg(RetryCmpVal);
7661+
.addReg(Dest).addReg(CmpVal);
76627662
BuildMI(MBB, DL, TII->get(SystemZ::BRC))
76637663
.addImm(SystemZ::CCMASK_ICMP)
76647664
.addImm(SystemZ::CCMASK_CMP_NE).addMBB(DoneMBB);
76657665
MBB->addSuccessor(DoneMBB);
76667666
MBB->addSuccessor(SetMBB);
76677667

76687668
// SetMBB:
7669-
// %RetrySwapVal = RISBG32 %SwapVal, %Dest, 32, 63-BitSize, 0
7670-
// ^^ Replace the upper 32-BitSize bits of the new
7671-
// value with those that we loaded.
7672-
// %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
7669+
// %StoreVal = RLL %RetrySwapVal, -BitSize(%NegBitShift)
76737670
// ^^ Rotate the new field to its proper position.
7674-
// %RetryOldVal = CS %Dest, %StoreVal, Disp(%Base)
7671+
// %RetryOldVal = CS %OldVal, %StoreVal, Disp(%Base)
76757672
// JNE LoopMBB
76767673
// # fall through to ExitMMB
76777674
MBB = SetMBB;
7678-
BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RetrySwapVal)
7679-
.addReg(SwapVal).addReg(Dest).addImm(32).addImm(63 - BitSize).addImm(0);
76807675
BuildMI(MBB, DL, TII->get(SystemZ::RLL), StoreVal)
76817676
.addReg(RetrySwapVal).addReg(NegBitShift).addImm(-BitSize);
76827677
BuildMI(MBB, DL, TII->get(CSOpcode), RetryOldVal)

‎llvm/lib/Target/SystemZ/SystemZISelLowering.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,10 @@ class SystemZTargetLowering : public TargetLowering {
550550
unsigned Depth) const override;
551551

552552
ISD::NodeType getExtendForAtomicOps() const override {
553-
return ISD::ANY_EXTEND;
553+
return ISD::ZERO_EXTEND;
554+
}
555+
ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
556+
return ISD::ZERO_EXTEND;
554557
}
555558

556559
bool supportSwiftError() const override {

‎llvm/test/CodeGen/SystemZ/cmpxchg-01.ll

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
1515
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
1616
; CHECK-MAIN-DAG: sll %r3, 3
1717
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
18+
; CHECK-MAIN-DAG: llcr %r4, %r4
1819
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
1920
; CHECK-MAIN: rll %r2, [[OLD]], 8(%r3)
20-
; CHECK-MAIN: risbg %r4, %r2, 32, 55, 0
21-
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
2221
; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0
22+
; CHECK-MAIN: llcr %r2, %r2
23+
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
2324
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
2425
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
2526
; CHECK-MAIN: jl [[LOOP]]
@@ -48,7 +49,6 @@ define i8 @f2(i8 *%src) {
4849
;
4950
; CHECK-SHIFT-LABEL: f2:
5051
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
51-
; CHECK-SHIFT: risbg
5252
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 55, 0
5353
; CHECK-SHIFT: br %r14
5454
%pair = cmpxchg i8 *%src, i8 42, i8 88 seq_cst seq_cst
@@ -62,12 +62,13 @@ define i32 @f3(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
6262
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
6363
; CHECK-MAIN-DAG: sll %r3, 3
6464
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
65+
; CHECK-MAIN-DAG: llcr %r2, %r4
6566
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
6667
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r3)
67-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
68-
; CHECK-MAIN: cr [[TMP]], %r4
69-
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
7068
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 55, 0
69+
; CHECK-MAIN: llcr [[TMP2:%r[0-9]+]], [[TMP]]
70+
; CHECK-MAIN: cr [[TMP2]], %r2
71+
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
7172
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
7273
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
7374
; CHECK-MAIN: jl [[LOOP]]
@@ -98,12 +99,13 @@ declare void @g()
9899
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
99100
; CHECK-MAIN-DAG: sll %r2, 3
100101
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
102+
; CHECK-MAIN-DAG: llcr %r3, %r3
101103
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
102104
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
103-
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
105+
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
106+
; CHECK-MAIN: llcr [[TMP]], [[TMP]]
104107
; CHECK-MAIN: cr [[TMP]], %r3
105108
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
106-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
107109
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
108110
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
109111
; CHECK-MAIN: jl [[LOOP]]
@@ -136,12 +138,13 @@ exit:
136138
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
137139
; CHECK-MAIN-DAG: sll %r2, 3
138140
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
141+
; CHECK-MAIN-DAG: llcr %r3, %r3
139142
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
140143
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 8(%r2)
141-
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 55, 0
144+
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
145+
; CHECK-MAIN: llcr [[TMP]], [[TMP]]
142146
; CHECK-MAIN: cr [[TMP]], %r3
143147
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
144-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 55, 0
145148
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -8({{%r[1-9]+}})
146149
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
147150
; CHECK-MAIN: jl [[LOOP]]

‎llvm/test/CodeGen/SystemZ/cmpxchg-02.ll

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
1515
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
1616
; CHECK-MAIN-DAG: sll %r3, 3
1717
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
18+
; CHECK-MAIN-DAG: llhr %r4, %r4
1819
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
1920
; CHECK-MAIN: rll %r2, [[OLD]], 16(%r3)
20-
; CHECK-MAIN: risbg %r4, %r2, 32, 47, 0
21-
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
2221
; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0
22+
; CHECK-MAIN: llhr %r2, %r2
23+
; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
2324
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
2425
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
2526
; CHECK-MAIN: jl [[LOOP]]
@@ -48,7 +49,6 @@ define i16 @f2(i16 *%src) {
4849
;
4950
; CHECK-SHIFT-LABEL: f2:
5051
; CHECK-SHIFT: lhi [[SWAP:%r[0-9]+]], 88
51-
; CHECK-SHIFT: risbg
5252
; CHECK-SHIFT: risbg [[SWAP]], {{%r[0-9]+}}, 32, 47, 0
5353
; CHECK-SHIFT: br %r14
5454
%pair = cmpxchg i16 *%src, i16 42, i16 88 seq_cst seq_cst
@@ -62,12 +62,13 @@ define i32 @f3(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
6262
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r3, 0, 189, 0{{$}}
6363
; CHECK-MAIN-DAG: sll %r3, 3
6464
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
65+
; CHECK-MAIN-DAG: llhr %r2, %r4
6566
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
6667
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r3)
67-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
68-
; CHECK-MAIN: cr [[TMP]], %r4
69-
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
7068
; CHECK-MAIN: risbg %r5, [[TMP]], 32, 47, 0
69+
; CHECK-MAIN: llhr %r14, %r14
70+
; CHECK-MAIN: cr [[TMP]], %r2
71+
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
7172
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
7273
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
7374
; CHECK-MAIN: jl [[LOOP]]
@@ -97,12 +98,13 @@ declare void @g()
9798
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
9899
; CHECK-MAIN-DAG: sll %r2, 3
99100
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
101+
; CHECK-MAIN-DAG: llhr %r3, %r3
100102
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
101103
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
102-
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
104+
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
105+
; CHECK-MAIN: llhr %r14, %r14
103106
; CHECK-MAIN: cr [[TMP]], %r3
104107
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
105-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
106108
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
107109
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
108110
; CHECK-MAIN: jl [[LOOP]]
@@ -135,12 +137,13 @@ exit:
135137
; CHECK-MAIN: risbg [[RISBG:%r[1-9]+]], %r2, 0, 189, 0{{$}}
136138
; CHECK-MAIN-DAG: sll %r2, 3
137139
; CHECK-MAIN-DAG: l [[OLD:%r[0-9]+]], 0([[RISBG]])
140+
; CHECK-MAIN-DAG: llhr %r3, %r3
138141
; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
139142
; CHECK-MAIN: rll [[TMP:%r[0-9]+]], [[OLD]], 16(%r2)
140-
; CHECK-MAIN: risbg %r3, [[TMP]], 32, 47, 0
143+
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
144+
; CHECK-MAIN: llhr %r14, %r14
141145
; CHECK-MAIN: cr [[TMP]], %r3
142146
; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
143-
; CHECK-MAIN: risbg %r4, [[TMP]], 32, 47, 0
144147
; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r4, -16({{%r[1-9]+}})
145148
; CHECK-MAIN: cs [[OLD]], [[NEW]], 0([[RISBG]])
146149
; CHECK-MAIN: jl [[LOOP]]

‎llvm/test/CodeGen/SystemZ/cmpxchg-05.ll

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
; CHECK: crjlh
77
; CHECK-NOT: llcr
88
; CHECK-NOT: cr
9-
; CHECK: llgcr %r2, [[RES:%r[0-9]+]]
9+
; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
1010
; CHECK-NOT: llcr
1111
; CHECK-NOT: cr
1212
define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
@@ -19,7 +19,7 @@ define zeroext i8 @f1(i8* nocapture, i8 zeroext, i8 zeroext) {
1919
; CHECK: crjlh
2020
; CHECK-NOT: llhr
2121
; CHECK-NOT: cr
22-
; CHECK: llghr %r2, [[RES:%r[0-9]+]]
22+
; CHECK: llgfr %r2, [[RES:%r[0-9]+]]
2323
; CHECK-NOT: llhr
2424
; CHECK-NOT: cr
2525
define zeroext i16 @f2(i16* nocapture, i16 zeroext, i16 zeroext) {
@@ -53,4 +53,3 @@ define signext i16 @f4(i16* nocapture, i16 signext, i16 signext) {
5353
%res = extractvalue { i16, i1 } %cx, 0
5454
ret i16 %res
5555
}
56-

0 commit comments

Comments
 (0)