Skip to content

Commit f44fc52

Browse files
jyknightjyknight
committed
Tweak some atomics functions in preparation for larger changes; NFC.
- Rename getATOMIC to getSYNC, as llvm will soon be able to emit both '__sync' libcalls and '__atomic' libcalls, and this function is for the '__sync' ones. - getInsertFencesForAtomic() has been replaced with shouldInsertFencesForAtomic(Instruction), so that the decision can be made per-instruction. This functionality will be used soon. - emitLeadingFence/emitTrailingFence are no longer called if shouldInsertFencesForAtomic returns false, and thus don't need to check the condition themselves. llvm-svn: 263665
1 parent b321050 commit f44fc52

17 files changed

+58
-56
lines changed

‎llvm/include/llvm/CodeGen/RuntimeLibcalls.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@ namespace RTLIB {
430430

431431
/// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
432432
/// UNKNOWN_LIBCALL if there is none.
433-
Libcall getATOMIC(unsigned Opc, MVT VT);
433+
Libcall getSYNC(unsigned Opc, MVT VT);
434434
}
435435
}
436436

‎llvm/include/llvm/Target/TargetLowering.h

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,12 +1003,6 @@ class TargetLoweringBase {
10031003
return PrefLoopAlignment;
10041004
}
10051005

1006-
/// Return whether the DAG builder should automatically insert fences and
1007-
/// reduce ordering for atomics.
1008-
bool getInsertFencesForAtomic() const {
1009-
return InsertFencesForAtomic;
1010-
}
1011-
10121006
/// Return true if the target stores stack protector cookies at a fixed offset
10131007
/// in some non-standard address space, and populates the address space and
10141008
/// offset as appropriate.
@@ -1052,6 +1046,13 @@ class TargetLoweringBase {
10521046
/// \name Helpers for atomic expansion.
10531047
/// @{
10541048

1049+
/// Whether AtomicExpandPass should automatically insert fences and reduce
1050+
/// ordering for this atomic. This should be true for most architectures with
1051+
/// weak memory ordering. Defaults to false.
1052+
virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
1053+
return false;
1054+
}
1055+
10551056
/// Perform a load-linked operation on Addr, returning a "Value *" with the
10561057
/// corresponding pointee type. This may entail some non-trivial operations to
10571058
/// truncate or reconstruct types that will be illegal in the backend. See
@@ -1070,12 +1071,12 @@ class TargetLoweringBase {
10701071

10711072
/// Inserts in the IR a target-specific intrinsic specifying a fence.
10721073
/// It is called by AtomicExpandPass before expanding an
1073-
/// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad.
1074+
/// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
1075+
/// if shouldInsertFencesForAtomic returns true.
10741076
/// RMW and CmpXchg set both IsStore and IsLoad to true.
10751077
/// This function should either return a nullptr, or a pointer to an IR-level
10761078
/// Instruction*. Even complex fence sequences can be represented by a
10771079
/// single Instruction* through an intrinsic to be lowered later.
1078-
/// Backends with !getInsertFencesForAtomic() should keep a no-op here.
10791080
/// Backends should override this method to produce target-specific intrinsic
10801081
/// for their fences.
10811082
/// FIXME: Please note that the default implementation here in terms of
@@ -1101,9 +1102,6 @@ class TargetLoweringBase {
11011102
virtual Instruction *emitLeadingFence(IRBuilder<> &Builder,
11021103
AtomicOrdering Ord, bool IsStore,
11031104
bool IsLoad) const {
1104-
if (!getInsertFencesForAtomic())
1105-
return nullptr;
1106-
11071105
if (isAtLeastRelease(Ord) && IsStore)
11081106
return Builder.CreateFence(Ord);
11091107
else
@@ -1113,9 +1111,6 @@ class TargetLoweringBase {
11131111
virtual Instruction *emitTrailingFence(IRBuilder<> &Builder,
11141112
AtomicOrdering Ord, bool IsStore,
11151113
bool IsLoad) const {
1116-
if (!getInsertFencesForAtomic())
1117-
return nullptr;
1118-
11191114
if (isAtLeastAcquire(Ord))
11201115
return Builder.CreateFence(Ord);
11211116
else
@@ -1441,12 +1436,6 @@ class TargetLoweringBase {
14411436
MinStackArgumentAlignment = Align;
14421437
}
14431438

1444-
/// Set if the DAG builder should automatically insert fences and reduce the
1445-
/// order of atomic memory operations to Monotonic.
1446-
void setInsertFencesForAtomic(bool fence) {
1447-
InsertFencesForAtomic = fence;
1448-
}
1449-
14501439
public:
14511440
//===--------------------------------------------------------------------===//
14521441
// Addressing mode description hooks (used by LSR etc).
@@ -1856,10 +1845,6 @@ class TargetLoweringBase {
18561845
/// The preferred loop alignment.
18571846
unsigned PrefLoopAlignment;
18581847

1859-
/// Whether the DAG builder should automatically insert fences and reduce
1860-
/// ordering for atomics. (This will be set for for most architectures with
1861-
/// weak memory ordering.)
1862-
bool InsertFencesForAtomic;
18631848

18641849
/// If set to a physical register, this specifies the register that
18651850
/// llvm.savestack/llvm.restorestack should save and restore.

‎llvm/lib/CodeGen/AtomicExpandPass.cpp

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ bool AtomicExpand::runOnFunction(Function &F) {
100100
assert((LI || SI || RMWI || CASI || isa<FenceInst>(I)) &&
101101
"Unknown atomic instruction");
102102

103-
if (TLI->getInsertFencesForAtomic()) {
103+
if (TLI->shouldInsertFencesForAtomic(I)) {
104104
auto FenceOrdering = Monotonic;
105105
bool IsStore, IsLoad;
106106
if (LI && isAtLeastAcquire(LI->getOrdering())) {
@@ -514,12 +514,13 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
514514
BasicBlock *BB = CI->getParent();
515515
Function *F = BB->getParent();
516516
LLVMContext &Ctx = F->getContext();
517-
// If getInsertFencesForAtomic() returns true, then the target does not want
518-
// to deal with memory orders, and emitLeading/TrailingFence should take care
519-
// of everything. Otherwise, emitLeading/TrailingFence are no-op and we
517+
// If shouldInsertFencesForAtomic() returns true, then the target does not
518+
// want to deal with memory orders, and emitLeading/TrailingFence should take
519+
// care of everything. Otherwise, emitLeading/TrailingFence are no-op and we
520520
// should preserve the ordering.
521+
bool ShouldInsertFencesForAtomic = TLI->shouldInsertFencesForAtomic(CI);
521522
AtomicOrdering MemOpOrder =
522-
TLI->getInsertFencesForAtomic() ? Monotonic : SuccessOrder;
523+
ShouldInsertFencesForAtomic ? Monotonic : SuccessOrder;
523524

524525
// In implementations which use a barrier to achieve release semantics, we can
525526
// delay emitting this barrier until we know a store is actually going to be
@@ -530,7 +531,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
530531
// since in other cases the extra blocks naturally collapse down to the
531532
// minimal loop. Unfortunately, this puts too much stress on later
532533
// optimisations so we avoid emitting the extra logic in those cases too.
533-
bool HasReleasedLoadBB = !CI->isWeak() && TLI->getInsertFencesForAtomic() &&
534+
bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic &&
534535
SuccessOrder != Monotonic &&
535536
SuccessOrder != Acquire && !F->optForMinSize();
536537

@@ -601,7 +602,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
601602
// the branch entirely.
602603
std::prev(BB->end())->eraseFromParent();
603604
Builder.SetInsertPoint(BB);
604-
if (UseUnconditionalReleaseBarrier)
605+
if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
605606
TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
606607
/*IsLoad=*/true);
607608
Builder.CreateBr(StartBB);
@@ -617,7 +618,7 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
617618
Builder.CreateCondBr(ShouldStore, ReleasingStoreBB, NoStoreBB);
618619

619620
Builder.SetInsertPoint(ReleasingStoreBB);
620-
if (!UseUnconditionalReleaseBarrier)
621+
if (ShouldInsertFencesForAtomic && !UseUnconditionalReleaseBarrier)
621622
TLI->emitLeadingFence(Builder, SuccessOrder, /*IsStore=*/true,
622623
/*IsLoad=*/true);
623624
Builder.CreateBr(TryStoreBB);
@@ -647,8 +648,9 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
647648
// Make sure later instructions don't get reordered with a fence if
648649
// necessary.
649650
Builder.SetInsertPoint(SuccessBB);
650-
TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true,
651-
/*IsLoad=*/true);
651+
if (ShouldInsertFencesForAtomic)
652+
TLI->emitTrailingFence(Builder, SuccessOrder, /*IsStore=*/true,
653+
/*IsLoad=*/true);
652654
Builder.CreateBr(ExitBB);
653655

654656
Builder.SetInsertPoint(NoStoreBB);
@@ -659,8 +661,9 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
659661
Builder.CreateBr(FailureBB);
660662

661663
Builder.SetInsertPoint(FailureBB);
662-
TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true,
663-
/*IsLoad=*/true);
664+
if (ShouldInsertFencesForAtomic)
665+
TLI->emitTrailingFence(Builder, FailureOrder, /*IsStore=*/true,
666+
/*IsLoad=*/true);
664667
Builder.CreateBr(ExitBB);
665668

666669
// Finally, we have control-flow based knowledge of whether the cmpxchg

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4035,7 +4035,7 @@ void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
40354035
case ISD::ATOMIC_LOAD_UMAX:
40364036
case ISD::ATOMIC_CMP_SWAP: {
40374037
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4038-
RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
4038+
RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
40394039
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
40404040

40414041
std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1404,7 +1404,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
14041404
std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
14051405
unsigned Opc = Node->getOpcode();
14061406
MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1407-
RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
1407+
RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
14081408
assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
14091409

14101410
return ExpandChainLibCall(LC, Node, false);

‎llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -667,7 +667,7 @@ RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
667667
return UNKNOWN_LIBCALL;
668668
}
669669

670-
RTLIB::Libcall RTLIB::getATOMIC(unsigned Opc, MVT VT) {
670+
RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
671671
#define OP_TO_LIBCALL(Name, Enum) \
672672
case Name: \
673673
switch (VT.SimpleTy) { \
@@ -774,7 +774,6 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
774774
PrefLoopAlignment = 0;
775775
GatherAllAliasesMaxDepth = 6;
776776
MinStackArgumentAlignment = 1;
777-
InsertFencesForAtomic = false;
778777
MinimumJumpTableEntries = 4;
779778

780779
InitLibcallNames(LibcallRoutineNames, TM.getTargetTriple());

‎llvm/lib/Target/ARM/ARMISelLowering.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -840,6 +840,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
840840
// the default expansion. If we are targeting a single threaded system,
841841
// then set them all for expand so we can lower them later into their
842842
// non-atomic form.
843+
InsertFencesForAtomic = false;
843844
if (TM.Options.ThreadModel == ThreadModel::Single)
844845
setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
845846
else if (Subtarget->hasAnyDataBarrier() && (!Subtarget->isThumb() ||
@@ -852,7 +853,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
852853
// if they can be combined with nearby atomic loads and stores.
853854
if (!Subtarget->hasV8Ops()) {
854855
// Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
855-
setInsertFencesForAtomic(true);
856+
InsertFencesForAtomic = true;
856857
}
857858
} else {
858859
// If there's anything we can use as a barrier, go through custom lowering
@@ -11997,9 +11998,6 @@ Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder,
1199711998
Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
1199811999
AtomicOrdering Ord, bool IsStore,
1199912000
bool IsLoad) const {
12000-
if (!getInsertFencesForAtomic())
12001-
return nullptr;
12002-
1200312001
switch (Ord) {
1200412002
case NotAtomic:
1200512003
case Unordered:
@@ -12025,9 +12023,6 @@ Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
1202512023
Instruction* ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
1202612024
AtomicOrdering Ord, bool IsStore,
1202712025
bool IsLoad) const {
12028-
if (!getInsertFencesForAtomic())
12029-
return nullptr;
12030-
1203112026
switch (Ord) {
1203212027
case NotAtomic:
1203312028
case Unordered:
@@ -12081,6 +12076,11 @@ bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
1208112076
return true;
1208212077
}
1208312078

12079+
bool ARMTargetLowering::shouldInsertFencesForAtomic(
12080+
const Instruction *I) const {
12081+
return InsertFencesForAtomic;
12082+
}
12083+
1208412084
// This has so far only been implemented for MachO.
1208512085
bool ARMTargetLowering::useLoadStackGuardNode() const {
1208612086
return Subtarget->isTargetMachO();

‎llvm/lib/Target/ARM/ARMISelLowering.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,7 @@ namespace llvm {
453453
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
454454
unsigned Factor) const override;
455455

456+
bool shouldInsertFencesForAtomic(const Instruction *I) const override;
456457
TargetLoweringBase::AtomicExpansionKind
457458
shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
458459
bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
@@ -486,6 +487,10 @@ namespace llvm {
486487
///
487488
unsigned ARMPCLabelIndex;
488489

490+
// TODO: remove this, and have shouldInsertFencesForAtomic do the proper
491+
// check.
492+
bool InsertFencesForAtomic;
493+
489494
void addTypeForNEON(MVT VT, MVT PromotedLdStVT, MVT PromotedBitwiseVT);
490495
void addDRTypeForNEON(MVT VT);
491496
void addQRTypeForNEON(MVT VT);

‎llvm/lib/Target/Hexagon/HexagonISelLowering.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1724,7 +1724,6 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
17241724
setPrefLoopAlignment(4);
17251725
setPrefFunctionAlignment(4);
17261726
setMinFunctionAlignment(2);
1727-
setInsertFencesForAtomic(false);
17281727
setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
17291728

17301729
if (EnableHexSDNodeSched)

‎llvm/lib/Target/Mips/MipsISelLowering.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,7 +396,6 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
396396
setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand);
397397
}
398398

399-
setInsertFencesForAtomic(true);
400399

401400
if (!Subtarget.hasMips32r2()) {
402401
setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);

‎llvm/lib/Target/Mips/MipsISelLowering.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,10 @@ namespace llvm {
561561
unsigned getJumpTableEncoding() const override;
562562
bool useSoftFloat() const override;
563563

564+
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
565+
return true;
566+
}
567+
564568
/// Emit a sign-extension using sll/sra, seb, or seh appropriately.
565569
MachineBasicBlock *emitSignExtendToI32InReg(MachineInstr *MI,
566570
MachineBasicBlock *BB,

‎llvm/lib/Target/PowerPC/PPCISelLowering.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -916,7 +916,6 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
916916
break;
917917
}
918918

919-
setInsertFencesForAtomic(true);
920919

921920
if (Subtarget.enableMachineScheduler())
922921
setSchedulingPreference(Sched::Source);

‎llvm/lib/Target/PowerPC/PPCISelLowering.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,10 @@ namespace llvm {
508508

509509
unsigned getPrefLoopAlignment(MachineLoop *ML) const override;
510510

511+
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
512+
return true;
513+
}
514+
511515
Instruction* emitLeadingFence(IRBuilder<> &Builder, AtomicOrdering Ord,
512516
bool IsStore, bool IsLoad) const override;
513517
Instruction* emitTrailingFence(IRBuilder<> &Builder, AtomicOrdering Ord,

‎llvm/lib/Target/Sparc/SparcISelLowering.cpp

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1603,10 +1603,6 @@ SparcTargetLowering::SparcTargetLowering(TargetMachine &TM,
16031603
}
16041604

16051605
// ATOMICs.
1606-
// FIXME: We insert fences for each atomics and generate sub-optimal code
1607-
// for PSO/TSO. Also, implement other atomicrmw operations.
1608-
1609-
setInsertFencesForAtomic(true);
16101606

16111607
setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Legal);
16121608
setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32,

‎llvm/lib/Target/Sparc/SparcISelLowering.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,13 @@ namespace llvm {
180180
return VT != MVT::f128;
181181
}
182182

183+
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
184+
// FIXME: We insert fences for each atomics and generate
185+
// sub-optimal code for PSO/TSO. (Approximately nobody uses any
186+
// mode but TSO, which makes this even more silly)
187+
return true;
188+
}
189+
183190
void ReplaceNodeResults(SDNode *N,
184191
SmallVectorImpl<SDValue>& Results,
185192
SelectionDAG &DAG) const override;

‎llvm/lib/Target/XCore/XCoreISelLowering.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,6 @@ XCoreTargetLowering::XCoreTargetLowering(const TargetMachine &TM,
156156
// Atomic operations
157157
// We request a fence for ATOMIC_* instructions, to reduce them to Monotonic.
158158
// As we are always Sequential Consistent, an ATOMIC_FENCE becomes a no OP.
159-
setInsertFencesForAtomic(true);
160159
setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
161160
setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
162161
setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);

‎llvm/lib/Target/XCore/XCoreISelLowering.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,9 @@ namespace llvm {
229229
bool isVarArg,
230230
const SmallVectorImpl<ISD::OutputArg> &ArgsFlags,
231231
LLVMContext &Context) const override;
232+
bool shouldInsertFencesForAtomic(const Instruction *I) const override {
233+
return true;
234+
}
232235
};
233236
}
234237

0 commit comments

Comments
 (0)