Skip to content

Commit 2d7a2c1

Browse files
arsenmarsenm
authored
AtomicExpand: Refactor atomic instruction handling (#102914)
Move the processing of an instruction into a helper function. Also avoid redundant checking for all types of atomic instructions. Including the assert, it was effectively performing the same check 3 times.
1 parent 55323ca commit 2d7a2c1

File tree

1 file changed

+143
-125
lines changed

1 file changed

+143
-125
lines changed

‎llvm/lib/CodeGen/AtomicExpandPass.cpp

Lines changed: 143 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ class AtomicExpandImpl {
119119
llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI,
120120
CreateCmpXchgInstFun CreateCmpXchg);
121121

122+
bool processAtomicInstr(Instruction *I);
123+
122124
public:
123125
bool run(Function &F, const TargetMachine *TM);
124126
};
@@ -203,13 +205,152 @@ static bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
203205
Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
204206
}
205207

208+
bool AtomicExpandImpl::processAtomicInstr(Instruction *I) {
209+
auto *LI = dyn_cast<LoadInst>(I);
210+
auto *SI = dyn_cast<StoreInst>(I);
211+
auto *RMWI = dyn_cast<AtomicRMWInst>(I);
212+
auto *CASI = dyn_cast<AtomicCmpXchgInst>(I);
213+
214+
bool MadeChange = false;
215+
216+
// If the Size/Alignment is not supported, replace with a libcall.
217+
if (LI) {
218+
if (!LI->isAtomic())
219+
return false;
220+
221+
if (!atomicSizeSupported(TLI, LI)) {
222+
expandAtomicLoadToLibcall(LI);
223+
return true;
224+
}
225+
226+
if (TLI->shouldCastAtomicLoadInIR(LI) ==
227+
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
228+
I = LI = convertAtomicLoadToIntegerType(LI);
229+
MadeChange = true;
230+
}
231+
} else if (SI) {
232+
if (!SI->isAtomic())
233+
return false;
234+
235+
if (!atomicSizeSupported(TLI, SI)) {
236+
expandAtomicStoreToLibcall(SI);
237+
return true;
238+
}
239+
240+
if (TLI->shouldCastAtomicStoreInIR(SI) ==
241+
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
242+
I = SI = convertAtomicStoreToIntegerType(SI);
243+
MadeChange = true;
244+
}
245+
} else if (RMWI) {
246+
if (!atomicSizeSupported(TLI, RMWI)) {
247+
expandAtomicRMWToLibcall(RMWI);
248+
return true;
249+
}
250+
251+
if (TLI->shouldCastAtomicRMWIInIR(RMWI) ==
252+
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
253+
I = RMWI = convertAtomicXchgToIntegerType(RMWI);
254+
MadeChange = true;
255+
}
256+
} else if (CASI) {
257+
if (!atomicSizeSupported(TLI, CASI)) {
258+
expandAtomicCASToLibcall(CASI);
259+
return true;
260+
}
261+
262+
// TODO: when we're ready to make the change at the IR level, we can
263+
// extend convertCmpXchgToInteger for floating point too.
264+
if (CASI->getCompareOperand()->getType()->isPointerTy()) {
265+
// TODO: add a TLI hook to control this so that each target can
266+
// convert to lowering the original type one at a time.
267+
I = CASI = convertCmpXchgToIntegerType(CASI);
268+
MadeChange = true;
269+
}
270+
} else
271+
return false;
272+
273+
if (TLI->shouldInsertFencesForAtomic(I)) {
274+
auto FenceOrdering = AtomicOrdering::Monotonic;
275+
if (LI && isAcquireOrStronger(LI->getOrdering())) {
276+
FenceOrdering = LI->getOrdering();
277+
LI->setOrdering(AtomicOrdering::Monotonic);
278+
} else if (SI && isReleaseOrStronger(SI->getOrdering())) {
279+
FenceOrdering = SI->getOrdering();
280+
SI->setOrdering(AtomicOrdering::Monotonic);
281+
} else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
282+
isAcquireOrStronger(RMWI->getOrdering()))) {
283+
FenceOrdering = RMWI->getOrdering();
284+
RMWI->setOrdering(AtomicOrdering::Monotonic);
285+
} else if (CASI &&
286+
TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
287+
TargetLoweringBase::AtomicExpansionKind::None &&
288+
(isReleaseOrStronger(CASI->getSuccessOrdering()) ||
289+
isAcquireOrStronger(CASI->getSuccessOrdering()) ||
290+
isAcquireOrStronger(CASI->getFailureOrdering()))) {
291+
// If a compare and swap is lowered to LL/SC, we can do smarter fence
292+
// insertion, with a stronger one on the success path than on the
293+
// failure path. As a result, fence insertion is directly done by
294+
// expandAtomicCmpXchg in that case.
295+
FenceOrdering = CASI->getMergedOrdering();
296+
CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
297+
CASI->setFailureOrdering(AtomicOrdering::Monotonic);
298+
}
299+
300+
if (FenceOrdering != AtomicOrdering::Monotonic) {
301+
MadeChange |= bracketInstWithFences(I, FenceOrdering);
302+
}
303+
} else if (I->hasAtomicStore() &&
304+
TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
305+
auto FenceOrdering = AtomicOrdering::Monotonic;
306+
if (SI)
307+
FenceOrdering = SI->getOrdering();
308+
else if (RMWI)
309+
FenceOrdering = RMWI->getOrdering();
310+
else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
311+
TargetLoweringBase::AtomicExpansionKind::LLSC)
312+
// LLSC is handled in expandAtomicCmpXchg().
313+
FenceOrdering = CASI->getSuccessOrdering();
314+
315+
IRBuilder Builder(I);
316+
if (auto TrailingFence =
317+
TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
318+
TrailingFence->moveAfter(I);
319+
MadeChange = true;
320+
}
321+
}
322+
323+
if (LI)
324+
MadeChange |= tryExpandAtomicLoad(LI);
325+
else if (SI)
326+
MadeChange |= tryExpandAtomicStore(SI);
327+
else if (RMWI) {
328+
// There are two different ways of expanding RMW instructions:
329+
// - into a load if it is idempotent
330+
// - into a Cmpxchg/LL-SC loop otherwise
331+
// we try them in that order.
332+
333+
if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
334+
MadeChange = true;
335+
336+
} else {
337+
MadeChange |= tryExpandAtomicRMW(RMWI);
338+
}
339+
} else if (CASI)
340+
MadeChange |= tryExpandAtomicCmpXchg(CASI);
341+
342+
return MadeChange;
343+
}
344+
206345
bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
207346
const auto *Subtarget = TM->getSubtargetImpl(F);
208347
if (!Subtarget->enableAtomicExpand())
209348
return false;
210349
TLI = Subtarget->getTargetLowering();
211350
DL = &F.getDataLayout();
212351

352+
bool MadeChange = false;
353+
213354
SmallVector<Instruction *, 1> AtomicInsts;
214355

215356
// Changing control-flow while iterating through it is a bad idea, so gather a
@@ -218,134 +359,11 @@ bool AtomicExpandImpl::run(Function &F, const TargetMachine *TM) {
218359
if (I.isAtomic() && !isa<FenceInst>(&I))
219360
AtomicInsts.push_back(&I);
220361

221-
bool MadeChange = false;
222362
for (auto *I : AtomicInsts) {
223-
auto LI = dyn_cast<LoadInst>(I);
224-
auto SI = dyn_cast<StoreInst>(I);
225-
auto RMWI = dyn_cast<AtomicRMWInst>(I);
226-
auto CASI = dyn_cast<AtomicCmpXchgInst>(I);
227-
assert((LI || SI || RMWI || CASI) && "Unknown atomic instruction");
228-
229-
// If the Size/Alignment is not supported, replace with a libcall.
230-
if (LI) {
231-
if (!atomicSizeSupported(TLI, LI)) {
232-
expandAtomicLoadToLibcall(LI);
233-
MadeChange = true;
234-
continue;
235-
}
236-
} else if (SI) {
237-
if (!atomicSizeSupported(TLI, SI)) {
238-
expandAtomicStoreToLibcall(SI);
239-
MadeChange = true;
240-
continue;
241-
}
242-
} else if (RMWI) {
243-
if (!atomicSizeSupported(TLI, RMWI)) {
244-
expandAtomicRMWToLibcall(RMWI);
245-
MadeChange = true;
246-
continue;
247-
}
248-
} else if (CASI) {
249-
if (!atomicSizeSupported(TLI, CASI)) {
250-
expandAtomicCASToLibcall(CASI);
251-
MadeChange = true;
252-
continue;
253-
}
254-
}
255-
256-
if (LI && TLI->shouldCastAtomicLoadInIR(LI) ==
257-
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
258-
I = LI = convertAtomicLoadToIntegerType(LI);
363+
if (processAtomicInstr(I))
259364
MadeChange = true;
260-
} else if (SI &&
261-
TLI->shouldCastAtomicStoreInIR(SI) ==
262-
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
263-
I = SI = convertAtomicStoreToIntegerType(SI);
264-
MadeChange = true;
265-
} else if (RMWI &&
266-
TLI->shouldCastAtomicRMWIInIR(RMWI) ==
267-
TargetLoweringBase::AtomicExpansionKind::CastToInteger) {
268-
I = RMWI = convertAtomicXchgToIntegerType(RMWI);
269-
MadeChange = true;
270-
} else if (CASI) {
271-
// TODO: when we're ready to make the change at the IR level, we can
272-
// extend convertCmpXchgToInteger for floating point too.
273-
if (CASI->getCompareOperand()->getType()->isPointerTy()) {
274-
// TODO: add a TLI hook to control this so that each target can
275-
// convert to lowering the original type one at a time.
276-
I = CASI = convertCmpXchgToIntegerType(CASI);
277-
MadeChange = true;
278-
}
279-
}
280-
281-
if (TLI->shouldInsertFencesForAtomic(I)) {
282-
auto FenceOrdering = AtomicOrdering::Monotonic;
283-
if (LI && isAcquireOrStronger(LI->getOrdering())) {
284-
FenceOrdering = LI->getOrdering();
285-
LI->setOrdering(AtomicOrdering::Monotonic);
286-
} else if (SI && isReleaseOrStronger(SI->getOrdering())) {
287-
FenceOrdering = SI->getOrdering();
288-
SI->setOrdering(AtomicOrdering::Monotonic);
289-
} else if (RMWI && (isReleaseOrStronger(RMWI->getOrdering()) ||
290-
isAcquireOrStronger(RMWI->getOrdering()))) {
291-
FenceOrdering = RMWI->getOrdering();
292-
RMWI->setOrdering(AtomicOrdering::Monotonic);
293-
} else if (CASI &&
294-
TLI->shouldExpandAtomicCmpXchgInIR(CASI) ==
295-
TargetLoweringBase::AtomicExpansionKind::None &&
296-
(isReleaseOrStronger(CASI->getSuccessOrdering()) ||
297-
isAcquireOrStronger(CASI->getSuccessOrdering()) ||
298-
isAcquireOrStronger(CASI->getFailureOrdering()))) {
299-
// If a compare and swap is lowered to LL/SC, we can do smarter fence
300-
// insertion, with a stronger one on the success path than on the
301-
// failure path. As a result, fence insertion is directly done by
302-
// expandAtomicCmpXchg in that case.
303-
FenceOrdering = CASI->getMergedOrdering();
304-
CASI->setSuccessOrdering(AtomicOrdering::Monotonic);
305-
CASI->setFailureOrdering(AtomicOrdering::Monotonic);
306-
}
307-
308-
if (FenceOrdering != AtomicOrdering::Monotonic) {
309-
MadeChange |= bracketInstWithFences(I, FenceOrdering);
310-
}
311-
} else if (I->hasAtomicStore() &&
312-
TLI->shouldInsertTrailingFenceForAtomicStore(I)) {
313-
auto FenceOrdering = AtomicOrdering::Monotonic;
314-
if (SI)
315-
FenceOrdering = SI->getOrdering();
316-
else if (RMWI)
317-
FenceOrdering = RMWI->getOrdering();
318-
else if (CASI && TLI->shouldExpandAtomicCmpXchgInIR(CASI) !=
319-
TargetLoweringBase::AtomicExpansionKind::LLSC)
320-
// LLSC is handled in expandAtomicCmpXchg().
321-
FenceOrdering = CASI->getSuccessOrdering();
322-
323-
IRBuilder Builder(I);
324-
if (auto TrailingFence =
325-
TLI->emitTrailingFence(Builder, I, FenceOrdering)) {
326-
TrailingFence->moveAfter(I);
327-
MadeChange = true;
328-
}
329-
}
330-
331-
if (LI)
332-
MadeChange |= tryExpandAtomicLoad(LI);
333-
else if (SI)
334-
MadeChange |= tryExpandAtomicStore(SI);
335-
else if (RMWI) {
336-
// There are two different ways of expanding RMW instructions:
337-
// - into a load if it is idempotent
338-
// - into a Cmpxchg/LL-SC loop otherwise
339-
// we try them in that order.
340-
341-
if (isIdempotentRMW(RMWI) && simplifyIdempotentRMW(RMWI)) {
342-
MadeChange = true;
343-
} else {
344-
MadeChange |= tryExpandAtomicRMW(RMWI);
345-
}
346-
} else if (CASI)
347-
MadeChange |= tryExpandAtomicCmpXchg(CASI);
348365
}
366+
349367
return MadeChange;
350368
}
351369

0 commit comments

Comments
 (0)