Skip to content

Commit 4d5e834

Browse files
JonPsson1JonPsson1
authored
[ClangFE] Improve handling of casting of atomic memory operations. (#86691)
- Factor out a shouldCastToInt() method. - Also pass through pointer type values to not be casted to integer. CC @uweigand
1 parent 668c1ea commit 4d5e834

File tree

2 files changed

+70
-51
lines changed

2 files changed

+70
-51
lines changed

‎clang/lib/CodeGen/CGAtomic.cpp

Lines changed: 43 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -197,11 +197,11 @@ namespace {
197197
llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
198198

199199
/// Converts an rvalue to integer value if needed.
200-
llvm::Value *convertRValueToInt(RValue RVal, bool CastFP = true) const;
200+
llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
201201

202202
RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
203203
SourceLocation Loc, bool AsValue,
204-
bool CastFP = true) const;
204+
bool CmpXchg = false) const;
205205

206206
/// Copy an atomic r-value into atomic-layout memory.
207207
void emitCopyIntoMemory(RValue rvalue) const;
@@ -264,7 +264,7 @@ namespace {
264264
llvm::AtomicOrdering AO, bool IsVolatile);
265265
/// Emits atomic load as LLVM instruction.
266266
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
267-
bool CastFP = true);
267+
bool CmpXchg = false);
268268
/// Emits atomic compare-and-exchange op as a libcall.
269269
llvm::Value *EmitAtomicCompareExchangeLibcall(
270270
llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
@@ -1401,13 +1401,26 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
14011401
LVal.getBaseInfo(), TBAAAccessInfo()));
14021402
}
14031403

1404+
/// Return true if \param ValTy is a type that should be casted to integer
1405+
/// around the atomic memory operation. If \param CmpXchg is true, then the
1406+
/// cast of a floating point type is made as that instruction can not have
1407+
/// floating point operands. TODO: Allow compare-and-exchange and FP - see
1408+
/// comment in AtomicExpandPass.cpp.
1409+
static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
1410+
if (ValTy->isFloatingPointTy())
1411+
return ValTy->isX86_FP80Ty() || CmpXchg;
1412+
return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1413+
}
1414+
14041415
RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
14051416
AggValueSlot ResultSlot,
14061417
SourceLocation Loc, bool AsValue,
1407-
bool CastFP) const {
1418+
bool CmpXchg) const {
14081419
// Try not to in some easy cases.
1409-
assert((Val->getType()->isIntegerTy() || Val->getType()->isIEEELikeFPTy()) &&
1410-
"Expected integer or floating point value");
1420+
assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1421+
Val->getType()->isIEEELikeFPTy()) &&
1422+
"Expected integer, pointer or floating point value when converting "
1423+
"result.");
14111424
if (getEvaluationKind() == TEK_Scalar &&
14121425
(((!LVal.isBitField() ||
14131426
LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
@@ -1416,13 +1429,12 @@ RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
14161429
auto *ValTy = AsValue
14171430
? CGF.ConvertTypeForMem(ValueTy)
14181431
: getAtomicAddress().getElementType();
1419-
if (ValTy->isIntegerTy() || (!CastFP && ValTy->isIEEELikeFPTy())) {
1432+
if (!shouldCastToInt(ValTy, CmpXchg)) {
14201433
assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
14211434
"Different integer types.");
14221435
return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
1423-
} else if (ValTy->isPointerTy())
1424-
return RValue::get(CGF.Builder.CreateIntToPtr(Val, ValTy));
1425-
else if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1436+
}
1437+
if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
14261438
return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));
14271439
}
14281440

@@ -1459,10 +1471,10 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
14591471
}
14601472

14611473
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1462-
bool IsVolatile, bool CastFP) {
1474+
bool IsVolatile, bool CmpXchg) {
14631475
// Okay, we're doing this natively.
14641476
Address Addr = getAtomicAddress();
1465-
if (!(Addr.getElementType()->isIEEELikeFPTy() && !CastFP))
1477+
if (shouldCastToInt(Addr.getElementType(), CmpXchg))
14661478
Addr = castToAtomicIntPointer(Addr);
14671479
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
14681480
Load->setAtomic(AO);
@@ -1523,16 +1535,15 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
15231535
}
15241536

15251537
// Okay, we're doing this natively.
1526-
auto *Load = EmitAtomicLoadOp(AO, IsVolatile, /*CastFP=*/false);
1538+
auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
15271539

15281540
// If we're ignoring an aggregate return, don't do anything.
15291541
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
15301542
return RValue::getAggregate(Address::invalid(), false);
15311543

15321544
// Okay, turn that back into the original value or atomic (for non-simple
15331545
// lvalues) type.
1534-
return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue,
1535-
/*CastFP=*/false);
1546+
return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
15361547
}
15371548

15381549
/// Emit a load from an l-value of atomic type. Note that the r-value
@@ -1601,20 +1612,17 @@ llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
16011612
return nullptr;
16021613
}
16031614

1604-
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CastFP) const {
1615+
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
16051616
// If we've got a scalar value of the right size, try to avoid going
16061617
// through memory. Floats get casted if needed by AtomicExpandPass.
16071618
if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1608-
if (isa<llvm::IntegerType>(Value->getType()) ||
1609-
(!CastFP && Value->getType()->isIEEELikeFPTy()))
1619+
if (!shouldCastToInt(Value->getType(), CmpXchg))
16101620
return CGF.EmitToMemory(Value, ValueTy);
16111621
else {
16121622
llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
16131623
CGF.getLLVMContext(),
16141624
LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1615-
if (isa<llvm::PointerType>(Value->getType()))
1616-
return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1617-
else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1625+
if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
16181626
return CGF.Builder.CreateBitCast(Value, InputIntTy);
16191627
}
16201628
}
@@ -1687,13 +1695,14 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
16871695

16881696
// If we've got a scalar value of the right size, try to avoid going
16891697
// through memory.
1690-
auto *ExpectedVal = convertRValueToInt(Expected);
1691-
auto *DesiredVal = convertRValueToInt(Desired);
1698+
auto *ExpectedVal = convertRValueToInt(Expected, /*CmpXchg=*/true);
1699+
auto *DesiredVal = convertRValueToInt(Desired, /*CmpXchg=*/true);
16921700
auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
16931701
Failure, IsWeak);
16941702
return std::make_pair(
16951703
ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1696-
SourceLocation(), /*AsValue=*/false),
1704+
SourceLocation(), /*AsValue=*/false,
1705+
/*CmpXchg=*/true),
16971706
Res.second);
16981707
}
16991708

@@ -1787,7 +1796,7 @@ void AtomicInfo::EmitAtomicUpdateOp(
17871796
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
17881797

17891798
// Do the atomic load.
1790-
auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1799+
auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
17911800
// For non-simple lvalues perform compare-and-swap procedure.
17921801
auto *ContBB = CGF.createBasicBlock("atomic_cont");
17931802
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
@@ -1803,7 +1812,8 @@ void AtomicInfo::EmitAtomicUpdateOp(
18031812
CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
18041813
}
18051814
auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),
1806-
SourceLocation(), /*AsValue=*/false);
1815+
SourceLocation(), /*AsValue=*/false,
1816+
/*CmpXchg=*/true);
18071817
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
18081818
auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
18091819
// Try to write new value using cmpxchg operation.
@@ -1869,7 +1879,7 @@ void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
18691879
auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
18701880

18711881
// Do the atomic load.
1872-
auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1882+
auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile, /*CmpXchg=*/true);
18731883
// For non-simple lvalues perform compare-and-swap procedure.
18741884
auto *ContBB = CGF.createBasicBlock("atomic_cont");
18751885
auto *ExitBB = CGF.createBasicBlock("atomic_exit");
@@ -1969,21 +1979,16 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
19691979
}
19701980

19711981
// Okay, we're doing this natively.
1972-
llvm::Value *ValToStore =
1973-
atomics.convertRValueToInt(rvalue, /*CastFP=*/false);
1982+
llvm::Value *ValToStore = atomics.convertRValueToInt(rvalue);
19741983

19751984
// Do the atomic store.
19761985
Address Addr = atomics.getAtomicAddress();
1977-
bool ShouldCastToInt = true;
19781986
if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
1979-
if (isa<llvm::IntegerType>(Value->getType()) ||
1980-
Value->getType()->isIEEELikeFPTy())
1981-
ShouldCastToInt = false;
1982-
if (ShouldCastToInt) {
1983-
Addr = atomics.castToAtomicIntPointer(Addr);
1984-
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
1985-
/*isSigned=*/false);
1986-
}
1987+
if (shouldCastToInt(Value->getType(), /*CmpXchg=*/false)) {
1988+
Addr = atomics.castToAtomicIntPointer(Addr);
1989+
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
1990+
/*isSigned=*/false);
1991+
}
19871992
llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);
19881993

19891994
if (AO == llvm::AtomicOrdering::Acquire)

‎clang/test/CodeGen/atomic.c

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1-
// RUN: %clang_cc1 %s -emit-llvm -o - -triple=i686-apple-darwin9 | FileCheck %s
1+
// RUN: %clang_cc1 %s -emit-llvm -o - -triple=i686-apple-darwin9 | FileCheck %s --check-prefixes=CHECK,X86
2+
// RUN: %clang_cc1 %s -emit-llvm -o - -triple=s390x-linux-gnu | FileCheck %s --check-prefixes=CHECK,SYSTEMZ
23

34
// CHECK: @[[NONSTATIC_GLOB_POINTER_FROM_INT:.+]] = global ptr null
45
// CHECK: @[[GLOB_POINTER:.+]] = internal global ptr null
56
// CHECK: @[[GLOB_POINTER_FROM_INT:.+]] = internal global ptr null
67
// CHECK: @[[GLOB_INT:.+]] = internal global i32 0
78
// CHECK: @[[GLOB_FLT:.+]] = internal global float {{[0e\+-\.]+}}, align
9+
// CHECK: @[[GLOB_DBL:.+]] = internal global double {{[0e\+-\.]+}}, align
10+
// X86: @[[GLOB_LONGDBL:.+]] = internal global x86_fp80 {{[0xK]+}}, align
11+
// SYSTEMZ: @[[GLOB_LONGDBL:.+]] = internal global fp128 {{[0xL]+}}, align
812

913
int atomic(void) {
1014
// non-sensical test for sync functions
@@ -79,8 +83,10 @@ int atomic(void) {
7983
// CHECK: atomicrmw nand ptr %valc, i8 6 seq_cst, align 1
8084

8185
__sync_val_compare_and_swap((void **)0, (void *)0, (void *)0);
82-
// CHECK: [[PAIR:%[a-z0-9_.]+]] = cmpxchg ptr null, i32 0, i32 0 seq_cst seq_cst, align 4
83-
// CHECK: extractvalue { i32, i1 } [[PAIR]], 0
86+
// X86: [[PAIR:%[a-z0-9_.]+]] = cmpxchg ptr null, i32 0, i32 0 seq_cst seq_cst, align 4
87+
// X86-NEXT: extractvalue { i32, i1 } [[PAIR]], 0
88+
// SYSTEMZ: [[PAIR:%[a-z0-9_.]+]] = cmpxchg ptr null, i64 0, i64 0 seq_cst seq_cst, align 8
89+
// SYSTEMZ-NEXT: extractvalue { i64, i1 } [[PAIR]], 0
8490

8591
if ( __sync_val_compare_and_swap(&valb, 0, 1)) {
8692
// CHECK: [[PAIR:%[a-z0-9_.]+]] = cmpxchg ptr %valb, i8 0, i8 1 seq_cst seq_cst, align 1
@@ -90,13 +96,15 @@ int atomic(void) {
9096
}
9197

9298
__sync_bool_compare_and_swap((void **)0, (void *)0, (void *)0);
93-
// CHECK: cmpxchg ptr null, i32 0, i32 0 seq_cst seq_cst, align 4
99+
// X86: cmpxchg ptr null, i32 0, i32 0 seq_cst seq_cst, align 4
100+
// SYSTEMZ: cmpxchg ptr null, i64 0, i64 0 seq_cst seq_cst, align 8
94101

95102
__sync_lock_release(&val);
96103
// CHECK: store atomic i32 0, {{.*}} release, align 4
97104

98105
__sync_lock_release(&ptrval);
99-
// CHECK: store atomic i32 0, {{.*}} release, align 4
106+
// X86: store atomic i32 0, {{.*}} release, align 4
107+
// SYSTEMZ: store atomic i64 0, {{.*}} release, align 8
100108

101109
__sync_synchronize ();
102110
// CHECK: fence seq_cst
@@ -131,19 +139,25 @@ static _Atomic(int *) glob_pointer_from_int = 0;
131139
_Atomic(int *) nonstatic_glob_pointer_from_int = 0LL;
132140
static _Atomic int glob_int = 0;
133141
static _Atomic float glob_flt = 0.0f;
142+
static _Atomic double glob_dbl = 0.0f;
143+
static _Atomic long double glob_longdbl = 0.0f;
134144

135145
void force_global_uses(void) {
146+
// X86: %atomic-temp = alloca x86_fp80, align 16
136147
(void)glob_pointer;
137-
// CHECK: %[[LOCAL_INT:.+]] = load atomic i32, ptr @[[GLOB_POINTER]] seq_cst
138-
// CHECK-NEXT: inttoptr i32 %[[LOCAL_INT]] to ptr
148+
// CHECK: load atomic ptr, ptr @[[GLOB_POINTER]] seq_cst
139149
(void)glob_pointer_from_int;
140-
// CHECK: %[[LOCAL_INT_2:.+]] = load atomic i32, ptr @[[GLOB_POINTER_FROM_INT]] seq_cst
141-
// CHECK-NEXT: inttoptr i32 %[[LOCAL_INT_2]] to ptr
150+
// CHECK-NEXT: load atomic ptr, ptr @[[GLOB_POINTER_FROM_INT]] seq_cst
142151
(void)nonstatic_glob_pointer_from_int;
143-
// CHECK: %[[LOCAL_INT_3:.+]] = load atomic i32, ptr @[[NONSTATIC_GLOB_POINTER_FROM_INT]] seq_cst
144-
// CHECK-NEXT: inttoptr i32 %[[LOCAL_INT_3]] to ptr
152+
// CHECK-NEXT: load atomic ptr, ptr @[[NONSTATIC_GLOB_POINTER_FROM_INT]] seq_cst
145153
(void)glob_int;
146-
// CHECK: load atomic i32, ptr @[[GLOB_INT]] seq_cst
154+
// CHECK-NEXT: load atomic i32, ptr @[[GLOB_INT]] seq_cst
147155
(void)glob_flt;
148-
// CHECK: load atomic float, ptr @[[GLOB_FLT]] seq_cst
156+
// CHECK-NEXT: load atomic float, ptr @[[GLOB_FLT]] seq_cst
157+
(void)glob_dbl;
158+
// CHECK-NEXT: load atomic double, ptr @[[GLOB_DBL]] seq_cst
159+
(void)glob_longdbl;
160+
// X86: call void @__atomic_load(i32 noundef 16, ptr noundef @glob_longdbl, ptr noundef %atomic-temp
161+
// X86-NEXT: %0 = load x86_fp80, ptr %atomic-temp, align 16
162+
// SYSTEMZ: load atomic fp128, ptr @[[GLOB_LONGDBL]] seq_cst
149163
}

0 commit comments

Comments
 (0)