Skip to content

Commit 9f7ed36

Browse files
JonPsson1JonPsson1
authored
Don't do casting of atomic FP loads/stores in FE. (#83446)
The casting of FP atomic loads and stores were always done by the front-end, even though the AtomicExpandPass will do it if the target requests it (which is the default). This patch removes this casting in the front-end entirely.
1 parent ffd31c5 commit 9f7ed36

File tree

6 files changed

+236
-58
lines changed

6 files changed

+236
-58
lines changed

‎clang/lib/CodeGen/CGAtomic.cpp

Lines changed: 59 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -194,12 +194,14 @@ namespace {
194194
RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195195
SourceLocation loc, bool AsValue) const;
196196

197-
/// Converts a rvalue to integer value.
198-
llvm::Value *convertRValueToInt(RValue RVal) const;
197+
llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
199198

200-
RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201-
AggValueSlot ResultSlot,
202-
SourceLocation Loc, bool AsValue) const;
199+
/// Converts an rvalue to integer value if needed.
200+
llvm::Value *convertRValueToInt(RValue RVal, bool CastFP = true) const;
201+
202+
RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
203+
SourceLocation Loc, bool AsValue,
204+
bool CastFP = true) const;
203205

204206
/// Copy an atomic r-value into atomic-layout memory.
205207
void emitCopyIntoMemory(RValue rvalue) const;
@@ -261,7 +263,8 @@ namespace {
261263
void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262264
llvm::AtomicOrdering AO, bool IsVolatile);
263265
/// Emits atomic load as LLVM instruction.
264-
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
266+
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
267+
bool CastFP = true);
265268
/// Emits atomic compare-and-exchange op as a libcall.
266269
llvm::Value *EmitAtomicCompareExchangeLibcall(
267270
llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
@@ -1396,12 +1399,13 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
13961399
LVal.getBaseInfo(), TBAAAccessInfo()));
13971400
}
13981401

1399-
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1400-
AggValueSlot ResultSlot,
1401-
SourceLocation Loc,
1402-
bool AsValue) const {
1402+
RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1403+
AggValueSlot ResultSlot,
1404+
SourceLocation Loc, bool AsValue,
1405+
bool CastFP) const {
14031406
// Try not to in some easy cases.
1404-
assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1407+
assert((Val->getType()->isIntegerTy() || Val->getType()->isIEEELikeFPTy()) &&
1408+
"Expected integer or floating point value");
14051409
if (getEvaluationKind() == TEK_Scalar &&
14061410
(((!LVal.isBitField() ||
14071411
LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
@@ -1410,13 +1414,14 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
14101414
auto *ValTy = AsValue
14111415
? CGF.ConvertTypeForMem(ValueTy)
14121416
: getAtomicAddress().getElementType();
1413-
if (ValTy->isIntegerTy()) {
1414-
assert(IntVal->getType() == ValTy && "Different integer types.");
1415-
return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1417+
if (ValTy->isIntegerTy() || (!CastFP && ValTy->isIEEELikeFPTy())) {
1418+
assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1419+
"Different integer types.");
1420+
return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
14161421
} else if (ValTy->isPointerTy())
1417-
return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1418-
else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1419-
return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1422+
return RValue::get(CGF.Builder.CreateIntToPtr(Val, ValTy));
1423+
else if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
1424+
return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));
14201425
}
14211426

14221427
// Create a temporary. This needs to be big enough to hold the
@@ -1433,8 +1438,7 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
14331438

14341439
// Slam the integer into the temporary.
14351440
Address CastTemp = castToAtomicIntPointer(Temp);
1436-
CGF.Builder.CreateStore(IntVal, CastTemp)
1437-
->setVolatile(TempIsVolatile);
1441+
CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile);
14381442

14391443
return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
14401444
}
@@ -1453,9 +1457,11 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
14531457
}
14541458

14551459
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1456-
bool IsVolatile) {
1460+
bool IsVolatile, bool CastFP) {
14571461
// Okay, we're doing this natively.
1458-
Address Addr = getAtomicAddressAsAtomicIntPointer();
1462+
Address Addr = getAtomicAddress();
1463+
if (!(Addr.getElementType()->isIEEELikeFPTy() && !CastFP))
1464+
Addr = castToAtomicIntPointer(Addr);
14591465
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
14601466
Load->setAtomic(AO);
14611467

@@ -1515,15 +1521,16 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
15151521
}
15161522

15171523
// Okay, we're doing this natively.
1518-
auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1524+
auto *Load = EmitAtomicLoadOp(AO, IsVolatile, /*CastFP=*/false);
15191525

15201526
// If we're ignoring an aggregate return, don't do anything.
15211527
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
15221528
return RValue::getAggregate(Address::invalid(), false);
15231529

15241530
// Okay, turn that back into the original value or atomic (for non-simple
15251531
// lvalues) type.
1526-
return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1532+
return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue,
1533+
/*CastFP=*/false);
15271534
}
15281535

15291536
/// Emit a load from an l-value of atomic type. Note that the r-value
@@ -1586,12 +1593,18 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const {
15861593
return TempLV.getAddress(CGF);
15871594
}
15881595

1589-
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1596+
llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1597+
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1598+
return RVal.getScalarVal();
1599+
return nullptr;
1600+
}
1601+
1602+
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CastFP) const {
15901603
// If we've got a scalar value of the right size, try to avoid going
1591-
// through memory.
1592-
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1593-
llvm::Value *Value = RVal.getScalarVal();
1594-
if (isa<llvm::IntegerType>(Value->getType()))
1604+
// through memory. Floats get casted if needed by AtomicExpandPass.
1605+
if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1606+
if (isa<llvm::IntegerType>(Value->getType()) ||
1607+
(!CastFP && Value->getType()->isIEEELikeFPTy()))
15951608
return CGF.EmitToMemory(Value, ValueTy);
15961609
else {
15971610
llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
@@ -1677,8 +1690,8 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
16771690
auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
16781691
Failure, IsWeak);
16791692
return std::make_pair(
1680-
ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1681-
SourceLocation(), /*AsValue=*/false),
1693+
ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1694+
SourceLocation(), /*AsValue=*/false),
16821695
Res.second);
16831696
}
16841697

@@ -1787,8 +1800,8 @@ void AtomicInfo::EmitAtomicUpdateOp(
17871800
requiresMemSetZero(getAtomicAddress().getElementType())) {
17881801
CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
17891802
}
1790-
auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1791-
SourceLocation(), /*AsValue=*/false);
1803+
auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),
1804+
SourceLocation(), /*AsValue=*/false);
17921805
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
17931806
auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
17941807
// Try to write new value using cmpxchg operation.
@@ -1953,13 +1966,22 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
19531966
}
19541967

19551968
// Okay, we're doing this natively.
1956-
llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1969+
llvm::Value *ValToStore =
1970+
atomics.convertRValueToInt(rvalue, /*CastFP=*/false);
19571971

19581972
// Do the atomic store.
1959-
Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress());
1960-
intValue = Builder.CreateIntCast(
1961-
intValue, addr.getElementType(), /*isSigned=*/false);
1962-
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1973+
Address Addr = atomics.getAtomicAddress();
1974+
bool ShouldCastToInt = true;
1975+
if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
1976+
if (isa<llvm::IntegerType>(Value->getType()) ||
1977+
Value->getType()->isIEEELikeFPTy())
1978+
ShouldCastToInt = false;
1979+
if (ShouldCastToInt) {
1980+
Addr = atomics.castToAtomicIntPointer(Addr);
1981+
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
1982+
/*isSigned=*/false);
1983+
}
1984+
llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);
19631985

19641986
if (AO == llvm::AtomicOrdering::Acquire)
19651987
AO = llvm::AtomicOrdering::Monotonic;

‎clang/test/CodeGen/SystemZ/atomic_fp_load_store.c

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s
2+
//
3+
// Test that floating point atomic stores and loads do not get casted to/from
4+
// integer.
5+
6+
#include <stdatomic.h>
7+
8+
_Atomic float Af;
9+
_Atomic double Ad;
10+
_Atomic long double Ald;
11+
12+
//// Atomic stores of floating point values.
13+
void fun0(float Arg) {
14+
// CHECK-LABEL: @fun0
15+
// CHECK: store atomic float %Arg, ptr @Af seq_cst, align 4
16+
Af = Arg;
17+
}
18+
19+
void fun1(double Arg) {
20+
// CHECK-LABEL: @fun1
21+
// CHECK: store atomic double %Arg, ptr @Ad seq_cst, align 8
22+
Ad = Arg;
23+
}
24+
25+
void fun2(long double Arg) {
26+
// CHECK-LABEL: @fun2
27+
// CHECK: store atomic fp128 %Arg, ptr @Ald seq_cst, align 16
28+
Ald = Arg;
29+
}
30+
31+
void fun3(_Atomic float *Dst, float Arg) {
32+
// CHECK-LABEL: @fun
33+
// CHECK: store atomic float %Arg, ptr %Dst seq_cst, align 4
34+
*Dst = Arg;
35+
}
36+
37+
void fun4(_Atomic double *Dst, double Arg) {
38+
// CHECK-LABEL: @fun4
39+
// CHECK: store atomic double %Arg, ptr %Dst seq_cst, align 8
40+
*Dst = Arg;
41+
}
42+
43+
void fun5(_Atomic long double *Dst, long double Arg) {
44+
// CHECK-LABEL: @fun5
45+
// CHECK: store atomic fp128 %Arg, ptr %Dst seq_cst, align 16
46+
*Dst = Arg;
47+
}
48+
49+
//// Atomic loads of floating point values.
50+
float fun6() {
51+
// CHECK-LABEL: @fun6
52+
// CHECK: %atomic-load = load atomic float, ptr @Af seq_cst, align 4
53+
return Af;
54+
}
55+
56+
float fun7() {
57+
// CHECK-LABEL: @fun7
58+
// CHECK: %atomic-load = load atomic double, ptr @Ad seq_cst, align 8
59+
return Ad;
60+
}
61+
62+
float fun8() {
63+
// CHECK-LABEL: @fun8
64+
// CHECK: %atomic-load = load atomic fp128, ptr @Ald seq_cst, align 16
65+
return Ald;
66+
}
67+
68+
float fun9(_Atomic float *Src) {
69+
// CHECK-LABEL: @fun9
70+
// CHECK: %atomic-load = load atomic float, ptr %Src seq_cst, align 4
71+
return *Src;
72+
}
73+
74+
double fun10(_Atomic double *Src) {
75+
// CHECK-LABEL: @fun10
76+
// CHECK: %atomic-load = load atomic double, ptr %Src seq_cst, align 8
77+
return *Src;
78+
}
79+
80+
long double fun11(_Atomic long double *Src) {
81+
// CHECK-LABEL: @fun11
82+
// CHECK: %atomic-load = load atomic fp128, ptr %Src seq_cst, align 16
83+
return *Src;
84+
}
85+
86+
//// Same, but with 'volatile' as well:
87+
88+
_Atomic volatile float Af_vol;
89+
_Atomic volatile double Ad_vol;
90+
_Atomic volatile long double Ald_vol;
91+
92+
//// Atomic volatile stores of floating point values.
93+
void fun0_vol(float Arg) {
94+
// CHECK-LABEL: @fun0_vol
95+
// CHECK: store atomic volatile float %Arg, ptr @Af_vol seq_cst, align 4
96+
Af_vol = Arg;
97+
}
98+
99+
void fun1_vol(double Arg) {
100+
// CHECK-LABEL: @fun1_vol
101+
// CHECK: store atomic volatile double %Arg, ptr @Ad_vol seq_cst, align 8
102+
Ad_vol = Arg;
103+
}
104+
105+
void fun2_vol(long double Arg) {
106+
// CHECK-LABEL: @fun2_vol
107+
// CHECK: store atomic volatile fp128 %Arg, ptr @Ald_vol seq_cst, align 16
108+
Ald_vol = Arg;
109+
}
110+
111+
void fun3_vol(_Atomic volatile float *Dst, float Arg) {
112+
// CHECK-LABEL: @fun3_vol
113+
// CHECK: store atomic volatile float %Arg, ptr %Dst seq_cst, align 4
114+
*Dst = Arg;
115+
}
116+
117+
void fun4_vol(_Atomic volatile double *Dst, double Arg) {
118+
// CHECK-LABEL: @fun4_vol
119+
// CHECK: store atomic volatile double %Arg, ptr %Dst seq_cst, align 8
120+
*Dst = Arg;
121+
}
122+
123+
void fun5_vol(_Atomic volatile long double *Dst, long double Arg) {
124+
// CHECK-LABEL: @fun5_vol
125+
// CHECK: store atomic volatile fp128 %Arg, ptr %Dst seq_cst, align 16
126+
*Dst = Arg;
127+
}
128+
129+
//// Atomic volatile loads of floating point values.
130+
float fun6_vol() {
131+
// CHECK-LABEL: @fun6_vol
132+
// CHECK: %atomic-load = load atomic volatile float, ptr @Af_vol seq_cst, align 4
133+
return Af_vol;
134+
}
135+
136+
float fun7_vol() {
137+
// CHECK-LABEL: @fun7_vol
138+
// CHECK: %atomic-load = load atomic volatile double, ptr @Ad_vol seq_cst, align 8
139+
return Ad_vol;
140+
}
141+
142+
float fun8_vol() {
143+
// CHECK-LABEL: @fun8_vol
144+
// CHECK: %atomic-load = load atomic volatile fp128, ptr @Ald_vol seq_cst, align 16
145+
return Ald_vol;
146+
}
147+
148+
float fun9_vol(_Atomic volatile float *Src) {
149+
// CHECK-LABEL: @fun9_vol
150+
// CHECK: %atomic-load = load atomic volatile float, ptr %Src seq_cst, align 4
151+
return *Src;
152+
}
153+
154+
double fun10_vol(_Atomic volatile double *Src) {
155+
// CHECK-LABEL: @fun10_vol
156+
// CHECK: %atomic-load = load atomic volatile double, ptr %Src seq_cst, align 8
157+
return *Src;
158+
}
159+
160+
long double fun11_vol(_Atomic volatile long double *Src) {
161+
// CHECK-LABEL: @fun11_vol
162+
// CHECK: %atomic-load = load atomic volatile fp128, ptr %Src seq_cst, align 16
163+
return *Src;
164+
}

‎clang/test/CodeGen/atomic.c

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,5 @@ void force_global_uses(void) {
145145
(void)glob_int;
146146
// CHECK: load atomic i32, ptr @[[GLOB_INT]] seq_cst
147147
(void)glob_flt;
148-
// CHECK: %[[LOCAL_FLT:.+]] = load atomic i32, ptr @[[GLOB_FLT]] seq_cst
149-
// CHECK-NEXT: bitcast i32 %[[LOCAL_FLT]] to float
148+
// CHECK: load atomic float, ptr @[[GLOB_FLT]] seq_cst
150149
}

‎clang/test/CodeGen/c11atomics-ios.c

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,13 @@ void testFloat(_Atomic(float) *fp) {
1919
_Atomic(float) x = 2.0f;
2020

2121
// CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[FP]]
22-
// CHECK-NEXT: [[T2:%.*]] = load atomic i32, ptr [[T0]] seq_cst, align 4
23-
// CHECK-NEXT: [[T3:%.*]] = bitcast i32 [[T2]] to float
24-
// CHECK-NEXT: store float [[T3]], ptr [[F]]
22+
// CHECK-NEXT: [[T2:%.*]] = load atomic float, ptr [[T0]] seq_cst, align 4
23+
// CHECK-NEXT: store float [[T2]], ptr [[F]]
2524
float f = *fp;
2625

2726
// CHECK-NEXT: [[T0:%.*]] = load float, ptr [[F]], align 4
2827
// CHECK-NEXT: [[T1:%.*]] = load ptr, ptr [[FP]], align 4
29-
// CHECK-NEXT: [[T2:%.*]] = bitcast float [[T0]] to i32
30-
// CHECK-NEXT: store atomic i32 [[T2]], ptr [[T1]] seq_cst, align 4
28+
// CHECK-NEXT: store atomic float [[T0]], ptr [[T1]] seq_cst, align 4
3129
*fp = f;
3230

3331
// CHECK-NEXT: ret void

0 commit comments

Comments
 (0)