-
Notifications
You must be signed in to change notification settings - Fork 14.3k
Don't do casting of atomic FP loads/stores in FE. #83446
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -194,12 +194,14 @@ namespace { | |||||
RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, | ||||||
SourceLocation loc, bool AsValue) const; | ||||||
|
||||||
/// Converts a rvalue to integer value. | ||||||
llvm::Value *convertRValueToInt(RValue RVal) const; | ||||||
llvm::Value *getScalarRValValueOrNull(RValue RVal) const; | ||||||
|
||||||
RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, | ||||||
AggValueSlot ResultSlot, | ||||||
SourceLocation Loc, bool AsValue) const; | ||||||
/// Converts an rvalue to integer value if needed. | ||||||
llvm::Value *convertRValueToInt(RValue RVal, bool CastFP = true) const; | ||||||
|
||||||
RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot, | ||||||
SourceLocation Loc, bool AsValue, | ||||||
bool CastFP = true) const; | ||||||
|
||||||
/// Copy an atomic r-value into atomic-layout memory. | ||||||
void emitCopyIntoMemory(RValue rvalue) const; | ||||||
|
@@ -261,7 +263,8 @@ namespace { | |||||
void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, | ||||||
llvm::AtomicOrdering AO, bool IsVolatile); | ||||||
/// Emits atomic load as LLVM instruction. | ||||||
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); | ||||||
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile, | ||||||
bool CastFP = true); | ||||||
/// Emits atomic compare-and-exchange op as a libcall. | ||||||
llvm::Value *EmitAtomicCompareExchangeLibcall( | ||||||
llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, | ||||||
|
@@ -1396,12 +1399,13 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr, | |||||
LVal.getBaseInfo(), TBAAAccessInfo())); | ||||||
} | ||||||
|
||||||
RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, | ||||||
AggValueSlot ResultSlot, | ||||||
SourceLocation Loc, | ||||||
bool AsValue) const { | ||||||
RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, | ||||||
AggValueSlot ResultSlot, | ||||||
SourceLocation Loc, bool AsValue, | ||||||
bool CastFP) const { | ||||||
// Try not to in some easy cases. | ||||||
assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); | ||||||
assert((Val->getType()->isIntegerTy() || Val->getType()->isIEEELikeFPTy()) && | ||||||
"Expected integer or floating point value"); | ||||||
if (getEvaluationKind() == TEK_Scalar && | ||||||
(((!LVal.isBitField() || | ||||||
LVal.getBitFieldInfo().Size == ValueSizeInBits) && | ||||||
|
@@ -1410,13 +1414,14 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, | |||||
auto *ValTy = AsValue | ||||||
? CGF.ConvertTypeForMem(ValueTy) | ||||||
: getAtomicAddress().getElementType(); | ||||||
if (ValTy->isIntegerTy()) { | ||||||
assert(IntVal->getType() == ValTy && "Different integer types."); | ||||||
return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); | ||||||
if (ValTy->isIntegerTy() || (!CastFP && ValTy->isIEEELikeFPTy())) { | ||||||
assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) && | ||||||
"Different integer types."); | ||||||
return RValue::get(CGF.EmitFromMemory(Val, ValueTy)); | ||||||
} else if (ValTy->isPointerTy()) | ||||||
return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); | ||||||
else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) | ||||||
return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); | ||||||
return RValue::get(CGF.Builder.CreateIntToPtr(Val, ValTy)); | ||||||
else if (llvm::CastInst::isBitCastable(Val->getType(), ValTy)) | ||||||
return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy)); | ||||||
} | ||||||
|
||||||
// Create a temporary. This needs to be big enough to hold the | ||||||
|
@@ -1433,8 +1438,7 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, | |||||
|
||||||
// Slam the integer into the temporary. | ||||||
Address CastTemp = castToAtomicIntPointer(Temp); | ||||||
CGF.Builder.CreateStore(IntVal, CastTemp) | ||||||
->setVolatile(TempIsVolatile); | ||||||
CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile); | ||||||
|
||||||
return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); | ||||||
} | ||||||
|
@@ -1453,9 +1457,11 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, | |||||
} | ||||||
|
||||||
llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, | ||||||
bool IsVolatile) { | ||||||
bool IsVolatile, bool CastFP) { | ||||||
// Okay, we're doing this natively. | ||||||
Address Addr = getAtomicAddressAsAtomicIntPointer(); | ||||||
Address Addr = getAtomicAddress(); | ||||||
if (!(Addr.getElementType()->isIEEELikeFPTy() && !CastFP)) | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Still should probably pass through the non-ieee type case through to the IR also, but step at a time I guess |
||||||
Addr = castToAtomicIntPointer(Addr); | ||||||
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); | ||||||
Load->setAtomic(AO); | ||||||
|
||||||
|
@@ -1515,15 +1521,16 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, | |||||
} | ||||||
|
||||||
// Okay, we're doing this natively. | ||||||
auto *Load = EmitAtomicLoadOp(AO, IsVolatile); | ||||||
auto *Load = EmitAtomicLoadOp(AO, IsVolatile, /*CastFP=*/false); | ||||||
|
||||||
// If we're ignoring an aggregate return, don't do anything. | ||||||
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) | ||||||
return RValue::getAggregate(Address::invalid(), false); | ||||||
|
||||||
// Okay, turn that back into the original value or atomic (for non-simple | ||||||
// lvalues) type. | ||||||
return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); | ||||||
return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue, | ||||||
/*CastFP=*/false); | ||||||
} | ||||||
|
||||||
/// Emit a load from an l-value of atomic type. Note that the r-value | ||||||
|
@@ -1586,12 +1593,18 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const { | |||||
return TempLV.getAddress(CGF); | ||||||
} | ||||||
|
||||||
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { | ||||||
llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const { | ||||||
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) | ||||||
return RVal.getScalarVal(); | ||||||
return nullptr; | ||||||
} | ||||||
|
||||||
llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CastFP) const { | ||||||
// If we've got a scalar value of the right size, try to avoid going | ||||||
// through memory. | ||||||
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { | ||||||
llvm::Value *Value = RVal.getScalarVal(); | ||||||
if (isa<llvm::IntegerType>(Value->getType())) | ||||||
// through memory. Floats get casted if needed by AtomicExpandPass. | ||||||
if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) { | ||||||
if (isa<llvm::IntegerType>(Value->getType()) || | ||||||
(!CastFP && Value->getType()->isIEEELikeFPTy())) | ||||||
return CGF.EmitToMemory(Value, ValueTy); | ||||||
else { | ||||||
llvm::IntegerType *InputIntTy = llvm::IntegerType::get( | ||||||
|
@@ -1677,8 +1690,8 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( | |||||
auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, | ||||||
Failure, IsWeak); | ||||||
return std::make_pair( | ||||||
ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), | ||||||
SourceLocation(), /*AsValue=*/false), | ||||||
ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(), | ||||||
SourceLocation(), /*AsValue=*/false), | ||||||
Res.second); | ||||||
} | ||||||
|
||||||
|
@@ -1787,8 +1800,8 @@ void AtomicInfo::EmitAtomicUpdateOp( | |||||
requiresMemSetZero(getAtomicAddress().getElementType())) { | ||||||
CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); | ||||||
} | ||||||
auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), | ||||||
SourceLocation(), /*AsValue=*/false); | ||||||
auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(), | ||||||
SourceLocation(), /*AsValue=*/false); | ||||||
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); | ||||||
auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); | ||||||
// Try to write new value using cmpxchg operation. | ||||||
|
@@ -1953,13 +1966,22 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, | |||||
} | ||||||
|
||||||
// Okay, we're doing this natively. | ||||||
llvm::Value *intValue = atomics.convertRValueToInt(rvalue); | ||||||
llvm::Value *ValToStore = | ||||||
atomics.convertRValueToInt(rvalue, /*CastFP=*/false); | ||||||
|
||||||
// Do the atomic store. | ||||||
Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress()); | ||||||
intValue = Builder.CreateIntCast( | ||||||
intValue, addr.getElementType(), /*isSigned=*/false); | ||||||
llvm::StoreInst *store = Builder.CreateStore(intValue, addr); | ||||||
Address Addr = atomics.getAtomicAddress(); | ||||||
bool ShouldCastToInt = true; | ||||||
if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue)) | ||||||
if (isa<llvm::IntegerType>(Value->getType()) || | ||||||
Value->getType()->isIEEELikeFPTy()) | ||||||
ShouldCastToInt = false; | ||||||
if (ShouldCastToInt) { | ||||||
Comment on lines
+1974
to
+1979
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Factoring this into a shouldCastToInt helper function would be better, but it matters little if you're just going to remove this soon anyway There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for review - I'll try to get back to this soon. |
||||||
Addr = atomics.castToAtomicIntPointer(Addr); | ||||||
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(), | ||||||
/*isSigned=*/false); | ||||||
} | ||||||
llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr); | ||||||
|
||||||
if (AO == llvm::AtomicOrdering::Acquire) | ||||||
AO = llvm::AtomicOrdering::Monotonic; | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,164 @@ | ||
// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s | ||
// | ||
// Test that floating point atomic stores and loads do not get casted to/from | ||
// integer. | ||
|
||
#include <stdatomic.h> | ||
|
||
_Atomic float Af; | ||
_Atomic double Ad; | ||
_Atomic long double Ald; | ||
|
||
//// Atomic stores of floating point values. | ||
void fun0(float Arg) { | ||
// CHECK-LABEL: @fun0 | ||
// CHECK: store atomic float %Arg, ptr @Af seq_cst, align 4 | ||
Af = Arg; | ||
} | ||
|
||
void fun1(double Arg) { | ||
// CHECK-LABEL: @fun1 | ||
// CHECK: store atomic double %Arg, ptr @Ad seq_cst, align 8 | ||
Ad = Arg; | ||
} | ||
|
||
void fun2(long double Arg) { | ||
// CHECK-LABEL: @fun2 | ||
// CHECK: store atomic fp128 %Arg, ptr @Ald seq_cst, align 16 | ||
Ald = Arg; | ||
} | ||
|
||
void fun3(_Atomic float *Dst, float Arg) { | ||
// CHECK-LABEL: @fun | ||
// CHECK: store atomic float %Arg, ptr %Dst seq_cst, align 4 | ||
*Dst = Arg; | ||
} | ||
|
||
void fun4(_Atomic double *Dst, double Arg) { | ||
// CHECK-LABEL: @fun4 | ||
// CHECK: store atomic double %Arg, ptr %Dst seq_cst, align 8 | ||
*Dst = Arg; | ||
} | ||
|
||
void fun5(_Atomic long double *Dst, long double Arg) { | ||
// CHECK-LABEL: @fun5 | ||
// CHECK: store atomic fp128 %Arg, ptr %Dst seq_cst, align 16 | ||
*Dst = Arg; | ||
} | ||
|
||
//// Atomic loads of floating point values. | ||
float fun6() { | ||
// CHECK-LABEL: @fun6 | ||
// CHECK: %atomic-load = load atomic float, ptr @Af seq_cst, align 4 | ||
return Af; | ||
} | ||
|
||
float fun7() { | ||
// CHECK-LABEL: @fun7 | ||
// CHECK: %atomic-load = load atomic double, ptr @Ad seq_cst, align 8 | ||
return Ad; | ||
} | ||
|
||
float fun8() { | ||
// CHECK-LABEL: @fun8 | ||
// CHECK: %atomic-load = load atomic fp128, ptr @Ald seq_cst, align 16 | ||
return Ald; | ||
} | ||
|
||
float fun9(_Atomic float *Src) { | ||
// CHECK-LABEL: @fun9 | ||
// CHECK: %atomic-load = load atomic float, ptr %Src seq_cst, align 4 | ||
return *Src; | ||
} | ||
|
||
double fun10(_Atomic double *Src) { | ||
// CHECK-LABEL: @fun10 | ||
// CHECK: %atomic-load = load atomic double, ptr %Src seq_cst, align 8 | ||
return *Src; | ||
} | ||
|
||
long double fun11(_Atomic long double *Src) { | ||
// CHECK-LABEL: @fun11 | ||
// CHECK: %atomic-load = load atomic fp128, ptr %Src seq_cst, align 16 | ||
return *Src; | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. test the volatile is preserved too? |
||
|
||
//// Same, but with 'volatile' as well: | ||
|
||
_Atomic volatile float Af_vol; | ||
_Atomic volatile double Ad_vol; | ||
_Atomic volatile long double Ald_vol; | ||
|
||
//// Atomic volatile stores of floating point values. | ||
void fun0_vol(float Arg) { | ||
// CHECK-LABEL: @fun0_vol | ||
// CHECK: store atomic volatile float %Arg, ptr @Af_vol seq_cst, align 4 | ||
Af_vol = Arg; | ||
} | ||
|
||
void fun1_vol(double Arg) { | ||
// CHECK-LABEL: @fun1_vol | ||
// CHECK: store atomic volatile double %Arg, ptr @Ad_vol seq_cst, align 8 | ||
Ad_vol = Arg; | ||
} | ||
|
||
void fun2_vol(long double Arg) { | ||
// CHECK-LABEL: @fun2_vol | ||
// CHECK: store atomic volatile fp128 %Arg, ptr @Ald_vol seq_cst, align 16 | ||
Ald_vol = Arg; | ||
} | ||
|
||
void fun3_vol(_Atomic volatile float *Dst, float Arg) { | ||
// CHECK-LABEL: @fun3_vol | ||
// CHECK: store atomic volatile float %Arg, ptr %Dst seq_cst, align 4 | ||
*Dst = Arg; | ||
} | ||
|
||
void fun4_vol(_Atomic volatile double *Dst, double Arg) { | ||
// CHECK-LABEL: @fun4_vol | ||
// CHECK: store atomic volatile double %Arg, ptr %Dst seq_cst, align 8 | ||
*Dst = Arg; | ||
} | ||
|
||
void fun5_vol(_Atomic volatile long double *Dst, long double Arg) { | ||
// CHECK-LABEL: @fun5_vol | ||
// CHECK: store atomic volatile fp128 %Arg, ptr %Dst seq_cst, align 16 | ||
*Dst = Arg; | ||
} | ||
|
||
//// Atomic volatile loads of floating point values. | ||
float fun6_vol() { | ||
// CHECK-LABEL: @fun6_vol | ||
// CHECK: %atomic-load = load atomic volatile float, ptr @Af_vol seq_cst, align 4 | ||
return Af_vol; | ||
} | ||
|
||
float fun7_vol() { | ||
// CHECK-LABEL: @fun7_vol | ||
// CHECK: %atomic-load = load atomic volatile double, ptr @Ad_vol seq_cst, align 8 | ||
return Ad_vol; | ||
} | ||
|
||
float fun8_vol() { | ||
// CHECK-LABEL: @fun8_vol | ||
// CHECK: %atomic-load = load atomic volatile fp128, ptr @Ald_vol seq_cst, align 16 | ||
return Ald_vol; | ||
} | ||
|
||
float fun9_vol(_Atomic volatile float *Src) { | ||
// CHECK-LABEL: @fun9_vol | ||
// CHECK: %atomic-load = load atomic volatile float, ptr %Src seq_cst, align 4 | ||
return *Src; | ||
} | ||
|
||
double fun10_vol(_Atomic volatile double *Src) { | ||
// CHECK-LABEL: @fun10_vol | ||
// CHECK: %atomic-load = load atomic volatile double, ptr %Src seq_cst, align 8 | ||
return *Src; | ||
} | ||
|
||
long double fun11_vol(_Atomic volatile long double *Src) { | ||
// CHECK-LABEL: @fun11_vol | ||
// CHECK: %atomic-load = load atomic volatile fp128, ptr %Src seq_cst, align 16 | ||
return *Src; | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shouldn't be casting pointers either. I think this should pass through all types directly to atomicrmw
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
As AtomicExpandPass doesn't do any casting of AtomicCmpXchg - at least currently - I think the other users of ConvertIntToValueOrAtomic() (which emit AtomicCmpXchg) need the current behavior.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
llvm-project/llvm/lib/CodeGen/AtomicExpandPass.cpp
Line 256 in d1538c1
It's wild how much junk clang ended up with to avoid writing proper backend support. It's fine to just do the FP in this patch, but I think all of this needs to be ripped out
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah, I see, AtomicExpandPass is actually casting pointers to int, but not with the target hook to control it...
I guess I could start with FP per this patch for now...