Skip to content

Don't do casting of atomic FP loads/stores in FE. #83446

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 59 additions & 37 deletions clang/lib/CodeGen/CGAtomic.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -194,12 +194,14 @@ namespace {
RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
SourceLocation loc, bool AsValue) const;

/// Converts a rvalue to integer value.
llvm::Value *convertRValueToInt(RValue RVal) const;
llvm::Value *getScalarRValValueOrNull(RValue RVal) const;

RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
AggValueSlot ResultSlot,
SourceLocation Loc, bool AsValue) const;
/// Converts an rvalue to integer value if needed.
llvm::Value *convertRValueToInt(RValue RVal, bool CastFP = true) const;

RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
SourceLocation Loc, bool AsValue,
bool CastFP = true) const;

/// Copy an atomic r-value into atomic-layout memory.
void emitCopyIntoMemory(RValue rvalue) const;
Expand Down Expand Up @@ -261,7 +263,8 @@ namespace {
void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
llvm::AtomicOrdering AO, bool IsVolatile);
/// Emits atomic load as LLVM instruction.
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
bool CastFP = true);
/// Emits atomic compare-and-exchange op as a libcall.
llvm::Value *EmitAtomicCompareExchangeLibcall(
llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
Expand Down Expand Up @@ -1396,12 +1399,13 @@ RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
LVal.getBaseInfo(), TBAAAccessInfo()));
}

RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
AggValueSlot ResultSlot,
SourceLocation Loc,
bool AsValue) const {
RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
AggValueSlot ResultSlot,
SourceLocation Loc, bool AsValue,
bool CastFP) const {
// Try not to in some easy cases.
assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
assert((Val->getType()->isIntegerTy() || Val->getType()->isIEEELikeFPTy()) &&
"Expected integer or floating point value");
if (getEvaluationKind() == TEK_Scalar &&
(((!LVal.isBitField() ||
LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
Expand All @@ -1410,13 +1414,14 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
auto *ValTy = AsValue
? CGF.ConvertTypeForMem(ValueTy)
: getAtomicAddress().getElementType();
if (ValTy->isIntegerTy()) {
assert(IntVal->getType() == ValTy && "Different integer types.");
return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
if (ValTy->isIntegerTy() || (!CastFP && ValTy->isIEEELikeFPTy())) {
assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
"Different integer types.");
return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
} else if (ValTy->isPointerTy())
return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
return RValue::get(CGF.Builder.CreateIntToPtr(Val, ValTy));
else if (llvm::CastInst::isBitCastable(Val->getType(), ValTy))
return RValue::get(CGF.Builder.CreateBitCast(Val, ValTy));
Comment on lines +1420 to +1424
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't be casting pointers either. I think this should pass through all types directly to atomicrmw

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As AtomicExpandPass doesn't do any casting of AtomicCmpXchg - at least currently - I think the other users of ConvertIntToValueOrAtomic() (which emit AtomicCmpXchg) need the current behavior.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if (CASI->getCompareOperand()->getType()->isPointerTy()) {

It's wild how much junk clang ended up with to avoid writing proper backend support. It's fine to just do the FP in this patch, but I think all of this needs to be ripped out

Copy link
Contributor Author

@JonPsson1 JonPsson1 Mar 1, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I see, AtomicExpandPass is actually casting pointers to int, but not with the target hook to control it...

I guess I could start with FP per this patch for now...

}

// Create a temporary. This needs to be big enough to hold the
Expand All @@ -1433,8 +1438,7 @@ RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,

// Slam the integer into the temporary.
Address CastTemp = castToAtomicIntPointer(Temp);
CGF.Builder.CreateStore(IntVal, CastTemp)
->setVolatile(TempIsVolatile);
CGF.Builder.CreateStore(Val, CastTemp)->setVolatile(TempIsVolatile);

return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
}
Expand All @@ -1453,9 +1457,11 @@ void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
}

llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
bool IsVolatile) {
bool IsVolatile, bool CastFP) {
// Okay, we're doing this natively.
Address Addr = getAtomicAddressAsAtomicIntPointer();
Address Addr = getAtomicAddress();
if (!(Addr.getElementType()->isIEEELikeFPTy() && !CastFP))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (!(Addr.getElementType()->isIEEELikeFPTy() && !CastFP))
if (CastFP || !Addr.getElementType()->isIEEELikeFPTy())

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still should probably pass through the non-ieee type case through to the IR also, but step at a time I guess

Addr = castToAtomicIntPointer(Addr);
llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
Load->setAtomic(AO);

Expand Down Expand Up @@ -1515,15 +1521,16 @@ RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
}

// Okay, we're doing this natively.
auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
auto *Load = EmitAtomicLoadOp(AO, IsVolatile, /*CastFP=*/false);

// If we're ignoring an aggregate return, don't do anything.
if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
return RValue::getAggregate(Address::invalid(), false);

// Okay, turn that back into the original value or atomic (for non-simple
// lvalues) type.
return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
return ConvertToValueOrAtomic(Load, ResultSlot, Loc, AsValue,
/*CastFP=*/false);
}

/// Emit a load from an l-value of atomic type. Note that the r-value
Expand Down Expand Up @@ -1586,12 +1593,18 @@ Address AtomicInfo::materializeRValue(RValue rvalue) const {
return TempLV.getAddress(CGF);
}

llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
return RVal.getScalarVal();
return nullptr;
}

llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CastFP) const {
// If we've got a scalar value of the right size, try to avoid going
// through memory.
if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
llvm::Value *Value = RVal.getScalarVal();
if (isa<llvm::IntegerType>(Value->getType()))
// through memory. Floats get casted if needed by AtomicExpandPass.
if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
if (isa<llvm::IntegerType>(Value->getType()) ||
(!CastFP && Value->getType()->isIEEELikeFPTy()))
return CGF.EmitToMemory(Value, ValueTy);
else {
llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
Expand Down Expand Up @@ -1677,8 +1690,8 @@ std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
Failure, IsWeak);
return std::make_pair(
ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false),
ConvertToValueOrAtomic(Res.first, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false),
Res.second);
}

Expand Down Expand Up @@ -1787,8 +1800,8 @@ void AtomicInfo::EmitAtomicUpdateOp(
requiresMemSetZero(getAtomicAddress().getElementType())) {
CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
}
auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false);
auto OldRVal = ConvertToValueOrAtomic(PHI, AggValueSlot::ignored(),
SourceLocation(), /*AsValue=*/false);
EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
// Try to write new value using cmpxchg operation.
Expand Down Expand Up @@ -1953,13 +1966,22 @@ void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
}

// Okay, we're doing this natively.
llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
llvm::Value *ValToStore =
atomics.convertRValueToInt(rvalue, /*CastFP=*/false);

// Do the atomic store.
Address addr = atomics.castToAtomicIntPointer(atomics.getAtomicAddress());
intValue = Builder.CreateIntCast(
intValue, addr.getElementType(), /*isSigned=*/false);
llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
Address Addr = atomics.getAtomicAddress();
bool ShouldCastToInt = true;
if (llvm::Value *Value = atomics.getScalarRValValueOrNull(rvalue))
if (isa<llvm::IntegerType>(Value->getType()) ||
Value->getType()->isIEEELikeFPTy())
ShouldCastToInt = false;
if (ShouldCastToInt) {
Comment on lines +1974 to +1979
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Factoring this into a shouldCastToInt helper function would be better, but it matters little if you're just going to remove this soon anyway

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for review - I'll try to get back to this soon.

Addr = atomics.castToAtomicIntPointer(Addr);
ValToStore = Builder.CreateIntCast(ValToStore, Addr.getElementType(),
/*isSigned=*/false);
}
llvm::StoreInst *store = Builder.CreateStore(ValToStore, Addr);

if (AO == llvm::AtomicOrdering::Acquire)
AO = llvm::AtomicOrdering::Monotonic;
Expand Down
164 changes: 164 additions & 0 deletions clang/test/CodeGen/SystemZ/atomic_fp_load_store.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
// RUN: %clang_cc1 -triple s390x-linux-gnu -O1 -emit-llvm %s -o - | FileCheck %s
//
// Test that floating point atomic stores and loads do not get casted to/from
// integer.

#include <stdatomic.h>

_Atomic float Af;
_Atomic double Ad;
_Atomic long double Ald;

//// Atomic stores of floating point values.
void fun0(float Arg) {
// CHECK-LABEL: @fun0
// CHECK: store atomic float %Arg, ptr @Af seq_cst, align 4
Af = Arg;
}

void fun1(double Arg) {
// CHECK-LABEL: @fun1
// CHECK: store atomic double %Arg, ptr @Ad seq_cst, align 8
Ad = Arg;
}

void fun2(long double Arg) {
// CHECK-LABEL: @fun2
// CHECK: store atomic fp128 %Arg, ptr @Ald seq_cst, align 16
Ald = Arg;
}

void fun3(_Atomic float *Dst, float Arg) {
// CHECK-LABEL: @fun
// CHECK: store atomic float %Arg, ptr %Dst seq_cst, align 4
*Dst = Arg;
}

void fun4(_Atomic double *Dst, double Arg) {
// CHECK-LABEL: @fun4
// CHECK: store atomic double %Arg, ptr %Dst seq_cst, align 8
*Dst = Arg;
}

void fun5(_Atomic long double *Dst, long double Arg) {
// CHECK-LABEL: @fun5
// CHECK: store atomic fp128 %Arg, ptr %Dst seq_cst, align 16
*Dst = Arg;
}

//// Atomic loads of floating point values.
float fun6() {
// CHECK-LABEL: @fun6
// CHECK: %atomic-load = load atomic float, ptr @Af seq_cst, align 4
return Af;
}

float fun7() {
// CHECK-LABEL: @fun7
// CHECK: %atomic-load = load atomic double, ptr @Ad seq_cst, align 8
return Ad;
}

float fun8() {
// CHECK-LABEL: @fun8
// CHECK: %atomic-load = load atomic fp128, ptr @Ald seq_cst, align 16
return Ald;
}

float fun9(_Atomic float *Src) {
// CHECK-LABEL: @fun9
// CHECK: %atomic-load = load atomic float, ptr %Src seq_cst, align 4
return *Src;
}

double fun10(_Atomic double *Src) {
// CHECK-LABEL: @fun10
// CHECK: %atomic-load = load atomic double, ptr %Src seq_cst, align 8
return *Src;
}

long double fun11(_Atomic long double *Src) {
// CHECK-LABEL: @fun11
// CHECK: %atomic-load = load atomic fp128, ptr %Src seq_cst, align 16
return *Src;
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test the volatile is preserved too?


//// Same, but with 'volatile' as well:

_Atomic volatile float Af_vol;
_Atomic volatile double Ad_vol;
_Atomic volatile long double Ald_vol;

//// Atomic volatile stores of floating point values.
void fun0_vol(float Arg) {
// CHECK-LABEL: @fun0_vol
// CHECK: store atomic volatile float %Arg, ptr @Af_vol seq_cst, align 4
Af_vol = Arg;
}

void fun1_vol(double Arg) {
// CHECK-LABEL: @fun1_vol
// CHECK: store atomic volatile double %Arg, ptr @Ad_vol seq_cst, align 8
Ad_vol = Arg;
}

void fun2_vol(long double Arg) {
// CHECK-LABEL: @fun2_vol
// CHECK: store atomic volatile fp128 %Arg, ptr @Ald_vol seq_cst, align 16
Ald_vol = Arg;
}

void fun3_vol(_Atomic volatile float *Dst, float Arg) {
// CHECK-LABEL: @fun3_vol
// CHECK: store atomic volatile float %Arg, ptr %Dst seq_cst, align 4
*Dst = Arg;
}

void fun4_vol(_Atomic volatile double *Dst, double Arg) {
// CHECK-LABEL: @fun4_vol
// CHECK: store atomic volatile double %Arg, ptr %Dst seq_cst, align 8
*Dst = Arg;
}

void fun5_vol(_Atomic volatile long double *Dst, long double Arg) {
// CHECK-LABEL: @fun5_vol
// CHECK: store atomic volatile fp128 %Arg, ptr %Dst seq_cst, align 16
*Dst = Arg;
}

//// Atomic volatile loads of floating point values.
float fun6_vol() {
// CHECK-LABEL: @fun6_vol
// CHECK: %atomic-load = load atomic volatile float, ptr @Af_vol seq_cst, align 4
return Af_vol;
}

float fun7_vol() {
// CHECK-LABEL: @fun7_vol
// CHECK: %atomic-load = load atomic volatile double, ptr @Ad_vol seq_cst, align 8
return Ad_vol;
}

float fun8_vol() {
// CHECK-LABEL: @fun8_vol
// CHECK: %atomic-load = load atomic volatile fp128, ptr @Ald_vol seq_cst, align 16
return Ald_vol;
}

float fun9_vol(_Atomic volatile float *Src) {
// CHECK-LABEL: @fun9_vol
// CHECK: %atomic-load = load atomic volatile float, ptr %Src seq_cst, align 4
return *Src;
}

double fun10_vol(_Atomic volatile double *Src) {
// CHECK-LABEL: @fun10_vol
// CHECK: %atomic-load = load atomic volatile double, ptr %Src seq_cst, align 8
return *Src;
}

long double fun11_vol(_Atomic volatile long double *Src) {
// CHECK-LABEL: @fun11_vol
// CHECK: %atomic-load = load atomic volatile fp128, ptr %Src seq_cst, align 16
return *Src;
}
3 changes: 1 addition & 2 deletions clang/test/CodeGen/atomic.c
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,5 @@ void force_global_uses(void) {
(void)glob_int;
// CHECK: load atomic i32, ptr @[[GLOB_INT]] seq_cst
(void)glob_flt;
// CHECK: %[[LOCAL_FLT:.+]] = load atomic i32, ptr @[[GLOB_FLT]] seq_cst
// CHECK-NEXT: bitcast i32 %[[LOCAL_FLT]] to float
// CHECK: load atomic float, ptr @[[GLOB_FLT]] seq_cst
}
8 changes: 3 additions & 5 deletions clang/test/CodeGen/c11atomics-ios.c
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,13 @@ void testFloat(_Atomic(float) *fp) {
_Atomic(float) x = 2.0f;

// CHECK-NEXT: [[T0:%.*]] = load ptr, ptr [[FP]]
// CHECK-NEXT: [[T2:%.*]] = load atomic i32, ptr [[T0]] seq_cst, align 4
// CHECK-NEXT: [[T3:%.*]] = bitcast i32 [[T2]] to float
// CHECK-NEXT: store float [[T3]], ptr [[F]]
// CHECK-NEXT: [[T2:%.*]] = load atomic float, ptr [[T0]] seq_cst, align 4
// CHECK-NEXT: store float [[T2]], ptr [[F]]
float f = *fp;

// CHECK-NEXT: [[T0:%.*]] = load float, ptr [[F]], align 4
// CHECK-NEXT: [[T1:%.*]] = load ptr, ptr [[FP]], align 4
// CHECK-NEXT: [[T2:%.*]] = bitcast float [[T0]] to i32
// CHECK-NEXT: store atomic i32 [[T2]], ptr [[T1]] seq_cst, align 4
// CHECK-NEXT: store atomic float [[T0]], ptr [[T1]] seq_cst, align 4
*fp = f;

// CHECK-NEXT: ret void
Expand Down
Loading