Skip to content

Use Address for CGBuilder's CreateAtomicRMW and CreateAtomicCmpXchg. #74349

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 4, 2023
Merged

Use Address for CGBuilder's CreateAtomicRMW and CreateAtomicCmpXchg. #74349

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fd7ebb
Use Address for CGBuilder's CreateAtomicRMW and CreateAtomicCmpXchg.
jyknight Dec 4, 2023
654911b
Fix formatting
jyknight Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions clang/lib/CodeGen/CGAtomic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,7 @@ static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);

llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
Scope);
Ptr, Expected, Desired, SuccessOrder, FailureOrder, Scope);
Pair->setVolatile(E->isVolatile());
Pair->setWeak(IsWeak);

Expand Down Expand Up @@ -699,7 +698,7 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,

llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
llvm::AtomicRMWInst *RMWI =
CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order, Scope);
RMWI->setVolatile(E->isVolatile());

// For __atomic_*_fetch operations, perform the operation again to
Expand Down Expand Up @@ -1740,8 +1739,7 @@ std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
// Do the atomic store.
Address Addr = getAtomicAddressAsAtomicIntPointer();
auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
ExpectedVal, DesiredVal,
auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, ExpectedVal, DesiredVal,
Success, Failure);
// Other decoration.
Inst->setVolatile(LVal.isVolatileQualified());
Expand Down
17 changes: 7 additions & 10 deletions clang/lib/CodeGen/CGBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,25 +126,22 @@ class CGBuilderTy : public CGBuilderBaseTy {
return CreateAlignedStore(getInt1(Value), Addr, CharUnits::One());
}

// Temporarily use old signature; clang will be updated to an Address overload
// in a subsequent patch.
llvm::AtomicCmpXchgInst *
CreateAtomicCmpXchg(llvm::Value *Ptr, llvm::Value *Cmp, llvm::Value *New,
CreateAtomicCmpXchg(Address Addr, llvm::Value *Cmp, llvm::Value *New,
llvm::AtomicOrdering SuccessOrdering,
llvm::AtomicOrdering FailureOrdering,
llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
return CGBuilderBaseTy::CreateAtomicCmpXchg(
Ptr, Cmp, New, llvm::MaybeAlign(), SuccessOrdering, FailureOrdering,
SSID);
Addr.getPointer(), Cmp, New, Addr.getAlignment().getAsAlign(),
SuccessOrdering, FailureOrdering, SSID);
}

// Temporarily use old signature; clang will be updated to an Address overload
// in a subsequent patch.
llvm::AtomicRMWInst *
CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, llvm::Value *Ptr,
llvm::Value *Val, llvm::AtomicOrdering Ordering,
CreateAtomicRMW(llvm::AtomicRMWInst::BinOp Op, Address Addr, llvm::Value *Val,
llvm::AtomicOrdering Ordering,
llvm::SyncScope::ID SSID = llvm::SyncScope::System) {
return CGBuilderBaseTy::CreateAtomicRMW(Op, Ptr, Val, llvm::MaybeAlign(),
return CGBuilderBaseTy::CreateAtomicRMW(Op, Addr.getPointer(), Val,
Addr.getAlignment().getAsAlign(),
Ordering, SSID);
}

Expand Down
136 changes: 67 additions & 69 deletions clang/lib/CodeGen/CGBuiltin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,7 @@ static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
return V;
}

static llvm::Value *CheckAtomicAlignment(CodeGenFunction &CGF,
const CallExpr *E) {
static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
ASTContext &Ctx = CGF.getContext();
Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
unsigned Bytes = Ptr.getElementType()->isPointerTy()
Expand All @@ -199,8 +198,10 @@ static llvm::Value *CheckAtomicAlignment(CodeGenFunction &CGF,
if (Align % Bytes != 0) {
DiagnosticsEngine &Diags = CGF.CGM.getDiags();
Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
// Force address to be at least naturally-aligned.
return Ptr.withAlignment(CharUnits::fromQuantity(Bytes));
}
return Ptr.getPointer();
return Ptr;
}

/// Utility to insert an atomic instruction based on Intrinsic::ID
Expand All @@ -215,19 +216,17 @@ static Value *MakeBinaryAtomicValue(
E->getArg(0)->getType()->getPointeeType()));
assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));

llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
Address DestAddr = CheckAtomicAlignment(CGF, E);

llvm::IntegerType *IntType = llvm::IntegerType::get(
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));

llvm::Value *Args[2];
Args[0] = DestPtr;
Args[1] = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Args[1]->getType();
Args[1] = EmitToInt(CGF, Args[1], T, IntType);
llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Val->getType();
Val = EmitToInt(CGF, Val, T, IntType);

llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
Kind, Args[0], Args[1], Ordering);
llvm::Value *Result =
CGF.Builder.CreateAtomicRMW(Kind, DestAddr, Val, Ordering);
return EmitFromInt(CGF, Result, T, ValueType);
}

Expand Down Expand Up @@ -270,20 +269,18 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
E->getArg(0)->getType()->getPointeeType()));
assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));

llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
Address DestAddr = CheckAtomicAlignment(CGF, E);

llvm::IntegerType *IntType = llvm::IntegerType::get(
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));

llvm::Value *Args[2];
Args[1] = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Args[1]->getType();
Args[1] = EmitToInt(CGF, Args[1], T, IntType);
Args[0] = DestPtr;
llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Val->getType();
Val = EmitToInt(CGF, Val, T, IntType);

llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
Kind, Args[0], Args[1], llvm::AtomicOrdering::SequentiallyConsistent);
Result = CGF.Builder.CreateBinOp(Op, Result, Args[1]);
Kind, DestAddr, Val, llvm::AtomicOrdering::SequentiallyConsistent);
Result = CGF.Builder.CreateBinOp(Op, Result, Val);
if (Invert)
Result =
CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
Expand All @@ -309,20 +306,18 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
bool ReturnBool) {
QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
Address DestAddr = CheckAtomicAlignment(CGF, E);

llvm::IntegerType *IntType = llvm::IntegerType::get(
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));

Value *Args[3];
Args[0] = DestPtr;
Args[1] = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Args[1]->getType();
Args[1] = EmitToInt(CGF, Args[1], T, IntType);
Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
llvm::Type *ValueType = Cmp->getType();
Cmp = EmitToInt(CGF, Cmp, T, IntType);
Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);

Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
Args[0], Args[1], Args[2], llvm::AtomicOrdering::SequentiallyConsistent,
DestAddr, Cmp, New, llvm::AtomicOrdering::SequentiallyConsistent,
llvm::AtomicOrdering::SequentiallyConsistent);
if (ReturnBool)
// Extract boolean success flag and zext it to int.
Expand Down Expand Up @@ -358,7 +353,8 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
E->getArg(2)->getType()));

auto *Destination = CGF.EmitScalarExpr(E->getArg(0));
Address DestAddr = CheckAtomicAlignment(CGF, E);

auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));

Expand All @@ -372,8 +368,7 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
auto *Result = CGF.Builder.CreateAtomicCmpXchg(
Destination, Comparand, Exchange,
SuccessOrdering, FailureOrdering);
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
Result->setVolatile(true);
return CGF.Builder.CreateExtractValue(Result, 0);
}
Expand All @@ -386,29 +381,34 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// __int64 _ExchangeHigh,
// __int64 _ExchangeLow,
// __int64 * _ComparandResult);
//
// Note that Destination is assumed to be at least 16-byte aligned, despite
// being typed int64.

static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
const CallExpr *E,
AtomicOrdering SuccessOrdering) {
assert(E->getNumArgs() == 4);
llvm::Value *Destination = CGF.EmitScalarExpr(E->getArg(0));
llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
llvm::Value *ComparandPtr = CGF.EmitScalarExpr(E->getArg(3));
Address ComparandAddr = CGF.EmitPointerWithAlignment(E->getArg(3));

assert(Destination->getType()->isPointerTy());
assert(DestPtr->getType()->isPointerTy());
assert(!ExchangeHigh->getType()->isPointerTy());
assert(!ExchangeLow->getType()->isPointerTy());
assert(ComparandPtr->getType()->isPointerTy());

// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
? AtomicOrdering::Monotonic
: SuccessOrdering;

// Convert to i128 pointers and values.
// Convert to i128 pointers and values. Alignment is also overridden for
// destination pointer.
llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);
Address ComparandResult(ComparandPtr, Int128Ty,
CGF.getContext().toCharUnitsFromBits(128));
Address DestAddr(DestPtr, Int128Ty,
CGF.getContext().toCharUnitsFromBits(128));
ComparandAddr = ComparandAddr.withElementType(Int128Ty);

// (((i128)hi) << 64) | ((i128)lo)
ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
Expand All @@ -418,9 +418,9 @@ static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);

// Load the comparand for the instruction.
llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandResult);
llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr);

auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
SuccessOrdering, FailureOrdering);

// The atomic instruction is marked volatile for consistency with MSVC. This
Expand All @@ -431,7 +431,7 @@ static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,

// Store the result as an outparameter.
CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
ComparandResult);
ComparandAddr);

// Get the success boolean and zero extend it to i8.
Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
Expand All @@ -443,24 +443,21 @@ static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
assert(E->getArg(0)->getType()->isPointerType());

auto *IntTy = CGF.ConvertType(E->getType());
Address DestAddr = CheckAtomicAlignment(CGF, E);
auto *Result = CGF.Builder.CreateAtomicRMW(
AtomicRMWInst::Add,
CGF.EmitScalarExpr(E->getArg(0)),
ConstantInt::get(IntTy, 1),
Ordering);
AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
}

static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E,
static Value *EmitAtomicDecrementValue(
CodeGenFunction &CGF, const CallExpr *E,
AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
assert(E->getArg(0)->getType()->isPointerType());

auto *IntTy = CGF.ConvertType(E->getType());
Address DestAddr = CheckAtomicAlignment(CGF, E);
auto *Result = CGF.Builder.CreateAtomicRMW(
AtomicRMWInst::Sub,
CGF.EmitScalarExpr(E->getArg(0)),
ConstantInt::get(IntTy, 1),
Ordering);
AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
}

Expand Down Expand Up @@ -1215,8 +1212,7 @@ static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
Mask = CGF.Builder.CreateNot(Mask);
RMWOp = llvm::AtomicRMWInst::And;
}
OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
Ordering);
OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
} else {
// Emit a plain load for the non-interlocked intrinsics.
OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
Expand Down Expand Up @@ -4456,14 +4452,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BI__sync_lock_release_4:
case Builtin::BI__sync_lock_release_8:
case Builtin::BI__sync_lock_release_16: {
Value *Ptr = CheckAtomicAlignment(*this, E);
Address Ptr = CheckAtomicAlignment(*this, E);
QualType ElTy = E->getArg(0)->getType()->getPointeeType();
CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
llvm::Type *ITy =
llvm::IntegerType::get(getLLVMContext(), StoreSize.getQuantity() * 8);

llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
getContext().getTypeSize(ElTy));
llvm::StoreInst *Store =
Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
StoreSize);
Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
Store->setAtomic(llvm::AtomicOrdering::Release);
return RValue::get(nullptr);
}
Expand Down Expand Up @@ -4514,7 +4509,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
bool Volatile =
PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();

Value *Ptr = EmitScalarExpr(E->getArg(0));
Address Ptr =
EmitPointerWithAlignment(E->getArg(0)).withElementType(Int8Ty);

Value *NewVal = Builder.getInt8(1);
Value *Order = EmitScalarExpr(E->getArg(1));
if (isa<llvm::ConstantInt>(Order)) {
Expand Down Expand Up @@ -5035,7 +5032,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
llvm::IntegerType *IntType = IntegerType::get(
getLLVMContext(), getContext().getTypeSize(E->getType()));

llvm::Value *Destination = EmitScalarExpr(E->getArg(0));
Address DestAddr = CheckAtomicAlignment(*this, E);

llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
RTy = Exchange->getType();
Expand All @@ -5048,7 +5045,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;

auto Result = Builder.CreateAtomicCmpXchg(Destination, Comparand, Exchange,
auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
Ordering, Ordering);
Result->setVolatile(true);

Expand Down Expand Up @@ -11901,12 +11898,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
}

case clang::AArch64::BI_InterlockedAdd: {
Value *Arg0 = EmitScalarExpr(E->getArg(0));
Value *Arg1 = EmitScalarExpr(E->getArg(1));
AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
AtomicRMWInst::Add, Arg0, Arg1,
llvm::AtomicOrdering::SequentiallyConsistent);
return Builder.CreateAdd(RMWI, Arg1);
Address DestAddr = CheckAtomicAlignment(*this, E);
Value *Val = EmitScalarExpr(E->getArg(1));
AtomicRMWInst *RMWI =
Builder.CreateAtomicRMW(AtomicRMWInst::Add, DestAddr, Val,
llvm::AtomicOrdering::SequentiallyConsistent);
return Builder.CreateAdd(RMWI, Val);
}
}

Expand Down Expand Up @@ -18219,7 +18216,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
break;
}

Value *Ptr = EmitScalarExpr(E->getArg(0));
Address Ptr = CheckAtomicAlignment(*this, E);
Value *Val = EmitScalarExpr(E->getArg(1));

ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
Expand Down Expand Up @@ -19082,9 +19079,10 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,

case NVPTX::BI__nvvm_atom_add_gen_f:
case NVPTX::BI__nvvm_atom_add_gen_d: {
Value *Ptr = EmitScalarExpr(E->getArg(0));
Address DestAddr = EmitPointerWithAlignment(E->getArg(0));
Value *Val = EmitScalarExpr(E->getArg(1));
return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, Ptr, Val,

return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, DestAddr, Val,
AtomicOrdering::SequentiallyConsistent);
}

Expand Down
Loading