@@ -188,8 +188,7 @@ static Value *EmitFromInt(CodeGenFunction &CGF, llvm::Value *V,
188
188
return V;
189
189
}
190
190
191
- static llvm::Value *CheckAtomicAlignment(CodeGenFunction &CGF,
192
- const CallExpr *E) {
191
+ static Address CheckAtomicAlignment(CodeGenFunction &CGF, const CallExpr *E) {
193
192
ASTContext &Ctx = CGF.getContext();
194
193
Address Ptr = CGF.EmitPointerWithAlignment(E->getArg(0));
195
194
unsigned Bytes = Ptr.getElementType()->isPointerTy()
@@ -199,8 +198,10 @@ static llvm::Value *CheckAtomicAlignment(CodeGenFunction &CGF,
199
198
if (Align % Bytes != 0) {
200
199
DiagnosticsEngine &Diags = CGF.CGM.getDiags();
201
200
Diags.Report(E->getBeginLoc(), diag::warn_sync_op_misaligned);
201
+ // Force address to be at least naturally-aligned.
202
+ return Ptr.withAlignment(CharUnits::fromQuantity(Bytes));
202
203
}
203
- return Ptr.getPointer() ;
204
+ return Ptr;
204
205
}
205
206
206
207
/// Utility to insert an atomic instruction based on Intrinsic::ID
@@ -215,19 +216,17 @@ static Value *MakeBinaryAtomicValue(
215
216
E->getArg(0)->getType()->getPointeeType()));
216
217
assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
217
218
218
- llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
219
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
219
220
220
221
llvm::IntegerType *IntType = llvm::IntegerType::get(
221
222
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
222
223
223
- llvm::Value *Args[2];
224
- Args[0] = DestPtr;
225
- Args[1] = CGF.EmitScalarExpr(E->getArg(1));
226
- llvm::Type *ValueType = Args[1]->getType();
227
- Args[1] = EmitToInt(CGF, Args[1], T, IntType);
224
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
225
+ llvm::Type *ValueType = Val->getType();
226
+ Val = EmitToInt(CGF, Val, T, IntType);
228
227
229
- llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
230
- Kind, Args[0], Args[1] , Ordering);
228
+ llvm::Value *Result =
229
+ CGF.Builder.CreateAtomicRMW( Kind, DestAddr, Val , Ordering);
231
230
return EmitFromInt(CGF, Result, T, ValueType);
232
231
}
233
232
@@ -270,20 +269,18 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
270
269
E->getArg(0)->getType()->getPointeeType()));
271
270
assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
272
271
273
- llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
272
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
274
273
275
274
llvm::IntegerType *IntType = llvm::IntegerType::get(
276
275
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
277
276
278
- llvm::Value *Args[2];
279
- Args[1] = CGF.EmitScalarExpr(E->getArg(1));
280
- llvm::Type *ValueType = Args[1]->getType();
281
- Args[1] = EmitToInt(CGF, Args[1], T, IntType);
282
- Args[0] = DestPtr;
277
+ llvm::Value *Val = CGF.EmitScalarExpr(E->getArg(1));
278
+ llvm::Type *ValueType = Val->getType();
279
+ Val = EmitToInt(CGF, Val, T, IntType);
283
280
284
281
llvm::Value *Result = CGF.Builder.CreateAtomicRMW(
285
- Kind, Args[0], Args[1] , llvm::AtomicOrdering::SequentiallyConsistent);
286
- Result = CGF.Builder.CreateBinOp(Op, Result, Args[1] );
282
+ Kind, DestAddr, Val , llvm::AtomicOrdering::SequentiallyConsistent);
283
+ Result = CGF.Builder.CreateBinOp(Op, Result, Val );
287
284
if (Invert)
288
285
Result =
289
286
CGF.Builder.CreateBinOp(llvm::Instruction::Xor, Result,
@@ -309,20 +306,18 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
309
306
static Value *MakeAtomicCmpXchgValue(CodeGenFunction &CGF, const CallExpr *E,
310
307
bool ReturnBool) {
311
308
QualType T = ReturnBool ? E->getArg(1)->getType() : E->getType();
312
- llvm::Value *DestPtr = CheckAtomicAlignment(CGF, E);
309
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
313
310
314
311
llvm::IntegerType *IntType = llvm::IntegerType::get(
315
312
CGF.getLLVMContext(), CGF.getContext().getTypeSize(T));
316
313
317
- Value *Args[3];
318
- Args[0] = DestPtr;
319
- Args[1] = CGF.EmitScalarExpr(E->getArg(1));
320
- llvm::Type *ValueType = Args[1]->getType();
321
- Args[1] = EmitToInt(CGF, Args[1], T, IntType);
322
- Args[2] = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
314
+ Value *Cmp = CGF.EmitScalarExpr(E->getArg(1));
315
+ llvm::Type *ValueType = Cmp->getType();
316
+ Cmp = EmitToInt(CGF, Cmp, T, IntType);
317
+ Value *New = EmitToInt(CGF, CGF.EmitScalarExpr(E->getArg(2)), T, IntType);
323
318
324
319
Value *Pair = CGF.Builder.CreateAtomicCmpXchg(
325
- Args[0], Args[1], Args[2] , llvm::AtomicOrdering::SequentiallyConsistent,
320
+ DestAddr, Cmp, New , llvm::AtomicOrdering::SequentiallyConsistent,
326
321
llvm::AtomicOrdering::SequentiallyConsistent);
327
322
if (ReturnBool)
328
323
// Extract boolean success flag and zext it to int.
@@ -358,7 +353,8 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
358
353
assert(CGF.getContext().hasSameUnqualifiedType(E->getType(),
359
354
E->getArg(2)->getType()));
360
355
361
- auto *Destination = CGF.EmitScalarExpr(E->getArg(0));
356
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
357
+
362
358
auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
363
359
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
364
360
@@ -372,8 +368,7 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
372
368
// _Interlocked* operations in the future, we will have to remove the volatile
373
369
// marker.
374
370
auto *Result = CGF.Builder.CreateAtomicCmpXchg(
375
- Destination, Comparand, Exchange,
376
- SuccessOrdering, FailureOrdering);
371
+ DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
377
372
Result->setVolatile(true);
378
373
return CGF.Builder.CreateExtractValue(Result, 0);
379
374
}
@@ -386,29 +381,34 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
386
381
// __int64 _ExchangeHigh,
387
382
// __int64 _ExchangeLow,
388
383
// __int64 * _ComparandResult);
384
+ //
385
+ // Note that Destination is assumed to be at least 16-byte aligned, despite
386
+ // being typed int64.
387
+
389
388
static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
390
389
const CallExpr *E,
391
390
AtomicOrdering SuccessOrdering) {
392
391
assert(E->getNumArgs() == 4);
393
- llvm::Value *Destination = CGF.EmitScalarExpr(E->getArg(0));
392
+ llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
394
393
llvm::Value *ExchangeHigh = CGF.EmitScalarExpr(E->getArg(1));
395
394
llvm::Value *ExchangeLow = CGF.EmitScalarExpr(E->getArg(2));
396
- llvm::Value *ComparandPtr = CGF.EmitScalarExpr (E->getArg(3));
395
+ Address ComparandAddr = CGF.EmitPointerWithAlignment (E->getArg(3));
397
396
398
- assert(Destination ->getType()->isPointerTy());
397
+ assert(DestPtr ->getType()->isPointerTy());
399
398
assert(!ExchangeHigh->getType()->isPointerTy());
400
399
assert(!ExchangeLow->getType()->isPointerTy());
401
- assert(ComparandPtr->getType()->isPointerTy());
402
400
403
401
// For Release ordering, the failure ordering should be Monotonic.
404
402
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release
405
403
? AtomicOrdering::Monotonic
406
404
: SuccessOrdering;
407
405
408
- // Convert to i128 pointers and values.
406
+ // Convert to i128 pointers and values. Alignment is also overridden for
407
+ // destination pointer.
409
408
llvm::Type *Int128Ty = llvm::IntegerType::get(CGF.getLLVMContext(), 128);
410
- Address ComparandResult(ComparandPtr, Int128Ty,
411
- CGF.getContext().toCharUnitsFromBits(128));
409
+ Address DestAddr(DestPtr, Int128Ty,
410
+ CGF.getContext().toCharUnitsFromBits(128));
411
+ ComparandAddr = ComparandAddr.withElementType(Int128Ty);
412
412
413
413
// (((i128)hi) << 64) | ((i128)lo)
414
414
ExchangeHigh = CGF.Builder.CreateZExt(ExchangeHigh, Int128Ty);
@@ -418,9 +418,9 @@ static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
418
418
llvm::Value *Exchange = CGF.Builder.CreateOr(ExchangeHigh, ExchangeLow);
419
419
420
420
// Load the comparand for the instruction.
421
- llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandResult );
421
+ llvm::Value *Comparand = CGF.Builder.CreateLoad(ComparandAddr );
422
422
423
- auto *CXI = CGF.Builder.CreateAtomicCmpXchg(Destination , Comparand, Exchange,
423
+ auto *CXI = CGF.Builder.CreateAtomicCmpXchg(DestAddr , Comparand, Exchange,
424
424
SuccessOrdering, FailureOrdering);
425
425
426
426
// The atomic instruction is marked volatile for consistency with MSVC. This
@@ -431,7 +431,7 @@ static Value *EmitAtomicCmpXchg128ForMSIntrin(CodeGenFunction &CGF,
431
431
432
432
// Store the result as an outparameter.
433
433
CGF.Builder.CreateStore(CGF.Builder.CreateExtractValue(CXI, 0),
434
- ComparandResult );
434
+ ComparandAddr );
435
435
436
436
// Get the success boolean and zero extend it to i8.
437
437
Value *Success = CGF.Builder.CreateExtractValue(CXI, 1);
@@ -443,24 +443,21 @@ static Value *EmitAtomicIncrementValue(CodeGenFunction &CGF, const CallExpr *E,
443
443
assert(E->getArg(0)->getType()->isPointerType());
444
444
445
445
auto *IntTy = CGF.ConvertType(E->getType());
446
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
446
447
auto *Result = CGF.Builder.CreateAtomicRMW(
447
- AtomicRMWInst::Add,
448
- CGF.EmitScalarExpr(E->getArg(0)),
449
- ConstantInt::get(IntTy, 1),
450
- Ordering);
448
+ AtomicRMWInst::Add, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
451
449
return CGF.Builder.CreateAdd(Result, ConstantInt::get(IntTy, 1));
452
450
}
453
451
454
- static Value *EmitAtomicDecrementValue(CodeGenFunction &CGF, const CallExpr *E,
452
+ static Value *EmitAtomicDecrementValue(
453
+ CodeGenFunction &CGF, const CallExpr *E,
455
454
AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent) {
456
455
assert(E->getArg(0)->getType()->isPointerType());
457
456
458
457
auto *IntTy = CGF.ConvertType(E->getType());
458
+ Address DestAddr = CheckAtomicAlignment(CGF, E);
459
459
auto *Result = CGF.Builder.CreateAtomicRMW(
460
- AtomicRMWInst::Sub,
461
- CGF.EmitScalarExpr(E->getArg(0)),
462
- ConstantInt::get(IntTy, 1),
463
- Ordering);
460
+ AtomicRMWInst::Sub, DestAddr, ConstantInt::get(IntTy, 1), Ordering);
464
461
return CGF.Builder.CreateSub(Result, ConstantInt::get(IntTy, 1));
465
462
}
466
463
@@ -1215,8 +1212,7 @@ static llvm::Value *EmitBitTestIntrinsic(CodeGenFunction &CGF,
1215
1212
Mask = CGF.Builder.CreateNot(Mask);
1216
1213
RMWOp = llvm::AtomicRMWInst::And;
1217
1214
}
1218
- OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr.getPointer(), Mask,
1219
- Ordering);
1215
+ OldByte = CGF.Builder.CreateAtomicRMW(RMWOp, ByteAddr, Mask, Ordering);
1220
1216
} else {
1221
1217
// Emit a plain load for the non-interlocked intrinsics.
1222
1218
OldByte = CGF.Builder.CreateLoad(ByteAddr, "bittest.byte");
@@ -4456,14 +4452,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4456
4452
case Builtin::BI__sync_lock_release_4:
4457
4453
case Builtin::BI__sync_lock_release_8:
4458
4454
case Builtin::BI__sync_lock_release_16: {
4459
- Value * Ptr = CheckAtomicAlignment(*this, E);
4455
+ Address Ptr = CheckAtomicAlignment(*this, E);
4460
4456
QualType ElTy = E->getArg(0)->getType()->getPointeeType();
4461
- CharUnits StoreSize = getContext().getTypeSizeInChars(ElTy);
4462
- llvm::Type *ITy =
4463
- llvm::IntegerType::get(getLLVMContext(), StoreSize.getQuantity() * 8 );
4457
+
4458
+ llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
4459
+ getContext().getTypeSize(ElTy) );
4464
4460
llvm::StoreInst *Store =
4465
- Builder.CreateAlignedStore(llvm::Constant::getNullValue(ITy), Ptr,
4466
- StoreSize);
4461
+ Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
4467
4462
Store->setAtomic(llvm::AtomicOrdering::Release);
4468
4463
return RValue::get(nullptr);
4469
4464
}
@@ -4514,7 +4509,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
4514
4509
bool Volatile =
4515
4510
PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
4516
4511
4517
- Value *Ptr = EmitScalarExpr(E->getArg(0));
4512
+ Address Ptr =
4513
+ EmitPointerWithAlignment(E->getArg(0)).withElementType(Int8Ty);
4514
+
4518
4515
Value *NewVal = Builder.getInt8(1);
4519
4516
Value *Order = EmitScalarExpr(E->getArg(1));
4520
4517
if (isa<llvm::ConstantInt>(Order)) {
@@ -5035,7 +5032,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
5035
5032
llvm::IntegerType *IntType = IntegerType::get(
5036
5033
getLLVMContext(), getContext().getTypeSize(E->getType()));
5037
5034
5038
- llvm::Value *Destination = EmitScalarExpr(E->getArg(0) );
5035
+ Address DestAddr = CheckAtomicAlignment(*this, E );
5039
5036
5040
5037
llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
5041
5038
RTy = Exchange->getType();
@@ -5048,7 +5045,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
5048
5045
BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
5049
5046
AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
5050
5047
5051
- auto Result = Builder.CreateAtomicCmpXchg(Destination , Comparand, Exchange,
5048
+ auto Result = Builder.CreateAtomicCmpXchg(DestAddr , Comparand, Exchange,
5052
5049
Ordering, Ordering);
5053
5050
Result->setVolatile(true);
5054
5051
@@ -11901,12 +11898,12 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID,
11901
11898
}
11902
11899
11903
11900
case clang::AArch64::BI_InterlockedAdd: {
11904
- Value *Arg0 = EmitScalarExpr(E->getArg(0) );
11905
- Value *Arg1 = EmitScalarExpr(E->getArg(1));
11906
- AtomicRMWInst *RMWI = Builder.CreateAtomicRMW(
11907
- AtomicRMWInst::Add, Arg0, Arg1 ,
11908
- llvm::AtomicOrdering::SequentiallyConsistent);
11909
- return Builder.CreateAdd(RMWI, Arg1 );
11901
+ Address DestAddr = CheckAtomicAlignment(*this, E );
11902
+ Value *Val = EmitScalarExpr(E->getArg(1));
11903
+ AtomicRMWInst *RMWI =
11904
+ Builder.CreateAtomicRMW( AtomicRMWInst::Add, DestAddr, Val ,
11905
+ llvm::AtomicOrdering::SequentiallyConsistent);
11906
+ return Builder.CreateAdd(RMWI, Val );
11910
11907
}
11911
11908
}
11912
11909
@@ -18219,7 +18216,7 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
18219
18216
break;
18220
18217
}
18221
18218
18222
- Value * Ptr = EmitScalarExpr(E->getArg(0) );
18219
+ Address Ptr = CheckAtomicAlignment(*this, E );
18223
18220
Value *Val = EmitScalarExpr(E->getArg(1));
18224
18221
18225
18222
ProcessOrderScopeAMDGCN(EmitScalarExpr(E->getArg(2)),
@@ -19108,9 +19105,10 @@ Value *CodeGenFunction::EmitNVPTXBuiltinExpr(unsigned BuiltinID,
19108
19105
19109
19106
case NVPTX::BI__nvvm_atom_add_gen_f:
19110
19107
case NVPTX::BI__nvvm_atom_add_gen_d: {
19111
- Value *Ptr = EmitScalarExpr (E->getArg(0));
19108
+ Address DestAddr = EmitPointerWithAlignment (E->getArg(0));
19112
19109
Value *Val = EmitScalarExpr(E->getArg(1));
19113
- return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, Ptr, Val,
19110
+
19111
+ return Builder.CreateAtomicRMW(llvm::AtomicRMWInst::FAdd, DestAddr, Val,
19114
19112
AtomicOrdering::SequentiallyConsistent);
19115
19113
}
19116
19114
0 commit comments