Skip to content

[Clang][CodeGen] Emit llvm.ptrmask for align_up and align_down #71238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[Clang][CodeGen] Emit llvm.ptrmask for align_up and align_down #71238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 20 additions & 24 deletions clang/lib/CodeGen/CGBuiltin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19668,44 +19668,40 @@ RValue CodeGenFunction::EmitBuiltinIsAligned(const CallExpr *E) {
/// Generate (x & ~(y-1)) to align down or ((x+(y-1)) & ~(y-1)) to align up.
/// Note: For pointer types we can avoid ptrtoint/inttoptr pairs by using the
/// llvm.ptrmask intrinsic (with a GEP before in the align_up case).
/// TODO: actually use ptrmask once most optimization passes know about it.
RValue CodeGenFunction::EmitBuiltinAlignTo(const CallExpr *E, bool AlignUp) {
BuiltinAlignArgs Args(E, *this);
llvm::Value *SrcAddr = Args.Src;
if (Args.Src->getType()->isPointerTy())
SrcAddr = Builder.CreatePtrToInt(Args.Src, Args.IntType, "intptr");
llvm::Value *SrcForMask = SrcAddr;
llvm::Value *SrcForMask = Args.Src;
if (AlignUp) {
// When aligning up we have to first add the mask to ensure we go over the
// next alignment value and then align down to the next valid multiple.
// By adding the mask, we ensure that align_up on an already aligned
// value will not change the value.
SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask, "over_boundary");
if (Args.Src->getType()->isPointerTy()) {
if (getLangOpts().isSignedOverflowDefined())
SrcForMask =
Builder.CreateGEP(Int8Ty, SrcForMask, Args.Mask, "over_boundary");
else
SrcForMask = EmitCheckedInBoundsGEP(Int8Ty, SrcForMask, Args.Mask,
/*SignedIndices=*/true,
/*isSubtraction=*/false,
E->getExprLoc(), "over_boundary");
} else {
SrcForMask = Builder.CreateAdd(SrcForMask, Args.Mask, "over_boundary");
}
}
// Invert the mask to only clear the lower bits.
llvm::Value *InvertedMask = Builder.CreateNot(Args.Mask, "inverted_mask");
llvm::Value *Result =
Builder.CreateAnd(SrcForMask, InvertedMask, "aligned_result");
llvm::Value *Result = nullptr;
if (Args.Src->getType()->isPointerTy()) {
/// TODO: Use ptrmask instead of ptrtoint+gep once it is optimized well.
// Result = Builder.CreateIntrinsic(
// Intrinsic::ptrmask, {Args.SrcType, SrcForMask->getType(), Args.IntType},
// {SrcForMask, NegatedMask}, nullptr, "aligned_result");
Result->setName("aligned_intptr");
llvm::Value *Difference = Builder.CreateSub(Result, SrcAddr, "diff");
// The result must point to the same underlying allocation. This means we
// can use an inbounds GEP to enable better optimization.
if (getLangOpts().isSignedOverflowDefined())
Result =
Builder.CreateGEP(Int8Ty, Args.Src, Difference, "aligned_result");
else
Result = EmitCheckedInBoundsGEP(Int8Ty, Args.Src, Difference,
/*SignedIndices=*/true,
/*isSubtraction=*/!AlignUp,
E->getExprLoc(), "aligned_result");
Result = Builder.CreateIntrinsic(
Intrinsic::ptrmask, {Args.SrcType, Args.IntType},
{SrcForMask, InvertedMask}, nullptr, "aligned_result");

// Emit an alignment assumption to ensure that the new alignment is
// propagated to loads/stores, etc.
emitAlignmentAssumption(Result, E, E->getExprLoc(), Args.Alignment);
} else {
Result = Builder.CreateAnd(SrcForMask, InvertedMask, "aligned_result");
}
assert(Result->getType() == Args.SrcType);
return RValue::get(Result);
Expand Down
36 changes: 12 additions & 24 deletions clang/test/CodeGen/builtin-align-array.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,16 @@ extern int func(char *c);
// CHECK-NEXT: entry:
// CHECK-NEXT: [[BUF:%.*]] = alloca [1024 x i8], align 16
// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 44
// CHECK-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[ARRAYIDX]] to i64
// CHECK-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], -16
// CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX]], i64 [[DIFF]]
// CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ARRAYIDX]], i64 -16)
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 16) ]
// CHECK-NEXT: [[CALL:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT]])
// CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 22
// CHECK-NEXT: [[INTPTR2:%.*]] = ptrtoint ptr [[ARRAYIDX1]] to i64
// CHECK-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR2]], 31
// CHECK-NEXT: [[ALIGNED_INTPTR4:%.*]] = and i64 [[OVER_BOUNDARY]], -32
// CHECK-NEXT: [[DIFF5:%.*]] = sub i64 [[ALIGNED_INTPTR4]], [[INTPTR2]]
// CHECK-NEXT: [[ALIGNED_RESULT6:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX1]], i64 [[DIFF5]]
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT6]], i64 32) ]
// CHECK-NEXT: [[CALL7:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT6]])
// CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 16
// CHECK-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[ARRAYIDX8]] to i64
// CHECK-NEXT: [[OVER_BOUNDARY:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX1]], i64 31
// CHECK-NEXT: [[ALIGNED_RESULT2:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[OVER_BOUNDARY]], i64 -32)
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT2]], i64 32) ]
// CHECK-NEXT: [[CALL3:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT2]])
// CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 16
// CHECK-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[ARRAYIDX4]] to i64
// CHECK-NEXT: [[SET_BITS:%.*]] = and i64 [[SRC_ADDR]], 63
// CHECK-NEXT: [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
// CHECK-NEXT: [[CONV:%.*]] = zext i1 [[IS_ALIGNED]] to i32
Expand All @@ -40,20 +34,14 @@ int test_array(void) {
// CHECK-NEXT: entry:
// CHECK-NEXT: [[BUF:%.*]] = alloca [1024 x i8], align 32
// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 64
// CHECK-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[ARRAYIDX]] to i64
// CHECK-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], -16
// CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX]], i64 [[DIFF]]
// CHECK-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[ARRAYIDX]], i64 -16)
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 16) ]
// CHECK-NEXT: [[CALL:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT]])
// CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BUF]], i64 0, i64 32
// CHECK-NEXT: [[INTPTR2:%.*]] = ptrtoint ptr [[ARRAYIDX1]] to i64
// CHECK-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR2]], 31
// CHECK-NEXT: [[ALIGNED_INTPTR4:%.*]] = and i64 [[OVER_BOUNDARY]], -32
// CHECK-NEXT: [[DIFF5:%.*]] = sub i64 [[ALIGNED_INTPTR4]], [[INTPTR2]]
// CHECK-NEXT: [[ALIGNED_RESULT6:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX1]], i64 [[DIFF5]]
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT6]], i64 32) ]
// CHECK-NEXT: [[CALL7:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT6]])
// CHECK-NEXT: [[OVER_BOUNDARY:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX1]], i64 31
// CHECK-NEXT: [[ALIGNED_RESULT2:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[OVER_BOUNDARY]], i64 -32)
// CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT2]], i64 32) ]
// CHECK-NEXT: [[CALL3:%.*]] = call i32 @func(ptr noundef [[ALIGNED_RESULT2]])
// CHECK-NEXT: ret i32 1
//
int test_array_should_not_mask(void) {
Expand Down
64 changes: 26 additions & 38 deletions clang/test/CodeGen/builtin-align.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,38 +71,38 @@ TYPE get_type(void) {
// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@is_aligned
// CHECK-VOID_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-VOID_PTR-NEXT: entry:
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-VOID_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-VOID_PTR-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-VOID_PTR-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
// CHECK-VOID_PTR-NEXT: [[SET_BITS:%.*]] = and i64 [[SRC_ADDR]], [[MASK]]
// CHECK-VOID_PTR-NEXT: [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
// CHECK-VOID_PTR-NEXT: ret i1 [[IS_ALIGNED]]
//
// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@is_aligned
// CHECK-FLOAT_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-FLOAT_PTR-NEXT: entry:
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-FLOAT_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-FLOAT_PTR-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-FLOAT_PTR-NEXT: [[SRC_ADDR:%.*]] = ptrtoint ptr [[PTR:%.*]] to i64
// CHECK-FLOAT_PTR-NEXT: [[SET_BITS:%.*]] = and i64 [[SRC_ADDR]], [[MASK]]
// CHECK-FLOAT_PTR-NEXT: [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
// CHECK-FLOAT_PTR-NEXT: ret i1 [[IS_ALIGNED]]
//
// CHECK-LONG-LABEL: define {{[^@]+}}@is_aligned
// CHECK-LONG-SAME: (i64 noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-LONG-NEXT: entry:
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-LONG-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-LONG-NEXT: [[SET_BITS:%.*]] = and i64 [[PTR]], [[MASK]]
// CHECK-LONG-NEXT: [[SET_BITS:%.*]] = and i64 [[PTR:%.*]], [[MASK]]
// CHECK-LONG-NEXT: [[IS_ALIGNED:%.*]] = icmp eq i64 [[SET_BITS]], 0
// CHECK-LONG-NEXT: ret i1 [[IS_ALIGNED]]
//
// CHECK-USHORT-LABEL: define {{[^@]+}}@is_aligned
// CHECK-USHORT-SAME: (i16 noundef zeroext [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-USHORT-NEXT: entry:
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN:%.*]] to i16
// CHECK-USHORT-NEXT: [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
// CHECK-USHORT-NEXT: [[SET_BITS:%.*]] = and i16 [[PTR]], [[MASK]]
// CHECK-USHORT-NEXT: [[SET_BITS:%.*]] = and i16 [[PTR:%.*]], [[MASK]]
// CHECK-USHORT-NEXT: [[IS_ALIGNED:%.*]] = icmp eq i16 [[SET_BITS]], 0
// CHECK-USHORT-NEXT: ret i1 [[IS_ALIGNED]]
//
Expand All @@ -114,47 +114,41 @@ _Bool is_aligned(TYPE ptr, unsigned align) {
// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@align_up
// CHECK-VOID_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-VOID_PTR-NEXT: entry:
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-VOID_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-VOID_PTR-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-VOID_PTR-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR]], [[MASK]]
// CHECK-VOID_PTR-NEXT: [[OVER_BOUNDARY:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[MASK]]
// CHECK-VOID_PTR-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-VOID_PTR-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
// CHECK-VOID_PTR-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-VOID_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[DIFF]]
// CHECK-VOID_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[OVER_BOUNDARY]], i64 [[INVERTED_MASK]])
// CHECK-VOID_PTR-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 [[ALIGNMENT]]) ]
// CHECK-VOID_PTR-NEXT: ret ptr [[ALIGNED_RESULT]]
//
// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@align_up
// CHECK-FLOAT_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-FLOAT_PTR-NEXT: entry:
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-FLOAT_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-FLOAT_PTR-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-FLOAT_PTR-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[INTPTR]], [[MASK]]
// CHECK-FLOAT_PTR-NEXT: [[OVER_BOUNDARY:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i64 [[MASK]]
// CHECK-FLOAT_PTR-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
// CHECK-FLOAT_PTR-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[DIFF]]
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[OVER_BOUNDARY]], i64 [[INVERTED_MASK]])
// CHECK-FLOAT_PTR-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 [[ALIGNMENT]]) ]
// CHECK-FLOAT_PTR-NEXT: ret ptr [[ALIGNED_RESULT]]
//
// CHECK-LONG-LABEL: define {{[^@]+}}@align_up
// CHECK-LONG-SAME: (i64 noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-LONG-NEXT: entry:
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-LONG-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-LONG-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[PTR]], [[MASK]]
// CHECK-LONG-NEXT: [[OVER_BOUNDARY:%.*]] = add i64 [[PTR:%.*]], [[MASK]]
// CHECK-LONG-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-LONG-NEXT: [[ALIGNED_RESULT:%.*]] = and i64 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
// CHECK-LONG-NEXT: ret i64 [[ALIGNED_RESULT]]
//
// CHECK-USHORT-LABEL: define {{[^@]+}}@align_up
// CHECK-USHORT-SAME: (i16 noundef zeroext [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-USHORT-NEXT: entry:
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN:%.*]] to i16
// CHECK-USHORT-NEXT: [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
// CHECK-USHORT-NEXT: [[OVER_BOUNDARY:%.*]] = add i16 [[PTR]], [[MASK]]
// CHECK-USHORT-NEXT: [[OVER_BOUNDARY:%.*]] = add i16 [[PTR:%.*]], [[MASK]]
// CHECK-USHORT-NEXT: [[INVERTED_MASK:%.*]] = xor i16 [[MASK]], -1
// CHECK-USHORT-NEXT: [[ALIGNED_RESULT:%.*]] = and i16 [[OVER_BOUNDARY]], [[INVERTED_MASK]]
// CHECK-USHORT-NEXT: ret i16 [[ALIGNED_RESULT]]
Expand All @@ -167,45 +161,39 @@ TYPE align_up(TYPE ptr, unsigned align) {
// CHECK-VOID_PTR-LABEL: define {{[^@]+}}@align_down
// CHECK-VOID_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-VOID_PTR-NEXT: entry:
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-VOID_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-VOID_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-VOID_PTR-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-VOID_PTR-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-VOID_PTR-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], [[INVERTED_MASK]]
// CHECK-VOID_PTR-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-VOID_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[DIFF]]
// CHECK-VOID_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 [[INVERTED_MASK]])
// CHECK-VOID_PTR-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 [[ALIGNMENT]]) ]
// CHECK-VOID_PTR-NEXT: ret ptr [[ALIGNED_RESULT]]
//
// CHECK-FLOAT_PTR-LABEL: define {{[^@]+}}@align_down
// CHECK-FLOAT_PTR-SAME: (ptr noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-FLOAT_PTR-NEXT: entry:
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-FLOAT_PTR-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-FLOAT_PTR-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-FLOAT_PTR-NEXT: [[INTPTR:%.*]] = ptrtoint ptr [[PTR]] to i64
// CHECK-FLOAT_PTR-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_INTPTR:%.*]] = and i64 [[INTPTR]], [[INVERTED_MASK]]
// CHECK-FLOAT_PTR-NEXT: [[DIFF:%.*]] = sub i64 [[ALIGNED_INTPTR]], [[INTPTR]]
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i64 [[DIFF]]
// CHECK-FLOAT_PTR-NEXT: [[ALIGNED_RESULT:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[PTR:%.*]], i64 [[INVERTED_MASK]])
// CHECK-FLOAT_PTR-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[ALIGNED_RESULT]], i64 [[ALIGNMENT]]) ]
// CHECK-FLOAT_PTR-NEXT: ret ptr [[ALIGNED_RESULT]]
//
// CHECK-LONG-LABEL: define {{[^@]+}}@align_down
// CHECK-LONG-SAME: (i64 noundef [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-LONG-NEXT: entry:
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN]] to i64
// CHECK-LONG-NEXT: [[ALIGNMENT:%.*]] = zext i32 [[ALIGN:%.*]] to i64
// CHECK-LONG-NEXT: [[MASK:%.*]] = sub i64 [[ALIGNMENT]], 1
// CHECK-LONG-NEXT: [[INVERTED_MASK:%.*]] = xor i64 [[MASK]], -1
// CHECK-LONG-NEXT: [[ALIGNED_RESULT:%.*]] = and i64 [[PTR]], [[INVERTED_MASK]]
// CHECK-LONG-NEXT: [[ALIGNED_RESULT:%.*]] = and i64 [[PTR:%.*]], [[INVERTED_MASK]]
// CHECK-LONG-NEXT: ret i64 [[ALIGNED_RESULT]]
//
// CHECK-USHORT-LABEL: define {{[^@]+}}@align_down
// CHECK-USHORT-SAME: (i16 noundef zeroext [[PTR:%.*]], i32 noundef [[ALIGN:%.*]]) #0
// CHECK-USHORT-NEXT: entry:
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN]] to i16
// CHECK-USHORT-NEXT: [[ALIGNMENT:%.*]] = trunc i32 [[ALIGN:%.*]] to i16
// CHECK-USHORT-NEXT: [[MASK:%.*]] = sub i16 [[ALIGNMENT]], 1
// CHECK-USHORT-NEXT: [[INVERTED_MASK:%.*]] = xor i16 [[MASK]], -1
// CHECK-USHORT-NEXT: [[ALIGNED_RESULT:%.*]] = and i16 [[PTR]], [[INVERTED_MASK]]
// CHECK-USHORT-NEXT: [[ALIGNED_RESULT:%.*]] = and i16 [[PTR:%.*]], [[INVERTED_MASK]]
// CHECK-USHORT-NEXT: ret i16 [[ALIGNED_RESULT]]
//
TYPE align_down(TYPE ptr, unsigned align) {
Expand Down