Skip to content

AtomicExpand: Fix creating invalid ptrmask for fat pointers #94955

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 12, 2024
Merged

AtomicExpand: Fix creating invalid ptrmask for fat pointers #94955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8f75750
AtomicExpand: Fix creating invalid ptrmask for fat pointers
arsenm Jun 10, 2024
f9472f4
AMDGPU: Add some codegen tests for ptrmask with fat pointers
arsenm Jun 10, 2024
1b0e274
AMDGPU: Add more tests for ptrmask with fat pointers
arsenm Jun 10, 2024
181835b
Merge branch 'main' into atomic-expand-fat-pointers
arsenm Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion llvm/lib/CodeGen/AtomicExpandPass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,7 @@ static PartwordMaskValues createMaskInstrs(IRBuilderBase &Builder,
assert(ValueSize < MinWordSize);

PointerType *PtrTy = cast<PointerType>(Addr->getType());
IntegerType *IntTy = DL.getIntPtrType(Ctx, PtrTy->getAddressSpace());
IntegerType *IntTy = DL.getIndexType(Ctx, PtrTy->getAddressSpace());
Value *PtrLSB;

if (AddrAlign < MinWordSize) {
Expand Down
154 changes: 154 additions & 0 deletions llvm/test/CodeGen/AMDGPU/ptrmask.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,160 @@ define amdgpu_ps ptr addrspace(3) @s_ptrmask_local_variable_i32(ptr addrspace(3)
ret ptr addrspace(3) %masked
}

define ptr addrspace(7) @v_ptrmask_buffer_fat_ptr_variable_i32(ptr addrspace(7) %ptr, i32 %mask) {
; GCN-LABEL: v_ptrmask_buffer_fat_ptr_variable_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v4, v4, v5
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_ptrmask_buffer_fat_ptr_variable_i32:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: v_and_b32_e32 v4, v4, v5
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%masked = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) %ptr, i32 %mask)
ret ptr addrspace(7) %masked
}

define ptr addrspace(7) @v_ptrmask_buffer_fat_ptr_i32_neg8(ptr addrspace(7) %ptr) {
; GCN-LABEL: v_ptrmask_buffer_fat_ptr_i32_neg8:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v4, -8, v4
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_ptrmask_buffer_fat_ptr_i32_neg8:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: v_and_b32_e32 v4, -8, v4
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%masked = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) %ptr, i32 -8)
ret ptr addrspace(7) %masked
}

define amdgpu_ps ptr addrspace(7) @s_ptrmask_buffer_fat_ptr_variable_i32(ptr addrspace(7) inreg %ptr, i32 inreg %mask) {
; GCN-LABEL: s_ptrmask_buffer_fat_ptr_variable_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b32 s8, s4
; GCN-NEXT: s_mov_b32 s1, s3
; GCN-NEXT: s_mov_b32 s0, s2
; GCN-NEXT: s_and_b32 s4, s6, s7
; GCN-NEXT: s_mov_b32 s2, s8
; GCN-NEXT: s_mov_b32 s3, s5
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ptrmask_buffer_fat_ptr_variable_i32:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_mov_b32 s8, s4
; GFX10PLUS-NEXT: s_mov_b32 s1, s3
; GFX10PLUS-NEXT: s_mov_b32 s0, s2
; GFX10PLUS-NEXT: s_and_b32 s4, s6, s7
; GFX10PLUS-NEXT: s_mov_b32 s2, s8
; GFX10PLUS-NEXT: s_mov_b32 s3, s5
; GFX10PLUS-NEXT: ; return to shader part epilog
%masked = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) %ptr, i32 %mask)
ret ptr addrspace(7) %masked
}

define amdgpu_ps ptr addrspace(7) @s_ptrmask_buffer_fat_ptr_i32_neg8(ptr addrspace(7) inreg %ptr) {
; GCN-LABEL: s_ptrmask_buffer_fat_ptr_i32_neg8:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b32 s7, s4
; GCN-NEXT: s_mov_b32 s1, s3
; GCN-NEXT: s_mov_b32 s0, s2
; GCN-NEXT: s_and_b32 s4, s6, -8
; GCN-NEXT: s_mov_b32 s2, s7
; GCN-NEXT: s_mov_b32 s3, s5
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ptrmask_buffer_fat_ptr_i32_neg8:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_mov_b32 s7, s4
; GFX10PLUS-NEXT: s_mov_b32 s1, s3
; GFX10PLUS-NEXT: s_mov_b32 s0, s2
; GFX10PLUS-NEXT: s_and_b32 s4, s6, -8
; GFX10PLUS-NEXT: s_mov_b32 s2, s7
; GFX10PLUS-NEXT: s_mov_b32 s3, s5
; GFX10PLUS-NEXT: ; return to shader part epilog
%masked = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) %ptr, i32 -8)
ret ptr addrspace(7) %masked
}

define ptr addrspace(8) @v_ptrmask_buffer_resource_variable_i128(ptr addrspace(8) %ptr, i128 %mask) {
; GCN-LABEL: v_ptrmask_buffer_resource_variable_i128:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v1, v1, v5
; GCN-NEXT: v_and_b32_e32 v0, v0, v4
; GCN-NEXT: v_and_b32_e32 v3, v3, v7
; GCN-NEXT: v_and_b32_e32 v2, v2, v6
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_ptrmask_buffer_resource_variable_i128:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v4
; GFX10PLUS-NEXT: v_and_b32_e32 v1, v1, v5
; GFX10PLUS-NEXT: v_and_b32_e32 v2, v2, v6
; GFX10PLUS-NEXT: v_and_b32_e32 v3, v3, v7
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%masked = call ptr addrspace(8) @llvm.ptrmask.p8.i128(ptr addrspace(8) %ptr, i128 %mask)
ret ptr addrspace(8) %masked
}

define ptr addrspace(8) @v_ptrmask_buffer_resource_variable_i128_neg8(ptr addrspace(8) %ptr) {
; GCN-LABEL: v_ptrmask_buffer_resource_variable_i128_neg8:
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: v_and_b32_e32 v0, -8, v0
; GCN-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_ptrmask_buffer_resource_variable_i128_neg8:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10PLUS-NEXT: v_and_b32_e32 v0, -8, v0
; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
%masked = call ptr addrspace(8) @llvm.ptrmask.p8.i128(ptr addrspace(8) %ptr, i128 -8)
ret ptr addrspace(8) %masked
}

define amdgpu_ps ptr addrspace(8) @s_ptrmask_buffer_resource_variable_i128(ptr addrspace(8) inreg %ptr, i128 inreg %mask) {
; GCN-LABEL: s_ptrmask_buffer_resource_variable_i128:
; GCN: ; %bb.0:
; GCN-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7]
; GCN-NEXT: s_and_b64 s[2:3], s[4:5], s[8:9]
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ptrmask_buffer_resource_variable_i128:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7]
; GFX10PLUS-NEXT: s_and_b64 s[2:3], s[4:5], s[8:9]
; GFX10PLUS-NEXT: ; return to shader part epilog
%masked = call ptr addrspace(8) @llvm.ptrmask.p8.i128(ptr addrspace(8) %ptr, i128 %mask)
ret ptr addrspace(8) %masked
}

define amdgpu_ps ptr addrspace(8) @s_ptrmask_buffer_resource_variable_i128_neg8(ptr addrspace(8) inreg %ptr) {
; GCN-LABEL: s_ptrmask_buffer_resource_variable_i128_neg8:
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b32 s1, s3
; GCN-NEXT: s_and_b32 s0, s2, -8
; GCN-NEXT: s_mov_b32 s2, s4
; GCN-NEXT: s_mov_b32 s3, s5
; GCN-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_ptrmask_buffer_resource_variable_i128_neg8:
; GFX10PLUS: ; %bb.0:
; GFX10PLUS-NEXT: s_mov_b32 s1, s3
; GFX10PLUS-NEXT: s_and_b32 s0, s2, -8
; GFX10PLUS-NEXT: s_mov_b32 s2, s4
; GFX10PLUS-NEXT: s_mov_b32 s3, s5
; GFX10PLUS-NEXT: ; return to shader part epilog
%masked = call ptr addrspace(8) @llvm.ptrmask.p8.i128(ptr addrspace(8) %ptr, i128 -8)
ret ptr addrspace(8) %masked
}

declare ptr addrspace(3) @llvm.ptrmask.p3.i32(ptr addrspace(3), i32) #0
declare ptr addrspace(1) @llvm.ptrmask.p1.i64(ptr addrspace(1), i64) #0

Expand Down
104 changes: 104 additions & 0 deletions llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1262,6 +1262,110 @@ define bfloat @test_atomicrmw_xchg_bf16_global_agent_align4(ptr addrspace(1) %pt
ret bfloat %res
}

define i16 @test_atomicrmw_xchg_i16_buffer_fat_agent(ptr addrspace(7) %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i16_buffer_fat_agent(
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure this is really sensible. I'm surprised this is i32 and not i64, and since this is zero extended, this is probably nonsense

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The codegen seems correct, but mostly because of the pre-processing IR pass splits these into the i128 + i32 structs

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't actually guarantee an aligned pointer, since a ptr addrspace(7) is a concat(i128 rsrc, i32 offset), or, more relevantly, an concat(i32 flags, i32 extent, i16 flags, i48/ptr base_address, i32 offset).

The address an atomic operates on is add(base_address, zext(offset)) for these pointers, so if base_address is unaligned, clearing the low bits of the offset won't do anything.

I'm not sure what, if anything, we'd want to do about this.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unsigned offsets are so problematic. I guess we could interpret this as a mask on the base address instead? But for that, we would want this to be the 64/48-bit mask and not the 32-bit indexing type

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AMD only supports unsigned offsets (unless you count those weird SRC2 LDS instructions), but they're also never larger than 32 bits.

Speaking of LDS, does any of this change for those instructions? IIUC, the AtomicExpansion works for both.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

global/scratch instructions are supposed to support signed offsets, but it was broken for a few generations.

None of this matters for LDS or those cases since they are normal pointers

; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i32
; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
; CHECK-NEXT: ret i16 [[EXTRACTED]]
;
%res = atomicrmw xchg ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst
ret i16 %res
}

define i16 @test_atomicrmw_xchg_i16_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i16_buffer_fat_agent_align4(
; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[LOADED]], -65536
; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP4]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
; CHECK-NEXT: ret i16 [[EXTRACTED]]
;
%res = atomicrmw xchg ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst, align 4
ret i16 %res
}

define i16 @test_atomicrmw_add_i16_buffer_fat_agent(ptr addrspace(7) %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i16_buffer_fat_agent(
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i32
; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[TMP2]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP3:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i16
; CHECK-NEXT: ret i16 [[EXTRACTED]]
;
%res = atomicrmw add ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst
ret i16 %res
}

define i16 @test_atomicrmw_add_i16_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i16_buffer_fat_agent_align4(
; CHECK-NEXT: [[TMP1:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[NEW]], 65535
; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -65536
; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i16
; CHECK-NEXT: ret i16 [[EXTRACTED]]
;
%res = atomicrmw add ptr addrspace(7) %ptr, i16 %value syncscope("agent") seq_cst, align 4
ret i16 %res
}

!0 = !{}
!1 = !{!"foo", !"bar"}
!2 = !{!3}
Expand Down
104 changes: 104 additions & 0 deletions llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1608,3 +1608,107 @@ define i8 @test_atomicrmw_dec_i8_flat_agent_align4(ptr %ptr, i8 %value) {
%res = atomicrmw udec_wrap ptr %ptr, i8 %value syncscope("agent") seq_cst, align 4
ret i8 %res
}

define i8 @test_atomicrmw_xchg_i8_buffer_fat_agent(ptr addrspace(7) %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i8_buffer_fat_agent(
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i32
; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[TMP2]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP6:%.*]] = or i32 [[TMP5]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP6]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP7]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP7]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
; CHECK-NEXT: ret i8 [[EXTRACTED]]
;
%res = atomicrmw xchg ptr addrspace(7) %ptr, i8 %value syncscope("agent") seq_cst
ret i8 %res
}

define i8 @test_atomicrmw_xchg_i8_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i8_buffer_fat_agent_align4(
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[LOADED]], -256
; CHECK-NEXT: [[TMP4:%.*]] = or i32 [[TMP3]], [[TMP1]]
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP4]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP5]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i8
; CHECK-NEXT: ret i8 [[EXTRACTED]]
;
%res = atomicrmw xchg ptr addrspace(7) %ptr, i8 %value syncscope("agent") seq_cst, align 4
ret i8 %res
}

define i8 @test_atomicrmw_add_i8_buffer_fat_agent(ptr addrspace(7) %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i8_buffer_fat_agent(
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = call ptr addrspace(7) @llvm.ptrmask.p7.i32(ptr addrspace(7) [[PTR:%.*]], i32 -4)
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr addrspace(7) [[PTR]] to i32
; CHECK-NEXT: [[PTRLSB:%.*]] = and i32 [[TMP1]], 3
; CHECK-NEXT: [[TMP2:%.*]] = shl i32 [[PTRLSB]], 3
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[TMP2]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP3:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr addrspace(7) [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP4]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP5:%.*]] = and i32 [[NEW]], [[MASK]]
; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP7:%.*]] = or i32 [[TMP6]], [[TMP5]]
; CHECK-NEXT: [[TMP8:%.*]] = cmpxchg ptr addrspace(7) [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP7]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP8]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP8]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[NEWLOADED]], [[TMP2]]
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[SHIFTED]] to i8
; CHECK-NEXT: ret i8 [[EXTRACTED]]
;
%res = atomicrmw add ptr addrspace(7) %ptr, i8 %value syncscope("agent") seq_cst
ret i8 %res
}

define i8 @test_atomicrmw_add_i8_buffer_fat_agent_align4(ptr addrspace(7) %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i8_buffer_fat_agent_align4(
; CHECK-NEXT: [[TMP1:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr addrspace(7) [[PTR:%.*]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[NEW:%.*]] = add i32 [[LOADED]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[NEW]], 255
; CHECK-NEXT: [[TMP4:%.*]] = and i32 [[LOADED]], -256
; CHECK-NEXT: [[TMP5:%.*]] = or i32 [[TMP4]], [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = cmpxchg ptr addrspace(7) [[PTR]], i32 [[LOADED]], i32 [[TMP5]] syncscope("agent") seq_cst seq_cst, align 4
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP6]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP6]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[EXTRACTED:%.*]] = trunc i32 [[NEWLOADED]] to i8
; CHECK-NEXT: ret i8 [[EXTRACTED]]
;
%res = atomicrmw add ptr addrspace(7) %ptr, i8 %value syncscope("agent") seq_cst, align 4
ret i8 %res
}
Loading