Skip to content

Commit 9e5fa33

Browse files
arsenmarsenm
committed
AMDGPU: Fix dropping memref for ds append/consume
The way SelectionDAG treats memory operands is very frustrating, and by default drops them unless a property is set on the pattern. There is no pattern for manually selected instructions, so this requires manually setting them. llvm-svn: 363455
1 parent 1c5a879 commit 9e5fa33

File tree

4 files changed

+24
-2
lines changed

4 files changed

+24
-2
lines changed

‎llvm/include/llvm/IR/IntrinsicsAMDGPU.td

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,9 @@ class AMDGPUDSAppendConsumedIntrinsic : Intrinsic<
417417
[llvm_i32_ty],
418418
[llvm_anyptr_ty, // LDS or GDS ptr
419419
llvm_i1_ty], // isVolatile
420-
[IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>]
420+
[IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>],
421+
"",
422+
[SDNPMemOperand]
421423
>;
422424

423425
def int_amdgcn_ds_ordered_add : AMDGPUDSOrderedIntrinsic;

‎llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1993,6 +1993,7 @@ void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
19931993
SDValue Chain = N->getOperand(0);
19941994
SDValue Ptr = N->getOperand(2);
19951995
MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
1996+
MachineMemOperand *MMO = M->getMemOperand();
19961997
bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
19971998

19981999
SDValue Offset;
@@ -2019,7 +2020,8 @@ void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) {
20192020
N->getOperand(N->getNumOperands() - 1) // New glue
20202021
};
20212022

2022-
CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2023+
SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
2024+
CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO});
20232025
}
20242026

20252027
void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {

‎llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
; GCN: s_load_dword [[PTR:s[0-9]+]]
88
; GCN: s_mov_b32 m0, [[PTR]]
99
; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
10+
; GCN-NOT: buffer_wbinvl1
1011
; GCN: {{.*}}store{{.*}} [[RESULT]]
1112
define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
1213
%val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
@@ -18,6 +19,7 @@ define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1
1819
; GCN: s_load_dword [[PTR:s[0-9]+]]
1920
; GCN: s_mov_b32 m0, [[PTR]]
2021
; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532{{$}}
22+
; GCN-NOT: buffer_wbinvl1
2123
; GCN: {{.*}}store{{.*}} [[RESULT]]
2224
define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
2325
%gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
@@ -36,6 +38,7 @@ define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32
3638
; CIPLUS: s_mov_b32 m0, [[PTR]]
3739
; CIPLUS: ds_append [[RESULT:v[0-9]+]] offset:16{{$}}
3840

41+
; GCN-NOT: buffer_wbinvl1
3942
; GCN: {{.*}}store{{.*}} [[RESULT]]
4043
define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
4144
%lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
@@ -53,6 +56,7 @@ define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspa
5356

5457
; GCN: s_mov_b32 m0, [[PTR]]
5558
; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
59+
; GCN-NOT: buffer_wbinvl1
5660
; GCN: {{.*}}store{{.*}} [[RESULT]]
5761
define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
5862
%gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
@@ -65,6 +69,7 @@ define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds,
6569
; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
6670
; GCN: s_mov_b32 m0, [[READLANE]]
6771
; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
72+
; GCN-NOT: buffer_wbinvl1
6873
; GCN: {{.*}}store{{.*}} [[RESULT]]
6974
define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
7075
%val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
@@ -76,6 +81,7 @@ define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %
7681
; GCN: s_load_dword [[PTR:s[0-9]+]]
7782
; GCN: s_mov_b32 m0, [[PTR]]
7883
; GCN: ds_append [[RESULT:v[0-9]+]] gds{{$}}
84+
; GCN-NOT: buffer_wbinvl1
7985
; GCN: {{.*}}store{{.*}} [[RESULT]]
8086
define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
8187
%val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gds, i1 false)
@@ -87,6 +93,7 @@ define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1
8793
; GCN: s_load_dword [[PTR:s[0-9]+]]
8894
; GCN: s_mov_b32 m0, [[PTR]]
8995
; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
96+
; GCN-NOT: buffer_wbinvl1
9097
; GCN: {{.*}}store{{.*}} [[RESULT]]
9198
define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
9299
%gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
@@ -96,6 +103,7 @@ define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32
96103
}
97104

98105
; GCN-LABEL: {{^}}ds_append_gds_over_max_offset:
106+
; GCN-NOT: buffer_wbinvl1
99107
define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
100108
%gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
101109
%val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false)
@@ -107,6 +115,7 @@ define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds,
107115
; GCN: s_load_dword [[PTR:s[0-9]+]]
108116
; GCN: s_mov_b32 m0, [[PTR]]
109117
; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
118+
; GCN-NOT: buffer_wbinvl1
110119
; NOTGFX9: s_mov_b32 m0, -1
111120
; GFX9-NOT: m0
112121
; GCN: _store_dword

‎llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
; GCN: s_load_dword [[PTR:s[0-9]+]]
88
; GCN: s_mov_b32 m0, [[PTR]]
99
; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
10+
; GCN-NOT: buffer_wbinvl1
1011
; GCN: {{.*}}store{{.*}} [[RESULT]]
1112
define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
1213
%val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
@@ -18,6 +19,7 @@ define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace(
1819
; GCN: s_load_dword [[PTR:s[0-9]+]]
1920
; GCN: s_mov_b32 m0, [[PTR]]
2021
; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532{{$}}
22+
; GCN-NOT: buffer_wbinvl1
2123
; GCN: {{.*}}store{{.*}} [[RESULT]]
2224
define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
2325
%gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
@@ -36,6 +38,7 @@ define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32
3638
; CIPLUS: s_mov_b32 m0, [[PTR]]
3739
; CIPLUS: ds_consume [[RESULT:v[0-9]+]] offset:16{{$}}
3840

41+
; GCN-NOT: buffer_wbinvl1
3942
; GCN: {{.*}}store{{.*}} [[RESULT]]
4043
define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
4144
%lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
@@ -53,6 +56,7 @@ define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrsp
5356

5457
; GCN: s_mov_b32 m0, [[PTR]]
5558
; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
59+
; GCN-NOT: buffer_wbinvl1
5660
; GCN: {{.*}}store{{.*}} [[RESULT]]
5761
define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
5862
%gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
@@ -65,6 +69,7 @@ define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds
6569
; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
6670
; GCN: s_mov_b32 m0, [[READLANE]]
6771
; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
72+
; GCN-NOT: buffer_wbinvl1
6873
; GCN: {{.*}}store{{.*}} [[RESULT]]
6974
define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
7075
%val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
@@ -76,6 +81,7 @@ define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)*
7681
; GCN: s_load_dword [[PTR:s[0-9]+]]
7782
; GCN: s_mov_b32 m0, [[PTR]]
7883
; GCN: ds_consume [[RESULT:v[0-9]+]] gds{{$}}
84+
; GCN-NOT: buffer_wbinvl1
7985
; GCN: {{.*}}store{{.*}} [[RESULT]]
8086
define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
8187
%val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gds, i1 false)
@@ -87,6 +93,7 @@ define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace(
8793
; GCN: s_load_dword [[PTR:s[0-9]+]]
8894
; GCN: s_mov_b32 m0, [[PTR]]
8995
; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
96+
; GCN-NOT: buffer_wbinvl1
9097
; GCN: {{.*}}store{{.*}} [[RESULT]]
9198
define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
9299
%gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
@@ -96,6 +103,7 @@ define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32
96103
}
97104

98105
; GCN-LABEL: {{^}}ds_consume_gds_over_max_offset:
106+
; GCN-NOT: buffer_wbinvl1
99107
define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
100108
%gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
101109
%val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false)
@@ -107,6 +115,7 @@ define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds
107115
; GCN: s_load_dword [[PTR:s[0-9]+]]
108116
; GCN: s_mov_b32 m0, [[PTR]]
109117
; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
118+
; GCN-NOT: buffer_wbinvl1
110119
; NOTGFX9: s_mov_b32 m0, -1
111120
; GFX9-NOT: m0
112121
; GCN: _store_dword

0 commit comments

Comments
 (0)