Skip to content

[AMDGPU] Disable VALU sinking and hoisting with WWM #123124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+7,764 −7,385
Open

[AMDGPU] Disable VALU sinking and hoisting with WWM #123124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2773,6 +2773,9 @@ void AMDGPUDAGToDAGISel::SelectINTRINSIC_WO_CHAIN(SDNode *N) {
case Intrinsic::amdgcn_wwm:
case Intrinsic::amdgcn_strict_wwm:
Opcode = AMDGPU::STRICT_WWM;
CurDAG->getMachineFunction()
.getInfo<SIMachineFunctionInfo>()
->setInitWholeWave();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would not recommend using HasInitWholeWave for this. That has a very narrow meaning referring only to the use of the llvm.amdgcn.init.whole.wave intrinsic, not WWM in general.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ack. I can create a separate property HasWWM, but I really want to hear if we even want to go that way.

break;
case Intrinsic::amdgcn_strict_wqm:
Opcode = AMDGPU::STRICT_WQM;
Expand Down
6 changes: 5 additions & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1055,8 +1055,12 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
case Intrinsic::amdgcn_softwqm:
return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM);
case Intrinsic::amdgcn_strict_wwm:
case Intrinsic::amdgcn_wwm:
case Intrinsic::amdgcn_wwm: {
MachineFunction *MF = I.getParent()->getParent();
SIMachineFunctionInfo *MFInfo = MF->getInfo<SIMachineFunctionInfo>();
MFInfo->setInitWholeWave();
return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WWM);
}
case Intrinsic::amdgcn_strict_wqm:
return constrainCopyLikeIntrin(I, AMDGPU::STRICT_WQM);
case Intrinsic::amdgcn_writelane:
Expand Down
6 changes: 5 additions & 1 deletion llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,11 @@ static bool resultDependsOnExec(const MachineInstr &MI) {
bool SIInstrInfo::isIgnorableUse(const MachineOperand &MO) const {
// Any implicit use of exec by VALU is not a real register read.
return MO.getReg() == AMDGPU::EXEC && MO.isImplicit() &&
isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent());
isVALU(*MO.getParent()) && !resultDependsOnExec(*MO.getParent()) &&
!MO.getParent()
->getMF()
->getInfo<SIMachineFunctionInfo>()
->hasInitWholeWave();
}

bool SIInstrInfo::isSafeToSink(MachineInstr &MI,
Expand Down
1,380 changes: 700 additions & 680 deletions llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
View file
Open in desktop

Large diffs are not rendered by default.

6,034 changes: 3,067 additions & 2,967 deletions llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 11 additions & 9 deletions llvm/test/CodeGen/AMDGPU/cse-convergent.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,31 +8,33 @@ define i32 @test(i32 %val, i32 %cond) {
; GCN-NEXT: s_xor_saveexec_b32 s4, -1
; GCN-NEXT: buffer_store_dword v2, off, s[0:3], s32 ; 4-byte Folded Spill
; GCN-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GCN-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GCN-NEXT: s_waitcnt_depctr 0xffe3
; GCN-NEXT: s_mov_b32 exec_lo, s4
; GCN-NEXT: s_or_saveexec_b32 s4, -1
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v0, s4
; GCN-NEXT: v_mov_b32_dpp v2, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: s_mov_b32 exec_lo, s4
; GCN-NEXT: v_mov_b32_e32 v5, 0
; GCN-NEXT: v_mov_b32_e32 v4, v2
; GCN-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: s_mov_b32 exec_lo, s4
; GCN-NEXT: v_mov_b32_e32 v5, v4
; GCN-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GCN-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GCN-NEXT: ; %bb.1: ; %if
; GCN-NEXT: s_or_saveexec_b32 s5, -1
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_cndmask_b32_e64 v3, 0, v0, s5
; GCN-NEXT: v_mov_b32_dpp v2, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v0, s5
; GCN-NEXT: v_mov_b32_dpp v3, v2 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: s_mov_b32 exec_lo, s5
; GCN-NEXT: v_mov_b32_e32 v5, v2
; GCN-NEXT: v_mov_b32_e32 v2, v3
; GCN-NEXT: ; %bb.2: ; %end
; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s4
; GCN-NEXT: v_add_nc_u32_e32 v0, v4, v5
; GCN-NEXT: v_add_nc_u32_e32 v0, v5, v2
; GCN-NEXT: s_xor_saveexec_b32 s4, -1
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_clause 0x2
; GCN-NEXT: buffer_load_dword v2, off, s[0:3], s32
; GCN-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:4
; GCN-NEXT: buffer_load_dword v4, off, s[0:3], s32 offset:8
; GCN-NEXT: s_waitcnt_depctr 0xffe3
; GCN-NEXT: s_mov_b32 exec_lo, s4
; GCN-NEXT: s_waitcnt vmcnt(0)
Expand Down
2,215 changes: 1,138 additions & 1,077 deletions llvm/test/CodeGen/AMDGPU/global_atomics_scan_fadd.ll
View file
Open in desktop

Large diffs are not rendered by default.

1,537 changes: 792 additions & 745 deletions llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
View file
Open in desktop

Large diffs are not rendered by default.

1,537 changes: 792 additions & 745 deletions llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
View file
Open in desktop

Large diffs are not rendered by default.

2,289 changes: 1,179 additions & 1,110 deletions llvm/test/CodeGen/AMDGPU/global_atomics_scan_fsub.ll
View file
Open in desktop

Large diffs are not rendered by default.

13 changes: 7 additions & 6 deletions llvm/test/CodeGen/AMDGPU/licm-wwm.mir
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,28 @@
# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -run-pass=early-machinelicm,si-wqm -o - %s | FileCheck -check-prefix=GCN %s

# Machine LICM may hoist an intruction from a WWM region, which will force SI-WQM pass
# to create a second WWM region. This is an unwanted hoisting.
# to create a second WWM region. This is an unwanted hoisting. Make sure it does not happen.

---
name: licm_move_wwm
tracksRegLiveness: true
machineFunctionInfo:
hasInitWholeWave: true

body: |
; GCN-LABEL: name: licm_move_wwm
; GCN: bb.0:
; GCN-NEXT: successors: %bb.1(0x80000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_32 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: $exec_lo = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
; GCN-NEXT: S_BRANCH %bb.1
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.1:
; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[ENTER_STRICT_WWM1:%[0-9]+]]:sreg_32 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
; GCN-NEXT: [[ENTER_STRICT_WWM:%[0-9]+]]:sreg_32 = ENTER_STRICT_WWM -1, implicit-def $exec, implicit-def $scc, implicit $exec
; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32 = V_READFIRSTLANE_B32 [[V_MOV_B32_e32_]], implicit $exec
; GCN-NEXT: $exec_lo = EXIT_STRICT_WWM [[ENTER_STRICT_WWM1]]
; GCN-NEXT: $exec_lo = EXIT_STRICT_WWM [[ENTER_STRICT_WWM]]
; GCN-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
; GCN-NEXT: $exec_lo = S_OR_B32 $exec_lo, [[COPY]], implicit-def $scc
; GCN-NEXT: S_CBRANCH_EXECNZ %bb.1, implicit $exec
Expand Down
21 changes: 11 additions & 10 deletions llvm/test/CodeGen/AMDGPU/set-inactive-wwm-overwrite.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,32 @@
define amdgpu_cs void @if_then(ptr addrspace(8) inreg %input, ptr addrspace(8) inreg %output, <3 x i32> %LocalInvocationId) {
; GCN-LABEL: if_then:
; GCN: ; %bb.0: ; %.entry
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
; GCN-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GCN-NEXT: ; %bb.1: ; %.bb0
; GCN-NEXT: v_mov_b32_e32 v3, 1
; GCN-NEXT: v_mov_b32_e32 v4, 1
; GCN-NEXT: ; %bb.2: ; %.merge
; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GCN-NEXT: v_cmp_lt_u32_e32 vcc_lo, 3, v0
; GCN-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GCN-NEXT: s_cbranch_execz .LBB0_4
; GCN-NEXT: ; %bb.3: ; %.then
; GCN-NEXT: s_or_saveexec_b32 s1, -1
; GCN-NEXT: v_cndmask_b32_e64 v1, 0, v3, s1
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: v_mov_b32_dpp v2, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_cndmask_b32_e64 v2, 0, v4, s1
; GCN-NEXT: v_mov_b32_e32 v3, v1
; GCN-NEXT: v_mov_b32_dpp v3, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: s_mov_b32 exec_lo, s1
; GCN-NEXT: v_mov_b32_e32 v0, v2
; GCN-NEXT: v_mov_b32_e32 v4, -1
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: buffer_store_dword v4, v0, s[4:7], 0 offen
; GCN-NEXT: v_mov_b32_e32 v0, v3
; GCN-NEXT: v_mov_b32_e32 v5, -1
; GCN-NEXT: v_mov_b32_e32 v4, v1
; GCN-NEXT: buffer_store_dword v5, v0, s[4:7], 0 offen
; GCN-NEXT: .LBB0_4: ; %.end
; GCN-NEXT: s_waitcnt_depctr 0xffe3
; GCN-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GCN-NEXT: v_mov_b32_e32 v0, -1
; GCN-NEXT: buffer_store_dword v0, v3, s[4:7], 0 offen
; GCN-NEXT: buffer_store_dword v0, v4, s[4:7], 0 offen
; GCN-NEXT: s_endpgm
.entry:
%LocalInvocationId.i0 = extractelement <3 x i32> %LocalInvocationId, i32 0
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/CodeGen/AMDGPU/should-not-hoist-set-inactive.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ define amdgpu_cs void @should_not_hoist_set_inactive(<4 x i32> inreg %i14, i32 i
; GCN-NEXT: ; %bb.3: ; %bb1
; GCN-NEXT: ; in Loop: Header=BB0_2 Depth=1
; GCN-NEXT: s_or_saveexec_b32 s9, -1
; GCN-NEXT: v_cndmask_b32_e64 v3, 0, s4, s9
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_mov_b32_dpp v4, v3 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_cndmask_b32_e64 v4, 0, s4, s9
; GCN-NEXT: v_mov_b32_dpp v3, v4 row_xmask:1 row_mask:0xf bank_mask:0xf
; GCN-NEXT: s_mov_b32 exec_lo, s9
; GCN-NEXT: v_mov_b32_e32 v0, v4
; GCN-NEXT: v_mov_b32_e32 v0, v3
; GCN-NEXT: s_and_b32 exec_lo, exec_lo, s5
; GCN-NEXT: s_cbranch_execz .LBB0_1
; GCN-NEXT: ; %bb.4: ; %bb2
Expand Down
22 changes: 12 additions & 10 deletions llvm/test/CodeGen/AMDGPU/wwm-reserved-spill.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,39 +250,41 @@ define amdgpu_gfx void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg)
; GFX9-O3-NEXT: s_xor_saveexec_b64 s[34:35], -1
; GFX9-O3-NEXT: buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O3-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-O3-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O3-NEXT: s_mov_b64 exec, s[34:35]
; GFX9-O3-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; GFX9-O3-NEXT: v_mov_b32_e32 v5, 0
; GFX9-O3-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; GFX9-O3-NEXT: s_or_saveexec_b64 s[34:35], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_mov_b32_e32 v2, v1
; GFX9-O3-NEXT: s_waitcnt vmcnt(0)
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v3, s[34:35]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v3, 0, v4, s[34:35]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_mov_b32_dpp v2, v3 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v2, v3, v2
; GFX9-O3-NEXT: s_mov_b64 exec, s[34:35]
; GFX9-O3-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v4, v2
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-O3-NEXT: s_and_saveexec_b64 s[34:35], vcc
; GFX9-O3-NEXT: ; %bb.1: ; %if
; GFX9-O3-NEXT: s_or_saveexec_b64 s[36:37], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[36:37]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v5, s[36:37]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-O3-NEXT: s_mov_b64 exec, s[36:37]
; GFX9-O3-NEXT: v_mov_b32_e32 v5, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O3-NEXT: ; %bb.2: ; %merge
; GFX9-O3-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX9-O3-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-O3-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-O3-NEXT: v_and_b32_e32 v0, 2, v0
; GFX9-O3-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:4
; GFX9-O3-NEXT: s_xor_saveexec_b64 s[34:35], -1
; GFX9-O3-NEXT: buffer_load_dword v1, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O3-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; GFX9-O3-NEXT: buffer_load_dword v3, off, s[0:3], s32 offset:8 ; 4-byte Folded Reload
; GFX9-O3-NEXT: s_mov_b64 exec, s[34:35]
; GFX9-O3-NEXT: s_waitcnt vmcnt(0)
; GFX9-O3-NEXT: s_setpc_b64 s[30:31]
Expand Down
40 changes: 20 additions & 20 deletions llvm/test/CodeGen/AMDGPU/wwm-reserved.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,31 +217,31 @@ define amdgpu_cs void @cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
;
; GFX9-O3-LABEL: cfg:
; GFX9-O3: ; %bb.0: ; %entry
; GFX9-O3-NEXT: buffer_load_dwordx2 v[3:4], off, s[0:3], 0
; GFX9-O3-NEXT: v_mov_b32_e32 v5, 0
; GFX9-O3-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0
; GFX9-O3-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_mov_b32_e32 v2, v1
; GFX9-O3-NEXT: s_waitcnt vmcnt(0)
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v3, s[4:5]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v3, 0, v4, s[4:5]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_mov_b32_dpp v2, v3 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v2, v3, v2
; GFX9-O3-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O3-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v4, v2
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-O3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-O3-NEXT: ; %bb.1: ; %if
; GFX9-O3-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[6:7]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v5, s[6:7]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-O3-NEXT: s_mov_b64 exec, s[6:7]
; GFX9-O3-NEXT: v_mov_b32_e32 v5, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O3-NEXT: ; %bb.2: ; %merge
; GFX9-O3-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX9-O3-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-O3-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-O3-NEXT: v_and_b32_e32 v0, 2, v0
Expand Down Expand Up @@ -1069,31 +1069,31 @@ define amdgpu_cs void @strict_wwm_cfg(ptr addrspace(8) inreg %tmp14, i32 %arg) {
;
; GFX9-O3-LABEL: strict_wwm_cfg:
; GFX9-O3: ; %bb.0: ; %entry
; GFX9-O3-NEXT: buffer_load_dwordx2 v[3:4], off, s[0:3], 0
; GFX9-O3-NEXT: v_mov_b32_e32 v5, 0
; GFX9-O3-NEXT: buffer_load_dwordx2 v[4:5], off, s[0:3], 0
; GFX9-O3-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_mov_b32_e32 v2, v1
; GFX9-O3-NEXT: s_waitcnt vmcnt(0)
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v3, s[4:5]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v3, 0, v4, s[4:5]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_mov_b32_dpp v2, v3 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v2, v3, v2
; GFX9-O3-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O3-NEXT: v_mov_b32_e32 v3, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v4, v2
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-O3-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-O3-NEXT: ; %bb.1: ; %if
; GFX9-O3-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, 0
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v4, s[6:7]
; GFX9-O3-NEXT: v_cndmask_b32_e64 v2, 0, v5, s[6:7]
; GFX9-O3-NEXT: s_nop 1
; GFX9-O3-NEXT: v_mov_b32_dpp v1, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9-O3-NEXT: v_add_u32_e32 v1, v2, v1
; GFX9-O3-NEXT: v_add_u32_e32 v2, v2, v1
; GFX9-O3-NEXT: s_mov_b64 exec, s[6:7]
; GFX9-O3-NEXT: v_mov_b32_e32 v5, v1
; GFX9-O3-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O3-NEXT: ; %bb.2: ; %merge
; GFX9-O3-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
; GFX9-O3-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX9-O3-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GFX9-O3-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX9-O3-NEXT: v_and_b32_e32 v0, 2, v0
Expand Down
18 changes: 18 additions & 0 deletions llvm/test/CodeGen/MIR/AMDGPU/init-whole.wave.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,21 @@ bb.1:
bb.2:
ret void
}

; GCN-LABEL: name: wwm
; GCN: hasInitWholeWave: true
define void @wwm(ptr addrspace(1) inreg %p) {
%val = load i32, ptr addrspace(1) %p
%wwm = tail call i32 @llvm.amdgcn.wwm.i32(i32 %val)
store i32 %wwm, ptr addrspace(1) %p
ret void
}

; GCN-LABEL: name: strict_wwm
; GCN: hasInitWholeWave: true
define void @strict_wwm(ptr addrspace(1) inreg %p) {
%val = load i32, ptr addrspace(1) %p
%wwm = tail call i32 @llvm.amdgcn.strict.wwm.i32(i32 %val)
store i32 %wwm, ptr addrspace(1) %p
ret void
}
Loading