Skip to content

[AMDGPU][True16][CodeGen] 16bit spill support in true16 mode #128060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 26, 2025
Merged

[AMDGPU][True16][CodeGen] 16bit spill support in true16 mode #128060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c76963
spill 16 with scratch load/store
broxigarchen Feb 20, 2025
8d19d36
added an assert for scratch flat
broxigarchen Feb 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1580,6 +1580,8 @@ static unsigned getSGPRSpillSaveOpcode(unsigned Size) {

static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
switch (Size) {
case 2:
return AMDGPU::SI_SPILL_V16_SAVE;
case 4:
return AMDGPU::SI_SPILL_V32_SAVE;
case 8:
Expand Down Expand Up @@ -1807,6 +1809,8 @@ static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {

static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
switch (Size) {
case 2:
return AMDGPU::SI_SPILL_V16_RESTORE;
case 4:
return AMDGPU::SI_SPILL_V32_RESTORE;
case 8:
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/AMDGPU/SIInstructions.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1003,6 +1003,7 @@ multiclass SI_SPILL_VGPR <RegisterClass vgpr_class, bit UsesTmp = 0> {
} // End UseNamedOperandTable = 1, Spill = 1, VALU = 1, SchedRW = [WriteVMEM]
}

defm SI_SPILL_V16 : SI_SPILL_VGPR <VGPR_16>;
defm SI_SPILL_V32 : SI_SPILL_VGPR <VGPR_32>;
defm SI_SPILL_V64 : SI_SPILL_VGPR <VReg_64>;
defm SI_SPILL_V96 : SI_SPILL_VGPR <VReg_96>;
Expand Down
25 changes: 21 additions & 4 deletions llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1280,6 +1280,8 @@ static unsigned getNumSubRegsForSpillOp(unsigned Op) {
case AMDGPU::SI_SPILL_WWM_V32_RESTORE:
case AMDGPU::SI_SPILL_WWM_AV32_SAVE:
case AMDGPU::SI_SPILL_WWM_AV32_RESTORE:
case AMDGPU::SI_SPILL_V16_SAVE:
case AMDGPU::SI_SPILL_V16_RESTORE:
return 1;
default: llvm_unreachable("Invalid spill opcode");
}
Expand Down Expand Up @@ -2350,6 +2352,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
case AMDGPU::SI_SPILL_V96_SAVE:
case AMDGPU::SI_SPILL_V64_SAVE:
case AMDGPU::SI_SPILL_V32_SAVE:
case AMDGPU::SI_SPILL_V16_SAVE:
case AMDGPU::SI_SPILL_A1024_SAVE:
case AMDGPU::SI_SPILL_A512_SAVE:
case AMDGPU::SI_SPILL_A384_SAVE:
Expand Down Expand Up @@ -2390,8 +2393,15 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
MFI->getStackPtrOffsetReg());

unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
: AMDGPU::BUFFER_STORE_DWORD_OFFSET;
unsigned Opc;
if (MI->getOpcode() == AMDGPU::SI_SPILL_V16_SAVE) {
assert(ST.enableFlatScratch() && "Flat Scratch is not enabled!");
Opc = AMDGPU::SCRATCH_STORE_SHORT_SADDR_t16;
Copy link
Contributor

@arsenm arsenm Feb 25, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the mubuf ABI does not work with true16? Or is this todo? Can you assert ST.enableFlatScratch at least?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure about this part. Might need some input from @Sisyph when he is back from vacation.

Added an assert first

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think there is a conceptual issue here, just we did not implement T16 BUF instructions or spilling using them yet. We have a ticket to track it. Do you have any idea of the priority of the BUF spilling? It looks like all the latest subtargets have ArchitectedFlatScratch and so use the scratch path.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't remember all the details, but I thought gfx11 still defaulted to mubuf path? I vaguely remember a few rounds of the addressing modes not actually working and/or being as good as mubuf with scratch

} else {
Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
: AMDGPU::BUFFER_STORE_DWORD_OFFSET;
}

auto *MBB = MI->getParent();
bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode());
if (IsWWMRegSpill) {
Expand All @@ -2409,6 +2419,7 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
MI->eraseFromParent();
return true;
}
case AMDGPU::SI_SPILL_V16_RESTORE:
case AMDGPU::SI_SPILL_V32_RESTORE:
case AMDGPU::SI_SPILL_V64_RESTORE:
case AMDGPU::SI_SPILL_V96_RESTORE:
Expand Down Expand Up @@ -2458,8 +2469,14 @@ bool SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==
MFI->getStackPtrOffsetReg());

unsigned Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
: AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
unsigned Opc;
if (MI->getOpcode() == AMDGPU::SI_SPILL_V16_RESTORE) {
assert(ST.enableFlatScratch() && "Flat Scratch is not enabled!");
Opc = AMDGPU::SCRATCH_LOAD_SHORT_D16_SADDR_t16;
} else {
Opc = ST.enableFlatScratch() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
: AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
}
auto *MBB = MI->getParent();
bool IsWWMRegSpill = TII->isWWMRegSpillOpcode(MI->getOpcode());
if (IsWWMRegSpill) {
Expand Down
140 changes: 140 additions & 0 deletions llvm/test/CodeGen/AMDGPU/spill_kill_v16.mir
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# RUN: llc -march=amdgcn -verify-machineinstrs -mcpu=gfx1100 -mattr=+real-true16 -run-pass=prologepilog -o - %s | FileCheck -check-prefix=EXPANDED %s

---
name: spill_restore_vgpr16
tracksRegLiveness: true
stack:
- { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
- { id: 1, name: '', type: spill-slot, offset: 4, size: 4, alignment: 4 }
machineFunctionInfo:
scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
stackPtrOffsetReg: $sgpr32
hasSpilledVGPRs: true
body: |
; EXPANDED-LABEL: name: spill_restore_vgpr16
; EXPANDED: bb.0:
; EXPANDED-NEXT: successors: %bb.1(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
; EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.1:
; EXPANDED-NEXT: successors: %bb.2(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 1
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.2:
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
; EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
bb.0:
successors: %bb.1(0x80000000)
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
SI_SPILL_V16_SAVE killed $vgpr0_hi16, %stack.1, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.1, addrspace 5)
S_NOP 0, implicit renamable $vgpr0_lo16
SI_SPILL_V16_SAVE killed $vgpr0_lo16, %stack.0, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.0, addrspace 5)
S_CBRANCH_SCC1 %bb.1, implicit undef $scc
bb.1:
successors: %bb.2(0x80000000)
S_NOP 1
bb.2:
$vgpr0_lo16 = SI_SPILL_V16_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.0, addrspace 5)
$vgpr0_hi16 = SI_SPILL_V16_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.1, addrspace 5)
S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
...

---
name: spill_restore_vgpr16_middle_of_block
tracksRegLiveness: true
stack:
- { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
- { id: 1, name: '', type: spill-slot, offset: 4, size: 4, alignment: 4 }
machineFunctionInfo:
scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
stackPtrOffsetReg: $sgpr32
hasSpilledVGPRs: true
body: |
; EXPANDED-LABEL: name: spill_restore_vgpr16_middle_of_block
; EXPANDED: bb.0:
; EXPANDED-NEXT: successors: %bb.1(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
; EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.1:
; EXPANDED-NEXT: successors: %bb.2(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 1
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.2:
; EXPANDED-NEXT: S_NOP 1
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
; EXPANDED-NEXT: S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
bb.0:
successors: %bb.1(0x80000000)
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
SI_SPILL_V16_SAVE killed $vgpr0_hi16, %stack.1, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.1, addrspace 5)
S_NOP 0, implicit renamable $vgpr0_lo16
SI_SPILL_V16_SAVE killed $vgpr0_lo16, %stack.0, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.0, addrspace 5)
S_CBRANCH_SCC1 %bb.1, implicit undef $scc
bb.1:
successors: %bb.2(0x80000000)
S_NOP 1
bb.2:
S_NOP 1
$vgpr0_lo16 = SI_SPILL_V16_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.0, addrspace 5)
$vgpr0_hi16 = SI_SPILL_V16_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.1, addrspace 5)
S_NOP 0, implicit killed renamable $vgpr0_lo16, implicit killed renamable $vgpr0_hi16
...

---
name: spill_restore_vgpr16_end_of_block
tracksRegLiveness: true
stack:
- { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4 }
- { id: 1, name: '', type: spill-slot, offset: 4, size: 4, alignment: 4 }
machineFunctionInfo:
scratchRSrcReg: $sgpr0_sgpr1_sgpr2_sgpr3
stackPtrOffsetReg: $sgpr32
hasSpilledVGPRs: true
body: |
; EXPANDED-LABEL: name: spill_restore_vgpr16_end_of_block
; EXPANDED: bb.0:
; EXPANDED-NEXT: successors: %bb.1(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_hi16, $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.1, align 4, addrspace 5)
; EXPANDED-NEXT: S_NOP 0, implicit renamable $vgpr0_lo16
; EXPANDED-NEXT: SCRATCH_STORE_SHORT_SADDR_t16 killed $vgpr0_lo16, $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: S_CBRANCH_SCC1 %bb.1, implicit undef $scc
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.1:
; EXPANDED-NEXT: successors: %bb.2(0x80000000)
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: S_NOP 1
; EXPANDED-NEXT: {{ $}}
; EXPANDED-NEXT: bb.2:
; EXPANDED-NEXT: $vgpr0_lo16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 0, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.0, align 4, addrspace 5)
; EXPANDED-NEXT: $vgpr0_hi16 = SCRATCH_LOAD_SHORT_D16_SADDR_t16 $sgpr32, 4, 0, implicit $exec, implicit $flat_scr :: (load (s16) from %stack.1, align 4, addrspace 5)
bb.0:
successors: %bb.1(0x80000000)
S_NOP 0, implicit-def renamable $vgpr0_lo16, implicit-def renamable $vgpr0_hi16
SI_SPILL_V16_SAVE killed $vgpr0_hi16, %stack.1, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.1, addrspace 5)
S_NOP 0, implicit renamable $vgpr0_lo16
SI_SPILL_V16_SAVE killed $vgpr0_lo16, %stack.0, $sgpr32, 0, implicit $exec :: (store (s16) into %stack.0, addrspace 5)
S_CBRANCH_SCC1 %bb.1, implicit undef $scc
bb.1:
successors: %bb.2(0x80000000)
S_NOP 1
bb.2:
$vgpr0_lo16 = SI_SPILL_V16_RESTORE %stack.0, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.0, addrspace 5)
$vgpr0_hi16 = SI_SPILL_V16_RESTORE %stack.1, $sgpr32, 0, implicit $exec :: (load (s16) from %stack.1, addrspace 5)
...
Loading