Skip to content

[AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo. #129103

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 28, 2025
Merged

[AMDGPU][NFC] Move isXDL and isDGEMM to SIInstrInfo. #129103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 21 additions & 38 deletions llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,25 +111,6 @@ static bool isSMovRel(unsigned Opcode) {
}
}

static bool isDGEMM(unsigned Opcode) {
return AMDGPU::getMAIIsDGEMM(Opcode);
}

static bool isXDL(const GCNSubtarget &ST, const MachineInstr &MI) {
unsigned Opcode = MI.getOpcode();

if (!SIInstrInfo::isMAI(MI) ||
isDGEMM(Opcode) ||
Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
return false;

if (!ST.hasGFX940Insts())
return true;

return AMDGPU::getMAIIsGFX940XDL(Opcode);
}

static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
const MachineInstr &MI) {
if (TII.isAlwaysGDS(MI.getOpcode()))
Expand Down Expand Up @@ -2375,7 +2356,8 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
unsigned Opc1 = MI1->getOpcode();
int NeedWaitStates = 0;
if (OpNo == SrcCIdx) {
if (!isDGEMM(Opc) && (!ST.hasGFX940Insts() && isDGEMM(Opc1))) {
if (!SIInstrInfo::isDGEMM(Opc) &&
(!ST.hasGFX940Insts() && SIInstrInfo::isDGEMM(Opc1))) {
NeedWaitStates = 0;
} else if (FullReg) {
if ((Opc == AMDGPU::V_MFMA_F64_4X4X4F64_e64 ||
Expand All @@ -2392,26 +2374,26 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
case AMDGPU::V_MFMA_F64_16X16X4F64_vgprcd_e64:
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_e64:
case AMDGPU::V_MFMA_F64_16X16X4F64_mac_vgprcd_e64:
if (!isXDL(ST, *MI))
if (!TII.isXDL(*MI))
NeedWaitStates =
ST.hasGFX950Insts()
? GFX950_DMFMA16x16WritesVGPROverlappedSrcCWaitStates
: DMFMA16x16WritesVGPROverlappedSrcCWaitStates;
break;
case AMDGPU::V_MFMA_F64_4X4X4F64_e64:
case AMDGPU::V_MFMA_F64_4X4X4F64_vgprcd_e64:
if (!isXDL(ST, *MI))
if (!TII.isXDL(*MI))
NeedWaitStates = DMFMA4x4WritesVGPROverlappedSrcCWaitStates;
break;
default:
int NumPasses = TSchedModel.computeInstrLatency(MI1);
if (ST.hasGFX940Insts()) {
if (isXDL(ST, *MI) && !isXDL(ST, *MI1))
if (TII.isXDL(*MI) && !TII.isXDL(*MI1))
break;

NeedWaitStates =
isXDL(ST, *MI1)
? (isXDL(ST, *MI)
TII.isXDL(*MI1)
? (TII.isXDL(*MI)
? GFX940_XDL_N_PassWritesVGPROverlappedXDLOrSMFMASrcCWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_XDL_N_PassWritesVGPROverlappedSGEMMDGEMMSrcCWaitStates(
Expand All @@ -2424,18 +2406,19 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {
switch (NumPasses) {
case 2:
NeedWaitStates =
isDGEMM(Opc) ? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
SIInstrInfo::isDGEMM(Opc)
? SMFMA4x4WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA4x4WritesVGPROverlappedSMFMASrcCWaitStates;
break;
case 8:
NeedWaitStates =
isDGEMM(Opc)
SIInstrInfo::isDGEMM(Opc)
? SMFMA16x16WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA16x16WritesVGPROverlappedSMFMASrcCWaitStates;
break;
case 16:
NeedWaitStates =
isDGEMM(Opc)
SIInstrInfo::isDGEMM(Opc)
? SMFMA32x32WritesVGPROverlappedDMFMASrcCWaitStates
: SMFMA32x32WritesVGPROverlappedSMFMASrcCWaitStates;
break;
Expand Down Expand Up @@ -2464,7 +2447,7 @@ int GCNHazardRecognizer::checkMAIHazards90A(MachineInstr *MI) {

if (ST.hasGFX940Insts()) {
NeedWaitStates =
isXDL(ST, *MI1)
TII.isXDL(*MI1)
? GFX940_XDL_N_PassWritesVGPROverlappedSrcABWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWritesVGPROverlappedSrcABWaitStates(
Expand Down Expand Up @@ -2631,7 +2614,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
return 0;

auto IsDGEMMFn = [](const MachineInstr &MI) -> bool {
return isDGEMM(MI.getOpcode());
return SIInstrInfo::isDGEMM(MI.getOpcode());
};

// This is checked in checkMAIHazards90A()
Expand Down Expand Up @@ -2670,7 +2653,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
bool DGEMMAfterVALUWrite = false;
auto IsDGEMMHazard = [&DGEMMAfterVALUWrite, this](const MachineInstr &MI) {
// Found DGEMM on reverse traversal to def.
if (isDGEMM(MI.getOpcode()))
if (SIInstrInfo::isDGEMM(MI.getOpcode()))
DGEMMAfterVALUWrite = true;

// Only hazard if register is defined by a VALU and a DGEMM is found after
Expand Down Expand Up @@ -2745,7 +2728,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
int NumPasses = HazardDefLatency;
int NeedWaitStates = MaxWaitStates;

if (isDGEMM(MFMA->getOpcode())) {
if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
switch (HazardDefLatency) {
case 4:
NeedWaitStates = IsMemOrExport ? DMFMA4x4WriteVgprMemExpReadWaitStates
Expand All @@ -2765,7 +2748,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}
} else if (ST.hasGFX940Insts()) {
NeedWaitStates =
isXDL(ST, *MFMA)
TII.isXDL(*MFMA)
? GFX940_XDL_N_PassWriteVgprVALUMemExpReadWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWriteVgprVALUMemExpReadWaitStates(
Expand Down Expand Up @@ -2838,7 +2821,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
int NeedWaitStates = MaxWaitStates;
int NumPasses = TSchedModel.computeInstrLatency(MFMA);

if (isDGEMM(MFMA->getOpcode())) {
if (SIInstrInfo::isDGEMM(MFMA->getOpcode())) {
switch (NumPasses) {
case 4:
NeedWaitStates = DMFMA4x4WriteVgprVALUWriteWaitStates;
Expand All @@ -2852,7 +2835,7 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}
} else if (ST.hasGFX940Insts()) {
NeedWaitStates =
isXDL(ST, *MFMA)
TII.isXDL(*MFMA)
? GFX940_XDL_N_PassWriteVgprVALUWawWaitStates(
NumPasses, ST.hasGFX950Insts())
: GFX940_SMFMA_N_PassWriteVgprVALUWawWaitStates(NumPasses);
Expand Down Expand Up @@ -2880,11 +2863,11 @@ int GCNHazardRecognizer::checkMAIVALUHazards(MachineInstr *MI) {
}

auto IsSMFMAReadAsCFn = [&Reg, &MFMA, this](const MachineInstr &MI) {
if (!SIInstrInfo::isMFMA(MI) || isDGEMM(MI.getOpcode()) ||
if (!SIInstrInfo::isMFMA(MI) || SIInstrInfo::isDGEMM(MI.getOpcode()) ||
!MI.readsRegister(Reg, &TRI))
return false;

if (ST.hasGFX940Insts() && !isXDL(ST, MI))
if (ST.hasGFX940Insts() && !TII.isXDL(MI))
return false;

const MachineOperand *SrcC =
Expand Down
14 changes: 14 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10251,3 +10251,17 @@ bool SIInstrInfo::isGlobalMemoryObject(const MachineInstr *MI) const {

return TargetInstrInfo::isGlobalMemoryObject(MI);
}

bool SIInstrInfo::isXDL(const MachineInstr &MI) const {
unsigned Opcode = MI.getOpcode();

if (!SIInstrInfo::isMAI(MI) || isDGEMM(Opcode) ||
Opcode == AMDGPU::V_ACCVGPR_WRITE_B32_e64 ||
Opcode == AMDGPU::V_ACCVGPR_READ_B32_e64)
return false;

if (!ST.hasGFX940Insts())
return true;

return AMDGPU::getMAIIsGFX940XDL(Opcode);
}
4 changes: 4 additions & 0 deletions llvm/lib/Target/AMDGPU/SIInstrInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -847,6 +847,10 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
}

bool isXDL(const MachineInstr &MI) const;

static bool isDGEMM(unsigned Opcode) { return AMDGPU::getMAIIsDGEMM(Opcode); }

static bool isLDSDIR(const MachineInstr &MI) {
return MI.getDesc().TSFlags & SIInstrFlags::LDSDIR;
}
Expand Down