Skip to content

[AMDGPU] Remove SIWholeQuadMode pass early exit #98450

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 17, 2024
Merged

[AMDGPU] Remove SIWholeQuadMode pass early exit #98450

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a7356db
[AMDGPU] Remove SIWholeQuadMode pass early exit
perlfu Jul 11, 2024
627b345
Address comments:
perlfu Jul 14, 2024
f3317a0
Match previous conditions for variable live mask register introduction.
perlfu Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 38 additions & 25 deletions llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,11 +219,12 @@ class SIWholeQuadMode : public MachineFunctionPass {
void lowerBlock(MachineBasicBlock &MBB);
void processBlock(MachineBasicBlock &MBB, bool IsEntry);

void lowerLiveMaskQueries();
void lowerCopyInstrs();
void lowerKillInstrs(bool IsWQM);
bool lowerLiveMaskQueries();
bool lowerCopyInstrs();
bool lowerKillInstrs(bool IsWQM);
void lowerInitExec(MachineInstr &MI);
MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry);
MachineBasicBlock::iterator lowerInitExecInstrs(MachineBasicBlock &Entry,
bool &Changed);

public:
static char ID;
Expand Down Expand Up @@ -797,6 +798,8 @@ MachineBasicBlock *SIWholeQuadMode::splitBlock(MachineBasicBlock *BB,

MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB,
MachineInstr &MI) {
assert(LiveMaskReg.isVirtual());

const DebugLoc &DL = MI.getDebugLoc();
unsigned Opcode = 0;

Expand Down Expand Up @@ -914,6 +917,8 @@ MachineInstr *SIWholeQuadMode::lowerKillF32(MachineBasicBlock &MBB,

MachineInstr *SIWholeQuadMode::lowerKillI1(MachineBasicBlock &MBB,
MachineInstr &MI, bool IsWQM) {
assert(LiveMaskReg.isVirtual());

const DebugLoc &DL = MI.getDebugLoc();
MachineInstr *MaskUpdateMI = nullptr;

Expand Down Expand Up @@ -1145,6 +1150,8 @@ MachineBasicBlock::iterator SIWholeQuadMode::prepareInsertion(
void SIWholeQuadMode::toExact(MachineBasicBlock &MBB,
MachineBasicBlock::iterator Before,
Register SaveWQM) {
assert(LiveMaskReg.isVirtual());

bool IsTerminator = Before == MBB.end();
if (!IsTerminator) {
auto FirstTerm = MBB.getFirstTerminator();
Expand Down Expand Up @@ -1424,7 +1431,7 @@ void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
assert(!SavedNonStrictReg);
}

void SIWholeQuadMode::lowerLiveMaskQueries() {
bool SIWholeQuadMode::lowerLiveMaskQueries() {
for (MachineInstr *MI : LiveMaskQueries) {
const DebugLoc &DL = MI->getDebugLoc();
Register Dest = MI->getOperand(0).getReg();
Expand All @@ -1436,9 +1443,10 @@ void SIWholeQuadMode::lowerLiveMaskQueries() {
LIS->ReplaceMachineInstrInMaps(*MI, *Copy);
MI->eraseFromParent();
}
return !LiveMaskQueries.empty();
}

void SIWholeQuadMode::lowerCopyInstrs() {
bool SIWholeQuadMode::lowerCopyInstrs() {
for (MachineInstr *MI : LowerToMovInstrs) {
assert(MI->getNumExplicitOperands() == 2);

Expand Down Expand Up @@ -1493,9 +1501,10 @@ void SIWholeQuadMode::lowerCopyInstrs() {
*MRI, MI->getOperand(0)));
MI->setDesc(TII->get(CopyOp));
}
return !LowerToCopyInstrs.empty() || !LowerToMovInstrs.empty();
}

void SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
bool SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
for (MachineInstr *MI : KillInstrs) {
MachineBasicBlock *MBB = MI->getParent();
MachineInstr *SplitPoint = nullptr;
Expand All @@ -1511,6 +1520,7 @@ void SIWholeQuadMode::lowerKillInstrs(bool IsWQM) {
if (SplitPoint)
splitBlock(MBB, SplitPoint);
}
return !KillInstrs.empty();
}

void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) {
Expand Down Expand Up @@ -1602,7 +1612,7 @@ void SIWholeQuadMode::lowerInitExec(MachineInstr &MI) {
/// Lower INIT_EXEC instructions. Return a suitable insert point in \p Entry
/// for instructions that depend on EXEC.
MachineBasicBlock::iterator
SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry) {
SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry, bool &Changed) {
MachineBasicBlock::iterator InsertPt = Entry.getFirstNonPHI();

for (MachineInstr *MI : InitExecInstrs) {
Expand All @@ -1613,6 +1623,7 @@ SIWholeQuadMode::lowerInitExecInstrs(MachineBasicBlock &Entry) {
InsertPt = std::next(MI->getIterator());

lowerInitExec(*MI);
Changed = true;
}

return InsertPt;
Expand Down Expand Up @@ -1665,48 +1676,50 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
}

const char GlobalFlags = analyzeFunction(MF);
const bool NeedsLiveMask = !(KillInstrs.empty() && LiveMaskQueries.empty());
bool Changed = false;

LiveMaskReg = Exec;

MachineBasicBlock &Entry = MF.front();
MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry);

// Shader is simple does not need any state changes or any complex lowering
if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
LowerToMovInstrs.empty() && KillInstrs.empty()) {
lowerLiveMaskQueries();
if (!InitExecInstrs.empty())
LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
return !InitExecInstrs.empty() || !LiveMaskQueries.empty();
}
MachineBasicBlock::iterator EntryMI = lowerInitExecInstrs(Entry, Changed);

// Store a copy of the original live mask when required
if (NeedsLiveMask || (GlobalFlags & StateWQM)) {
const bool HasLiveMaskQueries = !LiveMaskQueries.empty();
const bool HasWaveModes = GlobalFlags & ~StateExact;
const bool HasKills = !KillInstrs.empty();
const bool UsesWQM = GlobalFlags & StateWQM;
if (HasKills || UsesWQM || (HasWaveModes && HasLiveMaskQueries)) {
LiveMaskReg = MRI->createVirtualRegister(TRI->getBoolRC());
MachineInstr *MI =
BuildMI(Entry, EntryMI, DebugLoc(), TII->get(AMDGPU::COPY), LiveMaskReg)
.addReg(Exec);
LIS->InsertMachineInstrInMaps(*MI);
Changed = true;
}

LLVM_DEBUG(printInfo());

lowerLiveMaskQueries();
lowerCopyInstrs();
Changed |= lowerLiveMaskQueries();
Changed |= lowerCopyInstrs();

// Shader only needs WQM
if (GlobalFlags == StateWQM) {
if (!HasWaveModes) {
// No wave mode execution
Changed |= lowerKillInstrs(false);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before this patch we never called lowerKillInstrs(false). Was that a bug?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it wasn't necessary.
We ran the whole lowering for an Exact shader with kills.
In the whole lower lowerBlock performs the same function as lowerKillInstrs.

} else if (GlobalFlags == StateWQM) {
// Shader only needs WQM
auto MI = BuildMI(Entry, EntryMI, DebugLoc(), TII->get(WQMOpc), Exec)
.addReg(Exec);
LIS->InsertMachineInstrInMaps(*MI);
lowerKillInstrs(true);
Changed = true;
} else {
// Wave mode switching requires full lowering pass.
for (auto BII : Blocks)
processBlock(*BII.first, BII.first == &Entry);
// Lowering blocks causes block splitting so perform as a second pass.
for (auto BII : Blocks)
lowerBlock(*BII.first);
Changed = true;
}

// Compute live range for live mask
Expand All @@ -1722,5 +1735,5 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
if (!KillInstrs.empty() || !InitExecInstrs.empty())
LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);

return true;
return Changed;
}
Loading