Skip to content

[MacroFusion] Add IsPostRA to indicate whether running in post-ra scheduler #77567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[MacroFusion] Add IsPostRA to indicate whether running in post-ra scheduler #77567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion llvm/include/llvm/CodeGen/MacroFusion.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class SUnit;
using MacroFusionPredTy = bool (*)(const TargetInstrInfo &TII,
const TargetSubtargetInfo &STI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI);
const MachineInstr &SecondMI, bool IsPostRA);

/// Checks if the number of cluster edges between SU and its predecessors is
/// less than FuseLimit
Expand Down
15 changes: 9 additions & 6 deletions llvm/include/llvm/Target/TargetSchedule.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,7 @@ def both_fusion_target : FusionTarget;
// * const MachineRegisterInfo &MRI
// * const MachineInstr *FirstMI
// * const MachineInstr &SecondMI
// * bool IsPostRA
class FusionPredicate<FusionTarget target> {
FusionTarget Target = target;
}
Expand Down Expand Up @@ -642,10 +643,16 @@ def WildcardFalse : WildcardPred<0>;
def WildcardTrue : WildcardPred<1>;

// Indicates that the destination register of `FirstMI` should have one use if
// it is a virtual register.
// it is a virtual register (fusion is done in pre-ra scheduler).
class OneUsePred : FirstFusionPredicate;
def OneUse : OneUsePred;

// Indicates that the first register of `SecondMI` should be the same as the
// second register if it is a physical register (fusion is done in post-ra
// scheduler).
class SameRegisterPred : SecondFusionPredicate;
def SameRegister : SameRegisterPred;

// Handled by MacroFusionPredicatorEmitter backend.
// The generated predicator will be like:
// ```
Expand Down Expand Up @@ -688,11 +695,7 @@ class SimpleFusion<MCInstPredicate firstPred, MCInstPredicate secondPred,
SecondFusionPredicateWithMCInstPredicate<secondPred>,
WildcardTrue,
FirstFusionPredicateWithMCInstPredicate<firstPred>,
SecondFusionPredicateWithMCInstPredicate<
CheckAny<[
CheckIsVRegOperand<0>,
CheckSameRegOperand<0, 1>
]>>,
SameRegister,
OneUse,
TieReg<0, 1>,
],
Expand Down
13 changes: 8 additions & 5 deletions llvm/lib/CodeGen/MacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,17 +151,18 @@ class MacroFusion : public ScheduleDAGMutation {
bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &STI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI);
const MachineInstr &SecondMI, bool IsPostRA);
};

} // end anonymous namespace

bool MacroFusion::shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &STI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
return llvm::any_of(Predicates, [&](MacroFusionPredTy Predicate) {
return Predicate(TII, STI, FirstMI, SecondMI);
return Predicate(TII, STI, FirstMI, SecondMI, IsPostRA);
});
}

Expand All @@ -183,9 +184,11 @@ bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU)
const MachineInstr &AnchorMI = *AnchorSU.getInstr();
const TargetInstrInfo &TII = *DAG.TII;
const TargetSubtargetInfo &ST = DAG.MF.getSubtarget();
bool IsPostRA = DAG.MF.getProperties().hasProperty(
MachineFunctionProperties::Property::NoVRegs);

// Check if the anchor instr may be fused.
if (!shouldScheduleAdjacent(TII, ST, nullptr, AnchorMI))
if (!shouldScheduleAdjacent(TII, ST, nullptr, AnchorMI, IsPostRA))
return false;

// Explorer for fusion candidates among the dependencies of the anchor instr.
Expand All @@ -201,7 +204,7 @@ bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU)
// Only chain two instructions together at most.
const MachineInstr *DepMI = DepSU.getInstr();
if (!hasLessThanNumFused(DepSU, 2) ||
!shouldScheduleAdjacent(TII, ST, DepMI, AnchorMI))
!shouldScheduleAdjacent(TII, ST, DepMI, AnchorMI, IsPostRA))
continue;

if (fuseInstructionPair(DAG, DepSU, AnchorSU))
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,8 @@ static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);

// All checking functions assume that the 1st instr is a wildcard if it is
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ namespace {
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII_,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const SIInstrInfo &TII = static_cast<const SIInstrInfo&>(TII_);

switch (SecondMI.getOpcode()) {
Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII,
static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
unsigned Opc2 = SecondMI.getOpcode();
auto SecondCanBeVOPD = AMDGPU::getCanBeVOPD(Opc2);
Expand Down Expand Up @@ -165,7 +166,7 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
std::vector<SUnit>::iterator ISUI, JSUI;
for (ISUI = DAG->SUnits.begin(); ISUI != DAG->SUnits.end(); ++ISUI) {
const MachineInstr *IMI = ISUI->getInstr();
if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI))
if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI, /*IsPostRA=*/true))
continue;
if (!hasLessThanNumFused(*ISUI, 2))
continue;
Expand All @@ -175,7 +176,7 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
continue;
const MachineInstr *JMI = JSUI->getInstr();
if (!hasLessThanNumFused(*JSUI, 2) ||
!shouldScheduleAdjacent(TII, ST, IMI, *JMI))
!shouldScheduleAdjacent(TII, ST, IMI, *JMI, /*IsPostRA=*/true))
continue;
if (fuseInstructionPair(*DAG, *ISUI, *JSUI))
break;
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/ARM/ARMMacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ static bool isLiteralsPair(const MachineInstr *FirstMI,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const ARMSubtarget &ST = static_cast<const ARMSubtarget&>(TSI);

if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/PowerPC/PPCMacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,8 @@ static bool checkOpConstraints(FusionFeature::FusionKind Kd,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
// We use the PPC namespace to avoid the need to prefix opcodes with PPC:: in
// the def file.
using namespace PPC;
Expand Down
49 changes: 27 additions & 22 deletions llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,16 @@

using namespace llvm;

static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) {
static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI,
bool IsPostRA) {
if (!SecondMI.getOperand(1).isReg())
return false;

if (SecondMI.getOperand(1).getReg() != FirstDest)
return false;

// If the input is virtual make sure this is the only user.
if (FirstDest.isVirtual()) {
if (!IsPostRA) {
auto &MRI = SecondMI.getMF()->getRegInfo();
return MRI.hasOneNonDBGUse(FirstDest);
}
Expand All @@ -37,7 +38,8 @@ static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) {
// Fuse load with add:
// add rd, rs1, rs2
// ld rd, 0(rd)
static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::LD)
return false;

Expand All @@ -55,13 +57,14 @@ static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOpcode() != RISCV::ADD)
return true;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

// Fuse zero extension of halfword:
// slli rd, rs1, 48
// srli rd, rd, 48
static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;

Expand All @@ -82,13 +85,14 @@ static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOperand(2).getImm() != 48)
return false;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

// Fuse zero extension of word:
// slli rd, rs1, 32
// srli rd, rd, 32
static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;

Expand All @@ -109,15 +113,15 @@ static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOperand(2).getImm() != 32)
return false;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

// Fuse shifted zero extension of word:
// slli rd, rs1, 32
// srli rd, rd, x
// where 0 <= x < 32
static bool isShiftedZExtW(const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI, bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;

Expand All @@ -139,14 +143,14 @@ static bool isShiftedZExtW(const MachineInstr *FirstMI,
if (FirstMI->getOperand(2).getImm() != 32)
return false;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

// Fuse AUIPC followed by ADDI
// auipc rd, imm20
// addi rd, rd, imm12
static bool isAUIPCADDI(const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI, bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::ADDI)
return false;
// Assume the 1st instr to be a wildcard if it is unspecified.
Expand All @@ -156,15 +160,15 @@ static bool isAUIPCADDI(const MachineInstr *FirstMI,
if (FirstMI->getOpcode() != RISCV::AUIPC)
return false;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

// Fuse LUI followed by ADDI or ADDIW.
// rd = imm[31:0] which decomposes to
// lui rd, imm[31:12]
// addi(w) rd, rd, imm[11:0]
static bool isLUIADDI(const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
static bool isLUIADDI(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::ADDI &&
SecondMI.getOpcode() != RISCV::ADDIW)
return false;
Expand All @@ -175,31 +179,32 @@ static bool isLUIADDI(const MachineInstr *FirstMI,
if (FirstMI->getOpcode() != RISCV::LUI)
return false;

return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}

static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const RISCVSubtarget &ST = static_cast<const RISCVSubtarget &>(TSI);

if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI))
if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI, IsPostRA))
return true;

if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI))
if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI, IsPostRA))
return true;

if (ST.hasZExtHFusion() && isZExtH(FirstMI, SecondMI))
if (ST.hasZExtHFusion() && isZExtH(FirstMI, SecondMI, IsPostRA))
return true;

if (ST.hasZExtWFusion() && isZExtW(FirstMI, SecondMI))
if (ST.hasZExtWFusion() && isZExtW(FirstMI, SecondMI, IsPostRA))
return true;

if (ST.hasShiftedZExtWFusion() && isShiftedZExtW(FirstMI, SecondMI))
if (ST.hasShiftedZExtWFusion() && isShiftedZExtW(FirstMI, SecondMI, IsPostRA))
return true;

if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI))
if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI, IsPostRA))
return true;

return false;
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/X86/X86MacroFusion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ static X86::SecondMacroFusionInstKind classifySecond(const MachineInstr &MI) {
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
const MachineInstr &SecondMI,
bool IsPostRA) {
const X86Subtarget &ST = static_cast<const X86Subtarget &>(TSI);

// Check if this processor supports any kind of fusion.
Expand Down
22 changes: 7 additions & 15 deletions llvm/test/TableGen/MacroFusion.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_DECL
// CHECK-PREDICATOR-EMPTY:
// CHECK-PREDICATOR-NEXT: namespace llvm {
// CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &);
// CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &, bool);
// CHECK-PREDICATOR-NEXT: } // end namespace llvm
// CHECK-PREDICATOR-EMPTY:
// CHECK-PREDICATOR-NEXT: #endif
Expand All @@ -56,7 +56,8 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: const TargetInstrInfo &TII,
// CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI,
// CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI,
// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) {
// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI,
// CHECK-PREDICATOR-NEXT: bool IsPostRA) {
// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo();
// CHECK-PREDICATOR-NEXT: {
// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI;
Expand All @@ -73,19 +74,10 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: if (( MI->getOpcode() != Test::Inst0 ))
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: }
// CHECK-PREDICATOR-NEXT: {
// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI;
// CHECK-PREDICATOR-NEXT: if (!(
// CHECK-PREDICATOR-NEXT: MI->getOperand(0).getReg().isVirtual()
// CHECK-PREDICATOR-NEXT: || MI->getOperand(0).getReg() == MI->getOperand(1).getReg()
// CHECK-PREDICATOR-NEXT: ))
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: }
// CHECK-PREDICATOR-NEXT: {
// CHECK-PREDICATOR-NEXT: Register FirstDest = FirstMI->getOperand(0).getReg();
// CHECK-PREDICATOR-NEXT: if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: }
// CHECK-PREDICATOR-NEXT: if (IsPostRA && SecondMI.getOperand(0).getReg() != SecondMI.getOperand(1).getReg())
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: if (!IsPostRA && !MRI.hasOneNonDBGUse(FirstMI->getOperand(0).getReg()))
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: if (!(FirstMI->getOperand(0).isReg() &&
// CHECK-PREDICATOR-NEXT: SecondMI.getOperand(1).isReg() &&
// CHECK-PREDICATOR-NEXT: FirstMI->getOperand(0).getReg() == SecondMI.getOperand(1).getReg()))
Expand Down
Loading