@@ -849,7 +849,7 @@ class IGLPStrategy {
849
849
850
850
public:
851
851
// / Add SchedGroups to \p SyncedSchedGroups to implement this Strategy.
852
- virtual void applyIGLPStrategy (
852
+ virtual bool applyIGLPStrategy (
853
853
DenseMap<int , SUnitsToCandidateSGsMap> &SyncedInstrs,
854
854
DenseMap<int , SmallVector<SchedGroup, 4 >> &SyncedSchedGroups,
855
855
IGLPPhase Phase) = 0;
@@ -868,7 +868,7 @@ class IGLPStrategy {
868
868
class MFMASmallGemmOpt final : public IGLPStrategy {
869
869
private:
870
870
public:
871
- void applyIGLPStrategy (
871
+ bool applyIGLPStrategy (
872
872
DenseMap<int , SUnitsToCandidateSGsMap> &SyncedInstrs,
873
873
DenseMap<int , SmallVector<SchedGroup, 4 >> &SyncedSchedGroups,
874
874
IGLPPhase Phase) override ;
@@ -881,7 +881,7 @@ class MFMASmallGemmOpt final : public IGLPStrategy {
881
881
}
882
882
};
883
883
884
- void MFMASmallGemmOpt::applyIGLPStrategy (
884
+ bool MFMASmallGemmOpt::applyIGLPStrategy (
885
885
DenseMap<int , SUnitsToCandidateSGsMap> &SyncedInstrs,
886
886
DenseMap<int , SmallVector<SchedGroup, 4 >> &SyncedSchedGroups,
887
887
IGLPPhase Phase) {
@@ -902,6 +902,8 @@ void MFMASmallGemmOpt::applyIGLPStrategy(
902
902
SchedGroupMask::MFMA, 1 , PipelineSyncID, DAG, TII);
903
903
SG->initSchedGroup (SyncedInstrs[SG->getSyncID ()]);
904
904
}
905
+
906
+ return true ;
905
907
}
906
908
907
909
class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
@@ -1098,7 +1100,7 @@ class MFMASmallGemmSingleWaveOpt final : public IGLPStrategy {
1098
1100
};
1099
1101
1100
1102
public:
1101
- void applyIGLPStrategy (
1103
+ bool applyIGLPStrategy (
1102
1104
DenseMap<int , SUnitsToCandidateSGsMap> &SyncedInstrs,
1103
1105
DenseMap<int , SmallVector<SchedGroup, 4 >> &SyncedSchedGroups,
1104
1106
IGLPPhase Phase) override ;
@@ -1115,7 +1117,7 @@ static unsigned DSWCount = 0;
1115
1117
static unsigned DSWWithPermCount = 0 ;
1116
1118
static unsigned DSWWithSharedVMEMCount = 0 ;
1117
1119
1118
- void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy (
1120
+ bool MFMASmallGemmSingleWaveOpt::applyIGLPStrategy (
1119
1121
DenseMap<int , SUnitsToCandidateSGsMap> &SyncedInstrs,
1120
1122
DenseMap<int , SmallVector<SchedGroup, 4 >> &SyncedSchedGroups,
1121
1123
IGLPPhase Phase) {
@@ -1355,6 +1357,8 @@ void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy(
1355
1357
SchedGroupMask::MFMA, 1 , PipelineSyncID, DAG, TII);
1356
1358
SG->initSchedGroup (SyncedInstrs[SG->getSyncID ()]);
1357
1359
}
1360
+
1361
+ return true ;
1358
1362
}
1359
1363
1360
1364
static std::unique_ptr<IGLPStrategy>
@@ -1376,6 +1380,8 @@ class IGroupLPDAGMutation : public ScheduleDAGMutation {
1376
1380
1377
1381
ScheduleDAGMI *DAG;
1378
1382
1383
+ std::vector<std::unique_ptr<ScheduleDAGMutation>> *SavedMutations;
1384
+
1379
1385
// Organize lists of SchedGroups by their SyncID. SchedGroups /
1380
1386
// SCHED_GROUP_BARRIERs with different SyncIDs will have no edges added
1381
1387
// between then.
@@ -1402,7 +1408,7 @@ class IGroupLPDAGMutation : public ScheduleDAGMutation {
1402
1408
void initSchedGroupBarrierPipelineStage (
1403
1409
std::vector<SUnit>::reverse_iterator RIter);
1404
1410
1405
- void initIGLPOpt (SUnit &SU);
1411
+ bool initIGLPOpt (SUnit &SU);
1406
1412
1407
1413
public:
1408
1414
void apply (ScheduleDAGInstrs *DAGInstrs) override ;
@@ -1418,7 +1424,10 @@ class IGroupLPDAGMutation : public ScheduleDAGMutation {
1418
1424
IGLPPhase Phase = IGLPPhase::Initial;
1419
1425
1420
1426
IGroupLPDAGMutation () = default ;
1421
- IGroupLPDAGMutation (IGLPPhase Phase) : Phase(Phase) {}
1427
+ IGroupLPDAGMutation (
1428
+ IGLPPhase Phase,
1429
+ std::vector<std::unique_ptr<ScheduleDAGMutation>> *SavedMutations)
1430
+ : SavedMutations(SavedMutations), Phase(Phase) {}
1422
1431
};
1423
1432
1424
1433
unsigned SchedGroup::NumSchedGroups = 0 ;
@@ -1623,8 +1632,7 @@ void IGroupLPDAGMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
1623
1632
} else if (Opc == AMDGPU::IGLP_OPT) {
1624
1633
resetEdges (*R, DAG);
1625
1634
if (!foundSB && !foundIGLP)
1626
- initIGLPOpt (*R);
1627
- foundIGLP = true ;
1635
+ foundIGLP |= initIGLPOpt (*R);
1628
1636
}
1629
1637
}
1630
1638
@@ -1633,7 +1641,16 @@ void IGroupLPDAGMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
1633
1641
// PipelineSolver performs the mutation by adding the edges it
1634
1642
// determined as the best
1635
1643
PS.solve ();
1644
+ return ;
1636
1645
}
1646
+
1647
+ if (!SavedMutations)
1648
+ return ;
1649
+
1650
+ // !foundSB && !foundIGLP -- most likely we have an ILGP_OPT instruciton but
1651
+ // did not apply any mutation
1652
+ for (auto &m : *SavedMutations)
1653
+ m->apply (DAG);
1637
1654
}
1638
1655
1639
1656
void IGroupLPDAGMutation::addSchedBarrierEdges (SUnit &SchedBarrier) {
@@ -1712,14 +1729,15 @@ void IGroupLPDAGMutation::initSchedGroupBarrierPipelineStage(
1712
1729
SG.initSchedGroup (RIter, SyncedInstrs[SG.getSyncID ()]);
1713
1730
}
1714
1731
1715
- void IGroupLPDAGMutation::initIGLPOpt (SUnit &SU) {
1732
+ bool IGroupLPDAGMutation::initIGLPOpt (SUnit &SU) {
1716
1733
IGLPStrategyID StrategyID =
1717
1734
(IGLPStrategyID)SU.getInstr ()->getOperand (0 ).getImm ();
1718
1735
auto S = createIGLPStrategy (StrategyID, DAG, TII);
1719
- if (S->shouldApplyStrategy (DAG)) {
1720
- IsBottomUp = S->IsBottomUp ;
1721
- S->applyIGLPStrategy (SyncedInstrs, SyncedSchedGroups, Phase);
1722
- }
1736
+ if (!S->shouldApplyStrategy (DAG))
1737
+ return false ;
1738
+
1739
+ IsBottomUp = S->IsBottomUp ;
1740
+ return S->applyIGLPStrategy (SyncedInstrs, SyncedSchedGroups, Phase);
1723
1741
}
1724
1742
1725
1743
} // namespace
@@ -1731,9 +1749,10 @@ namespace llvm {
1731
1749
// / same scheduling region (e.g. pre and post-RA scheduling / multiple
1732
1750
// / scheduling "phases"), we can reenter this mutation framework more than once
1733
1751
// / for a given region.
1734
- std::unique_ptr<ScheduleDAGMutation>
1735
- createIGroupLPDAGMutation (IGLPPhase Phase) {
1736
- return std::make_unique<IGroupLPDAGMutation>(Phase);
1752
+ std::unique_ptr<ScheduleDAGMutation> createIGroupLPDAGMutation (
1753
+ IGLPPhase Phase,
1754
+ std::vector<std::unique_ptr<ScheduleDAGMutation>> *SavedMutations) {
1755
+ return std::make_unique<IGroupLPDAGMutation>(Phase, SavedMutations);
1737
1756
}
1738
1757
1739
1758
} // end namespace llvm
0 commit comments