@@ -125,36 +125,72 @@ SchedBundle *Scheduler::createBundle(ArrayRef<Instruction *> Instrs) {
125125void Scheduler::eraseBundle (SchedBundle *SB) { Bndls.erase (SB); }
126126
127127bool Scheduler::tryScheduleUntil (ArrayRef<Instruction *> Instrs) {
128- // Use a set of instructions, instead of `Instrs` for fast lookups.
129- DenseSet<Instruction *> InstrsToDefer (Instrs.begin (), Instrs.end ());
130- // This collects the nodes that correspond to instructions found in `Instrs`
131- // that have just become ready. These nodes won't be scheduled right away.
132- SmallVector<DGNode *, 8 > DeferredNodes;
133-
128+ // Create a bundle for Instrs. If it turns out the schedule is infeasible we
129+ // will dismantle it.
130+ auto *InstrsSB = createBundle (Instrs);
134131 // Keep scheduling ready nodes until we either run out of ready nodes (i.e.,
135132 // ReadyList is empty), or all nodes that correspond to `Instrs` (the nodes of
136133 // which are collected in DeferredNodes) are all ready to schedule.
137- while (!ReadyList.empty ()) {
138- auto *ReadyN = ReadyList.pop ();
139- if (InstrsToDefer.contains (ReadyN->getInstruction ())) {
140- // If the ready instruction is one of those in `Instrs`, then we don't
141- // schedule it right away. Instead we defer it until we can schedule it
142- // along with the rest of the instructions in `Instrs`, at the same
143- // time in a single scheduling bundle.
144- DeferredNodes.push_back (ReadyN);
145- bool ReadyToScheduleDeferred = DeferredNodes.size () == Instrs.size ();
146- if (ReadyToScheduleDeferred) {
147- scheduleAndUpdateReadyList (*createBundle (Instrs));
134+ SmallVector<DGNode *> Retry;
135+ bool KeepScheduling = true ;
136+ while (KeepScheduling) {
137+ enum class TryScheduleRes {
138+ Success, // /> We successfully scheduled the node.
139+ Failure, // /> We failed to schedule the node.
140+ Done, // /> We scheduled the `Instrs` bundle.
141+ };
142+ auto TrySchedule = [this , InstrsSB](DGNode *ReadyN) -> TryScheduleRes {
143+ auto *SB = ReadyN->getSchedBundle ();
144+ if (SB == nullptr ) {
145+ // If ReadyN does not belong to a bundle, create a singleton bundle
146+ // and schedule it.
147+ auto *SingletonSB = createBundle ({ReadyN->getInstruction ()});
148+ scheduleAndUpdateReadyList (*SingletonSB);
149+ return TryScheduleRes::Success;
150+ }
151+ if (SB->ready ()) {
152+ // Remove the rest of the bundle from the ready list.
153+ // TODO: Perhaps change the Scheduler + ReadyList to operate on
154+ // SchedBundles instead of DGNodes.
155+ for (auto *N : *SB) {
156+ if (N != ReadyN)
157+ ReadyList.remove (N);
158+ }
159+ // If all nodes in the bundle are ready.
160+ scheduleAndUpdateReadyList (*SB);
161+ if (SB == InstrsSB)
162+ // If this is bundle corresponding to `Instrs` we are done.
163+ return TryScheduleRes::Done;
164+ return TryScheduleRes::Success;
165+ }
166+ return TryScheduleRes::Failure;
167+ };
168+ while (!ReadyList.empty ()) {
169+ auto *ReadyN = ReadyList.pop ();
170+ auto Res = TrySchedule (ReadyN);
171+ switch (Res) {
172+ case TryScheduleRes::Success:
173+ continue ;
174+ case TryScheduleRes::Failure:
175+ Retry.push_back (ReadyN);
176+ continue ;
177+ case TryScheduleRes::Done:
148178 return true ;
149179 }
150- } else {
151- // If the ready instruction is not found in `Instrs`, then we wrap it in a
152- // scheduling bundle and schedule it right away.
153- scheduleAndUpdateReadyList (*createBundle ({ReadyN->getInstruction ()}));
180+ llvm_unreachable (" Unhandled TrySchedule() result"
181+ }
182+ // Try to schedule nodes from the Retry list.
183+ KeepScheduling = false ;
184+ for (auto *N : make_early_inc_range (Retry)) {
185+ auto Res = TrySchedule (N);
186+ if (Res == TryScheduleRes::Success) {
187+ Retry.erase (find (Retry, N));
188+ KeepScheduling = true ;
189+ }
154190 }
155191 }
156- assert (DeferredNodes. size () != Instrs. size () &&
157- " We should have succesfully scheduled and early-returned! "
192+
193+ eraseBundle (InstrsSB );
158194 return false ;
159195}
160196
@@ -275,6 +311,7 @@ bool Scheduler::trySchedule(ArrayRef<Instruction *> Instrs) {
275311 // If one or more instrs are already scheduled we need to destroy the
276312 // top-most part of the schedule that includes the instrs in the bundle and
277313 // re-schedule.
314+ DAG.extend (Instrs);
278315 trimSchedule (Instrs);
279316 ScheduleTopItOpt = std::next (VecUtils::getLowest (Instrs)->getIterator ());
280317 return tryScheduleUntil (Instrs);
0 commit comments