/// Sort the blocks, taking special care to make sure that loops are not /// interrupted by blocks not dominated by their header. /// TODO: There are many opportunities for improving the heuristics here. /// Explore them. static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI, const MachineDominatorTree &MDT) { // Prepare for a topological sort: Record the number of predecessors each // block has, ignoring loop backedges. MF.RenumberBlocks(); SmallVector<unsigned, 16> NumPredsLeft(MF.getNumBlockIDs(), 0); for (MachineBasicBlock &MBB : MF) { unsigned N = MBB.pred_size(); if (MachineLoop *L = MLI.getLoopFor(&MBB)) if (L->getHeader() == &MBB) for (const MachineBasicBlock *Pred : MBB.predecessors()) if (L->contains(Pred)) --N; NumPredsLeft[MBB.getNumber()] = N; } // Topological sort the CFG, with additional constraints: // - Between a loop header and the last block in the loop, there can be // no blocks not dominated by the loop header. // - It's desirable to preserve the original block order when possible. // We use two ready lists; Preferred and Ready. Preferred has recently // processed sucessors, to help preserve block sequences from the original // order. Ready has the remaining ready blocks. PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>, CompareBlockNumbers> Preferred; PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>, CompareBlockNumbersBackwards> Ready; SmallVector<Entry, 4> Loops; for (MachineBasicBlock *MBB = &MF.front();;) { const MachineLoop *L = MLI.getLoopFor(MBB); if (L) { // If MBB is a loop header, add it to the active loop list. We can't put // any blocks that it doesn't dominate until we see the end of the loop. if (L->getHeader() == MBB) Loops.push_back(Entry(L)); // For each active loop the block is in, decrement the count. If MBB is // the last block in an active loop, take it off the list and pick up any // blocks deferred because the header didn't dominate them. for (Entry &E : Loops) if (E.Loop->contains(MBB) && --E.NumBlocksLeft == 0) for (auto DeferredBlock : E.Deferred) Ready.push(DeferredBlock); while (!Loops.empty() && Loops.back().NumBlocksLeft == 0) Loops.pop_back(); } // The main topological sort logic. for (MachineBasicBlock *Succ : MBB->successors()) { // Ignore backedges. if (MachineLoop *SuccL = MLI.getLoopFor(Succ)) if (SuccL->getHeader() == Succ && SuccL->contains(MBB)) continue; // Decrement the predecessor count. If it's now zero, it's ready. if (--NumPredsLeft[Succ->getNumber()] == 0) Preferred.push(Succ); } // Determine the block to follow MBB. First try to find a preferred block, // to preserve the original block order when possible. MachineBasicBlock *Next = nullptr; while (!Preferred.empty()) { Next = Preferred.top(); Preferred.pop(); // If X isn't dominated by the top active loop header, defer it until that // loop is done. if (!Loops.empty() && !MDT.dominates(Loops.back().Loop->getHeader(), Next)) { Loops.back().Deferred.push_back(Next); Next = nullptr; continue; } // If Next was originally ordered before MBB, and it isn't because it was // loop-rotated above the header, it's not preferred. if (Next->getNumber() < MBB->getNumber() && (!L || !L->contains(Next) || L->getHeader()->getNumber() < Next->getNumber())) { Ready.push(Next); Next = nullptr; continue; } break; } // If we didn't find a suitable block in the Preferred list, check the // general Ready list. if (!Next) { // If there are no more blocks to process, we're done. if (Ready.empty()) { MaybeUpdateTerminator(MBB); break; } for (;;) { Next = Ready.top(); Ready.pop(); // If Next isn't dominated by the top active loop header, defer it until // that loop is done. if (!Loops.empty() && !MDT.dominates(Loops.back().Loop->getHeader(), Next)) { Loops.back().Deferred.push_back(Next); continue; } break; } } // Move the next block into place and iterate. Next->moveAfter(MBB); MaybeUpdateTerminator(MBB); MBB = Next; } assert(Loops.empty() && "Active loop list not finished"); MF.RenumberBlocks(); #ifndef NDEBUG SmallSetVector<MachineLoop *, 8> OnStack; // Insert a sentinel representing the degenerate loop that starts at the // function entry block and includes the entire function as a "loop" that // executes once. OnStack.insert(nullptr); for (auto &MBB : MF) { assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative."); MachineLoop *Loop = MLI.getLoopFor(&MBB); if (Loop && &MBB == Loop->getHeader()) { // Loop header. The loop predecessor should be sorted above, and the other // predecessors should be backedges below. for (auto Pred : MBB.predecessors()) assert( (Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) && "Loop header predecessors must be loop predecessors or backedges"); assert(OnStack.insert(Loop) && "Loops should be declared at most once."); } else { // Not a loop header. All predecessors should be sorted above. for (auto Pred : MBB.predecessors()) assert(Pred->getNumber() < MBB.getNumber() && "Non-loop-header predecessors should be topologically sorted"); assert(OnStack.count(MLI.getLoopFor(&MBB)) && "Blocks must be nested in their loops"); } while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back())) OnStack.pop_back(); } assert(OnStack.pop_back_val() == nullptr && "The function entry block shouldn't actually be a loop header"); assert(OnStack.empty() && "Control flow stack pushes and pops should be balanced."); #endif }
/// Sort the blocks in RPO, taking special care to make sure that loops are /// contiguous even in the case of split backedges. /// /// TODO: Determine whether RPO is actually worthwhile, or whether we should /// move to just a stable-topological-sort-based approach that would preserve /// more of the original order. static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) { // Note that we do our own RPO rather than using // "llvm/ADT/PostOrderIterator.h" because we want control over the order that // successors are visited in (see above). Also, we can sort the blocks in the // MachineFunction as we go. SmallPtrSet<MachineBasicBlock *, 16> Visited; SmallVector<POStackEntry, 16> Stack; MachineBasicBlock *EntryBlock = &*MF.begin(); Visited.insert(EntryBlock); Stack.push_back(POStackEntry(EntryBlock, MF, MLI)); for (;;) { POStackEntry &Entry = Stack.back(); SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs; if (!Succs.empty()) { MachineBasicBlock *Succ = Succs.pop_back_val(); if (Visited.insert(Succ).second) Stack.push_back(POStackEntry(Succ, MF, MLI)); continue; } // Put the block in its position in the MachineFunction. MachineBasicBlock &MBB = *Entry.MBB; MBB.moveBefore(&*MF.begin()); // Branch instructions may utilize a fallthrough, so update them if a // fallthrough has been added or removed. if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() && !MBB.back().isBarrier()) report_fatal_error( "Non-branch terminator with fallthrough cannot yet be rewritten"); if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch()) MBB.updateTerminator(); Stack.pop_back(); if (Stack.empty()) break; } // Now that we've sorted the blocks in RPO, renumber them. MF.RenumberBlocks(); #ifndef NDEBUG SmallSetVector<MachineLoop *, 8> OnStack; // Insert a sentinel representing the degenerate loop that starts at the // function entry block and includes the entire function as a "loop" that // executes once. OnStack.insert(nullptr); for (auto &MBB : MF) { assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative."); MachineLoop *Loop = MLI.getLoopFor(&MBB); if (Loop && &MBB == Loop->getHeader()) { // Loop header. The loop predecessor should be sorted above, and the other // predecessors should be backedges below. for (auto Pred : MBB.predecessors()) assert( (Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) && "Loop header predecessors must be loop predecessors or backedges"); assert(OnStack.insert(Loop) && "Loops should be declared at most once."); } else { // Not a loop header. All predecessors should be sorted above. for (auto Pred : MBB.predecessors()) assert(Pred->getNumber() < MBB.getNumber() && "Non-loop-header predecessors should be topologically sorted"); assert(OnStack.count(MLI.getLoopFor(&MBB)) && "Blocks must be nested in their loops"); } while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back())) OnStack.pop_back(); } assert(OnStack.pop_back_val() == nullptr && "The function entry block shouldn't actually be a loop header"); assert(OnStack.empty() && "Control flow stack pushes and pops should be balanced."); #endif }