/// Sort the blocks in RPO, taking special care to make sure that loops are /// contiguous even in the case of split backedges. static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) { // Note that we do our own RPO rather than using // "llvm/ADT/PostOrderIterator.h" because we want control over the order that // successors are visited in (see above). Also, we can sort the blocks in the // MachineFunction as we go. SmallPtrSet<MachineBasicBlock *, 16> Visited; SmallVector<POStackEntry, 16> Stack; MachineBasicBlock *Entry = &*MF.begin(); Visited.insert(Entry); Stack.push_back(POStackEntry(Entry, MF, MLI)); for (;;) { POStackEntry &Entry = Stack.back(); SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs; if (!Succs.empty()) { MachineBasicBlock *Succ = Succs.pop_back_val(); if (Visited.insert(Succ).second) Stack.push_back(POStackEntry(Succ, MF, MLI)); continue; } // Put the block in its position in the MachineFunction. MachineBasicBlock &MBB = *Entry.MBB; MBB.moveBefore(&*MF.begin()); // Branch instructions may utilize a fallthrough, so update them if a // fallthrough has been added or removed. if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() && !MBB.back().isBarrier()) report_fatal_error( "Non-branch terminator with fallthrough cannot yet be rewritten"); if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch()) MBB.updateTerminator(); Stack.pop_back(); if (Stack.empty()) break; } // Now that we've sorted the blocks in RPO, renumber them. MF.RenumberBlocks(); #ifndef NDEBUG for (auto &MBB : MF) if (MachineLoop *Loop = MLI.getLoopFor(&MBB)) { // Assert that loops are contiguous. assert(Loop->getHeader() == Loop->getTopBlock()); assert((Loop->getHeader() == &MBB || Loop->contains( MLI.getLoopFor(&*prev(MachineFunction::iterator(&MBB))))) && "Loop isn't contiguous"); } else { // Assert that non-loops have no backedge predecessors. for (auto Pred : MBB.predecessors()) assert(Pred->getNumber() < MBB.getNumber() && "CFG still has multiple-entry loops"); } #endif }
static void EliminateMultipleEntryLoops(MachineFunction &MF, const MachineLoopInfo &MLI) { SmallPtrSet<MachineBasicBlock *, 8> InSet; for (scc_iterator<MachineFunction *> I = scc_begin(&MF), E = scc_end(&MF); I != E; ++I) { const std::vector<MachineBasicBlock *> &CurrentSCC = *I; // Skip trivial SCCs. if (CurrentSCC.size() == 1) continue; InSet.insert(CurrentSCC.begin(), CurrentSCC.end()); MachineBasicBlock *Header = nullptr; for (MachineBasicBlock *MBB : CurrentSCC) { for (MachineBasicBlock *Pred : MBB->predecessors()) { if (InSet.count(Pred)) continue; if (!Header) { Header = MBB; break; } // TODO: Implement multiple-entry loops. report_fatal_error("multiple-entry loops are not supported yet"); } } assert(MLI.isLoopHeader(Header)); InSet.clear(); } }
/// Insert LOOP and BLOCK markers at appropriate places. static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI, const WebAssemblyInstrInfo &TII) { for (auto &MBB : MF) { // Place the LOOP for loops. if (MachineLoop *Loop = MLI.getLoopFor(&MBB)) if (Loop->getHeader() == &MBB) { // The operand of a LOOP is the first block after the loop. If the loop // is the bottom of the function, insert a dummy block at the end. MachineBasicBlock *Bottom = Loop->getBottomBlock(); auto Iter = next(MachineFunction::iterator(Bottom)); if (Iter == MF.end()) { MF.push_back(MF.CreateMachineBasicBlock()); Iter = next(MachineFunction::iterator(Bottom)); } BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP)) .addMBB(&*Iter); } // Check for forward branches and switches that need BLOCKS placed. for (auto &Term : MBB.terminators()) for (auto &MO : Term.operands()) if (MO.isMBB()) PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII); } }
/// Insert BLOCK markers at appropriate places. static void PlaceBlockMarkers(MachineBasicBlock &MBB, MachineBasicBlock &Succ, MachineFunction &MF, const MachineLoopInfo &MLI, const WebAssemblyInstrInfo &TII) { // Backward branches are loop backedges, and we place the LOOP markers // separately. So only consider forward branches here. if (Succ.getNumber() <= MBB.getNumber()) return; // Place the BLOCK for a forward branch. For simplicity, we just insert // blocks immediately inside loop boundaries. MachineLoop *Loop = MLI.getLoopFor(&Succ); MachineBasicBlock &Header = *(Loop ? Loop->getHeader() : &MF.front()); MachineBasicBlock::iterator InsertPos = Header.begin(), End = Header.end(); if (InsertPos != End) { if (InsertPos->getOpcode() == WebAssembly::LOOP) ++InsertPos; int SuccNumber = Succ.getNumber(); // Position the BLOCK in nesting order. for (; InsertPos != End && InsertPos->getOpcode() == WebAssembly::BLOCK; ++InsertPos) { int N = InsertPos->getOperand(0).getMBB()->getNumber(); if (N < SuccNumber) break; // If there's already a BLOCK for Succ, we don't need another. if (N == SuccNumber) return; } } BuildMI(Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK)) .addMBB(&Succ); }
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header). static void PlaceLoopMarker(MachineBasicBlock &MBB, MachineFunction &MF, SmallVectorImpl<MachineBasicBlock *> &ScopeTops, const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) { MachineLoop *Loop = MLI.getLoopFor(&MBB); if (!Loop || Loop->getHeader() != &MBB) return; // The operand of a LOOP is the first block after the loop. If the loop is the // bottom of the function, insert a dummy block at the end. MachineBasicBlock *Bottom = LoopBottom(Loop); auto Iter = next(MachineFunction::iterator(Bottom)); if (Iter == MF.end()) { MachineBasicBlock *Label = MF.CreateMachineBasicBlock(); // Give it a fake predecessor so that AsmPrinter prints its label. Label->addSuccessor(Label); MF.push_back(Label); Iter = next(MachineFunction::iterator(Bottom)); } MachineBasicBlock *AfterLoop = &*Iter; BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP)) .addMBB(AfterLoop); // Emit a special no-op telling the asm printer that we need a label to close // the loop scope, even though the destination is only reachable by // fallthrough. if (!Bottom->back().isBarrier()) BuildMI(*Bottom, Bottom->end(), DebugLoc(), TII.get(WebAssembly::LOOP_END)); assert((!ScopeTops[AfterLoop->getNumber()] || ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) && "With RPO we should visit the outer-most loop for a block first."); if (!ScopeTops[AfterLoop->getNumber()]) ScopeTops[AfterLoop->getNumber()] = &MBB; }
/// Check whether the edge (\p SrcBB, \p DestBB) is a backedge according to MLI. /// I.e., check if it exists a loop that contains SrcBB and where DestBB is the /// loop header. static bool isProperBackedge(const MachineLoopInfo &MLI, const MachineBasicBlock *SrcBB, const MachineBasicBlock *DestBB) { for (const MachineLoop *Loop = MLI.getLoopFor(SrcBB); Loop; Loop = Loop->getParentLoop()) { if (Loop->getHeader() == DestBB) return true; } return false; }
POStackEntry::POStackEntry(MachineBasicBlock *MBB, MachineFunction &MF, const MachineLoopInfo &MLI) : MBB(MBB), Succs(MBB->successors()) { // RPO is not a unique form, since at every basic block with multiple // successors, the DFS has to pick which order to visit the successors in. // Sort them strategically (see below). MachineLoop *Loop = MLI.getLoopFor(MBB); MachineFunction::iterator Next = next(MachineFunction::iterator(MBB)); MachineBasicBlock *LayoutSucc = Next == MF.end() ? nullptr : &*Next; std::stable_sort( Succs.begin(), Succs.end(), [=, &MLI](const MachineBasicBlock *A, const MachineBasicBlock *B) { if (A == B) return false; // Keep loops contiguous by preferring the block that's in the same // loop. MachineLoop *LoopA = MLI.getLoopFor(A); MachineLoop *LoopB = MLI.getLoopFor(B); if (LoopA == Loop && LoopB != Loop) return true; if (LoopA != Loop && LoopB == Loop) return false; // Minimize perturbation by preferring the block which is the immediate // layout successor. if (A == LayoutSucc) return true; if (B == LayoutSucc) return false; // TODO: More sophisticated orderings may be profitable here. return false; }); }
/// Insert LOOP and BLOCK markers at appropriate places. static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI, const WebAssemblyInstrInfo &TII) { for (auto &MBB : MF) { // Place the LOOP for loops. if (MachineLoop *Loop = MLI.getLoopFor(&MBB)) if (Loop->getHeader() == &MBB) BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP)) .addMBB(Loop->getBottomBlock()); // Check for forward branches and switches that need BLOCKS placed. for (auto &Term : MBB.terminators()) for (auto &MO : Term.operands()) if (MO.isMBB()) PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII); } }
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header). static void PlaceLoopMarker( MachineBasicBlock &MBB, MachineFunction &MF, SmallVectorImpl<MachineBasicBlock *> &ScopeTops, DenseMap<const MachineInstr *, const MachineBasicBlock *> &LoopTops, const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) { MachineLoop *Loop = MLI.getLoopFor(&MBB); if (!Loop || Loop->getHeader() != &MBB) return; // The operand of a LOOP is the first block after the loop. If the loop is the // bottom of the function, insert a dummy block at the end. MachineBasicBlock *Bottom = LoopBottom(Loop); auto Iter = next(MachineFunction::iterator(Bottom)); if (Iter == MF.end()) { MachineBasicBlock *Label = MF.CreateMachineBasicBlock(); // Give it a fake predecessor so that AsmPrinter prints its label. Label->addSuccessor(Label); MF.push_back(Label); Iter = next(MachineFunction::iterator(Bottom)); } MachineBasicBlock *AfterLoop = &*Iter; // Mark the beginning of the loop (after the end of any existing loop that // ends here). auto InsertPos = MBB.begin(); while (InsertPos != MBB.end() && InsertPos->getOpcode() == WebAssembly::END_LOOP) ++InsertPos; BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::LOOP)); // Mark the end of the loop. MachineInstr *End = BuildMI(*AfterLoop, AfterLoop->begin(), DebugLoc(), TII.get(WebAssembly::END_LOOP)); LoopTops[End] = &MBB; assert((!ScopeTops[AfterLoop->getNumber()] || ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) && "With block sorting the outermost loop for a block should be first."); if (!ScopeTops[AfterLoop->getNumber()]) ScopeTops[AfterLoop->getNumber()] = &MBB; }
bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) { DEBUG(dbgs() << "********** Compute Spill Weights **********\n" << "********** Function: " << fn.getFunction()->getName() << '\n'); LiveIntervals *lis = &getAnalysis<LiveIntervals>(); MachineLoopInfo *loopInfo = &getAnalysis<MachineLoopInfo>(); const TargetInstrInfo *tii = fn.getTarget().getInstrInfo(); MachineRegisterInfo *mri = &fn.getRegInfo(); SmallSet<unsigned, 4> processed; for (MachineFunction::iterator mbbi = fn.begin(), mbbe = fn.end(); mbbi != mbbe; ++mbbi) { MachineBasicBlock* mbb = mbbi; SlotIndex mbbEnd = lis->getMBBEndIdx(mbb); MachineLoop* loop = loopInfo->getLoopFor(mbb); unsigned loopDepth = loop ? loop->getLoopDepth() : 0; bool isExiting = loop ? loop->isLoopExiting(mbb) : false; for (MachineBasicBlock::const_iterator mii = mbb->begin(), mie = mbb->end(); mii != mie; ++mii) { const MachineInstr *mi = mii; if (tii->isIdentityCopy(*mi) || mi->isImplicitDef() || mi->isDebugValue()) continue; for (unsigned i = 0, e = mi->getNumOperands(); i != e; ++i) { const MachineOperand &mopi = mi->getOperand(i); if (!mopi.isReg() || mopi.getReg() == 0) continue; unsigned reg = mopi.getReg(); if (!TargetRegisterInfo::isVirtualRegister(mopi.getReg())) continue; // Multiple uses of reg by the same instruction. It should not // contribute to spill weight again. if (!processed.insert(reg)) continue; bool hasDef = mopi.isDef(); bool hasUse = !hasDef; for (unsigned j = i+1; j != e; ++j) { const MachineOperand &mopj = mi->getOperand(j); if (!mopj.isReg() || mopj.getReg() != reg) continue; hasDef |= mopj.isDef(); hasUse |= mopj.isUse(); if (hasDef && hasUse) break; } LiveInterval ®Int = lis->getInterval(reg); float weight = lis->getSpillWeight(hasDef, hasUse, loopDepth); if (hasDef && isExiting) { // Looks like this is a loop count variable update. SlotIndex defIdx = lis->getInstructionIndex(mi).getDefIndex(); const LiveRange *dlr = lis->getInterval(reg).getLiveRangeContaining(defIdx); if (dlr->end >= mbbEnd) weight *= 3.0F; } regInt.weight += weight; } processed.clear(); } } for (LiveIntervals::iterator I = lis->begin(), E = lis->end(); I != E; ++I) { LiveInterval &li = *I->second; if (TargetRegisterInfo::isVirtualRegister(li.reg)) { // If the live interval length is essentially zero, i.e. in every live // range the use follows def immediately, it doesn't make sense to spill // it and hope it will be easier to allocate for this li. if (isZeroLengthInterval(&li)) { li.weight = HUGE_VALF; continue; } bool isLoad = false; SmallVector<LiveInterval*, 4> spillIs; if (lis->isReMaterializable(li, spillIs, isLoad)) { // If all of the definitions of the interval are re-materializable, // it is a preferred candidate for spilling. If non of the defs are // loads, then it's potentially very cheap to re-materialize. // FIXME: this gets much more complicated once we support non-trivial // re-materialization. if (isLoad) li.weight *= 0.9F; else li.weight *= 0.5F; } // Slightly prefer live interval that has been assigned a preferred reg. std::pair<unsigned, unsigned> Hint = mri->getRegAllocationHint(li.reg); if (Hint.first || Hint.second) li.weight *= 1.01F; lis->normalizeSpillWeight(li); } } return false; }
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) { df_iterator_default_set<MachineBasicBlock*> Reachable; bool ModifiedPHI = false; MMI = getAnalysisIfAvailable<MachineModuleInfo>(); MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>(); MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); // Mark all reachable blocks. for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable)) (void)BB/* Mark all reachable blocks */; // Loop over all dead blocks, remembering them and deleting all instructions // in them. std::vector<MachineBasicBlock*> DeadBlocks; for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = &*I; // Test for deadness. if (!Reachable.count(BB)) { DeadBlocks.push_back(BB); // Update dominator and loop info. if (MLI) MLI->removeBlock(BB); if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB); while (BB->succ_begin() != BB->succ_end()) { MachineBasicBlock* succ = *BB->succ_begin(); MachineBasicBlock::iterator start = succ->begin(); while (start != succ->end() && start->isPHI()) { for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2) if (start->getOperand(i).isMBB() && start->getOperand(i).getMBB() == BB) { start->RemoveOperand(i); start->RemoveOperand(i-1); } start++; } BB->removeSuccessor(BB->succ_begin()); } } } // Actually remove the blocks now. for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) DeadBlocks[i]->eraseFromParent(); // Cleanup PHI nodes. for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = &*I; // Prune unneeded PHI entries. SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(), BB->pred_end()); MachineBasicBlock::iterator phi = BB->begin(); while (phi != BB->end() && phi->isPHI()) { for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2) if (!preds.count(phi->getOperand(i).getMBB())) { phi->RemoveOperand(i); phi->RemoveOperand(i-1); ModifiedPHI = true; } if (phi->getNumOperands() == 3) { const MachineOperand &Input = phi->getOperand(1); const MachineOperand &Output = phi->getOperand(0); unsigned InputReg = Input.getReg(); unsigned OutputReg = Output.getReg(); assert(Output.getSubReg() == 0 && "Cannot have output subregister"); ModifiedPHI = true; if (InputReg != OutputReg) { MachineRegisterInfo &MRI = F.getRegInfo(); unsigned InputSub = Input.getSubReg(); if (InputSub == 0 && MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) { MRI.replaceRegWith(OutputReg, InputReg); } else { // The input register to the PHI has a subregister or it can't be // constrained to the proper register class: // insert a COPY instead of simply replacing the output // with the input. const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo(); BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(), TII->get(TargetOpcode::COPY), OutputReg) .addReg(InputReg, getRegState(Input), InputSub); } phi++->eraseFromParent(); } continue; } ++phi; } } F.RenumberBlocks(); return (!DeadBlocks.empty() || ModifiedPHI); }
/// Insert a BLOCK marker for branches to MBB (if needed). static void PlaceBlockMarker(MachineBasicBlock &MBB, MachineFunction &MF, SmallVectorImpl<MachineBasicBlock *> &ScopeTops, const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI, MachineDominatorTree &MDT, WebAssemblyFunctionInfo &MFI) { // First compute the nearest common dominator of all forward non-fallthrough // predecessors so that we minimize the time that the BLOCK is on the stack, // which reduces overall stack height. MachineBasicBlock *Header = nullptr; bool IsBranchedTo = false; int MBBNumber = MBB.getNumber(); for (MachineBasicBlock *Pred : MBB.predecessors()) if (Pred->getNumber() < MBBNumber) { Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred; if (ExplicitlyBranchesTo(Pred, &MBB)) IsBranchedTo = true; } if (!Header) return; if (!IsBranchedTo) return; assert(&MBB != &MF.front() && "Header blocks shouldn't have predecessors"); MachineBasicBlock *LayoutPred = &*prev(MachineFunction::iterator(&MBB)); // If the nearest common dominator is inside a more deeply nested context, // walk out to the nearest scope which isn't more deeply nested. for (MachineFunction::iterator I(LayoutPred), E(Header); I != E; --I) { if (MachineBasicBlock *ScopeTop = ScopeTops[I->getNumber()]) { if (ScopeTop->getNumber() > Header->getNumber()) { // Skip over an intervening scope. I = next(MachineFunction::iterator(ScopeTop)); } else { // We found a scope level at an appropriate depth. Header = ScopeTop; break; } } } // If there's a loop which ends just before MBB which contains Header, we can // reuse its label instead of inserting a new BLOCK. for (MachineLoop *Loop = MLI.getLoopFor(LayoutPred); Loop && Loop->contains(LayoutPred); Loop = Loop->getParentLoop()) if (Loop && LoopBottom(Loop) == LayoutPred && Loop->contains(Header)) return; // Decide where in Header to put the BLOCK. MachineBasicBlock::iterator InsertPos; MachineLoop *HeaderLoop = MLI.getLoopFor(Header); if (HeaderLoop && MBB.getNumber() > LoopBottom(HeaderLoop)->getNumber()) { // Header is the header of a loop that does not lexically contain MBB, so // the BLOCK needs to be above the LOOP, after any END constructs. InsertPos = Header->begin(); while (InsertPos->getOpcode() != WebAssembly::LOOP) ++InsertPos; } else { // Otherwise, insert the BLOCK as late in Header as we can, but before the // beginning of the local expression tree and any nested BLOCKs. InsertPos = Header->getFirstTerminator(); while (InsertPos != Header->begin() && IsChild(prev(InsertPos), MFI) && prev(InsertPos)->getOpcode() != WebAssembly::LOOP && prev(InsertPos)->getOpcode() != WebAssembly::END_BLOCK && prev(InsertPos)->getOpcode() != WebAssembly::END_LOOP) --InsertPos; } // Add the BLOCK. BuildMI(*Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK)); // Mark the end of the block. InsertPos = MBB.begin(); while (InsertPos != MBB.end() && InsertPos->getOpcode() == WebAssembly::END_LOOP) ++InsertPos; BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::END_BLOCK)); // Track the farthest-spanning scope that ends at this point. int Number = MBB.getNumber(); if (!ScopeTops[Number] || ScopeTops[Number]->getNumber() > Header->getNumber()) ScopeTops[Number] = Header; }
/// Sort the blocks, taking special care to make sure that loops are not /// interrupted by blocks not dominated by their header. /// TODO: There are many opportunities for improving the heuristics here. /// Explore them. static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI, const MachineDominatorTree &MDT) { // Prepare for a topological sort: Record the number of predecessors each // block has, ignoring loop backedges. MF.RenumberBlocks(); SmallVector<unsigned, 16> NumPredsLeft(MF.getNumBlockIDs(), 0); for (MachineBasicBlock &MBB : MF) { unsigned N = MBB.pred_size(); if (MachineLoop *L = MLI.getLoopFor(&MBB)) if (L->getHeader() == &MBB) for (const MachineBasicBlock *Pred : MBB.predecessors()) if (L->contains(Pred)) --N; NumPredsLeft[MBB.getNumber()] = N; } // Topological sort the CFG, with additional constraints: // - Between a loop header and the last block in the loop, there can be // no blocks not dominated by the loop header. // - It's desirable to preserve the original block order when possible. // We use two ready lists; Preferred and Ready. Preferred has recently // processed sucessors, to help preserve block sequences from the original // order. Ready has the remaining ready blocks. PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>, CompareBlockNumbers> Preferred; PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>, CompareBlockNumbersBackwards> Ready; SmallVector<Entry, 4> Loops; for (MachineBasicBlock *MBB = &MF.front();;) { const MachineLoop *L = MLI.getLoopFor(MBB); if (L) { // If MBB is a loop header, add it to the active loop list. We can't put // any blocks that it doesn't dominate until we see the end of the loop. if (L->getHeader() == MBB) Loops.push_back(Entry(L)); // For each active loop the block is in, decrement the count. If MBB is // the last block in an active loop, take it off the list and pick up any // blocks deferred because the header didn't dominate them. for (Entry &E : Loops) if (E.Loop->contains(MBB) && --E.NumBlocksLeft == 0) for (auto DeferredBlock : E.Deferred) Ready.push(DeferredBlock); while (!Loops.empty() && Loops.back().NumBlocksLeft == 0) Loops.pop_back(); } // The main topological sort logic. for (MachineBasicBlock *Succ : MBB->successors()) { // Ignore backedges. if (MachineLoop *SuccL = MLI.getLoopFor(Succ)) if (SuccL->getHeader() == Succ && SuccL->contains(MBB)) continue; // Decrement the predecessor count. If it's now zero, it's ready. if (--NumPredsLeft[Succ->getNumber()] == 0) Preferred.push(Succ); } // Determine the block to follow MBB. First try to find a preferred block, // to preserve the original block order when possible. MachineBasicBlock *Next = nullptr; while (!Preferred.empty()) { Next = Preferred.top(); Preferred.pop(); // If X isn't dominated by the top active loop header, defer it until that // loop is done. if (!Loops.empty() && !MDT.dominates(Loops.back().Loop->getHeader(), Next)) { Loops.back().Deferred.push_back(Next); Next = nullptr; continue; } // If Next was originally ordered before MBB, and it isn't because it was // loop-rotated above the header, it's not preferred. if (Next->getNumber() < MBB->getNumber() && (!L || !L->contains(Next) || L->getHeader()->getNumber() < Next->getNumber())) { Ready.push(Next); Next = nullptr; continue; } break; } // If we didn't find a suitable block in the Preferred list, check the // general Ready list. if (!Next) { // If there are no more blocks to process, we're done. if (Ready.empty()) { MaybeUpdateTerminator(MBB); break; } for (;;) { Next = Ready.top(); Ready.pop(); // If Next isn't dominated by the top active loop header, defer it until // that loop is done. if (!Loops.empty() && !MDT.dominates(Loops.back().Loop->getHeader(), Next)) { Loops.back().Deferred.push_back(Next); continue; } break; } } // Move the next block into place and iterate. Next->moveAfter(MBB); MaybeUpdateTerminator(MBB); MBB = Next; } assert(Loops.empty() && "Active loop list not finished"); MF.RenumberBlocks(); #ifndef NDEBUG SmallSetVector<MachineLoop *, 8> OnStack; // Insert a sentinel representing the degenerate loop that starts at the // function entry block and includes the entire function as a "loop" that // executes once. OnStack.insert(nullptr); for (auto &MBB : MF) { assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative."); MachineLoop *Loop = MLI.getLoopFor(&MBB); if (Loop && &MBB == Loop->getHeader()) { // Loop header. The loop predecessor should be sorted above, and the other // predecessors should be backedges below. for (auto Pred : MBB.predecessors()) assert( (Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) && "Loop header predecessors must be loop predecessors or backedges"); assert(OnStack.insert(Loop) && "Loops should be declared at most once."); } else { // Not a loop header. All predecessors should be sorted above. for (auto Pred : MBB.predecessors()) assert(Pred->getNumber() < MBB.getNumber() && "Non-loop-header predecessors should be topologically sorted"); assert(OnStack.count(MLI.getLoopFor(&MBB)) && "Blocks must be nested in their loops"); } while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back())) OnStack.pop_back(); } assert(OnStack.pop_back_val() == nullptr && "The function entry block shouldn't actually be a loop header"); assert(OnStack.empty() && "Control flow stack pushes and pops should be balanced."); #endif }
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop) { MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin(); SetVector<MachineBasicBlock *> RewriteSuccs; // DFS through Loop's body, looking for for irreducible control flow. Loop is // natural, and we stay in its body, and we treat any nested loops // monolithically, so any cycles we encounter indicate irreducibility. SmallPtrSet<MachineBasicBlock *, 8> OnStack; SmallPtrSet<MachineBasicBlock *, 8> Visited; SmallVector<SuccessorList, 4> LoopWorklist; LoopWorklist.push_back(SuccessorList(Header)); OnStack.insert(Header); Visited.insert(Header); while (!LoopWorklist.empty()) { SuccessorList &Top = LoopWorklist.back(); if (Top.HasNext()) { MachineBasicBlock *Next = Top.Next(); if (Next == Header || (Loop && !Loop->contains(Next))) continue; if (LLVM_LIKELY(OnStack.insert(Next).second)) { if (!Visited.insert(Next).second) { OnStack.erase(Next); continue; } MachineLoop *InnerLoop = MLI.getLoopFor(Next); if (InnerLoop != Loop) LoopWorklist.push_back(SuccessorList(InnerLoop)); else LoopWorklist.push_back(SuccessorList(Next)); } else { RewriteSuccs.insert(Top.getBlock()); } continue; } OnStack.erase(Top.getBlock()); LoopWorklist.pop_back(); } // Most likely, we didn't find any irreducible control flow. if (LLVM_LIKELY(RewriteSuccs.empty())) return false; DEBUG(dbgs() << "Irreducible control flow detected!\n"); // Ok. We have irreducible control flow! Create a dispatch block which will // contains a jump table to any block in the problematic set of blocks. MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock(); MF.insert(MF.end(), Dispatch); MLI.changeLoopFor(Dispatch, Loop); // Add the jump table. const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo(); MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(), TII.get(WebAssembly::BR_TABLE_I32)); // Add the register which will be used to tell the jump table which block to // jump to. MachineRegisterInfo &MRI = MF.getRegInfo(); unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); MIB.addReg(Reg); // Collect all the blocks which need to have their successors rewritten, // add the successors to the jump table, and remember their index. DenseMap<MachineBasicBlock *, unsigned> Indices; SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(), RewriteSuccs.end()); while (!SuccWorklist.empty()) { MachineBasicBlock *MBB = SuccWorklist.pop_back_val(); auto Pair = Indices.insert(std::make_pair(MBB, 0)); if (!Pair.second) continue; unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1; DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n"); Pair.first->second = Index; for (auto Pred : MBB->predecessors()) RewriteSuccs.insert(Pred); MIB.addMBB(MBB); Dispatch->addSuccessor(MBB); MetaBlock Meta(MBB); for (auto *Succ : Meta.successors()) if (Succ != Header && (!Loop || Loop->contains(Succ))) SuccWorklist.push_back(Succ); } // Rewrite the problematic successors for every block in RewriteSuccs. // For simplicity, we just introduce a new block for every edge we need to // rewrite. Fancier things are possible. for (MachineBasicBlock *MBB : RewriteSuccs) { DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map; for (auto *Succ : MBB->successors()) { if (!Indices.count(Succ)) continue; MachineBasicBlock *Split = MF.CreateMachineBasicBlock(); MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ) : MF.end(), Split); MLI.changeLoopFor(Split, Loop); // Set the jump table's register of the index of the block we wish to // jump to, and jump to the jump table. BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32), Reg) .addImm(Indices[Succ]); BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR)) .addMBB(Dispatch); Split->addSuccessor(Dispatch); Map[Succ] = Split; } // Remap the terminator operands and the successor list. for (MachineInstr &Term : MBB->terminators()) for (auto &Op : Term.explicit_uses()) if (Op.isMBB() && Indices.count(Op.getMBB())) Op.setMBB(Map[Op.getMBB()]); for (auto Rewrite : Map) MBB->replaceSuccessor(Rewrite.first, Rewrite.second); } // Create a fake default label, because br_table requires one. MIB.addMBB(MIB.getInstr() ->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1) .getMBB()); return true; }
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) { SmallPtrSet<MachineBasicBlock*, 8> Reachable; MMI = getAnalysisIfAvailable<MachineModuleInfo>(); MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>(); MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); // Mark all reachable blocks. for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> > I = df_ext_begin(&F, Reachable), E = df_ext_end(&F, Reachable); I != E; ++I) /* Mark all reachable blocks */; // Loop over all dead blocks, remembering them and deleting all instructions // in them. std::vector<MachineBasicBlock*> DeadBlocks; for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = I; // Test for deadness. if (!Reachable.count(BB)) { DeadBlocks.push_back(BB); // Update dominator and loop info. if (MLI) MLI->removeBlock(BB); if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB); while (BB->succ_begin() != BB->succ_end()) { MachineBasicBlock* succ = *BB->succ_begin(); MachineBasicBlock::iterator start = succ->begin(); while (start != succ->end() && start->isPHI()) { for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2) if (start->getOperand(i).isMBB() && start->getOperand(i).getMBB() == BB) { start->RemoveOperand(i); start->RemoveOperand(i-1); } start++; } BB->removeSuccessor(BB->succ_begin()); } } } // Actually remove the blocks now. for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) DeadBlocks[i]->eraseFromParent(); // Cleanup PHI nodes. for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = I; // Prune unneeded PHI entries. SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(), BB->pred_end()); MachineBasicBlock::iterator phi = BB->begin(); while (phi != BB->end() && phi->isPHI()) { for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2) if (!preds.count(phi->getOperand(i).getMBB())) { phi->RemoveOperand(i); phi->RemoveOperand(i-1); } if (phi->getNumOperands() == 3) { unsigned Input = phi->getOperand(1).getReg(); unsigned Output = phi->getOperand(0).getReg(); MachineInstr* temp = phi; ++phi; temp->eraseFromParent(); if (Input != Output) F.getRegInfo().replaceRegWith(Output, Input); continue; } ++phi; } } F.RenumberBlocks(); return DeadBlocks.size(); }
/// Sort the blocks in RPO, taking special care to make sure that loops are /// contiguous even in the case of split backedges. /// /// TODO: Determine whether RPO is actually worthwhile, or whether we should /// move to just a stable-topological-sort-based approach that would preserve /// more of the original order. static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) { // Note that we do our own RPO rather than using // "llvm/ADT/PostOrderIterator.h" because we want control over the order that // successors are visited in (see above). Also, we can sort the blocks in the // MachineFunction as we go. SmallPtrSet<MachineBasicBlock *, 16> Visited; SmallVector<POStackEntry, 16> Stack; MachineBasicBlock *EntryBlock = &*MF.begin(); Visited.insert(EntryBlock); Stack.push_back(POStackEntry(EntryBlock, MF, MLI)); for (;;) { POStackEntry &Entry = Stack.back(); SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs; if (!Succs.empty()) { MachineBasicBlock *Succ = Succs.pop_back_val(); if (Visited.insert(Succ).second) Stack.push_back(POStackEntry(Succ, MF, MLI)); continue; } // Put the block in its position in the MachineFunction. MachineBasicBlock &MBB = *Entry.MBB; MBB.moveBefore(&*MF.begin()); // Branch instructions may utilize a fallthrough, so update them if a // fallthrough has been added or removed. if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() && !MBB.back().isBarrier()) report_fatal_error( "Non-branch terminator with fallthrough cannot yet be rewritten"); if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch()) MBB.updateTerminator(); Stack.pop_back(); if (Stack.empty()) break; } // Now that we've sorted the blocks in RPO, renumber them. MF.RenumberBlocks(); #ifndef NDEBUG SmallSetVector<MachineLoop *, 8> OnStack; // Insert a sentinel representing the degenerate loop that starts at the // function entry block and includes the entire function as a "loop" that // executes once. OnStack.insert(nullptr); for (auto &MBB : MF) { assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative."); MachineLoop *Loop = MLI.getLoopFor(&MBB); if (Loop && &MBB == Loop->getHeader()) { // Loop header. The loop predecessor should be sorted above, and the other // predecessors should be backedges below. for (auto Pred : MBB.predecessors()) assert( (Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) && "Loop header predecessors must be loop predecessors or backedges"); assert(OnStack.insert(Loop) && "Loops should be declared at most once."); } else { // Not a loop header. All predecessors should be sorted above. for (auto Pred : MBB.predecessors()) assert(Pred->getNumber() < MBB.getNumber() && "Non-loop-header predecessors should be topologically sorted"); assert(OnStack.count(MLI.getLoopFor(&MBB)) && "Blocks must be nested in their loops"); } while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back())) OnStack.pop_back(); } assert(OnStack.pop_back_val() == nullptr && "The function entry block shouldn't actually be a loop header"); assert(OnStack.empty() && "Control flow stack pushes and pops should be balanced."); #endif }