/// Sort the blocks in RPO, taking special care to make sure that loops are
/// contiguous even in the case of split backedges.
static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) {
// Note that we do our own RPO rather than using
// "llvm/ADT/PostOrderIterator.h" because we want control over the order that
// successors are visited in (see above). Also, we can sort the blocks in the
// MachineFunction as we go.
SmallPtrSet<MachineBasicBlock *, 16> Visited;
SmallVector<POStackEntry, 16> Stack;
MachineBasicBlock *Entry = &*MF.begin();
Visited.insert(Entry);
Stack.push_back(POStackEntry(Entry, MF, MLI));
for (;;) {
POStackEntry &Entry = Stack.back();
SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs;
if (!Succs.empty()) {
MachineBasicBlock *Succ = Succs.pop_back_val();
if (Visited.insert(Succ).second)
Stack.push_back(POStackEntry(Succ, MF, MLI));
continue;
}
// Put the block in its position in the MachineFunction.
MachineBasicBlock &MBB = *Entry.MBB;
MBB.moveBefore(&*MF.begin());
// Branch instructions may utilize a fallthrough, so update them if a
// fallthrough has been added or removed.
if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() &&
!MBB.back().isBarrier())
report_fatal_error(
"Non-branch terminator with fallthrough cannot yet be rewritten");
if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch())
MBB.updateTerminator();
Stack.pop_back();
if (Stack.empty())
break;
}
// Now that we've sorted the blocks in RPO, renumber them.
MF.RenumberBlocks();
#ifndef NDEBUG
for (auto &MBB : MF)
if (MachineLoop *Loop = MLI.getLoopFor(&MBB)) {
// Assert that loops are contiguous.
assert(Loop->getHeader() == Loop->getTopBlock());
assert((Loop->getHeader() == &MBB ||
Loop->contains(
MLI.getLoopFor(&*prev(MachineFunction::iterator(&MBB))))) &&
"Loop isn't contiguous");
} else {
// Assert that non-loops have no backedge predecessors.
for (auto Pred : MBB.predecessors())
assert(Pred->getNumber() < MBB.getNumber() &&
"CFG still has multiple-entry loops");
}
#endif
}
static void EliminateMultipleEntryLoops(MachineFunction &MF,
const MachineLoopInfo &MLI) {
SmallPtrSet<MachineBasicBlock *, 8> InSet;
for (scc_iterator<MachineFunction *> I = scc_begin(&MF), E = scc_end(&MF);
I != E; ++I) {
const std::vector<MachineBasicBlock *> &CurrentSCC = *I;
// Skip trivial SCCs.
if (CurrentSCC.size() == 1)
continue;
InSet.insert(CurrentSCC.begin(), CurrentSCC.end());
MachineBasicBlock *Header = nullptr;
for (MachineBasicBlock *MBB : CurrentSCC) {
for (MachineBasicBlock *Pred : MBB->predecessors()) {
if (InSet.count(Pred))
continue;
if (!Header) {
Header = MBB;
break;
}
// TODO: Implement multiple-entry loops.
report_fatal_error("multiple-entry loops are not supported yet");
}
}
assert(MLI.isLoopHeader(Header));
InSet.clear();
}
}
/// Insert LOOP and BLOCK markers at appropriate places.
static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
for (auto &MBB : MF) {
// Place the LOOP for loops.
if (MachineLoop *Loop = MLI.getLoopFor(&MBB))
if (Loop->getHeader() == &MBB) {
// The operand of a LOOP is the first block after the loop. If the loop
// is the bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = Loop->getBottomBlock();
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MF.push_back(MF.CreateMachineBasicBlock());
Iter = next(MachineFunction::iterator(Bottom));
}
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(&*Iter);
}
// Check for forward branches and switches that need BLOCKS placed.
for (auto &Term : MBB.terminators())
for (auto &MO : Term.operands())
if (MO.isMBB())
PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII);
}
}
/// Insert BLOCK markers at appropriate places.
static void PlaceBlockMarkers(MachineBasicBlock &MBB, MachineBasicBlock &Succ,
MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
// Backward branches are loop backedges, and we place the LOOP markers
// separately. So only consider forward branches here.
if (Succ.getNumber() <= MBB.getNumber())
return;
// Place the BLOCK for a forward branch. For simplicity, we just insert
// blocks immediately inside loop boundaries.
MachineLoop *Loop = MLI.getLoopFor(&Succ);
MachineBasicBlock &Header = *(Loop ? Loop->getHeader() : &MF.front());
MachineBasicBlock::iterator InsertPos = Header.begin(), End = Header.end();
if (InsertPos != End) {
if (InsertPos->getOpcode() == WebAssembly::LOOP)
++InsertPos;
int SuccNumber = Succ.getNumber();
// Position the BLOCK in nesting order.
for (; InsertPos != End && InsertPos->getOpcode() == WebAssembly::BLOCK;
++InsertPos) {
int N = InsertPos->getOperand(0).getMBB()->getNumber();
if (N < SuccNumber)
break;
// If there's already a BLOCK for Succ, we don't need another.
if (N == SuccNumber)
return;
}
}
BuildMI(Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK))
.addMBB(&Succ);
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(AfterLoop);
// Emit a special no-op telling the asm printer that we need a label to close
// the loop scope, even though the destination is only reachable by
// fallthrough.
if (!Bottom->back().isBarrier())
BuildMI(*Bottom, Bottom->end(), DebugLoc(), TII.get(WebAssembly::LOOP_END));
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With RPO we should visit the outer-most loop for a block first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
/// Check whether the edge (\p SrcBB, \p DestBB) is a backedge according to MLI.
/// I.e., check if it exists a loop that contains SrcBB and where DestBB is the
/// loop header.
static bool isProperBackedge(const MachineLoopInfo &MLI,
const MachineBasicBlock *SrcBB,
const MachineBasicBlock *DestBB) {
for (const MachineLoop *Loop = MLI.getLoopFor(SrcBB); Loop;
Loop = Loop->getParentLoop()) {
if (Loop->getHeader() == DestBB)
return true;
}
return false;
}
POStackEntry::POStackEntry(MachineBasicBlock *MBB, MachineFunction &MF,
const MachineLoopInfo &MLI)
: MBB(MBB), Succs(MBB->successors()) {
// RPO is not a unique form, since at every basic block with multiple
// successors, the DFS has to pick which order to visit the successors in.
// Sort them strategically (see below).
MachineLoop *Loop = MLI.getLoopFor(MBB);
MachineFunction::iterator Next = next(MachineFunction::iterator(MBB));
MachineBasicBlock *LayoutSucc = Next == MF.end() ? nullptr : &*Next;
std::stable_sort(
Succs.begin(), Succs.end(),
[=, &MLI](const MachineBasicBlock *A, const MachineBasicBlock *B) {
if (A == B)
return false;
// Keep loops contiguous by preferring the block that's in the same
// loop.
MachineLoop *LoopA = MLI.getLoopFor(A);
MachineLoop *LoopB = MLI.getLoopFor(B);
if (LoopA == Loop && LoopB != Loop)
return true;
if (LoopA != Loop && LoopB == Loop)
return false;
// Minimize perturbation by preferring the block which is the immediate
// layout successor.
if (A == LayoutSucc)
return true;
if (B == LayoutSucc)
return false;
// TODO: More sophisticated orderings may be profitable here.
return false;
});
}
/// Insert LOOP and BLOCK markers at appropriate places.
static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
for (auto &MBB : MF) {
// Place the LOOP for loops.
if (MachineLoop *Loop = MLI.getLoopFor(&MBB))
if (Loop->getHeader() == &MBB)
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(Loop->getBottomBlock());
// Check for forward branches and switches that need BLOCKS placed.
for (auto &Term : MBB.terminators())
for (auto &MO : Term.operands())
if (MO.isMBB())
PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII);
}
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(
MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
DenseMap<const MachineInstr *, const MachineBasicBlock *> &LoopTops,
const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
// Mark the beginning of the loop (after the end of any existing loop that
// ends here).
auto InsertPos = MBB.begin();
while (InsertPos != MBB.end() &&
InsertPos->getOpcode() == WebAssembly::END_LOOP)
++InsertPos;
BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::LOOP));
// Mark the end of the loop.
MachineInstr *End = BuildMI(*AfterLoop, AfterLoop->begin(), DebugLoc(),
TII.get(WebAssembly::END_LOOP));
LoopTops[End] = &MBB;
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With block sorting the outermost loop for a block should be first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &fn) {
DEBUG(dbgs() << "********** Compute Spill Weights **********\n"
<< "********** Function: "
<< fn.getFunction()->getName() << '\n');
LiveIntervals *lis = &getAnalysis<LiveIntervals>();
MachineLoopInfo *loopInfo = &getAnalysis<MachineLoopInfo>();
const TargetInstrInfo *tii = fn.getTarget().getInstrInfo();
MachineRegisterInfo *mri = &fn.getRegInfo();
SmallSet<unsigned, 4> processed;
for (MachineFunction::iterator mbbi = fn.begin(), mbbe = fn.end();
mbbi != mbbe; ++mbbi) {
MachineBasicBlock* mbb = mbbi;
SlotIndex mbbEnd = lis->getMBBEndIdx(mbb);
MachineLoop* loop = loopInfo->getLoopFor(mbb);
unsigned loopDepth = loop ? loop->getLoopDepth() : 0;
bool isExiting = loop ? loop->isLoopExiting(mbb) : false;
for (MachineBasicBlock::const_iterator mii = mbb->begin(), mie = mbb->end();
mii != mie; ++mii) {
const MachineInstr *mi = mii;
if (tii->isIdentityCopy(*mi) || mi->isImplicitDef() || mi->isDebugValue())
continue;
for (unsigned i = 0, e = mi->getNumOperands(); i != e; ++i) {
const MachineOperand &mopi = mi->getOperand(i);
if (!mopi.isReg() || mopi.getReg() == 0)
continue;
unsigned reg = mopi.getReg();
if (!TargetRegisterInfo::isVirtualRegister(mopi.getReg()))
continue;
// Multiple uses of reg by the same instruction. It should not
// contribute to spill weight again.
if (!processed.insert(reg))
continue;
bool hasDef = mopi.isDef();
bool hasUse = !hasDef;
for (unsigned j = i+1; j != e; ++j) {
const MachineOperand &mopj = mi->getOperand(j);
if (!mopj.isReg() || mopj.getReg() != reg)
continue;
hasDef |= mopj.isDef();
hasUse |= mopj.isUse();
if (hasDef && hasUse)
break;
}
LiveInterval ®Int = lis->getInterval(reg);
float weight = lis->getSpillWeight(hasDef, hasUse, loopDepth);
if (hasDef && isExiting) {
// Looks like this is a loop count variable update.
SlotIndex defIdx = lis->getInstructionIndex(mi).getDefIndex();
const LiveRange *dlr =
lis->getInterval(reg).getLiveRangeContaining(defIdx);
if (dlr->end >= mbbEnd)
weight *= 3.0F;
}
regInt.weight += weight;
}
processed.clear();
}
}
for (LiveIntervals::iterator I = lis->begin(), E = lis->end(); I != E; ++I) {
LiveInterval &li = *I->second;
if (TargetRegisterInfo::isVirtualRegister(li.reg)) {
// If the live interval length is essentially zero, i.e. in every live
// range the use follows def immediately, it doesn't make sense to spill
// it and hope it will be easier to allocate for this li.
if (isZeroLengthInterval(&li)) {
li.weight = HUGE_VALF;
continue;
}
bool isLoad = false;
SmallVector<LiveInterval*, 4> spillIs;
if (lis->isReMaterializable(li, spillIs, isLoad)) {
// If all of the definitions of the interval are re-materializable,
// it is a preferred candidate for spilling. If non of the defs are
// loads, then it's potentially very cheap to re-materialize.
// FIXME: this gets much more complicated once we support non-trivial
// re-materialization.
if (isLoad)
li.weight *= 0.9F;
else
li.weight *= 0.5F;
}
// Slightly prefer live interval that has been assigned a preferred reg.
std::pair<unsigned, unsigned> Hint = mri->getRegAllocationHint(li.reg);
if (Hint.first || Hint.second)
li.weight *= 1.01F;
lis->normalizeSpillWeight(li);
}
}
return false;
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
df_iterator_default_set<MachineBasicBlock*> Reachable;
bool ModifiedPHI = false;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
ModifiedPHI = true;
}
if (phi->getNumOperands() == 3) {
const MachineOperand &Input = phi->getOperand(1);
const MachineOperand &Output = phi->getOperand(0);
unsigned InputReg = Input.getReg();
unsigned OutputReg = Output.getReg();
assert(Output.getSubReg() == 0 && "Cannot have output subregister");
ModifiedPHI = true;
if (InputReg != OutputReg) {
MachineRegisterInfo &MRI = F.getRegInfo();
unsigned InputSub = Input.getSubReg();
if (InputSub == 0 &&
MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) {
MRI.replaceRegWith(OutputReg, InputReg);
} else {
// The input register to the PHI has a subregister or it can't be
// constrained to the proper register class:
// insert a COPY instead of simply replacing the output
// with the input.
const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(),
TII->get(TargetOpcode::COPY), OutputReg)
.addReg(InputReg, getRegState(Input), InputSub);
}
phi++->eraseFromParent();
}
continue;
}
++phi;
}
}
F.RenumberBlocks();
return (!DeadBlocks.empty() || ModifiedPHI);
}
/// Insert a BLOCK marker for branches to MBB (if needed).
static void PlaceBlockMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI,
MachineDominatorTree &MDT,
WebAssemblyFunctionInfo &MFI) {
// First compute the nearest common dominator of all forward non-fallthrough
// predecessors so that we minimize the time that the BLOCK is on the stack,
// which reduces overall stack height.
MachineBasicBlock *Header = nullptr;
bool IsBranchedTo = false;
int MBBNumber = MBB.getNumber();
for (MachineBasicBlock *Pred : MBB.predecessors())
if (Pred->getNumber() < MBBNumber) {
Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
if (ExplicitlyBranchesTo(Pred, &MBB))
IsBranchedTo = true;
}
if (!Header)
return;
if (!IsBranchedTo)
return;
assert(&MBB != &MF.front() && "Header blocks shouldn't have predecessors");
MachineBasicBlock *LayoutPred = &*prev(MachineFunction::iterator(&MBB));
// If the nearest common dominator is inside a more deeply nested context,
// walk out to the nearest scope which isn't more deeply nested.
for (MachineFunction::iterator I(LayoutPred), E(Header); I != E; --I) {
if (MachineBasicBlock *ScopeTop = ScopeTops[I->getNumber()]) {
if (ScopeTop->getNumber() > Header->getNumber()) {
// Skip over an intervening scope.
I = next(MachineFunction::iterator(ScopeTop));
} else {
// We found a scope level at an appropriate depth.
Header = ScopeTop;
break;
}
}
}
// If there's a loop which ends just before MBB which contains Header, we can
// reuse its label instead of inserting a new BLOCK.
for (MachineLoop *Loop = MLI.getLoopFor(LayoutPred);
Loop && Loop->contains(LayoutPred); Loop = Loop->getParentLoop())
if (Loop && LoopBottom(Loop) == LayoutPred && Loop->contains(Header))
return;
// Decide where in Header to put the BLOCK.
MachineBasicBlock::iterator InsertPos;
MachineLoop *HeaderLoop = MLI.getLoopFor(Header);
if (HeaderLoop && MBB.getNumber() > LoopBottom(HeaderLoop)->getNumber()) {
// Header is the header of a loop that does not lexically contain MBB, so
// the BLOCK needs to be above the LOOP, after any END constructs.
InsertPos = Header->begin();
while (InsertPos->getOpcode() != WebAssembly::LOOP)
++InsertPos;
} else {
// Otherwise, insert the BLOCK as late in Header as we can, but before the
// beginning of the local expression tree and any nested BLOCKs.
InsertPos = Header->getFirstTerminator();
while (InsertPos != Header->begin() && IsChild(prev(InsertPos), MFI) &&
prev(InsertPos)->getOpcode() != WebAssembly::LOOP &&
prev(InsertPos)->getOpcode() != WebAssembly::END_BLOCK &&
prev(InsertPos)->getOpcode() != WebAssembly::END_LOOP)
--InsertPos;
}
// Add the BLOCK.
BuildMI(*Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK));
// Mark the end of the block.
InsertPos = MBB.begin();
while (InsertPos != MBB.end() &&
InsertPos->getOpcode() == WebAssembly::END_LOOP)
++InsertPos;
BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::END_BLOCK));
// Track the farthest-spanning scope that ends at this point.
int Number = MBB.getNumber();
if (!ScopeTops[Number] ||
ScopeTops[Number]->getNumber() > Header->getNumber())
ScopeTops[Number] = Header;
}
/// Sort the blocks, taking special care to make sure that loops are not
/// interrupted by blocks not dominated by their header.
/// TODO: There are many opportunities for improving the heuristics here.
/// Explore them.
static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
const MachineDominatorTree &MDT) {
// Prepare for a topological sort: Record the number of predecessors each
// block has, ignoring loop backedges.
MF.RenumberBlocks();
SmallVector<unsigned, 16> NumPredsLeft(MF.getNumBlockIDs(), 0);
for (MachineBasicBlock &MBB : MF) {
unsigned N = MBB.pred_size();
if (MachineLoop *L = MLI.getLoopFor(&MBB))
if (L->getHeader() == &MBB)
for (const MachineBasicBlock *Pred : MBB.predecessors())
if (L->contains(Pred))
--N;
NumPredsLeft[MBB.getNumber()] = N;
}
// Topological sort the CFG, with additional constraints:
// - Between a loop header and the last block in the loop, there can be
// no blocks not dominated by the loop header.
// - It's desirable to preserve the original block order when possible.
// We use two ready lists; Preferred and Ready. Preferred has recently
// processed sucessors, to help preserve block sequences from the original
// order. Ready has the remaining ready blocks.
PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
CompareBlockNumbers>
Preferred;
PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
CompareBlockNumbersBackwards>
Ready;
SmallVector<Entry, 4> Loops;
for (MachineBasicBlock *MBB = &MF.front();;) {
const MachineLoop *L = MLI.getLoopFor(MBB);
if (L) {
// If MBB is a loop header, add it to the active loop list. We can't put
// any blocks that it doesn't dominate until we see the end of the loop.
if (L->getHeader() == MBB)
Loops.push_back(Entry(L));
// For each active loop the block is in, decrement the count. If MBB is
// the last block in an active loop, take it off the list and pick up any
// blocks deferred because the header didn't dominate them.
for (Entry &E : Loops)
if (E.Loop->contains(MBB) && --E.NumBlocksLeft == 0)
for (auto DeferredBlock : E.Deferred)
Ready.push(DeferredBlock);
while (!Loops.empty() && Loops.back().NumBlocksLeft == 0)
Loops.pop_back();
}
// The main topological sort logic.
for (MachineBasicBlock *Succ : MBB->successors()) {
// Ignore backedges.
if (MachineLoop *SuccL = MLI.getLoopFor(Succ))
if (SuccL->getHeader() == Succ && SuccL->contains(MBB))
continue;
// Decrement the predecessor count. If it's now zero, it's ready.
if (--NumPredsLeft[Succ->getNumber()] == 0)
Preferred.push(Succ);
}
// Determine the block to follow MBB. First try to find a preferred block,
// to preserve the original block order when possible.
MachineBasicBlock *Next = nullptr;
while (!Preferred.empty()) {
Next = Preferred.top();
Preferred.pop();
// If X isn't dominated by the top active loop header, defer it until that
// loop is done.
if (!Loops.empty() &&
!MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
Loops.back().Deferred.push_back(Next);
Next = nullptr;
continue;
}
// If Next was originally ordered before MBB, and it isn't because it was
// loop-rotated above the header, it's not preferred.
if (Next->getNumber() < MBB->getNumber() &&
(!L || !L->contains(Next) ||
L->getHeader()->getNumber() < Next->getNumber())) {
Ready.push(Next);
Next = nullptr;
continue;
}
break;
}
// If we didn't find a suitable block in the Preferred list, check the
// general Ready list.
if (!Next) {
// If there are no more blocks to process, we're done.
if (Ready.empty()) {
MaybeUpdateTerminator(MBB);
break;
}
for (;;) {
Next = Ready.top();
Ready.pop();
// If Next isn't dominated by the top active loop header, defer it until
// that loop is done.
if (!Loops.empty() &&
!MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
Loops.back().Deferred.push_back(Next);
continue;
}
break;
}
}
// Move the next block into place and iterate.
Next->moveAfter(MBB);
MaybeUpdateTerminator(MBB);
MBB = Next;
}
assert(Loops.empty() && "Active loop list not finished");
MF.RenumberBlocks();
#ifndef NDEBUG
SmallSetVector<MachineLoop *, 8> OnStack;
// Insert a sentinel representing the degenerate loop that starts at the
// function entry block and includes the entire function as a "loop" that
// executes once.
OnStack.insert(nullptr);
for (auto &MBB : MF) {
assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (Loop && &MBB == Loop->getHeader()) {
// Loop header. The loop predecessor should be sorted above, and the other
// predecessors should be backedges below.
for (auto Pred : MBB.predecessors())
assert(
(Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) &&
"Loop header predecessors must be loop predecessors or backedges");
assert(OnStack.insert(Loop) && "Loops should be declared at most once.");
} else {
// Not a loop header. All predecessors should be sorted above.
for (auto Pred : MBB.predecessors())
assert(Pred->getNumber() < MBB.getNumber() &&
"Non-loop-header predecessors should be topologically sorted");
assert(OnStack.count(MLI.getLoopFor(&MBB)) &&
"Blocks must be nested in their loops");
}
while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back()))
OnStack.pop_back();
}
assert(OnStack.pop_back_val() == nullptr &&
"The function entry block shouldn't actually be a loop header");
assert(OnStack.empty() &&
"Control flow stack pushes and pops should be balanced.");
#endif
}
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
MachineLoopInfo &MLI,
MachineLoop *Loop) {
MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
SetVector<MachineBasicBlock *> RewriteSuccs;
// DFS through Loop's body, looking for for irreducible control flow. Loop is
// natural, and we stay in its body, and we treat any nested loops
// monolithically, so any cycles we encounter indicate irreducibility.
SmallPtrSet<MachineBasicBlock *, 8> OnStack;
SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<SuccessorList, 4> LoopWorklist;
LoopWorklist.push_back(SuccessorList(Header));
OnStack.insert(Header);
Visited.insert(Header);
while (!LoopWorklist.empty()) {
SuccessorList &Top = LoopWorklist.back();
if (Top.HasNext()) {
MachineBasicBlock *Next = Top.Next();
if (Next == Header || (Loop && !Loop->contains(Next)))
continue;
if (LLVM_LIKELY(OnStack.insert(Next).second)) {
if (!Visited.insert(Next).second) {
OnStack.erase(Next);
continue;
}
MachineLoop *InnerLoop = MLI.getLoopFor(Next);
if (InnerLoop != Loop)
LoopWorklist.push_back(SuccessorList(InnerLoop));
else
LoopWorklist.push_back(SuccessorList(Next));
} else {
RewriteSuccs.insert(Top.getBlock());
}
continue;
}
OnStack.erase(Top.getBlock());
LoopWorklist.pop_back();
}
// Most likely, we didn't find any irreducible control flow.
if (LLVM_LIKELY(RewriteSuccs.empty()))
return false;
DEBUG(dbgs() << "Irreducible control flow detected!\n");
// Ok. We have irreducible control flow! Create a dispatch block which will
// contains a jump table to any block in the problematic set of blocks.
MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
MF.insert(MF.end(), Dispatch);
MLI.changeLoopFor(Dispatch, Loop);
// Add the jump table.
const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(),
TII.get(WebAssembly::BR_TABLE_I32));
// Add the register which will be used to tell the jump table which block to
// jump to.
MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
MIB.addReg(Reg);
// Collect all the blocks which need to have their successors rewritten,
// add the successors to the jump table, and remember their index.
DenseMap<MachineBasicBlock *, unsigned> Indices;
SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
RewriteSuccs.end());
while (!SuccWorklist.empty()) {
MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
auto Pair = Indices.insert(std::make_pair(MBB, 0));
if (!Pair.second)
continue;
unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n");
Pair.first->second = Index;
for (auto Pred : MBB->predecessors())
RewriteSuccs.insert(Pred);
MIB.addMBB(MBB);
Dispatch->addSuccessor(MBB);
MetaBlock Meta(MBB);
for (auto *Succ : Meta.successors())
if (Succ != Header && (!Loop || Loop->contains(Succ)))
SuccWorklist.push_back(Succ);
}
// Rewrite the problematic successors for every block in RewriteSuccs.
// For simplicity, we just introduce a new block for every edge we need to
// rewrite. Fancier things are possible.
for (MachineBasicBlock *MBB : RewriteSuccs) {
DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
for (auto *Succ : MBB->successors()) {
if (!Indices.count(Succ))
continue;
MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
: MF.end(),
Split);
MLI.changeLoopFor(Split, Loop);
// Set the jump table's register of the index of the block we wish to
// jump to, and jump to the jump table.
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32),
Reg)
.addImm(Indices[Succ]);
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR))
.addMBB(Dispatch);
Split->addSuccessor(Dispatch);
Map[Succ] = Split;
}
// Remap the terminator operands and the successor list.
for (MachineInstr &Term : MBB->terminators())
for (auto &Op : Term.explicit_uses())
if (Op.isMBB() && Indices.count(Op.getMBB()))
Op.setMBB(Map[Op.getMBB()]);
for (auto Rewrite : Map)
MBB->replaceSuccessor(Rewrite.first, Rewrite.second);
}
// Create a fake default label, because br_table requires one.
MIB.addMBB(MIB.getInstr()
->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1)
.getMBB());
return true;
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
SmallPtrSet<MachineBasicBlock*, 8> Reachable;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> >
I = df_ext_begin(&F, Reachable), E = df_ext_end(&F, Reachable);
I != E; ++I)
/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
}
if (phi->getNumOperands() == 3) {
unsigned Input = phi->getOperand(1).getReg();
unsigned Output = phi->getOperand(0).getReg();
MachineInstr* temp = phi;
++phi;
temp->eraseFromParent();
if (Input != Output)
F.getRegInfo().replaceRegWith(Output, Input);
continue;
}
++phi;
}
}
F.RenumberBlocks();
return DeadBlocks.size();
}
/// Sort the blocks in RPO, taking special care to make sure that loops are
/// contiguous even in the case of split backedges.
///
/// TODO: Determine whether RPO is actually worthwhile, or whether we should
/// move to just a stable-topological-sort-based approach that would preserve
/// more of the original order.
static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) {
// Note that we do our own RPO rather than using
// "llvm/ADT/PostOrderIterator.h" because we want control over the order that
// successors are visited in (see above). Also, we can sort the blocks in the
// MachineFunction as we go.
SmallPtrSet<MachineBasicBlock *, 16> Visited;
SmallVector<POStackEntry, 16> Stack;
MachineBasicBlock *EntryBlock = &*MF.begin();
Visited.insert(EntryBlock);
Stack.push_back(POStackEntry(EntryBlock, MF, MLI));
for (;;) {
POStackEntry &Entry = Stack.back();
SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs;
if (!Succs.empty()) {
MachineBasicBlock *Succ = Succs.pop_back_val();
if (Visited.insert(Succ).second)
Stack.push_back(POStackEntry(Succ, MF, MLI));
continue;
}
// Put the block in its position in the MachineFunction.
MachineBasicBlock &MBB = *Entry.MBB;
MBB.moveBefore(&*MF.begin());
// Branch instructions may utilize a fallthrough, so update them if a
// fallthrough has been added or removed.
if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() &&
!MBB.back().isBarrier())
report_fatal_error(
"Non-branch terminator with fallthrough cannot yet be rewritten");
if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch())
MBB.updateTerminator();
Stack.pop_back();
if (Stack.empty())
break;
}
// Now that we've sorted the blocks in RPO, renumber them.
MF.RenumberBlocks();
#ifndef NDEBUG
SmallSetVector<MachineLoop *, 8> OnStack;
// Insert a sentinel representing the degenerate loop that starts at the
// function entry block and includes the entire function as a "loop" that
// executes once.
OnStack.insert(nullptr);
for (auto &MBB : MF) {
assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (Loop && &MBB == Loop->getHeader()) {
// Loop header. The loop predecessor should be sorted above, and the other
// predecessors should be backedges below.
for (auto Pred : MBB.predecessors())
assert(
(Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) &&
"Loop header predecessors must be loop predecessors or backedges");
assert(OnStack.insert(Loop) && "Loops should be declared at most once.");
} else {
// Not a loop header. All predecessors should be sorted above.
for (auto Pred : MBB.predecessors())
assert(Pred->getNumber() < MBB.getNumber() &&
"Non-loop-header predecessors should be topologically sorted");
assert(OnStack.count(MLI.getLoopFor(&MBB)) &&
"Blocks must be nested in their loops");
}
while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back()))
OnStack.pop_back();
}
assert(OnStack.pop_back_val() == nullptr &&
"The function entry block shouldn't actually be a loop header");
assert(OnStack.empty() &&
"Control flow stack pushes and pops should be balanced.");
#endif
}
