static bool MaybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
const MachineFunction &MF,
WebAssemblyFunctionInfo &MFI,
MachineRegisterInfo &MRI,
const WebAssemblyInstrInfo &TII,
unsigned FallthroughOpc,
unsigned CopyLocalOpc) {
if (DisableWebAssemblyFallthroughReturnOpt)
return false;
if (&MBB != &MF.back())
return false;
if (&MI != &MBB.back())
return false;
// If the operand isn't stackified, insert a COPY to read the operand and
// stackify it.
MachineOperand &MO = MI.getOperand(0);
unsigned Reg = MO.getReg();
if (!MFI.isVRegStackified(Reg)) {
unsigned NewReg = MRI.createVirtualRegister(MRI.getRegClass(Reg));
BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(CopyLocalOpc), NewReg)
.addReg(Reg);
MO.setReg(NewReg);
MFI.stackifyVReg(NewReg);
}
// Rewrite the return.
MI.setDesc(TII.get(FallthroughOpc));
return true;
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(AfterLoop);
// Emit a special no-op telling the asm printer that we need a label to close
// the loop scope, even though the destination is only reachable by
// fallthrough.
if (!Bottom->back().isBarrier())
BuildMI(*Bottom, Bottom->end(), DebugLoc(), TII.get(WebAssembly::LOOP_END));
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With RPO we should visit the outer-most loop for a block first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
/// Insert LOOP and BLOCK markers at appropriate places.
static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
for (auto &MBB : MF) {
// Place the LOOP for loops.
if (MachineLoop *Loop = MLI.getLoopFor(&MBB))
if (Loop->getHeader() == &MBB) {
// The operand of a LOOP is the first block after the loop. If the loop
// is the bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = Loop->getBottomBlock();
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MF.push_back(MF.CreateMachineBasicBlock());
Iter = next(MachineFunction::iterator(Bottom));
}
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(&*Iter);
}
// Check for forward branches and switches that need BLOCKS placed.
for (auto &Term : MBB.terminators())
for (auto &MO : Term.operands())
if (MO.isMBB())
PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII);
}
}
/// Insert BLOCK markers at appropriate places.
static void PlaceBlockMarkers(MachineBasicBlock &MBB, MachineBasicBlock &Succ,
MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
// Backward branches are loop backedges, and we place the LOOP markers
// separately. So only consider forward branches here.
if (Succ.getNumber() <= MBB.getNumber())
return;
// Place the BLOCK for a forward branch. For simplicity, we just insert
// blocks immediately inside loop boundaries.
MachineLoop *Loop = MLI.getLoopFor(&Succ);
MachineBasicBlock &Header = *(Loop ? Loop->getHeader() : &MF.front());
MachineBasicBlock::iterator InsertPos = Header.begin(), End = Header.end();
if (InsertPos != End) {
if (InsertPos->getOpcode() == WebAssembly::LOOP)
++InsertPos;
int SuccNumber = Succ.getNumber();
// Position the BLOCK in nesting order.
for (; InsertPos != End && InsertPos->getOpcode() == WebAssembly::BLOCK;
++InsertPos) {
int N = InsertPos->getOperand(0).getMBB()->getNumber();
if (N < SuccNumber)
break;
// If there's already a BLOCK for Succ, we don't need another.
if (N == SuccNumber)
return;
}
}
BuildMI(Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK))
.addMBB(&Succ);
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(
MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
DenseMap<const MachineInstr *, const MachineBasicBlock *> &LoopTops,
const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
// Mark the beginning of the loop (after the end of any existing loop that
// ends here).
auto InsertPos = MBB.begin();
while (InsertPos != MBB.end() &&
InsertPos->getOpcode() == WebAssembly::END_LOOP)
++InsertPos;
BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::LOOP));
// Mark the end of the loop.
MachineInstr *End = BuildMI(*AfterLoop, AfterLoop->begin(), DebugLoc(),
TII.get(WebAssembly::END_LOOP));
LoopTops[End] = &MBB;
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With block sorting the outermost loop for a block should be first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
/// Insert LOOP and BLOCK markers at appropriate places.
static void PlaceMarkers(MachineFunction &MF, const MachineLoopInfo &MLI,
const WebAssemblyInstrInfo &TII) {
for (auto &MBB : MF) {
// Place the LOOP for loops.
if (MachineLoop *Loop = MLI.getLoopFor(&MBB))
if (Loop->getHeader() == &MBB)
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(Loop->getBottomBlock());
// Check for forward branches and switches that need BLOCKS placed.
for (auto &Term : MBB.terminators())
for (auto &MO : Term.operands())
if (MO.isMBB())
PlaceBlockMarkers(MBB, *MO.getMBB(), MF, MLI, TII);
}
}
/// Insert a BLOCK marker for branches to MBB (if needed).
static void PlaceBlockMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI,
MachineDominatorTree &MDT,
WebAssemblyFunctionInfo &MFI) {
// First compute the nearest common dominator of all forward non-fallthrough
// predecessors so that we minimize the time that the BLOCK is on the stack,
// which reduces overall stack height.
MachineBasicBlock *Header = nullptr;
bool IsBranchedTo = false;
int MBBNumber = MBB.getNumber();
for (MachineBasicBlock *Pred : MBB.predecessors())
if (Pred->getNumber() < MBBNumber) {
Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
if (ExplicitlyBranchesTo(Pred, &MBB))
IsBranchedTo = true;
}
if (!Header)
return;
if (!IsBranchedTo)
return;
assert(&MBB != &MF.front() && "Header blocks shouldn't have predecessors");
MachineBasicBlock *LayoutPred = &*prev(MachineFunction::iterator(&MBB));
// If the nearest common dominator is inside a more deeply nested context,
// walk out to the nearest scope which isn't more deeply nested.
for (MachineFunction::iterator I(LayoutPred), E(Header); I != E; --I) {
if (MachineBasicBlock *ScopeTop = ScopeTops[I->getNumber()]) {
if (ScopeTop->getNumber() > Header->getNumber()) {
// Skip over an intervening scope.
I = next(MachineFunction::iterator(ScopeTop));
} else {
// We found a scope level at an appropriate depth.
Header = ScopeTop;
break;
}
}
}
// If there's a loop which ends just before MBB which contains Header, we can
// reuse its label instead of inserting a new BLOCK.
for (MachineLoop *Loop = MLI.getLoopFor(LayoutPred);
Loop && Loop->contains(LayoutPred); Loop = Loop->getParentLoop())
if (Loop && LoopBottom(Loop) == LayoutPred && Loop->contains(Header))
return;
// Decide where in Header to put the BLOCK.
MachineBasicBlock::iterator InsertPos;
MachineLoop *HeaderLoop = MLI.getLoopFor(Header);
if (HeaderLoop && MBB.getNumber() > LoopBottom(HeaderLoop)->getNumber()) {
// Header is the header of a loop that does not lexically contain MBB, so
// the BLOCK needs to be above the LOOP, after any END constructs.
InsertPos = Header->begin();
while (InsertPos->getOpcode() != WebAssembly::LOOP)
++InsertPos;
} else {
// Otherwise, insert the BLOCK as late in Header as we can, but before the
// beginning of the local expression tree and any nested BLOCKs.
InsertPos = Header->getFirstTerminator();
while (InsertPos != Header->begin() && IsChild(prev(InsertPos), MFI) &&
prev(InsertPos)->getOpcode() != WebAssembly::LOOP &&
prev(InsertPos)->getOpcode() != WebAssembly::END_BLOCK &&
prev(InsertPos)->getOpcode() != WebAssembly::END_LOOP)
--InsertPos;
}
// Add the BLOCK.
BuildMI(*Header, InsertPos, DebugLoc(), TII.get(WebAssembly::BLOCK));
// Mark the end of the block.
InsertPos = MBB.begin();
while (InsertPos != MBB.end() &&
InsertPos->getOpcode() == WebAssembly::END_LOOP)
++InsertPos;
BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::END_BLOCK));
// Track the farthest-spanning scope that ends at this point.
int Number = MBB.getNumber();
if (!ScopeTops[Number] ||
ScopeTops[Number]->getNumber() > Header->getNumber())
ScopeTops[Number] = Header;
}
