void
LoopUnroller::go(LoopIterationBound *bound)
{
// For now we always unroll loops the same number of times.
static const size_t UnrollCount = 10;
JitSpew(JitSpew_Unrolling, "Attempting to unroll loop");
header = bound->header;
// UCE might have determined this isn't actually a loop.
if (!header->isLoopHeader())
return;
backedge = header->backedge();
oldPreheader = header->loopPredecessor();
JS_ASSERT(oldPreheader->numSuccessors() == 1);
// Only unroll loops with two blocks: an initial one ending with the
// bound's test, and the body ending with the backedge.
MTest *test = bound->test;
if (header->lastIns() != test)
return;
if (test->ifTrue() == backedge) {
if (test->ifFalse()->id() <= backedge->id())
return;
} else if (test->ifFalse() == backedge) {
if (test->ifTrue()->id() <= backedge->id())
return;
} else {
return;
}
if (backedge->numPredecessors() != 1 || backedge->numSuccessors() != 1)
return;
JS_ASSERT(backedge->phisEmpty());
MBasicBlock *bodyBlocks[] = { header, backedge };
// All instructions in the header and body must be clonable.
for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) {
MBasicBlock *block = bodyBlocks[i];
for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) {
MInstruction *ins = *iter;
if (ins->canClone())
continue;
if (ins->isTest() || ins->isGoto() || ins->isInterruptCheck())
continue;
#ifdef DEBUG
JitSpew(JitSpew_Unrolling, "Aborting: can't clone instruction %s", ins->opName());
#endif
return;
}
}
// Compute the linear inequality we will use for exiting the unrolled loop:
//
// iterationBound - iterationCount - UnrollCount >= 0
//
LinearSum remainingIterationsInequality(bound->boundSum);
if (!remainingIterationsInequality.add(bound->currentSum, -1))
return;
if (!remainingIterationsInequality.add(-int32_t(UnrollCount)))
return;
// Terms in the inequality need to be either loop invariant or phis from
// the original header.
for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) {
MDefinition *def = remainingIterationsInequality.term(i).term;
if (def->block()->id() < header->id())
continue;
if (def->block() == header && def->isPhi())
continue;
return;
}
// OK, we've checked everything, now unroll the loop.
JitSpew(JitSpew_Unrolling, "Unrolling loop");
// The old preheader will go before the unrolled loop, and the old loop
// will need a new empty preheader.
CompileInfo &info = oldPreheader->info();
if (header->trackedSite().pc()) {
unrolledHeader =
MBasicBlock::New(graph, nullptr, info,
oldPreheader, header->trackedSite(), MBasicBlock::LOOP_HEADER);
unrolledBackedge =
MBasicBlock::New(graph, nullptr, info,
unrolledHeader, backedge->trackedSite(), MBasicBlock::NORMAL);
newPreheader =
MBasicBlock::New(graph, nullptr, info,
unrolledHeader, oldPreheader->trackedSite(), MBasicBlock::NORMAL);
} else {
unrolledHeader = MBasicBlock::NewAsmJS(graph, info, oldPreheader, MBasicBlock::LOOP_HEADER);
unrolledBackedge = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL);
newPreheader = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL);
}
unrolledHeader->discardAllResumePoints();
unrolledBackedge->discardAllResumePoints();
newPreheader->discardAllResumePoints();
// Insert new blocks at their RPO position, and update block ids.
graph.insertBlockAfter(oldPreheader, unrolledHeader);
graph.insertBlockAfter(unrolledHeader, unrolledBackedge);
graph.insertBlockAfter(unrolledBackedge, newPreheader);
graph.renumberBlocksAfter(oldPreheader);
if (!unrolledDefinitions.init())
CrashAtUnhandlableOOM("LoopUnroller::go");
// Add phis to the unrolled loop header which correspond to the phis in the
// original loop header.
JS_ASSERT(header->getPredecessor(0) == oldPreheader);
for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) {
MPhi *old = *iter;
JS_ASSERT(old->numOperands() == 2);
MPhi *phi = MPhi::New(alloc);
phi->setResultType(old->type());
phi->setResultTypeSet(old->resultTypeSet());
phi->setRange(old->range());
unrolledHeader->addPhi(phi);
if (!phi->reserveLength(2))
CrashAtUnhandlableOOM("LoopUnroller::go");
// Set the first input for the phi for now. We'll set the second after
// finishing the unroll.
phi->addInput(old->getOperand(0));
// The old phi will now take the value produced by the unrolled loop.
old->replaceOperand(0, phi);
if (!unrolledDefinitions.putNew(old, phi))
CrashAtUnhandlableOOM("LoopUnroller::go");
}
// The loop condition can bail out on e.g. integer overflow, so make a
// resume point based on the initial resume point of the original header.
MResumePoint *headerResumePoint = header->entryResumePoint();
if (headerResumePoint) {
MResumePoint *rp = makeReplacementResumePoint(unrolledHeader, headerResumePoint);
unrolledHeader->setEntryResumePoint(rp);
// Perform an interrupt check at the start of the unrolled loop.
unrolledHeader->add(MInterruptCheck::New(alloc));
}
// Generate code for the test in the unrolled loop.
for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) {
MDefinition *def = remainingIterationsInequality.term(i).term;
MDefinition *replacement = getReplacementDefinition(def);
remainingIterationsInequality.replaceTerm(i, replacement);
}
MCompare *compare = ConvertLinearInequality(alloc, unrolledHeader, remainingIterationsInequality);
MTest *unrolledTest = MTest::New(alloc, compare, unrolledBackedge, newPreheader);
unrolledHeader->end(unrolledTest);
// Make an entry resume point for the unrolled body. The unrolled header
// does not have side effects on stack values, even if the original loop
// header does, so use the same resume point as for the unrolled header.
if (headerResumePoint) {
MResumePoint *rp = makeReplacementResumePoint(unrolledBackedge, headerResumePoint);
unrolledBackedge->setEntryResumePoint(rp);
}
// Make an entry resume point for the new preheader. There are no
// instructions which use this but some other stuff wants one to be here.
if (headerResumePoint) {
MResumePoint *rp = makeReplacementResumePoint(newPreheader, headerResumePoint);
newPreheader->setEntryResumePoint(rp);
}
// Generate the unrolled code.
JS_ASSERT(UnrollCount > 1);
size_t unrollIndex = 0;
while (true) {
// Clone the contents of the original loop into the unrolled loop body.
for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) {
MBasicBlock *block = bodyBlocks[i];
for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) {
MInstruction *ins = *iter;
if (ins->canClone()) {
makeReplacementInstruction(*iter);
} else {
// Control instructions are handled separately.
JS_ASSERT(ins->isTest() || ins->isGoto() || ins->isInterruptCheck());
}
}
}
// Compute the value of each loop header phi after the execution of
// this unrolled iteration.
MDefinitionVector phiValues(alloc);
JS_ASSERT(header->getPredecessor(1) == backedge);
for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) {
MPhi *old = *iter;
MDefinition *oldInput = old->getOperand(1);
if (!phiValues.append(getReplacementDefinition(oldInput)))
CrashAtUnhandlableOOM("LoopUnroller::go");
}
unrolledDefinitions.clear();
if (unrollIndex == UnrollCount - 1) {
// We're at the end of the last unrolled iteration, set the
// backedge input for the unrolled loop phis.
size_t phiIndex = 0;
for (MPhiIterator iter(unrolledHeader->phisBegin()); iter != unrolledHeader->phisEnd(); iter++) {
MPhi *phi = *iter;
phi->addInput(phiValues[phiIndex++]);
}
JS_ASSERT(phiIndex == phiValues.length());
break;
}
// Update the map for the phis in the next iteration.
size_t phiIndex = 0;
for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) {
MPhi *old = *iter;
if (!unrolledDefinitions.putNew(old, phiValues[phiIndex++]))
CrashAtUnhandlableOOM("LoopUnroller::go");
}
JS_ASSERT(phiIndex == phiValues.length());
unrollIndex++;
}
MGoto *backedgeJump = MGoto::New(alloc, unrolledHeader);
unrolledBackedge->end(backedgeJump);
// Place the old preheader before the unrolled loop.
JS_ASSERT(oldPreheader->lastIns()->isGoto());
oldPreheader->discardLastIns();
oldPreheader->end(MGoto::New(alloc, unrolledHeader));
// Place the new preheader before the original loop.
newPreheader->end(MGoto::New(alloc, header));
// Cleanup the MIR graph.
if (!unrolledHeader->addPredecessorWithoutPhis(unrolledBackedge))
CrashAtUnhandlableOOM("LoopUnroller::go");
header->replacePredecessor(oldPreheader, newPreheader);
oldPreheader->setSuccessorWithPhis(unrolledHeader, 0);
newPreheader->setSuccessorWithPhis(header, 0);
unrolledBackedge->setSuccessorWithPhis(unrolledHeader, 1);
}
bool
Sink(MIRGenerator* mir, MIRGraph& graph)
{
TempAllocator& alloc = graph.alloc();
bool sinkEnabled = mir->optimizationInfo().sinkEnabled();
for (PostorderIterator block = graph.poBegin(); block != graph.poEnd(); block++) {
if (mir->shouldCancel("Sink"))
return false;
for (MInstructionReverseIterator iter = block->rbegin(); iter != block->rend(); ) {
MInstruction* ins = *iter++;
// Only instructions which can be recovered on bailout can be moved
// into the bailout paths.
if (ins->isGuard() || ins->isGuardRangeBailouts() ||
ins->isRecoveredOnBailout() || !ins->canRecoverOnBailout())
{
continue;
}
// Compute a common dominator for all uses of the current
// instruction.
bool hasLiveUses = false;
bool hasUses = false;
MBasicBlock* usesDominator = nullptr;
for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; i++) {
hasUses = true;
MNode* consumerNode = (*i)->consumer();
if (consumerNode->isResumePoint())
continue;
MDefinition* consumer = consumerNode->toDefinition();
if (consumer->isRecoveredOnBailout())
continue;
hasLiveUses = true;
// If the instruction is a Phi, then we should dominate the
// predecessor from which the value is coming from.
MBasicBlock* consumerBlock = consumer->block();
if (consumer->isPhi())
consumerBlock = consumerBlock->getPredecessor(consumer->indexOf(*i));
usesDominator = CommonDominator(usesDominator, consumerBlock);
if (usesDominator == *block)
break;
}
// Leave this instruction for DCE.
if (!hasUses)
continue;
// We have no uses, so sink this instruction in all the bailout
// paths.
if (!hasLiveUses) {
MOZ_ASSERT(!usesDominator);
ins->setRecoveredOnBailout();
JitSpewDef(JitSpew_Sink, " No live uses, recover the instruction on bailout\n", ins);
continue;
}
// This guard is temporarly moved here as the above code deals with
// Dead Code elimination, which got moved into this Sink phase, as
// the Dead Code elimination used to move instructions with no-live
// uses to the bailout path.
if (!sinkEnabled)
continue;
// To move an effectful instruction, we would have to verify that the
// side-effect is not observed. In the mean time, we just inhibit
// this optimization on effectful instructions.
if (ins->isEffectful())
continue;
// If all the uses are under a loop, we might not want to work
// against LICM by moving everything back into the loop, but if the
// loop is it-self inside an if, then we still want to move the
// computation under this if statement.
while (block->loopDepth() < usesDominator->loopDepth()) {
MOZ_ASSERT(usesDominator != usesDominator->immediateDominator());
usesDominator = usesDominator->immediateDominator();
}
// Only move instructions if there is a branch between the dominator
// of the uses and the original instruction. This prevent moving the
// computation of the arguments into an inline function if there is
// no major win.
MBasicBlock* lastJoin = usesDominator;
while (*block != lastJoin && lastJoin->numPredecessors() == 1) {
MOZ_ASSERT(lastJoin != lastJoin->immediateDominator());
MBasicBlock* next = lastJoin->immediateDominator();
if (next->numSuccessors() > 1)
break;
lastJoin = next;
}
if (*block == lastJoin)
continue;
// Skip to the next instruction if we cannot find a common dominator
// for all the uses of this instruction, or if the common dominator
// correspond to the block of the current instruction.
if (!usesDominator || usesDominator == *block)
continue;
// Only instruction which can be recovered on bailout and which are
// sinkable can be moved into blocks which are below while filling
// the resume points with a clone which is recovered on bailout.
// If the instruction has live uses and if it is clonable, then we
// can clone the instruction for all non-dominated uses and move the
// instruction into the block which is dominating all live uses.
if (!ins->canClone())
continue;
// If the block is a split-edge block, which is created for folding
// test conditions, then the block has no resume point and has
// multiple predecessors. In such case, we cannot safely move
// bailing instruction to these blocks as we have no way to bailout.
if (!usesDominator->entryResumePoint() && usesDominator->numPredecessors() != 1)
continue;
JitSpewDef(JitSpew_Sink, " Can Clone & Recover, sink instruction\n", ins);
JitSpew(JitSpew_Sink, " into Block %u", usesDominator->id());
// Copy the arguments and clone the instruction.
MDefinitionVector operands(alloc);
for (size_t i = 0, end = ins->numOperands(); i < end; i++) {
if (!operands.append(ins->getOperand(i)))
return false;
}
MInstruction* clone = ins->clone(alloc, operands);
ins->block()->insertBefore(ins, clone);
clone->setRecoveredOnBailout();
// We should not update the producer of the entry resume point, as
// it cannot refer to any instruction within the basic block excepts
// for Phi nodes.
MResumePoint* entry = usesDominator->entryResumePoint();
// Replace the instruction by its clone in all the resume points /
// recovered-on-bailout instructions which are not in blocks which
// are dominated by the usesDominator block.
for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; ) {
MUse* use = *i++;
MNode* consumer = use->consumer();
// If the consumer is a Phi, then we look for the index of the
// use to find the corresponding predecessor block, which is
// then used as the consumer block.
MBasicBlock* consumerBlock = consumer->block();
if (consumer->isDefinition() && consumer->toDefinition()->isPhi()) {
consumerBlock = consumerBlock->getPredecessor(
consumer->toDefinition()->toPhi()->indexOf(use));
}
// Keep the current instruction for all dominated uses, except
// for the entry resume point of the block in which the
// instruction would be moved into.
if (usesDominator->dominates(consumerBlock) &&
(!consumer->isResumePoint() || consumer->toResumePoint() != entry))
{
continue;
}
use->replaceProducer(clone);
}
// As we move this instruction in a different block, we should
// verify that we do not carry over a resume point which would refer
// to an outdated state of the control flow.
if (ins->resumePoint())
ins->clearResumePoint();
// Now, that all uses which are not dominated by usesDominator are
// using the cloned instruction, we can safely move the instruction
// into the usesDominator block.
MInstruction* at = usesDominator->safeInsertTop(nullptr, MBasicBlock::IgnoreRecover);
block->moveBefore(at, ins);
}
}
return true;
}
