// Discard |def| and mine its operands for any subsequently dead defs.
bool
ValueNumberer::discardDef(MDefinition* def)
{
#ifdef JS_JITSPEW
JitSpew(JitSpew_GVN, " Discarding %s %s%u",
def->block()->isMarked() ? "unreachable" : "dead",
def->opName(), def->id());
#endif
#ifdef DEBUG
MOZ_ASSERT(def != nextDef_, "Invalidating the MDefinition iterator");
if (def->block()->isMarked()) {
MOZ_ASSERT(!def->hasUses(), "Discarding def that still has uses");
} else {
MOZ_ASSERT(IsDiscardable(def), "Discarding non-discardable definition");
MOZ_ASSERT(!values_.has(def), "Discarding a definition still in the set");
}
#endif
MBasicBlock* block = def->block();
if (def->isPhi()) {
MPhi* phi = def->toPhi();
if (!releaseAndRemovePhiOperands(phi))
return false;
block->discardPhi(phi);
} else {
MInstruction* ins = def->toInstruction();
if (MResumePoint* resume = ins->resumePoint()) {
if (!releaseResumePointOperands(resume))
return false;
}
if (!releaseOperands(ins))
return false;
block->discardIgnoreOperands(ins);
}
// If that was the last definition in the block, it can be safely removed
// from the graph.
if (block->phisEmpty() && block->begin() == block->end()) {
MOZ_ASSERT(block->isMarked(), "Reachable block lacks at least a control instruction");
// As a special case, don't remove a block which is a dominator tree
// root so that we don't invalidate the iterator in visitGraph. We'll
// check for this and remove it later.
if (block->immediateDominator() != block) {
JitSpew(JitSpew_GVN, " Block block%u is now empty; discarding", block->id());
graph_.removeBlock(block);
blocksRemoved_ = true;
} else {
JitSpew(JitSpew_GVN, " Dominator root block%u is now empty; will discard later",
block->id());
}
}
return true;
}
bool
Sink(MIRGenerator* mir, MIRGraph& graph)
{
TempAllocator& alloc = graph.alloc();
bool sinkEnabled = mir->optimizationInfo().sinkEnabled();
for (PostorderIterator block = graph.poBegin(); block != graph.poEnd(); block++) {
if (mir->shouldCancel("Sink"))
return false;
for (MInstructionReverseIterator iter = block->rbegin(); iter != block->rend(); ) {
MInstruction* ins = *iter++;
// Only instructions which can be recovered on bailout can be moved
// into the bailout paths.
if (ins->isGuard() || ins->isGuardRangeBailouts() ||
ins->isRecoveredOnBailout() || !ins->canRecoverOnBailout())
{
continue;
}
// Compute a common dominator for all uses of the current
// instruction.
bool hasLiveUses = false;
bool hasUses = false;
MBasicBlock* usesDominator = nullptr;
for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; i++) {
hasUses = true;
MNode* consumerNode = (*i)->consumer();
if (consumerNode->isResumePoint())
continue;
MDefinition* consumer = consumerNode->toDefinition();
if (consumer->isRecoveredOnBailout())
continue;
hasLiveUses = true;
// If the instruction is a Phi, then we should dominate the
// predecessor from which the value is coming from.
MBasicBlock* consumerBlock = consumer->block();
if (consumer->isPhi())
consumerBlock = consumerBlock->getPredecessor(consumer->indexOf(*i));
usesDominator = CommonDominator(usesDominator, consumerBlock);
if (usesDominator == *block)
break;
}
// Leave this instruction for DCE.
if (!hasUses)
continue;
// We have no uses, so sink this instruction in all the bailout
// paths.
if (!hasLiveUses) {
MOZ_ASSERT(!usesDominator);
ins->setRecoveredOnBailout();
JitSpewDef(JitSpew_Sink, " No live uses, recover the instruction on bailout\n", ins);
continue;
}
// This guard is temporarly moved here as the above code deals with
// Dead Code elimination, which got moved into this Sink phase, as
// the Dead Code elimination used to move instructions with no-live
// uses to the bailout path.
if (!sinkEnabled)
continue;
// To move an effectful instruction, we would have to verify that the
// side-effect is not observed. In the mean time, we just inhibit
// this optimization on effectful instructions.
if (ins->isEffectful())
continue;
// If all the uses are under a loop, we might not want to work
// against LICM by moving everything back into the loop, but if the
// loop is it-self inside an if, then we still want to move the
// computation under this if statement.
while (block->loopDepth() < usesDominator->loopDepth()) {
MOZ_ASSERT(usesDominator != usesDominator->immediateDominator());
usesDominator = usesDominator->immediateDominator();
}
// Only move instructions if there is a branch between the dominator
// of the uses and the original instruction. This prevent moving the
// computation of the arguments into an inline function if there is
// no major win.
MBasicBlock* lastJoin = usesDominator;
while (*block != lastJoin && lastJoin->numPredecessors() == 1) {
MOZ_ASSERT(lastJoin != lastJoin->immediateDominator());
MBasicBlock* next = lastJoin->immediateDominator();
if (next->numSuccessors() > 1)
break;
lastJoin = next;
}
if (*block == lastJoin)
continue;
// Skip to the next instruction if we cannot find a common dominator
// for all the uses of this instruction, or if the common dominator
// correspond to the block of the current instruction.
if (!usesDominator || usesDominator == *block)
continue;
// Only instruction which can be recovered on bailout and which are
// sinkable can be moved into blocks which are below while filling
// the resume points with a clone which is recovered on bailout.
// If the instruction has live uses and if it is clonable, then we
// can clone the instruction for all non-dominated uses and move the
// instruction into the block which is dominating all live uses.
if (!ins->canClone())
continue;
// If the block is a split-edge block, which is created for folding
// test conditions, then the block has no resume point and has
// multiple predecessors. In such case, we cannot safely move
// bailing instruction to these blocks as we have no way to bailout.
if (!usesDominator->entryResumePoint() && usesDominator->numPredecessors() != 1)
continue;
JitSpewDef(JitSpew_Sink, " Can Clone & Recover, sink instruction\n", ins);
JitSpew(JitSpew_Sink, " into Block %u", usesDominator->id());
// Copy the arguments and clone the instruction.
MDefinitionVector operands(alloc);
for (size_t i = 0, end = ins->numOperands(); i < end; i++) {
if (!operands.append(ins->getOperand(i)))
return false;
}
MInstruction* clone = ins->clone(alloc, operands);
ins->block()->insertBefore(ins, clone);
clone->setRecoveredOnBailout();
// We should not update the producer of the entry resume point, as
// it cannot refer to any instruction within the basic block excepts
// for Phi nodes.
MResumePoint* entry = usesDominator->entryResumePoint();
// Replace the instruction by its clone in all the resume points /
// recovered-on-bailout instructions which are not in blocks which
// are dominated by the usesDominator block.
for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; ) {
MUse* use = *i++;
MNode* consumer = use->consumer();
// If the consumer is a Phi, then we look for the index of the
// use to find the corresponding predecessor block, which is
// then used as the consumer block.
MBasicBlock* consumerBlock = consumer->block();
if (consumer->isDefinition() && consumer->toDefinition()->isPhi()) {
consumerBlock = consumerBlock->getPredecessor(
consumer->toDefinition()->toPhi()->indexOf(use));
}
// Keep the current instruction for all dominated uses, except
// for the entry resume point of the block in which the
// instruction would be moved into.
if (usesDominator->dominates(consumerBlock) &&
(!consumer->isResumePoint() || consumer->toResumePoint() != entry))
{
continue;
}
use->replaceProducer(clone);
}
// As we move this instruction in a different block, we should
// verify that we do not carry over a resume point which would refer
// to an outdated state of the control flow.
if (ins->resumePoint())
ins->clearResumePoint();
// Now, that all uses which are not dominated by usesDominator are
// using the cloned instruction, we can safely move the instruction
// into the usesDominator block.
MInstruction* at = usesDominator->safeInsertTop(nullptr, MBasicBlock::IgnoreRecover);
block->moveBefore(at, ins);
}
}
return true;
}
// Remove the CFG edge between |pred| and |block|, and if this makes |block|
// unreachable, mark it so, and remove the rest of its incoming edges too. And
// discard any instructions made dead by the entailed release of any phi
// operands.
bool
ValueNumberer::removePredecessorAndCleanUp(MBasicBlock* block, MBasicBlock* pred)
{
MOZ_ASSERT(!block->isMarked(), "Removing predecessor on block already marked unreachable");
// We'll be removing a predecessor, so anything we know about phis in this
// block will be wrong.
for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd()); iter != end; ++iter)
values_.forget(*iter);
// If this is a loop header, test whether it will become an unreachable
// loop, or whether it needs special OSR-related fixups.
bool isUnreachableLoop = false;
MBasicBlock* origBackedgeForOSRFixup = nullptr;
if (block->isLoopHeader()) {
if (block->loopPredecessor() == pred) {
if (MOZ_UNLIKELY(hasNonDominatingPredecessor(block, pred))) {
JitSpew(JitSpew_GVN, " "
"Loop with header block%u is now only reachable through an "
"OSR entry into the middle of the loop!!", block->id());
origBackedgeForOSRFixup = block->backedge();
} else {
// Deleting the entry into the loop makes the loop unreachable.
isUnreachableLoop = true;
JitSpew(JitSpew_GVN, " "
"Loop with header block%u is no longer reachable",
block->id());
}
#ifdef DEBUG
} else if (block->hasUniqueBackedge() && block->backedge() == pred) {
JitSpew(JitSpew_GVN, " Loop with header block%u is no longer a loop",
block->id());
#endif
}
}
// Actually remove the CFG edge.
if (!removePredecessorAndDoDCE(block, pred, block->getPredecessorIndex(pred)))
return false;
// We've now edited the CFG; check to see if |block| became unreachable.
if (block->numPredecessors() == 0 || isUnreachableLoop) {
JitSpew(JitSpew_GVN, " Disconnecting block%u", block->id());
// Remove |block| from its dominator parent's subtree. This is the only
// immediately-dominated-block information we need to update, because
// everything dominated by this block is about to be swept away.
MBasicBlock* parent = block->immediateDominator();
if (parent != block)
parent->removeImmediatelyDominatedBlock(block);
// Completely disconnect it from the CFG. We do this now rather than
// just doing it later when we arrive there in visitUnreachableBlock
// so that we don't leave a partially broken loop sitting around. This
// also lets visitUnreachableBlock assert that numPredecessors() == 0,
// which is a nice invariant.
if (block->isLoopHeader())
block->clearLoopHeader();
for (size_t i = 0, e = block->numPredecessors(); i < e; ++i) {
if (!removePredecessorAndDoDCE(block, block->getPredecessor(i), i))
return false;
}
// Clear out the resume point operands, as they can hold things that
// don't appear to dominate them live.
if (MResumePoint* resume = block->entryResumePoint()) {
if (!releaseResumePointOperands(resume) || !processDeadDefs())
return false;
if (MResumePoint* outer = block->outerResumePoint()) {
if (!releaseResumePointOperands(outer) || !processDeadDefs())
return false;
}
MOZ_ASSERT(nextDef_ == nullptr);
for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ) {
MInstruction* ins = *iter++;
nextDef_ = *iter;
if (MResumePoint* resume = ins->resumePoint()) {
if (!releaseResumePointOperands(resume) || !processDeadDefs())
return false;
}
}
nextDef_ = nullptr;
} else {
#ifdef DEBUG
MOZ_ASSERT(block->outerResumePoint() == nullptr,
"Outer resume point in block without an entry resume point");
for (MInstructionIterator iter(block->begin()), end(block->end());
iter != end;
++iter)
{
MOZ_ASSERT(iter->resumePoint() == nullptr,
"Instruction with resume point in block without entry resume point");
}
#endif
}
// Use the mark to note that we've already removed all its predecessors,
// and we know it's unreachable.
block->mark();
} else if (MOZ_UNLIKELY(origBackedgeForOSRFixup != nullptr)) {
// The loop is no only reachable through OSR into the middle. Fix it
// up so that the CFG can remain valid.
if (!fixupOSROnlyLoop(block, origBackedgeForOSRFixup))
return false;
}
return true;
}
bool
jit::ReorderInstructions(MIRGraph& graph)
{
// Renumber all instructions in the graph as we go.
size_t nextId = 0;
// List of the headers of any loops we are in.
Vector<MBasicBlock*, 4, SystemAllocPolicy> loopHeaders;
for (ReversePostorderIterator block(graph.rpoBegin()); block != graph.rpoEnd(); block++) {
// Renumber all definitions inside the basic blocks.
for (MPhiIterator iter(block->phisBegin()); iter != block->phisEnd(); iter++)
iter->setId(nextId++);
for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++)
iter->setId(nextId++);
// Don't reorder instructions within entry blocks, which have special requirements.
if (*block == graph.entryBlock() || *block == graph.osrBlock())
continue;
if (block->isLoopHeader()) {
if (!loopHeaders.append(*block))
return false;
}
MBasicBlock* innerLoop = loopHeaders.empty() ? nullptr : loopHeaders.back();
MInstruction* top = block->safeInsertTop();
MInstructionReverseIterator rtop = ++block->rbegin(top);
for (MInstructionIterator iter(block->begin(top)); iter != block->end(); ) {
MInstruction* ins = *iter;
// Filter out some instructions which are never reordered.
if (ins->isEffectful() ||
!ins->isMovable() ||
ins->resumePoint() ||
ins == block->lastIns())
{
iter++;
continue;
}
// Move constants with a single use in the current block to the
// start of the block. Constants won't be reordered by the logic
// below, as they have no inputs. Moving them up as high as
// possible can allow their use to be moved up further, though,
// and has no cost if the constant is emitted at its use.
if (ins->isConstant() &&
ins->hasOneUse() &&
ins->usesBegin()->consumer()->block() == *block &&
!IsFloatingPointType(ins->type()))
{
iter++;
MInstructionIterator targetIter = block->begin();
while (targetIter->isConstant() || targetIter->isInterruptCheck()) {
if (*targetIter == ins)
break;
targetIter++;
}
MoveBefore(*block, *targetIter, ins);
continue;
}
// Look for inputs where this instruction is the last use of that
// input. If we move this instruction up, the input's lifetime will
// be shortened, modulo resume point uses (which don't need to be
// stored in a register, and can be handled by the register
// allocator by just spilling at some point with no reload).
Vector<MDefinition*, 4, SystemAllocPolicy> lastUsedInputs;
for (size_t i = 0; i < ins->numOperands(); i++) {
MDefinition* input = ins->getOperand(i);
if (!input->isConstant() && IsLastUse(ins, input, innerLoop)) {
if (!lastUsedInputs.append(input))
return false;
}
}
// Don't try to move instructions which aren't the last use of any
// of their inputs (we really ought to move these down instead).
if (lastUsedInputs.length() < 2) {
iter++;
continue;
}
MInstruction* target = ins;
for (MInstructionReverseIterator riter = ++block->rbegin(ins); riter != rtop; riter++) {
MInstruction* prev = *riter;
if (prev->isInterruptCheck())
break;
// The instruction can't be moved before any of its uses.
bool isUse = false;
for (size_t i = 0; i < ins->numOperands(); i++) {
if (ins->getOperand(i) == prev) {
isUse = true;
break;
}
}
if (isUse)
break;
// The instruction can't be moved before an instruction that
// stores to a location read by the instruction.
if (prev->isEffectful() &&
(ins->getAliasSet().flags() & prev->getAliasSet().flags()) &&
ins->mightAlias(prev) != MDefinition::AliasType::NoAlias)
{
break;
}
// Make sure the instruction will still be the last use of one
// of its inputs when moved up this far.
for (size_t i = 0; i < lastUsedInputs.length(); ) {
bool found = false;
for (size_t j = 0; j < prev->numOperands(); j++) {
if (prev->getOperand(j) == lastUsedInputs[i]) {
found = true;
break;
}
}
if (found) {
lastUsedInputs[i] = lastUsedInputs.back();
lastUsedInputs.popBack();
} else {
i++;
}
}
if (lastUsedInputs.length() < 2)
break;
// We can move the instruction before this one.
target = prev;
}
iter++;
MoveBefore(*block, target, ins);
}
if (block->isLoopBackedge())
loopHeaders.popBack();
}
return true;
}
