// If an equivalent and dominating value already exists in the set, return it. // Otherwise insert |def| into the set and return it. MDefinition* ValueNumberer::leader(MDefinition* def) { // If the value isn't suitable for eliminating, don't bother hashing it. The // convention is that congruentTo returns false for node kinds that wish to // opt out of redundance elimination. // TODO: It'd be nice to clean up that convention (bug 1031406). if (!def->isEffectful() && def->congruentTo(def)) { // Look for a match. VisibleValues::AddPtr p = values_.findLeaderForAdd(def); if (p) { MDefinition* rep = *p; if (!rep->isDiscarded() && rep->block()->dominates(def->block())) { // We found a dominating congruent value. return rep; } // The congruent value doesn't dominate. It never will again in this // dominator tree, so overwrite it. values_.overwrite(p, def); } else { // No match. Add a new entry. if (!values_.add(p, def)) return nullptr; } #ifdef DEBUG JitSpew(JitSpew_GVN, " Recording %s%u", def->opName(), def->id()); #endif } return def; }
// Assuming |resume| is unreachable, release its operands. // It might be nice to integrate this code with prepareForDiscard, however GVN // needs it to call handleUseReleased so that it can observe when a definition // becomes unused, so it isn't trivial to do. bool ValueNumberer::releaseResumePointOperands(MResumePoint *resume) { for (size_t i = 0, e = resume->numOperands(); i < e; ++i) { if (!resume->hasOperand(i)) continue; MDefinition *op = resume->getOperand(i); // TODO: We shouldn't leave discarded operands sitting around // (bug 1055690). if (op->isDiscarded()) continue; resume->releaseOperand(i); // We set the UseRemoved flag when removing resume point operands, // because even though we may think we're certain that a particular // branch might not be taken, the type information might be incomplete. if (!handleUseReleased(op, SetUseRemoved)) return false; } return true; }
void LoopUnroller::go(LoopIterationBound* bound) { // For now we always unroll loops the same number of times. static const size_t UnrollCount = 10; JitSpew(JitSpew_Unrolling, "Attempting to unroll loop"); header = bound->header; // UCE might have determined this isn't actually a loop. if (!header->isLoopHeader()) return; backedge = header->backedge(); oldPreheader = header->loopPredecessor(); MOZ_ASSERT(oldPreheader->numSuccessors() == 1); // Only unroll loops with two blocks: an initial one ending with the // bound's test, and the body ending with the backedge. MTest* test = bound->test; if (header->lastIns() != test) return; if (test->ifTrue() == backedge) { if (test->ifFalse()->id() <= backedge->id()) return; } else if (test->ifFalse() == backedge) { if (test->ifTrue()->id() <= backedge->id()) return; } else { return; } if (backedge->numPredecessors() != 1 || backedge->numSuccessors() != 1) return; MOZ_ASSERT(backedge->phisEmpty()); MBasicBlock* bodyBlocks[] = { header, backedge }; // All instructions in the header and body must be clonable. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock* block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction* ins = *iter; if (ins->canClone()) continue; if (ins->isTest() || ins->isGoto() || ins->isInterruptCheck()) continue; #ifdef DEBUG JitSpew(JitSpew_Unrolling, "Aborting: can't clone instruction %s", ins->opName()); #endif return; } } // Compute the linear inequality we will use for exiting the unrolled loop: // // iterationBound - iterationCount - UnrollCount >= 0 // LinearSum remainingIterationsInequality(bound->boundSum); if (!remainingIterationsInequality.add(bound->currentSum, -1)) return; if (!remainingIterationsInequality.add(-int32_t(UnrollCount))) return; // Terms in the inequality need to be either loop invariant or phis from // the original header. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition* def = remainingIterationsInequality.term(i).term; if (def->isDiscarded()) return; if (def->block()->id() < header->id()) continue; if (def->block() == header && def->isPhi()) continue; return; } // OK, we've checked everything, now unroll the loop. JitSpew(JitSpew_Unrolling, "Unrolling loop"); // The old preheader will go before the unrolled loop, and the old loop // will need a new empty preheader. CompileInfo& info = oldPreheader->info(); if (header->trackedPc()) { unrolledHeader = MBasicBlock::New(graph, nullptr, info, oldPreheader, header->trackedSite(), MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::New(graph, nullptr, info, unrolledHeader, backedge->trackedSite(), MBasicBlock::NORMAL); newPreheader = MBasicBlock::New(graph, nullptr, info, unrolledHeader, oldPreheader->trackedSite(), MBasicBlock::NORMAL); } else { unrolledHeader = MBasicBlock::NewAsmJS(graph, info, oldPreheader, MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); newPreheader = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); } unrolledHeader->discardAllResumePoints(); unrolledBackedge->discardAllResumePoints(); newPreheader->discardAllResumePoints(); // Insert new blocks at their RPO position, and update block ids. graph.insertBlockAfter(oldPreheader, unrolledHeader); graph.insertBlockAfter(unrolledHeader, unrolledBackedge); graph.insertBlockAfter(unrolledBackedge, newPreheader); graph.renumberBlocksAfter(oldPreheader); if (!unrolledDefinitions.init()) CrashAtUnhandlableOOM("LoopUnroller::go"); // Add phis to the unrolled loop header which correspond to the phis in the // original loop header. MOZ_ASSERT(header->getPredecessor(0) == oldPreheader); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi* old = *iter; MOZ_ASSERT(old->numOperands() == 2); MPhi* phi = MPhi::New(alloc); phi->setResultType(old->type()); phi->setResultTypeSet(old->resultTypeSet()); phi->setRange(old->range()); unrolledHeader->addPhi(phi); if (!phi->reserveLength(2)) CrashAtUnhandlableOOM("LoopUnroller::go"); // Set the first input for the phi for now. We'll set the second after // finishing the unroll. phi->addInput(old->getOperand(0)); // The old phi will now take the value produced by the unrolled loop. old->replaceOperand(0, phi); if (!unrolledDefinitions.putNew(old, phi)) CrashAtUnhandlableOOM("LoopUnroller::go"); } // The loop condition can bail out on e.g. integer overflow, so make a // resume point based on the initial resume point of the original header. MResumePoint* headerResumePoint = header->entryResumePoint(); if (headerResumePoint) { MResumePoint* rp = makeReplacementResumePoint(unrolledHeader, headerResumePoint); unrolledHeader->setEntryResumePoint(rp); // Perform an interrupt check at the start of the unrolled loop. unrolledHeader->add(MInterruptCheck::New(alloc)); } // Generate code for the test in the unrolled loop. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition* def = remainingIterationsInequality.term(i).term; MDefinition* replacement = getReplacementDefinition(def); remainingIterationsInequality.replaceTerm(i, replacement); } MCompare* compare = ConvertLinearInequality(alloc, unrolledHeader, remainingIterationsInequality); MTest* unrolledTest = MTest::New(alloc, compare, unrolledBackedge, newPreheader); unrolledHeader->end(unrolledTest); // Make an entry resume point for the unrolled body. The unrolled header // does not have side effects on stack values, even if the original loop // header does, so use the same resume point as for the unrolled header. if (headerResumePoint) { MResumePoint* rp = makeReplacementResumePoint(unrolledBackedge, headerResumePoint); unrolledBackedge->setEntryResumePoint(rp); } // Make an entry resume point for the new preheader. There are no // instructions which use this but some other stuff wants one to be here. if (headerResumePoint) { MResumePoint* rp = makeReplacementResumePoint(newPreheader, headerResumePoint); newPreheader->setEntryResumePoint(rp); } // Generate the unrolled code. MOZ_ASSERT(UnrollCount > 1); size_t unrollIndex = 0; while (true) { // Clone the contents of the original loop into the unrolled loop body. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock* block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction* ins = *iter; if (ins->canClone()) { makeReplacementInstruction(*iter); } else { // Control instructions are handled separately. MOZ_ASSERT(ins->isTest() || ins->isGoto() || ins->isInterruptCheck()); } } } // Compute the value of each loop header phi after the execution of // this unrolled iteration. MDefinitionVector phiValues(alloc); MOZ_ASSERT(header->getPredecessor(1) == backedge); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi* old = *iter; MDefinition* oldInput = old->getOperand(1); if (!phiValues.append(getReplacementDefinition(oldInput))) CrashAtUnhandlableOOM("LoopUnroller::go"); } unrolledDefinitions.clear(); if (unrollIndex == UnrollCount - 1) { // We're at the end of the last unrolled iteration, set the // backedge input for the unrolled loop phis. size_t phiIndex = 0; for (MPhiIterator iter(unrolledHeader->phisBegin()); iter != unrolledHeader->phisEnd(); iter++) { MPhi* phi = *iter; phi->addInput(phiValues[phiIndex++]); } MOZ_ASSERT(phiIndex == phiValues.length()); break; } // Update the map for the phis in the next iteration. size_t phiIndex = 0; for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi* old = *iter; if (!unrolledDefinitions.putNew(old, phiValues[phiIndex++])) CrashAtUnhandlableOOM("LoopUnroller::go"); } MOZ_ASSERT(phiIndex == phiValues.length()); unrollIndex++; } MGoto* backedgeJump = MGoto::New(alloc, unrolledHeader); unrolledBackedge->end(backedgeJump); // Place the old preheader before the unrolled loop. MOZ_ASSERT(oldPreheader->lastIns()->isGoto()); oldPreheader->discardLastIns(); oldPreheader->end(MGoto::New(alloc, unrolledHeader)); // Place the new preheader before the original loop. newPreheader->end(MGoto::New(alloc, header)); // Cleanup the MIR graph. if (!unrolledHeader->addPredecessorWithoutPhis(unrolledBackedge)) CrashAtUnhandlableOOM("LoopUnroller::go"); header->replacePredecessor(oldPreheader, newPreheader); oldPreheader->setSuccessorWithPhis(unrolledHeader, 0); newPreheader->setSuccessorWithPhis(header, 0); unrolledBackedge->setSuccessorWithPhis(unrolledHeader, 1); }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition* def) { // Nop does not fit in any of the previous optimization, as its only purpose // is to reduce the register pressure by keeping additional resume // point. Still, there is no need consecutive list of MNop instructions, and // this will slow down every other iteration on the Graph. if (def->isNop()) { MNop* nop = def->toNop(); MBasicBlock* block = nop->block(); // We look backward to know if we can remove the previous Nop, we do not // look forward as we would not benefit from the folding made by GVN. MInstructionReverseIterator iter = ++block->rbegin(nop); // This nop is at the beginning of the basic block, just replace the // resume point of the basic block by the one from the resume point. if (iter == block->rend()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); nop->moveResumePointAsEntry(); block->discard(nop); return true; } // The previous instruction is also a Nop, no need to keep it anymore. MInstruction* prev = *iter; if (prev->isNop()) { JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id()); block->discard(prev); return true; } return true; } // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MInstruction* dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition* sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; // If |sim| doesn't belong to a block, insert it next to |def|. if (sim->block() == nullptr) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef DEBUG JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif MOZ_ASSERT(!sim->isDiscarded()); ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; // If that ended up discarding |sim|, then we're done here. if (sim->isDiscarded()) return true; } // Otherwise, procede to optimize with |sim| in place of |def|. def = sim; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition* rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef DEBUG JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition *def) { // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MDefinition *dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition *sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; // If |sim| doesn't belong to a block, insert it next to |def|. if (sim->block() == nullptr) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef DEBUG JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; } def = sim; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition *rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef DEBUG JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition* def) { // Nop does not fit in any of the previous optimization, as its only purpose // is to reduce the register pressure by keeping additional resume // point. Still, there is no need consecutive list of MNop instructions, and // this will slow down every other iteration on the Graph. if (def->isNop()) { MNop* nop = def->toNop(); MBasicBlock* block = nop->block(); // We look backward to know if we can remove the previous Nop, we do not // look forward as we would not benefit from the folding made by GVN. MInstructionReverseIterator iter = ++block->rbegin(nop); // This nop is at the beginning of the basic block, just replace the // resume point of the basic block by the one from the resume point. if (iter == block->rend()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); nop->moveResumePointAsEntry(); block->discard(nop); return true; } // The previous instruction is also a Nop, no need to keep it anymore. MInstruction* prev = *iter; if (prev->isNop()) { JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id()); block->discard(prev); return true; } // The Nop is introduced to capture the result and make sure the operands // are not live anymore when there are no further uses. Though when // all operands are still needed the Nop doesn't decrease the liveness // and can get removed. MResumePoint* rp = nop->resumePoint(); if (rp && rp->numOperands() > 0 && rp->getOperand(rp->numOperands() - 1) == prev && !nop->block()->lastIns()->isThrow() && !prev->isAssertRecoveredOnBailout()) { size_t numOperandsLive = 0; for (size_t j = 0; j < prev->numOperands(); j++) { for (size_t i = 0; i < rp->numOperands(); i++) { if (prev->getOperand(j) == rp->getOperand(i)) { numOperandsLive++; break; } } } if (numOperandsLive == prev->numOperands()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); block->discard(nop); } } return true; } // Skip optimizations on instructions which are recovered on bailout, to // avoid mixing instructions which are recovered on bailouts with // instructions which are not. if (def->isRecoveredOnBailout()) return true; // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MDefinition* dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition* sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; bool isNewInstruction = sim->block() == nullptr; // If |sim| doesn't belong to a block, insert it next to |def|. if (isNewInstruction) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef JS_JITSPEW JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif MOZ_ASSERT(!sim->isDiscarded()); ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (def->isGuardRangeBailouts()) sim->setGuardRangeBailoutsUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; // If that ended up discarding |sim|, then we're done here. if (sim->isDiscarded()) return true; } if (!rerun_ && def->isPhi() && !sim->isPhi()) { rerun_ = true; JitSpew(JitSpew_GVN, " Replacing phi%u may have enabled cascading optimisations; " "will re-run", def->id()); } // Otherwise, procede to optimize with |sim| in place of |def|. def = sim; // If the simplified instruction was already part of the graph, then we // probably already visited and optimized this instruction. if (!isNewInstruction) return true; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition* rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef JS_JITSPEW JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }