static void VisitLoopBlock(MBasicBlock *block, MBasicBlock *header, MInstruction *hoistPoint, bool hasCalls) { for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ) { MInstruction *ins = *insIter++; if (!IsHoistable(ins, header, hasCalls)) { #ifdef DEBUG if (IsHoistableIgnoringDependency(ins, hasCalls)) { JitSpew(JitSpew_LICM, " %s%u isn't hoistable due to dependency on %s%u", ins->opName(), ins->id(), ins->dependency()->opName(), ins->dependency()->id()); } #endif continue; } // Don't hoist a cheap constant if it doesn't enable us to hoist one of // its uses. We want those instructions as close as possible to their // use, to minimize register pressure. if (RequiresHoistedUse(ins, hasCalls)) { JitSpew(JitSpew_LICM, " %s%u will be hoisted only if its users are", ins->opName(), ins->id()); continue; } // Hoist operands which were too cheap to hoist on their own. MoveDeferredOperands(ins, hoistPoint, hasCalls); JitSpew(JitSpew_LICM, " Hoisting %s%u", ins->opName(), ins->id()); // Move the instruction to the hoistPoint. block->moveBefore(hoistPoint, ins); } }
static void VisitLoop(MIRGraph &graph, MBasicBlock *header) { MInstruction *hoistPoint = header->loopPredecessor()->lastIns(); JitSpew(JitSpew_LICM, " Visiting loop with header block%u, hoisting to %s%u", header->id(), hoistPoint->opName(), hoistPoint->id()); MBasicBlock *backedge = header->backedge(); // This indicates whether the loop contains calls or other things which // clobber most or all floating-point registers. In such loops, // floating-point constants should not be hoisted unless it enables further // hoisting. bool hasCalls = LoopContainsPossibleCall(graph, header, backedge); for (auto i(graph.rpoBegin(header)); ; ++i) { MOZ_ASSERT(i != graph.rpoEnd(), "Reached end of graph searching for blocks in loop"); MBasicBlock *block = *i; if (!block->isMarked()) continue; VisitLoopBlock(block, header, hoistPoint, hasCalls); if (block == backedge) break; } }
// In preparation for hoisting an instruction, hoist any of its operands which // were too cheap to hoist on their own. static void MoveDeferredOperands(MInstruction* ins, MInstruction* hoistPoint, bool hasCalls) { // If any of our operands were waiting for a user to be hoisted, make a note // to hoist them. for (size_t i = 0, e = ins->numOperands(); i != e; ++i) { MDefinition* op = ins->getOperand(i); if (!IsInLoop(op)) continue; MOZ_ASSERT(RequiresHoistedUse(op, hasCalls), "Deferred loop-invariant operand is not cheap"); MInstruction* opIns = op->toInstruction(); // Recursively move the operands. Note that the recursion is bounded // because we require RequiresHoistedUse to be set at each level. MoveDeferredOperands(opIns, hoistPoint, hasCalls); #ifdef DEBUG JitSpew(JitSpew_LICM, " Hoisting %s%u (now that a user will be hoisted)", opIns->opName(), opIns->id()); #endif opIns->block()->moveBefore(hoistPoint, opIns); } }
// Fold AddIs with one variable and two or more constants into one AddI. static void AnalyzeAdd(TempAllocator& alloc, MAdd* add) { if (add->specialization() != MIRType::Int32 || add->isRecoveredOnBailout()) { return; } if (!add->hasUses()) { return; } JitSpew(JitSpew_FLAC, "analyze add: %s%u", add->opName(), add->id()); SimpleLinearSum sum = ExtractLinearSum(add); if (sum.constant == 0 || !sum.term) { return; } // Determine which operand is the constant. int idx = add->getOperand(0)->isConstant() ? 0 : 1; if (add->getOperand(idx)->isConstant()) { // Do not replace an add where the outcome is the same add instruction. MOZ_ASSERT(add->getOperand(idx)->toConstant()->type() == MIRType::Int32); if (sum.term == add->getOperand(1 - idx) || sum.constant == add->getOperand(idx)->toConstant()->toInt32()) { return; } } MInstruction* rhs = MConstant::New(alloc, Int32Value(sum.constant)); add->block()->insertBefore(add, rhs); MAdd* addNew = MAdd::New(alloc, sum.term, rhs, MIRType::Int32, add->truncateKind()); add->replaceAllLiveUsesWith(addNew); add->block()->insertBefore(add, addNew); JitSpew(JitSpew_FLAC, "replaced with: %s%u", addNew->opName(), addNew->id()); JitSpew(JitSpew_FLAC, "and constant: %s%u (%d)", rhs->opName(), rhs->id(), sum.constant); // Mark the stale nodes as RecoveredOnBailout since the Sink pass has // been run before this pass. DCE will then remove the unused nodes. markNodesAsRecoveredOnBailout(add); }
// This pass annotates every load instruction with the last store instruction // on which it depends. The algorithm is optimistic in that it ignores explicit // dependencies and only considers loads and stores. // // Loads inside loops only have an implicit dependency on a store before the // loop header if no instruction inside the loop body aliases it. To calculate // this efficiently, we maintain a list of maybe-invariant loads and the combined // alias set for all stores inside the loop. When we see the loop's backedge, this // information is used to mark every load we wrongly assumed to be loop invariant as // having an implicit dependency on the last instruction of the loop header, so that // it's never moved before the loop header. // // The algorithm depends on the invariant that both control instructions and effectful // instructions (stores) are never hoisted. bool AliasAnalysis::analyze() { Vector<MInstructionVector, AliasSet::NumCategories, JitAllocPolicy> stores(alloc()); // Initialize to the first instruction. MInstruction* firstIns = *graph_.entryBlock()->begin(); for (unsigned i = 0; i < AliasSet::NumCategories; i++) { MInstructionVector defs(alloc()); if (!defs.append(firstIns)) return false; if (!stores.append(Move(defs))) return false; } // Type analysis may have inserted new instructions. Since this pass depends // on the instruction number ordering, all instructions are renumbered. uint32_t newId = 0; for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { if (mir->shouldCancel("Alias Analysis (main loop)")) return false; if (block->isLoopHeader()) { JitSpew(JitSpew_Alias, "Processing loop header %d", block->id()); loop_ = new(alloc()) LoopAliasInfo(alloc(), loop_, *block); } for (MPhiIterator def(block->phisBegin()), end(block->phisEnd()); def != end; ++def) def->setId(newId++); for (MInstructionIterator def(block->begin()), end(block->begin(block->lastIns())); def != end; ++def) { def->setId(newId++); AliasSet set = def->getAliasSet(); if (set.isNone()) continue; // For the purposes of alias analysis, all recoverable operations // are treated as effect free as the memory represented by these // operations cannot be aliased by others. if (def->canRecoverOnBailout()) continue; if (set.isStore()) { for (AliasSetIterator iter(set); iter; iter++) { if (!stores[*iter].append(*def)) return false; } if (JitSpewEnabled(JitSpew_Alias)) { Fprinter& out = JitSpewPrinter(); out.printf("Processing store "); def->printName(out); out.printf(" (flags %x)\n", set.flags()); } } else { // Find the most recent store on which this instruction depends. MInstruction* lastStore = firstIns; for (AliasSetIterator iter(set); iter; iter++) { MInstructionVector& aliasedStores = stores[*iter]; for (int i = aliasedStores.length() - 1; i >= 0; i--) { MInstruction* store = aliasedStores[i]; if (genericMightAlias(*def, store) != MDefinition::AliasType::NoAlias && def->mightAlias(store) != MDefinition::AliasType::NoAlias && BlockMightReach(store->block(), *block)) { if (lastStore->id() < store->id()) lastStore = store; break; } } } def->setDependency(lastStore); IonSpewDependency(*def, lastStore, "depends", ""); // If the last store was before the current loop, we assume this load // is loop invariant. If a later instruction writes to the same location, // we will fix this at the end of the loop. if (loop_ && lastStore->id() < loop_->firstInstruction()->id()) { if (!loop_->addInvariantLoad(*def)) return false; } } } // Renumber the last instruction, as the analysis depends on this and the order. block->lastIns()->setId(newId++); if (block->isLoopBackedge()) { MOZ_ASSERT(loop_->loopHeader() == block->loopHeaderOfBackedge()); JitSpew(JitSpew_Alias, "Processing loop backedge %d (header %d)", block->id(), loop_->loopHeader()->id()); LoopAliasInfo* outerLoop = loop_->outer(); MInstruction* firstLoopIns = *loop_->loopHeader()->begin(); const MInstructionVector& invariant = loop_->invariantLoads(); for (unsigned i = 0; i < invariant.length(); i++) { MInstruction* ins = invariant[i]; AliasSet set = ins->getAliasSet(); MOZ_ASSERT(set.isLoad()); bool hasAlias = false; for (AliasSetIterator iter(set); iter; iter++) { MInstructionVector& aliasedStores = stores[*iter]; for (int i = aliasedStores.length() - 1;; i--) { MInstruction* store = aliasedStores[i]; if (store->id() < firstLoopIns->id()) break; if (genericMightAlias(ins, store) != MDefinition::AliasType::NoAlias && ins->mightAlias(store) != MDefinition::AliasType::NoAlias) { hasAlias = true; IonSpewDependency(ins, store, "aliases", "store in loop body"); break; } } if (hasAlias) break; } if (hasAlias) { // This instruction depends on stores inside the loop body. Mark it as having a // dependency on the last instruction of the loop header. The last instruction is a // control instruction and these are never hoisted. MControlInstruction* controlIns = loop_->loopHeader()->lastIns(); IonSpewDependency(ins, controlIns, "depends", "due to stores in loop body"); ins->setDependency(controlIns); } else { IonSpewAliasInfo("Load", ins, "does not depend on any stores in this loop"); if (outerLoop && ins->dependency()->id() < outerLoop->firstInstruction()->id()) { IonSpewAliasInfo("Load", ins, "may be invariant in outer loop"); if (!outerLoop->addInvariantLoad(ins)) return false; } } } loop_ = loop_->outer(); } } spewDependencyList(); MOZ_ASSERT(loop_ == nullptr); return true; }
// Test whether any instruction in the loop possiblyCalls(). static bool LoopContainsPossibleCall(MIRGraph &graph, MBasicBlock *header, MBasicBlock *backedge) { for (auto i(graph.rpoBegin(header)); ; ++i) { MOZ_ASSERT(i != graph.rpoEnd(), "Reached end of graph searching for blocks in loop"); MBasicBlock *block = *i; if (!block->isMarked()) continue; for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ++insIter) { MInstruction *ins = *insIter; if (ins->possiblyCalls()) { JitSpew(JitSpew_LICM, " Possile call found at %s%u", ins->opName(), ins->id()); return true; } } if (block == backedge) break; } return false; }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition* def) { // Nop does not fit in any of the previous optimization, as its only purpose // is to reduce the register pressure by keeping additional resume // point. Still, there is no need consecutive list of MNop instructions, and // this will slow down every other iteration on the Graph. if (def->isNop()) { MNop* nop = def->toNop(); MBasicBlock* block = nop->block(); // We look backward to know if we can remove the previous Nop, we do not // look forward as we would not benefit from the folding made by GVN. MInstructionReverseIterator iter = ++block->rbegin(nop); // This nop is at the beginning of the basic block, just replace the // resume point of the basic block by the one from the resume point. if (iter == block->rend()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); nop->moveResumePointAsEntry(); block->discard(nop); return true; } // The previous instruction is also a Nop, no need to keep it anymore. MInstruction* prev = *iter; if (prev->isNop()) { JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id()); block->discard(prev); return true; } return true; } // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MInstruction* dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition* sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; // If |sim| doesn't belong to a block, insert it next to |def|. if (sim->block() == nullptr) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef DEBUG JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif MOZ_ASSERT(!sim->isDiscarded()); ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; // If that ended up discarding |sim|, then we're done here. if (sim->isDiscarded()) return true; } // Otherwise, procede to optimize with |sim| in place of |def|. def = sim; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition* rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef DEBUG JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition* def) { // Nop does not fit in any of the previous optimization, as its only purpose // is to reduce the register pressure by keeping additional resume // point. Still, there is no need consecutive list of MNop instructions, and // this will slow down every other iteration on the Graph. if (def->isNop()) { MNop* nop = def->toNop(); MBasicBlock* block = nop->block(); // We look backward to know if we can remove the previous Nop, we do not // look forward as we would not benefit from the folding made by GVN. MInstructionReverseIterator iter = ++block->rbegin(nop); // This nop is at the beginning of the basic block, just replace the // resume point of the basic block by the one from the resume point. if (iter == block->rend()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); nop->moveResumePointAsEntry(); block->discard(nop); return true; } // The previous instruction is also a Nop, no need to keep it anymore. MInstruction* prev = *iter; if (prev->isNop()) { JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id()); block->discard(prev); return true; } // The Nop is introduced to capture the result and make sure the operands // are not live anymore when there are no further uses. Though when // all operands are still needed the Nop doesn't decrease the liveness // and can get removed. MResumePoint* rp = nop->resumePoint(); if (rp && rp->numOperands() > 0 && rp->getOperand(rp->numOperands() - 1) == prev && !nop->block()->lastIns()->isThrow() && !prev->isAssertRecoveredOnBailout()) { size_t numOperandsLive = 0; for (size_t j = 0; j < prev->numOperands(); j++) { for (size_t i = 0; i < rp->numOperands(); i++) { if (prev->getOperand(j) == rp->getOperand(i)) { numOperandsLive++; break; } } } if (numOperandsLive == prev->numOperands()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); block->discard(nop); } } return true; } // Skip optimizations on instructions which are recovered on bailout, to // avoid mixing instructions which are recovered on bailouts with // instructions which are not. if (def->isRecoveredOnBailout()) return true; // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MDefinition* dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition* sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; bool isNewInstruction = sim->block() == nullptr; // If |sim| doesn't belong to a block, insert it next to |def|. if (isNewInstruction) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef JS_JITSPEW JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif MOZ_ASSERT(!sim->isDiscarded()); ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (def->isGuardRangeBailouts()) sim->setGuardRangeBailoutsUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; // If that ended up discarding |sim|, then we're done here. if (sim->isDiscarded()) return true; } if (!rerun_ && def->isPhi() && !sim->isPhi()) { rerun_ = true; JitSpew(JitSpew_GVN, " Replacing phi%u may have enabled cascading optimisations; " "will re-run", def->id()); } // Otherwise, procede to optimize with |sim| in place of |def|. def = sim; // If the simplified instruction was already part of the graph, then we // probably already visited and optimized this instruction. if (!isNewInstruction) return true; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition* rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef JS_JITSPEW JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }