bool FilterTypeSetPolicy::adjustInputs(TempAllocator &alloc, MInstruction *ins) { MOZ_ASSERT(ins->numOperands() == 1); MIRType inputType = ins->getOperand(0)->type(); MIRType outputType = ins->type(); // Input and output type are already in accordance. if (inputType == outputType) return true; // Output is a value, box the input. if (outputType == MIRType_Value) { MOZ_ASSERT(inputType != MIRType_Value); ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0))); return true; } // The outputType should be a subset of the inputType else we are in code // that has never executed yet. Bail to see the new type (if that hasn't // happened yet). if (inputType != MIRType_Value) { MBail *bail = MBail::New(alloc); ins->block()->insertBefore(ins, bail); bail->setDependency(ins->dependency()); ins->setDependency(bail); ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0))); } // We can't unbox a value to null/undefined/lazyargs. So keep output // also a value. // Note: Using setResultType shouldn't be done in TypePolicies, // Here it is fine, since the type barrier has no uses. if (IsNullOrUndefined(outputType) || outputType == MIRType_MagicOptimizedArguments) { MOZ_ASSERT(!ins->hasDefUses()); ins->setResultType(MIRType_Value); return true; } // Unbox / propagate the right type. MUnbox::Mode mode = MUnbox::Infallible; MInstruction *replace = MUnbox::New(alloc, ins->getOperand(0), ins->type(), mode); ins->block()->insertBefore(ins, replace); ins->replaceOperand(0, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; // Carry over the dependency the MFilterTypeSet had. replace->setDependency(ins->dependency()); return true; }
bool TypeBarrierPolicy::adjustInputs(TempAllocator &alloc, MInstruction *def) { MTypeBarrier *ins = def->toTypeBarrier(); MIRType inputType = ins->getOperand(0)->type(); MIRType outputType = ins->type(); // Input and output type are already in accordance. if (inputType == outputType) return true; // Output is a value, currently box the input. if (outputType == MIRType_Value) { // XXX: Possible optimization: decrease resultTypeSet to only include // the inputType. This will remove the need for boxing. MOZ_ASSERT(inputType != MIRType_Value); ins->replaceOperand(0, boxAt(alloc, ins, ins->getOperand(0))); return true; } // Box input if needed. if (inputType != MIRType_Value) { MOZ_ASSERT(ins->alwaysBails()); ins->replaceOperand(0, boxAt(alloc, ins, ins->getOperand(0))); } // We can't unbox a value to null/undefined/lazyargs. So keep output // also a value. // Note: Using setResultType shouldn't be done in TypePolicies, // Here it is fine, since the type barrier has no uses. if (IsNullOrUndefined(outputType) || outputType == MIRType_MagicOptimizedArguments) { MOZ_ASSERT(!ins->hasDefUses()); ins->setResultType(MIRType_Value); return true; } // Unbox / propagate the right type. MUnbox::Mode mode = MUnbox::TypeBarrier; MInstruction *replace = MUnbox::New(alloc, ins->getOperand(0), ins->type(), mode); ins->block()->insertBefore(ins, replace); ins->replaceOperand(0, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; // The TypeBarrier is equivalent to removing branches with unexpected // types. The unexpected types would have changed Range Analysis // predictions. As such, we need to prevent destructive optimizations. ins->block()->flagOperandsOfPrunedBranches(replace); return true; }
MConstant* MBasicBlock::optimizedOutConstant(TempAllocator& alloc) { // If the first instruction is a MConstant(MagicValue(JS_OPTIMIZED_OUT)) // then reuse it. MInstruction* ins = *begin(); if (ins->type() == MIRType_MagicOptimizedOut) return ins->toConstant(); MConstant* constant = MConstant::New(alloc, MagicValue(JS_OPTIMIZED_OUT)); insertBefore(ins, constant); return constant; }
bool MBasicBlock::linkOsrValues(MStart* start) { MOZ_ASSERT(start->startType() == MStart::StartType_Osr); MResumePoint* res = start->resumePoint(); for (uint32_t i = 0; i < stackDepth(); i++) { MDefinition* def = slots_[i]; MInstruction* cloneRp = nullptr; if (i == info().scopeChainSlot()) { if (def->isOsrScopeChain()) cloneRp = def->toOsrScopeChain(); } else if (i == info().returnValueSlot()) { if (def->isOsrReturnValue()) cloneRp = def->toOsrReturnValue(); } else if (info().hasArguments() && i == info().argsObjSlot()) { MOZ_ASSERT(def->isConstant() || def->isOsrArgumentsObject()); MOZ_ASSERT_IF(def->isConstant(), def->toConstant()->value() == UndefinedValue()); if (def->isOsrArgumentsObject()) cloneRp = def->toOsrArgumentsObject(); } else { MOZ_ASSERT(def->isOsrValue() || def->isGetArgumentsObjectArg() || def->isConstant() || def->isParameter()); // A constant Undefined can show up here for an argument slot when // the function has an arguments object, but the argument in // question is stored on the scope chain. MOZ_ASSERT_IF(def->isConstant(), def->toConstant()->value() == UndefinedValue()); if (def->isOsrValue()) cloneRp = def->toOsrValue(); else if (def->isGetArgumentsObjectArg()) cloneRp = def->toGetArgumentsObjectArg(); else if (def->isParameter()) cloneRp = def->toParameter(); } if (cloneRp) { MResumePoint* clone = MResumePoint::Copy(graph().alloc(), res); if (!clone) return false; cloneRp->setResumePoint(clone); } } return true; }
bool Loop::hoistInstructions(InstructionQueue &toHoist) { // Iterate in post-order (uses before definitions) for (int32_t i = toHoist.length() - 1; i >= 0; i--) { MInstruction *ins = toHoist[i]; // Don't hoist a cheap constant if it doesn't enable us to hoist one of // its uses. We want those instructions as close as possible to their // use, to facilitate folding and minimize register pressure. if (requiresHoistedUse(ins)) { bool loopInvariantUse = false; for (MUseDefIterator use(ins); use; use++) { if (use.def()->isLoopInvariant()) { loopInvariantUse = true; break; } } if (!loopInvariantUse) ins->setNotLoopInvariant(); } } // Move all instructions to the preLoop_ block just before the control instruction. for (size_t i = 0; i < toHoist.length(); i++) { MInstruction *ins = toHoist[i]; // Loads may have an implicit dependency on either stores (effectful instructions) or // control instructions so we should never move these. JS_ASSERT(!ins->isControlInstruction()); JS_ASSERT(!ins->isEffectful()); JS_ASSERT(ins->isMovable()); if (!ins->isLoopInvariant()) continue; if (checkHotness(ins->block())) { ins->block()->moveBefore(preLoop_->lastIns(), ins); ins->setNotLoopInvariant(); } } return true; }
bool AllDoublePolicy::adjustInputs(TempAllocator &alloc, MInstruction *ins) { for (size_t i = 0, e = ins->numOperands(); i < e; i++) { MDefinition *in = ins->getOperand(i); if (in->type() == MIRType_Double) continue; MInstruction *replace = MToDouble::New(alloc, in); ins->block()->insertBefore(ins, replace); ins->replaceOperand(i, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; } return true; }
// Fold AddIs with one variable and two or more constants into one AddI. static void AnalyzeAdd(TempAllocator& alloc, MAdd* add) { if (add->specialization() != MIRType::Int32 || add->isRecoveredOnBailout()) { return; } if (!add->hasUses()) { return; } JitSpew(JitSpew_FLAC, "analyze add: %s%u", add->opName(), add->id()); SimpleLinearSum sum = ExtractLinearSum(add); if (sum.constant == 0 || !sum.term) { return; } // Determine which operand is the constant. int idx = add->getOperand(0)->isConstant() ? 0 : 1; if (add->getOperand(idx)->isConstant()) { // Do not replace an add where the outcome is the same add instruction. MOZ_ASSERT(add->getOperand(idx)->toConstant()->type() == MIRType::Int32); if (sum.term == add->getOperand(1 - idx) || sum.constant == add->getOperand(idx)->toConstant()->toInt32()) { return; } } MInstruction* rhs = MConstant::New(alloc, Int32Value(sum.constant)); add->block()->insertBefore(add, rhs); MAdd* addNew = MAdd::New(alloc, sum.term, rhs, MIRType::Int32, add->truncateKind()); add->replaceAllLiveUsesWith(addNew); add->block()->insertBefore(add, addNew); JitSpew(JitSpew_FLAC, "replaced with: %s%u", addNew->opName(), addNew->id()); JitSpew(JitSpew_FLAC, "and constant: %s%u (%d)", rhs->opName(), rhs->id(), sum.constant); // Mark the stale nodes as RecoveredOnBailout since the Sink pass has // been run before this pass. DCE will then remove the unused nodes. markNodesAsRecoveredOnBailout(add); }
// Iterate backward on all instruction and attempt to truncate operations for // each instruction which respect the following list of predicates: Has been // analyzed by range analysis, the range has no rounding errors, all uses cases // are truncating the result. // // If the truncation of the operation is successful, then the instruction is // queue for later updating the graph to restore the type correctness by // converting the operands that need to be truncated. // // We iterate backward because it is likely that a truncated operation truncates // some of its operands. bool RangeAnalysis::truncate() { IonSpew(IonSpew_Range, "Do range-base truncation (backward loop)"); Vector<MInstruction *, 16, SystemAllocPolicy> worklist; for (PostorderIterator block(graph_.poBegin()); block != graph_.poEnd(); block++) { for (MInstructionReverseIterator iter(block->rbegin()); iter != block->rend(); iter++) { // Set truncated flag if range analysis ensure that it has no // rounding errors and no freactional part. const Range *r = iter->range(); if (!r || r->hasRoundingErrors()) continue; // Ensure all observable uses are truncated. if (!AllUsesTruncate(*iter)) continue; // Truncate this instruction if possible. if (!iter->truncate()) continue; // Delay updates of inputs/outputs to avoid creating node which // would be removed by the truncation of the next operations. iter->setInWorklist(); if (!worklist.append(*iter)) return false; } } // Update inputs/outputs of truncated instructions. IonSpew(IonSpew_Range, "Do graph type fixup (dequeue)"); while (!worklist.empty()) { MInstruction *ins = worklist.popCopy(); ins->setNotInWorklist(); RemoveTruncatesOnOutput(ins); AdjustTruncatedInputs(ins); } return true; }
bool CallPolicy::adjustInputs(TempAllocator &alloc, MInstruction *ins) { MCall *call = ins->toCall(); MDefinition *func = call->getFunction(); if (func->type() != MIRType_Object) { MInstruction *unbox = MUnbox::New(alloc, func, MIRType_Object, MUnbox::Fallible); call->block()->insertBefore(call, unbox); call->replaceFunction(unbox); if (!unbox->typePolicy()->adjustInputs(alloc, unbox)) return false; } for (uint32_t i = 0; i < call->numStackArgs(); i++) EnsureOperandNotFloat32(alloc, call, MCall::IndexOfStackArg(i)); return true; }
// Test whether any instruction in the loop possiblyCalls(). static bool LoopContainsPossibleCall(MIRGraph &graph, MBasicBlock *header, MBasicBlock *backedge) { for (auto i(graph.rpoBegin(header)); ; ++i) { MOZ_ASSERT(i != graph.rpoEnd(), "Reached end of graph searching for blocks in loop"); MBasicBlock *block = *i; if (!block->isMarked()) continue; for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ++insIter) { MInstruction *ins = *insIter; if (ins->possiblyCalls()) { JitSpew(JitSpew_LICM, " Possile call found at %s%u", ins->opName(), ins->id()); return true; } } if (block == backedge) break; } return false; }
Loop::LoopReturn Loop::iterateLoopBlocks(MBasicBlock *current) { // Visited. current->mark(); // Hoisting requires more finesse if the loop contains a block that // self-dominates: there exists control flow that may enter the loop // without passing through the loop preheader. // // Rather than perform a complicated analysis of the dominance graph, // just return a soft error to ignore this loop. if (current->immediateDominator() == current) return LoopReturn_Skip; // If we haven't reached the loop header yet, recursively explore predecessors // if we haven't seen them already. if (current != header_) { for (size_t i = 0; i < current->numPredecessors(); i++) { if (current->getPredecessor(i)->isMarked()) continue; LoopReturn lr = iterateLoopBlocks(current->getPredecessor(i)); if (lr != LoopReturn_Success) return lr; } } // Add all instructions in this block (but the control instruction) to the worklist for (MInstructionIterator i = current->begin(); i != current->end(); i++) { MInstruction *ins = *i; if (ins->isMovable() && !ins->isEffectful()) { if (!insertInWorklist(ins)) return LoopReturn_Error; } } return LoopReturn_Success; }
// In preparation for hoisting an instruction, hoist any of its operands which // were too cheap to hoist on their own. static void MoveDeferredOperands(MInstruction *ins, MInstruction *hoistPoint, bool hasCalls) { // If any of our operands were waiting for a user to be hoisted, make a note // to hoist them. for (size_t i = 0, e = ins->numOperands(); i != e; ++i) { MDefinition *op = ins->getOperand(i); if (!IsInLoop(op)) continue; MOZ_ASSERT(RequiresHoistedUse(op, hasCalls), "Deferred loop-invariant operand is not cheap"); MInstruction *opIns = op->toInstruction(); // Recursively move the operands. Note that the recursion is bounded // because we require RequiresHoistedUse to be set at each level. MoveDeferredOperands(opIns, hoistPoint, hasCalls); JitSpew(JitSpew_LICM, " Hoisting %s%u (now that a user will be hoisted)", opIns->opName(), opIns->id()); opIns->block()->moveBefore(hoistPoint, opIns); } }
bool SimdShufflePolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) { MSimdGeneralShuffle* s = ins->toSimdGeneralShuffle(); for (unsigned i = 0; i < s->numVectors(); i++) MOZ_ASSERT(ins->getOperand(i)->type() == ins->typePolicySpecialization()); // Next inputs are the lanes, which need to be int32 for (unsigned i = 0; i < s->numLanes(); i++) { MDefinition* in = ins->getOperand(s->numVectors() + i); if (in->type() == MIRType_Int32) continue; MInstruction* replace = MToInt32::New(alloc, in, MacroAssembler::IntConversion_NumbersOnly); ins->block()->insertBefore(ins, replace); ins->replaceOperand(s->numVectors() + i, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; } return true; }
bool SimdScalarPolicy<Op>::staticAdjustInputs(TempAllocator &alloc, MInstruction *ins) { MOZ_ASSERT(IsSimdType(ins->type())); MIRType scalarType = SimdTypeToScalarType(ins->type()); MDefinition *in = ins->getOperand(Op); if (in->type() == scalarType) return true; MInstruction *replace; if (scalarType == MIRType_Int32) { replace = MTruncateToInt32::New(alloc, in); } else { MOZ_ASSERT(scalarType == MIRType_Float32); replace = MToFloat32::New(alloc, in); } ins->block()->insertBefore(ins, replace); ins->replaceOperand(Op, replace); return replace->typePolicy()->adjustInputs(alloc, replace); }
void LoopUnroller::makeReplacementInstruction(MInstruction *ins) { MDefinitionVector inputs(alloc); for (size_t i = 0; i < ins->numOperands(); i++) { MDefinition *old = ins->getOperand(i); MDefinition *replacement = getReplacementDefinition(old); if (!inputs.append(replacement)) CrashAtUnhandlableOOM("LoopUnroller::makeReplacementDefinition"); } MInstruction *clone = ins->clone(alloc, inputs); unrolledBackedge->add(clone); if (!unrolledDefinitions.putNew(ins, clone)) CrashAtUnhandlableOOM("LoopUnroller::makeReplacementDefinition"); if (MResumePoint *old = ins->resumePoint()) { MResumePoint *rp = makeReplacementResumePoint(unrolledBackedge, old); clone->setResumePoint(rp); } }
bool Loop::optimize() { InstructionQueue invariantInstructions; IonSpew(IonSpew_LICM, "These instructions are in the loop: "); while (!worklist_.empty()) { if (mir->shouldCancel("LICM (worklist)")) return false; MInstruction *ins = popFromWorklist(); IonSpewHeader(IonSpew_LICM); if (IonSpewEnabled(IonSpew_LICM)) { ins->printName(IonSpewFile); fprintf(IonSpewFile, " <- "); ins->printOpcode(IonSpewFile); fprintf(IonSpewFile, ": "); } if (isLoopInvariant(ins)) { // Flag this instruction as loop invariant. ins->setLoopInvariant(); if (!invariantInstructions.append(ins)) return false; if (IonSpewEnabled(IonSpew_LICM)) fprintf(IonSpewFile, " Loop Invariant!\n"); } } if (!hoistInstructions(invariantInstructions)) return false; return true; }
bool Loop::optimize() { InstructionQueue invariantInstructions; InstructionQueue boundsChecks; IonSpew(IonSpew_LICM, "These instructions are in the loop: "); while (!worklist_.empty()) { if (mir->shouldCancel("LICM (worklist)")) return false; MInstruction *ins = popFromWorklist(); IonSpewHeader(IonSpew_LICM); if (IonSpewEnabled(IonSpew_LICM)) { ins->printName(IonSpewFile); fprintf(IonSpewFile, " <- "); ins->printOpcode(IonSpewFile); fprintf(IonSpewFile, ": "); } if (isLoopInvariant(ins)) { // Flag this instruction as loop invariant. ins->setLoopInvariant(); if (!invariantInstructions.append(ins)) return false; // Loop through uses of invariant instruction and add back to work list. for (MUseDefIterator iter(ins->toDefinition()); iter; iter++) { MDefinition *consumer = iter.def(); if (consumer->isInWorklist()) continue; // if the consumer of this invariant instruction is in the // loop, and it is also worth hoisting, then process it. if (isInLoop(consumer) && isHoistable(consumer)) { if (!insertInWorklist(consumer->toInstruction())) return false; } } if (IonSpewEnabled(IonSpew_LICM)) fprintf(IonSpewFile, " Loop Invariant!\n"); } else if (ins->isBoundsCheck()) { if (!boundsChecks.append(ins)) return false; } } if (!hoistInstructions(invariantInstructions, boundsChecks)) return false; return true; }
static void VisitLoopBlock(MBasicBlock* block, MBasicBlock* header, MInstruction* hoistPoint, bool hasCalls) { for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ) { MInstruction* ins = *insIter++; if (!IsHoistable(ins, header, hasCalls)) { #ifdef DEBUG if (IsHoistableIgnoringDependency(ins, hasCalls)) { JitSpew(JitSpew_LICM, " %s%u isn't hoistable due to dependency on %s%u", ins->opName(), ins->id(), ins->dependency()->opName(), ins->dependency()->id()); } #endif continue; } // Don't hoist a cheap constant if it doesn't enable us to hoist one of // its uses. We want those instructions as close as possible to their // use, to minimize register pressure. if (RequiresHoistedUse(ins, hasCalls)) { #ifdef DEBUG JitSpew(JitSpew_LICM, " %s%u will be hoisted only if its users are", ins->opName(), ins->id()); #endif continue; } // Hoist operands which were too cheap to hoist on their own. MoveDeferredOperands(ins, hoistPoint, hasCalls); #ifdef DEBUG JitSpew(JitSpew_LICM, " Hoisting %s%u", ins->opName(), ins->id()); #endif // Move the instruction to the hoistPoint. block->moveBefore(hoistPoint, ins); } }
// Visit |def|. bool ValueNumberer::visitDefinition(MDefinition* def) { // Nop does not fit in any of the previous optimization, as its only purpose // is to reduce the register pressure by keeping additional resume // point. Still, there is no need consecutive list of MNop instructions, and // this will slow down every other iteration on the Graph. if (def->isNop()) { MNop* nop = def->toNop(); MBasicBlock* block = nop->block(); // We look backward to know if we can remove the previous Nop, we do not // look forward as we would not benefit from the folding made by GVN. MInstructionReverseIterator iter = ++block->rbegin(nop); // This nop is at the beginning of the basic block, just replace the // resume point of the basic block by the one from the resume point. if (iter == block->rend()) { JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id()); nop->moveResumePointAsEntry(); block->discard(nop); return true; } // The previous instruction is also a Nop, no need to keep it anymore. MInstruction* prev = *iter; if (prev->isNop()) { JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id()); block->discard(prev); return true; } return true; } // If this instruction has a dependency() into an unreachable block, we'll // need to update AliasAnalysis. MInstruction* dep = def->dependency(); if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) { JitSpew(JitSpew_GVN, " AliasAnalysis invalidated"); if (updateAliasAnalysis_ && !dependenciesBroken_) { // TODO: Recomputing alias-analysis could theoretically expose more // GVN opportunities. JitSpew(JitSpew_GVN, " Will recompute!"); dependenciesBroken_ = true; } // Temporarily clear its dependency, to protect foldsTo, which may // wish to use the dependency to do store-to-load forwarding. def->setDependency(def->toInstruction()); } else { dep = nullptr; } // Look for a simplified form of |def|. MDefinition* sim = simplified(def); if (sim != def) { if (sim == nullptr) return false; // If |sim| doesn't belong to a block, insert it next to |def|. if (sim->block() == nullptr) def->block()->insertAfter(def->toInstruction(), sim->toInstruction()); #ifdef DEBUG JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(), sim->opName(), sim->id()); #endif MOZ_ASSERT(!sim->isDiscarded()); ReplaceAllUsesWith(def, sim); // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a // guard, then either |rep| is also a guard, or a guard isn't actually // needed, so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { if (!discardDefsRecursively(def)) return false; // If that ended up discarding |sim|, then we're done here. if (sim->isDiscarded()) return true; } // Otherwise, procede to optimize with |sim| in place of |def|. def = sim; } // Now that foldsTo is done, re-enable the original dependency. Even though // it may be pointing into a discarded block, it's still valid for the // purposes of detecting congruent loads. if (dep != nullptr) def->setDependency(dep); // Look for a dominating def which makes |def| redundant. MDefinition* rep = leader(def); if (rep != def) { if (rep == nullptr) return false; if (rep->updateForReplacement(def)) { #ifdef DEBUG JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(), def->id(), rep->opName(), rep->id()); #endif ReplaceAllUsesWith(def, rep); // The node's congruentTo said |def| is congruent to |rep|, and it's // dominated by |rep|. If |def| is a guard, it's covered by |rep|, // so we can clear |def|'s guard flag and let it be discarded. def->setNotGuardUnchecked(); if (DeadIfUnused(def)) { // discardDef should not add anything to the deadDefs, as the // redundant operation should have the same input operands. mozilla::DebugOnly<bool> r = discardDef(def); MOZ_ASSERT(r, "discardDef shouldn't have tried to add anything to the worklist, " "so it shouldn't have failed"); MOZ_ASSERT(deadDefs_.empty(), "discardDef shouldn't have added anything to the worklist"); } def = rep; } } return true; }
void LoopUnroller::go(LoopIterationBound *bound) { // For now we always unroll loops the same number of times. static const size_t UnrollCount = 10; JitSpew(JitSpew_Unrolling, "Attempting to unroll loop"); header = bound->header; // UCE might have determined this isn't actually a loop. if (!header->isLoopHeader()) return; backedge = header->backedge(); oldPreheader = header->loopPredecessor(); JS_ASSERT(oldPreheader->numSuccessors() == 1); // Only unroll loops with two blocks: an initial one ending with the // bound's test, and the body ending with the backedge. MTest *test = bound->test; if (header->lastIns() != test) return; if (test->ifTrue() == backedge) { if (test->ifFalse()->id() <= backedge->id()) return; } else if (test->ifFalse() == backedge) { if (test->ifTrue()->id() <= backedge->id()) return; } else { return; } if (backedge->numPredecessors() != 1 || backedge->numSuccessors() != 1) return; JS_ASSERT(backedge->phisEmpty()); MBasicBlock *bodyBlocks[] = { header, backedge }; // All instructions in the header and body must be clonable. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock *block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction *ins = *iter; if (ins->canClone()) continue; if (ins->isTest() || ins->isGoto() || ins->isInterruptCheck()) continue; #ifdef DEBUG JitSpew(JitSpew_Unrolling, "Aborting: can't clone instruction %s", ins->opName()); #endif return; } } // Compute the linear inequality we will use for exiting the unrolled loop: // // iterationBound - iterationCount - UnrollCount >= 0 // LinearSum remainingIterationsInequality(bound->boundSum); if (!remainingIterationsInequality.add(bound->currentSum, -1)) return; if (!remainingIterationsInequality.add(-int32_t(UnrollCount))) return; // Terms in the inequality need to be either loop invariant or phis from // the original header. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition *def = remainingIterationsInequality.term(i).term; if (def->block()->id() < header->id()) continue; if (def->block() == header && def->isPhi()) continue; return; } // OK, we've checked everything, now unroll the loop. JitSpew(JitSpew_Unrolling, "Unrolling loop"); // The old preheader will go before the unrolled loop, and the old loop // will need a new empty preheader. CompileInfo &info = oldPreheader->info(); if (header->trackedSite().pc()) { unrolledHeader = MBasicBlock::New(graph, nullptr, info, oldPreheader, header->trackedSite(), MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::New(graph, nullptr, info, unrolledHeader, backedge->trackedSite(), MBasicBlock::NORMAL); newPreheader = MBasicBlock::New(graph, nullptr, info, unrolledHeader, oldPreheader->trackedSite(), MBasicBlock::NORMAL); } else { unrolledHeader = MBasicBlock::NewAsmJS(graph, info, oldPreheader, MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); newPreheader = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); } unrolledHeader->discardAllResumePoints(); unrolledBackedge->discardAllResumePoints(); newPreheader->discardAllResumePoints(); // Insert new blocks at their RPO position, and update block ids. graph.insertBlockAfter(oldPreheader, unrolledHeader); graph.insertBlockAfter(unrolledHeader, unrolledBackedge); graph.insertBlockAfter(unrolledBackedge, newPreheader); graph.renumberBlocksAfter(oldPreheader); if (!unrolledDefinitions.init()) CrashAtUnhandlableOOM("LoopUnroller::go"); // Add phis to the unrolled loop header which correspond to the phis in the // original loop header. JS_ASSERT(header->getPredecessor(0) == oldPreheader); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; JS_ASSERT(old->numOperands() == 2); MPhi *phi = MPhi::New(alloc); phi->setResultType(old->type()); phi->setResultTypeSet(old->resultTypeSet()); phi->setRange(old->range()); unrolledHeader->addPhi(phi); if (!phi->reserveLength(2)) CrashAtUnhandlableOOM("LoopUnroller::go"); // Set the first input for the phi for now. We'll set the second after // finishing the unroll. phi->addInput(old->getOperand(0)); // The old phi will now take the value produced by the unrolled loop. old->replaceOperand(0, phi); if (!unrolledDefinitions.putNew(old, phi)) CrashAtUnhandlableOOM("LoopUnroller::go"); } // The loop condition can bail out on e.g. integer overflow, so make a // resume point based on the initial resume point of the original header. MResumePoint *headerResumePoint = header->entryResumePoint(); if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(unrolledHeader, headerResumePoint); unrolledHeader->setEntryResumePoint(rp); // Perform an interrupt check at the start of the unrolled loop. unrolledHeader->add(MInterruptCheck::New(alloc)); } // Generate code for the test in the unrolled loop. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition *def = remainingIterationsInequality.term(i).term; MDefinition *replacement = getReplacementDefinition(def); remainingIterationsInequality.replaceTerm(i, replacement); } MCompare *compare = ConvertLinearInequality(alloc, unrolledHeader, remainingIterationsInequality); MTest *unrolledTest = MTest::New(alloc, compare, unrolledBackedge, newPreheader); unrolledHeader->end(unrolledTest); // Make an entry resume point for the unrolled body. The unrolled header // does not have side effects on stack values, even if the original loop // header does, so use the same resume point as for the unrolled header. if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(unrolledBackedge, headerResumePoint); unrolledBackedge->setEntryResumePoint(rp); } // Make an entry resume point for the new preheader. There are no // instructions which use this but some other stuff wants one to be here. if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(newPreheader, headerResumePoint); newPreheader->setEntryResumePoint(rp); } // Generate the unrolled code. JS_ASSERT(UnrollCount > 1); size_t unrollIndex = 0; while (true) { // Clone the contents of the original loop into the unrolled loop body. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock *block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction *ins = *iter; if (ins->canClone()) { makeReplacementInstruction(*iter); } else { // Control instructions are handled separately. JS_ASSERT(ins->isTest() || ins->isGoto() || ins->isInterruptCheck()); } } } // Compute the value of each loop header phi after the execution of // this unrolled iteration. MDefinitionVector phiValues(alloc); JS_ASSERT(header->getPredecessor(1) == backedge); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; MDefinition *oldInput = old->getOperand(1); if (!phiValues.append(getReplacementDefinition(oldInput))) CrashAtUnhandlableOOM("LoopUnroller::go"); } unrolledDefinitions.clear(); if (unrollIndex == UnrollCount - 1) { // We're at the end of the last unrolled iteration, set the // backedge input for the unrolled loop phis. size_t phiIndex = 0; for (MPhiIterator iter(unrolledHeader->phisBegin()); iter != unrolledHeader->phisEnd(); iter++) { MPhi *phi = *iter; phi->addInput(phiValues[phiIndex++]); } JS_ASSERT(phiIndex == phiValues.length()); break; } // Update the map for the phis in the next iteration. size_t phiIndex = 0; for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; if (!unrolledDefinitions.putNew(old, phiValues[phiIndex++])) CrashAtUnhandlableOOM("LoopUnroller::go"); } JS_ASSERT(phiIndex == phiValues.length()); unrollIndex++; } MGoto *backedgeJump = MGoto::New(alloc, unrolledHeader); unrolledBackedge->end(backedgeJump); // Place the old preheader before the unrolled loop. JS_ASSERT(oldPreheader->lastIns()->isGoto()); oldPreheader->discardLastIns(); oldPreheader->end(MGoto::New(alloc, unrolledHeader)); // Place the new preheader before the original loop. newPreheader->end(MGoto::New(alloc, header)); // Cleanup the MIR graph. if (!unrolledHeader->addPredecessorWithoutPhis(unrolledBackedge)) CrashAtUnhandlableOOM("LoopUnroller::go"); header->replacePredecessor(oldPreheader, newPreheader); oldPreheader->setSuccessorWithPhis(unrolledHeader, 0); newPreheader->setSuccessorWithPhis(header, 0); unrolledBackedge->setSuccessorWithPhis(unrolledHeader, 1); }
bool ComparePolicy::adjustInputs(TempAllocator &alloc, MInstruction *def) { MOZ_ASSERT(def->isCompare()); MCompare *compare = def->toCompare(); // Convert Float32 operands to doubles for (size_t i = 0; i < 2; i++) { MDefinition *in = def->getOperand(i); if (in->type() == MIRType_Float32) { MInstruction *replace = MToDouble::New(alloc, in); def->block()->insertBefore(def, replace); def->replaceOperand(i, replace); } } // Box inputs to get value if (compare->compareType() == MCompare::Compare_Unknown || compare->compareType() == MCompare::Compare_Value) { return BoxInputsPolicy::adjustInputs(alloc, def); } // Compare_Boolean specialization is done for "Anything === Bool" // If the LHS is boolean, we set the specialization to Compare_Int32. // This matches other comparisons of the form bool === bool and // generated code of Compare_Int32 is more efficient. if (compare->compareType() == MCompare::Compare_Boolean && def->getOperand(0)->type() == MIRType_Boolean) { compare->setCompareType(MCompare::Compare_Int32MaybeCoerceBoth); } // Compare_Boolean specialization is done for "Anything === Bool" // As of previous line Anything can't be Boolean if (compare->compareType() == MCompare::Compare_Boolean) { // Unbox rhs that is definitely Boolean MDefinition *rhs = def->getOperand(1); if (rhs->type() != MIRType_Boolean) { MInstruction *unbox = MUnbox::New(alloc, rhs, MIRType_Boolean, MUnbox::Infallible); def->block()->insertBefore(def, unbox); def->replaceOperand(1, unbox); if (!unbox->typePolicy()->adjustInputs(alloc, unbox)) return false; } MOZ_ASSERT(def->getOperand(0)->type() != MIRType_Boolean); MOZ_ASSERT(def->getOperand(1)->type() == MIRType_Boolean); return true; } // Compare_StrictString specialization is done for "Anything === String" // If the LHS is string, we set the specialization to Compare_String. if (compare->compareType() == MCompare::Compare_StrictString && def->getOperand(0)->type() == MIRType_String) { compare->setCompareType(MCompare::Compare_String); } // Compare_StrictString specialization is done for "Anything === String" // As of previous line Anything can't be String if (compare->compareType() == MCompare::Compare_StrictString) { // Unbox rhs that is definitely String MDefinition *rhs = def->getOperand(1); if (rhs->type() != MIRType_String) { MInstruction *unbox = MUnbox::New(alloc, rhs, MIRType_String, MUnbox::Infallible); def->block()->insertBefore(def, unbox); def->replaceOperand(1, unbox); if (!unbox->typePolicy()->adjustInputs(alloc, unbox)) return false; } MOZ_ASSERT(def->getOperand(0)->type() != MIRType_String); MOZ_ASSERT(def->getOperand(1)->type() == MIRType_String); return true; } if (compare->compareType() == MCompare::Compare_Undefined || compare->compareType() == MCompare::Compare_Null) { // Nothing to do for undefined and null, lowering handles all types. return true; } // Convert all inputs to the right input type MIRType type = compare->inputType(); MOZ_ASSERT(type == MIRType_Int32 || type == MIRType_Double || type == MIRType_Object || type == MIRType_String || type == MIRType_Float32); for (size_t i = 0; i < 2; i++) { MDefinition *in = def->getOperand(i); if (in->type() == type) continue; MInstruction *replace; switch (type) { case MIRType_Double: { MToFPInstruction::ConversionKind convert = MToFPInstruction::NumbersOnly; if (compare->compareType() == MCompare::Compare_DoubleMaybeCoerceLHS && i == 0) convert = MToFPInstruction::NonNullNonStringPrimitives; else if (compare->compareType() == MCompare::Compare_DoubleMaybeCoerceRHS && i == 1) convert = MToFPInstruction::NonNullNonStringPrimitives; replace = MToDouble::New(alloc, in, convert); break; } case MIRType_Float32: { MToFPInstruction::ConversionKind convert = MToFPInstruction::NumbersOnly; if (compare->compareType() == MCompare::Compare_DoubleMaybeCoerceLHS && i == 0) convert = MToFPInstruction::NonNullNonStringPrimitives; else if (compare->compareType() == MCompare::Compare_DoubleMaybeCoerceRHS && i == 1) convert = MToFPInstruction::NonNullNonStringPrimitives; replace = MToFloat32::New(alloc, in, convert); break; } case MIRType_Int32: { MacroAssembler::IntConversionInputKind convert = MacroAssembler::IntConversion_NumbersOnly; if (compare->compareType() == MCompare::Compare_Int32MaybeCoerceBoth || (compare->compareType() == MCompare::Compare_Int32MaybeCoerceLHS && i == 0) || (compare->compareType() == MCompare::Compare_Int32MaybeCoerceRHS && i == 1)) { convert = MacroAssembler::IntConversion_NumbersOrBoolsOnly; } replace = MToInt32::New(alloc, in, convert); break; } case MIRType_Object: replace = MUnbox::New(alloc, in, MIRType_Object, MUnbox::Infallible); break; case MIRType_String: replace = MUnbox::New(alloc, in, MIRType_String, MUnbox::Infallible); break; default: MOZ_CRASH("Unknown compare specialization"); } def->block()->insertBefore(def, replace); def->replaceOperand(i, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; } return true; }
static void AnalyzeLsh(MBasicBlock *block, MLsh *lsh) { if (lsh->specialization() != MIRType_Int32) return; MDefinition *index = lsh->lhs(); JS_ASSERT(index->type() == MIRType_Int32); MDefinition *shift = lsh->rhs(); if (!shift->isConstant()) return; Value shiftValue = shift->toConstant()->value(); if (!shiftValue.isInt32() || !IsShiftInScaleRange(shiftValue.toInt32())) return; Scale scale = ShiftToScale(shiftValue.toInt32()); int32_t displacement = 0; MInstruction *last = lsh; MDefinition *base = nullptr; while (true) { if (!last->hasOneUse()) break; MUseIterator use = last->usesBegin(); if (!use->consumer()->isDefinition() || !use->consumer()->toDefinition()->isAdd()) break; MAdd *add = use->consumer()->toDefinition()->toAdd(); if (add->specialization() != MIRType_Int32 || !add->isTruncated()) break; MDefinition *other = add->getOperand(1 - use->index()); if (other->isConstant()) { displacement += other->toConstant()->value().toInt32(); } else { if (base) break; base = other; } last = add; } if (!base) { uint32_t elemSize = 1 << ScaleToShift(scale); if (displacement % elemSize != 0) return; if (!last->hasOneUse()) return; MUseIterator use = last->usesBegin(); if (!use->consumer()->isDefinition() || !use->consumer()->toDefinition()->isBitAnd()) return; MBitAnd *bitAnd = use->consumer()->toDefinition()->toBitAnd(); MDefinition *other = bitAnd->getOperand(1 - use->index()); if (!other->isConstant() || !other->toConstant()->value().isInt32()) return; uint32_t bitsClearedByShift = elemSize - 1; uint32_t bitsClearedByMask = ~uint32_t(other->toConstant()->value().toInt32()); if ((bitsClearedByShift & bitsClearedByMask) != bitsClearedByMask) return; bitAnd->replaceAllUsesWith(last); return; } MEffectiveAddress *eaddr = MEffectiveAddress::New(base, index, scale, displacement); last->replaceAllUsesWith(eaddr); block->insertAfter(last, eaddr); }
bool Loop::hoistInstructions(InstructionQueue &toHoist, InstructionQueue &boundsChecks) { // Hoist bounds checks first, so that hoistBoundsCheck can test for // invariant instructions, but delay actual insertion until the end to // handle dependencies on loop invariant instructions. InstructionQueue hoistedChecks; for (size_t i = 0; i < boundsChecks.length(); i++) { MBoundsCheck *ins = boundsChecks[i]->toBoundsCheck(); if (isLoopInvariant(ins) || !isInLoop(ins)) continue; // Try to find a test dominating the bounds check which can be // transformed into a hoistable check. Stop after the first such check // which could be transformed (the one which will be the closest to the // access in the source). MBasicBlock *block = ins->block(); while (true) { BranchDirection direction; MTest *branch = block->immediateDominatorBranch(&direction); if (branch) { MInstruction *upper, *lower; tryHoistBoundsCheck(ins, branch, direction, &upper, &lower); if (upper && !hoistedChecks.append(upper)) return false; if (lower && !hoistedChecks.append(lower)) return false; if (upper || lower) { ins->block()->discard(ins); break; } } MBasicBlock *dom = block->immediateDominator(); if (dom == block) break; block = dom; } } // Move all instructions to the preLoop_ block just before the control instruction. for (size_t i = 0; i < toHoist.length(); i++) { MInstruction *ins = toHoist[i]; // Loads may have an implicit dependency on either stores (effectful instructions) or // control instructions so we should never move these. JS_ASSERT(!ins->isControlInstruction()); JS_ASSERT(!ins->isEffectful()); JS_ASSERT(ins->isMovable()); if (checkHotness(ins->block())) { ins->block()->moveBefore(preLoop_->lastIns(), ins); ins->setNotLoopInvariant(); } } for (size_t i = 0; i < hoistedChecks.length(); i++) { MInstruction *ins = hoistedChecks[i]; preLoop_->insertBefore(preLoop_->lastIns(), ins); } return true; }
bool jit::ReorderInstructions(MIRGraph& graph) { // Renumber all instructions in the graph as we go. size_t nextId = 0; // List of the headers of any loops we are in. Vector<MBasicBlock*, 4, SystemAllocPolicy> loopHeaders; for (ReversePostorderIterator block(graph.rpoBegin()); block != graph.rpoEnd(); block++) { // Renumber all definitions inside the basic blocks. for (MPhiIterator iter(block->phisBegin()); iter != block->phisEnd(); iter++) iter->setId(nextId++); for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) iter->setId(nextId++); // Don't reorder instructions within entry blocks, which have special requirements. if (*block == graph.entryBlock() || *block == graph.osrBlock()) continue; if (block->isLoopHeader()) { if (!loopHeaders.append(*block)) return false; } MBasicBlock* innerLoop = loopHeaders.empty() ? nullptr : loopHeaders.back(); MInstruction* top = block->safeInsertTop(); MInstructionReverseIterator rtop = ++block->rbegin(top); for (MInstructionIterator iter(block->begin(top)); iter != block->end(); ) { MInstruction* ins = *iter; // Filter out some instructions which are never reordered. if (ins->isEffectful() || !ins->isMovable() || ins->resumePoint() || ins == block->lastIns()) { iter++; continue; } // Move constants with a single use in the current block to the // start of the block. Constants won't be reordered by the logic // below, as they have no inputs. Moving them up as high as // possible can allow their use to be moved up further, though, // and has no cost if the constant is emitted at its use. if (ins->isConstant() && ins->hasOneUse() && ins->usesBegin()->consumer()->block() == *block && !IsFloatingPointType(ins->type())) { iter++; MInstructionIterator targetIter = block->begin(); while (targetIter->isConstant() || targetIter->isInterruptCheck()) { if (*targetIter == ins) break; targetIter++; } MoveBefore(*block, *targetIter, ins); continue; } // Look for inputs where this instruction is the last use of that // input. If we move this instruction up, the input's lifetime will // be shortened, modulo resume point uses (which don't need to be // stored in a register, and can be handled by the register // allocator by just spilling at some point with no reload). Vector<MDefinition*, 4, SystemAllocPolicy> lastUsedInputs; for (size_t i = 0; i < ins->numOperands(); i++) { MDefinition* input = ins->getOperand(i); if (!input->isConstant() && IsLastUse(ins, input, innerLoop)) { if (!lastUsedInputs.append(input)) return false; } } // Don't try to move instructions which aren't the last use of any // of their inputs (we really ought to move these down instead). if (lastUsedInputs.length() < 2) { iter++; continue; } MInstruction* target = ins; for (MInstructionReverseIterator riter = ++block->rbegin(ins); riter != rtop; riter++) { MInstruction* prev = *riter; if (prev->isInterruptCheck()) break; // The instruction can't be moved before any of its uses. bool isUse = false; for (size_t i = 0; i < ins->numOperands(); i++) { if (ins->getOperand(i) == prev) { isUse = true; break; } } if (isUse) break; // The instruction can't be moved before an instruction that // stores to a location read by the instruction. if (prev->isEffectful() && (ins->getAliasSet().flags() & prev->getAliasSet().flags()) && ins->mightAlias(prev) != MDefinition::AliasType::NoAlias) { break; } // Make sure the instruction will still be the last use of one // of its inputs when moved up this far. for (size_t i = 0; i < lastUsedInputs.length(); ) { bool found = false; for (size_t j = 0; j < prev->numOperands(); j++) { if (prev->getOperand(j) == lastUsedInputs[i]) { found = true; break; } } if (found) { lastUsedInputs[i] = lastUsedInputs.back(); lastUsedInputs.popBack(); } else { i++; } } if (lastUsedInputs.length() < 2) break; // We can move the instruction before this one. target = prev; } iter++; MoveBefore(*block, target, ins); } if (block->isLoopBackedge()) loopHeaders.popBack(); } return true; }
bool FilterTypeSetPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) { MOZ_ASSERT(ins->numOperands() == 1); MIRType inputType = ins->getOperand(0)->type(); MIRType outputType = ins->type(); // Special case when output is a Float32, but input isn't. if (outputType == MIRType_Float32 && inputType != MIRType_Float32) { // Create a MToFloat32 to add between the MFilterTypeSet and // its uses. MInstruction* replace = MToFloat32::New(alloc, ins); ins->justReplaceAllUsesWithExcept(replace); ins->block()->insertAfter(ins, replace); // Reset the type to not MIRType_Float32 // Note: setResultType shouldn't happen in TypePolicies, // Here it is fine, since there is just one use we just // added ourself. And the resulting type after MToFloat32 // equals the original type. ins->setResultType(ins->resultTypeSet()->getKnownMIRType()); outputType = ins->type(); // Do the type analysis if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; // Fall through to let the MFilterTypeSet adjust its input based // on its new type. } // Input and output type are already in accordance. if (inputType == outputType) return true; // Output is a value, box the input. if (outputType == MIRType_Value) { MOZ_ASSERT(inputType != MIRType_Value); ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0))); return true; } // The outputType should be a subset of the inputType else we are in code // that has never executed yet. Bail to see the new type (if that hasn't // happened yet). if (inputType != MIRType_Value) { MBail* bail = MBail::New(alloc); ins->block()->insertBefore(ins, bail); bail->setDependency(ins->dependency()); ins->setDependency(bail); ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0))); } // We can't unbox a value to null/undefined/lazyargs. So keep output // also a value. // Note: Using setResultType shouldn't be done in TypePolicies, // Here it is fine, since the type barrier has no uses. if (IsNullOrUndefined(outputType) || outputType == MIRType_MagicOptimizedArguments) { MOZ_ASSERT(!ins->hasDefUses()); ins->setResultType(MIRType_Value); return true; } // Unbox / propagate the right type. MUnbox::Mode mode = MUnbox::Infallible; MInstruction* replace = MUnbox::New(alloc, ins->getOperand(0), ins->type(), mode); ins->block()->insertBefore(ins, replace); ins->replaceOperand(0, replace); if (!replace->typePolicy()->adjustInputs(alloc, replace)) return false; // Carry over the dependency the MFilterTypeSet had. replace->setDependency(ins->dependency()); return true; }
Loop::LoopReturn Loop::init() { IonSpew(IonSpew_LICM, "Loop identified, headed by block %d", header_->id()); IonSpew(IonSpew_LICM, "footer is block %d", header_->backedge()->id()); // The first predecessor of the loop header must dominate the header. JS_ASSERT(header_->id() > header_->getPredecessor(0)->id()); // Loops from backedge to header and marks all visited blocks // as part of the loop. At the same time add all hoistable instructions // (in RPO order) to the instruction worklist. Vector<MBasicBlock *, 1, IonAllocPolicy> inlooplist; if (!inlooplist.append(header_->backedge())) return LoopReturn_Error; header_->backedge()->mark(); while (!inlooplist.empty()) { MBasicBlock *block = inlooplist.back(); // Hoisting requires more finesse if the loop contains a block that // self-dominates: there exists control flow that may enter the loop // without passing through the loop preheader. // // Rather than perform a complicated analysis of the dominance graph, // just return a soft error to ignore this loop. if (block->immediateDominator() == block) { while (!worklist_.empty()) popFromWorklist(); return LoopReturn_Skip; } // Add not yet visited predecessors to the inlooplist. if (block != header_) { for (size_t i = 0; i < block->numPredecessors(); i++) { MBasicBlock *pred = block->getPredecessor(i); if (pred->isMarked()) continue; if (!inlooplist.append(pred)) return LoopReturn_Error; pred->mark(); } } // If any block was added, process them first. if (block != inlooplist.back()) continue; // Add all instructions in this block (but the control instruction) to the worklist for (MInstructionIterator i = block->begin(); i != block->end(); i++) { MInstruction *ins = *i; // Remember whether this loop contains anything which clobbers most // or all floating-point registers. This is just a rough heuristic. if (ins->possiblyCalls()) containsPossibleCall_ = true; if (isHoistable(ins)) { if (!insertInWorklist(ins)) return LoopReturn_Error; } } // All successors of this block are visited. inlooplist.popBack(); } return LoopReturn_Success; }
// This pass annotates every load instruction with the last store instruction // on which it depends. The algorithm is optimistic in that it ignores explicit // dependencies and only considers loads and stores. // // Loads inside loops only have an implicit dependency on a store before the // loop header if no instruction inside the loop body aliases it. To calculate // this efficiently, we maintain a list of maybe-invariant loads and the combined // alias set for all stores inside the loop. When we see the loop's backedge, this // information is used to mark every load we wrongly assumed to be loop invariant as // having an implicit dependency on the last instruction of the loop header, so that // it's never moved before the loop header. // // The algorithm depends on the invariant that both control instructions and effectful // instructions (stores) are never hoisted. bool AliasAnalysis::analyze() { Vector<MInstructionVector, AliasSet::NumCategories, JitAllocPolicy> stores(alloc()); // Initialize to the first instruction. MInstruction* firstIns = *graph_.entryBlock()->begin(); for (unsigned i = 0; i < AliasSet::NumCategories; i++) { MInstructionVector defs(alloc()); if (!defs.append(firstIns)) return false; if (!stores.append(Move(defs))) return false; } // Type analysis may have inserted new instructions. Since this pass depends // on the instruction number ordering, all instructions are renumbered. uint32_t newId = 0; for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) { if (mir->shouldCancel("Alias Analysis (main loop)")) return false; if (block->isLoopHeader()) { JitSpew(JitSpew_Alias, "Processing loop header %d", block->id()); loop_ = new(alloc()) LoopAliasInfo(alloc(), loop_, *block); } for (MPhiIterator def(block->phisBegin()), end(block->phisEnd()); def != end; ++def) def->setId(newId++); for (MInstructionIterator def(block->begin()), end(block->begin(block->lastIns())); def != end; ++def) { def->setId(newId++); AliasSet set = def->getAliasSet(); if (set.isNone()) continue; // For the purposes of alias analysis, all recoverable operations // are treated as effect free as the memory represented by these // operations cannot be aliased by others. if (def->canRecoverOnBailout()) continue; if (set.isStore()) { for (AliasSetIterator iter(set); iter; iter++) { if (!stores[*iter].append(*def)) return false; } if (JitSpewEnabled(JitSpew_Alias)) { Fprinter& out = JitSpewPrinter(); out.printf("Processing store "); def->printName(out); out.printf(" (flags %x)\n", set.flags()); } } else { // Find the most recent store on which this instruction depends. MInstruction* lastStore = firstIns; for (AliasSetIterator iter(set); iter; iter++) { MInstructionVector& aliasedStores = stores[*iter]; for (int i = aliasedStores.length() - 1; i >= 0; i--) { MInstruction* store = aliasedStores[i]; if (genericMightAlias(*def, store) != MDefinition::AliasType::NoAlias && def->mightAlias(store) != MDefinition::AliasType::NoAlias && BlockMightReach(store->block(), *block)) { if (lastStore->id() < store->id()) lastStore = store; break; } } } def->setDependency(lastStore); IonSpewDependency(*def, lastStore, "depends", ""); // If the last store was before the current loop, we assume this load // is loop invariant. If a later instruction writes to the same location, // we will fix this at the end of the loop. if (loop_ && lastStore->id() < loop_->firstInstruction()->id()) { if (!loop_->addInvariantLoad(*def)) return false; } } } // Renumber the last instruction, as the analysis depends on this and the order. block->lastIns()->setId(newId++); if (block->isLoopBackedge()) { MOZ_ASSERT(loop_->loopHeader() == block->loopHeaderOfBackedge()); JitSpew(JitSpew_Alias, "Processing loop backedge %d (header %d)", block->id(), loop_->loopHeader()->id()); LoopAliasInfo* outerLoop = loop_->outer(); MInstruction* firstLoopIns = *loop_->loopHeader()->begin(); const MInstructionVector& invariant = loop_->invariantLoads(); for (unsigned i = 0; i < invariant.length(); i++) { MInstruction* ins = invariant[i]; AliasSet set = ins->getAliasSet(); MOZ_ASSERT(set.isLoad()); bool hasAlias = false; for (AliasSetIterator iter(set); iter; iter++) { MInstructionVector& aliasedStores = stores[*iter]; for (int i = aliasedStores.length() - 1;; i--) { MInstruction* store = aliasedStores[i]; if (store->id() < firstLoopIns->id()) break; if (genericMightAlias(ins, store) != MDefinition::AliasType::NoAlias && ins->mightAlias(store) != MDefinition::AliasType::NoAlias) { hasAlias = true; IonSpewDependency(ins, store, "aliases", "store in loop body"); break; } } if (hasAlias) break; } if (hasAlias) { // This instruction depends on stores inside the loop body. Mark it as having a // dependency on the last instruction of the loop header. The last instruction is a // control instruction and these are never hoisted. MControlInstruction* controlIns = loop_->loopHeader()->lastIns(); IonSpewDependency(ins, controlIns, "depends", "due to stores in loop body"); ins->setDependency(controlIns); } else { IonSpewAliasInfo("Load", ins, "does not depend on any stores in this loop"); if (outerLoop && ins->dependency()->id() < outerLoop->firstInstruction()->id()) { IonSpewAliasInfo("Load", ins, "may be invariant in outer loop"); if (!outerLoop->addInvariantLoad(ins)) return false; } } } loop_ = loop_->outer(); } } spewDependencyList(); MOZ_ASSERT(loop_ == nullptr); return true; }
static void AnalyzeLsh(TempAllocator& alloc, MLsh* lsh) { if (lsh->specialization() != MIRType::Int32) return; if (lsh->isRecoveredOnBailout()) return; MDefinition* index = lsh->lhs(); MOZ_ASSERT(index->type() == MIRType::Int32); MConstant* shiftValue = lsh->rhs()->maybeConstantValue(); if (!shiftValue) return; if (shiftValue->type() != MIRType::Int32 || !IsShiftInScaleRange(shiftValue->toInt32())) return; Scale scale = ShiftToScale(shiftValue->toInt32()); int32_t displacement = 0; MInstruction* last = lsh; MDefinition* base = nullptr; while (true) { if (!last->hasOneUse()) break; MUseIterator use = last->usesBegin(); if (!use->consumer()->isDefinition() || !use->consumer()->toDefinition()->isAdd()) break; MAdd* add = use->consumer()->toDefinition()->toAdd(); if (add->specialization() != MIRType::Int32 || !add->isTruncated()) break; MDefinition* other = add->getOperand(1 - add->indexOf(*use)); if (MConstant* otherConst = other->maybeConstantValue()) { displacement += otherConst->toInt32(); } else { if (base) break; base = other; } last = add; if (last->isRecoveredOnBailout()) return; } if (!base) { uint32_t elemSize = 1 << ScaleToShift(scale); if (displacement % elemSize != 0) return; if (!last->hasOneUse()) return; MUseIterator use = last->usesBegin(); if (!use->consumer()->isDefinition() || !use->consumer()->toDefinition()->isBitAnd()) return; MBitAnd* bitAnd = use->consumer()->toDefinition()->toBitAnd(); if (bitAnd->isRecoveredOnBailout()) return; MDefinition* other = bitAnd->getOperand(1 - bitAnd->indexOf(*use)); MConstant* otherConst = other->maybeConstantValue(); if (!otherConst || otherConst->type() != MIRType::Int32) return; uint32_t bitsClearedByShift = elemSize - 1; uint32_t bitsClearedByMask = ~uint32_t(otherConst->toInt32()); if ((bitsClearedByShift & bitsClearedByMask) != bitsClearedByMask) return; bitAnd->replaceAllUsesWith(last); return; } if (base->isRecoveredOnBailout()) return; MEffectiveAddress* eaddr = MEffectiveAddress::New(alloc, base, index, scale, displacement); last->replaceAllUsesWith(eaddr); last->block()->insertAfter(last, eaddr); }
bool Sink(MIRGenerator* mir, MIRGraph& graph) { TempAllocator& alloc = graph.alloc(); bool sinkEnabled = mir->optimizationInfo().sinkEnabled(); for (PostorderIterator block = graph.poBegin(); block != graph.poEnd(); block++) { if (mir->shouldCancel("Sink")) return false; for (MInstructionReverseIterator iter = block->rbegin(); iter != block->rend(); ) { MInstruction* ins = *iter++; // Only instructions which can be recovered on bailout can be moved // into the bailout paths. if (ins->isGuard() || ins->isGuardRangeBailouts() || ins->isRecoveredOnBailout() || !ins->canRecoverOnBailout()) { continue; } // Compute a common dominator for all uses of the current // instruction. bool hasLiveUses = false; bool hasUses = false; MBasicBlock* usesDominator = nullptr; for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; i++) { hasUses = true; MNode* consumerNode = (*i)->consumer(); if (consumerNode->isResumePoint()) continue; MDefinition* consumer = consumerNode->toDefinition(); if (consumer->isRecoveredOnBailout()) continue; hasLiveUses = true; // If the instruction is a Phi, then we should dominate the // predecessor from which the value is coming from. MBasicBlock* consumerBlock = consumer->block(); if (consumer->isPhi()) consumerBlock = consumerBlock->getPredecessor(consumer->indexOf(*i)); usesDominator = CommonDominator(usesDominator, consumerBlock); if (usesDominator == *block) break; } // Leave this instruction for DCE. if (!hasUses) continue; // We have no uses, so sink this instruction in all the bailout // paths. if (!hasLiveUses) { MOZ_ASSERT(!usesDominator); ins->setRecoveredOnBailout(); JitSpewDef(JitSpew_Sink, " No live uses, recover the instruction on bailout\n", ins); continue; } // This guard is temporarly moved here as the above code deals with // Dead Code elimination, which got moved into this Sink phase, as // the Dead Code elimination used to move instructions with no-live // uses to the bailout path. if (!sinkEnabled) continue; // To move an effectful instruction, we would have to verify that the // side-effect is not observed. In the mean time, we just inhibit // this optimization on effectful instructions. if (ins->isEffectful()) continue; // If all the uses are under a loop, we might not want to work // against LICM by moving everything back into the loop, but if the // loop is it-self inside an if, then we still want to move the // computation under this if statement. while (block->loopDepth() < usesDominator->loopDepth()) { MOZ_ASSERT(usesDominator != usesDominator->immediateDominator()); usesDominator = usesDominator->immediateDominator(); } // Only move instructions if there is a branch between the dominator // of the uses and the original instruction. This prevent moving the // computation of the arguments into an inline function if there is // no major win. MBasicBlock* lastJoin = usesDominator; while (*block != lastJoin && lastJoin->numPredecessors() == 1) { MOZ_ASSERT(lastJoin != lastJoin->immediateDominator()); MBasicBlock* next = lastJoin->immediateDominator(); if (next->numSuccessors() > 1) break; lastJoin = next; } if (*block == lastJoin) continue; // Skip to the next instruction if we cannot find a common dominator // for all the uses of this instruction, or if the common dominator // correspond to the block of the current instruction. if (!usesDominator || usesDominator == *block) continue; // Only instruction which can be recovered on bailout and which are // sinkable can be moved into blocks which are below while filling // the resume points with a clone which is recovered on bailout. // If the instruction has live uses and if it is clonable, then we // can clone the instruction for all non-dominated uses and move the // instruction into the block which is dominating all live uses. if (!ins->canClone()) continue; // If the block is a split-edge block, which is created for folding // test conditions, then the block has no resume point and has // multiple predecessors. In such case, we cannot safely move // bailing instruction to these blocks as we have no way to bailout. if (!usesDominator->entryResumePoint() && usesDominator->numPredecessors() != 1) continue; JitSpewDef(JitSpew_Sink, " Can Clone & Recover, sink instruction\n", ins); JitSpew(JitSpew_Sink, " into Block %u", usesDominator->id()); // Copy the arguments and clone the instruction. MDefinitionVector operands(alloc); for (size_t i = 0, end = ins->numOperands(); i < end; i++) { if (!operands.append(ins->getOperand(i))) return false; } MInstruction* clone = ins->clone(alloc, operands); ins->block()->insertBefore(ins, clone); clone->setRecoveredOnBailout(); // We should not update the producer of the entry resume point, as // it cannot refer to any instruction within the basic block excepts // for Phi nodes. MResumePoint* entry = usesDominator->entryResumePoint(); // Replace the instruction by its clone in all the resume points / // recovered-on-bailout instructions which are not in blocks which // are dominated by the usesDominator block. for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; ) { MUse* use = *i++; MNode* consumer = use->consumer(); // If the consumer is a Phi, then we look for the index of the // use to find the corresponding predecessor block, which is // then used as the consumer block. MBasicBlock* consumerBlock = consumer->block(); if (consumer->isDefinition() && consumer->toDefinition()->isPhi()) { consumerBlock = consumerBlock->getPredecessor( consumer->toDefinition()->toPhi()->indexOf(use)); } // Keep the current instruction for all dominated uses, except // for the entry resume point of the block in which the // instruction would be moved into. if (usesDominator->dominates(consumerBlock) && (!consumer->isResumePoint() || consumer->toResumePoint() != entry)) { continue; } use->replaceProducer(clone); } // As we move this instruction in a different block, we should // verify that we do not carry over a resume point which would refer // to an outdated state of the control flow. if (ins->resumePoint()) ins->clearResumePoint(); // Now, that all uses which are not dominated by usesDominator are // using the cloned instruction, we can safely move the instruction // into the usesDominator block. MInstruction* at = usesDominator->safeInsertTop(nullptr, MBasicBlock::IgnoreRecover); block->moveBefore(at, ins); } } return true; }
// Remove the CFG edge between |pred| and |block|, and if this makes |block| // unreachable, mark it so, and remove the rest of its incoming edges too. And // discard any instructions made dead by the entailed release of any phi // operands. bool ValueNumberer::removePredecessorAndCleanUp(MBasicBlock* block, MBasicBlock* pred) { MOZ_ASSERT(!block->isMarked(), "Removing predecessor on block already marked unreachable"); // We'll be removing a predecessor, so anything we know about phis in this // block will be wrong. for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd()); iter != end; ++iter) values_.forget(*iter); // If this is a loop header, test whether it will become an unreachable // loop, or whether it needs special OSR-related fixups. bool isUnreachableLoop = false; MBasicBlock* origBackedgeForOSRFixup = nullptr; if (block->isLoopHeader()) { if (block->loopPredecessor() == pred) { if (MOZ_UNLIKELY(hasNonDominatingPredecessor(block, pred))) { JitSpew(JitSpew_GVN, " " "Loop with header block%u is now only reachable through an " "OSR entry into the middle of the loop!!", block->id()); origBackedgeForOSRFixup = block->backedge(); } else { // Deleting the entry into the loop makes the loop unreachable. isUnreachableLoop = true; JitSpew(JitSpew_GVN, " " "Loop with header block%u is no longer reachable", block->id()); } #ifdef DEBUG } else if (block->hasUniqueBackedge() && block->backedge() == pred) { JitSpew(JitSpew_GVN, " Loop with header block%u is no longer a loop", block->id()); #endif } } // Actually remove the CFG edge. if (!removePredecessorAndDoDCE(block, pred, block->getPredecessorIndex(pred))) return false; // We've now edited the CFG; check to see if |block| became unreachable. if (block->numPredecessors() == 0 || isUnreachableLoop) { JitSpew(JitSpew_GVN, " Disconnecting block%u", block->id()); // Remove |block| from its dominator parent's subtree. This is the only // immediately-dominated-block information we need to update, because // everything dominated by this block is about to be swept away. MBasicBlock* parent = block->immediateDominator(); if (parent != block) parent->removeImmediatelyDominatedBlock(block); // Completely disconnect it from the CFG. We do this now rather than // just doing it later when we arrive there in visitUnreachableBlock // so that we don't leave a partially broken loop sitting around. This // also lets visitUnreachableBlock assert that numPredecessors() == 0, // which is a nice invariant. if (block->isLoopHeader()) block->clearLoopHeader(); for (size_t i = 0, e = block->numPredecessors(); i < e; ++i) { if (!removePredecessorAndDoDCE(block, block->getPredecessor(i), i)) return false; } // Clear out the resume point operands, as they can hold things that // don't appear to dominate them live. if (MResumePoint* resume = block->entryResumePoint()) { if (!releaseResumePointOperands(resume) || !processDeadDefs()) return false; if (MResumePoint* outer = block->outerResumePoint()) { if (!releaseResumePointOperands(outer) || !processDeadDefs()) return false; } MOZ_ASSERT(nextDef_ == nullptr); for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ) { MInstruction* ins = *iter++; nextDef_ = *iter; if (MResumePoint* resume = ins->resumePoint()) { if (!releaseResumePointOperands(resume) || !processDeadDefs()) return false; } } nextDef_ = nullptr; } else { #ifdef DEBUG MOZ_ASSERT(block->outerResumePoint() == nullptr, "Outer resume point in block without an entry resume point"); for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ++iter) { MOZ_ASSERT(iter->resumePoint() == nullptr, "Instruction with resume point in block without entry resume point"); } #endif } // Use the mark to note that we've already removed all its predecessors, // and we know it's unreachable. block->mark(); } else if (MOZ_UNLIKELY(origBackedgeForOSRFixup != nullptr)) { // The loop is no only reachable through OSR into the middle. Fix it // up so that the CFG can remain valid. if (!fixupOSROnlyLoop(block, origBackedgeForOSRFixup)) return false; } return true; }