static void VisitLoopBlock(MBasicBlock *block, MBasicBlock *header, MInstruction *hoistPoint, bool hasCalls) { for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ) { MInstruction *ins = *insIter++; if (!IsHoistable(ins, header, hasCalls)) { #ifdef DEBUG if (IsHoistableIgnoringDependency(ins, hasCalls)) { JitSpew(JitSpew_LICM, " %s%u isn't hoistable due to dependency on %s%u", ins->opName(), ins->id(), ins->dependency()->opName(), ins->dependency()->id()); } #endif continue; } // Don't hoist a cheap constant if it doesn't enable us to hoist one of // its uses. We want those instructions as close as possible to their // use, to minimize register pressure. if (RequiresHoistedUse(ins, hasCalls)) { JitSpew(JitSpew_LICM, " %s%u will be hoisted only if its users are", ins->opName(), ins->id()); continue; } // Hoist operands which were too cheap to hoist on their own. MoveDeferredOperands(ins, hoistPoint, hasCalls); JitSpew(JitSpew_LICM, " Hoisting %s%u", ins->opName(), ins->id()); // Move the instruction to the hoistPoint. block->moveBefore(hoistPoint, ins); } }
static void VisitLoop(MIRGraph &graph, MBasicBlock *header) { MInstruction *hoistPoint = header->loopPredecessor()->lastIns(); JitSpew(JitSpew_LICM, " Visiting loop with header block%u, hoisting to %s%u", header->id(), hoistPoint->opName(), hoistPoint->id()); MBasicBlock *backedge = header->backedge(); // This indicates whether the loop contains calls or other things which // clobber most or all floating-point registers. In such loops, // floating-point constants should not be hoisted unless it enables further // hoisting. bool hasCalls = LoopContainsPossibleCall(graph, header, backedge); for (auto i(graph.rpoBegin(header)); ; ++i) { MOZ_ASSERT(i != graph.rpoEnd(), "Reached end of graph searching for blocks in loop"); MBasicBlock *block = *i; if (!block->isMarked()) continue; VisitLoopBlock(block, header, hoistPoint, hasCalls); if (block == backedge) break; } }
// In preparation for hoisting an instruction, hoist any of its operands which // were too cheap to hoist on their own. static void MoveDeferredOperands(MInstruction* ins, MInstruction* hoistPoint, bool hasCalls) { // If any of our operands were waiting for a user to be hoisted, make a note // to hoist them. for (size_t i = 0, e = ins->numOperands(); i != e; ++i) { MDefinition* op = ins->getOperand(i); if (!IsInLoop(op)) continue; MOZ_ASSERT(RequiresHoistedUse(op, hasCalls), "Deferred loop-invariant operand is not cheap"); MInstruction* opIns = op->toInstruction(); // Recursively move the operands. Note that the recursion is bounded // because we require RequiresHoistedUse to be set at each level. MoveDeferredOperands(opIns, hoistPoint, hasCalls); #ifdef DEBUG JitSpew(JitSpew_LICM, " Hoisting %s%u (now that a user will be hoisted)", opIns->opName(), opIns->id()); #endif opIns->block()->moveBefore(hoistPoint, opIns); } }
// Fold AddIs with one variable and two or more constants into one AddI. static void AnalyzeAdd(TempAllocator& alloc, MAdd* add) { if (add->specialization() != MIRType::Int32 || add->isRecoveredOnBailout()) { return; } if (!add->hasUses()) { return; } JitSpew(JitSpew_FLAC, "analyze add: %s%u", add->opName(), add->id()); SimpleLinearSum sum = ExtractLinearSum(add); if (sum.constant == 0 || !sum.term) { return; } // Determine which operand is the constant. int idx = add->getOperand(0)->isConstant() ? 0 : 1; if (add->getOperand(idx)->isConstant()) { // Do not replace an add where the outcome is the same add instruction. MOZ_ASSERT(add->getOperand(idx)->toConstant()->type() == MIRType::Int32); if (sum.term == add->getOperand(1 - idx) || sum.constant == add->getOperand(idx)->toConstant()->toInt32()) { return; } } MInstruction* rhs = MConstant::New(alloc, Int32Value(sum.constant)); add->block()->insertBefore(add, rhs); MAdd* addNew = MAdd::New(alloc, sum.term, rhs, MIRType::Int32, add->truncateKind()); add->replaceAllLiveUsesWith(addNew); add->block()->insertBefore(add, addNew); JitSpew(JitSpew_FLAC, "replaced with: %s%u", addNew->opName(), addNew->id()); JitSpew(JitSpew_FLAC, "and constant: %s%u (%d)", rhs->opName(), rhs->id(), sum.constant); // Mark the stale nodes as RecoveredOnBailout since the Sink pass has // been run before this pass. DCE will then remove the unused nodes. markNodesAsRecoveredOnBailout(add); }
// Test whether any instruction in the loop possiblyCalls(). static bool LoopContainsPossibleCall(MIRGraph &graph, MBasicBlock *header, MBasicBlock *backedge) { for (auto i(graph.rpoBegin(header)); ; ++i) { MOZ_ASSERT(i != graph.rpoEnd(), "Reached end of graph searching for blocks in loop"); MBasicBlock *block = *i; if (!block->isMarked()) continue; for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd; ++insIter) { MInstruction *ins = *insIter; if (ins->possiblyCalls()) { JitSpew(JitSpew_LICM, " Possile call found at %s%u", ins->opName(), ins->id()); return true; } } if (block == backedge) break; } return false; }
void LoopUnroller::go(LoopIterationBound *bound) { // For now we always unroll loops the same number of times. static const size_t UnrollCount = 10; JitSpew(JitSpew_Unrolling, "Attempting to unroll loop"); header = bound->header; // UCE might have determined this isn't actually a loop. if (!header->isLoopHeader()) return; backedge = header->backedge(); oldPreheader = header->loopPredecessor(); JS_ASSERT(oldPreheader->numSuccessors() == 1); // Only unroll loops with two blocks: an initial one ending with the // bound's test, and the body ending with the backedge. MTest *test = bound->test; if (header->lastIns() != test) return; if (test->ifTrue() == backedge) { if (test->ifFalse()->id() <= backedge->id()) return; } else if (test->ifFalse() == backedge) { if (test->ifTrue()->id() <= backedge->id()) return; } else { return; } if (backedge->numPredecessors() != 1 || backedge->numSuccessors() != 1) return; JS_ASSERT(backedge->phisEmpty()); MBasicBlock *bodyBlocks[] = { header, backedge }; // All instructions in the header and body must be clonable. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock *block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction *ins = *iter; if (ins->canClone()) continue; if (ins->isTest() || ins->isGoto() || ins->isInterruptCheck()) continue; #ifdef DEBUG JitSpew(JitSpew_Unrolling, "Aborting: can't clone instruction %s", ins->opName()); #endif return; } } // Compute the linear inequality we will use for exiting the unrolled loop: // // iterationBound - iterationCount - UnrollCount >= 0 // LinearSum remainingIterationsInequality(bound->boundSum); if (!remainingIterationsInequality.add(bound->currentSum, -1)) return; if (!remainingIterationsInequality.add(-int32_t(UnrollCount))) return; // Terms in the inequality need to be either loop invariant or phis from // the original header. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition *def = remainingIterationsInequality.term(i).term; if (def->block()->id() < header->id()) continue; if (def->block() == header && def->isPhi()) continue; return; } // OK, we've checked everything, now unroll the loop. JitSpew(JitSpew_Unrolling, "Unrolling loop"); // The old preheader will go before the unrolled loop, and the old loop // will need a new empty preheader. CompileInfo &info = oldPreheader->info(); if (header->trackedSite().pc()) { unrolledHeader = MBasicBlock::New(graph, nullptr, info, oldPreheader, header->trackedSite(), MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::New(graph, nullptr, info, unrolledHeader, backedge->trackedSite(), MBasicBlock::NORMAL); newPreheader = MBasicBlock::New(graph, nullptr, info, unrolledHeader, oldPreheader->trackedSite(), MBasicBlock::NORMAL); } else { unrolledHeader = MBasicBlock::NewAsmJS(graph, info, oldPreheader, MBasicBlock::LOOP_HEADER); unrolledBackedge = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); newPreheader = MBasicBlock::NewAsmJS(graph, info, unrolledHeader, MBasicBlock::NORMAL); } unrolledHeader->discardAllResumePoints(); unrolledBackedge->discardAllResumePoints(); newPreheader->discardAllResumePoints(); // Insert new blocks at their RPO position, and update block ids. graph.insertBlockAfter(oldPreheader, unrolledHeader); graph.insertBlockAfter(unrolledHeader, unrolledBackedge); graph.insertBlockAfter(unrolledBackedge, newPreheader); graph.renumberBlocksAfter(oldPreheader); if (!unrolledDefinitions.init()) CrashAtUnhandlableOOM("LoopUnroller::go"); // Add phis to the unrolled loop header which correspond to the phis in the // original loop header. JS_ASSERT(header->getPredecessor(0) == oldPreheader); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; JS_ASSERT(old->numOperands() == 2); MPhi *phi = MPhi::New(alloc); phi->setResultType(old->type()); phi->setResultTypeSet(old->resultTypeSet()); phi->setRange(old->range()); unrolledHeader->addPhi(phi); if (!phi->reserveLength(2)) CrashAtUnhandlableOOM("LoopUnroller::go"); // Set the first input for the phi for now. We'll set the second after // finishing the unroll. phi->addInput(old->getOperand(0)); // The old phi will now take the value produced by the unrolled loop. old->replaceOperand(0, phi); if (!unrolledDefinitions.putNew(old, phi)) CrashAtUnhandlableOOM("LoopUnroller::go"); } // The loop condition can bail out on e.g. integer overflow, so make a // resume point based on the initial resume point of the original header. MResumePoint *headerResumePoint = header->entryResumePoint(); if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(unrolledHeader, headerResumePoint); unrolledHeader->setEntryResumePoint(rp); // Perform an interrupt check at the start of the unrolled loop. unrolledHeader->add(MInterruptCheck::New(alloc)); } // Generate code for the test in the unrolled loop. for (size_t i = 0; i < remainingIterationsInequality.numTerms(); i++) { MDefinition *def = remainingIterationsInequality.term(i).term; MDefinition *replacement = getReplacementDefinition(def); remainingIterationsInequality.replaceTerm(i, replacement); } MCompare *compare = ConvertLinearInequality(alloc, unrolledHeader, remainingIterationsInequality); MTest *unrolledTest = MTest::New(alloc, compare, unrolledBackedge, newPreheader); unrolledHeader->end(unrolledTest); // Make an entry resume point for the unrolled body. The unrolled header // does not have side effects on stack values, even if the original loop // header does, so use the same resume point as for the unrolled header. if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(unrolledBackedge, headerResumePoint); unrolledBackedge->setEntryResumePoint(rp); } // Make an entry resume point for the new preheader. There are no // instructions which use this but some other stuff wants one to be here. if (headerResumePoint) { MResumePoint *rp = makeReplacementResumePoint(newPreheader, headerResumePoint); newPreheader->setEntryResumePoint(rp); } // Generate the unrolled code. JS_ASSERT(UnrollCount > 1); size_t unrollIndex = 0; while (true) { // Clone the contents of the original loop into the unrolled loop body. for (size_t i = 0; i < ArrayLength(bodyBlocks); i++) { MBasicBlock *block = bodyBlocks[i]; for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) { MInstruction *ins = *iter; if (ins->canClone()) { makeReplacementInstruction(*iter); } else { // Control instructions are handled separately. JS_ASSERT(ins->isTest() || ins->isGoto() || ins->isInterruptCheck()); } } } // Compute the value of each loop header phi after the execution of // this unrolled iteration. MDefinitionVector phiValues(alloc); JS_ASSERT(header->getPredecessor(1) == backedge); for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; MDefinition *oldInput = old->getOperand(1); if (!phiValues.append(getReplacementDefinition(oldInput))) CrashAtUnhandlableOOM("LoopUnroller::go"); } unrolledDefinitions.clear(); if (unrollIndex == UnrollCount - 1) { // We're at the end of the last unrolled iteration, set the // backedge input for the unrolled loop phis. size_t phiIndex = 0; for (MPhiIterator iter(unrolledHeader->phisBegin()); iter != unrolledHeader->phisEnd(); iter++) { MPhi *phi = *iter; phi->addInput(phiValues[phiIndex++]); } JS_ASSERT(phiIndex == phiValues.length()); break; } // Update the map for the phis in the next iteration. size_t phiIndex = 0; for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd(); iter++) { MPhi *old = *iter; if (!unrolledDefinitions.putNew(old, phiValues[phiIndex++])) CrashAtUnhandlableOOM("LoopUnroller::go"); } JS_ASSERT(phiIndex == phiValues.length()); unrollIndex++; } MGoto *backedgeJump = MGoto::New(alloc, unrolledHeader); unrolledBackedge->end(backedgeJump); // Place the old preheader before the unrolled loop. JS_ASSERT(oldPreheader->lastIns()->isGoto()); oldPreheader->discardLastIns(); oldPreheader->end(MGoto::New(alloc, unrolledHeader)); // Place the new preheader before the original loop. newPreheader->end(MGoto::New(alloc, header)); // Cleanup the MIR graph. if (!unrolledHeader->addPredecessorWithoutPhis(unrolledBackedge)) CrashAtUnhandlableOOM("LoopUnroller::go"); header->replacePredecessor(oldPreheader, newPreheader); oldPreheader->setSuccessorWithPhis(unrolledHeader, 0); newPreheader->setSuccessorWithPhis(header, 0); unrolledBackedge->setSuccessorWithPhis(unrolledHeader, 1); }