bool Loop::hoistInstructions(InstructionQueue &toHoist) { // Iterate in post-order (uses before definitions) for (int32_t i = toHoist.length() - 1; i >= 0; i--) { MInstruction *ins = toHoist[i]; // Don't hoist MConstantElements, MConstant and MBox // if it doesn't enable us to hoist one of its uses. // We want those instructions as close as possible to their use. if (ins->isConstantElements() || ins->isConstant() || ins->isBox()) { bool loopInvariantUse = false; for (MUseDefIterator use(ins); use; use++) { if (use.def()->isLoopInvariant()) { loopInvariantUse = true; break; } } if (!loopInvariantUse) ins->setNotLoopInvariant(); } } // Move all instructions to the preLoop_ block just before the control instruction. for (size_t i = 0; i < toHoist.length(); i++) { MInstruction *ins = toHoist[i]; // Loads may have an implicit dependency on either stores (effectful instructions) or // control instructions so we should never move these. JS_ASSERT(!ins->isControlInstruction()); JS_ASSERT(!ins->isEffectful()); JS_ASSERT(ins->isMovable()); if (!ins->isLoopInvariant()) continue; if (checkHotness(ins->block())) { ins->block()->moveBefore(preLoop_->lastIns(), ins); ins->setNotLoopInvariant(); } } return true; }
Loop::LoopReturn Loop::iterateLoopBlocks(MBasicBlock *current) { // Visited. current->mark(); // Hoisting requires more finesse if the loop contains a block that // self-dominates: there exists control flow that may enter the loop // without passing through the loop preheader. // // Rather than perform a complicated analysis of the dominance graph, // just return a soft error to ignore this loop. if (current->immediateDominator() == current) return LoopReturn_Skip; // If we haven't reached the loop header yet, recursively explore predecessors // if we haven't seen them already. if (current != header_) { for (size_t i = 0; i < current->numPredecessors(); i++) { if (current->getPredecessor(i)->isMarked()) continue; LoopReturn lr = iterateLoopBlocks(current->getPredecessor(i)); if (lr != LoopReturn_Success) return lr; } } // Add all instructions in this block (but the control instruction) to the worklist for (MInstructionIterator i = current->begin(); i != current->end(); i++) { MInstruction *ins = *i; if (ins->isMovable() && !ins->isEffectful()) { if (!insertInWorklist(ins)) return LoopReturn_Error; } } return LoopReturn_Success; }
bool Sink(MIRGenerator* mir, MIRGraph& graph) { TempAllocator& alloc = graph.alloc(); bool sinkEnabled = mir->optimizationInfo().sinkEnabled(); for (PostorderIterator block = graph.poBegin(); block != graph.poEnd(); block++) { if (mir->shouldCancel("Sink")) return false; for (MInstructionReverseIterator iter = block->rbegin(); iter != block->rend(); ) { MInstruction* ins = *iter++; // Only instructions which can be recovered on bailout can be moved // into the bailout paths. if (ins->isGuard() || ins->isGuardRangeBailouts() || ins->isRecoveredOnBailout() || !ins->canRecoverOnBailout()) { continue; } // Compute a common dominator for all uses of the current // instruction. bool hasLiveUses = false; bool hasUses = false; MBasicBlock* usesDominator = nullptr; for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; i++) { hasUses = true; MNode* consumerNode = (*i)->consumer(); if (consumerNode->isResumePoint()) continue; MDefinition* consumer = consumerNode->toDefinition(); if (consumer->isRecoveredOnBailout()) continue; hasLiveUses = true; // If the instruction is a Phi, then we should dominate the // predecessor from which the value is coming from. MBasicBlock* consumerBlock = consumer->block(); if (consumer->isPhi()) consumerBlock = consumerBlock->getPredecessor(consumer->indexOf(*i)); usesDominator = CommonDominator(usesDominator, consumerBlock); if (usesDominator == *block) break; } // Leave this instruction for DCE. if (!hasUses) continue; // We have no uses, so sink this instruction in all the bailout // paths. if (!hasLiveUses) { MOZ_ASSERT(!usesDominator); ins->setRecoveredOnBailout(); JitSpewDef(JitSpew_Sink, " No live uses, recover the instruction on bailout\n", ins); continue; } // This guard is temporarly moved here as the above code deals with // Dead Code elimination, which got moved into this Sink phase, as // the Dead Code elimination used to move instructions with no-live // uses to the bailout path. if (!sinkEnabled) continue; // To move an effectful instruction, we would have to verify that the // side-effect is not observed. In the mean time, we just inhibit // this optimization on effectful instructions. if (ins->isEffectful()) continue; // If all the uses are under a loop, we might not want to work // against LICM by moving everything back into the loop, but if the // loop is it-self inside an if, then we still want to move the // computation under this if statement. while (block->loopDepth() < usesDominator->loopDepth()) { MOZ_ASSERT(usesDominator != usesDominator->immediateDominator()); usesDominator = usesDominator->immediateDominator(); } // Only move instructions if there is a branch between the dominator // of the uses and the original instruction. This prevent moving the // computation of the arguments into an inline function if there is // no major win. MBasicBlock* lastJoin = usesDominator; while (*block != lastJoin && lastJoin->numPredecessors() == 1) { MOZ_ASSERT(lastJoin != lastJoin->immediateDominator()); MBasicBlock* next = lastJoin->immediateDominator(); if (next->numSuccessors() > 1) break; lastJoin = next; } if (*block == lastJoin) continue; // Skip to the next instruction if we cannot find a common dominator // for all the uses of this instruction, or if the common dominator // correspond to the block of the current instruction. if (!usesDominator || usesDominator == *block) continue; // Only instruction which can be recovered on bailout and which are // sinkable can be moved into blocks which are below while filling // the resume points with a clone which is recovered on bailout. // If the instruction has live uses and if it is clonable, then we // can clone the instruction for all non-dominated uses and move the // instruction into the block which is dominating all live uses. if (!ins->canClone()) continue; // If the block is a split-edge block, which is created for folding // test conditions, then the block has no resume point and has // multiple predecessors. In such case, we cannot safely move // bailing instruction to these blocks as we have no way to bailout. if (!usesDominator->entryResumePoint() && usesDominator->numPredecessors() != 1) continue; JitSpewDef(JitSpew_Sink, " Can Clone & Recover, sink instruction\n", ins); JitSpew(JitSpew_Sink, " into Block %u", usesDominator->id()); // Copy the arguments and clone the instruction. MDefinitionVector operands(alloc); for (size_t i = 0, end = ins->numOperands(); i < end; i++) { if (!operands.append(ins->getOperand(i))) return false; } MInstruction* clone = ins->clone(alloc, operands); ins->block()->insertBefore(ins, clone); clone->setRecoveredOnBailout(); // We should not update the producer of the entry resume point, as // it cannot refer to any instruction within the basic block excepts // for Phi nodes. MResumePoint* entry = usesDominator->entryResumePoint(); // Replace the instruction by its clone in all the resume points / // recovered-on-bailout instructions which are not in blocks which // are dominated by the usesDominator block. for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; ) { MUse* use = *i++; MNode* consumer = use->consumer(); // If the consumer is a Phi, then we look for the index of the // use to find the corresponding predecessor block, which is // then used as the consumer block. MBasicBlock* consumerBlock = consumer->block(); if (consumer->isDefinition() && consumer->toDefinition()->isPhi()) { consumerBlock = consumerBlock->getPredecessor( consumer->toDefinition()->toPhi()->indexOf(use)); } // Keep the current instruction for all dominated uses, except // for the entry resume point of the block in which the // instruction would be moved into. if (usesDominator->dominates(consumerBlock) && (!consumer->isResumePoint() || consumer->toResumePoint() != entry)) { continue; } use->replaceProducer(clone); } // As we move this instruction in a different block, we should // verify that we do not carry over a resume point which would refer // to an outdated state of the control flow. if (ins->resumePoint()) ins->clearResumePoint(); // Now, that all uses which are not dominated by usesDominator are // using the cloned instruction, we can safely move the instruction // into the usesDominator block. MInstruction* at = usesDominator->safeInsertTop(nullptr, MBasicBlock::IgnoreRecover); block->moveBefore(at, ins); } } return true; }
bool jit::ReorderInstructions(MIRGraph& graph) { // Renumber all instructions in the graph as we go. size_t nextId = 0; // List of the headers of any loops we are in. Vector<MBasicBlock*, 4, SystemAllocPolicy> loopHeaders; for (ReversePostorderIterator block(graph.rpoBegin()); block != graph.rpoEnd(); block++) { // Renumber all definitions inside the basic blocks. for (MPhiIterator iter(block->phisBegin()); iter != block->phisEnd(); iter++) iter->setId(nextId++); for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++) iter->setId(nextId++); // Don't reorder instructions within entry blocks, which have special requirements. if (*block == graph.entryBlock() || *block == graph.osrBlock()) continue; if (block->isLoopHeader()) { if (!loopHeaders.append(*block)) return false; } MBasicBlock* innerLoop = loopHeaders.empty() ? nullptr : loopHeaders.back(); MInstruction* top = block->safeInsertTop(); MInstructionReverseIterator rtop = ++block->rbegin(top); for (MInstructionIterator iter(block->begin(top)); iter != block->end(); ) { MInstruction* ins = *iter; // Filter out some instructions which are never reordered. if (ins->isEffectful() || !ins->isMovable() || ins->resumePoint() || ins == block->lastIns()) { iter++; continue; } // Move constants with a single use in the current block to the // start of the block. Constants won't be reordered by the logic // below, as they have no inputs. Moving them up as high as // possible can allow their use to be moved up further, though, // and has no cost if the constant is emitted at its use. if (ins->isConstant() && ins->hasOneUse() && ins->usesBegin()->consumer()->block() == *block && !IsFloatingPointType(ins->type())) { iter++; MInstructionIterator targetIter = block->begin(); while (targetIter->isConstant() || targetIter->isInterruptCheck()) { if (*targetIter == ins) break; targetIter++; } MoveBefore(*block, *targetIter, ins); continue; } // Look for inputs where this instruction is the last use of that // input. If we move this instruction up, the input's lifetime will // be shortened, modulo resume point uses (which don't need to be // stored in a register, and can be handled by the register // allocator by just spilling at some point with no reload). Vector<MDefinition*, 4, SystemAllocPolicy> lastUsedInputs; for (size_t i = 0; i < ins->numOperands(); i++) { MDefinition* input = ins->getOperand(i); if (!input->isConstant() && IsLastUse(ins, input, innerLoop)) { if (!lastUsedInputs.append(input)) return false; } } // Don't try to move instructions which aren't the last use of any // of their inputs (we really ought to move these down instead). if (lastUsedInputs.length() < 2) { iter++; continue; } MInstruction* target = ins; for (MInstructionReverseIterator riter = ++block->rbegin(ins); riter != rtop; riter++) { MInstruction* prev = *riter; if (prev->isInterruptCheck()) break; // The instruction can't be moved before any of its uses. bool isUse = false; for (size_t i = 0; i < ins->numOperands(); i++) { if (ins->getOperand(i) == prev) { isUse = true; break; } } if (isUse) break; // The instruction can't be moved before an instruction that // stores to a location read by the instruction. if (prev->isEffectful() && (ins->getAliasSet().flags() & prev->getAliasSet().flags()) && ins->mightAlias(prev) != MDefinition::AliasType::NoAlias) { break; } // Make sure the instruction will still be the last use of one // of its inputs when moved up this far. for (size_t i = 0; i < lastUsedInputs.length(); ) { bool found = false; for (size_t j = 0; j < prev->numOperands(); j++) { if (prev->getOperand(j) == lastUsedInputs[i]) { found = true; break; } } if (found) { lastUsedInputs[i] = lastUsedInputs.back(); lastUsedInputs.popBack(); } else { i++; } } if (lastUsedInputs.length() < 2) break; // We can move the instruction before this one. target = prev; } iter++; MoveBefore(*block, target, ins); } if (block->isLoopBackedge()) loopHeaders.popBack(); } return true; }
bool Loop::hoistInstructions(InstructionQueue &toHoist, InstructionQueue &boundsChecks) { // Hoist bounds checks first, so that hoistBoundsCheck can test for // invariant instructions, but delay actual insertion until the end to // handle dependencies on loop invariant instructions. InstructionQueue hoistedChecks; for (size_t i = 0; i < boundsChecks.length(); i++) { MBoundsCheck *ins = boundsChecks[i]->toBoundsCheck(); if (isLoopInvariant(ins) || !isInLoop(ins)) continue; // Try to find a test dominating the bounds check which can be // transformed into a hoistable check. Stop after the first such check // which could be transformed (the one which will be the closest to the // access in the source). MBasicBlock *block = ins->block(); while (true) { BranchDirection direction; MTest *branch = block->immediateDominatorBranch(&direction); if (branch) { MInstruction *upper, *lower; tryHoistBoundsCheck(ins, branch, direction, &upper, &lower); if (upper && !hoistedChecks.append(upper)) return false; if (lower && !hoistedChecks.append(lower)) return false; if (upper || lower) { ins->block()->discard(ins); break; } } MBasicBlock *dom = block->immediateDominator(); if (dom == block) break; block = dom; } } // Move all instructions to the preLoop_ block just before the control instruction. for (size_t i = 0; i < toHoist.length(); i++) { MInstruction *ins = toHoist[i]; // Loads may have an implicit dependency on either stores (effectful instructions) or // control instructions so we should never move these. JS_ASSERT(!ins->isControlInstruction()); JS_ASSERT(!ins->isEffectful()); JS_ASSERT(ins->isMovable()); if (checkHotness(ins->block())) { ins->block()->moveBefore(preLoop_->lastIns(), ins); ins->setNotLoopInvariant(); } } for (size_t i = 0; i < hoistedChecks.length(); i++) { MInstruction *ins = hoistedChecks[i]; preLoop_->insertBefore(preLoop_->lastIns(), ins); } return true; }