// This pass annotates every load instruction with the last store instruction
// on which it depends. The algorithm is optimistic in that it ignores explicit
// dependencies and only considers loads and stores.
//
// Loads inside loops only have an implicit dependency on a store before the
// loop header if no instruction inside the loop body aliases it. To calculate
// this efficiently, we maintain a list of maybe-invariant loads and the combined
// alias set for all stores inside the loop. When we see the loop's backedge, this
// information is used to mark every load we wrongly assumed to be loop invariant as
// having an implicit dependency on the last instruction of the loop header, so that
// it's never moved before the loop header.
//
// The algorithm depends on the invariant that both control instructions and effectful
// instructions (stores) are never hoisted.
bool
AliasAnalysis::analyze()
{
Vector<MInstructionVector, AliasSet::NumCategories, JitAllocPolicy> stores(alloc());
// Initialize to the first instruction.
MInstruction* firstIns = *graph_.entryBlock()->begin();
for (unsigned i = 0; i < AliasSet::NumCategories; i++) {
MInstructionVector defs(alloc());
if (!defs.append(firstIns))
return false;
if (!stores.append(Move(defs)))
return false;
}
// Type analysis may have inserted new instructions. Since this pass depends
// on the instruction number ordering, all instructions are renumbered.
uint32_t newId = 0;
for (ReversePostorderIterator block(graph_.rpoBegin()); block != graph_.rpoEnd(); block++) {
if (mir->shouldCancel("Alias Analysis (main loop)"))
return false;
if (block->isLoopHeader()) {
JitSpew(JitSpew_Alias, "Processing loop header %d", block->id());
loop_ = new(alloc()) LoopAliasInfo(alloc(), loop_, *block);
}
for (MPhiIterator def(block->phisBegin()), end(block->phisEnd()); def != end; ++def)
def->setId(newId++);
for (MInstructionIterator def(block->begin()), end(block->begin(block->lastIns()));
def != end;
++def)
{
def->setId(newId++);
AliasSet set = def->getAliasSet();
if (set.isNone())
continue;
// For the purposes of alias analysis, all recoverable operations
// are treated as effect free as the memory represented by these
// operations cannot be aliased by others.
if (def->canRecoverOnBailout())
continue;
if (set.isStore()) {
for (AliasSetIterator iter(set); iter; iter++) {
if (!stores[*iter].append(*def))
return false;
}
if (JitSpewEnabled(JitSpew_Alias)) {
Fprinter& out = JitSpewPrinter();
out.printf("Processing store ");
def->printName(out);
out.printf(" (flags %x)\n", set.flags());
}
} else {
// Find the most recent store on which this instruction depends.
MInstruction* lastStore = firstIns;
for (AliasSetIterator iter(set); iter; iter++) {
MInstructionVector& aliasedStores = stores[*iter];
for (int i = aliasedStores.length() - 1; i >= 0; i--) {
MInstruction* store = aliasedStores[i];
if (genericMightAlias(*def, store) != MDefinition::AliasType::NoAlias &&
def->mightAlias(store) != MDefinition::AliasType::NoAlias &&
BlockMightReach(store->block(), *block))
{
if (lastStore->id() < store->id())
lastStore = store;
break;
}
}
}
def->setDependency(lastStore);
IonSpewDependency(*def, lastStore, "depends", "");
// If the last store was before the current loop, we assume this load
// is loop invariant. If a later instruction writes to the same location,
// we will fix this at the end of the loop.
if (loop_ && lastStore->id() < loop_->firstInstruction()->id()) {
if (!loop_->addInvariantLoad(*def))
return false;
}
}
}
// Renumber the last instruction, as the analysis depends on this and the order.
block->lastIns()->setId(newId++);
if (block->isLoopBackedge()) {
MOZ_ASSERT(loop_->loopHeader() == block->loopHeaderOfBackedge());
JitSpew(JitSpew_Alias, "Processing loop backedge %d (header %d)", block->id(),
loop_->loopHeader()->id());
LoopAliasInfo* outerLoop = loop_->outer();
MInstruction* firstLoopIns = *loop_->loopHeader()->begin();
const MInstructionVector& invariant = loop_->invariantLoads();
for (unsigned i = 0; i < invariant.length(); i++) {
MInstruction* ins = invariant[i];
AliasSet set = ins->getAliasSet();
MOZ_ASSERT(set.isLoad());
bool hasAlias = false;
for (AliasSetIterator iter(set); iter; iter++) {
MInstructionVector& aliasedStores = stores[*iter];
for (int i = aliasedStores.length() - 1;; i--) {
MInstruction* store = aliasedStores[i];
if (store->id() < firstLoopIns->id())
break;
if (genericMightAlias(ins, store) != MDefinition::AliasType::NoAlias &&
ins->mightAlias(store) != MDefinition::AliasType::NoAlias)
{
hasAlias = true;
IonSpewDependency(ins, store, "aliases", "store in loop body");
break;
}
}
if (hasAlias)
break;
}
if (hasAlias) {
// This instruction depends on stores inside the loop body. Mark it as having a
// dependency on the last instruction of the loop header. The last instruction is a
// control instruction and these are never hoisted.
MControlInstruction* controlIns = loop_->loopHeader()->lastIns();
IonSpewDependency(ins, controlIns, "depends", "due to stores in loop body");
ins->setDependency(controlIns);
} else {
IonSpewAliasInfo("Load", ins, "does not depend on any stores in this loop");
if (outerLoop && ins->dependency()->id() < outerLoop->firstInstruction()->id()) {
IonSpewAliasInfo("Load", ins, "may be invariant in outer loop");
if (!outerLoop->addInvariantLoad(ins))
return false;
}
}
}
loop_ = loop_->outer();
}
}
spewDependencyList();
MOZ_ASSERT(loop_ == nullptr);
return true;
}
bool
jit::ReorderInstructions(MIRGraph& graph)
{
// Renumber all instructions in the graph as we go.
size_t nextId = 0;
// List of the headers of any loops we are in.
Vector<MBasicBlock*, 4, SystemAllocPolicy> loopHeaders;
for (ReversePostorderIterator block(graph.rpoBegin()); block != graph.rpoEnd(); block++) {
// Renumber all definitions inside the basic blocks.
for (MPhiIterator iter(block->phisBegin()); iter != block->phisEnd(); iter++)
iter->setId(nextId++);
for (MInstructionIterator iter(block->begin()); iter != block->end(); iter++)
iter->setId(nextId++);
// Don't reorder instructions within entry blocks, which have special requirements.
if (*block == graph.entryBlock() || *block == graph.osrBlock())
continue;
if (block->isLoopHeader()) {
if (!loopHeaders.append(*block))
return false;
}
MBasicBlock* innerLoop = loopHeaders.empty() ? nullptr : loopHeaders.back();
MInstruction* top = block->safeInsertTop();
MInstructionReverseIterator rtop = ++block->rbegin(top);
for (MInstructionIterator iter(block->begin(top)); iter != block->end(); ) {
MInstruction* ins = *iter;
// Filter out some instructions which are never reordered.
if (ins->isEffectful() ||
!ins->isMovable() ||
ins->resumePoint() ||
ins == block->lastIns())
{
iter++;
continue;
}
// Move constants with a single use in the current block to the
// start of the block. Constants won't be reordered by the logic
// below, as they have no inputs. Moving them up as high as
// possible can allow their use to be moved up further, though,
// and has no cost if the constant is emitted at its use.
if (ins->isConstant() &&
ins->hasOneUse() &&
ins->usesBegin()->consumer()->block() == *block &&
!IsFloatingPointType(ins->type()))
{
iter++;
MInstructionIterator targetIter = block->begin();
while (targetIter->isConstant() || targetIter->isInterruptCheck()) {
if (*targetIter == ins)
break;
targetIter++;
}
MoveBefore(*block, *targetIter, ins);
continue;
}
// Look for inputs where this instruction is the last use of that
// input. If we move this instruction up, the input's lifetime will
// be shortened, modulo resume point uses (which don't need to be
// stored in a register, and can be handled by the register
// allocator by just spilling at some point with no reload).
Vector<MDefinition*, 4, SystemAllocPolicy> lastUsedInputs;
for (size_t i = 0; i < ins->numOperands(); i++) {
MDefinition* input = ins->getOperand(i);
if (!input->isConstant() && IsLastUse(ins, input, innerLoop)) {
if (!lastUsedInputs.append(input))
return false;
}
}
// Don't try to move instructions which aren't the last use of any
// of their inputs (we really ought to move these down instead).
if (lastUsedInputs.length() < 2) {
iter++;
continue;
}
MInstruction* target = ins;
for (MInstructionReverseIterator riter = ++block->rbegin(ins); riter != rtop; riter++) {
MInstruction* prev = *riter;
if (prev->isInterruptCheck())
break;
// The instruction can't be moved before any of its uses.
bool isUse = false;
for (size_t i = 0; i < ins->numOperands(); i++) {
if (ins->getOperand(i) == prev) {
isUse = true;
break;
}
}
if (isUse)
break;
// The instruction can't be moved before an instruction that
// stores to a location read by the instruction.
if (prev->isEffectful() &&
(ins->getAliasSet().flags() & prev->getAliasSet().flags()) &&
ins->mightAlias(prev) != MDefinition::AliasType::NoAlias)
{
break;
}
// Make sure the instruction will still be the last use of one
// of its inputs when moved up this far.
for (size_t i = 0; i < lastUsedInputs.length(); ) {
bool found = false;
for (size_t j = 0; j < prev->numOperands(); j++) {
if (prev->getOperand(j) == lastUsedInputs[i]) {
found = true;
break;
}
}
if (found) {
lastUsedInputs[i] = lastUsedInputs.back();
lastUsedInputs.popBack();
} else {
i++;
}
}
if (lastUsedInputs.length() < 2)
break;
// We can move the instruction before this one.
target = prev;
}
iter++;
MoveBefore(*block, target, ins);
}
if (block->isLoopBackedge())
loopHeaders.popBack();
}
return true;
}
