void MemMap::sinkStores(StoreList& stores) { // sink dead stores into exit edges that occur between the dead store and the // next store StoreList::reverse_iterator it, end; for (it = stores.rbegin(), end = stores.rend(); it != end; ++it) { IRInstruction* store = it->first; if (store->getId() != DEAD) { continue; } std::vector<IRInstruction*>::iterator i, e; for (i = it->second.begin(), e = it->second.end(); i != e; ++i) { IRInstruction* guard = *i; IRInstruction* clone = store->clone(factory); if (store->getDst() != NULL) { factory->getSSATmp(clone); } guard->getLabel()->getParent()->prependInstruction(clone); } // StRefs cannot just be removed, they have to be converted into Movs // as the destination of the StRef still has the DecRef attached to it. if (store->getOpcode() == StRef || store->getOpcode() == StRefNT) { store->setOpcode(Mov); store->setSrc(1, NULL); store->setNumSrcs(1); store->setId(LIVE); } } }
void MemMap::sinkStores(StoreList& stores) { // sink dead stores into exit edges that occur between the dead store and the // next store StoreList::reverse_iterator it, end; for (it = stores.rbegin(), end = stores.rend(); it != end; ++it) { IRInstruction* store = it->first; if (isLive(store)) continue; for (IRInstruction* guard : it->second) { Block* exit = guard->getTaken(); exit->prepend(store->clone(m_factory)); } // StRefs cannot just be removed, they have to be converted into Movs // as the destination of the StRef still has the DecRef attached to it. if (store->getOpcode() == StRef || store->getOpcode() == StRefNT) { store->setOpcode(Mov); store->setSrc(1, nullptr); store->setNumSrcs(1); setLive(*store, true); } } }
void LinearScan::rematerializeAux(Trace* trace, SSATmp* curSp, SSATmp* curFp, std::vector<SSATmp*> localValues) { IRInstruction::List& instList = trace->getInstructionList(); for (IRInstruction::Iterator it = instList.begin(); it != instList.end(); ++it) { IRInstruction* inst = *it; Opcode opc = inst->getOpcode(); SSATmp* dst = inst->getDst(); if (opc == DefFP || opc == FreeActRec) { curFp = dst; ASSERT(dst && dst->getReg() == rVmFp); } if (opc == Reload) { // s = Spill t0 // t = Reload s SSATmp* spilledTmp = getSpilledTmp(dst); IRInstruction* spilledInst = spilledTmp->getInstruction(); IRInstruction* newInst = NULL; if (spilledInst->isRematerializable() || (spilledInst->getOpcode() == LdStack && spilledInst->getSrc(0) == curSp)) { // XXX: could change <newInst> to the non-check version. // Rematerialize those rematerializable instructions (i.e., // isRematerializable returns true) and LdStack. newInst = spilledInst->clone(m_irFactory); // The new instruction needn't have an exit label, because it is always // dominated by the original instruction. newInst->setLabel(NULL); } else { // Rematerialize LdLoc. std::vector<SSATmp*>::iterator pos = std::find(localValues.begin(), localValues.end(), canonicalize(spilledTmp)); // Search for a local that stores the value of <spilledTmp>. if (pos != localValues.end()) { size_t locId = pos - localValues.begin(); ASSERT(curFp != NULL); ConstInstruction constInst(curFp, Local(locId)); IRInstruction* ldHomeInst = m_irFactory->cloneInstruction(&constInst); newInst = m_irFactory->ldLoc(m_irFactory->getSSATmp(ldHomeInst), dst->getType(), NULL); } } if (newInst) { newInst->setDst(dst); newInst->getDst()->setInstruction(newInst); *it = newInst; newInst->setParent(trace); } } // Updating <curSp> and <localValues>. if (dst && dst->getReg() == rVmSp) { // <inst> modifies the stack pointer. curSp = dst; } if (opc == LdLoc || opc == StLoc || opc == StLocNT) { // dst = LdLoc home // StLoc/StLocNT home, src int locId = getLocalIdFromHomeOpnd(inst->getSrc(0)); SSATmp* localValue = (opc == LdLoc ? dst : inst->getSrc(1)); if (int(localValues.size()) < locId + 1) { localValues.resize(locId + 1); } localValues[locId] = canonicalize(localValue); } if (inst->isControlFlowInstruction()) { LabelInstruction* label = inst->getLabel(); if (label != NULL && label->getId() == inst->getId() + 1) { rematerializeAux(label->getTrace(), curSp, curFp, localValues); } } } }
void LinearScan::rematerializeAux() { struct State { SSATmp *sp, *fp; std::vector<SSATmp*> values; }; StateVector<Block, State*> states(m_irFactory, nullptr); SCOPE_EXIT { for (State* s : states) delete s; }; SSATmp* curSp = nullptr; SSATmp* curFp = nullptr; std::vector<SSATmp*> localValues; auto killLocal = [&](IRInstruction& inst, unsigned src) { if (src < inst.getNumSrcs()) { unsigned loc = inst.getSrc(src)->getValInt(); if (loc < localValues.size()) localValues[loc] = nullptr; } }; auto setLocal = [&](unsigned loc, SSATmp* value) { // Note that when we implement inlining, we will need to deal // with the new local id space of the inlined function. if (loc >= localValues.size()) localValues.resize(loc + 1); localValues[loc] = canonicalize(value); }; // Search for a local that stores <value> auto findLocal = [&](SSATmp* value) -> int { auto pos = std::find(localValues.begin(), localValues.end(), canonicalize(value)); return pos != localValues.end() ? pos - localValues.begin() : -1; }; // save the current state for future use by block; merge if necessary. auto saveState = [&](Block* block) { if (State* state = states[block]) { // merge with saved state assert(curFp == state->fp); if (curSp != state->sp) state->sp = nullptr; for (unsigned i = 0; i < state->values.size(); ++i) { if (i >= localValues.size() || localValues[i] != state->values[i]) { state->values[i] = nullptr; } } } else { // snapshot state for use at target. state = states[block] = new State; state->sp = curSp; state->fp = curFp; state->values = localValues; } }; for (Block* block : m_blocks) { if (State* state = states[block]) { states[block] = nullptr; localValues = state->values; curSp = state->sp; curFp = state->fp; delete state; } for (auto it = block->begin(); it != block->end(); ++it) { IRInstruction& inst = *it; Opcode opc = inst.getOpcode(); if (opc == DefFP || opc == FreeActRec) { assert(inst.getDst()->getReg() == rVmFp); curFp = inst.getDst(); } else if (opc == Reload) { // s = Spill t0 // t = Reload s SSATmp* dst = inst.getDst(); SSATmp* spilledTmp = getSpilledTmp(dst); IRInstruction* spilledInst = spilledTmp->getInstruction(); IRInstruction* newInst = NULL; if (spilledInst->isRematerializable() || (spilledInst->getOpcode() == LdStack && spilledInst->getSrc(0) == curSp)) { // XXX: could change <newInst> to the non-check version. // Rematerialize those rematerializable instructions (i.e., // isRematerializable returns true) and LdStack. newInst = spilledInst->clone(m_irFactory); // The new instruction needn't have an exit label; it must always // be dominated by the original instruction because reloads are // inserted just before uses, which must be dominated by the // original (spilled) def. newInst->setTaken(nullptr); } else if (curFp) { // Rematerialize LdLoc. int loc = findLocal(spilledTmp); if (loc != -1) { LocalId localId(loc); newInst = m_irFactory->gen(LdLoc, dst->getType(), &localId, curFp); } } if (newInst) { UNUSED Type oldType = dst->getType(); newInst->setDst(dst); dst->setInstruction(newInst); assert(outputType(newInst) == oldType); auto* block = inst.getBlock(); auto newIt = block->insert(it, newInst); block->erase(it); it = newIt; } } // Updating curSp and localValues if (inst.hasDst() && inst.getDst()->getReg() == rVmSp) { // inst modifies the stack pointer. curSp = inst.getDst(); } if (opc == LdLoc || opc == StLoc || opc == StLocNT) { setLocal(inst.getExtra<LocalId>()->locId, opc == LdLoc ? inst.getDst() : inst.getSrc(1)); } // Other instructions that may have side effects on locals must // kill the local variable values. else if (opc == IterInit) { killLocal(inst, 3); } else if (opc == IterInitK) { killLocal(inst, 3); killLocal(inst, 4); } else if (opc == IterNext) { killLocal(inst, 2); } else if (opc == IterNextK) { killLocal(inst, 2); killLocal(inst, 3); } } if (Block* taken = block->getTaken()) saveState(taken); if (Block* next = block->getNext()) saveState(next); } }
void LinearScan::rematerializeAux(Trace* trace, SSATmp* curSp, SSATmp* curFp, std::vector<SSATmp*> localValues) { IRInstruction::List& instList = trace->getInstructionList(); for (IRInstruction::Iterator it = instList.begin(); it != instList.end(); ++it) { IRInstruction* inst = *it; Opcode opc = inst->getOpcode(); SSATmp* dst = inst->getDst(); if (opc == DefFP || opc == FreeActRec) { curFp = dst; assert(dst && dst->getReg() == rVmFp); } if (opc == Reload) { // s = Spill t0 // t = Reload s SSATmp* spilledTmp = getSpilledTmp(dst); IRInstruction* spilledInst = spilledTmp->getInstruction(); IRInstruction* newInst = NULL; if (spilledInst->isRematerializable() || (spilledInst->getOpcode() == LdStack && spilledInst->getSrc(0) == curSp)) { // XXX: could change <newInst> to the non-check version. // Rematerialize those rematerializable instructions (i.e., // isRematerializable returns true) and LdStack. newInst = spilledInst->clone(m_irFactory); // The new instruction needn't have an exit label, because it is always // dominated by the original instruction. newInst->setLabel(NULL); } else { // Rematerialize LdLoc. std::vector<SSATmp*>::iterator pos = std::find(localValues.begin(), localValues.end(), canonicalize(spilledTmp)); // Search for a local that stores the value of <spilledTmp>. if (pos != localValues.end()) { size_t locId = pos - localValues.begin(); assert(curFp != NULL); ConstInstruction constInst(curFp, Local(locId)); IRInstruction* ldHomeInst = m_irFactory->cloneInstruction(&constInst); newInst = m_irFactory->gen(LdLoc, dst->getType(), m_irFactory->getSSATmp(ldHomeInst)); } } if (newInst) { UNUSED Type::Tag oldType = dst->getType(); newInst->setDst(dst); dst->setInstruction(newInst); assert(outputType(newInst) == oldType); *it = newInst; newInst->setParent(trace); } } // Updating <curSp> and <localValues>. if (dst && dst->getReg() == rVmSp) { // <inst> modifies the stack pointer. curSp = dst; } if (opc == LdLoc || opc == StLoc || opc == StLocNT) { // dst = LdLoc home // StLoc/StLocNT home, src int locId = getLocalIdFromHomeOpnd(inst->getSrc(0)); // Note that when we implement inlining, we will need to deal // with the new local id space of the inlined function. SSATmp* localValue = (opc == LdLoc ? dst : inst->getSrc(1)); if (int(localValues.size()) < locId + 1) { localValues.resize(locId + 1); } localValues[locId] = canonicalize(localValue); } // Other instructions that may have side effects on locals must // kill the local variable values. else if (opc == IterInit) { int valLocId = inst->getSrc(3)->getConstValAsInt(); localValues[valLocId] = NULL; if (inst->getNumSrcs() == 5) { int keyLocId = inst->getSrc(4)->getConstValAsInt(); localValues[keyLocId] = NULL; } } else if (opc == IterNext) { int valLocId = inst->getSrc(2)->getConstValAsInt(); localValues[valLocId] = NULL; if (inst->getNumSrcs() == 4) { int keyLocId = inst->getSrc(3)->getConstValAsInt(); localValues[keyLocId] = NULL; } } if (inst->isControlFlowInstruction()) { LabelInstruction* label = inst->getLabel(); if (label != NULL && label->getId() == inst->getId() + 1) { rematerializeAux(label->getParent(), curSp, curFp, localValues); } } } }
void LinearScan::allocRegsOneTrace(BlockList::iterator& blockIt, ExitTraceMap& etm) { auto const trace = (*blockIt)->trace(); collectInfo(blockIt, trace); computePreColoringHint(); auto v = etm.find(*blockIt); if (v != etm.end()) { assert(!trace->isMain()); v->second.restore(this); } else { assert(blockIt == m_blocks.begin() && trace->isMain()); initFreeList(); } // First, visit every instruction, allocating registers as we go, // and inserting Reload instructions where necessary. bool isMain = trace->isMain(); size_t sz = m_slots.size(); while (blockIt != m_blocks.end()) { Block* block = *blockIt; if (block->trace() != trace) { if (!isMain) { break; } else { ++blockIt; continue; } } FTRACE(5, "Block{}: {} ({})\n", trace->isMain() ? "" : " (exit trace)", (*blockIt)->id(), (*blockIt)->postId()); // clear remembered reloads that don't dominate this block for (SlotInfo& slot : m_slots) { if (SSATmp* reload = slot.latestReload) { if (!dominates(reload->inst()->block(), block, m_idoms)) { slot.latestReload = nullptr; } } } for (auto it = block->begin(), end = block->end(); it != end; ++it) { allocRegToInstruction(it); dumpIR<IRInstruction, kExtraLevel>(&*it, "allocated to instruction "); } if (isMain) { assert(block->trace()->isMain()); if (block->taken() && !block->taken()->trace()->isMain()) { etm[block->taken()].save(this); } } ++blockIt; } // Now that we have visited all instructions on this trace, // and inserted Reloads for SSATmps which needed to be spilled, // we can go back and insert the spills. // On the main trace, insert the spill right after the instruction // that generated the value (without traversing everything else). // On exit traces, if the instruction that generated the value // is on the main trace, insert the spill at the start of the trace, // otherwise, after the instruction that generated the value size_t begin = sz; size_t end = m_slots.size(); while (begin < end) { SlotInfo& slot = m_slots[begin++]; IRInstruction* spill = slot.spillTmp->inst(); IRInstruction* inst = spill->src(0)->inst(); Block* block = inst->block(); if (!isMain && block->trace()->isMain()) { // We're on an exit trace, but the def is on the // main trace, so put it at the start of this trace if (spill->block()) { // its already been inserted in another exit trace assert(!spill->block()->trace()->isMain()); spill = spill->clone(m_irFactory); } trace->front()->prepend(spill); } else if (inst->isBlockEnd()) { block->next()->prepend(spill); } else { auto pos = block->iteratorTo(inst); block->insert(++pos, spill); } } }