void MemMap::sinkStores(StoreList& stores) {
// sink dead stores into exit edges that occur between the dead store and the
// next store
StoreList::reverse_iterator it, end;
for (it = stores.rbegin(), end = stores.rend(); it != end; ++it) {
IRInstruction* store = it->first;
if (store->getId() != DEAD) {
continue;
}
std::vector<IRInstruction*>::iterator i, e;
for (i = it->second.begin(), e = it->second.end(); i != e; ++i) {
IRInstruction* guard = *i;
IRInstruction* clone = store->clone(factory);
if (store->getDst() != NULL) {
factory->getSSATmp(clone);
}
guard->getLabel()->getParent()->prependInstruction(clone);
}
// StRefs cannot just be removed, they have to be converted into Movs
// as the destination of the StRef still has the DecRef attached to it.
if (store->getOpcode() == StRef || store->getOpcode() == StRefNT) {
store->setOpcode(Mov);
store->setSrc(1, NULL);
store->setNumSrcs(1);
store->setId(LIVE);
}
}
}
void MemMap::sinkStores(StoreList& stores) {
// sink dead stores into exit edges that occur between the dead store and the
// next store
StoreList::reverse_iterator it, end;
for (it = stores.rbegin(), end = stores.rend(); it != end; ++it) {
IRInstruction* store = it->first;
if (isLive(store)) continue;
for (IRInstruction* guard : it->second) {
Block* exit = guard->getTaken();
exit->prepend(store->clone(m_factory));
}
// StRefs cannot just be removed, they have to be converted into Movs
// as the destination of the StRef still has the DecRef attached to it.
if (store->getOpcode() == StRef || store->getOpcode() == StRefNT) {
store->setOpcode(Mov);
store->setSrc(1, nullptr);
store->setNumSrcs(1);
setLive(*store, true);
}
}
}
void LinearScan::rematerializeAux(Trace* trace,
SSATmp* curSp,
SSATmp* curFp,
std::vector<SSATmp*> localValues) {
IRInstruction::List& instList = trace->getInstructionList();
for (IRInstruction::Iterator it = instList.begin();
it != instList.end();
++it) {
IRInstruction* inst = *it;
Opcode opc = inst->getOpcode();
SSATmp* dst = inst->getDst();
if (opc == DefFP || opc == FreeActRec) {
curFp = dst;
ASSERT(dst && dst->getReg() == rVmFp);
}
if (opc == Reload) {
// s = Spill t0
// t = Reload s
SSATmp* spilledTmp = getSpilledTmp(dst);
IRInstruction* spilledInst = spilledTmp->getInstruction();
IRInstruction* newInst = NULL;
if (spilledInst->isRematerializable() ||
(spilledInst->getOpcode() == LdStack &&
spilledInst->getSrc(0) == curSp)) {
// XXX: could change <newInst> to the non-check version.
// Rematerialize those rematerializable instructions (i.e.,
// isRematerializable returns true) and LdStack.
newInst = spilledInst->clone(m_irFactory);
// The new instruction needn't have an exit label, because it is always
// dominated by the original instruction.
newInst->setLabel(NULL);
} else {
// Rematerialize LdLoc.
std::vector<SSATmp*>::iterator pos =
std::find(localValues.begin(),
localValues.end(),
canonicalize(spilledTmp));
// Search for a local that stores the value of <spilledTmp>.
if (pos != localValues.end()) {
size_t locId = pos - localValues.begin();
ASSERT(curFp != NULL);
ConstInstruction constInst(curFp, Local(locId));
IRInstruction* ldHomeInst =
m_irFactory->cloneInstruction(&constInst);
newInst = m_irFactory->ldLoc(m_irFactory->getSSATmp(ldHomeInst),
dst->getType(),
NULL);
}
}
if (newInst) {
newInst->setDst(dst);
newInst->getDst()->setInstruction(newInst);
*it = newInst;
newInst->setParent(trace);
}
}
// Updating <curSp> and <localValues>.
if (dst && dst->getReg() == rVmSp) {
// <inst> modifies the stack pointer.
curSp = dst;
}
if (opc == LdLoc || opc == StLoc || opc == StLocNT) {
// dst = LdLoc home
// StLoc/StLocNT home, src
int locId = getLocalIdFromHomeOpnd(inst->getSrc(0));
SSATmp* localValue = (opc == LdLoc ? dst : inst->getSrc(1));
if (int(localValues.size()) < locId + 1) {
localValues.resize(locId + 1);
}
localValues[locId] = canonicalize(localValue);
}
if (inst->isControlFlowInstruction()) {
LabelInstruction* label = inst->getLabel();
if (label != NULL && label->getId() == inst->getId() + 1) {
rematerializeAux(label->getTrace(), curSp, curFp, localValues);
}
}
}
}
void LinearScan::rematerializeAux() {
struct State {
SSATmp *sp, *fp;
std::vector<SSATmp*> values;
};
StateVector<Block, State*> states(m_irFactory, nullptr);
SCOPE_EXIT { for (State* s : states) delete s; };
SSATmp* curSp = nullptr;
SSATmp* curFp = nullptr;
std::vector<SSATmp*> localValues;
auto killLocal = [&](IRInstruction& inst, unsigned src) {
if (src < inst.getNumSrcs()) {
unsigned loc = inst.getSrc(src)->getValInt();
if (loc < localValues.size()) localValues[loc] = nullptr;
}
};
auto setLocal = [&](unsigned loc, SSATmp* value) {
// Note that when we implement inlining, we will need to deal
// with the new local id space of the inlined function.
if (loc >= localValues.size()) localValues.resize(loc + 1);
localValues[loc] = canonicalize(value);
};
// Search for a local that stores <value>
auto findLocal = [&](SSATmp* value) -> int {
auto pos = std::find(localValues.begin(), localValues.end(),
canonicalize(value));
return pos != localValues.end() ? pos - localValues.begin() : -1;
};
// save the current state for future use by block; merge if necessary.
auto saveState = [&](Block* block) {
if (State* state = states[block]) {
// merge with saved state
assert(curFp == state->fp);
if (curSp != state->sp) state->sp = nullptr;
for (unsigned i = 0; i < state->values.size(); ++i) {
if (i >= localValues.size() || localValues[i] != state->values[i]) {
state->values[i] = nullptr;
}
}
} else {
// snapshot state for use at target.
state = states[block] = new State;
state->sp = curSp;
state->fp = curFp;
state->values = localValues;
}
};
for (Block* block : m_blocks) {
if (State* state = states[block]) {
states[block] = nullptr;
localValues = state->values;
curSp = state->sp;
curFp = state->fp;
delete state;
}
for (auto it = block->begin(); it != block->end(); ++it) {
IRInstruction& inst = *it;
Opcode opc = inst.getOpcode();
if (opc == DefFP || opc == FreeActRec) {
assert(inst.getDst()->getReg() == rVmFp);
curFp = inst.getDst();
}
else if (opc == Reload) {
// s = Spill t0
// t = Reload s
SSATmp* dst = inst.getDst();
SSATmp* spilledTmp = getSpilledTmp(dst);
IRInstruction* spilledInst = spilledTmp->getInstruction();
IRInstruction* newInst = NULL;
if (spilledInst->isRematerializable() ||
(spilledInst->getOpcode() == LdStack &&
spilledInst->getSrc(0) == curSp)) {
// XXX: could change <newInst> to the non-check version.
// Rematerialize those rematerializable instructions (i.e.,
// isRematerializable returns true) and LdStack.
newInst = spilledInst->clone(m_irFactory);
// The new instruction needn't have an exit label; it must always
// be dominated by the original instruction because reloads are
// inserted just before uses, which must be dominated by the
// original (spilled) def.
newInst->setTaken(nullptr);
} else if (curFp) {
// Rematerialize LdLoc.
int loc = findLocal(spilledTmp);
if (loc != -1) {
LocalId localId(loc);
newInst = m_irFactory->gen(LdLoc, dst->getType(), &localId, curFp);
}
}
if (newInst) {
UNUSED Type oldType = dst->getType();
newInst->setDst(dst);
dst->setInstruction(newInst);
assert(outputType(newInst) == oldType);
auto* block = inst.getBlock();
auto newIt = block->insert(it, newInst);
block->erase(it);
it = newIt;
}
}
// Updating curSp and localValues
if (inst.hasDst() && inst.getDst()->getReg() == rVmSp) {
// inst modifies the stack pointer.
curSp = inst.getDst();
}
if (opc == LdLoc || opc == StLoc || opc == StLocNT) {
setLocal(inst.getExtra<LocalId>()->locId,
opc == LdLoc ? inst.getDst() : inst.getSrc(1));
}
// Other instructions that may have side effects on locals must
// kill the local variable values.
else if (opc == IterInit) {
killLocal(inst, 3);
} else if (opc == IterInitK) {
killLocal(inst, 3);
killLocal(inst, 4);
} else if (opc == IterNext) {
killLocal(inst, 2);
} else if (opc == IterNextK) {
killLocal(inst, 2);
killLocal(inst, 3);
}
}
if (Block* taken = block->getTaken()) saveState(taken);
if (Block* next = block->getNext()) saveState(next);
}
}
void LinearScan::rematerializeAux(Trace* trace,
SSATmp* curSp,
SSATmp* curFp,
std::vector<SSATmp*> localValues) {
IRInstruction::List& instList = trace->getInstructionList();
for (IRInstruction::Iterator it = instList.begin();
it != instList.end();
++it) {
IRInstruction* inst = *it;
Opcode opc = inst->getOpcode();
SSATmp* dst = inst->getDst();
if (opc == DefFP || opc == FreeActRec) {
curFp = dst;
assert(dst && dst->getReg() == rVmFp);
}
if (opc == Reload) {
// s = Spill t0
// t = Reload s
SSATmp* spilledTmp = getSpilledTmp(dst);
IRInstruction* spilledInst = spilledTmp->getInstruction();
IRInstruction* newInst = NULL;
if (spilledInst->isRematerializable() ||
(spilledInst->getOpcode() == LdStack &&
spilledInst->getSrc(0) == curSp)) {
// XXX: could change <newInst> to the non-check version.
// Rematerialize those rematerializable instructions (i.e.,
// isRematerializable returns true) and LdStack.
newInst = spilledInst->clone(m_irFactory);
// The new instruction needn't have an exit label, because it is always
// dominated by the original instruction.
newInst->setLabel(NULL);
} else {
// Rematerialize LdLoc.
std::vector<SSATmp*>::iterator pos =
std::find(localValues.begin(),
localValues.end(),
canonicalize(spilledTmp));
// Search for a local that stores the value of <spilledTmp>.
if (pos != localValues.end()) {
size_t locId = pos - localValues.begin();
assert(curFp != NULL);
ConstInstruction constInst(curFp, Local(locId));
IRInstruction* ldHomeInst =
m_irFactory->cloneInstruction(&constInst);
newInst = m_irFactory->gen(LdLoc,
dst->getType(),
m_irFactory->getSSATmp(ldHomeInst));
}
}
if (newInst) {
UNUSED Type::Tag oldType = dst->getType();
newInst->setDst(dst);
dst->setInstruction(newInst);
assert(outputType(newInst) == oldType);
*it = newInst;
newInst->setParent(trace);
}
}
// Updating <curSp> and <localValues>.
if (dst && dst->getReg() == rVmSp) {
// <inst> modifies the stack pointer.
curSp = dst;
}
if (opc == LdLoc || opc == StLoc || opc == StLocNT) {
// dst = LdLoc home
// StLoc/StLocNT home, src
int locId = getLocalIdFromHomeOpnd(inst->getSrc(0));
// Note that when we implement inlining, we will need to deal
// with the new local id space of the inlined function.
SSATmp* localValue = (opc == LdLoc ? dst : inst->getSrc(1));
if (int(localValues.size()) < locId + 1) {
localValues.resize(locId + 1);
}
localValues[locId] = canonicalize(localValue);
}
// Other instructions that may have side effects on locals must
// kill the local variable values.
else if (opc == IterInit) {
int valLocId = inst->getSrc(3)->getConstValAsInt();
localValues[valLocId] = NULL;
if (inst->getNumSrcs() == 5) {
int keyLocId = inst->getSrc(4)->getConstValAsInt();
localValues[keyLocId] = NULL;
}
} else if (opc == IterNext) {
int valLocId = inst->getSrc(2)->getConstValAsInt();
localValues[valLocId] = NULL;
if (inst->getNumSrcs() == 4) {
int keyLocId = inst->getSrc(3)->getConstValAsInt();
localValues[keyLocId] = NULL;
}
}
if (inst->isControlFlowInstruction()) {
LabelInstruction* label = inst->getLabel();
if (label != NULL && label->getId() == inst->getId() + 1) {
rematerializeAux(label->getParent(), curSp, curFp, localValues);
}
}
}
}
void LinearScan::allocRegsOneTrace(BlockList::iterator& blockIt,
ExitTraceMap& etm) {
auto const trace = (*blockIt)->trace();
collectInfo(blockIt, trace);
computePreColoringHint();
auto v = etm.find(*blockIt);
if (v != etm.end()) {
assert(!trace->isMain());
v->second.restore(this);
} else {
assert(blockIt == m_blocks.begin() && trace->isMain());
initFreeList();
}
// First, visit every instruction, allocating registers as we go,
// and inserting Reload instructions where necessary.
bool isMain = trace->isMain();
size_t sz = m_slots.size();
while (blockIt != m_blocks.end()) {
Block* block = *blockIt;
if (block->trace() != trace) {
if (!isMain) {
break;
} else {
++blockIt;
continue;
}
}
FTRACE(5, "Block{}: {} ({})\n",
trace->isMain() ? "" : " (exit trace)",
(*blockIt)->id(), (*blockIt)->postId());
// clear remembered reloads that don't dominate this block
for (SlotInfo& slot : m_slots) {
if (SSATmp* reload = slot.latestReload) {
if (!dominates(reload->inst()->block(), block, m_idoms)) {
slot.latestReload = nullptr;
}
}
}
for (auto it = block->begin(), end = block->end(); it != end; ++it) {
allocRegToInstruction(it);
dumpIR<IRInstruction, kExtraLevel>(&*it, "allocated to instruction ");
}
if (isMain) {
assert(block->trace()->isMain());
if (block->taken() &&
!block->taken()->trace()->isMain()) {
etm[block->taken()].save(this);
}
}
++blockIt;
}
// Now that we have visited all instructions on this trace,
// and inserted Reloads for SSATmps which needed to be spilled,
// we can go back and insert the spills.
// On the main trace, insert the spill right after the instruction
// that generated the value (without traversing everything else).
// On exit traces, if the instruction that generated the value
// is on the main trace, insert the spill at the start of the trace,
// otherwise, after the instruction that generated the value
size_t begin = sz;
size_t end = m_slots.size();
while (begin < end) {
SlotInfo& slot = m_slots[begin++];
IRInstruction* spill = slot.spillTmp->inst();
IRInstruction* inst = spill->src(0)->inst();
Block* block = inst->block();
if (!isMain && block->trace()->isMain()) {
// We're on an exit trace, but the def is on the
// main trace, so put it at the start of this trace
if (spill->block()) {
// its already been inserted in another exit trace
assert(!spill->block()->trace()->isMain());
spill = spill->clone(m_irFactory);
}
trace->front()->prepend(spill);
} else if (inst->isBlockEnd()) {
block->next()->prepend(spill);
} else {
auto pos = block->iteratorTo(inst);
block->insert(++pos, spill);
}
}
}
