void spillEverything(Code& code)
{
PhaseScope phaseScope(code, "spillEverything");
// We want to know the set of registers used at every point in every basic block.
IndexMap<BasicBlock, Vector<RegisterSet>> usedRegisters(code.size());
Liveness<Tmp> liveness(code);
for (BasicBlock* block : code) {
Liveness<Tmp>::LocalCalc localCalc(liveness, block);
usedRegisters[block].resize(block->size() + 1);
auto setUsedRegisters = [&] (unsigned index, Inst& inst) {
RegisterSet& registerSet = usedRegisters[block][index];
for (Tmp tmp : localCalc.live()) {
if (tmp.isReg())
registerSet.set(tmp.reg());
}
// Gotta account for dead assignments to registers. These may happen because the input
// code is suboptimal.
inst.forEachArg(
[&] (Arg& arg, Arg::Role role, Arg::Type) {
if (Arg::isDef(role) && arg.isReg())
registerSet.set(arg.reg());
});
};
for (unsigned instIndex = block->size(); instIndex--;) {
Inst& inst = block->at(instIndex);
setUsedRegisters(instIndex + 1, inst);
localCalc.execute(inst);
}
Inst nop;
setUsedRegisters(0, nop);
}
// Allocate a stack slot for each tmp.
Vector<StackSlot*> allStackSlots[Arg::numTypes];
for (unsigned typeIndex = 0; typeIndex < Arg::numTypes; ++typeIndex) {
Vector<StackSlot*>& stackSlots = allStackSlots[typeIndex];
Arg::Type type = static_cast<Arg::Type>(typeIndex);
stackSlots.resize(code.numTmps(type));
for (unsigned tmpIndex = code.numTmps(type); tmpIndex--;)
stackSlots[tmpIndex] = code.addStackSlot(8, StackSlotKind::Anonymous);
}
InsertionSet insertionSet(code);
for (BasicBlock* block : code) {
for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
RegisterSet& setBefore = usedRegisters[block][instIndex];
RegisterSet& setAfter = usedRegisters[block][instIndex + 1];
Inst& inst = block->at(instIndex);
inst.forEachTmp(
[&] (Tmp& tmp, Arg::Role role, Arg::Type type) {
if (tmp.isReg())
return;
StackSlot* stackSlot = allStackSlots[type][tmp.tmpIndex()];
Arg arg = Arg::stack(stackSlot);
// Need to figure out a register to use. How we do that depends on the role.
Reg chosenReg;
switch (role) {
case Arg::Use:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setBefore.get(reg)) {
setBefore.set(reg);
chosenReg = reg;
break;
}
}
break;
case Arg::Def:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setAfter.get(reg)) {
setAfter.set(reg);
chosenReg = reg;
break;
}
}
break;
case Arg::UseDef:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setBefore.get(reg) && !setAfter.get(reg)) {
setAfter.set(reg);
setBefore.set(reg);
chosenReg = reg;
break;
}
}
break;
}
RELEASE_ASSERT(chosenReg);
tmp = Tmp(chosenReg);
Opcode move = type == Arg::GP ? Move : MoveDouble;
if (Arg::isUse(role)) {
insertionSet.insert(
instIndex, move, inst.origin, arg, tmp);
}
if (Arg::isDef(role)) {
insertionSet.insert(
instIndex + 1, move, inst.origin, tmp, arg);
}
});
}
insertionSet.execute(block);
}
}
void spillEverything(Code& code)
{
PhaseScope phaseScope(code, "spillEverything");
// We want to know the set of registers used at every point in every basic block.
IndexMap<BasicBlock, Vector<RegisterSet>> usedRegisters(code.size());
GPLiveness gpLiveness(code);
FPLiveness fpLiveness(code);
for (BasicBlock* block : code) {
GPLiveness::LocalCalc gpLocalCalc(gpLiveness, block);
FPLiveness::LocalCalc fpLocalCalc(fpLiveness, block);
usedRegisters[block].resize(block->size() + 1);
auto setUsedRegisters = [&] (unsigned index) {
RegisterSet& registerSet = usedRegisters[block][index];
for (Tmp tmp : gpLocalCalc.live()) {
if (tmp.isReg())
registerSet.set(tmp.reg());
}
for (Tmp tmp : fpLocalCalc.live()) {
if (tmp.isReg())
registerSet.set(tmp.reg());
}
// Gotta account for dead assignments to registers. These may happen because the input
// code is suboptimal.
Inst::forEachDefWithExtraClobberedRegs<Tmp>(
block->get(index - 1), block->get(index),
[&] (const Tmp& tmp, Arg::Role, Arg::Type, Arg::Width) {
if (tmp.isReg())
registerSet.set(tmp.reg());
});
};
for (unsigned instIndex = block->size(); instIndex--;) {
setUsedRegisters(instIndex + 1);
gpLocalCalc.execute(instIndex);
fpLocalCalc.execute(instIndex);
}
setUsedRegisters(0);
}
// Allocate a stack slot for each tmp.
Vector<StackSlot*> allStackSlots[Arg::numTypes];
for (unsigned typeIndex = 0; typeIndex < Arg::numTypes; ++typeIndex) {
Vector<StackSlot*>& stackSlots = allStackSlots[typeIndex];
Arg::Type type = static_cast<Arg::Type>(typeIndex);
stackSlots.resize(code.numTmps(type));
for (unsigned tmpIndex = code.numTmps(type); tmpIndex--;)
stackSlots[tmpIndex] = code.addStackSlot(8, StackSlotKind::Anonymous);
}
InsertionSet insertionSet(code);
for (BasicBlock* block : code) {
for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
RegisterSet& setBefore = usedRegisters[block][instIndex];
RegisterSet& setAfter = usedRegisters[block][instIndex + 1];
Inst& inst = block->at(instIndex);
// First try to spill directly.
for (unsigned i = 0; i < inst.args.size(); ++i) {
Arg& arg = inst.args[i];
if (arg.isTmp()) {
if (arg.isReg())
continue;
if (inst.admitsStack(i)) {
StackSlot* stackSlot = allStackSlots[arg.type()][arg.tmpIndex()];
arg = Arg::stack(stackSlot);
continue;
}
}
}
// Now fall back on spilling using separate Move's to load/store the tmp.
inst.forEachTmp(
[&] (Tmp& tmp, Arg::Role role, Arg::Type type, Arg::Width) {
if (tmp.isReg())
return;
StackSlot* stackSlot = allStackSlots[type][tmp.tmpIndex()];
Arg arg = Arg::stack(stackSlot);
// Need to figure out a register to use. How we do that depends on the role.
Reg chosenReg;
switch (role) {
case Arg::Use:
case Arg::ColdUse:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setBefore.get(reg)) {
setBefore.set(reg);
chosenReg = reg;
break;
}
}
break;
case Arg::Def:
case Arg::ZDef:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setAfter.get(reg)) {
setAfter.set(reg);
chosenReg = reg;
break;
}
}
break;
case Arg::UseDef:
case Arg::UseZDef:
case Arg::LateUse:
case Arg::LateColdUse:
case Arg::Scratch:
case Arg::EarlyDef:
for (Reg reg : regsInPriorityOrder(type)) {
if (!setBefore.get(reg) && !setAfter.get(reg)) {
setAfter.set(reg);
setBefore.set(reg);
chosenReg = reg;
break;
}
}
break;
case Arg::UseAddr:
// We will never UseAddr a Tmp, that doesn't make sense.
RELEASE_ASSERT_NOT_REACHED();
break;
}
RELEASE_ASSERT(chosenReg);
tmp = Tmp(chosenReg);
Opcode move = type == Arg::GP ? Move : MoveDouble;
if (Arg::isAnyUse(role) && role != Arg::Scratch)
insertionSet.insert(instIndex, move, inst.origin, arg, tmp);
if (Arg::isAnyDef(role))
insertionSet.insert(instIndex + 1, move, inst.origin, tmp, arg);
});
}
insertionSet.execute(block);
}
}
void handleCalleeSaves(Code& code)
{
PhaseScope phaseScope(code, "handleCalleeSaves");
RegisterSet usedCalleeSaves;
for (BasicBlock* block : code) {
for (Inst& inst : *block) {
inst.forEachTmpFast(
[&] (Tmp& tmp) {
// At first we just record all used regs.
usedCalleeSaves.set(tmp.reg());
});
if (inst.hasSpecial())
usedCalleeSaves.merge(inst.extraClobberedRegs());
}
}
// Now we filter to really get the callee saves.
usedCalleeSaves.filter(RegisterSet::calleeSaveRegisters());
usedCalleeSaves.exclude(RegisterSet::stackRegisters()); // We don't need to save FP here.
if (!usedCalleeSaves.numberOfSetRegisters())
return;
code.calleeSaveRegisters() = RegisterAtOffsetList(usedCalleeSaves);
size_t byteSize = 0;
for (const RegisterAtOffset& entry : code.calleeSaveRegisters())
byteSize = std::max(static_cast<size_t>(-entry.offset()), byteSize);
StackSlot* savesArea = code.addStackSlot(byteSize, StackSlotKind::Locked);
// This is a bit weird since we could have already pinned a different stack slot to this
// area. Also, our runtime does not require us to pin the saves area. Maybe we shouldn't pin it?
savesArea->setOffsetFromFP(-byteSize);
auto argFor = [&] (const RegisterAtOffset& entry) -> Arg {
return Arg::stack(savesArea, entry.offset() + byteSize);
};
InsertionSet insertionSet(code);
// First insert saving code in the prologue.
for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
insertionSet.insert(
0, entry.reg().isGPR() ? Move : MoveDouble, code[0]->at(0).origin,
Tmp(entry.reg()), argFor(entry));
}
insertionSet.execute(code[0]);
// Now insert restore code at epilogues.
for (BasicBlock* block : code) {
Inst& last = block->last();
if (!isReturn(last.opcode))
continue;
for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
insertionSet.insert(
block->size() - 1, entry.reg().isGPR() ? Move : MoveDouble, last.origin,
argFor(entry), Tmp(entry.reg()));
}
insertionSet.execute(block);
}
}
