void StackColoring::removeInvalidSlotRanges() {
MachineFunction::const_iterator BB, BBE;
MachineBasicBlock::const_iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
I->getOpcode() == TargetOpcode::LIFETIME_END || I->isDebugValue())
continue;
// Some intervals are suspicious! In some cases we find address
// calculations outside of the lifetime zone, but not actual memory
// read or write. Memory accesses outside of the lifetime zone are a clear
// violation, but address calculations are okay. This can happen when
// GEPs are hoisted outside of the lifetime zone.
// So, in here we only check instructions which can read or write memory.
if (!I->mayLoad() && !I->mayStore())
continue;
// Check all of the machine operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
const MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot<0)
continue;
if (Intervals[Slot]->empty())
continue;
// Check that the used slot is inside the calculated lifetime range.
// If it is not, warn about it and invalidate the range.
LiveInterval *Interval = Intervals[Slot];
SlotIndex Index = Indexes->getInstructionIndex(I);
if (Interval->find(Index) == Interval->end()) {
Intervals[Slot]->clear();
DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
EscapedAllocas++;
}
}
}
}
void StackColoring::removeInvalidSlotRanges() {
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END || I.isDebugValue())
continue;
// Some intervals are suspicious! In some cases we find address
// calculations outside of the lifetime zone, but not actual memory
// read or write. Memory accesses outside of the lifetime zone are a clear
// violation, but address calculations are okay. This can happen when
// GEPs are hoisted outside of the lifetime zone.
// So, in here we only check instructions which can read or write memory.
if (!I.mayLoad() && !I.mayStore())
continue;
// Check all of the machine operands.
for (const MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int Slot = MO.getIndex();
if (Slot<0)
continue;
if (Intervals[Slot]->empty())
continue;
// Check that the used slot is inside the calculated lifetime range.
// If it is not, warn about it and invalidate the range.
LiveInterval *Interval = &*Intervals[Slot];
SlotIndex Index = Indexes->getInstructionIndex(I);
if (Interval->find(Index) == Interval->end()) {
Interval->clear();
DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
EscapedAllocas++;
}
}
}
}
bool LiveInterval::overlaps(const LiveInterval &Other,
const CoalescerPair &CP,
const SlotIndexes &Indexes) const {
assert(!empty() && "empty interval");
if (Other.empty())
return false;
// Use binary searches to find initial positions.
const_iterator I = find(Other.beginIndex());
const_iterator IE = end();
if (I == IE)
return false;
const_iterator J = Other.find(I->start);
const_iterator JE = Other.end();
if (J == JE)
return false;
for (;;) {
// J has just been advanced to satisfy:
assert(J->end >= I->start);
// Check for an overlap.
if (J->start < I->end) {
// I and J are overlapping. Find the later start.
SlotIndex Def = std::max(I->start, J->start);
// Allow the overlap if Def is a coalescable copy.
if (Def.isBlock() ||
!CP.isCoalescable(Indexes.getInstructionFromIndex(Def)))
return true;
}
// Advance the iterator that ends first to check for more overlaps.
if (J->end > I->end) {
std::swap(I, J);
std::swap(IE, JE);
}
// Advance J until J->end >= I->start.
do
if (++J == JE)
return false;
while (J->end < I->start);
}
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
// Remap debug information that refers to stack slots.
MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
VE = VMap.end(); VI != VE; ++VI) {
const MDNode *Var = VI->first;
if (!Var) continue;
std::pair<unsigned, DebugLoc> &VP = VI->second;
if (SlotRemap.count(VP.first)) {
DEBUG(dbgs()<<"Remapping debug info for ["<<Var->getName()<<"].\n");
VP.first = SlotRemap[VP.first];
FixedDbg++;
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
for (DenseMap<int, int>::const_iterator it = SlotRemap.begin(),
e = SlotRemap.end(); it != e; ++it) {
const AllocaInst *From = MFI->getObjectAllocation(it->first);
const AllocaInst *To = MFI->getObjectAllocation(it->second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
}
// Remap all instructions to the new stack slots.
MachineFunction::iterator BB, BBE;
MachineBasicBlock::iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
// Skip lifetime markers. We'll remove them soon.
if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
I->getOpcode() == TargetOpcode::LIFETIME_END)
continue;
// Update the MachineMemOperand to use the new alloca.
for (MachineInstr::mmo_iterator MM = I->memoperands_begin(),
E = I->memoperands_end(); MM != E; ++MM) {
MachineMemOperand *MMO = *MM;
const Value *V = MMO->getValue();
if (!V)
continue;
const PseudoSourceValue *PSV = dyn_cast<const PseudoSourceValue>(V);
if (PSV && PSV->isConstant(MFI))
continue;
// Climb up and find the original alloca.
V = GetUnderlyingObject(V);
// If we did not find one, or if the one that we found is not in our
// map, then move on.
if (!V || !isa<AllocaInst>(V)) {
// Clear mem operand since we don't know for sure that it doesn't
// alias a merged alloca.
MMO->setValue(0);
continue;
}
const AllocaInst *AI= cast<AllocaInst>(V);
if (!Allocas.count(AI))
continue;
MMO->setValue(Allocas[AI]);
FixedMemOp++;
}
// Update all of the machine instruction operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
// Don't touch arguments.
if (FromSlot<0)
continue;
// Only look at mapped slots.
if (!SlotRemap.count(FromSlot))
continue;
// In a debug build, check that the instruction that we are modifying is
// inside the expected live range. If the instruction is not inside
// the calculated range then it means that the alloca usage moved
// outside of the lifetime markers, or that the user has a bug.
// NOTE: Alloca address calculations which happen outside the lifetime
// zone are are okay, despite the fact that we don't have a good way
// for validating all of the usages of the calculation.
#ifndef NDEBUG
bool TouchesMemory = I->mayLoad() || I->mayStore();
// If we *don't* protect the user from escaped allocas, don't bother
// validating the instructions.
if (!I->isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
SlotIndex Index = Indexes->getInstructionIndex(I);
LiveInterval *Interval = Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
}
#endif
// Fix the machine instructions.
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}
void LiveIntervals::addKillFlags(const VirtRegMap *VRM) {
// Keep track of regunit ranges.
SmallVector<std::pair<LiveInterval*, LiveInterval::iterator>, 8> RU;
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
if (MRI->reg_nodbg_empty(Reg))
continue;
LiveInterval *LI = &getInterval(Reg);
if (LI->empty())
continue;
// Find the regunit intervals for the assigned register. They may overlap
// the virtual register live range, cancelling any kills.
RU.clear();
for (MCRegUnitIterator Units(VRM->getPhys(Reg), TRI); Units.isValid();
++Units) {
LiveInterval *RUInt = &getRegUnit(*Units);
if (RUInt->empty())
continue;
RU.push_back(std::make_pair(RUInt, RUInt->find(LI->begin()->end)));
}
// Every instruction that kills Reg corresponds to a live range end point.
for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE;
++RI) {
// A block index indicates an MBB edge.
if (RI->end.isBlock())
continue;
MachineInstr *MI = getInstructionFromIndex(RI->end);
if (!MI)
continue;
// Check if any of the regunits are live beyond the end of RI. That could
// happen when a physreg is defined as a copy of a virtreg:
//
// %EAX = COPY %vreg5
// FOO %vreg5 <--- MI, cancel kill because %EAX is live.
// BAR %EAX<kill>
//
// There should be no kill flag on FOO when %vreg5 is rewritten as %EAX.
bool CancelKill = false;
for (unsigned u = 0, e = RU.size(); u != e; ++u) {
LiveInterval *RInt = RU[u].first;
LiveInterval::iterator &I = RU[u].second;
if (I == RInt->end())
continue;
I = RInt->advanceTo(I, RI->end);
if (I == RInt->end() || I->start >= RI->end)
continue;
// I is overlapping RI.
CancelKill = true;
break;
}
if (CancelKill)
MI->clearRegisterKills(Reg, NULL);
else
MI->addRegisterKilled(Reg, NULL);
}
}
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
// Remap debug information that refers to stack slots.
MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
VE = VMap.end(); VI != VE; ++VI) {
const MDNode *Var = VI->first;
if (!Var) continue;
std::pair<unsigned, DebugLoc> &VP = VI->second;
if (SlotRemap.count(VP.first)) {
DEBUG(dbgs()<<"Remapping debug info for ["<<Var->getName()<<"].\n");
VP.first = SlotRemap[VP.first];
FixedDbg++;
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const Value*, const Value*> Allocas;
for (DenseMap<int, int>::iterator it = SlotRemap.begin(),
e = SlotRemap.end(); it != e; ++it) {
const Value *From = MFI->getObjectAllocation(it->first);
const Value *To = MFI->getObjectAllocation(it->second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
}
// Remap all instructions to the new stack slots.
MachineFunction::iterator BB, BBE;
MachineBasicBlock::iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
// Skip lifetime markers. We'll remove them soon.
if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
I->getOpcode() == TargetOpcode::LIFETIME_END)
continue;
// Update the MachineMemOperand to use the new alloca.
for (MachineInstr::mmo_iterator MM = I->memoperands_begin(),
E = I->memoperands_end(); MM != E; ++MM) {
MachineMemOperand *MMO = *MM;
const Value *V = MMO->getValue();
if (!V)
continue;
// Climb up and find the original alloca.
V = GetUnderlyingObject(V);
// If we did not find one, or if the one that we found is not in our
// map, then move on.
if (!V || !Allocas.count(V))
continue;
MMO->setValue(Allocas[V]);
FixedMemOp++;
}
// Update all of the machine instruction operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
// Don't touch arguments.
if (FromSlot<0)
continue;
// Only look at mapped slots.
if (!SlotRemap.count(FromSlot))
continue;
// In a debug build, check that the instruction that we are modifying is
// inside the expected live range. If the instruction is not inside
// the calculated range then it means that the alloca usage moved
// outside of the lifetime markers.
#ifndef NDEBUG
SlotIndex Index = Indexes->getInstructionIndex(I);
LiveInterval* Interval = Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
#endif
// Fix the machine instructions.
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}