/// RenumberValues - Renumber all values in order of appearance and delete the
/// remaining unused values.
void LiveInterval::RenumberValues(LiveIntervals &lis) {
SmallPtrSet<VNInfo*, 8> Seen;
bool seenPHIDef = false;
valnos.clear();
for (const_iterator I = begin(), E = end(); I != E; ++I) {
VNInfo *VNI = I->valno;
if (!Seen.insert(VNI))
continue;
assert(!VNI->isUnused() && "Unused valno used by live range");
VNI->id = (unsigned)valnos.size();
valnos.push_back(VNI);
VNI->setHasPHIKill(false);
if (VNI->isPHIDef())
seenPHIDef = true;
}
// Recompute phi kill flags.
if (!seenPHIDef)
return;
for (const_vni_iterator I = vni_begin(), E = vni_end(); I != E; ++I) {
VNInfo *VNI = *I;
if (!VNI->isPHIDef())
continue;
const MachineBasicBlock *PHIBB = lis.getMBBFromIndex(VNI->def);
assert(PHIBB && "No basic block for phi-def");
for (MachineBasicBlock::const_pred_iterator PI = PHIBB->pred_begin(),
PE = PHIBB->pred_end(); PI != PE; ++PI) {
VNInfo *KVNI = getVNInfoAt(lis.getMBBEndIdx(*PI).getPrevSlot());
if (KVNI)
KVNI->setHasPHIKill(true);
}
}
}
/// analyzeSiblingValues - Trace values defined by sibling copies back to
/// something that isn't a sibling copy.
///
/// Keep track of values that may be rematerializable.
void InlineSpiller::analyzeSiblingValues() {
SibValues.clear();
// No siblings at all?
if (Edit->getReg() == Original)
return;
LiveInterval &OrigLI = LIS.getInterval(Original);
for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) {
unsigned Reg = RegsToSpill[i];
LiveInterval &LI = LIS.getInterval(Reg);
for (LiveInterval::const_vni_iterator VI = LI.vni_begin(),
VE = LI.vni_end(); VI != VE; ++VI) {
VNInfo *VNI = *VI;
if (VNI->isUnused())
continue;
MachineInstr *DefMI = 0;
if (!VNI->isPHIDef()) {
DefMI = LIS.getInstructionFromIndex(VNI->def);
assert(DefMI && "No defining instruction");
}
// Check possible sibling copies.
if (VNI->isPHIDef() || DefMI->isCopy()) {
VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def);
assert(OrigVNI && "Def outside original live range");
if (OrigVNI->def != VNI->def)
DefMI = traceSiblingValue(Reg, VNI, OrigVNI);
}
if (DefMI && Edit->checkRematerializable(VNI, DefMI, AA)) {
DEBUG(dbgs() << "Value " << PrintReg(Reg) << ':' << VNI->id << '@'
<< VNI->def << " may remat from " << *DefMI);
}
}
}
}
/// getCriticalExits - It may be necessary to partially break critical edges
/// leaving the loop if an exit block has phi uses of curli. Collect the exit
/// blocks that need special treatment into CriticalExits.
void SplitAnalysis::getCriticalExits(const SplitAnalysis::LoopBlocks &Blocks,
BlockPtrSet &CriticalExits) {
CriticalExits.clear();
// A critical exit block contains a phi def of curli, and has a predecessor
// that is not in the loop nor a loop predecessor.
// For such an exit block, the edges carrying the new variable must be moved
// to a new pre-exit block.
for (BlockPtrSet::iterator I = Blocks.Exits.begin(), E = Blocks.Exits.end();
I != E; ++I) {
const MachineBasicBlock *Succ = *I;
SlotIndex SuccIdx = lis_.getMBBStartIdx(Succ);
VNInfo *SuccVNI = curli_->getVNInfoAt(SuccIdx);
// This exit may not have curli live in at all. No need to split.
if (!SuccVNI)
continue;
// If this is not a PHI def, it is either using a value from before the
// loop, or a value defined inside the loop. Both are safe.
if (!SuccVNI->isPHIDef() || SuccVNI->def.getBaseIndex() != SuccIdx)
continue;
// This exit block does have a PHI. Does it also have a predecessor that is
// not a loop block or loop predecessor?
for (MachineBasicBlock::const_pred_iterator PI = Succ->pred_begin(),
PE = Succ->pred_end(); PI != PE; ++PI) {
const MachineBasicBlock *Pred = *PI;
if (Blocks.Loop.count(Pred) || Blocks.Preds.count(Pred))
continue;
// This is a critical exit block, and we need to split the exit edge.
CriticalExits.insert(Succ);
break;
}
}
}
static void extendSegmentsToUses(LiveRange &LR, const SlotIndexes &Indexes,
ShrinkToUsesWorkList &WorkList,
const LiveRange &OldRange) {
// Keep track of the PHIs that are in use.
SmallPtrSet<VNInfo*, 8> UsedPHIs;
// Blocks that have already been added to WorkList as live-out.
SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
// Extend intervals to reach all uses in WorkList.
while (!WorkList.empty()) {
SlotIndex Idx = WorkList.back().first;
VNInfo *VNI = WorkList.back().second;
WorkList.pop_back();
const MachineBasicBlock *MBB = Indexes.getMBBFromIndex(Idx.getPrevSlot());
SlotIndex BlockStart = Indexes.getMBBStartIdx(MBB);
// Extend the live range for VNI to be live at Idx.
if (VNInfo *ExtVNI = LR.extendInBlock(BlockStart, Idx)) {
assert(ExtVNI == VNI && "Unexpected existing value number");
(void)ExtVNI;
// Is this a PHIDef we haven't seen before?
if (!VNI->isPHIDef() || VNI->def != BlockStart ||
!UsedPHIs.insert(VNI).second)
continue;
// The PHI is live, make sure the predecessors are live-out.
for (auto &Pred : MBB->predecessors()) {
if (!LiveOut.insert(Pred).second)
continue;
SlotIndex Stop = Indexes.getMBBEndIdx(Pred);
// A predecessor is not required to have a live-out value for a PHI.
if (VNInfo *PVNI = OldRange.getVNInfoBefore(Stop))
WorkList.push_back(std::make_pair(Stop, PVNI));
}
continue;
}
// VNI is live-in to MBB.
DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
LR.addSegment(LiveRange::Segment(BlockStart, Idx, VNI));
// Make sure VNI is live-out from the predecessors.
for (auto &Pred : MBB->predecessors()) {
if (!LiveOut.insert(Pred).second)
continue;
SlotIndex Stop = Indexes.getMBBEndIdx(Pred);
assert(OldRange.getVNInfoBefore(Stop) == VNI &&
"Wrong value out of predecessor");
WorkList.push_back(std::make_pair(Stop, VNI));
}
}
}
/// reMaterializeAll - Try to rematerialize as many uses as possible,
/// and trim the live ranges after.
void InlineSpiller::reMaterializeAll() {
// analyzeSiblingValues has already tested all relevant defining instructions.
if (!Edit->anyRematerializable(AA))
return;
UsedValues.clear();
// Try to remat before all uses of snippets.
bool anyRemat = false;
for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) {
unsigned Reg = RegsToSpill[i];
LiveInterval &LI = LIS.getInterval(Reg);
for (MachineRegisterInfo::use_nodbg_iterator
RI = MRI.use_nodbg_begin(Reg);
MachineInstr *MI = RI.skipBundle();)
anyRemat |= reMaterializeFor(LI, MI);
}
if (!anyRemat)
return;
// Remove any values that were completely rematted.
for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) {
unsigned Reg = RegsToSpill[i];
LiveInterval &LI = LIS.getInterval(Reg);
for (LiveInterval::vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused() || VNI->isPHIDef() || UsedValues.count(VNI))
continue;
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
MI->addRegisterDead(Reg, &TRI);
if (!MI->allDefsAreDead())
continue;
DEBUG(dbgs() << "All defs dead: " << *MI);
DeadDefs.push_back(MI);
}
}
// Eliminate dead code after remat. Note that some snippet copies may be
// deleted here.
if (DeadDefs.empty())
return;
DEBUG(dbgs() << "Remat created " << DeadDefs.size() << " dead defs.\n");
Edit->eliminateDeadDefs(DeadDefs, RegsToSpill);
// Get rid of deleted and empty intervals.
unsigned ResultPos = 0;
for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) {
unsigned Reg = RegsToSpill[i];
if (!LIS.hasInterval(Reg))
continue;
LiveInterval &LI = LIS.getInterval(Reg);
if (LI.empty()) {
Edit->eraseVirtReg(Reg);
continue;
}
RegsToSpill[ResultPos++] = Reg;
}
RegsToSpill.erase(RegsToSpill.begin() + ResultPos, RegsToSpill.end());
DEBUG(dbgs() << RegsToSpill.size() << " registers to spill after remat.\n");
}
/// traceSiblingValue - Trace a value that is about to be spilled back to the
/// real defining instructions by looking through sibling copies. Always stay
/// within the range of OrigVNI so the registers are known to carry the same
/// value.
///
/// Determine if the value is defined by all reloads, so spilling isn't
/// necessary - the value is already in the stack slot.
///
/// Return a defining instruction that may be a candidate for rematerialization.
///
MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
VNInfo *OrigVNI) {
// Check if a cached value already exists.
SibValueMap::iterator SVI;
bool Inserted;
tie(SVI, Inserted) =
SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI)));
if (!Inserted) {
DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second);
return SVI->second.DefMI;
}
DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << '\n');
// List of (Reg, VNI) that have been inserted into SibValues, but need to be
// processed.
SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList;
WorkList.push_back(std::make_pair(UseReg, UseVNI));
do {
unsigned Reg;
VNInfo *VNI;
tie(Reg, VNI) = WorkList.pop_back_val();
DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def
<< ":\t");
// First check if this value has already been computed.
SVI = SibValues.find(VNI);
assert(SVI != SibValues.end() && "Missing SibValues entry");
// Trace through PHI-defs created by live range splitting.
if (VNI->isPHIDef()) {
// Stop at original PHIs. We don't know the value at the predecessors.
if (VNI->def == OrigVNI->def) {
DEBUG(dbgs() << "orig phi value\n");
SVI->second.DefByOrigPHI = true;
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
continue;
}
// This is a PHI inserted by live range splitting. We could trace the
// live-out value from predecessor blocks, but that search can be very
// expensive if there are many predecessors and many more PHIs as
// generated by tail-dup when it sees an indirectbr. Instead, look at
// all the non-PHI defs that have the same value as OrigVNI. They must
// jointly dominate VNI->def. This is not optimal since VNI may actually
// be jointly dominated by a smaller subset of defs, so there is a change
// we will miss a AllDefsAreReloads optimization.
// Separate all values dominated by OrigVNI into PHIs and non-PHIs.
SmallVector<VNInfo*, 8> PHIs, NonPHIs;
LiveInterval &LI = LIS.getInterval(Reg);
LiveInterval &OrigLI = LIS.getInterval(Original);
for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end();
VI != VE; ++VI) {
VNInfo *VNI2 = *VI;
if (VNI2->isUnused())
continue;
if (!OrigLI.containsOneValue() &&
OrigLI.getVNInfoAt(VNI2->def) != OrigVNI)
continue;
if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def)
PHIs.push_back(VNI2);
else
NonPHIs.push_back(VNI2);
}
DEBUG(dbgs() << "split phi value, checking " << PHIs.size()
<< " phi-defs, and " << NonPHIs.size()
<< " non-phi/orig defs\n");
// Create entries for all the PHIs. Don't add them to the worklist, we
// are processing all of them in one go here.
for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i])));
// Add every PHI as a dependent of all the non-PHIs.
for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) {
VNInfo *NonPHI = NonPHIs[i];
// Known value? Try an insertion.
tie(SVI, Inserted) =
SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI)));
// Add all the PHIs as dependents of NonPHI.
for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi)
SVI->second.Deps.push_back(PHIs[pi]);
// This is the first time we see NonPHI, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(Reg, NonPHI));
else
// Propagate to all inserted PHIs, not just VNI.
propagateSiblingValue(SVI);
}
// Next work list item.
continue;
}
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Missing def");
// Trace through sibling copies.
if (unsigned SrcReg = isFullCopyOf(MI, Reg)) {
if (isSibling(SrcReg)) {
LiveInterval &SrcLI = LIS.getInterval(SrcReg);
LiveRangeQuery SrcQ(SrcLI, VNI->def);
assert(SrcQ.valueIn() && "Copy from non-existing value");
// Check if this COPY kills its source.
SVI->second.KillsSource = SrcQ.isKill();
VNInfo *SrcVNI = SrcQ.valueIn();
DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':'
<< SrcVNI->id << '@' << SrcVNI->def
<< " kill=" << unsigned(SVI->second.KillsSource) << '\n');
// Known sibling source value? Try an insertion.
tie(SVI, Inserted) = SibValues.insert(std::make_pair(SrcVNI,
SibValueInfo(SrcReg, SrcVNI)));
// This is the first time we see Src, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(SrcReg, SrcVNI));
propagateSiblingValue(SVI, VNI);
// Next work list item.
continue;
}
}
// Track reachable reloads.
SVI->second.DefMI = MI;
SVI->second.SpillMBB = MI->getParent();
int FI;
if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) {
DEBUG(dbgs() << "reload\n");
propagateSiblingValue(SVI);
// Next work list item.
continue;
}
// Potential remat candidate.
DEBUG(dbgs() << "def " << *MI);
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
} while (!WorkList.empty());
// Look up the value we were looking for. We already did this lookup at the
// top of the function, but SibValues may have been invalidated.
SVI = SibValues.find(UseVNI);
assert(SVI != SibValues.end() && "Didn't compute requested info");
DEBUG(dbgs() << " traced to:\t" << SVI->second);
return SVI->second.DefMI;
}
void MachineVerifier::verifyLiveIntervals() {
assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
for (LiveIntervals::const_iterator LVI = LiveInts->begin(),
LVE = LiveInts->end(); LVI != LVE; ++LVI) {
const LiveInterval &LI = *LVI->second;
// Spilling and splitting may leave unused registers around. Skip them.
if (MRI->use_empty(LI.reg))
continue;
// Physical registers have much weirdness going on, mostly from coalescing.
// We should probably fix it, but for now just ignore them.
if (TargetRegisterInfo::isPhysicalRegister(LI.reg))
continue;
assert(LVI->first == LI.reg && "Invalid reg to interval mapping");
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I!=E; ++I) {
VNInfo *VNI = *I;
const VNInfo *DefVNI = LI.getVNInfoAt(VNI->def);
if (!DefVNI) {
if (!VNI->isUnused()) {
report("Valno not live at def and not marked unused", MF);
*OS << "Valno #" << VNI->id << " in " << LI << '\n';
}
continue;
}
if (VNI->isUnused())
continue;
if (DefVNI != VNI) {
report("Live range at def has different valno", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " where valno #" << DefVNI->id << " is live in " << LI << '\n';
continue;
}
const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);
if (!MBB) {
report("Invalid definition index", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
continue;
}
if (VNI->isPHIDef()) {
if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {
report("PHIDef value is not defined at MBB start", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< ", not at the beginning of BB#" << MBB->getNumber()
<< " in " << LI << '\n';
}
} else {
// Non-PHI def.
const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);
if (!MI) {
report("No instruction at def index", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
} else if (!MI->modifiesRegister(LI.reg, TRI)) {
report("Defining instruction does not modify register", MI);
*OS << "Valno #" << VNI->id << " in " << LI << '\n';
}
bool isEarlyClobber = false;
if (MI) {
for (MachineInstr::const_mop_iterator MOI = MI->operands_begin(),
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
if (MOI->isReg() && MOI->getReg() == LI.reg && MOI->isDef() &&
MOI->isEarlyClobber()) {
isEarlyClobber = true;
break;
}
}
}
// Early clobber defs begin at USE slots, but other defs must begin at
// DEF slots.
if (isEarlyClobber) {
if (!VNI->def.isUse()) {
report("Early clobber def must be at a USE slot", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
}
} else if (!VNI->def.isDef()) {
report("Non-PHI, non-early clobber def must be at a DEF slot", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
}
}
}
for (LiveInterval::const_iterator I = LI.begin(), E = LI.end(); I!=E; ++I) {
const VNInfo *VNI = I->valno;
assert(VNI && "Live range has no valno");
if (VNI->id >= LI.getNumValNums() || VNI != LI.getValNumInfo(VNI->id)) {
report("Foreign valno in live range", MF);
I->print(*OS);
*OS << " has a valno not in " << LI << '\n';
}
if (VNI->isUnused()) {
report("Live range valno is marked unused", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
}
const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(I->start);
if (!MBB) {
report("Bad start of live segment, no basic block", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
continue;
}
SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);
if (I->start != MBBStartIdx && I->start != VNI->def) {
report("Live segment must begin at MBB entry or valno def", MBB);
I->print(*OS);
*OS << " in " << LI << '\n' << "Basic block starts at "
<< MBBStartIdx << '\n';
}
const MachineBasicBlock *EndMBB =
LiveInts->getMBBFromIndex(I->end.getPrevSlot());
if (!EndMBB) {
report("Bad end of live segment, no basic block", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
continue;
}
if (I->end != LiveInts->getMBBEndIdx(EndMBB)) {
// The live segment is ending inside EndMBB
const MachineInstr *MI =
LiveInts->getInstructionFromIndex(I->end.getPrevSlot());
if (!MI) {
report("Live segment doesn't end at a valid instruction", EndMBB);
I->print(*OS);
*OS << " in " << LI << '\n' << "Basic block starts at "
<< MBBStartIdx << '\n';
} else if (TargetRegisterInfo::isVirtualRegister(LI.reg) &&
!MI->readsVirtualRegister(LI.reg)) {
// A live range can end with either a redefinition, a kill flag on a
// use, or a dead flag on a def.
// FIXME: Should we check for each of these?
bool hasDeadDef = false;
for (MachineInstr::const_mop_iterator MOI = MI->operands_begin(),
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
if (MOI->isReg() && MOI->getReg() == LI.reg && MOI->isDef() && MOI->isDead()) {
hasDeadDef = true;
break;
}
}
if (!hasDeadDef) {
report("Instruction killing live segment neither defines nor reads "
"register", MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
}
}
// Now check all the basic blocks in this live segment.
MachineFunction::const_iterator MFI = MBB;
// Is this live range the beginning of a non-PHIDef VN?
if (I->start == VNI->def && !VNI->isPHIDef()) {
// Not live-in to any blocks.
if (MBB == EndMBB)
continue;
// Skip this block.
++MFI;
}
for (;;) {
assert(LiveInts->isLiveInToMBB(LI, MFI));
// We don't know how to track physregs into a landing pad.
if (TargetRegisterInfo::isPhysicalRegister(LI.reg) &&
MFI->isLandingPad()) {
if (&*MFI == EndMBB)
break;
++MFI;
continue;
}
// Check that VNI is live-out of all predecessors.
for (MachineBasicBlock::const_pred_iterator PI = MFI->pred_begin(),
PE = MFI->pred_end(); PI != PE; ++PI) {
SlotIndex PEnd = LiveInts->getMBBEndIdx(*PI).getPrevSlot();
const VNInfo *PVNI = LI.getVNInfoAt(PEnd);
if (VNI->isPHIDef() && VNI->def == LiveInts->getMBBStartIdx(MFI))
continue;
if (!PVNI) {
report("Register not marked live out of predecessor", *PI);
*OS << "Valno #" << VNI->id << " live into BB#" << MFI->getNumber()
<< '@' << LiveInts->getMBBStartIdx(MFI) << ", not live at "
<< PEnd << " in " << LI << '\n';
continue;
}
if (PVNI != VNI) {
report("Different value live out of predecessor", *PI);
*OS << "Valno #" << PVNI->id << " live out of BB#"
<< (*PI)->getNumber() << '@' << PEnd
<< "\nValno #" << VNI->id << " live into BB#" << MFI->getNumber()
<< '@' << LiveInts->getMBBStartIdx(MFI) << " in " << LI << '\n';
}
}
if (&*MFI == EndMBB)
break;
++MFI;
}
}
// Check the LI only has one connected component.
if (TargetRegisterInfo::isVirtualRegister(LI.reg)) {
ConnectedVNInfoEqClasses ConEQ(*LiveInts);
unsigned NumComp = ConEQ.Classify(&LI);
if (NumComp > 1) {
report("Multiple connected components in live interval", MF);
*OS << NumComp << " components in " << LI << '\n';
for (unsigned comp = 0; comp != NumComp; ++comp) {
*OS << comp << ": valnos";
for (LiveInterval::const_vni_iterator I = LI.vni_begin(),
E = LI.vni_end(); I!=E; ++I)
if (comp == ConEQ.getEqClass(*I))
*OS << ' ' << (*I)->id;
*OS << '\n';
}
}
}
}
}
/// shrinkToUses - After removing some uses of a register, shrink its live
/// range to just the remaining uses. This method does not compute reaching
/// defs for new uses, and it doesn't remove dead defs.
bool LiveIntervals::shrinkToUses(LiveInterval *li,
SmallVectorImpl<MachineInstr*> *dead) {
DEBUG(dbgs() << "Shrink: " << *li << '\n');
assert(TargetRegisterInfo::isVirtualRegister(li->reg)
&& "Can only shrink virtual registers");
// Find all the values used, including PHI kills.
SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
// Blocks that have already been added to WorkList as live-out.
SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
// Visit all instructions reading li->reg.
for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg);
MachineInstr *UseMI = I.skipInstruction();) {
if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
continue;
SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
LiveRangeQuery LRQ(*li, Idx);
VNInfo *VNI = LRQ.valueIn();
if (!VNI) {
// This shouldn't happen: readsVirtualRegister returns true, but there is
// no live value. It is likely caused by a target getting <undef> flags
// wrong.
DEBUG(dbgs() << Idx << '\t' << *UseMI
<< "Warning: Instr claims to read non-existent value in "
<< *li << '\n');
continue;
}
// Special case: An early-clobber tied operand reads and writes the
// register one slot early.
if (VNInfo *DefVNI = LRQ.valueDefined())
Idx = DefVNI->def;
WorkList.push_back(std::make_pair(Idx, VNI));
}
// Create a new live interval with only minimal live segments per def.
LiveInterval NewLI(li->reg, 0);
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI));
}
// Keep track of the PHIs that are in use.
SmallPtrSet<VNInfo*, 8> UsedPHIs;
// Extend intervals to reach all uses in WorkList.
while (!WorkList.empty()) {
SlotIndex Idx = WorkList.back().first;
VNInfo *VNI = WorkList.back().second;
WorkList.pop_back();
const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
SlotIndex BlockStart = getMBBStartIdx(MBB);
// Extend the live range for VNI to be live at Idx.
if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) {
(void)ExtVNI;
assert(ExtVNI == VNI && "Unexpected existing value number");
// Is this a PHIDef we haven't seen before?
if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
continue;
// The PHI is live, make sure the predecessors are live-out.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
// A predecessor is not required to have a live-out value for a PHI.
if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
WorkList.push_back(std::make_pair(Stop, PVNI));
}
continue;
}
// VNI is live-in to MBB.
DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
NewLI.addRange(LiveRange(BlockStart, Idx, VNI));
// Make sure VNI is live-out from the predecessors.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
assert(li->getVNInfoBefore(Stop) == VNI &&
"Wrong value out of predecessor");
WorkList.push_back(std::make_pair(Stop, VNI));
}
}
// Handle dead values.
bool CanSeparate = false;
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def);
assert(LII != NewLI.end() && "Missing live range for PHI");
if (LII->end != VNI->def.getDeadSlot())
continue;
if (VNI->isPHIDef()) {
// This is a dead PHI. Remove it.
VNI->markUnused();
NewLI.removeRange(*LII);
DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
CanSeparate = true;
} else {
// This is a dead def. Make sure the instruction knows.
MachineInstr *MI = getInstructionFromIndex(VNI->def);
assert(MI && "No instruction defining live value");
MI->addRegisterDead(li->reg, TRI);
if (dead && MI->allDefsAreDead()) {
DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
dead->push_back(MI);
}
}
}
// Move the trimmed ranges back.
li->ranges.swap(NewLI.ranges);
DEBUG(dbgs() << "Shrunk: " << *li << '\n');
return CanSeparate;
}
void MachineVerifier::verifyLiveIntervals() {
assert(LiveInts && "Don't call verifyLiveIntervals without LiveInts");
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
// Spilling and splitting may leave unused registers around. Skip them.
if (MRI->reg_nodbg_empty(Reg))
continue;
if (!LiveInts->hasInterval(Reg)) {
report("Missing live interval for virtual register", MF);
*OS << PrintReg(Reg, TRI) << " still has defs or uses\n";
continue;
}
const LiveInterval &LI = LiveInts->getInterval(Reg);
assert(Reg == LI.reg && "Invalid reg to interval mapping");
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I!=E; ++I) {
VNInfo *VNI = *I;
const VNInfo *DefVNI = LI.getVNInfoAt(VNI->def);
if (!DefVNI) {
if (!VNI->isUnused()) {
report("Valno not live at def and not marked unused", MF);
*OS << "Valno #" << VNI->id << " in " << LI << '\n';
}
continue;
}
if (VNI->isUnused())
continue;
if (DefVNI != VNI) {
report("Live range at def has different valno", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " where valno #" << DefVNI->id << " is live in " << LI << '\n';
continue;
}
const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def);
if (!MBB) {
report("Invalid definition index", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
continue;
}
if (VNI->isPHIDef()) {
if (VNI->def != LiveInts->getMBBStartIdx(MBB)) {
report("PHIDef value is not defined at MBB start", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< ", not at the beginning of BB#" << MBB->getNumber()
<< " in " << LI << '\n';
}
} else {
// Non-PHI def.
const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def);
if (!MI) {
report("No instruction at def index", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
continue;
}
bool hasDef = false;
bool isEarlyClobber = false;
for (ConstMIBundleOperands MOI(MI); MOI.isValid(); ++MOI) {
if (!MOI->isReg() || !MOI->isDef())
continue;
if (TargetRegisterInfo::isVirtualRegister(LI.reg)) {
if (MOI->getReg() != LI.reg)
continue;
} else {
if (!TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) ||
!TRI->regsOverlap(LI.reg, MOI->getReg()))
continue;
}
hasDef = true;
if (MOI->isEarlyClobber())
isEarlyClobber = true;
}
if (!hasDef) {
report("Defining instruction does not modify register", MI);
*OS << "Valno #" << VNI->id << " in " << LI << '\n';
}
// Early clobber defs begin at USE slots, but other defs must begin at
// DEF slots.
if (isEarlyClobber) {
if (!VNI->def.isEarlyClobber()) {
report("Early clobber def must be at an early-clobber slot", MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
}
} else if (!VNI->def.isRegister()) {
report("Non-PHI, non-early clobber def must be at a register slot",
MF);
*OS << "Valno #" << VNI->id << " is defined at " << VNI->def
<< " in " << LI << '\n';
}
}
}
for (LiveInterval::const_iterator I = LI.begin(), E = LI.end(); I!=E; ++I) {
const VNInfo *VNI = I->valno;
assert(VNI && "Live range has no valno");
if (VNI->id >= LI.getNumValNums() || VNI != LI.getValNumInfo(VNI->id)) {
report("Foreign valno in live range", MF);
I->print(*OS);
*OS << " has a valno not in " << LI << '\n';
}
if (VNI->isUnused()) {
report("Live range valno is marked unused", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
}
const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(I->start);
if (!MBB) {
report("Bad start of live segment, no basic block", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
continue;
}
SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB);
if (I->start != MBBStartIdx && I->start != VNI->def) {
report("Live segment must begin at MBB entry or valno def", MBB);
I->print(*OS);
*OS << " in " << LI << '\n' << "Basic block starts at "
<< MBBStartIdx << '\n';
}
const MachineBasicBlock *EndMBB =
LiveInts->getMBBFromIndex(I->end.getPrevSlot());
if (!EndMBB) {
report("Bad end of live segment, no basic block", MF);
I->print(*OS);
*OS << " in " << LI << '\n';
continue;
}
// No more checks for live-out segments.
if (I->end == LiveInts->getMBBEndIdx(EndMBB))
continue;
// The live segment is ending inside EndMBB
const MachineInstr *MI =
LiveInts->getInstructionFromIndex(I->end.getPrevSlot());
if (!MI) {
report("Live segment doesn't end at a valid instruction", EndMBB);
I->print(*OS);
*OS << " in " << LI << '\n' << "Basic block starts at "
<< MBBStartIdx << '\n';
continue;
}
// The block slot must refer to a basic block boundary.
if (I->end.isBlock()) {
report("Live segment ends at B slot of an instruction", MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
if (I->end.isDead()) {
// Segment ends on the dead slot.
// That means there must be a dead def.
if (!SlotIndex::isSameInstr(I->start, I->end)) {
report("Live segment ending at dead slot spans instructions", MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
}
// A live segment can only end at an early-clobber slot if it is being
// redefined by an early-clobber def.
if (I->end.isEarlyClobber()) {
if (I+1 == E || (I+1)->start != I->end) {
report("Live segment ending at early clobber slot must be "
"redefined by an EC def in the same instruction", MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
}
// The following checks only apply to virtual registers. Physreg liveness
// is too weird to check.
if (TargetRegisterInfo::isVirtualRegister(LI.reg)) {
// A live range can end with either a redefinition, a kill flag on a
// use, or a dead flag on a def.
bool hasRead = false;
bool hasDeadDef = false;
for (ConstMIBundleOperands MOI(MI); MOI.isValid(); ++MOI) {
if (!MOI->isReg() || MOI->getReg() != LI.reg)
continue;
if (MOI->readsReg())
hasRead = true;
if (MOI->isDef() && MOI->isDead())
hasDeadDef = true;
}
if (I->end.isDead()) {
if (!hasDeadDef) {
report("Instruction doesn't have a dead def operand", MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
} else {
if (!hasRead) {
report("Instruction ending live range doesn't read the register",
MI);
I->print(*OS);
*OS << " in " << LI << '\n';
}
}
}
// Now check all the basic blocks in this live segment.
MachineFunction::const_iterator MFI = MBB;
// Is this live range the beginning of a non-PHIDef VN?
if (I->start == VNI->def && !VNI->isPHIDef()) {
// Not live-in to any blocks.
if (MBB == EndMBB)
continue;
// Skip this block.
++MFI;
}
for (;;) {
assert(LiveInts->isLiveInToMBB(LI, MFI));
// We don't know how to track physregs into a landing pad.
if (TargetRegisterInfo::isPhysicalRegister(LI.reg) &&
MFI->isLandingPad()) {
if (&*MFI == EndMBB)
break;
++MFI;
continue;
}
// Is VNI a PHI-def in the current block?
bool IsPHI = VNI->isPHIDef() &&
VNI->def == LiveInts->getMBBStartIdx(MFI);
// Check that VNI is live-out of all predecessors.
for (MachineBasicBlock::const_pred_iterator PI = MFI->pred_begin(),
PE = MFI->pred_end(); PI != PE; ++PI) {
SlotIndex PEnd = LiveInts->getMBBEndIdx(*PI);
const VNInfo *PVNI = LI.getVNInfoBefore(PEnd);
// All predecessors must have a live-out value.
if (!PVNI) {
report("Register not marked live out of predecessor", *PI);
*OS << "Valno #" << VNI->id << " live into BB#" << MFI->getNumber()
<< '@' << LiveInts->getMBBStartIdx(MFI) << ", not live before "
<< PEnd << " in " << LI << '\n';
continue;
}
// Only PHI-defs can take different predecessor values.
if (!IsPHI && PVNI != VNI) {
report("Different value live out of predecessor", *PI);
*OS << "Valno #" << PVNI->id << " live out of BB#"
<< (*PI)->getNumber() << '@' << PEnd
<< "\nValno #" << VNI->id << " live into BB#" << MFI->getNumber()
<< '@' << LiveInts->getMBBStartIdx(MFI) << " in "
<< PrintReg(Reg) << ": " << LI << '\n';
}
}
if (&*MFI == EndMBB)
break;
++MFI;
}
}
// Check the LI only has one connected component.
if (TargetRegisterInfo::isVirtualRegister(LI.reg)) {
ConnectedVNInfoEqClasses ConEQ(*LiveInts);
unsigned NumComp = ConEQ.Classify(&LI);
if (NumComp > 1) {
report("Multiple connected components in live interval", MF);
*OS << NumComp << " components in " << LI << '\n';
for (unsigned comp = 0; comp != NumComp; ++comp) {
*OS << comp << ": valnos";
for (LiveInterval::const_vni_iterator I = LI.vni_begin(),
E = LI.vni_end(); I!=E; ++I)
if (comp == ConEQ.getEqClass(*I))
*OS << ' ' << (*I)->id;
*OS << '\n';
}
}
}
}
}
/// transferValues - Transfer all possible values to the new live ranges.
/// Values that were rematerialized are left alone, they need LRCalc.extend().
bool SplitEditor::transferValues() {
bool Skipped = false;
RegAssignMap::const_iterator AssignI = RegAssign.begin();
for (LiveInterval::const_iterator ParentI = Edit->getParent().begin(),
ParentE = Edit->getParent().end(); ParentI != ParentE; ++ParentI) {
DEBUG(dbgs() << " blit " << *ParentI << ':');
VNInfo *ParentVNI = ParentI->valno;
// RegAssign has holes where RegIdx 0 should be used.
SlotIndex Start = ParentI->start;
AssignI.advanceTo(Start);
do {
unsigned RegIdx;
SlotIndex End = ParentI->end;
if (!AssignI.valid()) {
RegIdx = 0;
} else if (AssignI.start() <= Start) {
RegIdx = AssignI.value();
if (AssignI.stop() < End) {
End = AssignI.stop();
++AssignI;
}
} else {
RegIdx = 0;
End = std::min(End, AssignI.start());
}
// The interval [Start;End) is continuously mapped to RegIdx, ParentVNI.
DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx);
LiveInterval *LI = Edit->get(RegIdx);
// Check for a simply defined value that can be blitted directly.
ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
if (VNInfo *VNI = VFP.getPointer()) {
DEBUG(dbgs() << ':' << VNI->id);
LI->addRange(LiveRange(Start, End, VNI));
Start = End;
continue;
}
// Skip values with forced recomputation.
if (VFP.getInt()) {
DEBUG(dbgs() << "(recalc)");
Skipped = true;
Start = End;
continue;
}
LiveRangeCalc &LRC = getLRCalc(RegIdx);
// This value has multiple defs in RegIdx, but it wasn't rematerialized,
// so the live range is accurate. Add live-in blocks in [Start;End) to the
// LiveInBlocks.
MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start);
SlotIndex BlockStart, BlockEnd;
tie(BlockStart, BlockEnd) = LIS.getSlotIndexes()->getMBBRange(MBB);
// The first block may be live-in, or it may have its own def.
if (Start != BlockStart) {
VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped value");
DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber());
// MBB has its own def. Is it also live-out?
if (BlockEnd <= End)
LRC.setLiveOutValue(MBB, VNI);
// Skip to the next block for live-in.
++MBB;
BlockStart = BlockEnd;
}
// Handle the live-in blocks covered by [Start;End).
assert(Start <= BlockStart && "Expected live-in block");
while (BlockStart < End) {
DEBUG(dbgs() << ">BB#" << MBB->getNumber());
BlockEnd = LIS.getMBBEndIdx(MBB);
if (BlockStart == ParentVNI->def) {
// This block has the def of a parent PHI, so it isn't live-in.
assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
VNInfo *VNI = LI->extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped parent PHI");
if (End >= BlockEnd)
LRC.setLiveOutValue(MBB, VNI); // Live-out as well.
} else {
// This block needs a live-in value. The last block covered may not
// be live-out.
if (End < BlockEnd)
LRC.addLiveInBlock(LI, MDT[MBB], End);
else {
// Live-through, and we don't know the value.
LRC.addLiveInBlock(LI, MDT[MBB]);
LRC.setLiveOutValue(MBB, 0);
}
}
BlockStart = BlockEnd;
++MBB;
}
Start = End;
} while (Start != ParentI->end);
DEBUG(dbgs() << '\n');
}
LRCalc[0].calculateValues(LIS.getSlotIndexes(), &MDT,
&LIS.getVNInfoAllocator());
if (SpillMode)
LRCalc[1].calculateValues(LIS.getSlotIndexes(), &MDT,
&LIS.getVNInfoAllocator());
return Skipped;
}
