void ConnectedVNInfoEqClasses::Distribute(LiveInterval *LIV[],
MachineRegisterInfo &MRI) {
assert(LIV[0] && "LIV[0] must be set");
LiveInterval &LI = *LIV[0];
// Rewrite instructions.
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LI.reg),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = *RI;
MachineInstr *MI = RI->getParent();
++RI;
// DBG_VALUE instructions don't have slot indexes, so get the index of the
// instruction before them.
// Normally, DBG_VALUE instructions are removed before this function is
// called, but it is not a requirement.
SlotIndex Idx;
if (MI->isDebugValue())
Idx = LIS.getSlotIndexes()->getIndexBefore(MI);
else
Idx = LIS.getInstructionIndex(MI);
LiveQueryResult LRQ = LI.Query(Idx);
const VNInfo *VNI = MO.readsReg() ? LRQ.valueIn() : LRQ.valueDefined();
// In the case of an <undef> use that isn't tied to any def, VNI will be
// NULL. If the use is tied to a def, VNI will be the defined value.
if (!VNI)
continue;
MO.setReg(LIV[getEqClass(VNI)]->reg);
}
// Move runs to new intervals.
LiveInterval::iterator J = LI.begin(), E = LI.end();
while (J != E && EqClass[J->valno->id] == 0)
++J;
for (LiveInterval::iterator I = J; I != E; ++I) {
if (unsigned eq = EqClass[I->valno->id]) {
assert((LIV[eq]->empty() || LIV[eq]->expiredAt(I->start)) &&
"New intervals should be empty");
LIV[eq]->segments.push_back(*I);
} else
*J++ = *I;
}
LI.segments.erase(J, E);
// Transfer VNInfos to their new owners and renumber them.
unsigned j = 0, e = LI.getNumValNums();
while (j != e && EqClass[j] == 0)
++j;
for (unsigned i = j; i != e; ++i) {
VNInfo *VNI = LI.getValNumInfo(i);
if (unsigned eq = EqClass[i]) {
VNI->id = LIV[eq]->getNumValNums();
LIV[eq]->valnos.push_back(VNI);
} else {
VNI->id = j;
LI.valnos[j++] = VNI;
}
}
LI.valnos.resize(j);
}
void LiveIntervals::pruneValue(LiveRange &LR, SlotIndex Kill,
SmallVectorImpl<SlotIndex> *EndPoints) {
LiveQueryResult LRQ = LR.Query(Kill);
VNInfo *VNI = LRQ.valueOutOrDead();
if (!VNI)
return;
MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(Kill);
SlotIndex MBBEnd = Indexes->getMBBEndIdx(KillMBB);
// If VNI isn't live out from KillMBB, the value is trivially pruned.
if (LRQ.endPoint() < MBBEnd) {
LR.removeSegment(Kill, LRQ.endPoint());
if (EndPoints) EndPoints->push_back(LRQ.endPoint());
return;
}
// VNI is live out of KillMBB.
LR.removeSegment(Kill, MBBEnd);
if (EndPoints) EndPoints->push_back(MBBEnd);
// Find all blocks that are reachable from KillMBB without leaving VNI's live
// range. It is possible that KillMBB itself is reachable, so start a DFS
// from each successor.
typedef SmallPtrSet<MachineBasicBlock*, 9> VisitedTy;
VisitedTy Visited;
for (MachineBasicBlock::succ_iterator
SuccI = KillMBB->succ_begin(), SuccE = KillMBB->succ_end();
SuccI != SuccE; ++SuccI) {
for (df_ext_iterator<MachineBasicBlock*, VisitedTy>
I = df_ext_begin(*SuccI, Visited), E = df_ext_end(*SuccI, Visited);
I != E;) {
MachineBasicBlock *MBB = *I;
// Check if VNI is live in to MBB.
SlotIndex MBBStart, MBBEnd;
std::tie(MBBStart, MBBEnd) = Indexes->getMBBRange(MBB);
LiveQueryResult LRQ = LR.Query(MBBStart);
if (LRQ.valueIn() != VNI) {
// This block isn't part of the VNI segment. Prune the search.
I.skipChildren();
continue;
}
// Prune the search if VNI is killed in MBB.
if (LRQ.endPoint() < MBBEnd) {
LR.removeSegment(MBBStart, LRQ.endPoint());
if (EndPoints) EndPoints->push_back(LRQ.endPoint());
I.skipChildren();
continue;
}
// VNI is live through MBB.
LR.removeSegment(MBBStart, MBBEnd);
if (EndPoints) EndPoints->push_back(MBBEnd);
++I;
}
}
}
/// shrinkToUses - After removing some uses of a register, shrink its live
/// range to just the remaining uses. This method does not compute reaching
/// defs for new uses, and it doesn't remove dead defs.
bool LiveIntervals::shrinkToUses(LiveInterval *li,
SmallVectorImpl<MachineInstr*> *dead) {
DEBUG(dbgs() << "Shrink: " << *li << '\n');
assert(TargetRegisterInfo::isVirtualRegister(li->reg)
&& "Can only shrink virtual registers");
// Shrink subregister live ranges.
for (LiveInterval::SubRange &S : li->subranges()) {
shrinkToUses(S, li->reg);
}
// Find all the values used, including PHI kills.
ShrinkToUsesWorkList WorkList;
// Visit all instructions reading li->reg.
for (MachineRegisterInfo::reg_instr_iterator
I = MRI->reg_instr_begin(li->reg), E = MRI->reg_instr_end();
I != E; ) {
MachineInstr *UseMI = &*(I++);
if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
continue;
SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
LiveQueryResult LRQ = li->Query(Idx);
VNInfo *VNI = LRQ.valueIn();
if (!VNI) {
// This shouldn't happen: readsVirtualRegister returns true, but there is
// no live value. It is likely caused by a target getting <undef> flags
// wrong.
DEBUG(dbgs() << Idx << '\t' << *UseMI
<< "Warning: Instr claims to read non-existent value in "
<< *li << '\n');
continue;
}
// Special case: An early-clobber tied operand reads and writes the
// register one slot early.
if (VNInfo *DefVNI = LRQ.valueDefined())
Idx = DefVNI->def;
WorkList.push_back(std::make_pair(Idx, VNI));
}
// Create new live ranges with only minimal live segments per def.
LiveRange NewLR;
createSegmentsForValues(NewLR, make_range(li->vni_begin(), li->vni_end()));
extendSegmentsToUses(NewLR, *Indexes, WorkList, *li);
// Move the trimmed segments back.
li->segments.swap(NewLR.segments);
// Handle dead values.
bool CanSeparate = computeDeadValues(*li, dead);
DEBUG(dbgs() << "Shrunk: " << *li << '\n');
return CanSeparate;
}
// Check if all values in LI are rematerializable
static bool isRematerializable(const LiveInterval &LI,
const LiveIntervals &LIS,
VirtRegMap *VRM,
const TargetInstrInfo &TII) {
unsigned Reg = LI.reg;
unsigned Original = VRM ? VRM->getOriginal(Reg) : 0;
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I != E; ++I) {
const VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
if (VNI->isPHIDef())
return false;
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Dead valno in interval");
// Trace copies introduced by live range splitting. The inline
// spiller can rematerialize through these copies, so the spill
// weight must reflect this.
if (VRM) {
while (MI->isFullCopy()) {
// The copy destination must match the interval register.
if (MI->getOperand(0).getReg() != Reg)
return false;
// Get the source register.
Reg = MI->getOperand(1).getReg();
// If the original (pre-splitting) registers match this
// copy came from a split.
if (!TargetRegisterInfo::isVirtualRegister(Reg) ||
VRM->getOriginal(Reg) != Original)
return false;
// Follow the copy live-in value.
const LiveInterval &SrcLI = LIS.getInterval(Reg);
LiveQueryResult SrcQ = SrcLI.Query(VNI->def);
VNI = SrcQ.valueIn();
assert(VNI && "Copy from non-existing value");
if (VNI->isPHIDef())
return false;
MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Dead valno in interval");
}
}
if (!TII.isTriviallyReMaterializable(*MI, LIS.getAliasAnalysis()))
return false;
}
return true;
}
void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
const LiveIntervals &LIS) {
SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
unsigned Reg = RI->RegUnit;
const LiveRange *LR = getLiveRange(LIS, Reg);
if (LR != nullptr) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (LRQ.isDeadDef()) {
// LiveIntervals knows this is a dead even though it's MachineOperand is
// not flagged as such.
DeadDefs.push_back(*RI);
RI = Defs.erase(RI);
continue;
}
}
++RI;
}
}
/// Record the downward impact of a single instruction on current register
/// pressure. Unlike the advance/recede pressure tracking interface, this does
/// not discover live in/outs.
///
/// This is intended for speculative queries. It leaves pressure inconsistent
/// with the current position, so must be restored by the caller.
void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers(TRI, MRI);
collectOperands(MI, RegOpers);
// Kill liveness at last uses. Assume allocatable physregs are single-use
// rather than checking LiveIntervals.
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
unsigned Reg = RegOpers.Uses[i];
if (RequireIntervals) {
// FIXME: allow the caller to pass in the list of vreg uses that remain
// to be bottom-scheduled to avoid searching uses at each query.
SlotIndex CurrIdx = getCurrSlot();
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (LRQ.isKill() && !findUseBetween(Reg, CurrIdx, SlotIdx, MRI, LIS)) {
decreaseRegPressure(Reg);
}
}
}
else if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
// Allocatable physregs are always single-use before register rewriting.
decreaseRegPressure(Reg);
}
}
// Generate liveness for defs.
increaseRegPressure(RegOpers.Defs);
// Boost pressure for all dead defs together.
increaseRegPressure(RegOpers.DeadDefs);
decreaseRegPressure(RegOpers.DeadDefs);
}
/// Record the upward impact of a single instruction on current register
/// pressure. Unlike the advance/recede pressure tracking interface, this does
/// not discover live in/outs.
///
/// This is intended for speculative queries. It leaves pressure inconsistent
/// with the current position, so must be restored by the caller.
void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers(TRI, MRI, /*IgnoreDead=*/true);
collectOperands(MI, RegOpers);
// Boost max pressure for all dead defs together.
// Since CurrSetPressure and MaxSetPressure
increaseRegPressure(RegOpers.DeadDefs);
decreaseRegPressure(RegOpers.DeadDefs);
// Kill liveness at live defs.
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
unsigned Reg = RegOpers.Defs[i];
bool DeadDef = false;
if (RequireIntervals) {
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
SlotIndex SlotIdx = LIS->getInstructionIndex(MI);
LiveQueryResult LRQ = LR->Query(SlotIdx);
DeadDef = LRQ.isDeadDef();
}
}
if (!DeadDef) {
if (!containsReg(RegOpers.Uses, Reg))
decreaseRegPressure(Reg);
}
}
// Generate liveness for uses.
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
unsigned Reg = RegOpers.Uses[i];
if (!LiveRegs.contains(Reg))
increaseRegPressure(Reg);
}
}
/// traceSiblingValue - Trace a value that is about to be spilled back to the
/// real defining instructions by looking through sibling copies. Always stay
/// within the range of OrigVNI so the registers are known to carry the same
/// value.
///
/// Determine if the value is defined by all reloads, so spilling isn't
/// necessary - the value is already in the stack slot.
///
/// Return a defining instruction that may be a candidate for rematerialization.
///
MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
VNInfo *OrigVNI) {
// Check if a cached value already exists.
SibValueMap::iterator SVI;
bool Inserted;
std::tie(SVI, Inserted) =
SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI)));
if (!Inserted) {
DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second);
return SVI->second.DefMI;
}
DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << '\n');
// List of (Reg, VNI) that have been inserted into SibValues, but need to be
// processed.
SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList;
WorkList.push_back(std::make_pair(UseReg, UseVNI));
do {
unsigned Reg;
VNInfo *VNI;
std::tie(Reg, VNI) = WorkList.pop_back_val();
DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def
<< ":\t");
// First check if this value has already been computed.
SVI = SibValues.find(VNI);
assert(SVI != SibValues.end() && "Missing SibValues entry");
// Trace through PHI-defs created by live range splitting.
if (VNI->isPHIDef()) {
// Stop at original PHIs. We don't know the value at the predecessors.
if (VNI->def == OrigVNI->def) {
DEBUG(dbgs() << "orig phi value\n");
SVI->second.DefByOrigPHI = true;
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
continue;
}
// This is a PHI inserted by live range splitting. We could trace the
// live-out value from predecessor blocks, but that search can be very
// expensive if there are many predecessors and many more PHIs as
// generated by tail-dup when it sees an indirectbr. Instead, look at
// all the non-PHI defs that have the same value as OrigVNI. They must
// jointly dominate VNI->def. This is not optimal since VNI may actually
// be jointly dominated by a smaller subset of defs, so there is a change
// we will miss a AllDefsAreReloads optimization.
// Separate all values dominated by OrigVNI into PHIs and non-PHIs.
SmallVector<VNInfo*, 8> PHIs, NonPHIs;
LiveInterval &LI = LIS.getInterval(Reg);
LiveInterval &OrigLI = LIS.getInterval(Original);
for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end();
VI != VE; ++VI) {
VNInfo *VNI2 = *VI;
if (VNI2->isUnused())
continue;
if (!OrigLI.containsOneValue() &&
OrigLI.getVNInfoAt(VNI2->def) != OrigVNI)
continue;
if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def)
PHIs.push_back(VNI2);
else
NonPHIs.push_back(VNI2);
}
DEBUG(dbgs() << "split phi value, checking " << PHIs.size()
<< " phi-defs, and " << NonPHIs.size()
<< " non-phi/orig defs\n");
// Create entries for all the PHIs. Don't add them to the worklist, we
// are processing all of them in one go here.
for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i])));
// Add every PHI as a dependent of all the non-PHIs.
for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) {
VNInfo *NonPHI = NonPHIs[i];
// Known value? Try an insertion.
std::tie(SVI, Inserted) =
SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI)));
// Add all the PHIs as dependents of NonPHI.
for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi)
SVI->second.Deps.push_back(PHIs[pi]);
// This is the first time we see NonPHI, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(Reg, NonPHI));
else
// Propagate to all inserted PHIs, not just VNI.
propagateSiblingValue(SVI);
}
// Next work list item.
continue;
}
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Missing def");
// Trace through sibling copies.
if (unsigned SrcReg = isFullCopyOf(MI, Reg)) {
if (isSibling(SrcReg)) {
LiveInterval &SrcLI = LIS.getInterval(SrcReg);
LiveQueryResult SrcQ = SrcLI.Query(VNI->def);
assert(SrcQ.valueIn() && "Copy from non-existing value");
// Check if this COPY kills its source.
SVI->second.KillsSource = SrcQ.isKill();
VNInfo *SrcVNI = SrcQ.valueIn();
DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':'
<< SrcVNI->id << '@' << SrcVNI->def
<< " kill=" << unsigned(SVI->second.KillsSource) << '\n');
// Known sibling source value? Try an insertion.
std::tie(SVI, Inserted) = SibValues.insert(
std::make_pair(SrcVNI, SibValueInfo(SrcReg, SrcVNI)));
// This is the first time we see Src, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(SrcReg, SrcVNI));
propagateSiblingValue(SVI, VNI);
// Next work list item.
continue;
}
}
// Track reachable reloads.
SVI->second.DefMI = MI;
SVI->second.SpillMBB = MI->getParent();
int FI;
if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) {
DEBUG(dbgs() << "reload\n");
propagateSiblingValue(SVI);
// Next work list item.
continue;
}
// Potential remat candidate.
DEBUG(dbgs() << "def " << *MI);
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
} while (!WorkList.empty());
// Look up the value we were looking for. We already did this lookup at the
// top of the function, but SibValues may have been invalidated.
SVI = SibValues.find(UseVNI);
assert(SVI != SibValues.end() && "Didn't compute requested info");
DEBUG(dbgs() << " traced to:\t" << SVI->second);
return SVI->second.DefMI;
}
void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg)
{
DEBUG(dbgs() << "Shrink: " << SR << '\n');
assert(TargetRegisterInfo::isVirtualRegister(Reg)
&& "Can only shrink virtual registers");
// Find all the values used, including PHI kills.
ShrinkToUsesWorkList WorkList;
// Visit all instructions reading Reg.
SlotIndex LastIdx;
for (MachineOperand &MO : MRI->reg_operands(Reg)) {
MachineInstr *UseMI = MO.getParent();
if (UseMI->isDebugValue())
continue;
// Maybe the operand is for a subregister we don't care about.
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
unsigned SubRegMask = TRI->getSubRegIndexLaneMask(SubReg);
if ((SubRegMask & SR.LaneMask) == 0)
continue;
}
// We only need to visit each instruction once.
SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
if (Idx == LastIdx)
continue;
LastIdx = Idx;
LiveQueryResult LRQ = SR.Query(Idx);
VNInfo *VNI = LRQ.valueIn();
// For Subranges it is possible that only undef values are left in that
// part of the subregister, so there is no real liverange at the use
if (!VNI)
continue;
// Special case: An early-clobber tied operand reads and writes the
// register one slot early.
if (VNInfo *DefVNI = LRQ.valueDefined())
Idx = DefVNI->def;
WorkList.push_back(std::make_pair(Idx, VNI));
}
// Create a new live ranges with only minimal live segments per def.
LiveRange NewLR;
createSegmentsForValues(NewLR, make_range(SR.vni_begin(), SR.vni_end()));
extendSegmentsToUses(NewLR, *Indexes, WorkList, SR);
// Move the trimmed ranges back.
SR.segments.swap(NewLR.segments);
// Remove dead PHI value numbers
for (auto VNI : SR.valnos) {
if (VNI->isUnused())
continue;
const LiveRange::Segment *Segment = SR.getSegmentContaining(VNI->def);
assert(Segment != nullptr && "Missing segment for VNI");
if (Segment->end != VNI->def.getDeadSlot())
continue;
if (VNI->isPHIDef()) {
// This is a dead PHI. Remove it.
VNI->markUnused();
SR.removeSegment(*Segment);
DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
}
}
DEBUG(dbgs() << "Shrunk: " << SR << '\n');
}
/// Recede across the previous instruction. If LiveUses is provided, record any
/// RegUnits that are made live by the current instruction's uses. This includes
/// registers that are both defined and used by the instruction. If a pressure
/// difference pointer is provided record the changes is pressure caused by this
/// instruction independent of liveness.
bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
PressureDiff *PDiff) {
// Check for the top of the analyzable region.
if (CurrPos == MBB->begin()) {
closeRegion();
return false;
}
if (!isBottomClosed())
closeBottom();
// Open the top of the region using block iterators.
if (!RequireIntervals && isTopClosed())
static_cast<RegionPressure&>(P).openTop(CurrPos);
// Find the previous instruction.
do
--CurrPos;
while (CurrPos != MBB->begin() && CurrPos->isDebugValue());
if (CurrPos->isDebugValue()) {
closeRegion();
return false;
}
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot();
// Open the top of the region using slot indexes.
if (RequireIntervals && isTopClosed())
static_cast<IntervalPressure&>(P).openTop(SlotIdx);
RegisterOperands RegOpers(TRI, MRI);
collectOperands(CurrPos, RegOpers);
if (PDiff)
collectPDiff(*PDiff, RegOpers, MRI);
// Boost pressure for all dead defs together.
increaseRegPressure(RegOpers.DeadDefs);
decreaseRegPressure(RegOpers.DeadDefs);
// Kill liveness at live defs.
// TODO: consider earlyclobbers?
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
unsigned Reg = RegOpers.Defs[i];
bool DeadDef = false;
if (RequireIntervals) {
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
DeadDef = LRQ.isDeadDef();
}
}
if (DeadDef) {
// LiveIntervals knows this is a dead even though it's MachineOperand is
// not flagged as such. Since this register will not be recorded as
// live-out, increase its PDiff value to avoid underflowing pressure.
if (PDiff)
PDiff->addPressureChange(Reg, false, MRI);
} else {
if (LiveRegs.erase(Reg))
decreaseRegPressure(Reg);
else
discoverLiveOut(Reg);
}
}
// Generate liveness for uses.
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
unsigned Reg = RegOpers.Uses[i];
if (!LiveRegs.contains(Reg)) {
// Adjust liveouts if LiveIntervals are available.
if (RequireIntervals) {
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (!LRQ.isKill() && !LRQ.valueDefined())
discoverLiveOut(Reg);
}
}
increaseRegPressure(Reg);
LiveRegs.insert(Reg);
if (LiveUses && !containsReg(*LiveUses, Reg))
LiveUses->push_back(Reg);
}
}
if (TrackUntiedDefs) {
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
unsigned Reg = RegOpers.Defs[i];
if (TargetRegisterInfo::isVirtualRegister(Reg) && !LiveRegs.contains(Reg))
UntiedDefs.insert(Reg);
}
}
return true;
}
void ConnectedVNInfoEqClasses::Distribute(LiveInterval &LI, LiveInterval *LIV[],
MachineRegisterInfo &MRI) {
// Rewrite instructions.
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LI.reg),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = *RI;
MachineInstr *MI = RI->getParent();
++RI;
// DBG_VALUE instructions don't have slot indexes, so get the index of the
// instruction before them.
// Normally, DBG_VALUE instructions are removed before this function is
// called, but it is not a requirement.
SlotIndex Idx;
if (MI->isDebugValue())
Idx = LIS.getSlotIndexes()->getIndexBefore(*MI);
else
Idx = LIS.getInstructionIndex(*MI);
LiveQueryResult LRQ = LI.Query(Idx);
const VNInfo *VNI = MO.readsReg() ? LRQ.valueIn() : LRQ.valueDefined();
// In the case of an <undef> use that isn't tied to any def, VNI will be
// NULL. If the use is tied to a def, VNI will be the defined value.
if (!VNI)
continue;
if (unsigned EqClass = getEqClass(VNI))
MO.setReg(LIV[EqClass-1]->reg);
}
// Distribute subregister liveranges.
if (LI.hasSubRanges()) {
unsigned NumComponents = EqClass.getNumClasses();
SmallVector<unsigned, 8> VNIMapping;
SmallVector<LiveInterval::SubRange*, 8> SubRanges;
BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator();
for (LiveInterval::SubRange &SR : LI.subranges()) {
// Create new subranges in the split intervals and construct a mapping
// for the VNInfos in the subrange.
unsigned NumValNos = SR.valnos.size();
VNIMapping.clear();
VNIMapping.reserve(NumValNos);
SubRanges.clear();
SubRanges.resize(NumComponents-1, nullptr);
for (unsigned I = 0; I < NumValNos; ++I) {
const VNInfo &VNI = *SR.valnos[I];
unsigned ComponentNum;
if (VNI.isUnused()) {
ComponentNum = 0;
} else {
const VNInfo *MainRangeVNI = LI.getVNInfoAt(VNI.def);
assert(MainRangeVNI != nullptr
&& "SubRange def must have corresponding main range def");
ComponentNum = getEqClass(MainRangeVNI);
if (ComponentNum > 0 && SubRanges[ComponentNum-1] == nullptr) {
SubRanges[ComponentNum-1]
= LIV[ComponentNum-1]->createSubRange(Allocator, SR.LaneMask);
}
}
VNIMapping.push_back(ComponentNum);
}
DistributeRange(SR, SubRanges.data(), VNIMapping);
}
LI.removeEmptySubRanges();
}
// Distribute main liverange.
DistributeRange(LI, LIV, EqClass);
}
/// shrinkToUses - After removing some uses of a register, shrink its live
/// range to just the remaining uses. This method does not compute reaching
/// defs for new uses, and it doesn't remove dead defs.
bool LiveIntervals::shrinkToUses(LiveInterval *li,
SmallVectorImpl<MachineInstr*> *dead) {
DEBUG(dbgs() << "Shrink: " << *li << '\n');
assert(TargetRegisterInfo::isVirtualRegister(li->reg)
&& "Can only shrink virtual registers");
// Find all the values used, including PHI kills.
SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
// Blocks that have already been added to WorkList as live-out.
SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
// Visit all instructions reading li->reg.
for (MachineRegisterInfo::reg_instr_iterator
I = MRI->reg_instr_begin(li->reg), E = MRI->reg_instr_end();
I != E; ) {
MachineInstr *UseMI = &*(I++);
if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
continue;
SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
LiveQueryResult LRQ = li->Query(Idx);
VNInfo *VNI = LRQ.valueIn();
if (!VNI) {
// This shouldn't happen: readsVirtualRegister returns true, but there is
// no live value. It is likely caused by a target getting <undef> flags
// wrong.
DEBUG(dbgs() << Idx << '\t' << *UseMI
<< "Warning: Instr claims to read non-existent value in "
<< *li << '\n');
continue;
}
// Special case: An early-clobber tied operand reads and writes the
// register one slot early.
if (VNInfo *DefVNI = LRQ.valueDefined())
Idx = DefVNI->def;
WorkList.push_back(std::make_pair(Idx, VNI));
}
// Create new live ranges with only minimal live segments per def.
LiveRange NewLR;
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
NewLR.addSegment(LiveRange::Segment(VNI->def, VNI->def.getDeadSlot(), VNI));
}
// Keep track of the PHIs that are in use.
SmallPtrSet<VNInfo*, 8> UsedPHIs;
// Extend intervals to reach all uses in WorkList.
while (!WorkList.empty()) {
SlotIndex Idx = WorkList.back().first;
VNInfo *VNI = WorkList.back().second;
WorkList.pop_back();
const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
SlotIndex BlockStart = getMBBStartIdx(MBB);
// Extend the live range for VNI to be live at Idx.
if (VNInfo *ExtVNI = NewLR.extendInBlock(BlockStart, Idx)) {
(void)ExtVNI;
assert(ExtVNI == VNI && "Unexpected existing value number");
// Is this a PHIDef we haven't seen before?
if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
continue;
// The PHI is live, make sure the predecessors are live-out.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
// A predecessor is not required to have a live-out value for a PHI.
if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
WorkList.push_back(std::make_pair(Stop, PVNI));
}
continue;
}
// VNI is live-in to MBB.
DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
NewLR.addSegment(LiveRange::Segment(BlockStart, Idx, VNI));
// Make sure VNI is live-out from the predecessors.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
assert(li->getVNInfoBefore(Stop) == VNI &&
"Wrong value out of predecessor");
WorkList.push_back(std::make_pair(Stop, VNI));
}
}
// Handle dead values.
bool CanSeparate = false;
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
LiveRange::iterator LRI = NewLR.FindSegmentContaining(VNI->def);
assert(LRI != NewLR.end() && "Missing segment for PHI");
if (LRI->end != VNI->def.getDeadSlot())
continue;
if (VNI->isPHIDef()) {
// This is a dead PHI. Remove it.
VNI->markUnused();
NewLR.removeSegment(LRI->start, LRI->end);
DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
CanSeparate = true;
} else {
// This is a dead def. Make sure the instruction knows.
MachineInstr *MI = getInstructionFromIndex(VNI->def);
assert(MI && "No instruction defining live value");
MI->addRegisterDead(li->reg, TRI);
if (dead && MI->allDefsAreDead()) {
DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
dead->push_back(MI);
}
}
}
// Move the trimmed segments back.
li->segments.swap(NewLR.segments);
DEBUG(dbgs() << "Shrunk: " << *li << '\n');
return CanSeparate;
}
