/// Record the downward impact of a single instruction on current register
/// pressure. Unlike the advance/recede pressure tracking interface, this does
/// not discover live in/outs.
///
/// This is intended for speculative queries. It leaves pressure inconsistent
/// with the current position, so must be restored by the caller.
void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers(TRI, MRI);
collectOperands(MI, RegOpers);
// Kill liveness at last uses. Assume allocatable physregs are single-use
// rather than checking LiveIntervals.
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(MI).getRegSlot();
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
unsigned Reg = RegOpers.Uses[i];
if (RequireIntervals) {
// FIXME: allow the caller to pass in the list of vreg uses that remain
// to be bottom-scheduled to avoid searching uses at each query.
SlotIndex CurrIdx = getCurrSlot();
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (LRQ.isKill() && !findUseBetween(Reg, CurrIdx, SlotIdx, MRI, LIS)) {
decreaseRegPressure(Reg);
}
}
}
else if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
// Allocatable physregs are always single-use before register rewriting.
decreaseRegPressure(Reg);
}
}
// Generate liveness for defs.
increaseRegPressure(RegOpers.Defs);
// Boost pressure for all dead defs together.
increaseRegPressure(RegOpers.DeadDefs);
decreaseRegPressure(RegOpers.DeadDefs);
}
/// traceSiblingValue - Trace a value that is about to be spilled back to the
/// real defining instructions by looking through sibling copies. Always stay
/// within the range of OrigVNI so the registers are known to carry the same
/// value.
///
/// Determine if the value is defined by all reloads, so spilling isn't
/// necessary - the value is already in the stack slot.
///
/// Return a defining instruction that may be a candidate for rematerialization.
///
MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
VNInfo *OrigVNI) {
// Check if a cached value already exists.
SibValueMap::iterator SVI;
bool Inserted;
std::tie(SVI, Inserted) =
SibValues.insert(std::make_pair(UseVNI, SibValueInfo(UseReg, UseVNI)));
if (!Inserted) {
DEBUG(dbgs() << "Cached value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << ' ' << SVI->second);
return SVI->second.DefMI;
}
DEBUG(dbgs() << "Tracing value " << PrintReg(UseReg) << ':'
<< UseVNI->id << '@' << UseVNI->def << '\n');
// List of (Reg, VNI) that have been inserted into SibValues, but need to be
// processed.
SmallVector<std::pair<unsigned, VNInfo*>, 8> WorkList;
WorkList.push_back(std::make_pair(UseReg, UseVNI));
do {
unsigned Reg;
VNInfo *VNI;
std::tie(Reg, VNI) = WorkList.pop_back_val();
DEBUG(dbgs() << " " << PrintReg(Reg) << ':' << VNI->id << '@' << VNI->def
<< ":\t");
// First check if this value has already been computed.
SVI = SibValues.find(VNI);
assert(SVI != SibValues.end() && "Missing SibValues entry");
// Trace through PHI-defs created by live range splitting.
if (VNI->isPHIDef()) {
// Stop at original PHIs. We don't know the value at the predecessors.
if (VNI->def == OrigVNI->def) {
DEBUG(dbgs() << "orig phi value\n");
SVI->second.DefByOrigPHI = true;
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
continue;
}
// This is a PHI inserted by live range splitting. We could trace the
// live-out value from predecessor blocks, but that search can be very
// expensive if there are many predecessors and many more PHIs as
// generated by tail-dup when it sees an indirectbr. Instead, look at
// all the non-PHI defs that have the same value as OrigVNI. They must
// jointly dominate VNI->def. This is not optimal since VNI may actually
// be jointly dominated by a smaller subset of defs, so there is a change
// we will miss a AllDefsAreReloads optimization.
// Separate all values dominated by OrigVNI into PHIs and non-PHIs.
SmallVector<VNInfo*, 8> PHIs, NonPHIs;
LiveInterval &LI = LIS.getInterval(Reg);
LiveInterval &OrigLI = LIS.getInterval(Original);
for (LiveInterval::vni_iterator VI = LI.vni_begin(), VE = LI.vni_end();
VI != VE; ++VI) {
VNInfo *VNI2 = *VI;
if (VNI2->isUnused())
continue;
if (!OrigLI.containsOneValue() &&
OrigLI.getVNInfoAt(VNI2->def) != OrigVNI)
continue;
if (VNI2->isPHIDef() && VNI2->def != OrigVNI->def)
PHIs.push_back(VNI2);
else
NonPHIs.push_back(VNI2);
}
DEBUG(dbgs() << "split phi value, checking " << PHIs.size()
<< " phi-defs, and " << NonPHIs.size()
<< " non-phi/orig defs\n");
// Create entries for all the PHIs. Don't add them to the worklist, we
// are processing all of them in one go here.
for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
SibValues.insert(std::make_pair(PHIs[i], SibValueInfo(Reg, PHIs[i])));
// Add every PHI as a dependent of all the non-PHIs.
for (unsigned i = 0, e = NonPHIs.size(); i != e; ++i) {
VNInfo *NonPHI = NonPHIs[i];
// Known value? Try an insertion.
std::tie(SVI, Inserted) =
SibValues.insert(std::make_pair(NonPHI, SibValueInfo(Reg, NonPHI)));
// Add all the PHIs as dependents of NonPHI.
for (unsigned pi = 0, pe = PHIs.size(); pi != pe; ++pi)
SVI->second.Deps.push_back(PHIs[pi]);
// This is the first time we see NonPHI, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(Reg, NonPHI));
else
// Propagate to all inserted PHIs, not just VNI.
propagateSiblingValue(SVI);
}
// Next work list item.
continue;
}
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Missing def");
// Trace through sibling copies.
if (unsigned SrcReg = isFullCopyOf(MI, Reg)) {
if (isSibling(SrcReg)) {
LiveInterval &SrcLI = LIS.getInterval(SrcReg);
LiveQueryResult SrcQ = SrcLI.Query(VNI->def);
assert(SrcQ.valueIn() && "Copy from non-existing value");
// Check if this COPY kills its source.
SVI->second.KillsSource = SrcQ.isKill();
VNInfo *SrcVNI = SrcQ.valueIn();
DEBUG(dbgs() << "copy of " << PrintReg(SrcReg) << ':'
<< SrcVNI->id << '@' << SrcVNI->def
<< " kill=" << unsigned(SVI->second.KillsSource) << '\n');
// Known sibling source value? Try an insertion.
std::tie(SVI, Inserted) = SibValues.insert(
std::make_pair(SrcVNI, SibValueInfo(SrcReg, SrcVNI)));
// This is the first time we see Src, add it to the worklist.
if (Inserted)
WorkList.push_back(std::make_pair(SrcReg, SrcVNI));
propagateSiblingValue(SVI, VNI);
// Next work list item.
continue;
}
}
// Track reachable reloads.
SVI->second.DefMI = MI;
SVI->second.SpillMBB = MI->getParent();
int FI;
if (Reg == TII.isLoadFromStackSlot(MI, FI) && FI == StackSlot) {
DEBUG(dbgs() << "reload\n");
propagateSiblingValue(SVI);
// Next work list item.
continue;
}
// Potential remat candidate.
DEBUG(dbgs() << "def " << *MI);
SVI->second.AllDefsAreReloads = false;
propagateSiblingValue(SVI);
} while (!WorkList.empty());
// Look up the value we were looking for. We already did this lookup at the
// top of the function, but SibValues may have been invalidated.
SVI = SibValues.find(UseVNI);
assert(SVI != SibValues.end() && "Didn't compute requested info");
DEBUG(dbgs() << " traced to:\t" << SVI->second);
return SVI->second.DefMI;
}
/// Recede across the previous instruction. If LiveUses is provided, record any
/// RegUnits that are made live by the current instruction's uses. This includes
/// registers that are both defined and used by the instruction. If a pressure
/// difference pointer is provided record the changes is pressure caused by this
/// instruction independent of liveness.
bool RegPressureTracker::recede(SmallVectorImpl<unsigned> *LiveUses,
PressureDiff *PDiff) {
// Check for the top of the analyzable region.
if (CurrPos == MBB->begin()) {
closeRegion();
return false;
}
if (!isBottomClosed())
closeBottom();
// Open the top of the region using block iterators.
if (!RequireIntervals && isTopClosed())
static_cast<RegionPressure&>(P).openTop(CurrPos);
// Find the previous instruction.
do
--CurrPos;
while (CurrPos != MBB->begin() && CurrPos->isDebugValue());
if (CurrPos->isDebugValue()) {
closeRegion();
return false;
}
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(CurrPos).getRegSlot();
// Open the top of the region using slot indexes.
if (RequireIntervals && isTopClosed())
static_cast<IntervalPressure&>(P).openTop(SlotIdx);
RegisterOperands RegOpers(TRI, MRI);
collectOperands(CurrPos, RegOpers);
if (PDiff)
collectPDiff(*PDiff, RegOpers, MRI);
// Boost pressure for all dead defs together.
increaseRegPressure(RegOpers.DeadDefs);
decreaseRegPressure(RegOpers.DeadDefs);
// Kill liveness at live defs.
// TODO: consider earlyclobbers?
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
unsigned Reg = RegOpers.Defs[i];
bool DeadDef = false;
if (RequireIntervals) {
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
DeadDef = LRQ.isDeadDef();
}
}
if (DeadDef) {
// LiveIntervals knows this is a dead even though it's MachineOperand is
// not flagged as such. Since this register will not be recorded as
// live-out, increase its PDiff value to avoid underflowing pressure.
if (PDiff)
PDiff->addPressureChange(Reg, false, MRI);
} else {
if (LiveRegs.erase(Reg))
decreaseRegPressure(Reg);
else
discoverLiveOut(Reg);
}
}
// Generate liveness for uses.
for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) {
unsigned Reg = RegOpers.Uses[i];
if (!LiveRegs.contains(Reg)) {
// Adjust liveouts if LiveIntervals are available.
if (RequireIntervals) {
const LiveRange *LR = getLiveRange(Reg);
if (LR) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (!LRQ.isKill() && !LRQ.valueDefined())
discoverLiveOut(Reg);
}
}
increaseRegPressure(Reg);
LiveRegs.insert(Reg);
if (LiveUses && !containsReg(*LiveUses, Reg))
LiveUses->push_back(Reg);
}
}
if (TrackUntiedDefs) {
for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) {
unsigned Reg = RegOpers.Defs[i];
if (TargetRegisterInfo::isVirtualRegister(Reg) && !LiveRegs.contains(Reg))
UntiedDefs.insert(Reg);
}
}
return true;
}
