// Merge a LiveInterval's segments. Guarantee no overlaps.
void LiveIntervalUnion::unify(LiveInterval &VirtReg, const LiveRange &Range) {
if (Range.empty())
return;
++Tag;
// Insert each of the virtual register's live segments into the map.
LiveRange::const_iterator RegPos = Range.begin();
LiveRange::const_iterator RegEnd = Range.end();
SegmentIter SegPos = Segments.find(RegPos->start);
while (SegPos.valid()) {
SegPos.insert(RegPos->start, RegPos->end, &VirtReg);
if (++RegPos == RegEnd)
return;
SegPos.advanceTo(RegPos->start);
}
// We have reached the end of Segments, so it is no longer necessary to search
// for the insertion position.
// It is faster to insert the end first.
--RegEnd;
SegPos.insert(RegEnd->start, RegEnd->end, &VirtReg);
for (; RegPos != RegEnd; ++RegPos, ++SegPos)
SegPos.insert(RegPos->start, RegPos->end, &VirtReg);
}
/// MergeValueInAsValue - Merge all of the live segments of a specific val#
/// in RHS into this live range as the specified value number.
/// The segments in RHS are allowed to overlap with segments in the
/// current range, it will replace the value numbers of the overlaped
/// segments with the specified value number.
void LiveRange::MergeValueInAsValue(const LiveRange &RHS,
const VNInfo *RHSValNo,
VNInfo *LHSValNo) {
LiveRangeUpdater Updater(this);
for (const_iterator I = RHS.begin(), E = RHS.end(); I != E; ++I)
if (I->valno == RHSValNo)
Updater.add(I->start, I->end, LHSValNo);
}
// overlaps - Return true if the intersection of the two live ranges is
// not empty.
//
// An example for overlaps():
//
// 0: A = ...
// 4: B = ...
// 8: C = A + B ;; last use of A
//
// The live ranges should look like:
//
// A = [3, 11)
// B = [7, x)
// C = [11, y)
//
// A->overlaps(C) should return false since we want to be able to join
// A and C.
//
bool LiveRange::overlapsFrom(const LiveRange& other,
const_iterator StartPos) const {
assert(!empty() && "empty range");
const_iterator i = begin();
const_iterator ie = end();
const_iterator j = StartPos;
const_iterator je = other.end();
assert((StartPos->start <= i->start || StartPos == other.begin()) &&
StartPos != other.end() && "Bogus start position hint!");
if (i->start < j->start) {
i = std::upper_bound(i, ie, j->start);
if (i != begin()) --i;
} else if (j->start < i->start) {
++StartPos;
if (StartPos != other.end() && StartPos->start <= i->start) {
assert(StartPos < other.end() && i < end());
j = std::upper_bound(j, je, i->start);
if (j != other.begin()) --j;
}
} else {
return true;
}
if (j == je) return false;
while (i != ie) {
if (i->start > j->start) {
std::swap(i, j);
std::swap(ie, je);
}
if (i->end > j->start)
return true;
++i;
}
return false;
}
bool LiveRange::overlaps(const LiveRange &Other, const CoalescerPair &CP,
const SlotIndexes &Indexes) const {
assert(!empty() && "empty range");
if (Other.empty())
return false;
// Use binary searches to find initial positions.
const_iterator I = find(Other.beginIndex());
const_iterator IE = end();
if (I == IE)
return false;
const_iterator J = Other.find(I->start);
const_iterator JE = Other.end();
if (J == JE)
return false;
for (;;) {
// J has just been advanced to satisfy:
assert(J->end >= I->start);
// Check for an overlap.
if (J->start < I->end) {
// I and J are overlapping. Find the later start.
SlotIndex Def = std::max(I->start, J->start);
// Allow the overlap if Def is a coalescable copy.
if (Def.isBlock() ||
!CP.isCoalescable(Indexes.getInstructionFromIndex(Def)))
return true;
}
// Advance the iterator that ends first to check for more overlaps.
if (J->end > I->end) {
std::swap(I, J);
std::swap(IE, JE);
}
// Advance J until J->end >= I->start.
do
if (++J == JE)
return false;
while (J->end < I->start);
}
}
// Remove a live virtual register's segments from this union.
void LiveIntervalUnion::extract(LiveInterval &VirtReg, const LiveRange &Range) {
if (Range.empty())
return;
++Tag;
// Remove each of the virtual register's live segments from the map.
LiveRange::const_iterator RegPos = Range.begin();
LiveRange::const_iterator RegEnd = Range.end();
SegmentIter SegPos = Segments.find(RegPos->start);
while (true) {
assert(SegPos.value() == &VirtReg && "Inconsistent LiveInterval");
SegPos.erase();
if (!SegPos.valid())
return;
// Skip all segments that may have been coalesced.
RegPos = Range.advanceTo(RegPos, SegPos.start());
if (RegPos == RegEnd)
return;
SegPos.advanceTo(RegPos->start);
}
}
/// Merge all of the segments in RHS into this live range as the specified
/// value number. The segments in RHS are allowed to overlap with segments in
/// the current range, but only if the overlapping segments have the
/// specified value number.
void LiveRange::MergeSegmentsInAsValue(const LiveRange &RHS,
VNInfo *LHSValNo) {
LiveRangeUpdater Updater(this);
for (const_iterator I = RHS.begin(), E = RHS.end(); I != E; ++I)
Updater.add(I->start, I->end, LHSValNo);
}
void LiveRange::join(LiveRange &Other,
const int *LHSValNoAssignments,
const int *RHSValNoAssignments,
SmallVectorImpl<VNInfo *> &NewVNInfo) {
verify();
// Determine if any of our values are mapped. This is uncommon, so we want
// to avoid the range scan if not.
bool MustMapCurValNos = false;
unsigned NumVals = getNumValNums();
unsigned NumNewVals = NewVNInfo.size();
for (unsigned i = 0; i != NumVals; ++i) {
unsigned LHSValID = LHSValNoAssignments[i];
if (i != LHSValID ||
(NewVNInfo[LHSValID] && NewVNInfo[LHSValID] != getValNumInfo(i))) {
MustMapCurValNos = true;
break;
}
}
// If we have to apply a mapping to our base range assignment, rewrite it now.
if (MustMapCurValNos && !empty()) {
// Map the first live range.
iterator OutIt = begin();
OutIt->valno = NewVNInfo[LHSValNoAssignments[OutIt->valno->id]];
for (iterator I = std::next(OutIt), E = end(); I != E; ++I) {
VNInfo* nextValNo = NewVNInfo[LHSValNoAssignments[I->valno->id]];
assert(nextValNo != 0 && "Huh?");
// If this live range has the same value # as its immediate predecessor,
// and if they are neighbors, remove one Segment. This happens when we
// have [0,4:0)[4,7:1) and map 0/1 onto the same value #.
if (OutIt->valno == nextValNo && OutIt->end == I->start) {
OutIt->end = I->end;
} else {
// Didn't merge. Move OutIt to the next segment,
++OutIt;
OutIt->valno = nextValNo;
if (OutIt != I) {
OutIt->start = I->start;
OutIt->end = I->end;
}
}
}
// If we merge some segments, chop off the end.
++OutIt;
segments.erase(OutIt, end());
}
// Rewrite Other values before changing the VNInfo ids.
// This can leave Other in an invalid state because we're not coalescing
// touching segments that now have identical values. That's OK since Other is
// not supposed to be valid after calling join();
for (iterator I = Other.begin(), E = Other.end(); I != E; ++I)
I->valno = NewVNInfo[RHSValNoAssignments[I->valno->id]];
// Update val# info. Renumber them and make sure they all belong to this
// LiveRange now. Also remove dead val#'s.
unsigned NumValNos = 0;
for (unsigned i = 0; i < NumNewVals; ++i) {
VNInfo *VNI = NewVNInfo[i];
if (VNI) {
if (NumValNos >= NumVals)
valnos.push_back(VNI);
else
valnos[NumValNos] = VNI;
VNI->id = NumValNos++; // Renumber val#.
}
}
if (NumNewVals < NumVals)
valnos.resize(NumNewVals); // shrinkify
// Okay, now insert the RHS live segments into the LHS.
LiveRangeUpdater Updater(this);
for (iterator I = Other.begin(), E = Other.end(); I != E; ++I)
Updater.add(*I);
}
bool LiveRangeCalc::isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs,
MachineBasicBlock &MBB, BitVector &DefOnEntry,
BitVector &UndefOnEntry) {
unsigned BN = MBB.getNumber();
if (DefOnEntry[BN])
return true;
if (UndefOnEntry[BN])
return false;
auto MarkDefined = [BN, &DefOnEntry](MachineBasicBlock &B) -> bool {
for (MachineBasicBlock *S : B.successors())
DefOnEntry[S->getNumber()] = true;
DefOnEntry[BN] = true;
return true;
};
SetVector<unsigned> WorkList;
// Checking if the entry of MBB is reached by some def: add all predecessors
// that are potentially defined-on-exit to the work list.
for (MachineBasicBlock *P : MBB.predecessors())
WorkList.insert(P->getNumber());
for (unsigned i = 0; i != WorkList.size(); ++i) {
// Determine if the exit from the block is reached by some def.
unsigned N = WorkList[i];
MachineBasicBlock &B = *MF->getBlockNumbered(N);
if (Seen[N]) {
const LiveOutPair &LOB = Map[&B];
if (LOB.first != nullptr && LOB.first != &UndefVNI)
return MarkDefined(B);
}
SlotIndex Begin, End;
std::tie(Begin, End) = Indexes->getMBBRange(&B);
// Treat End as not belonging to B.
// If LR has a segment S that starts at the next block, i.e. [End, ...),
// std::upper_bound will return the segment following S. Instead,
// S should be treated as the first segment that does not overlap B.
LiveRange::iterator UB = std::upper_bound(LR.begin(), LR.end(),
End.getPrevSlot());
if (UB != LR.begin()) {
LiveRange::Segment &Seg = *std::prev(UB);
if (Seg.end > Begin) {
// There is a segment that overlaps B. If the range is not explicitly
// undefined between the end of the segment and the end of the block,
// treat the block as defined on exit. If it is, go to the next block
// on the work list.
if (LR.isUndefIn(Undefs, Seg.end, End))
continue;
return MarkDefined(B);
}
}
// No segment overlaps with this block. If this block is not defined on
// entry, or it undefines the range, do not process its predecessors.
if (UndefOnEntry[N] || LR.isUndefIn(Undefs, Begin, End)) {
UndefOnEntry[N] = true;
continue;
}
if (DefOnEntry[N])
return MarkDefined(B);
// Still don't know: add all predecessors to the work list.
for (MachineBasicBlock *P : B.predecessors())
WorkList.insert(P->getNumber());
}
UndefOnEntry[BN] = true;
return false;
}
/// shrinkToUses - After removing some uses of a register, shrink its live
/// range to just the remaining uses. This method does not compute reaching
/// defs for new uses, and it doesn't remove dead defs.
bool LiveIntervals::shrinkToUses(LiveInterval *li,
SmallVectorImpl<MachineInstr*> *dead) {
DEBUG(dbgs() << "Shrink: " << *li << '\n');
assert(TargetRegisterInfo::isVirtualRegister(li->reg)
&& "Can only shrink virtual registers");
// Find all the values used, including PHI kills.
SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList;
// Blocks that have already been added to WorkList as live-out.
SmallPtrSet<MachineBasicBlock*, 16> LiveOut;
// Visit all instructions reading li->reg.
for (MachineRegisterInfo::reg_instr_iterator
I = MRI->reg_instr_begin(li->reg), E = MRI->reg_instr_end();
I != E; ) {
MachineInstr *UseMI = &*(I++);
if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg))
continue;
SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot();
LiveQueryResult LRQ = li->Query(Idx);
VNInfo *VNI = LRQ.valueIn();
if (!VNI) {
// This shouldn't happen: readsVirtualRegister returns true, but there is
// no live value. It is likely caused by a target getting <undef> flags
// wrong.
DEBUG(dbgs() << Idx << '\t' << *UseMI
<< "Warning: Instr claims to read non-existent value in "
<< *li << '\n');
continue;
}
// Special case: An early-clobber tied operand reads and writes the
// register one slot early.
if (VNInfo *DefVNI = LRQ.valueDefined())
Idx = DefVNI->def;
WorkList.push_back(std::make_pair(Idx, VNI));
}
// Create new live ranges with only minimal live segments per def.
LiveRange NewLR;
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
NewLR.addSegment(LiveRange::Segment(VNI->def, VNI->def.getDeadSlot(), VNI));
}
// Keep track of the PHIs that are in use.
SmallPtrSet<VNInfo*, 8> UsedPHIs;
// Extend intervals to reach all uses in WorkList.
while (!WorkList.empty()) {
SlotIndex Idx = WorkList.back().first;
VNInfo *VNI = WorkList.back().second;
WorkList.pop_back();
const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot());
SlotIndex BlockStart = getMBBStartIdx(MBB);
// Extend the live range for VNI to be live at Idx.
if (VNInfo *ExtVNI = NewLR.extendInBlock(BlockStart, Idx)) {
(void)ExtVNI;
assert(ExtVNI == VNI && "Unexpected existing value number");
// Is this a PHIDef we haven't seen before?
if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI))
continue;
// The PHI is live, make sure the predecessors are live-out.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
// A predecessor is not required to have a live-out value for a PHI.
if (VNInfo *PVNI = li->getVNInfoBefore(Stop))
WorkList.push_back(std::make_pair(Stop, PVNI));
}
continue;
}
// VNI is live-in to MBB.
DEBUG(dbgs() << " live-in at " << BlockStart << '\n');
NewLR.addSegment(LiveRange::Segment(BlockStart, Idx, VNI));
// Make sure VNI is live-out from the predecessors.
for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
if (!LiveOut.insert(*PI))
continue;
SlotIndex Stop = getMBBEndIdx(*PI);
assert(li->getVNInfoBefore(Stop) == VNI &&
"Wrong value out of predecessor");
WorkList.push_back(std::make_pair(Stop, VNI));
}
}
// Handle dead values.
bool CanSeparate = false;
for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end();
I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
LiveRange::iterator LRI = NewLR.FindSegmentContaining(VNI->def);
assert(LRI != NewLR.end() && "Missing segment for PHI");
if (LRI->end != VNI->def.getDeadSlot())
continue;
if (VNI->isPHIDef()) {
// This is a dead PHI. Remove it.
VNI->markUnused();
NewLR.removeSegment(LRI->start, LRI->end);
DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n");
CanSeparate = true;
} else {
// This is a dead def. Make sure the instruction knows.
MachineInstr *MI = getInstructionFromIndex(VNI->def);
assert(MI && "No instruction defining live value");
MI->addRegisterDead(li->reg, TRI);
if (dead && MI->allDefsAreDead()) {
DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI);
dead->push_back(MI);
}
}
}
// Move the trimmed segments back.
li->segments.swap(NewLR.segments);
DEBUG(dbgs() << "Shrunk: " << *li << '\n');
return CanSeparate;
}
void InterferenceCache::Entry::update(unsigned MBBNum) {
SlotIndex Start, Stop;
tie(Start, Stop) = Indexes->getMBBRange(MBBNum);
// Use advanceTo only when possible.
if (PrevPos != Start) {
if (!PrevPos.isValid() || Start < PrevPos) {
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
RegUnitInfo &RUI = RegUnits[i];
RUI.VirtI.find(Start);
RUI.FixedI = RUI.Fixed->find(Start);
}
} else {
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
RegUnitInfo &RUI = RegUnits[i];
RUI.VirtI.advanceTo(Start);
if (RUI.FixedI != RUI.Fixed->end())
RUI.FixedI = RUI.Fixed->advanceTo(RUI.FixedI, Start);
}
}
PrevPos = Start;
}
MachineFunction::const_iterator MFI = MF->getBlockNumbered(MBBNum);
BlockInterference *BI = &Blocks[MBBNum];
ArrayRef<SlotIndex> RegMaskSlots;
ArrayRef<const uint32_t*> RegMaskBits;
for (;;) {
BI->Tag = Tag;
BI->First = BI->Last = SlotIndex();
// Check for first interference from virtregs.
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
LiveIntervalUnion::SegmentIter &I = RegUnits[i].VirtI;
if (!I.valid())
continue;
SlotIndex StartI = I.start();
if (StartI >= Stop)
continue;
if (!BI->First.isValid() || StartI < BI->First)
BI->First = StartI;
}
// Same thing for fixed interference.
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
LiveInterval::const_iterator I = RegUnits[i].FixedI;
LiveInterval::const_iterator E = RegUnits[i].Fixed->end();
if (I == E)
continue;
SlotIndex StartI = I->start;
if (StartI >= Stop)
continue;
if (!BI->First.isValid() || StartI < BI->First)
BI->First = StartI;
}
// Also check for register mask interference.
RegMaskSlots = LIS->getRegMaskSlotsInBlock(MBBNum);
RegMaskBits = LIS->getRegMaskBitsInBlock(MBBNum);
SlotIndex Limit = BI->First.isValid() ? BI->First : Stop;
for (unsigned i = 0, e = RegMaskSlots.size();
i != e && RegMaskSlots[i] < Limit; ++i)
if (MachineOperand::clobbersPhysReg(RegMaskBits[i], PhysReg)) {
// Register mask i clobbers PhysReg before the LIU interference.
BI->First = RegMaskSlots[i];
break;
}
PrevPos = Stop;
if (BI->First.isValid())
break;
// No interference in this block? Go ahead and precompute the next block.
if (++MFI == MF->end())
return;
MBBNum = MFI->getNumber();
BI = &Blocks[MBBNum];
if (BI->Tag == Tag)
return;
tie(Start, Stop) = Indexes->getMBBRange(MBBNum);
}
// Check for last interference in block.
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
LiveIntervalUnion::SegmentIter &I = RegUnits[i].VirtI;
if (!I.valid() || I.start() >= Stop)
continue;
I.advanceTo(Stop);
bool Backup = !I.valid() || I.start() >= Stop;
if (Backup)
--I;
SlotIndex StopI = I.stop();
if (!BI->Last.isValid() || StopI > BI->Last)
BI->Last = StopI;
if (Backup)
++I;
}
// Fixed interference.
for (unsigned i = 0, e = RegUnits.size(); i != e; ++i) {
LiveInterval::iterator &I = RegUnits[i].FixedI;
LiveRange *LR = RegUnits[i].Fixed;
if (I == LR->end() || I->start >= Stop)
continue;
I = LR->advanceTo(I, Stop);
bool Backup = I == LR->end() || I->start >= Stop;
if (Backup)
--I;
SlotIndex StopI = I->end;
if (!BI->Last.isValid() || StopI > BI->Last)
BI->Last = StopI;
if (Backup)
++I;
}
// Also check for register mask interference.
SlotIndex Limit = BI->Last.isValid() ? BI->Last : Start;
for (unsigned i = RegMaskSlots.size();
i && RegMaskSlots[i-1].getDeadSlot() > Limit; --i)
if (MachineOperand::clobbersPhysReg(RegMaskBits[i-1], PhysReg)) {
// Register mask i-1 clobbers PhysReg after the LIU interference.
// Model the regmask clobber as a dead def.
BI->Last = RegMaskSlots[i-1].getDeadSlot();
break;
}
}