LaneBitmask RegPressureTracker::getLastUsedLanes(unsigned RegUnit,
SlotIndex Pos) const {
assert(RequireIntervals);
return getLanesWithProperty(*LIS, *MRI, TrackLaneMasks, RegUnit,
Pos.getBaseIndex(), LaneBitmask::getNone(),
[](const LiveRange &LR, SlotIndex Pos) {
const LiveRange::Segment *S = LR.getSegmentContaining(Pos);
return S != nullptr && S->end == Pos.getRegSlot();
});
}
bool RenameIndependentSubregs::findComponents(IntEqClasses &Classes,
SmallVectorImpl<RenameIndependentSubregs::SubRangeInfo> &SubRangeInfos,
LiveInterval &LI) const {
// First step: Create connected components for the VNInfos inside the
// subranges and count the global number of such components.
unsigned NumComponents = 0;
for (LiveInterval::SubRange &SR : LI.subranges()) {
SubRangeInfos.push_back(SubRangeInfo(*LIS, SR, NumComponents));
ConnectedVNInfoEqClasses &ConEQ = SubRangeInfos.back().ConEQ;
unsigned NumSubComponents = ConEQ.Classify(SR);
NumComponents += NumSubComponents;
}
// Shortcut: With only 1 subrange, the normal separate component tests are
// enough and we do not need to perform the union-find on the subregister
// segments.
if (SubRangeInfos.size() < 2)
return false;
// Next step: Build union-find structure over all subranges and merge classes
// across subranges when they are affected by the same MachineOperand.
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
Classes.grow(NumComponents);
unsigned Reg = LI.reg;
for (const MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
if (!MO.isDef() && !MO.readsReg())
continue;
unsigned SubRegIdx = MO.getSubReg();
LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
unsigned MergedID = ~0u;
for (RenameIndependentSubregs::SubRangeInfo &SRInfo : SubRangeInfos) {
const LiveInterval::SubRange &SR = *SRInfo.SR;
if ((SR.LaneMask & LaneMask) == 0)
continue;
SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent());
Pos = MO.isDef() ? Pos.getRegSlot(MO.isEarlyClobber())
: Pos.getBaseIndex();
const VNInfo *VNI = SR.getVNInfoAt(Pos);
if (VNI == nullptr)
continue;
// Map to local representant ID.
unsigned LocalID = SRInfo.ConEQ.getEqClass(VNI);
// Global ID
unsigned ID = LocalID + SRInfo.Index;
// Merge other sets
MergedID = MergedID == ~0u ? ID : Classes.join(MergedID, ID);
}
}
// Early exit if we ended up with a single equivalence class.
Classes.compress();
unsigned NumClasses = Classes.getNumClasses();
return NumClasses > 1;
}
SlotIndex SplitEditor::enterIntvBefore(SlotIndex Idx) {
assert(OpenIdx && "openIntv not called before enterIntvBefore");
DEBUG(dbgs() << " enterIntvBefore " << Idx);
Idx = Idx.getBaseIndex();
VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
if (!ParentVNI) {
DEBUG(dbgs() << ": not live\n");
return Idx;
}
DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
assert(MI && "enterIntvBefore called with invalid index");
VNInfo *VNI = defFromParent(OpenIdx, ParentVNI, Idx, *MI->getParent(), MI);
return VNI->def;
}
void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
const MachineRegisterInfo &MRI,
SlotIndex Pos,
MachineInstr *AddFlagsMI) {
for (auto I = Defs.begin(); I != Defs.end(); ) {
LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
Pos.getDeadSlot());
// If the the def is all that is live after the instruction, then in case
// of a subregister def we need a read-undef flag.
unsigned RegUnit = I->RegUnit;
if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask) == 0)
AddFlagsMI->setRegisterDefReadUndef(RegUnit);
LaneBitmask LaneMask = I->LaneMask & LiveAfter;
if (LaneMask == 0) {
I = Defs.erase(I);
// Make sure the operand is properly marked as Dead.
if (AddFlagsMI != nullptr)
AddFlagsMI->addRegisterDead(RegUnit, MRI.getTargetRegisterInfo());
} else {
I->LaneMask = LaneMask;
++I;
}
}
for (auto I = Uses.begin(); I != Uses.end(); ) {
LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
Pos.getBaseIndex());
LaneBitmask LaneMask = I->LaneMask & LiveBefore;
if (LaneMask == 0) {
I = Uses.erase(I);
} else {
I->LaneMask = LaneMask;
++I;
}
}
if (AddFlagsMI != nullptr) {
for (const RegisterMaskPair &P : DeadDefs) {
unsigned RegUnit = P.RegUnit;
LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
Pos.getDeadSlot());
if (LiveAfter == 0)
AddFlagsMI->setRegisterDefReadUndef(RegUnit);
}
}
}
SlotIndex SplitEditor::leaveIntvBefore(SlotIndex Idx) {
assert(OpenIdx && "openIntv not called before leaveIntvBefore");
DEBUG(dbgs() << " leaveIntvBefore " << Idx);
// The interval must be live into the instruction at Idx.
Idx = Idx.getBaseIndex();
VNInfo *ParentVNI = Edit->getParent().getVNInfoAt(Idx);
if (!ParentVNI) {
DEBUG(dbgs() << ": not live\n");
return Idx.getNextSlot();
}
DEBUG(dbgs() << ": valno " << ParentVNI->id << '\n');
MachineInstr *MI = LIS.getInstructionFromIndex(Idx);
assert(MI && "No instruction at index");
VNInfo *VNI = defFromParent(0, ParentVNI, Idx, *MI->getParent(), MI);
return VNI->def;
}
void RenameIndependentSubregs::rewriteOperands(const IntEqClasses &Classes,
const SmallVectorImpl<SubRangeInfo> &SubRangeInfos,
const SmallVectorImpl<LiveInterval*> &Intervals) const {
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
unsigned Reg = Intervals[0]->reg;;
for (MachineRegisterInfo::reg_nodbg_iterator I = MRI->reg_nodbg_begin(Reg),
E = MRI->reg_nodbg_end(); I != E; ) {
MachineOperand &MO = *I++;
if (!MO.isDef() && !MO.readsReg())
continue;
SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent());
Pos = MO.isDef() ? Pos.getRegSlot(MO.isEarlyClobber())
: Pos.getBaseIndex();
unsigned SubRegIdx = MO.getSubReg();
LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
unsigned ID = ~0u;
for (const SubRangeInfo &SRInfo : SubRangeInfos) {
const LiveInterval::SubRange &SR = *SRInfo.SR;
if ((SR.LaneMask & LaneMask) == 0)
continue;
const VNInfo *VNI = SR.getVNInfoAt(Pos);
if (VNI == nullptr)
continue;
// Map to local representant ID.
unsigned LocalID = SRInfo.ConEQ.getEqClass(VNI);
// Global ID
ID = Classes[LocalID + SRInfo.Index];
break;
}
unsigned VReg = Intervals[ID]->reg;
MO.setReg(VReg);
}
// TODO: We could attempt to recompute new register classes while visiting
// the operands: Some of the split register may be fine with less constraint
// classes than the original vreg.
}
/// Check whether (part of) \p SuperPhysReg is live through \p MI.
/// \pre \p MI defines a subregister of a virtual register that
/// has been assigned to \p SuperPhysReg.
bool VirtRegRewriter::subRegLiveThrough(const MachineInstr &MI,
unsigned SuperPhysReg) const {
SlotIndex MIIndex = LIS->getInstructionIndex(MI);
SlotIndex BeforeMIUses = MIIndex.getBaseIndex();
SlotIndex AfterMIDefs = MIIndex.getBoundaryIndex();
for (MCRegUnitIterator Unit(SuperPhysReg, TRI); Unit.isValid(); ++Unit) {
const LiveRange &UnitRange = LIS->getRegUnit(*Unit);
// If the regunit is live both before and after MI,
// we assume it is live through.
// Generally speaking, this is not true, because something like
// "RU = op RU" would match that description.
// However, we know that we are trying to assess whether
// a def of a virtual reg, vreg, is live at the same time of RU.
// If we are in the "RU = op RU" situation, that means that vreg
// is defined at the same time as RU (i.e., "vreg, RU = op RU").
// Thus, vreg and RU interferes and vreg cannot be assigned to
// SuperPhysReg. Therefore, this situation cannot happen.
if (UnitRange.liveAt(AfterMIDefs) && UnitRange.liveAt(BeforeMIUses))
return true;
}
return false;
}
/// findInsertLocation - Find an iterator and DebugLoc for inserting a DBG_VALUE
/// instruction.
static MachineBasicBlock::iterator
findInsertLocation(MachineBasicBlock *MBB, SlotIndex Idx, DebugLoc &DL,
LiveIntervals &LIS) {
SlotIndex Start = LIS.getMBBStartIdx(MBB);
Idx = Idx.getBaseIndex();
// Try to find an insert location by going backwards from Idx.
MachineInstr *MI;
while (!(MI = LIS.getInstructionFromIndex(Idx))) {
// We've reached the beginning of MBB.
if (Idx == Start) {
MachineBasicBlock::iterator I = MBB->SkipPHIsAndLabels(MBB->begin());
if (I != MBB->end())
DL = I->getDebugLoc();
return I;
}
Idx = Idx.getPrevIndex();
}
// We found an instruction. The insert point is after the instr.
DL = MI->getDebugLoc();
// Don't insert anything after the first terminator, though.
return MI->getDesc().isTerminator() ? MBB->getFirstTerminator() :
llvm::next(MachineBasicBlock::iterator(MI));
}
/// reMaterializeFor - Attempt to rematerialize before MI instead of reloading.
bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg,
MachineBasicBlock::iterator MI) {
SlotIndex UseIdx = LIS.getInstructionIndex(MI).getRegSlot(true);
VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex());
if (!ParentVNI) {
DEBUG(dbgs() << "\tadding <undef> flags: ");
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg)
MO.setIsUndef();
}
DEBUG(dbgs() << UseIdx << '\t' << *MI);
return true;
}
if (SnippetCopies.count(MI))
return false;
// Use an OrigVNI from traceSiblingValue when ParentVNI is a sibling copy.
LiveRangeEdit::Remat RM(ParentVNI);
SibValueMap::const_iterator SibI = SibValues.find(ParentVNI);
if (SibI != SibValues.end())
RM.OrigMI = SibI->second.DefMI;
if (!Edit->canRematerializeAt(RM, UseIdx, false)) {
markValueUsed(&VirtReg, ParentVNI);
DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI);
return false;
}
// If the instruction also writes VirtReg.reg, it had better not require the
// same register for uses and defs.
SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops;
MIBundleOperands::VirtRegInfo RI =
MIBundleOperands(MI).analyzeVirtReg(VirtReg.reg, &Ops);
if (RI.Tied) {
markValueUsed(&VirtReg, ParentVNI);
DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << *MI);
return false;
}
// Before rematerializing into a register for a single instruction, try to
// fold a load into the instruction. That avoids allocating a new register.
if (RM.OrigMI->canFoldAsLoad() &&
foldMemoryOperand(Ops, RM.OrigMI)) {
Edit->markRematerialized(RM.ParentVNI);
++NumFoldedLoads;
return true;
}
// Alocate a new register for the remat.
LiveInterval &NewLI = Edit->createFrom(Original);
NewLI.markNotSpillable();
// Finally we can rematerialize OrigMI before MI.
SlotIndex DefIdx = Edit->rematerializeAt(*MI->getParent(), MI, NewLI.reg, RM,
TRI);
DEBUG(dbgs() << "\tremat: " << DefIdx << '\t'
<< *LIS.getInstructionFromIndex(DefIdx));
// Replace operands
for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(Ops[i].second);
if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) {
MO.setReg(NewLI.reg);
MO.setIsKill();
}
}
DEBUG(dbgs() << "\t " << UseIdx << '\t' << *MI);
VNInfo *DefVNI = NewLI.getNextValue(DefIdx, LIS.getVNInfoAllocator());
NewLI.addRange(LiveRange(DefIdx, UseIdx.getRegSlot(), DefVNI));
DEBUG(dbgs() << "\tinterval: " << NewLI << '\n');
++NumRemats;
return true;
}
void SplitEditor::splitLiveThroughBlock(unsigned MBBNum,
unsigned IntvIn, SlotIndex LeaveBefore,
unsigned IntvOut, SlotIndex EnterAfter){
SlotIndex Start, Stop;
tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum);
DEBUG(dbgs() << "BB#" << MBBNum << " [" << Start << ';' << Stop
<< ") intf " << LeaveBefore << '-' << EnterAfter
<< ", live-through " << IntvIn << " -> " << IntvOut);
assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks");
assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block");
assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf");
assert((!EnterAfter || EnterAfter >= Start) && "Interference before block");
MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum);
if (!IntvOut) {
DEBUG(dbgs() << ", spill on entry.\n");
//
// <<<<<<<<< Possible LeaveBefore interference.
// |-----------| Live through.
// -____________ Spill on entry.
//
selectIntv(IntvIn);
SlotIndex Idx = leaveIntvAtTop(*MBB);
assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
(void)Idx;
return;
}
if (!IntvIn) {
DEBUG(dbgs() << ", reload on exit.\n");
//
// >>>>>>> Possible EnterAfter interference.
// |-----------| Live through.
// ___________-- Reload on exit.
//
selectIntv(IntvOut);
SlotIndex Idx = enterIntvAtEnd(*MBB);
assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
(void)Idx;
return;
}
if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) {
DEBUG(dbgs() << ", straight through.\n");
//
// |-----------| Live through.
// ------------- Straight through, same intv, no interference.
//
selectIntv(IntvOut);
useIntv(Start, Stop);
return;
}
// We cannot legally insert splits after LSP.
SlotIndex LSP = SA.getLastSplitPoint(MBBNum);
assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf");
if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter ||
LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) {
DEBUG(dbgs() << ", switch avoiding interference.\n");
//
// >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference.
// |-----------| Live through.
// ------======= Switch intervals between interference.
//
selectIntv(IntvOut);
SlotIndex Idx;
if (LeaveBefore && LeaveBefore < LSP) {
Idx = enterIntvBefore(LeaveBefore);
useIntv(Idx, Stop);
} else {
Idx = enterIntvAtEnd(*MBB);
}
selectIntv(IntvIn);
useIntv(Start, Idx);
assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
return;
}
DEBUG(dbgs() << ", create local intv for interference.\n");
//
// >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference.
// |-----------| Live through.
// ==---------== Switch intervals before/after interference.
//
assert(LeaveBefore <= EnterAfter && "Missed case");
selectIntv(IntvOut);
SlotIndex Idx = enterIntvAfter(EnterAfter);
useIntv(Idx, Stop);
assert((!EnterAfter || Idx >= EnterAfter) && "Interference");
selectIntv(IntvIn);
Idx = leaveIntvBefore(LeaveBefore);
useIntv(Start, Idx);
assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference");
}
