/// allUsesAvailableAt - Return true if all registers used by OrigMI at
/// OrigIdx are also available with the same value at UseIdx.
bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI,
SlotIndex OrigIdx,
SlotIndex UseIdx,
LiveIntervals &lis) {
OrigIdx = OrigIdx.getUseIndex();
UseIdx = UseIdx.getUseIndex();
for (unsigned i = 0, e = OrigMI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = OrigMI->getOperand(i);
if (!MO.isReg() || !MO.getReg() || MO.isDef())
continue;
// Reserved registers are OK.
if (MO.isUndef() || !lis.hasInterval(MO.getReg()))
continue;
// We cannot depend on virtual registers in uselessRegs_.
if (uselessRegs_)
for (unsigned ui = 0, ue = uselessRegs_->size(); ui != ue; ++ui)
if ((*uselessRegs_)[ui]->reg == MO.getReg())
return false;
LiveInterval &li = lis.getInterval(MO.getReg());
const VNInfo *OVNI = li.getVNInfoAt(OrigIdx);
if (!OVNI)
continue;
if (OVNI != li.getVNInfoAt(UseIdx))
return false;
}
return true;
}
static LaneBitmask getLanesWithProperty(const LiveIntervals &LIS,
const MachineRegisterInfo &MRI, bool TrackLaneMasks, unsigned RegUnit,
SlotIndex Pos,
bool(*Property)(const LiveRange &LR, SlotIndex Pos)) {
if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
const LiveInterval &LI = LIS.getInterval(RegUnit);
LaneBitmask Result = 0;
if (TrackLaneMasks && LI.hasSubRanges()) {
for (const LiveInterval::SubRange &SR : LI.subranges()) {
if (Property(SR, Pos))
Result |= SR.LaneMask;
}
} else if (Property(LI, Pos)) {
Result = TrackLaneMasks ? MRI.getMaxLaneMaskForVReg(RegUnit) : ~0u;
}
return Result;
} else {
const LiveRange *LR = LIS.getCachedRegUnit(RegUnit);
// Be prepared for missing liveranges: We usually do not compute liveranges
// for physical registers on targets with many registers (GPUs).
if (LR == nullptr)
return 0;
return Property(*LR, Pos) ? ~0u : 0;
}
}
/// RenumberValues - Renumber all values in order of appearance and delete the
/// remaining unused values.
void LiveInterval::RenumberValues(LiveIntervals &lis) {
SmallPtrSet<VNInfo*, 8> Seen;
bool seenPHIDef = false;
valnos.clear();
for (const_iterator I = begin(), E = end(); I != E; ++I) {
VNInfo *VNI = I->valno;
if (!Seen.insert(VNI))
continue;
assert(!VNI->isUnused() && "Unused valno used by live range");
VNI->id = (unsigned)valnos.size();
valnos.push_back(VNI);
VNI->setHasPHIKill(false);
if (VNI->isPHIDef())
seenPHIDef = true;
}
// Recompute phi kill flags.
if (!seenPHIDef)
return;
for (const_vni_iterator I = vni_begin(), E = vni_end(); I != E; ++I) {
VNInfo *VNI = *I;
if (!VNI->isPHIDef())
continue;
const MachineBasicBlock *PHIBB = lis.getMBBFromIndex(VNI->def);
assert(PHIBB && "No basic block for phi-def");
for (MachineBasicBlock::const_pred_iterator PI = PHIBB->pred_begin(),
PE = PHIBB->pred_end(); PI != PE; ++PI) {
VNInfo *KVNI = getVNInfoAt(lis.getMBBEndIdx(*PI).getPrevSlot());
if (KVNI)
KVNI->setHasPHIKill(true);
}
}
}
void
UserValue::computeIntervals(MachineRegisterInfo &MRI,
LiveIntervals &LIS,
MachineDominatorTree &MDT) {
SmallVector<std::pair<SlotIndex, unsigned>, 16> Defs;
// Collect all defs to be extended (Skipping undefs).
for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I)
if (I.value() != ~0u)
Defs.push_back(std::make_pair(I.start(), I.value()));
// Extend all defs, and possibly add new ones along the way.
for (unsigned i = 0; i != Defs.size(); ++i) {
SlotIndex Idx = Defs[i].first;
unsigned LocNo = Defs[i].second;
const MachineOperand &Loc = locations[LocNo];
// Register locations are constrained to where the register value is live.
if (Loc.isReg() && LIS.hasInterval(Loc.getReg())) {
LiveInterval *LI = &LIS.getInterval(Loc.getReg());
const VNInfo *VNI = LI->getVNInfoAt(Idx);
SmallVector<SlotIndex, 16> Kills;
extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT);
addDefsFromCopies(LI, LocNo, Kills, Defs, MRI, LIS);
} else
extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT);
}
// Finally, erase all the undefs.
for (LocMap::iterator I = locInts.begin(); I.valid();)
if (I.value() == ~0u)
I.erase();
else
++I;
}
LLVM_DUMP_METHOD
void llvm::printLivesAt(SlotIndex SI,
const LiveIntervals &LIS,
const MachineRegisterInfo &MRI) {
dbgs() << "Live regs at " << SI << ": "
<< *LIS.getInstructionFromIndex(SI);
unsigned Num = 0;
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
const unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
if (!LIS.hasInterval(Reg))
continue;
const auto &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
bool firstTime = true;
for (const auto &S : LI.subranges()) {
if (!S.liveAt(SI)) continue;
if (firstTime) {
dbgs() << " " << PrintReg(Reg, MRI.getTargetRegisterInfo())
<< '\n';
firstTime = false;
}
dbgs() << " " << S << '\n';
++Num;
}
} else if (LI.liveAt(SI)) {
dbgs() << " " << LI << '\n';
++Num;
}
}
if (!Num) dbgs() << " <none>\n";
}
void UserValue::extendDef(SlotIndex Idx, unsigned LocNo,
LiveRange *LR, const VNInfo *VNI,
SmallVectorImpl<SlotIndex> *Kills,
LiveIntervals &LIS, MachineDominatorTree &MDT,
UserValueScopes &UVS) {
SmallVector<SlotIndex, 16> Todo;
Todo.push_back(Idx);
do {
SlotIndex Start = Todo.pop_back_val();
MachineBasicBlock *MBB = LIS.getMBBFromIndex(Start);
SlotIndex Stop = LIS.getMBBEndIdx(MBB);
LocMap::iterator I = locInts.find(Start);
// Limit to VNI's live range.
bool ToEnd = true;
if (LR && VNI) {
LiveInterval::Segment *Segment = LR->getSegmentContaining(Start);
if (!Segment || Segment->valno != VNI) {
if (Kills)
Kills->push_back(Start);
continue;
}
if (Segment->end < Stop)
Stop = Segment->end, ToEnd = false;
}
// There could already be a short def at Start.
if (I.valid() && I.start() <= Start) {
// Stop when meeting a different location or an already extended interval.
Start = Start.getNextSlot();
if (I.value() != LocNo || I.stop() != Start)
continue;
// This is a one-slot placeholder. Just skip it.
++I;
}
// Limited by the next def.
if (I.valid() && I.start() < Stop)
Stop = I.start(), ToEnd = false;
// Limited by VNI's live range.
else if (!ToEnd && Kills)
Kills->push_back(Stop);
if (Start >= Stop)
continue;
I.insert(Start, Stop, LocNo);
// If we extended to the MBB end, propagate down the dominator tree.
if (!ToEnd)
continue;
const std::vector<MachineDomTreeNode*> &Children =
MDT.getNode(MBB)->getChildren();
for (unsigned i = 0, e = Children.size(); i != e; ++i) {
MachineBasicBlock *MBB = Children[i]->getBlock();
if (UVS.dominates(MBB))
Todo.push_back(LIS.getMBBStartIdx(MBB));
}
} while (!Todo.empty());
}
bool LiveRangeEdit::canRematerializeAt(Remat &RM,
SlotIndex UseIdx,
bool cheapAsAMove,
LiveIntervals &lis) {
assert(scannedRemattable_ && "Call anyRematerializable first");
// Use scanRemattable info.
if (!remattable_.count(RM.ParentVNI))
return false;
// No defining instruction provided.
SlotIndex DefIdx;
if (RM.OrigMI)
DefIdx = lis.getInstructionIndex(RM.OrigMI);
else {
DefIdx = RM.ParentVNI->def;
RM.OrigMI = lis.getInstructionFromIndex(DefIdx);
assert(RM.OrigMI && "No defining instruction for remattable value");
}
// If only cheap remats were requested, bail out early.
if (cheapAsAMove && !RM.OrigMI->getDesc().isAsCheapAsAMove())
return false;
// Verify that all used registers are available with the same values.
if (!allUsesAvailableAt(RM.OrigMI, DefIdx, UseIdx, lis))
return false;
return true;
}
void RegAllocPBQP::spillVReg(unsigned VReg,
SmallVectorImpl<unsigned> &NewIntervals,
MachineFunction &MF, LiveIntervals &LIS,
VirtRegMap &VRM, Spiller &VRegSpiller) {
VRegsToAlloc.erase(VReg);
LiveRangeEdit LRE(&LIS.getInterval(VReg), NewIntervals, MF, LIS, &VRM,
nullptr, &DeadRemats);
VRegSpiller.spill(LRE);
const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
(void)TRI;
DEBUG(dbgs() << "VREG " << PrintReg(VReg, &TRI) << " -> SPILLED (Cost: "
<< LRE.getParent().weight << ", New vregs: ");
// Copy any newly inserted live intervals into the list of regs to
// allocate.
for (LiveRangeEdit::iterator I = LRE.begin(), E = LRE.end();
I != E; ++I) {
const LiveInterval &LI = LIS.getInterval(*I);
assert(!LI.empty() && "Empty spill range.");
DEBUG(dbgs() << PrintReg(LI.reg, &TRI) << " ");
VRegsToAlloc.insert(LI.reg);
}
DEBUG(dbgs() << ")\n");
}
/// Test whether OneUse, a use of Reg, dominates all of Reg's other uses.
static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
const MachineBasicBlock &MBB,
const MachineRegisterInfo &MRI,
const MachineDominatorTree &MDT,
LiveIntervals &LIS,
WebAssemblyFunctionInfo &MFI) {
const LiveInterval &LI = LIS.getInterval(Reg);
const MachineInstr *OneUseInst = OneUse.getParent();
VNInfo *OneUseVNI = LI.getVNInfoBefore(LIS.getInstructionIndex(*OneUseInst));
for (const MachineOperand &Use : MRI.use_nodbg_operands(Reg)) {
if (&Use == &OneUse)
continue;
const MachineInstr *UseInst = Use.getParent();
VNInfo *UseVNI = LI.getVNInfoBefore(LIS.getInstructionIndex(*UseInst));
if (UseVNI != OneUseVNI)
continue;
const MachineInstr *OneUseInst = OneUse.getParent();
if (UseInst == OneUseInst) {
// Another use in the same instruction. We need to ensure that the one
// selected use happens "before" it.
if (&OneUse > &Use)
return false;
} else {
// Test that the use is dominated by the one selected use.
while (!MDT.dominates(OneUseInst, UseInst)) {
// Actually, dominating is over-conservative. Test that the use would
// happen after the one selected use in the stack evaluation order.
//
// This is needed as a consequence of using implicit get_locals for
// uses and implicit set_locals for defs.
if (UseInst->getDesc().getNumDefs() == 0)
return false;
const MachineOperand &MO = UseInst->getOperand(0);
if (!MO.isReg())
return false;
unsigned DefReg = MO.getReg();
if (!TargetRegisterInfo::isVirtualRegister(DefReg) ||
!MFI.isVRegStackified(DefReg))
return false;
assert(MRI.hasOneUse(DefReg));
const MachineOperand &NewUse = *MRI.use_begin(DefReg);
const MachineInstr *NewUseInst = NewUse.getParent();
if (NewUseInst == OneUseInst) {
if (&OneUse > &NewUse)
return false;
break;
}
UseInst = NewUseInst;
}
}
}
return true;
}
void
UserValue::computeIntervals(MachineRegisterInfo &MRI,
const TargetRegisterInfo &TRI,
LiveIntervals &LIS,
MachineDominatorTree &MDT,
UserValueScopes &UVS) {
SmallVector<std::pair<SlotIndex, unsigned>, 16> Defs;
// Collect all defs to be extended (Skipping undefs).
for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I)
if (I.value() != ~0u)
Defs.push_back(std::make_pair(I.start(), I.value()));
// Extend all defs, and possibly add new ones along the way.
for (unsigned i = 0; i != Defs.size(); ++i) {
SlotIndex Idx = Defs[i].first;
unsigned LocNo = Defs[i].second;
const MachineOperand &Loc = locations[LocNo];
if (!Loc.isReg()) {
extendDef(Idx, LocNo, 0, 0, 0, LIS, MDT, UVS);
continue;
}
// Register locations are constrained to where the register value is live.
if (TargetRegisterInfo::isVirtualRegister(Loc.getReg())) {
LiveInterval *LI = 0;
const VNInfo *VNI = 0;
if (LIS.hasInterval(Loc.getReg())) {
LI = &LIS.getInterval(Loc.getReg());
VNI = LI->getVNInfoAt(Idx);
}
SmallVector<SlotIndex, 16> Kills;
extendDef(Idx, LocNo, LI, VNI, &Kills, LIS, MDT, UVS);
if (LI)
addDefsFromCopies(LI, LocNo, Kills, Defs, MRI, LIS);
continue;
}
// For physregs, use the live range of the first regunit as a guide.
unsigned Unit = *MCRegUnitIterator(Loc.getReg(), &TRI);
LiveRange *LR = &LIS.getRegUnit(Unit);
const VNInfo *VNI = LR->getVNInfoAt(Idx);
// Don't track copies from physregs, it is too expensive.
extendDef(Idx, LocNo, LR, VNI, 0, LIS, MDT, UVS);
}
// Finally, erase all the undefs.
for (LocMap::iterator I = locInts.begin(); I.valid();)
if (I.value() == ~0u)
I.erase();
else
++I;
}
// Check if all values in LI are rematerializable
static bool isRematerializable(const LiveInterval &LI,
const LiveIntervals &LIS,
VirtRegMap *VRM,
const TargetInstrInfo &TII) {
unsigned Reg = LI.reg;
unsigned Original = VRM ? VRM->getOriginal(Reg) : 0;
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I != E; ++I) {
const VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
if (VNI->isPHIDef())
return false;
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Dead valno in interval");
// Trace copies introduced by live range splitting. The inline
// spiller can rematerialize through these copies, so the spill
// weight must reflect this.
if (VRM) {
while (MI->isFullCopy()) {
// The copy destination must match the interval register.
if (MI->getOperand(0).getReg() != Reg)
return false;
// Get the source register.
Reg = MI->getOperand(1).getReg();
// If the original (pre-splitting) registers match this
// copy came from a split.
if (!TargetRegisterInfo::isVirtualRegister(Reg) ||
VRM->getOriginal(Reg) != Original)
return false;
// Follow the copy live-in value.
const LiveInterval &SrcLI = LIS.getInterval(Reg);
LiveQueryResult SrcQ = SrcLI.Query(VNI->def);
VNI = SrcQ.valueIn();
assert(VNI && "Copy from non-existing value");
if (VNI->isPHIDef())
return false;
MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Dead valno in interval");
}
}
if (!TII.isTriviallyReMaterializable(*MI, LIS.getAliasAnalysis()))
return false;
}
return true;
}
void LiveInterval::MergeInClobberRange(LiveIntervals &li_,
SlotIndex Start,
SlotIndex End,
VNInfo::Allocator &VNInfoAllocator) {
// Find a value # to use for the clobber ranges. If there is already a value#
// for unknown values, use it.
VNInfo *ClobberValNo =
getNextValue(li_.getInvalidIndex(), 0, false, VNInfoAllocator);
iterator IP = begin();
IP = std::upper_bound(IP, end(), Start);
// If the start of this range overlaps with an existing liverange, trim it.
if (IP != begin() && IP[-1].end > Start) {
Start = IP[-1].end;
// Trimmed away the whole range?
if (Start >= End) return;
}
// If the end of this range overlaps with an existing liverange, trim it.
if (IP != end() && End > IP->start) {
End = IP->start;
// If this trimmed away the whole range, ignore it.
if (Start == End) return;
}
// Insert the clobber interval.
addRangeFrom(LiveRange(Start, End, ClobberValNo), IP);
}
// Dump the range of instructions from B to E with their slot indexes.
static void dumpMachineInstrRangeWithSlotIndex(MachineBasicBlock::iterator B,
MachineBasicBlock::iterator E,
LiveIntervals const &LIS,
const char *const header,
unsigned VReg =0) {
char NextLine = '\n';
char SlotIndent = '\t';
if (std::next(B) == E) {
NextLine = ' ';
SlotIndent = ' ';
}
dbgs() << '\t' << header << ": " << NextLine;
for (MachineBasicBlock::iterator I = B; I != E; ++I) {
SlotIndex Idx = LIS.getInstructionIndex(I).getRegSlot();
// If a register was passed in and this instruction has it as a
// destination that is marked as an early clobber, print the
// early-clobber slot index.
if (VReg) {
MachineOperand *MO = I->findRegisterDefOperand(VReg);
if (MO && MO->isEarlyClobber())
Idx = Idx.getRegSlot(true);
}
dbgs() << SlotIndent << Idx << '\t' << *I;
}
}
bool HexagonRegisterInfo::shouldCoalesce(MachineInstr *MI,
const TargetRegisterClass *SrcRC, unsigned SubReg,
const TargetRegisterClass *DstRC, unsigned DstSubReg,
const TargetRegisterClass *NewRC, LiveIntervals &LIS) const {
// Coalescing will extend the live interval of the destination register.
// If the destination register is a vector pair, avoid introducing function
// calls into the interval, since it could result in a spilling of a pair
// instead of a single vector.
MachineFunction &MF = *MI->getParent()->getParent();
const HexagonSubtarget &HST = MF.getSubtarget<HexagonSubtarget>();
if (!HST.useHVXOps() || NewRC->getID() != Hexagon::HvxWRRegClass.getID())
return true;
bool SmallSrc = SrcRC->getID() == Hexagon::HvxVRRegClass.getID();
bool SmallDst = DstRC->getID() == Hexagon::HvxVRRegClass.getID();
if (!SmallSrc && !SmallDst)
return true;
unsigned DstReg = MI->getOperand(0).getReg();
unsigned SrcReg = MI->getOperand(1).getReg();
const SlotIndexes &Indexes = *LIS.getSlotIndexes();
auto HasCall = [&Indexes] (const LiveInterval::Segment &S) {
for (SlotIndex I = S.start.getBaseIndex(), E = S.end.getBaseIndex();
I != E; I = I.getNextIndex()) {
if (const MachineInstr *MI = Indexes.getInstructionFromIndex(I))
if (MI->isCall())
return true;
}
return false;
};
if (SmallSrc == SmallDst) {
// Both must be true, because the case for both being false was
// checked earlier. Both registers will be coalesced into a register
// of a wider class (HvxWR), and we don't want its live range to
// span over calls.
return !any_of(LIS.getInterval(DstReg), HasCall) &&
!any_of(LIS.getInterval(SrcReg), HasCall);
}
// If one register is large (HvxWR) and the other is small (HvxVR), then
// coalescing is ok if the large is already live across a function call,
// or if the small one is not.
unsigned SmallReg = SmallSrc ? SrcReg : DstReg;
unsigned LargeReg = SmallSrc ? DstReg : SrcReg;
return any_of(LIS.getInterval(LargeReg), HasCall) ||
!any_of(LIS.getInterval(SmallReg), HasCall);
}
/// We only propagate DBG_VALUES locally here. LiveDebugValues performs a
/// data-flow analysis to propagate them beyond basic block boundaries.
void UserValue::extendDef(SlotIndex Idx, unsigned LocNo, LiveRange *LR,
const VNInfo *VNI, SmallVectorImpl<SlotIndex> *Kills,
LiveIntervals &LIS, MachineDominatorTree &MDT,
UserValueScopes &UVS) {
SlotIndex Start = Idx;
MachineBasicBlock *MBB = LIS.getMBBFromIndex(Start);
SlotIndex Stop = LIS.getMBBEndIdx(MBB);
LocMap::iterator I = locInts.find(Start);
// Limit to VNI's live range.
bool ToEnd = true;
if (LR && VNI) {
LiveInterval::Segment *Segment = LR->getSegmentContaining(Start);
if (!Segment || Segment->valno != VNI) {
if (Kills)
Kills->push_back(Start);
return;
}
if (Segment->end < Stop) {
Stop = Segment->end;
ToEnd = false;
}
}
// There could already be a short def at Start.
if (I.valid() && I.start() <= Start) {
// Stop when meeting a different location or an already extended interval.
Start = Start.getNextSlot();
if (I.value() != LocNo || I.stop() != Start)
return;
// This is a one-slot placeholder. Just skip it.
++I;
}
// Limited by the next def.
if (I.valid() && I.start() < Stop) {
Stop = I.start();
ToEnd = false;
}
// Limited by VNI's live range.
else if (!ToEnd && Kills)
Kills->push_back(Stop);
if (Start < Stop)
I.insert(Start, Stop, LocNo);
}
void RegAllocPBQP::postOptimization(Spiller &VRegSpiller, LiveIntervals &LIS) {
VRegSpiller.postOptimization();
/// Remove dead defs because of rematerialization.
for (auto DeadInst : DeadRemats) {
LIS.RemoveMachineInstrFromMaps(*DeadInst);
DeadInst->eraseFromParent();
}
DeadRemats.clear();
}
// Test whether it's safe to move Def to just before Insert.
// TODO: Compute memory dependencies in a way that doesn't require always
// walking the block.
// TODO: Compute memory dependencies in a way that uses AliasAnalysis to be
// more precise.
static bool IsSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
AliasAnalysis &AA, LiveIntervals &LIS,
MachineRegisterInfo &MRI) {
assert(Def->getParent() == Insert->getParent());
bool SawStore = false, SawSideEffects = false;
MachineBasicBlock::const_iterator D(Def), I(Insert);
// Check for register dependencies.
for (const MachineOperand &MO : Def->operands()) {
if (!MO.isReg() || MO.isUndef())
continue;
unsigned Reg = MO.getReg();
// If the register is dead here and at Insert, ignore it.
if (MO.isDead() && Insert->definesRegister(Reg) &&
!Insert->readsRegister(Reg))
continue;
if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
// If the physical register is never modified, ignore it.
if (!MRI.isPhysRegModified(Reg))
continue;
// Otherwise, it's a physical register with unknown liveness.
return false;
}
// Ask LiveIntervals whether moving this virtual register use or def to
// Insert will change value numbers are seen.
const LiveInterval &LI = LIS.getInterval(Reg);
VNInfo *DefVNI =
MO.isDef() ? LI.getVNInfoAt(LIS.getInstructionIndex(Def).getRegSlot())
: LI.getVNInfoBefore(LIS.getInstructionIndex(Def));
assert(DefVNI && "Instruction input missing value number");
VNInfo *InsVNI = LI.getVNInfoBefore(LIS.getInstructionIndex(Insert));
if (InsVNI && DefVNI != InsVNI)
return false;
}
// Check for memory dependencies and side effects.
for (--I; I != D; --I)
SawSideEffects |= I->isSafeToMove(&AA, SawStore);
return !(SawStore && Def->mayLoad() && !Def->isInvariantLoad(&AA)) &&
!(SawSideEffects && !Def->isSafeToMove(&AA, SawStore));
}
// Check if all values in LI are rematerializable
static bool isRematerializable(const LiveInterval &LI,
const LiveIntervals &LIS,
const TargetInstrInfo &TII) {
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
I != E; ++I) {
const VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
if (VNI->isPHIDef())
return false;
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
assert(MI && "Dead valno in interval");
if (!TII.isTriviallyReMaterializable(MI, LIS.getAliasAnalysis()))
return false;
}
return true;
}
void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS,
const TargetInstrInfo &TII) {
MachineFunction::iterator MFEnd = VRM->getMachineFunction().end();
for (LocMap::const_iterator I = locInts.begin(); I.valid();) {
SlotIndex Start = I.start();
SlotIndex Stop = I.stop();
unsigned LocNo = I.value();
DEBUG(dbgs() << "\t[" << Start << ';' << Stop << "):" << LocNo);
MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start);
SlotIndex MBBEnd = LIS.getMBBEndIdx(MBB);
DEBUG(dbgs() << " BB#" << MBB->getNumber() << '-' << MBBEnd);
insertDebugValue(MBB, Start, LocNo, LIS, TII);
// This interval may span multiple basic blocks.
// Insert a DBG_VALUE into each one.
while(Stop > MBBEnd) {
// Move to the next block.
Start = MBBEnd;
if (++MBB == MFEnd)
break;
MBBEnd = LIS.getMBBEndIdx(MBB);
DEBUG(dbgs() << " BB#" << MBB->getNumber() << '-' << MBBEnd);
insertDebugValue(MBB, Start, LocNo, LIS, TII);
}
DEBUG(dbgs() << '\n');
if (MBB == MFEnd)
break;
++I;
if (Stop == MBBEnd)
continue;
// The current interval ends before MBB.
// Insert a kill if there is a gap.
if (!I.valid() || I.start() > Stop)
insertDebugKill(MBB, Stop, LIS, TII);
}
}
GCNRPTracker::LiveRegSet llvm::getLiveRegs(SlotIndex SI,
const LiveIntervals &LIS,
const MachineRegisterInfo &MRI) {
GCNRPTracker::LiveRegSet LiveRegs;
for (unsigned I = 0, E = MRI.getNumVirtRegs(); I != E; ++I) {
auto Reg = TargetRegisterInfo::index2VirtReg(I);
if (!LIS.hasInterval(Reg))
continue;
auto LiveMask = getLiveLaneMask(Reg, SI, LIS, MRI);
if (LiveMask.any())
LiveRegs[Reg] = LiveMask;
}
return LiveRegs;
}
SlotIndex LiveRangeEdit::rematerializeAt(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
unsigned DestReg,
const Remat &RM,
LiveIntervals &lis,
const TargetInstrInfo &tii,
const TargetRegisterInfo &tri,
bool Late) {
assert(RM.OrigMI && "Invalid remat");
tii.reMaterialize(MBB, MI, DestReg, 0, RM.OrigMI, tri);
rematted_.insert(RM.ParentVNI);
return lis.getSlotIndexes()->insertMachineInstrInMaps(--MI, Late)
.getDefIndex();
}
bool SIFixSGPRLiveRanges::runOnMachineFunction(MachineFunction &MF) {
MachineRegisterInfo &MRI = MF.getRegInfo();
const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(
MF.getTarget().getRegisterInfo());
LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
BI != BE; ++BI) {
MachineBasicBlock &MBB = *BI;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
I != E; ++I) {
MachineInstr &MI = *I;
MachineOperand *ExecUse = MI.findRegisterUseOperand(AMDGPU::EXEC);
if (ExecUse)
continue;
for (const MachineOperand &Def : MI.operands()) {
if (!Def.isReg() || !Def.isDef() ||!TargetRegisterInfo::isVirtualRegister(Def.getReg()))
continue;
const TargetRegisterClass *RC = MRI.getRegClass(Def.getReg());
if (!TRI->isSGPRClass(RC))
continue;
LiveInterval &LI = LIS->getInterval(Def.getReg());
for (unsigned i = 0, e = LI.size() - 1; i != e; ++i) {
LiveRange::Segment &Seg = LI.segments[i];
LiveRange::Segment &Next = LI.segments[i + 1];
Seg.end = Next.start;
}
}
}
}
return false;
}
void LiveRangeEdit::scanRemattable(LiveIntervals &lis,
const TargetInstrInfo &tii,
AliasAnalysis *aa) {
for (LiveInterval::vni_iterator I = parent_.vni_begin(),
E = parent_.vni_end(); I != E; ++I) {
VNInfo *VNI = *I;
if (VNI->isUnused())
continue;
MachineInstr *DefMI = lis.getInstructionFromIndex(VNI->def);
if (!DefMI)
continue;
checkRematerializable(VNI, DefMI, tii, aa);
}
scannedRemattable_ = true;
}
// Test whether Reg, as defined at Def, has exactly one use. This is a
// generalization of MachineRegisterInfo::hasOneUse that uses LiveIntervals
// to handle complex cases.
static bool HasOneUse(unsigned Reg, MachineInstr *Def,
MachineRegisterInfo &MRI, MachineDominatorTree &MDT,
LiveIntervals &LIS) {
// Most registers are in SSA form here so we try a quick MRI query first.
if (MRI.hasOneUse(Reg))
return true;
bool HasOne = false;
const LiveInterval &LI = LIS.getInterval(Reg);
const VNInfo *DefVNI = LI.getVNInfoAt(
LIS.getInstructionIndex(*Def).getRegSlot());
assert(DefVNI);
for (auto &I : MRI.use_nodbg_operands(Reg)) {
const auto &Result = LI.Query(LIS.getInstructionIndex(*I.getParent()));
if (Result.valueIn() == DefVNI) {
if (!Result.isKill())
return false;
if (HasOne)
return false;
HasOne = true;
}
}
return HasOne;
}
/// findInsertLocation - Find an iterator and DebugLoc for inserting a DBG_VALUE
/// instruction.
static MachineBasicBlock::iterator
findInsertLocation(MachineBasicBlock *MBB, SlotIndex Idx, DebugLoc &DL,
LiveIntervals &LIS) {
SlotIndex Start = LIS.getMBBStartIdx(MBB);
Idx = Idx.getBaseIndex();
// Try to find an insert location by going backwards from Idx.
MachineInstr *MI;
while (!(MI = LIS.getInstructionFromIndex(Idx))) {
// We've reached the beginning of MBB.
if (Idx == Start) {
MachineBasicBlock::iterator I = MBB->SkipPHIsAndLabels(MBB->begin());
if (I != MBB->end())
DL = I->getDebugLoc();
return I;
}
Idx = Idx.getPrevIndex();
}
// We found an instruction. The insert point is after the instr.
DL = MI->getDebugLoc();
// Don't insert anything after the first terminator, though.
return MI->getDesc().isTerminator() ? MBB->getFirstTerminator() :
llvm::next(MachineBasicBlock::iterator(MI));
}
bool LiveRangeEdit::foldAsLoad(LiveInterval *LI,
SmallVectorImpl<MachineInstr*> &Dead,
MachineRegisterInfo &MRI,
LiveIntervals &LIS,
const TargetInstrInfo &TII) {
MachineInstr *DefMI = 0, *UseMI = 0;
// Check that there is a single def and a single use.
for (MachineRegisterInfo::reg_nodbg_iterator I = MRI.reg_nodbg_begin(LI->reg),
E = MRI.reg_nodbg_end(); I != E; ++I) {
MachineOperand &MO = I.getOperand();
MachineInstr *MI = MO.getParent();
if (MO.isDef()) {
if (DefMI && DefMI != MI)
return false;
if (!MI->getDesc().canFoldAsLoad())
return false;
DefMI = MI;
} else if (!MO.isUndef()) {
if (UseMI && UseMI != MI)
return false;
// FIXME: Targets don't know how to fold subreg uses.
if (MO.getSubReg())
return false;
UseMI = MI;
}
}
if (!DefMI || !UseMI)
return false;
DEBUG(dbgs() << "Try to fold single def: " << *DefMI
<< " into single use: " << *UseMI);
SmallVector<unsigned, 8> Ops;
if (UseMI->readsWritesVirtualRegister(LI->reg, &Ops).second)
return false;
MachineInstr *FoldMI = TII.foldMemoryOperand(UseMI, Ops, DefMI);
if (!FoldMI)
return false;
DEBUG(dbgs() << " folded: " << *FoldMI);
LIS.ReplaceMachineInstrInMaps(UseMI, FoldMI);
UseMI->eraseFromParent();
DefMI->addRegisterDead(LI->reg, 0);
Dead.push_back(DefMI);
++NumDCEFoldedLoads;
return true;
}
void llvm::calculateSpillWeightsAndHints(LiveIntervals &LIS,
MachineFunction &MF,
const MachineLoopInfo &MLI,
const MachineBlockFrequencyInfo &MBFI,
VirtRegAuxInfo::NormalizingFn norm) {
DEBUG(dbgs() << "********** Compute Spill Weights **********\n"
<< "********** Function: " << MF.getName() << '\n');
MachineRegisterInfo &MRI = MF.getRegInfo();
VirtRegAuxInfo VRAI(MF, LIS, MLI, MBFI, norm);
for (unsigned i = 0, e = MRI.getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
if (MRI.reg_nodbg_empty(Reg))
continue;
VRAI.calculateSpillWeightAndHint(LIS.getInterval(Reg));
}
}
LaneBitmask llvm::getLiveLaneMask(unsigned Reg,
SlotIndex SI,
const LiveIntervals &LIS,
const MachineRegisterInfo &MRI) {
LaneBitmask LiveMask;
const auto &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
for (const auto &S : LI.subranges())
if (S.liveAt(SI)) {
LiveMask |= S.LaneMask;
assert(LiveMask < MRI.getMaxLaneMaskForVReg(Reg) ||
LiveMask == MRI.getMaxLaneMaskForVReg(Reg));
}
} else if (LI.liveAt(SI)) {
LiveMask = MRI.getMaxLaneMaskForVReg(Reg);
}
return LiveMask;
}
static LaneBitmask getUsedRegMask(const MachineOperand &MO,
const MachineRegisterInfo &MRI,
const LiveIntervals &LIS) {
assert(MO.isUse() && MO.isReg() &&
TargetRegisterInfo::isVirtualRegister(MO.getReg()));
if (auto SubReg = MO.getSubReg())
return MRI.getTargetRegisterInfo()->getSubRegIndexLaneMask(SubReg);
auto MaxMask = MRI.getMaxLaneMaskForVReg(MO.getReg());
if (MaxMask == LaneBitmask::getLane(0)) // cannot have subregs
return MaxMask;
// For a tentative schedule LIS isn't updated yet but livemask should remain
// the same on any schedule. Subreg defs can be reordered but they all must
// dominate uses anyway.
auto SI = LIS.getInstructionIndex(*MO.getParent()).getBaseIndex();
return getLiveLaneMask(MO.getReg(), SI, LIS, MRI);
}
void RegisterOperands::detectDeadDefs(const MachineInstr &MI,
const LiveIntervals &LIS) {
SlotIndex SlotIdx = LIS.getInstructionIndex(MI);
for (auto RI = Defs.begin(); RI != Defs.end(); /*empty*/) {
unsigned Reg = RI->RegUnit;
const LiveRange *LR = getLiveRange(LIS, Reg);
if (LR != nullptr) {
LiveQueryResult LRQ = LR->Query(SlotIdx);
if (LRQ.isDeadDef()) {
// LiveIntervals knows this is a dead even though it's MachineOperand is
// not flagged as such.
DeadDefs.push_back(*RI);
RI = Defs.erase(RI);
continue;
}
}
++RI;
}
}
