bool PhysicalRegisterInfo::aliasRM(RegisterRef RR, RegisterRef RM) const {
assert(TargetRegisterInfo::isPhysicalRegister(RR.Reg) && isRegMaskId(RM.Reg));
const uint32_t *MB = getRegMaskBits(RM.Reg);
bool Preserved = MB[RR.Reg/32] & (1u << (RR.Reg%32));
// If the lane mask information is "full", e.g. when the given lane mask
// is a superset of the lane mask from the register class, check the regmask
// bit directly.
if (RR.Mask == LaneBitmask::getAll())
return !Preserved;
const TargetRegisterClass *RC = RegInfos[RR.Reg].RegClass;
if (RC != nullptr && (RR.Mask & RC->LaneMask) == RC->LaneMask)
return !Preserved;
// Otherwise, check all subregisters whose lane mask overlaps the given
// mask. For each such register, if it is preserved by the regmask, then
// clear the corresponding bits in the given mask. If at the end, all
// bits have been cleared, the register does not alias the regmask (i.e.
// is it preserved by it).
LaneBitmask M = RR.Mask;
for (MCSubRegIndexIterator SI(RR.Reg, &TRI); SI.isValid(); ++SI) {
LaneBitmask SM = TRI.getSubRegIndexLaneMask(SI.getSubRegIndex());
if ((SM & RR.Mask).none())
continue;
unsigned SR = SI.getSubReg();
if (!(MB[SR/32] & (1u << (SR%32))))
continue;
// The subregister SR is preserved.
M &= ~SM;
if (M.none())
return false;
}
return true;
}
void RegScavenger::setRegUsed(unsigned Reg, LaneBitmask LaneMask) {
for (MCRegUnitMaskIterator RUI(Reg, TRI); RUI.isValid(); ++RUI) {
LaneBitmask UnitMask = (*RUI).second;
if (UnitMask.none() || (LaneMask & UnitMask).any())
RegUnitsAvailable.reset((*RUI).first);
}
}
void LiveInterval::refineSubRanges(BumpPtrAllocator &Allocator,
LaneBitmask LaneMask, std::function<void(LiveInterval::SubRange&)> Apply) {
LaneBitmask ToApply = LaneMask;
for (SubRange &SR : subranges()) {
LaneBitmask SRMask = SR.LaneMask;
LaneBitmask Matching = SRMask & LaneMask;
if (Matching.none())
continue;
SubRange *MatchingRange;
if (SRMask == Matching) {
// The subrange fits (it does not cover bits outside \p LaneMask).
MatchingRange = &SR;
} else {
// We have to split the subrange into a matching and non-matching part.
// Reduce lanemask of existing lane to non-matching part.
SR.LaneMask = SRMask & ~Matching;
// Create a new subrange for the matching part
MatchingRange = createSubRangeFrom(Allocator, Matching, SR);
}
Apply(*MatchingRange);
ToApply &= ~Matching;
}
// Create a new subrange if there are uncovered bits left.
if (ToApply.any()) {
SubRange *NewRange = createSubRange(Allocator, ToApply);
Apply(*NewRange);
}
}
/// Increase pressure for each pressure set provided by TargetRegisterInfo.
static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure,
const MachineRegisterInfo &MRI, unsigned Reg,
LaneBitmask PrevMask, LaneBitmask NewMask) {
assert((PrevMask & ~NewMask).none() && "Must not remove bits");
if (PrevMask.any() || NewMask.none())
return;
PSetIterator PSetI = MRI.getPressureSets(Reg);
unsigned Weight = PSetI.getWeight();
for (; PSetI.isValid(); ++PSetI)
CurrSetPressure[*PSetI] += Weight;
}
/// Advance across the current instruction.
void RegPressureTracker::advance(const RegisterOperands &RegOpers) {
assert(!TrackUntiedDefs && "unsupported mode");
assert(CurrPos != MBB->end());
if (!isTopClosed())
closeTop();
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = getCurrSlot();
// Open the bottom of the region using slot indexes.
if (isBottomClosed()) {
if (RequireIntervals)
static_cast<IntervalPressure&>(P).openBottom(SlotIdx);
else
static_cast<RegionPressure&>(P).openBottom(CurrPos);
}
for (const RegisterMaskPair &Use : RegOpers.Uses) {
unsigned Reg = Use.RegUnit;
LaneBitmask LiveMask = LiveRegs.contains(Reg);
LaneBitmask LiveIn = Use.LaneMask & ~LiveMask;
if (LiveIn.any()) {
discoverLiveIn(RegisterMaskPair(Reg, LiveIn));
increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn);
LiveRegs.insert(RegisterMaskPair(Reg, LiveIn));
}
// Kill liveness at last uses.
if (RequireIntervals) {
LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
if (LastUseMask.any()) {
LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask));
decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask);
}
}
}
// Generate liveness for defs.
for (const RegisterMaskPair &Def : RegOpers.Defs) {
LaneBitmask PreviousMask = LiveRegs.insert(Def);
LaneBitmask NewMask = PreviousMask | Def.LaneMask;
increaseRegPressure(Def.RegUnit, PreviousMask, NewMask);
}
// Boost pressure for all dead defs together.
bumpDeadDefs(RegOpers.DeadDefs);
// Find the next instruction.
CurrPos = skipDebugInstructionsForward(std::next(CurrPos), MBB->end());
}
/// Decrease pressure for each pressure set provided by TargetRegisterInfo.
static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure,
const MachineRegisterInfo &MRI, unsigned Reg,
LaneBitmask PrevMask, LaneBitmask NewMask) {
//assert((NewMask & !PrevMask) == 0 && "Must not add bits");
if (NewMask.any() || PrevMask.none())
return;
PSetIterator PSetI = MRI.getPressureSets(Reg);
unsigned Weight = PSetI.getWeight();
for (; PSetI.isValid(); ++PSetI) {
assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow");
CurrSetPressure[*PSetI] -= Weight;
}
}
void RegPressureTracker::increaseRegPressure(unsigned RegUnit,
LaneBitmask PreviousMask,
LaneBitmask NewMask) {
if (PreviousMask.any() || NewMask.none())
return;
PSetIterator PSetI = MRI->getPressureSets(RegUnit);
unsigned Weight = PSetI.getWeight();
for (; PSetI.isValid(); ++PSetI) {
CurrSetPressure[*PSetI] += Weight;
P.MaxSetPressure[*PSetI] =
std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]);
}
}
/// Record the downward impact of a single instruction on current register
/// pressure. Unlike the advance/recede pressure tracking interface, this does
/// not discover live in/outs.
///
/// This is intended for speculative queries. It leaves pressure inconsistent
/// with the current position, so must be restored by the caller.
void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
assert(!MI->isDebugValue() && "Expect a nondebug instruction.");
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
// Account for register pressure similar to RegPressureTracker::recede().
RegisterOperands RegOpers;
RegOpers.collect(*MI, *TRI, *MRI, TrackLaneMasks, false);
if (TrackLaneMasks)
RegOpers.adjustLaneLiveness(*LIS, *MRI, SlotIdx);
if (RequireIntervals) {
for (const RegisterMaskPair &Use : RegOpers.Uses) {
unsigned Reg = Use.RegUnit;
LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx);
if (LastUseMask.none())
continue;
// The LastUseMask is queried from the liveness information of instruction
// which may be further down the schedule. Some lanes may actually not be
// last uses for the current position.
// FIXME: allow the caller to pass in the list of vreg uses that remain
// to be bottom-scheduled to avoid searching uses at each query.
SlotIndex CurrIdx = getCurrSlot();
LastUseMask
= findUseBetween(Reg, LastUseMask, CurrIdx, SlotIdx, *MRI, LIS);
if (LastUseMask.none())
continue;
LaneBitmask LiveMask = LiveRegs.contains(Reg);
LaneBitmask NewMask = LiveMask & ~LastUseMask;
decreaseRegPressure(Reg, LiveMask, NewMask);
}
}
// Generate liveness for defs.
for (const RegisterMaskPair &Def : RegOpers.Defs) {
unsigned Reg = Def.RegUnit;
LaneBitmask LiveMask = LiveRegs.contains(Reg);
LaneBitmask NewMask = LiveMask | Def.LaneMask;
increaseRegPressure(Reg, LiveMask, NewMask);
}
// Boost pressure for all dead defs together.
bumpDeadDefs(RegOpers.DeadDefs);
}
void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask Mask,
LiveInterval *LI) {
SmallVector<SlotIndex, 4> Undefs;
if (LI != nullptr)
LI->computeSubRangeUndefs(Undefs, Mask, *MRI, *Indexes);
// Visit all operands that read Reg. This may include partial defs.
bool IsSubRange = !Mask.all();
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
// Clear all kill flags. They will be reinserted after register allocation
// by LiveIntervals::addKillFlags().
if (MO.isUse())
MO.setIsKill(false);
// MO::readsReg returns "true" for subregister defs. This is for keeping
// liveness of the entire register (i.e. for the main range of the live
// interval). For subranges, definitions of non-overlapping subregisters
// do not count as uses.
if (!MO.readsReg() || (IsSubRange && MO.isDef()))
continue;
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
LaneBitmask SLM = TRI.getSubRegIndexLaneMask(SubReg);
if (MO.isDef())
SLM = ~SLM;
// Ignore uses not reading the current (sub)range.
if ((SLM & Mask).none())
continue;
}
// Determine the actual place of the use.
const MachineInstr *MI = MO.getParent();
unsigned OpNo = (&MO - &MI->getOperand(0));
SlotIndex UseIdx;
if (MI->isPHI()) {
assert(!MO.isDef() && "Cannot handle PHI def of partial register.");
// The actual place where a phi operand is used is the end of the pred
// MBB. PHI operands are paired: (Reg, PredMBB).
UseIdx = Indexes->getMBBEndIdx(MI->getOperand(OpNo+1).getMBB());
} else {
// Check for early-clobber redefs.
bool isEarlyClobber = false;
unsigned DefIdx;
if (MO.isDef())
isEarlyClobber = MO.isEarlyClobber();
else if (MI->isRegTiedToDefOperand(OpNo, &DefIdx)) {
// FIXME: This would be a lot easier if tied early-clobber uses also
// had an early-clobber flag.
isEarlyClobber = MI->getOperand(DefIdx).isEarlyClobber();
}
UseIdx = Indexes->getInstructionIndex(*MI).getRegSlot(isEarlyClobber);
}
// MI is reading Reg. We may have visited MI before if it happens to be
// reading Reg multiple times. That is OK, extend() is idempotent.
extend(LR, UseIdx, Reg, Undefs);
}
}
void VirtRegRewriter::addLiveInsForSubRanges(const LiveInterval &LI,
unsigned PhysReg) const {
assert(!LI.empty());
assert(LI.hasSubRanges());
using SubRangeIteratorPair =
std::pair<const LiveInterval::SubRange *, LiveInterval::const_iterator>;
SmallVector<SubRangeIteratorPair, 4> SubRanges;
SlotIndex First;
SlotIndex Last;
for (const LiveInterval::SubRange &SR : LI.subranges()) {
SubRanges.push_back(std::make_pair(&SR, SR.begin()));
if (!First.isValid() || SR.segments.front().start < First)
First = SR.segments.front().start;
if (!Last.isValid() || SR.segments.back().end > Last)
Last = SR.segments.back().end;
}
// Check all mbb start positions between First and Last while
// simulatenously advancing an iterator for each subrange.
for (SlotIndexes::MBBIndexIterator MBBI = Indexes->findMBBIndex(First);
MBBI != Indexes->MBBIndexEnd() && MBBI->first <= Last; ++MBBI) {
SlotIndex MBBBegin = MBBI->first;
// Advance all subrange iterators so that their end position is just
// behind MBBBegin (or the iterator is at the end).
LaneBitmask LaneMask;
for (auto &RangeIterPair : SubRanges) {
const LiveInterval::SubRange *SR = RangeIterPair.first;
LiveInterval::const_iterator &SRI = RangeIterPair.second;
while (SRI != SR->end() && SRI->end <= MBBBegin)
++SRI;
if (SRI == SR->end())
continue;
if (SRI->start <= MBBBegin)
LaneMask |= SR->LaneMask;
}
if (LaneMask.none())
continue;
MachineBasicBlock *MBB = MBBI->second;
MBB->addLiveIn(PhysReg, LaneMask);
}
}
void GCNRegPressure::inc(unsigned Reg,
LaneBitmask PrevMask,
LaneBitmask NewMask,
const MachineRegisterInfo &MRI) {
if (NewMask == PrevMask)
return;
int Sign = 1;
if (NewMask < PrevMask) {
std::swap(NewMask, PrevMask);
Sign = -1;
}
#ifndef NDEBUG
const auto MaxMask = MRI.getMaxLaneMaskForVReg(Reg);
#endif
switch (auto Kind = getRegKind(Reg, MRI)) {
case SGPR32:
case VGPR32:
assert(PrevMask.none() && NewMask == MaxMask);
Value[Kind] += Sign;
break;
case SGPR_TUPLE:
case VGPR_TUPLE:
assert(NewMask < MaxMask || NewMask == MaxMask);
assert(PrevMask < NewMask);
Value[Kind == SGPR_TUPLE ? SGPR32 : VGPR32] +=
Sign * (~PrevMask & NewMask).getNumLanes();
if (PrevMask.none()) {
assert(NewMask.any());
Value[Kind] += Sign * MRI.getPressureSets(Reg).getWeight();
}
break;
default: llvm_unreachable("Unknown register kind");
}
}
/// Helper to find a vreg use between two indices [PriorUseIdx, NextUseIdx).
/// The query starts with a lane bitmask which gets lanes/bits removed for every
/// use we find.
static LaneBitmask findUseBetween(unsigned Reg, LaneBitmask LastUseMask,
SlotIndex PriorUseIdx, SlotIndex NextUseIdx,
const MachineRegisterInfo &MRI,
const LiveIntervals *LIS) {
const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
for (const MachineOperand &MO : MRI.use_nodbg_operands(Reg)) {
if (MO.isUndef())
continue;
const MachineInstr *MI = MO.getParent();
SlotIndex InstSlot = LIS->getInstructionIndex(*MI).getRegSlot();
if (InstSlot >= PriorUseIdx && InstSlot < NextUseIdx) {
unsigned SubRegIdx = MO.getSubReg();
LaneBitmask UseMask = TRI.getSubRegIndexLaneMask(SubRegIdx);
LastUseMask &= ~UseMask;
if (LastUseMask.none())
return LaneBitmask::getNone();
}
}
return LastUseMask;
}
void RegisterOperands::adjustLaneLiveness(const LiveIntervals &LIS,
const MachineRegisterInfo &MRI,
SlotIndex Pos,
MachineInstr *AddFlagsMI) {
for (auto I = Defs.begin(); I != Defs.end(); ) {
LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
Pos.getDeadSlot());
// If the the def is all that is live after the instruction, then in case
// of a subregister def we need a read-undef flag.
unsigned RegUnit = I->RegUnit;
if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
AddFlagsMI != nullptr && (LiveAfter & ~I->LaneMask).none())
AddFlagsMI->setRegisterDefReadUndef(RegUnit);
LaneBitmask ActualDef = I->LaneMask & LiveAfter;
if (ActualDef.none()) {
I = Defs.erase(I);
} else {
I->LaneMask = ActualDef;
++I;
}
}
for (auto I = Uses.begin(); I != Uses.end(); ) {
LaneBitmask LiveBefore = getLiveLanesAt(LIS, MRI, true, I->RegUnit,
Pos.getBaseIndex());
LaneBitmask LaneMask = I->LaneMask & LiveBefore;
if (LaneMask.none()) {
I = Uses.erase(I);
} else {
I->LaneMask = LaneMask;
++I;
}
}
if (AddFlagsMI != nullptr) {
for (const RegisterMaskPair &P : DeadDefs) {
unsigned RegUnit = P.RegUnit;
if (!TargetRegisterInfo::isVirtualRegister(RegUnit))
continue;
LaneBitmask LiveAfter = getLiveLanesAt(LIS, MRI, true, RegUnit,
Pos.getDeadSlot());
if (LiveAfter.none())
AddFlagsMI->setRegisterDefReadUndef(RegUnit);
}
}
}
/// Recede across the previous instruction. If LiveUses is provided, record any
/// RegUnits that are made live by the current instruction's uses. This includes
/// registers that are both defined and used by the instruction. If a pressure
/// difference pointer is provided record the changes is pressure caused by this
/// instruction independent of liveness.
void RegPressureTracker::recede(const RegisterOperands &RegOpers,
SmallVectorImpl<RegisterMaskPair> *LiveUses) {
assert(!CurrPos->isDebugValue());
// Boost pressure for all dead defs together.
bumpDeadDefs(RegOpers.DeadDefs);
// Kill liveness at live defs.
// TODO: consider earlyclobbers?
for (const RegisterMaskPair &Def : RegOpers.Defs) {
unsigned Reg = Def.RegUnit;
LaneBitmask PreviousMask = LiveRegs.erase(Def);
LaneBitmask NewMask = PreviousMask & ~Def.LaneMask;
LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask;
if (LiveOut.any()) {
discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
// Retroactively model effects on pressure of the live out lanes.
increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(),
LiveOut);
PreviousMask = LiveOut;
}
if (NewMask.none()) {
// Add a 0 entry to LiveUses as a marker that the complete vreg has become
// dead.
if (TrackLaneMasks && LiveUses != nullptr)
setRegZero(*LiveUses, Reg);
}
decreaseRegPressure(Reg, PreviousMask, NewMask);
}
SlotIndex SlotIdx;
if (RequireIntervals)
SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot();
// Generate liveness for uses.
for (const RegisterMaskPair &Use : RegOpers.Uses) {
unsigned Reg = Use.RegUnit;
assert(Use.LaneMask.any());
LaneBitmask PreviousMask = LiveRegs.insert(Use);
LaneBitmask NewMask = PreviousMask | Use.LaneMask;
if (NewMask == PreviousMask)
continue;
// Did the register just become live?
if (PreviousMask.none()) {
if (LiveUses != nullptr) {
if (!TrackLaneMasks) {
addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
} else {
auto I = find_if(*LiveUses, [Reg](const RegisterMaskPair Other) {
return Other.RegUnit == Reg;
});
bool IsRedef = I != LiveUses->end();
if (IsRedef) {
// ignore re-defs here...
assert(I->LaneMask.none());
removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
} else {
addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask));
}
}
}
// Discover live outs if this may be the first occurance of this register.
if (RequireIntervals) {
LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx);
if (LiveOut.any())
discoverLiveOut(RegisterMaskPair(Reg, LiveOut));
}
}
increaseRegPressure(Reg, PreviousMask, NewMask);
}
if (TrackUntiedDefs) {
for (const RegisterMaskPair &Def : RegOpers.Defs) {
unsigned RegUnit = Def.RegUnit;
if (TargetRegisterInfo::isVirtualRegister(RegUnit) &&
(LiveRegs.contains(RegUnit) & Def.LaneMask).none())
UntiedDefs.insert(RegUnit);
}
}
}
void HexagonExpandCondsets::updateDeadsInRange(unsigned Reg, LaneBitmask LM,
LiveRange &Range) {
assert(TargetRegisterInfo::isVirtualRegister(Reg));
if (Range.empty())
return;
// Return two booleans: { def-modifes-reg, def-covers-reg }.
auto IsRegDef = [this,Reg,LM] (MachineOperand &Op) -> std::pair<bool,bool> {
if (!Op.isReg() || !Op.isDef())
return { false, false };
unsigned DR = Op.getReg(), DSR = Op.getSubReg();
if (!TargetRegisterInfo::isVirtualRegister(DR) || DR != Reg)
return { false, false };
LaneBitmask SLM = getLaneMask(DR, DSR);
LaneBitmask A = SLM & LM;
return { A.any(), A == SLM };
};
// The splitting step will create pairs of predicated definitions without
// any implicit uses (since implicit uses would interfere with predication).
// This can cause the reaching defs to become dead after live range
// recomputation, even though they are not really dead.
// We need to identify predicated defs that need implicit uses, and
// dead defs that are not really dead, and correct both problems.
auto Dominate = [this] (SetVector<MachineBasicBlock*> &Defs,
MachineBasicBlock *Dest) -> bool {
for (MachineBasicBlock *D : Defs)
if (D != Dest && MDT->dominates(D, Dest))
return true;
MachineBasicBlock *Entry = &Dest->getParent()->front();
SetVector<MachineBasicBlock*> Work(Dest->pred_begin(), Dest->pred_end());
for (unsigned i = 0; i < Work.size(); ++i) {
MachineBasicBlock *B = Work[i];
if (Defs.count(B))
continue;
if (B == Entry)
return false;
for (auto *P : B->predecessors())
Work.insert(P);
}
return true;
};
// First, try to extend live range within individual basic blocks. This
// will leave us only with dead defs that do not reach any predicated
// defs in the same block.
SetVector<MachineBasicBlock*> Defs;
SmallVector<SlotIndex,4> PredDefs;
for (auto &Seg : Range) {
if (!Seg.start.isRegister())
continue;
MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start);
Defs.insert(DefI->getParent());
if (HII->isPredicated(*DefI))
PredDefs.push_back(Seg.start);
}
SmallVector<SlotIndex,8> Undefs;
LiveInterval &LI = LIS->getInterval(Reg);
LI.computeSubRangeUndefs(Undefs, LM, *MRI, *LIS->getSlotIndexes());
for (auto &SI : PredDefs) {
MachineBasicBlock *BB = LIS->getMBBFromIndex(SI);
auto P = Range.extendInBlock(Undefs, LIS->getMBBStartIdx(BB), SI);
if (P.first != nullptr || P.second)
SI = SlotIndex();
}
// Calculate reachability for those predicated defs that were not handled
// by the in-block extension.
SmallVector<SlotIndex,4> ExtTo;
for (auto &SI : PredDefs) {
if (!SI.isValid())
continue;
MachineBasicBlock *BB = LIS->getMBBFromIndex(SI);
if (BB->pred_empty())
continue;
// If the defs from this range reach SI via all predecessors, it is live.
// It can happen that SI is reached by the defs through some paths, but
// not all. In the IR coming into this optimization, SI would not be
// considered live, since the defs would then not jointly dominate SI.
// That means that SI is an overwriting def, and no implicit use is
// needed at this point. Do not add SI to the extension points, since
// extendToIndices will abort if there is no joint dominance.
// If the abort was avoided by adding extra undefs added to Undefs,
// extendToIndices could actually indicate that SI is live, contrary
// to the original IR.
if (Dominate(Defs, BB))
ExtTo.push_back(SI);
}
if (!ExtTo.empty())
LIS->extendToIndices(Range, ExtTo, Undefs);
// Remove <dead> flags from all defs that are not dead after live range
// extension, and collect all def operands. They will be used to generate
// the necessary implicit uses.
// At the same time, add <dead> flag to all defs that are actually dead.
// This can happen, for example, when a mux with identical inputs is
// replaced with a COPY: the use of the predicate register disappears and
// the dead can become dead.
std::set<RegisterRef> DefRegs;
for (auto &Seg : Range) {
if (!Seg.start.isRegister())
continue;
MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start);
for (auto &Op : DefI->operands()) {
auto P = IsRegDef(Op);
if (P.second && Seg.end.isDead()) {
Op.setIsDead(true);
} else if (P.first) {
DefRegs.insert(Op);
Op.setIsDead(false);
}
}
}
// Now, add implicit uses to each predicated def that is reached
// by other defs.
for (auto &Seg : Range) {
if (!Seg.start.isRegister() || !Range.liveAt(Seg.start.getPrevSlot()))
continue;
MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start);
if (!HII->isPredicated(*DefI))
continue;
// Construct the set of all necessary implicit uses, based on the def
// operands in the instruction.
std::set<RegisterRef> ImpUses;
for (auto &Op : DefI->operands())
if (Op.isReg() && Op.isDef() && DefRegs.count(Op))
ImpUses.insert(Op);
if (ImpUses.empty())
continue;
MachineFunction &MF = *DefI->getParent()->getParent();
for (RegisterRef R : ImpUses)
MachineInstrBuilder(MF, DefI).addReg(R.Reg, RegState::Implicit, R.Sub);
}
}
void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) {
assert(MRI && Indexes && "call reset() first");
// Step 1: Create minimal live segments for every definition of Reg.
// Visit all def operands. If the same instruction has multiple defs of Reg,
// createDeadDef() will deduplicate.
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
unsigned Reg = LI.reg;
for (const MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
if (!MO.isDef() && !MO.readsReg())
continue;
unsigned SubReg = MO.getSubReg();
if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) {
LaneBitmask SubMask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg)
: MRI->getMaxLaneMaskForVReg(Reg);
// If this is the first time we see a subregister def, initialize
// subranges by creating a copy of the main range.
if (!LI.hasSubRanges() && !LI.empty()) {
LaneBitmask ClassMask = MRI->getMaxLaneMaskForVReg(Reg);
LI.createSubRangeFrom(*Alloc, ClassMask, LI);
}
LaneBitmask Mask = SubMask;
for (LiveInterval::SubRange &S : LI.subranges()) {
// A Mask for subregs common to the existing subrange and current def.
LaneBitmask Common = S.LaneMask & Mask;
if (Common.none())
continue;
LiveInterval::SubRange *CommonRange;
// A Mask for subregs covered by the subrange but not the current def.
LaneBitmask RM = S.LaneMask & ~Mask;
if (RM.any()) {
// Split the subrange S into two parts: one covered by the current
// def (CommonRange), and the one not affected by it (updated S).
S.LaneMask = RM;
CommonRange = LI.createSubRangeFrom(*Alloc, Common, S);
} else {
assert(Common == S.LaneMask);
CommonRange = &S;
}
if (MO.isDef())
createDeadDef(*Indexes, *Alloc, *CommonRange, MO);
Mask &= ~Common;
}
// Create a new SubRange for subregs we did not cover yet.
if (Mask.any()) {
LiveInterval::SubRange *NewRange = LI.createSubRange(*Alloc, Mask);
if (MO.isDef())
createDeadDef(*Indexes, *Alloc, *NewRange, MO);
}
}
// Create the def in the main liverange. We do not have to do this if
// subranges are tracked as we recreate the main range later in this case.
if (MO.isDef() && !LI.hasSubRanges())
createDeadDef(*Indexes, *Alloc, LI, MO);
}
// We may have created empty live ranges for partially undefined uses, we
// can't keep them because we won't find defs in them later.
LI.removeEmptySubRanges();
// Step 2: Extend live segments to all uses, constructing SSA form as
// necessary.
if (LI.hasSubRanges()) {
for (LiveInterval::SubRange &S : LI.subranges()) {
LiveRangeCalc SubLRC;
SubLRC.reset(MF, Indexes, DomTree, Alloc);
SubLRC.extendToUses(S, Reg, S.LaneMask, &LI);
}
LI.clear();
constructMainRangeFromSubranges(LI);
} else {
resetLiveOutMap();
extendToUses(LI, Reg, LaneBitmask::getAll());
}
}
void Liveness::computePhiInfo() {
RealUseMap.clear();
NodeList Phis;
NodeAddr<FuncNode*> FA = DFG.getFunc();
NodeList Blocks = FA.Addr->members(DFG);
for (NodeAddr<BlockNode*> BA : Blocks) {
auto Ps = BA.Addr->members_if(DFG.IsCode<NodeAttrs::Phi>, DFG);
Phis.insert(Phis.end(), Ps.begin(), Ps.end());
}
// phi use -> (map: reaching phi -> set of registers defined in between)
std::map<NodeId,std::map<NodeId,RegisterAggr>> PhiUp;
std::vector<NodeId> PhiUQ; // Work list of phis for upward propagation.
std::map<NodeId,RegisterAggr> PhiDRs; // Phi -> registers defined by it.
// Go over all phis.
for (NodeAddr<PhiNode*> PhiA : Phis) {
// Go over all defs and collect the reached uses that are non-phi uses
// (i.e. the "real uses").
RefMap &RealUses = RealUseMap[PhiA.Id];
NodeList PhiRefs = PhiA.Addr->members(DFG);
// Have a work queue of defs whose reached uses need to be found.
// For each def, add to the queue all reached (non-phi) defs.
SetVector<NodeId> DefQ;
NodeSet PhiDefs;
RegisterAggr DRs(PRI);
for (NodeAddr<RefNode*> R : PhiRefs) {
if (!DFG.IsRef<NodeAttrs::Def>(R))
continue;
DRs.insert(R.Addr->getRegRef(DFG));
DefQ.insert(R.Id);
PhiDefs.insert(R.Id);
}
PhiDRs.insert(std::make_pair(PhiA.Id, DRs));
// Collect the super-set of all possible reached uses. This set will
// contain all uses reached from this phi, either directly from the
// phi defs, or (recursively) via non-phi defs reached by the phi defs.
// This set of uses will later be trimmed to only contain these uses that
// are actually reached by the phi defs.
for (unsigned i = 0; i < DefQ.size(); ++i) {
NodeAddr<DefNode*> DA = DFG.addr<DefNode*>(DefQ[i]);
// Visit all reached uses. Phi defs should not really have the "dead"
// flag set, but check it anyway for consistency.
bool IsDead = DA.Addr->getFlags() & NodeAttrs::Dead;
NodeId UN = !IsDead ? DA.Addr->getReachedUse() : 0;
while (UN != 0) {
NodeAddr<UseNode*> A = DFG.addr<UseNode*>(UN);
uint16_t F = A.Addr->getFlags();
if ((F & (NodeAttrs::Undef | NodeAttrs::PhiRef)) == 0) {
RegisterRef R = PRI.normalize(A.Addr->getRegRef(DFG));
RealUses[R.Reg].insert({A.Id,R.Mask});
}
UN = A.Addr->getSibling();
}
// Visit all reached defs, and add them to the queue. These defs may
// override some of the uses collected here, but that will be handled
// later.
NodeId DN = DA.Addr->getReachedDef();
while (DN != 0) {
NodeAddr<DefNode*> A = DFG.addr<DefNode*>(DN);
for (auto T : DFG.getRelatedRefs(A.Addr->getOwner(DFG), A)) {
uint16_t Flags = NodeAddr<DefNode*>(T).Addr->getFlags();
// Must traverse the reached-def chain. Consider:
// def(D0) -> def(R0) -> def(R0) -> use(D0)
// The reachable use of D0 passes through a def of R0.
if (!(Flags & NodeAttrs::PhiRef))
DefQ.insert(T.Id);
}
DN = A.Addr->getSibling();
}
}
// Filter out these uses that appear to be reachable, but really
// are not. For example:
//
// R1:0 = d1
// = R1:0 u2 Reached by d1.
// R0 = d3
// = R1:0 u4 Still reached by d1: indirectly through
// the def d3.
// R1 = d5
// = R1:0 u6 Not reached by d1 (covered collectively
// by d3 and d5), but following reached
// defs and uses from d1 will lead here.
for (auto UI = RealUses.begin(), UE = RealUses.end(); UI != UE; ) {
// For each reached register UI->first, there is a set UI->second, of
// uses of it. For each such use, check if it is reached by this phi,
// i.e. check if the set of its reaching uses intersects the set of
// this phi's defs.
NodeRefSet Uses = UI->second;
UI->second.clear();
for (std::pair<NodeId,LaneBitmask> I : Uses) {
auto UA = DFG.addr<UseNode*>(I.first);
// Undef flag is checked above.
assert((UA.Addr->getFlags() & NodeAttrs::Undef) == 0);
RegisterRef R(UI->first, I.second);
// Calculate the exposed part of the reached use.
RegisterAggr Covered(PRI);
for (NodeAddr<DefNode*> DA : getAllReachingDefs(R, UA)) {
if (PhiDefs.count(DA.Id))
break;
Covered.insert(DA.Addr->getRegRef(DFG));
}
if (RegisterRef RC = Covered.clearIn(R)) {
// We are updating the map for register UI->first, so we need
// to map RC to be expressed in terms of that register.
RegisterRef S = PRI.mapTo(RC, UI->first);
UI->second.insert({I.first, S.Mask});
}
}
UI = UI->second.empty() ? RealUses.erase(UI) : std::next(UI);
}
// If this phi reaches some "real" uses, add it to the queue for upward
// propagation.
if (!RealUses.empty())
PhiUQ.push_back(PhiA.Id);
// Go over all phi uses and check if the reaching def is another phi.
// Collect the phis that are among the reaching defs of these uses.
// While traversing the list of reaching defs for each phi use, accumulate
// the set of registers defined between this phi (PhiA) and the owner phi
// of the reaching def.
NodeSet SeenUses;
for (auto I : PhiRefs) {
if (!DFG.IsRef<NodeAttrs::Use>(I) || SeenUses.count(I.Id))
continue;
NodeAddr<PhiUseNode*> PUA = I;
if (PUA.Addr->getReachingDef() == 0)
continue;
RegisterRef UR = PUA.Addr->getRegRef(DFG);
NodeList Ds = getAllReachingDefs(UR, PUA, true, false, NoRegs);
RegisterAggr DefRRs(PRI);
for (NodeAddr<DefNode*> D : Ds) {
if (D.Addr->getFlags() & NodeAttrs::PhiRef) {
NodeId RP = D.Addr->getOwner(DFG).Id;
std::map<NodeId,RegisterAggr> &M = PhiUp[PUA.Id];
auto F = M.find(RP);
if (F == M.end())
M.insert(std::make_pair(RP, DefRRs));
else
F->second.insert(DefRRs);
}
DefRRs.insert(D.Addr->getRegRef(DFG));
}
for (NodeAddr<PhiUseNode*> T : DFG.getRelatedRefs(PhiA, PUA))
SeenUses.insert(T.Id);
}
}
if (Trace) {
dbgs() << "Phi-up-to-phi map with intervening defs:\n";
for (auto I : PhiUp) {
dbgs() << "phi " << Print<NodeId>(I.first, DFG) << " -> {";
for (auto R : I.second)
dbgs() << ' ' << Print<NodeId>(R.first, DFG)
<< Print<RegisterAggr>(R.second, DFG);
dbgs() << " }\n";
}
}
// Propagate the reached registers up in the phi chain.
//
// The following type of situation needs careful handling:
//
// phi d1<R1:0> (1)
// |
// ... d2<R1>
// |
// phi u3<R1:0> (2)
// |
// ... u4<R1>
//
// The phi node (2) defines a register pair R1:0, and reaches a "real"
// use u4 of just R1. The same phi node is also known to reach (upwards)
// the phi node (1). However, the use u4 is not reached by phi (1),
// because of the intervening definition d2 of R1. The data flow between
// phis (1) and (2) is restricted to R1:0 minus R1, i.e. R0.
//
// When propagating uses up the phi chains, get the all reaching defs
// for a given phi use, and traverse the list until the propagated ref
// is covered, or until reaching the final phi. Only assume that the
// reference reaches the phi in the latter case.
for (unsigned i = 0; i < PhiUQ.size(); ++i) {
auto PA = DFG.addr<PhiNode*>(PhiUQ[i]);
NodeList PUs = PA.Addr->members_if(DFG.IsRef<NodeAttrs::Use>, DFG);
RefMap &RUM = RealUseMap[PA.Id];
for (NodeAddr<UseNode*> UA : PUs) {
std::map<NodeId,RegisterAggr> &PUM = PhiUp[UA.Id];
RegisterRef UR = PRI.normalize(UA.Addr->getRegRef(DFG));
for (const std::pair<NodeId,RegisterAggr> &P : PUM) {
bool Changed = false;
const RegisterAggr &MidDefs = P.second;
// Collect the set PropUp of uses that are reached by the current
// phi PA, and are not covered by any intervening def between the
// currently visited use UA and the the upward phi P.
if (MidDefs.hasCoverOf(UR))
continue;
// General algorithm:
// for each (R,U) : U is use node of R, U is reached by PA
// if MidDefs does not cover (R,U)
// then add (R-MidDefs,U) to RealUseMap[P]
//
for (const std::pair<RegisterId,NodeRefSet> &T : RUM) {
RegisterRef R(T.first);
// The current phi (PA) could be a phi for a regmask. It could
// reach a whole variety of uses that are not related to the
// specific upward phi (P.first).
const RegisterAggr &DRs = PhiDRs.at(P.first);
if (!DRs.hasAliasOf(R))
continue;
R = PRI.mapTo(DRs.intersectWith(R), T.first);
for (std::pair<NodeId,LaneBitmask> V : T.second) {
LaneBitmask M = R.Mask & V.second;
if (M.none())
continue;
if (RegisterRef SS = MidDefs.clearIn(RegisterRef(R.Reg, M))) {
NodeRefSet &RS = RealUseMap[P.first][SS.Reg];
Changed |= RS.insert({V.first,SS.Mask}).second;
}
}
}
if (Changed)
PhiUQ.push_back(P.first);
}
}
}
if (Trace) {
dbgs() << "Real use map:\n";
for (auto I : RealUseMap) {
dbgs() << "phi " << Print<NodeId>(I.first, DFG);
NodeAddr<PhiNode*> PA = DFG.addr<PhiNode*>(I.first);
NodeList Ds = PA.Addr->members_if(DFG.IsRef<NodeAttrs::Def>, DFG);
if (!Ds.empty()) {
RegisterRef RR = NodeAddr<DefNode*>(Ds[0]).Addr->getRegRef(DFG);
dbgs() << '<' << Print<RegisterRef>(RR, DFG) << '>';
} else {
dbgs() << "<noreg>";
}
dbgs() << " -> " << Print<RefMap>(I.second, DFG) << '\n';
}
}
}
