/// Advance across the current instruction. void RegPressureTracker::advance(const RegisterOperands &RegOpers) { assert(!TrackUntiedDefs && "unsupported mode"); assert(CurrPos != MBB->end()); if (!isTopClosed()) closeTop(); SlotIndex SlotIdx; if (RequireIntervals) SlotIdx = getCurrSlot(); // Open the bottom of the region using slot indexes. if (isBottomClosed()) { if (RequireIntervals) static_cast<IntervalPressure&>(P).openBottom(SlotIdx); else static_cast<RegionPressure&>(P).openBottom(CurrPos); } for (const RegisterMaskPair &Use : RegOpers.Uses) { unsigned Reg = Use.RegUnit; LaneBitmask LiveMask = LiveRegs.contains(Reg); LaneBitmask LiveIn = Use.LaneMask & ~LiveMask; if (LiveIn.any()) { discoverLiveIn(RegisterMaskPair(Reg, LiveIn)); increaseRegPressure(Reg, LiveMask, LiveMask | LiveIn); LiveRegs.insert(RegisterMaskPair(Reg, LiveIn)); } // Kill liveness at last uses. if (RequireIntervals) { LaneBitmask LastUseMask = getLastUsedLanes(Reg, SlotIdx); if (LastUseMask.any()) { LiveRegs.erase(RegisterMaskPair(Reg, LastUseMask)); decreaseRegPressure(Reg, LiveMask, LiveMask & ~LastUseMask); } } } // Generate liveness for defs. for (const RegisterMaskPair &Def : RegOpers.Defs) { LaneBitmask PreviousMask = LiveRegs.insert(Def); LaneBitmask NewMask = PreviousMask | Def.LaneMask; increaseRegPressure(Def.RegUnit, PreviousMask, NewMask); } // Boost pressure for all dead defs together. bumpDeadDefs(RegOpers.DeadDefs); // Find the next instruction. CurrPos = skipDebugInstructionsForward(std::next(CurrPos), MBB->end()); }
void LiveInterval::refineSubRanges(BumpPtrAllocator &Allocator, LaneBitmask LaneMask, std::function<void(LiveInterval::SubRange&)> Apply) { LaneBitmask ToApply = LaneMask; for (SubRange &SR : subranges()) { LaneBitmask SRMask = SR.LaneMask; LaneBitmask Matching = SRMask & LaneMask; if (Matching.none()) continue; SubRange *MatchingRange; if (SRMask == Matching) { // The subrange fits (it does not cover bits outside \p LaneMask). MatchingRange = &SR; } else { // We have to split the subrange into a matching and non-matching part. // Reduce lanemask of existing lane to non-matching part. SR.LaneMask = SRMask & ~Matching; // Create a new subrange for the matching part MatchingRange = createSubRangeFrom(Allocator, Matching, SR); } Apply(*MatchingRange); ToApply &= ~Matching; } // Create a new subrange if there are uncovered bits left. if (ToApply.any()) { SubRange *NewRange = createSubRange(Allocator, ToApply); Apply(*NewRange); } }
/// Increase pressure for each pressure set provided by TargetRegisterInfo. static void increaseSetPressure(std::vector<unsigned> &CurrSetPressure, const MachineRegisterInfo &MRI, unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask) { assert((PrevMask & ~NewMask).none() && "Must not remove bits"); if (PrevMask.any() || NewMask.none()) return; PSetIterator PSetI = MRI.getPressureSets(Reg); unsigned Weight = PSetI.getWeight(); for (; PSetI.isValid(); ++PSetI) CurrSetPressure[*PSetI] += Weight; }
/// Decrease pressure for each pressure set provided by TargetRegisterInfo. static void decreaseSetPressure(std::vector<unsigned> &CurrSetPressure, const MachineRegisterInfo &MRI, unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask) { //assert((NewMask & !PrevMask) == 0 && "Must not add bits"); if (NewMask.any() || PrevMask.none()) return; PSetIterator PSetI = MRI.getPressureSets(Reg); unsigned Weight = PSetI.getWeight(); for (; PSetI.isValid(); ++PSetI) { assert(CurrSetPressure[*PSetI] >= Weight && "register pressure underflow"); CurrSetPressure[*PSetI] -= Weight; } }
void RegPressureTracker::increaseRegPressure(unsigned RegUnit, LaneBitmask PreviousMask, LaneBitmask NewMask) { if (PreviousMask.any() || NewMask.none()) return; PSetIterator PSetI = MRI->getPressureSets(RegUnit); unsigned Weight = PSetI.getWeight(); for (; PSetI.isValid(); ++PSetI) { CurrSetPressure[*PSetI] += Weight; P.MaxSetPressure[*PSetI] = std::max(P.MaxSetPressure[*PSetI], CurrSetPressure[*PSetI]); } }
void GCNRegPressure::inc(unsigned Reg, LaneBitmask PrevMask, LaneBitmask NewMask, const MachineRegisterInfo &MRI) { if (NewMask == PrevMask) return; int Sign = 1; if (NewMask < PrevMask) { std::swap(NewMask, PrevMask); Sign = -1; } #ifndef NDEBUG const auto MaxMask = MRI.getMaxLaneMaskForVReg(Reg); #endif switch (auto Kind = getRegKind(Reg, MRI)) { case SGPR32: case VGPR32: assert(PrevMask.none() && NewMask == MaxMask); Value[Kind] += Sign; break; case SGPR_TUPLE: case VGPR_TUPLE: assert(NewMask < MaxMask || NewMask == MaxMask); assert(PrevMask < NewMask); Value[Kind == SGPR_TUPLE ? SGPR32 : VGPR32] += Sign * (~PrevMask & NewMask).getNumLanes(); if (PrevMask.none()) { assert(NewMask.any()); Value[Kind] += Sign * MRI.getPressureSets(Reg).getWeight(); } break; default: llvm_unreachable("Unknown register kind"); } }
void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) { assert(MRI && Indexes && "call reset() first"); // Step 1: Create minimal live segments for every definition of Reg. // Visit all def operands. If the same instruction has multiple defs of Reg, // createDeadDef() will deduplicate. const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo(); unsigned Reg = LI.reg; for (const MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) { if (!MO.isDef() && !MO.readsReg()) continue; unsigned SubReg = MO.getSubReg(); if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) { LaneBitmask SubMask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg) : MRI->getMaxLaneMaskForVReg(Reg); // If this is the first time we see a subregister def, initialize // subranges by creating a copy of the main range. if (!LI.hasSubRanges() && !LI.empty()) { LaneBitmask ClassMask = MRI->getMaxLaneMaskForVReg(Reg); LI.createSubRangeFrom(*Alloc, ClassMask, LI); } LaneBitmask Mask = SubMask; for (LiveInterval::SubRange &S : LI.subranges()) { // A Mask for subregs common to the existing subrange and current def. LaneBitmask Common = S.LaneMask & Mask; if (Common.none()) continue; LiveInterval::SubRange *CommonRange; // A Mask for subregs covered by the subrange but not the current def. LaneBitmask RM = S.LaneMask & ~Mask; if (RM.any()) { // Split the subrange S into two parts: one covered by the current // def (CommonRange), and the one not affected by it (updated S). S.LaneMask = RM; CommonRange = LI.createSubRangeFrom(*Alloc, Common, S); } else { assert(Common == S.LaneMask); CommonRange = &S; } if (MO.isDef()) createDeadDef(*Indexes, *Alloc, *CommonRange, MO); Mask &= ~Common; } // Create a new SubRange for subregs we did not cover yet. if (Mask.any()) { LiveInterval::SubRange *NewRange = LI.createSubRange(*Alloc, Mask); if (MO.isDef()) createDeadDef(*Indexes, *Alloc, *NewRange, MO); } } // Create the def in the main liverange. We do not have to do this if // subranges are tracked as we recreate the main range later in this case. if (MO.isDef() && !LI.hasSubRanges()) createDeadDef(*Indexes, *Alloc, LI, MO); } // We may have created empty live ranges for partially undefined uses, we // can't keep them because we won't find defs in them later. LI.removeEmptySubRanges(); // Step 2: Extend live segments to all uses, constructing SSA form as // necessary. if (LI.hasSubRanges()) { for (LiveInterval::SubRange &S : LI.subranges()) { LiveRangeCalc SubLRC; SubLRC.reset(MF, Indexes, DomTree, Alloc); SubLRC.extendToUses(S, Reg, S.LaneMask, &LI); } LI.clear(); constructMainRangeFromSubranges(LI); } else { resetLiveOutMap(); extendToUses(LI, Reg, LaneBitmask::getAll()); } }
void HexagonExpandCondsets::updateDeadsInRange(unsigned Reg, LaneBitmask LM, LiveRange &Range) { assert(TargetRegisterInfo::isVirtualRegister(Reg)); if (Range.empty()) return; // Return two booleans: { def-modifes-reg, def-covers-reg }. auto IsRegDef = [this,Reg,LM] (MachineOperand &Op) -> std::pair<bool,bool> { if (!Op.isReg() || !Op.isDef()) return { false, false }; unsigned DR = Op.getReg(), DSR = Op.getSubReg(); if (!TargetRegisterInfo::isVirtualRegister(DR) || DR != Reg) return { false, false }; LaneBitmask SLM = getLaneMask(DR, DSR); LaneBitmask A = SLM & LM; return { A.any(), A == SLM }; }; // The splitting step will create pairs of predicated definitions without // any implicit uses (since implicit uses would interfere with predication). // This can cause the reaching defs to become dead after live range // recomputation, even though they are not really dead. // We need to identify predicated defs that need implicit uses, and // dead defs that are not really dead, and correct both problems. auto Dominate = [this] (SetVector<MachineBasicBlock*> &Defs, MachineBasicBlock *Dest) -> bool { for (MachineBasicBlock *D : Defs) if (D != Dest && MDT->dominates(D, Dest)) return true; MachineBasicBlock *Entry = &Dest->getParent()->front(); SetVector<MachineBasicBlock*> Work(Dest->pred_begin(), Dest->pred_end()); for (unsigned i = 0; i < Work.size(); ++i) { MachineBasicBlock *B = Work[i]; if (Defs.count(B)) continue; if (B == Entry) return false; for (auto *P : B->predecessors()) Work.insert(P); } return true; }; // First, try to extend live range within individual basic blocks. This // will leave us only with dead defs that do not reach any predicated // defs in the same block. SetVector<MachineBasicBlock*> Defs; SmallVector<SlotIndex,4> PredDefs; for (auto &Seg : Range) { if (!Seg.start.isRegister()) continue; MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start); Defs.insert(DefI->getParent()); if (HII->isPredicated(*DefI)) PredDefs.push_back(Seg.start); } SmallVector<SlotIndex,8> Undefs; LiveInterval &LI = LIS->getInterval(Reg); LI.computeSubRangeUndefs(Undefs, LM, *MRI, *LIS->getSlotIndexes()); for (auto &SI : PredDefs) { MachineBasicBlock *BB = LIS->getMBBFromIndex(SI); auto P = Range.extendInBlock(Undefs, LIS->getMBBStartIdx(BB), SI); if (P.first != nullptr || P.second) SI = SlotIndex(); } // Calculate reachability for those predicated defs that were not handled // by the in-block extension. SmallVector<SlotIndex,4> ExtTo; for (auto &SI : PredDefs) { if (!SI.isValid()) continue; MachineBasicBlock *BB = LIS->getMBBFromIndex(SI); if (BB->pred_empty()) continue; // If the defs from this range reach SI via all predecessors, it is live. // It can happen that SI is reached by the defs through some paths, but // not all. In the IR coming into this optimization, SI would not be // considered live, since the defs would then not jointly dominate SI. // That means that SI is an overwriting def, and no implicit use is // needed at this point. Do not add SI to the extension points, since // extendToIndices will abort if there is no joint dominance. // If the abort was avoided by adding extra undefs added to Undefs, // extendToIndices could actually indicate that SI is live, contrary // to the original IR. if (Dominate(Defs, BB)) ExtTo.push_back(SI); } if (!ExtTo.empty()) LIS->extendToIndices(Range, ExtTo, Undefs); // Remove <dead> flags from all defs that are not dead after live range // extension, and collect all def operands. They will be used to generate // the necessary implicit uses. // At the same time, add <dead> flag to all defs that are actually dead. // This can happen, for example, when a mux with identical inputs is // replaced with a COPY: the use of the predicate register disappears and // the dead can become dead. std::set<RegisterRef> DefRegs; for (auto &Seg : Range) { if (!Seg.start.isRegister()) continue; MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start); for (auto &Op : DefI->operands()) { auto P = IsRegDef(Op); if (P.second && Seg.end.isDead()) { Op.setIsDead(true); } else if (P.first) { DefRegs.insert(Op); Op.setIsDead(false); } } } // Now, add implicit uses to each predicated def that is reached // by other defs. for (auto &Seg : Range) { if (!Seg.start.isRegister() || !Range.liveAt(Seg.start.getPrevSlot())) continue; MachineInstr *DefI = LIS->getInstructionFromIndex(Seg.start); if (!HII->isPredicated(*DefI)) continue; // Construct the set of all necessary implicit uses, based on the def // operands in the instruction. std::set<RegisterRef> ImpUses; for (auto &Op : DefI->operands()) if (Op.isReg() && Op.isDef() && DefRegs.count(Op)) ImpUses.insert(Op); if (ImpUses.empty()) continue; MachineFunction &MF = *DefI->getParent()->getParent(); for (RegisterRef R : ImpUses) MachineInstrBuilder(MF, DefI).addReg(R.Reg, RegState::Implicit, R.Sub); } }
/// Recede across the previous instruction. If LiveUses is provided, record any /// RegUnits that are made live by the current instruction's uses. This includes /// registers that are both defined and used by the instruction. If a pressure /// difference pointer is provided record the changes is pressure caused by this /// instruction independent of liveness. void RegPressureTracker::recede(const RegisterOperands &RegOpers, SmallVectorImpl<RegisterMaskPair> *LiveUses) { assert(!CurrPos->isDebugValue()); // Boost pressure for all dead defs together. bumpDeadDefs(RegOpers.DeadDefs); // Kill liveness at live defs. // TODO: consider earlyclobbers? for (const RegisterMaskPair &Def : RegOpers.Defs) { unsigned Reg = Def.RegUnit; LaneBitmask PreviousMask = LiveRegs.erase(Def); LaneBitmask NewMask = PreviousMask & ~Def.LaneMask; LaneBitmask LiveOut = Def.LaneMask & ~PreviousMask; if (LiveOut.any()) { discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); // Retroactively model effects on pressure of the live out lanes. increaseSetPressure(CurrSetPressure, *MRI, Reg, LaneBitmask::getNone(), LiveOut); PreviousMask = LiveOut; } if (NewMask.none()) { // Add a 0 entry to LiveUses as a marker that the complete vreg has become // dead. if (TrackLaneMasks && LiveUses != nullptr) setRegZero(*LiveUses, Reg); } decreaseRegPressure(Reg, PreviousMask, NewMask); } SlotIndex SlotIdx; if (RequireIntervals) SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot(); // Generate liveness for uses. for (const RegisterMaskPair &Use : RegOpers.Uses) { unsigned Reg = Use.RegUnit; assert(Use.LaneMask.any()); LaneBitmask PreviousMask = LiveRegs.insert(Use); LaneBitmask NewMask = PreviousMask | Use.LaneMask; if (NewMask == PreviousMask) continue; // Did the register just become live? if (PreviousMask.none()) { if (LiveUses != nullptr) { if (!TrackLaneMasks) { addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); } else { auto I = find_if(*LiveUses, [Reg](const RegisterMaskPair Other) { return Other.RegUnit == Reg; }); bool IsRedef = I != LiveUses->end(); if (IsRedef) { // ignore re-defs here... assert(I->LaneMask.none()); removeRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); } else { addRegLanes(*LiveUses, RegisterMaskPair(Reg, NewMask)); } } } // Discover live outs if this may be the first occurance of this register. if (RequireIntervals) { LaneBitmask LiveOut = getLiveThroughAt(Reg, SlotIdx); if (LiveOut.any()) discoverLiveOut(RegisterMaskPair(Reg, LiveOut)); } } increaseRegPressure(Reg, PreviousMask, NewMask); } if (TrackUntiedDefs) { for (const RegisterMaskPair &Def : RegOpers.Defs) { unsigned RegUnit = Def.RegUnit; if (TargetRegisterInfo::isVirtualRegister(RegUnit) && (LiveRegs.contains(RegUnit) & Def.LaneMask).none()) UntiedDefs.insert(RegUnit); } } }