/// collectRegsToSpill - Collect live range snippets that only have a single /// real use. void InlineSpiller::collectRegsToSpill() { unsigned Reg = Edit->getReg(); // Main register always spills. RegsToSpill.assign(1, Reg); SnippetCopies.clear(); // Snippets all have the same original, so there can't be any for an original // register. if (Original == Reg) return; for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Reg); MachineInstr *MI = RI.skipInstruction();) { unsigned SnipReg = isFullCopyOf(MI, Reg); if (!isSibling(SnipReg)) continue; LiveInterval &SnipLI = LIS.getInterval(SnipReg); if (!isSnippet(SnipLI)) continue; SnippetCopies.insert(MI); if (isRegToSpill(SnipReg)) continue; RegsToSpill.push_back(SnipReg); DEBUG(dbgs() << "\talso spill snippet " << SnipLI << '\n'); ++NumSnippets; } }
/// analyzeUses - Count instructions, basic blocks, and loops using curli. void SplitAnalysis::analyzeUses() { const MachineRegisterInfo &MRI = mf_.getRegInfo(); for (MachineRegisterInfo::reg_iterator I = MRI.reg_begin(curli_->reg); MachineInstr *MI = I.skipInstruction();) { if (MI->isDebugValue() || !usingInstrs_.insert(MI)) continue; MachineBasicBlock *MBB = MI->getParent(); if (usingBlocks_[MBB]++) continue; if (MachineLoop *Loop = loops_.getLoopFor(MBB)) usingLoops_[Loop]++; } DEBUG(dbgs() << " counted " << usingInstrs_.size() << " instrs, " << usingBlocks_.size() << " blocks, " << usingLoops_.size() << " loops.\n"); }
/// spillAll - Spill all registers remaining after rematerialization. void InlineSpiller::spillAll() { // Update LiveStacks now that we are committed to spilling. if (StackSlot == VirtRegMap::NO_STACK_SLOT) { StackSlot = VRM.assignVirt2StackSlot(Original); StackInt = &LSS.getOrCreateInterval(StackSlot, MRI.getRegClass(Original)); StackInt->getNextValue(SlotIndex(), LSS.getVNInfoAllocator()); } else StackInt = &LSS.getInterval(StackSlot); if (Original != Edit->getReg()) VRM.assignVirt2StackSlot(Edit->getReg(), StackSlot); assert(StackInt->getNumValNums() == 1 && "Bad stack interval values"); for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) StackInt->MergeRangesInAsValue(LIS.getInterval(RegsToSpill[i]), StackInt->getValNumInfo(0)); DEBUG(dbgs() << "Merged spilled regs: " << *StackInt << '\n'); // Spill around uses of all RegsToSpill. for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) spillAroundUses(RegsToSpill[i]); // Hoisted spills may cause dead code. if (!DeadDefs.empty()) { DEBUG(dbgs() << "Eliminating " << DeadDefs.size() << " dead defs\n"); Edit->eliminateDeadDefs(DeadDefs, RegsToSpill); } // Finally delete the SnippetCopies. for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) { for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(RegsToSpill[i]); MachineInstr *MI = RI.skipInstruction();) { assert(SnippetCopies.count(MI) && "Remaining use wasn't a snippet copy"); // FIXME: Do this with a LiveRangeEdit callback. LIS.RemoveMachineInstrFromMaps(MI); MI->eraseFromParent(); } } // Delete all spilled registers. for (unsigned i = 0, e = RegsToSpill.size(); i != e; ++i) Edit->eraseVirtReg(RegsToSpill[i]); }
/// analyzeUses - Count instructions, basic blocks, and loops using CurLI. void SplitAnalysis::analyzeUses() { const MachineRegisterInfo &MRI = MF.getRegInfo(); for (MachineRegisterInfo::reg_iterator I = MRI.reg_begin(CurLI->reg); MachineInstr *MI = I.skipInstruction();) { if (MI->isDebugValue() || !UsingInstrs.insert(MI)) continue; UseSlots.push_back(LIS.getInstructionIndex(MI).getDefIndex()); MachineBasicBlock *MBB = MI->getParent(); if (UsingBlocks[MBB]++) continue; for (MachineLoop *Loop = Loops.getLoopFor(MBB); Loop; Loop = Loop->getParentLoop()) UsingLoops[Loop]++; } array_pod_sort(UseSlots.begin(), UseSlots.end()); DEBUG(dbgs() << " counted " << UsingInstrs.size() << " instrs, " << UsingBlocks.size() << " blocks, " << UsingLoops.size() << " loops.\n"); }
/// shrinkToUses - After removing some uses of a register, shrink its live /// range to just the remaining uses. This method does not compute reaching /// defs for new uses, and it doesn't remove dead defs. bool LiveIntervals::shrinkToUses(LiveInterval *li, SmallVectorImpl<MachineInstr*> *dead) { DEBUG(dbgs() << "Shrink: " << *li << '\n'); assert(TargetRegisterInfo::isVirtualRegister(li->reg) && "Can only shrink virtual registers"); // Find all the values used, including PHI kills. SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList; // Blocks that have already been added to WorkList as live-out. SmallPtrSet<MachineBasicBlock*, 16> LiveOut; // Visit all instructions reading li->reg. for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg); MachineInstr *UseMI = I.skipInstruction();) { if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg)) continue; SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot(); LiveRangeQuery LRQ(*li, Idx); VNInfo *VNI = LRQ.valueIn(); if (!VNI) { // This shouldn't happen: readsVirtualRegister returns true, but there is // no live value. It is likely caused by a target getting <undef> flags // wrong. DEBUG(dbgs() << Idx << '\t' << *UseMI << "Warning: Instr claims to read non-existent value in " << *li << '\n'); continue; } // Special case: An early-clobber tied operand reads and writes the // register one slot early. if (VNInfo *DefVNI = LRQ.valueDefined()) Idx = DefVNI->def; WorkList.push_back(std::make_pair(Idx, VNI)); } // Create a new live interval with only minimal live segments per def. LiveInterval NewLI(li->reg, 0); for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end(); I != E; ++I) { VNInfo *VNI = *I; if (VNI->isUnused()) continue; NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI)); } // Keep track of the PHIs that are in use. SmallPtrSet<VNInfo*, 8> UsedPHIs; // Extend intervals to reach all uses in WorkList. while (!WorkList.empty()) { SlotIndex Idx = WorkList.back().first; VNInfo *VNI = WorkList.back().second; WorkList.pop_back(); const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot()); SlotIndex BlockStart = getMBBStartIdx(MBB); // Extend the live range for VNI to be live at Idx. if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) { (void)ExtVNI; assert(ExtVNI == VNI && "Unexpected existing value number"); // Is this a PHIDef we haven't seen before? if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI)) continue; // The PHI is live, make sure the predecessors are live-out. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { if (!LiveOut.insert(*PI)) continue; SlotIndex Stop = getMBBEndIdx(*PI); // A predecessor is not required to have a live-out value for a PHI. if (VNInfo *PVNI = li->getVNInfoBefore(Stop)) WorkList.push_back(std::make_pair(Stop, PVNI)); } continue; } // VNI is live-in to MBB. DEBUG(dbgs() << " live-in at " << BlockStart << '\n'); NewLI.addRange(LiveRange(BlockStart, Idx, VNI)); // Make sure VNI is live-out from the predecessors. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { if (!LiveOut.insert(*PI)) continue; SlotIndex Stop = getMBBEndIdx(*PI); assert(li->getVNInfoBefore(Stop) == VNI && "Wrong value out of predecessor"); WorkList.push_back(std::make_pair(Stop, VNI)); } } // Handle dead values. bool CanSeparate = false; for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end(); I != E; ++I) { VNInfo *VNI = *I; if (VNI->isUnused()) continue; LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def); assert(LII != NewLI.end() && "Missing live range for PHI"); if (LII->end != VNI->def.getDeadSlot()) continue; if (VNI->isPHIDef()) { // This is a dead PHI. Remove it. VNI->markUnused(); NewLI.removeRange(*LII); DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n"); CanSeparate = true; } else { // This is a dead def. Make sure the instruction knows. MachineInstr *MI = getInstructionFromIndex(VNI->def); assert(MI && "No instruction defining live value"); MI->addRegisterDead(li->reg, TRI); if (dead && MI->allDefsAreDead()) { DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI); dead->push_back(MI); } } } // Move the trimmed ranges back. li->ranges.swap(NewLI.ranges); DEBUG(dbgs() << "Shrunk: " << *li << '\n'); return CanSeparate; }
// Top-level driver to manage the queue of unassigned VirtRegs and call the // selectOrSplit implementation. void RegAllocBase::allocatePhysRegs() { seedLiveRegs(); // Continue assigning vregs one at a time to available physical registers. while (LiveInterval *VirtReg = dequeue()) { assert(!VRM->hasPhys(VirtReg->reg) && "Register already assigned"); // Unused registers can appear when the spiller coalesces snippets. if (MRI->reg_nodbg_empty(VirtReg->reg)) { DEBUG(dbgs() << "Dropping unused " << *VirtReg << '\n'); LIS->removeInterval(VirtReg->reg); continue; } // Invalidate all interference queries, live ranges could have changed. invalidateVirtRegs(); // selectOrSplit requests the allocator to return an available physical // register if possible and populate a list of new live intervals that // result from splitting. DEBUG(dbgs() << "\nselectOrSplit " << MRI->getRegClass(VirtReg->reg)->getName() << ':' << *VirtReg << '\n'); typedef SmallVector<LiveInterval*, 4> VirtRegVec; VirtRegVec SplitVRegs; unsigned AvailablePhysReg = selectOrSplit(*VirtReg, SplitVRegs); if (AvailablePhysReg == ~0u) { // selectOrSplit failed to find a register! std::string msg; raw_string_ostream Msg(msg); Msg << "Ran out of registers during register allocation!" "\nCannot allocate: " << *VirtReg; for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(VirtReg->reg); MachineInstr *MI = I.skipInstruction();) { if (!MI->isInlineAsm()) continue; Msg << "\nPlease check your inline asm statement for " "invalid constraints:\n"; MI->print(Msg, &VRM->getMachineFunction().getTarget()); } report_fatal_error(Msg.str()); } if (AvailablePhysReg) assign(*VirtReg, AvailablePhysReg); for (VirtRegVec::iterator I = SplitVRegs.begin(), E = SplitVRegs.end(); I != E; ++I) { LiveInterval *SplitVirtReg = *I; assert(!VRM->hasPhys(SplitVirtReg->reg) && "Register already assigned"); if (MRI->reg_nodbg_empty(SplitVirtReg->reg)) { DEBUG(dbgs() << "not queueing unused " << *SplitVirtReg << '\n'); LIS->removeInterval(SplitVirtReg->reg); continue; } DEBUG(dbgs() << "queuing new interval: " << *SplitVirtReg << "\n"); assert(TargetRegisterInfo::isVirtualRegister(SplitVirtReg->reg) && "expect split value in virtual register"); enqueue(SplitVirtReg); ++NumNewQueued; } } }
// Top-level driver to manage the queue of unassigned VirtRegs and call the // selectOrSplit implementation. void RegAllocBase::allocatePhysRegs() { seedLiveRegs(); // Continue assigning vregs one at a time to available physical registers. while (LiveInterval *VirtReg = dequeue()) { assert(!VRM->hasPhys(VirtReg->reg) && "Register already assigned"); // Unused registers can appear when the spiller coalesces snippets. if (MRI->reg_nodbg_empty(VirtReg->reg)) { DEBUG(dbgs() << "Dropping unused " << *VirtReg << '\n'); LIS->removeInterval(VirtReg->reg); continue; } // Invalidate all interference queries, live ranges could have changed. invalidateVirtRegs(); // selectOrSplit requests the allocator to return an available physical // register if possible and populate a list of new live intervals that // result from splitting. DEBUG(dbgs() << "\nselectOrSplit " << MRI->getRegClass(VirtReg->reg)->getName() << ':' << *VirtReg << '\n'); typedef SmallVector<LiveInterval*, 4> VirtRegVec; VirtRegVec SplitVRegs; unsigned AvailablePhysReg = selectOrSplit(*VirtReg, SplitVRegs); if (AvailablePhysReg == ~0u) { // selectOrSplit failed to find a register! const char *Msg = "ran out of registers during register allocation"; // Probably caused by an inline asm. MachineInstr *MI; for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(VirtReg->reg); (MI = I.skipInstruction());) if (MI->isInlineAsm()) break; if (MI) MI->emitError(Msg); else report_fatal_error(Msg); // Keep going after reporting the error. VRM->assignVirt2Phys(VirtReg->reg, RegClassInfo.getOrder(MRI->getRegClass(VirtReg->reg)).front()); continue; } if (AvailablePhysReg) assign(*VirtReg, AvailablePhysReg); for (VirtRegVec::iterator I = SplitVRegs.begin(), E = SplitVRegs.end(); I != E; ++I) { LiveInterval *SplitVirtReg = *I; assert(!VRM->hasPhys(SplitVirtReg->reg) && "Register already assigned"); if (MRI->reg_nodbg_empty(SplitVirtReg->reg)) { DEBUG(dbgs() << "not queueing unused " << *SplitVirtReg << '\n'); LIS->removeInterval(SplitVirtReg->reg); continue; } DEBUG(dbgs() << "queuing new interval: " << *SplitVirtReg << "\n"); assert(TargetRegisterInfo::isVirtualRegister(SplitVirtReg->reg) && "expect split value in virtual register"); enqueue(SplitVirtReg); ++NumNewQueued; } } }
void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) { MachineRegisterInfo &mri = MF.getRegInfo(); const TargetRegisterInfo &tri = *MF.getTarget().getRegisterInfo(); MachineBasicBlock *mbb = 0; MachineLoop *loop = 0; unsigned loopDepth = 0; bool isExiting = false; float totalWeight = 0; SmallPtrSet<MachineInstr*, 8> visited; // Find the best physreg hist and the best virtreg hint. float bestPhys = 0, bestVirt = 0; unsigned hintPhys = 0, hintVirt = 0; // Don't recompute a target specific hint. bool noHint = mri.getRegAllocationHint(li.reg).first != 0; // Don't recompute spill weight for an unspillable register. bool Spillable = li.isSpillable(); for (MachineRegisterInfo::reg_iterator I = mri.reg_begin(li.reg); MachineInstr *mi = I.skipInstruction();) { if (mi->isIdentityCopy() || mi->isImplicitDef() || mi->isDebugValue()) continue; if (!visited.insert(mi)) continue; float weight = 1.0f; if (Spillable) { // Get loop info for mi. if (mi->getParent() != mbb) { mbb = mi->getParent(); loop = Loops.getLoopFor(mbb); loopDepth = loop ? loop->getLoopDepth() : 0; isExiting = loop ? loop->isLoopExiting(mbb) : false; } // Calculate instr weight. bool reads, writes; tie(reads, writes) = mi->readsWritesVirtualRegister(li.reg); weight = LiveIntervals::getSpillWeight(writes, reads, loopDepth); // Give extra weight to what looks like a loop induction variable update. if (writes && isExiting && LIS.isLiveOutOfMBB(li, mbb)) weight *= 3; totalWeight += weight; } // Get allocation hints from copies. if (noHint || !mi->isCopy()) continue; unsigned hint = copyHint(mi, li.reg, tri, mri); if (!hint) continue; float hweight = Hint[hint] += weight; if (TargetRegisterInfo::isPhysicalRegister(hint)) { if (hweight > bestPhys && mri.isAllocatable(hint)) bestPhys = hweight, hintPhys = hint; } else { if (hweight > bestVirt) bestVirt = hweight, hintVirt = hint; } } Hint.clear(); // Always prefer the physreg hint. if (unsigned hint = hintPhys ? hintPhys : hintVirt) { mri.setRegAllocationHint(li.reg, 0, hint); // Weakly boost the spill weight of hinted registers. totalWeight *= 1.01F; } // If the live interval was already unspillable, leave it that way. if (!Spillable) return; // Mark li as unspillable if all live ranges are tiny. if (li.isZeroLength(LIS.getSlotIndexes())) { li.markNotSpillable(); return; } // If all of the definitions of the interval are re-materializable, // it is a preferred candidate for spilling. // FIXME: this gets much more complicated once we support non-trivial // re-materialization. if (isRematerializable(li, LIS, *MF.getTarget().getInstrInfo())) totalWeight *= 0.5F; li.weight = normalizeSpillWeight(totalWeight, li.getSize()); }
void InlineSpiller::spill(LiveInterval *li, SmallVectorImpl<LiveInterval*> &newIntervals, SmallVectorImpl<LiveInterval*> &spillIs) { DEBUG(dbgs() << "Inline spilling " << *li << "\n"); assert(li->isSpillable() && "Attempting to spill already spilled value."); assert(!li->isStackSlot() && "Trying to spill a stack slot."); li_ = li; newIntervals_ = &newIntervals; rc_ = mri_.getRegClass(li->reg); spillIs_ = &spillIs; if (split()) return; reMaterializeAll(); // Remat may handle everything. if (li_->empty()) return; stackSlot_ = vrm_.getStackSlot(li->reg); if (stackSlot_ == VirtRegMap::NO_STACK_SLOT) stackSlot_ = vrm_.assignVirt2StackSlot(li->reg); // Iterate over instructions using register. for (MachineRegisterInfo::reg_iterator RI = mri_.reg_begin(li->reg); MachineInstr *MI = RI.skipInstruction();) { // Debug values are not allowed to affect codegen. if (MI->isDebugValue()) { // Modify DBG_VALUE now that the value is in a spill slot. uint64_t Offset = MI->getOperand(1).getImm(); const MDNode *MDPtr = MI->getOperand(2).getMetadata(); DebugLoc DL = MI->getDebugLoc(); if (MachineInstr *NewDV = tii_.emitFrameIndexDebugValue(mf_, stackSlot_, Offset, MDPtr, DL)) { DEBUG(dbgs() << "Modifying debug info due to spill:" << "\t" << *MI); MachineBasicBlock *MBB = MI->getParent(); MBB->insert(MBB->erase(MI), NewDV); } else { DEBUG(dbgs() << "Removing debug info due to spill:" << "\t" << *MI); MI->eraseFromParent(); } continue; } // Stack slot accesses may coalesce away. if (coalesceStackAccess(MI)) continue; // Analyze instruction. bool Reads, Writes; SmallVector<unsigned, 8> Ops; tie(Reads, Writes) = MI->readsWritesVirtualRegister(li->reg, &Ops); // Attempt to fold memory ops. if (foldMemoryOperand(MI, Ops)) continue; // Allocate interval around instruction. // FIXME: Infer regclass from instruction alone. unsigned NewVReg = mri_.createVirtualRegister(rc_); vrm_.grow(); LiveInterval &NewLI = lis_.getOrCreateInterval(NewVReg); NewLI.markNotSpillable(); if (Reads) insertReload(NewLI, MI); // Rewrite instruction operands. bool hasLiveDef = false; for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); MO.setReg(NewVReg); if (MO.isUse()) { if (!MI->isRegTiedToDefOperand(Ops[i])) MO.setIsKill(); } else { if (!MO.isDead()) hasLiveDef = true; } } // FIXME: Use a second vreg if instruction has no tied ops. if (Writes && hasLiveDef) insertSpill(NewLI, MI); DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); newIntervals.push_back(&NewLI); } }