/// leaveIntvAfter - Leave openli after the instruction at Idx. void SplitEditor::leaveIntvAfter(SlotIndex Idx) { assert(openli_ && "openIntv not called before leaveIntvAfter"); const LiveRange *CurLR = curli_->getLiveRangeContaining(Idx.getDefIndex()); if (!CurLR || CurLR->end <= Idx.getBoundaryIndex()) { DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": not live\n"); return; } // Was this value of curli live through openli? if (!openli_->liveAt(CurLR->valno->def)) { DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": using external value\n"); liveThrough_ = true; return; } // We are going to insert a back copy, so we must have a dupli_. LiveRange *DupLR = getDupLI()->getLiveRangeContaining(Idx.getDefIndex()); assert(DupLR && "dupli not live into black, but curli is?"); // Insert the COPY instruction. MachineBasicBlock::iterator I = lis_.getInstructionFromIndex(Idx); MachineInstr *MI = BuildMI(*I->getParent(), llvm::next(I), I->getDebugLoc(), tii_.get(TargetOpcode::COPY), dupli_->reg) .addReg(openli_->reg); SlotIndex CopyIdx = lis_.InsertMachineInstrInMaps(MI).getDefIndex(); openli_->addRange(LiveRange(Idx.getDefIndex(), CopyIdx, mapValue(CurLR->valno))); DupLR->valno->def = CopyIdx; DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": " << *openli_ << '\n'); }
/// rewrite - after all the new live ranges have been created, rewrite /// instructions using curli to use the new intervals. void SplitEditor::rewrite() { assert(!openli_ && "Previous LI not closed before rewrite"); const LiveInterval *curli = sa_.getCurLI(); for (MachineRegisterInfo::reg_iterator RI = mri_.reg_begin(curli->reg), RE = mri_.reg_end(); RI != RE;) { MachineOperand &MO = RI.getOperand(); MachineInstr *MI = MO.getParent(); ++RI; if (MI->isDebugValue()) { DEBUG(dbgs() << "Zapping " << *MI); // FIXME: We can do much better with debug values. MO.setReg(0); continue; } SlotIndex Idx = lis_.getInstructionIndex(MI); Idx = MO.isUse() ? Idx.getUseIndex() : Idx.getDefIndex(); LiveInterval *LI = dupli_; for (unsigned i = firstInterval, e = intervals_.size(); i != e; ++i) { LiveInterval *testli = intervals_[i]; if (testli->liveAt(Idx)) { LI = testli; break; } } if (LI) { MO.setReg(LI->reg); sa_.removeUse(MI); DEBUG(dbgs() << " rewrite " << Idx << '\t' << *MI); } } // dupli_ goes in last, after rewriting. if (dupli_) { if (dupli_->empty()) { DEBUG(dbgs() << " dupli became empty?\n"); lis_.removeInterval(dupli_->reg); dupli_ = 0; } else { dupli_->RenumberValues(lis_); intervals_.push_back(dupli_); } } // Calculate spill weight and allocation hints for new intervals. VirtRegAuxInfo vrai(vrm_.getMachineFunction(), lis_, sa_.loops_); for (unsigned i = firstInterval, e = intervals_.size(); i != e; ++i) { LiveInterval &li = *intervals_[i]; vrai.CalculateRegClass(li.reg); vrai.CalculateWeightAndHint(li); DEBUG(dbgs() << " new interval " << mri_.getRegClass(li.reg)->getName() << ":" << li << '\n'); } }
void ConnectedVNInfoEqClasses::Distribute(LiveInterval *LIV[], MachineRegisterInfo &MRI) { assert(LIV[0] && "LIV[0] must be set"); LiveInterval &LI = *LIV[0]; // Rewrite instructions. for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LI.reg), RE = MRI.reg_end(); RI != RE;) { MachineOperand &MO = RI.getOperand(); MachineInstr *MI = MO.getParent(); ++RI; if (MO.isUse() && MO.isUndef()) continue; // DBG_VALUE instructions should have been eliminated earlier. SlotIndex Idx = LIS.getInstructionIndex(MI); Idx = MO.isUse() ? Idx.getUseIndex() : Idx.getDefIndex(); const VNInfo *VNI = LI.getVNInfoAt(Idx); assert(VNI && "Interval not live at use."); MO.setReg(LIV[getEqClass(VNI)]->reg); } // Move runs to new intervals. LiveInterval::iterator J = LI.begin(), E = LI.end(); while (J != E && EqClass[J->valno->id] == 0) ++J; for (LiveInterval::iterator I = J; I != E; ++I) { if (unsigned eq = EqClass[I->valno->id]) { assert((LIV[eq]->empty() || LIV[eq]->expiredAt(I->start)) && "New intervals should be empty"); LIV[eq]->ranges.push_back(*I); } else *J++ = *I; } LI.ranges.erase(J, E); // Transfer VNInfos to their new owners and renumber them. unsigned j = 0, e = LI.getNumValNums(); while (j != e && EqClass[j] == 0) ++j; for (unsigned i = j; i != e; ++i) { VNInfo *VNI = LI.getValNumInfo(i); if (unsigned eq = EqClass[i]) { VNI->id = LIV[eq]->getNumValNums(); LIV[eq]->valnos.push_back(VNI); } else { VNI->id = j; LI.valnos[j++] = VNI; } } LI.valnos.resize(j); }
/// rewriteAssigned - Rewrite all uses of Edit->getReg(). void SplitEditor::rewriteAssigned(bool ExtendRanges) { for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()), RE = MRI.reg_end(); RI != RE;) { MachineOperand &MO = RI.getOperand(); MachineInstr *MI = MO.getParent(); ++RI; // LiveDebugVariables should have handled all DBG_VALUE instructions. if (MI->isDebugValue()) { DEBUG(dbgs() << "Zapping " << *MI); MO.setReg(0); continue; } // <undef> operands don't really read the register, so it doesn't matter // which register we choose. When the use operand is tied to a def, we must // use the same register as the def, so just do that always. SlotIndex Idx = LIS.getInstructionIndex(MI); if (MO.isDef() || MO.isUndef()) Idx = MO.isEarlyClobber() ? Idx.getUseIndex() : Idx.getDefIndex(); // Rewrite to the mapped register at Idx. unsigned RegIdx = RegAssign.lookup(Idx); LiveInterval *LI = Edit->get(RegIdx); MO.setReg(LI->reg); DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t' << Idx << ':' << RegIdx << '\t' << *MI); // Extend liveness to Idx if the instruction reads reg. if (!ExtendRanges || MO.isUndef()) continue; // Skip instructions that don't read Reg. if (MO.isDef()) { if (!MO.getSubReg() && !MO.isEarlyClobber()) continue; // We may wan't to extend a live range for a partial redef, or for a use // tied to an early clobber. Idx = Idx.getPrevSlot(); if (!Edit->getParent().liveAt(Idx)) continue; } else Idx = Idx.getUseIndex(); getLRCalc(RegIdx).extend(LI, Idx.getNextSlot(), LIS.getSlotIndexes(), &MDT, &LIS.getVNInfoAllocator()); } }
/// enterIntvBefore - Enter openli before the instruction at Idx. If curli is /// not live before Idx, a COPY is not inserted. void SplitEditor::enterIntvBefore(SlotIndex Idx) { assert(openli_ && "openIntv not called before enterIntvBefore"); // Copy from curli_ if it is live. if (VNInfo *CurVNI = curli_->getVNInfoAt(Idx.getUseIndex())) { MachineInstr *MI = lis_.getInstructionFromIndex(Idx); assert(MI && "enterIntvBefore called with invalid index"); VNInfo *VNI = insertCopy(*openli_, *MI->getParent(), MI); openli_->addRange(LiveRange(VNI->def, Idx.getDefIndex(), VNI)); // Make sure CurVNI is properly mapped. VNInfo *&mapVNI = valueMap_[CurVNI]; // We dont have SSA update yet, so only one entry per value is allowed. assert(!mapVNI && "enterIntvBefore called more than once for the same value"); mapVNI = VNI; } DEBUG(dbgs() << " enterIntvBefore " << Idx << ": " << *openli_ << '\n'); }
void UserValue::addDefsFromCopies(LiveInterval *LI, unsigned LocNo, const SmallVectorImpl<SlotIndex> &Kills, SmallVectorImpl<std::pair<SlotIndex, unsigned> > &NewDefs, MachineRegisterInfo &MRI, LiveIntervals &LIS) { if (Kills.empty()) return; // Don't track copies from physregs, there are too many uses. if (!TargetRegisterInfo::isVirtualRegister(LI->reg)) return; // Collect all the (vreg, valno) pairs that are copies of LI. SmallVector<std::pair<LiveInterval*, const VNInfo*>, 8> CopyValues; for (MachineRegisterInfo::use_nodbg_iterator UI = MRI.use_nodbg_begin(LI->reg), UE = MRI.use_nodbg_end(); UI != UE; ++UI) { // Copies of the full value. if (UI.getOperand().getSubReg() || !UI->isCopy()) continue; MachineInstr *MI = &*UI; unsigned DstReg = MI->getOperand(0).getReg(); // Don't follow copies to physregs. These are usually setting up call // arguments, and the argument registers are always call clobbered. We are // better off in the source register which could be a callee-saved register, // or it could be spilled. if (!TargetRegisterInfo::isVirtualRegister(DstReg)) continue; // Is LocNo extended to reach this copy? If not, another def may be blocking // it, or we are looking at a wrong value of LI. SlotIndex Idx = LIS.getInstructionIndex(MI); LocMap::iterator I = locInts.find(Idx.getUseIndex()); if (!I.valid() || I.value() != LocNo) continue; if (!LIS.hasInterval(DstReg)) continue; LiveInterval *DstLI = &LIS.getInterval(DstReg); const VNInfo *DstVNI = DstLI->getVNInfoAt(Idx.getDefIndex()); assert(DstVNI && DstVNI->def == Idx.getDefIndex() && "Bad copy value"); CopyValues.push_back(std::make_pair(DstLI, DstVNI)); } if (CopyValues.empty()) return; DEBUG(dbgs() << "Got " << CopyValues.size() << " copies of " << *LI << '\n'); // Try to add defs of the copied values for each kill point. for (unsigned i = 0, e = Kills.size(); i != e; ++i) { SlotIndex Idx = Kills[i]; for (unsigned j = 0, e = CopyValues.size(); j != e; ++j) { LiveInterval *DstLI = CopyValues[j].first; const VNInfo *DstVNI = CopyValues[j].second; if (DstLI->getVNInfoAt(Idx) != DstVNI) continue; // Check that there isn't already a def at Idx LocMap::iterator I = locInts.find(Idx); if (I.valid() && I.start() <= Idx) continue; DEBUG(dbgs() << "Kill at " << Idx << " covered by valno #" << DstVNI->id << " in " << *DstLI << '\n'); MachineInstr *CopyMI = LIS.getInstructionFromIndex(DstVNI->def); assert(CopyMI && CopyMI->isCopy() && "Bad copy value"); unsigned LocNo = getLocationNo(CopyMI->getOperand(0)); I.insert(Idx, Idx.getNextSlot(), LocNo); NewDefs.push_back(std::make_pair(Idx, LocNo)); break; } } }
void MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); const MCInstrDesc &MCID = MI->getDesc(); const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; // The first MCID.NumDefs operands must be explicit register defines if (MONum < MCID.getNumDefs()) { if (!MO->isReg()) report("Explicit definition must be a register", MO, MONum); else if (!MO->isDef()) report("Explicit definition marked as use", MO, MONum); else if (MO->isImplicit()) report("Explicit definition marked as implicit", MO, MONum); } else if (MONum < MCID.getNumOperands()) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. if (MO->isReg() && !(MCID.isVariadic() && MONum == MCID.getNumOperands()-1)) { if (MO->isDef() && !MCOI.isOptionalDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) report("Explicit operand marked as implicit", MO, MONum); } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. if (MO->isReg() && !MO->isImplicit() && !MCID.isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } switch (MO->getType()) { case MachineOperand::MO_Register: { const unsigned Reg = MO->getReg(); if (!Reg) return; // Check Live Variables. if (MI->isDebugValue()) { // Liveness checks are not valid for debug values. } else if (MO->isUse() && !MO->isUndef()) { regsLiveInButUnused.erase(Reg); bool isKill = false; unsigned defIdx; if (MI->isRegTiedToDefOperand(MONum, &defIdx)) { // A two-addr use counts as a kill if use and def are the same. unsigned DefReg = MI->getOperand(defIdx).getReg(); if (Reg == DefReg) isKill = true; else if (TargetRegisterInfo::isPhysicalRegister(Reg)) { report("Two-address instruction operands must be identical", MO, MONum); } } else isKill = MO->isKill(); if (isKill) addRegWithSubRegs(regsKilled, Reg); // Check that LiveVars knows this kill. if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) && MO->isKill()) { LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); if (std::find(VI.Kills.begin(), VI.Kills.end(), MI) == VI.Kills.end()) report("Kill missing from LiveVariables", MO, MONum); } // Check LiveInts liveness and kill. if (TargetRegisterInfo::isVirtualRegister(Reg) && LiveInts && !LiveInts->isNotInMIMap(MI)) { SlotIndex UseIdx = LiveInts->getInstructionIndex(MI).getUseIndex(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (!LI.liveAt(UseIdx)) { report("No live range at use", MO, MONum); *OS << UseIdx << " is not live in " << LI << '\n'; } // Check for extra kill flags. // Note that we allow missing kill flags for now. if (MO->isKill() && !LI.killedAt(UseIdx.getDefIndex())) { report("Live range continues after kill flag", MO, MONum); *OS << "Live range: " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } // Use of a dead register. if (!regsLive.count(Reg)) { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { // Reserved registers may be used even when 'dead'. if (!isReserved(Reg)) report("Using an undefined physical register", MO, MONum); } else { BBInfo &MInfo = MBBInfoMap[MI->getParent()]; // We don't know which virtual registers are live in, so only complain // if vreg was killed in this MBB. Otherwise keep track of vregs that // must be live in. PHI instructions are handled separately. if (MInfo.regsKilled.count(Reg)) report("Using a killed virtual register", MO, MONum); else if (!MI->isPHI()) MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI)); } } } else if (MO->isDef()) { // Register defined. // TODO: verify that earlyclobber ops are not used. if (MO->isDead()) addRegWithSubRegs(regsDead, Reg); else addRegWithSubRegs(regsDefined, Reg); // Verify SSA form. if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) && llvm::next(MRI->def_begin(Reg)) != MRI->def_end()) report("Multiple virtual register defs in SSA form", MO, MONum); // Check LiveInts for a live range, but only for virtual registers. if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) && !LiveInts->isNotInMIMap(MI)) { SlotIndex DefIdx = LiveInts->getInstructionIndex(MI).getDefIndex(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (const VNInfo *VNI = LI.getVNInfoAt(DefIdx)) { assert(VNI && "NULL valno is not allowed"); if (VNI->def != DefIdx && !MO->isEarlyClobber()) { report("Inconsistent valno->def", MO, MONum); *OS << "Valno " << VNI->id << " is not defined at " << DefIdx << " in " << LI << '\n'; } } else { report("No live range at def", MO, MONum); *OS << DefIdx << " is not live in " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } } // Check register classes. if (MONum < MCID.getNumOperands() && !MO->isImplicit()) { unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { if (SubIdx) { report("Illegal subregister index for physical register", MO, MONum); return; } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (!DRC->contains(Reg)) { report("Illegal physical register for instruction", MO, MONum); *OS << TRI->getName(Reg) << " is not a " << DRC->getName() << " register.\n"; } } } else { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { const TargetRegisterClass *SRC = TRI->getSubClassWithSubReg(RC, SubIdx); if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); *OS << "Register class " << RC->getName() << " does not support subreg index " << SubIdx << "\n"; return; } if (RC != SRC) { report("Invalid register class for subregister index", MO, MONum); *OS << "Register class " << RC->getName() << " does not fully support subreg index " << SubIdx << "\n"; return; } } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (SubIdx) { const TargetRegisterClass *SuperRC = TRI->getLargestLegalSuperClass(RC); if (!SuperRC) { report("No largest legal super class exists.", MO, MONum); return; } DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); if (!DRC) { report("No matching super-reg register class.", MO, MONum); return; } } if (!RC->hasSuperClassEq(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " << RC->getName() << " register\n"; } } } } break; } case MachineOperand::MO_MachineBasicBlock: if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) report("PHI operand is not in the CFG", MO, MONum); break; case MachineOperand::MO_FrameIndex: if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); if (MCID.mayLoad() && !LI.liveAt(Idx.getUseIndex())) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } if (MCID.mayStore() && !LI.liveAt(Idx.getDefIndex())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } } break; default: break; } }
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &Fn) { LiveIntervals& LI = getAnalysis<LiveIntervals>(); // Compute DFS numbers of each block computeDFS(Fn); // Determine which phi node operands need copies for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) if (!I->empty() && I->begin()->isPHI()) processBlock(I); // Break interferences where two different phis want to coalesce // in the same register. std::set<unsigned> seen; typedef std::map<unsigned, std::map<unsigned, MachineBasicBlock*> > RenameSetType; for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end(); I != E; ++I) { for (std::map<unsigned, MachineBasicBlock*>::iterator OI = I->second.begin(), OE = I->second.end(); OI != OE; ) { if (!seen.count(OI->first)) { seen.insert(OI->first); ++OI; } else { Waiting[OI->second].insert(std::make_pair(OI->first, I->first)); unsigned reg = OI->first; ++OI; I->second.erase(reg); DEBUG(dbgs() << "Removing Renaming: " << reg << " -> " << I->first << "\n"); } } } // Insert copies // FIXME: This process should probably preserve LiveIntervals SmallPtrSet<MachineBasicBlock*, 16> visited; MachineDominatorTree& MDT = getAnalysis<MachineDominatorTree>(); InsertCopies(MDT.getRootNode(), visited); // Perform renaming for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end(); I != E; ++I) while (I->second.size()) { std::map<unsigned, MachineBasicBlock*>::iterator SI = I->second.begin(); DEBUG(dbgs() << "Renaming: " << SI->first << " -> " << I->first << "\n"); if (SI->first != I->first) { if (mergeLiveIntervals(I->first, SI->first)) { Fn.getRegInfo().replaceRegWith(SI->first, I->first); if (RenameSets.count(SI->first)) { I->second.insert(RenameSets[SI->first].begin(), RenameSets[SI->first].end()); RenameSets.erase(SI->first); } } else { // Insert a last-minute copy if a conflict was detected. const TargetInstrInfo *TII = Fn.getTarget().getInstrInfo(); const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(I->first); TII->copyRegToReg(*SI->second, SI->second->getFirstTerminator(), I->first, SI->first, RC, RC, DebugLoc()); LI.renumber(); LiveInterval& Int = LI.getOrCreateInterval(I->first); SlotIndex instrIdx = LI.getInstructionIndex(--SI->second->getFirstTerminator()); if (Int.liveAt(instrIdx.getDefIndex())) Int.removeRange(instrIdx.getDefIndex(), LI.getMBBEndIdx(SI->second).getNextSlot(), true); LiveRange R = LI.addLiveRangeToEndOfBlock(I->first, --SI->second->getFirstTerminator()); R.valno->setCopy(--SI->second->getFirstTerminator()); R.valno->def = instrIdx.getDefIndex(); DEBUG(dbgs() << "Renaming failed: " << SI->first << " -> " << I->first << "\n"); } } LiveInterval& Int = LI.getOrCreateInterval(I->first); const LiveRange* LR = Int.getLiveRangeContaining(LI.getMBBEndIdx(SI->second)); LR->valno->setHasPHIKill(true); I->second.erase(SI->first); } // Remove PHIs std::vector<MachineInstr*> phis; for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { for (MachineBasicBlock::iterator BI = I->begin(), BE = I->end(); BI != BE; ++BI) if (BI->isPHI()) phis.push_back(BI); } for (std::vector<MachineInstr*>::iterator I = phis.begin(), E = phis.end(); I != E; ) { MachineInstr* PInstr = *(I++); // If this is a dead PHI node, then remove it from LiveIntervals. unsigned DestReg = PInstr->getOperand(0).getReg(); LiveInterval& PI = LI.getInterval(DestReg); if (PInstr->registerDefIsDead(DestReg)) { if (PI.containsOneValue()) { LI.removeInterval(DestReg); } else { SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex(); PI.removeRange(*PI.getLiveRangeContaining(idx), true); } } else { // Trim live intervals of input registers. They are no longer live into // this block if they died after the PHI. If they lived after it, don't // trim them because they might have other legitimate uses. for (unsigned i = 1; i < PInstr->getNumOperands(); i += 2) { unsigned reg = PInstr->getOperand(i).getReg(); MachineBasicBlock* MBB = PInstr->getOperand(i+1).getMBB(); LiveInterval& InputI = LI.getInterval(reg); if (MBB != PInstr->getParent() && InputI.liveAt(LI.getMBBStartIdx(PInstr->getParent())) && InputI.expiredAt(LI.getInstructionIndex(PInstr).getNextIndex())) InputI.removeRange(LI.getMBBStartIdx(PInstr->getParent()), LI.getInstructionIndex(PInstr), true); } // If the PHI is not dead, then the valno defined by the PHI // now has an unknown def. SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex(); const LiveRange* PLR = PI.getLiveRangeContaining(idx); PLR->valno->setIsPHIDef(true); LiveRange R (LI.getMBBStartIdx(PInstr->getParent()), PLR->start, PLR->valno); PI.addRange(R); } LI.RemoveMachineInstrFromMaps(PInstr); PInstr->eraseFromParent(); } LI.renumber(); return true; }
/// ScheduleCopies - Insert copies into predecessor blocks, scheduling /// them properly so as to avoid the 'lost copy' and the 'virtual swap' /// problems. /// /// Based on "Practical Improvements to the Construction and Destruction /// of Static Single Assignment Form" by Briggs, et al. void StrongPHIElimination::ScheduleCopies(MachineBasicBlock* MBB, std::set<unsigned>& pushed) { // FIXME: This function needs to update LiveIntervals std::multimap<unsigned, unsigned>& copy_set= Waiting[MBB]; std::multimap<unsigned, unsigned> worklist; std::map<unsigned, unsigned> map; // Setup worklist of initial copies for (std::multimap<unsigned, unsigned>::iterator I = copy_set.begin(), E = copy_set.end(); I != E; ) { map.insert(std::make_pair(I->first, I->first)); map.insert(std::make_pair(I->second, I->second)); if (!UsedByAnother.count(I->second)) { worklist.insert(*I); // Avoid iterator invalidation std::multimap<unsigned, unsigned>::iterator OI = I; ++I; copy_set.erase(OI); } else { ++I; } } LiveIntervals& LI = getAnalysis<LiveIntervals>(); MachineFunction* MF = MBB->getParent(); MachineRegisterInfo& MRI = MF->getRegInfo(); const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); SmallVector<std::pair<unsigned, MachineInstr*>, 4> InsertedPHIDests; // Iterate over the worklist, inserting copies while (!worklist.empty() || !copy_set.empty()) { while (!worklist.empty()) { std::multimap<unsigned, unsigned>::iterator WI = worklist.begin(); std::pair<unsigned, unsigned> curr = *WI; worklist.erase(WI); const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(curr.first); if (isLiveOut(curr.second, MBB, LI)) { // Create a temporary unsigned t = MF->getRegInfo().createVirtualRegister(RC); // Insert copy from curr.second to a temporary at // the Phi defining curr.second MachineBasicBlock::iterator PI = MRI.getVRegDef(curr.second); TII->copyRegToReg(*PI->getParent(), PI, t, curr.second, RC, RC, DebugLoc()); DEBUG(dbgs() << "Inserted copy from " << curr.second << " to " << t << "\n"); // Push temporary on Stacks Stacks[curr.second].push_back(t); // Insert curr.second in pushed pushed.insert(curr.second); // Create a live interval for this temporary InsertedPHIDests.push_back(std::make_pair(t, --PI)); } // Insert copy from map[curr.first] to curr.second TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), curr.second, map[curr.first], RC, RC, DebugLoc()); map[curr.first] = curr.second; DEBUG(dbgs() << "Inserted copy from " << curr.first << " to " << curr.second << "\n"); // Push this copy onto InsertedPHICopies so we can // update LiveIntervals with it. MachineBasicBlock::iterator MI = MBB->getFirstTerminator(); InsertedPHIDests.push_back(std::make_pair(curr.second, --MI)); // If curr.first is a destination in copy_set... for (std::multimap<unsigned, unsigned>::iterator I = copy_set.begin(), E = copy_set.end(); I != E; ) if (curr.first == I->second) { std::pair<unsigned, unsigned> temp = *I; worklist.insert(temp); // Avoid iterator invalidation std::multimap<unsigned, unsigned>::iterator OI = I; ++I; copy_set.erase(OI); break; } else { ++I; } } if (!copy_set.empty()) { std::multimap<unsigned, unsigned>::iterator CI = copy_set.begin(); std::pair<unsigned, unsigned> curr = *CI; worklist.insert(curr); copy_set.erase(CI); LiveInterval& I = LI.getInterval(curr.second); MachineBasicBlock::iterator term = MBB->getFirstTerminator(); SlotIndex endIdx = SlotIndex(); if (term != MBB->end()) endIdx = LI.getInstructionIndex(term); else endIdx = LI.getMBBEndIdx(MBB); if (I.liveAt(endIdx)) { const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(curr.first); // Insert a copy from dest to a new temporary t at the end of b unsigned t = MF->getRegInfo().createVirtualRegister(RC); TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), t, curr.second, RC, RC, DebugLoc()); map[curr.second] = t; MachineBasicBlock::iterator TI = MBB->getFirstTerminator(); InsertedPHIDests.push_back(std::make_pair(t, --TI)); } } } // Renumber the instructions so that we can perform the index computations // needed to create new live intervals. LI.renumber(); // For copies that we inserted at the ends of predecessors, we construct // live intervals. This is pretty easy, since we know that the destination // register cannot have be in live at that point previously. We just have // to make sure that, for registers that serve as inputs to more than one // PHI, we don't create multiple overlapping live intervals. std::set<unsigned> RegHandled; for (SmallVector<std::pair<unsigned, MachineInstr*>, 4>::iterator I = InsertedPHIDests.begin(), E = InsertedPHIDests.end(); I != E; ++I) { if (RegHandled.insert(I->first).second) { LiveInterval& Int = LI.getOrCreateInterval(I->first); SlotIndex instrIdx = LI.getInstructionIndex(I->second); if (Int.liveAt(instrIdx.getDefIndex())) Int.removeRange(instrIdx.getDefIndex(), LI.getMBBEndIdx(I->second->getParent()).getNextSlot(), true); LiveRange R = LI.addLiveRangeToEndOfBlock(I->first, I->second); R.valno->setCopy(I->second); R.valno->def = LI.getInstructionIndex(I->second).getDefIndex(); } } }
void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI, MachineBasicBlock *MBB) { assert(PHI->isPHI()); unsigned PHIColor = getPHIColor(PHI); for (unsigned i = 1; i < PHI->getNumOperands(); i += 2) { MachineOperand &SrcMO = PHI->getOperand(i); // If a source is defined by an implicit def, there is no need to insert a // copy in the predecessor. if (SrcMO.isUndef()) continue; unsigned SrcReg = SrcMO.getReg(); assert(TargetRegisterInfo::isVirtualRegister(SrcReg) && "Machine PHI Operands must all be virtual registers!"); MachineBasicBlock *PredBB = PHI->getOperand(i + 1).getMBB(); unsigned SrcColor = getRegColor(SrcReg); // If neither the PHI nor the operand were isolated, then we only need to // set the phi-kill flag on the VNInfo at this PHI. if (PHIColor && SrcColor == PHIColor) { LiveInterval &SrcInterval = LI->getInterval(SrcReg); SlotIndex PredIndex = LI->getMBBEndIdx(PredBB); VNInfo *SrcVNI = SrcInterval.getVNInfoAt(PredIndex.getPrevIndex()); assert(SrcVNI); SrcVNI->setHasPHIKill(true); continue; } unsigned CopyReg = 0; if (PHIColor) { SrcCopyMap::const_iterator I = InsertedSrcCopyMap.find(std::make_pair(PredBB, PHIColor)); CopyReg = I != InsertedSrcCopyMap.end() ? I->second->getOperand(0).getReg() : 0; } if (!CopyReg) { const TargetRegisterClass *RC = MRI->getRegClass(SrcReg); CopyReg = MRI->createVirtualRegister(RC); MachineBasicBlock::iterator CopyInsertPoint = findPHICopyInsertPoint(PredBB, MBB, SrcReg); unsigned SrcSubReg = SrcMO.getSubReg(); MachineInstr *CopyInstr = BuildMI(*PredBB, CopyInsertPoint, PHI->getDebugLoc(), TII->get(TargetOpcode::COPY), CopyReg).addReg(SrcReg, 0, SrcSubReg); LI->InsertMachineInstrInMaps(CopyInstr); // addLiveRangeToEndOfBlock() also adds the phikill flag to the VNInfo for // the newly added range. LI->addLiveRangeToEndOfBlock(CopyReg, CopyInstr); InsertedSrcCopySet.insert(std::make_pair(PredBB, SrcReg)); addReg(CopyReg); if (PHIColor) { unionRegs(PHIColor, CopyReg); assert(getRegColor(CopyReg) != CopyReg); } else { PHIColor = CopyReg; assert(getRegColor(CopyReg) == CopyReg); } if (!InsertedSrcCopyMap.count(std::make_pair(PredBB, PHIColor))) InsertedSrcCopyMap[std::make_pair(PredBB, PHIColor)] = CopyInstr; } SrcMO.setReg(CopyReg); // If SrcReg is not live beyond the PHI, trim its interval so that it is no // longer live-in to MBB. Note that SrcReg may appear in other PHIs that are // processed later, but this is still correct to do at this point because we // never rely on LiveIntervals being correct while inserting copies. // FIXME: Should this just count uses at PHIs like the normal PHIElimination // pass does? LiveInterval &SrcLI = LI->getInterval(SrcReg); SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); SlotIndex PHIIndex = LI->getInstructionIndex(PHI); SlotIndex NextInstrIndex = PHIIndex.getNextIndex(); if (SrcLI.liveAt(MBBStartIndex) && SrcLI.expiredAt(NextInstrIndex)) SrcLI.removeRange(MBBStartIndex, PHIIndex, true); } unsigned DestReg = PHI->getOperand(0).getReg(); unsigned DestColor = getRegColor(DestReg); if (PHIColor && DestColor == PHIColor) { LiveInterval &DestLI = LI->getInterval(DestReg); // Set the phi-def flag for the VN at this PHI. SlotIndex PHIIndex = LI->getInstructionIndex(PHI); VNInfo *DestVNI = DestLI.getVNInfoAt(PHIIndex.getDefIndex()); assert(DestVNI); DestVNI->setIsPHIDef(true); // Prior to PHI elimination, the live ranges of PHIs begin at their defining // instruction. After PHI elimination, PHI instructions are replaced by VNs // with the phi-def flag set, and the live ranges of these VNs start at the // beginning of the basic block. SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); DestVNI->def = MBBStartIndex; DestLI.addRange(LiveRange(MBBStartIndex, PHIIndex.getDefIndex(), DestVNI)); return; } const TargetRegisterClass *RC = MRI->getRegClass(DestReg); unsigned CopyReg = MRI->createVirtualRegister(RC); MachineInstr *CopyInstr = BuildMI(*MBB, MBB->SkipPHIsAndLabels(MBB->begin()), PHI->getDebugLoc(), TII->get(TargetOpcode::COPY), DestReg).addReg(CopyReg); LI->InsertMachineInstrInMaps(CopyInstr); PHI->getOperand(0).setReg(CopyReg); // Add the region from the beginning of MBB to the copy instruction to // CopyReg's live interval, and give the VNInfo the phidef flag. LiveInterval &CopyLI = LI->getOrCreateInterval(CopyReg); SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); SlotIndex DestCopyIndex = LI->getInstructionIndex(CopyInstr); VNInfo *CopyVNI = CopyLI.getNextValue(MBBStartIndex, CopyInstr, LI->getVNInfoAllocator()); CopyVNI->setIsPHIDef(true); CopyLI.addRange(LiveRange(MBBStartIndex, DestCopyIndex.getDefIndex(), CopyVNI)); // Adjust DestReg's live interval to adjust for its new definition at // CopyInstr. LiveInterval &DestLI = LI->getOrCreateInterval(DestReg); SlotIndex PHIIndex = LI->getInstructionIndex(PHI); DestLI.removeRange(PHIIndex.getDefIndex(), DestCopyIndex.getDefIndex()); VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getDefIndex()); assert(DestVNI); DestVNI->def = DestCopyIndex.getDefIndex(); InsertedDestCopies[CopyReg] = CopyInstr; }
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &MF) { MRI = &MF.getRegInfo(); TII = MF.getTarget().getInstrInfo(); DT = &getAnalysis<MachineDominatorTree>(); LI = &getAnalysis<LiveIntervals>(); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { unsigned DestReg = BBI->getOperand(0).getReg(); addReg(DestReg); PHISrcDefs[I].push_back(BBI); for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) { MachineOperand &SrcMO = BBI->getOperand(i); unsigned SrcReg = SrcMO.getReg(); addReg(SrcReg); unionRegs(DestReg, SrcReg); MachineInstr *DefMI = MRI->getVRegDef(SrcReg); if (DefMI) PHISrcDefs[DefMI->getParent()].push_back(DefMI); } } } // Perform a depth-first traversal of the dominator tree, splitting // interferences amongst PHI-congruence classes. DenseMap<unsigned, unsigned> CurrentDominatingParent; DenseMap<unsigned, unsigned> ImmediateDominatingParent; for (df_iterator<MachineDomTreeNode*> DI = df_begin(DT->getRootNode()), DE = df_end(DT->getRootNode()); DI != DE; ++DI) { SplitInterferencesForBasicBlock(*DI->getBlock(), CurrentDominatingParent, ImmediateDominatingParent); } // Insert copies for all PHI source and destination registers. for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { InsertCopiesForPHI(BBI, I); } } // FIXME: Preserve the equivalence classes during copy insertion and use // the preversed equivalence classes instead of recomputing them. RegNodeMap.clear(); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { unsigned DestReg = BBI->getOperand(0).getReg(); addReg(DestReg); for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) { unsigned SrcReg = BBI->getOperand(i).getReg(); addReg(SrcReg); unionRegs(DestReg, SrcReg); } } } DenseMap<unsigned, unsigned> RegRenamingMap; bool Changed = false; for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); while (BBI != BBE && BBI->isPHI()) { MachineInstr *PHI = BBI; assert(PHI->getNumOperands() > 0); unsigned SrcReg = PHI->getOperand(1).getReg(); unsigned SrcColor = getRegColor(SrcReg); unsigned NewReg = RegRenamingMap[SrcColor]; if (!NewReg) { NewReg = SrcReg; RegRenamingMap[SrcColor] = SrcReg; } MergeLIsAndRename(SrcReg, NewReg); unsigned DestReg = PHI->getOperand(0).getReg(); if (!InsertedDestCopies.count(DestReg)) MergeLIsAndRename(DestReg, NewReg); for (unsigned i = 3; i < PHI->getNumOperands(); i += 2) { unsigned SrcReg = PHI->getOperand(i).getReg(); MergeLIsAndRename(SrcReg, NewReg); } ++BBI; LI->RemoveMachineInstrFromMaps(PHI); PHI->eraseFromParent(); Changed = true; } } // Due to the insertion of copies to split live ranges, the live intervals are // guaranteed to not overlap, except in one case: an original PHI source and a // PHI destination copy. In this case, they have the same value and thus don't // truly intersect, so we merge them into the value live at that point. // FIXME: Is there some better way we can handle this? for (DestCopyMap::iterator I = InsertedDestCopies.begin(), E = InsertedDestCopies.end(); I != E; ++I) { unsigned DestReg = I->first; unsigned DestColor = getRegColor(DestReg); unsigned NewReg = RegRenamingMap[DestColor]; LiveInterval &DestLI = LI->getInterval(DestReg); LiveInterval &NewLI = LI->getInterval(NewReg); assert(DestLI.ranges.size() == 1 && "PHI destination copy's live interval should be a single live " "range from the beginning of the BB to the copy instruction."); LiveRange *DestLR = DestLI.begin(); VNInfo *NewVNI = NewLI.getVNInfoAt(DestLR->start); if (!NewVNI) { NewVNI = NewLI.createValueCopy(DestLR->valno, LI->getVNInfoAllocator()); MachineInstr *CopyInstr = I->second; CopyInstr->getOperand(1).setIsKill(true); } LiveRange NewLR(DestLR->start, DestLR->end, NewVNI); NewLI.addRange(NewLR); LI->removeInterval(DestReg); MRI->replaceRegWith(DestReg, NewReg); } // Adjust the live intervals of all PHI source registers to handle the case // where the PHIs in successor blocks were the only later uses of the source // register. for (SrcCopySet::iterator I = InsertedSrcCopySet.begin(), E = InsertedSrcCopySet.end(); I != E; ++I) { MachineBasicBlock *MBB = I->first; unsigned SrcReg = I->second; if (unsigned RenamedRegister = RegRenamingMap[getRegColor(SrcReg)]) SrcReg = RenamedRegister; LiveInterval &SrcLI = LI->getInterval(SrcReg); bool isLiveOut = false; for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { if (SrcLI.liveAt(LI->getMBBStartIdx(*SI))) { isLiveOut = true; break; } } if (isLiveOut) continue; MachineOperand *LastUse = findLastUse(MBB, SrcReg); assert(LastUse); SlotIndex LastUseIndex = LI->getInstructionIndex(LastUse->getParent()); SrcLI.removeRange(LastUseIndex.getDefIndex(), LI->getMBBEndIdx(MBB)); LastUse->setIsKill(true); } LI->renumber(); Allocator.Reset(); RegNodeMap.clear(); PHISrcDefs.clear(); InsertedSrcCopySet.clear(); InsertedSrcCopyMap.clear(); InsertedDestCopies.clear(); return Changed; }
/// reMaterializeFor - Attempt to rematerialize li_->reg before MI instead of /// reloading it. bool InlineSpiller::reMaterializeFor(MachineBasicBlock::iterator MI) { SlotIndex UseIdx = lis_.getInstructionIndex(MI).getUseIndex(); VNInfo *OrigVNI = li_->getVNInfoAt(UseIdx); if (!OrigVNI) { DEBUG(dbgs() << "\tadding <undef> flags: "); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg) MO.setIsUndef(); } DEBUG(dbgs() << UseIdx << '\t' << *MI); return true; } if (!reMattable_.count(OrigVNI)) { DEBUG(dbgs() << "\tusing non-remat valno " << OrigVNI->id << ": " << UseIdx << '\t' << *MI); return false; } MachineInstr *OrigMI = lis_.getInstructionFromIndex(OrigVNI->def); if (!allUsesAvailableAt(OrigMI, OrigVNI->def, UseIdx)) { usedValues_.insert(OrigVNI); DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI); return false; } // If the instruction also writes li_->reg, it had better not require the same // register for uses and defs. bool Reads, Writes; SmallVector<unsigned, 8> Ops; tie(Reads, Writes) = MI->readsWritesVirtualRegister(li_->reg, &Ops); if (Writes) { for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); if (MO.isUse() ? MI->isRegTiedToDefOperand(Ops[i]) : MO.getSubReg()) { usedValues_.insert(OrigVNI); DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << *MI); return false; } } } // Alocate a new register for the remat. unsigned NewVReg = mri_.createVirtualRegister(rc_); vrm_.grow(); LiveInterval &NewLI = lis_.getOrCreateInterval(NewVReg); NewLI.markNotSpillable(); newIntervals_->push_back(&NewLI); // Finally we can rematerialize OrigMI before MI. MachineBasicBlock &MBB = *MI->getParent(); tii_.reMaterialize(MBB, MI, NewLI.reg, 0, OrigMI, tri_); MachineBasicBlock::iterator RematMI = MI; SlotIndex DefIdx = lis_.InsertMachineInstrInMaps(--RematMI).getDefIndex(); DEBUG(dbgs() << "\tremat: " << DefIdx << '\t' << *RematMI); // Replace operands for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg) { MO.setReg(NewVReg); MO.setIsKill(); } } DEBUG(dbgs() << "\t " << UseIdx << '\t' << *MI); VNInfo *DefVNI = NewLI.getNextValue(DefIdx, 0, true, lis_.getVNInfoAllocator()); NewLI.addRange(LiveRange(DefIdx, UseIdx.getDefIndex(), DefVNI)); DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); return true; }