/// LowerPHINode - Lower the PHI node at the top of the specified block, /// void PHIElimination::LowerPHINode(MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt) { ++NumLowered; MachineBasicBlock::iterator AfterPHIsIt = std::next(LastPHIIt); // Unlink the PHI node from the basic block, but don't delete the PHI yet. MachineInstr *MPhi = MBB.remove(MBB.begin()); unsigned NumSrcs = (MPhi->getNumOperands() - 1) / 2; unsigned DestReg = MPhi->getOperand(0).getReg(); assert(MPhi->getOperand(0).getSubReg() == 0 && "Can't handle sub-reg PHIs"); bool isDead = MPhi->getOperand(0).isDead(); // Create a new register for the incoming PHI arguments. MachineFunction &MF = *MBB.getParent(); unsigned IncomingReg = 0; bool reusedIncoming = false; // Is IncomingReg reused from an earlier PHI? // Insert a register to register copy at the top of the current block (but // after any remaining phi nodes) which copies the new incoming register // into the phi node destination. const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); if (isSourceDefinedByImplicitDef(MPhi, MRI)) // If all sources of a PHI node are implicit_def, just emit an // implicit_def instead of a copy. BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), DestReg); else { // Can we reuse an earlier PHI node? This only happens for critical edges, // typically those created by tail duplication. unsigned &entry = LoweredPHIs[MPhi]; if (entry) { // An identical PHI node was already lowered. Reuse the incoming register. IncomingReg = entry; reusedIncoming = true; ++NumReused; DEBUG(dbgs() << "Reusing " << PrintReg(IncomingReg) << " for " << *MPhi); } else { const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg); entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC); } BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), DestReg) .addReg(IncomingReg); } // Update live variable information if there is any. if (LV) { MachineInstr *PHICopy = std::prev(AfterPHIsIt); if (IncomingReg) { LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg); // Increment use count of the newly created virtual register. LV->setPHIJoin(IncomingReg); // When we are reusing the incoming register, it may already have been // killed in this block. The old kill will also have been inserted at // AfterPHIsIt, so it appears before the current PHICopy. if (reusedIncoming) if (MachineInstr *OldKill = VI.findKill(&MBB)) { DEBUG(dbgs() << "Remove old kill from " << *OldKill); LV->removeVirtualRegisterKilled(IncomingReg, OldKill); DEBUG(MBB.dump()); } // Add information to LiveVariables to know that the incoming value is // killed. Note that because the value is defined in several places (once // each for each incoming block), the "def" block and instruction fields // for the VarInfo is not filled in. LV->addVirtualRegisterKilled(IncomingReg, PHICopy); } // Since we are going to be deleting the PHI node, if it is the last use of // any registers, or if the value itself is dead, we need to move this // information over to the new copy we just inserted. LV->removeVirtualRegistersKilled(MPhi); // If the result is dead, update LV. if (isDead) { LV->addVirtualRegisterDead(DestReg, PHICopy); LV->removeVirtualRegisterDead(DestReg, MPhi); } } // Update LiveIntervals for the new copy or implicit def. if (LIS) { MachineInstr *NewInstr = std::prev(AfterPHIsIt); SlotIndex DestCopyIndex = LIS->InsertMachineInstrInMaps(NewInstr); SlotIndex MBBStartIndex = LIS->getMBBStartIdx(&MBB); if (IncomingReg) { // Add the region from the beginning of MBB to the copy instruction to // IncomingReg's live interval. LiveInterval &IncomingLI = LIS->createEmptyInterval(IncomingReg); VNInfo *IncomingVNI = IncomingLI.getVNInfoAt(MBBStartIndex); if (!IncomingVNI) IncomingVNI = IncomingLI.getNextValue(MBBStartIndex, LIS->getVNInfoAllocator()); IncomingLI.addSegment(LiveInterval::Segment(MBBStartIndex, DestCopyIndex.getRegSlot(), IncomingVNI)); } LiveInterval &DestLI = LIS->getInterval(DestReg); assert(DestLI.begin() != DestLI.end() && "PHIs should have nonempty LiveIntervals."); if (DestLI.endIndex().isDead()) { // A dead PHI's live range begins and ends at the start of the MBB, but // the lowered copy, which will still be dead, needs to begin and end at // the copy instruction. VNInfo *OrigDestVNI = DestLI.getVNInfoAt(MBBStartIndex); assert(OrigDestVNI && "PHI destination should be live at block entry."); DestLI.removeSegment(MBBStartIndex, MBBStartIndex.getDeadSlot()); DestLI.createDeadDef(DestCopyIndex.getRegSlot(), LIS->getVNInfoAllocator()); DestLI.removeValNo(OrigDestVNI); } else { // Otherwise, remove the region from the beginning of MBB to the copy // instruction from DestReg's live interval. DestLI.removeSegment(MBBStartIndex, DestCopyIndex.getRegSlot()); VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getRegSlot()); assert(DestVNI && "PHI destination should be live at its definition."); DestVNI->def = DestCopyIndex.getRegSlot(); } } // Adjust the VRegPHIUseCount map to account for the removal of this PHI node. for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(), MPhi->getOperand(i).getReg())]; // Now loop over all of the incoming arguments, changing them to copy into the // IncomingReg register in the corresponding predecessor basic block. SmallPtrSet<MachineBasicBlock*, 8> MBBsInsertedInto; for (int i = NumSrcs - 1; i >= 0; --i) { unsigned SrcReg = MPhi->getOperand(i*2+1).getReg(); unsigned SrcSubReg = MPhi->getOperand(i*2+1).getSubReg(); bool SrcUndef = MPhi->getOperand(i*2+1).isUndef() || isImplicitlyDefined(SrcReg, MRI); assert(TargetRegisterInfo::isVirtualRegister(SrcReg) && "Machine PHI Operands must all be virtual registers!"); // Get the MachineBasicBlock equivalent of the BasicBlock that is the source // path the PHI. MachineBasicBlock &opBlock = *MPhi->getOperand(i*2+2).getMBB(); // Check to make sure we haven't already emitted the copy for this block. // This can happen because PHI nodes may have multiple entries for the same // basic block. if (!MBBsInsertedInto.insert(&opBlock)) continue; // If the copy has already been emitted, we're done. // Find a safe location to insert the copy, this may be the first terminator // in the block (or end()). MachineBasicBlock::iterator InsertPos = findPHICopyInsertPoint(&opBlock, &MBB, SrcReg); // Insert the copy. MachineInstr *NewSrcInstr = nullptr; if (!reusedIncoming && IncomingReg) { if (SrcUndef) { // The source register is undefined, so there is no need for a real // COPY, but we still need to ensure joint dominance by defs. // Insert an IMPLICIT_DEF instruction. NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), IncomingReg); // Clean up the old implicit-def, if there even was one. if (MachineInstr *DefMI = MRI->getVRegDef(SrcReg)) if (DefMI->isImplicitDef()) ImpDefs.insert(DefMI); } else { NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), IncomingReg) .addReg(SrcReg, 0, SrcSubReg); } } // We only need to update the LiveVariables kill of SrcReg if this was the // last PHI use of SrcReg to be lowered on this CFG edge and it is not live // out of the predecessor. We can also ignore undef sources. if (LV && !SrcUndef && !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)] && !LV->isLiveOut(SrcReg, opBlock)) { // We want to be able to insert a kill of the register if this PHI (aka, // the copy we just inserted) is the last use of the source value. Live // variable analysis conservatively handles this by saying that the value // is live until the end of the block the PHI entry lives in. If the value // really is dead at the PHI copy, there will be no successor blocks which // have the value live-in. // Okay, if we now know that the value is not live out of the block, we // can add a kill marker in this block saying that it kills the incoming // value! // In our final twist, we have to decide which instruction kills the // register. In most cases this is the copy, however, terminator // instructions at the end of the block may also use the value. In this // case, we should mark the last such terminator as being the killing // block, not the copy. MachineBasicBlock::iterator KillInst = opBlock.end(); MachineBasicBlock::iterator FirstTerm = opBlock.getFirstTerminator(); for (MachineBasicBlock::iterator Term = FirstTerm; Term != opBlock.end(); ++Term) { if (Term->readsRegister(SrcReg)) KillInst = Term; } if (KillInst == opBlock.end()) { // No terminator uses the register. if (reusedIncoming || !IncomingReg) { // We may have to rewind a bit if we didn't insert a copy this time. KillInst = FirstTerm; while (KillInst != opBlock.begin()) { --KillInst; if (KillInst->isDebugValue()) continue; if (KillInst->readsRegister(SrcReg)) break; } } else { // We just inserted this copy. KillInst = std::prev(InsertPos); } } assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); // Finally, mark it killed. LV->addVirtualRegisterKilled(SrcReg, KillInst); // This vreg no longer lives all of the way through opBlock. unsigned opBlockNum = opBlock.getNumber(); LV->getVarInfo(SrcReg).AliveBlocks.reset(opBlockNum); } if (LIS) { if (NewSrcInstr) { LIS->InsertMachineInstrInMaps(NewSrcInstr); LIS->addSegmentToEndOfBlock(IncomingReg, NewSrcInstr); } if (!SrcUndef && !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]) { LiveInterval &SrcLI = LIS->getInterval(SrcReg); bool isLiveOut = false; for (MachineBasicBlock::succ_iterator SI = opBlock.succ_begin(), SE = opBlock.succ_end(); SI != SE; ++SI) { SlotIndex startIdx = LIS->getMBBStartIdx(*SI); VNInfo *VNI = SrcLI.getVNInfoAt(startIdx); // Definitions by other PHIs are not truly live-in for our purposes. if (VNI && VNI->def != startIdx) { isLiveOut = true; break; } } if (!isLiveOut) { MachineBasicBlock::iterator KillInst = opBlock.end(); MachineBasicBlock::iterator FirstTerm = opBlock.getFirstTerminator(); for (MachineBasicBlock::iterator Term = FirstTerm; Term != opBlock.end(); ++Term) { if (Term->readsRegister(SrcReg)) KillInst = Term; } if (KillInst == opBlock.end()) { // No terminator uses the register. if (reusedIncoming || !IncomingReg) { // We may have to rewind a bit if we didn't just insert a copy. KillInst = FirstTerm; while (KillInst != opBlock.begin()) { --KillInst; if (KillInst->isDebugValue()) continue; if (KillInst->readsRegister(SrcReg)) break; } } else { // We just inserted this copy. KillInst = std::prev(InsertPos); } } assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); SlotIndex LastUseIndex = LIS->getInstructionIndex(KillInst); SrcLI.removeSegment(LastUseIndex.getRegSlot(), LIS->getMBBEndIdx(&opBlock)); } } } } // Really delete the PHI instruction now, if it is not in the LoweredPHIs map. if (reusedIncoming || !IncomingReg) { if (LIS) LIS->RemoveMachineInstrFromMaps(MPhi); MF.DeleteMachineInstr(MPhi); } }
MachineBasicBlock * MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Splitting the critical edge to a landing pad block is non-trivial. Don't do // it in this generic function. if (Succ->isLandingPad()) return nullptr; MachineFunction *MF = getParent(); DebugLoc dl; // FIXME: this is nowhere // Performance might be harmed on HW that implements branching using exec mask // where both sides of the branches are always executed. if (MF->getTarget().requiresStructuredCFG()) return nullptr; // We may need to update this's terminator, but we can't do that if // AnalyzeBranch fails. If this uses a jump table, we won't touch it. const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector<MachineOperand, 4> Cond; if (TII->AnalyzeBranch(*this, TBB, FBB, Cond)) return nullptr; // Avoid bugpoint weirdness: A block may end with a conditional branch but // jumps to the same MBB is either case. We have duplicate CFG edges in that // case that we can't handle. Since this never happens in properly optimized // code, just skip those edges. if (TBB && TBB == FBB) { DEBUG(dbgs() << "Won't split critical edge after degenerate BB#" << getNumber() << '\n'); return nullptr; } MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); MF->insert(std::next(MachineFunction::iterator(this)), NMBB); DEBUG(dbgs() << "Splitting critical edge:" " BB#" << getNumber() << " -- BB#" << NMBB->getNumber() << " -- BB#" << Succ->getNumber() << '\n'); LiveIntervals *LIS = P->getAnalysisIfAvailable<LiveIntervals>(); SlotIndexes *Indexes = P->getAnalysisIfAvailable<SlotIndexes>(); if (LIS) LIS->insertMBBInMaps(NMBB); else if (Indexes) Indexes->insertMBBInMaps(NMBB); // On some targets like Mips, branches may kill virtual registers. Make sure // that LiveVariables is properly updated after updateTerminator replaces the // terminators. LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>(); // Collect a list of virtual registers killed by the terminators. SmallVector<unsigned, 4> KilledRegs; if (LV) for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { if (!OI->isReg() || OI->getReg() == 0 || !OI->isUse() || !OI->isKill() || OI->isUndef()) continue; unsigned Reg = OI->getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg) || LV->getVarInfo(Reg).removeKill(MI)) { KilledRegs.push_back(Reg); DEBUG(dbgs() << "Removing terminator kill: " << *MI); OI->setIsKill(false); } } } SmallVector<unsigned, 4> UsedRegs; if (LIS) { for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { if (!OI->isReg() || OI->getReg() == 0) continue; unsigned Reg = OI->getReg(); if (std::find(UsedRegs.begin(), UsedRegs.end(), Reg) == UsedRegs.end()) UsedRegs.push_back(Reg); } } } ReplaceUsesOfBlockWith(Succ, NMBB); // If updateTerminator() removes instructions, we need to remove them from // SlotIndexes. SmallVector<MachineInstr*, 4> Terminators; if (Indexes) { for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) Terminators.push_back(I); } updateTerminator(); if (Indexes) { SmallVector<MachineInstr*, 4> NewTerminators; for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) NewTerminators.push_back(I); for (SmallVectorImpl<MachineInstr*>::iterator I = Terminators.begin(), E = Terminators.end(); I != E; ++I) { if (std::find(NewTerminators.begin(), NewTerminators.end(), *I) == NewTerminators.end()) Indexes->removeMachineInstrFromMaps(*I); } } // Insert unconditional "jump Succ" instruction in NMBB if necessary. NMBB->addSuccessor(Succ); if (!NMBB->isLayoutSuccessor(Succ)) { Cond.clear(); MF->getSubtarget().getInstrInfo()->InsertBranch(*NMBB, Succ, nullptr, Cond, dl); if (Indexes) { for (instr_iterator I = NMBB->instr_begin(), E = NMBB->instr_end(); I != E; ++I) { // Some instructions may have been moved to NMBB by updateTerminator(), // so we first remove any instruction that already has an index. if (Indexes->hasIndex(I)) Indexes->removeMachineInstrFromMaps(I); Indexes->insertMachineInstrInMaps(I); } } } // Fix PHI nodes in Succ so they refer to NMBB instead of this for (MachineBasicBlock::instr_iterator i = Succ->instr_begin(),e = Succ->instr_end(); i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == this) i->getOperand(ni+1).setMBB(NMBB); // Inherit live-ins from the successor for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(), E = Succ->livein_end(); I != E; ++I) NMBB->addLiveIn(*I); // Update LiveVariables. const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); if (LV) { // Restore kills of virtual registers that were killed by the terminators. while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) { if (!(--I)->addRegisterKilled(Reg, TRI, /* addIfNotFound= */ false)) continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) LV->getVarInfo(Reg).Kills.push_back(I); DEBUG(dbgs() << "Restored terminator kill: " << *I); break; } } // Update relevant live-through information. LV->addNewBlock(NMBB, this, Succ); } if (LIS) { // After splitting the edge and updating SlotIndexes, live intervals may be // in one of two situations, depending on whether this block was the last in // the function. If the original block was the last in the function, all live // intervals will end prior to the beginning of the new split block. If the // original block was not at the end of the function, all live intervals will // extend to the end of the new split block. bool isLastMBB = std::next(MachineFunction::iterator(NMBB)) == getParent()->end(); SlotIndex StartIndex = Indexes->getMBBEndIdx(this); SlotIndex PrevIndex = StartIndex.getPrevSlot(); SlotIndex EndIndex = Indexes->getMBBEndIdx(NMBB); // Find the registers used from NMBB in PHIs in Succ. SmallSet<unsigned, 8> PHISrcRegs; for (MachineBasicBlock::instr_iterator I = Succ->instr_begin(), E = Succ->instr_end(); I != E && I->isPHI(); ++I) { for (unsigned ni = 1, ne = I->getNumOperands(); ni != ne; ni += 2) { if (I->getOperand(ni+1).getMBB() == NMBB) { MachineOperand &MO = I->getOperand(ni); unsigned Reg = MO.getReg(); PHISrcRegs.insert(Reg); if (MO.isUndef()) continue; LiveInterval &LI = LIS->getInterval(Reg); VNInfo *VNI = LI.getVNInfoAt(PrevIndex); assert(VNI && "PHI sources should be live out of their predecessors."); LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI)); } } } MachineRegisterInfo *MRI = &getParent()->getRegInfo(); for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { unsigned Reg = TargetRegisterInfo::index2VirtReg(i); if (PHISrcRegs.count(Reg) || !LIS->hasInterval(Reg)) continue; LiveInterval &LI = LIS->getInterval(Reg); if (!LI.liveAt(PrevIndex)) continue; bool isLiveOut = LI.liveAt(LIS->getMBBStartIdx(Succ)); if (isLiveOut && isLastMBB) { VNInfo *VNI = LI.getVNInfoAt(PrevIndex); assert(VNI && "LiveInterval should have VNInfo where it is live."); LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI)); } else if (!isLiveOut && !isLastMBB) { LI.removeSegment(StartIndex, EndIndex); } } // Update all intervals for registers whose uses may have been modified by // updateTerminator(). LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs); } if (MachineDominatorTree *MDT = P->getAnalysisIfAvailable<MachineDominatorTree>()) MDT->recordSplitCriticalEdge(this, Succ, NMBB); if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>()) if (MachineLoop *TIL = MLI->getLoopFor(this)) { // If one or the other blocks were not in a loop, the new block is not // either, and thus LI doesn't need to be updated. if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) { if (TIL == DestLoop) { // Both in the same loop, the NMBB joins loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (TIL->contains(DestLoop)) { // Edge from an outer loop to an inner loop. Add to the outer loop. TIL->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (DestLoop->contains(TIL)) { // Edge from an inner loop to an outer loop. Add to the outer loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else { // Edge from two loops with no containment relation. Because these // are natural loops, we know that the destination block must be the // header of its loop (adding a branch into a loop elsewhere would // create an irreducible loop). assert(DestLoop->getHeader() == Succ && "Should not create irreducible loops!"); if (MachineLoop *P = DestLoop->getParentLoop()) P->addBasicBlockToLoop(NMBB, MLI->getBase()); } } } return NMBB; }
void MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); const TargetInstrDesc &TI = MI->getDesc(); // The first TI.NumDefs operands must be explicit register defines if (MONum < TI.getNumDefs()) { if (!MO->isReg()) report("Explicit definition must be a register", MO, MONum); else if (!MO->isDef()) report("Explicit definition marked as use", MO, MONum); else if (MO->isImplicit()) report("Explicit definition marked as implicit", MO, MONum); } else if (MONum < TI.getNumOperands()) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. if (MO->isReg() && !(TI.isVariadic() && MONum == TI.getNumOperands()-1)) { if (MO->isDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) report("Explicit operand marked as implicit", MO, MONum); } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. if (MO->isReg() && !MO->isImplicit() && !TI.isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } switch (MO->getType()) { case MachineOperand::MO_Register: { const unsigned Reg = MO->getReg(); if (!Reg) return; // Check Live Variables. if (MO->isUndef()) { // An <undef> doesn't refer to any register, so just skip it. } else if (MO->isUse()) { regsLiveInButUnused.erase(Reg); bool isKill = false; unsigned defIdx; if (MI->isRegTiedToDefOperand(MONum, &defIdx)) { // A two-addr use counts as a kill if use and def are the same. unsigned DefReg = MI->getOperand(defIdx).getReg(); if (Reg == DefReg) { isKill = true; // And in that case an explicit kill flag is not allowed. if (MO->isKill()) report("Illegal kill flag on two-address instruction operand", MO, MONum); } else if (TargetRegisterInfo::isPhysicalRegister(Reg)) { report("Two-address instruction operands must be identical", MO, MONum); } } else isKill = MO->isKill(); if (isKill) addRegWithSubRegs(regsKilled, Reg); // Check that LiveVars knows this kill. if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) && MO->isKill()) { LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); if (std::find(VI.Kills.begin(), VI.Kills.end(), MI) == VI.Kills.end()) report("Kill missing from LiveVariables", MO, MONum); } // Check LiveInts liveness and kill. if (TargetRegisterInfo::isVirtualRegister(Reg) && LiveInts && !LiveInts->isNotInMIMap(MI)) { SlotIndex UseIdx = LiveInts->getInstructionIndex(MI).getUseIndex(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (!LI.liveAt(UseIdx)) { report("No live range at use", MO, MONum); *OS << UseIdx << " is not live in " << LI << '\n'; } // Verify isKill == LI.killedAt. if (!MI->isRegTiedToDefOperand(MONum)) { // MI could kill register without a kill flag on MO. bool miKill = MI->killsRegister(Reg); bool liKill = LI.killedAt(UseIdx.getDefIndex()); if (miKill && !liKill) { report("Live range continues after kill flag", MO, MONum); *OS << "Live range: " << LI << '\n'; } if (!miKill && liKill) { report("Live range ends without kill flag", MO, MONum); *OS << "Live range: " << LI << '\n'; } } } else { report("Virtual register has no Live interval", MO, MONum); } } // Use of a dead register. if (!regsLive.count(Reg)) { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { // Reserved registers may be used even when 'dead'. if (!isReserved(Reg)) report("Using an undefined physical register", MO, MONum); } else { BBInfo &MInfo = MBBInfoMap[MI->getParent()]; // We don't know which virtual registers are live in, so only complain // if vreg was killed in this MBB. Otherwise keep track of vregs that // must be live in. PHI instructions are handled separately. if (MInfo.regsKilled.count(Reg)) report("Using a killed virtual register", MO, MONum); else if (!MI->isPHI()) MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI)); } } } else { assert(MO->isDef()); // Register defined. // TODO: verify that earlyclobber ops are not used. if (MO->isDead()) addRegWithSubRegs(regsDead, Reg); else addRegWithSubRegs(regsDefined, Reg); // Check LiveInts for a live range, but only for virtual registers. if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) && !LiveInts->isNotInMIMap(MI)) { SlotIndex DefIdx = LiveInts->getInstructionIndex(MI).getDefIndex(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (const VNInfo *VNI = LI.getVNInfoAt(DefIdx)) { assert(VNI && "NULL valno is not allowed"); if (VNI->def != DefIdx) { report("Inconsistent valno->def", MO, MONum); *OS << "Valno " << VNI->id << " is not defined at " << DefIdx << " in " << LI << '\n'; } } else { report("No live range at def", MO, MONum); *OS << DefIdx << " is not live in " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } } // Check register classes. if (MONum < TI.getNumOperands() && !MO->isImplicit()) { const TargetOperandInfo &TOI = TI.OpInfo[MONum]; unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { unsigned sr = Reg; if (SubIdx) { unsigned s = TRI->getSubReg(Reg, SubIdx); if (!s) { report("Invalid subregister index for physical register", MO, MONum); return; } sr = s; } if (const TargetRegisterClass *DRC = TOI.getRegClass(TRI)) { if (!DRC->contains(sr)) { report("Illegal physical register for instruction", MO, MONum); *OS << TRI->getName(sr) << " is not a " << DRC->getName() << " register.\n"; } } } else { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { const TargetRegisterClass *SRC = RC->getSubRegisterRegClass(SubIdx); if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); *OS << "Register class " << RC->getName() << " does not support subreg index " << SubIdx << "\n"; return; } RC = SRC; } if (const TargetRegisterClass *DRC = TOI.getRegClass(TRI)) { if (RC != DRC && !RC->hasSuperClass(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " << RC->getName() << " register\n"; } } } } break; } case MachineOperand::MO_MachineBasicBlock: if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) report("PHI operand is not in the CFG", MO, MONum); break; case MachineOperand::MO_FrameIndex: if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); if (TI.mayLoad() && !LI.liveAt(Idx.getUseIndex())) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } if (TI.mayStore() && !LI.liveAt(Idx.getDefIndex())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } } break; default: break; } }
void LiveInterval::constructMainRangeFromSubranges( const SlotIndexes &Indexes, VNInfo::Allocator &VNIAllocator) { // The basic observations on which this algorithm is based: // - Each Def/ValNo in a subrange must have a corresponding def on the main // range, but not further defs/valnos are necessary. // - If any of the subranges is live at a point the main liverange has to be // live too, conversily if no subrange is live the main range mustn't be // live either. // We do this by scannig through all the subranges simultaneously creating new // segments in the main range as segments start/ends come up in the subranges. assert(hasSubRanges() && "expected subranges to be present"); assert(segments.empty() && valnos.empty() && "expected empty main range"); // Collect subrange, iterator pairs for the walk and determine first and last // SlotIndex involved. SmallVector<std::pair<const SubRange*, const_iterator>, 4> SRs; SlotIndex First; SlotIndex Last; for (const SubRange &SR : subranges()) { if (SR.empty()) continue; SRs.push_back(std::make_pair(&SR, SR.begin())); if (!First.isValid() || SR.segments.front().start < First) First = SR.segments.front().start; if (!Last.isValid() || SR.segments.back().end > Last) Last = SR.segments.back().end; } // Walk over all subranges simultaneously. Segment CurrentSegment; bool ConstructingSegment = false; bool NeedVNIFixup = false; unsigned ActiveMask = 0; SlotIndex Pos = First; while (true) { SlotIndex NextPos = Last; enum { NOTHING, BEGIN_SEGMENT, END_SEGMENT, } Event = NOTHING; // Which subregister lanes are affected by the current event. unsigned EventMask = 0; // Whether a BEGIN_SEGMENT is also a valno definition point. bool IsDef = false; // Find the next begin or end of a subrange segment. Combine masks if we // have multiple begins/ends at the same position. Ends take precedence over // Begins. for (auto &SRP : SRs) { const SubRange &SR = *SRP.first; const_iterator &I = SRP.second; // Advance iterator of subrange to a segment involving Pos; the earlier // segments are already merged at this point. while (I != SR.end() && (I->end < Pos || (I->end == Pos && (ActiveMask & SR.LaneMask) == 0))) ++I; if (I == SR.end()) continue; if ((ActiveMask & SR.LaneMask) == 0 && Pos <= I->start && I->start <= NextPos) { // Merge multiple begins at the same position. if (I->start == NextPos && Event == BEGIN_SEGMENT) { EventMask |= SR.LaneMask; IsDef |= I->valno->def == I->start; } else if (I->start < NextPos || Event != END_SEGMENT) { Event = BEGIN_SEGMENT; NextPos = I->start; EventMask = SR.LaneMask; IsDef = I->valno->def == I->start; } } if ((ActiveMask & SR.LaneMask) != 0 && Pos <= I->end && I->end <= NextPos) { // Merge multiple ends at the same position. if (I->end == NextPos && Event == END_SEGMENT) EventMask |= SR.LaneMask; else { Event = END_SEGMENT; NextPos = I->end; EventMask = SR.LaneMask; } } } // Advance scan position. Pos = NextPos; if (Event == BEGIN_SEGMENT) { if (ConstructingSegment && IsDef) { // Finish previous segment because we have to start a new one. CurrentSegment.end = Pos; append(CurrentSegment); ConstructingSegment = false; } // Start a new segment if necessary. if (!ConstructingSegment) { // Determine value number for the segment. VNInfo *VNI; if (IsDef) { VNI = getNextValue(Pos, VNIAllocator); } else { // We have to reuse an existing value number, if we are lucky // then we already passed one of the predecessor blocks and determined // its value number (with blocks in reverse postorder this would be // always true but we have no such guarantee). assert(Pos.isBlock()); const MachineBasicBlock *MBB = Indexes.getMBBFromIndex(Pos); // See if any of the predecessor blocks has a lower number and a VNI for (const MachineBasicBlock *Pred : MBB->predecessors()) { SlotIndex PredEnd = Indexes.getMBBEndIdx(Pred); VNI = getVNInfoBefore(PredEnd); if (VNI != nullptr) break; } // Def will come later: We have to do an extra fixup pass. if (VNI == nullptr) NeedVNIFixup = true; } // In rare cases we can produce adjacent segments with the same value // number (if they come from different subranges, but happen to have // the same defining instruction). VNIFixup will fix those cases. if (!empty() && segments.back().end == Pos && segments.back().valno == VNI) NeedVNIFixup = true; CurrentSegment.start = Pos; CurrentSegment.valno = VNI; ConstructingSegment = true; } ActiveMask |= EventMask; } else if (Event == END_SEGMENT) { assert(ConstructingSegment); // Finish segment if no lane is active anymore. ActiveMask &= ~EventMask; if (ActiveMask == 0) { CurrentSegment.end = Pos; append(CurrentSegment); ConstructingSegment = false; } } else { // We reached the end of the last subranges and can stop. assert(Event == NOTHING); break; } } // We might not be able to assign new valnos for all segments if the basic // block containing the definition comes after a segment using the valno. // Do a fixup pass for this uncommon case. if (NeedVNIFixup) determineMissingVNIs(Indexes, *this); assert(ActiveMask == 0 && !ConstructingSegment && "all segments ended"); verify(); }
// mapValue - Find the mapped value for ParentVNI at Idx. // Potentially create phi-def values. VNInfo *LiveIntervalMap::mapValue(const VNInfo *ParentVNI, SlotIndex Idx) { assert(ParentVNI && "Mapping NULL value"); assert(Idx.isValid() && "Invalid SlotIndex"); assert(parentli_.getVNInfoAt(Idx) == ParentVNI && "Bad ParentVNI"); // Use insert for lookup, so we can add missing values with a second lookup. std::pair<ValueMap::iterator,bool> InsP = valueMap_.insert(ValueMap::value_type(ParentVNI, static_cast<VNInfo *>(0))); // The static_cast<VNInfo *> is only needed to work around a bug in an // old version of the C++0x standard which the following compilers // implemented and have yet to fix: // // Microsoft Visual Studio 2010 Version 10.0.30319.1 RTMRel // Microsoft (R) 32-bit C/C++ Optimizing Compiler Version 16.00.30319.01 // // If/When we move to C++0x, this can be replaced by nullptr. // This was an unknown value. Create a simple mapping. if (InsP.second) return InsP.first->second = li_.createValueCopy(ParentVNI, lis_.getVNInfoAllocator()); // This was a simple mapped value. if (InsP.first->second) return InsP.first->second; // This is a complex mapped value. There may be multiple defs, and we may need // to create phi-defs. MachineBasicBlock *IdxMBB = lis_.getMBBFromIndex(Idx); assert(IdxMBB && "No MBB at Idx"); // Is there a def in the same MBB we can extend? if (VNInfo *VNI = extendTo(IdxMBB, Idx)) return VNI; // Now for the fun part. We know that ParentVNI potentially has multiple defs, // and we may need to create even more phi-defs to preserve VNInfo SSA form. // Perform a depth-first search for predecessor blocks where we know the // dominating VNInfo. Insert phi-def VNInfos along the path back to IdxMBB. // Track MBBs where we have created or learned the dominating value. // This may change during the DFS as we create new phi-defs. typedef DenseMap<MachineBasicBlock*, VNInfo*> MBBValueMap; MBBValueMap DomValue; for (idf_iterator<MachineBasicBlock*> IDFI = idf_begin(IdxMBB), IDFE = idf_end(IdxMBB); IDFI != IDFE;) { MachineBasicBlock *MBB = *IDFI; SlotIndex End = lis_.getMBBEndIdx(MBB); // We are operating on the restricted CFG where ParentVNI is live. if (parentli_.getVNInfoAt(End.getPrevSlot()) != ParentVNI) { IDFI.skipChildren(); continue; } // Do we have a dominating value in this block? VNInfo *VNI = extendTo(MBB, End); if (!VNI) { ++IDFI; continue; } // Yes, VNI dominates MBB. Track the path back to IdxMBB, creating phi-defs // as needed along the way. for (unsigned PI = IDFI.getPathLength()-1; PI != 0; --PI) { // Start from MBB's immediate successor. End at IdxMBB. MachineBasicBlock *Succ = IDFI.getPath(PI-1); std::pair<MBBValueMap::iterator, bool> InsP = DomValue.insert(MBBValueMap::value_type(Succ, VNI)); // This is the first time we backtrack to Succ. if (InsP.second) continue; // We reached Succ again with the same VNI. Nothing is going to change. VNInfo *OVNI = InsP.first->second; if (OVNI == VNI) break; // Succ already has a phi-def. No need to continue. SlotIndex Start = lis_.getMBBStartIdx(Succ); if (OVNI->def == Start) break; // We have a collision between the old and new VNI at Succ. That means // neither dominates and we need a new phi-def. VNI = li_.getNextValue(Start, 0, true, lis_.getVNInfoAllocator()); VNI->setIsPHIDef(true); InsP.first->second = VNI; // Replace OVNI with VNI in the remaining path. for (; PI > 1 ; --PI) { MBBValueMap::iterator I = DomValue.find(IDFI.getPath(PI-2)); if (I == DomValue.end() || I->second != OVNI) break; I->second = VNI; } } // No need to search the children, we found a dominating value. IDFI.skipChildren(); } // The search should at least find a dominating value for IdxMBB. assert(!DomValue.empty() && "Couldn't find a reaching definition"); // Since we went through the trouble of a full DFS visiting all reaching defs, // the values in DomValue are now accurate. No more phi-defs are needed for // these blocks, so we can color the live ranges. // This makes the next mapValue call much faster. VNInfo *IdxVNI = 0; for (MBBValueMap::iterator I = DomValue.begin(), E = DomValue.end(); I != E; ++I) { MachineBasicBlock *MBB = I->first; VNInfo *VNI = I->second; SlotIndex Start = lis_.getMBBStartIdx(MBB); if (MBB == IdxMBB) { // Don't add full liveness to IdxMBB, stop at Idx. if (Start != Idx) li_.addRange(LiveRange(Start, Idx, VNI)); // The caller had better add some liveness to IdxVNI, or it leaks. IdxVNI = VNI; } else li_.addRange(LiveRange(Start, lis_.getMBBEndIdx(MBB), VNI)); } assert(IdxVNI && "Didn't find value for Idx"); return IdxVNI; }
/// spillAroundUses - insert spill code around each use of Reg. void InlineSpiller::spillAroundUses(unsigned Reg) { DEBUG(dbgs() << "spillAroundUses " << PrintReg(Reg) << '\n'); LiveInterval &OldLI = LIS.getInterval(Reg); // Iterate over instructions using Reg. for (MachineRegisterInfo::reg_bundle_iterator RegI = MRI.reg_bundle_begin(Reg), E = MRI.reg_bundle_end(); RegI != E; ) { MachineInstr *MI = &*(RegI++); // Debug values are not allowed to affect codegen. if (MI->isDebugValue()) { // Modify DBG_VALUE now that the value is in a spill slot. bool IsIndirect = MI->isIndirectDebugValue(); uint64_t Offset = IsIndirect ? MI->getOperand(1).getImm() : 0; const MDNode *Var = MI->getDebugVariable(); const MDNode *Expr = MI->getDebugExpression(); DebugLoc DL = MI->getDebugLoc(); DEBUG(dbgs() << "Modifying debug info due to spill:" << "\t" << *MI); MachineBasicBlock *MBB = MI->getParent(); assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) && "Expected inlined-at fields to agree"); BuildMI(*MBB, MBB->erase(MI), DL, TII.get(TargetOpcode::DBG_VALUE)) .addFrameIndex(StackSlot) .addImm(Offset) .addMetadata(Var) .addMetadata(Expr); continue; } // Ignore copies to/from snippets. We'll delete them. if (SnippetCopies.count(MI)) continue; // Stack slot accesses may coalesce away. if (coalesceStackAccess(MI, Reg)) continue; // Analyze instruction. SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops; MIBundleOperands::VirtRegInfo RI = MIBundleOperands(MI).analyzeVirtReg(Reg, &Ops); // Find the slot index where this instruction reads and writes OldLI. // This is usually the def slot, except for tied early clobbers. SlotIndex Idx = LIS.getInstructionIndex(MI).getRegSlot(); if (VNInfo *VNI = OldLI.getVNInfoAt(Idx.getRegSlot(true))) if (SlotIndex::isSameInstr(Idx, VNI->def)) Idx = VNI->def; // Check for a sibling copy. unsigned SibReg = isFullCopyOf(MI, Reg); if (SibReg && isSibling(SibReg)) { // This may actually be a copy between snippets. if (isRegToSpill(SibReg)) { DEBUG(dbgs() << "Found new snippet copy: " << *MI); SnippetCopies.insert(MI); continue; } if (RI.Writes) { // Hoist the spill of a sib-reg copy. if (hoistSpill(OldLI, MI)) { // This COPY is now dead, the value is already in the stack slot. MI->getOperand(0).setIsDead(); DeadDefs.push_back(MI); continue; } } else { // This is a reload for a sib-reg copy. Drop spills downstream. LiveInterval &SibLI = LIS.getInterval(SibReg); eliminateRedundantSpills(SibLI, SibLI.getVNInfoAt(Idx)); // The COPY will fold to a reload below. } } // Attempt to fold memory ops. if (foldMemoryOperand(Ops)) continue; // Create a new virtual register for spill/fill. // FIXME: Infer regclass from instruction alone. unsigned NewVReg = Edit->createFrom(Reg); if (RI.Reads) insertReload(NewVReg, Idx, MI); // Rewrite instruction operands. bool hasLiveDef = false; for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = Ops[i].first->getOperand(Ops[i].second); MO.setReg(NewVReg); if (MO.isUse()) { if (!Ops[i].first->isRegTiedToDefOperand(Ops[i].second)) MO.setIsKill(); } else { if (!MO.isDead()) hasLiveDef = true; } } DEBUG(dbgs() << "\trewrite: " << Idx << '\t' << *MI << '\n'); // FIXME: Use a second vreg if instruction has no tied ops. if (RI.Writes) if (hasLiveDef) insertSpill(NewVReg, true, MI); } }
/// reMaterializeFor - Attempt to rematerialize before MI instead of reloading. bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineBasicBlock::iterator MI) { // Analyze instruction SmallVector<std::pair<MachineInstr *, unsigned>, 8> Ops; MIBundleOperands::VirtRegInfo RI = MIBundleOperands(MI).analyzeVirtReg(VirtReg.reg, &Ops); if (!RI.Reads) return false; SlotIndex UseIdx = LIS.getInstructionIndex(MI).getRegSlot(true); VNInfo *ParentVNI = VirtReg.getVNInfoAt(UseIdx.getBaseIndex()); if (!ParentVNI) { DEBUG(dbgs() << "\tadding <undef> flags: "); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) MO.setIsUndef(); } DEBUG(dbgs() << UseIdx << '\t' << *MI); return true; } if (SnippetCopies.count(MI)) return false; // Use an OrigVNI from traceSiblingValue when ParentVNI is a sibling copy. LiveRangeEdit::Remat RM(ParentVNI); SibValueMap::const_iterator SibI = SibValues.find(ParentVNI); if (SibI != SibValues.end()) RM.OrigMI = SibI->second.DefMI; if (!Edit->canRematerializeAt(RM, UseIdx, false)) { markValueUsed(&VirtReg, ParentVNI); DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI); return false; } // If the instruction also writes VirtReg.reg, it had better not require the // same register for uses and defs. if (RI.Tied) { markValueUsed(&VirtReg, ParentVNI); DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << *MI); return false; } // Before rematerializing into a register for a single instruction, try to // fold a load into the instruction. That avoids allocating a new register. if (RM.OrigMI->canFoldAsLoad() && foldMemoryOperand(Ops, RM.OrigMI)) { Edit->markRematerialized(RM.ParentVNI); ++NumFoldedLoads; return true; } // Alocate a new register for the remat. unsigned NewVReg = Edit->createFrom(Original); // Finally we can rematerialize OrigMI before MI. SlotIndex DefIdx = Edit->rematerializeAt(*MI->getParent(), MI, NewVReg, RM, TRI); (void)DefIdx; DEBUG(dbgs() << "\tremat: " << DefIdx << '\t' << *LIS.getInstructionFromIndex(DefIdx)); // Replace operands for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i].second); if (MO.isReg() && MO.isUse() && MO.getReg() == VirtReg.reg) { MO.setReg(NewVReg); MO.setIsKill(); } } DEBUG(dbgs() << "\t " << UseIdx << '\t' << *MI << '\n'); ++NumRemats; return true; }
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &MF) { MRI = &MF.getRegInfo(); TII = MF.getTarget().getInstrInfo(); DT = &getAnalysis<MachineDominatorTree>(); LI = &getAnalysis<LiveIntervals>(); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { unsigned DestReg = BBI->getOperand(0).getReg(); addReg(DestReg); PHISrcDefs[I].push_back(BBI); for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) { MachineOperand &SrcMO = BBI->getOperand(i); unsigned SrcReg = SrcMO.getReg(); addReg(SrcReg); unionRegs(DestReg, SrcReg); MachineInstr *DefMI = MRI->getVRegDef(SrcReg); if (DefMI) PHISrcDefs[DefMI->getParent()].push_back(DefMI); } } } // Perform a depth-first traversal of the dominator tree, splitting // interferences amongst PHI-congruence classes. DenseMap<unsigned, unsigned> CurrentDominatingParent; DenseMap<unsigned, unsigned> ImmediateDominatingParent; for (df_iterator<MachineDomTreeNode*> DI = df_begin(DT->getRootNode()), DE = df_end(DT->getRootNode()); DI != DE; ++DI) { SplitInterferencesForBasicBlock(*DI->getBlock(), CurrentDominatingParent, ImmediateDominatingParent); } // Insert copies for all PHI source and destination registers. for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { InsertCopiesForPHI(BBI, I); } } // FIXME: Preserve the equivalence classes during copy insertion and use // the preversed equivalence classes instead of recomputing them. RegNodeMap.clear(); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); BBI != BBE && BBI->isPHI(); ++BBI) { unsigned DestReg = BBI->getOperand(0).getReg(); addReg(DestReg); for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) { unsigned SrcReg = BBI->getOperand(i).getReg(); addReg(SrcReg); unionRegs(DestReg, SrcReg); } } } DenseMap<unsigned, unsigned> RegRenamingMap; bool Changed = false; for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end(); while (BBI != BBE && BBI->isPHI()) { MachineInstr *PHI = BBI; assert(PHI->getNumOperands() > 0); unsigned SrcReg = PHI->getOperand(1).getReg(); unsigned SrcColor = getRegColor(SrcReg); unsigned NewReg = RegRenamingMap[SrcColor]; if (!NewReg) { NewReg = SrcReg; RegRenamingMap[SrcColor] = SrcReg; } MergeLIsAndRename(SrcReg, NewReg); unsigned DestReg = PHI->getOperand(0).getReg(); if (!InsertedDestCopies.count(DestReg)) MergeLIsAndRename(DestReg, NewReg); for (unsigned i = 3; i < PHI->getNumOperands(); i += 2) { unsigned SrcReg = PHI->getOperand(i).getReg(); MergeLIsAndRename(SrcReg, NewReg); } ++BBI; LI->RemoveMachineInstrFromMaps(PHI); PHI->eraseFromParent(); Changed = true; } } // Due to the insertion of copies to split live ranges, the live intervals are // guaranteed to not overlap, except in one case: an original PHI source and a // PHI destination copy. In this case, they have the same value and thus don't // truly intersect, so we merge them into the value live at that point. // FIXME: Is there some better way we can handle this? for (DestCopyMap::iterator I = InsertedDestCopies.begin(), E = InsertedDestCopies.end(); I != E; ++I) { unsigned DestReg = I->first; unsigned DestColor = getRegColor(DestReg); unsigned NewReg = RegRenamingMap[DestColor]; LiveInterval &DestLI = LI->getInterval(DestReg); LiveInterval &NewLI = LI->getInterval(NewReg); assert(DestLI.ranges.size() == 1 && "PHI destination copy's live interval should be a single live " "range from the beginning of the BB to the copy instruction."); LiveRange *DestLR = DestLI.begin(); VNInfo *NewVNI = NewLI.getVNInfoAt(DestLR->start); if (!NewVNI) { NewVNI = NewLI.createValueCopy(DestLR->valno, LI->getVNInfoAllocator()); MachineInstr *CopyInstr = I->second; CopyInstr->getOperand(1).setIsKill(true); } LiveRange NewLR(DestLR->start, DestLR->end, NewVNI); NewLI.addRange(NewLR); LI->removeInterval(DestReg); MRI->replaceRegWith(DestReg, NewReg); } // Adjust the live intervals of all PHI source registers to handle the case // where the PHIs in successor blocks were the only later uses of the source // register. for (SrcCopySet::iterator I = InsertedSrcCopySet.begin(), E = InsertedSrcCopySet.end(); I != E; ++I) { MachineBasicBlock *MBB = I->first; unsigned SrcReg = I->second; if (unsigned RenamedRegister = RegRenamingMap[getRegColor(SrcReg)]) SrcReg = RenamedRegister; LiveInterval &SrcLI = LI->getInterval(SrcReg); bool isLiveOut = false; for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(), SE = MBB->succ_end(); SI != SE; ++SI) { if (SrcLI.liveAt(LI->getMBBStartIdx(*SI))) { isLiveOut = true; break; } } if (isLiveOut) continue; MachineOperand *LastUse = findLastUse(MBB, SrcReg); assert(LastUse); SlotIndex LastUseIndex = LI->getInstructionIndex(LastUse->getParent()); SrcLI.removeRange(LastUseIndex.getDefIndex(), LI->getMBBEndIdx(MBB)); LastUse->setIsKill(true); } LI->renumber(); Allocator.Reset(); RegNodeMap.clear(); PHISrcDefs.clear(); InsertedSrcCopySet.clear(); InsertedSrcCopyMap.clear(); InsertedDestCopies.clear(); return Changed; }
void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI, MachineBasicBlock *MBB) { assert(PHI->isPHI()); ++NumPHIsLowered; unsigned PHIColor = getPHIColor(PHI); for (unsigned i = 1; i < PHI->getNumOperands(); i += 2) { MachineOperand &SrcMO = PHI->getOperand(i); // If a source is defined by an implicit def, there is no need to insert a // copy in the predecessor. if (SrcMO.isUndef()) continue; unsigned SrcReg = SrcMO.getReg(); assert(TargetRegisterInfo::isVirtualRegister(SrcReg) && "Machine PHI Operands must all be virtual registers!"); MachineBasicBlock *PredBB = PHI->getOperand(i + 1).getMBB(); unsigned SrcColor = getRegColor(SrcReg); // If neither the PHI nor the operand were isolated, then we only need to // set the phi-kill flag on the VNInfo at this PHI. if (PHIColor && SrcColor == PHIColor) { LiveInterval &SrcInterval = LI->getInterval(SrcReg); SlotIndex PredIndex = LI->getMBBEndIdx(PredBB); VNInfo *SrcVNI = SrcInterval.getVNInfoBefore(PredIndex); assert(SrcVNI); SrcVNI->setHasPHIKill(true); continue; } unsigned CopyReg = 0; if (PHIColor) { SrcCopyMap::const_iterator I = InsertedSrcCopyMap.find(std::make_pair(PredBB, PHIColor)); CopyReg = I != InsertedSrcCopyMap.end() ? I->second->getOperand(0).getReg() : 0; } if (!CopyReg) { const TargetRegisterClass *RC = MRI->getRegClass(SrcReg); CopyReg = MRI->createVirtualRegister(RC); MachineBasicBlock::iterator CopyInsertPoint = findPHICopyInsertPoint(PredBB, MBB, SrcReg); unsigned SrcSubReg = SrcMO.getSubReg(); MachineInstr *CopyInstr = BuildMI(*PredBB, CopyInsertPoint, PHI->getDebugLoc(), TII->get(TargetOpcode::COPY), CopyReg).addReg(SrcReg, 0, SrcSubReg); LI->InsertMachineInstrInMaps(CopyInstr); ++NumSrcCopiesInserted; // addLiveRangeToEndOfBlock() also adds the phikill flag to the VNInfo for // the newly added range. LI->addLiveRangeToEndOfBlock(CopyReg, CopyInstr); InsertedSrcCopySet.insert(std::make_pair(PredBB, SrcReg)); addReg(CopyReg); if (PHIColor) { unionRegs(PHIColor, CopyReg); assert(getRegColor(CopyReg) != CopyReg); } else { PHIColor = CopyReg; assert(getRegColor(CopyReg) == CopyReg); } if (!InsertedSrcCopyMap.count(std::make_pair(PredBB, PHIColor))) InsertedSrcCopyMap[std::make_pair(PredBB, PHIColor)] = CopyInstr; } SrcMO.setReg(CopyReg); // If SrcReg is not live beyond the PHI, trim its interval so that it is no // longer live-in to MBB. Note that SrcReg may appear in other PHIs that are // processed later, but this is still correct to do at this point because we // never rely on LiveIntervals being correct while inserting copies. // FIXME: Should this just count uses at PHIs like the normal PHIElimination // pass does? LiveInterval &SrcLI = LI->getInterval(SrcReg); SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); SlotIndex PHIIndex = LI->getInstructionIndex(PHI); SlotIndex NextInstrIndex = PHIIndex.getNextIndex(); if (SrcLI.liveAt(MBBStartIndex) && SrcLI.expiredAt(NextInstrIndex)) SrcLI.removeRange(MBBStartIndex, PHIIndex, true); } unsigned DestReg = PHI->getOperand(0).getReg(); unsigned DestColor = getRegColor(DestReg); if (PHIColor && DestColor == PHIColor) { LiveInterval &DestLI = LI->getInterval(DestReg); // Set the phi-def flag for the VN at this PHI. SlotIndex PHIIndex = LI->getInstructionIndex(PHI); VNInfo *DestVNI = DestLI.getVNInfoAt(PHIIndex.getDefIndex()); assert(DestVNI); DestVNI->setIsPHIDef(true); // Prior to PHI elimination, the live ranges of PHIs begin at their defining // instruction. After PHI elimination, PHI instructions are replaced by VNs // with the phi-def flag set, and the live ranges of these VNs start at the // beginning of the basic block. SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); DestVNI->def = MBBStartIndex; DestLI.addRange(LiveRange(MBBStartIndex, PHIIndex.getDefIndex(), DestVNI)); return; } const TargetRegisterClass *RC = MRI->getRegClass(DestReg); unsigned CopyReg = MRI->createVirtualRegister(RC); MachineInstr *CopyInstr = BuildMI(*MBB, MBB->SkipPHIsAndLabels(MBB->begin()), PHI->getDebugLoc(), TII->get(TargetOpcode::COPY), DestReg).addReg(CopyReg); LI->InsertMachineInstrInMaps(CopyInstr); PHI->getOperand(0).setReg(CopyReg); ++NumDestCopiesInserted; // Add the region from the beginning of MBB to the copy instruction to // CopyReg's live interval, and give the VNInfo the phidef flag. LiveInterval &CopyLI = LI->getOrCreateInterval(CopyReg); SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB); SlotIndex DestCopyIndex = LI->getInstructionIndex(CopyInstr); VNInfo *CopyVNI = CopyLI.getNextValue(MBBStartIndex, CopyInstr, LI->getVNInfoAllocator()); CopyVNI->setIsPHIDef(true); CopyLI.addRange(LiveRange(MBBStartIndex, DestCopyIndex.getDefIndex(), CopyVNI)); // Adjust DestReg's live interval to adjust for its new definition at // CopyInstr. LiveInterval &DestLI = LI->getOrCreateInterval(DestReg); SlotIndex PHIIndex = LI->getInstructionIndex(PHI); DestLI.removeRange(PHIIndex.getDefIndex(), DestCopyIndex.getDefIndex()); VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getDefIndex()); assert(DestVNI); DestVNI->def = DestCopyIndex.getDefIndex(); InsertedDestCopies[CopyReg] = CopyInstr; }
/// reMaterializeFor - Attempt to rematerialize li_->reg before MI instead of /// reloading it. bool InlineSpiller::reMaterializeFor(MachineBasicBlock::iterator MI) { SlotIndex UseIdx = lis_.getInstructionIndex(MI).getUseIndex(); VNInfo *OrigVNI = li_->getVNInfoAt(UseIdx); if (!OrigVNI) { DEBUG(dbgs() << "\tadding <undef> flags: "); for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg) MO.setIsUndef(); } DEBUG(dbgs() << UseIdx << '\t' << *MI); return true; } if (!reMattable_.count(OrigVNI)) { DEBUG(dbgs() << "\tusing non-remat valno " << OrigVNI->id << ": " << UseIdx << '\t' << *MI); return false; } MachineInstr *OrigMI = lis_.getInstructionFromIndex(OrigVNI->def); if (!allUsesAvailableAt(OrigMI, OrigVNI->def, UseIdx)) { usedValues_.insert(OrigVNI); DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI); return false; } // If the instruction also writes li_->reg, it had better not require the same // register for uses and defs. bool Reads, Writes; SmallVector<unsigned, 8> Ops; tie(Reads, Writes) = MI->readsWritesVirtualRegister(li_->reg, &Ops); if (Writes) { for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); if (MO.isUse() ? MI->isRegTiedToDefOperand(Ops[i]) : MO.getSubReg()) { usedValues_.insert(OrigVNI); DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << *MI); return false; } } } // Alocate a new register for the remat. unsigned NewVReg = mri_.createVirtualRegister(rc_); vrm_.grow(); LiveInterval &NewLI = lis_.getOrCreateInterval(NewVReg); NewLI.markNotSpillable(); newIntervals_->push_back(&NewLI); // Finally we can rematerialize OrigMI before MI. MachineBasicBlock &MBB = *MI->getParent(); tii_.reMaterialize(MBB, MI, NewLI.reg, 0, OrigMI, tri_); MachineBasicBlock::iterator RematMI = MI; SlotIndex DefIdx = lis_.InsertMachineInstrInMaps(--RematMI).getDefIndex(); DEBUG(dbgs() << "\tremat: " << DefIdx << '\t' << *RematMI); // Replace operands for (unsigned i = 0, e = Ops.size(); i != e; ++i) { MachineOperand &MO = MI->getOperand(Ops[i]); if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg) { MO.setReg(NewVReg); MO.setIsKill(); } } DEBUG(dbgs() << "\t " << UseIdx << '\t' << *MI); VNInfo *DefVNI = NewLI.getNextValue(DefIdx, 0, true, lis_.getVNInfoAllocator()); NewLI.addRange(LiveRange(DefIdx, UseIdx.getDefIndex(), DefVNI)); DEBUG(dbgs() << "\tinterval: " << NewLI << '\n'); return true; }
void MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); const MCInstrDesc &MCID = MI->getDesc(); const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; // The first MCID.NumDefs operands must be explicit register defines if (MONum < MCID.getNumDefs()) { if (!MO->isReg()) report("Explicit definition must be a register", MO, MONum); else if (!MO->isDef()) report("Explicit definition marked as use", MO, MONum); else if (MO->isImplicit()) report("Explicit definition marked as implicit", MO, MONum); } else if (MONum < MCID.getNumOperands()) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. if (MO->isReg() && !(MI->isVariadic() && MONum == MCID.getNumOperands()-1)) { if (MO->isDef() && !MCOI.isOptionalDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) report("Explicit operand marked as implicit", MO, MONum); } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } switch (MO->getType()) { case MachineOperand::MO_Register: { const unsigned Reg = MO->getReg(); if (!Reg) return; // Check Live Variables. if (MI->isDebugValue()) { // Liveness checks are not valid for debug values. } else if (MO->isUse() && !MO->isUndef()) { regsLiveInButUnused.erase(Reg); bool isKill = false; unsigned defIdx; if (MI->isRegTiedToDefOperand(MONum, &defIdx)) { // A two-addr use counts as a kill if use and def are the same. unsigned DefReg = MI->getOperand(defIdx).getReg(); if (Reg == DefReg) isKill = true; else if (TargetRegisterInfo::isPhysicalRegister(Reg)) { report("Two-address instruction operands must be identical", MO, MONum); } } else isKill = MO->isKill(); if (isKill) addRegWithSubRegs(regsKilled, Reg); // Check that LiveVars knows this kill. if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) && MO->isKill()) { LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); if (std::find(VI.Kills.begin(), VI.Kills.end(), MI) == VI.Kills.end()) report("Kill missing from LiveVariables", MO, MONum); } // Check LiveInts liveness and kill. if (TargetRegisterInfo::isVirtualRegister(Reg) && LiveInts && !LiveInts->isNotInMIMap(MI)) { SlotIndex UseIdx = LiveInts->getInstructionIndex(MI).getRegSlot(true); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (!LI.liveAt(UseIdx)) { report("No live range at use", MO, MONum); *OS << UseIdx << " is not live in " << LI << '\n'; } // Check for extra kill flags. // Note that we allow missing kill flags for now. if (MO->isKill() && !LI.killedAt(UseIdx.getRegSlot())) { report("Live range continues after kill flag", MO, MONum); *OS << "Live range: " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } // Use of a dead register. if (!regsLive.count(Reg)) { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { // Reserved registers may be used even when 'dead'. if (!isReserved(Reg)) report("Using an undefined physical register", MO, MONum); } else { BBInfo &MInfo = MBBInfoMap[MI->getParent()]; // We don't know which virtual registers are live in, so only complain // if vreg was killed in this MBB. Otherwise keep track of vregs that // must be live in. PHI instructions are handled separately. if (MInfo.regsKilled.count(Reg)) report("Using a killed virtual register", MO, MONum); else if (!MI->isPHI()) MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI)); } } } else if (MO->isDef()) { // Register defined. // TODO: verify that earlyclobber ops are not used. if (MO->isDead()) addRegWithSubRegs(regsDead, Reg); else addRegWithSubRegs(regsDefined, Reg); // Verify SSA form. if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) && llvm::next(MRI->def_begin(Reg)) != MRI->def_end()) report("Multiple virtual register defs in SSA form", MO, MONum); // Check LiveInts for a live range, but only for virtual registers. if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) && !LiveInts->isNotInMIMap(MI)) { SlotIndex DefIdx = LiveInts->getInstructionIndex(MI).getRegSlot(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (const VNInfo *VNI = LI.getVNInfoAt(DefIdx)) { assert(VNI && "NULL valno is not allowed"); if (VNI->def != DefIdx && !MO->isEarlyClobber()) { report("Inconsistent valno->def", MO, MONum); *OS << "Valno " << VNI->id << " is not defined at " << DefIdx << " in " << LI << '\n'; } } else { report("No live range at def", MO, MONum); *OS << DefIdx << " is not live in " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } } // Check register classes. if (MONum < MCID.getNumOperands() && !MO->isImplicit()) { unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { if (SubIdx) { report("Illegal subregister index for physical register", MO, MONum); return; } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (!DRC->contains(Reg)) { report("Illegal physical register for instruction", MO, MONum); *OS << TRI->getName(Reg) << " is not a " << DRC->getName() << " register.\n"; } } } else { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { const TargetRegisterClass *SRC = TRI->getSubClassWithSubReg(RC, SubIdx); if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); *OS << "Register class " << RC->getName() << " does not support subreg index " << SubIdx << "\n"; return; } if (RC != SRC) { report("Invalid register class for subregister index", MO, MONum); *OS << "Register class " << RC->getName() << " does not fully support subreg index " << SubIdx << "\n"; return; } } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) { if (SubIdx) { const TargetRegisterClass *SuperRC = TRI->getLargestLegalSuperClass(RC); if (!SuperRC) { report("No largest legal super class exists.", MO, MONum); return; } DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); if (!DRC) { report("No matching super-reg register class.", MO, MONum); return; } } if (!RC->hasSuperClassEq(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " << RC->getName() << " register\n"; } } } } break; } case MachineOperand::MO_RegisterMask: regMasks.push_back(MO->getRegMask()); break; case MachineOperand::MO_MachineBasicBlock: if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) report("PHI operand is not in the CFG", MO, MONum); break; case MachineOperand::MO_FrameIndex: if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); if (MI->mayLoad() && !LI.liveAt(Idx.getRegSlot(true))) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } if (MI->mayStore() && !LI.liveAt(Idx.getRegSlot())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } } break; default: break; } }
MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair( MachineBasicBlock::iterator I, MachineBasicBlock::iterator Paired, unsigned EltSize) { MachineBasicBlock *MBB = I->getParent(); // Be sure to use .addOperand(), and not .addReg() with these. We want to be // sure we preserve the subregister index and any register flags set on them. const MachineOperand *Addr = TII->getNamedOperand(*I, AMDGPU::OpName::addr); const MachineOperand *Data0 = TII->getNamedOperand(*I, AMDGPU::OpName::data0); const MachineOperand *Data1 = TII->getNamedOperand(*Paired, AMDGPU::OpName::data0); unsigned Offset0 = TII->getNamedOperand(*I, AMDGPU::OpName::offset)->getImm() & 0xffff; unsigned Offset1 = TII->getNamedOperand(*Paired, AMDGPU::OpName::offset)->getImm() & 0xffff; unsigned NewOffset0 = Offset0 / EltSize; unsigned NewOffset1 = Offset1 / EltSize; unsigned Opc = (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64; // Prefer the st64 form if we can use it, even if we can fit the offset in the // non st64 version. I'm not sure if there's any real reason to do this. bool UseST64 = (NewOffset0 % 64 == 0) && (NewOffset1 % 64 == 0); if (UseST64) { NewOffset0 /= 64; NewOffset1 /= 64; Opc = (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32 : AMDGPU::DS_WRITE2ST64_B64; } assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) && (NewOffset0 != NewOffset1) && "Computed offset doesn't fit"); const MCInstrDesc &Write2Desc = TII->get(Opc); DebugLoc DL = I->getDebugLoc(); // repairLiveintervalsInRange() doesn't handle physical register, so we have // to update the M0 range manually. SlotIndex PairedIndex = LIS->getInstructionIndex(*Paired); LiveRange &M0Range = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::M0, TRI)); LiveRange::Segment *M0Segment = M0Range.getSegmentContaining(PairedIndex); bool UpdateM0Range = M0Segment->end == PairedIndex.getRegSlot(); MachineInstrBuilder Write2 = BuildMI(*MBB, I, DL, Write2Desc) .addOperand(*Addr) // addr .addOperand(*Data0) // data0 .addOperand(*Data1) // data1 .addImm(NewOffset0) // offset0 .addImm(NewOffset1) // offset1 .addImm(0) // gds .addMemOperand(*I->memoperands_begin()) .addMemOperand(*Paired->memoperands_begin()); // XXX - How do we express subregisters here? unsigned OrigRegs[] = { Data0->getReg(), Data1->getReg(), Addr->getReg() }; LIS->RemoveMachineInstrFromMaps(*I); LIS->RemoveMachineInstrFromMaps(*Paired); I->eraseFromParent(); Paired->eraseFromParent(); // This doesn't handle physical registers like M0 LIS->repairIntervalsInRange(MBB, Write2, Write2, OrigRegs); if (UpdateM0Range) { SlotIndex Write2Index = LIS->getInstructionIndex(*Write2); M0Segment->end = Write2Index.getRegSlot(); } DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n'); return Write2.getInstr(); }
MachineBasicBlock::iterator SILoadStoreOptimizer::mergeRead2Pair( MachineBasicBlock::iterator I, MachineBasicBlock::iterator Paired, unsigned EltSize) { MachineBasicBlock *MBB = I->getParent(); // Be careful, since the addresses could be subregisters themselves in weird // cases, like vectors of pointers. const MachineOperand *AddrReg = TII->getNamedOperand(*I, AMDGPU::OpName::addr); const MachineOperand *Dest0 = TII->getNamedOperand(*I, AMDGPU::OpName::vdst); const MachineOperand *Dest1 = TII->getNamedOperand(*Paired, AMDGPU::OpName::vdst); unsigned Offset0 = TII->getNamedOperand(*I, AMDGPU::OpName::offset)->getImm() & 0xffff; unsigned Offset1 = TII->getNamedOperand(*Paired, AMDGPU::OpName::offset)->getImm() & 0xffff; unsigned NewOffset0 = Offset0 / EltSize; unsigned NewOffset1 = Offset1 / EltSize; unsigned Opc = (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64; // Prefer the st64 form if we can use it, even if we can fit the offset in the // non st64 version. I'm not sure if there's any real reason to do this. bool UseST64 = (NewOffset0 % 64 == 0) && (NewOffset1 % 64 == 0); if (UseST64) { NewOffset0 /= 64; NewOffset1 /= 64; Opc = (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64; } assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) && (NewOffset0 != NewOffset1) && "Computed offset doesn't fit"); const MCInstrDesc &Read2Desc = TII->get(Opc); const TargetRegisterClass *SuperRC = (EltSize == 4) ? &AMDGPU::VReg_64RegClass : &AMDGPU::VReg_128RegClass; unsigned DestReg = MRI->createVirtualRegister(SuperRC); DebugLoc DL = I->getDebugLoc(); MachineInstrBuilder Read2 = BuildMI(*MBB, I, DL, Read2Desc, DestReg) .addOperand(*AddrReg) // addr .addImm(NewOffset0) // offset0 .addImm(NewOffset1) // offset1 .addImm(0) // gds .addMemOperand(*I->memoperands_begin()) .addMemOperand(*Paired->memoperands_begin()); unsigned SubRegIdx0 = (EltSize == 4) ? AMDGPU::sub0 : AMDGPU::sub0_sub1; unsigned SubRegIdx1 = (EltSize == 4) ? AMDGPU::sub1 : AMDGPU::sub2_sub3; const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); // Copy to the old destination registers. MachineInstr *Copy0 = BuildMI(*MBB, I, DL, CopyDesc) .addOperand(*Dest0) // Copy to same destination including flags and sub reg. .addReg(DestReg, 0, SubRegIdx0); MachineInstr *Copy1 = BuildMI(*MBB, I, DL, CopyDesc) .addOperand(*Dest1) .addReg(DestReg, RegState::Kill, SubRegIdx1); LIS->InsertMachineInstrInMaps(*Read2); // repairLiveintervalsInRange() doesn't handle physical register, so we have // to update the M0 range manually. SlotIndex PairedIndex = LIS->getInstructionIndex(*Paired); LiveRange &M0Range = LIS->getRegUnit(*MCRegUnitIterator(AMDGPU::M0, TRI)); LiveRange::Segment *M0Segment = M0Range.getSegmentContaining(PairedIndex); bool UpdateM0Range = M0Segment->end == PairedIndex.getRegSlot(); // The new write to the original destination register is now the copy. Steal // the old SlotIndex. LIS->ReplaceMachineInstrInMaps(*I, *Copy0); LIS->ReplaceMachineInstrInMaps(*Paired, *Copy1); I->eraseFromParent(); Paired->eraseFromParent(); LiveInterval &AddrRegLI = LIS->getInterval(AddrReg->getReg()); LIS->shrinkToUses(&AddrRegLI); LIS->createAndComputeVirtRegInterval(DestReg); if (UpdateM0Range) { SlotIndex Read2Index = LIS->getInstructionIndex(*Read2); M0Segment->end = Read2Index.getRegSlot(); } DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n'); return Read2.getInstr(); }
// Replace uses of FromReg with ToReg if they are dominated by MI. static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI, unsigned FromReg, unsigned ToReg, const MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS) { bool Changed = false; LiveInterval *FromLI = &LIS.getInterval(FromReg); LiveInterval *ToLI = &LIS.getInterval(ToReg); SlotIndex FromIdx = LIS.getInstructionIndex(MI).getRegSlot(); VNInfo *FromVNI = FromLI->getVNInfoAt(FromIdx); SmallVector<SlotIndex, 4> Indices; for (auto I = MRI.use_nodbg_begin(FromReg), E = MRI.use_nodbg_end(); I != E;) { MachineOperand &O = *I++; MachineInstr *Where = O.getParent(); // Check that MI dominates the instruction in the normal way. if (&MI == Where || !MDT.dominates(&MI, Where)) continue; // If this use gets a different value, skip it. SlotIndex WhereIdx = LIS.getInstructionIndex(*Where); VNInfo *WhereVNI = FromLI->getVNInfoAt(WhereIdx); if (WhereVNI && WhereVNI != FromVNI) continue; // Make sure ToReg isn't clobbered before it gets there. VNInfo *ToVNI = ToLI->getVNInfoAt(WhereIdx); if (ToVNI && ToVNI != FromVNI) continue; Changed = true; LLVM_DEBUG(dbgs() << "Setting operand " << O << " in " << *Where << " from " << MI << "\n"); O.setReg(ToReg); // If the store's def was previously dead, it is no longer. if (!O.isUndef()) { MI.getOperand(0).setIsDead(false); Indices.push_back(WhereIdx.getRegSlot()); } } if (Changed) { // Extend ToReg's liveness. LIS.extendToIndices(*ToLI, Indices); // Shrink FromReg's liveness. LIS.shrinkToUses(FromLI); // If we replaced all dominated uses, FromReg is now killed at MI. if (!FromLI->liveAt(FromIdx.getDeadSlot())) MI.addRegisterKilled(FromReg, MBB.getParent() ->getSubtarget<WebAssemblySubtarget>() .getRegisterInfo()); } return Changed; }
void LiveIntervals::handleVirtualRegisterDef(MachineBasicBlock *mbb, MachineBasicBlock::iterator mi, SlotIndex MIIdx, MachineOperand& MO, unsigned MOIdx, LiveInterval &interval) { DEBUG(dbgs() << "\t\tregister: " << PrintReg(interval.reg, TRI)); // Virtual registers may be defined multiple times (due to phi // elimination and 2-addr elimination). Much of what we do only has to be // done once for the vreg. We use an empty interval to detect the first // time we see a vreg. LiveVariables::VarInfo& vi = LV->getVarInfo(interval.reg); if (interval.empty()) { // Get the Idx of the defining instructions. SlotIndex defIndex = MIIdx.getRegSlot(MO.isEarlyClobber()); // Make sure the first definition is not a partial redefinition. assert(!MO.readsReg() && "First def cannot also read virtual register " "missing <undef> flag?"); VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator); assert(ValNo->id == 0 && "First value in interval is not 0?"); // Loop over all of the blocks that the vreg is defined in. There are // two cases we have to handle here. The most common case is a vreg // whose lifetime is contained within a basic block. In this case there // will be a single kill, in MBB, which comes after the definition. if (vi.Kills.size() == 1 && vi.Kills[0]->getParent() == mbb) { // FIXME: what about dead vars? SlotIndex killIdx; if (vi.Kills[0] != mi) killIdx = getInstructionIndex(vi.Kills[0]).getRegSlot(); else killIdx = defIndex.getDeadSlot(); // If the kill happens after the definition, we have an intra-block // live range. if (killIdx > defIndex) { assert(vi.AliveBlocks.empty() && "Shouldn't be alive across any blocks!"); LiveRange LR(defIndex, killIdx, ValNo); interval.addRange(LR); DEBUG(dbgs() << " +" << LR << "\n"); return; } } // The other case we handle is when a virtual register lives to the end // of the defining block, potentially live across some blocks, then is // live into some number of blocks, but gets killed. Start by adding a // range that goes from this definition to the end of the defining block. LiveRange NewLR(defIndex, getMBBEndIdx(mbb), ValNo); DEBUG(dbgs() << " +" << NewLR); interval.addRange(NewLR); bool PHIJoin = LV->isPHIJoin(interval.reg); if (PHIJoin) { // A phi join register is killed at the end of the MBB and revived as a // new valno in the killing blocks. assert(vi.AliveBlocks.empty() && "Phi join can't pass through blocks"); DEBUG(dbgs() << " phi-join"); } else { // Iterate over all of the blocks that the variable is completely // live in, adding [insrtIndex(begin), instrIndex(end)+4) to the // live interval. for (SparseBitVector<>::iterator I = vi.AliveBlocks.begin(), E = vi.AliveBlocks.end(); I != E; ++I) { MachineBasicBlock *aliveBlock = MF->getBlockNumbered(*I); LiveRange LR(getMBBStartIdx(aliveBlock), getMBBEndIdx(aliveBlock), ValNo); interval.addRange(LR); DEBUG(dbgs() << " +" << LR); } } // Finally, this virtual register is live from the start of any killing // block to the 'use' slot of the killing instruction. for (unsigned i = 0, e = vi.Kills.size(); i != e; ++i) { MachineInstr *Kill = vi.Kills[i]; SlotIndex Start = getMBBStartIdx(Kill->getParent()); SlotIndex killIdx = getInstructionIndex(Kill).getRegSlot(); // Create interval with one of a NEW value number. Note that this value // number isn't actually defined by an instruction, weird huh? :) if (PHIJoin) { assert(getInstructionFromIndex(Start) == 0 && "PHI def index points at actual instruction."); ValNo = interval.getNextValue(Start, VNInfoAllocator); } LiveRange LR(Start, killIdx, ValNo); interval.addRange(LR); DEBUG(dbgs() << " +" << LR); } } else { if (MultipleDefsBySameMI(*mi, MOIdx)) // Multiple defs of the same virtual register by the same instruction. // e.g. %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ... // This is likely due to elimination of REG_SEQUENCE instructions. Return // here since there is nothing to do. return; // If this is the second time we see a virtual register definition, it // must be due to phi elimination or two addr elimination. If this is // the result of two address elimination, then the vreg is one of the // def-and-use register operand. // It may also be partial redef like this: // 80 %reg1041:6<def> = VSHRNv4i16 %reg1034<kill>, 12, pred:14, pred:%reg0 // 120 %reg1041:5<def> = VSHRNv4i16 %reg1039<kill>, 12, pred:14, pred:%reg0 bool PartReDef = isPartialRedef(MIIdx, MO, interval); if (PartReDef || mi->isRegTiedToUseOperand(MOIdx)) { // If this is a two-address definition, then we have already processed // the live range. The only problem is that we didn't realize there // are actually two values in the live interval. Because of this we // need to take the LiveRegion that defines this register and split it // into two values. SlotIndex RedefIndex = MIIdx.getRegSlot(MO.isEarlyClobber()); const LiveRange *OldLR = interval.getLiveRangeContaining(RedefIndex.getRegSlot(true)); VNInfo *OldValNo = OldLR->valno; SlotIndex DefIndex = OldValNo->def.getRegSlot(); // Delete the previous value, which should be short and continuous, // because the 2-addr copy must be in the same MBB as the redef. interval.removeRange(DefIndex, RedefIndex); // The new value number (#1) is defined by the instruction we claimed // defined value #0. VNInfo *ValNo = interval.createValueCopy(OldValNo, VNInfoAllocator); // Value#0 is now defined by the 2-addr instruction. OldValNo->def = RedefIndex; // Add the new live interval which replaces the range for the input copy. LiveRange LR(DefIndex, RedefIndex, ValNo); DEBUG(dbgs() << " replace range with " << LR); interval.addRange(LR); // If this redefinition is dead, we need to add a dummy unit live // range covering the def slot. if (MO.isDead()) interval.addRange(LiveRange(RedefIndex, RedefIndex.getDeadSlot(), OldValNo)); DEBUG(dbgs() << " RESULT: " << interval); } else if (LV->isPHIJoin(interval.reg)) { // In the case of PHI elimination, each variable definition is only // live until the end of the block. We've already taken care of the // rest of the live range. SlotIndex defIndex = MIIdx.getRegSlot(); if (MO.isEarlyClobber()) defIndex = MIIdx.getRegSlot(true); VNInfo *ValNo = interval.getNextValue(defIndex, VNInfoAllocator); SlotIndex killIndex = getMBBEndIdx(mbb); LiveRange LR(defIndex, killIndex, ValNo); interval.addRange(LR); DEBUG(dbgs() << " phi-join +" << LR); } else { llvm_unreachable("Multiply defined register"); } } DEBUG(dbgs() << '\n'); }
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, SlotIndex Use, unsigned PhysReg, ArrayRef<SlotIndex> Undefs) { unsigned UseMBBNum = UseMBB.getNumber(); // Block numbers where LR should be live-in. SmallVector<unsigned, 16> WorkList(1, UseMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = nullptr; bool FoundUndef = false; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); errs() << "Use of " << printReg(PhysReg) << " does not have a corresponding definition on every path:\n"; const MachineInstr *MI = Indexes->getInstructionFromIndex(Use); if (MI != nullptr) errs() << Use << " " << *MI; report_fatal_error("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo(); errs() << "The register " << printReg(PhysReg, TRI) << " needs to be live in to " << printMBBReference(*MBB) << ", but is missing from the live-in list.\n"; report_fatal_error("Invalid global physical register"); } #endif FoundUndef |= MBB->pred_empty(); for (MachineBasicBlock *Pred : MBB->predecessors()) { // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. auto EP = LR.extendInBlock(Undefs, Start, End); VNInfo *VNI = EP.first; FoundUndef |= EP.second; setLiveOutValue(Pred, EP.second ? &UndefVNI : VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } if (VNI || EP.second) continue; // No, we need a live-in value for Pred as well if (Pred != &UseMBB) WorkList.push_back(Pred->getNumber()); else // Loopback to UseMBB, so value is really live through. Use = SlotIndex(); } } LiveIn.clear(); FoundUndef |= (TheVNI == nullptr || TheVNI == &UndefVNI); if (!Undefs.empty() && FoundUndef) UniqueVNI = false; // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but // neither require it. Skip the sorting overhead for small updates. if (WorkList.size() > 4) array_pod_sort(WorkList.begin(), WorkList.end()); // If a unique reaching def was found, blit in the live ranges immediately. if (UniqueVNI) { assert(TheVNI != nullptr && TheVNI != &UndefVNI); LiveRangeUpdater Updater(&LR); for (unsigned BN : WorkList) { SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(BN); // Trim the live range in UseMBB. if (BN == UseMBBNum && Use.isValid()) End = Use; else Map[MF->getBlockNumbered(BN)] = LiveOutPair(TheVNI, nullptr); Updater.add(Start, End, TheVNI); } return true; } // Prepare the defined/undefined bit vectors. EntryInfoMap::iterator Entry; bool DidInsert; std::tie(Entry, DidInsert) = EntryInfos.insert( std::make_pair(&LR, std::make_pair(BitVector(), BitVector()))); if (DidInsert) { // Initialize newly inserted entries. unsigned N = MF->getNumBlockIDs(); Entry->second.first.resize(N); Entry->second.second.resize(N); } BitVector &DefOnEntry = Entry->second.first; BitVector &UndefOnEntry = Entry->second.second; // Multiple values were found, so transfer the work list to the LiveIn array // where UpdateSSA will use it as a work list. LiveIn.reserve(WorkList.size()); for (unsigned BN : WorkList) { MachineBasicBlock *MBB = MF->getBlockNumbered(BN); if (!Undefs.empty() && !isDefOnEntry(LR, Undefs, *MBB, DefOnEntry, UndefOnEntry)) continue; addLiveInBlock(LR, DomTree->getNode(MBB)); if (MBB == &UseMBB) LiveIn.back().Kill = Use; } return false; }
/// computeIntervals - computes the live intervals for virtual /// registers. for some ordering of the machine instructions [1,N] a /// live interval is an interval [i, j) where 1 <= i <= j < N for /// which a variable is live void LiveIntervals::computeIntervals() { DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n" << "********** Function: " << MF->getName() << '\n'); RegMaskBlocks.resize(MF->getNumBlockIDs()); SmallVector<unsigned, 8> UndefUses; for (MachineFunction::iterator MBBI = MF->begin(), E = MF->end(); MBBI != E; ++MBBI) { MachineBasicBlock *MBB = MBBI; RegMaskBlocks[MBB->getNumber()].first = RegMaskSlots.size(); if (MBB->empty()) continue; // Track the index of the current machine instr. SlotIndex MIIndex = getMBBStartIdx(MBB); DEBUG(dbgs() << "BB#" << MBB->getNumber() << ":\t\t# derived from " << MBB->getName() << "\n"); // Skip over empty initial indices. if (getInstructionFromIndex(MIIndex) == 0) MIIndex = Indexes->getNextNonNullIndex(MIIndex); for (MachineBasicBlock::iterator MI = MBB->begin(), miEnd = MBB->end(); MI != miEnd; ++MI) { DEBUG(dbgs() << MIIndex << "\t" << *MI); if (MI->isDebugValue()) continue; assert(Indexes->getInstructionFromIndex(MIIndex) == MI && "Lost SlotIndex synchronization"); // Handle defs. for (int i = MI->getNumOperands() - 1; i >= 0; --i) { MachineOperand &MO = MI->getOperand(i); // Collect register masks. if (MO.isRegMask()) { RegMaskSlots.push_back(MIIndex.getRegSlot()); RegMaskBits.push_back(MO.getRegMask()); continue; } if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) continue; // handle register defs - build intervals if (MO.isDef()) handleRegisterDef(MBB, MI, MIIndex, MO, i); else if (MO.isUndef()) UndefUses.push_back(MO.getReg()); } // Move to the next instr slot. MIIndex = Indexes->getNextNonNullIndex(MIIndex); } // Compute the number of register mask instructions in this block. std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()]; RMB.second = RegMaskSlots.size() - RMB.first; } // Create empty intervals for registers defined by implicit_def's (except // for those implicit_def that define values which are liveout of their // blocks. for (unsigned i = 0, e = UndefUses.size(); i != e; ++i) { unsigned UndefReg = UndefUses[i]; (void)getOrCreateInterval(UndefReg); } }
void MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); const MCInstrDesc &MCID = MI->getDesc(); const MCOperandInfo &MCOI = MCID.OpInfo[MONum]; // The first MCID.NumDefs operands must be explicit register defines if (MONum < MCID.getNumDefs()) { if (!MO->isReg()) report("Explicit definition must be a register", MO, MONum); else if (!MO->isDef()) report("Explicit definition marked as use", MO, MONum); else if (MO->isImplicit()) report("Explicit definition marked as implicit", MO, MONum); } else if (MONum < MCID.getNumOperands()) { // Don't check if it's the last operand in a variadic instruction. See, // e.g., LDM_RET in the arm back end. if (MO->isReg() && !(MI->isVariadic() && MONum == MCID.getNumOperands()-1)) { if (MO->isDef() && !MCOI.isOptionalDef()) report("Explicit operand marked as def", MO, MONum); if (MO->isImplicit()) report("Explicit operand marked as implicit", MO, MONum); } } else { // ARM adds %reg0 operands to indicate predicates. We'll allow that. if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg()) report("Extra explicit operand on non-variadic instruction", MO, MONum); } switch (MO->getType()) { case MachineOperand::MO_Register: { const unsigned Reg = MO->getReg(); if (!Reg) return; if (MRI->tracksLiveness() && !MI->isDebugValue()) checkLiveness(MO, MONum); // Check register classes. if (MONum < MCID.getNumOperands() && !MO->isImplicit()) { unsigned SubIdx = MO->getSubReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { if (SubIdx) { report("Illegal subregister index for physical register", MO, MONum); return; } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, MONum, TRI, *MF)) { if (!DRC->contains(Reg)) { report("Illegal physical register for instruction", MO, MONum); *OS << TRI->getName(Reg) << " is not a " << DRC->getName() << " register.\n"; } } } else { // Virtual register. const TargetRegisterClass *RC = MRI->getRegClass(Reg); if (SubIdx) { const TargetRegisterClass *SRC = TRI->getSubClassWithSubReg(RC, SubIdx); if (!SRC) { report("Invalid subregister index for virtual register", MO, MONum); *OS << "Register class " << RC->getName() << " does not support subreg index " << SubIdx << "\n"; return; } if (RC != SRC) { report("Invalid register class for subregister index", MO, MONum); *OS << "Register class " << RC->getName() << " does not fully support subreg index " << SubIdx << "\n"; return; } } if (const TargetRegisterClass *DRC = TII->getRegClass(MCID, MONum, TRI, *MF)) { if (SubIdx) { const TargetRegisterClass *SuperRC = TRI->getLargestLegalSuperClass(RC); if (!SuperRC) { report("No largest legal super class exists.", MO, MONum); return; } DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx); if (!DRC) { report("No matching super-reg register class.", MO, MONum); return; } } if (!RC->hasSuperClassEq(DRC)) { report("Illegal virtual register for instruction", MO, MONum); *OS << "Expected a " << DRC->getName() << " register, but got a " << RC->getName() << " register\n"; } } } } break; } case MachineOperand::MO_RegisterMask: regMasks.push_back(MO->getRegMask()); break; case MachineOperand::MO_MachineBasicBlock: if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent())) report("PHI operand is not in the CFG", MO, MONum); break; case MachineOperand::MO_FrameIndex: if (LiveStks && LiveStks->hasInterval(MO->getIndex()) && LiveInts && !LiveInts->isNotInMIMap(MI)) { LiveInterval &LI = LiveStks->getInterval(MO->getIndex()); SlotIndex Idx = LiveInts->getInstructionIndex(MI); if (MI->mayLoad() && !LI.liveAt(Idx.getRegSlot(true))) { report("Instruction loads from dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } if (MI->mayStore() && !LI.liveAt(Idx.getRegSlot())) { report("Instruction stores to dead spill slot", MO, MONum); *OS << "Live stack: " << LI << '\n'; } } break; default: break; } }
/// hoistSpill - Given a sibling copy that defines a value to be spilled, insert /// a spill at a better location. bool InlineSpiller::hoistSpill(LiveInterval &SpillLI, MachineInstr *CopyMI) { SlotIndex Idx = LIS.getInstructionIndex(CopyMI); VNInfo *VNI = SpillLI.getVNInfoAt(Idx.getRegSlot()); assert(VNI && VNI->def == Idx.getRegSlot() && "Not defined by copy"); SibValueMap::iterator I = SibValues.find(VNI); if (I == SibValues.end()) return false; const SibValueInfo &SVI = I->second; // Let the normal folding code deal with the boring case. if (!SVI.AllDefsAreReloads && SVI.SpillVNI == VNI) return false; // SpillReg may have been deleted by remat and DCE. if (!LIS.hasInterval(SVI.SpillReg)) { DEBUG(dbgs() << "Stale interval: " << PrintReg(SVI.SpillReg) << '\n'); SibValues.erase(I); return false; } LiveInterval &SibLI = LIS.getInterval(SVI.SpillReg); if (!SibLI.containsValue(SVI.SpillVNI)) { DEBUG(dbgs() << "Stale value: " << PrintReg(SVI.SpillReg) << '\n'); SibValues.erase(I); return false; } // Conservatively extend the stack slot range to the range of the original // value. We may be able to do better with stack slot coloring by being more // careful here. assert(StackInt && "No stack slot assigned yet."); LiveInterval &OrigLI = LIS.getInterval(Original); VNInfo *OrigVNI = OrigLI.getVNInfoAt(Idx); StackInt->MergeValueInAsValue(OrigLI, OrigVNI, StackInt->getValNumInfo(0)); DEBUG(dbgs() << "\tmerged orig valno " << OrigVNI->id << ": " << *StackInt << '\n'); // Already spilled everywhere. if (SVI.AllDefsAreReloads) { DEBUG(dbgs() << "\tno spill needed: " << SVI); ++NumOmitReloadSpill; return true; } // We are going to spill SVI.SpillVNI immediately after its def, so clear out // any later spills of the same value. eliminateRedundantSpills(SibLI, SVI.SpillVNI); MachineBasicBlock *MBB = LIS.getMBBFromIndex(SVI.SpillVNI->def); MachineBasicBlock::iterator MII; if (SVI.SpillVNI->isPHIDef()) MII = MBB->SkipPHIsAndLabels(MBB->begin()); else { MachineInstr *DefMI = LIS.getInstructionFromIndex(SVI.SpillVNI->def); assert(DefMI && "Defining instruction disappeared"); MII = DefMI; ++MII; } // Insert spill without kill flag immediately after def. TII.storeRegToStackSlot(*MBB, MII, SVI.SpillReg, false, StackSlot, MRI.getRegClass(SVI.SpillReg), &TRI); --MII; // Point to store instruction. LIS.InsertMachineInstrInMaps(MII); DEBUG(dbgs() << "\thoisted: " << SVI.SpillVNI->def << '\t' << *MII); ++NumSpills; ++NumHoists; return true; }
void MachineVerifier::checkLiveness(const MachineOperand *MO, unsigned MONum) { const MachineInstr *MI = MO->getParent(); const unsigned Reg = MO->getReg(); // Both use and def operands can read a register. if (MO->readsReg()) { regsLiveInButUnused.erase(Reg); bool isKill = false; unsigned defIdx; if (MI->isRegTiedToDefOperand(MONum, &defIdx)) { // A two-addr use counts as a kill if use and def are the same. unsigned DefReg = MI->getOperand(defIdx).getReg(); if (Reg == DefReg) isKill = true; else if (TargetRegisterInfo::isPhysicalRegister(Reg)) { report("Two-address instruction operands must be identical", MO, MONum); } } else isKill = MO->isKill(); if (isKill) addRegWithSubRegs(regsKilled, Reg); // Check that LiveVars knows this kill. if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) && MO->isKill()) { LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg); if (std::find(VI.Kills.begin(), VI.Kills.end(), MI) == VI.Kills.end()) report("Kill missing from LiveVariables", MO, MONum); } // Check LiveInts liveness and kill. if (TargetRegisterInfo::isVirtualRegister(Reg) && LiveInts && !LiveInts->isNotInMIMap(MI)) { SlotIndex UseIdx = LiveInts->getInstructionIndex(MI).getRegSlot(true); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (!LI.liveAt(UseIdx)) { report("No live range at use", MO, MONum); *OS << UseIdx << " is not live in " << LI << '\n'; } // Check for extra kill flags. // Note that we allow missing kill flags for now. if (MO->isKill() && !LI.killedAt(UseIdx.getRegSlot())) { report("Live range continues after kill flag", MO, MONum); *OS << "Live range: " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } // Use of a dead register. if (!regsLive.count(Reg)) { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { // Reserved registers may be used even when 'dead'. if (!isReserved(Reg)) report("Using an undefined physical register", MO, MONum); } else { BBInfo &MInfo = MBBInfoMap[MI->getParent()]; // We don't know which virtual registers are live in, so only complain // if vreg was killed in this MBB. Otherwise keep track of vregs that // must be live in. PHI instructions are handled separately. if (MInfo.regsKilled.count(Reg)) report("Using a killed virtual register", MO, MONum); else if (!MI->isPHI()) MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI)); } } } if (MO->isDef()) { // Register defined. // TODO: verify that earlyclobber ops are not used. if (MO->isDead()) addRegWithSubRegs(regsDead, Reg); else addRegWithSubRegs(regsDefined, Reg); // Verify SSA form. if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) && llvm::next(MRI->def_begin(Reg)) != MRI->def_end()) report("Multiple virtual register defs in SSA form", MO, MONum); // Check LiveInts for a live range, but only for virtual registers. if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) && !LiveInts->isNotInMIMap(MI)) { SlotIndex DefIdx = LiveInts->getInstructionIndex(MI).getRegSlot(); if (LiveInts->hasInterval(Reg)) { const LiveInterval &LI = LiveInts->getInterval(Reg); if (const VNInfo *VNI = LI.getVNInfoAt(DefIdx)) { assert(VNI && "NULL valno is not allowed"); if (VNI->def != DefIdx && !MO->isEarlyClobber()) { report("Inconsistent valno->def", MO, MONum); *OS << "Valno " << VNI->id << " is not defined at " << DefIdx << " in " << LI << '\n'; } } else { report("No live range at def", MO, MONum); *OS << DefIdx << " is not live in " << LI << '\n'; } } else { report("Virtual register has no Live interval", MO, MONum); } } } }
void UserValue::addDefsFromCopies(LiveInterval *LI, unsigned LocNo, const SmallVectorImpl<SlotIndex> &Kills, SmallVectorImpl<std::pair<SlotIndex, unsigned> > &NewDefs, MachineRegisterInfo &MRI, LiveIntervals &LIS) { if (Kills.empty()) return; // Don't track copies from physregs, there are too many uses. if (!TargetRegisterInfo::isVirtualRegister(LI->reg)) return; // Collect all the (vreg, valno) pairs that are copies of LI. SmallVector<std::pair<LiveInterval*, const VNInfo*>, 8> CopyValues; for (MachineOperand &MO : MRI.use_nodbg_operands(LI->reg)) { MachineInstr *MI = MO.getParent(); // Copies of the full value. if (MO.getSubReg() || !MI->isCopy()) continue; unsigned DstReg = MI->getOperand(0).getReg(); // Don't follow copies to physregs. These are usually setting up call // arguments, and the argument registers are always call clobbered. We are // better off in the source register which could be a callee-saved register, // or it could be spilled. if (!TargetRegisterInfo::isVirtualRegister(DstReg)) continue; // Is LocNo extended to reach this copy? If not, another def may be blocking // it, or we are looking at a wrong value of LI. SlotIndex Idx = LIS.getInstructionIndex(MI); LocMap::iterator I = locInts.find(Idx.getRegSlot(true)); if (!I.valid() || I.value() != LocNo) continue; if (!LIS.hasInterval(DstReg)) continue; LiveInterval *DstLI = &LIS.getInterval(DstReg); const VNInfo *DstVNI = DstLI->getVNInfoAt(Idx.getRegSlot()); assert(DstVNI && DstVNI->def == Idx.getRegSlot() && "Bad copy value"); CopyValues.push_back(std::make_pair(DstLI, DstVNI)); } if (CopyValues.empty()) return; DEBUG(dbgs() << "Got " << CopyValues.size() << " copies of " << *LI << '\n'); // Try to add defs of the copied values for each kill point. for (unsigned i = 0, e = Kills.size(); i != e; ++i) { SlotIndex Idx = Kills[i]; for (unsigned j = 0, e = CopyValues.size(); j != e; ++j) { LiveInterval *DstLI = CopyValues[j].first; const VNInfo *DstVNI = CopyValues[j].second; if (DstLI->getVNInfoAt(Idx) != DstVNI) continue; // Check that there isn't already a def at Idx LocMap::iterator I = locInts.find(Idx); if (I.valid() && I.start() <= Idx) continue; DEBUG(dbgs() << "Kill at " << Idx << " covered by valno #" << DstVNI->id << " in " << *DstLI << '\n'); MachineInstr *CopyMI = LIS.getInstructionFromIndex(DstVNI->def); assert(CopyMI && CopyMI->isCopy() && "Bad copy value"); unsigned LocNo = getLocationNo(CopyMI->getOperand(0)); I.insert(Idx, Idx.getNextSlot(), LocNo); NewDefs.push_back(std::make_pair(Idx, LocNo)); break; } } }
// Repair indexes after adding and removing instructions. void SlotIndexes::repairIndexesInRange(MachineBasicBlock *MBB, MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End) { // FIXME: Is this really necessary? The only caller repairIntervalsForRange() // does the same thing. // Find anchor points, which are at the beginning/end of blocks or at // instructions that already have indexes. while (Begin != MBB->begin() && !hasIndex(Begin)) --Begin; while (End != MBB->end() && !hasIndex(End)) ++End; bool includeStart = (Begin == MBB->begin()); SlotIndex startIdx; if (includeStart) startIdx = getMBBStartIdx(MBB); else startIdx = getInstructionIndex(Begin); SlotIndex endIdx; if (End == MBB->end()) endIdx = getMBBEndIdx(MBB); else endIdx = getInstructionIndex(End); // FIXME: Conceptually, this code is implementing an iterator on MBB that // optionally includes an additional position prior to MBB->begin(), indicated // by the includeStart flag. This is done so that we can iterate MIs in a MBB // in parallel with SlotIndexes, but there should be a better way to do this. IndexList::iterator ListB = startIdx.listEntry(); IndexList::iterator ListI = endIdx.listEntry(); MachineBasicBlock::iterator MBBI = End; bool pastStart = false; while (ListI != ListB || MBBI != Begin || (includeStart && !pastStart)) { assert(ListI->getIndex() >= startIdx.getIndex() && (includeStart || !pastStart) && "Decremented past the beginning of region to repair."); MachineInstr *SlotMI = ListI->getInstr(); MachineInstr *MI = (MBBI != MBB->end() && !pastStart) ? MBBI : 0; bool MBBIAtBegin = MBBI == Begin && (!includeStart || pastStart); if (SlotMI == MI && !MBBIAtBegin) { --ListI; if (MBBI != Begin) --MBBI; else pastStart = true; } else if (MI && mi2iMap.find(MI) == mi2iMap.end()) { if (MBBI != Begin) --MBBI; else pastStart = true; } else { --ListI; if (SlotMI) removeMachineInstrFromMaps(SlotMI); } } // In theory this could be combined with the previous loop, but it is tricky // to update the IndexList while we are iterating it. for (MachineBasicBlock::iterator I = End; I != Begin;) { --I; MachineInstr *MI = I; if (!MI->isDebugValue() && mi2iMap.find(MI) == mi2iMap.end()) insertMachineInstrInMaps(MI); } }
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, SlotIndex Use, unsigned PhysReg) { unsigned UseMBBNum = UseMBB.getNumber(); // Block numbers where LR should be live-in. SmallVector<unsigned, 16> WorkList(1, UseMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = nullptr; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); errs() << "Use of " << PrintReg(PhysReg) << " does not have a corresponding definition on every path:\n"; const MachineInstr *MI = Indexes->getInstructionFromIndex(Use); if (MI != nullptr) errs() << Use << " " << *MI; llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); errs() << "The register " << PrintReg(PhysReg) << " needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LR.extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; continue; } // No, we need a live-in value for Pred as well if (Pred != &UseMBB) WorkList.push_back(Pred->getNumber()); else // Loopback to UseMBB, so value is really live through. Use = SlotIndex(); } } LiveIn.clear(); // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but // neither require it. Skip the sorting overhead for small updates. if (WorkList.size() > 4) array_pod_sort(WorkList.begin(), WorkList.end()); // If a unique reaching def was found, blit in the live ranges immediately. if (UniqueVNI) { LiveRangeUpdater Updater(&LR); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(*I); // Trim the live range in UseMBB. if (*I == UseMBBNum && Use.isValid()) End = Use; else Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr); Updater.add(Start, End, TheVNI); } return true; } // Multiple values were found, so transfer the work list to the LiveIn array // where UpdateSSA will use it as a work list. LiveIn.reserve(WorkList.size()); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { MachineBasicBlock *MBB = MF->getBlockNumbered(*I); addLiveInBlock(LR, DomTree->getNode(MBB)); if (MBB == &UseMBB) LiveIn.back().Kill = Use; } return false; }
/// Add the specified instruction to live intervals. This function is used /// to update the live intervals while the program code is being changed. /// Neither the expansion of a MUX, nor the predication are atomic, and this /// function is used to update the live intervals while these transformations /// are being done. void HexagonExpandCondsets::addInstrToLiveness(MachineInstr *MI) { SlotIndex MX = LIS->isNotInMIMap(*MI) ? LIS->InsertMachineInstrInMaps(*MI) : LIS->getInstructionIndex(*MI); DEBUG(dbgs() << "adding liveness info for instr\n " << MX << " " << *MI); MX = MX.getRegSlot(); bool Predicated = HII->isPredicated(*MI); MachineBasicBlock *MB = MI->getParent(); // Strip all implicit uses from predicated instructions. They will be // added again, according to the updated information. if (Predicated) removeImplicitUses(MI); // For each def in MI we need to insert a new live segment starting at MX // into the interval. If there already exists a live segment in the interval // that contains MX, we need to terminate it at MX. SmallVector<RegisterRef,2> Defs; for (auto &Op : MI->operands()) if (Op.isReg() && Op.isDef()) Defs.push_back(RegisterRef(Op)); for (unsigned i = 0, n = Defs.size(); i < n; ++i) { unsigned DefR = Defs[i].Reg; LiveInterval &LID = LIS->getInterval(DefR); DEBUG(dbgs() << "adding def " << PrintReg(DefR, TRI) << " with interval\n " << LID << "\n"); // If MX falls inside of an existing live segment, terminate it. LiveInterval::iterator LT = LID.FindSegmentContaining(MX); if (LT != LID.end()) terminateSegment(LT, MX, LID); DEBUG(dbgs() << "after terminating segment\n " << LID << "\n"); // Create a new segment starting from MX. LiveInterval::iterator P = prevSegment(LID, MX), N = nextSegment(LID, MX); SlotIndex EX; VNInfo *VN = LID.getNextValue(MX, LIS->getVNInfoAllocator()); if (N == LID.end()) { // There is no live segment after MX. End this segment at the end of // the block. EX = LIS->getMBBEndIdx(MB); } else { // If the next segment starts at the block boundary, end the new segment // at the boundary of the preceding block (i.e. the previous index). // Otherwise, end the segment at the beginning of the next segment. In // either case it will be "shrunk-to-uses" later. EX = N->start.isBlock() ? N->start.getPrevIndex() : N->start; } if (Predicated) { // Predicated instruction will have an implicit use of the defined // register. This is necessary so that this definition will not make // any previous definitions dead. If there are no previous live // segments, still add the implicit use, but make it "undef". // Because of the implicit use, the preceding definition is not // dead. Mark is as such (if necessary). MachineOperand ImpUse = MachineOperand::CreateReg(DefR, false, true); ImpUse.setSubReg(Defs[i].Sub); bool Undef = false; if (P == LID.end()) Undef = true; else { // If the previous segment extends to the end of the previous block, // the end index may actually be the beginning of this block. If // the previous segment ends at a block boundary, move it back by one, // to get the proper block for it. SlotIndex PE = P->end.isBlock() ? P->end.getPrevIndex() : P->end; MachineBasicBlock *PB = LIS->getMBBFromIndex(PE); if (PB != MB && !LIS->isLiveInToMBB(LID, MB)) Undef = true; } if (!Undef) { makeUndead(DefR, P->valno->def); // We are adding a live use, so extend the previous segment to // include it. P->end = MX; } else { ImpUse.setIsUndef(true); } if (!MI->readsRegister(DefR)) MI->addOperand(ImpUse); if (N != LID.end()) makeDefined(DefR, N->start, true); } LiveRange::Segment NR = LiveRange::Segment(MX, EX, VN); LID.addSegment(NR); DEBUG(dbgs() << "added a new segment " << NR << "\n " << LID << "\n"); shrinkToUses(DefR, LID); DEBUG(dbgs() << "updated imp-uses: " << *MI); LID.verify(); } // For each use in MI: // - If there is no live segment that contains MX for the used register, // extend the previous one. Ignore implicit uses. for (auto &Op : MI->operands()) { if (!Op.isReg() || !Op.isUse() || Op.isImplicit() || Op.isUndef()) continue; unsigned UseR = Op.getReg(); LiveInterval &LIU = LIS->getInterval(UseR); // Find the last segment P that starts before MX. LiveInterval::iterator P = LIU.FindSegmentContaining(MX); if (P == LIU.end()) P = prevSegment(LIU, MX); assert(P != LIU.end() && "MI uses undefined register?"); SlotIndex EX = P->end; // If P contains MX, there is not much to do. if (EX > MX) { Op.setIsKill(false); continue; } // Otherwise, extend P to "next(MX)". P->end = MX.getNextIndex(); Op.setIsKill(true); // Get the old "kill" instruction, and remove the kill flag. if (MachineInstr *KI = LIS->getInstructionFromIndex(MX)) KI->clearRegisterKills(UseR, nullptr); shrinkToUses(UseR, LIU); LIU.verify(); } }
LiveInterval::iterator LiveInterval::find(SlotIndex Pos) { assert(Pos.isValid() && "Cannot search for an invalid index"); return std::upper_bound(begin(), end(), Pos, CompEnd()); }
void SplitEditor::splitLiveThroughBlock(unsigned MBBNum, unsigned IntvIn, SlotIndex LeaveBefore, unsigned IntvOut, SlotIndex EnterAfter){ SlotIndex Start, Stop; tie(Start, Stop) = LIS.getSlotIndexes()->getMBBRange(MBBNum); DEBUG(dbgs() << "BB#" << MBBNum << " [" << Start << ';' << Stop << ") intf " << LeaveBefore << '-' << EnterAfter << ", live-through " << IntvIn << " -> " << IntvOut); assert((IntvIn || IntvOut) && "Use splitSingleBlock for isolated blocks"); assert((!LeaveBefore || LeaveBefore < Stop) && "Interference after block"); assert((!IntvIn || !LeaveBefore || LeaveBefore > Start) && "Impossible intf"); assert((!EnterAfter || EnterAfter >= Start) && "Interference before block"); MachineBasicBlock *MBB = VRM.getMachineFunction().getBlockNumbered(MBBNum); if (!IntvOut) { DEBUG(dbgs() << ", spill on entry.\n"); // // <<<<<<<<< Possible LeaveBefore interference. // |-----------| Live through. // -____________ Spill on entry. // selectIntv(IntvIn); SlotIndex Idx = leaveIntvAtTop(*MBB); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); (void)Idx; return; } if (!IntvIn) { DEBUG(dbgs() << ", reload on exit.\n"); // // >>>>>>> Possible EnterAfter interference. // |-----------| Live through. // ___________-- Reload on exit. // selectIntv(IntvOut); SlotIndex Idx = enterIntvAtEnd(*MBB); assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); (void)Idx; return; } if (IntvIn == IntvOut && !LeaveBefore && !EnterAfter) { DEBUG(dbgs() << ", straight through.\n"); // // |-----------| Live through. // ------------- Straight through, same intv, no interference. // selectIntv(IntvOut); useIntv(Start, Stop); return; } // We cannot legally insert splits after LSP. SlotIndex LSP = SA.getLastSplitPoint(MBBNum); assert((!IntvOut || !EnterAfter || EnterAfter < LSP) && "Impossible intf"); if (IntvIn != IntvOut && (!LeaveBefore || !EnterAfter || LeaveBefore.getBaseIndex() > EnterAfter.getBoundaryIndex())) { DEBUG(dbgs() << ", switch avoiding interference.\n"); // // >>>> <<<< Non-overlapping EnterAfter/LeaveBefore interference. // |-----------| Live through. // ------======= Switch intervals between interference. // selectIntv(IntvOut); SlotIndex Idx; if (LeaveBefore && LeaveBefore < LSP) { Idx = enterIntvBefore(LeaveBefore); useIntv(Idx, Stop); } else { Idx = enterIntvAtEnd(*MBB); } selectIntv(IntvIn); useIntv(Start, Idx); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); return; } DEBUG(dbgs() << ", create local intv for interference.\n"); // // >>><><><><<<< Overlapping EnterAfter/LeaveBefore interference. // |-----------| Live through. // ==---------== Switch intervals before/after interference. // assert(LeaveBefore <= EnterAfter && "Missed case"); selectIntv(IntvOut); SlotIndex Idx = enterIntvAfter(EnterAfter); useIntv(Idx, Stop); assert((!EnterAfter || Idx >= EnterAfter) && "Interference"); selectIntv(IntvIn); Idx = leaveIntvBefore(LeaveBefore); useIntv(Start, Idx); assert((!LeaveBefore || Idx <= LeaveBefore) && "Interference"); }
/// shrinkToUses - After removing some uses of a register, shrink its live /// range to just the remaining uses. This method does not compute reaching /// defs for new uses, and it doesn't remove dead defs. bool LiveIntervals::shrinkToUses(LiveInterval *li, SmallVectorImpl<MachineInstr*> *dead) { DEBUG(dbgs() << "Shrink: " << *li << '\n'); assert(TargetRegisterInfo::isVirtualRegister(li->reg) && "Can only shrink virtual registers"); // Find all the values used, including PHI kills. SmallVector<std::pair<SlotIndex, VNInfo*>, 16> WorkList; // Blocks that have already been added to WorkList as live-out. SmallPtrSet<MachineBasicBlock*, 16> LiveOut; // Visit all instructions reading li->reg. for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(li->reg); MachineInstr *UseMI = I.skipInstruction();) { if (UseMI->isDebugValue() || !UseMI->readsVirtualRegister(li->reg)) continue; SlotIndex Idx = getInstructionIndex(UseMI).getRegSlot(); LiveRangeQuery LRQ(*li, Idx); VNInfo *VNI = LRQ.valueIn(); if (!VNI) { // This shouldn't happen: readsVirtualRegister returns true, but there is // no live value. It is likely caused by a target getting <undef> flags // wrong. DEBUG(dbgs() << Idx << '\t' << *UseMI << "Warning: Instr claims to read non-existent value in " << *li << '\n'); continue; } // Special case: An early-clobber tied operand reads and writes the // register one slot early. if (VNInfo *DefVNI = LRQ.valueDefined()) Idx = DefVNI->def; WorkList.push_back(std::make_pair(Idx, VNI)); } // Create a new live interval with only minimal live segments per def. LiveInterval NewLI(li->reg, 0); for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end(); I != E; ++I) { VNInfo *VNI = *I; if (VNI->isUnused()) continue; NewLI.addRange(LiveRange(VNI->def, VNI->def.getDeadSlot(), VNI)); } // Keep track of the PHIs that are in use. SmallPtrSet<VNInfo*, 8> UsedPHIs; // Extend intervals to reach all uses in WorkList. while (!WorkList.empty()) { SlotIndex Idx = WorkList.back().first; VNInfo *VNI = WorkList.back().second; WorkList.pop_back(); const MachineBasicBlock *MBB = getMBBFromIndex(Idx.getPrevSlot()); SlotIndex BlockStart = getMBBStartIdx(MBB); // Extend the live range for VNI to be live at Idx. if (VNInfo *ExtVNI = NewLI.extendInBlock(BlockStart, Idx)) { (void)ExtVNI; assert(ExtVNI == VNI && "Unexpected existing value number"); // Is this a PHIDef we haven't seen before? if (!VNI->isPHIDef() || VNI->def != BlockStart || !UsedPHIs.insert(VNI)) continue; // The PHI is live, make sure the predecessors are live-out. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { if (!LiveOut.insert(*PI)) continue; SlotIndex Stop = getMBBEndIdx(*PI); // A predecessor is not required to have a live-out value for a PHI. if (VNInfo *PVNI = li->getVNInfoBefore(Stop)) WorkList.push_back(std::make_pair(Stop, PVNI)); } continue; } // VNI is live-in to MBB. DEBUG(dbgs() << " live-in at " << BlockStart << '\n'); NewLI.addRange(LiveRange(BlockStart, Idx, VNI)); // Make sure VNI is live-out from the predecessors. for (MachineBasicBlock::const_pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { if (!LiveOut.insert(*PI)) continue; SlotIndex Stop = getMBBEndIdx(*PI); assert(li->getVNInfoBefore(Stop) == VNI && "Wrong value out of predecessor"); WorkList.push_back(std::make_pair(Stop, VNI)); } } // Handle dead values. bool CanSeparate = false; for (LiveInterval::vni_iterator I = li->vni_begin(), E = li->vni_end(); I != E; ++I) { VNInfo *VNI = *I; if (VNI->isUnused()) continue; LiveInterval::iterator LII = NewLI.FindLiveRangeContaining(VNI->def); assert(LII != NewLI.end() && "Missing live range for PHI"); if (LII->end != VNI->def.getDeadSlot()) continue; if (VNI->isPHIDef()) { // This is a dead PHI. Remove it. VNI->markUnused(); NewLI.removeRange(*LII); DEBUG(dbgs() << "Dead PHI at " << VNI->def << " may separate interval\n"); CanSeparate = true; } else { // This is a dead def. Make sure the instruction knows. MachineInstr *MI = getInstructionFromIndex(VNI->def); assert(MI && "No instruction defining live value"); MI->addRegisterDead(li->reg, TRI); if (dead && MI->allDefsAreDead()) { DEBUG(dbgs() << "All defs dead: " << VNI->def << '\t' << *MI); dead->push_back(MI); } } } // Move the trimmed ranges back. li->ranges.swap(NewLI.ranges); DEBUG(dbgs() << "Shrunk: " << *li << '\n'); return CanSeparate; }
void RenameIndependentSubregs::computeMainRangesFixFlags( const IntEqClasses &Classes, const SmallVectorImpl<SubRangeInfo> &SubRangeInfos, const SmallVectorImpl<LiveInterval*> &Intervals) const { BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); const SlotIndexes &Indexes = *LIS->getSlotIndexes(); for (size_t I = 0, E = Intervals.size(); I < E; ++I) { LiveInterval &LI = *Intervals[I]; unsigned Reg = LI.reg; LI.removeEmptySubRanges(); // There must be a def (or live-in) before every use. Splitting vregs may // violate this principle as the splitted vreg may not have a definition on // every path. Fix this by creating IMPLICIT_DEF instruction as necessary. for (const LiveInterval::SubRange &SR : LI.subranges()) { // Search for "PHI" value numbers in the subranges. We must find a live // value in each predecessor block, add an IMPLICIT_DEF where it is // missing. for (unsigned I = 0; I < SR.valnos.size(); ++I) { const VNInfo &VNI = *SR.valnos[I]; if (VNI.isUnused() || !VNI.isPHIDef()) continue; SlotIndex Def = VNI.def; MachineBasicBlock &MBB = *Indexes.getMBBFromIndex(Def); for (MachineBasicBlock *PredMBB : MBB.predecessors()) { SlotIndex PredEnd = Indexes.getMBBEndIdx(PredMBB); if (subRangeLiveAt(LI, PredEnd.getPrevSlot())) continue; MachineBasicBlock::iterator InsertPos = llvm::findPHICopyInsertPoint(PredMBB, &MBB, Reg); const MCInstrDesc &MCDesc = TII->get(TargetOpcode::IMPLICIT_DEF); MachineInstrBuilder ImpDef = BuildMI(*PredMBB, InsertPos, DebugLoc(), MCDesc, Reg); SlotIndex DefIdx = LIS->InsertMachineInstrInMaps(*ImpDef); SlotIndex RegDefIdx = DefIdx.getRegSlot(); for (LiveInterval::SubRange &SR : LI.subranges()) { VNInfo *SRVNI = SR.getNextValue(RegDefIdx, Allocator); SR.addSegment(LiveRange::Segment(RegDefIdx, PredEnd, SRVNI)); } } } } for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) { if (!MO.isDef()) continue; unsigned SubRegIdx = MO.getSubReg(); if (SubRegIdx == 0) continue; // After assigning the new vreg we may not have any other sublanes living // in and out of the instruction anymore. We need to add new dead and // undef flags in these cases. if (!MO.isUndef()) { SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent()); if (!subRangeLiveAt(LI, Pos)) MO.setIsUndef(); } if (!MO.isDead()) { SlotIndex Pos = LIS->getInstructionIndex(*MO.getParent()).getDeadSlot(); if (!subRangeLiveAt(LI, Pos)) MO.setIsDead(); } } if (I == 0) LI.clear(); LIS->constructMainRangeFromSubranges(LI); // A def of a subregister may be a use of other register lanes. Replacing // such a def with a def of a different register will eliminate the use, // and may cause the recorded live range to be larger than the actual // liveness in the program IR. LIS->shrinkToUses(&LI); } }