/// Changes operand OpNum in MI the refer the PhysReg, considering subregs. This /// may invalidate any operand pointers. Return true if the operand kills its /// register. bool RegAllocFast::setPhysReg(MachineInstr &MI, MachineOperand &MO, MCPhysReg PhysReg) { bool Dead = MO.isDead(); if (!MO.getSubReg()) { MO.setReg(PhysReg); MO.setIsRenamable(true); return MO.isKill() || Dead; } // Handle subregister index. MO.setReg(PhysReg ? TRI->getSubReg(PhysReg, MO.getSubReg()) : 0); MO.setIsRenamable(true); MO.setSubReg(0); // A kill flag implies killing the full register. Add corresponding super // register kill. if (MO.isKill()) { MI.addRegisterKilled(PhysReg, TRI, true); return true; } // A <def,read-undef> of a sub-register requires an implicit def of the full // register. if (MO.isDef() && MO.isUndef()) MI.addRegisterDefined(PhysReg, TRI); return Dead; }
// Split MI into the comparison given by CompareOpcode followed // a BRCL on the result. void SystemZLongBranch::splitCompareBranch(MachineInstr *MI, unsigned CompareOpcode) { MachineBasicBlock *MBB = MI->getParent(); DebugLoc DL = MI->getDebugLoc(); BuildMI(*MBB, MI, DL, TII->get(CompareOpcode)) .addOperand(MI->getOperand(0)) .addOperand(MI->getOperand(1)); MachineInstr *BRCL = BuildMI(*MBB, MI, DL, TII->get(SystemZ::BRCL)) .addImm(SystemZ::CCMASK_ICMP) .addOperand(MI->getOperand(2)) .addOperand(MI->getOperand(3)); // The implicit use of CC is a killing use. BRCL->addRegisterKilled(SystemZ::CC, &TII->getRegisterInfo()); MI->eraseFromParent(); }
void VirtRegRewriter::rewrite() { bool NoSubRegLiveness = !MRI->subRegLivenessEnabled(); SmallVector<unsigned, 8> SuperDeads; SmallVector<unsigned, 8> SuperDefs; SmallVector<unsigned, 8> SuperKills; for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end(); MBBI != MBBE; ++MBBI) { DEBUG(MBBI->print(dbgs(), Indexes)); for (MachineBasicBlock::instr_iterator MII = MBBI->instr_begin(), MIE = MBBI->instr_end(); MII != MIE;) { MachineInstr *MI = &*MII; ++MII; for (MachineInstr::mop_iterator MOI = MI->operands_begin(), MOE = MI->operands_end(); MOI != MOE; ++MOI) { MachineOperand &MO = *MOI; // Make sure MRI knows about registers clobbered by regmasks. if (MO.isRegMask()) MRI->addPhysRegsUsedFromRegMask(MO.getRegMask()); if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) continue; unsigned VirtReg = MO.getReg(); unsigned PhysReg = VRM->getPhys(VirtReg); assert(PhysReg != VirtRegMap::NO_PHYS_REG && "Instruction uses unmapped VirtReg"); assert(!MRI->isReserved(PhysReg) && "Reserved register assignment"); // Preserve semantics of sub-register operands. unsigned SubReg = MO.getSubReg(); if (SubReg != 0) { if (NoSubRegLiveness) { // A virtual register kill refers to the whole register, so we may // have to add <imp-use,kill> operands for the super-register. A // partial redef always kills and redefines the super-register. if (MO.readsReg() && (MO.isDef() || MO.isKill())) SuperKills.push_back(PhysReg); if (MO.isDef()) { // Also add implicit defs for the super-register. if (MO.isDead()) SuperDeads.push_back(PhysReg); else SuperDefs.push_back(PhysReg); } } else { if (MO.isUse()) { if (readsUndefSubreg(MO)) // We need to add an <undef> flag if the subregister is // completely undefined (and we are not adding super-register // defs). MO.setIsUndef(true); } else if (!MO.isDead()) { assert(MO.isDef()); } } // The <def,undef> flag only makes sense for sub-register defs, and // we are substituting a full physreg. An <imp-use,kill> operand // from the SuperKills list will represent the partial read of the // super-register. if (MO.isDef()) MO.setIsUndef(false); // PhysReg operands cannot have subregister indexes. PhysReg = TRI->getSubReg(PhysReg, SubReg); assert(PhysReg && "Invalid SubReg for physical register"); MO.setSubReg(0); } // Rewrite. Note we could have used MachineOperand::substPhysReg(), but // we need the inlining here. MO.setReg(PhysReg); } // Add any missing super-register kills after rewriting the whole // instruction. while (!SuperKills.empty()) MI->addRegisterKilled(SuperKills.pop_back_val(), TRI, true); while (!SuperDeads.empty()) MI->addRegisterDead(SuperDeads.pop_back_val(), TRI, true); while (!SuperDefs.empty()) MI->addRegisterDefined(SuperDefs.pop_back_val(), TRI); DEBUG(dbgs() << "> " << *MI); // Finally, remove any identity copies. if (MI->isIdentityCopy()) { ++NumIdCopies; DEBUG(dbgs() << "Deleting identity copy.\n"); if (Indexes) Indexes->removeMachineInstrFromMaps(*MI); // It's safe to erase MI because MII has already been incremented. MI->eraseFromParent(); } } } }
void LiveIntervals::addKillFlags(const VirtRegMap *VRM) { // Keep track of regunit ranges. SmallVector<std::pair<const LiveRange*, LiveRange::const_iterator>, 8> RU; // Keep track of subregister ranges. SmallVector<std::pair<const LiveInterval::SubRange*, LiveRange::const_iterator>, 4> SRs; for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { unsigned Reg = TargetRegisterInfo::index2VirtReg(i); if (MRI->reg_nodbg_empty(Reg)) continue; const LiveInterval &LI = getInterval(Reg); if (LI.empty()) continue; // Find the regunit intervals for the assigned register. They may overlap // the virtual register live range, cancelling any kills. RU.clear(); for (MCRegUnitIterator Units(VRM->getPhys(Reg), TRI); Units.isValid(); ++Units) { const LiveRange &RURange = getRegUnit(*Units); if (RURange.empty()) continue; RU.push_back(std::make_pair(&RURange, RURange.find(LI.begin()->end))); } if (MRI->tracksSubRegLiveness()) { SRs.clear(); for (const LiveInterval::SubRange &SR : LI.subranges()) { SRs.push_back(std::make_pair(&SR, SR.find(LI.begin()->end))); } } // Every instruction that kills Reg corresponds to a segment range end // point. for (LiveInterval::const_iterator RI = LI.begin(), RE = LI.end(); RI != RE; ++RI) { // A block index indicates an MBB edge. if (RI->end.isBlock()) continue; MachineInstr *MI = getInstructionFromIndex(RI->end); if (!MI) continue; // Check if any of the regunits are live beyond the end of RI. That could // happen when a physreg is defined as a copy of a virtreg: // // %EAX = COPY %vreg5 // FOO %vreg5 <--- MI, cancel kill because %EAX is live. // BAR %EAX<kill> // // There should be no kill flag on FOO when %vreg5 is rewritten as %EAX. for (auto &RUP : RU) { const LiveRange &RURange = *RUP.first; LiveRange::const_iterator &I = RUP.second; if (I == RURange.end()) continue; I = RURange.advanceTo(I, RI->end); if (I == RURange.end() || I->start >= RI->end) continue; // I is overlapping RI. goto CancelKill; } if (MRI->tracksSubRegLiveness()) { // When reading a partial undefined value we must not add a kill flag. // The regalloc might have used the undef lane for something else. // Example: // %vreg1 = ... ; R32: %vreg1 // %vreg2:high16 = ... ; R64: %vreg2 // = read %vreg2<kill> ; R64: %vreg2 // = read %vreg1 ; R32: %vreg1 // The <kill> flag is correct for %vreg2, but the register allocator may // assign R0L to %vreg1, and R0 to %vreg2 because the low 32bits of R0 // are actually never written by %vreg2. After assignment the <kill> // flag at the read instruction is invalid. unsigned DefinedLanesMask; if (!SRs.empty()) { // Compute a mask of lanes that are defined. DefinedLanesMask = 0; for (auto &SRP : SRs) { const LiveInterval::SubRange &SR = *SRP.first; LiveRange::const_iterator &I = SRP.second; if (I == SR.end()) continue; I = SR.advanceTo(I, RI->end); if (I == SR.end() || I->start >= RI->end) continue; // I is overlapping RI DefinedLanesMask |= SR.LaneMask; } } else DefinedLanesMask = ~0u; bool IsFullWrite = false; for (const MachineOperand &MO : MI->operands()) { if (!MO.isReg() || MO.getReg() != Reg) continue; if (MO.isUse()) { // Reading any undefined lanes? unsigned UseMask = TRI->getSubRegIndexLaneMask(MO.getSubReg()); if ((UseMask & ~DefinedLanesMask) != 0) goto CancelKill; } else if (MO.getSubReg() == 0) { // Writing to the full register? assert(MO.isDef()); IsFullWrite = true; } } // If an instruction writes to a subregister, a new segment starts in // the LiveInterval. But as this is only overriding part of the register // adding kill-flags is not correct here after registers have been // assigned. if (!IsFullWrite) { // Next segment has to be adjacent in the subregister write case. LiveRange::const_iterator N = std::next(RI); if (N != LI.end() && N->start == RI->end) goto CancelKill; } } MI->addRegisterKilled(Reg, nullptr); continue; CancelKill: MI->clearRegisterKills(Reg, nullptr); } } }
void VirtRegRewriter::rewrite() { SmallVector<unsigned, 8> SuperDeads; SmallVector<unsigned, 8> SuperDefs; SmallVector<unsigned, 8> SuperKills; SmallPtrSet<const MachineInstr *, 4> NoReturnInsts; // Here we have a SparseSet to hold which PhysRegs are actually encountered // in the MF we are about to iterate over so that later when we call // setPhysRegUsed, we are only doing it for physRegs that were actually found // in the program and not for all of the possible physRegs for the given // target architecture. If the target has a lot of physRegs, then for a small // program there will be a significant compile time reduction here. PhysRegs.clear(); PhysRegs.setUniverse(TRI->getNumRegs()); // The function with uwtable should guarantee that the stack unwinder // can unwind the stack to the previous frame. Thus, we can't apply the // noreturn optimization if the caller function has uwtable attribute. bool HasUWTable = MF->getFunction()->hasFnAttribute(Attribute::UWTable); for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end(); MBBI != MBBE; ++MBBI) { DEBUG(MBBI->print(dbgs(), Indexes)); bool IsExitBB = MBBI->succ_empty(); for (MachineBasicBlock::instr_iterator MII = MBBI->instr_begin(), MIE = MBBI->instr_end(); MII != MIE;) { MachineInstr *MI = MII; ++MII; // Check if this instruction is a call to a noreturn function. If this // is a call to noreturn function and we don't need the stack unwinding // functionality (i.e. this function does not have uwtable attribute and // the callee function has the nounwind attribute), then we can ignore // the definitions set by this instruction. if (!HasUWTable && IsExitBB && MI->isCall()) { for (MachineInstr::mop_iterator MOI = MI->operands_begin(), MOE = MI->operands_end(); MOI != MOE; ++MOI) { MachineOperand &MO = *MOI; if (!MO.isGlobal()) continue; const Function *Func = dyn_cast<Function>(MO.getGlobal()); if (!Func || !Func->hasFnAttribute(Attribute::NoReturn) || // We need to keep correct unwind information // even if the function will not return, since the // runtime may need it. !Func->hasFnAttribute(Attribute::NoUnwind)) continue; NoReturnInsts.insert(MI); break; } } for (MachineInstr::mop_iterator MOI = MI->operands_begin(), MOE = MI->operands_end(); MOI != MOE; ++MOI) { MachineOperand &MO = *MOI; // Make sure MRI knows about registers clobbered by regmasks. if (MO.isRegMask()) MRI->addPhysRegsUsedFromRegMask(MO.getRegMask()); // If we encounter a VirtReg or PhysReg then get at the PhysReg and add // it to the physreg bitset. Later we use only the PhysRegs that were // actually encountered in the MF to populate the MRI's used physregs. if (MO.isReg() && MO.getReg()) PhysRegs.insert( TargetRegisterInfo::isVirtualRegister(MO.getReg()) ? VRM->getPhys(MO.getReg()) : MO.getReg()); if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) continue; unsigned VirtReg = MO.getReg(); unsigned PhysReg = VRM->getPhys(VirtReg); assert(PhysReg != VirtRegMap::NO_PHYS_REG && "Instruction uses unmapped VirtReg"); assert(!MRI->isReserved(PhysReg) && "Reserved register assignment"); // Preserve semantics of sub-register operands. if (MO.getSubReg()) { // A virtual register kill refers to the whole register, so we may // have to add <imp-use,kill> operands for the super-register. A // partial redef always kills and redefines the super-register. if (MO.readsReg() && (MO.isDef() || MO.isKill())) SuperKills.push_back(PhysReg); if (MO.isDef()) { // The <def,undef> flag only makes sense for sub-register defs, and // we are substituting a full physreg. An <imp-use,kill> operand // from the SuperKills list will represent the partial read of the // super-register. MO.setIsUndef(false); // Also add implicit defs for the super-register. if (MO.isDead()) SuperDeads.push_back(PhysReg); else SuperDefs.push_back(PhysReg); } // PhysReg operands cannot have subregister indexes. PhysReg = TRI->getSubReg(PhysReg, MO.getSubReg()); assert(PhysReg && "Invalid SubReg for physical register"); MO.setSubReg(0); } // Rewrite. Note we could have used MachineOperand::substPhysReg(), but // we need the inlining here. MO.setReg(PhysReg); } // Add any missing super-register kills after rewriting the whole // instruction. while (!SuperKills.empty()) MI->addRegisterKilled(SuperKills.pop_back_val(), TRI, true); while (!SuperDeads.empty()) MI->addRegisterDead(SuperDeads.pop_back_val(), TRI, true); while (!SuperDefs.empty()) MI->addRegisterDefined(SuperDefs.pop_back_val(), TRI); DEBUG(dbgs() << "> " << *MI); // Finally, remove any identity copies. if (MI->isIdentityCopy()) { ++NumIdCopies; if (MI->getNumOperands() == 2) { DEBUG(dbgs() << "Deleting identity copy.\n"); if (Indexes) Indexes->removeMachineInstrFromMaps(MI); // It's safe to erase MI because MII has already been incremented. MI->eraseFromParent(); } else { // Transform identity copy to a KILL to deal with subregisters. MI->setDesc(TII->get(TargetOpcode::KILL)); DEBUG(dbgs() << "Identity copy: " << *MI); } } } } // Tell MRI about physical registers in use. if (NoReturnInsts.empty()) { for (SparseSet<unsigned>::iterator RegI = PhysRegs.begin(), E = PhysRegs.end(); RegI != E; ++RegI) if (!MRI->reg_nodbg_empty(*RegI)) MRI->setPhysRegUsed(*RegI); } else { for (SparseSet<unsigned>::iterator I = PhysRegs.begin(), E = PhysRegs.end(); I != E; ++I) { unsigned Reg = *I; if (MRI->reg_nodbg_empty(Reg)) continue; // Check if this register has a use that will impact the rest of the // code. Uses in debug and noreturn instructions do not impact the // generated code. for (MachineInstr &It : MRI->reg_nodbg_instructions(Reg)) { if (!NoReturnInsts.count(&It)) { MRI->setPhysRegUsed(Reg); break; } } } } }
bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *MI) { MachineInstr *LastDef = PhysRegDef[Reg]; MachineInstr *LastUse = PhysRegUse[Reg]; if (!LastDef && !LastUse) return false; MachineInstr *LastRefOrPartRef = LastUse ? LastUse : LastDef; unsigned LastRefOrPartRefDist = DistanceMap[LastRefOrPartRef]; // The whole register is used. // AL = // AH = // // = AX // = AL, AX<imp-use, kill> // AX = // // Or whole register is defined, but not used at all. // AX<dead> = // ... // AX = // // Or whole register is defined, but only partly used. // AX<dead> = AL<imp-def> // = AL<kill> // AX = MachineInstr *LastPartDef = nullptr; unsigned LastPartDefDist = 0; SmallSet<unsigned, 8> PartUses; for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) { unsigned SubReg = *SubRegs; MachineInstr *Def = PhysRegDef[SubReg]; if (Def && Def != LastDef) { // There was a def of this sub-register in between. This is a partial // def, keep track of the last one. unsigned Dist = DistanceMap[Def]; if (Dist > LastPartDefDist) { LastPartDefDist = Dist; LastPartDef = Def; } continue; } if (MachineInstr *Use = PhysRegUse[SubReg]) { for (MCSubRegIterator SS(SubReg, TRI, /*IncludeSelf=*/true); SS.isValid(); ++SS) PartUses.insert(*SS); unsigned Dist = DistanceMap[Use]; if (Dist > LastRefOrPartRefDist) { LastRefOrPartRefDist = Dist; LastRefOrPartRef = Use; } } } if (!PhysRegUse[Reg]) { // Partial uses. Mark register def dead and add implicit def of // sub-registers which are used. // EAX<dead> = op AL<imp-def> // That is, EAX def is dead but AL def extends pass it. PhysRegDef[Reg]->addRegisterDead(Reg, TRI, true); for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) { unsigned SubReg = *SubRegs; if (!PartUses.count(SubReg)) continue; bool NeedDef = true; if (PhysRegDef[Reg] == PhysRegDef[SubReg]) { MachineOperand *MO = PhysRegDef[Reg]->findRegisterDefOperand(SubReg); if (MO) { NeedDef = false; assert(!MO->isDead()); } } if (NeedDef) PhysRegDef[Reg]->addOperand(MachineOperand::CreateReg(SubReg, true/*IsDef*/, true/*IsImp*/)); MachineInstr *LastSubRef = FindLastRefOrPartRef(SubReg); if (LastSubRef) LastSubRef->addRegisterKilled(SubReg, TRI, true); else { LastRefOrPartRef->addRegisterKilled(SubReg, TRI, true); for (MCSubRegIterator SS(SubReg, TRI, /*IncludeSelf=*/true); SS.isValid(); ++SS) PhysRegUse[*SS] = LastRefOrPartRef; } for (MCSubRegIterator SS(SubReg, TRI); SS.isValid(); ++SS) PartUses.erase(*SS); } } else if (LastRefOrPartRef == PhysRegDef[Reg] && LastRefOrPartRef != MI) { if (LastPartDef) // The last partial def kills the register. LastPartDef->addOperand(MachineOperand::CreateReg(Reg, false/*IsDef*/, true/*IsImp*/, true/*IsKill*/)); else { MachineOperand *MO = LastRefOrPartRef->findRegisterDefOperand(Reg, false, TRI); bool NeedEC = MO->isEarlyClobber() && MO->getReg() != Reg; // If the last reference is the last def, then it's not used at all. // That is, unless we are currently processing the last reference itself. LastRefOrPartRef->addRegisterDead(Reg, TRI, true); if (NeedEC) { // If we are adding a subreg def and the superreg def is marked early // clobber, add an early clobber marker to the subreg def. MO = LastRefOrPartRef->findRegisterDefOperand(Reg); if (MO) MO->setIsEarlyClobber(); } } } else LastRefOrPartRef->addRegisterKilled(Reg, TRI, true); return true; }
void LiveIntervals::addKillFlags(const VirtRegMap *VRM) { // Keep track of regunit ranges. SmallVector<std::pair<LiveInterval*, LiveInterval::iterator>, 8> RU; for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { unsigned Reg = TargetRegisterInfo::index2VirtReg(i); if (MRI->reg_nodbg_empty(Reg)) continue; LiveInterval *LI = &getInterval(Reg); if (LI->empty()) continue; // Find the regunit intervals for the assigned register. They may overlap // the virtual register live range, cancelling any kills. RU.clear(); for (MCRegUnitIterator Units(VRM->getPhys(Reg), TRI); Units.isValid(); ++Units) { LiveInterval *RUInt = &getRegUnit(*Units); if (RUInt->empty()) continue; RU.push_back(std::make_pair(RUInt, RUInt->find(LI->begin()->end))); } // Every instruction that kills Reg corresponds to a live range end point. for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE; ++RI) { // A block index indicates an MBB edge. if (RI->end.isBlock()) continue; MachineInstr *MI = getInstructionFromIndex(RI->end); if (!MI) continue; // Check if any of the regunits are live beyond the end of RI. That could // happen when a physreg is defined as a copy of a virtreg: // // %EAX = COPY %vreg5 // FOO %vreg5 <--- MI, cancel kill because %EAX is live. // BAR %EAX<kill> // // There should be no kill flag on FOO when %vreg5 is rewritten as %EAX. bool CancelKill = false; for (unsigned u = 0, e = RU.size(); u != e; ++u) { LiveInterval *RInt = RU[u].first; LiveInterval::iterator &I = RU[u].second; if (I == RInt->end()) continue; I = RInt->advanceTo(I, RI->end); if (I == RInt->end() || I->start >= RI->end) continue; // I is overlapping RI. CancelKill = true; break; } if (CancelKill) MI->clearRegisterKills(Reg, NULL); else MI->addRegisterKilled(Reg, NULL); } } }
/// isSafeToMoveTogether - Returns true if it is safe to move I1 next to I2 such /// that the two instructions can be paired in a combine. bool HexagonCopyToCombine::isSafeToMoveTogether(MachineInstr *I1, MachineInstr *I2, unsigned I1DestReg, unsigned I2DestReg, bool &DoInsertAtI1) { unsigned I2UseReg = UseReg(I2->getOperand(1)); // It is not safe to move I1 and I2 into one combine if I2 has a true // dependence on I1. if (I2UseReg && I1->modifiesRegister(I2UseReg, TRI)) return false; bool isSafe = true; // First try to move I2 towards I1. { // A reverse_iterator instantiated like below starts before I2, and I1 // respectively. // Look at instructions I in between I2 and (excluding) I1. MachineBasicBlock::reverse_iterator I(I2), End = --(MachineBasicBlock::reverse_iterator(I1)); // At 03 we got better results (dhrystone!) by being more conservative. if (!ShouldCombineAggressively) End = MachineBasicBlock::reverse_iterator(I1); // If I2 kills its operand and we move I2 over an instruction that also // uses I2's use reg we need to modify that (first) instruction to now kill // this reg. unsigned KilledOperand = 0; if (I2->killsRegister(I2UseReg)) KilledOperand = I2UseReg; MachineInstr *KillingInstr = nullptr; for (; I != End; ++I) { // If the intervening instruction I: // * modifies I2's use reg // * modifies I2's def reg // * reads I2's def reg // * or has unmodelled side effects // we can't move I2 across it. if (isUnsafeToMoveAcross(&*I, I2UseReg, I2DestReg, TRI)) { isSafe = false; break; } // Update first use of the killed operand. if (!KillingInstr && KilledOperand && I->readsRegister(KilledOperand, TRI)) KillingInstr = &*I; } if (isSafe) { // Update the intermediate instruction to with the kill flag. if (KillingInstr) { bool Added = KillingInstr->addRegisterKilled(KilledOperand, TRI, true); (void)Added; // suppress compiler warning assert(Added && "Must successfully update kill flag"); removeKillInfo(I2, KilledOperand); } DoInsertAtI1 = true; return true; } } // Try to move I1 towards I2. { // Look at instructions I in between I1 and (excluding) I2. MachineBasicBlock::iterator I(I1), End(I2); // At O3 we got better results (dhrystone) by being more conservative here. if (!ShouldCombineAggressively) End = std::next(MachineBasicBlock::iterator(I2)); unsigned I1UseReg = UseReg(I1->getOperand(1)); // Track killed operands. If we move across an instruction that kills our // operand, we need to update the kill information on the moved I1. It kills // the operand now. MachineInstr *KillingInstr = nullptr; unsigned KilledOperand = 0; while(++I != End) { // If the intervening instruction I: // * modifies I1's use reg // * modifies I1's def reg // * reads I1's def reg // * or has unmodelled side effects // We introduce this special case because llvm has no api to remove a // kill flag for a register (a removeRegisterKilled() analogous to // addRegisterKilled) that handles aliased register correctly. // * or has a killed aliased register use of I1's use reg // %D4<def> = TFRI64 16 // %R6<def> = TFR %R9 // %R8<def> = KILL %R8, %D4<imp-use,kill> // If we want to move R6 = across the KILL instruction we would have // to remove the %D4<imp-use,kill> operand. For now, we are // conservative and disallow the move. // we can't move I1 across it. if (isUnsafeToMoveAcross(I, I1UseReg, I1DestReg, TRI) || // Check for an aliased register kill. Bail out if we see one. (!I->killsRegister(I1UseReg) && I->killsRegister(I1UseReg, TRI))) return false; // Check for an exact kill (registers match). if (I1UseReg && I->killsRegister(I1UseReg)) { assert(!KillingInstr && "Should only see one killing instruction"); KilledOperand = I1UseReg; KillingInstr = &*I; } } if (KillingInstr) { removeKillInfo(KillingInstr, KilledOperand); // Update I1 to set the kill flag. This flag will later be picked up by // the new COMBINE instruction. bool Added = I1->addRegisterKilled(KilledOperand, TRI); (void)Added; // suppress compiler warning assert(Added && "Must successfully update kill flag"); } DoInsertAtI1 = false; } return true; }
bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *MI) { if (!PhysRegUse[Reg] && !PhysRegDef[Reg]) return false; MachineInstr *LastRefOrPartRef = PhysRegUse[Reg] ? PhysRegUse[Reg] : PhysRegDef[Reg]; unsigned LastRefOrPartRefDist = DistanceMap[LastRefOrPartRef]; // The whole register is used. // AL = // AH = // // = AX // = AL, AX<imp-use, kill> // AX = // // Or whole register is defined, but not used at all. // AX<dead> = // ... // AX = // // Or whole register is defined, but only partly used. // AX<dead> = AL<imp-def> // = AL<kill> // AX = SmallSet<unsigned, 8> PartUses; for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); unsigned SubReg = *SubRegs; ++SubRegs) { if (MachineInstr *Use = PhysRegUse[SubReg]) { PartUses.insert(SubReg); for (const unsigned *SS = TRI->getSubRegisters(SubReg); *SS; ++SS) PartUses.insert(*SS); unsigned Dist = DistanceMap[Use]; if (Dist > LastRefOrPartRefDist) { LastRefOrPartRefDist = Dist; LastRefOrPartRef = Use; } } } if (LastRefOrPartRef == PhysRegDef[Reg] && LastRefOrPartRef != MI) // If the last reference is the last def, then it's not used at all. // That is, unless we are currently processing the last reference itself. LastRefOrPartRef->addRegisterDead(Reg, TRI, true); /* Partial uses. Mark register def dead and add implicit def of sub-registers which are used. FIXME: LiveIntervalAnalysis can't handle this yet! EAX<dead> = op AL<imp-def> That is, EAX def is dead but AL def extends pass it. Enable this after live interval analysis is fixed to improve codegen! else if (!PhysRegUse[Reg]) { PhysRegDef[Reg]->addRegisterDead(Reg, TRI, true); for (const unsigned *SubRegs = TRI->getSubRegisters(Reg); unsigned SubReg = *SubRegs; ++SubRegs) { if (PartUses.count(SubReg)) { PhysRegDef[Reg]->addOperand(MachineOperand::CreateReg(SubReg, true, true)); LastRefOrPartRef->addRegisterKilled(SubReg, TRI, true); for (const unsigned *SS = TRI->getSubRegisters(SubReg); *SS; ++SS) PartUses.erase(*SS); } } } */ else LastRefOrPartRef->addRegisterKilled(Reg, TRI, true); return true; }
void Mos6502InstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, DebugLoc DL, unsigned DestReg, unsigned SrcReg, bool KillSrc) const { unsigned numSubRegs = 0; unsigned movOpc = 0; const unsigned *subRegIdx = nullptr; const unsigned DFP_FP_SubRegsIdx[] = { M6502::sub_even, M6502::sub_odd }; const unsigned QFP_DFP_SubRegsIdx[] = { M6502::sub_even64, M6502::sub_odd64 }; const unsigned QFP_FP_SubRegsIdx[] = { M6502::sub_even, M6502::sub_odd, M6502::sub_odd64_then_sub_even, M6502::sub_odd64_then_sub_odd }; if (M6502::IntRegsRegClass.contains(DestReg, SrcReg)) BuildMI(MBB, I, DL, get(M6502::ORrr), DestReg).addReg(M6502::G0) .addReg(SrcReg, getKillRegState(KillSrc)); else if (M6502::FPRegsRegClass.contains(DestReg, SrcReg)) BuildMI(MBB, I, DL, get(M6502::FMOVS), DestReg) .addReg(SrcReg, getKillRegState(KillSrc)); else if (M6502::DFPRegsRegClass.contains(DestReg, SrcReg)) { if (Subtarget.isV9()) { BuildMI(MBB, I, DL, get(M6502::FMOVD), DestReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else { // Use two FMOVS instructions. subRegIdx = DFP_FP_SubRegsIdx; numSubRegs = 2; movOpc = M6502::FMOVS; } } else if (M6502::QFPRegsRegClass.contains(DestReg, SrcReg)) { if (Subtarget.isV9()) { if (Subtarget.hasHardQuad()) { BuildMI(MBB, I, DL, get(M6502::FMOVQ), DestReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else { // Use two FMOVD instructions. subRegIdx = QFP_DFP_SubRegsIdx; numSubRegs = 2; movOpc = M6502::FMOVD; } } else { // Use four FMOVS instructions. subRegIdx = QFP_FP_SubRegsIdx; numSubRegs = 4; movOpc = M6502::FMOVS; } } else if (M6502::ASRRegsRegClass.contains(DestReg) && M6502::IntRegsRegClass.contains(SrcReg)) { BuildMI(MBB, I, DL, get(M6502::WRASRrr), DestReg) .addReg(M6502::G0) .addReg(SrcReg, getKillRegState(KillSrc)); } else if (M6502::IntRegsRegClass.contains(DestReg) && M6502::ASRRegsRegClass.contains(SrcReg)) { BuildMI(MBB, I, DL, get(M6502::RDASR), DestReg) .addReg(SrcReg, getKillRegState(KillSrc)); } else llvm_unreachable("Impossible reg-to-reg copy"); if (numSubRegs == 0 || subRegIdx == nullptr || movOpc == 0) return; const TargetRegisterInfo *TRI = &getRegisterInfo(); MachineInstr *MovMI = nullptr; for (unsigned i = 0; i != numSubRegs; ++i) { unsigned Dst = TRI->getSubReg(DestReg, subRegIdx[i]); unsigned Src = TRI->getSubReg(SrcReg, subRegIdx[i]); assert(Dst && Src && "Bad sub-register"); MovMI = BuildMI(MBB, I, DL, get(movOpc), Dst).addReg(Src); } // Add implicit super-register defs and kills to the last MovMI. MovMI->addRegisterDefined(DestReg, TRI); if (KillSrc) MovMI->addRegisterKilled(SrcReg, TRI); }
void VirtRegMap::rewrite(SlotIndexes *Indexes) { DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n" << "********** Function: " << MF->getFunction()->getName() << '\n'); DEBUG(dump()); SmallVector<unsigned, 8> SuperDeads; SmallVector<unsigned, 8> SuperDefs; SmallVector<unsigned, 8> SuperKills; for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end(); MBBI != MBBE; ++MBBI) { DEBUG(MBBI->print(dbgs(), Indexes)); for (MachineBasicBlock::iterator MII = MBBI->begin(), MIE = MBBI->end(); MII != MIE;) { MachineInstr *MI = MII; ++MII; for (MachineInstr::mop_iterator MOI = MI->operands_begin(), MOE = MI->operands_end(); MOI != MOE; ++MOI) { MachineOperand &MO = *MOI; if (!MO.isReg() || !TargetRegisterInfo::isVirtualRegister(MO.getReg())) continue; unsigned VirtReg = MO.getReg(); unsigned PhysReg = getPhys(VirtReg); assert(PhysReg != NO_PHYS_REG && "Instruction uses unmapped VirtReg"); // Preserve semantics of sub-register operands. if (MO.getSubReg()) { // A virtual register kill refers to the whole register, so we may // have to add <imp-use,kill> operands for the super-register. A // partial redef always kills and redefines the super-register. if (MO.readsReg() && (MO.isDef() || MO.isKill())) SuperKills.push_back(PhysReg); if (MO.isDef()) { // The <def,undef> flag only makes sense for sub-register defs, and // we are substituting a full physreg. An <imp-use,kill> operand // from the SuperKills list will represent the partial read of the // super-register. MO.setIsUndef(false); // Also add implicit defs for the super-register. if (MO.isDead()) SuperDeads.push_back(PhysReg); else SuperDefs.push_back(PhysReg); } // PhysReg operands cannot have subregister indexes. PhysReg = TRI->getSubReg(PhysReg, MO.getSubReg()); assert(PhysReg && "Invalid SubReg for physical register"); MO.setSubReg(0); } // Rewrite. Note we could have used MachineOperand::substPhysReg(), but // we need the inlining here. MO.setReg(PhysReg); } // Add any missing super-register kills after rewriting the whole // instruction. while (!SuperKills.empty()) MI->addRegisterKilled(SuperKills.pop_back_val(), TRI, true); while (!SuperDeads.empty()) MI->addRegisterDead(SuperDeads.pop_back_val(), TRI, true); while (!SuperDefs.empty()) MI->addRegisterDefined(SuperDefs.pop_back_val(), TRI); DEBUG(dbgs() << "> " << *MI); // Finally, remove any identity copies. if (MI->isIdentityCopy()) { ++NumIdCopies; if (MI->getNumOperands() == 2) { DEBUG(dbgs() << "Deleting identity copy.\n"); RemoveMachineInstrFromMaps(MI); if (Indexes) Indexes->removeMachineInstrFromMaps(MI); // It's safe to erase MI because MII has already been incremented. MI->eraseFromParent(); } else { // Transform identity copy to a KILL to deal with subregisters. MI->setDesc(TII->get(TargetOpcode::KILL)); DEBUG(dbgs() << "Identity copy: " << *MI); } } } } // Tell MRI about physical registers in use. for (unsigned Reg = 1, RegE = TRI->getNumRegs(); Reg != RegE; ++Reg) if (!MRI->reg_nodbg_empty(Reg)) MRI->setPhysRegUsed(Reg); }
void RAFast::AllocateBasicBlock() { DEBUG(dbgs() << "\nAllocating " << *MBB); // FIXME: This should probably be added by instruction selection instead? // If the last instruction in the block is a return, make sure to mark it as // using all of the live-out values in the function. Things marked both call // and return are tail calls; do not do this for them. The tail callee need // not take the same registers as input that it produces as output, and there // are dependencies for its input registers elsewhere. if (!MBB->empty() && MBB->back().getDesc().isReturn() && !MBB->back().getDesc().isCall()) { MachineInstr *Ret = &MBB->back(); for (MachineRegisterInfo::liveout_iterator I = MF->getRegInfo().liveout_begin(), E = MF->getRegInfo().liveout_end(); I != E; ++I) { assert(TargetRegisterInfo::isPhysicalRegister(*I) && "Cannot have a live-out virtual register."); // Add live-out registers as implicit uses. Ret->addRegisterKilled(*I, TRI, true); } } PhysRegState.assign(TRI->getNumRegs(), regDisabled); assert(LiveVirtRegs.empty() && "Mapping not cleared form last block?"); MachineBasicBlock::iterator MII = MBB->begin(); // Add live-in registers as live. for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(), E = MBB->livein_end(); I != E; ++I) if (Allocatable.test(*I)) definePhysReg(MII, *I, regReserved); SmallVector<unsigned, 8> VirtDead; SmallVector<MachineInstr*, 32> Coalesced; // Otherwise, sequentially allocate each instruction in the MBB. while (MII != MBB->end()) { MachineInstr *MI = MII++; const TargetInstrDesc &TID = MI->getDesc(); DEBUG({ dbgs() << "\n>> " << *MI << "Regs:"; for (unsigned Reg = 1, E = TRI->getNumRegs(); Reg != E; ++Reg) { if (PhysRegState[Reg] == regDisabled) continue; dbgs() << " " << TRI->getName(Reg); switch(PhysRegState[Reg]) { case regFree: break; case regReserved: dbgs() << "*"; break; default: dbgs() << '=' << PrintReg(PhysRegState[Reg]); if (LiveVirtRegs[PhysRegState[Reg]].Dirty) dbgs() << "*"; assert(LiveVirtRegs[PhysRegState[Reg]].PhysReg == Reg && "Bad inverse map"); break; } } dbgs() << '\n'; // Check that LiveVirtRegs is the inverse. for (LiveRegMap::iterator i = LiveVirtRegs.begin(), e = LiveVirtRegs.end(); i != e; ++i) { assert(TargetRegisterInfo::isVirtualRegister(i->first) && "Bad map key"); assert(TargetRegisterInfo::isPhysicalRegister(i->second.PhysReg) && "Bad map value"); assert(PhysRegState[i->second.PhysReg] == i->first && "Bad inverse map"); } }); // Debug values are not allowed to change codegen in any way. if (MI->isDebugValue()) { bool ScanDbgValue = true; while (ScanDbgValue) { ScanDbgValue = false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; LiveDbgValueMap[Reg] = MI; LiveRegMap::iterator LRI = LiveVirtRegs.find(Reg); if (LRI != LiveVirtRegs.end()) setPhysReg(MI, i, LRI->second.PhysReg); else { int SS = StackSlotForVirtReg[Reg]; if (SS == -1) { // We can't allocate a physreg for a DebugValue, sorry! DEBUG(dbgs() << "Unable to allocate vreg used by DBG_VALUE"); MO.setReg(0); } else { // Modify DBG_VALUE now that the value is in a spill slot. int64_t Offset = MI->getOperand(1).getImm(); const MDNode *MDPtr = MI->getOperand(MI->getNumOperands()-1).getMetadata(); DebugLoc DL = MI->getDebugLoc(); if (MachineInstr *NewDV = TII->emitFrameIndexDebugValue(*MF, SS, Offset, MDPtr, DL)) { DEBUG(dbgs() << "Modifying debug info due to spill:" << "\t" << *MI); MachineBasicBlock *MBB = MI->getParent(); MBB->insert(MBB->erase(MI), NewDV); // Scan NewDV operands from the beginning. MI = NewDV; ScanDbgValue = true; break; } else { // We can't allocate a physreg for a DebugValue; sorry! DEBUG(dbgs() << "Unable to allocate vreg used by DBG_VALUE"); MO.setReg(0); } } } } } // Next instruction. continue; } // If this is a copy, we may be able to coalesce. unsigned CopySrc = 0, CopyDst = 0, CopySrcSub = 0, CopyDstSub = 0; if (MI->isCopy()) { CopyDst = MI->getOperand(0).getReg(); CopySrc = MI->getOperand(1).getReg(); CopyDstSub = MI->getOperand(0).getSubReg(); CopySrcSub = MI->getOperand(1).getSubReg(); } // Track registers used by instruction. UsedInInstr.reset(); // First scan. // Mark physreg uses and early clobbers as used. // Find the end of the virtreg operands unsigned VirtOpEnd = 0; bool hasTiedOps = false; bool hasEarlyClobbers = false; bool hasPartialRedefs = false; bool hasPhysDefs = false; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) { VirtOpEnd = i+1; if (MO.isUse()) { hasTiedOps = hasTiedOps || TID.getOperandConstraint(i, TOI::TIED_TO) != -1; } else { if (MO.isEarlyClobber()) hasEarlyClobbers = true; if (MO.getSubReg() && MI->readsVirtualRegister(Reg)) hasPartialRedefs = true; } continue; } if (!Allocatable.test(Reg)) continue; if (MO.isUse()) { usePhysReg(MO); } else if (MO.isEarlyClobber()) { definePhysReg(MI, Reg, (MO.isImplicit() || MO.isDead()) ? regFree : regReserved); hasEarlyClobbers = true; } else hasPhysDefs = true; } // The instruction may have virtual register operands that must be allocated // the same register at use-time and def-time: early clobbers and tied // operands. If there are also physical defs, these registers must avoid // both physical defs and uses, making them more constrained than normal // operands. // Similarly, if there are multiple defs and tied operands, we must make // sure the same register is allocated to uses and defs. // We didn't detect inline asm tied operands above, so just make this extra // pass for all inline asm. if (MI->isInlineAsm() || hasEarlyClobbers || hasPartialRedefs || (hasTiedOps && (hasPhysDefs || TID.getNumDefs() > 1))) { handleThroughOperands(MI, VirtDead); // Don't attempt coalescing when we have funny stuff going on. CopyDst = 0; // Pretend we have early clobbers so the use operands get marked below. // This is not necessary for the common case of a single tied use. hasEarlyClobbers = true; } // Second scan. // Allocate virtreg uses. for (unsigned i = 0; i != VirtOpEnd; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; if (MO.isUse()) { LiveRegMap::iterator LRI = reloadVirtReg(MI, i, Reg, CopyDst); unsigned PhysReg = LRI->second.PhysReg; CopySrc = (CopySrc == Reg || CopySrc == PhysReg) ? PhysReg : 0; if (setPhysReg(MI, i, PhysReg)) killVirtReg(LRI); } } MRI->addPhysRegsUsed(UsedInInstr); // Track registers defined by instruction - early clobbers and tied uses at // this point. UsedInInstr.reset(); if (hasEarlyClobbers) { for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue; // Look for physreg defs and tied uses. if (!MO.isDef() && !MI->isRegTiedToDefOperand(i)) continue; UsedInInstr.set(Reg); for (const unsigned *AS = TRI->getAliasSet(Reg); *AS; ++AS) UsedInInstr.set(*AS); } } unsigned DefOpEnd = MI->getNumOperands(); if (TID.isCall()) { // Spill all virtregs before a call. This serves two purposes: 1. If an // exception is thrown, the landing pad is going to expect to find // registers in their spill slots, and 2. we don't have to wade through // all the <imp-def> operands on the call instruction. DefOpEnd = VirtOpEnd; DEBUG(dbgs() << " Spilling remaining registers before call.\n"); spillAll(MI); // The imp-defs are skipped below, but we still need to mark those // registers as used by the function. SkippedInstrs.insert(&TID); } // Third scan. // Allocate defs and collect dead defs. for (unsigned i = 0; i != DefOpEnd; ++i) { MachineOperand &MO = MI->getOperand(i); if (!MO.isReg() || !MO.isDef() || !MO.getReg() || MO.isEarlyClobber()) continue; unsigned Reg = MO.getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { if (!Allocatable.test(Reg)) continue; definePhysReg(MI, Reg, (MO.isImplicit() || MO.isDead()) ? regFree : regReserved); continue; } LiveRegMap::iterator LRI = defineVirtReg(MI, i, Reg, CopySrc); unsigned PhysReg = LRI->second.PhysReg; if (setPhysReg(MI, i, PhysReg)) { VirtDead.push_back(Reg); CopyDst = 0; // cancel coalescing; } else CopyDst = (CopyDst == Reg || CopyDst == PhysReg) ? PhysReg : 0; } // Kill dead defs after the scan to ensure that multiple defs of the same // register are allocated identically. We didn't need to do this for uses // because we are crerating our own kill flags, and they are always at the // last use. for (unsigned i = 0, e = VirtDead.size(); i != e; ++i) killVirtReg(VirtDead[i]); VirtDead.clear(); MRI->addPhysRegsUsed(UsedInInstr); if (CopyDst && CopyDst == CopySrc && CopyDstSub == CopySrcSub) { DEBUG(dbgs() << "-- coalescing: " << *MI); Coalesced.push_back(MI); } else { DEBUG(dbgs() << "<< " << *MI); } }
// Replace uses of FromReg with ToReg if they are dominated by MI. static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI, unsigned FromReg, unsigned ToReg, const MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS) { bool Changed = false; LiveInterval *FromLI = &LIS.getInterval(FromReg); LiveInterval *ToLI = &LIS.getInterval(ToReg); SlotIndex FromIdx = LIS.getInstructionIndex(MI).getRegSlot(); VNInfo *FromVNI = FromLI->getVNInfoAt(FromIdx); SmallVector<SlotIndex, 4> Indices; for (auto I = MRI.use_nodbg_begin(FromReg), E = MRI.use_nodbg_end(); I != E;) { MachineOperand &O = *I++; MachineInstr *Where = O.getParent(); // Check that MI dominates the instruction in the normal way. if (&MI == Where || !MDT.dominates(&MI, Where)) continue; // If this use gets a different value, skip it. SlotIndex WhereIdx = LIS.getInstructionIndex(*Where); VNInfo *WhereVNI = FromLI->getVNInfoAt(WhereIdx); if (WhereVNI && WhereVNI != FromVNI) continue; // Make sure ToReg isn't clobbered before it gets there. VNInfo *ToVNI = ToLI->getVNInfoAt(WhereIdx); if (ToVNI && ToVNI != FromVNI) continue; Changed = true; LLVM_DEBUG(dbgs() << "Setting operand " << O << " in " << *Where << " from " << MI << "\n"); O.setReg(ToReg); // If the store's def was previously dead, it is no longer. if (!O.isUndef()) { MI.getOperand(0).setIsDead(false); Indices.push_back(WhereIdx.getRegSlot()); } } if (Changed) { // Extend ToReg's liveness. LIS.extendToIndices(*ToLI, Indices); // Shrink FromReg's liveness. LIS.shrinkToUses(FromLI); // If we replaced all dominated uses, FromReg is now killed at MI. if (!FromLI->liveAt(FromIdx.getDeadSlot())) MI.addRegisterKilled(FromReg, MBB.getParent() ->getSubtarget<WebAssemblySubtarget>() .getRegisterInfo()); } return Changed; }