bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB, SlotIndex Kill, unsigned PhysReg) { unsigned KillMBBNum = KillMBB.getNumber(); // Block numbers where LR should be live-in. SmallVector<unsigned, 16> WorkList(1, KillMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = 0; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); errs() << "The register needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = LiveOut[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LR.extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; continue; } // No, we need a live-in value for Pred as well if (Pred != &KillMBB) WorkList.push_back(Pred->getNumber()); else // Loopback to KillMBB, so value is really live through. Kill = SlotIndex(); } } LiveIn.clear(); // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but // neither require it. Skip the sorting overhead for small updates. if (WorkList.size() > 4) array_pod_sort(WorkList.begin(), WorkList.end()); // If a unique reaching def was found, blit in the live ranges immediately. if (UniqueVNI) { LiveRangeUpdater Updater(&LR); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(*I); // Trim the live range in KillMBB. if (*I == KillMBBNum && Kill.isValid()) End = Kill; else LiveOut[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, (MachineDomTreeNode *)0); Updater.add(Start, End, TheVNI); } return true; } // Multiple values were found, so transfer the work list to the LiveIn array // where UpdateSSA will use it as a work list. LiveIn.reserve(WorkList.size()); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { MachineBasicBlock *MBB = MF->getBlockNumbered(*I); addLiveInBlock(LR, DomTree->getNode(MBB)); if (MBB == &KillMBB) LiveIn.back().Kill = Kill; } return false; }
MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops, int FI, LiveIntervals *LIS) const { auto Flags = MachineMemOperand::MONone; for (unsigned OpIdx : Ops) Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; MachineBasicBlock *MBB = MI.getParent(); assert(MBB && "foldMemoryOperand needs an inserted instruction"); MachineFunction &MF = *MBB->getParent(); // If we're not folding a load into a subreg, the size of the load is the // size of the spill slot. But if we are, we need to figure out what the // actual load size is. int64_t MemSize = 0; const MachineFrameInfo &MFI = MF.getFrameInfo(); const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); if (Flags & MachineMemOperand::MOStore) { MemSize = MFI.getObjectSize(FI); } else { for (unsigned OpIdx : Ops) { int64_t OpSize = MFI.getObjectSize(FI); if (auto SubReg = MI.getOperand(OpIdx).getSubReg()) { unsigned SubRegSize = TRI->getSubRegIdxSize(SubReg); if (SubRegSize > 0 && !(SubRegSize % 8)) OpSize = SubRegSize / 8; } MemSize = std::max(MemSize, OpSize); } } assert(MemSize && "Did not expect a zero-sized stack slot"); MachineInstr *NewMI = nullptr; if (MI.getOpcode() == TargetOpcode::STACKMAP || MI.getOpcode() == TargetOpcode::PATCHPOINT || MI.getOpcode() == TargetOpcode::STATEPOINT) { // Fold stackmap/patchpoint. NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); if (NewMI) MBB->insert(MI, NewMI); } else { // Ask the target to do the actual folding. NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS); } if (NewMI) { NewMI->setMemRefs(MF, MI.memoperands()); // Add a memory operand, foldMemoryOperandImpl doesn't do that. assert((!(Flags & MachineMemOperand::MOStore) || NewMI->mayStore()) && "Folded a def to a non-store!"); assert((!(Flags & MachineMemOperand::MOLoad) || NewMI->mayLoad()) && "Folded a use to a non-load!"); assert(MFI.getObjectOffset(FI) != -1); MachineMemOperand *MMO = MF.getMachineMemOperand( MachinePointerInfo::getFixedStack(MF, FI), Flags, MemSize, MFI.getObjectAlignment(FI)); NewMI->addMemOperand(MF, MMO); return NewMI; } // Straight COPY may fold as load/store. if (!MI.isCopy() || Ops.size() != 1) return nullptr; const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); if (!RC) return nullptr; const MachineOperand &MO = MI.getOperand(1 - Ops[0]); MachineBasicBlock::iterator Pos = MI; if (Flags == MachineMemOperand::MOStore) storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); else loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); return &*--Pos; }
bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { const unsigned Size = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8; MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC, ZERO, BNE, BEQ, MOVE; if (Size == 4) { if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); BNE = Mips::BNE; BEQ = Mips::BEQ; } ZERO = Mips::ZERO; MOVE = Mips::OR; } else { LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; ZERO = Mips::ZERO_64; BNE = Mips::BNE64; BEQ = Mips::BEQ64; MOVE = Mips::OR64; } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned OldVal = I->getOperand(2).getReg(); unsigned NewVal = I->getOperand(3).getReg(); unsigned Scratch = I->getOperand(4).getReg(); // insert new blocks after the current block const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loop1MBB); MF->insert(It, loop2MBB); MF->insert(It, exitMBB); // Transfer the remainder of BB and its successor edges to exitMBB. exitMBB->splice(exitMBB->begin(), &BB, std::next(MachineBasicBlock::iterator(I)), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); // thisMBB: // ... // fallthrough --> loop1MBB BB.addSuccessor(loop1MBB, BranchProbability::getOne()); loop1MBB->addSuccessor(exitMBB); loop1MBB->addSuccessor(loop2MBB); loop1MBB->normalizeSuccProbs(); loop2MBB->addSuccessor(loop1MBB); loop2MBB->addSuccessor(exitMBB); loop2MBB->normalizeSuccProbs(); // loop1MBB: // ll dest, 0(ptr) // bne dest, oldval, exitMBB BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0); BuildMI(loop1MBB, DL, TII->get(BNE)) .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB); // loop2MBB: // move scratch, NewVal // sc Scratch, Scratch, 0(ptr) // beq Scratch, $0, loop1MBB BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO); BuildMI(loop2MBB, DL, TII->get(SC), Scratch) .addReg(Scratch).addReg(Ptr).addImm(0); BuildMI(loop2MBB, DL, TII->get(BEQ)) .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB); LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loop1MBB); computeAndAddLiveIns(LiveRegs, *loop2MBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }
bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI, unsigned Size) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC, ZERO, BEQ; if (Size == 4) { if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); BEQ = Mips::BEQ; } ZERO = Mips::ZERO; } else { LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; ZERO = Mips::ZERO_64; BEQ = Mips::BEQ64; } unsigned OldVal = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Incr = I->getOperand(2).getReg(); unsigned Scratch = I->getOperand(3).getReg(); unsigned Opcode = 0; unsigned OR = 0; unsigned AND = 0; unsigned NOR = 0; bool IsNand = false; switch (I->getOpcode()) { case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: Opcode = Mips::ADDu; break; case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: Opcode = Mips::SUBu; break; case Mips::ATOMIC_LOAD_AND_I32_POSTRA: Opcode = Mips::AND; break; case Mips::ATOMIC_LOAD_OR_I32_POSTRA: Opcode = Mips::OR; break; case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: Opcode = Mips::XOR; break; case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: IsNand = true; AND = Mips::AND; NOR = Mips::NOR; break; case Mips::ATOMIC_SWAP_I32_POSTRA: OR = Mips::OR; break; case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: Opcode = Mips::DADDu; break; case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: Opcode = Mips::DSUBu; break; case Mips::ATOMIC_LOAD_AND_I64_POSTRA: Opcode = Mips::AND64; break; case Mips::ATOMIC_LOAD_OR_I64_POSTRA: Opcode = Mips::OR64; break; case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: Opcode = Mips::XOR64; break; case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: IsNand = true; AND = Mips::AND64; NOR = Mips::NOR64; break; case Mips::ATOMIC_SWAP_I64_POSTRA: OR = Mips::OR64; break; default: llvm_unreachable("Unknown pseudo atomic!"); } const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); MF->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); loopMBB->addSuccessor(exitMBB); loopMBB->addSuccessor(loopMBB); loopMBB->normalizeSuccProbs(); BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); assert((OldVal != Incr) && "Clobbered the wrong reg!"); if (Opcode) { BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr); } else if (IsNand) { assert(AND && NOR && "Unknown nand instruction for atomic pseudo expansion"); BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr); BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch); } else { assert(OR && "Unknown instruction for atomic pseudo expansion!"); BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); } BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0); BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB); NMBBI = BB.end(); I->eraseFromParent(); LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); return true; }
bool AArch64FrameLowering::restoreCalleeSavedRegisters( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); DebugLoc DL; SmallVector<RegPairInfo, 8> RegPairs; if (MI != MBB.end()) DL = MI->getDebugLoc(); computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs); for (auto RPII = RegPairs.begin(), RPIE = RegPairs.end(); RPII != RPIE; ++RPII) { RegPairInfo RPI = *RPII; unsigned Reg1 = RPI.Reg1; unsigned Reg2 = RPI.Reg2; // Issue sequence of non-sp increment and sp-pi restores for cs regs. Only // the last load is sp-pi post-increment and de-allocates the stack: // For example: // ldp fp, lr, [sp, #32] // addImm(+4) // ldp x20, x19, [sp, #16] // addImm(+2) // ldp x22, x21, [sp], #48 // addImm(+6) // Note: see comment in spillCalleeSavedRegisters() unsigned LdrOpc; bool BumpSP = RPII == std::prev(RegPairs.end()); if (RPI.IsGPR) { if (BumpSP) LdrOpc = RPI.isPaired() ? AArch64::LDPXpost : AArch64::LDRXpost; else LdrOpc = RPI.isPaired() ? AArch64::LDPXi : AArch64::LDRXui; } else { if (BumpSP) LdrOpc = RPI.isPaired() ? AArch64::LDPDpost : AArch64::LDRDpost; else LdrOpc = RPI.isPaired() ? AArch64::LDPDi : AArch64::LDRDui; } DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1); if (RPI.isPaired()) dbgs() << ", " << TRI->getName(Reg2); dbgs() << ") -> fi#(" << RPI.FrameIdx; if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx+1; dbgs() << ")\n"); const int Offset = RPI.Offset; MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc)); if (BumpSP) MIB.addReg(AArch64::SP, RegState::Define); if (RPI.isPaired()) MIB.addReg(Reg2, getDefRegState(true)) .addReg(Reg1, getDefRegState(true)) .addReg(AArch64::SP) .addImm(Offset) // [sp], #offset * 8 or [sp, #offset * 8] // where the factor * 8 is implicit .setMIFlag(MachineInstr::FrameDestroy); else MIB.addReg(Reg1, getDefRegState(true)) .addReg(AArch64::SP) .addImm(BumpSP ? Offset * 8 : Offset) // post-dec version is unscaled .setMIFlag(MachineInstr::FrameDestroy); } return true; }
/// finalizeBundle - Finalize a machine instruction bundle which includes /// a sequence of instructions starting from FirstMI to LastMI (exclusive). /// This routine adds a BUNDLE instruction to represent the bundle, it adds /// IsInternalRead markers to MachineOperands which are defined inside the /// bundle, and it copies externally visible defs and uses to the BUNDLE /// instruction. void llvm::finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI) { assert(FirstMI != LastMI && "Empty bundle?"); MIBundleBuilder Bundle(MBB, FirstMI, LastMI); const TargetMachine &TM = MBB.getParent()->getTarget(); const TargetInstrInfo *TII = TM.getInstrInfo(); const TargetRegisterInfo *TRI = TM.getRegisterInfo(); MachineInstrBuilder MIB = BuildMI(*MBB.getParent(), FirstMI->getDebugLoc(), TII->get(TargetOpcode::BUNDLE)); Bundle.prepend(MIB); SmallVector<unsigned, 32> LocalDefs; SmallSet<unsigned, 32> LocalDefSet; SmallSet<unsigned, 8> DeadDefSet; SmallSet<unsigned, 16> KilledDefSet; SmallVector<unsigned, 8> ExternUses; SmallSet<unsigned, 8> ExternUseSet; SmallSet<unsigned, 8> KilledUseSet; SmallSet<unsigned, 8> UndefUseSet; SmallVector<MachineOperand*, 4> Defs; for (; FirstMI != LastMI; ++FirstMI) { for (unsigned i = 0, e = FirstMI->getNumOperands(); i != e; ++i) { MachineOperand &MO = FirstMI->getOperand(i); if (!MO.isReg()) continue; if (MO.isDef()) { Defs.push_back(&MO); continue; } unsigned Reg = MO.getReg(); if (!Reg) continue; assert(TargetRegisterInfo::isPhysicalRegister(Reg)); if (LocalDefSet.count(Reg)) { MO.setIsInternalRead(); if (MO.isKill()) // Internal def is now killed. KilledDefSet.insert(Reg); } else { if (ExternUseSet.insert(Reg)) { ExternUses.push_back(Reg); if (MO.isUndef()) UndefUseSet.insert(Reg); } if (MO.isKill()) // External def is now killed. KilledUseSet.insert(Reg); } } for (unsigned i = 0, e = Defs.size(); i != e; ++i) { MachineOperand &MO = *Defs[i]; unsigned Reg = MO.getReg(); if (!Reg) continue; if (LocalDefSet.insert(Reg)) { LocalDefs.push_back(Reg); if (MO.isDead()) { DeadDefSet.insert(Reg); } } else { // Re-defined inside the bundle, it's no longer killed. KilledDefSet.erase(Reg); if (!MO.isDead()) // Previously defined but dead. DeadDefSet.erase(Reg); } if (!MO.isDead()) { for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) { unsigned SubReg = *SubRegs; if (LocalDefSet.insert(SubReg)) LocalDefs.push_back(SubReg); } } } Defs.clear(); } SmallSet<unsigned, 32> Added; for (unsigned i = 0, e = LocalDefs.size(); i != e; ++i) { unsigned Reg = LocalDefs[i]; if (Added.insert(Reg)) { // If it's not live beyond end of the bundle, mark it dead. bool isDead = DeadDefSet.count(Reg) || KilledDefSet.count(Reg); MIB.addReg(Reg, getDefRegState(true) | getDeadRegState(isDead) | getImplRegState(true)); } } for (unsigned i = 0, e = ExternUses.size(); i != e; ++i) { unsigned Reg = ExternUses[i]; bool isKill = KilledUseSet.count(Reg); bool isUndef = UndefUseSet.count(Reg); MIB.addReg(Reg, getKillRegState(isKill) | getUndefRegState(isUndef) | getImplRegState(true)); } }
void ARMFrameLowering::emitPushInst(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, unsigned StmOpc, unsigned StrOpc, bool NoGap, bool(*Func)(unsigned, bool), unsigned MIFlags) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo(); DebugLoc DL; if (MI != MBB.end()) DL = MI->getDebugLoc(); SmallVector<std::pair<unsigned,bool>, 4> Regs; unsigned i = CSI.size(); while (i != 0) { unsigned LastReg = 0; for (; i != 0; --i) { unsigned Reg = CSI[i-1].getReg(); if (!(Func)(Reg, STI.isTargetDarwin())) continue; // Add the callee-saved register as live-in unless it's LR and // @llvm.returnaddress is called. If LR is returned for // @llvm.returnaddress then it's already added to the function and // entry block live-in sets. bool isKill = true; if (Reg == ARM::LR) { if (MF.getFrameInfo()->isReturnAddressTaken() && MF.getRegInfo().isLiveIn(Reg)) isKill = false; } if (isKill) MBB.addLiveIn(Reg); // If NoGap is true, push consecutive registers and then leave the rest // for other instructions. e.g. // vpush {d8, d10, d11} -> vpush {d8}, vpush {d10, d11} if (NoGap && LastReg && LastReg != Reg-1) break; LastReg = Reg; Regs.push_back(std::make_pair(Reg, isKill)); } if (Regs.empty()) continue; if (Regs.size() > 1 || StrOpc== 0) { MachineInstrBuilder MIB = AddDefaultPred(BuildMI(MBB, MI, DL, TII.get(StmOpc), ARM::SP) .addReg(ARM::SP).setMIFlags(MIFlags)); for (unsigned i = 0, e = Regs.size(); i < e; ++i) MIB.addReg(Regs[i].first, getKillRegState(Regs[i].second)); } else if (Regs.size() == 1) { MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc), ARM::SP) .addReg(Regs[0].first, getKillRegState(Regs[0].second)) .addReg(ARM::SP).setMIFlags(MIFlags); // ARM mode needs an extra reg0 here due to addrmode2. Will go away once // that refactoring is complete (eventually). if (StrOpc == ARM::STR_PRE) { MIB.addReg(0); MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::sub, 4, ARM_AM::no_shift)); } else MIB.addImm(-4); AddDefaultPred(MIB); } Regs.clear(); } }
void MipsSEInstrInfo:: loadRegFromStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, unsigned DestReg, int FI, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, int64_t Offset) const { DebugLoc DL; if (I != MBB.end()) DL = I->getDebugLoc(); MachineMemOperand *MMO = GetMemOperand(MBB, FI, MachineMemOperand::MOLoad); unsigned Opc = 0; const Function *Func = MBB.getParent()->getFunction(); bool ReqIndirectLoad = Func->hasFnAttribute("interrupt") && (DestReg == Mips::LO0 || DestReg == Mips::LO0_64 || DestReg == Mips::HI0 || DestReg == Mips::HI0_64); if (Mips::GPR32RegClass.hasSubClassEq(RC)) Opc = Mips::LW; else if (Mips::GPR64RegClass.hasSubClassEq(RC)) Opc = Mips::LD; else if (Mips::ACC64RegClass.hasSubClassEq(RC)) Opc = Mips::LOAD_ACC64; else if (Mips::ACC64DSPRegClass.hasSubClassEq(RC)) Opc = Mips::LOAD_ACC64DSP; else if (Mips::ACC128RegClass.hasSubClassEq(RC)) Opc = Mips::LOAD_ACC128; else if (Mips::DSPCCRegClass.hasSubClassEq(RC)) Opc = Mips::LOAD_CCOND_DSP; else if (Mips::FGR32RegClass.hasSubClassEq(RC)) Opc = Mips::LWC1; else if (Mips::AFGR64RegClass.hasSubClassEq(RC)) Opc = Mips::LDC1; else if (Mips::FGR64RegClass.hasSubClassEq(RC)) Opc = Mips::LDC164; else if (TRI->isTypeLegalForClass(*RC, MVT::v16i8)) Opc = Mips::LD_B; else if (TRI->isTypeLegalForClass(*RC, MVT::v8i16) || TRI->isTypeLegalForClass(*RC, MVT::v8f16)) Opc = Mips::LD_H; else if (TRI->isTypeLegalForClass(*RC, MVT::v4i32) || TRI->isTypeLegalForClass(*RC, MVT::v4f32)) Opc = Mips::LD_W; else if (TRI->isTypeLegalForClass(*RC, MVT::v2i64) || TRI->isTypeLegalForClass(*RC, MVT::v2f64)) Opc = Mips::LD_D; else if (Mips::HI32RegClass.hasSubClassEq(RC)) Opc = Mips::LW; else if (Mips::HI64RegClass.hasSubClassEq(RC)) Opc = Mips::LD; else if (Mips::LO32RegClass.hasSubClassEq(RC)) Opc = Mips::LW; else if (Mips::LO64RegClass.hasSubClassEq(RC)) Opc = Mips::LD; assert(Opc && "Register class not handled!"); if (!ReqIndirectLoad) BuildMI(MBB, I, DL, get(Opc), DestReg) .addFrameIndex(FI) .addImm(Offset) .addMemOperand(MMO); else { // Load HI/LO through K0. Notably the DestReg is encoded into the // instruction itself. unsigned Reg = Mips::K0; unsigned LdOp = Mips::MTLO; if (DestReg == Mips::HI0) LdOp = Mips::MTHI; if (Subtarget.getABI().ArePtrs64bit()) { Reg = Mips::K0_64; if (DestReg == Mips::HI0_64) LdOp = Mips::MTHI64; else LdOp = Mips::MTLO64; } BuildMI(MBB, I, DL, get(Opc), Reg) .addFrameIndex(FI) .addImm(Offset) .addMemOperand(MMO); BuildMI(MBB, I, DL, get(LdOp)).addReg(Reg); } }
// Replace uses of FromReg with ToReg if they are dominated by MI. static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI, unsigned FromReg, unsigned ToReg, const MachineRegisterInfo &MRI, MachineDominatorTree &MDT, LiveIntervals &LIS) { bool Changed = false; LiveInterval *FromLI = &LIS.getInterval(FromReg); LiveInterval *ToLI = &LIS.getInterval(ToReg); SlotIndex FromIdx = LIS.getInstructionIndex(MI).getRegSlot(); VNInfo *FromVNI = FromLI->getVNInfoAt(FromIdx); SmallVector<SlotIndex, 4> Indices; for (auto I = MRI.use_nodbg_begin(FromReg), E = MRI.use_nodbg_end(); I != E;) { MachineOperand &O = *I++; MachineInstr *Where = O.getParent(); // Check that MI dominates the instruction in the normal way. if (&MI == Where || !MDT.dominates(&MI, Where)) continue; // If this use gets a different value, skip it. SlotIndex WhereIdx = LIS.getInstructionIndex(*Where); VNInfo *WhereVNI = FromLI->getVNInfoAt(WhereIdx); if (WhereVNI && WhereVNI != FromVNI) continue; // Make sure ToReg isn't clobbered before it gets there. VNInfo *ToVNI = ToLI->getVNInfoAt(WhereIdx); if (ToVNI && ToVNI != FromVNI) continue; Changed = true; LLVM_DEBUG(dbgs() << "Setting operand " << O << " in " << *Where << " from " << MI << "\n"); O.setReg(ToReg); // If the store's def was previously dead, it is no longer. if (!O.isUndef()) { MI.getOperand(0).setIsDead(false); Indices.push_back(WhereIdx.getRegSlot()); } } if (Changed) { // Extend ToReg's liveness. LIS.extendToIndices(*ToLI, Indices); // Shrink FromReg's liveness. LIS.shrinkToUses(FromLI); // If we replaced all dominated uses, FromReg is now killed at MI. if (!FromLI->liveAt(FromIdx.getDeadSlot())) MI.addRegisterKilled(FromReg, MBB.getParent() ->getSubtarget<WebAssemblySubtarget>() .getRegisterInfo()); } return Changed; }
void MIPrinter::print(const MachineBasicBlock &MBB) { assert(MBB.getNumber() >= 0 && "Invalid MBB number"); OS << "bb." << MBB.getNumber(); bool HasAttributes = false; if (const auto *BB = MBB.getBasicBlock()) { if (BB->hasName()) { OS << "." << BB->getName(); } else { HasAttributes = true; OS << " ("; int Slot = MST.getLocalSlot(BB); if (Slot == -1) OS << "<ir-block badref>"; else OS << (Twine("%ir-block.") + Twine(Slot)).str(); } } if (MBB.hasAddressTaken()) { OS << (HasAttributes ? ", " : " ("); OS << "address-taken"; HasAttributes = true; } if (MBB.isEHPad()) { OS << (HasAttributes ? ", " : " ("); OS << "landing-pad"; HasAttributes = true; } if (MBB.getAlignment()) { OS << (HasAttributes ? ", " : " ("); OS << "align " << MBB.getAlignment(); HasAttributes = true; } if (HasAttributes) OS << ")"; OS << ":\n"; bool HasLineAttributes = false; // Print the successors if (!MBB.succ_empty()) { OS.indent(2) << "successors: "; for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I) { if (I != MBB.succ_begin()) OS << ", "; printMBBReference(**I); if (MBB.hasSuccessorProbabilities()) OS << '(' << MBB.getSuccProbability(I) << ')'; } OS << "\n"; HasLineAttributes = true; } // Print the live in registers. const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo(); assert(TRI && "Expected target register info"); if (!MBB.livein_empty()) { OS.indent(2) << "liveins: "; bool First = true; for (const auto &LI : MBB.liveins()) { if (!First) OS << ", "; First = false; printReg(LI.PhysReg, OS, TRI); if (LI.LaneMask != ~0u) OS << ':' << PrintLaneMask(LI.LaneMask); } OS << "\n"; HasLineAttributes = true; } if (HasLineAttributes) OS << "\n"; bool IsInBundle = false; for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; ++I) { const MachineInstr &MI = *I; if (IsInBundle && !MI.isInsideBundle()) { OS.indent(2) << "}\n"; IsInBundle = false; } OS.indent(IsInBundle ? 4 : 2); print(MI); if (!IsInBundle && MI.getFlag(MachineInstr::BundledSucc)) { OS << " {"; IsInBundle = true; } OS << "\n"; } if (IsInBundle) OS.indent(2) << "}\n"; }
void MIPrinter::print(const MachineBasicBlock &MBB) { assert(MBB.getNumber() >= 0 && "Invalid MBB number"); OS << "bb." << MBB.getNumber(); bool HasAttributes = false; if (const auto *BB = MBB.getBasicBlock()) { if (BB->hasName()) { OS << "." << BB->getName(); } else { HasAttributes = true; OS << " ("; int Slot = MST.getLocalSlot(BB); if (Slot == -1) OS << "<ir-block badref>"; else OS << (Twine("%ir-block.") + Twine(Slot)).str(); } } if (MBB.hasAddressTaken()) { OS << (HasAttributes ? ", " : " ("); OS << "address-taken"; HasAttributes = true; } if (MBB.isEHPad()) { OS << (HasAttributes ? ", " : " ("); OS << "landing-pad"; HasAttributes = true; } if (MBB.getAlignment()) { OS << (HasAttributes ? ", " : " ("); OS << "align " << MBB.getAlignment(); HasAttributes = true; } if (HasAttributes) OS << ")"; OS << ":\n"; bool HasLineAttributes = false; // Print the successors bool canPredictProbs = canPredictBranchProbabilities(MBB); if (!MBB.succ_empty() && (!SimplifyMIR || !canPredictProbs || !canPredictSuccessors(MBB))) { OS.indent(2) << "successors: "; for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I) { if (I != MBB.succ_begin()) OS << ", "; printMBBReference(**I); if (!SimplifyMIR || !canPredictProbs) OS << '(' << format("0x%08" PRIx32, MBB.getSuccProbability(I).getNumerator()) << ')'; } OS << "\n"; HasLineAttributes = true; } // Print the live in registers. const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); if (MRI.tracksLiveness() && !MBB.livein_empty()) { const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo(); OS.indent(2) << "liveins: "; bool First = true; for (const auto &LI : MBB.liveins()) { if (!First) OS << ", "; First = false; printReg(LI.PhysReg, OS, &TRI); if (!LI.LaneMask.all()) OS << ":0x" << PrintLaneMask(LI.LaneMask); } OS << "\n"; HasLineAttributes = true; } if (HasLineAttributes) OS << "\n"; bool IsInBundle = false; for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; ++I) { const MachineInstr &MI = *I; if (IsInBundle && !MI.isInsideBundle()) { OS.indent(2) << "}\n"; IsInBundle = false; } OS.indent(IsInBundle ? 4 : 2); print(MI); if (!IsInBundle && MI.getFlag(MachineInstr::BundledSucc)) { OS << " {"; IsInBundle = true; } OS << "\n"; } if (IsInBundle) OS.indent(2) << "}\n"; }
/// If \p MBBI is a pseudo instruction, this method expands /// it to the corresponding (sequence of) actual instruction(s). /// \returns true if \p MBBI has been expanded. bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) { MachineInstr &MI = *MBBI; unsigned Opcode = MI.getOpcode(); DebugLoc DL = MBBI->getDebugLoc(); switch (Opcode) { default: return false; case X86::TCRETURNdi: case X86::TCRETURNdicc: case X86::TCRETURNri: case X86::TCRETURNmi: case X86::TCRETURNdi64: case X86::TCRETURNdi64cc: case X86::TCRETURNri64: case X86::TCRETURNmi64: { bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64; MachineOperand &JumpTarget = MBBI->getOperand(0); MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1); assert(StackAdjust.isImm() && "Expecting immediate value."); // Adjust stack pointer. int StackAdj = StackAdjust.getImm(); int MaxTCDelta = X86FI->getTCReturnAddrDelta(); int Offset = 0; assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive"); // Incoporate the retaddr area. Offset = StackAdj - MaxTCDelta; assert(Offset >= 0 && "Offset should never be negative"); if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) { assert(Offset == 0 && "Conditional tail call cannot adjust the stack."); } if (Offset) { // Check for possible merge with preceding ADD instruction. Offset += X86FL->mergeSPUpdates(MBB, MBBI, true); X86FL->emitSPUpdate(MBB, MBBI, Offset, /*InEpilogue=*/true); } // Jump to label or value in register. bool IsWin64 = STI->isTargetWin64(); if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) { unsigned Op; switch (Opcode) { case X86::TCRETURNdi: Op = X86::TAILJMPd; break; case X86::TCRETURNdicc: Op = X86::TAILJMPd_CC; break; case X86::TCRETURNdi64cc: assert(!IsWin64 && "Conditional tail calls confuse the Win64 unwinder."); // TODO: We could do it for Win64 "leaf" functions though; PR30337. Op = X86::TAILJMPd64_CC; break; default: // Note: Win64 uses REX prefixes indirect jumps out of functions, but // not direct ones. Op = X86::TAILJMPd64; break; } MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op)); if (JumpTarget.isGlobal()) { MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(), JumpTarget.getTargetFlags()); } else { assert(JumpTarget.isSymbol()); MIB.addExternalSymbol(JumpTarget.getSymbolName(), JumpTarget.getTargetFlags()); } if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) { MIB.addImm(MBBI->getOperand(2).getImm()); } } else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) { unsigned Op = (Opcode == X86::TCRETURNmi) ? X86::TAILJMPm : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64); MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op)); for (unsigned i = 0; i != 5; ++i) MIB.addOperand(MBBI->getOperand(i)); } else if (Opcode == X86::TCRETURNri64) { BuildMI(MBB, MBBI, DL, TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64)) .addReg(JumpTarget.getReg(), RegState::Kill); } else { BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr)) .addReg(JumpTarget.getReg(), RegState::Kill); } MachineInstr &NewMI = *std::prev(MBBI); NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI); // Delete the pseudo instruction TCRETURN. MBB.erase(MBBI); return true; } case X86::EH_RETURN: case X86::EH_RETURN64: { MachineOperand &DestAddr = MBBI->getOperand(0); assert(DestAddr.isReg() && "Offset should be in register!"); const bool Uses64BitFramePtr = STI->isTarget64BitLP64() || STI->isTargetNaCl64(); unsigned StackPtr = TRI->getStackRegister(); BuildMI(MBB, MBBI, DL, TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr) .addReg(DestAddr.getReg()); // The EH_RETURN pseudo is really removed during the MC Lowering. return true; } case X86::IRET: { // Adjust stack to erase error code int64_t StackAdj = MBBI->getOperand(0).getImm(); X86FL->emitSPUpdate(MBB, MBBI, StackAdj, true); // Replace pseudo with machine iret BuildMI(MBB, MBBI, DL, TII->get(STI->is64Bit() ? X86::IRET64 : X86::IRET32)); MBB.erase(MBBI); return true; } case X86::RET: { // Adjust stack to erase error code int64_t StackAdj = MBBI->getOperand(0).getImm(); MachineInstrBuilder MIB; if (StackAdj == 0) { MIB = BuildMI(MBB, MBBI, DL, TII->get(STI->is64Bit() ? X86::RETQ : X86::RETL)); } else if (isUInt<16>(StackAdj)) { MIB = BuildMI(MBB, MBBI, DL, TII->get(STI->is64Bit() ? X86::RETIQ : X86::RETIL)) .addImm(StackAdj); } else { assert(!STI->is64Bit() && "shouldn't need to do this for x86_64 targets!"); // A ret can only handle immediates as big as 2**16-1. If we need to pop // off bytes before the return address, we must do it manually. BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define); X86FL->emitSPUpdate(MBB, MBBI, StackAdj, /*InEpilogue=*/true); BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX); MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RETL)); } for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I) MIB.addOperand(MBBI->getOperand(I)); MBB.erase(MBBI); return true; } case X86::EH_RESTORE: { // Restore ESP and EBP, and optionally ESI if required. bool IsSEH = isAsynchronousEHPersonality(classifyEHPersonality( MBB.getParent()->getFunction()->getPersonalityFn())); X86FL->restoreWin32EHStackPointers(MBB, MBBI, DL, /*RestoreSP=*/IsSEH); MBBI->eraseFromParent(); return true; } case X86::LCMPXCHG8B_SAVE_EBX: case X86::LCMPXCHG16B_SAVE_RBX: { // Perform the following transformation. // SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx // => // [E|R]BX = InArg // actualcmpxchg Addr // [E|R]BX = SaveRbx const MachineOperand &InArg = MBBI->getOperand(6); unsigned SaveRbx = MBBI->getOperand(7).getReg(); unsigned ActualInArg = Opcode == X86::LCMPXCHG8B_SAVE_EBX ? X86::EBX : X86::RBX; // Copy the input argument of the pseudo into the argument of the // actual instruction. TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, InArg.getReg(), InArg.isKill()); // Create the actual instruction. unsigned ActualOpc = Opcode == X86::LCMPXCHG8B_SAVE_EBX ? X86::LCMPXCHG8B : X86::LCMPXCHG16B; MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(ActualOpc)); // Copy the operands related to the address. for (unsigned Idx = 1; Idx < 6; ++Idx) NewInstr->addOperand(MBBI->getOperand(Idx)); // Finally, restore the value of RBX. TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, SaveRbx, /*SrcIsKill*/ true); // Delete the pseudo. MBBI->eraseFromParent(); return true; } } llvm_unreachable("Previous switch has a fallthrough?"); }
/// runOnMachineBasicBlock - Fill in delay slots for the given basic block. /// We assume there is only one delay slot per delayed instruction. bool Filler::runOnMachineBasicBlock(MachineBasicBlock &MBB) { bool Changed = false; const MipsSubtarget &STI = MBB.getParent()->getSubtarget<MipsSubtarget>(); bool InMicroMipsMode = STI.inMicroMipsMode(); const MipsInstrInfo *TII = STI.getInstrInfo(); for (Iter I = MBB.begin(); I != MBB.end(); ++I) { if (!hasUnoccupiedSlot(&*I)) continue; ++FilledSlots; Changed = true; // Delay slot filling is disabled at -O0. if (!DisableDelaySlotFiller && (TM.getOptLevel() != CodeGenOpt::None)) { bool Filled = false; if (searchBackward(MBB, I)) { Filled = true; } else if (I->isTerminator()) { if (searchSuccBBs(MBB, I)) { Filled = true; } } else if (searchForward(MBB, I)) { Filled = true; } if (Filled) { // Get instruction with delay slot. MachineBasicBlock::instr_iterator DSI(I); if (InMicroMipsMode && TII->GetInstSizeInBytes(std::next(DSI)) == 2 && DSI->isCall()) { // If instruction in delay slot is 16b change opcode to // corresponding instruction with short delay slot. DSI->setDesc(TII->get(getEquivalentCallShort(DSI->getOpcode()))); } continue; } } // If instruction is BEQ or BNE with one ZERO register, then instead of // adding NOP replace this instruction with the corresponding compact // branch instruction, i.e. BEQZC or BNEZC. unsigned Opcode = I->getOpcode(); if (InMicroMipsMode && (Opcode == Mips::BEQ || Opcode == Mips::BNE) && ((unsigned) I->getOperand(1).getReg()) == Mips::ZERO) { I = replaceWithCompactBranch(MBB, I, I->getDebugLoc()); } else { // Bundle the NOP to the instruction with the delay slot. BuildMI(MBB, std::next(I), I->getDebugLoc(), TII->get(Mips::NOP)); MIBundleBuilder(MBB, I, std::next(I, 2)); } } return Changed; }
/// converToHardwareLoop - check if the loop is a candidate for /// converting to a hardware loop. If so, then perform the /// transformation. /// /// This function works on innermost loops first. A loop can /// be converted if it is a counting loop; either a register /// value or an immediate. /// /// The code makes several assumptions about the representation /// of the loop in llvm. bool HexagonHardwareLoops::convertToHardwareLoop(MachineLoop *L) { bool Changed = false; // Process nested loops first. for (MachineLoop::iterator I = L->begin(), E = L->end(); I != E; ++I) { Changed |= convertToHardwareLoop(*I); } // If a nested loop has been converted, then we can't convert this loop. if (Changed) { return Changed; } // Are we able to determine the trip count for the loop? CountValue *TripCount = getTripCount(L); if (TripCount == 0) { return false; } // Does the loop contain any invalid instructions? if (containsInvalidInstruction(L)) { return false; } MachineBasicBlock *Preheader = L->getLoopPreheader(); // No preheader means there's not place for the loop instr. if (Preheader == 0) { return false; } MachineBasicBlock::iterator InsertPos = Preheader->getFirstTerminator(); MachineBasicBlock *LastMBB = L->getExitingBlock(); // Don't generate hw loop if the loop has more than one exit. if (LastMBB == 0) { return false; } MachineBasicBlock::iterator LastI = LastMBB->getFirstTerminator(); // Determine the loop start. MachineBasicBlock *LoopStart = L->getTopBlock(); if (L->getLoopLatch() != LastMBB) { // When the exit and latch are not the same, use the latch block as the // start. // The loop start address is used only after the 1st iteration, and the loop // latch may contains instrs. that need to be executed after the 1st iter. LoopStart = L->getLoopLatch(); // Make sure the latch is a successor of the exit, otherwise it won't work. if (!LastMBB->isSuccessor(LoopStart)) { return false; } } // Convert the loop to a hardware loop DEBUG(dbgs() << "Change to hardware loop at "; L->dump()); if (TripCount->isReg()) { // Create a copy of the loop count register. MachineFunction *MF = LastMBB->getParent(); const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(TripCount->getReg()); unsigned CountReg = MF->getRegInfo().createVirtualRegister(RC); BuildMI(*Preheader, InsertPos, InsertPos->getDebugLoc(), TII->get(TargetOpcode::COPY), CountReg).addReg(TripCount->getReg()); if (TripCount->isNeg()) { unsigned CountReg1 = CountReg; CountReg = MF->getRegInfo().createVirtualRegister(RC); BuildMI(*Preheader, InsertPos, InsertPos->getDebugLoc(), TII->get(Hexagon::NEG), CountReg).addReg(CountReg1); } // Add the Loop instruction to the begining of the loop. BuildMI(*Preheader, InsertPos, InsertPos->getDebugLoc(), TII->get(Hexagon::LOOP0_r)).addMBB(LoopStart).addReg(CountReg); } else { assert(TripCount->isImm() && "Expecting immedate vaule for trip count"); // Add the Loop immediate instruction to the beginning of the loop. int64_t CountImm = TripCount->getImm(); BuildMI(*Preheader, InsertPos, InsertPos->getDebugLoc(), TII->get(Hexagon::LOOP0_i)).addMBB(LoopStart).addImm(CountImm); } // Make sure the loop start always has a reference in the CFG. We need to // create a BlockAddress operand to get this mechanism to work both the // MachineBasicBlock and BasicBlock objects need the flag set. LoopStart->setHasAddressTaken(); // This line is needed to set the hasAddressTaken flag on the BasicBlock // object BlockAddress::get(const_cast<BasicBlock *>(LoopStart->getBasicBlock())); // Replace the loop branch with an endloop instruction. DebugLoc dl = LastI->getDebugLoc(); BuildMI(*LastMBB, LastI, dl, TII->get(Hexagon::ENDLOOP0)).addMBB(LoopStart); // The loop ends with either: // - a conditional branch followed by an unconditional branch, or // - a conditional branch to the loop start. if (LastI->getOpcode() == Hexagon::JMP_c || LastI->getOpcode() == Hexagon::JMP_cNot) { // delete one and change/add an uncond. branch to out of the loop MachineBasicBlock *BranchTarget = LastI->getOperand(1).getMBB(); LastI = LastMBB->erase(LastI); if (!L->contains(BranchTarget)) { if (LastI != LastMBB->end()) { TII->RemoveBranch(*LastMBB); } SmallVector<MachineOperand, 0> Cond; TII->InsertBranch(*LastMBB, BranchTarget, 0, Cond, dl); } } else { // Conditional branch to loop start; just delete it. LastMBB->erase(LastI); } delete TripCount; ++NumHWLoops; return true; }
void ARMFrameLowering::emitPopInst(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, unsigned LdmOpc, unsigned LdrOpc, bool isVarArg, bool NoGap, bool(*Func)(unsigned, bool)) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); DebugLoc DL = MI->getDebugLoc(); unsigned RetOpcode = MI->getOpcode(); bool isTailCall = (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNdiND || RetOpcode == ARM::TCRETURNri || RetOpcode == ARM::TCRETURNriND); SmallVector<unsigned, 4> Regs; unsigned i = CSI.size(); while (i != 0) { unsigned LastReg = 0; bool DeleteRet = false; for (; i != 0; --i) { unsigned Reg = CSI[i-1].getReg(); if (!(Func)(Reg, STI.isTargetDarwin())) continue; if (Reg == ARM::LR && !isTailCall && !isVarArg && STI.hasV5TOps()) { Reg = ARM::PC; LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA_RET : ARM::LDMIA_RET; // Fold the return instruction into the LDM. DeleteRet = true; } // If NoGap is true, pop consecutive registers and then leave the rest // for other instructions. e.g. // vpop {d8, d10, d11} -> vpop {d8}, vpop {d10, d11} if (NoGap && LastReg && LastReg != Reg-1) break; LastReg = Reg; Regs.push_back(Reg); } if (Regs.empty()) continue; if (Regs.size() > 1 || LdrOpc == 0) { MachineInstrBuilder MIB = AddDefaultPred(BuildMI(MBB, MI, DL, TII.get(LdmOpc), ARM::SP) .addReg(ARM::SP)); for (unsigned i = 0, e = Regs.size(); i < e; ++i) MIB.addReg(Regs[i], getDefRegState(true)); if (DeleteRet) MI->eraseFromParent(); MI = MIB; } else if (Regs.size() == 1) { // If we adjusted the reg to PC from LR above, switch it back here. We // only do that for LDM. if (Regs[0] == ARM::PC) Regs[0] = ARM::LR; MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc), Regs[0]) .addReg(ARM::SP, RegState::Define) .addReg(ARM::SP); // ARM mode needs an extra reg0 here due to addrmode2. Will go away once // that refactoring is complete (eventually). if (LdrOpc == ARM::LDR_POST) { MIB.addReg(0); MIB.addImm(ARM_AM::getAM2Opc(ARM_AM::add, 4, ARM_AM::no_shift)); } else MIB.addImm(4); AddDefaultPred(MIB); } Regs.clear(); } }
void MipsSEInstrInfo:: storeRegToStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, unsigned SrcReg, bool isKill, int FI, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, int64_t Offset) const { DebugLoc DL; MachineMemOperand *MMO = GetMemOperand(MBB, FI, MachineMemOperand::MOStore); unsigned Opc = 0; if (Mips::GPR32RegClass.hasSubClassEq(RC)) Opc = Mips::SW; else if (Mips::GPR64RegClass.hasSubClassEq(RC)) Opc = Mips::SD; else if (Mips::ACC64RegClass.hasSubClassEq(RC)) Opc = Mips::STORE_ACC64; else if (Mips::ACC64DSPRegClass.hasSubClassEq(RC)) Opc = Mips::STORE_ACC64DSP; else if (Mips::ACC128RegClass.hasSubClassEq(RC)) Opc = Mips::STORE_ACC128; else if (Mips::DSPCCRegClass.hasSubClassEq(RC)) Opc = Mips::STORE_CCOND_DSP; else if (Mips::FGR32RegClass.hasSubClassEq(RC)) Opc = Mips::SWC1; else if (Mips::AFGR64RegClass.hasSubClassEq(RC)) Opc = Mips::SDC1; else if (Mips::FGR64RegClass.hasSubClassEq(RC)) Opc = Mips::SDC164; else if (TRI->isTypeLegalForClass(*RC, MVT::v16i8)) Opc = Mips::ST_B; else if (TRI->isTypeLegalForClass(*RC, MVT::v8i16) || TRI->isTypeLegalForClass(*RC, MVT::v8f16)) Opc = Mips::ST_H; else if (TRI->isTypeLegalForClass(*RC, MVT::v4i32) || TRI->isTypeLegalForClass(*RC, MVT::v4f32)) Opc = Mips::ST_W; else if (TRI->isTypeLegalForClass(*RC, MVT::v2i64) || TRI->isTypeLegalForClass(*RC, MVT::v2f64)) Opc = Mips::ST_D; else if (Mips::LO32RegClass.hasSubClassEq(RC)) Opc = Mips::SW; else if (Mips::LO64RegClass.hasSubClassEq(RC)) Opc = Mips::SD; else if (Mips::HI32RegClass.hasSubClassEq(RC)) Opc = Mips::SW; else if (Mips::HI64RegClass.hasSubClassEq(RC)) Opc = Mips::SD; // Hi, Lo are normally caller save but they are callee save // for interrupt handling. const Function *Func = MBB.getParent()->getFunction(); if (Func->hasFnAttribute("interrupt")) { if (Mips::HI32RegClass.hasSubClassEq(RC)) { BuildMI(MBB, I, DL, get(Mips::MFHI), Mips::K0); SrcReg = Mips::K0; } else if (Mips::HI64RegClass.hasSubClassEq(RC)) { BuildMI(MBB, I, DL, get(Mips::MFHI64), Mips::K0_64); SrcReg = Mips::K0_64; } else if (Mips::LO32RegClass.hasSubClassEq(RC)) { BuildMI(MBB, I, DL, get(Mips::MFLO), Mips::K0); SrcReg = Mips::K0; } else if (Mips::LO64RegClass.hasSubClassEq(RC)) { BuildMI(MBB, I, DL, get(Mips::MFLO64), Mips::K0_64); SrcReg = Mips::K0_64; } } assert(Opc && "Register class not handled!"); BuildMI(MBB, I, DL, get(Opc)).addReg(SrcReg, getKillRegState(isKill)) .addFrameIndex(FI).addImm(Offset).addMemOperand(MMO); }
/// LowerAtomicPHINode - Lower the PHI node at the top of the specified block, /// under the assuption that it needs to be lowered in a way that supports /// atomic execution of PHIs. This lowering method is always correct all of the /// time. /// void PHIElimination::LowerAtomicPHINode( MachineBasicBlock &MBB, MachineBasicBlock::iterator AfterPHIsIt) { ++NumAtomic; // Unlink the PHI node from the basic block, but don't delete the PHI yet. MachineInstr *MPhi = MBB.remove(MBB.begin()); unsigned NumSrcs = (MPhi->getNumOperands() - 1) / 2; unsigned DestReg = MPhi->getOperand(0).getReg(); assert(MPhi->getOperand(0).getSubReg() == 0 && "Can't handle sub-reg PHIs"); bool isDead = MPhi->getOperand(0).isDead(); // Create a new register for the incoming PHI arguments. MachineFunction &MF = *MBB.getParent(); unsigned IncomingReg = 0; bool reusedIncoming = false; // Is IncomingReg reused from an earlier PHI? // Insert a register to register copy at the top of the current block (but // after any remaining phi nodes) which copies the new incoming register // into the phi node destination. const TargetInstrInfo *TII = MF.getTarget().getInstrInfo(); if (isSourceDefinedByImplicitDef(MPhi, MRI)) // If all sources of a PHI node are implicit_def, just emit an // implicit_def instead of a copy. BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), DestReg); else { // Can we reuse an earlier PHI node? This only happens for critical edges, // typically those created by tail duplication. unsigned &entry = LoweredPHIs[MPhi]; if (entry) { // An identical PHI node was already lowered. Reuse the incoming register. IncomingReg = entry; reusedIncoming = true; ++NumReused; DEBUG(dbgs() << "Reusing %reg" << IncomingReg << " for " << *MPhi); } else { const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg); entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC); } BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), DestReg) .addReg(IncomingReg); } // Update live variable information if there is any. LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>(); if (LV) { MachineInstr *PHICopy = prior(AfterPHIsIt); if (IncomingReg) { LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg); // Increment use count of the newly created virtual register. VI.NumUses++; LV->setPHIJoin(IncomingReg); // When we are reusing the incoming register, it may already have been // killed in this block. The old kill will also have been inserted at // AfterPHIsIt, so it appears before the current PHICopy. if (reusedIncoming) if (MachineInstr *OldKill = VI.findKill(&MBB)) { DEBUG(dbgs() << "Remove old kill from " << *OldKill); LV->removeVirtualRegisterKilled(IncomingReg, OldKill); DEBUG(MBB.dump()); } // Add information to LiveVariables to know that the incoming value is // killed. Note that because the value is defined in several places (once // each for each incoming block), the "def" block and instruction fields // for the VarInfo is not filled in. LV->addVirtualRegisterKilled(IncomingReg, PHICopy); } // Since we are going to be deleting the PHI node, if it is the last use of // any registers, or if the value itself is dead, we need to move this // information over to the new copy we just inserted. LV->removeVirtualRegistersKilled(MPhi); // If the result is dead, update LV. if (isDead) { LV->addVirtualRegisterDead(DestReg, PHICopy); LV->removeVirtualRegisterDead(DestReg, MPhi); } } // Adjust the VRegPHIUseCount map to account for the removal of this PHI node. for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(), MPhi->getOperand(i).getReg())]; // Now loop over all of the incoming arguments, changing them to copy into the // IncomingReg register in the corresponding predecessor basic block. SmallPtrSet<MachineBasicBlock*, 8> MBBsInsertedInto; for (int i = NumSrcs - 1; i >= 0; --i) { unsigned SrcReg = MPhi->getOperand(i*2+1).getReg(); unsigned SrcSubReg = MPhi->getOperand(i*2+1).getSubReg(); assert(TargetRegisterInfo::isVirtualRegister(SrcReg) && "Machine PHI Operands must all be virtual registers!"); // Get the MachineBasicBlock equivalent of the BasicBlock that is the source // path the PHI. MachineBasicBlock &opBlock = *MPhi->getOperand(i*2+2).getMBB(); // If source is defined by an implicit def, there is no need to insert a // copy. MachineInstr *DefMI = MRI->getVRegDef(SrcReg); if (DefMI->isImplicitDef()) { ImpDefs.insert(DefMI); continue; } // Check to make sure we haven't already emitted the copy for this block. // This can happen because PHI nodes may have multiple entries for the same // basic block. if (!MBBsInsertedInto.insert(&opBlock)) continue; // If the copy has already been emitted, we're done. // Find a safe location to insert the copy, this may be the first terminator // in the block (or end()). MachineBasicBlock::iterator InsertPos = findPHICopyInsertPoint(&opBlock, &MBB, SrcReg); // Insert the copy. if (!reusedIncoming && IncomingReg) BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), IncomingReg).addReg(SrcReg, 0, SrcSubReg); // Now update live variable information if we have it. Otherwise we're done if (!LV) continue; // We want to be able to insert a kill of the register if this PHI (aka, the // copy we just inserted) is the last use of the source value. Live // variable analysis conservatively handles this by saying that the value is // live until the end of the block the PHI entry lives in. If the value // really is dead at the PHI copy, there will be no successor blocks which // have the value live-in. // Also check to see if this register is in use by another PHI node which // has not yet been eliminated. If so, it will be killed at an appropriate // point later. // Is it used by any PHI instructions in this block? bool ValueIsUsed = VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]; // Okay, if we now know that the value is not live out of the block, we can // add a kill marker in this block saying that it kills the incoming value! if (!ValueIsUsed && !LV->isLiveOut(SrcReg, opBlock)) { // In our final twist, we have to decide which instruction kills the // register. In most cases this is the copy, however, the first // terminator instruction at the end of the block may also use the value. // In this case, we should mark *it* as being the killing block, not the // copy. MachineBasicBlock::iterator KillInst; MachineBasicBlock::iterator Term = opBlock.getFirstTerminator(); if (Term != opBlock.end() && Term->readsRegister(SrcReg)) { KillInst = Term; // Check that no other terminators use values. #ifndef NDEBUG for (MachineBasicBlock::iterator TI = llvm::next(Term); TI != opBlock.end(); ++TI) { assert(!TI->readsRegister(SrcReg) && "Terminator instructions cannot use virtual registers unless" "they are the first terminator in a block!"); } #endif } else if (reusedIncoming || !IncomingReg) { // We may have to rewind a bit if we didn't insert a copy this time. KillInst = Term; while (KillInst != opBlock.begin()) if ((--KillInst)->readsRegister(SrcReg)) break; } else { // We just inserted this copy. KillInst = prior(InsertPos); } assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); // Finally, mark it killed. LV->addVirtualRegisterKilled(SrcReg, KillInst); // This vreg no longer lives all of the way through opBlock. unsigned opBlockNum = opBlock.getNumber(); LV->getVarInfo(SrcReg).AliveBlocks.reset(opBlockNum); } } // Really delete the PHI instruction now, if it is not in the LoweredPHIs map. if (reusedIncoming || !IncomingReg) MF.DeleteMachineInstr(MPhi); }
bool AArch64FrameLowering::spillCalleeSavedRegisters( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); DebugLoc DL; SmallVector<RegPairInfo, 8> RegPairs; computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs); for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE; ++RPII) { RegPairInfo RPI = *RPII; unsigned Reg1 = RPI.Reg1; unsigned Reg2 = RPI.Reg2; unsigned StrOpc; // Issue sequence of non-sp increment and pi sp spills for cs regs. The // first spill is a pre-increment that allocates the stack. // For example: // stp x22, x21, [sp, #-48]! // addImm(-6) // stp x20, x19, [sp, #16] // addImm(+2) // stp fp, lr, [sp, #32] // addImm(+4) // Rationale: This sequence saves uop updates compared to a sequence of // pre-increment spills like stp xi,xj,[sp,#-16]! // Note: Similar rationale and sequence for restores in epilog. bool BumpSP = RPII == RegPairs.rbegin(); if (RPI.IsGPR) { // For first spill use pre-increment store. if (BumpSP) StrOpc = RPI.isPaired() ? AArch64::STPXpre : AArch64::STRXpre; else StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui; } else { // For first spill use pre-increment store. if (BumpSP) StrOpc = RPI.isPaired() ? AArch64::STPDpre : AArch64::STRDpre; else StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui; } DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1); if (RPI.isPaired()) dbgs() << ", " << TRI->getName(Reg2); dbgs() << ") -> fi#(" << RPI.FrameIdx; if (RPI.isPaired()) dbgs() << ", " << RPI.FrameIdx+1; dbgs() << ")\n"); const int Offset = BumpSP ? -RPI.Offset : RPI.Offset; MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc)); if (BumpSP) MIB.addReg(AArch64::SP, RegState::Define); if (RPI.isPaired()) { MBB.addLiveIn(Reg1); MBB.addLiveIn(Reg2); MIB.addReg(Reg2, getPrologueDeath(MF, Reg2)) .addReg(Reg1, getPrologueDeath(MF, Reg1)) .addReg(AArch64::SP) .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit .setMIFlag(MachineInstr::FrameSetup); } else { MBB.addLiveIn(Reg1); MIB.addReg(Reg1, getPrologueDeath(MF, Reg1)) .addReg(AArch64::SP) .addImm(BumpSP ? Offset * 8 : Offset) // pre-inc version is unscaled .setMIFlag(MachineInstr::FrameSetup); } } return true; }
/// Splits a MachineBasicBlock to branch before \p SplitBefore. The original /// branch is \p OrigBranch. The target of the new branch can either be the same /// as the target of the original branch or the fallthrough successor of the /// original block as determined by \p BranchToFallThrough. The branch /// conditions will be inverted according to \p InvertNewBranch and /// \p InvertOrigBranch. If an instruction that previously fed the branch is to /// be deleted, it is provided in \p MIToDelete and \p NewCond will be used as /// the branch condition. The branch probabilities will be set if the /// MachineBranchProbabilityInfo isn't null. static bool splitMBB(BlockSplitInfo &BSI) { assert(BSI.allInstrsInSameMBB() && "All instructions must be in the same block."); MachineBasicBlock *ThisMBB = BSI.OrigBranch->getParent(); MachineFunction *MF = ThisMBB->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); assert(MRI->isSSA() && "Can only do this while the function is in SSA form."); if (ThisMBB->succ_size() != 2) { LLVM_DEBUG( dbgs() << "Don't know how to handle blocks that don't have exactly" << " two succesors.\n"); return false; } const PPCInstrInfo *TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo(); unsigned OrigBROpcode = BSI.OrigBranch->getOpcode(); unsigned InvertedOpcode = OrigBROpcode == PPC::BC ? PPC::BCn : OrigBROpcode == PPC::BCn ? PPC::BC : OrigBROpcode == PPC::BCLR ? PPC::BCLRn : PPC::BCLR; unsigned NewBROpcode = BSI.InvertNewBranch ? InvertedOpcode : OrigBROpcode; MachineBasicBlock *OrigTarget = BSI.OrigBranch->getOperand(1).getMBB(); MachineBasicBlock *OrigFallThrough = OrigTarget == *ThisMBB->succ_begin() ? *ThisMBB->succ_rbegin() : *ThisMBB->succ_begin(); MachineBasicBlock *NewBRTarget = BSI.BranchToFallThrough ? OrigFallThrough : OrigTarget; BranchProbability ProbToNewTarget = !BSI.MBPI ? BranchProbability::getUnknown() : BSI.MBPI->getEdgeProbability(ThisMBB, NewBRTarget); // Create a new basic block. MachineBasicBlock::iterator InsertPoint = BSI.SplitBefore; const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock(); MachineFunction::iterator It = ThisMBB->getIterator(); MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(LLVM_BB); MF->insert(++It, NewMBB); // Move everything after SplitBefore into the new block. NewMBB->splice(NewMBB->end(), ThisMBB, InsertPoint, ThisMBB->end()); NewMBB->transferSuccessors(ThisMBB); // Add the two successors to ThisMBB. The probabilities come from the // existing blocks if available. ThisMBB->addSuccessor(NewBRTarget, ProbToNewTarget); ThisMBB->addSuccessor(NewMBB, ProbToNewTarget.getCompl()); // Add the branches to ThisMBB. BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(), TII->get(NewBROpcode)) .addReg(BSI.SplitCond->getOperand(0).getReg()) .addMBB(NewBRTarget); BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(), TII->get(PPC::B)) .addMBB(NewMBB); if (BSI.MIToDelete) BSI.MIToDelete->eraseFromParent(); // Change the condition on the original branch and invert it if requested. auto FirstTerminator = NewMBB->getFirstTerminator(); if (BSI.NewCond) { assert(FirstTerminator->getOperand(0).isReg() && "Can't update condition of unconditional branch."); FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg()); } if (BSI.InvertOrigBranch) FirstTerminator->setDesc(TII->get(InvertedOpcode)); // If any of the PHIs in the successors of NewMBB reference values that // now come from NewMBB, they need to be updated. for (auto *Succ : NewMBB->successors()) { updatePHIs(Succ, ThisMBB, NewMBB, MRI); } addIncomingValuesToPHIs(NewBRTarget, ThisMBB, NewMBB, MRI); LLVM_DEBUG(dbgs() << "After splitting, ThisMBB:\n"; ThisMBB->dump()); LLVM_DEBUG(dbgs() << "NewMBB:\n"; NewMBB->dump()); LLVM_DEBUG(dbgs() << "New branch-to block:\n"; NewBRTarget->dump()); return true; }
/// foldMemoryOperand - Attempt to fold a load or store of the specified stack /// slot into the specified machine instruction for the specified operand(s). /// If this is possible, a new instruction is returned with the specified /// operand folded, otherwise NULL is returned. The client is responsible for /// removing the old instruction and adding the new one in the instruction /// stream. MachineInstr* TargetInstrInfo::foldMemoryOperand(MachineBasicBlock::iterator MI, const SmallVectorImpl<unsigned> &Ops, int FI) const { unsigned Flags = 0; for (unsigned i = 0, e = Ops.size(); i != e; ++i) if (MI->getOperand(Ops[i]).isDef()) Flags |= MachineMemOperand::MOStore; else Flags |= MachineMemOperand::MOLoad; MachineBasicBlock *MBB = MI->getParent(); assert(MBB && "foldMemoryOperand needs an inserted instruction"); MachineFunction &MF = *MBB->getParent(); MachineInstr *NewMI = nullptr; if (MI->getOpcode() == TargetOpcode::STACKMAP || MI->getOpcode() == TargetOpcode::PATCHPOINT) { // Fold stackmap/patchpoint. NewMI = foldPatchpoint(MF, MI, Ops, FI, *this); } else { // Ask the target to do the actual folding. NewMI =foldMemoryOperandImpl(MF, MI, Ops, FI); } if (NewMI) { NewMI->setMemRefs(MI->memoperands_begin(), MI->memoperands_end()); // Add a memory operand, foldMemoryOperandImpl doesn't do that. assert((!(Flags & MachineMemOperand::MOStore) || NewMI->mayStore()) && "Folded a def to a non-store!"); assert((!(Flags & MachineMemOperand::MOLoad) || NewMI->mayLoad()) && "Folded a use to a non-load!"); const MachineFrameInfo &MFI = *MF.getFrameInfo(); assert(MFI.getObjectOffset(FI) != -1); MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FI), Flags, MFI.getObjectSize(FI), MFI.getObjectAlignment(FI)); NewMI->addMemOperand(MF, MMO); // FIXME: change foldMemoryOperandImpl semantics to also insert NewMI. return MBB->insert(MI, NewMI); } // Straight COPY may fold as load/store. if (!MI->isCopy() || Ops.size() != 1) return nullptr; const TargetRegisterClass *RC = canFoldCopy(MI, Ops[0]); if (!RC) return nullptr; const MachineOperand &MO = MI->getOperand(1-Ops[0]); MachineBasicBlock::iterator Pos = MI; const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); if (Flags == MachineMemOperand::MOStore) storeRegToStackSlot(*MBB, Pos, MO.getReg(), MO.isKill(), FI, RC, TRI); else loadRegFromStackSlot(*MBB, Pos, MO.getReg(), FI, RC, TRI); return --Pos; }
/// LowerPHINode - Lower the PHI node at the top of the specified block, /// void PHIElimination::LowerPHINode(MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt) { ++NumLowered; MachineBasicBlock::iterator AfterPHIsIt = llvm::next(LastPHIIt); // Unlink the PHI node from the basic block, but don't delete the PHI yet. MachineInstr *MPhi = MBB.remove(MBB.begin()); unsigned NumSrcs = (MPhi->getNumOperands() - 1) / 2; unsigned DestReg = MPhi->getOperand(0).getReg(); assert(MPhi->getOperand(0).getSubReg() == 0 && "Can't handle sub-reg PHIs"); bool isDead = MPhi->getOperand(0).isDead(); // Create a new register for the incoming PHI arguments. MachineFunction &MF = *MBB.getParent(); unsigned IncomingReg = 0; bool reusedIncoming = false; // Is IncomingReg reused from an earlier PHI? // Insert a register to register copy at the top of the current block (but // after any remaining phi nodes) which copies the new incoming register // into the phi node destination. const TargetInstrInfo *TII = MF.getTarget().getInstrInfo(); if (isSourceDefinedByImplicitDef(MPhi, MRI)) // If all sources of a PHI node are implicit_def, just emit an // implicit_def instead of a copy. BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), DestReg); else { // Can we reuse an earlier PHI node? This only happens for critical edges, // typically those created by tail duplication. unsigned &entry = LoweredPHIs[MPhi]; if (entry) { // An identical PHI node was already lowered. Reuse the incoming register. IncomingReg = entry; reusedIncoming = true; ++NumReused; DEBUG(dbgs() << "Reusing " << PrintReg(IncomingReg) << " for " << *MPhi); } else { const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg); entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC); } BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), DestReg) .addReg(IncomingReg); } // Update live variable information if there is any. if (LV) { MachineInstr *PHICopy = prior(AfterPHIsIt); if (IncomingReg) { LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg); // Increment use count of the newly created virtual register. LV->setPHIJoin(IncomingReg); // When we are reusing the incoming register, it may already have been // killed in this block. The old kill will also have been inserted at // AfterPHIsIt, so it appears before the current PHICopy. if (reusedIncoming) if (MachineInstr *OldKill = VI.findKill(&MBB)) { DEBUG(dbgs() << "Remove old kill from " << *OldKill); LV->removeVirtualRegisterKilled(IncomingReg, OldKill); DEBUG(MBB.dump()); } // Add information to LiveVariables to know that the incoming value is // killed. Note that because the value is defined in several places (once // each for each incoming block), the "def" block and instruction fields // for the VarInfo is not filled in. LV->addVirtualRegisterKilled(IncomingReg, PHICopy); } // Since we are going to be deleting the PHI node, if it is the last use of // any registers, or if the value itself is dead, we need to move this // information over to the new copy we just inserted. LV->removeVirtualRegistersKilled(MPhi); // If the result is dead, update LV. if (isDead) { LV->addVirtualRegisterDead(DestReg, PHICopy); LV->removeVirtualRegisterDead(DestReg, MPhi); } } // Update LiveIntervals for the new copy or implicit def. if (LIS) { MachineInstr *NewInstr = prior(AfterPHIsIt); SlotIndex DestCopyIndex = LIS->InsertMachineInstrInMaps(NewInstr); SlotIndex MBBStartIndex = LIS->getMBBStartIdx(&MBB); if (IncomingReg) { // Add the region from the beginning of MBB to the copy instruction to // IncomingReg's live interval. LiveInterval &IncomingLI = LIS->createEmptyInterval(IncomingReg); VNInfo *IncomingVNI = IncomingLI.getVNInfoAt(MBBStartIndex); if (!IncomingVNI) IncomingVNI = IncomingLI.getNextValue(MBBStartIndex, LIS->getVNInfoAllocator()); IncomingLI.addSegment(LiveInterval::Segment(MBBStartIndex, DestCopyIndex.getRegSlot(), IncomingVNI)); } LiveInterval &DestLI = LIS->getInterval(DestReg); assert(DestLI.begin() != DestLI.end() && "PHIs should have nonempty LiveIntervals."); if (DestLI.endIndex().isDead()) { // A dead PHI's live range begins and ends at the start of the MBB, but // the lowered copy, which will still be dead, needs to begin and end at // the copy instruction. VNInfo *OrigDestVNI = DestLI.getVNInfoAt(MBBStartIndex); assert(OrigDestVNI && "PHI destination should be live at block entry."); DestLI.removeSegment(MBBStartIndex, MBBStartIndex.getDeadSlot()); DestLI.createDeadDef(DestCopyIndex.getRegSlot(), LIS->getVNInfoAllocator()); DestLI.removeValNo(OrigDestVNI); } else { // Otherwise, remove the region from the beginning of MBB to the copy // instruction from DestReg's live interval. DestLI.removeSegment(MBBStartIndex, DestCopyIndex.getRegSlot()); VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getRegSlot()); assert(DestVNI && "PHI destination should be live at its definition."); DestVNI->def = DestCopyIndex.getRegSlot(); } } // Adjust the VRegPHIUseCount map to account for the removal of this PHI node. for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) --VRegPHIUseCount[BBVRegPair(MPhi->getOperand(i+1).getMBB()->getNumber(), MPhi->getOperand(i).getReg())]; // Now loop over all of the incoming arguments, changing them to copy into the // IncomingReg register in the corresponding predecessor basic block. SmallPtrSet<MachineBasicBlock*, 8> MBBsInsertedInto; for (int i = NumSrcs - 1; i >= 0; --i) { unsigned SrcReg = MPhi->getOperand(i*2+1).getReg(); unsigned SrcSubReg = MPhi->getOperand(i*2+1).getSubReg(); bool SrcUndef = MPhi->getOperand(i*2+1).isUndef() || isImplicitlyDefined(SrcReg, MRI); assert(TargetRegisterInfo::isVirtualRegister(SrcReg) && "Machine PHI Operands must all be virtual registers!"); // Get the MachineBasicBlock equivalent of the BasicBlock that is the source // path the PHI. MachineBasicBlock &opBlock = *MPhi->getOperand(i*2+2).getMBB(); // Check to make sure we haven't already emitted the copy for this block. // This can happen because PHI nodes may have multiple entries for the same // basic block. if (!MBBsInsertedInto.insert(&opBlock)) continue; // If the copy has already been emitted, we're done. // Find a safe location to insert the copy, this may be the first terminator // in the block (or end()). MachineBasicBlock::iterator InsertPos = findPHICopyInsertPoint(&opBlock, &MBB, SrcReg); // Insert the copy. MachineInstr *NewSrcInstr = 0; if (!reusedIncoming && IncomingReg) { if (SrcUndef) { // The source register is undefined, so there is no need for a real // COPY, but we still need to ensure joint dominance by defs. // Insert an IMPLICIT_DEF instruction. NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), IncomingReg); // Clean up the old implicit-def, if there even was one. if (MachineInstr *DefMI = MRI->getVRegDef(SrcReg)) if (DefMI->isImplicitDef()) ImpDefs.insert(DefMI); } else { NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(), TII->get(TargetOpcode::COPY), IncomingReg) .addReg(SrcReg, 0, SrcSubReg); } } // We only need to update the LiveVariables kill of SrcReg if this was the // last PHI use of SrcReg to be lowered on this CFG edge and it is not live // out of the predecessor. We can also ignore undef sources. if (LV && !SrcUndef && !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)] && !LV->isLiveOut(SrcReg, opBlock)) { // We want to be able to insert a kill of the register if this PHI (aka, // the copy we just inserted) is the last use of the source value. Live // variable analysis conservatively handles this by saying that the value // is live until the end of the block the PHI entry lives in. If the value // really is dead at the PHI copy, there will be no successor blocks which // have the value live-in. // Okay, if we now know that the value is not live out of the block, we // can add a kill marker in this block saying that it kills the incoming // value! // In our final twist, we have to decide which instruction kills the // register. In most cases this is the copy, however, terminator // instructions at the end of the block may also use the value. In this // case, we should mark the last such terminator as being the killing // block, not the copy. MachineBasicBlock::iterator KillInst = opBlock.end(); MachineBasicBlock::iterator FirstTerm = opBlock.getFirstTerminator(); for (MachineBasicBlock::iterator Term = FirstTerm; Term != opBlock.end(); ++Term) { if (Term->readsRegister(SrcReg)) KillInst = Term; } if (KillInst == opBlock.end()) { // No terminator uses the register. if (reusedIncoming || !IncomingReg) { // We may have to rewind a bit if we didn't insert a copy this time. KillInst = FirstTerm; while (KillInst != opBlock.begin()) { --KillInst; if (KillInst->isDebugValue()) continue; if (KillInst->readsRegister(SrcReg)) break; } } else { // We just inserted this copy. KillInst = prior(InsertPos); } } assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); // Finally, mark it killed. LV->addVirtualRegisterKilled(SrcReg, KillInst); // This vreg no longer lives all of the way through opBlock. unsigned opBlockNum = opBlock.getNumber(); LV->getVarInfo(SrcReg).AliveBlocks.reset(opBlockNum); } if (LIS) { if (NewSrcInstr) { LIS->InsertMachineInstrInMaps(NewSrcInstr); LIS->addSegmentToEndOfBlock(IncomingReg, NewSrcInstr); } if (!SrcUndef && !VRegPHIUseCount[BBVRegPair(opBlock.getNumber(), SrcReg)]) { LiveInterval &SrcLI = LIS->getInterval(SrcReg); bool isLiveOut = false; for (MachineBasicBlock::succ_iterator SI = opBlock.succ_begin(), SE = opBlock.succ_end(); SI != SE; ++SI) { SlotIndex startIdx = LIS->getMBBStartIdx(*SI); VNInfo *VNI = SrcLI.getVNInfoAt(startIdx); // Definitions by other PHIs are not truly live-in for our purposes. if (VNI && VNI->def != startIdx) { isLiveOut = true; break; } } if (!isLiveOut) { MachineBasicBlock::iterator KillInst = opBlock.end(); MachineBasicBlock::iterator FirstTerm = opBlock.getFirstTerminator(); for (MachineBasicBlock::iterator Term = FirstTerm; Term != opBlock.end(); ++Term) { if (Term->readsRegister(SrcReg)) KillInst = Term; } if (KillInst == opBlock.end()) { // No terminator uses the register. if (reusedIncoming || !IncomingReg) { // We may have to rewind a bit if we didn't just insert a copy. KillInst = FirstTerm; while (KillInst != opBlock.begin()) { --KillInst; if (KillInst->isDebugValue()) continue; if (KillInst->readsRegister(SrcReg)) break; } } else { // We just inserted this copy. KillInst = prior(InsertPos); } } assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction"); SlotIndex LastUseIndex = LIS->getInstructionIndex(KillInst); SrcLI.removeSegment(LastUseIndex.getRegSlot(), LIS->getMBBEndIdx(&opBlock)); } } } } // Really delete the PHI instruction now, if it is not in the LoweredPHIs map. if (reusedIncoming || !IncomingReg) { if (LIS) LIS->RemoveMachineInstrFromMaps(MPhi); MF.DeleteMachineInstr(MPhi); } }
void AArch64FrameLowering::emitFrameMemOps(bool isPrologue, MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI, LoadStoreMethod PossClasses[], unsigned NumClasses) const { DebugLoc DL = MBB.findDebugLoc(MBBI); MachineFunction &MF = *MBB.getParent(); MachineFrameInfo &MFI = *MF.getFrameInfo(); const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo(); // A certain amount of implicit contract is present here. The actual stack // offsets haven't been allocated officially yet, so for strictly correct code // we rely on the fact that the elements of CSI are allocated in order // starting at SP, purely as dictated by size and alignment. In practice since // this function handles the only accesses to those slots it's not quite so // important. // // We have also ordered the Callee-saved register list in AArch64CallingConv // so that the above scheme puts registers in order: in particular we want // &X30 to be &X29+8 for an ABI-correct frame record (PCS 5.2.2) for (unsigned i = 0, e = CSI.size(); i < e; ++i) { unsigned Reg = CSI[i].getReg(); // First we need to find out which register class the register belongs to so // that we can use the correct load/store instrucitons. unsigned ClassIdx; for (ClassIdx = 0; ClassIdx < NumClasses; ++ClassIdx) { if (PossClasses[ClassIdx].RegClass->contains(Reg)) break; } assert(ClassIdx != NumClasses && "Asked to store register in unexpected class"); const TargetRegisterClass &TheClass = *PossClasses[ClassIdx].RegClass; // Now we need to decide whether it's possible to emit a paired instruction: // for this we want the next register to be in the same class. MachineInstrBuilder NewMI; bool Pair = false; if (i + 1 < CSI.size() && TheClass.contains(CSI[i+1].getReg())) { Pair = true; unsigned StLow = 0, StHigh = 0; if (isPrologue) { // Most of these registers will be live-in to the MBB and killed by our // store, though there are exceptions (see determinePrologueDeath). StLow = getKillRegState(determinePrologueDeath(MBB, CSI[i+1].getReg())); StHigh = getKillRegState(determinePrologueDeath(MBB, CSI[i].getReg())); } else { StLow = RegState::Define; StHigh = RegState::Define; } NewMI = BuildMI(MBB, MBBI, DL, TII.get(PossClasses[ClassIdx].PairOpcode)) .addReg(CSI[i+1].getReg(), StLow) .addReg(CSI[i].getReg(), StHigh); // If it's a paired op, we've consumed two registers ++i; } else { unsigned State; if (isPrologue) { State = getKillRegState(determinePrologueDeath(MBB, CSI[i].getReg())); } else { State = RegState::Define; } NewMI = BuildMI(MBB, MBBI, DL, TII.get(PossClasses[ClassIdx].SingleOpcode)) .addReg(CSI[i].getReg(), State); } // Note that the FrameIdx refers to the second register in a pair: it will // be allocated the smaller numeric address and so is the one an LDP/STP // address must use. int FrameIdx = CSI[i].getFrameIdx(); MachineMemOperand::MemOperandFlags Flags; Flags = isPrologue ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; MachineMemOperand *MMO = MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(FrameIdx), Flags, Pair ? TheClass.getSize() * 2 : TheClass.getSize(), MFI.getObjectAlignment(FrameIdx)); NewMI.addFrameIndex(FrameIdx) .addImm(0) // address-register offset .addMemOperand(MMO); if (isPrologue) NewMI.setMIFlags(MachineInstr::FrameSetup); // For aesthetic reasons, during an epilogue we want to emit complementary // operations to the prologue, but in the opposite order. So we still // iterate through the CalleeSavedInfo list in order, but we put the // instructions successively earlier in the MBB. if (!isPrologue) --MBBI; } }
bool Thumb1FrameLowering::emitPopSpecialFixUp(MachineBasicBlock &MBB, bool DoIt) const { MachineFunction &MF = *MBB.getParent(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize(); const TargetInstrInfo &TII = *STI.getInstrInfo(); const ThumbRegisterInfo *RegInfo = static_cast<const ThumbRegisterInfo *>(STI.getRegisterInfo()); // If MBBI is a return instruction, or is a tPOP followed by a return // instruction in the successor BB, we may be able to directly restore // LR in the PC. // This is only possible with v5T ops (v4T can't change the Thumb bit via // a POP PC instruction), and only if we do not need to emit any SP update. // Otherwise, we need a temporary register to pop the value // and copy that value into LR. auto MBBI = MBB.getFirstTerminator(); bool CanRestoreDirectly = STI.hasV5TOps() && !ArgRegsSaveSize; if (CanRestoreDirectly) { if (MBBI != MBB.end()) CanRestoreDirectly = (MBBI->getOpcode() == ARM::tBX_RET || MBBI->getOpcode() == ARM::tPOP_RET); else { assert(MBB.back().getOpcode() == ARM::tPOP); assert(MBB.succ_size() == 1); if ((*MBB.succ_begin())->begin()->getOpcode() == ARM::tBX_RET) MBBI--; // Replace the final tPOP with a tPOP_RET. else CanRestoreDirectly = false; } } if (CanRestoreDirectly) { if (!DoIt || MBBI->getOpcode() == ARM::tPOP_RET) return true; MachineInstrBuilder MIB = AddDefaultPred( BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII.get(ARM::tPOP_RET))); // Copy implicit ops and popped registers, if any. for (auto MO: MBBI->operands()) if (MO.isReg() && (MO.isImplicit() || MO.isDef()) && MO.getReg() != ARM::LR) MIB.addOperand(MO); MIB.addReg(ARM::PC, RegState::Define); // Erase the old instruction (tBX_RET or tPOP). MBB.erase(MBBI); return true; } // Look for a temporary register to use. // First, compute the liveness information. LivePhysRegs UsedRegs(STI.getRegisterInfo()); UsedRegs.addLiveOuts(&MBB, /*AddPristines*/ true); // The semantic of pristines changed recently and now, // the callee-saved registers that are touched in the function // are not part of the pristines set anymore. // Add those callee-saved now. const TargetRegisterInfo *TRI = STI.getRegisterInfo(); const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF); for (unsigned i = 0; CSRegs[i]; ++i) UsedRegs.addReg(CSRegs[i]); DebugLoc dl = DebugLoc(); if (MBBI != MBB.end()) { dl = MBBI->getDebugLoc(); auto InstUpToMBBI = MBB.end(); while (InstUpToMBBI != MBBI) // The pre-decrement is on purpose here. // We want to have the liveness right before MBBI. UsedRegs.stepBackward(*--InstUpToMBBI); } // Look for a register that can be directly use in the POP. unsigned PopReg = 0; // And some temporary register, just in case. unsigned TemporaryReg = 0; BitVector PopFriendly = TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::tGPRRegClassID)); assert(PopFriendly.any() && "No allocatable pop-friendly register?!"); // Rebuild the GPRs from the high registers because they are removed // form the GPR reg class for thumb1. BitVector GPRsNoLRSP = TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::hGPRRegClassID)); GPRsNoLRSP |= PopFriendly; GPRsNoLRSP.reset(ARM::LR); GPRsNoLRSP.reset(ARM::SP); GPRsNoLRSP.reset(ARM::PC); for (int Register = GPRsNoLRSP.find_first(); Register != -1; Register = GPRsNoLRSP.find_next(Register)) { if (!UsedRegs.contains(Register)) { // Remember the first pop-friendly register and exit. if (PopFriendly.test(Register)) { PopReg = Register; TemporaryReg = 0; break; } // Otherwise, remember that the register will be available to // save a pop-friendly register. TemporaryReg = Register; } } if (!DoIt && !PopReg && !TemporaryReg) return false; assert((PopReg || TemporaryReg) && "Cannot get LR"); if (TemporaryReg) { assert(!PopReg && "Unnecessary MOV is about to be inserted"); PopReg = PopFriendly.find_first(); AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr)) .addReg(TemporaryReg, RegState::Define) .addReg(PopReg, RegState::Kill)); } if (MBBI == MBB.end()) { MachineInstr& Pop = MBB.back(); assert(Pop.getOpcode() == ARM::tPOP); Pop.RemoveOperand(Pop.findRegisterDefOperandIdx(ARM::LR)); } assert(PopReg && "Do not know how to get LR"); AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP))) .addReg(PopReg, RegState::Define); emitSPUpdate(MBB, MBBI, TII, dl, *RegInfo, ArgRegsSaveSize); if (!TemporaryReg && MBBI != MBB.end() && MBBI->getOpcode() == ARM::tBX_RET) { MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(ARM::tBX)) .addReg(PopReg, RegState::Kill); AddDefaultPred(MIB); MIB.copyImplicitOps(&*MBBI); // erase the old tBX_RET instruction MBB.erase(MBBI); return true; } AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr)) .addReg(ARM::LR, RegState::Define) .addReg(PopReg, RegState::Kill)); if (TemporaryReg) { AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr)) .addReg(PopReg, RegState::Define) .addReg(TemporaryReg, RegState::Kill)); } return true; }
bool MipsExpandPseudo::expandAtomicBinOpSubword( MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC; unsigned BEQ = Mips::BEQ; unsigned SEOp = Mips::SEH; if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); } bool IsSwap = false; bool IsNand = false; unsigned Opcode = 0; switch (I->getOpcode()) { case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: IsNand = true; break; case Mips::ATOMIC_SWAP_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_SWAP_I16_POSTRA: IsSwap = true; break; case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: Opcode = Mips::ADDu; break; case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: Opcode = Mips::SUBu; break; case Mips::ATOMIC_LOAD_AND_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_AND_I16_POSTRA: Opcode = Mips::AND; break; case Mips::ATOMIC_LOAD_OR_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_OR_I16_POSTRA: Opcode = Mips::OR; break; case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: Opcode = Mips::XOR; break; default: llvm_unreachable("Unknown subword atomic pseudo for expansion!"); } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Incr = I->getOperand(2).getReg(); unsigned Mask = I->getOperand(3).getReg(); unsigned Mask2 = I->getOperand(4).getReg(); unsigned ShiftAmnt = I->getOperand(5).getReg(); unsigned OldVal = I->getOperand(6).getReg(); unsigned BinOpRes = I->getOperand(7).getReg(); unsigned StoreVal = I->getOperand(8).getReg(); const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); MF->insert(It, sinkMBB); MF->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); loopMBB->addSuccessor(sinkMBB); loopMBB->addSuccessor(loopMBB); loopMBB->normalizeSuccProbs(); BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); if (IsNand) { // and andres, oldval, incr2 // nor binopres, $0, andres // and newval, binopres, mask BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(OldVal) .addReg(Incr); BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes) .addReg(Mips::ZERO) .addReg(BinOpRes); BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(BinOpRes) .addReg(Mask); } else if (!IsSwap) { // <binop> binopres, oldval, incr2 // and newval, binopres, mask BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes) .addReg(OldVal) .addReg(Incr); BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(BinOpRes) .addReg(Mask); } else { // atomic.swap // and newval, incr2, mask BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(Incr) .addReg(Mask); } // and StoreVal, OlddVal, Mask2 // or StoreVal, StoreVal, BinOpRes // StoreVal<tied1> = sc StoreVal, 0(Ptr) // beq StoreVal, zero, loopMBB BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) .addReg(OldVal).addReg(Mask2); BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) .addReg(StoreVal).addReg(BinOpRes); BuildMI(loopMBB, DL, TII->get(SC), StoreVal) .addReg(StoreVal).addReg(Ptr).addImm(0); BuildMI(loopMBB, DL, TII->get(BEQ)) .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); // sinkMBB: // and maskedoldval1,oldval,mask // srl srlres,maskedoldval1,shiftamt // sign_extend dest,srlres sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest) .addReg(OldVal).addReg(Mask); BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) .addReg(Dest).addReg(ShiftAmnt); if (STI->hasMips32r2()) { BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); } else { const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24; BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); } LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); computeAndAddLiveIns(LiveRegs, *sinkMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }
bool AArch64FrameLowering::spillCalleeSavedRegisters( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); unsigned Count = CSI.size(); DebugLoc DL; assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!"); if (MI != MBB.end()) DL = MI->getDebugLoc(); for (unsigned i = 0; i < Count; i += 2) { unsigned idx = Count - i - 2; unsigned Reg1 = CSI[idx].getReg(); unsigned Reg2 = CSI[idx + 1].getReg(); // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI // list to come in sorted by frame index so that we can issue the store // pair instructions directly. Assert if we see anything otherwise. // // The order of the registers in the list is controlled by // getCalleeSavedRegs(), so they will always be in-order, as well. assert(CSI[idx].getFrameIdx() + 1 == CSI[idx + 1].getFrameIdx() && "Out of order callee saved regs!"); unsigned StrOpc; assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!"); assert((i & 1) == 0 && "Odd index for callee-saved reg spill!"); // Issue sequence of non-sp increment and pi sp spills for cs regs. The // first spill is a pre-increment that allocates the stack. // For example: // stp x22, x21, [sp, #-48]! // addImm(-6) // stp x20, x19, [sp, #16] // addImm(+2) // stp fp, lr, [sp, #32] // addImm(+4) // Rationale: This sequence saves uop updates compared to a sequence of // pre-increment spills like stp xi,xj,[sp,#-16]! // Note: Similar rational and sequence for restores in epilog. if (AArch64::GPR64RegClass.contains(Reg1)) { assert(AArch64::GPR64RegClass.contains(Reg2) && "Expected GPR64 callee-saved register pair!"); // For first spill use pre-increment store. if (i == 0) StrOpc = AArch64::STPXpre; else StrOpc = AArch64::STPXi; } else if (AArch64::FPR64RegClass.contains(Reg1)) { assert(AArch64::FPR64RegClass.contains(Reg2) && "Expected FPR64 callee-saved register pair!"); // For first spill use pre-increment store. if (i == 0) StrOpc = AArch64::STPDpre; else StrOpc = AArch64::STPDi; } else llvm_unreachable("Unexpected callee saved register!"); DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1) << ", " << TRI->getName(Reg2) << ") -> fi#(" << CSI[idx].getFrameIdx() << ", " << CSI[idx + 1].getFrameIdx() << ")\n"); // Compute offset: i = 0 => offset = -Count; // i = 2 => offset = -(Count - 2) + Count = 2 = i; etc. const int Offset = (i == 0) ? -Count : i; assert((Offset >= -64 && Offset <= 63) && "Offset out of bounds for STP immediate"); MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc)); if (StrOpc == AArch64::STPDpre || StrOpc == AArch64::STPXpre) MIB.addReg(AArch64::SP, RegState::Define); MIB.addReg(Reg2, getPrologueDeath(MF, Reg2)) .addReg(Reg1, getPrologueDeath(MF, Reg1)) .addReg(AArch64::SP) .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit .setMIFlag(MachineInstr::FrameSetup); } return true; }
bool MipsExpandPseudo::expandAtomicCmpSwapSubword( MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC; unsigned ZERO = Mips::ZERO; unsigned BNE = Mips::BNE; unsigned BEQ = Mips::BEQ; unsigned SEOp = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH; if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Mask = I->getOperand(2).getReg(); unsigned ShiftCmpVal = I->getOperand(3).getReg(); unsigned Mask2 = I->getOperand(4).getReg(); unsigned ShiftNewVal = I->getOperand(5).getReg(); unsigned ShiftAmnt = I->getOperand(6).getReg(); unsigned Scratch = I->getOperand(7).getReg(); unsigned Scratch2 = I->getOperand(8).getReg(); // insert new blocks after the current block const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loop1MBB); MF->insert(It, loop2MBB); MF->insert(It, sinkMBB); MF->insert(It, exitMBB); // Transfer the remainder of BB and its successor edges to exitMBB. exitMBB->splice(exitMBB->begin(), &BB, std::next(MachineBasicBlock::iterator(I)), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); // thisMBB: // ... // fallthrough --> loop1MBB BB.addSuccessor(loop1MBB, BranchProbability::getOne()); loop1MBB->addSuccessor(sinkMBB); loop1MBB->addSuccessor(loop2MBB); loop1MBB->normalizeSuccProbs(); loop2MBB->addSuccessor(loop1MBB); loop2MBB->addSuccessor(sinkMBB); loop2MBB->normalizeSuccProbs(); sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); // loop1MBB: // ll dest, 0(ptr) // and Mask', dest, Mask // bne Mask', ShiftCmpVal, exitMBB BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0); BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2) .addReg(Scratch) .addReg(Mask); BuildMI(loop1MBB, DL, TII->get(BNE)) .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB); // loop2MBB: // and dest, dest, mask2 // or dest, dest, ShiftNewVal // sc dest, dest, 0(ptr) // beq dest, $0, loop1MBB BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch) .addReg(Scratch, RegState::Kill) .addReg(Mask2); BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch) .addReg(Scratch, RegState::Kill) .addReg(ShiftNewVal); BuildMI(loop2MBB, DL, TII->get(SC), Scratch) .addReg(Scratch, RegState::Kill) .addReg(Ptr) .addImm(0); BuildMI(loop2MBB, DL, TII->get(BEQ)) .addReg(Scratch, RegState::Kill) .addReg(ZERO) .addMBB(loop1MBB); // sinkMBB: // srl srlres, Mask', shiftamt // sign_extend dest,srlres BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) .addReg(Scratch2) .addReg(ShiftAmnt); if (STI->hasMips32r2()) { BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); } else { const unsigned ShiftImm = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24; BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); } LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loop1MBB); computeAndAddLiveIns(LiveRegs, *loop2MBB); computeAndAddLiveIns(LiveRegs, *sinkMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }
bool AArch64FrameLowering::restoreCalleeSavedRegisters( MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const std::vector<CalleeSavedInfo> &CSI, const TargetRegisterInfo *TRI) const { MachineFunction &MF = *MBB.getParent(); const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); unsigned Count = CSI.size(); DebugLoc DL; assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!"); if (MI != MBB.end()) DL = MI->getDebugLoc(); for (unsigned i = 0; i < Count; i += 2) { unsigned Reg1 = CSI[i].getReg(); unsigned Reg2 = CSI[i + 1].getReg(); // GPRs and FPRs are saved in pairs of 64-bit regs. We expect the CSI // list to come in sorted by frame index so that we can issue the store // pair instructions directly. Assert if we see anything otherwise. assert(CSI[i].getFrameIdx() + 1 == CSI[i + 1].getFrameIdx() && "Out of order callee saved regs!"); // Issue sequence of non-sp increment and sp-pi restores for cs regs. Only // the last load is sp-pi post-increment and de-allocates the stack: // For example: // ldp fp, lr, [sp, #32] // addImm(+4) // ldp x20, x19, [sp, #16] // addImm(+2) // ldp x22, x21, [sp], #48 // addImm(+6) // Note: see comment in spillCalleeSavedRegisters() unsigned LdrOpc; assert((Count & 1) == 0 && "Odd number of callee-saved regs to spill!"); assert((i & 1) == 0 && "Odd index for callee-saved reg spill!"); if (AArch64::GPR64RegClass.contains(Reg1)) { assert(AArch64::GPR64RegClass.contains(Reg2) && "Expected GPR64 callee-saved register pair!"); if (i == Count - 2) LdrOpc = AArch64::LDPXpost; else LdrOpc = AArch64::LDPXi; } else if (AArch64::FPR64RegClass.contains(Reg1)) { assert(AArch64::FPR64RegClass.contains(Reg2) && "Expected FPR64 callee-saved register pair!"); if (i == Count - 2) LdrOpc = AArch64::LDPDpost; else LdrOpc = AArch64::LDPDi; } else llvm_unreachable("Unexpected callee saved register!"); DEBUG(dbgs() << "CSR restore: (" << TRI->getName(Reg1) << ", " << TRI->getName(Reg2) << ") -> fi#(" << CSI[i].getFrameIdx() << ", " << CSI[i + 1].getFrameIdx() << ")\n"); // Compute offset: i = 0 => offset = Count - 2; i = 2 => offset = Count - 4; // etc. const int Offset = (i == Count - 2) ? Count : Count - i - 2; assert((Offset >= -64 && Offset <= 63) && "Offset out of bounds for LDP immediate"); MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(LdrOpc)); if (LdrOpc == AArch64::LDPXpost || LdrOpc == AArch64::LDPDpost) MIB.addReg(AArch64::SP, RegState::Define); MIB.addReg(Reg2, getDefRegState(true)) .addReg(Reg1, getDefRegState(true)) .addReg(AArch64::SP) .addImm(Offset); // [sp], #offset * 8 or [sp, #offset * 8] // where the factor * 8 is implicit } return true; }
VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, MachineBasicBlock *KillMBB, SlotIndex Kill, unsigned PhysReg) { // Blocks where LI should be live-in. SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = 0; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = WorkList[i]; #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); errs() << "The register needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = LiveOut[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LI->extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; continue; } // No, we need a live-in value for Pred as well if (Pred != KillMBB) WorkList.push_back(Pred); else // Loopback to KillMBB, so value is really live through. Kill = SlotIndex(); } } // Transfer WorkList to LiveInBlocks in reverse order. // This ordering works best with updateSSA(). LiveIn.clear(); LiveIn.reserve(WorkList.size()); while(!WorkList.empty()) addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val())); // The kill block may not be live-through. assert(LiveIn.back().DomNode->getBlock() == KillMBB); LiveIn.back().Kill = Kill; return UniqueVNI ? TheVNI : 0; }
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, SlotIndex Use, unsigned PhysReg, ArrayRef<SlotIndex> Undefs) { unsigned UseMBBNum = UseMBB.getNumber(); // Block numbers where LR should be live-in. SmallVector<unsigned, 16> WorkList(1, UseMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = nullptr; bool FoundUndef = false; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (Undefs.size() > 0 && MBB->pred_empty()) { MBB->getParent()->verify(); errs() << "Use of " << PrintReg(PhysReg) << " does not have a corresponding definition on every path:\n"; const MachineInstr *MI = Indexes->getInstructionFromIndex(Use); if (MI != nullptr) errs() << Use << " " << *MI; llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo(); errs() << "The register " << PrintReg(PhysReg, TRI) << " needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif FoundUndef |= MBB->pred_empty(); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. auto EP = LR.extendInBlock(Undefs, Start, End); VNInfo *VNI = EP.first; FoundUndef |= EP.second; setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } if (VNI || EP.second) continue; // No, we need a live-in value for Pred as well if (Pred != &UseMBB) WorkList.push_back(Pred->getNumber()); else // Loopback to UseMBB, so value is really live through. Use = SlotIndex(); } } LiveIn.clear(); FoundUndef |= (TheVNI == nullptr); if (Undefs.size() > 0 && FoundUndef) UniqueVNI = false; // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but // neither require it. Skip the sorting overhead for small updates. if (WorkList.size() > 4) array_pod_sort(WorkList.begin(), WorkList.end()); // If a unique reaching def was found, blit in the live ranges immediately. if (UniqueVNI) { assert(TheVNI != nullptr); LiveRangeUpdater Updater(&LR); for (unsigned BN : WorkList) { SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(BN); // Trim the live range in UseMBB. if (BN == UseMBBNum && Use.isValid()) End = Use; else Map[MF->getBlockNumbered(BN)] = LiveOutPair(TheVNI, nullptr); Updater.add(Start, End, TheVNI); } return true; } // Prepare the defined/undefined bit vectors. auto EF = EntryInfoMap.find(&LR); if (EF == EntryInfoMap.end()) { unsigned N = MF->getNumBlockIDs(); EF = EntryInfoMap.insert({&LR, {BitVector(), BitVector()}}).first; EF->second.first.resize(N); EF->second.second.resize(N); } BitVector &DefOnEntry = EF->second.first; BitVector &UndefOnEntry = EF->second.second; // Multiple values were found, so transfer the work list to the LiveIn array // where UpdateSSA will use it as a work list. LiveIn.reserve(WorkList.size()); for (unsigned BN : WorkList) { MachineBasicBlock *MBB = MF->getBlockNumbered(BN); if (Undefs.size() > 0 && !isDefOnEntry(LR, Undefs, *MBB, DefOnEntry, UndefOnEntry)) continue; addLiveInBlock(LR, DomTree->getNode(MBB)); if (MBB == &UseMBB) LiveIn.back().Kill = Use; } return false; }