bool AArch64FrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction &MF = *MBB.getParent();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
DebugLoc DL;
SmallVector<RegPairInfo, 8> RegPairs;
computeCalleeSaveRegisterPairs(MF, CSI, TRI, RegPairs);
for (auto RPII = RegPairs.rbegin(), RPIE = RegPairs.rend(); RPII != RPIE;
++RPII) {
RegPairInfo RPI = *RPII;
unsigned Reg1 = RPI.Reg1;
unsigned Reg2 = RPI.Reg2;
unsigned StrOpc;
// Issue sequence of spills for cs regs. The first spill may be converted
// to a pre-decrement store later by emitPrologue if the callee-save stack
// area allocation can't be combined with the local stack area allocation.
// For example:
// stp x22, x21, [sp, #0] // addImm(+0)
// stp x20, x19, [sp, #16] // addImm(+2)
// stp fp, lr, [sp, #32] // addImm(+4)
// Rationale: This sequence saves uop updates compared to a sequence of
// pre-increment spills like stp xi,xj,[sp,#-16]!
// Note: Similar rationale and sequence for restores in epilog.
if (RPI.IsGPR)
StrOpc = RPI.isPaired() ? AArch64::STPXi : AArch64::STRXui;
else
StrOpc = RPI.isPaired() ? AArch64::STPDi : AArch64::STRDui;
DEBUG(dbgs() << "CSR spill: (" << TRI->getName(Reg1);
if (RPI.isPaired())
dbgs() << ", " << TRI->getName(Reg2);
dbgs() << ") -> fi#(" << RPI.FrameIdx;
if (RPI.isPaired())
dbgs() << ", " << RPI.FrameIdx+1;
dbgs() << ")\n");
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(StrOpc));
MBB.addLiveIn(Reg1);
if (RPI.isPaired()) {
MBB.addLiveIn(Reg2);
MIB.addReg(Reg2, getPrologueDeath(MF, Reg2));
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
MachineMemOperand::MOStore, 8, 8));
}
MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
.addReg(AArch64::SP)
.addImm(RPI.Offset) // [sp, #offset*8], where factor*8 is implicit
.setMIFlag(MachineInstr::FrameSetup);
MIB.addMemOperand(MF.getMachineMemOperand(
MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
MachineMemOperand::MOStore, 8, 8));
}
return true;
}
bool
HexagonFrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction *MF = MBB.getParent();
const TargetInstrInfo &TII = *MF->getTarget().getInstrInfo();
if (CSI.empty()) {
return false;
}
// We can only schedule double loads if we spill contiguous callee-saved regs
// For instance, we cannot scheduled double-word loads if we spill r24,
// r26, and r27.
// Hexagon_TODO: We can try to double-word align odd registers for -O2 and
// above.
bool ContiguousRegs = true;
for (unsigned i = 0; i < CSI.size(); ++i) {
unsigned Reg = CSI[i].getReg();
//
// Check if we can use a double-word store.
//
const unsigned* SuperReg = TRI->getSuperRegisters(Reg);
// Assume that there is exactly one superreg.
assert(SuperReg[0] && !SuperReg[1] && "Expected exactly one superreg");
bool CanUseDblStore = false;
const TargetRegisterClass* SuperRegClass = 0;
if (ContiguousRegs && (i < CSI.size()-1)) {
const unsigned* SuperRegNext = TRI->getSuperRegisters(CSI[i+1].getReg());
assert(SuperRegNext[0] && !SuperRegNext[1] &&
"Expected exactly one superreg");
SuperRegClass = TRI->getMinimalPhysRegClass(SuperReg[0]);
CanUseDblStore = (SuperRegNext[0] == SuperReg[0]);
}
if (CanUseDblStore) {
TII.storeRegToStackSlot(MBB, MI, SuperReg[0], true,
CSI[i+1].getFrameIdx(), SuperRegClass, TRI);
MBB.addLiveIn(SuperReg[0]);
++i;
} else {
// Cannot use a double-word store.
ContiguousRegs = false;
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i].getFrameIdx(), RC,
TRI);
MBB.addLiveIn(Reg);
}
}
return true;
}
/// UnfoldAndRewriteInstruction - Rewrite specified instruction by unfolding
/// folded memory references and replacing those references with register
/// references instead.
void
StackSlotColoring::UnfoldAndRewriteInstruction(MachineInstr *MI, int OldFI,
unsigned Reg,
const TargetRegisterClass *RC,
SmallSet<unsigned, 4> &Defs,
MachineFunction &MF) {
MachineBasicBlock *MBB = MI->getParent();
if (unsigned DstReg = TII->isLoadFromStackSlot(MI, OldFI)) {
if (PropagateForward(MI, MBB, DstReg, Reg)) {
DEBUG(dbgs() << "Eliminated load: ");
DEBUG(MI->dump());
++NumLoadElim;
} else {
BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(TargetOpcode::COPY),
DstReg).addReg(Reg);
++NumRegRepl;
}
if (!Defs.count(Reg)) {
// If this is the first use of Reg in this MBB and it wasn't previously
// defined in MBB, add it to livein.
MBB->addLiveIn(Reg);
Defs.insert(Reg);
}
} else if (unsigned SrcReg = TII->isStoreToStackSlot(MI, OldFI)) {
if (MI->killsRegister(SrcReg) && PropagateBackward(MI, MBB, SrcReg, Reg)) {
DEBUG(dbgs() << "Eliminated store: ");
DEBUG(MI->dump());
++NumStoreElim;
} else {
BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(TargetOpcode::COPY), Reg)
.addReg(SrcReg);
++NumRegRepl;
}
// Remember reg has been defined in MBB.
Defs.insert(Reg);
} else {
SmallVector<MachineInstr*, 4> NewMIs;
bool Success = TII->unfoldMemoryOperand(MF, MI, Reg, false, false, NewMIs);
Success = Success; // Silence compiler warning.
assert(Success && "Failed to unfold!");
MachineInstr *NewMI = NewMIs[0];
MBB->insert(MI, NewMI);
++NumRegRepl;
if (NewMI->readsRegister(Reg)) {
if (!Defs.count(Reg))
// If this is the first use of Reg in this MBB and it wasn't previously
// defined in MBB, add it to livein.
MBB->addLiveIn(Reg);
Defs.insert(Reg);
}
}
MBB->erase(MI);
}
bool HexagonFrameLowering::restoreCalleeSavedRegisters(
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction *MF = MBB.getParent();
const TargetInstrInfo &TII = *MF->getTarget().getInstrInfo();
if (CSI.empty()) {
return false;
}
// We can only schedule double loads if we spill contiguous callee-saved regs
// For instance, we cannot scheduled double-word loads if we spill r24,
// r26, and r27.
// Hexagon_TODO: We can try to double-word align odd registers for -O2 and
// above.
bool ContiguousRegs = true;
for (unsigned i = 0; i < CSI.size(); ++i) {
unsigned Reg = CSI[i].getReg();
//
// Check if we can use a double-word load.
//
unsigned SuperReg = uniqueSuperReg(Reg, TRI);
const TargetRegisterClass* SuperRegClass = 0;
bool CanUseDblLoad = false;
if (ContiguousRegs && (i < CSI.size()-1)) {
unsigned SuperRegNext = uniqueSuperReg(CSI[i+1].getReg(), TRI);
SuperRegClass = TRI->getMinimalPhysRegClass(SuperReg);
CanUseDblLoad = (SuperRegNext == SuperReg);
}
if (CanUseDblLoad) {
TII.loadRegFromStackSlot(MBB, MI, SuperReg, CSI[i+1].getFrameIdx(),
SuperRegClass, TRI);
MBB.addLiveIn(SuperReg);
++i;
} else {
// Cannot use a double-word load.
ContiguousRegs = false;
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.loadRegFromStackSlot(MBB, MI, Reg, CSI[i].getFrameIdx(), RC, TRI);
MBB.addLiveIn(Reg);
}
}
return true;
}
void X86RetpolineThunks::populateThunk(MachineFunction &MF,
unsigned Reg) {
// Set MF properties. We never use vregs...
MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
// Grab the entry MBB and erase any other blocks. O0 codegen appears to
// generate two bbs for the entry block.
MachineBasicBlock *Entry = &MF.front();
Entry->clear();
while (MF.size() > 1)
MF.erase(std::next(MF.begin()));
MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MCSymbol *TargetSym = MF.getContext().createTempSymbol();
MF.push_back(CaptureSpec);
MF.push_back(CallTarget);
const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
Entry->addLiveIn(Reg);
BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
// The MIR verifier thinks that the CALL in the entry block will fall through
// to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
// the successor, but the MIR verifier doesn't know how to cope with that.
Entry->addSuccessor(CaptureSpec);
// In the capture loop for speculation, we want to stop the processor from
// speculating as fast as possible. On Intel processors, the PAUSE instruction
// will block speculation without consuming any execution resources. On AMD
// processors, the PAUSE instruction is (essentially) a nop, so we also use an
// LFENCE instruction which they have advised will stop speculation as well
// with minimal resource utilization. We still end the capture with a jump to
// form an infinite loop to fully guarantee that no matter what implementation
// of the x86 ISA, speculating this code path never escapes.
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
CaptureSpec->setHasAddressTaken();
CaptureSpec->addSuccessor(CaptureSpec);
CallTarget->addLiveIn(Reg);
CallTarget->setHasAddressTaken();
CallTarget->setAlignment(4);
insertRegReturnAddrClobber(*CallTarget, Reg);
CallTarget->back().setPreInstrSymbol(MF, TargetSym);
BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
}
// MI is a load-register-on-condition pseudo instruction that could not be
// handled as a single hardware instruction. Replace it by a branch sequence.
bool SystemZExpandPseudo::expandLOCRMux(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI) {
MachineFunction &MF = *MBB.getParent();
const BasicBlock *BB = MBB.getBasicBlock();
MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
unsigned DestReg = MI.getOperand(0).getReg();
unsigned SrcReg = MI.getOperand(2).getReg();
unsigned CCValid = MI.getOperand(3).getImm();
unsigned CCMask = MI.getOperand(4).getImm();
LivePhysRegs LiveRegs(TII->getRegisterInfo());
LiveRegs.addLiveOuts(MBB);
for (auto I = std::prev(MBB.end()); I != MBBI; --I)
LiveRegs.stepBackward(*I);
// Splice MBB at MI, moving the rest of the block into RestMBB.
MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB);
MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB);
RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end());
RestMBB->transferSuccessors(&MBB);
for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
RestMBB->addLiveIn(*I);
// Create a new block MoveMBB to hold the move instruction.
MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB);
MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB);
MoveMBB->addLiveIn(SrcReg);
for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
MoveMBB->addLiveIn(*I);
// At the end of MBB, create a conditional branch to RestMBB if the
// condition is false, otherwise fall through to MoveMBB.
BuildMI(&MBB, DL, TII->get(SystemZ::BRC))
.addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB);
MBB.addSuccessor(RestMBB);
MBB.addSuccessor(MoveMBB);
// In MoveMBB, emit an instruction to move SrcReg into DestReg,
// then fall through to RestMBB.
TII->copyPhysReg(*MoveMBB, MoveMBB->end(), DL, DestReg, SrcReg,
MI.getOperand(2).isKill());
MoveMBB->addSuccessor(RestMBB);
NextMBBI = MBB.end();
MI.eraseFromParent();
return true;
}
bool XCoreFrameLowering::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return true;
MachineFunction *MF = MBB.getParent();
const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
XCoreFunctionInfo *XFI = MF->getInfo<XCoreFunctionInfo>();
bool emitFrameMoves = XCoreRegisterInfo::needsFrameMoves(*MF);
DebugLoc DL;
if (MI != MBB.end() && !MI->isDebugValue())
DL = MI->getDebugLoc();
for (std::vector<CalleeSavedInfo>::const_iterator it = CSI.begin();
it != CSI.end(); ++it) {
unsigned Reg = it->getReg();
assert(Reg != XCore::LR && !(Reg == XCore::R10 && hasFP(*MF)) &&
"LR & FP are always handled in emitPrologue");
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(Reg);
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(MBB, MI, Reg, true, it->getFrameIdx(), RC, TRI);
if (emitFrameMoves) {
auto Store = MI;
--Store;
XFI->getSpillLabels().push_back(std::make_pair(Store, *it));
}
}
return true;
}
bool XCoreFrameLowering::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return true;
MachineFunction *MF = MBB.getParent();
const TargetInstrInfo &TII = *MF->getTarget().getInstrInfo();
XCoreFunctionInfo *XFI = MF->getInfo<XCoreFunctionInfo>();
bool emitFrameMoves = XCoreRegisterInfo::needsFrameMoves(*MF);
DebugLoc DL;
if (MI != MBB.end())
DL = MI->getDebugLoc();
for (std::vector<CalleeSavedInfo>::const_iterator it = CSI.begin();
it != CSI.end(); ++it) {
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(it->getReg());
unsigned Reg = it->getReg();
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(MBB, MI, Reg, true,
it->getFrameIdx(), RC, TRI);
if (emitFrameMoves) {
MCSymbol *SaveLabel = MF->getContext().CreateTempSymbol();
BuildMI(MBB, MI, DL, TII.get(XCore::PROLOG_LABEL)).addSym(SaveLabel);
XFI->getSpillLabels().push_back(std::make_pair(SaveLabel, *it));
}
}
return true;
}
bool Thumb1FrameLowering::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
DebugLoc DL;
const TargetInstrInfo &TII = *STI.getInstrInfo();
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, TII.get(ARM::tPUSH));
AddDefaultPred(MIB);
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
bool isKill = true;
// Add the callee-saved register as live-in unless it's LR and
// @llvm.returnaddress is called. If LR is returned for @llvm.returnaddress
// then it's already added to the function and entry block live-in sets.
if (Reg == ARM::LR) {
MachineFunction &MF = *MBB.getParent();
if (MF.getFrameInfo()->isReturnAddressTaken() &&
MF.getRegInfo().isLiveIn(Reg))
isKill = false;
}
if (isKill)
MBB.addLiveIn(Reg);
MIB.addReg(Reg, getKillRegState(isKill));
}
MIB.setMIFlags(MachineInstr::FrameSetup);
return true;
}
// Compute MBB live-in lists from virtual register live ranges and their
// assignments.
void VirtRegRewriter::addMBBLiveIns() {
for (unsigned Idx = 0, IdxE = MRI->getNumVirtRegs(); Idx != IdxE; ++Idx) {
unsigned VirtReg = TargetRegisterInfo::index2VirtReg(Idx);
if (MRI->reg_nodbg_empty(VirtReg))
continue;
LiveInterval &LI = LIS->getInterval(VirtReg);
if (LI.empty() || LIS->intervalIsInOneMBB(LI))
continue;
// This is a virtual register that is live across basic blocks. Its
// assigned PhysReg must be marked as live-in to those blocks.
unsigned PhysReg = VRM->getPhys(VirtReg);
assert(PhysReg != VirtRegMap::NO_PHYS_REG && "Unmapped virtual register.");
if (LI.hasSubRanges()) {
addLiveInsForSubRanges(LI, PhysReg);
} else {
// Go over MBB begin positions and see if we have segments covering them.
// The following works because segments and the MBBIndex list are both
// sorted by slot indexes.
SlotIndexes::MBBIndexIterator I = Indexes->MBBIndexBegin();
for (const auto &Seg : LI) {
I = Indexes->advanceMBBIndex(I, Seg.start);
for (; I != Indexes->MBBIndexEnd() && I->first < Seg.end; ++I) {
MachineBasicBlock *MBB = I->second;
MBB->addLiveIn(PhysReg);
}
}
}
}
// Sort and unique MBB LiveIns as we've not checked if SubReg/PhysReg were in
// each MBB's LiveIns set before calling addLiveIn on them.
for (MachineBasicBlock &MBB : *MF)
MBB.sortUniqueLiveIns();
}
// FIXME: Can we eleminate these in favour of generic code?
bool
MSP430FrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
DebugLoc DL;
if (MI != MBB.end()) DL = MI->getDebugLoc();
MachineFunction &MF = *MBB.getParent();
const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
MSP430MachineFunctionInfo *MFI = MF.getInfo<MSP430MachineFunctionInfo>();
MFI->setCalleeSavedFrameSize(CSI.size() * 2);
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(Reg);
BuildMI(MBB, MI, DL, TII.get(MSP430::PUSH16r))
.addReg(Reg, RegState::Kill);
}
return true;
}
bool Cpu0FrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction *MF = MBB.getParent();
MachineBasicBlock *EntryBlock = MF->begin();
const TargetInstrInfo &TII = *MF->getTarget().getInstrInfo();
for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
// Add the callee-saved register as live-in. Do not add if the register is
// RA and return address is taken, because it has already been added in
// method Cpu0TargetLowering::LowerRETURNADDR.
// It's killed at the spill, unless the register is RA and return address
// is taken.
unsigned Reg = CSI[i].getReg();
bool IsRAAndRetAddrIsTaken = (Reg == Cpu0::LR)
&& MF->getFrameInfo()->isReturnAddressTaken();
if (!IsRAAndRetAddrIsTaken)
EntryBlock->addLiveIn(Reg);
// Insert the spill to the stack frame.
bool IsKill = !IsRAAndRetAddrIsTaken;
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(*EntryBlock, MI, Reg, IsKill,
CSI[i].getFrameIdx(), RC, TRI);
}
return true;
}
bool MIRParserImpl::initializeMachineBasicBlock(
MachineFunction &MF, MachineBasicBlock &MBB,
const yaml::MachineBasicBlock &YamlMBB,
const PerFunctionMIParsingState &PFS) {
MBB.setAlignment(YamlMBB.Alignment);
if (YamlMBB.AddressTaken)
MBB.setHasAddressTaken();
MBB.setIsLandingPad(YamlMBB.IsLandingPad);
SMDiagnostic Error;
// Parse the successors.
for (const auto &MBBSource : YamlMBB.Successors) {
MachineBasicBlock *SuccMBB = nullptr;
if (parseMBBReference(SuccMBB, MBBSource, MF, PFS))
return true;
// TODO: Report an error when adding the same successor more than once.
MBB.addSuccessor(SuccMBB);
}
// Parse the liveins.
for (const auto &LiveInSource : YamlMBB.LiveIns) {
unsigned Reg = 0;
if (parseNamedRegisterReference(Reg, SM, MF, LiveInSource.Value, PFS,
IRSlots, Error))
return error(Error, LiveInSource.SourceRange);
MBB.addLiveIn(Reg);
}
// Parse the instructions.
for (const auto &MISource : YamlMBB.Instructions) {
MachineInstr *MI = nullptr;
if (parseMachineInstr(MI, SM, MF, MISource.Value, PFS, IRSlots, Error))
return error(Error, MISource.SourceRange);
MBB.insert(MBB.end(), MI);
}
return false;
}
bool Mips16FrameLowering::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction *MF = MBB.getParent();
MachineBasicBlock *EntryBlock = MF->begin();
//
// Registers RA, S0,S1 are the callee saved registers and they
// will be saved with the "save" instruction
// during emitPrologue
//
for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
// Add the callee-saved register as live-in. Do not add if the register is
// RA and return address is taken, because it has already been added in
// method MipsTargetLowering::LowerRETURNADDR.
// It's killed at the spill, unless the register is RA and return address
// is taken.
unsigned Reg = CSI[i].getReg();
bool IsRAAndRetAddrIsTaken = (Reg == Mips::RA)
&& MF->getFrameInfo()->isReturnAddressTaken();
if (!IsRAAndRetAddrIsTaken)
EntryBlock->addLiveIn(Reg);
}
return true;
}
bool XCoreInstrInfo::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI) const
{
if (CSI.empty()) {
return true;
}
MachineFunction *MF = MBB.getParent();
const MachineFrameInfo *MFI = MF->getFrameInfo();
MachineModuleInfo *MMI = MFI->getMachineModuleInfo();
XCoreFunctionInfo *XFI = MF->getInfo<XCoreFunctionInfo>();
bool emitFrameMoves = XCoreRegisterInfo::needsFrameMoves(*MF);
DebugLoc DL = DebugLoc::getUnknownLoc();
if (MI != MBB.end()) DL = MI->getDebugLoc();
for (std::vector<CalleeSavedInfo>::const_iterator it = CSI.begin();
it != CSI.end(); ++it) {
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(it->getReg());
storeRegToStackSlot(MBB, MI, it->getReg(), true,
it->getFrameIdx(), it->getRegClass());
if (emitFrameMoves) {
unsigned SaveLabelId = MMI->NextLabelID();
BuildMI(MBB, MI, DL, get(XCore::DBG_LABEL)).addImm(SaveLabelId);
XFI->getSpillLabels().push_back(
std::pair<unsigned, CalleeSavedInfo>(SaveLabelId, *it));
}
}
return true;
}
// Add GPR64 to the save instruction being built by MIB, which is in basic
// block MBB. IsImplicit says whether this is an explicit operand to the
// instruction, or an implicit one that comes between the explicit start
// and end registers.
static void addSavedGPR(MachineBasicBlock &MBB, MachineInstrBuilder &MIB,
unsigned GPR64, bool IsImplicit) {
const TargetRegisterInfo *RI = MBB.getParent()->getTarget().getRegisterInfo();
unsigned GPR32 = RI->getSubReg(GPR64, SystemZ::subreg_l32);
bool IsLive = MBB.isLiveIn(GPR64) || MBB.isLiveIn(GPR32);
if (!IsLive || !IsImplicit) {
MIB.addReg(GPR64, getImplRegState(IsImplicit) | getKillRegState(!IsLive));
if (!IsLive)
MBB.addLiveIn(GPR64);
}
}
/// Rewrite the null checks in NullCheckList into implicit null checks.
void ImplicitNullChecks::rewriteNullChecks(
ArrayRef<ImplicitNullChecks::NullCheck> NullCheckList) {
DebugLoc DL;
for (auto &NC : NullCheckList) {
// Remove the conditional branch dependent on the null check.
unsigned BranchesRemoved = TII->removeBranch(*NC.getCheckBlock());
(void)BranchesRemoved;
assert(BranchesRemoved > 0 && "expected at least one branch!");
if (auto *DepMI = NC.getOnlyDependency()) {
DepMI->removeFromParent();
NC.getCheckBlock()->insert(NC.getCheckBlock()->end(), DepMI);
}
// Insert a faulting instruction where the conditional branch was
// originally. We check earlier ensures that this bit of code motion
// is legal. We do not touch the successors list for any basic block
// since we haven't changed control flow, we've just made it implicit.
MachineInstr *FaultingInstr = insertFaultingInstr(
NC.getMemOperation(), NC.getCheckBlock(), NC.getNullSucc());
// Now the values defined by MemOperation, if any, are live-in of
// the block of MemOperation.
// The original operation may define implicit-defs alongside
// the value.
MachineBasicBlock *MBB = NC.getMemOperation()->getParent();
for (const MachineOperand &MO : FaultingInstr->operands()) {
if (!MO.isReg() || !MO.isDef())
continue;
unsigned Reg = MO.getReg();
if (!Reg || MBB->isLiveIn(Reg))
continue;
MBB->addLiveIn(Reg);
}
if (auto *DepMI = NC.getOnlyDependency()) {
for (auto &MO : DepMI->operands()) {
if (!MO.isReg() || !MO.getReg() || !MO.isDef())
continue;
if (!NC.getNotNullSucc()->isLiveIn(MO.getReg()))
NC.getNotNullSucc()->addLiveIn(MO.getReg());
}
}
NC.getMemOperation()->eraseFromParent();
NC.getCheckOperation()->eraseFromParent();
// Insert an *unconditional* branch to not-null successor.
TII->insertBranch(*NC.getCheckBlock(), NC.getNotNullSucc(), nullptr,
/*Cond=*/None, DL);
NumImplicitNullChecks++;
}
}
bool
PatmosFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
DebugLoc DL;
if (MI != MBB.end()) DL = MI->getDebugLoc();
MachineFunction &MF = *MBB.getParent();
const TargetInstrInfo &TII = *TM.getInstrInfo();
PatmosMachineFunctionInfo &PMFI =
*MF.getInfo<PatmosMachineFunctionInfo>();
unsigned spilledSize = 0;
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(Reg);
// as all PRegs are aliased with S0, a spill of a Preg will cause
// a spill of S0
if (Patmos::PRegsRegClass.contains(Reg))
continue;
// Spill S0 to a register instead to a slot if there is a free register
if (Reg == Patmos::S0 && PMFI.getS0SpillReg()) {
TII.copyPhysReg(MBB, MI, DL, PMFI.getS0SpillReg(), Reg, true);
prior(MI)->setFlag(MachineInstr::FrameSetup);
continue;
}
// copy to R register first, then spill
if (Patmos::SRegsRegClass.contains(Reg)) {
TII.copyPhysReg(MBB, MI, DL, Patmos::R9, Reg, true);
prior(MI)->setFlag(MachineInstr::FrameSetup);
Reg = Patmos::R9;
}
// spill
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(MBB, MI, Reg, true,
CSI[i-1].getFrameIdx(), RC, TRI);
prior(MI)->setFlag(MachineInstr::FrameSetup);
// increment spilled size
spilledSize += 4;
}
return true;
}
// All currently live registers must remain so in the remainder block.
void SILowerControlFlow::splitBlockLiveIns(const MachineBasicBlock &MBB,
const MachineInstr &MI,
MachineBasicBlock &LoopBB,
MachineBasicBlock &RemainderBB,
unsigned SaveReg,
const MachineOperand &IdxReg) {
LivePhysRegs RemainderLiveRegs(TRI);
RemainderLiveRegs.addLiveOuts(MBB);
for (MachineBasicBlock::const_reverse_iterator I = MBB.rbegin(), E(&MI);
I != E; ++I) {
RemainderLiveRegs.stepBackward(*I);
}
// Add reg defined in loop body.
RemainderLiveRegs.addReg(SaveReg);
if (const MachineOperand *Val = TII->getNamedOperand(MI, AMDGPU::OpName::val)) {
if (!Val->isUndef()) {
RemainderLiveRegs.addReg(Val->getReg());
LoopBB.addLiveIn(Val->getReg());
}
}
const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
for (unsigned Reg : RemainderLiveRegs) {
if (MRI.isAllocatable(Reg))
RemainderBB.addLiveIn(Reg);
}
const MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src);
if (!Src->isUndef())
LoopBB.addLiveIn(Src->getReg());
if (!IdxReg.isUndef())
LoopBB.addLiveIn(IdxReg.getReg());
LoopBB.sortUniqueLiveIns();
}
bool MipsSEFrameLowering::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
MachineFunction *MF = MBB.getParent();
MachineBasicBlock *EntryBlock = &MF->front();
const TargetInstrInfo &TII = *STI.getInstrInfo();
for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
// Add the callee-saved register as live-in. Do not add if the register is
// RA and return address is taken, because it has already been added in
// method MipsTargetLowering::LowerRETURNADDR.
// It's killed at the spill, unless the register is RA and return address
// is taken.
unsigned Reg = CSI[i].getReg();
bool IsRAAndRetAddrIsTaken = (Reg == Mips::RA || Reg == Mips::RA_64)
&& MF->getFrameInfo()->isReturnAddressTaken();
if (!IsRAAndRetAddrIsTaken)
EntryBlock->addLiveIn(Reg);
// ISRs require HI/LO to be spilled into kernel registers to be then
// spilled to the stack frame.
bool IsLOHI = (Reg == Mips::LO0 || Reg == Mips::LO0_64 ||
Reg == Mips::HI0 || Reg == Mips::HI0_64);
const Function *Func = MBB.getParent()->getFunction();
if (IsLOHI && Func->hasFnAttribute("interrupt")) {
DebugLoc DL = MI->getDebugLoc();
unsigned Op = 0;
if (!STI.getABI().ArePtrs64bit()) {
Op = (Reg == Mips::HI0) ? Mips::MFHI : Mips::MFLO;
Reg = Mips::K0;
} else {
Op = (Reg == Mips::HI0) ? Mips::MFHI64 : Mips::MFLO64;
Reg = Mips::K0_64;
}
BuildMI(MBB, MI, DL, TII.get(Op), Mips::K0)
.setMIFlag(MachineInstr::FrameSetup);
}
// Insert the spill to the stack frame.
bool IsKill = !IsRAAndRetAddrIsTaken;
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(*EntryBlock, MI, Reg, IsKill,
CSI[i].getFrameIdx(), RC, TRI);
}
return true;
}
bool Z80FrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
const MachineFunction &MF = *MBB.getParent();
const MachineRegisterInfo &MRI = MF.getRegInfo();
bool UseShadow = shouldUseShadow(MF);
DebugLoc DL = MBB.findDebugLoc(MI);
if (UseShadow)
shadowCalleeSavedRegisters(MBB, MI, DL, MachineInstr::FrameSetup, CSI);
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i - 1].getReg();
// Non-index registers can be spilled to shadow registers.
if (UseShadow && !Z80::I24RegClass.contains(Reg) &&
!Z80::I16RegClass.contains(Reg))
continue;
bool isLiveIn = MRI.isLiveIn(Reg);
if (!isLiveIn)
MBB.addLiveIn(Reg);
// Decide whether we can add a kill flag to the use.
bool CanKill = !isLiveIn;
// Check if any subregister is live-in
if (CanKill) {
for (MCRegAliasIterator AReg(Reg, TRI, false); AReg.isValid(); ++AReg) {
if (MRI.isLiveIn(*AReg)) {
CanKill = false;
break;
}
}
}
// Do not set a kill flag on values that are also marked as live-in. This
// happens with the @llvm-returnaddress intrinsic and with arguments
// passed in callee saved registers.
// Omitting the kill flags is conservatively correct even if the live-in
// is not used after all.
MachineInstrBuilder MIB;
if (Reg == Z80::AF)
MIB = BuildMI(MBB, MI, DL, TII.get(Is24Bit ? Z80::PUSH24AF
: Z80::PUSH16AF));
else
MIB = BuildMI(MBB, MI, DL, TII.get(Is24Bit ? Z80::PUSH24r : Z80::PUSH16r))
.addReg(Reg, getKillRegState(CanKill));
MIB.setMIFlag(MachineInstr::FrameSetup);
}
return true;
}
bool AArch64FrameLowering::determinePrologueDeath(MachineBasicBlock &MBB,
unsigned Reg) const {
// If @llvm.returnaddress is called then it will refer to X30 by some means;
// the prologue store does not kill the register.
if (Reg == AArch64::X30) {
if (MBB.getParent()->getFrameInfo()->isReturnAddressTaken()
&& MBB.getParent()->getRegInfo().isLiveIn(Reg))
return false;
}
// In all other cases, physical registers are dead after they've been saved
// but live at the beginning of the prologue block.
MBB.addLiveIn(Reg);
return true;
}
bool
TMS320C64XFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const
{
const MachineFunction *MF;
unsigned int i, reg;
bool is_kill;
MF = MBB.getParent();
const MachineRegisterInfo &MRI = MF->getRegInfo();
for (i = 0; i < CSI.size(); ++i) {
// Should this be a kill? Unfortunately the argument registers
// and nonvolatile registers on the target overlap, which leads
// to a situation where we spill a nonvolatile register,
// killing it, and then try and use it as an argument register
// -> "Using undefined register". So, check whether this reg is
// a _function_ LiveIn too.
is_kill = true;
reg = CSI[i].getReg();
MachineRegisterInfo::livein_iterator li = MRI.livein_begin();
for (; li != MRI.livein_end(); li++) {
if (li->first == reg) {
is_kill = false;
break;
}
}
MBB.addLiveIn(reg);
const TargetInstrInfo &TII = *(MF->getTarget().getInstrInfo());
// register class and target reg info unused
TII.storeRegToStackSlot(
MBB, MBBI, reg, is_kill, CSI[i].getFrameIdx(), 0, 0);
}
return true;
}
/// AddToLiveIns - Add register 'Reg' to the livein sets of BBs in the current
/// loop, and make sure it is not killed by any instructions in the loop.
void MachineLICM::AddToLiveIns(unsigned Reg) {
const std::vector<MachineBasicBlock*> Blocks = CurLoop->getBlocks();
for (unsigned i = 0, e = Blocks.size(); i != e; ++i) {
MachineBasicBlock *BB = Blocks[i];
if (!BB->isLiveIn(Reg))
BB->addLiveIn(Reg);
for (MachineBasicBlock::iterator
MII = BB->begin(), E = BB->end(); MII != E; ++MII) {
MachineInstr *MI = &*MII;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || !MO.getReg() || MO.isDef()) continue;
if (MO.getReg() == Reg || TRI->isSuperRegister(Reg, MO.getReg()))
MO.setIsKill(false);
}
}
}
}
void VirtRegRewriter::addLiveInsForSubRanges(const LiveInterval &LI,
unsigned PhysReg) const {
assert(!LI.empty());
assert(LI.hasSubRanges());
using SubRangeIteratorPair =
std::pair<const LiveInterval::SubRange *, LiveInterval::const_iterator>;
SmallVector<SubRangeIteratorPair, 4> SubRanges;
SlotIndex First;
SlotIndex Last;
for (const LiveInterval::SubRange &SR : LI.subranges()) {
SubRanges.push_back(std::make_pair(&SR, SR.begin()));
if (!First.isValid() || SR.segments.front().start < First)
First = SR.segments.front().start;
if (!Last.isValid() || SR.segments.back().end > Last)
Last = SR.segments.back().end;
}
// Check all mbb start positions between First and Last while
// simulatenously advancing an iterator for each subrange.
for (SlotIndexes::MBBIndexIterator MBBI = Indexes->findMBBIndex(First);
MBBI != Indexes->MBBIndexEnd() && MBBI->first <= Last; ++MBBI) {
SlotIndex MBBBegin = MBBI->first;
// Advance all subrange iterators so that their end position is just
// behind MBBBegin (or the iterator is at the end).
LaneBitmask LaneMask;
for (auto &RangeIterPair : SubRanges) {
const LiveInterval::SubRange *SR = RangeIterPair.first;
LiveInterval::const_iterator &SRI = RangeIterPair.second;
while (SRI != SR->end() && SRI->end <= MBBBegin)
++SRI;
if (SRI == SR->end())
continue;
if (SRI->start <= MBBBegin)
LaneMask |= SR->LaneMask;
}
if (LaneMask.none())
continue;
MachineBasicBlock *MBB = MBBI->second;
MBB->addLiveIn(PhysReg, LaneMask);
}
}
/// This function adds registers Filler defines to MBB's live-in register list.
static void addLiveInRegs(Iter Filler, MachineBasicBlock &MBB) {
for (unsigned I = 0, E = Filler->getNumOperands(); I != E; ++I) {
const MachineOperand &MO = Filler->getOperand(I);
unsigned R;
if (!MO.isReg() || !MO.isDef() || !(R = MO.getReg()))
continue;
#ifndef NDEBUG
const MachineFunction &MF = *MBB.getParent();
assert(MF.getSubtarget().getRegisterInfo()->getAllocatableSet(MF).test(R) &&
"Shouldn't move an instruction with unallocatable registers across "
"basic block boundaries.");
#endif
if (!MBB.isLiveIn(R))
MBB.addLiveIn(R);
}
}
bool Thumb1InstrInfo::
spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI) const {
if (CSI.empty())
return false;
DebugLoc DL = DebugLoc::getUnknownLoc();
if (MI != MBB.end()) DL = MI->getDebugLoc();
MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, get(ARM::tPUSH));
AddDefaultPred(MIB);
MIB.addReg(0); // No write back.
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(Reg);
MIB.addReg(Reg, RegState::Kill);
}
return true;
}
bool X86FrameInfo::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
DebugLoc DL = MBB.findDebugLoc(MI);
MachineFunction &MF = *MBB.getParent();
bool isWin64 = STI.isTargetWin64();
unsigned SlotSize = STI.is64Bit() ? 8 : 4;
unsigned FPReg = TRI->getFrameRegister(MF);
unsigned CalleeFrameSize = 0;
const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
unsigned Opc = STI.is64Bit() ? X86::PUSH64r : X86::PUSH32r;
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
// Add the callee-saved register as live-in. It's killed at the spill.
MBB.addLiveIn(Reg);
if (Reg == FPReg)
// X86RegisterInfo::emitPrologue will handle spilling of frame register.
continue;
if (!X86::VR128RegClass.contains(Reg) && !isWin64) {
CalleeFrameSize += SlotSize;
BuildMI(MBB, MI, DL, TII.get(Opc)).addReg(Reg, RegState::Kill);
} else {
const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
TII.storeRegToStackSlot(MBB, MI, Reg, true, CSI[i-1].getFrameIdx(),
RC, TRI);
}
}
X86FI->setCalleeSavedFrameSize(CalleeFrameSize);
return true;
}
bool AVRFrameLowering::spillCalleeSavedRegisters(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty()) {
return false;
}
unsigned CalleeFrameSize = 0;
DebugLoc DL = MBB.findDebugLoc(MI);
MachineFunction &MF = *MBB.getParent();
const AVRSubtarget &STI = MF.getSubtarget<AVRSubtarget>();
const TargetInstrInfo &TII = *STI.getInstrInfo();
AVRMachineFunctionInfo *AVRFI = MF.getInfo<AVRMachineFunctionInfo>();
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i - 1].getReg();
bool IsNotLiveIn = !MBB.isLiveIn(Reg);
assert(TRI->getRegSizeInBits(*TRI->getMinimalPhysRegClass(Reg)) == 8 &&
"Invalid register size");
// Add the callee-saved register as live-in only if it is not already a
// live-in register, this usually happens with arguments that are passed
// through callee-saved registers.
if (IsNotLiveIn) {
MBB.addLiveIn(Reg);
}
// Do not kill the register when it is an input argument.
BuildMI(MBB, MI, DL, TII.get(AVR::PUSHRr))
.addReg(Reg, getKillRegState(IsNotLiveIn))
.setMIFlag(MachineInstr::FrameSetup);
++CalleeFrameSize;
}
AVRFI->setCalleeSavedFrameSize(CalleeFrameSize);
return true;
}
bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
unsigned OldBrSize = TII->getInstSizeInBytes(MI);
MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
int64_t DestOffset = BlockInfo[DestBB->getNumber()].Offset;
int64_t SrcOffset = getInstrOffset(MI);
assert(!TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - SrcOffset));
BlockInfo[MBB->getNumber()].Size -= OldBrSize;
MachineBasicBlock *BranchBB = MBB;
// If this was an expanded conditional branch, there is already a single
// unconditional branch in a block.
if (!MBB->empty()) {
BranchBB = createNewBlockAfter(*MBB);
// Add live outs.
for (const MachineBasicBlock *Succ : MBB->successors()) {
for (const MachineBasicBlock::RegisterMaskPair &LiveIn : Succ->liveins())
BranchBB->addLiveIn(LiveIn);
}
BranchBB->sortUniqueLiveIns();
BranchBB->addSuccessor(DestBB);
MBB->replaceSuccessor(DestBB, BranchBB);
}
DebugLoc DL = MI.getDebugLoc();
MI.eraseFromParent();
BlockInfo[BranchBB->getNumber()].Size += TII->insertIndirectBranch(
*BranchBB, *DestBB, DL, DestOffset - SrcOffset, RS.get());
adjustBlockOffsets(*MBB);
return true;
}
