/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(AfterLoop);
// Emit a special no-op telling the asm printer that we need a label to close
// the loop scope, even though the destination is only reachable by
// fallthrough.
if (!Bottom->back().isBarrier())
BuildMI(*Bottom, Bottom->end(), DebugLoc(), TII.get(WebAssembly::LOOP_END));
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With RPO we should visit the outer-most loop for a block first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
// ProcessSourceNode - Process nodes with source order numbers. These are added
// to a vector which EmitSchedule uses to determine how to insert dbg_value
// instructions in the right order.
static void
ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
DenseMap<SDValue, unsigned> &VRBaseMap,
SmallVectorImpl<std::pair<unsigned, MachineInstr*> > &Orders,
SmallSet<unsigned, 8> &Seen) {
unsigned Order = N->getIROrder();
if (!Order || !Seen.insert(Order).second) {
// Process any valid SDDbgValues even if node does not have any order
// assigned.
ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, 0);
return;
}
MachineBasicBlock *BB = Emitter.getBlock();
if (Emitter.getInsertPos() == BB->begin() || BB->back().isPHI() ||
// Fast-isel may have inserted some instructions, in which case the
// BB->back().isPHI() test will not fire when we want it to.
std::prev(Emitter.getInsertPos())->isPHI()) {
// Did not insert any instruction.
Orders.push_back(std::make_pair(Order, (MachineInstr*)nullptr));
return;
}
Orders.push_back(std::make_pair(Order, std::prev(Emitter.getInsertPos())));
ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order);
}
static bool MaybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
const MachineFunction &MF,
WebAssemblyFunctionInfo &MFI,
MachineRegisterInfo &MRI,
const WebAssemblyInstrInfo &TII,
unsigned FallthroughOpc,
unsigned CopyLocalOpc) {
if (DisableWebAssemblyFallthroughReturnOpt)
return false;
if (&MBB != &MF.back())
return false;
if (&MI != &MBB.back())
return false;
// If the operand isn't stackified, insert a COPY to read the operand and
// stackify it.
MachineOperand &MO = MI.getOperand(0);
unsigned Reg = MO.getReg();
if (!MFI.isVRegStackified(Reg)) {
unsigned NewReg = MRI.createVirtualRegister(MRI.getRegClass(Reg));
BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(CopyLocalOpc), NewReg)
.addReg(Reg);
MO.setReg(NewReg);
MFI.stackifyVReg(NewReg);
}
// Rewrite the return.
MI.setDesc(TII.get(FallthroughOpc));
return true;
}
bool MSP430InstrInfo::BlockHasNoFallThrough(const MachineBasicBlock &MBB)const{
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case MSP430::RET: // Return.
case MSP430::JMP: // Uncond branch.
return true;
default: return false;
}
}
bool PPCInstrInfo::BlockHasNoFallThrough(MachineBasicBlock &MBB) const {
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case PPC::B: // Uncond branch.
case PPC::BCTR: // Indirect branch.
return true;
default: return false;
}
}
bool AlphaInstrInfo::BlockHasNoFallThrough(const MachineBasicBlock &MBB) const {
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case Alpha::RETDAG: // Return.
case Alpha::RETDAGp:
case Alpha::BR: // Uncond branch.
case Alpha::JMP: // Indirect branch.
return true;
default: return false;
}
}
unsigned PTXInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
unsigned count = 0;
while (!MBB.empty())
if (IsAnyKindOfBranch(MBB.back())) {
MBB.pop_back();
++count;
} else
break;
DEBUG(dbgs() << "RemoveBranch: MBB: " << MBB.getName().str() << "\n");
DEBUG(dbgs() << "RemoveBranch: remove " << count << " branch inst\n");
return count;
}
void X86RetpolineThunks::populateThunk(MachineFunction &MF,
unsigned Reg) {
// Set MF properties. We never use vregs...
MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
// Grab the entry MBB and erase any other blocks. O0 codegen appears to
// generate two bbs for the entry block.
MachineBasicBlock *Entry = &MF.front();
Entry->clear();
while (MF.size() > 1)
MF.erase(std::next(MF.begin()));
MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MCSymbol *TargetSym = MF.getContext().createTempSymbol();
MF.push_back(CaptureSpec);
MF.push_back(CallTarget);
const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
Entry->addLiveIn(Reg);
BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
// The MIR verifier thinks that the CALL in the entry block will fall through
// to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
// the successor, but the MIR verifier doesn't know how to cope with that.
Entry->addSuccessor(CaptureSpec);
// In the capture loop for speculation, we want to stop the processor from
// speculating as fast as possible. On Intel processors, the PAUSE instruction
// will block speculation without consuming any execution resources. On AMD
// processors, the PAUSE instruction is (essentially) a nop, so we also use an
// LFENCE instruction which they have advised will stop speculation as well
// with minimal resource utilization. We still end the capture with a jump to
// form an infinite loop to fully guarantee that no matter what implementation
// of the x86 ISA, speculating this code path never escapes.
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
CaptureSpec->setHasAddressTaken();
CaptureSpec->addSuccessor(CaptureSpec);
CallTarget->addLiveIn(Reg);
CallTarget->setHasAddressTaken();
CallTarget->setAlignment(4);
insertRegReturnAddrClobber(*CallTarget, Reg);
CallTarget->back().setPreInstrSymbol(MF, TargetSym);
BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
}
// ProcessSourceNode - Process nodes with source order numbers. These are added
// to a vector which EmitSchedule uses to determine how to insert dbg_value
// instructions in the right order.
static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG,
InstrEmitter &Emitter,
DenseMap<SDValue, unsigned> &VRBaseMap,
SmallVector<std::pair<unsigned, MachineInstr*>, 32> &Orders,
SmallSet<unsigned, 8> &Seen) {
unsigned Order = DAG->GetOrdering(N);
if (!Order || !Seen.insert(Order))
return;
MachineBasicBlock *BB = Emitter.getBlock();
if (BB->empty() || BB->back().isPHI()) {
// Did not insert any instruction.
Orders.push_back(std::make_pair(Order, (MachineInstr*)0));
return;
}
Orders.push_back(std::make_pair(Order, &BB->back()));
if (!N->getHasDebugValue())
return;
// Opportunistically insert immediate dbg_value uses, i.e. those with source
// order number right after the N.
MachineBasicBlock::iterator InsertPos = Emitter.getInsertPos();
SmallVector<SDDbgValue*,2> &DVs = DAG->GetDbgValues(N);
for (unsigned i = 0, e = DVs.size(); i != e; ++i) {
if (DVs[i]->isInvalidated())
continue;
unsigned DVOrder = DVs[i]->getOrder();
if (DVOrder == ++Order) {
MachineInstr *DbgMI = Emitter.EmitDbgValue(DVs[i], VRBaseMap);
if (DbgMI) {
Orders.push_back(std::make_pair(DVOrder, DbgMI));
BB->insert(InsertPos, DbgMI);
}
DVs[i]->setIsInvalidated();
}
}
}
/// BlockHasNoFallThrough - Analyse if MachineBasicBlock does not
/// fall-through into its successor block.
bool XCoreInstrInfo::
BlockHasNoFallThrough(const MachineBasicBlock &MBB) const
{
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case XCore::RETSP_u6: // Return.
case XCore::RETSP_lu6:
case XCore::BAU_1r: // Indirect branch.
case XCore::BRFU_u6: // Uncond branch.
case XCore::BRFU_lu6:
case XCore::BRBU_u6:
case XCore::BRBU_lu6:
return true;
default: return false;
}
}
bool ARMInstrInfo::BlockHasNoFallThrough(const MachineBasicBlock &MBB) const {
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case ARM::BX_RET: // Return.
case ARM::LDM_RET:
case ARM::B:
case ARM::BRIND:
case ARM::BR_JTr: // Jumptable branch.
case ARM::BR_JTm: // Jumptable branch through mem.
case ARM::BR_JTadd: // Jumptable branch add to pc.
return true;
default:
break;
}
return false;
}
bool
Thumb2InstrInfo::BlockHasNoFallThrough(const MachineBasicBlock &MBB) const {
if (MBB.empty()) return false;
switch (MBB.back().getOpcode()) {
case ARM::t2LDM_RET:
case ARM::t2B: // Uncond branch.
case ARM::t2BR_JT: // Jumptable branch.
case ARM::t2TBB: // Table branch byte.
case ARM::t2TBH: // Table branch halfword.
case ARM::tBR_JTr: // Jumptable branch (16-bit version).
case ARM::tBX_RET:
case ARM::tBX_RET_vararg:
case ARM::tPOP_RET:
case ARM::tB:
case ARM::tBRIND:
return true;
default:
break;
}
return false;
}
// ProcessSourceNode - Process nodes with source order numbers. These are added
// to a vector which EmitSchedule uses to determine how to insert dbg_value
// instructions in the right order.
static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG,
InstrEmitter &Emitter,
DenseMap<SDValue, unsigned> &VRBaseMap,
SmallVector<std::pair<unsigned, MachineInstr*>, 32> &Orders,
SmallSet<unsigned, 8> &Seen) {
unsigned Order = DAG->GetOrdering(N);
if (!Order || !Seen.insert(Order)) {
// Process any valid SDDbgValues even if node does not have any order
// assigned.
ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, 0);
return;
}
MachineBasicBlock *BB = Emitter.getBlock();
if (Emitter.getInsertPos() == BB->begin() || BB->back().isPHI()) {
// Did not insert any instruction.
Orders.push_back(std::make_pair(Order, (MachineInstr*)0));
return;
}
Orders.push_back(std::make_pair(Order, prior(Emitter.getInsertPos())));
ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order);
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
DebugLoc DL = MI.getDebugLoc();
MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
assert(!Fail && "branches to be relaxed must be analyzable");
(void)Fail;
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
if (FBB && isBlockInRange(MI, *FBB)) {
// Last MI in the BB is an unconditional branch. We can simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
DEBUG(dbgs() << " Invert condition and swap "
"its destination with " << MBB->back());
TII->reverseBranchCondition(Cond);
int OldSize = 0, NewSize = 0;
TII->removeBranch(*MBB, &OldSize);
TII->insertBranch(*MBB, FBB, TBB, Cond, DL, &NewSize);
BlockInfo[MBB->getNumber()].Size += (NewSize - OldSize);
return true;
} else if (FBB) {
// We need to split the basic block here to obtain two long-range
// unconditional branches.
auto &NewBB = *MF->CreateMachineBasicBlock(MBB->getBasicBlock());
MF->insert(++MBB->getIterator(), &NewBB);
// Insert an entry into BlockInfo to align it properly with the block
// numbers.
BlockInfo.insert(BlockInfo.begin() + NewBB.getNumber(), BasicBlockInfo());
unsigned &NewBBSize = BlockInfo[NewBB.getNumber()].Size;
int NewBrSize;
TII->insertUnconditionalBranch(NewBB, FBB, DL, &NewBrSize);
NewBBSize += NewBrSize;
// Update the successor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, &NewBB);
NewBB.addSuccessor(FBB);
}
// We now have an appropriate fall-through block in place (either naturally or
// just created), so we can invert the condition.
MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << TBB->getNumber()
<< ", invert condition and change dest. to BB#"
<< NextBB.getNumber() << '\n');
unsigned &MBBSize = BlockInfo[MBB->getNumber()].Size;
// Insert a new conditional branch and a new unconditional branch.
int RemovedSize = 0;
TII->reverseBranchCondition(Cond);
TII->removeBranch(*MBB, &RemovedSize);
MBBSize -= RemovedSize;
int AddedSize = 0;
TII->insertBranch(*MBB, &NextBB, TBB, Cond, DL, &AddedSize);
MBBSize += AddedSize;
// Finally, keep the block offsets up to date.
adjustBlockOffsets(*MBB);
return true;
}
bool Thumb1FrameLowering::emitPopSpecialFixUp(MachineBasicBlock &MBB,
bool DoIt) const {
MachineFunction &MF = *MBB.getParent();
ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
unsigned ArgRegsSaveSize = AFI->getArgRegsSaveSize();
const TargetInstrInfo &TII = *STI.getInstrInfo();
const ThumbRegisterInfo *RegInfo =
static_cast<const ThumbRegisterInfo *>(STI.getRegisterInfo());
// If MBBI is a return instruction, or is a tPOP followed by a return
// instruction in the successor BB, we may be able to directly restore
// LR in the PC.
// This is only possible with v5T ops (v4T can't change the Thumb bit via
// a POP PC instruction), and only if we do not need to emit any SP update.
// Otherwise, we need a temporary register to pop the value
// and copy that value into LR.
auto MBBI = MBB.getFirstTerminator();
bool CanRestoreDirectly = STI.hasV5TOps() && !ArgRegsSaveSize;
if (CanRestoreDirectly) {
if (MBBI != MBB.end())
CanRestoreDirectly = (MBBI->getOpcode() == ARM::tBX_RET ||
MBBI->getOpcode() == ARM::tPOP_RET);
else {
assert(MBB.back().getOpcode() == ARM::tPOP);
assert(MBB.succ_size() == 1);
if ((*MBB.succ_begin())->begin()->getOpcode() == ARM::tBX_RET)
MBBI--; // Replace the final tPOP with a tPOP_RET.
else
CanRestoreDirectly = false;
}
}
if (CanRestoreDirectly) {
if (!DoIt || MBBI->getOpcode() == ARM::tPOP_RET)
return true;
MachineInstrBuilder MIB =
AddDefaultPred(
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII.get(ARM::tPOP_RET)));
// Copy implicit ops and popped registers, if any.
for (auto MO: MBBI->operands())
if (MO.isReg() && (MO.isImplicit() || MO.isDef()) &&
MO.getReg() != ARM::LR)
MIB.addOperand(MO);
MIB.addReg(ARM::PC, RegState::Define);
// Erase the old instruction (tBX_RET or tPOP).
MBB.erase(MBBI);
return true;
}
// Look for a temporary register to use.
// First, compute the liveness information.
LivePhysRegs UsedRegs(STI.getRegisterInfo());
UsedRegs.addLiveOuts(&MBB, /*AddPristines*/ true);
// The semantic of pristines changed recently and now,
// the callee-saved registers that are touched in the function
// are not part of the pristines set anymore.
// Add those callee-saved now.
const TargetRegisterInfo *TRI = STI.getRegisterInfo();
const MCPhysReg *CSRegs = TRI->getCalleeSavedRegs(&MF);
for (unsigned i = 0; CSRegs[i]; ++i)
UsedRegs.addReg(CSRegs[i]);
DebugLoc dl = DebugLoc();
if (MBBI != MBB.end()) {
dl = MBBI->getDebugLoc();
auto InstUpToMBBI = MBB.end();
while (InstUpToMBBI != MBBI)
// The pre-decrement is on purpose here.
// We want to have the liveness right before MBBI.
UsedRegs.stepBackward(*--InstUpToMBBI);
}
// Look for a register that can be directly use in the POP.
unsigned PopReg = 0;
// And some temporary register, just in case.
unsigned TemporaryReg = 0;
BitVector PopFriendly =
TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::tGPRRegClassID));
assert(PopFriendly.any() && "No allocatable pop-friendly register?!");
// Rebuild the GPRs from the high registers because they are removed
// form the GPR reg class for thumb1.
BitVector GPRsNoLRSP =
TRI->getAllocatableSet(MF, TRI->getRegClass(ARM::hGPRRegClassID));
GPRsNoLRSP |= PopFriendly;
GPRsNoLRSP.reset(ARM::LR);
GPRsNoLRSP.reset(ARM::SP);
GPRsNoLRSP.reset(ARM::PC);
for (int Register = GPRsNoLRSP.find_first(); Register != -1;
Register = GPRsNoLRSP.find_next(Register)) {
if (!UsedRegs.contains(Register)) {
// Remember the first pop-friendly register and exit.
if (PopFriendly.test(Register)) {
PopReg = Register;
TemporaryReg = 0;
break;
}
// Otherwise, remember that the register will be available to
// save a pop-friendly register.
TemporaryReg = Register;
}
}
if (!DoIt && !PopReg && !TemporaryReg)
return false;
assert((PopReg || TemporaryReg) && "Cannot get LR");
if (TemporaryReg) {
assert(!PopReg && "Unnecessary MOV is about to be inserted");
PopReg = PopFriendly.find_first();
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
.addReg(TemporaryReg, RegState::Define)
.addReg(PopReg, RegState::Kill));
}
if (MBBI == MBB.end()) {
MachineInstr& Pop = MBB.back();
assert(Pop.getOpcode() == ARM::tPOP);
Pop.RemoveOperand(Pop.findRegisterDefOperandIdx(ARM::LR));
}
assert(PopReg && "Do not know how to get LR");
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP)))
.addReg(PopReg, RegState::Define);
emitSPUpdate(MBB, MBBI, TII, dl, *RegInfo, ArgRegsSaveSize);
if (!TemporaryReg && MBBI != MBB.end() && MBBI->getOpcode() == ARM::tBX_RET) {
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(ARM::tBX))
.addReg(PopReg, RegState::Kill);
AddDefaultPred(MIB);
MIB.copyImplicitOps(&*MBBI);
// erase the old tBX_RET instruction
MBB.erase(MBBI);
return true;
}
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
.addReg(ARM::LR, RegState::Define)
.addReg(PopReg, RegState::Kill));
if (TemporaryReg) {
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr))
.addReg(PopReg, RegState::Define)
.addReg(TemporaryReg, RegState::Kill));
}
return true;
}
bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MRI = &mf.getRegInfo();
TRI = MF->getTarget().getRegisterInfo();
ReservedRegisters = TRI->getReservedRegs(mf);
unsigned NumRegs = TRI->getNumRegs();
PhysRegDef = new MachineInstr*[NumRegs];
PhysRegUse = new MachineInstr*[NumRegs];
PHIVarInfo = new SmallVector<unsigned, 4>[MF->getNumBlockIDs()];
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
/// Get some space for a respectable number of registers.
VirtRegInfo.resize(64);
analyzePHINodes(mf);
// Calculate live variable information in depth first order on the CFG of the
// function. This guarantees that we will see the definition of a virtual
// register before its uses due to dominance properties of SSA (except for PHI
// nodes, which are treated as a special case).
MachineBasicBlock *Entry = MF->begin();
SmallPtrSet<MachineBasicBlock*,16> Visited;
for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> >
DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited);
DFI != E; ++DFI) {
MachineBasicBlock *MBB = *DFI;
// Mark live-in registers as live-in.
for (MachineBasicBlock::const_livein_iterator II = MBB->livein_begin(),
EE = MBB->livein_end(); II != EE; ++II) {
assert(TargetRegisterInfo::isPhysicalRegister(*II) &&
"Cannot have a live-in virtual register!");
HandlePhysRegDef(*II, 0);
}
// Loop over all of the instructions, processing them.
DistanceMap.clear();
unsigned Dist = 0;
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
I != E; ++I) {
MachineInstr *MI = I;
DistanceMap.insert(std::make_pair(MI, Dist++));
// Process all of the operands of the instruction...
unsigned NumOperandsToProcess = MI->getNumOperands();
// Unless it is a PHI node. In this case, ONLY process the DEF, not any
// of the uses. They will be handled in other basic blocks.
if (MI->getOpcode() == TargetInstrInfo::PHI)
NumOperandsToProcess = 1;
SmallVector<unsigned, 4> UseRegs;
SmallVector<unsigned, 4> DefRegs;
for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || MO.getReg() == 0)
continue;
unsigned MOReg = MO.getReg();
if (MO.isUse())
UseRegs.push_back(MOReg);
if (MO.isDef())
DefRegs.push_back(MOReg);
}
// Process all uses.
for (unsigned i = 0, e = UseRegs.size(); i != e; ++i) {
unsigned MOReg = UseRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegUse(MOReg, MBB, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegUse(MOReg, MI);
}
// Process all defs.
for (unsigned i = 0, e = DefRegs.size(); i != e; ++i) {
unsigned MOReg = DefRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegDef(MOReg, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegDef(MOReg, MI);
}
}
// Handle any virtual assignments from PHI nodes which might be at the
// bottom of this basic block. We check all of our successor blocks to see
// if they have PHI nodes, and if so, we simulate an assignment at the end
// of the current block.
if (!PHIVarInfo[MBB->getNumber()].empty()) {
SmallVector<unsigned, 4>& VarInfoVec = PHIVarInfo[MBB->getNumber()];
for (SmallVector<unsigned, 4>::iterator I = VarInfoVec.begin(),
E = VarInfoVec.end(); I != E; ++I)
// Mark it alive only in the block we are representing.
MarkVirtRegAliveInBlock(getVarInfo(*I),MRI->getVRegDef(*I)->getParent(),
MBB);
}
// Finally, if the last instruction in the block is a return, make sure to
// mark it as using all of the live-out values in the function.
if (!MBB->empty() && MBB->back().getDesc().isReturn()) {
MachineInstr *Ret = &MBB->back();
for (MachineRegisterInfo::liveout_iterator
I = MF->getRegInfo().liveout_begin(),
E = MF->getRegInfo().liveout_end(); I != E; ++I) {
assert(TargetRegisterInfo::isPhysicalRegister(*I) &&
"Cannot have a live-out virtual register!");
HandlePhysRegUse(*I, Ret);
// Add live-out registers as implicit uses.
if (!Ret->readsRegister(*I))
Ret->addOperand(MachineOperand::CreateReg(*I, false, true));
}
}
// Loop over PhysRegDef / PhysRegUse, killing any registers that are
// available at the end of the basic block.
for (unsigned i = 0; i != NumRegs; ++i)
if (PhysRegDef[i] || PhysRegUse[i])
HandlePhysRegDef(i, 0);
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
}
// Convert and transfer the dead / killed information we have gathered into
// VirtRegInfo onto MI's.
for (unsigned i = 0, e1 = VirtRegInfo.size(); i != e1; ++i)
for (unsigned j = 0, e2 = VirtRegInfo[i].Kills.size(); j != e2; ++j)
if (VirtRegInfo[i].Kills[j] ==
MRI->getVRegDef(i + TargetRegisterInfo::FirstVirtualRegister))
VirtRegInfo[i]
.Kills[j]->addRegisterDead(i +
TargetRegisterInfo::FirstVirtualRegister,
TRI);
else
VirtRegInfo[i]
.Kills[j]->addRegisterKilled(i +
TargetRegisterInfo::FirstVirtualRegister,
TRI);
// Check to make sure there are no unreachable blocks in the MC CFG for the
// function. If so, it is due to a bug in the instruction selector or some
// other part of the code generator if this happens.
#ifndef NDEBUG
for(MachineFunction::iterator i = MF->begin(), e = MF->end(); i != e; ++i)
assert(Visited.count(&*i) != 0 && "unreachable basic block found");
#endif
delete[] PhysRegDef;
delete[] PhysRegUse;
delete[] PHIVarInfo;
return false;
}
bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MRI = &mf.getRegInfo();
TRI = MF->getTarget().getRegisterInfo();
ReservedRegisters = TRI->getReservedRegs(mf);
unsigned NumRegs = TRI->getNumRegs();
PhysRegDef = new MachineInstr*[NumRegs];
PhysRegUse = new MachineInstr*[NumRegs];
PHIVarInfo = new SmallVector<unsigned, 4>[MF->getNumBlockIDs()];
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
PHIJoins.clear();
// FIXME: LiveIntervals will be updated to remove its dependence on
// LiveVariables to improve compilation time and eliminate bizarre pass
// dependencies. Until then, we can't change much in -O0.
if (!MRI->isSSA())
report_fatal_error("regalloc=... not currently supported with -O0");
analyzePHINodes(mf);
// Calculate live variable information in depth first order on the CFG of the
// function. This guarantees that we will see the definition of a virtual
// register before its uses due to dominance properties of SSA (except for PHI
// nodes, which are treated as a special case).
MachineBasicBlock *Entry = MF->begin();
SmallPtrSet<MachineBasicBlock*,16> Visited;
for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> >
DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited);
DFI != E; ++DFI) {
MachineBasicBlock *MBB = *DFI;
// Mark live-in registers as live-in.
SmallVector<unsigned, 4> Defs;
for (MachineBasicBlock::livein_iterator II = MBB->livein_begin(),
EE = MBB->livein_end(); II != EE; ++II) {
assert(TargetRegisterInfo::isPhysicalRegister(*II) &&
"Cannot have a live-in virtual register!");
HandlePhysRegDef(*II, 0, Defs);
}
// Loop over all of the instructions, processing them.
DistanceMap.clear();
unsigned Dist = 0;
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
I != E; ++I) {
MachineInstr *MI = I;
if (MI->isDebugValue())
continue;
DistanceMap.insert(std::make_pair(MI, Dist++));
// Process all of the operands of the instruction...
unsigned NumOperandsToProcess = MI->getNumOperands();
// Unless it is a PHI node. In this case, ONLY process the DEF, not any
// of the uses. They will be handled in other basic blocks.
if (MI->isPHI())
NumOperandsToProcess = 1;
// Clear kill and dead markers. LV will recompute them.
SmallVector<unsigned, 4> UseRegs;
SmallVector<unsigned, 4> DefRegs;
SmallVector<unsigned, 1> RegMasks;
for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isRegMask()) {
RegMasks.push_back(i);
continue;
}
if (!MO.isReg() || MO.getReg() == 0)
continue;
unsigned MOReg = MO.getReg();
if (MO.isUse()) {
MO.setIsKill(false);
UseRegs.push_back(MOReg);
} else /*MO.isDef()*/ {
MO.setIsDead(false);
DefRegs.push_back(MOReg);
}
}
// Process all uses.
for (unsigned i = 0, e = UseRegs.size(); i != e; ++i) {
unsigned MOReg = UseRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegUse(MOReg, MBB, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegUse(MOReg, MI);
}
// Process all masked registers. (Call clobbers).
for (unsigned i = 0, e = RegMasks.size(); i != e; ++i)
HandleRegMask(MI->getOperand(RegMasks[i]));
// Process all defs.
for (unsigned i = 0, e = DefRegs.size(); i != e; ++i) {
unsigned MOReg = DefRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegDef(MOReg, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegDef(MOReg, MI, Defs);
}
UpdatePhysRegDefs(MI, Defs);
}
// Handle any virtual assignments from PHI nodes which might be at the
// bottom of this basic block. We check all of our successor blocks to see
// if they have PHI nodes, and if so, we simulate an assignment at the end
// of the current block.
if (!PHIVarInfo[MBB->getNumber()].empty()) {
SmallVector<unsigned, 4>& VarInfoVec = PHIVarInfo[MBB->getNumber()];
for (SmallVector<unsigned, 4>::iterator I = VarInfoVec.begin(),
E = VarInfoVec.end(); I != E; ++I)
// Mark it alive only in the block we are representing.
MarkVirtRegAliveInBlock(getVarInfo(*I),MRI->getVRegDef(*I)->getParent(),
MBB);
}
// Finally, if the last instruction in the block is a return, make sure to
// mark it as using all of the live-out values in the function.
// Things marked both call and return are tail calls; do not do this for
// them. The tail callee need not take the same registers as input
// that it produces as output, and there are dependencies for its input
// registers elsewhere.
if (!MBB->empty() && MBB->back().isReturn()
&& !MBB->back().isCall()) {
MachineInstr *Ret = &MBB->back();
for (MachineRegisterInfo::liveout_iterator
I = MF->getRegInfo().liveout_begin(),
E = MF->getRegInfo().liveout_end(); I != E; ++I) {
assert(TargetRegisterInfo::isPhysicalRegister(*I) &&
"Cannot have a live-out virtual register!");
HandlePhysRegUse(*I, Ret);
// Add live-out registers as implicit uses.
if (!Ret->readsRegister(*I))
Ret->addOperand(MachineOperand::CreateReg(*I, false, true));
}
}
// MachineCSE may CSE instructions which write to non-allocatable physical
// registers across MBBs. Remember if any reserved register is liveout.
SmallSet<unsigned, 4> LiveOuts;
for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
SE = MBB->succ_end(); SI != SE; ++SI) {
MachineBasicBlock *SuccMBB = *SI;
if (SuccMBB->isLandingPad())
continue;
for (MachineBasicBlock::livein_iterator LI = SuccMBB->livein_begin(),
LE = SuccMBB->livein_end(); LI != LE; ++LI) {
unsigned LReg = *LI;
if (!TRI->isInAllocatableClass(LReg))
// Ignore other live-ins, e.g. those that are live into landing pads.
LiveOuts.insert(LReg);
}
}
// Loop over PhysRegDef / PhysRegUse, killing any registers that are
// available at the end of the basic block.
for (unsigned i = 0; i != NumRegs; ++i)
if ((PhysRegDef[i] || PhysRegUse[i]) && !LiveOuts.count(i))
HandlePhysRegDef(i, 0, Defs);
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
}
// Convert and transfer the dead / killed information we have gathered into
// VirtRegInfo onto MI's.
for (unsigned i = 0, e1 = VirtRegInfo.size(); i != e1; ++i) {
const unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
for (unsigned j = 0, e2 = VirtRegInfo[Reg].Kills.size(); j != e2; ++j)
if (VirtRegInfo[Reg].Kills[j] == MRI->getVRegDef(Reg))
VirtRegInfo[Reg].Kills[j]->addRegisterDead(Reg, TRI);
else
VirtRegInfo[Reg].Kills[j]->addRegisterKilled(Reg, TRI);
}
// Check to make sure there are no unreachable blocks in the MC CFG for the
// function. If so, it is due to a bug in the instruction selector or some
// other part of the code generator if this happens.
#ifndef NDEBUG
for(MachineFunction::iterator i = MF->begin(), e = MF->end(); i != e; ++i)
assert(Visited.count(&*i) != 0 && "unreachable basic block found");
#endif
delete[] PhysRegDef;
delete[] PhysRegUse;
delete[] PHIVarInfo;
return false;
}
/// Determine if it is profitable to duplicate this block.
bool TailDuplicator::shouldTailDuplicate(bool IsSimple,
MachineBasicBlock &TailBB) {
// When doing tail-duplication during layout, the block ordering is in flux,
// so canFallThrough returns a result based on incorrect information and
// should just be ignored.
if (!LayoutMode && TailBB.canFallThrough())
return false;
// Don't try to tail-duplicate single-block loops.
if (TailBB.isSuccessor(&TailBB))
return false;
// Set the limit on the cost to duplicate. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDupSize == 0 &&
TailDuplicateSize.getNumOccurrences() == 0 &&
MF->getFunction()->optForSize())
MaxDuplicateCount = 1;
else if (TailDupSize == 0)
MaxDuplicateCount = TailDuplicateSize;
else
MaxDuplicateCount = TailDupSize;
// If the block to be duplicated ends in an unanalyzable fallthrough, don't
// duplicate it.
// A similar check is necessary in MachineBlockPlacement to make sure pairs of
// blocks with unanalyzable fallthrough get layed out contiguously.
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (TII->analyzeBranch(TailBB, PredTBB, PredFBB, PredCond) &&
TailBB.canFallThrough())
return false;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
bool HasIndirectbr = false;
if (!TailBB.empty())
HasIndirectbr = TailBB.back().isIndirectBranch();
if (HasIndirectbr && PreRegAlloc)
MaxDuplicateCount = TailDupIndirectBranchSize;
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
for (MachineInstr &MI : TailBB) {
// Non-duplicable things shouldn't be tail-duplicated.
if (MI.isNotDuplicable())
return false;
// Convergent instructions can be duplicated only if doing so doesn't add
// new control dependencies, which is what we're going to do here.
if (MI.isConvergent())
return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && MI.isReturn())
return false;
// Avoid duplicating calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (PreRegAlloc && MI.isCall())
return false;
if (!MI.isPHI() && !MI.isDebugValue())
InstrCount += 1;
if (InstrCount > MaxDuplicateCount)
return false;
}
// Check if any of the successors of TailBB has a PHI node in which the
// value corresponding to TailBB uses a subregister.
// If a phi node uses a register paired with a subregister, the actual
// "value type" of the phi may differ from the type of the register without
// any subregisters. Due to a bug, tail duplication may add a new operand
// without a necessary subregister, producing an invalid code. This is
// demonstrated by test/CodeGen/Hexagon/tail-dup-subreg-abort.ll.
// Disable tail duplication for this case for now, until the problem is
// fixed.
for (auto SB : TailBB.successors()) {
for (auto &I : *SB) {
if (!I.isPHI())
break;
unsigned Idx = getPHISrcRegOpIdx(&I, &TailBB);
assert(Idx != 0);
MachineOperand &PU = I.getOperand(Idx);
if (PU.getSubReg() != 0)
return false;
}
}
if (HasIndirectbr && PreRegAlloc)
return true;
if (IsSimple)
return true;
if (!PreRegAlloc)
return true;
return canCompletelyDuplicateBB(TailBB);
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
MachineBasicBlock *DestBB = getDestBlock(MI);
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
// If the branch is at the end of its MBB and that has a fall-through block,
// direct the updated conditional branch to the fall-through block. Otherwise,
// split the MBB before the next instruction.
MachineBasicBlock *MBB = MI.getParent();
MachineInstr *BMI = &MBB->back();
bool NeedSplit = (BMI != &MI) || !hasFallthrough(*MBB);
if (BMI != &MI) {
if (std::next(MachineBasicBlock::iterator(MI)) ==
std::prev(MBB->getLastNonDebugInstr()) &&
BMI->isUnconditionalBranch()) {
// Last MI in the BB is an unconditional branch. We can simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
MachineBasicBlock *NewDest = getDestBlock(*BMI);
if (isBlockInRange(MI, *NewDest)) {
DEBUG(dbgs() << " Invert condition and swap its destination with "
<< *BMI);
changeBranchDestBlock(*BMI, *DestBB);
int NewSize =
insertInvertedConditionalBranch(*MBB, MI.getIterator(),
MI.getDebugLoc(), MI, *NewDest);
int OldSize = TII->getInstSizeInBytes(MI);
BlockInfo[MBB->getNumber()].Size += (NewSize - OldSize);
MI.eraseFromParent();
return true;
}
}
}
if (NeedSplit) {
// Analyze the branch so we know how to update the successor lists.
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
SmallVector<MachineOperand, 2> Cond;
bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond, false);
assert(!Fail && "branches to relax should be analyzable");
(void)Fail;
MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI);
// No need for the branch to the next block. We're adding an unconditional
// branch to the destination.
int delta = TII->getInstSizeInBytes(MBB->back());
BlockInfo[MBB->getNumber()].Size -= delta;
MBB->back().eraseFromParent();
// BlockInfo[SplitBB].Offset is wrong temporarily, fixed below
// Update the successor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
<< ", invert condition and change dest. to BB#"
<< NextBB.getNumber() << '\n');
unsigned &MBBSize = BlockInfo[MBB->getNumber()].Size;
// Insert a new conditional branch and a new unconditional branch.
MBBSize += insertInvertedConditionalBranch(*MBB, MBB->end(),
MI.getDebugLoc(), MI, NextBB);
MBBSize += insertUnconditionalBranch(*MBB, *DestBB, MI.getDebugLoc());
// Remove the old conditional branch. It may or may not still be in MBB.
MBBSize -= TII->getInstSizeInBytes(MI);
MI.eraseFromParent();
// Finally, keep the block offsets up to date.
adjustBlockOffsets(*MBB);
return true;
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
DebugLoc DL = MI.getDebugLoc();
MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
MachineBasicBlock *NewBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
auto insertUncondBranch = [&](MachineBasicBlock *MBB,
MachineBasicBlock *DestBB) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int NewBrSize = 0;
TII->insertUnconditionalBranch(*MBB, DestBB, DL, &NewBrSize);
BBSize += NewBrSize;
};
auto insertBranch = [&](MachineBasicBlock *MBB, MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
SmallVectorImpl<MachineOperand>& Cond) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int NewBrSize = 0;
TII->insertBranch(*MBB, TBB, FBB, Cond, DL, &NewBrSize);
BBSize += NewBrSize;
};
auto removeBranch = [&](MachineBasicBlock *MBB) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int RemovedSize = 0;
TII->removeBranch(*MBB, &RemovedSize);
BBSize -= RemovedSize;
};
auto finalizeBlockChanges = [&](MachineBasicBlock *MBB,
MachineBasicBlock *NewBB) {
// Keep the block offsets up to date.
adjustBlockOffsets(*MBB);
// Need to fix live-in lists if we track liveness.
if (NewBB && TRI->trackLivenessAfterRegAlloc(*MF))
computeAndAddLiveIns(LiveRegs, *NewBB);
};
bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
assert(!Fail && "branches to be relaxed must be analyzable");
(void)Fail;
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
bool ReversedCond = !TII->reverseBranchCondition(Cond);
if (ReversedCond) {
if (FBB && isBlockInRange(MI, *FBB)) {
// Last MI in the BB is an unconditional branch. We can simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
LLVM_DEBUG(dbgs() << " Invert condition and swap "
"its destination with "
<< MBB->back());
removeBranch(MBB);
insertBranch(MBB, FBB, TBB, Cond);
finalizeBlockChanges(MBB, nullptr);
return true;
}
if (FBB) {
// We need to split the basic block here to obtain two long-range
// unconditional branches.
NewBB = createNewBlockAfter(*MBB);
insertUncondBranch(NewBB, FBB);
// Update the succesor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
// We now have an appropriate fall-through block in place (either naturally or
// just created), so we can use the inverted the condition.
MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
LLVM_DEBUG(dbgs() << " Insert B to " << printMBBReference(*TBB)
<< ", invert condition and change dest. to "
<< printMBBReference(NextBB) << '\n');
removeBranch(MBB);
// Insert a new conditional branch and a new unconditional branch.
insertBranch(MBB, &NextBB, TBB, Cond);
finalizeBlockChanges(MBB, NewBB);
return true;
}
// Branch cond can't be inverted.
// In this case we always add a block after the MBB.
LLVM_DEBUG(dbgs() << " The branch condition can't be inverted. "
<< " Insert a new BB after " << MBB->back());
if (!FBB)
FBB = &(*std::next(MachineFunction::iterator(MBB)));
// This is the block with cond. branch and the distance to TBB is too long.
// beq L1
// L2:
// We do the following transformation:
// beq NewBB
// b L2
// NewBB:
// b L1
// L2:
NewBB = createNewBlockAfter(*MBB);
insertUncondBranch(NewBB, TBB);
LLVM_DEBUG(dbgs() << " Insert cond B to the new BB "
<< printMBBReference(*NewBB)
<< " Keep the exiting condition.\n"
<< " Insert B to " << printMBBReference(*FBB) << ".\n"
<< " In the new BB: Insert B to "
<< printMBBReference(*TBB) << ".\n");
// Update the successor lists according to the transformation to follow.
MBB->replaceSuccessor(TBB, NewBB);
NewBB->addSuccessor(TBB);
// Replace branch in the current (MBB) block.
removeBranch(MBB);
insertBranch(MBB, NewBB, FBB, Cond);
finalizeBlockChanges(MBB, NewBB);
return true;
}
/// shouldTailDuplicate - Determine if it is profitable to duplicate this block.
bool
TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF,
MachineBasicBlock &TailBB) {
// Only duplicate blocks that end with unconditional branches.
if (TailBB.canFallThrough())
return false;
// Don't try to tail-duplicate single-block loops.
if (TailBB.isSuccessor(&TailBB))
return false;
// Set the limit on the cost to duplicate. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDuplicateSize.getNumOccurrences() == 0 &&
MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
MaxDuplicateCount = 1;
else
MaxDuplicateCount = TailDuplicateSize;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
if (PreRegAlloc && !TailBB.empty()) {
const TargetInstrDesc &TID = TailBB.back().getDesc();
if (TID.isIndirectBranch())
MaxDuplicateCount = 20;
}
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
for (MachineBasicBlock::const_iterator I = TailBB.begin(); I != TailBB.end();
++I) {
// Non-duplicable things shouldn't be tail-duplicated.
if (I->getDesc().isNotDuplicable())
return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && I->getDesc().isReturn())
return false;
// Avoid duplicating calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (PreRegAlloc && I->getDesc().isCall())
return false;
if (!I->isPHI() && !I->isDebugValue())
InstrCount += 1;
if (InstrCount > MaxDuplicateCount)
return false;
}
return true;
}
bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) {
bool AnyChanges = false;
MRI = &MF.getRegInfo();
TRI = MF.getTarget().getRegisterInfo();
TII = MF.getTarget().getInstrInfo();
// Treat reserved registers as always live.
BitVector ReservedRegs = TRI->getReservedRegs(MF);
// Loop over all instructions in all blocks, from bottom to top, so that it's
// more likely that chains of dependent but ultimately dead instructions will
// be cleaned up.
for (MachineFunction::reverse_iterator I = MF.rbegin(), E = MF.rend();
I != E; ++I) {
MachineBasicBlock *MBB = &*I;
// Start out assuming that reserved registers are live out of this block.
LivePhysRegs = ReservedRegs;
// Also add any explicit live-out physregs for this block.
if (!MBB->empty() && MBB->back().getDesc().isReturn())
for (MachineRegisterInfo::liveout_iterator LOI = MRI->liveout_begin(),
LOE = MRI->liveout_end(); LOI != LOE; ++LOI) {
unsigned Reg = *LOI;
if (TargetRegisterInfo::isPhysicalRegister(Reg))
LivePhysRegs.set(Reg);
}
// FIXME: Add live-ins from sucessors to LivePhysRegs. Normally, physregs
// are not live across blocks, but some targets (x86) can have flags live
// out of a block.
// Now scan the instructions and delete dead ones, tracking physreg
// liveness as we go.
for (MachineBasicBlock::reverse_iterator MII = MBB->rbegin(),
MIE = MBB->rend(); MII != MIE; ) {
MachineInstr *MI = &*MII;
// If the instruction is dead, delete it!
if (isDead(MI)) {
DEBUG(dbgs() << "DeadMachineInstructionElim: DELETING: " << *MI);
// It is possible that some DBG_VALUE instructions refer to this
// instruction. Examine each def operand for such references;
// if found, mark the DBG_VALUE as undef (but don't delete it).
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || !MO.isDef())
continue;
unsigned Reg = MO.getReg();
if (!TargetRegisterInfo::isVirtualRegister(Reg))
continue;
MachineRegisterInfo::use_iterator nextI;
for (MachineRegisterInfo::use_iterator I = MRI->use_begin(Reg),
E = MRI->use_end(); I!=E; I=nextI) {
nextI = llvm::next(I); // I is invalidated by the setReg
MachineOperand& Use = I.getOperand();
MachineInstr *UseMI = Use.getParent();
if (UseMI==MI)
continue;
assert(Use.isDebug());
UseMI->getOperand(0).setReg(0U);
}
}
AnyChanges = true;
MI->eraseFromParent();
++NumDeletes;
MIE = MBB->rend();
// MII is now pointing to the next instruction to process,
// so don't increment it.
continue;
}
// Record the physreg defs.
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (MO.isReg() && MO.isDef()) {
unsigned Reg = MO.getReg();
if (Reg != 0 && TargetRegisterInfo::isPhysicalRegister(Reg)) {
LivePhysRegs.reset(Reg);
// Check the subreg set, not the alias set, because a def
// of a super-register may still be partially live after
// this def.
for (const unsigned *SubRegs = TRI->getSubRegisters(Reg);
*SubRegs; ++SubRegs)
LivePhysRegs.reset(*SubRegs);
}
}
}
// Record the physreg uses, after the defs, in case a physreg is
// both defined and used in the same instruction.
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (MO.isReg() && MO.isUse()) {
unsigned Reg = MO.getReg();
if (Reg != 0 && TargetRegisterInfo::isPhysicalRegister(Reg)) {
LivePhysRegs.set(Reg);
for (const unsigned *AliasSet = TRI->getAliasSet(Reg);
*AliasSet; ++AliasSet)
LivePhysRegs.set(*AliasSet);
}
}
}
// We didn't delete the current instruction, so increment MII to
// the next one.
++MII;
}
}
LivePhysRegs.clear();
return AnyChanges;
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr *MI) {
MachineBasicBlock *DestBB = getDestBlock(MI);
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
// If the branch is at the end of its MBB and that has a fall-through block,
// direct the updated conditional branch to the fall-through block. Otherwise,
// split the MBB before the next instruction.
MachineBasicBlock *MBB = MI->getParent();
MachineInstr *BMI = &MBB->back();
bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB);
if (BMI != MI) {
if (std::next(MachineBasicBlock::iterator(MI)) ==
std::prev(MBB->getLastNonDebugInstr()) &&
BMI->getOpcode() == AArch64::B) {
// Last MI in the BB is an unconditional branch. Can we simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB();
if (isBlockInRange(MI, NewDest,
getBranchDisplacementBits(MI->getOpcode()))) {
DEBUG(dbgs() << " Invert condition and swap its destination with "
<< *BMI);
BMI->getOperand(0).setMBB(DestBB);
unsigned OpNum = (MI->getOpcode() == AArch64::TBZW ||
MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX ||
MI->getOpcode() == AArch64::TBNZX)
? 2
: 1;
MI->getOperand(OpNum).setMBB(NewDest);
MI->setDesc(TII->get(getOppositeConditionOpcode(MI->getOpcode())));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MI);
return true;
}
}
}
if (NeedSplit) {
// Analyze the branch so we know how to update the successor lists.
MachineBasicBlock *TBB, *FBB;
SmallVector<MachineOperand, 2> Cond;
TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, false);
MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI);
// No need for the branch to the next block. We're adding an unconditional
// branch to the destination.
int delta = TII->GetInstSizeInBytes(&MBB->back());
BlockInfo[MBB->getNumber()].Size -= delta;
MBB->back().eraseFromParent();
// BlockInfo[SplitBB].Offset is wrong temporarily, fixed below
// Update the successor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
<< ", invert condition and change dest. to BB#"
<< NextBB->getNumber() << "\n");
// Insert a new conditional branch and a new unconditional branch.
MachineInstrBuilder MIB = BuildMI(
MBB, DebugLoc(), TII->get(getOppositeConditionOpcode(MI->getOpcode())))
.addOperand(MI->getOperand(0));
if (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX)
MIB.addOperand(MI->getOperand(1));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MIB);
MIB.addMBB(NextBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
// Remove the old conditional branch. It may or may not still be in MBB.
BlockInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
MI->eraseFromParent();
// Finally, keep the block offsets up to date.
adjustBlockOffsets(*MBB);
return true;
}
bool
SPUInstrInfo::BlockHasNoFallThrough(const MachineBasicBlock &MBB) const {
return (!MBB.empty() && isUncondBranch(&MBB.back()));
}
