/// Compute the set of return blocks
void PEI::calculateSets(MachineFunction &Fn) {
const MachineFrameInfo *MFI = Fn.getFrameInfo();
// Even when we do not change any CSR, we still want to insert the
// prologue and epilogue of the function.
// So set the save points for those.
// Use the points found by shrink-wrapping, if any.
if (MFI->getSavePoint()) {
SaveBlock = MFI->getSavePoint();
assert(MFI->getRestorePoint() && "Both restore and save must be set");
MachineBasicBlock *RestoreBlock = MFI->getRestorePoint();
// If RestoreBlock does not have any successor and is not a return block
// then the end point is unreachable and we do not need to insert any
// epilogue.
if (!RestoreBlock->succ_empty() || isReturnBlock(RestoreBlock))
RestoreBlocks.push_back(RestoreBlock);
return;
}
// Save refs to entry and return blocks.
SaveBlock = Fn.begin();
for (MachineFunction::iterator MBB = Fn.begin(), E = Fn.end();
MBB != E; ++MBB)
if (isReturnBlock(MBB))
RestoreBlocks.push_back(MBB);
return;
}
/// Compute the sets of entry and return blocks for saving and restoring
/// callee-saved registers, and placing prolog and epilog code.
void PEI::calculateSaveRestoreBlocks(MachineFunction &MF) {
const MachineFrameInfo &MFI = MF.getFrameInfo();
// Even when we do not change any CSR, we still want to insert the
// prologue and epilogue of the function.
// So set the save points for those.
// Use the points found by shrink-wrapping, if any.
if (MFI.getSavePoint()) {
SaveBlocks.push_back(MFI.getSavePoint());
assert(MFI.getRestorePoint() && "Both restore and save must be set");
MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
// If RestoreBlock does not have any successor and is not a return block
// then the end point is unreachable and we do not need to insert any
// epilogue.
if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
RestoreBlocks.push_back(RestoreBlock);
return;
}
// Save refs to entry and return blocks.
SaveBlocks.push_back(&MF.front());
for (MachineBasicBlock &MBB : MF) {
if (MBB.isEHFuncletEntry())
SaveBlocks.push_back(&MBB);
if (MBB.isReturnBlock())
RestoreBlocks.push_back(&MBB);
}
}
bool Thumb1FrameLowering::
restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
const std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
MachineFunction &MF = *MBB.getParent();
ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
const TargetInstrInfo &TII = *STI.getInstrInfo();
bool isVarArg = AFI->getArgRegsSaveSize() > 0;
DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
MachineInstrBuilder MIB = BuildMI(MF, DL, TII.get(ARM::tPOP));
AddDefaultPred(MIB);
bool NeedsPop = false;
for (unsigned i = CSI.size(); i != 0; --i) {
unsigned Reg = CSI[i-1].getReg();
if (Reg == ARM::LR) {
if (MBB.succ_empty()) {
// Special epilogue for vararg functions. See emitEpilogue
if (isVarArg)
continue;
// ARMv4T requires BX, see emitEpilogue
if (!STI.hasV5TOps())
continue;
Reg = ARM::PC;
(*MIB).setDesc(TII.get(ARM::tPOP_RET));
if (MI != MBB.end())
MIB.copyImplicitOps(*MI);
MI = MBB.erase(MI);
} else
// LR may only be popped into PC, as part of return sequence.
// If this isn't the return sequence, we'll need emitPopSpecialFixUp
// to restore LR the hard way.
continue;
}
MIB.addReg(Reg, getDefRegState(true));
NeedsPop = true;
}
// It's illegal to emit pop instruction without operands.
if (NeedsPop)
MBB.insert(MI, &*MIB);
else
MF.DeleteMachineInstr(MIB);
return true;
}
bool SIInsertSkips::shouldSkip(const MachineBasicBlock &From,
const MachineBasicBlock &To) const {
if (From.succ_empty())
return false;
unsigned NumInstr = 0;
const MachineFunction *MF = From.getParent();
for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
MBBI != End && MBBI != ToI; ++MBBI) {
const MachineBasicBlock &MBB = *MBBI;
for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end();
NumInstr < SkipThreshold && I != E; ++I) {
if (opcodeEmitsNoInsts(I->getOpcode()))
continue;
// FIXME: Since this is required for correctness, this should be inserted
// during SILowerControlFlow.
// When a uniform loop is inside non-uniform control flow, the branch
// leaving the loop might be an S_CBRANCH_VCCNZ, which is never taken
// when EXEC = 0. We should skip the loop lest it becomes infinite.
if (I->getOpcode() == AMDGPU::S_CBRANCH_VCCNZ ||
I->getOpcode() == AMDGPU::S_CBRANCH_VCCZ)
return true;
if (I->isInlineAsm()) {
const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
const char *AsmStr = I->getOperand(0).getSymbolName();
// inlineasm length estimate is number of bytes assuming the longest
// instruction.
uint64_t MaxAsmSize = TII->getInlineAsmLength(AsmStr, *MAI);
NumInstr += MaxAsmSize / MAI->getMaxInstLength();
} else {
++NumInstr;
}
if (NumInstr >= SkipThreshold)
return true;
}
}
return false;
}
/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
/// after it, replacing it with an unconditional branch to NewDest.
void
TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
MachineBasicBlock *NewDest) const {
MachineBasicBlock *MBB = Tail->getParent();
// Remove all the old successors of MBB from the CFG.
while (!MBB->succ_empty())
MBB->removeSuccessor(MBB->succ_begin());
// Remove all the dead instructions from the end of MBB.
MBB->erase(Tail, MBB->end());
// If MBB isn't immediately before MBB, insert a branch to it.
if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
InsertBranch(*MBB, NewDest, 0, SmallVector<MachineOperand, 0>(),
Tail->getDebugLoc());
MBB->addSuccessor(NewDest);
}
/// Compute the sets of entry and return blocks for saving and restoring
/// callee-saved registers, and placing prolog and epilog code.
void PEI::calculateSaveRestoreBlocks(MachineFunction &Fn) {
MachineFrameInfo &MFI = Fn.getFrameInfo();
const TargetFrameLowering *TFI = Fn.getSubtarget().getFrameLowering();
// Even when we do not change any CSR, we still want to insert the
// prologue and epilogue of the function.
// So set the save points for those.
// Use the points found by shrink-wrapping, if any.
if (MFI.getSavePoint()) {
SaveBlocks.push_back(MFI.getSavePoint());
assert(MFI.getRestorePoint() && "Both restore and save must be set");
MachineBasicBlock *RestoreBlock = MFI.getRestorePoint();
// If RestoreBlock does not have any successor and is not a return block
// then the end point is unreachable and we do not need to insert any
// epilogue.
if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
RestoreBlocks.push_back(RestoreBlock);
// If we are adding return protectors ensure we can find a free register
if (MFI.hasReturnProtector() &&
!TFI->determineReturnProtectorTempRegister(Fn, SaveBlocks, RestoreBlocks)) {
// Shrinkwrapping will prevent finding a free register
SaveBlocks.clear();
RestoreBlocks.clear();
MFI.setSavePoint(nullptr);
MFI.setRestorePoint(nullptr);
} else {
return;
}
}
// Save refs to entry and return blocks.
SaveBlocks.push_back(&Fn.front());
for (MachineBasicBlock &MBB : Fn) {
if (MBB.isEHFuncletEntry())
SaveBlocks.push_back(&MBB);
if (MBB.isReturnBlock())
RestoreBlocks.push_back(&MBB);
}
if (MFI.hasReturnProtector())
TFI->determineReturnProtectorTempRegister(Fn, SaveBlocks, RestoreBlocks);
}
bool SILowerControlFlowPass::shouldSkip(MachineBasicBlock *From,
MachineBasicBlock *To) {
unsigned NumInstr = 0;
for (MachineBasicBlock *MBB = From; MBB != To && !MBB->succ_empty();
MBB = *MBB->succ_begin()) {
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
NumInstr < SkipThreshold && I != E; ++I) {
if (I->isBundle() || !I->isBundled())
if (++NumInstr >= SkipThreshold)
return true;
}
}
return false;
}
bool SIInsertSkips::shouldSkip(const MachineBasicBlock &From,
const MachineBasicBlock &To) const {
if (From.succ_empty())
return false;
unsigned NumInstr = 0;
const MachineFunction *MF = From.getParent();
for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end();
MBBI != End && MBBI != ToI; ++MBBI) {
const MachineBasicBlock &MBB = *MBBI;
for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end();
NumInstr < SkipThreshold && I != E; ++I) {
if (opcodeEmitsNoInsts(I->getOpcode()))
continue;
// FIXME: Since this is required for correctness, this should be inserted
// during SILowerControlFlow.
// When a uniform loop is inside non-uniform control flow, the branch
// leaving the loop might be an S_CBRANCH_VCCNZ, which is never taken
// when EXEC = 0. We should skip the loop lest it becomes infinite.
if (I->getOpcode() == AMDGPU::S_CBRANCH_VCCNZ ||
I->getOpcode() == AMDGPU::S_CBRANCH_VCCZ)
return true;
if (TII->hasUnwantedEffectsWhenEXECEmpty(*I))
return true;
++NumInstr;
if (NumInstr >= SkipThreshold)
return true;
}
}
return false;
}
void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB,
RegScavenger *RS) {
// Get rid of the easy cases first.
if (!Save)
Save = &MBB;
else
Save = MDT->findNearestCommonDominator(Save, &MBB);
if (!Save) {
LLVM_DEBUG(dbgs() << "Found a block that is not reachable from Entry\n");
return;
}
if (!Restore)
Restore = &MBB;
else if (MPDT->getNode(&MBB)) // If the block is not in the post dom tree, it
// means the block never returns. If that's the
// case, we don't want to call
// `findNearestCommonDominator`, which will
// return `Restore`.
Restore = MPDT->findNearestCommonDominator(Restore, &MBB);
else
Restore = nullptr; // Abort, we can't find a restore point in this case.
// Make sure we would be able to insert the restore code before the
// terminator.
if (Restore == &MBB) {
for (const MachineInstr &Terminator : MBB.terminators()) {
if (!useOrDefCSROrFI(Terminator, RS))
continue;
// One of the terminator needs to happen before the restore point.
if (MBB.succ_empty()) {
Restore = nullptr; // Abort, we can't find a restore point in this case.
break;
}
// Look for a restore point that post-dominates all the successors.
// The immediate post-dominator is what we are looking for.
Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
break;
}
}
if (!Restore) {
LLVM_DEBUG(
dbgs() << "Restore point needs to be spanned on several blocks\n");
return;
}
// Make sure Save and Restore are suitable for shrink-wrapping:
// 1. all path from Save needs to lead to Restore before exiting.
// 2. all path to Restore needs to go through Save from Entry.
// We achieve that by making sure that:
// A. Save dominates Restore.
// B. Restore post-dominates Save.
// C. Save and Restore are in the same loop.
bool SaveDominatesRestore = false;
bool RestorePostDominatesSave = false;
while (Save && Restore &&
(!(SaveDominatesRestore = MDT->dominates(Save, Restore)) ||
!(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) ||
// Post-dominance is not enough in loops to ensure that all uses/defs
// are after the prologue and before the epilogue at runtime.
// E.g.,
// while(1) {
// Save
// Restore
// if (...)
// break;
// use/def CSRs
// }
// All the uses/defs of CSRs are dominated by Save and post-dominated
// by Restore. However, the CSRs uses are still reachable after
// Restore and before Save are executed.
//
// For now, just push the restore/save points outside of loops.
// FIXME: Refine the criteria to still find interesting cases
// for loops.
MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
// Fix (A).
if (!SaveDominatesRestore) {
Save = MDT->findNearestCommonDominator(Save, Restore);
continue;
}
// Fix (B).
if (!RestorePostDominatesSave)
Restore = MPDT->findNearestCommonDominator(Restore, Save);
// Fix (C).
if (Save && Restore &&
(MLI->getLoopFor(Save) || MLI->getLoopFor(Restore))) {
if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore)) {
// Push Save outside of this loop if immediate dominator is different
// from save block. If immediate dominator is not different, bail out.
Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
if (!Save)
break;
} else {
// If the loop does not exit, there is no point in looking
// for a post-dominator outside the loop.
SmallVector<MachineBasicBlock*, 4> ExitBlocks;
MLI->getLoopFor(Restore)->getExitingBlocks(ExitBlocks);
// Push Restore outside of this loop.
// Look for the immediate post-dominator of the loop exits.
MachineBasicBlock *IPdom = Restore;
for (MachineBasicBlock *LoopExitBB: ExitBlocks) {
IPdom = FindIDom<>(*IPdom, LoopExitBB->successors(), *MPDT);
if (!IPdom)
break;
}
// If the immediate post-dominator is not in a less nested loop,
// then we are stuck in a program with an infinite loop.
// In that case, we will not find a safe point, hence, bail out.
if (IPdom && MLI->getLoopDepth(IPdom) < MLI->getLoopDepth(Restore))
Restore = IPdom;
else {
Restore = nullptr;
break;
}
}
}
}
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
if (!shouldTailDuplicate(MF, *TailBB))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() > 1)
continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
// Simplify
TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true);
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (PrevBB->succ_size() == 1 &&
!TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) &&
PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end())
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
}
return Changed;
}
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion
DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map
DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map
SourceMap SrcMap; // Src -> Def map
bool Changed = false;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) {
MachineInstr *MI = &*I;
++I;
if (MI->isCopy()) {
unsigned Def = MI->getOperand(0).getReg();
unsigned Src = MI->getOperand(1).getReg();
if (TargetRegisterInfo::isVirtualRegister(Def) ||
TargetRegisterInfo::isVirtualRegister(Src))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src);
if (CI != AvailCopyMap.end()) {
MachineInstr *CopyMI = CI->second;
if (!MRI->isReserved(Def) &&
(!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
isNopCopy(CopyMI, Def, Src, TRI)) {
// The two copies cancel out and the source of the first copy
// hasn't been overridden, eliminate the second one. e.g.
// %ECX<def> = COPY %EAX<kill>
// ... nothing clobbered EAX.
// %EAX<def> = COPY %ECX
// =>
// %ECX<def> = COPY %EAX
//
// Also avoid eliminating a copy from reserved registers unless the
// definition is proven not clobbered. e.g.
// %RSP<def> = COPY %RAX
// CALL
// %RAX<def> = COPY %RSP
// Clear any kills of Def between CopyMI and MI. This extends the
// live range.
for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I)
I->clearRegisterKills(Def, TRI);
removeCopy(MI);
Changed = true;
++NumDeletes;
continue;
}
}
// If Src is defined by a previous copy, it cannot be eliminated.
for (MCRegAliasIterator AI(Src, TRI, true); AI.isValid(); ++AI) {
CI = CopyMap.find(*AI);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
// Copy is now a candidate for deletion.
MaybeDeadCopies.insert(MI);
// If 'Src' is previously source of another copy, then this earlier copy's
// source is no longer available. e.g.
// %xmm9<def> = copy %xmm2
// ...
// %xmm2<def> = copy %xmm0
// ...
// %xmm2<def> = copy %xmm9
SourceNoLongerAvailable(Def, SrcMap, AvailCopyMap);
// Remember Def is defined by the copy.
// ... Make sure to clear the def maps of aliases first.
for (MCRegAliasIterator AI(Def, TRI, false); AI.isValid(); ++AI) {
CopyMap.erase(*AI);
AvailCopyMap.erase(*AI);
}
CopyMap[Def] = MI;
AvailCopyMap[Def] = MI;
for (MCSubRegIterator SR(Def, TRI); SR.isValid(); ++SR) {
CopyMap[*SR] = MI;
AvailCopyMap[*SR] = MI;
}
// Remember source that's copied to Def. Once it's clobbered, then
// it's no longer available for copy propagation.
if (std::find(SrcMap[Src].begin(), SrcMap[Src].end(), Def) ==
SrcMap[Src].end()) {
SrcMap[Src].push_back(Def);
}
continue;
}
// Not a copy.
SmallVector<unsigned, 2> Defs;
int RegMaskOpNum = -1;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isRegMask())
RegMaskOpNum = i;
if (!MO.isReg())
continue;
unsigned Reg = MO.getReg();
if (!Reg)
continue;
if (TargetRegisterInfo::isVirtualRegister(Reg))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
if (MO.isDef()) {
Defs.push_back(Reg);
continue;
}
// If 'Reg' is defined by a copy, the copy is no longer a candidate
// for elimination.
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
DenseMap<unsigned, MachineInstr*>::iterator CI = CopyMap.find(*AI);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
}
// The instruction has a register mask operand which means that it clobbers
// a large set of registers. It is possible to use the register mask to
// prune the available copies, but treat it like a basic block boundary for
// now.
if (RegMaskOpNum >= 0) {
// Erase any MaybeDeadCopies whose destination register is clobbered.
const MachineOperand &MaskMO = MI->getOperand(RegMaskOpNum);
for (SmallSetVector<MachineInstr*, 8>::iterator
DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
DI != DE; ++DI) {
unsigned Reg = (*DI)->getOperand(0).getReg();
if (MRI->isReserved(Reg) || !MaskMO.clobbersPhysReg(Reg))
continue;
removeCopy(*DI);
Changed = true;
++NumDeletes;
}
// Clear all data structures as if we were beginning a new basic block.
MaybeDeadCopies.clear();
AvailCopyMap.clear();
CopyMap.clear();
SrcMap.clear();
continue;
}
for (unsigned i = 0, e = Defs.size(); i != e; ++i) {
unsigned Reg = Defs[i];
// No longer defined by a copy.
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
CopyMap.erase(*AI);
AvailCopyMap.erase(*AI);
}
// If 'Reg' is previously source of a copy, it is no longer available for
// copy propagation.
SourceNoLongerAvailable(Reg, SrcMap, AvailCopyMap);
}
}
// If MBB doesn't have successors, delete the copies whose defs are not used.
// If MBB does have successors, then conservative assume the defs are live-out
// since we don't want to trust live-in lists.
if (MBB.succ_empty()) {
for (SmallSetVector<MachineInstr*, 8>::iterator
DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
DI != DE; ++DI) {
if (!MRI->isReserved((*DI)->getOperand(0).getReg())) {
removeCopy(*DI);
Changed = true;
++NumDeletes;
}
}
}
return Changed;
}
bool Thumb1FrameLowering::
restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MI,
std::vector<CalleeSavedInfo> &CSI,
const TargetRegisterInfo *TRI) const {
if (CSI.empty())
return false;
MachineFunction &MF = *MBB.getParent();
ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
const TargetInstrInfo &TII = *STI.getInstrInfo();
const ARMBaseRegisterInfo *RegInfo = static_cast<const ARMBaseRegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
bool isVarArg = AFI->getArgRegsSaveSize() > 0;
DebugLoc DL = MI != MBB.end() ? MI->getDebugLoc() : DebugLoc();
ARMRegSet LoRegsToRestore;
ARMRegSet HiRegsToRestore;
// Low registers (r0-r7) which can be used to restore the high registers.
ARMRegSet CopyRegs;
for (CalleeSavedInfo I : CSI) {
unsigned Reg = I.getReg();
if (ARM::tGPRRegClass.contains(Reg) || Reg == ARM::LR) {
LoRegsToRestore[Reg] = true;
} else if (ARM::hGPRRegClass.contains(Reg) && Reg != ARM::LR) {
HiRegsToRestore[Reg] = true;
} else {
llvm_unreachable("callee-saved register of unexpected class");
}
// If this is a low register not used as the frame pointer, we may want to
// use it for restoring the high registers.
if ((ARM::tGPRRegClass.contains(Reg)) &&
!(hasFP(MF) && Reg == RegInfo->getFrameRegister(MF)))
CopyRegs[Reg] = true;
}
// If this is a return block, we may be able to use some unused return value
// registers for restoring the high regs.
auto Terminator = MBB.getFirstTerminator();
if (Terminator != MBB.end() && Terminator->getOpcode() == ARM::tBX_RET) {
CopyRegs[ARM::R0] = true;
CopyRegs[ARM::R1] = true;
CopyRegs[ARM::R2] = true;
CopyRegs[ARM::R3] = true;
for (auto Op : Terminator->implicit_operands()) {
if (Op.isReg())
CopyRegs[Op.getReg()] = false;
}
}
static const unsigned AllCopyRegs[] = {ARM::R0, ARM::R1, ARM::R2, ARM::R3,
ARM::R4, ARM::R5, ARM::R6, ARM::R7};
static const unsigned AllHighRegs[] = {ARM::R8, ARM::R9, ARM::R10, ARM::R11};
const unsigned *AllCopyRegsEnd = std::end(AllCopyRegs);
const unsigned *AllHighRegsEnd = std::end(AllHighRegs);
// Find the first register to restore.
auto HiRegToRestore = findNextOrderedReg(std::begin(AllHighRegs),
HiRegsToRestore, AllHighRegsEnd);
while (HiRegToRestore != AllHighRegsEnd) {
assert(!CopyRegs.none());
// Find the first low register to use.
auto CopyReg =
findNextOrderedReg(std::begin(AllCopyRegs), CopyRegs, AllCopyRegsEnd);
// Create the POP instruction.
MachineInstrBuilder PopMIB =
BuildMI(MBB, MI, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));
while (HiRegToRestore != AllHighRegsEnd && CopyReg != AllCopyRegsEnd) {
// Add the low register to the POP.
PopMIB.addReg(*CopyReg, RegState::Define);
// Create the MOV from low to high register.
BuildMI(MBB, MI, DL, TII.get(ARM::tMOVr))
.addReg(*HiRegToRestore, RegState::Define)
.addReg(*CopyReg, RegState::Kill)
.add(predOps(ARMCC::AL));
CopyReg = findNextOrderedReg(++CopyReg, CopyRegs, AllCopyRegsEnd);
HiRegToRestore =
findNextOrderedReg(++HiRegToRestore, HiRegsToRestore, AllHighRegsEnd);
}
}
MachineInstrBuilder MIB =
BuildMI(MF, DL, TII.get(ARM::tPOP)).add(predOps(ARMCC::AL));
bool NeedsPop = false;
for (unsigned i = CSI.size(); i != 0; --i) {
CalleeSavedInfo &Info = CSI[i-1];
unsigned Reg = Info.getReg();
// High registers (excluding lr) have already been dealt with
if (!(ARM::tGPRRegClass.contains(Reg) || Reg == ARM::LR))
continue;
if (Reg == ARM::LR) {
Info.setRestored(false);
if (!MBB.succ_empty() ||
MI->getOpcode() == ARM::TCRETURNdi ||
MI->getOpcode() == ARM::TCRETURNri)
// LR may only be popped into PC, as part of return sequence.
// If this isn't the return sequence, we'll need emitPopSpecialFixUp
// to restore LR the hard way.
// FIXME: if we don't pass any stack arguments it would be actually
// advantageous *and* correct to do the conversion to an ordinary call
// instruction here.
continue;
// Special epilogue for vararg functions. See emitEpilogue
if (isVarArg)
continue;
// ARMv4T requires BX, see emitEpilogue
if (!STI.hasV5TOps())
continue;
// Pop LR into PC.
Reg = ARM::PC;
(*MIB).setDesc(TII.get(ARM::tPOP_RET));
if (MI != MBB.end())
MIB.copyImplicitOps(*MI);
MI = MBB.erase(MI);
}
MIB.addReg(Reg, getDefRegState(true));
NeedsPop = true;
}
// It's illegal to emit pop instruction without operands.
if (NeedsPop)
MBB.insert(MI, &*MIB);
else
MF.DeleteMachineInstr(MIB);
return true;
}
/// If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
/// \p IsSimple result of isSimpleBB
/// \p TailBB Block to be duplicated.
/// \p ForcedLayoutPred When non-null, use this block as the layout predecessor
/// instead of the previous block in MF's order.
/// \p TDBBs A vector to keep track of all blocks tail-duplicated
/// into.
/// \p Copies A vector of copy instructions inserted. Used later to
/// walk all the inserted copies and remove redundant ones.
bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
MachineBasicBlock *ForcedLayoutPred,
SmallVectorImpl<MachineBasicBlock *> &TDBBs,
SmallVectorImpl<MachineInstr *> &Copies) {
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
if (IsSimple)
return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies);
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock *, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
for (MachineBasicBlock *PredBB : Preds) {
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
if (!canTailDuplicate(TailBB, PredBB))
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
bool IsLayoutSuccessor = false;
if (ForcedLayoutPred)
IsLayoutSuccessor = (ForcedLayoutPred == PredBB);
else if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
IsLayoutSuccessor = true;
if (IsLayoutSuccessor)
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->removeBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
for (MachineBasicBlock::iterator I = TailBB->begin(), E = TailBB->end();
I != E; /* empty */) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
duplicateInstruction(MI, TailBB, PredBB, LocalVRMap, UsedByPhi);
}
}
appendCopies(PredBB, CopyInfos, Copies);
// Simplify
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond);
NumTailDupAdded += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock *Succ : TailBB->successors())
PredBB->addSuccessor(Succ, MBPI->getEdgeProbability(TailBB, Succ));
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = ForcedLayoutPred;
if (!PrevBB)
PrevBB = &*std::prev(TailBB->getIterator());
MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// analyzeBranch.
if (PrevBB->succ_size() == 1 &&
// Layout preds are not always CFG preds. Check.
*PrevBB->succ_begin() == TailBB &&
!TII->analyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond) &&
PriorCond.empty() &&
(!PriorTBB || PriorTBB == TailBB) &&
TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
// There may be a branch to the layout successor. This is unlikely but it
// happens. The correct thing to do is to remove the branch before
// duplicating the instructions in all cases.
TII->removeBranch(*PrevBB);
if (PreRegAlloc) {
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
processPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
assert(!MI->isBundle() && "Not expecting bundles before regalloc!");
duplicateInstruction(MI, TailBB, PrevBB, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
appendCopies(PrevBB, CopyInfos, Copies);
} else {
TII->removeBranch(*PrevBB);
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (MachineBasicBlock *PredBB : Preds) {
if (is_contained(TDBBs, PredBB))
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
appendCopies(PredBB, CopyInfos, Copies);
}
return Changed;
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
// Set the limit on the number of instructions to duplicate, with a default
// of one less than the tail-merge threshold. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDuplicateSize.getNumOccurrences() == 0 &&
MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
MaxDuplicateCount = 1;
else
MaxDuplicateCount = TailDuplicateSize;
if (PreRegAlloc) {
if (TailBB->empty())
return false;
const TargetInstrDesc &TID = TailBB->back().getDesc();
// Pre-regalloc tail duplication hurts compile time and doesn't help
// much except for indirect branches and returns.
if (!TID.isIndirectBranch() && !TID.isReturn())
return false;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
MaxDuplicateCount = 20;
}
// Don't try to tail-duplicate single-block loops.
if (TailBB->isSuccessor(TailBB))
return false;
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
bool HasCall = false;
for (MachineBasicBlock::iterator I = TailBB->begin();
I != TailBB->end(); ++I) {
// Non-duplicable things shouldn't be tail-duplicated.
if (I->getDesc().isNotDuplicable()) return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && I->getDesc().isReturn()) return false;
// Don't duplicate more than the threshold.
if (InstrCount == MaxDuplicateCount) return false;
// Remember if we saw a call.
if (I->getDesc().isCall()) HasCall = true;
if (!I->isPHI() && !I->isDebugValue())
InstrCount += 1;
}
// Don't tail-duplicate calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (InstrCount > 1 && (PreRegAlloc && HasCall))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
if (PredBB->succ_size() > 1) continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() != 1)
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
bool PriorUnAnalyzable =
TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true);
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (!PriorUnAnalyzable && PriorCond.empty() && !PriorTBB &&
TailBB->pred_size() == 1 && PrevBB->succ_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
return Changed;
}
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion
DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map
DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map
DenseMap<unsigned, unsigned> SrcMap; // Src -> Def map
bool Changed = false;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) {
MachineInstr *MI = &*I;
++I;
if (MI->isCopy()) {
unsigned Def = MI->getOperand(0).getReg();
unsigned Src = MI->getOperand(1).getReg();
if (TargetRegisterInfo::isVirtualRegister(Def) ||
TargetRegisterInfo::isVirtualRegister(Src))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src);
if (CI != AvailCopyMap.end()) {
MachineInstr *CopyMI = CI->second;
unsigned SrcSrc = CopyMI->getOperand(1).getReg();
if (!ReservedRegs.test(Def) &&
(!ReservedRegs.test(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
(SrcSrc == Def || TRI->isSubRegister(SrcSrc, Def))) {
// The two copies cancel out and the source of the first copy
// hasn't been overridden, eliminate the second one. e.g.
// %ECX<def> = COPY %EAX<kill>
// ... nothing clobbered EAX.
// %EAX<def> = COPY %ECX
// =>
// %ECX<def> = COPY %EAX
//
// Also avoid eliminating a copy from reserved registers unless the
// definition is proven not clobbered. e.g.
// %RSP<def> = COPY %RAX
// CALL
// %RAX<def> = COPY %RSP
CopyMI->getOperand(1).setIsKill(false);
MI->eraseFromParent();
Changed = true;
++NumDeletes;
continue;
}
}
// If Src is defined by a previous copy, it cannot be eliminated.
CI = CopyMap.find(Src);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
for (const unsigned *AS = TRI->getAliasSet(Src); *AS; ++AS) {
CI = CopyMap.find(*AS);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
// Copy is now a candidate for deletion.
MaybeDeadCopies.insert(MI);
// If 'Src' is previously source of another copy, then this earlier copy's
// source is no longer available. e.g.
// %xmm9<def> = copy %xmm2
// ...
// %xmm2<def> = copy %xmm0
// ...
// %xmm2<def> = copy %xmm9
SourceNoLongerAvailable(Def, SrcMap, AvailCopyMap);
// Remember Def is defined by the copy.
CopyMap[Def] = MI;
AvailCopyMap[Def] = MI;
for (const unsigned *SR = TRI->getSubRegisters(Def); *SR; ++SR) {
CopyMap[*SR] = MI;
AvailCopyMap[*SR] = MI;
}
// Remember source that's copied to Def. Once it's clobbered, then
// it's no longer available for copy propagation.
SrcMap[Src] = Def;
continue;
}
// Not a copy.
SmallVector<unsigned, 2> Defs;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg())
continue;
unsigned Reg = MO.getReg();
if (!Reg)
continue;
if (TargetRegisterInfo::isVirtualRegister(Reg))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
if (MO.isDef()) {
Defs.push_back(Reg);
continue;
}
// If 'Reg' is defined by a copy, the copy is no longer a candidate
// for elimination.
DenseMap<unsigned, MachineInstr*>::iterator CI = CopyMap.find(Reg);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
for (const unsigned *AS = TRI->getAliasSet(Reg); *AS; ++AS) {
CI = CopyMap.find(*AS);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
}
for (unsigned i = 0, e = Defs.size(); i != e; ++i) {
unsigned Reg = Defs[i];
// No longer defined by a copy.
CopyMap.erase(Reg);
AvailCopyMap.erase(Reg);
for (const unsigned *AS = TRI->getAliasSet(Reg); *AS; ++AS) {
CopyMap.erase(*AS);
AvailCopyMap.erase(*AS);
}
// If 'Reg' is previously source of a copy, it is no longer available for
// copy propagation.
SourceNoLongerAvailable(Reg, SrcMap, AvailCopyMap);
}
}
// If MBB doesn't have successors, delete the copies whose defs are not used.
// If MBB does have successors, then conservative assume the defs are live-out
// since we don't want to trust live-in lists.
if (MBB.succ_empty()) {
for (SmallSetVector<MachineInstr*, 8>::iterator
DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
DI != DE; ++DI) {
if (!ReservedRegs.test((*DI)->getOperand(0).getReg())) {
(*DI)->eraseFromParent();
Changed = true;
++NumDeletes;
}
}
}
return Changed;
}
void MIPrinter::print(const MachineBasicBlock &MBB) {
assert(MBB.getNumber() >= 0 && "Invalid MBB number");
OS << "bb." << MBB.getNumber();
bool HasAttributes = false;
if (const auto *BB = MBB.getBasicBlock()) {
if (BB->hasName()) {
OS << "." << BB->getName();
} else {
HasAttributes = true;
OS << " (";
int Slot = MST.getLocalSlot(BB);
if (Slot == -1)
OS << "<ir-block badref>";
else
OS << (Twine("%ir-block.") + Twine(Slot)).str();
}
}
if (MBB.hasAddressTaken()) {
OS << (HasAttributes ? ", " : " (");
OS << "address-taken";
HasAttributes = true;
}
if (MBB.isEHPad()) {
OS << (HasAttributes ? ", " : " (");
OS << "landing-pad";
HasAttributes = true;
}
if (MBB.getAlignment()) {
OS << (HasAttributes ? ", " : " (");
OS << "align " << MBB.getAlignment();
HasAttributes = true;
}
if (HasAttributes)
OS << ")";
OS << ":\n";
bool HasLineAttributes = false;
// Print the successors
bool canPredictProbs = canPredictBranchProbabilities(MBB);
// Even if the list of successors is empty, if we cannot guess it,
// we need to print it to tell the parser that the list is empty.
// This is needed, because MI model unreachable as empty blocks
// with an empty successor list. If the parser would see that
// without the successor list, it would guess the code would
// fallthrough.
if ((!MBB.succ_empty() && !SimplifyMIR) || !canPredictProbs ||
!canPredictSuccessors(MBB)) {
OS.indent(2) << "successors: ";
for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I) {
if (I != MBB.succ_begin())
OS << ", ";
OS << printMBBReference(**I);
if (!SimplifyMIR || !canPredictProbs)
OS << '('
<< format("0x%08" PRIx32, MBB.getSuccProbability(I).getNumerator())
<< ')';
}
OS << "\n";
HasLineAttributes = true;
}
// Print the live in registers.
const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
if (MRI.tracksLiveness() && !MBB.livein_empty()) {
const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
OS.indent(2) << "liveins: ";
bool First = true;
for (const auto &LI : MBB.liveins()) {
if (!First)
OS << ", ";
First = false;
OS << printReg(LI.PhysReg, &TRI);
if (!LI.LaneMask.all())
OS << ":0x" << PrintLaneMask(LI.LaneMask);
}
OS << "\n";
HasLineAttributes = true;
}
if (HasLineAttributes)
OS << "\n";
bool IsInBundle = false;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; ++I) {
const MachineInstr &MI = *I;
if (IsInBundle && !MI.isInsideBundle()) {
OS.indent(2) << "}\n";
IsInBundle = false;
}
OS.indent(IsInBundle ? 4 : 2);
print(MI);
if (!IsInBundle && MI.getFlag(MachineInstr::BundledSucc)) {
OS << " {";
IsInBundle = true;
}
OS << "\n";
}
if (IsInBundle)
OS.indent(2) << "}\n";
}
void MIPrinter::print(const MachineBasicBlock &MBB) {
assert(MBB.getNumber() >= 0 && "Invalid MBB number");
OS << "bb." << MBB.getNumber();
bool HasAttributes = false;
if (const auto *BB = MBB.getBasicBlock()) {
if (BB->hasName()) {
OS << "." << BB->getName();
} else {
HasAttributes = true;
OS << " (";
int Slot = MST.getLocalSlot(BB);
if (Slot == -1)
OS << "<ir-block badref>";
else
OS << (Twine("%ir-block.") + Twine(Slot)).str();
}
}
if (MBB.hasAddressTaken()) {
OS << (HasAttributes ? ", " : " (");
OS << "address-taken";
HasAttributes = true;
}
if (MBB.isEHPad()) {
OS << (HasAttributes ? ", " : " (");
OS << "landing-pad";
HasAttributes = true;
}
if (MBB.getAlignment()) {
OS << (HasAttributes ? ", " : " (");
OS << "align " << MBB.getAlignment();
HasAttributes = true;
}
if (HasAttributes)
OS << ")";
OS << ":\n";
bool HasLineAttributes = false;
// Print the successors
if (!MBB.succ_empty()) {
OS.indent(2) << "successors: ";
for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I) {
if (I != MBB.succ_begin())
OS << ", ";
printMBBReference(**I);
if (MBB.hasSuccessorWeights())
OS << '(' << MBB.getSuccWeight(I) << ')';
}
OS << "\n";
HasLineAttributes = true;
}
// Print the live in registers.
const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
assert(TRI && "Expected target register info");
if (!MBB.livein_empty()) {
OS.indent(2) << "liveins: ";
bool First = true;
for (unsigned LI : MBB.liveins()) {
if (!First)
OS << ", ";
First = false;
printReg(LI, OS, TRI);
}
OS << "\n";
HasLineAttributes = true;
}
if (HasLineAttributes)
OS << "\n";
bool IsInBundle = false;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; ++I) {
const MachineInstr &MI = *I;
if (IsInBundle && !MI.isInsideBundle()) {
OS.indent(2) << "}\n";
IsInBundle = false;
}
OS.indent(IsInBundle ? 4 : 2);
print(MI);
if (!IsInBundle && MI.getFlag(MachineInstr::BundledSucc)) {
OS << " {";
IsInBundle = true;
}
OS << "\n";
}
if (IsInBundle)
OS.indent(2) << "}\n";
}
void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB) {
// Get rid of the easy cases first.
if (!Save)
Save = &MBB;
else
Save = MDT->findNearestCommonDominator(Save, &MBB);
if (!Save) {
DEBUG(dbgs() << "Found a block that is not reachable from Entry\n");
return;
}
if (!Restore)
Restore = &MBB;
else
Restore = MPDT->findNearestCommonDominator(Restore, &MBB);
// Make sure we would be able to insert the restore code before the
// terminator.
if (Restore == &MBB) {
for (const MachineInstr &Terminator : MBB.terminators()) {
if (!useOrDefCSROrFI(Terminator))
continue;
// One of the terminator needs to happen before the restore point.
if (MBB.succ_empty()) {
Restore = nullptr;
break;
}
// Look for a restore point that post-dominates all the successors.
// The immediate post-dominator is what we are looking for.
Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
break;
}
}
if (!Restore) {
DEBUG(dbgs() << "Restore point needs to be spanned on several blocks\n");
return;
}
// Make sure Save and Restore are suitable for shrink-wrapping:
// 1. all path from Save needs to lead to Restore before exiting.
// 2. all path to Restore needs to go through Save from Entry.
// We achieve that by making sure that:
// A. Save dominates Restore.
// B. Restore post-dominates Save.
// C. Save and Restore are in the same loop.
bool SaveDominatesRestore = false;
bool RestorePostDominatesSave = false;
while (Save && Restore &&
(!(SaveDominatesRestore = MDT->dominates(Save, Restore)) ||
!(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) ||
MLI->getLoopFor(Save) != MLI->getLoopFor(Restore))) {
// Fix (A).
if (!SaveDominatesRestore) {
Save = MDT->findNearestCommonDominator(Save, Restore);
continue;
}
// Fix (B).
if (!RestorePostDominatesSave)
Restore = MPDT->findNearestCommonDominator(Restore, Save);
// Fix (C).
if (Save && Restore && Save != Restore &&
MLI->getLoopFor(Save) != MLI->getLoopFor(Restore)) {
if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore))
// Push Save outside of this loop.
Save = FindIDom<>(*Save, Save->predecessors(), *MDT);
else
// Push Restore outside of this loop.
Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT);
}
}
}
