static
unsigned LookForIdenticalPHI(MachineBasicBlock *BB,
SmallVector<std::pair<MachineBasicBlock*, unsigned>, 8> &PredValues) {
if (BB->empty())
return 0;
MachineBasicBlock::iterator I = BB->front();
if (!I->isPHI())
return 0;
AvailableValsTy AVals;
for (unsigned i = 0, e = PredValues.size(); i != e; ++i)
AVals[PredValues[i].first] = PredValues[i].second;
while (I != BB->end() && I->isPHI()) {
bool Same = true;
for (unsigned i = 1, e = I->getNumOperands(); i != e; i += 2) {
unsigned SrcReg = I->getOperand(i).getReg();
MachineBasicBlock *SrcBB = I->getOperand(i+1).getMBB();
if (AVals[SrcBB] != SrcReg) {
Same = false;
break;
}
}
if (Same)
return I->getOperand(0).getReg();
++I;
}
return 0;
}
bool PHIElimination::SplitPHIEdges(MachineFunction &MF,
MachineBasicBlock &MBB,
LiveVariables &LV,
MachineLoopInfo *MLI) {
if (MBB.empty() || !MBB.front().isPHI() || MBB.isLandingPad())
return false; // Quick exit for basic blocks without PHIs.
bool Changed = false;
for (MachineBasicBlock::iterator BBI = MBB.begin(), BBE = MBB.end();
BBI != BBE && BBI->isPHI(); ++BBI) {
for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2) {
unsigned Reg = BBI->getOperand(i).getReg();
MachineBasicBlock *PreMBB = BBI->getOperand(i+1).getMBB();
// We break edges when registers are live out from the predecessor block
// (not considering PHI nodes). If the register is live in to this block
// anyway, we would gain nothing from splitting.
// Avoid splitting backedges of loops. It would introduce small
// out-of-line blocks into the loop which is very bad for code placement.
if (PreMBB != &MBB &&
!LV.isLiveIn(Reg, MBB) && LV.isLiveOut(Reg, *PreMBB)) {
if (!MLI ||
!(MLI->getLoopFor(PreMBB) == MLI->getLoopFor(&MBB) &&
MLI->isLoopHeader(&MBB))) {
if (PreMBB->SplitCriticalEdge(&MBB, this)) {
Changed = true;
++NumCriticalEdgesSplit;
}
}
}
}
}
return Changed;
}
MachineBasicBlock::iterator
MachineBasicBlock::SkipPHIsAndLabels(MachineBasicBlock::iterator I) {
while (I != end() && (I->isPHI() || I->isLabel() || I->isDebugValue()))
++I;
// FIXME: This needs to change if we wish to bundle labels / dbg_values
// inside the bundle.
assert(!I->isInsideBundle() &&
"First non-phi / non-label instruction is inside a bundle!");
return I;
}
MachineBasicBlock::iterator
MachineBasicBlock::SkipPHIsAndLabels(MachineBasicBlock::iterator I) {
iterator E = end();
while (I != E && (I->isPHI() || I->isPosition()))
++I;
// FIXME: This needs to change if we wish to bundle labels
// inside the bundle.
assert((I == E || !I->isInsideBundle()) &&
"First non-phi / non-label instruction is inside a bundle!");
return I;
}
/// InsertCopies - insert copies into MBB and all of its successors
void StrongPHIElimination::InsertCopies(MachineDomTreeNode* MDTN,
SmallPtrSet<MachineBasicBlock*, 16>& visited) {
MachineBasicBlock* MBB = MDTN->getBlock();
visited.insert(MBB);
std::set<unsigned> pushed;
LiveIntervals& LI = getAnalysis<LiveIntervals>();
// Rewrite register uses from Stacks
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
I != E; ++I) {
if (I->isPHI())
continue;
for (unsigned i = 0; i < I->getNumOperands(); ++i)
if (I->getOperand(i).isReg() &&
Stacks[I->getOperand(i).getReg()].size()) {
// Remove the live range for the old vreg.
LiveInterval& OldInt = LI.getInterval(I->getOperand(i).getReg());
LiveInterval::iterator OldLR =
OldInt.FindLiveRangeContaining(LI.getInstructionIndex(I).getUseIndex());
if (OldLR != OldInt.end())
OldInt.removeRange(*OldLR, true);
// Change the register
I->getOperand(i).setReg(Stacks[I->getOperand(i).getReg()].back());
// Add a live range for the new vreg
LiveInterval& Int = LI.getInterval(I->getOperand(i).getReg());
VNInfo* FirstVN = *Int.vni_begin();
FirstVN->setHasPHIKill(false);
LiveRange LR (LI.getMBBStartIdx(I->getParent()),
LI.getInstructionIndex(I).getUseIndex().getNextSlot(),
FirstVN);
Int.addRange(LR);
}
}
// Schedule the copies for this block
ScheduleCopies(MBB, pushed);
// Recur down the dominator tree.
for (MachineDomTreeNode::iterator I = MDTN->begin(),
E = MDTN->end(); I != E; ++I)
if (!visited.count((*I)->getBlock()))
InsertCopies(*I, visited);
// As we exit this block, pop the names we pushed while processing it
for (std::set<unsigned>::iterator I = pushed.begin(),
E = pushed.end(); I != E; ++I)
Stacks[*I].pop_back();
}
MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A,
MachineBasicBlock *B) {
assert(A && B && "Missing MBB end point");
MachineFunction *MF = A->getParent();
// We may need to update A's terminator, but we can't do that if AnalyzeBranch
// fails. If A uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
MachineBasicBlock *TBB = 0, *FBB = 0;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*A, TBB, FBB, Cond))
return NULL;
++NumSplits;
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(llvm::next(MachineFunction::iterator(A)), NMBB);
DEBUG(dbgs() << "PHIElimination splitting critical edge:"
" BB#" << A->getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << B->getNumber() << '\n');
A->ReplaceUsesOfBlockWith(B, NMBB);
A->updateTerminator();
// Insert unconditional "jump B" instruction in NMBB if necessary.
NMBB->addSuccessor(B);
if (!NMBB->isLayoutSuccessor(B)) {
Cond.clear();
MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, B, NULL, Cond);
}
// Fix PHI nodes in B so they refer to NMBB instead of A
for (MachineBasicBlock::iterator i = B->begin(), e = B->end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == A)
i->getOperand(ni+1).setMBB(NMBB);
if (LiveVariables *LV=getAnalysisIfAvailable<LiveVariables>())
LV->addNewBlock(NMBB, A, B);
if (MachineDominatorTree *MDT=getAnalysisIfAvailable<MachineDominatorTree>())
MDT->addNewBlock(NMBB, A);
return NMBB;
}
static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) {
for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) {
MachineBasicBlock *MBB = I;
SmallSetVector<MachineBasicBlock*, 8> Preds(MBB->pred_begin(),
MBB->pred_end());
MachineBasicBlock::iterator MI = MBB->begin();
while (MI != MBB->end()) {
if (!MI->isPHI())
break;
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
bool Found = false;
for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
if (PHIBB == PredBB) {
Found = true;
break;
}
}
if (!Found) {
dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
dbgs() << " missing input from predecessor BB#"
<< PredBB->getNumber() << '\n';
llvm_unreachable(0);
}
}
for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
if (CheckExtra && !Preds.count(PHIBB)) {
dbgs() << "Warning: malformed PHI in BB#" << MBB->getNumber()
<< ": " << *MI;
dbgs() << " extra input from predecessor BB#"
<< PHIBB->getNumber() << '\n';
llvm_unreachable(0);
}
if (PHIBB->getNumber() < 0) {
dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n';
llvm_unreachable(0);
}
}
++MI;
}
}
}
/// addNewBlock - Add a new basic block BB as an empty succcessor to DomBB. All
/// variables that are live out of DomBB will be marked as passing live through
/// BB.
void LiveVariables::addNewBlock(MachineBasicBlock *BB,
MachineBasicBlock *DomBB,
MachineBasicBlock *SuccBB) {
const unsigned NumNew = BB->getNumber();
SmallSet<unsigned, 16> Defs, Kills;
MachineBasicBlock::iterator BBI = SuccBB->begin(), BBE = SuccBB->end();
for (; BBI != BBE && BBI->isPHI(); ++BBI) {
// Record the def of the PHI node.
Defs.insert(BBI->getOperand(0).getReg());
// All registers used by PHI nodes in SuccBB must be live through BB.
for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
if (BBI->getOperand(i+1).getMBB() == BB)
getVarInfo(BBI->getOperand(i).getReg()).AliveBlocks.set(NumNew);
}
// Record all vreg defs and kills of all instructions in SuccBB.
for (; BBI != BBE; ++BBI) {
for (MachineInstr::mop_iterator I = BBI->operands_begin(),
E = BBI->operands_end(); I != E; ++I) {
if (I->isReg() && TargetRegisterInfo::isVirtualRegister(I->getReg())) {
if (I->isDef())
Defs.insert(I->getReg());
else if (I->isKill())
Kills.insert(I->getReg());
}
}
}
// Update info for all live variables
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
// If the Defs is defined in the successor it can't be live in BB.
if (Defs.count(Reg))
continue;
// If the register is either killed in or live through SuccBB it's also live
// through BB.
VarInfo &VI = getVarInfo(Reg);
if (Kills.count(Reg) || VI.AliveBlocks.test(SuccBB->getNumber()))
VI.AliveBlocks.set(NumNew);
}
}
/// getCanonicalInductionVariable - Check to see if the loop has a canonical
/// induction variable. We check for a simple recurrence pattern - an
/// integer recurrence that decrements by one each time through the loop and
/// ends at zero. If so, return the phi node that corresponds to it.
///
/// Based upon the similar code in LoopInfo except this code is specific to
/// the machine.
/// This method assumes that the IndVarSimplify pass has been run by 'opt'.
///
void
PPCCTRLoops::getCanonicalInductionVariable(MachineLoop *L,
SmallVector<MachineInstr *, 4> &IVars,
SmallVector<MachineInstr *, 4> &IOps) const {
MachineBasicBlock *TopMBB = L->getTopBlock();
MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin();
assert(PI != TopMBB->pred_end() &&
"Loop must have more than one incoming edge!");
MachineBasicBlock *Backedge = *PI++;
if (PI == TopMBB->pred_end()) return; // dead loop
MachineBasicBlock *Incoming = *PI++;
if (PI != TopMBB->pred_end()) return; // multiple backedges?
// make sure there is one incoming and one backedge and determine which
// is which.
if (L->contains(Incoming)) {
if (L->contains(Backedge))
return;
std::swap(Incoming, Backedge);
} else if (!L->contains(Backedge))
return;
// Loop over all of the PHI nodes, looking for a canonical induction variable:
// - The PHI node is "reg1 = PHI reg2, BB1, reg3, BB2".
// - The recurrence comes from the backedge.
// - the definition is an induction operatio.n
for (MachineBasicBlock::iterator I = TopMBB->begin(), E = TopMBB->end();
I != E && I->isPHI(); ++I) {
MachineInstr *MPhi = &*I;
unsigned DefReg = MPhi->getOperand(0).getReg();
for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) {
// Check each operand for the value from the backedge.
MachineBasicBlock *MBB = MPhi->getOperand(i+1).getMBB();
if (L->contains(MBB)) { // operands comes from the backedge
// Check if the definition is an induction operation.
MachineInstr *DI = MRI->getVRegDef(MPhi->getOperand(i).getReg());
if (isInductionOperation(DI, DefReg)) {
IOps.push_back(DI);
IVars.push_back(MPhi);
}
}
}
}
return;
}
void
MachineBasicBlock::transferSuccessorsAndUpdatePHIs(MachineBasicBlock *fromMBB) {
if (this == fromMBB)
return;
while (!fromMBB->succ_empty()) {
MachineBasicBlock *Succ = *fromMBB->succ_begin();
addSuccessor(Succ);
fromMBB->removeSuccessor(Succ);
// Fix up any PHI nodes in the successor.
for (MachineBasicBlock::iterator MI = Succ->begin(), ME = Succ->end();
MI != ME && MI->isPHI(); ++MI)
for (unsigned i = 2, e = MI->getNumOperands()+1; i != e; i += 2) {
MachineOperand &MO = MI->getOperand(i);
if (MO.getMBB() == fromMBB)
MO.setMBB(this);
}
}
}
/// addNewBlock - Add a new basic block BB as an empty succcessor to DomBB. All
/// variables that are live out of DomBB will be marked as passing live through
/// BB.
void LiveVariables::addNewBlock(MachineBasicBlock *BB,
MachineBasicBlock *DomBB,
MachineBasicBlock *SuccBB) {
const unsigned NumNew = BB->getNumber();
// All registers used by PHI nodes in SuccBB must be live through BB.
for (MachineBasicBlock::iterator BBI = SuccBB->begin(),
BBE = SuccBB->end(); BBI != BBE && BBI->isPHI(); ++BBI)
for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
if (BBI->getOperand(i+1).getMBB() == BB)
getVarInfo(BBI->getOperand(i).getReg()).AliveBlocks.set(NumNew);
// Update info for all live variables
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
VarInfo &VI = getVarInfo(Reg);
if (!VI.AliveBlocks.test(NumNew) && VI.isLiveIn(*SuccBB, Reg, *MRI))
VI.AliveBlocks.set(NumNew);
}
}
MachineBasicBlock *
MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
MachineFunction *MF = getParent();
DebugLoc dl; // FIXME: this is nowhere
// We may need to update this's terminator, but we can't do that if
// AnalyzeBranch fails. If this uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
MachineBasicBlock *TBB = 0, *FBB = 0;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*this, TBB, FBB, Cond))
return NULL;
// Avoid bugpoint weirdness: A block may end with a conditional branch but
// jumps to the same MBB is either case. We have duplicate CFG edges in that
// case that we can't handle. Since this never happens in properly optimized
// code, just skip those edges.
if (TBB && TBB == FBB) {
DEBUG(dbgs() << "Won't split critical edge after degenerate BB#"
<< getNumber() << '\n');
return NULL;
}
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(llvm::next(MachineFunction::iterator(this)), NMBB);
DEBUG(dbgs() << "Splitting critical edge:"
" BB#" << getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << Succ->getNumber() << '\n');
// On some targets like Mips, branches may kill virtual registers. Make sure
// that LiveVariables is properly updated after updateTerminator replaces the
// terminators.
LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>();
// Collect a list of virtual registers killed by the terminators.
SmallVector<unsigned, 4> KilledRegs;
if (LV)
for (iterator I = getFirstTerminator(), E = end(); I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || !OI->isUse() || !OI->isKill() || OI->isUndef())
continue;
unsigned Reg = OI->getReg();
if (TargetRegisterInfo::isVirtualRegister(Reg) &&
LV->getVarInfo(Reg).removeKill(MI)) {
KilledRegs.push_back(Reg);
DEBUG(dbgs() << "Removing terminator kill: " << *MI);
OI->setIsKill(false);
}
}
}
ReplaceUsesOfBlockWith(Succ, NMBB);
updateTerminator();
// Insert unconditional "jump Succ" instruction in NMBB if necessary.
NMBB->addSuccessor(Succ);
if (!NMBB->isLayoutSuccessor(Succ)) {
Cond.clear();
MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl);
}
// Fix PHI nodes in Succ so they refer to NMBB instead of this
for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == this)
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
E = Succ->livein_end(); I != E; ++I)
NMBB->addLiveIn(*I);
// Update LiveVariables.
if (LV) {
// Restore kills of virtual registers that were killed by the terminators.
while (!KilledRegs.empty()) {
unsigned Reg = KilledRegs.pop_back_val();
for (iterator I = end(), E = begin(); I != E;) {
if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false))
continue;
LV->getVarInfo(Reg).Kills.push_back(I);
DEBUG(dbgs() << "Restored terminator kill: " << *I);
break;
}
}
// Update relevant live-through information.
LV->addNewBlock(NMBB, this, Succ);
}
if (MachineDominatorTree *MDT =
P->getAnalysisIfAvailable<MachineDominatorTree>()) {
// Update dominator information.
MachineDomTreeNode *SucccDTNode = MDT->getNode(Succ);
bool IsNewIDom = true;
for (const_pred_iterator PI = Succ->pred_begin(), E = Succ->pred_end();
PI != E; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB == NMBB)
continue;
if (!MDT->dominates(SucccDTNode, MDT->getNode(PredBB))) {
IsNewIDom = false;
break;
}
}
// We know "this" dominates the newly created basic block.
MachineDomTreeNode *NewDTNode = MDT->addNewBlock(NMBB, this);
// If all the other predecessors of "Succ" are dominated by "Succ" itself
// then the new block is the new immediate dominator of "Succ". Otherwise,
// the new block doesn't dominate anything.
if (IsNewIDom)
MDT->changeImmediateDominator(SucccDTNode, NewDTNode);
}
if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>())
if (MachineLoop *TIL = MLI->getLoopFor(this)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) {
if (TIL == DestLoop) {
// Both in the same loop, the NMBB joins loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (TIL->contains(DestLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
TIL->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (DestLoop->contains(TIL)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(DestLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (MachineLoop *P = DestLoop->getParentLoop())
P->addBasicBlockToLoop(NMBB, MLI->getBase());
}
}
}
return NMBB;
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
// Set the limit on the number of instructions to duplicate, with a default
// of one less than the tail-merge threshold. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDuplicateSize.getNumOccurrences() == 0 &&
MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
MaxDuplicateCount = 1;
else
MaxDuplicateCount = TailDuplicateSize;
if (PreRegAlloc) {
if (TailBB->empty())
return false;
const TargetInstrDesc &TID = TailBB->back().getDesc();
// Pre-regalloc tail duplication hurts compile time and doesn't help
// much except for indirect branches and returns.
if (!TID.isIndirectBranch() && !TID.isReturn())
return false;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
MaxDuplicateCount = 20;
}
// Don't try to tail-duplicate single-block loops.
if (TailBB->isSuccessor(TailBB))
return false;
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
bool HasCall = false;
for (MachineBasicBlock::iterator I = TailBB->begin();
I != TailBB->end(); ++I) {
// Non-duplicable things shouldn't be tail-duplicated.
if (I->getDesc().isNotDuplicable()) return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && I->getDesc().isReturn()) return false;
// Don't duplicate more than the threshold.
if (InstrCount == MaxDuplicateCount) return false;
// Remember if we saw a call.
if (I->getDesc().isCall()) HasCall = true;
if (!I->isPHI() && !I->isDebugValue())
InstrCount += 1;
}
// Don't tail-duplicate calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (InstrCount > 1 && (PreRegAlloc && HasCall))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
if (PredBB->succ_size() > 1) continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() != 1)
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
bool PriorUnAnalyzable =
TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true);
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (!PriorUnAnalyzable && PriorCond.empty() && !PriorTBB &&
TailBB->pred_size() == 1 && PrevBB->succ_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
return Changed;
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
if (!shouldTailDuplicate(MF, *TailBB))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() > 1)
continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
// Simplify
TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true);
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (PrevBB->succ_size() == 1 &&
!TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) &&
PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end())
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
}
return Changed;
}
/// shouldTailDuplicate - Determine if it is profitable to duplicate this block.
bool
TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF,
bool IsSimple,
MachineBasicBlock &TailBB) {
// Only duplicate blocks that end with unconditional branches.
if (TailBB.canFallThrough())
return false;
// Don't try to tail-duplicate single-block loops.
if (TailBB.isSuccessor(&TailBB))
return false;
// Set the limit on the cost to duplicate. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDuplicateSize.getNumOccurrences() == 0 &&
MF.getFunction()->hasFnAttribute(Attribute::OptimizeForSize))
MaxDuplicateCount = 1;
else
MaxDuplicateCount = TailDuplicateSize;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
bool HasIndirectbr = false;
if (!TailBB.empty())
HasIndirectbr = TailBB.back().isIndirectBranch();
if (HasIndirectbr && PreRegAlloc)
MaxDuplicateCount = 20;
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
for (MachineBasicBlock::iterator I = TailBB.begin(); I != TailBB.end(); ++I) {
// Non-duplicable things shouldn't be tail-duplicated.
if (I->isNotDuplicable())
return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && I->isReturn())
return false;
// Avoid duplicating calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (PreRegAlloc && I->isCall())
return false;
if (!I->isPHI() && !I->isDebugValue())
InstrCount += 1;
if (InstrCount > MaxDuplicateCount)
return false;
}
if (HasIndirectbr && PreRegAlloc)
return true;
if (IsSimple)
return true;
if (!PreRegAlloc)
return true;
return canCompletelyDuplicateBB(TailBB);
}
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &MF) {
MRI = &MF.getRegInfo();
TII = MF.getTarget().getInstrInfo();
DT = &getAnalysis<MachineDominatorTree>();
LI = &getAnalysis<LiveIntervals>();
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
unsigned DestReg = BBI->getOperand(0).getReg();
addReg(DestReg);
PHISrcDefs[I].push_back(BBI);
for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) {
MachineOperand &SrcMO = BBI->getOperand(i);
unsigned SrcReg = SrcMO.getReg();
addReg(SrcReg);
unionRegs(DestReg, SrcReg);
MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
if (DefMI)
PHISrcDefs[DefMI->getParent()].push_back(DefMI);
}
}
}
// Perform a depth-first traversal of the dominator tree, splitting
// interferences amongst PHI-congruence classes.
DenseMap<unsigned, unsigned> CurrentDominatingParent;
DenseMap<unsigned, unsigned> ImmediateDominatingParent;
for (df_iterator<MachineDomTreeNode*> DI = df_begin(DT->getRootNode()),
DE = df_end(DT->getRootNode()); DI != DE; ++DI) {
SplitInterferencesForBasicBlock(*DI->getBlock(),
CurrentDominatingParent,
ImmediateDominatingParent);
}
// Insert copies for all PHI source and destination registers.
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
InsertCopiesForPHI(BBI, I);
}
}
// FIXME: Preserve the equivalence classes during copy insertion and use
// the preversed equivalence classes instead of recomputing them.
RegNodeMap.clear();
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
unsigned DestReg = BBI->getOperand(0).getReg();
addReg(DestReg);
for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) {
unsigned SrcReg = BBI->getOperand(i).getReg();
addReg(SrcReg);
unionRegs(DestReg, SrcReg);
}
}
}
DenseMap<unsigned, unsigned> RegRenamingMap;
bool Changed = false;
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
while (BBI != BBE && BBI->isPHI()) {
MachineInstr *PHI = BBI;
assert(PHI->getNumOperands() > 0);
unsigned SrcReg = PHI->getOperand(1).getReg();
unsigned SrcColor = getRegColor(SrcReg);
unsigned NewReg = RegRenamingMap[SrcColor];
if (!NewReg) {
NewReg = SrcReg;
RegRenamingMap[SrcColor] = SrcReg;
}
MergeLIsAndRename(SrcReg, NewReg);
unsigned DestReg = PHI->getOperand(0).getReg();
if (!InsertedDestCopies.count(DestReg))
MergeLIsAndRename(DestReg, NewReg);
for (unsigned i = 3; i < PHI->getNumOperands(); i += 2) {
unsigned SrcReg = PHI->getOperand(i).getReg();
MergeLIsAndRename(SrcReg, NewReg);
}
++BBI;
LI->RemoveMachineInstrFromMaps(PHI);
PHI->eraseFromParent();
Changed = true;
}
}
// Due to the insertion of copies to split live ranges, the live intervals are
// guaranteed to not overlap, except in one case: an original PHI source and a
// PHI destination copy. In this case, they have the same value and thus don't
// truly intersect, so we merge them into the value live at that point.
// FIXME: Is there some better way we can handle this?
for (DestCopyMap::iterator I = InsertedDestCopies.begin(),
E = InsertedDestCopies.end(); I != E; ++I) {
unsigned DestReg = I->first;
unsigned DestColor = getRegColor(DestReg);
unsigned NewReg = RegRenamingMap[DestColor];
LiveInterval &DestLI = LI->getInterval(DestReg);
LiveInterval &NewLI = LI->getInterval(NewReg);
assert(DestLI.ranges.size() == 1
&& "PHI destination copy's live interval should be a single live "
"range from the beginning of the BB to the copy instruction.");
LiveRange *DestLR = DestLI.begin();
VNInfo *NewVNI = NewLI.getVNInfoAt(DestLR->start);
if (!NewVNI) {
NewVNI = NewLI.createValueCopy(DestLR->valno, LI->getVNInfoAllocator());
MachineInstr *CopyInstr = I->second;
CopyInstr->getOperand(1).setIsKill(true);
}
LiveRange NewLR(DestLR->start, DestLR->end, NewVNI);
NewLI.addRange(NewLR);
LI->removeInterval(DestReg);
MRI->replaceRegWith(DestReg, NewReg);
}
// Adjust the live intervals of all PHI source registers to handle the case
// where the PHIs in successor blocks were the only later uses of the source
// register.
for (SrcCopySet::iterator I = InsertedSrcCopySet.begin(),
E = InsertedSrcCopySet.end(); I != E; ++I) {
MachineBasicBlock *MBB = I->first;
unsigned SrcReg = I->second;
if (unsigned RenamedRegister = RegRenamingMap[getRegColor(SrcReg)])
SrcReg = RenamedRegister;
LiveInterval &SrcLI = LI->getInterval(SrcReg);
bool isLiveOut = false;
for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
SE = MBB->succ_end(); SI != SE; ++SI) {
if (SrcLI.liveAt(LI->getMBBStartIdx(*SI))) {
isLiveOut = true;
break;
}
}
if (isLiveOut)
continue;
MachineOperand *LastUse = findLastUse(MBB, SrcReg);
assert(LastUse);
SlotIndex LastUseIndex = LI->getInstructionIndex(LastUse->getParent());
SrcLI.removeRange(LastUseIndex.getRegSlot(), LI->getMBBEndIdx(MBB));
LastUse->setIsKill(true);
}
Allocator.Reset();
RegNodeMap.clear();
PHISrcDefs.clear();
InsertedSrcCopySet.clear();
InsertedSrcCopyMap.clear();
InsertedDestCopies.clear();
return Changed;
}
/// EmitSchedule - Emit the machine code in scheduled order.
MachineBasicBlock *ScheduleDAGSDNodes::
EmitSchedule(DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
InstrEmitter Emitter(BB, InsertPos);
DenseMap<SDValue, unsigned> VRBaseMap;
DenseMap<SUnit*, unsigned> CopyVRBaseMap;
SmallVector<std::pair<unsigned, MachineInstr*>, 32> Orders;
SmallSet<unsigned, 8> Seen;
bool HasDbg = DAG->hasDebugValues();
for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
SUnit *SU = Sequence[i];
if (!SU) {
// Null SUnit* is a noop.
EmitNoop();
continue;
}
// For pre-regalloc scheduling, create instructions corresponding to the
// SDNode and any flagged SDNodes and append them to the block.
if (!SU->getNode()) {
// Emit a copy.
EmitPhysRegCopy(SU, CopyVRBaseMap);
continue;
}
SmallVector<SDNode *, 4> FlaggedNodes;
for (SDNode *N = SU->getNode()->getFlaggedNode(); N;
N = N->getFlaggedNode())
FlaggedNodes.push_back(N);
while (!FlaggedNodes.empty()) {
SDNode *N = FlaggedNodes.back();
Emitter.EmitNode(FlaggedNodes.back(), SU->OrigNode != SU, SU->isCloned,
VRBaseMap, EM);
// Remember the the source order of the inserted instruction.
if (HasDbg)
ProcessSourceNode(N, DAG, Emitter, EM, VRBaseMap, Orders, Seen);
FlaggedNodes.pop_back();
}
Emitter.EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned,
VRBaseMap, EM);
// Remember the the source order of the inserted instruction.
if (HasDbg)
ProcessSourceNode(SU->getNode(), DAG, Emitter, EM, VRBaseMap, Orders,
Seen);
}
// Insert all the dbg_value which have not already been inserted in source
// order sequence.
if (HasDbg) {
MachineBasicBlock::iterator BBBegin = BB->empty() ? BB->end() : BB->begin();
while (BBBegin != BB->end() && BBBegin->isPHI())
++BBBegin;
// Sort the source order instructions and use the order to insert debug
// values.
std::sort(Orders.begin(), Orders.end(), OrderSorter());
SDDbgInfo::DbgIterator DI = DAG->DbgBegin();
SDDbgInfo::DbgIterator DE = DAG->DbgEnd();
// Now emit the rest according to source order.
unsigned LastOrder = 0;
MachineInstr *LastMI = 0;
for (unsigned i = 0, e = Orders.size(); i != e && DI != DE; ++i) {
unsigned Order = Orders[i].first;
MachineInstr *MI = Orders[i].second;
// Insert all SDDbgValue's whose order(s) are before "Order".
if (!MI)
continue;
MachineBasicBlock *MIBB = MI->getParent();
#ifndef NDEBUG
unsigned LastDIOrder = 0;
#endif
for (; DI != DE &&
(*DI)->getOrder() >= LastOrder && (*DI)->getOrder() < Order; ++DI) {
#ifndef NDEBUG
assert((*DI)->getOrder() >= LastDIOrder &&
"SDDbgValue nodes must be in source order!");
LastDIOrder = (*DI)->getOrder();
#endif
if ((*DI)->isInvalidated())
continue;
MachineInstr *DbgMI = Emitter.EmitDbgValue(*DI, MIBB, VRBaseMap, EM);
if (!LastOrder)
// Insert to start of the BB (after PHIs).
BB->insert(BBBegin, DbgMI);
else {
MachineBasicBlock::iterator Pos = MI;
MIBB->insert(llvm::next(Pos), DbgMI);
}
}
LastOrder = Order;
LastMI = MI;
}
// Add trailing DbgValue's before the terminator. FIXME: May want to add
// some of them before one or more conditional branches?
while (DI != DE) {
MachineBasicBlock *InsertBB = Emitter.getBlock();
MachineBasicBlock::iterator Pos= Emitter.getBlock()->getFirstTerminator();
if (!(*DI)->isInvalidated()) {
MachineInstr *DbgMI= Emitter.EmitDbgValue(*DI, InsertBB, VRBaseMap, EM);
InsertBB->insert(Pos, DbgMI);
}
++DI;
}
}
BB = Emitter.getBlock();
InsertPos = Emitter.getInsertPos();
return BB;
}
/// SinkInstruction - Determine whether it is safe to sink the specified machine
/// instruction out of its current block into a successor.
bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
// Check if it's safe to move the instruction.
if (!MI->isSafeToMove(TII, AA, SawStore))
return false;
// FIXME: This should include support for sinking instructions within the
// block they are currently in to shorten the live ranges. We often get
// instructions sunk into the top of a large block, but it would be better to
// also sink them down before their first use in the block. This xform has to
// be careful not to *increase* register pressure though, e.g. sinking
// "x = y + z" down if it kills y and z would increase the live ranges of y
// and z and only shrink the live range of x.
// Loop over all the operands of the specified instruction. If there is
// anything we can't handle, bail out.
MachineBasicBlock *ParentBlock = MI->getParent();
// SuccToSinkTo - This is the successor to sink this instruction to, once we
// decide.
MachineBasicBlock *SuccToSinkTo = 0;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg()) continue; // Ignore non-register operands.
unsigned Reg = MO.getReg();
if (Reg == 0) continue;
if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
if (MO.isUse()) {
// If the physreg has no defs anywhere, it's just an ambient register
// and we can freely move its uses. Alternatively, if it's allocatable,
// it could get allocated to something with a def during allocation.
if (!RegInfo->def_empty(Reg))
return false;
if (AllocatableSet.test(Reg))
return false;
// Check for a def among the register's aliases too.
for (const unsigned *Alias = TRI->getAliasSet(Reg); *Alias; ++Alias) {
unsigned AliasReg = *Alias;
if (!RegInfo->def_empty(AliasReg))
return false;
if (AllocatableSet.test(AliasReg))
return false;
}
} else if (!MO.isDead()) {
// A def that isn't dead. We can't move it.
return false;
}
} else {
// Virtual register uses are always safe to sink.
if (MO.isUse()) continue;
// If it's not safe to move defs of the register class, then abort.
if (!TII->isSafeToMoveRegClassDefs(RegInfo->getRegClass(Reg)))
return false;
// FIXME: This picks a successor to sink into based on having one
// successor that dominates all the uses. However, there are cases where
// sinking can happen but where the sink point isn't a successor. For
// example:
// x = computation
// if () {} else {}
// use x
// the instruction could be sunk over the whole diamond for the
// if/then/else (or loop, etc), allowing it to be sunk into other blocks
// after that.
// Virtual register defs can only be sunk if all their uses are in blocks
// dominated by one of the successors.
if (SuccToSinkTo) {
// If a previous operand picked a block to sink to, then this operand
// must be sinkable to the same block.
if (!AllUsesDominatedByBlock(Reg, SuccToSinkTo))
return false;
continue;
}
// Otherwise, we should look at all the successors and decide which one
// we should sink to.
for (MachineBasicBlock::succ_iterator SI = ParentBlock->succ_begin(),
E = ParentBlock->succ_end(); SI != E; ++SI) {
if (AllUsesDominatedByBlock(Reg, *SI)) {
SuccToSinkTo = *SI;
break;
}
}
// If we couldn't find a block to sink to, ignore this instruction.
if (SuccToSinkTo == 0)
return false;
}
}
// If there are no outputs, it must have side-effects.
if (SuccToSinkTo == 0)
return false;
// It's not safe to sink instructions to EH landing pad. Control flow into
// landing pad is implicitly defined.
if (SuccToSinkTo->isLandingPad())
return false;
// It is not possible to sink an instruction into its own block. This can
// happen with loops.
if (MI->getParent() == SuccToSinkTo)
return false;
DEBUG(dbgs() << "Sink instr " << *MI);
DEBUG(dbgs() << "to block " << *SuccToSinkTo);
// If the block has multiple predecessors, this would introduce computation on
// a path that it doesn't already exist. We could split the critical edge,
// but for now we just punt.
// FIXME: Split critical edges if not backedges.
if (SuccToSinkTo->pred_size() > 1) {
DEBUG(dbgs() << " *** PUNTING: Critical edge found\n");
return false;
}
// Determine where to insert into. Skip phi nodes.
MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
++InsertPos;
// Move the instruction.
SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
++MachineBasicBlock::iterator(MI));
return true;
}
/// If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
/// \p IsSimple result of isSimpleBB
/// \p TailBB Block to be duplicated.
/// \p ForcedLayoutPred When non-null, use this block as the layout predecessor
/// instead of the previous block in MF's order.
/// \p TDBBs A vector to keep track of all blocks tail-duplicated
/// into.
/// \p Copies A vector of copy instructions inserted. Used later to
/// walk all the inserted copies and remove redundant ones.
bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB,
MachineBasicBlock *ForcedLayoutPred,
SmallVectorImpl<MachineBasicBlock *> &TDBBs,
SmallVectorImpl<MachineInstr *> &Copies) {
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
if (IsSimple)
return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies);
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock *, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
for (MachineBasicBlock *PredBB : Preds) {
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
if (!canTailDuplicate(TailBB, PredBB))
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
bool IsLayoutSuccessor = false;
if (ForcedLayoutPred)
IsLayoutSuccessor = (ForcedLayoutPred == PredBB);
else if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
IsLayoutSuccessor = true;
if (IsLayoutSuccessor)
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->removeBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
for (MachineBasicBlock::iterator I = TailBB->begin(), E = TailBB->end();
I != E; /* empty */) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
duplicateInstruction(MI, TailBB, PredBB, LocalVRMap, UsedByPhi);
}
}
appendCopies(PredBB, CopyInfos, Copies);
// Simplify
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond);
NumTailDupAdded += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock *Succ : TailBB->successors())
PredBB->addSuccessor(Succ, MBPI->getEdgeProbability(TailBB, Succ));
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = ForcedLayoutPred;
if (!PrevBB)
PrevBB = &*std::prev(TailBB->getIterator());
MachineBasicBlock *PriorTBB = nullptr, *PriorFBB = nullptr;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// analyzeBranch.
if (PrevBB->succ_size() == 1 &&
// Layout preds are not always CFG preds. Check.
*PrevBB->succ_begin() == TailBB &&
!TII->analyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond) &&
PriorCond.empty() &&
(!PriorTBB || PriorTBB == TailBB) &&
TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
// There may be a branch to the layout successor. This is unlikely but it
// happens. The correct thing to do is to remove the branch before
// duplicating the instructions in all cases.
TII->removeBranch(*PrevBB);
if (PreRegAlloc) {
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
processPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
assert(!MI->isBundle() && "Not expecting bundles before regalloc!");
duplicateInstruction(MI, TailBB, PrevBB, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
appendCopies(PrevBB, CopyInfos, Copies);
} else {
TII->removeBranch(*PrevBB);
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (MachineBasicBlock *PredBB : Preds) {
if (is_contained(TDBBs, PredBB))
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, RegSubRegPair> LocalVRMap;
SmallVector<std::pair<unsigned, RegSubRegPair>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
processPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
appendCopies(PredBB, CopyInfos, Copies);
}
return Changed;
}
/// SinkInstruction - Determine whether it is safe to sink the specified machine
/// instruction out of its current block into a successor.
bool MachineSinking::SinkInstruction(MachineInstr *MI, bool &SawStore) {
// Don't sink insert_subreg, subreg_to_reg, reg_sequence. These are meant to
// be close to the source to make it easier to coalesce.
if (AvoidsSinking(MI, MRI))
return false;
// Check if it's safe to move the instruction.
if (!MI->isSafeToMove(AA, SawStore))
return false;
// FIXME: This should include support for sinking instructions within the
// block they are currently in to shorten the live ranges. We often get
// instructions sunk into the top of a large block, but it would be better to
// also sink them down before their first use in the block. This xform has to
// be careful not to *increase* register pressure though, e.g. sinking
// "x = y + z" down if it kills y and z would increase the live ranges of y
// and z and only shrink the live range of x.
bool BreakPHIEdge = false;
MachineBasicBlock *ParentBlock = MI->getParent();
MachineBasicBlock *SuccToSinkTo = FindSuccToSinkTo(MI, ParentBlock,
BreakPHIEdge);
// If there are no outputs, it must have side-effects.
if (!SuccToSinkTo)
return false;
// If the instruction to move defines a dead physical register which is live
// when leaving the basic block, don't move it because it could turn into a
// "zombie" define of that preg. E.g., EFLAGS. (<rdar://problem/8030636>)
for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
const MachineOperand &MO = MI->getOperand(I);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0 || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
if (SuccToSinkTo->isLiveIn(Reg))
return false;
}
DEBUG(dbgs() << "Sink instr " << *MI << "\tinto block " << *SuccToSinkTo);
// If the block has multiple predecessors, this is a critical edge.
// Decide if we can sink along it or need to break the edge.
if (SuccToSinkTo->pred_size() > 1) {
// We cannot sink a load across a critical edge - there may be stores in
// other code paths.
bool TryBreak = false;
bool store = true;
if (!MI->isSafeToMove(AA, store)) {
DEBUG(dbgs() << " *** NOTE: Won't sink load along critical edge.\n");
TryBreak = true;
}
// We don't want to sink across a critical edge if we don't dominate the
// successor. We could be introducing calculations to new code paths.
if (!TryBreak && !DT->dominates(ParentBlock, SuccToSinkTo)) {
DEBUG(dbgs() << " *** NOTE: Critical edge found\n");
TryBreak = true;
}
// Don't sink instructions into a loop.
if (!TryBreak && LI->isLoopHeader(SuccToSinkTo)) {
DEBUG(dbgs() << " *** NOTE: Loop header found\n");
TryBreak = true;
}
// Otherwise we are OK with sinking along a critical edge.
if (!TryBreak)
DEBUG(dbgs() << "Sinking along critical edge.\n");
else {
// Mark this edge as to be split.
// If the edge can actually be split, the next iteration of the main loop
// will sink MI in the newly created block.
bool Status =
PostponeSplitCriticalEdge(MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
if (!Status)
DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
"break critical edge\n");
// The instruction will not be sunk this time.
return false;
}
}
if (BreakPHIEdge) {
// BreakPHIEdge is true if all the uses are in the successor MBB being
// sunken into and they are all PHI nodes. In this case, machine-sink must
// break the critical edge first.
bool Status = PostponeSplitCriticalEdge(MI, ParentBlock,
SuccToSinkTo, BreakPHIEdge);
if (!Status)
DEBUG(dbgs() << " *** PUNTING: Not legal or profitable to "
"break critical edge\n");
// The instruction will not be sunk this time.
return false;
}
// Determine where to insert into. Skip phi nodes.
MachineBasicBlock::iterator InsertPos = SuccToSinkTo->begin();
while (InsertPos != SuccToSinkTo->end() && InsertPos->isPHI())
++InsertPos;
// collect matching debug values.
SmallVector<MachineInstr *, 2> DbgValuesToSink;
collectDebugValues(MI, DbgValuesToSink);
// Move the instruction.
SuccToSinkTo->splice(InsertPos, ParentBlock, MI,
++MachineBasicBlock::iterator(MI));
// Move debug values.
for (SmallVectorImpl<MachineInstr *>::iterator DBI = DbgValuesToSink.begin(),
DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) {
MachineInstr *DbgMI = *DBI;
SuccToSinkTo->splice(InsertPos, ParentBlock, DbgMI,
++MachineBasicBlock::iterator(DbgMI));
}
// Conservatively, clear any kill flags, since it's possible that they are no
// longer correct.
// Note that we have to clear the kill flags for any register this instruction
// uses as we may sink over another instruction which currently kills the
// used registers.
for (MachineOperand &MO : MI->operands()) {
if (MO.isReg() && MO.isUse())
RegsToClearKillFlags.set(MO.getReg()); // Remember to clear kill flags.
}
return true;
}
/// SplitInterferencesForBasicBlock - traverses a basic block, splitting any
/// interferences found between registers in the same congruence class. It
/// takes two DenseMaps as arguments that it also updates:
///
/// 1) CurrentDominatingParent, which maps a color to the register in that
/// congruence class whose definition was most recently seen.
///
/// 2) ImmediateDominatingParent, which maps a register to the register in the
/// same congruence class that most immediately dominates it.
///
/// This function assumes that it is being called in a depth-first traversal
/// of the dominator tree.
///
/// The algorithm used here is a generalization of the dominance-based SSA test
/// for two variables. If there are variables a_1, ..., a_n such that
///
/// def(a_1) dom ... dom def(a_n),
///
/// then we can test for an interference between any two a_i by only using O(n)
/// interference tests between pairs of variables. If i < j and a_i and a_j
/// interfere, then a_i is alive at def(a_j), so it is also alive at def(a_i+1).
/// Thus, in order to test for an interference involving a_i, we need only check
/// for a potential interference with a_i+1.
///
/// This method can be generalized to arbitrary sets of variables by performing
/// a depth-first traversal of the dominator tree. As we traverse down a branch
/// of the dominator tree, we keep track of the current dominating variable and
/// only perform an interference test with that variable. However, when we go to
/// another branch of the dominator tree, the definition of the current dominating
/// variable may no longer dominate the current block. In order to correct this,
/// we need to use a stack of past choices of the current dominating variable
/// and pop from this stack until we find a variable whose definition actually
/// dominates the current block.
///
/// There will be one push on this stack for each variable that has become the
/// current dominating variable, so instead of using an explicit stack we can
/// simply associate the previous choice for a current dominating variable with
/// the new choice. This works better in our implementation, where we test for
/// interference in multiple distinct sets at once.
void
StrongPHIElimination::SplitInterferencesForBasicBlock(
MachineBasicBlock &MBB,
DenseMap<unsigned, unsigned> &CurrentDominatingParent,
DenseMap<unsigned, unsigned> &ImmediateDominatingParent) {
// Sort defs by their order in the original basic block, as the code below
// assumes that it is processing definitions in dominance order.
std::vector<MachineInstr*> &DefInstrs = PHISrcDefs[&MBB];
std::sort(DefInstrs.begin(), DefInstrs.end(), MIIndexCompare(LI));
for (std::vector<MachineInstr*>::const_iterator BBI = DefInstrs.begin(),
BBE = DefInstrs.end(); BBI != BBE; ++BBI) {
for (MachineInstr::const_mop_iterator I = (*BBI)->operands_begin(),
E = (*BBI)->operands_end(); I != E; ++I) {
const MachineOperand &MO = *I;
// FIXME: This would be faster if it were possible to bail out of checking
// an instruction's operands after the explicit defs, but this is incorrect
// for variadic instructions, which may appear before register allocation
// in the future.
if (!MO.isReg() || !MO.isDef())
continue;
unsigned DestReg = MO.getReg();
if (!DestReg || !TargetRegisterInfo::isVirtualRegister(DestReg))
continue;
// If the virtual register being defined is not used in any PHI or has
// already been isolated, then there are no more interferences to check.
unsigned DestColor = getRegColor(DestReg);
if (!DestColor)
continue;
// The input to this pass sometimes is not in SSA form in every basic
// block, as some virtual registers have redefinitions. We could eliminate
// this by fixing the passes that generate the non-SSA code, or we could
// handle it here by tracking defining machine instructions rather than
// virtual registers. For now, we just handle the situation conservatively
// in a way that will possibly lead to false interferences.
unsigned &CurrentParent = CurrentDominatingParent[DestColor];
unsigned NewParent = CurrentParent;
if (NewParent == DestReg)
continue;
// Pop registers from the stack represented by ImmediateDominatingParent
// until we find a parent that dominates the current instruction.
while (NewParent && (!DT->dominates(MRI->getVRegDef(NewParent), *BBI)
|| !getRegColor(NewParent)))
NewParent = ImmediateDominatingParent[NewParent];
// If NewParent is nonzero, then its definition dominates the current
// instruction, so it is only necessary to check for the liveness of
// NewParent in order to check for an interference.
if (NewParent
&& LI->getInterval(NewParent).liveAt(LI->getInstructionIndex(*BBI))) {
// If there is an interference, always isolate the new register. This
// could be improved by using a heuristic that decides which of the two
// registers to isolate.
isolateReg(DestReg);
CurrentParent = NewParent;
} else {
// If there is no interference, update ImmediateDominatingParent and set
// the CurrentDominatingParent for this color to the current register.
ImmediateDominatingParent[DestReg] = NewParent;
CurrentParent = DestReg;
}
}
}
// We now walk the PHIs in successor blocks and check for interferences. This
// is necessary because the use of a PHI's operands are logically contained in
// the predecessor block. The def of a PHI's destination register is processed
// along with the other defs in a basic block.
CurrentPHIForColor.clear();
for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end(); SI != SE; ++SI) {
for (MachineBasicBlock::iterator BBI = (*SI)->begin(), BBE = (*SI)->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
MachineInstr *PHI = BBI;
// If a PHI is already isolated, either by being isolated directly or
// having all of its operands isolated, ignore it.
unsigned Color = getPHIColor(PHI);
if (!Color)
continue;
// Find the index of the PHI operand that corresponds to this basic block.
unsigned PredIndex;
for (PredIndex = 1; PredIndex < PHI->getNumOperands(); PredIndex += 2) {
if (PHI->getOperand(PredIndex + 1).getMBB() == &MBB)
break;
}
assert(PredIndex < PHI->getNumOperands());
unsigned PredOperandReg = PHI->getOperand(PredIndex).getReg();
// Pop registers from the stack represented by ImmediateDominatingParent
// until we find a parent that dominates the current instruction.
unsigned &CurrentParent = CurrentDominatingParent[Color];
unsigned NewParent = CurrentParent;
while (NewParent
&& (!DT->dominates(MRI->getVRegDef(NewParent)->getParent(), &MBB)
|| !getRegColor(NewParent)))
NewParent = ImmediateDominatingParent[NewParent];
CurrentParent = NewParent;
// If there is an interference with a register, always isolate the
// register rather than the PHI. It is also possible to isolate the
// PHI, but that introduces copies for all of the registers involved
// in that PHI.
if (NewParent && LI->isLiveOutOfMBB(LI->getInterval(NewParent), &MBB)
&& NewParent != PredOperandReg)
isolateReg(NewParent);
std::pair<MachineInstr*, unsigned>
&CurrentPHI = CurrentPHIForColor[Color];
// If two PHIs have the same operand from every shared predecessor, then
// they don't actually interfere. Otherwise, isolate the current PHI. This
// could possibly be improved, e.g. we could isolate the PHI with the
// fewest operands.
if (CurrentPHI.first && CurrentPHI.second != PredOperandReg)
isolatePHI(PHI);
else
CurrentPHI = std::make_pair(PHI, PredOperandReg);
}
}
}
/// processBlock - Determine how to break up PHIs in the current block. Each
/// PHI is broken up by some combination of renaming its operands and inserting
/// copies. This method is responsible for determining which operands receive
/// which treatment.
void StrongPHIElimination::processBlock(MachineBasicBlock* MBB) {
LiveIntervals& LI = getAnalysis<LiveIntervals>();
MachineRegisterInfo& MRI = MBB->getParent()->getRegInfo();
// Holds names that have been added to a set in any PHI within this block
// before the current one.
std::set<unsigned> ProcessedNames;
// Iterate over all the PHI nodes in this block
MachineBasicBlock::iterator P = MBB->begin();
while (P != MBB->end() && P->isPHI()) {
unsigned DestReg = P->getOperand(0).getReg();
// Don't both doing PHI elimination for dead PHI's.
if (P->registerDefIsDead(DestReg)) {
++P;
continue;
}
LiveInterval& PI = LI.getOrCreateInterval(DestReg);
SlotIndex pIdx = LI.getInstructionIndex(P).getDefIndex();
VNInfo* PVN = PI.getLiveRangeContaining(pIdx)->valno;
PhiValueNumber.insert(std::make_pair(DestReg, PVN->id));
// PHIUnion is the set of incoming registers to the PHI node that
// are going to be renames rather than having copies inserted. This set
// is refinded over the course of this function. UnionedBlocks is the set
// of corresponding MBBs.
std::map<unsigned, MachineBasicBlock*> PHIUnion;
SmallPtrSet<MachineBasicBlock*, 8> UnionedBlocks;
// Iterate over the operands of the PHI node
for (int i = P->getNumOperands() - 1; i >= 2; i-=2) {
unsigned SrcReg = P->getOperand(i-1).getReg();
// Don't need to try to coalesce a register with itself.
if (SrcReg == DestReg) {
ProcessedNames.insert(SrcReg);
continue;
}
// We don't need to insert copies for implicit_defs.
MachineInstr* DefMI = MRI.getVRegDef(SrcReg);
if (DefMI->isImplicitDef())
ProcessedNames.insert(SrcReg);
// Check for trivial interferences via liveness information, allowing us
// to avoid extra work later. Any registers that interfere cannot both
// be in the renaming set, so choose one and add copies for it instead.
// The conditions are:
// 1) if the operand is live into the PHI node's block OR
// 2) if the PHI node is live out of the operand's defining block OR
// 3) if the operand is itself a PHI node and the original PHI is
// live into the operand's defining block OR
// 4) if the operand is already being renamed for another PHI node
// in this block OR
// 5) if any two operands are defined in the same block, insert copies
// for one of them
if (isLiveIn(SrcReg, P->getParent(), LI) ||
isLiveOut(P->getOperand(0).getReg(),
MRI.getVRegDef(SrcReg)->getParent(), LI) ||
( MRI.getVRegDef(SrcReg)->isPHI() &&
isLiveIn(P->getOperand(0).getReg(),
MRI.getVRegDef(SrcReg)->getParent(), LI) ) ||
ProcessedNames.count(SrcReg) ||
UnionedBlocks.count(MRI.getVRegDef(SrcReg)->getParent())) {
// Add a copy for the selected register
MachineBasicBlock* From = P->getOperand(i).getMBB();
Waiting[From].insert(std::make_pair(SrcReg, DestReg));
UsedByAnother.insert(SrcReg);
} else {
// Otherwise, add it to the renaming set
PHIUnion.insert(std::make_pair(SrcReg,P->getOperand(i).getMBB()));
UnionedBlocks.insert(MRI.getVRegDef(SrcReg)->getParent());
}
}
// Compute the dominator forest for the renaming set. This is a forest
// where the nodes are the registers and the edges represent dominance
// relations between the defining blocks of the registers
std::vector<StrongPHIElimination::DomForestNode*> DF =
computeDomForest(PHIUnion, MRI);
// Walk DomForest to resolve interferences at an inter-block level. This
// will remove registers from the renaming set (and insert copies for them)
// if interferences are found.
std::vector<std::pair<unsigned, unsigned> > localInterferences;
processPHIUnion(P, PHIUnion, DF, localInterferences);
// If one of the inputs is defined in the same block as the current PHI
// then we need to check for a local interference between that input and
// the PHI.
for (std::map<unsigned, MachineBasicBlock*>::iterator I = PHIUnion.begin(),
E = PHIUnion.end(); I != E; ++I)
if (MRI.getVRegDef(I->first)->getParent() == P->getParent())
localInterferences.push_back(std::make_pair(I->first,
P->getOperand(0).getReg()));
// The dominator forest walk may have returned some register pairs whose
// interference cannot be determined from dominator analysis. We now
// examine these pairs for local interferences.
for (std::vector<std::pair<unsigned, unsigned> >::iterator I =
localInterferences.begin(), E = localInterferences.end(); I != E; ++I) {
std::pair<unsigned, unsigned> p = *I;
MachineDominatorTree& MDT = getAnalysis<MachineDominatorTree>();
// Determine the block we need to scan and the relationship between
// the two registers
MachineBasicBlock* scan = 0;
unsigned mode = 0;
if (MRI.getVRegDef(p.first)->getParent() ==
MRI.getVRegDef(p.second)->getParent()) {
scan = MRI.getVRegDef(p.first)->getParent();
mode = 0; // Same block
} else if (MDT.dominates(MRI.getVRegDef(p.first)->getParent(),
MRI.getVRegDef(p.second)->getParent())) {
scan = MRI.getVRegDef(p.second)->getParent();
mode = 1; // First dominates second
} else {
scan = MRI.getVRegDef(p.first)->getParent();
mode = 2; // Second dominates first
}
// If there's an interference, we need to insert copies
if (interferes(p.first, p.second, scan, LI, mode)) {
// Insert copies for First
for (int i = P->getNumOperands() - 1; i >= 2; i-=2) {
if (P->getOperand(i-1).getReg() == p.first) {
unsigned SrcReg = p.first;
MachineBasicBlock* From = P->getOperand(i).getMBB();
Waiting[From].insert(std::make_pair(SrcReg,
P->getOperand(0).getReg()));
UsedByAnother.insert(SrcReg);
PHIUnion.erase(SrcReg);
}
}
}
}
// Add the renaming set for this PHI node to our overall renaming information
for (std::map<unsigned, MachineBasicBlock*>::iterator QI = PHIUnion.begin(),
QE = PHIUnion.end(); QI != QE; ++QI) {
DEBUG(dbgs() << "Adding Renaming: " << QI->first << " -> "
<< P->getOperand(0).getReg() << "\n");
}
RenameSets.insert(std::make_pair(P->getOperand(0).getReg(), PHIUnion));
// Remember which registers are already renamed, so that we don't try to
// rename them for another PHI node in this block
for (std::map<unsigned, MachineBasicBlock*>::iterator I = PHIUnion.begin(),
E = PHIUnion.end(); I != E; ++I)
ProcessedNames.insert(I->first);
++P;
}
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
df_iterator_default_set<MachineBasicBlock*> Reachable;
bool ModifiedPHI = false;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
ModifiedPHI = true;
}
if (phi->getNumOperands() == 3) {
const MachineOperand &Input = phi->getOperand(1);
const MachineOperand &Output = phi->getOperand(0);
unsigned InputReg = Input.getReg();
unsigned OutputReg = Output.getReg();
assert(Output.getSubReg() == 0 && "Cannot have output subregister");
ModifiedPHI = true;
if (InputReg != OutputReg) {
MachineRegisterInfo &MRI = F.getRegInfo();
unsigned InputSub = Input.getSubReg();
if (InputSub == 0 &&
MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) {
MRI.replaceRegWith(OutputReg, InputReg);
} else {
// The input register to the PHI has a subregister or it can't be
// constrained to the proper register class:
// insert a COPY instead of simply replacing the output
// with the input.
const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(),
TII->get(TargetOpcode::COPY), OutputReg)
.addReg(InputReg, getRegState(Input), InputSub);
}
phi++->eraseFromParent();
}
continue;
}
++phi;
}
}
F.RenumberBlocks();
return (!DeadBlocks.empty() || ModifiedPHI);
}
bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
MachineRegisterInfo &MRI = MF.getRegInfo();
LLVM_DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n');
InstOrderMap IOM;
// Map from register to instruction order (value of IOM) where the
// register is used last. When moving instructions up, we need to
// make sure all its defs (including dead def) will not cross its
// last use when moving up.
DenseMap<unsigned, std::pair<unsigned, MachineInstr *>> UseMap;
for (MachineBasicBlock &MBB : MF) {
if (MBB.empty())
continue;
bool SawStore = false;
BuildInstOrderMap(MBB.begin(), IOM);
UseMap.clear();
for (MachineBasicBlock::iterator Next = MBB.begin(); Next != MBB.end();) {
MachineInstr &MI = *Next;
++Next;
if (MI.isPHI() || MI.isDebugInstr())
continue;
if (MI.mayStore())
SawStore = true;
unsigned CurrentOrder = IOM[&MI];
unsigned Barrier = 0;
MachineInstr *BarrierMI = nullptr;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || MO.isDebug())
continue;
if (MO.isUse())
UseMap[MO.getReg()] = std::make_pair(CurrentOrder, &MI);
else if (MO.isDead() && UseMap.count(MO.getReg()))
// Barrier is the last instruction where MO get used. MI should not
// be moved above Barrier.
if (Barrier < UseMap[MO.getReg()].first) {
Barrier = UseMap[MO.getReg()].first;
BarrierMI = UseMap[MO.getReg()].second;
}
}
if (!MI.isSafeToMove(nullptr, SawStore)) {
// If MI has side effects, it should become a barrier for code motion.
// IOM is rebuild from the next instruction to prevent later
// instructions from being moved before this MI.
if (MI.hasUnmodeledSideEffects() && Next != MBB.end()) {
BuildInstOrderMap(Next, IOM);
SawStore = false;
}
continue;
}
const MachineOperand *DefMO = nullptr;
MachineInstr *Insert = nullptr;
// Number of live-ranges that will be shortened. We do not count
// live-ranges that are defined by a COPY as it could be coalesced later.
unsigned NumEligibleUse = 0;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || MO.isDead() || MO.isDebug())
continue;
unsigned Reg = MO.getReg();
// Do not move the instruction if it def/uses a physical register,
// unless it is a constant physical register or a noreg.
if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
if (!Reg || MRI.isConstantPhysReg(Reg))
continue;
Insert = nullptr;
break;
}
if (MO.isDef()) {
// Do not move if there is more than one def.
if (DefMO) {
Insert = nullptr;
break;
}
DefMO = &MO;
} else if (MRI.hasOneNonDBGUse(Reg) && MRI.hasOneDef(Reg) && DefMO &&
MRI.getRegClass(DefMO->getReg()) ==
MRI.getRegClass(MO.getReg())) {
// The heuristic does not handle different register classes yet
// (registers of different sizes, looser/tighter constraints). This
// is because it needs more accurate model to handle register
// pressure correctly.
MachineInstr &DefInstr = *MRI.def_instr_begin(Reg);
if (!DefInstr.isCopy())
NumEligibleUse++;
Insert = FindDominatedInstruction(DefInstr, Insert, IOM);
} else {
Insert = nullptr;
break;
}
}
// If Barrier equals IOM[I], traverse forward to find if BarrierMI is
// after Insert, if yes, then we should not hoist.
for (MachineInstr *I = Insert; I && IOM[I] == Barrier;
I = I->getNextNode())
if (I == BarrierMI) {
Insert = nullptr;
break;
}
// Move the instruction when # of shrunk live range > 1.
if (DefMO && Insert && NumEligibleUse > 1 && Barrier <= IOM[Insert]) {
MachineBasicBlock::iterator I = std::next(Insert->getIterator());
// Skip all the PHI and debug instructions.
while (I != MBB.end() && (I->isPHI() || I->isDebugInstr()))
I = std::next(I);
if (I == MI.getIterator())
continue;
// Update the dominator order to be the same as the insertion point.
// We do this to maintain a non-decreasing order without need to update
// all instruction orders after the insertion point.
unsigned NewOrder = IOM[&*I];
IOM[&MI] = NewOrder;
NumInstrsHoistedToShrinkLiveRange++;
// Find MI's debug value following MI.
MachineBasicBlock::iterator EndIter = std::next(MI.getIterator());
if (MI.getOperand(0).isReg())
for (; EndIter != MBB.end() && EndIter->isDebugValue() &&
EndIter->getOperand(0).isReg() &&
EndIter->getOperand(0).getReg() == MI.getOperand(0).getReg();
++EndIter, ++Next)
IOM[&*EndIter] = NewOrder;
MBB.splice(I, &MBB, MI.getIterator(), EndIter);
}
}
}
return false;
}
bool PHIElimination::SplitPHIEdges(MachineFunction &MF,
MachineBasicBlock &MBB,
MachineLoopInfo *MLI) {
if (MBB.empty() || !MBB.front().isPHI() || MBB.isLandingPad())
return false; // Quick exit for basic blocks without PHIs.
const MachineLoop *CurLoop = MLI ? MLI->getLoopFor(&MBB) : 0;
bool IsLoopHeader = CurLoop && &MBB == CurLoop->getHeader();
bool Changed = false;
for (MachineBasicBlock::iterator BBI = MBB.begin(), BBE = MBB.end();
BBI != BBE && BBI->isPHI(); ++BBI) {
for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2) {
unsigned Reg = BBI->getOperand(i).getReg();
MachineBasicBlock *PreMBB = BBI->getOperand(i+1).getMBB();
// Is there a critical edge from PreMBB to MBB?
if (PreMBB->succ_size() == 1)
continue;
// Avoid splitting backedges of loops. It would introduce small
// out-of-line blocks into the loop which is very bad for code placement.
if (PreMBB == &MBB && !SplitAllCriticalEdges)
continue;
const MachineLoop *PreLoop = MLI ? MLI->getLoopFor(PreMBB) : 0;
if (IsLoopHeader && PreLoop == CurLoop && !SplitAllCriticalEdges)
continue;
// LV doesn't consider a phi use live-out, so isLiveOut only returns true
// when the source register is live-out for some other reason than a phi
// use. That means the copy we will insert in PreMBB won't be a kill, and
// there is a risk it may not be coalesced away.
//
// If the copy would be a kill, there is no need to split the edge.
if (!isLiveOutPastPHIs(Reg, PreMBB) && !SplitAllCriticalEdges)
continue;
DEBUG(dbgs() << PrintReg(Reg) << " live-out before critical edge BB#"
<< PreMBB->getNumber() << " -> BB#" << MBB.getNumber()
<< ": " << *BBI);
// If Reg is not live-in to MBB, it means it must be live-in to some
// other PreMBB successor, and we can avoid the interference by splitting
// the edge.
//
// If Reg *is* live-in to MBB, the interference is inevitable and a copy
// is likely to be left after coalescing. If we are looking at a loop
// exiting edge, split it so we won't insert code in the loop, otherwise
// don't bother.
bool ShouldSplit = !isLiveIn(Reg, &MBB) || SplitAllCriticalEdges;
// Check for a loop exiting edge.
if (!ShouldSplit && CurLoop != PreLoop) {
DEBUG({
dbgs() << "Split wouldn't help, maybe avoid loop copies?\n";
if (PreLoop) dbgs() << "PreLoop: " << *PreLoop;
if (CurLoop) dbgs() << "CurLoop: " << *CurLoop;
});
// This edge could be entering a loop, exiting a loop, or it could be
// both: Jumping directly form one loop to the header of a sibling
// loop.
// Split unless this edge is entering CurLoop from an outer loop.
ShouldSplit = PreLoop && !PreLoop->contains(CurLoop);
}
if (!ShouldSplit)
continue;
if (!PreMBB->SplitCriticalEdge(&MBB, this)) {
DEBUG(dbgs() << "Failed to split ciritcal edge.\n");
continue;
}
Changed = true;
++NumCriticalEdgesSplit;
}
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &Fn) {
LiveIntervals& LI = getAnalysis<LiveIntervals>();
// Compute DFS numbers of each block
computeDFS(Fn);
// Determine which phi node operands need copies
for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
if (!I->empty() && I->begin()->isPHI())
processBlock(I);
// Break interferences where two different phis want to coalesce
// in the same register.
std::set<unsigned> seen;
typedef std::map<unsigned, std::map<unsigned, MachineBasicBlock*> >
RenameSetType;
for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end();
I != E; ++I) {
for (std::map<unsigned, MachineBasicBlock*>::iterator
OI = I->second.begin(), OE = I->second.end(); OI != OE; ) {
if (!seen.count(OI->first)) {
seen.insert(OI->first);
++OI;
} else {
Waiting[OI->second].insert(std::make_pair(OI->first, I->first));
unsigned reg = OI->first;
++OI;
I->second.erase(reg);
DEBUG(dbgs() << "Removing Renaming: " << reg << " -> " << I->first
<< "\n");
}
}
}
// Insert copies
// FIXME: This process should probably preserve LiveIntervals
SmallPtrSet<MachineBasicBlock*, 16> visited;
MachineDominatorTree& MDT = getAnalysis<MachineDominatorTree>();
InsertCopies(MDT.getRootNode(), visited);
// Perform renaming
for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end();
I != E; ++I)
while (I->second.size()) {
std::map<unsigned, MachineBasicBlock*>::iterator SI = I->second.begin();
DEBUG(dbgs() << "Renaming: " << SI->first << " -> " << I->first << "\n");
if (SI->first != I->first) {
if (mergeLiveIntervals(I->first, SI->first)) {
Fn.getRegInfo().replaceRegWith(SI->first, I->first);
if (RenameSets.count(SI->first)) {
I->second.insert(RenameSets[SI->first].begin(),
RenameSets[SI->first].end());
RenameSets.erase(SI->first);
}
} else {
// Insert a last-minute copy if a conflict was detected.
const TargetInstrInfo *TII = Fn.getTarget().getInstrInfo();
const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(I->first);
TII->copyRegToReg(*SI->second, SI->second->getFirstTerminator(),
I->first, SI->first, RC, RC, DebugLoc());
LI.renumber();
LiveInterval& Int = LI.getOrCreateInterval(I->first);
SlotIndex instrIdx =
LI.getInstructionIndex(--SI->second->getFirstTerminator());
if (Int.liveAt(instrIdx.getDefIndex()))
Int.removeRange(instrIdx.getDefIndex(),
LI.getMBBEndIdx(SI->second).getNextSlot(), true);
LiveRange R = LI.addLiveRangeToEndOfBlock(I->first,
--SI->second->getFirstTerminator());
R.valno->setCopy(--SI->second->getFirstTerminator());
R.valno->def = instrIdx.getDefIndex();
DEBUG(dbgs() << "Renaming failed: " << SI->first << " -> "
<< I->first << "\n");
}
}
LiveInterval& Int = LI.getOrCreateInterval(I->first);
const LiveRange* LR =
Int.getLiveRangeContaining(LI.getMBBEndIdx(SI->second));
LR->valno->setHasPHIKill(true);
I->second.erase(SI->first);
}
// Remove PHIs
std::vector<MachineInstr*> phis;
for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
for (MachineBasicBlock::iterator BI = I->begin(), BE = I->end();
BI != BE; ++BI)
if (BI->isPHI())
phis.push_back(BI);
}
for (std::vector<MachineInstr*>::iterator I = phis.begin(), E = phis.end();
I != E; ) {
MachineInstr* PInstr = *(I++);
// If this is a dead PHI node, then remove it from LiveIntervals.
unsigned DestReg = PInstr->getOperand(0).getReg();
LiveInterval& PI = LI.getInterval(DestReg);
if (PInstr->registerDefIsDead(DestReg)) {
if (PI.containsOneValue()) {
LI.removeInterval(DestReg);
} else {
SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex();
PI.removeRange(*PI.getLiveRangeContaining(idx), true);
}
} else {
// Trim live intervals of input registers. They are no longer live into
// this block if they died after the PHI. If they lived after it, don't
// trim them because they might have other legitimate uses.
for (unsigned i = 1; i < PInstr->getNumOperands(); i += 2) {
unsigned reg = PInstr->getOperand(i).getReg();
MachineBasicBlock* MBB = PInstr->getOperand(i+1).getMBB();
LiveInterval& InputI = LI.getInterval(reg);
if (MBB != PInstr->getParent() &&
InputI.liveAt(LI.getMBBStartIdx(PInstr->getParent())) &&
InputI.expiredAt(LI.getInstructionIndex(PInstr).getNextIndex()))
InputI.removeRange(LI.getMBBStartIdx(PInstr->getParent()),
LI.getInstructionIndex(PInstr),
true);
}
// If the PHI is not dead, then the valno defined by the PHI
// now has an unknown def.
SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex();
const LiveRange* PLR = PI.getLiveRangeContaining(idx);
PLR->valno->setIsPHIDef(true);
LiveRange R (LI.getMBBStartIdx(PInstr->getParent()),
PLR->start, PLR->valno);
PI.addRange(R);
}
LI.RemoveMachineInstrFromMaps(PInstr);
PInstr->eraseFromParent();
}
LI.renumber();
return true;
}
/// UpdateSuccessorsPHIs - After FromBB is tail duplicated into its predecessor
/// blocks, the successors have gained new predecessors. Update the PHI
/// instructions in them accordingly.
void
TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallSetVector<MachineBasicBlock*,8> &Succs) {
for (SmallSetVector<MachineBasicBlock*, 8>::iterator SI = Succs.begin(),
SE = Succs.end(); SI != SE; ++SI) {
MachineBasicBlock *SuccBB = *SI;
for (MachineBasicBlock::iterator II = SuccBB->begin(), EE = SuccBB->end();
II != EE; ++II) {
if (!II->isPHI())
break;
unsigned Idx = 0;
for (unsigned i = 1, e = II->getNumOperands(); i != e; i += 2) {
MachineOperand &MO = II->getOperand(i+1);
if (MO.getMBB() == FromBB) {
Idx = i;
break;
}
}
assert(Idx != 0);
MachineOperand &MO0 = II->getOperand(Idx);
unsigned Reg = MO0.getReg();
if (isDead) {
// Folded into the previous BB.
// There could be duplicate phi source entries. FIXME: Should sdisel
// or earlier pass fixed this?
for (unsigned i = II->getNumOperands()-2; i != Idx; i -= 2) {
MachineOperand &MO = II->getOperand(i+1);
if (MO.getMBB() == FromBB) {
II->RemoveOperand(i+1);
II->RemoveOperand(i);
}
}
} else
Idx = 0;
// If Idx is set, the operands at Idx and Idx+1 must be removed.
// We reuse the location to avoid expensive RemoveOperand calls.
DenseMap<unsigned,AvailableValsTy>::iterator LI=SSAUpdateVals.find(Reg);
if (LI != SSAUpdateVals.end()) {
// This register is defined in the tail block.
for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
MachineBasicBlock *SrcBB = LI->second[j].first;
// If we didn't duplicate a bb into a particular predecessor, we
// might still have added an entry to SSAUpdateVals to correcly
// recompute SSA. If that case, avoid adding a dummy extra argument
// this PHI.
if (!SrcBB->isSuccessor(SuccBB))
continue;
unsigned SrcReg = LI->second[j].second;
if (Idx != 0) {
II->getOperand(Idx).setReg(SrcReg);
II->getOperand(Idx+1).setMBB(SrcBB);
Idx = 0;
} else {
II->addOperand(MachineOperand::CreateReg(SrcReg, false));
II->addOperand(MachineOperand::CreateMBB(SrcBB));
}
}
} else {
// Live in tail block, must also be live in predecessors.
for (unsigned j = 0, ee = TDBBs.size(); j != ee; ++j) {
MachineBasicBlock *SrcBB = TDBBs[j];
if (Idx != 0) {
II->getOperand(Idx).setReg(Reg);
II->getOperand(Idx+1).setMBB(SrcBB);
Idx = 0;
} else {
II->addOperand(MachineOperand::CreateReg(Reg, false));
II->addOperand(MachineOperand::CreateMBB(SrcBB));
}
}
}
if (Idx != 0) {
II->RemoveOperand(Idx+1);
II->RemoveOperand(Idx);
}
}
}
}
MachineBasicBlock::iterator
MachineBasicBlock::SkipPHIsAndLabels(MachineBasicBlock::iterator I) {
while (I != end() && (I->isPHI() || I->isLabel() || I->isDebugValue()))
++I;
return I;
}
/// canSpeculateInstrs - Returns true if all the instructions in MBB can safely
/// be speculated. The terminators are not considered.
///
/// If instructions use any values that are defined in the head basic block,
/// the defining instructions are added to InsertAfter.
///
/// Any clobbered regunits are added to ClobberedRegUnits.
///
bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
// Reject any live-in physregs. It's probably CPSR/EFLAGS, and very hard to
// get right.
if (!MBB->livein_empty()) {
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has live-ins.\n");
return false;
}
unsigned InstrCount = 0;
// Check all instructions, except the terminators. It is assumed that
// terminators never have side effects or define any used register values.
for (MachineBasicBlock::iterator I = MBB->begin(),
E = MBB->getFirstTerminator(); I != E; ++I) {
if (I->isDebugValue())
continue;
if (++InstrCount > BlockInstrLimit && !Stress) {
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has more than "
<< BlockInstrLimit << " instructions.\n");
return false;
}
// There shouldn't normally be any phis in a single-predecessor block.
if (I->isPHI()) {
DEBUG(dbgs() << "Can't hoist: " << *I);
return false;
}
// Don't speculate loads. Note that it may be possible and desirable to
// speculate GOT or constant pool loads that are guaranteed not to trap,
// but we don't support that for now.
if (I->mayLoad()) {
DEBUG(dbgs() << "Won't speculate load: " << *I);
return false;
}
// We never speculate stores, so an AA pointer isn't necessary.
bool DontMoveAcrossStore = true;
if (!I->isSafeToMove(nullptr, DontMoveAcrossStore)) {
DEBUG(dbgs() << "Can't speculate: " << *I);
return false;
}
// Check for any dependencies on Head instructions.
for (MIOperands MO(I); MO.isValid(); ++MO) {
if (MO->isRegMask()) {
DEBUG(dbgs() << "Won't speculate regmask: " << *I);
return false;
}
if (!MO->isReg())
continue;
unsigned Reg = MO->getReg();
// Remember clobbered regunits.
if (MO->isDef() && TargetRegisterInfo::isPhysicalRegister(Reg))
for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
ClobberedRegUnits.set(*Units);
if (!MO->readsReg() || !TargetRegisterInfo::isVirtualRegister(Reg))
continue;
MachineInstr *DefMI = MRI->getVRegDef(Reg);
if (!DefMI || DefMI->getParent() != Head)
continue;
if (InsertAfter.insert(DefMI).second)
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " depends on " << *DefMI);
if (DefMI->isTerminator()) {
DEBUG(dbgs() << "Can't insert instructions below terminator.\n");
return false;
}
}
}
return true;
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
SmallPtrSet<MachineBasicBlock*, 8> Reachable;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> >
I = df_ext_begin(&F, Reachable), E = df_ext_end(&F, Reachable);
I != E; ++I)
/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
}
if (phi->getNumOperands() == 3) {
unsigned Input = phi->getOperand(1).getReg();
unsigned Output = phi->getOperand(0).getReg();
MachineInstr* temp = phi;
++phi;
temp->eraseFromParent();
if (Input != Output)
F.getRegInfo().replaceRegWith(Output, Input);
continue;
}
++phi;
}
}
F.RenumberBlocks();
return DeadBlocks.size();
}