bool searchPredecessors(const MachineBasicBlock *MBB,
const MachineBasicBlock *CutOff,
UnaryPredicate Predicate) {
if (MBB == CutOff)
return false;
DenseSet<const MachineBasicBlock*> Visited;
SmallVector<MachineBasicBlock*, 4> Worklist(MBB->pred_begin(),
MBB->pred_end());
while (!Worklist.empty()) {
MachineBasicBlock *MBB = Worklist.pop_back_val();
if (!Visited.insert(MBB).second)
continue;
if (MBB == CutOff)
continue;
if (Predicate(MBB))
return true;
Worklist.append(MBB->pred_begin(), MBB->pred_end());
}
return false;
}
void SplitEditor::extendPHIKillRanges() {
// Extend live ranges to be live-out for successor PHI values.
for (LiveInterval::const_vni_iterator I = Edit->getParent().vni_begin(),
E = Edit->getParent().vni_end(); I != E; ++I) {
const VNInfo *PHIVNI = *I;
if (PHIVNI->isUnused() || !PHIVNI->isPHIDef())
continue;
unsigned RegIdx = RegAssign.lookup(PHIVNI->def);
LiveInterval *LI = Edit->get(RegIdx);
LiveRangeCalc &LRC = getLRCalc(RegIdx);
MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def);
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
SlotIndex End = LIS.getMBBEndIdx(*PI);
SlotIndex LastUse = End.getPrevSlot();
// The predecessor may not have a live-out value. That is OK, like an
// undef PHI operand.
if (Edit->getParent().liveAt(LastUse)) {
assert(RegAssign.lookup(LastUse) == RegIdx &&
"Different register assignment in phi predecessor");
LRC.extend(LI, End,
LIS.getSlotIndexes(), &MDT, &LIS.getVNInfoAllocator());
}
}
}
}
/// markValueUsed - Remember that VNI failed to rematerialize, so its defining
/// instruction cannot be eliminated. See through snippet copies
void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList;
WorkList.push_back(std::make_pair(LI, VNI));
do {
tie(LI, VNI) = WorkList.pop_back_val();
if (!UsedValues.insert(VNI))
continue;
if (VNI->isPHIDef()) {
MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def);
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
VNInfo *PVNI = LI->getVNInfoBefore(LIS.getMBBEndIdx(*PI));
if (PVNI)
WorkList.push_back(std::make_pair(LI, PVNI));
}
continue;
}
// Follow snippet copies.
MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def);
if (!SnippetCopies.count(MI))
continue;
LiveInterval &SnipLI = LIS.getInterval(MI->getOperand(1).getReg());
assert(isRegToSpill(SnipLI.reg) && "Unexpected register in copy");
VNInfo *SnipVNI = SnipLI.getVNInfoAt(VNI->def.getRegSlot(true));
assert(SnipVNI && "Snippet undefined before copy");
WorkList.push_back(std::make_pair(&SnipLI, SnipVNI));
} while (!WorkList.empty());
}
static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) {
for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) {
MachineBasicBlock *MBB = I;
SmallSetVector<MachineBasicBlock*, 8> Preds(MBB->pred_begin(),
MBB->pred_end());
MachineBasicBlock::iterator MI = MBB->begin();
while (MI != MBB->end()) {
if (!MI->isPHI())
break;
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
bool Found = false;
for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
if (PHIBB == PredBB) {
Found = true;
break;
}
}
if (!Found) {
dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
dbgs() << " missing input from predecessor BB#"
<< PredBB->getNumber() << '\n';
llvm_unreachable(0);
}
}
for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) {
MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB();
if (CheckExtra && !Preds.count(PHIBB)) {
dbgs() << "Warning: malformed PHI in BB#" << MBB->getNumber()
<< ": " << *MI;
dbgs() << " extra input from predecessor BB#"
<< PHIBB->getNumber() << '\n';
llvm_unreachable(0);
}
if (PHIBB->getNumber() < 0) {
dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI;
dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n';
llvm_unreachable(0);
}
}
++MI;
}
}
}
bool Filler::searchSuccBBs(MachineBasicBlock &MBB, Iter Slot) const {
if (DisableSuccBBSearch)
return false;
MachineBasicBlock *SuccBB = selectSuccBB(MBB);
if (!SuccBB)
return false;
RegDefsUses RegDU(*MBB.getParent()->getSubtarget().getRegisterInfo());
bool HasMultipleSuccs = false;
BB2BrMap BrMap;
std::unique_ptr<InspectMemInstr> IM;
Iter Filler;
auto *Fn = MBB.getParent();
// Iterate over SuccBB's predecessor list.
for (MachineBasicBlock::pred_iterator PI = SuccBB->pred_begin(),
PE = SuccBB->pred_end(); PI != PE; ++PI)
if (!examinePred(**PI, *SuccBB, RegDU, HasMultipleSuccs, BrMap))
return false;
// Do not allow moving instructions which have unallocatable register operands
// across basic block boundaries.
RegDU.setUnallocatableRegs(*Fn);
// Only allow moving loads from stack or constants if any of the SuccBB's
// predecessors have multiple successors.
if (HasMultipleSuccs) {
IM.reset(new LoadFromStackOrConst());
} else {
const MachineFrameInfo *MFI = Fn->getFrameInfo();
IM.reset(new MemDefsUses(Fn->getDataLayout(), MFI));
}
if (!searchRange(MBB, SuccBB->begin(), SuccBB->end(), RegDU, *IM, Slot,
Filler))
return false;
insertDelayFiller(Filler, BrMap);
addLiveInRegs(Filler, *SuccBB);
Filler->eraseFromParent();
return true;
}
/// getCanonicalInductionVariable - Check to see if the loop has a canonical
/// induction variable. We check for a simple recurrence pattern - an
/// integer recurrence that decrements by one each time through the loop and
/// ends at zero. If so, return the phi node that corresponds to it.
///
/// Based upon the similar code in LoopInfo except this code is specific to
/// the machine.
/// This method assumes that the IndVarSimplify pass has been run by 'opt'.
///
void
PPCCTRLoops::getCanonicalInductionVariable(MachineLoop *L,
SmallVector<MachineInstr *, 4> &IVars,
SmallVector<MachineInstr *, 4> &IOps) const {
MachineBasicBlock *TopMBB = L->getTopBlock();
MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin();
assert(PI != TopMBB->pred_end() &&
"Loop must have more than one incoming edge!");
MachineBasicBlock *Backedge = *PI++;
if (PI == TopMBB->pred_end()) return; // dead loop
MachineBasicBlock *Incoming = *PI++;
if (PI != TopMBB->pred_end()) return; // multiple backedges?
// make sure there is one incoming and one backedge and determine which
// is which.
if (L->contains(Incoming)) {
if (L->contains(Backedge))
return;
std::swap(Incoming, Backedge);
} else if (!L->contains(Backedge))
return;
// Loop over all of the PHI nodes, looking for a canonical induction variable:
// - The PHI node is "reg1 = PHI reg2, BB1, reg3, BB2".
// - The recurrence comes from the backedge.
// - the definition is an induction operatio.n
for (MachineBasicBlock::iterator I = TopMBB->begin(), E = TopMBB->end();
I != E && I->isPHI(); ++I) {
MachineInstr *MPhi = &*I;
unsigned DefReg = MPhi->getOperand(0).getReg();
for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) {
// Check each operand for the value from the backedge.
MachineBasicBlock *MBB = MPhi->getOperand(i+1).getMBB();
if (L->contains(MBB)) { // operands comes from the backedge
// Check if the definition is an induction operation.
MachineInstr *DI = MRI->getVRegDef(MPhi->getOperand(i).getReg());
if (isInductionOperation(DI, DefReg)) {
IOps.push_back(DI);
IVars.push_back(MPhi);
}
}
}
}
return;
}
bool
TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) {
for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(),
PE = BB.pred_end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB->succ_size() > 1)
return false;
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
return false;
if (!PredCond.empty())
return false;
}
return true;
}
bool LembergInstrInfo::
ComplexPredecessorPredicates(MachineBasicBlock &MBB) const {
for (MachineBasicBlock::pred_iterator PI = MBB.pred_begin(), PE = MBB.pred_end();
PI != PE; ++PI) {
MachineBasicBlock *TBB = NULL, *FBB = NULL;
SmallVector<MachineOperand, 4> Cond;
AnalyzeBranch(**PI, TBB, FBB, Cond, false);
if (!Cond.empty()
&& !(Cond[1].getImm() == LembergCC::FALSE
|| Cond[1].getImm() == LembergCC::TRUE)) {
return true;
}
}
return false;
}
bool LoopSplitter::canInsertPreHeader(MachineLoop &loop) {
MachineBasicBlock *header = loop.getHeader();
MachineBasicBlock *a = 0, *b = 0;
SmallVector<MachineOperand, 4> c;
for (MachineBasicBlock::pred_iterator pbItr = header->pred_begin(),
pbEnd = header->pred_end();
pbItr != pbEnd; ++pbItr) {
MachineBasicBlock *predBlock = *pbItr;
if (!!tii->AnalyzeBranch(*predBlock, a, b, c)) {
return false;
}
}
MachineFunction::iterator headerItr(header);
if (headerItr == mf->begin())
return true;
MachineBasicBlock *headerLayoutPred = llvm::prior(headerItr);
assert(headerLayoutPred != 0 && "Header should have layout pred.");
return (!tii->AnalyzeBranch(*headerLayoutPred, a, b, c));
}
bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) {
bool Modified = false;
// Yes, CPSR could be livein.
bool LiveCPSR = MBB.isLiveIn(ARM::CPSR);
MachineInstr *BundleMI = 0;
CPSRDef = 0;
HighLatencyCPSR = false;
// Check predecessors for the latest CPSRDef.
for (MachineBasicBlock::pred_iterator
I = MBB.pred_begin(), E = MBB.pred_end(); I != E; ++I) {
const MBBInfo &PInfo = BlockInfo[(*I)->getNumber()];
if (!PInfo.Visited) {
// Since blocks are visited in RPO, this must be a back-edge.
continue;
}
if (PInfo.HighLatencyCPSR) {
HighLatencyCPSR = true;
break;
}
}
// If this BB loops back to itself, conservatively avoid narrowing the
// first instruction that does partial flag update.
bool IsSelfLoop = MBB.isSuccessor(&MBB);
MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),E = MBB.instr_end();
MachineBasicBlock::instr_iterator NextMII;
for (; MII != E; MII = NextMII) {
NextMII = llvm::next(MII);
MachineInstr *MI = &*MII;
if (MI->isBundle()) {
BundleMI = MI;
continue;
}
if (MI->isDebugValue())
continue;
LiveCPSR = UpdateCPSRUse(*MI, LiveCPSR);
// Does NextMII belong to the same bundle as MI?
bool NextInSameBundle = NextMII != E && NextMII->isBundledWithPred();
if (ReduceMI(MBB, MI, LiveCPSR, IsSelfLoop)) {
Modified = true;
MachineBasicBlock::instr_iterator I = prior(NextMII);
MI = &*I;
// Removing and reinserting the first instruction in a bundle will break
// up the bundle. Fix the bundling if it was broken.
if (NextInSameBundle && !NextMII->isBundledWithPred())
NextMII->bundleWithPred();
}
if (!NextInSameBundle && MI->isInsideBundle()) {
// FIXME: Since post-ra scheduler operates on bundles, the CPSR kill
// marker is only on the BUNDLE instruction. Process the BUNDLE
// instruction as we finish with the bundled instruction to work around
// the inconsistency.
if (BundleMI->killsRegister(ARM::CPSR))
LiveCPSR = false;
MachineOperand *MO = BundleMI->findRegisterDefOperand(ARM::CPSR);
if (MO && !MO->isDead())
LiveCPSR = true;
}
bool DefCPSR = false;
LiveCPSR = UpdateCPSRDef(*MI, LiveCPSR, DefCPSR);
if (MI->isCall()) {
// Calls don't really set CPSR.
CPSRDef = 0;
HighLatencyCPSR = false;
IsSelfLoop = false;
} else if (DefCPSR) {
// This is the last CPSR defining instruction.
CPSRDef = MI;
HighLatencyCPSR = isHighLatencyCPSR(CPSRDef);
IsSelfLoop = false;
}
}
MBBInfo &Info = BlockInfo[MBB.getNumber()];
Info.HighLatencyCPSR = HighLatencyCPSR;
Info.Visited = true;
return Modified;
}
// This is essentially the same iterative algorithm that SSAUpdater uses,
// except we already have a dominator tree, so we don't have to recompute it.
void LiveRangeCalc::updateSSA() {
assert(Indexes && "Missing SlotIndexes");
assert(DomTree && "Missing dominator tree");
// Interate until convergence.
unsigned Changes;
do {
Changes = 0;
// Propagate live-out values down the dominator tree, inserting phi-defs
// when necessary.
for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(),
E = LiveIn.end(); I != E; ++I) {
MachineDomTreeNode *Node = I->DomNode;
// Skip block if the live-in value has already been determined.
if (!Node)
continue;
MachineBasicBlock *MBB = Node->getBlock();
MachineDomTreeNode *IDom = Node->getIDom();
LiveOutPair IDomValue;
// We need a live-in value to a block with no immediate dominator?
// This is probably an unreachable block that has survived somehow.
bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber());
// IDom dominates all of our predecessors, but it may not be their
// immediate dominator. Check if any of them have live-out values that are
// properly dominated by IDom. If so, we need a phi-def here.
if (!needPHI) {
IDomValue = LiveOut[IDom->getBlock()];
// Cache the DomTree node that defined the value.
if (IDomValue.first && !IDomValue.second)
LiveOut[IDom->getBlock()].second = IDomValue.second =
DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def));
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
LiveOutPair &Value = LiveOut[*PI];
if (!Value.first || Value.first == IDomValue.first)
continue;
// Cache the DomTree node that defined the value.
if (!Value.second)
Value.second =
DomTree->getNode(Indexes->getMBBFromIndex(Value.first->def));
// This predecessor is carrying something other than IDomValue.
// It could be because IDomValue hasn't propagated yet, or it could be
// because MBB is in the dominance frontier of that value.
if (DomTree->dominates(IDom, Value.second)) {
needPHI = true;
break;
}
}
}
// The value may be live-through even if Kill is set, as can happen when
// we are called from extendRange. In that case LiveOutSeen is true, and
// LiveOut indicates a foreign or missing value.
LiveOutPair &LOP = LiveOut[MBB];
// Create a phi-def if required.
if (needPHI) {
++Changes;
assert(Alloc && "Need VNInfo allocator to create PHI-defs");
SlotIndex Start, End;
tie(Start, End) = Indexes->getMBBRange(MBB);
VNInfo *VNI = I->LI->getNextValue(Start, *Alloc);
I->Value = VNI;
// This block is done, we know the final value.
I->DomNode = 0;
// Add liveness since updateLiveIns now skips this node.
if (I->Kill.isValid())
I->LI->addRange(LiveRange(Start, I->Kill, VNI));
else {
I->LI->addRange(LiveRange(Start, End, VNI));
LOP = LiveOutPair(VNI, Node);
}
} else if (IDomValue.first) {
// No phi-def here. Remember incoming value.
I->Value = IDomValue.first;
// If the IDomValue is killed in the block, don't propagate through.
if (I->Kill.isValid())
continue;
// Propagate IDomValue if it isn't killed:
// MBB is live-out and doesn't define its own value.
if (LOP.first == IDomValue.first)
continue;
++Changes;
LOP = IDomValue;
}
}
} while (Changes);
}
VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI,
MachineBasicBlock *KillMBB,
SlotIndex Kill,
unsigned PhysReg) {
// Blocks where LI should be live-in.
SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB);
// Remember if we have seen more than one value.
bool UniqueVNI = true;
VNInfo *TheVNI = 0;
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = WorkList[i];
#ifndef NDEBUG
if (MBB->pred_empty()) {
MBB->getParent()->verify();
llvm_unreachable("Use not jointly dominated by defs.");
}
if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
!MBB->isLiveIn(PhysReg)) {
MBB->getParent()->verify();
errs() << "The register needs to be live in to BB#" << MBB->getNumber()
<< ", but is missing from the live-in list.\n";
llvm_unreachable("Invalid global physical register");
}
#endif
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
// Is this a known live-out block?
if (Seen.test(Pred->getNumber())) {
if (VNInfo *VNI = LiveOut[Pred].first) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
continue;
}
SlotIndex Start, End;
tie(Start, End) = Indexes->getMBBRange(Pred);
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
VNInfo *VNI = LI->extendInBlock(Start, End);
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
continue;
}
// No, we need a live-in value for Pred as well
if (Pred != KillMBB)
WorkList.push_back(Pred);
else
// Loopback to KillMBB, so value is really live through.
Kill = SlotIndex();
}
}
// Transfer WorkList to LiveInBlocks in reverse order.
// This ordering works best with updateSSA().
LiveIn.clear();
LiveIn.reserve(WorkList.size());
while(!WorkList.empty())
addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val()));
// The kill block may not be live-through.
assert(LiveIn.back().DomNode->getBlock() == KillMBB);
LiveIn.back().Kill = Kill;
return UniqueVNI ? TheVNI : 0;
}
void StackColoring::calculateLocalLiveness() {
// Perform a standard reverse dataflow computation to solve for
// global liveness. The BEGIN set here is equivalent to KILL in the standard
// formulation, and END is equivalent to GEN. The result of this computation
// is a map from blocks to bitvectors where the bitvectors represent which
// allocas are live in/out of that block.
SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
BasicBlockNumbering.end());
unsigned NumSSMIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumSSMIters;
SmallPtrSet<MachineBasicBlock*, 8> NextBBSet;
for (SmallVector<MachineBasicBlock*, 8>::iterator
PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
PI != PE; ++PI) {
MachineBasicBlock *BB = *PI;
if (!BBSet.count(BB)) continue;
BitVector LocalLiveIn;
BitVector LocalLiveOut;
// Forward propagation from begins to ends.
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
LocalLiveIn |= BlockLiveness[*PI].LiveOut;
LocalLiveIn |= BlockLiveness[BB].End;
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// Reverse propagation from ends to begins.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
LocalLiveOut |= BlockLiveness[*SI].LiveIn;
LocalLiveOut |= BlockLiveness[BB].Begin;
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn |= LocalLiveOut;
LocalLiveOut |= LocalLiveIn;
// After adopting the live bits, we need to turn-off the bits which
// are de-activated in this block.
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// If we have both BEGIN and END markers in the same basic block then
// we know that the BEGIN marker comes after the END, because we already
// handle the case where the BEGIN comes before the END when collecting
// the markers (and building the BEGIN/END vectore).
// Want to enable the LIVE_IN and LIVE_OUT of slots that have both
// BEGIN and END because it means that the value lives before and after
// this basic block.
BitVector LocalEndBegin = BlockLiveness[BB].End;
LocalEndBegin &= BlockLiveness[BB].Begin;
LocalLiveIn |= LocalEndBegin;
LocalLiveOut |= LocalEndBegin;
if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) {
changed = true;
BlockLiveness[BB].LiveIn |= LocalLiveIn;
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
NextBBSet.insert(*PI);
}
if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) {
changed = true;
BlockLiveness[BB].LiveOut |= LocalLiveOut;
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
NextBBSet.insert(*SI);
}
}
BBSet = NextBBSet;
}// while changed.
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
df_iterator_default_set<MachineBasicBlock*> Reachable;
bool ModifiedPHI = false;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
ModifiedPHI = true;
}
if (phi->getNumOperands() == 3) {
const MachineOperand &Input = phi->getOperand(1);
const MachineOperand &Output = phi->getOperand(0);
unsigned InputReg = Input.getReg();
unsigned OutputReg = Output.getReg();
assert(Output.getSubReg() == 0 && "Cannot have output subregister");
ModifiedPHI = true;
if (InputReg != OutputReg) {
MachineRegisterInfo &MRI = F.getRegInfo();
unsigned InputSub = Input.getSubReg();
if (InputSub == 0 &&
MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) {
MRI.replaceRegWith(OutputReg, InputReg);
} else {
// The input register to the PHI has a subregister or it can't be
// constrained to the proper register class:
// insert a COPY instead of simply replacing the output
// with the input.
const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(),
TII->get(TargetOpcode::COPY), OutputReg)
.addReg(InputReg, getRegState(Input), InputSub);
}
phi++->eraseFromParent();
}
continue;
}
++phi;
}
}
F.RenumberBlocks();
return (!DeadBlocks.empty() || ModifiedPHI);
}
bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB,
MachineFunction &MF,
const TargetRegisterInfo *TRI,
const TargetInstrInfo *TII) {
SmallPtrSet<MachineBasicBlock *, 2> SinkableBBs;
// FIXME: For now, we sink only to a successor which has a single predecessor
// so that we can directly sink COPY instructions to the successor without
// adding any new block or branch instruction.
for (MachineBasicBlock *SI : CurBB.successors())
if (!SI->livein_empty() && SI->pred_size() == 1)
SinkableBBs.insert(SI);
if (SinkableBBs.empty())
return false;
bool Changed = false;
// Track which registers have been modified and used between the end of the
// block and the current instruction.
ModifiedRegUnits.clear();
UsedRegUnits.clear();
for (auto I = CurBB.rbegin(), E = CurBB.rend(); I != E;) {
MachineInstr *MI = &*I;
++I;
if (MI->isDebugInstr())
continue;
// Do not move any instruction across function call.
if (MI->isCall())
return false;
if (!MI->isCopy() || !MI->getOperand(0).isRenamable()) {
LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits,
TRI);
continue;
}
// Track the operand index for use in Copy.
SmallVector<unsigned, 2> UsedOpsInCopy;
// Track the register number defed in Copy.
SmallVector<unsigned, 2> DefedRegsInCopy;
// Don't sink the COPY if it would violate a register dependency.
if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy,
ModifiedRegUnits, UsedRegUnits)) {
LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits,
TRI);
continue;
}
assert((!UsedOpsInCopy.empty() && !DefedRegsInCopy.empty()) &&
"Unexpect SrcReg or DefReg");
MachineBasicBlock *SuccBB =
getSingleLiveInSuccBB(CurBB, SinkableBBs, DefedRegsInCopy, TRI);
// Don't sink if we cannot find a single sinkable successor in which Reg
// is live-in.
if (!SuccBB) {
LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits,
TRI);
continue;
}
assert((SuccBB->pred_size() == 1 && *SuccBB->pred_begin() == &CurBB) &&
"Unexpected predecessor");
// Clear the kill flag if SrcReg is killed between MI and the end of the
// block.
clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI);
MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI();
performSink(*MI, *SuccBB, InsertPos);
updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy);
Changed = true;
++NumPostRACopySink;
}
return Changed;
}
/// EliminateUnconditionalJumpsToTop - Move blocks which unconditionally jump
/// to the loop top to the top of the loop so that they have a fall through.
/// This can introduce a branch on entry to the loop, but it can eliminate a
/// branch within the loop. See the @simple case in
/// test/CodeGen/X86/loop_blocks.ll for an example of this.
bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF,
MachineLoop *L) {
bool Changed = false;
MachineBasicBlock *TopMBB = L->getTopBlock();
bool BotHasFallthrough = HasFallthrough(L->getBottomBlock());
if (TopMBB == MF.begin() ||
HasAnalyzableTerminator(prior(MachineFunction::iterator(TopMBB)))) {
new_top:
for (MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(),
PE = TopMBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
if (Pred == TopMBB) continue;
if (HasFallthrough(Pred)) continue;
if (!L->contains(Pred)) continue;
// Verify that we can analyze all the loop entry edges before beginning
// any changes which will require us to be able to analyze them.
if (Pred == MF.begin())
continue;
if (!HasAnalyzableTerminator(Pred))
continue;
if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Pred))))
continue;
// Move the block.
DEBUG(dbgs() << "CGP: Moving blocks starting at BB#" << Pred->getNumber()
<< " to top of loop.\n");
Changed = true;
// Move it and all the blocks that can reach it via fallthrough edges
// exclusively, to keep existing fallthrough edges intact.
MachineFunction::iterator Begin = Pred;
MachineFunction::iterator End = llvm::next(Begin);
while (Begin != MF.begin()) {
MachineFunction::iterator Prior = prior(Begin);
if (Prior == MF.begin())
break;
// Stop when a non-fallthrough edge is found.
if (!HasFallthrough(Prior))
break;
// Stop if a block which could fall-through out of the loop is found.
if (Prior->isSuccessor(End))
break;
// If we've reached the top, stop scanning.
if (Prior == MachineFunction::iterator(TopMBB)) {
// We know top currently has a fall through (because we just checked
// it) which would be lost if we do the transformation, so it isn't
// worthwhile to do the transformation unless it would expose a new
// fallthrough edge.
if (!Prior->isSuccessor(End))
goto next_pred;
// Otherwise we can stop scanning and procede to move the blocks.
break;
}
// If we hit a switch or something complicated, don't move anything
// for this predecessor.
if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Prior))))
break;
// Ok, the block prior to Begin will be moved along with the rest.
// Extend the range to include it.
Begin = Prior;
++NumIntraMoved;
}
// Move the blocks.
Splice(MF, TopMBB, Begin, End);
// Update TopMBB.
TopMBB = L->getTopBlock();
// We have a new loop top. Iterate on it. We shouldn't have to do this
// too many times if BranchFolding has done a reasonable job.
goto new_top;
next_pred:;
}
}
// If the loop previously didn't exit with a fall-through and it now does,
// we eliminated a branch.
if (Changed &&
!BotHasFallthrough &&
HasFallthrough(L->getBottomBlock())) {
++NumIntraElim;
}
return Changed;
}
MachineBasicBlock& LoopSplitter::insertPreHeader(MachineLoop &loop) {
assert(loop.getLoopPreheader() == 0 && "Loop already has preheader.");
MachineBasicBlock &header = *loop.getHeader();
// Save the preds - we'll need to update them once we insert the preheader.
typedef std::set<MachineBasicBlock*> HeaderPreds;
HeaderPreds headerPreds;
for (MachineBasicBlock::pred_iterator predItr = header.pred_begin(),
predEnd = header.pred_end();
predItr != predEnd; ++predItr) {
if (!loop.contains(*predItr))
headerPreds.insert(*predItr);
}
assert(!headerPreds.empty() && "No predecessors for header?");
//dbgs() << fqn << " MBB#" << header.getNumber() << " inserting preheader...";
MachineBasicBlock *preHeader =
mf->CreateMachineBasicBlock(header.getBasicBlock());
assert(preHeader != 0 && "Failed to create pre-header.");
mf->insert(header, preHeader);
for (HeaderPreds::iterator hpItr = headerPreds.begin(),
hpEnd = headerPreds.end();
hpItr != hpEnd; ++hpItr) {
assert(*hpItr != 0 && "How'd a null predecessor get into this set?");
MachineBasicBlock &hp = **hpItr;
hp.ReplaceUsesOfBlockWith(&header, preHeader);
}
preHeader->addSuccessor(&header);
MachineBasicBlock *oldLayoutPred =
llvm::prior(MachineFunction::iterator(preHeader));
if (oldLayoutPred != 0) {
updateTerminators(*oldLayoutPred);
}
lis->InsertMBBInMaps(preHeader);
if (MachineLoop *parentLoop = loop.getParentLoop()) {
assert(parentLoop->getHeader() != loop.getHeader() &&
"Parent loop has same header?");
parentLoop->addBasicBlockToLoop(preHeader, mli->getBase());
// Invalidate all parent loop ranges.
while (parentLoop != 0) {
loopRangeMap.erase(parentLoop);
parentLoop = parentLoop->getParentLoop();
}
}
for (LiveIntervals::iterator liItr = lis->begin(),
liEnd = lis->end();
liItr != liEnd; ++liItr) {
LiveInterval &li = *liItr->second;
// Is this safe for physregs?
// TargetRegisterInfo::isPhysicalRegister(li.reg) ||
if (!lis->isLiveInToMBB(li, &header))
continue;
if (lis->isLiveInToMBB(li, preHeader)) {
assert(lis->isLiveOutOfMBB(li, preHeader) &&
"Range terminates in newly added preheader?");
continue;
}
bool insertRange = false;
for (MachineBasicBlock::pred_iterator predItr = preHeader->pred_begin(),
predEnd = preHeader->pred_end();
predItr != predEnd; ++predItr) {
MachineBasicBlock *predMBB = *predItr;
if (lis->isLiveOutOfMBB(li, predMBB)) {
insertRange = true;
break;
}
}
if (!insertRange)
continue;
SlotIndex newDefIdx = lis->getMBBStartIdx(preHeader);
assert(lis->getInstructionFromIndex(newDefIdx) == 0 &&
"PHI def index points at actual instruction.");
VNInfo *newVal = li.getNextValue(newDefIdx, 0, lis->getVNInfoAllocator());
li.addRange(LiveRange(lis->getMBBStartIdx(preHeader),
lis->getMBBEndIdx(preHeader),
newVal));
}
//dbgs() << "Dumping SlotIndexes:\n";
//sis->dump();
//dbgs() << "done. (Added MBB#" << preHeader->getNumber() << ")\n";
return *preHeader;
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
SmallPtrSet<MachineBasicBlock*, 8> Reachable;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> >
I = df_ext_begin(&F, Reachable), E = df_ext_end(&F, Reachable);
I != E; ++I)
/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
}
if (phi->getNumOperands() == 3) {
unsigned Input = phi->getOperand(1).getReg();
unsigned Output = phi->getOperand(0).getReg();
MachineInstr* temp = phi;
++phi;
temp->eraseFromParent();
if (Input != Output)
F.getRegInfo().replaceRegWith(Output, Input);
continue;
}
++phi;
}
}
F.RenumberBlocks();
return DeadBlocks.size();
}
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
SlotIndex Use, unsigned PhysReg) {
unsigned UseMBBNum = UseMBB.getNumber();
// Block numbers where LR should be live-in.
SmallVector<unsigned, 16> WorkList(1, UseMBBNum);
// Remember if we have seen more than one value.
bool UniqueVNI = true;
VNInfo *TheVNI = nullptr;
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]);
#ifndef NDEBUG
if (MBB->pred_empty()) {
MBB->getParent()->verify();
errs() << "Use of " << PrintReg(PhysReg)
<< " does not have a corresponding definition on every path:\n";
const MachineInstr *MI = Indexes->getInstructionFromIndex(Use);
if (MI != nullptr)
errs() << Use << " " << *MI;
llvm_unreachable("Use not jointly dominated by defs.");
}
if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
!MBB->isLiveIn(PhysReg)) {
MBB->getParent()->verify();
errs() << "The register " << PrintReg(PhysReg)
<< " needs to be live in to BB#" << MBB->getNumber()
<< ", but is missing from the live-in list.\n";
llvm_unreachable("Invalid global physical register");
}
#endif
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
// Is this a known live-out block?
if (Seen.test(Pred->getNumber())) {
if (VNInfo *VNI = Map[Pred].first) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
continue;
}
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(Pred);
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
VNInfo *VNI = LR.extendInBlock(Start, End);
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
continue;
}
// No, we need a live-in value for Pred as well
if (Pred != &UseMBB)
WorkList.push_back(Pred->getNumber());
else
// Loopback to UseMBB, so value is really live through.
Use = SlotIndex();
}
}
LiveIn.clear();
// Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but
// neither require it. Skip the sorting overhead for small updates.
if (WorkList.size() > 4)
array_pod_sort(WorkList.begin(), WorkList.end());
// If a unique reaching def was found, blit in the live ranges immediately.
if (UniqueVNI) {
LiveRangeUpdater Updater(&LR);
for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(),
E = WorkList.end(); I != E; ++I) {
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(*I);
// Trim the live range in UseMBB.
if (*I == UseMBBNum && Use.isValid())
End = Use;
else
Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr);
Updater.add(Start, End, TheVNI);
}
return true;
}
// Multiple values were found, so transfer the work list to the LiveIn array
// where UpdateSSA will use it as a work list.
LiveIn.reserve(WorkList.size());
for (SmallVectorImpl<unsigned>::const_iterator
I = WorkList.begin(), E = WorkList.end(); I != E; ++I) {
MachineBasicBlock *MBB = MF->getBlockNumbered(*I);
addLiveInBlock(LR, DomTree->getNode(MBB));
if (MBB == &UseMBB)
LiveIn.back().Kill = Use;
}
return false;
}
/// processBasicBlock - Loop over all of the instructions in the basic block,
/// inserting vzero upper instructions before function calls.
bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
MachineBasicBlock &BB) {
bool Changed = false;
unsigned BBNum = BB.getNumber();
MBB = &BB;
// Don't process already solved BBs
if (BBSolved[BBNum])
return false; // No changes
// Check the state of all predecessors
unsigned EntryState = ST_INIT;
for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(),
PE = BB.pred_end(); PI != PE; ++PI) {
EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]);
if (EntryState == ST_DIRTY)
break;
}
// The entry MBB for the function may set the inital state to dirty if
// the function receives any YMM incoming arguments
if (MBB == MF.begin()) {
EntryState = ST_CLEAN;
if (FnHasLiveInYmm)
EntryState = ST_DIRTY;
}
// The current state is initialized according to the predecessors
unsigned CurState = EntryState;
bool BBHasCall = false;
for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) {
MachineInstr *MI = I;
DebugLoc dl = I->getDebugLoc();
bool isControlFlow = MI->isCall() || MI->isReturn();
// Shortcut: don't need to check regular instructions in dirty state.
if (!isControlFlow && CurState == ST_DIRTY)
continue;
if (hasYmmReg(MI)) {
// We found a ymm-using instruction; this could be an AVX instruction,
// or it could be control flow.
CurState = ST_DIRTY;
continue;
}
// Check for control-flow out of the current function (which might
// indirectly execute SSE instructions).
if (!isControlFlow)
continue;
BBHasCall = true;
// The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX
// registers. This instruction has zero latency. In addition, the processor
// changes back to Clean state, after which execution of Intel SSE
// instructions or Intel AVX instructions has no transition penalty. Add
// the VZEROUPPER instruction before any function call/return that might
// execute SSE code.
// FIXME: In some cases, we may want to move the VZEROUPPER into a
// predecessor block.
if (CurState == ST_DIRTY) {
// Only insert the VZEROUPPER in case the entry state isn't unknown.
// When unknown, only compute the information within the block to have
// it available in the exit if possible, but don't change the block.
if (EntryState != ST_UNKNOWN) {
BuildMI(*MBB, I, dl, TII->get(X86::VZEROUPPER));
++NumVZU;
}
// After the inserted VZEROUPPER the state becomes clean again, but
// other YMM may appear before other subsequent calls or even before
// the end of the BB.
CurState = ST_CLEAN;
}
}
DEBUG(dbgs() << "MBB #" << BBNum
<< ", current state: " << CurState << '\n');
// A BB can only be considered solved when we both have done all the
// necessary transformations, and have computed the exit state. This happens
// in two cases:
// 1) We know the entry state: this immediately implies the exit state and
// all the necessary transformations.
// 2) There are no calls, and and a non-call instruction marks this block:
// no transformations are necessary, and we know the exit state.
if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN))
BBSolved[BBNum] = true;
if (CurState != BBState[BBNum])
Changed = true;
BBState[BBNum] = CurState;
return Changed;
}
/// \brief Merge a chain with any viable successor.
///
/// This routine walks the predecessors of the current block, looking for
/// viable merge candidates. It has strict rules it uses to determine when
/// a predecessor can be merged with the current block, which center around
/// preserving the CFG structure. It performs the merge if any viable candidate
/// is found.
void MachineBlockPlacement::mergeSuccessor(MachineBasicBlock *BB,
BlockChain *Chain,
BlockFilterSet *Filter) {
assert(BB);
assert(Chain);
// If this block is not at the end of its chain, it cannot merge with any
// other chain.
if (Chain && *llvm::prior(Chain->end()) != BB)
return;
// Walk through the successors looking for the highest probability edge.
MachineBasicBlock *Successor = 0;
BranchProbability BestProb = BranchProbability::getZero();
DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n");
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end();
SI != SE; ++SI) {
if (BB == *SI || (Filter && !Filter->count(*SI)))
continue;
BranchProbability SuccProb = MBPI->getEdgeProbability(BB, *SI);
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> " << SuccProb << "\n");
if (!Successor || SuccProb > BestProb || (!(SuccProb < BestProb) &&
BB->isLayoutSuccessor(*SI))) {
Successor = *SI;
BestProb = SuccProb;
}
}
if (!Successor)
return;
// Grab a chain if it exists already for this successor and make sure the
// successor is at the start of the chain as we can't merge mid-chain. Also,
// if the successor chain is the same as our chain, we're already merged.
BlockChain *SuccChain = BlockToChain[Successor];
if (SuccChain && (SuccChain == Chain || Successor != *SuccChain->begin()))
return;
// We only merge chains across a CFG merge when the desired merge path is
// significantly hotter than the incoming edge. We define a hot edge more
// strictly than the BranchProbabilityInfo does, as the two predecessor
// blocks may have dramatically different incoming probabilities we need to
// account for. Therefor we use the "global" edge weight which is the
// branch's probability times the block frequency of the predecessor.
BlockFrequency MergeWeight = MBFI->getBlockFreq(BB);
MergeWeight *= MBPI->getEdgeProbability(BB, Successor);
// We only want to consider breaking the CFG when the merge weight is much
// higher (80% vs. 20%), so multiply it by 1/4. This will require the merged
// edge to be 4x more likely before we disrupt the CFG. This number matches
// the definition of "hot" in BranchProbabilityAnalysis (80% vs. 20%).
MergeWeight *= BranchProbability(1, 4);
for (MachineBasicBlock::pred_iterator PI = Successor->pred_begin(),
PE = Successor->pred_end();
PI != PE; ++PI) {
if (BB == *PI || Successor == *PI) continue;
BlockFrequency PredWeight = MBFI->getBlockFreq(*PI);
PredWeight *= MBPI->getEdgeProbability(*PI, Successor);
// Return on the first predecessor we find which outstrips our merge weight.
if (MergeWeight < PredWeight)
return;
DEBUG(dbgs() << "Breaking CFG edge!\n"
<< " Edge from " << getBlockNum(BB) << " to "
<< getBlockNum(Successor) << ": " << MergeWeight << "\n"
<< " vs. " << getBlockNum(BB) << " to "
<< getBlockNum(*PI) << ": " << PredWeight << "\n");
}
DEBUG(dbgs() << "Merging from " << getBlockNum(BB) << " to "
<< getBlockNum(Successor) << "\n");
Chain->merge(Successor, SuccChain);
}
