bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) {
bool Modified = false;
// Yes, CPSR could be livein.
bool LiveCPSR = MBB.isLiveIn(ARM::CPSR);
MachineInstr *BundleMI = 0;
CPSRDef = 0;
HighLatencyCPSR = false;
// Check predecessors for the latest CPSRDef.
bool HasBackEdges = false;
for (MachineBasicBlock::pred_iterator
I = MBB.pred_begin(), E = MBB.pred_end(); I != E; ++I) {
const MBBInfo &PInfo = BlockInfo[(*I)->getNumber()];
if (!PInfo.Visited) {
// Since blocks are visited in RPO, this must be a back-edge.
HasBackEdges = true;
continue;
}
if (PInfo.HighLatencyCPSR) {
HighLatencyCPSR = true;
break;
}
}
// If this BB loops back to itself, conservatively avoid narrowing the
// first instruction that does partial flag update.
bool IsSelfLoop = MBB.isSuccessor(&MBB);
MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),E = MBB.instr_end();
MachineBasicBlock::instr_iterator NextMII;
for (; MII != E; MII = NextMII) {
NextMII = llvm::next(MII);
MachineInstr *MI = &*MII;
if (MI->isBundle()) {
BundleMI = MI;
continue;
}
if (MI->isDebugValue())
continue;
LiveCPSR = UpdateCPSRUse(*MI, LiveCPSR);
// Does NextMII belong to the same bundle as MI?
bool NextInSameBundle = NextMII != E && NextMII->isBundledWithPred();
if (ReduceMI(MBB, MI, LiveCPSR, IsSelfLoop)) {
Modified = true;
MachineBasicBlock::instr_iterator I = prior(NextMII);
MI = &*I;
// Removing and reinserting the first instruction in a bundle will break
// up the bundle. Fix the bundling if it was broken.
if (NextInSameBundle && !NextMII->isBundledWithPred())
NextMII->bundleWithPred();
}
if (!NextInSameBundle && MI->isInsideBundle()) {
// FIXME: Since post-ra scheduler operates on bundles, the CPSR kill
// marker is only on the BUNDLE instruction. Process the BUNDLE
// instruction as we finish with the bundled instruction to work around
// the inconsistency.
if (BundleMI->killsRegister(ARM::CPSR))
LiveCPSR = false;
MachineOperand *MO = BundleMI->findRegisterDefOperand(ARM::CPSR);
if (MO && !MO->isDead())
LiveCPSR = true;
}
bool DefCPSR = false;
LiveCPSR = UpdateCPSRDef(*MI, LiveCPSR, DefCPSR);
if (MI->isCall()) {
// Calls don't really set CPSR.
CPSRDef = 0;
HighLatencyCPSR = false;
IsSelfLoop = false;
} else if (DefCPSR) {
// This is the last CPSR defining instruction.
CPSRDef = MI;
HighLatencyCPSR = isHighLatencyCPSR(CPSRDef);
IsSelfLoop = false;
}
}
MBBInfo &Info = BlockInfo[MBB.getNumber()];
Info.HighLatencyCPSR = HighLatencyCPSR;
Info.Visited = true;
return Modified;
}
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
SlotIndex Use, unsigned PhysReg,
ArrayRef<SlotIndex> Undefs) {
unsigned UseMBBNum = UseMBB.getNumber();
// Block numbers where LR should be live-in.
SmallVector<unsigned, 16> WorkList(1, UseMBBNum);
// Remember if we have seen more than one value.
bool UniqueVNI = true;
VNInfo *TheVNI = nullptr;
bool FoundUndef = false;
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]);
#ifndef NDEBUG
if (MBB->pred_empty()) {
MBB->getParent()->verify();
errs() << "Use of " << PrintReg(PhysReg)
<< " does not have a corresponding definition on every path:\n";
const MachineInstr *MI = Indexes->getInstructionFromIndex(Use);
if (MI != nullptr)
errs() << Use << " " << *MI;
report_fatal_error("Use not jointly dominated by defs.");
}
if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
!MBB->isLiveIn(PhysReg)) {
MBB->getParent()->verify();
const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
errs() << "The register " << PrintReg(PhysReg, TRI)
<< " needs to be live in to BB#" << MBB->getNumber()
<< ", but is missing from the live-in list.\n";
report_fatal_error("Invalid global physical register");
}
#endif
FoundUndef |= MBB->pred_empty();
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
// Is this a known live-out block?
if (Seen.test(Pred->getNumber())) {
if (VNInfo *VNI = Map[Pred].first) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
continue;
}
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(Pred);
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
auto EP = LR.extendInBlock(Undefs, Start, End);
VNInfo *VNI = EP.first;
FoundUndef |= EP.second;
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
if (VNI || EP.second)
continue;
// No, we need a live-in value for Pred as well
if (Pred != &UseMBB)
WorkList.push_back(Pred->getNumber());
else
// Loopback to UseMBB, so value is really live through.
Use = SlotIndex();
}
}
LiveIn.clear();
FoundUndef |= (TheVNI == nullptr);
if (Undefs.size() > 0 && FoundUndef)
UniqueVNI = false;
// Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but
// neither require it. Skip the sorting overhead for small updates.
if (WorkList.size() > 4)
array_pod_sort(WorkList.begin(), WorkList.end());
// If a unique reaching def was found, blit in the live ranges immediately.
if (UniqueVNI) {
assert(TheVNI != nullptr);
LiveRangeUpdater Updater(&LR);
for (unsigned BN : WorkList) {
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(BN);
// Trim the live range in UseMBB.
if (BN == UseMBBNum && Use.isValid())
End = Use;
else
Map[MF->getBlockNumbered(BN)] = LiveOutPair(TheVNI, nullptr);
Updater.add(Start, End, TheVNI);
}
return true;
}
// Prepare the defined/undefined bit vectors.
auto EF = EntryInfoMap.find(&LR);
if (EF == EntryInfoMap.end()) {
unsigned N = MF->getNumBlockIDs();
EF = EntryInfoMap.insert({&LR, {BitVector(), BitVector()}}).first;
EF->second.first.resize(N);
EF->second.second.resize(N);
}
BitVector &DefOnEntry = EF->second.first;
BitVector &UndefOnEntry = EF->second.second;
// Multiple values were found, so transfer the work list to the LiveIn array
// where UpdateSSA will use it as a work list.
LiveIn.reserve(WorkList.size());
for (unsigned BN : WorkList) {
MachineBasicBlock *MBB = MF->getBlockNumbered(BN);
if (Undefs.size() > 0 && !isDefOnEntry(LR, Undefs, *MBB, DefOnEntry, UndefOnEntry))
continue;
addLiveInBlock(LR, DomTree->getNode(MBB));
if (MBB == &UseMBB)
LiveIn.back().Kill = Use;
}
return false;
}
// This is essentially the same iterative algorithm that SSAUpdater uses,
// except we already have a dominator tree, so we don't have to recompute it.
void LiveRangeCalc::updateSSA() {
assert(Indexes && "Missing SlotIndexes");
assert(DomTree && "Missing dominator tree");
// Interate until convergence.
unsigned Changes;
do {
Changes = 0;
// Propagate live-out values down the dominator tree, inserting phi-defs
// when necessary.
for (LiveInBlock &I : LiveIn) {
MachineDomTreeNode *Node = I.DomNode;
// Skip block if the live-in value has already been determined.
if (!Node)
continue;
MachineBasicBlock *MBB = Node->getBlock();
MachineDomTreeNode *IDom = Node->getIDom();
LiveOutPair IDomValue;
// We need a live-in value to a block with no immediate dominator?
// This is probably an unreachable block that has survived somehow.
bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber());
// IDom dominates all of our predecessors, but it may not be their
// immediate dominator. Check if any of them have live-out values that are
// properly dominated by IDom. If so, we need a phi-def here.
if (!needPHI) {
IDomValue = Map[IDom->getBlock()];
// Cache the DomTree node that defined the value.
if (IDomValue.first && !IDomValue.second)
Map[IDom->getBlock()].second = IDomValue.second =
DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def));
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
LiveOutPair &Value = Map[*PI];
if (!Value.first || Value.first == IDomValue.first)
continue;
// Cache the DomTree node that defined the value.
if (!Value.second)
Value.second =
DomTree->getNode(Indexes->getMBBFromIndex(Value.first->def));
// This predecessor is carrying something other than IDomValue.
// It could be because IDomValue hasn't propagated yet, or it could be
// because MBB is in the dominance frontier of that value.
if (DomTree->dominates(IDom, Value.second)) {
needPHI = true;
break;
}
}
}
// The value may be live-through even if Kill is set, as can happen when
// we are called from extendRange. In that case LiveOutSeen is true, and
// LiveOut indicates a foreign or missing value.
LiveOutPair &LOP = Map[MBB];
// Create a phi-def if required.
if (needPHI) {
++Changes;
assert(Alloc && "Need VNInfo allocator to create PHI-defs");
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(MBB);
LiveRange &LR = I.LR;
VNInfo *VNI = LR.getNextValue(Start, *Alloc);
I.Value = VNI;
// This block is done, we know the final value.
I.DomNode = nullptr;
// Add liveness since updateFromLiveIns now skips this node.
if (I.Kill.isValid()) {
if (VNI)
LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI));
} else {
if (VNI)
LR.addSegment(LiveInterval::Segment(Start, End, VNI));
LOP = LiveOutPair(VNI, Node);
}
} else if (IDomValue.first) {
// No phi-def here. Remember incoming value.
I.Value = IDomValue.first;
// If the IDomValue is killed in the block, don't propagate through.
if (I.Kill.isValid())
continue;
// Propagate IDomValue if it isn't killed:
// MBB is live-out and doesn't define its own value.
if (LOP.first == IDomValue.first)
continue;
++Changes;
LOP = IDomValue;
}
}
} while (Changes);
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
SmallPtrSet<MachineBasicBlock*, 8> Reachable;
bool ModifiedPHI = false;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
ModifiedPHI = true;
}
if (phi->getNumOperands() == 3) {
unsigned Input = phi->getOperand(1).getReg();
unsigned Output = phi->getOperand(0).getReg();
MachineInstr* temp = phi;
++phi;
temp->eraseFromParent();
ModifiedPHI = true;
if (Input != Output) {
MachineRegisterInfo &MRI = F.getRegInfo();
MRI.constrainRegClass(Input, MRI.getRegClass(Output));
MRI.replaceRegWith(Output, Input);
}
continue;
}
++phi;
}
}
F.RenumberBlocks();
return (DeadBlocks.size() || ModifiedPHI);
}
void StackColoring::calculateLocalLiveness() {
// Perform a standard reverse dataflow computation to solve for
// global liveness. The BEGIN set here is equivalent to KILL in the standard
// formulation, and END is equivalent to GEN. The result of this computation
// is a map from blocks to bitvectors where the bitvectors represent which
// allocas are live in/out of that block.
SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
BasicBlockNumbering.end());
unsigned NumSSMIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumSSMIters;
SmallPtrSet<MachineBasicBlock*, 8> NextBBSet;
for (SmallVector<MachineBasicBlock*, 8>::iterator
PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
PI != PE; ++PI) {
MachineBasicBlock *BB = *PI;
if (!BBSet.count(BB)) continue;
BitVector LocalLiveIn;
BitVector LocalLiveOut;
// Forward propagation from begins to ends.
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
LocalLiveIn |= BlockLiveness[*PI].LiveOut;
LocalLiveIn |= BlockLiveness[BB].End;
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// Reverse propagation from ends to begins.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
LocalLiveOut |= BlockLiveness[*SI].LiveIn;
LocalLiveOut |= BlockLiveness[BB].Begin;
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn |= LocalLiveOut;
LocalLiveOut |= LocalLiveIn;
// After adopting the live bits, we need to turn-off the bits which
// are de-activated in this block.
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// If we have both BEGIN and END markers in the same basic block then
// we know that the BEGIN marker comes after the END, because we already
// handle the case where the BEGIN comes before the END when collecting
// the markers (and building the BEGIN/END vectore).
// Want to enable the LIVE_IN and LIVE_OUT of slots that have both
// BEGIN and END because it means that the value lives before and after
// this basic block.
BitVector LocalEndBegin = BlockLiveness[BB].End;
LocalEndBegin &= BlockLiveness[BB].Begin;
LocalLiveIn |= LocalEndBegin;
LocalLiveOut |= LocalEndBegin;
if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) {
changed = true;
BlockLiveness[BB].LiveIn |= LocalLiveIn;
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
NextBBSet.insert(*PI);
}
if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) {
changed = true;
BlockLiveness[BB].LiveOut |= LocalLiveOut;
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
NextBBSet.insert(*SI);
}
}
BBSet = NextBBSet;
}// while changed.
}
MachineBasicBlock& LoopSplitter::insertPreHeader(MachineLoop &loop) {
assert(loop.getLoopPreheader() == 0 && "Loop already has preheader.");
MachineBasicBlock &header = *loop.getHeader();
// Save the preds - we'll need to update them once we insert the preheader.
typedef std::set<MachineBasicBlock*> HeaderPreds;
HeaderPreds headerPreds;
for (MachineBasicBlock::pred_iterator predItr = header.pred_begin(),
predEnd = header.pred_end();
predItr != predEnd; ++predItr) {
if (!loop.contains(*predItr))
headerPreds.insert(*predItr);
}
assert(!headerPreds.empty() && "No predecessors for header?");
//dbgs() << fqn << " MBB#" << header.getNumber() << " inserting preheader...";
MachineBasicBlock *preHeader =
mf->CreateMachineBasicBlock(header.getBasicBlock());
assert(preHeader != 0 && "Failed to create pre-header.");
mf->insert(header, preHeader);
for (HeaderPreds::iterator hpItr = headerPreds.begin(),
hpEnd = headerPreds.end();
hpItr != hpEnd; ++hpItr) {
assert(*hpItr != 0 && "How'd a null predecessor get into this set?");
MachineBasicBlock &hp = **hpItr;
hp.ReplaceUsesOfBlockWith(&header, preHeader);
}
preHeader->addSuccessor(&header);
MachineBasicBlock *oldLayoutPred =
llvm::prior(MachineFunction::iterator(preHeader));
if (oldLayoutPred != 0) {
updateTerminators(*oldLayoutPred);
}
lis->InsertMBBInMaps(preHeader);
if (MachineLoop *parentLoop = loop.getParentLoop()) {
assert(parentLoop->getHeader() != loop.getHeader() &&
"Parent loop has same header?");
parentLoop->addBasicBlockToLoop(preHeader, mli->getBase());
// Invalidate all parent loop ranges.
while (parentLoop != 0) {
loopRangeMap.erase(parentLoop);
parentLoop = parentLoop->getParentLoop();
}
}
for (LiveIntervals::iterator liItr = lis->begin(),
liEnd = lis->end();
liItr != liEnd; ++liItr) {
LiveInterval &li = *liItr->second;
// Is this safe for physregs?
// TargetRegisterInfo::isPhysicalRegister(li.reg) ||
if (!lis->isLiveInToMBB(li, &header))
continue;
if (lis->isLiveInToMBB(li, preHeader)) {
assert(lis->isLiveOutOfMBB(li, preHeader) &&
"Range terminates in newly added preheader?");
continue;
}
bool insertRange = false;
for (MachineBasicBlock::pred_iterator predItr = preHeader->pred_begin(),
predEnd = preHeader->pred_end();
predItr != predEnd; ++predItr) {
MachineBasicBlock *predMBB = *predItr;
if (lis->isLiveOutOfMBB(li, predMBB)) {
insertRange = true;
break;
}
}
if (!insertRange)
continue;
SlotIndex newDefIdx = lis->getMBBStartIdx(preHeader);
assert(lis->getInstructionFromIndex(newDefIdx) == 0 &&
"PHI def index points at actual instruction.");
VNInfo *newVal = li.getNextValue(newDefIdx, 0, lis->getVNInfoAllocator());
li.addRange(LiveRange(lis->getMBBStartIdx(preHeader),
lis->getMBBEndIdx(preHeader),
newVal));
}
//dbgs() << "Dumping SlotIndexes:\n";
//sis->dump();
//dbgs() << "done. (Added MBB#" << preHeader->getNumber() << ")\n";
return *preHeader;
}
VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI,
MachineBasicBlock *KillMBB,
SlotIndex Kill,
unsigned PhysReg) {
// Blocks where LI should be live-in.
SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB);
// Remember if we have seen more than one value.
bool UniqueVNI = true;
VNInfo *TheVNI = 0;
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = WorkList[i];
#ifndef NDEBUG
if (MBB->pred_empty()) {
MBB->getParent()->verify();
llvm_unreachable("Use not jointly dominated by defs.");
}
if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
!MBB->isLiveIn(PhysReg)) {
MBB->getParent()->verify();
errs() << "The register needs to be live in to BB#" << MBB->getNumber()
<< ", but is missing from the live-in list.\n";
llvm_unreachable("Invalid global physical register");
}
#endif
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
// Is this a known live-out block?
if (Seen.test(Pred->getNumber())) {
if (VNInfo *VNI = LiveOut[Pred].first) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
continue;
}
SlotIndex Start, End;
tie(Start, End) = Indexes->getMBBRange(Pred);
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
VNInfo *VNI = LI->extendInBlock(Start, End);
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
continue;
}
// No, we need a live-in value for Pred as well
if (Pred != KillMBB)
WorkList.push_back(Pred);
else
// Loopback to KillMBB, so value is really live through.
Kill = SlotIndex();
}
}
// Transfer WorkList to LiveInBlocks in reverse order.
// This ordering works best with updateSSA().
LiveIn.clear();
LiveIn.reserve(WorkList.size());
while(!WorkList.empty())
addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val()));
// The kill block may not be live-through.
assert(LiveIn.back().DomNode->getBlock() == KillMBB);
LiveIn.back().Kill = Kill;
return UniqueVNI ? TheVNI : 0;
}
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &KillMBB,
SlotIndex Kill, unsigned PhysReg) {
unsigned KillMBBNum = KillMBB.getNumber();
// Block numbers where LR should be live-in.
SmallVector<unsigned, 16> WorkList(1, KillMBBNum);
// Remember if we have seen more than one value.
bool UniqueVNI = true;
VNInfo *TheVNI = 0;
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]);
#ifndef NDEBUG
if (MBB->pred_empty()) {
MBB->getParent()->verify();
llvm_unreachable("Use not jointly dominated by defs.");
}
if (TargetRegisterInfo::isPhysicalRegister(PhysReg) &&
!MBB->isLiveIn(PhysReg)) {
MBB->getParent()->verify();
errs() << "The register needs to be live in to BB#" << MBB->getNumber()
<< ", but is missing from the live-in list.\n";
llvm_unreachable("Invalid global physical register");
}
#endif
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
// Is this a known live-out block?
if (Seen.test(Pred->getNumber())) {
if (VNInfo *VNI = LiveOut[Pred].first) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
}
continue;
}
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(Pred);
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
VNInfo *VNI = LR.extendInBlock(Start, End);
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
continue;
}
// No, we need a live-in value for Pred as well
if (Pred != &KillMBB)
WorkList.push_back(Pred->getNumber());
else
// Loopback to KillMBB, so value is really live through.
Kill = SlotIndex();
}
}
LiveIn.clear();
// Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but
// neither require it. Skip the sorting overhead for small updates.
if (WorkList.size() > 4)
array_pod_sort(WorkList.begin(), WorkList.end());
// If a unique reaching def was found, blit in the live ranges immediately.
if (UniqueVNI) {
LiveRangeUpdater Updater(&LR);
for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(),
E = WorkList.end(); I != E; ++I) {
SlotIndex Start, End;
std::tie(Start, End) = Indexes->getMBBRange(*I);
// Trim the live range in KillMBB.
if (*I == KillMBBNum && Kill.isValid())
End = Kill;
else
LiveOut[MF->getBlockNumbered(*I)] =
LiveOutPair(TheVNI, (MachineDomTreeNode *)0);
Updater.add(Start, End, TheVNI);
}
return true;
}
// Multiple values were found, so transfer the work list to the LiveIn array
// where UpdateSSA will use it as a work list.
LiveIn.reserve(WorkList.size());
for (SmallVectorImpl<unsigned>::const_iterator
I = WorkList.begin(), E = WorkList.end(); I != E; ++I) {
MachineBasicBlock *MBB = MF->getBlockNumbered(*I);
addLiveInBlock(LR, DomTree->getNode(MBB));
if (MBB == &KillMBB)
LiveIn.back().Kill = Kill;
}
return false;
}
/// EliminateUnconditionalJumpsToTop - Move blocks which unconditionally jump
/// to the loop top to the top of the loop so that they have a fall through.
/// This can introduce a branch on entry to the loop, but it can eliminate a
/// branch within the loop. See the @simple case in
/// test/CodeGen/X86/loop_blocks.ll for an example of this.
bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF,
MachineLoop *L) {
bool Changed = false;
MachineBasicBlock *TopMBB = L->getTopBlock();
bool BotHasFallthrough = HasFallthrough(L->getBottomBlock());
if (TopMBB == MF.begin() ||
HasAnalyzableTerminator(prior(MachineFunction::iterator(TopMBB)))) {
new_top:
for (MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(),
PE = TopMBB->pred_end(); PI != PE; ++PI) {
MachineBasicBlock *Pred = *PI;
if (Pred == TopMBB) continue;
if (HasFallthrough(Pred)) continue;
if (!L->contains(Pred)) continue;
// Verify that we can analyze all the loop entry edges before beginning
// any changes which will require us to be able to analyze them.
if (Pred == MF.begin())
continue;
if (!HasAnalyzableTerminator(Pred))
continue;
if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Pred))))
continue;
// Move the block.
DEBUG(dbgs() << "CGP: Moving blocks starting at BB#" << Pred->getNumber()
<< " to top of loop.\n");
Changed = true;
// Move it and all the blocks that can reach it via fallthrough edges
// exclusively, to keep existing fallthrough edges intact.
MachineFunction::iterator Begin = Pred;
MachineFunction::iterator End = llvm::next(Begin);
while (Begin != MF.begin()) {
MachineFunction::iterator Prior = prior(Begin);
if (Prior == MF.begin())
break;
// Stop when a non-fallthrough edge is found.
if (!HasFallthrough(Prior))
break;
// Stop if a block which could fall-through out of the loop is found.
if (Prior->isSuccessor(End))
break;
// If we've reached the top, stop scanning.
if (Prior == MachineFunction::iterator(TopMBB)) {
// We know top currently has a fall through (because we just checked
// it) which would be lost if we do the transformation, so it isn't
// worthwhile to do the transformation unless it would expose a new
// fallthrough edge.
if (!Prior->isSuccessor(End))
goto next_pred;
// Otherwise we can stop scanning and procede to move the blocks.
break;
}
// If we hit a switch or something complicated, don't move anything
// for this predecessor.
if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Prior))))
break;
// Ok, the block prior to Begin will be moved along with the rest.
// Extend the range to include it.
Begin = Prior;
++NumIntraMoved;
}
// Move the blocks.
Splice(MF, TopMBB, Begin, End);
// Update TopMBB.
TopMBB = L->getTopBlock();
// We have a new loop top. Iterate on it. We shouldn't have to do this
// too many times if BranchFolding has done a reasonable job.
goto new_top;
next_pred:;
}
}
// If the loop previously didn't exit with a fall-through and it now does,
// we eliminated a branch.
if (Changed &&
!BotHasFallthrough &&
HasFallthrough(L->getBottomBlock())) {
++NumIntraElim;
}
return Changed;
}
/// \brief Merge a chain with any viable successor.
///
/// This routine walks the predecessors of the current block, looking for
/// viable merge candidates. It has strict rules it uses to determine when
/// a predecessor can be merged with the current block, which center around
/// preserving the CFG structure. It performs the merge if any viable candidate
/// is found.
void MachineBlockPlacement::mergeSuccessor(MachineBasicBlock *BB,
BlockChain *Chain,
BlockFilterSet *Filter) {
assert(BB);
assert(Chain);
// If this block is not at the end of its chain, it cannot merge with any
// other chain.
if (Chain && *llvm::prior(Chain->end()) != BB)
return;
// Walk through the successors looking for the highest probability edge.
MachineBasicBlock *Successor = 0;
BranchProbability BestProb = BranchProbability::getZero();
DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n");
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end();
SI != SE; ++SI) {
if (BB == *SI || (Filter && !Filter->count(*SI)))
continue;
BranchProbability SuccProb = MBPI->getEdgeProbability(BB, *SI);
DEBUG(dbgs() << " " << getBlockName(*SI) << " -> " << SuccProb << "\n");
if (!Successor || SuccProb > BestProb || (!(SuccProb < BestProb) &&
BB->isLayoutSuccessor(*SI))) {
Successor = *SI;
BestProb = SuccProb;
}
}
if (!Successor)
return;
// Grab a chain if it exists already for this successor and make sure the
// successor is at the start of the chain as we can't merge mid-chain. Also,
// if the successor chain is the same as our chain, we're already merged.
BlockChain *SuccChain = BlockToChain[Successor];
if (SuccChain && (SuccChain == Chain || Successor != *SuccChain->begin()))
return;
// We only merge chains across a CFG merge when the desired merge path is
// significantly hotter than the incoming edge. We define a hot edge more
// strictly than the BranchProbabilityInfo does, as the two predecessor
// blocks may have dramatically different incoming probabilities we need to
// account for. Therefor we use the "global" edge weight which is the
// branch's probability times the block frequency of the predecessor.
BlockFrequency MergeWeight = MBFI->getBlockFreq(BB);
MergeWeight *= MBPI->getEdgeProbability(BB, Successor);
// We only want to consider breaking the CFG when the merge weight is much
// higher (80% vs. 20%), so multiply it by 1/4. This will require the merged
// edge to be 4x more likely before we disrupt the CFG. This number matches
// the definition of "hot" in BranchProbabilityAnalysis (80% vs. 20%).
MergeWeight *= BranchProbability(1, 4);
for (MachineBasicBlock::pred_iterator PI = Successor->pred_begin(),
PE = Successor->pred_end();
PI != PE; ++PI) {
if (BB == *PI || Successor == *PI) continue;
BlockFrequency PredWeight = MBFI->getBlockFreq(*PI);
PredWeight *= MBPI->getEdgeProbability(*PI, Successor);
// Return on the first predecessor we find which outstrips our merge weight.
if (MergeWeight < PredWeight)
return;
DEBUG(dbgs() << "Breaking CFG edge!\n"
<< " Edge from " << getBlockNum(BB) << " to "
<< getBlockNum(Successor) << ": " << MergeWeight << "\n"
<< " vs. " << getBlockNum(BB) << " to "
<< getBlockNum(*PI) << ": " << PredWeight << "\n");
}
DEBUG(dbgs() << "Merging from " << getBlockNum(BB) << " to "
<< getBlockNum(Successor) << "\n");
Chain->merge(Successor, SuccChain);
}
