MDNode *MDNode::getMostGenericTBAA(MDNode *A, MDNode *B) {
if (!A || !B)
return nullptr;
if (A == B)
return A;
// For struct-path aware TBAA, we use the access type of the tag.
bool StructPath = isStructPathTBAA(A) && isStructPathTBAA(B);
if (StructPath) {
A = cast_or_null<MDNode>(A->getOperand(1));
if (!A) return nullptr;
B = cast_or_null<MDNode>(B->getOperand(1));
if (!B) return nullptr;
}
SmallSetVector<MDNode *, 4> PathA;
MDNode *T = A;
while (T) {
if (PathA.count(T))
report_fatal_error("Cycle found in TBAA metadata.");
PathA.insert(T);
T = T->getNumOperands() >= 2 ? cast_or_null<MDNode>(T->getOperand(1))
: nullptr;
}
SmallSetVector<MDNode *, 4> PathB;
T = B;
while (T) {
if (PathB.count(T))
report_fatal_error("Cycle found in TBAA metadata.");
PathB.insert(T);
T = T->getNumOperands() >= 2 ? cast_or_null<MDNode>(T->getOperand(1))
: nullptr;
}
int IA = PathA.size() - 1;
int IB = PathB.size() - 1;
MDNode *Ret = nullptr;
while (IA >= 0 && IB >=0) {
if (PathA[IA] == PathB[IB])
Ret = PathA[IA];
else
break;
--IA;
--IB;
}
if (!StructPath)
return Ret;
if (!Ret)
return nullptr;
// We need to convert from a type node to a tag node.
Type *Int64 = IntegerType::get(A->getContext(), 64);
Metadata *Ops[3] = {Ret, Ret,
ConstantAsMetadata::get(ConstantInt::get(Int64, 0))};
return MDNode::get(A->getContext(), Ops);
}
MDNode *MDNode::getMostGenericTBAA(MDNode *A, MDNode *B) {
if (!A || !B)
return nullptr;
if (A == B)
return A;
// For struct-path aware TBAA, we use the access type of the tag.
assert(isStructPathTBAA(A) && isStructPathTBAA(B) &&
"Auto upgrade should have taken care of this!");
A = cast_or_null<MDNode>(MutableTBAAStructTagNode(A).getAccessType());
if (!A)
return nullptr;
B = cast_or_null<MDNode>(MutableTBAAStructTagNode(B).getAccessType());
if (!B)
return nullptr;
SmallSetVector<MDNode *, 4> PathA;
MutableTBAANode TA(A);
while (TA.getNode()) {
if (PathA.count(TA.getNode()))
report_fatal_error("Cycle found in TBAA metadata.");
PathA.insert(TA.getNode());
TA = TA.getParent();
}
SmallSetVector<MDNode *, 4> PathB;
MutableTBAANode TB(B);
while (TB.getNode()) {
if (PathB.count(TB.getNode()))
report_fatal_error("Cycle found in TBAA metadata.");
PathB.insert(TB.getNode());
TB = TB.getParent();
}
int IA = PathA.size() - 1;
int IB = PathB.size() - 1;
MDNode *Ret = nullptr;
while (IA >= 0 && IB >= 0) {
if (PathA[IA] == PathB[IB])
Ret = PathA[IA];
else
break;
--IA;
--IB;
}
if (!Ret)
return nullptr;
// We need to convert from a type node to a tag node.
Type *Int64 = IntegerType::get(A->getContext(), 64);
Metadata *Ops[3] = {Ret, Ret,
ConstantAsMetadata::get(ConstantInt::get(Int64, 0))};
return MDNode::get(A->getContext(), Ops);
}
/// Sort the blocks, taking special care to make sure that loops are not
/// interrupted by blocks not dominated by their header.
/// TODO: There are many opportunities for improving the heuristics here.
/// Explore them.
static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
const MachineDominatorTree &MDT) {
// Prepare for a topological sort: Record the number of predecessors each
// block has, ignoring loop backedges.
MF.RenumberBlocks();
SmallVector<unsigned, 16> NumPredsLeft(MF.getNumBlockIDs(), 0);
for (MachineBasicBlock &MBB : MF) {
unsigned N = MBB.pred_size();
if (MachineLoop *L = MLI.getLoopFor(&MBB))
if (L->getHeader() == &MBB)
for (const MachineBasicBlock *Pred : MBB.predecessors())
if (L->contains(Pred))
--N;
NumPredsLeft[MBB.getNumber()] = N;
}
// Topological sort the CFG, with additional constraints:
// - Between a loop header and the last block in the loop, there can be
// no blocks not dominated by the loop header.
// - It's desirable to preserve the original block order when possible.
// We use two ready lists; Preferred and Ready. Preferred has recently
// processed sucessors, to help preserve block sequences from the original
// order. Ready has the remaining ready blocks.
PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
CompareBlockNumbers>
Preferred;
PriorityQueue<MachineBasicBlock *, std::vector<MachineBasicBlock *>,
CompareBlockNumbersBackwards>
Ready;
SmallVector<Entry, 4> Loops;
for (MachineBasicBlock *MBB = &MF.front();;) {
const MachineLoop *L = MLI.getLoopFor(MBB);
if (L) {
// If MBB is a loop header, add it to the active loop list. We can't put
// any blocks that it doesn't dominate until we see the end of the loop.
if (L->getHeader() == MBB)
Loops.push_back(Entry(L));
// For each active loop the block is in, decrement the count. If MBB is
// the last block in an active loop, take it off the list and pick up any
// blocks deferred because the header didn't dominate them.
for (Entry &E : Loops)
if (E.Loop->contains(MBB) && --E.NumBlocksLeft == 0)
for (auto DeferredBlock : E.Deferred)
Ready.push(DeferredBlock);
while (!Loops.empty() && Loops.back().NumBlocksLeft == 0)
Loops.pop_back();
}
// The main topological sort logic.
for (MachineBasicBlock *Succ : MBB->successors()) {
// Ignore backedges.
if (MachineLoop *SuccL = MLI.getLoopFor(Succ))
if (SuccL->getHeader() == Succ && SuccL->contains(MBB))
continue;
// Decrement the predecessor count. If it's now zero, it's ready.
if (--NumPredsLeft[Succ->getNumber()] == 0)
Preferred.push(Succ);
}
// Determine the block to follow MBB. First try to find a preferred block,
// to preserve the original block order when possible.
MachineBasicBlock *Next = nullptr;
while (!Preferred.empty()) {
Next = Preferred.top();
Preferred.pop();
// If X isn't dominated by the top active loop header, defer it until that
// loop is done.
if (!Loops.empty() &&
!MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
Loops.back().Deferred.push_back(Next);
Next = nullptr;
continue;
}
// If Next was originally ordered before MBB, and it isn't because it was
// loop-rotated above the header, it's not preferred.
if (Next->getNumber() < MBB->getNumber() &&
(!L || !L->contains(Next) ||
L->getHeader()->getNumber() < Next->getNumber())) {
Ready.push(Next);
Next = nullptr;
continue;
}
break;
}
// If we didn't find a suitable block in the Preferred list, check the
// general Ready list.
if (!Next) {
// If there are no more blocks to process, we're done.
if (Ready.empty()) {
MaybeUpdateTerminator(MBB);
break;
}
for (;;) {
Next = Ready.top();
Ready.pop();
// If Next isn't dominated by the top active loop header, defer it until
// that loop is done.
if (!Loops.empty() &&
!MDT.dominates(Loops.back().Loop->getHeader(), Next)) {
Loops.back().Deferred.push_back(Next);
continue;
}
break;
}
}
// Move the next block into place and iterate.
Next->moveAfter(MBB);
MaybeUpdateTerminator(MBB);
MBB = Next;
}
assert(Loops.empty() && "Active loop list not finished");
MF.RenumberBlocks();
#ifndef NDEBUG
SmallSetVector<MachineLoop *, 8> OnStack;
// Insert a sentinel representing the degenerate loop that starts at the
// function entry block and includes the entire function as a "loop" that
// executes once.
OnStack.insert(nullptr);
for (auto &MBB : MF) {
assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (Loop && &MBB == Loop->getHeader()) {
// Loop header. The loop predecessor should be sorted above, and the other
// predecessors should be backedges below.
for (auto Pred : MBB.predecessors())
assert(
(Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) &&
"Loop header predecessors must be loop predecessors or backedges");
assert(OnStack.insert(Loop) && "Loops should be declared at most once.");
} else {
// Not a loop header. All predecessors should be sorted above.
for (auto Pred : MBB.predecessors())
assert(Pred->getNumber() < MBB.getNumber() &&
"Non-loop-header predecessors should be topologically sorted");
assert(OnStack.count(MLI.getLoopFor(&MBB)) &&
"Blocks must be nested in their loops");
}
while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back()))
OnStack.pop_back();
}
assert(OnStack.pop_back_val() == nullptr &&
"The function entry block shouldn't actually be a loop header");
assert(OnStack.empty() &&
"Control flow stack pushes and pops should be balanced.");
#endif
}
/// handleEndBlock - Remove dead stores to stack-allocated locations in the
/// function end block. Ex:
/// %A = alloca i32
/// ...
/// store i32 1, i32* %A
/// ret void
bool DSE::handleEndBlock(BasicBlock &BB) {
bool MadeChange = false;
// Keep track of all of the stack objects that are dead at the end of the
// function.
SmallSetVector<Value*, 16> DeadStackObjects;
// Find all of the alloca'd pointers in the entry block.
BasicBlock *Entry = BB.getParent()->begin();
for (BasicBlock::iterator I = Entry->begin(), E = Entry->end(); I != E; ++I) {
if (isa<AllocaInst>(I))
DeadStackObjects.insert(I);
// Okay, so these are dead heap objects, but if the pointer never escapes
// then it's leaked by this function anyways.
else if (isAllocLikeFn(I, TLI) && !PointerMayBeCaptured(I, true, true))
DeadStackObjects.insert(I);
}
// Treat byval or inalloca arguments the same, stores to them are dead at the
// end of the function.
for (Function::arg_iterator AI = BB.getParent()->arg_begin(),
AE = BB.getParent()->arg_end(); AI != AE; ++AI)
if (AI->hasByValOrInAllocaAttr())
DeadStackObjects.insert(AI);
// Scan the basic block backwards
for (BasicBlock::iterator BBI = BB.end(); BBI != BB.begin(); ){
--BBI;
// If we find a store, check to see if it points into a dead stack value.
if (hasMemoryWrite(BBI, TLI) && isRemovable(BBI)) {
// See through pointer-to-pointer bitcasts
SmallVector<Value *, 4> Pointers;
GetUnderlyingObjects(getStoredPointerOperand(BBI), Pointers);
// Stores to stack values are valid candidates for removal.
bool AllDead = true;
for (SmallVectorImpl<Value *>::iterator I = Pointers.begin(),
E = Pointers.end(); I != E; ++I)
if (!DeadStackObjects.count(*I)) {
AllDead = false;
break;
}
if (AllDead) {
Instruction *Dead = BBI++;
DEBUG(dbgs() << "DSE: Dead Store at End of Block:\n DEAD: "
<< *Dead << "\n Objects: ";
for (SmallVectorImpl<Value *>::iterator I = Pointers.begin(),
E = Pointers.end(); I != E; ++I) {
dbgs() << **I;
if (std::next(I) != E)
dbgs() << ", ";
}
dbgs() << '\n');
// DCE instructions only used to calculate that store.
DeleteDeadInstruction(Dead, *MD, TLI, &DeadStackObjects);
++NumFastStores;
MadeChange = true;
continue;
}
}
// Remove any dead non-memory-mutating instructions.
if (isInstructionTriviallyDead(BBI, TLI)) {
Instruction *Inst = BBI++;
DeleteDeadInstruction(Inst, *MD, TLI, &DeadStackObjects);
++NumFastOther;
MadeChange = true;
continue;
}
if (isa<AllocaInst>(BBI)) {
// Remove allocas from the list of dead stack objects; there can't be
// any references before the definition.
DeadStackObjects.remove(BBI);
continue;
}
if (CallSite CS = cast<Value>(BBI)) {
// Remove allocation function calls from the list of dead stack objects;
// there can't be any references before the definition.
if (isAllocLikeFn(BBI, TLI))
DeadStackObjects.remove(BBI);
// If this call does not access memory, it can't be loading any of our
// pointers.
if (AA->doesNotAccessMemory(CS))
continue;
// If the call might load from any of our allocas, then any store above
// the call is live.
DeadStackObjects.remove_if([&](Value *I) {
// See if the call site touches the value.
AliasAnalysis::ModRefResult A =
AA->getModRefInfo(CS, I, getPointerSize(I, *AA));
return A == AliasAnalysis::ModRef || A == AliasAnalysis::Ref;
});
// If all of the allocas were clobbered by the call then we're not going
// to find anything else to process.
if (DeadStackObjects.empty())
break;
continue;
}
AliasAnalysis::Location LoadedLoc;
// If we encounter a use of the pointer, it is no longer considered dead
if (LoadInst *L = dyn_cast<LoadInst>(BBI)) {
if (!L->isUnordered()) // Be conservative with atomic/volatile load
break;
LoadedLoc = AA->getLocation(L);
} else if (VAArgInst *V = dyn_cast<VAArgInst>(BBI)) {
LoadedLoc = AA->getLocation(V);
} else if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(BBI)) {
LoadedLoc = AA->getLocationForSource(MTI);
} else if (!BBI->mayReadFromMemory()) {
// Instruction doesn't read memory. Note that stores that weren't removed
// above will hit this case.
continue;
} else {
// Unknown inst; assume it clobbers everything.
break;
}
// Remove any allocas from the DeadPointer set that are loaded, as this
// makes any stores above the access live.
RemoveAccessedObjects(LoadedLoc, DeadStackObjects);
// If all of the allocas were clobbered by the access then we're not going
// to find anything else to process.
if (DeadStackObjects.empty())
break;
}
/// performStoreOnlyObjectElimination - Scan the graph of uses of the specified
/// object allocation. If the object does not escape and is only stored to
/// (this happens because GVN and other optimizations hoists forward substitutes
/// all stores to the object to eliminate all loads from it), then zap the
/// object and all accesses related to it.
static bool performStoreOnlyObjectElimination(CallInst &Allocation,
BasicBlock::iterator &BBI) {
DtorKind DtorInfo = analyzeDestructor(Allocation.getArgOperand(0));
// We can't delete the object if its destructor has side effects.
if (DtorInfo != DtorKind::NoSideEffects)
return false;
// Do a depth first search exploring all of the uses of the object pointer,
// following through casts, pointer adjustments etc. If we find any loads or
// any escape sites of the object, we give up. If we succeed in walking the
// entire graph of uses, we can remove the resultant set.
SmallSetVector<Instruction*, 16> InvolvedInstructions;
SmallVector<Instruction*, 16> Worklist;
Worklist.push_back(&Allocation);
// Stores - Keep track of all of the store instructions we see.
SmallVector<StoreInst*, 16> Stores;
while (!Worklist.empty()) {
Instruction *I = Worklist.pop_back_val();
// Insert the instruction into our InvolvedInstructions set. If we have
// already seen it, then don't reprocess all of the uses.
if (!InvolvedInstructions.insert(I)) continue;
// Okay, this is the first time we've seen this instruction, proceed.
switch (classifyInstruction(*I)) {
// These instructions should not reach here based on the pass ordering.
// i.e. LLVMARCOpt -> LLVMContractOpt.
case RT_RetainN:
case RT_UnknownRetainN:
case RT_BridgeRetainN:
case RT_ReleaseN:
case RT_UnknownReleaseN:
case RT_BridgeReleaseN:
llvm_unreachable("These are only created by LLVMARCContract !");
case RT_AllocObject:
// If this is a different swift_allocObject than we started with, then
// there is some computation feeding into a size or alignment computation
// that we have to keep... unless we can delete *that* entire object as
// well.
break;
case RT_NoMemoryAccessed:
// If no memory is accessed, then something is being done with the
// pointer: maybe it is bitcast or GEP'd. Since there are no side effects,
// it is perfectly fine to delete this instruction if all uses of the
// instruction are also eliminable.
if (I->mayHaveSideEffects() || isa<TerminatorInst>(I))
return false;
break;
case RT_Release:
case RT_Retain:
case RT_FixLifetime:
case RT_CheckUnowned:
// It is perfectly fine to eliminate various retains and releases of this
// object: we are zapping all accesses or none.
break;
// If this is an unknown instruction, we have more interesting things to
// consider.
case RT_Unknown:
case RT_ObjCRelease:
case RT_ObjCRetain:
case RT_UnknownRetain:
case RT_UnknownRelease:
case RT_BridgeRetain:
case RT_BridgeRelease:
case RT_RetainUnowned:
// Otherwise, this really is some unhandled instruction. Bail out.
return false;
}
// Okay, if we got here, the instruction can be eaten so-long as all of its
// uses can be. Scan through the uses and add them to the worklist for
// recursive processing.
for (auto UI = I->user_begin(), E = I->user_end(); UI != E; ++UI) {
Instruction *User = cast<Instruction>(*UI);
// Handle stores as a special case here: we want to make sure that the
// object is being stored *to*, not itself being stored (which would be an
// escape point). Since stores themselves don't have any uses, we can
// short-cut the classification scheme above.
if (StoreInst *SI = dyn_cast<StoreInst>(User)) {
// If this is a store *to* the object, we can zap it.
if (UI.getUse().getOperandNo() == StoreInst::getPointerOperandIndex()) {
InvolvedInstructions.insert(SI);
continue;
}
// Otherwise, using the object as a source (or size) is an escape.
return false;
}
if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(User)) {
// If this is a memset/memcpy/memmove *to* the object, we can zap it.
if (UI.getUse().getOperandNo() == 0) {
InvolvedInstructions.insert(MI);
continue;
}
// Otherwise, using the object as a source (or size) is an escape.
return false;
}
// Otherwise, normal instructions just go on the worklist for processing.
Worklist.push_back(User);
}
}
// Ok, we succeeded! This means we can zap all of the instructions that use
// the object. One thing we have to be careful of is to make sure that we
// don't invalidate "BBI" (the iterator the outer walk of the optimization
// pass is using, and indicates the next instruction to process). This would
// happen if we delete the instruction it is pointing to. Advance the
// iterator if that would happen.
while (InvolvedInstructions.count(&*BBI))
++BBI;
// Zap all of the instructions.
for (auto I : InvolvedInstructions) {
if (!I->use_empty())
I->replaceAllUsesWith(UndefValue::get(I->getType()));
I->eraseFromParent();
}
++NumStoreOnlyObjectsEliminated;
return true;
}
/// Remove dead stores to stack-allocated locations in the function end block.
/// Ex:
/// %A = alloca i32
/// ...
/// store i32 1, i32* %A
/// ret void
static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
MemoryDependenceResults *MD,
const TargetLibraryInfo *TLI,
InstOverlapIntervalsTy &IOL,
DenseMap<Instruction*, size_t> *InstrOrdering) {
bool MadeChange = false;
// Keep track of all of the stack objects that are dead at the end of the
// function.
SmallSetVector<Value*, 16> DeadStackObjects;
// Find all of the alloca'd pointers in the entry block.
BasicBlock &Entry = BB.getParent()->front();
for (Instruction &I : Entry) {
if (isa<AllocaInst>(&I))
DeadStackObjects.insert(&I);
// Okay, so these are dead heap objects, but if the pointer never escapes
// then it's leaked by this function anyways.
else if (isAllocLikeFn(&I, TLI) && !PointerMayBeCaptured(&I, true, true))
DeadStackObjects.insert(&I);
}
// Treat byval or inalloca arguments the same, stores to them are dead at the
// end of the function.
for (Argument &AI : BB.getParent()->args())
if (AI.hasByValOrInAllocaAttr())
DeadStackObjects.insert(&AI);
const DataLayout &DL = BB.getModule()->getDataLayout();
// Scan the basic block backwards
for (BasicBlock::iterator BBI = BB.end(); BBI != BB.begin(); ){
--BBI;
// If we find a store, check to see if it points into a dead stack value.
if (hasMemoryWrite(&*BBI, *TLI) && isRemovable(&*BBI)) {
// See through pointer-to-pointer bitcasts
SmallVector<Value *, 4> Pointers;
GetUnderlyingObjects(getStoredPointerOperand(&*BBI), Pointers, DL);
// Stores to stack values are valid candidates for removal.
bool AllDead = true;
for (Value *Pointer : Pointers)
if (!DeadStackObjects.count(Pointer)) {
AllDead = false;
break;
}
if (AllDead) {
Instruction *Dead = &*BBI;
DEBUG(dbgs() << "DSE: Dead Store at End of Block:\n DEAD: "
<< *Dead << "\n Objects: ";
for (SmallVectorImpl<Value *>::iterator I = Pointers.begin(),
E = Pointers.end(); I != E; ++I) {
dbgs() << **I;
if (std::next(I) != E)
dbgs() << ", ";
}
dbgs() << '\n');
// DCE instructions only used to calculate that store.
deleteDeadInstruction(Dead, &BBI, *MD, *TLI, IOL, InstrOrdering, &DeadStackObjects);
++NumFastStores;
MadeChange = true;
continue;
}
}
// Remove any dead non-memory-mutating instructions.
if (isInstructionTriviallyDead(&*BBI, TLI)) {
DEBUG(dbgs() << "DSE: Removing trivially dead instruction:\n DEAD: "
<< *&*BBI << '\n');
deleteDeadInstruction(&*BBI, &BBI, *MD, *TLI, IOL, InstrOrdering, &DeadStackObjects);
++NumFastOther;
MadeChange = true;
continue;
}
if (isa<AllocaInst>(BBI)) {
// Remove allocas from the list of dead stack objects; there can't be
// any references before the definition.
DeadStackObjects.remove(&*BBI);
continue;
}
if (auto CS = CallSite(&*BBI)) {
// Remove allocation function calls from the list of dead stack objects;
// there can't be any references before the definition.
if (isAllocLikeFn(&*BBI, TLI))
DeadStackObjects.remove(&*BBI);
// If this call does not access memory, it can't be loading any of our
// pointers.
if (AA->doesNotAccessMemory(CS))
continue;
// If the call might load from any of our allocas, then any store above
// the call is live.
DeadStackObjects.remove_if([&](Value *I) {
// See if the call site touches the value.
ModRefInfo A = AA->getModRefInfo(CS, I, getPointerSize(I, DL, *TLI));
return A == MRI_ModRef || A == MRI_Ref;
});
// If all of the allocas were clobbered by the call then we're not going
// to find anything else to process.
if (DeadStackObjects.empty())
break;
continue;
}
// We can remove the dead stores, irrespective of the fence and its ordering
// (release/acquire/seq_cst). Fences only constraints the ordering of
// already visible stores, it does not make a store visible to other
// threads. So, skipping over a fence does not change a store from being
// dead.
if (isa<FenceInst>(*BBI))
continue;
MemoryLocation LoadedLoc;
// If we encounter a use of the pointer, it is no longer considered dead
if (LoadInst *L = dyn_cast<LoadInst>(BBI)) {
if (!L->isUnordered()) // Be conservative with atomic/volatile load
break;
LoadedLoc = MemoryLocation::get(L);
} else if (VAArgInst *V = dyn_cast<VAArgInst>(BBI)) {
LoadedLoc = MemoryLocation::get(V);
} else if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(BBI)) {
LoadedLoc = MemoryLocation::getForSource(MTI);
} else if (!BBI->mayReadFromMemory()) {
// Instruction doesn't read memory. Note that stores that weren't removed
// above will hit this case.
continue;
} else {
// Unknown inst; assume it clobbers everything.
break;
}
// Remove any allocas from the DeadPointer set that are loaded, as this
// makes any stores above the access live.
removeAccessedObjects(LoadedLoc, DeadStackObjects, DL, AA, TLI);
// If all of the allocas were clobbered by the access then we're not going
// to find anything else to process.
if (DeadStackObjects.empty())
break;
}
/// Sort the blocks in RPO, taking special care to make sure that loops are
/// contiguous even in the case of split backedges.
///
/// TODO: Determine whether RPO is actually worthwhile, or whether we should
/// move to just a stable-topological-sort-based approach that would preserve
/// more of the original order.
static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI) {
// Note that we do our own RPO rather than using
// "llvm/ADT/PostOrderIterator.h" because we want control over the order that
// successors are visited in (see above). Also, we can sort the blocks in the
// MachineFunction as we go.
SmallPtrSet<MachineBasicBlock *, 16> Visited;
SmallVector<POStackEntry, 16> Stack;
MachineBasicBlock *EntryBlock = &*MF.begin();
Visited.insert(EntryBlock);
Stack.push_back(POStackEntry(EntryBlock, MF, MLI));
for (;;) {
POStackEntry &Entry = Stack.back();
SmallVectorImpl<MachineBasicBlock *> &Succs = Entry.Succs;
if (!Succs.empty()) {
MachineBasicBlock *Succ = Succs.pop_back_val();
if (Visited.insert(Succ).second)
Stack.push_back(POStackEntry(Succ, MF, MLI));
continue;
}
// Put the block in its position in the MachineFunction.
MachineBasicBlock &MBB = *Entry.MBB;
MBB.moveBefore(&*MF.begin());
// Branch instructions may utilize a fallthrough, so update them if a
// fallthrough has been added or removed.
if (!MBB.empty() && MBB.back().isTerminator() && !MBB.back().isBranch() &&
!MBB.back().isBarrier())
report_fatal_error(
"Non-branch terminator with fallthrough cannot yet be rewritten");
if (MBB.empty() || !MBB.back().isTerminator() || MBB.back().isBranch())
MBB.updateTerminator();
Stack.pop_back();
if (Stack.empty())
break;
}
// Now that we've sorted the blocks in RPO, renumber them.
MF.RenumberBlocks();
#ifndef NDEBUG
SmallSetVector<MachineLoop *, 8> OnStack;
// Insert a sentinel representing the degenerate loop that starts at the
// function entry block and includes the entire function as a "loop" that
// executes once.
OnStack.insert(nullptr);
for (auto &MBB : MF) {
assert(MBB.getNumber() >= 0 && "Renumbered blocks should be non-negative.");
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (Loop && &MBB == Loop->getHeader()) {
// Loop header. The loop predecessor should be sorted above, and the other
// predecessors should be backedges below.
for (auto Pred : MBB.predecessors())
assert(
(Pred->getNumber() < MBB.getNumber() || Loop->contains(Pred)) &&
"Loop header predecessors must be loop predecessors or backedges");
assert(OnStack.insert(Loop) && "Loops should be declared at most once.");
} else {
// Not a loop header. All predecessors should be sorted above.
for (auto Pred : MBB.predecessors())
assert(Pred->getNumber() < MBB.getNumber() &&
"Non-loop-header predecessors should be topologically sorted");
assert(OnStack.count(MLI.getLoopFor(&MBB)) &&
"Blocks must be nested in their loops");
}
while (OnStack.size() > 1 && &MBB == LoopBottom(OnStack.back()))
OnStack.pop_back();
}
assert(OnStack.pop_back_val() == nullptr &&
"The function entry block shouldn't actually be a loop header");
assert(OnStack.empty() &&
"Control flow stack pushes and pops should be balanced.");
#endif
}
