/// \return true if the given block is dominated by a _slowPath branch hint.
///
/// Cache all blocks visited to avoid introducing quadratic behavior.
bool ColdBlockInfo::isCold(const SILBasicBlock *BB, int recursionDepth) {
auto I = ColdBlockMap.find(BB);
if (I != ColdBlockMap.end())
return I->second;
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DominanceInfo *DT = DA->get(const_cast<SILFunction*>(BB->getParent()));
DomTreeNode *Node = DT->getNode(const_cast<SILBasicBlock*>(BB));
// Always consider unreachable code cold.
if (!Node)
return true;
std::vector<const SILBasicBlock*> DomChain;
DomChain.push_back(BB);
bool IsCold = false;
Node = Node->getIDom();
while (Node) {
if (isSlowPath(Node->getBlock(), DomChain.back(), recursionDepth)) {
IsCold = true;
break;
}
auto I = ColdBlockMap.find(Node->getBlock());
if (I != ColdBlockMap.end()) {
IsCold = I->second;
break;
}
DomChain.push_back(Node->getBlock());
Node = Node->getIDom();
}
for (auto *ChainBB : DomChain)
ColdBlockMap[ChainBB] = IsCold;
return IsCold;
}
void TempScopInfo::buildCondition(BasicBlock *BB, BasicBlock *RegionEntry) {
BBCond Cond;
DomTreeNode *BBNode = DT->getNode(BB), *EntryNode = DT->getNode(RegionEntry);
assert(BBNode && EntryNode && "Get null node while building condition!");
// Walk up the dominance tree until reaching the entry node. Add all
// conditions on the path to BB except if BB postdominates the block
// containing the condition.
while (BBNode != EntryNode) {
BasicBlock *CurBB = BBNode->getBlock();
BBNode = BBNode->getIDom();
assert(BBNode && "BBNode should not reach the root node!");
if (PDT->dominates(CurBB, BBNode->getBlock()))
continue;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getBlock()->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
if (Br->isUnconditional())
continue;
// Is BB on the ELSE side of the branch?
bool inverted = DT->dominates(Br->getSuccessor(1), BB);
Comparison *Cmp;
buildAffineCondition(*(Br->getCondition()), inverted, &Cmp);
Cond.push_back(*Cmp);
}
if (!Cond.empty())
BBConds[BB] = Cond;
}
void ControlDependenceGraphBase::computeDependencies(Function &F, PostDominatorTree &pdt) {
root = new ControlDependenceNode();
nodes.insert(root);
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
ControlDependenceNode *bn = new ControlDependenceNode(BB);
nodes.insert(bn);
bbMap[BB] = bn;
}
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
BasicBlock *A = BB;
ControlDependenceNode *AN = bbMap[A];
for (succ_iterator succ = succ_begin(A), end = succ_end(A); succ != end; ++succ) {
BasicBlock *B = *succ;
assert(A && B);
if (A == B || !pdt.dominates(B,A)) {
BasicBlock *L = pdt.findNearestCommonDominator(A,B);
ControlDependenceNode::EdgeType type = ControlDependenceGraphBase::getEdgeType(A,B);
if (A == L) {
switch (type) {
case ControlDependenceNode::TRUE:
AN->addTrue(AN); break;
case ControlDependenceNode::FALSE:
AN->addFalse(AN); break;
case ControlDependenceNode::OTHER:
AN->addOther(AN); break;
}
AN->addParent(AN);
}
for (DomTreeNode *cur = pdt[B]; cur && cur != pdt[L]; cur = cur->getIDom()) {
ControlDependenceNode *CN = bbMap[cur->getBlock()];
switch (type) {
case ControlDependenceNode::TRUE:
AN->addTrue(CN); break;
case ControlDependenceNode::FALSE:
AN->addFalse(CN); break;
case ControlDependenceNode::OTHER:
AN->addOther(CN); break;
}
assert(CN);
CN->addParent(AN);
}
}
}
}
// ENTRY -> START
for (DomTreeNode *cur = pdt[&F.getEntryBlock()]; cur; cur = cur->getIDom()) {
if (cur->getBlock()) {
ControlDependenceNode *CN = bbMap[cur->getBlock()];
assert(CN);
root->addOther(CN); CN->addParent(root);
}
}
}
SILValue
StackAllocationPromoter::getLiveOutValue(BlockSet &PhiBlocks,
SILBasicBlock *StartBB) {
DEBUG(llvm::dbgs() << "*** Searching for a value definition.\n");
// Walk the Dom tree in search of a defining value:
for (DomTreeNode *Node = DT->getNode(StartBB); Node; Node = Node->getIDom()) {
SILBasicBlock *BB = Node->getBlock();
// If there is a store (that must come after the phi), use its value.
BlockToInstMap::iterator it = LastStoreInBlock.find(BB);
if (it != LastStoreInBlock.end())
if (auto *St = dyn_cast_or_null<StoreInst>(it->second)) {
DEBUG(llvm::dbgs() << "*** Found Store def " << *St->getSrc());
return St->getSrc();
}
// If there is a Phi definition in this block:
if (PhiBlocks.count(BB)) {
// Return the dummy instruction that represents the new value that we will
// add to the basic block.
SILValue Phi = BB->getArgument(BB->getNumArguments() - 1);
DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *Phi);
return Phi;
}
// Move to the next dominating block.
DEBUG(llvm::dbgs() << "*** Walking up the iDOM.\n");
}
DEBUG(llvm::dbgs() << "*** Could not find a Def. Using Undef.\n");
return SILUndef::get(ASI->getElementType(), ASI->getModule());
}
/// isExitBlockDominatedByBlockInLoop - This method checks to see if the
/// specified exit block of the loop is dominated by the specified block
/// that is in the body of the loop. We use these constraints to
/// dramatically limit the amount of the dominator tree that needs to be
/// searched.
bool isExitBlockDominatedByBlockInLoop(BasicBlock *ExitBlock,
BasicBlock *BlockInLoop) const {
// If the block in the loop is the loop header, it must be dominated!
BasicBlock *LoopHeader = CurLoop->getHeader();
if (BlockInLoop == LoopHeader)
return true;
DomTreeNode *BlockInLoopNode = DT->getNode(BlockInLoop);
DomTreeNode *IDom = DT->getNode(ExitBlock);
// Because the exit block is not in the loop, we know we have to get _at
// least_ its immediate dominator.
do {
// Get next Immediate Dominator.
IDom = IDom->getIDom();
// If we have got to the header of the loop, then the instructions block
// did not dominate the exit node, so we can't hoist it.
if (IDom->getBlock() == LoopHeader)
return false;
} while (IDom != BlockInLoopNode);
return true;
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst *> Calls) {
LLVM_DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< Demangle::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction *, ApplyInst *> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
ApplyInst *HoistAI =
getHoistedApplyForInitializer(AI, DT, InitF, ParentF, ParentFuncs);
// If we were unable to find anything, just go onto the next apply.
if (!HoistAI) {
continue;
}
// Otherwise, move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
LLVM_DEBUG(llvm::dbgs() << " don't hoist above availability check "
"at bb"
<< DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
LLVM_DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName()
<< "\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI << " in "
<< HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
/**
* removeUndefBranches -- remove branches with undef condition
*
* These are irrelevant to the code, so may be removed completely with their
* bodies.
*/
void FunctionStaticSlicer::removeUndefBranches(ModulePass *MP, Function &F) {
#ifdef DEBUG_SLICE
errs() << __func__ << " ============ Removing unused branches\n";
#endif
PostDominatorTree &PDT = MP->getAnalysis<PostDominatorTree>(F);
typedef llvm::SmallVector<const BasicBlock *, 10> Unsafe;
Unsafe unsafe;
for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
BasicBlock &bb = *I;
if (std::distance(succ_begin(&bb), succ_end(&bb)) <= 1)
continue;
Instruction &back = bb.back();
if (back.getOpcode() != Instruction::Br &&
back.getOpcode() != Instruction::Switch)
continue;
const Value *cond = back.getOperand(0);
if (cond->getValueID() != Value::UndefValueVal)
continue;
DomTreeNode *node = PDT.getNode(&bb);
if (!node) /* this bb is unreachable */
continue;
DomTreeNode *idom = node->getIDom();
assert(idom);
/* if (!idom)
continue;*/
BasicBlock *dest = idom->getBlock();
if (!dest) /* TODO when there are nodes with noreturn calls */
continue;
#ifdef DEBUG_SLICE
errs() << " considering branch: " << bb.getName() << '\n';
errs() << " dest=" << dest->getName() << "\n";
#endif
if (PHINode *PHI = dyn_cast<PHINode>(&dest->front()))
if (PHI->getBasicBlockIndex(&bb) == -1) {
/* TODO this is unsafe! */
unsafe.push_back(&bb);
PHI->addIncoming(Constant::getNullValue(PHI->getType()), &bb);
}
BasicBlock::iterator ii(back);
Instruction *newI = BranchInst::Create(dest);
ReplaceInstWithInst(bb.getInstList(), ii, newI);
}
for (Unsafe::const_iterator I = unsafe.begin(), E = unsafe.end();
I != E; ++I) {
const BasicBlock *bb = *I;
if (std::distance(pred_begin(bb), pred_end(bb)) > 1)
errs() << "WARNING: PHI node with added value which is zero\n";
}
#ifdef DEBUG_SLICE
errs() << __func__ << " ============ END\n";
#endif
}
void GCPtrTracker::gatherDominatingDefs(const BasicBlock *BB,
AvailableValueSet &Result,
const DominatorTree &DT) {
DomTreeNode *DTN = DT[const_cast<BasicBlock *>(BB)];
while (DTN->getIDom()) {
DTN = DTN->getIDom();
const auto &Defs = BlockMap[DTN->getBlock()]->Contribution;
Result.insert(Defs.begin(), Defs.end());
// If this block is 'Cleared', then nothing LiveIn to this block can be
// available after this block completes. Note: This turns out to be
// really important for reducing memory consuption of the initial available
// sets and thus peak memory usage by this verifier.
if (BlockMap[DTN->getBlock()]->Cleared)
return;
}
for (const Argument &A : BB->getParent()->args())
if (containsGCPtrType(A.getType()))
Result.insert(&A);
}
void RegionInfo::findRegionsWithEntry(BasicBlock *entry, BBtoBBMap *ShortCut) {
assert(entry);
DomTreeNode *N = PDT->getNode(entry);
if (!N)
return;
Region *lastRegion= 0;
BasicBlock *lastExit = entry;
// As only a BasicBlock that postdominates entry can finish a region, walk the
// post dominance tree upwards.
while ((N = getNextPostDom(N, ShortCut))) {
BasicBlock *exit = N->getBlock();
if (!exit)
break;
if (isRegion(entry, exit)) {
Region *newRegion = createRegion(entry, exit);
if (lastRegion)
newRegion->addSubRegion(lastRegion);
lastRegion = newRegion;
lastExit = exit;
}
// This can never be a region, so stop the search.
if (!DT->dominates(entry, exit))
break;
}
// Tried to create regions from entry to lastExit. Next time take a
// shortcut from entry to lastExit.
if (lastExit != entry)
insertShortCut(entry, lastExit, ShortCut);
}
SILValue
StackAllocationPromoter::getLiveInValue(BlockSet &PhiBlocks,
SILBasicBlock *BB) {
// First, check if there is a Phi value in the current block. We know that
// our loads happen before stores, so we need to first check for Phi nodes
// in the first block, but stores first in all other stores in the idom
// chain.
if (PhiBlocks.count(BB)) {
DEBUG(llvm::dbgs() << "*** Found a local Phi definition.\n");
return BB->getArgument(BB->getNumArguments() - 1);
}
if (BB->pred_empty() || !DT->getNode(BB))
return SILUndef::get(ASI->getElementType(), ASI->getModule());
// No phi for this value in this block means that the value flowing
// out of the immediate dominator reaches here.
DomTreeNode *IDom = DT->getNode(BB)->getIDom();
assert(IDom &&
"Attempt to get live-in value for alloc_stack in entry block!");
return getLiveOutValue(PhiBlocks, IDom->getBlock());
}
void TempScopInfo::buildCondition(BasicBlock *BB, Region &R) {
BasicBlock *RegionEntry = R.getEntry();
BBCond Cond;
DomTreeNode *BBNode = DT->getNode(BB), *EntryNode = DT->getNode(RegionEntry);
assert(BBNode && EntryNode && "Get null node while building condition!");
// Walk up the dominance tree until reaching the entry node. Collect all
// branching blocks on the path to BB except if BB postdominates the block
// containing the condition.
SmallVector<BasicBlock *, 4> DominatorBrBlocks;
while (BBNode != EntryNode) {
BasicBlock *CurBB = BBNode->getBlock();
BBNode = BBNode->getIDom();
assert(BBNode && "BBNode should not reach the root node!");
if (PDT->dominates(CurBB, BBNode->getBlock()))
continue;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getBlock()->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
if (Br->isUnconditional())
continue;
DominatorBrBlocks.push_back(BBNode->getBlock());
}
RegionInfo *RI = R.getRegionInfo();
// Iterate in reverse order over the dominating blocks. Until a non-affine
// branch was encountered add all conditions collected. If a non-affine branch
// was encountered, stop as we overapproximate from here on anyway.
for (auto BIt = DominatorBrBlocks.rbegin(), BEnd = DominatorBrBlocks.rend();
BIt != BEnd; BIt++) {
BasicBlock *BBNode = *BIt;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
assert(Br->isConditional() && "Assumed a conditional branch");
if (SD->isNonAffineSubRegion(RI->getRegionFor(BBNode), &R))
break;
BasicBlock *TrueBB = Br->getSuccessor(0), *FalseBB = Br->getSuccessor(1);
// Is BB on the ELSE side of the branch?
bool inverted = DT->dominates(FalseBB, BB);
// If both TrueBB and FalseBB dominate BB, one of them must be the target of
// a back-edge, i.e. a loop header.
if (inverted && DT->dominates(TrueBB, BB)) {
assert(
(DT->dominates(TrueBB, FalseBB) || DT->dominates(FalseBB, TrueBB)) &&
"One of the successors should be the loop header and dominate the"
"other!");
// It is not an invert if the FalseBB is the header.
if (DT->dominates(FalseBB, TrueBB))
inverted = false;
}
Comparison *Cmp;
buildAffineCondition(*(Br->getCondition()), inverted, &Cmp);
Cond.push_back(*Cmp);
}
if (!Cond.empty())
BBConds[BB] = Cond;
}
/// getImmediateDominator - A handy utility for the specific DominatorTree
/// query that we need here.
///
static const BasicBlock *getImmediateDominator(const BasicBlock *BB,
const DominatorTree *DT) {
DomTreeNode *Node = DT->getNode(const_cast<BasicBlock *>(BB))->getIDom();
return Node ? Node->getBlock() : 0;
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst*> Calls) {
DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< demangle_wrappers::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction*, ApplyInst*> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
auto PFI = ParentFuncs.find(ParentF);
ApplyInst *HoistAI = nullptr;
if (PFI != ParentFuncs.end()) {
// Found a replacement for this init call.
// Ensure the replacement dominates the original call site.
ApplyInst *CommonAI = PFI->second;
assert(cast<FunctionRefInst>(CommonAI->getCallee())
->getReferencedFunction() == InitF &&
"ill-formed global init call");
SILBasicBlock *DomBB =
DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
// We must not move initializers around availability-checks.
if (!isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT)) {
if (DomBB != CommonAI->getParent()) {
CommonAI->moveBefore(&*DomBB->begin());
placeFuncRef(CommonAI, DT);
// Try to hoist the existing AI again if we move it to another block,
// e.g. from a loop exit into the loop.
HoistAI = CommonAI;
}
AI->replaceAllUsesWith(CommonAI);
AI->eraseFromParent();
HasChanged = true;
}
} else {
ParentFuncs[ParentF] = AI;
// It's the first time we found a call to InitF in this function, so we
// try to hoist it out of any loop.
HoistAI = AI;
}
if (HoistAI) {
// Move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
DEBUG(llvm::dbgs() << " don't hoist above availability check at bb" <<
DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
}
else {
DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
}
}
void idenRegion::computeAntidependencePaths() {
// Iterate through every pair of antidependency
// Record all the store along the path
for (AntiDepPairs::iterator I = AntiDepPairs_.begin(), E = AntiDepPairs_.end(); I != E; I++) {
BasicBlock::iterator Load, Store;
tie(Load, Store) = *I;
// create a new path
AntiDepPaths_.resize(AntiDepPaths_.size()+1);
AntiDepPathTy &newPath = AntiDepPaths_.back();
// Always record current store
newPath.push_back(Store);
// Load and store in the same basic block
BasicBlock::iterator curInst = Store;
BasicBlock *LoadBB = Load->getParent(), *StoreBB = Store->getParent();
if (LoadBB == StoreBB && DT->dominates(Load, Store)) {
while(--curInst != Load) {
if (isa<StoreInst>(curInst))
newPath.push_back(curInst);
}
errs() << "@@@ Local BB: \n@@@ ";
printPath(newPath);
errs() << "\n";
continue;
}
// Load and store in different basic block
BasicBlock *curBB = StoreBB;
DomTreeNode *curDTNode = DT->getNode(StoreBB), *LoadDTNode = DT->getNode(LoadBB);
// loop until load is not
while (DT->dominates(LoadDTNode, curDTNode)) {
errs() << "^^^^^ Current BB is " << curBB->getName() << "\n";
BasicBlock::iterator E;
// check if Load and current node in the same BB
if (curBB == LoadBB) {
E = Load;
} else {
E = curBB->begin();
}
// scan current BB
while(curInst != E) {
if (isa<StoreInst>(--curInst)) {
newPath.push_back(curInst);
}
}
// find current Node's iDOM
curDTNode = curDTNode->getIDom();
if (curDTNode == NULL)
break;
curBB = curDTNode->getBlock();
curInst = curBB->end();
}
errs() << "@@@ Inter BB: \n@@@ ";
printPath(newPath);
errs() << "\n";
}
errs() << "Path cap is " << AntiDepPaths_.capacity() << "\n";
errs() << "Path size is " << AntiDepPaths_.size() << "\n";
errs() << "#########################################################\n";
errs() << "#################### Paths Summary ######################\n";
errs() << "#########################################################\n";
printCollection(AntiDepPaths_);
errs() << "\n";
}
void IDFCalculator<NodeTy>::calculate(
SmallVectorImpl<BasicBlock *> &PHIBlocks) {
// Use a priority queue keyed on dominator tree level so that inserted nodes
// are handled from the bottom of the dominator tree upwards.
typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair;
typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>,
less_second> IDFPriorityQueue;
IDFPriorityQueue PQ;
for (BasicBlock *BB : *DefBlocks) {
if (DomTreeNode *Node = DT.getNode(BB))
PQ.push({Node, Node->getLevel()});
}
SmallVector<DomTreeNode *, 32> Worklist;
SmallPtrSet<DomTreeNode *, 32> VisitedPQ;
SmallPtrSet<DomTreeNode *, 32> VisitedWorklist;
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dominator tree children of Root, inspecting their CFG edges with
// targets elsewhere on the dominator tree. Only targets whose level is at
// most Root's level are added to the iterated dominance frontier of the
// definition set.
Worklist.clear();
Worklist.push_back(Root);
VisitedWorklist.insert(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
BasicBlock *BB = Node->getBlock();
// Succ is the successor in the direction we are calculating IDF, so it is
// successor for IDF, and predecessor for Reverse IDF.
for (auto *Succ : children<NodeTy>(BB)) {
DomTreeNode *SuccNode = DT.getNode(Succ);
// Quickly skip all CFG edges that are also dominator tree edges instead
// of catching them below.
if (SuccNode->getIDom() == Node)
continue;
const unsigned SuccLevel = SuccNode->getLevel();
if (SuccLevel > RootLevel)
continue;
if (!VisitedPQ.insert(SuccNode).second)
continue;
BasicBlock *SuccBB = SuccNode->getBlock();
if (useLiveIn && !LiveInBlocks->count(SuccBB))
continue;
PHIBlocks.emplace_back(SuccBB);
if (!DefBlocks->count(SuccBB))
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
for (auto DomChild : *Node) {
if (VisitedWorklist.insert(DomChild).second)
Worklist.push_back(DomChild);
}
}
}
}
void LLVMDependenceGraph::computePostDominators(bool addPostDomFrontiers)
{
using namespace llvm;
// iterate over all functions
for (auto& F : getConstructedFunctions()) {
analysis::PostDominanceFrontiers<LLVMNode> pdfrontiers;
// root of post-dominator tree
LLVMBBlock *root = nullptr;
Value *val = const_cast<Value *>(F.first);
Function& f = *cast<Function>(val);
PostDominatorTree *pdtree;
#if ((LLVM_VERSION_MAJOR == 3) && (LLVM_VERSION_MINOR < 9))
pdtree = new PostDominatorTree();
// compute post-dominator tree for this function
pdtree->runOnFunction(f);
#else
PostDominatorTreeWrapperPass wrapper;
wrapper.runOnFunction(f);
pdtree = &wrapper.getPostDomTree();
#ifndef NDEBUG
wrapper.verifyAnalysis();
#endif
#endif
// add immediate post-dominator edges
auto& our_blocks = F.second->getBlocks();
bool built = false;
for (auto& it : our_blocks) {
LLVMBBlock *BB = it.second;
BasicBlock *B = cast<BasicBlock>(const_cast<Value *>(it.first));
DomTreeNode *N = pdtree->getNode(B);
// when function contains infinite loop, we're screwed
// and we don't have anything
// FIXME: just check for the root,
// don't iterate over all blocks, stupid...
if (!N)
continue;
DomTreeNode *idom = N->getIDom();
BasicBlock *idomBB = idom ? idom->getBlock() : nullptr;
built = true;
if (idomBB) {
LLVMBBlock *pb = our_blocks[idomBB];
assert(pb && "Do not have constructed BB");
BB->setIPostDom(pb);
assert(cast<BasicBlock>(BB->getKey())->getParent()
== cast<BasicBlock>(pb->getKey())->getParent()
&& "BBs are from diferent functions");
// if we do not have idomBB, then the idomBB is a root BB
} else {
// PostDominatorTree may has special root without BB set
// or it is the node without immediate post-dominator
if (!root) {
root = new LLVMBBlock();
root->setKey(nullptr);
F.second->setPostDominatorTreeRoot(root);
}
BB->setIPostDom(root);
}
}
// well, if we haven't built the pdtree, this is probably infinite loop
// that has no pdtree. Until we have anything better, just add sound control
// edges that are not so precise - to predecessors.
if (!built && addPostDomFrontiers) {
for (auto& it : our_blocks) {
LLVMBBlock *BB = it.second;
for (const LLVMBBlock::BBlockEdge& succ : BB->successors()) {
// in this case we add only the control dependencies,
// since we have no pd frontiers
BB->addControlDependence(succ.target);
}
}
}
if (addPostDomFrontiers) {
// assert(root && "BUG: must have root");
if (root)
pdfrontiers.compute(root, true /* store also control depend. */);
}
#if ((LLVM_VERSION_MAJOR == 3) && (LLVM_VERSION_MINOR < 9))
delete pdtree;
#endif
}
}
void StackAllocationPromoter::promoteAllocationToPhi() {
DEBUG(llvm::dbgs() << "*** Placing Phis for : " << *ASI);
// A list of blocks that will require new Phi values.
BlockSet PhiBlocks;
// The "piggy-bank" data-structure that we use for processing the dom-tree
// bottom-up.
NodePriorityQueue PQ;
// Collect all of the stores into the AllocStack. We know that at this point
// we have at most one store per block.
for (auto UI = ASI->use_begin(), E = ASI->use_end(); UI != E; ++UI) {
SILInstruction *II = UI->getUser();
// We need to place Phis for this block.
if (isa<StoreInst>(II)) {
// If the block is in the dom tree (dominated by the entry block).
if (DomTreeNode *Node = DT->getNode(II->getParent()))
PQ.push(std::make_pair(Node, DomTreeLevels[Node]));
}
}
DEBUG(llvm::dbgs() << "*** Found: " << PQ.size() << " Defs\n");
// A list of nodes for which we already calculated the dominator frontier.
llvm::SmallPtrSet<DomTreeNode *, 32> Visited;
SmallVector<DomTreeNode *, 32> Worklist;
// Scan all of the definitions in the function bottom-up using the priority
// queue.
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dom tree children of Root, inspecting their successors. Only
// J-edges, whose target level is at most Root's level are added to the
// dominance frontier.
Worklist.clear();
Worklist.push_back(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
SILBasicBlock *BB = Node->getBlock();
// For all successors of the node:
for (auto &Succ : BB->getSuccessors()) {
DomTreeNode *SuccNode = DT->getNode(Succ);
// Skip D-edges (edges that are dom-tree edges).
if (SuccNode->getIDom() == Node)
continue;
// Ignore J-edges that point to nodes that are not smaller or equal
// to the root level.
unsigned SuccLevel = DomTreeLevels[SuccNode];
if (SuccLevel > RootLevel)
continue;
// Ignore visited nodes.
if (!Visited.insert(SuccNode).second)
continue;
// If the new PHInode is not dominated by the allocation then it's dead.
if (!DT->dominates(ASI->getParent(), SuccNode->getBlock()))
continue;
// If the new PHInode is properly dominated by the deallocation then it
// is obviously a dead PHInode, so we don't need to insert it.
if (DSI && DT->properlyDominates(DSI->getParent(),
SuccNode->getBlock()))
continue;
// The successor node is a new PHINode. If this is a new PHI node
// then it may require additional definitions, so add it to the PQ.
if (PhiBlocks.insert(Succ).second)
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
// Add the children in the dom-tree to the worklist.
for (auto CI = Node->begin(), CE = Node->end(); CI != CE; ++CI)
if (!Visited.count(*CI))
Worklist.push_back(*CI);
}
}
DEBUG(llvm::dbgs() << "*** Found: " << PhiBlocks.size() << " new PHIs\n");
NumPhiPlaced += PhiBlocks.size();
// At this point we calculated the locations of all of the new Phi values.
// Next, add the Phi values and promote all of the loads and stores into the
// new locations.
// Replace the dummy values with new block arguments.
addBlockArguments(PhiBlocks);
// Hook up the Phi nodes, loads, and debug_value_addr with incoming values.
fixBranchesAndUses(PhiBlocks);
DEBUG(llvm::dbgs() << "*** Finished placing Phis ***\n");
}
/// Clones the body of the loop L, putting it between \p InsertTop and \p
/// InsertBot.
/// \param IterNumber The serial number of the iteration currently being
/// peeled off.
/// \param Exit The exit block of the original loop.
/// \param[out] NewBlocks A list of the blocks in the newly created clone
/// \param[out] VMap The value map between the loop and the new clone.
/// \param LoopBlocks A helper for DFS-traversal of the loop.
/// \param LVMap A value-map that maps instructions from the original loop to
/// instructions in the last peeled-off iteration.
static void cloneLoopBlocks(Loop *L, unsigned IterNumber, BasicBlock *InsertTop,
BasicBlock *InsertBot, BasicBlock *Exit,
SmallVectorImpl<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
ValueToValueMapTy &LVMap, DominatorTree *DT,
LoopInfo *LI) {
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *PreHeader = L->getLoopPreheader();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *ParentLoop = L->getParentLoop();
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".peel", F);
NewBlocks.push_back(NewBB);
if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewBB, *LI);
VMap[*BB] = NewBB;
// If dominator tree is available, insert nodes to represent cloned blocks.
if (DT) {
if (Header == *BB)
DT->addNewBlock(NewBB, InsertTop);
else {
DomTreeNode *IDom = DT->getNode(*BB)->getIDom();
// VMap must contain entry for IDom, as the iteration order is RPO.
DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDom->getBlock()]));
}
}
}
// Hook-up the control flow for the newly inserted blocks.
// The new header is hooked up directly to the "top", which is either
// the original loop preheader (for the first iteration) or the previous
// iteration's exiting block (for every other iteration)
InsertTop->getTerminator()->setSuccessor(0, cast<BasicBlock>(VMap[Header]));
// Similarly, for the latch:
// The original exiting edge is still hooked up to the loop exit.
// The backedge now goes to the "bottom", which is either the loop's real
// header (for the last peeled iteration) or the copied header of the next
// iteration (for every other iteration)
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
BranchInst *LatchBR = cast<BranchInst>(NewLatch->getTerminator());
unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1);
LatchBR->setSuccessor(HeaderIdx, InsertBot);
LatchBR->setSuccessor(1 - HeaderIdx, Exit);
if (DT)
DT->changeImmediateDominator(InsertBot, NewLatch);
// The new copy of the loop body starts with a bunch of PHI nodes
// that pick an incoming value from either the preheader, or the previous
// loop iteration. Since this copy is no longer part of the loop, we
// resolve this statically:
// For the first iteration, we use the value from the preheader directly.
// For any other iteration, we replace the phi with the value generated by
// the immediately preceding clone of the loop body (which represents
// the previous iteration).
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
if (IterNumber == 0) {
VMap[&*I] = NewPHI->getIncomingValueForBlock(PreHeader);
} else {
Value *LatchVal = NewPHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(LatchVal);
if (LatchInst && L->contains(LatchInst))
VMap[&*I] = LVMap[LatchInst];
else
VMap[&*I] = LatchVal;
}
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
}
// Fix up the outgoing values - we need to add a value for the iteration
// we've just created. Note that this must happen *after* the incoming
// values are adjusted, since the value going out of the latch may also be
// a value coming into the header.
for (BasicBlock::iterator I = Exit->begin(); isa<PHINode>(I); ++I) {
PHINode *PHI = cast<PHINode>(I);
Value *LatchVal = PHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(LatchVal);
if (LatchInst && L->contains(LatchInst))
LatchVal = VMap[LatchVal];
PHI->addIncoming(LatchVal, cast<BasicBlock>(VMap[Latch]));
}
// LastValueMap is updated with the values for the current loop
// which are used the next time this function is called.
for (const auto &KV : VMap)
LVMap[KV.first] = KV.second;
}
const DominanceFrontier::DomSetType &
DominanceFrontier::calculate(const DominatorTree &DT,
const DomTreeNode *Node) {
BasicBlock *BB = Node->getBlock();
DomSetType *Result = NULL;
std::vector<DFCalculateWorkObject> workList;
SmallPtrSet<BasicBlock *, 32> visited;
workList.push_back(DFCalculateWorkObject(BB, NULL, Node, NULL));
do {
DFCalculateWorkObject *currentW = &workList.back();
assert (currentW && "Missing work object.");
BasicBlock *currentBB = currentW->currentBB;
BasicBlock *parentBB = currentW->parentBB;
const DomTreeNode *currentNode = currentW->Node;
const DomTreeNode *parentNode = currentW->parentNode;
assert (currentBB && "Invalid work object. Missing current Basic Block");
assert (currentNode && "Invalid work object. Missing current Node");
DomSetType &S = Frontiers[currentBB];
// Visit each block only once.
if (visited.count(currentBB) == 0) {
visited.insert(currentBB);
// Loop over CFG successors to calculate DFlocal[currentNode]
for (succ_iterator SI = succ_begin(currentBB), SE = succ_end(currentBB);
SI != SE; ++SI) {
// Does Node immediately dominate this successor?
if (DT[*SI]->getIDom() != currentNode)
S.insert(*SI);
}
}
// At this point, S is DFlocal. Now we union in DFup's of our children...
// Loop through and visit the nodes that Node immediately dominates (Node's
// children in the IDomTree)
bool visitChild = false;
for (DomTreeNode::const_iterator NI = currentNode->begin(),
NE = currentNode->end(); NI != NE; ++NI) {
DomTreeNode *IDominee = *NI;
BasicBlock *childBB = IDominee->getBlock();
if (visited.count(childBB) == 0) {
workList.push_back(DFCalculateWorkObject(childBB, currentBB,
IDominee, currentNode));
visitChild = true;
}
}
// If all children are visited or there is any child then pop this block
// from the workList.
if (!visitChild) {
if (!parentBB) {
Result = &S;
break;
}
DomSetType::const_iterator CDFI = S.begin(), CDFE = S.end();
DomSetType &parentSet = Frontiers[parentBB];
for (; CDFI != CDFE; ++CDFI) {
if (!DT.properlyDominates(parentNode, DT[*CDFI]))
parentSet.insert(*CDFI);
}
workList.pop_back();
}
} while (!workList.empty());
return *Result;
}
/// At this point, we're committed to promoting the alloca using IDF's, and the
/// standard SSA construction algorithm. Determine which blocks need phi nodes
/// and see if we can optimize out some work by avoiding insertion of dead phi
/// nodes.
void PromoteMem2Reg::DetermineInsertionPoint(AllocaInst *AI, unsigned AllocaNum,
AllocaInfo &Info) {
// Unique the set of defining blocks for efficient lookup.
SmallPtrSet<BasicBlock *, 32> DefBlocks;
DefBlocks.insert(Info.DefiningBlocks.begin(), Info.DefiningBlocks.end());
// Determine which blocks the value is live in. These are blocks which lead
// to uses.
SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
ComputeLiveInBlocks(AI, Info, DefBlocks, LiveInBlocks);
// Use a priority queue keyed on dominator tree level so that inserted nodes
// are handled from the bottom of the dominator tree upwards.
typedef std::priority_queue<DomTreeNodePair,
SmallVector<DomTreeNodePair, 32>,
DomTreeNodeCompare> IDFPriorityQueue;
IDFPriorityQueue PQ;
for (SmallPtrSet<BasicBlock *, 32>::const_iterator I = DefBlocks.begin(),
E = DefBlocks.end();
I != E; ++I) {
if (DomTreeNode *Node = DT.getNode(*I))
PQ.push(std::make_pair(Node, DomLevels[Node]));
}
SmallVector<std::pair<unsigned, BasicBlock *>, 32> DFBlocks;
SmallPtrSet<DomTreeNode *, 32> Visited;
SmallVector<DomTreeNode *, 32> Worklist;
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dominator tree children of Root, inspecting their CFG edges with
// targets elsewhere on the dominator tree. Only targets whose level is at
// most Root's level are added to the iterated dominance frontier of the
// definition set.
Worklist.clear();
Worklist.push_back(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
BasicBlock *BB = Node->getBlock();
for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB); SI != SE;
++SI) {
DomTreeNode *SuccNode = DT.getNode(*SI);
// Quickly skip all CFG edges that are also dominator tree edges instead
// of catching them below.
if (SuccNode->getIDom() == Node)
continue;
unsigned SuccLevel = DomLevels[SuccNode];
if (SuccLevel > RootLevel)
continue;
if (!Visited.insert(SuccNode))
continue;
BasicBlock *SuccBB = SuccNode->getBlock();
if (!LiveInBlocks.count(SuccBB))
continue;
DFBlocks.push_back(std::make_pair(BBNumbers[SuccBB], SuccBB));
if (!DefBlocks.count(SuccBB))
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
for (DomTreeNode::iterator CI = Node->begin(), CE = Node->end(); CI != CE;
++CI) {
if (!Visited.count(*CI))
Worklist.push_back(*CI);
}
}
}
if (DFBlocks.size() > 1)
std::sort(DFBlocks.begin(), DFBlocks.end());
unsigned CurrentVersion = 0;
for (unsigned i = 0, e = DFBlocks.size(); i != e; ++i)
QueuePhiNode(DFBlocks[i].second, AllocaNum, CurrentVersion);
}
