/// \return true if the given block is dominated by a _slowPath branch hint.
///
/// Cache all blocks visited to avoid introducing quadratic behavior.
bool ColdBlockInfo::isCold(const SILBasicBlock *BB, int recursionDepth) {
auto I = ColdBlockMap.find(BB);
if (I != ColdBlockMap.end())
return I->second;
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DominanceInfo *DT = DA->get(const_cast<SILFunction*>(BB->getParent()));
DomTreeNode *Node = DT->getNode(const_cast<SILBasicBlock*>(BB));
// Always consider unreachable code cold.
if (!Node)
return true;
std::vector<const SILBasicBlock*> DomChain;
DomChain.push_back(BB);
bool IsCold = false;
Node = Node->getIDom();
while (Node) {
if (isSlowPath(Node->getBlock(), DomChain.back(), recursionDepth)) {
IsCold = true;
break;
}
auto I = ColdBlockMap.find(Node->getBlock());
if (I != ColdBlockMap.end()) {
IsCold = I->second;
break;
}
DomChain.push_back(Node->getBlock());
Node = Node->getIDom();
}
for (auto *ChainBB : DomChain)
ColdBlockMap[ChainBB] = IsCold;
return IsCold;
}
void ControlDependenceGraphBase::computeDependencies(Function &F, PostDominatorTree &pdt) {
root = new ControlDependenceNode();
nodes.insert(root);
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
ControlDependenceNode *bn = new ControlDependenceNode(BB);
nodes.insert(bn);
bbMap[BB] = bn;
}
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
BasicBlock *A = BB;
ControlDependenceNode *AN = bbMap[A];
for (succ_iterator succ = succ_begin(A), end = succ_end(A); succ != end; ++succ) {
BasicBlock *B = *succ;
assert(A && B);
if (A == B || !pdt.dominates(B,A)) {
BasicBlock *L = pdt.findNearestCommonDominator(A,B);
ControlDependenceNode::EdgeType type = ControlDependenceGraphBase::getEdgeType(A,B);
if (A == L) {
switch (type) {
case ControlDependenceNode::TRUE:
AN->addTrue(AN); break;
case ControlDependenceNode::FALSE:
AN->addFalse(AN); break;
case ControlDependenceNode::OTHER:
AN->addOther(AN); break;
}
AN->addParent(AN);
}
for (DomTreeNode *cur = pdt[B]; cur && cur != pdt[L]; cur = cur->getIDom()) {
ControlDependenceNode *CN = bbMap[cur->getBlock()];
switch (type) {
case ControlDependenceNode::TRUE:
AN->addTrue(CN); break;
case ControlDependenceNode::FALSE:
AN->addFalse(CN); break;
case ControlDependenceNode::OTHER:
AN->addOther(CN); break;
}
assert(CN);
CN->addParent(AN);
}
}
}
}
// ENTRY -> START
for (DomTreeNode *cur = pdt[&F.getEntryBlock()]; cur; cur = cur->getIDom()) {
if (cur->getBlock()) {
ControlDependenceNode *CN = bbMap[cur->getBlock()];
assert(CN);
root->addOther(CN); CN->addParent(root);
}
}
}
/// isExitBlockDominatedByBlockInLoop - This method checks to see if the
/// specified exit block of the loop is dominated by the specified block
/// that is in the body of the loop. We use these constraints to
/// dramatically limit the amount of the dominator tree that needs to be
/// searched.
bool isExitBlockDominatedByBlockInLoop(BasicBlock *ExitBlock,
BasicBlock *BlockInLoop) const {
// If the block in the loop is the loop header, it must be dominated!
BasicBlock *LoopHeader = CurLoop->getHeader();
if (BlockInLoop == LoopHeader)
return true;
DomTreeNode *BlockInLoopNode = DT->getNode(BlockInLoop);
DomTreeNode *IDom = DT->getNode(ExitBlock);
// Because the exit block is not in the loop, we know we have to get _at
// least_ its immediate dominator.
do {
// Get next Immediate Dominator.
IDom = IDom->getIDom();
// If we have got to the header of the loop, then the instructions block
// did not dominate the exit node, so we can't hoist it.
if (IDom->getBlock() == LoopHeader)
return false;
} while (IDom != BlockInLoopNode);
return true;
}
void TempScopInfo::buildCondition(BasicBlock *BB, BasicBlock *RegionEntry) {
BBCond Cond;
DomTreeNode *BBNode = DT->getNode(BB), *EntryNode = DT->getNode(RegionEntry);
assert(BBNode && EntryNode && "Get null node while building condition!");
// Walk up the dominance tree until reaching the entry node. Add all
// conditions on the path to BB except if BB postdominates the block
// containing the condition.
while (BBNode != EntryNode) {
BasicBlock *CurBB = BBNode->getBlock();
BBNode = BBNode->getIDom();
assert(BBNode && "BBNode should not reach the root node!");
if (PDT->dominates(CurBB, BBNode->getBlock()))
continue;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getBlock()->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
if (Br->isUnconditional())
continue;
// Is BB on the ELSE side of the branch?
bool inverted = DT->dominates(Br->getSuccessor(1), BB);
Comparison *Cmp;
buildAffineCondition(*(Br->getCondition()), inverted, &Cmp);
Cond.push_back(*Cmp);
}
if (!Cond.empty())
BBConds[BB] = Cond;
}
SILValue
StackAllocationPromoter::getLiveOutValue(BlockSet &PhiBlocks,
SILBasicBlock *StartBB) {
DEBUG(llvm::dbgs() << "*** Searching for a value definition.\n");
// Walk the Dom tree in search of a defining value:
for (DomTreeNode *Node = DT->getNode(StartBB); Node; Node = Node->getIDom()) {
SILBasicBlock *BB = Node->getBlock();
// If there is a store (that must come after the phi), use its value.
BlockToInstMap::iterator it = LastStoreInBlock.find(BB);
if (it != LastStoreInBlock.end())
if (auto *St = dyn_cast_or_null<StoreInst>(it->second)) {
DEBUG(llvm::dbgs() << "*** Found Store def " << *St->getSrc());
return St->getSrc();
}
// If there is a Phi definition in this block:
if (PhiBlocks.count(BB)) {
// Return the dummy instruction that represents the new value that we will
// add to the basic block.
SILValue Phi = BB->getArgument(BB->getNumArguments() - 1);
DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *Phi);
return Phi;
}
// Move to the next dominating block.
DEBUG(llvm::dbgs() << "*** Walking up the iDOM.\n");
}
DEBUG(llvm::dbgs() << "*** Could not find a Def. Using Undef.\n");
return SILUndef::get(ASI->getElementType(), ASI->getModule());
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst *> Calls) {
LLVM_DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< Demangle::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction *, ApplyInst *> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
ApplyInst *HoistAI =
getHoistedApplyForInitializer(AI, DT, InitF, ParentF, ParentFuncs);
// If we were unable to find anything, just go onto the next apply.
if (!HoistAI) {
continue;
}
// Otherwise, move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
LLVM_DEBUG(llvm::dbgs() << " don't hoist above availability check "
"at bb"
<< DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
LLVM_DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName()
<< "\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI << " in "
<< HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
/**
* removeUndefBranches -- remove branches with undef condition
*
* These are irrelevant to the code, so may be removed completely with their
* bodies.
*/
void FunctionStaticSlicer::removeUndefBranches(ModulePass *MP, Function &F) {
#ifdef DEBUG_SLICE
errs() << __func__ << " ============ Removing unused branches\n";
#endif
PostDominatorTree &PDT = MP->getAnalysis<PostDominatorTree>(F);
typedef llvm::SmallVector<const BasicBlock *, 10> Unsafe;
Unsafe unsafe;
for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) {
BasicBlock &bb = *I;
if (std::distance(succ_begin(&bb), succ_end(&bb)) <= 1)
continue;
Instruction &back = bb.back();
if (back.getOpcode() != Instruction::Br &&
back.getOpcode() != Instruction::Switch)
continue;
const Value *cond = back.getOperand(0);
if (cond->getValueID() != Value::UndefValueVal)
continue;
DomTreeNode *node = PDT.getNode(&bb);
if (!node) /* this bb is unreachable */
continue;
DomTreeNode *idom = node->getIDom();
assert(idom);
/* if (!idom)
continue;*/
BasicBlock *dest = idom->getBlock();
if (!dest) /* TODO when there are nodes with noreturn calls */
continue;
#ifdef DEBUG_SLICE
errs() << " considering branch: " << bb.getName() << '\n';
errs() << " dest=" << dest->getName() << "\n";
#endif
if (PHINode *PHI = dyn_cast<PHINode>(&dest->front()))
if (PHI->getBasicBlockIndex(&bb) == -1) {
/* TODO this is unsafe! */
unsafe.push_back(&bb);
PHI->addIncoming(Constant::getNullValue(PHI->getType()), &bb);
}
BasicBlock::iterator ii(back);
Instruction *newI = BranchInst::Create(dest);
ReplaceInstWithInst(bb.getInstList(), ii, newI);
}
for (Unsafe::const_iterator I = unsafe.begin(), E = unsafe.end();
I != E; ++I) {
const BasicBlock *bb = *I;
if (std::distance(pred_begin(bb), pred_end(bb)) > 1)
errs() << "WARNING: PHI node with added value which is zero\n";
}
#ifdef DEBUG_SLICE
errs() << __func__ << " ============ END\n";
#endif
}
void GCPtrTracker::gatherDominatingDefs(const BasicBlock *BB,
AvailableValueSet &Result,
const DominatorTree &DT) {
DomTreeNode *DTN = DT[const_cast<BasicBlock *>(BB)];
while (DTN->getIDom()) {
DTN = DTN->getIDom();
const auto &Defs = BlockMap[DTN->getBlock()]->Contribution;
Result.insert(Defs.begin(), Defs.end());
// If this block is 'Cleared', then nothing LiveIn to this block can be
// available after this block completes. Note: This turns out to be
// really important for reducing memory consuption of the initial available
// sets and thus peak memory usage by this verifier.
if (BlockMap[DTN->getBlock()]->Cleared)
return;
}
for (const Argument &A : BB->getParent()->args())
if (containsGCPtrType(A.getType()))
Result.insert(&A);
}
const DominanceFrontier::DomSetType &
PostDominanceFrontier::calculate(const PostDominatorTree &DT,
const DomTreeNode *Node) {
// Loop over CFG successors to calculate DFlocal[Node]
BasicBlock *BB = Node->getBlock();
DomSetType &S = Frontiers[BB]; // The new set to fill in...
if (getRoots().empty()) return S;
if (BB)
for (pred_iterator SI = pred_begin(BB), SE = pred_end(BB);
SI != SE; ++SI) {
BasicBlock *P = *SI;
// Does Node immediately dominate this predecessor?
DomTreeNode *SINode = DT[P];
if (SINode && SINode->getIDom() != Node)
S.insert(P);
}
// At this point, S is DFlocal. Now we union in DFup's of our children...
// Loop through and visit the nodes that Node immediately dominates (Node's
// children in the IDomTree)
//
for (DomTreeNode::const_iterator
NI = Node->begin(), NE = Node->end(); NI != NE; ++NI) {
DomTreeNode *IDominee = *NI;
const DomSetType &ChildDF = calculate(DT, IDominee);
DomSetType::const_iterator CDFI = ChildDF.begin(), CDFE = ChildDF.end();
for (; CDFI != CDFE; ++CDFI) {
if (!DT.properlyDominates(Node, DT[*CDFI]))
S.insert(*CDFI);
}
}
return S;
}
/// \brief Simplify one loop and queue further loops for simplification.
///
/// FIXME: Currently this accepts both lots of analyses that it uses and a raw
/// Pass pointer. The Pass pointer is used by numerous utilities to update
/// specific analyses. Rather than a pass it would be much cleaner and more
/// explicit if they accepted the analysis directly and then updated it.
static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, Pass *PP,
AssumptionCache *AC) {
bool Changed = false;
ReprocessLoop:
// Check to see that no blocks (other than the header) in this loop have
// predecessors that are not in the loop. This is not valid for natural
// loops, but can occur if the blocks are unreachable. Since they are
// unreachable we can just shamelessly delete those CFG edges!
for (Loop::block_iterator BB = L->block_begin(), E = L->block_end();
BB != E; ++BB) {
if (*BB == L->getHeader()) continue;
SmallPtrSet<BasicBlock*, 4> BadPreds;
for (pred_iterator PI = pred_begin(*BB),
PE = pred_end(*BB); PI != PE; ++PI) {
BasicBlock *P = *PI;
if (!L->contains(P))
BadPreds.insert(P);
}
// Delete each unique out-of-loop (and thus dead) predecessor.
for (BasicBlock *P : BadPreds) {
DEBUG(dbgs() << "LoopSimplify: Deleting edge from dead predecessor "
<< P->getName() << "\n");
// Inform each successor of each dead pred.
for (succ_iterator SI = succ_begin(P), SE = succ_end(P); SI != SE; ++SI)
(*SI)->removePredecessor(P);
// Zap the dead pred's terminator and replace it with unreachable.
TerminatorInst *TI = P->getTerminator();
TI->replaceAllUsesWith(UndefValue::get(TI->getType()));
P->getTerminator()->eraseFromParent();
new UnreachableInst(P->getContext(), P);
Changed = true;
}
}
// If there are exiting blocks with branches on undef, resolve the undef in
// the direction which will exit the loop. This will help simplify loop
// trip count computations.
SmallVector<BasicBlock*, 8> ExitingBlocks;
L->getExitingBlocks(ExitingBlocks);
for (SmallVectorImpl<BasicBlock *>::iterator I = ExitingBlocks.begin(),
E = ExitingBlocks.end(); I != E; ++I)
if (BranchInst *BI = dyn_cast<BranchInst>((*I)->getTerminator()))
if (BI->isConditional()) {
if (UndefValue *Cond = dyn_cast<UndefValue>(BI->getCondition())) {
DEBUG(dbgs() << "LoopSimplify: Resolving \"br i1 undef\" to exit in "
<< (*I)->getName() << "\n");
BI->setCondition(ConstantInt::get(Cond->getType(),
!L->contains(BI->getSuccessor(0))));
// This may make the loop analyzable, force SCEV recomputation.
if (SE)
SE->forgetLoop(L);
Changed = true;
}
}
// Does the loop already have a preheader? If so, don't insert one.
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) {
Preheader = InsertPreheaderForLoop(L, PP);
if (Preheader) {
++NumInserted;
Changed = true;
}
}
// Next, check to make sure that all exit nodes of the loop only have
// predecessors that are inside of the loop. This check guarantees that the
// loop preheader/header will dominate the exit blocks. If the exit block has
// predecessors from outside of the loop, split the edge now.
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getExitBlocks(ExitBlocks);
SmallSetVector<BasicBlock *, 8> ExitBlockSet(ExitBlocks.begin(),
ExitBlocks.end());
for (SmallSetVector<BasicBlock *, 8>::iterator I = ExitBlockSet.begin(),
E = ExitBlockSet.end(); I != E; ++I) {
BasicBlock *ExitBlock = *I;
for (pred_iterator PI = pred_begin(ExitBlock), PE = pred_end(ExitBlock);
PI != PE; ++PI)
// Must be exactly this loop: no subloops, parent loops, or non-loop preds
// allowed.
if (!L->contains(*PI)) {
if (rewriteLoopExitBlock(L, ExitBlock, DT, LI, PP)) {
++NumInserted;
Changed = true;
}
break;
}
}
// If the header has more than two predecessors at this point (from the
// preheader and from multiple backedges), we must adjust the loop.
BasicBlock *LoopLatch = L->getLoopLatch();
if (!LoopLatch) {
// If this is really a nested loop, rip it out into a child loop. Don't do
// this for loops with a giant number of backedges, just factor them into a
// common backedge instead.
if (L->getNumBackEdges() < 8) {
if (Loop *OuterL = separateNestedLoop(L, Preheader, DT, LI, SE, PP, AC)) {
++NumNested;
// Enqueue the outer loop as it should be processed next in our
// depth-first nest walk.
Worklist.push_back(OuterL);
// This is a big restructuring change, reprocess the whole loop.
Changed = true;
// GCC doesn't tail recursion eliminate this.
// FIXME: It isn't clear we can't rely on LLVM to TRE this.
goto ReprocessLoop;
}
}
// If we either couldn't, or didn't want to, identify nesting of the loops,
// insert a new block that all backedges target, then make it jump to the
// loop header.
LoopLatch = insertUniqueBackedgeBlock(L, Preheader, DT, LI);
if (LoopLatch) {
++NumInserted;
Changed = true;
}
}
const DataLayout &DL = L->getHeader()->getModule()->getDataLayout();
// Scan over the PHI nodes in the loop header. Since they now have only two
// incoming values (the loop is canonicalized), we may have simplified the PHI
// down to 'X = phi [X, Y]', which should be replaced with 'Y'.
PHINode *PN;
for (BasicBlock::iterator I = L->getHeader()->begin();
(PN = dyn_cast<PHINode>(I++)); )
if (Value *V = SimplifyInstruction(PN, DL, nullptr, DT, AC)) {
if (SE) SE->forgetValue(PN);
PN->replaceAllUsesWith(V);
PN->eraseFromParent();
}
// If this loop has multiple exits and the exits all go to the same
// block, attempt to merge the exits. This helps several passes, such
// as LoopRotation, which do not support loops with multiple exits.
// SimplifyCFG also does this (and this code uses the same utility
// function), however this code is loop-aware, where SimplifyCFG is
// not. That gives it the advantage of being able to hoist
// loop-invariant instructions out of the way to open up more
// opportunities, and the disadvantage of having the responsibility
// to preserve dominator information.
bool UniqueExit = true;
if (!ExitBlocks.empty())
for (unsigned i = 1, e = ExitBlocks.size(); i != e; ++i)
if (ExitBlocks[i] != ExitBlocks[0]) {
UniqueExit = false;
break;
}
if (UniqueExit) {
for (unsigned i = 0, e = ExitingBlocks.size(); i != e; ++i) {
BasicBlock *ExitingBlock = ExitingBlocks[i];
if (!ExitingBlock->getSinglePredecessor()) continue;
BranchInst *BI = dyn_cast<BranchInst>(ExitingBlock->getTerminator());
if (!BI || !BI->isConditional()) continue;
CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition());
if (!CI || CI->getParent() != ExitingBlock) continue;
// Attempt to hoist out all instructions except for the
// comparison and the branch.
bool AllInvariant = true;
bool AnyInvariant = false;
for (BasicBlock::iterator I = ExitingBlock->begin(); &*I != BI; ) {
Instruction *Inst = I++;
// Skip debug info intrinsics.
if (isa<DbgInfoIntrinsic>(Inst))
continue;
if (Inst == CI)
continue;
if (!L->makeLoopInvariant(Inst, AnyInvariant,
Preheader ? Preheader->getTerminator()
: nullptr)) {
AllInvariant = false;
break;
}
}
if (AnyInvariant) {
Changed = true;
// The loop disposition of all SCEV expressions that depend on any
// hoisted values have also changed.
if (SE)
SE->forgetLoopDispositions(L);
}
if (!AllInvariant) continue;
// The block has now been cleared of all instructions except for
// a comparison and a conditional branch. SimplifyCFG may be able
// to fold it now.
if (!FoldBranchToCommonDest(BI))
continue;
// Success. The block is now dead, so remove it from the loop,
// update the dominator tree and delete it.
DEBUG(dbgs() << "LoopSimplify: Eliminating exiting block "
<< ExitingBlock->getName() << "\n");
// Notify ScalarEvolution before deleting this block. Currently assume the
// parent loop doesn't change (spliting edges doesn't count). If blocks,
// CFG edges, or other values in the parent loop change, then we need call
// to forgetLoop() for the parent instead.
if (SE)
SE->forgetLoop(L);
assert(pred_begin(ExitingBlock) == pred_end(ExitingBlock));
Changed = true;
LI->removeBlock(ExitingBlock);
DomTreeNode *Node = DT->getNode(ExitingBlock);
const std::vector<DomTreeNodeBase<BasicBlock> *> &Children =
Node->getChildren();
while (!Children.empty()) {
DomTreeNode *Child = Children.front();
DT->changeImmediateDominator(Child, Node->getIDom());
}
DT->eraseNode(ExitingBlock);
BI->getSuccessor(0)->removePredecessor(ExitingBlock);
BI->getSuccessor(1)->removePredecessor(ExitingBlock);
ExitingBlock->eraseFromParent();
}
}
return Changed;
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst*> Calls) {
DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< demangle_wrappers::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction*, ApplyInst*> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
auto PFI = ParentFuncs.find(ParentF);
ApplyInst *HoistAI = nullptr;
if (PFI != ParentFuncs.end()) {
// Found a replacement for this init call.
// Ensure the replacement dominates the original call site.
ApplyInst *CommonAI = PFI->second;
assert(cast<FunctionRefInst>(CommonAI->getCallee())
->getReferencedFunction() == InitF &&
"ill-formed global init call");
SILBasicBlock *DomBB =
DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
// We must not move initializers around availability-checks.
if (!isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT)) {
if (DomBB != CommonAI->getParent()) {
CommonAI->moveBefore(&*DomBB->begin());
placeFuncRef(CommonAI, DT);
// Try to hoist the existing AI again if we move it to another block,
// e.g. from a loop exit into the loop.
HoistAI = CommonAI;
}
AI->replaceAllUsesWith(CommonAI);
AI->eraseFromParent();
HasChanged = true;
}
} else {
ParentFuncs[ParentF] = AI;
// It's the first time we found a call to InitF in this function, so we
// try to hoist it out of any loop.
HoistAI = AI;
}
if (HoistAI) {
// Move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
DEBUG(llvm::dbgs() << " don't hoist above availability check at bb" <<
DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
}
else {
DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
}
}
void idenRegion::computeAntidependencePaths() {
// Iterate through every pair of antidependency
// Record all the store along the path
for (AntiDepPairs::iterator I = AntiDepPairs_.begin(), E = AntiDepPairs_.end(); I != E; I++) {
BasicBlock::iterator Load, Store;
tie(Load, Store) = *I;
// create a new path
AntiDepPaths_.resize(AntiDepPaths_.size()+1);
AntiDepPathTy &newPath = AntiDepPaths_.back();
// Always record current store
newPath.push_back(Store);
// Load and store in the same basic block
BasicBlock::iterator curInst = Store;
BasicBlock *LoadBB = Load->getParent(), *StoreBB = Store->getParent();
if (LoadBB == StoreBB && DT->dominates(Load, Store)) {
while(--curInst != Load) {
if (isa<StoreInst>(curInst))
newPath.push_back(curInst);
}
errs() << "@@@ Local BB: \n@@@ ";
printPath(newPath);
errs() << "\n";
continue;
}
// Load and store in different basic block
BasicBlock *curBB = StoreBB;
DomTreeNode *curDTNode = DT->getNode(StoreBB), *LoadDTNode = DT->getNode(LoadBB);
// loop until load is not
while (DT->dominates(LoadDTNode, curDTNode)) {
errs() << "^^^^^ Current BB is " << curBB->getName() << "\n";
BasicBlock::iterator E;
// check if Load and current node in the same BB
if (curBB == LoadBB) {
E = Load;
} else {
E = curBB->begin();
}
// scan current BB
while(curInst != E) {
if (isa<StoreInst>(--curInst)) {
newPath.push_back(curInst);
}
}
// find current Node's iDOM
curDTNode = curDTNode->getIDom();
if (curDTNode == NULL)
break;
curBB = curDTNode->getBlock();
curInst = curBB->end();
}
errs() << "@@@ Inter BB: \n@@@ ";
printPath(newPath);
errs() << "\n";
}
errs() << "Path cap is " << AntiDepPaths_.capacity() << "\n";
errs() << "Path size is " << AntiDepPaths_.size() << "\n";
errs() << "#########################################################\n";
errs() << "#################### Paths Summary ######################\n";
errs() << "#########################################################\n";
printCollection(AntiDepPaths_);
errs() << "\n";
}
// NewBB is split and now it has one successor. Update dominance frontier to
// reflect this change.
void DominanceFrontier::splitBlock(BasicBlock *NewBB) {
assert(NewBB->getTerminator()->getNumSuccessors() == 1 &&
"NewBB should have a single successor!");
BasicBlock *NewBBSucc = NewBB->getTerminator()->getSuccessor(0);
// NewBBSucc inherits original NewBB frontier.
DominanceFrontier::iterator NewBBI = find(NewBB);
if (NewBBI != end())
addBasicBlock(NewBBSucc, NewBBI->second);
// If NewBB dominates NewBBSucc, then DF(NewBB) is now going to be the
// DF(NewBBSucc) without the stuff that the new block does not dominate
// a predecessor of.
DominatorTree &DT = getAnalysis<DominatorTree>();
DomTreeNode *NewBBNode = DT.getNode(NewBB);
DomTreeNode *NewBBSuccNode = DT.getNode(NewBBSucc);
if (DT.dominates(NewBBNode, NewBBSuccNode)) {
DominanceFrontier::iterator DFI = find(NewBBSucc);
if (DFI != end()) {
DominanceFrontier::DomSetType Set = DFI->second;
// Filter out stuff in Set that we do not dominate a predecessor of.
for (DominanceFrontier::DomSetType::iterator SetI = Set.begin(),
E = Set.end(); SetI != E;) {
bool DominatesPred = false;
for (pred_iterator PI = pred_begin(*SetI), E = pred_end(*SetI);
PI != E; ++PI)
if (DT.dominates(NewBBNode, DT.getNode(*PI))) {
DominatesPred = true;
break;
}
if (!DominatesPred)
Set.erase(SetI++);
else
++SetI;
}
if (NewBBI != end()) {
for (DominanceFrontier::DomSetType::iterator SetI = Set.begin(),
E = Set.end(); SetI != E; ++SetI) {
BasicBlock *SB = *SetI;
addToFrontier(NewBBI, SB);
}
} else
addBasicBlock(NewBB, Set);
}
} else {
// DF(NewBB) is {NewBBSucc} because NewBB does not strictly dominate
// NewBBSucc, but it does dominate itself (and there is an edge (NewBB ->
// NewBBSucc)). NewBBSucc is the single successor of NewBB.
DominanceFrontier::DomSetType NewDFSet;
NewDFSet.insert(NewBBSucc);
addBasicBlock(NewBB, NewDFSet);
}
// Now update dominance frontiers which either used to contain NewBBSucc
// or which now need to include NewBB.
// Collect the set of blocks which dominate a predecessor of NewBB or
// NewSuccBB and which don't dominate both. This is an initial
// approximation of the blocks whose dominance frontiers will need updates.
SmallVector<DomTreeNode *, 16> AllPredDoms;
// Compute the block which dominates both NewBBSucc and NewBB. This is
// the immediate dominator of NewBBSucc unless NewBB dominates NewBBSucc.
// The code below which climbs dominator trees will stop at this point,
// because from this point up, dominance frontiers are unaffected.
DomTreeNode *DominatesBoth = 0;
if (NewBBSuccNode) {
DominatesBoth = NewBBSuccNode->getIDom();
if (DominatesBoth == NewBBNode)
DominatesBoth = NewBBNode->getIDom();
}
// Collect the set of all blocks which dominate a predecessor of NewBB.
SmallPtrSet<DomTreeNode *, 8> NewBBPredDoms;
for (pred_iterator PI = pred_begin(NewBB), E = pred_end(NewBB); PI != E; ++PI)
for (DomTreeNode *DTN = DT.getNode(*PI); DTN; DTN = DTN->getIDom()) {
if (DTN == DominatesBoth)
break;
if (!NewBBPredDoms.insert(DTN))
break;
AllPredDoms.push_back(DTN);
}
// Collect the set of all blocks which dominate a predecessor of NewSuccBB.
SmallPtrSet<DomTreeNode *, 8> NewBBSuccPredDoms;
for (pred_iterator PI = pred_begin(NewBBSucc),
E = pred_end(NewBBSucc); PI != E; ++PI)
for (DomTreeNode *DTN = DT.getNode(*PI); DTN; DTN = DTN->getIDom()) {
if (DTN == DominatesBoth)
break;
if (!NewBBSuccPredDoms.insert(DTN))
break;
if (!NewBBPredDoms.count(DTN))
AllPredDoms.push_back(DTN);
}
// Visit all relevant dominance frontiers and make any needed updates.
for (SmallVectorImpl<DomTreeNode *>::const_iterator I = AllPredDoms.begin(),
E = AllPredDoms.end(); I != E; ++I) {
DomTreeNode *DTN = *I;
iterator DFI = find((*I)->getBlock());
// Only consider nodes that have NewBBSucc in their dominator frontier.
if (DFI == end() || !DFI->second.count(NewBBSucc)) continue;
// If the block dominates a predecessor of NewBB but does not properly
// dominate NewBB itself, add NewBB to its dominance frontier.
if (NewBBPredDoms.count(DTN) &&
!DT.properlyDominates(DTN, NewBBNode))
addToFrontier(DFI, NewBB);
// If the block does not dominate a predecessor of NewBBSucc or
// properly dominates NewBBSucc itself, remove NewBBSucc from its
// dominance frontier.
if (!NewBBSuccPredDoms.count(DTN) ||
DT.properlyDominates(DTN, NewBBSuccNode))
removeFromFrontier(DFI, NewBBSucc);
}
}
void LLVMDependenceGraph::computePostDominators(bool addPostDomFrontiers)
{
using namespace llvm;
// iterate over all functions
for (auto& F : getConstructedFunctions()) {
analysis::PostDominanceFrontiers<LLVMNode> pdfrontiers;
// root of post-dominator tree
LLVMBBlock *root = nullptr;
Value *val = const_cast<Value *>(F.first);
Function& f = *cast<Function>(val);
PostDominatorTree *pdtree;
#if ((LLVM_VERSION_MAJOR == 3) && (LLVM_VERSION_MINOR < 9))
pdtree = new PostDominatorTree();
// compute post-dominator tree for this function
pdtree->runOnFunction(f);
#else
PostDominatorTreeWrapperPass wrapper;
wrapper.runOnFunction(f);
pdtree = &wrapper.getPostDomTree();
#ifndef NDEBUG
wrapper.verifyAnalysis();
#endif
#endif
// add immediate post-dominator edges
auto& our_blocks = F.second->getBlocks();
bool built = false;
for (auto& it : our_blocks) {
LLVMBBlock *BB = it.second;
BasicBlock *B = cast<BasicBlock>(const_cast<Value *>(it.first));
DomTreeNode *N = pdtree->getNode(B);
// when function contains infinite loop, we're screwed
// and we don't have anything
// FIXME: just check for the root,
// don't iterate over all blocks, stupid...
if (!N)
continue;
DomTreeNode *idom = N->getIDom();
BasicBlock *idomBB = idom ? idom->getBlock() : nullptr;
built = true;
if (idomBB) {
LLVMBBlock *pb = our_blocks[idomBB];
assert(pb && "Do not have constructed BB");
BB->setIPostDom(pb);
assert(cast<BasicBlock>(BB->getKey())->getParent()
== cast<BasicBlock>(pb->getKey())->getParent()
&& "BBs are from diferent functions");
// if we do not have idomBB, then the idomBB is a root BB
} else {
// PostDominatorTree may has special root without BB set
// or it is the node without immediate post-dominator
if (!root) {
root = new LLVMBBlock();
root->setKey(nullptr);
F.second->setPostDominatorTreeRoot(root);
}
BB->setIPostDom(root);
}
}
// well, if we haven't built the pdtree, this is probably infinite loop
// that has no pdtree. Until we have anything better, just add sound control
// edges that are not so precise - to predecessors.
if (!built && addPostDomFrontiers) {
for (auto& it : our_blocks) {
LLVMBBlock *BB = it.second;
for (const LLVMBBlock::BBlockEdge& succ : BB->successors()) {
// in this case we add only the control dependencies,
// since we have no pd frontiers
BB->addControlDependence(succ.target);
}
}
}
if (addPostDomFrontiers) {
// assert(root && "BUG: must have root");
if (root)
pdfrontiers.compute(root, true /* store also control depend. */);
}
#if ((LLVM_VERSION_MAJOR == 3) && (LLVM_VERSION_MINOR < 9))
delete pdtree;
#endif
}
}
void IDFCalculator<NodeTy>::calculate(
SmallVectorImpl<BasicBlock *> &PHIBlocks) {
// Use a priority queue keyed on dominator tree level so that inserted nodes
// are handled from the bottom of the dominator tree upwards.
typedef std::pair<DomTreeNode *, unsigned> DomTreeNodePair;
typedef std::priority_queue<DomTreeNodePair, SmallVector<DomTreeNodePair, 32>,
less_second> IDFPriorityQueue;
IDFPriorityQueue PQ;
for (BasicBlock *BB : *DefBlocks) {
if (DomTreeNode *Node = DT.getNode(BB))
PQ.push({Node, Node->getLevel()});
}
SmallVector<DomTreeNode *, 32> Worklist;
SmallPtrSet<DomTreeNode *, 32> VisitedPQ;
SmallPtrSet<DomTreeNode *, 32> VisitedWorklist;
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dominator tree children of Root, inspecting their CFG edges with
// targets elsewhere on the dominator tree. Only targets whose level is at
// most Root's level are added to the iterated dominance frontier of the
// definition set.
Worklist.clear();
Worklist.push_back(Root);
VisitedWorklist.insert(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
BasicBlock *BB = Node->getBlock();
// Succ is the successor in the direction we are calculating IDF, so it is
// successor for IDF, and predecessor for Reverse IDF.
for (auto *Succ : children<NodeTy>(BB)) {
DomTreeNode *SuccNode = DT.getNode(Succ);
// Quickly skip all CFG edges that are also dominator tree edges instead
// of catching them below.
if (SuccNode->getIDom() == Node)
continue;
const unsigned SuccLevel = SuccNode->getLevel();
if (SuccLevel > RootLevel)
continue;
if (!VisitedPQ.insert(SuccNode).second)
continue;
BasicBlock *SuccBB = SuccNode->getBlock();
if (useLiveIn && !LiveInBlocks->count(SuccBB))
continue;
PHIBlocks.emplace_back(SuccBB);
if (!DefBlocks->count(SuccBB))
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
for (auto DomChild : *Node) {
if (VisitedWorklist.insert(DomChild).second)
Worklist.push_back(DomChild);
}
}
}
}
void TempScopInfo::buildCondition(BasicBlock *BB, Region &R) {
BasicBlock *RegionEntry = R.getEntry();
BBCond Cond;
DomTreeNode *BBNode = DT->getNode(BB), *EntryNode = DT->getNode(RegionEntry);
assert(BBNode && EntryNode && "Get null node while building condition!");
// Walk up the dominance tree until reaching the entry node. Collect all
// branching blocks on the path to BB except if BB postdominates the block
// containing the condition.
SmallVector<BasicBlock *, 4> DominatorBrBlocks;
while (BBNode != EntryNode) {
BasicBlock *CurBB = BBNode->getBlock();
BBNode = BBNode->getIDom();
assert(BBNode && "BBNode should not reach the root node!");
if (PDT->dominates(CurBB, BBNode->getBlock()))
continue;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getBlock()->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
if (Br->isUnconditional())
continue;
DominatorBrBlocks.push_back(BBNode->getBlock());
}
RegionInfo *RI = R.getRegionInfo();
// Iterate in reverse order over the dominating blocks. Until a non-affine
// branch was encountered add all conditions collected. If a non-affine branch
// was encountered, stop as we overapproximate from here on anyway.
for (auto BIt = DominatorBrBlocks.rbegin(), BEnd = DominatorBrBlocks.rend();
BIt != BEnd; BIt++) {
BasicBlock *BBNode = *BIt;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
assert(Br->isConditional() && "Assumed a conditional branch");
if (SD->isNonAffineSubRegion(RI->getRegionFor(BBNode), &R))
break;
BasicBlock *TrueBB = Br->getSuccessor(0), *FalseBB = Br->getSuccessor(1);
// Is BB on the ELSE side of the branch?
bool inverted = DT->dominates(FalseBB, BB);
// If both TrueBB and FalseBB dominate BB, one of them must be the target of
// a back-edge, i.e. a loop header.
if (inverted && DT->dominates(TrueBB, BB)) {
assert(
(DT->dominates(TrueBB, FalseBB) || DT->dominates(FalseBB, TrueBB)) &&
"One of the successors should be the loop header and dominate the"
"other!");
// It is not an invert if the FalseBB is the header.
if (DT->dominates(FalseBB, TrueBB))
inverted = false;
}
Comparison *Cmp;
buildAffineCondition(*(Br->getCondition()), inverted, &Cmp);
Cond.push_back(*Cmp);
}
if (!Cond.empty())
BBConds[BB] = Cond;
}
void StackAllocationPromoter::promoteAllocationToPhi() {
DEBUG(llvm::dbgs() << "*** Placing Phis for : " << *ASI);
// A list of blocks that will require new Phi values.
BlockSet PhiBlocks;
// The "piggy-bank" data-structure that we use for processing the dom-tree
// bottom-up.
NodePriorityQueue PQ;
// Collect all of the stores into the AllocStack. We know that at this point
// we have at most one store per block.
for (auto UI = ASI->use_begin(), E = ASI->use_end(); UI != E; ++UI) {
SILInstruction *II = UI->getUser();
// We need to place Phis for this block.
if (isa<StoreInst>(II)) {
// If the block is in the dom tree (dominated by the entry block).
if (DomTreeNode *Node = DT->getNode(II->getParent()))
PQ.push(std::make_pair(Node, DomTreeLevels[Node]));
}
}
DEBUG(llvm::dbgs() << "*** Found: " << PQ.size() << " Defs\n");
// A list of nodes for which we already calculated the dominator frontier.
llvm::SmallPtrSet<DomTreeNode *, 32> Visited;
SmallVector<DomTreeNode *, 32> Worklist;
// Scan all of the definitions in the function bottom-up using the priority
// queue.
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dom tree children of Root, inspecting their successors. Only
// J-edges, whose target level is at most Root's level are added to the
// dominance frontier.
Worklist.clear();
Worklist.push_back(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
SILBasicBlock *BB = Node->getBlock();
// For all successors of the node:
for (auto &Succ : BB->getSuccessors()) {
DomTreeNode *SuccNode = DT->getNode(Succ);
// Skip D-edges (edges that are dom-tree edges).
if (SuccNode->getIDom() == Node)
continue;
// Ignore J-edges that point to nodes that are not smaller or equal
// to the root level.
unsigned SuccLevel = DomTreeLevels[SuccNode];
if (SuccLevel > RootLevel)
continue;
// Ignore visited nodes.
if (!Visited.insert(SuccNode).second)
continue;
// If the new PHInode is not dominated by the allocation then it's dead.
if (!DT->dominates(ASI->getParent(), SuccNode->getBlock()))
continue;
// If the new PHInode is properly dominated by the deallocation then it
// is obviously a dead PHInode, so we don't need to insert it.
if (DSI && DT->properlyDominates(DSI->getParent(),
SuccNode->getBlock()))
continue;
// The successor node is a new PHINode. If this is a new PHI node
// then it may require additional definitions, so add it to the PQ.
if (PhiBlocks.insert(Succ).second)
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
// Add the children in the dom-tree to the worklist.
for (auto CI = Node->begin(), CE = Node->end(); CI != CE; ++CI)
if (!Visited.count(*CI))
Worklist.push_back(*CI);
}
}
DEBUG(llvm::dbgs() << "*** Found: " << PhiBlocks.size() << " new PHIs\n");
NumPhiPlaced += PhiBlocks.size();
// At this point we calculated the locations of all of the new Phi values.
// Next, add the Phi values and promote all of the loads and stores into the
// new locations.
// Replace the dummy values with new block arguments.
addBlockArguments(PhiBlocks);
// Hook up the Phi nodes, loads, and debug_value_addr with incoming values.
fixBranchesAndUses(PhiBlocks);
DEBUG(llvm::dbgs() << "*** Finished placing Phis ***\n");
}
/// At this point, we're committed to promoting the alloca using IDF's, and the
/// standard SSA construction algorithm. Determine which blocks need phi nodes
/// and see if we can optimize out some work by avoiding insertion of dead phi
/// nodes.
void PromoteMem2Reg::DetermineInsertionPoint(AllocaInst *AI, unsigned AllocaNum,
AllocaInfo &Info) {
// Unique the set of defining blocks for efficient lookup.
SmallPtrSet<BasicBlock *, 32> DefBlocks;
DefBlocks.insert(Info.DefiningBlocks.begin(), Info.DefiningBlocks.end());
// Determine which blocks the value is live in. These are blocks which lead
// to uses.
SmallPtrSet<BasicBlock *, 32> LiveInBlocks;
ComputeLiveInBlocks(AI, Info, DefBlocks, LiveInBlocks);
// Use a priority queue keyed on dominator tree level so that inserted nodes
// are handled from the bottom of the dominator tree upwards.
typedef std::priority_queue<DomTreeNodePair,
SmallVector<DomTreeNodePair, 32>,
DomTreeNodeCompare> IDFPriorityQueue;
IDFPriorityQueue PQ;
for (SmallPtrSet<BasicBlock *, 32>::const_iterator I = DefBlocks.begin(),
E = DefBlocks.end();
I != E; ++I) {
if (DomTreeNode *Node = DT.getNode(*I))
PQ.push(std::make_pair(Node, DomLevels[Node]));
}
SmallVector<std::pair<unsigned, BasicBlock *>, 32> DFBlocks;
SmallPtrSet<DomTreeNode *, 32> Visited;
SmallVector<DomTreeNode *, 32> Worklist;
while (!PQ.empty()) {
DomTreeNodePair RootPair = PQ.top();
PQ.pop();
DomTreeNode *Root = RootPair.first;
unsigned RootLevel = RootPair.second;
// Walk all dominator tree children of Root, inspecting their CFG edges with
// targets elsewhere on the dominator tree. Only targets whose level is at
// most Root's level are added to the iterated dominance frontier of the
// definition set.
Worklist.clear();
Worklist.push_back(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
BasicBlock *BB = Node->getBlock();
for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB); SI != SE;
++SI) {
DomTreeNode *SuccNode = DT.getNode(*SI);
// Quickly skip all CFG edges that are also dominator tree edges instead
// of catching them below.
if (SuccNode->getIDom() == Node)
continue;
unsigned SuccLevel = DomLevels[SuccNode];
if (SuccLevel > RootLevel)
continue;
if (!Visited.insert(SuccNode))
continue;
BasicBlock *SuccBB = SuccNode->getBlock();
if (!LiveInBlocks.count(SuccBB))
continue;
DFBlocks.push_back(std::make_pair(BBNumbers[SuccBB], SuccBB));
if (!DefBlocks.count(SuccBB))
PQ.push(std::make_pair(SuccNode, SuccLevel));
}
for (DomTreeNode::iterator CI = Node->begin(), CE = Node->end(); CI != CE;
++CI) {
if (!Visited.count(*CI))
Worklist.push_back(*CI);
}
}
}
if (DFBlocks.size() > 1)
std::sort(DFBlocks.begin(), DFBlocks.end());
unsigned CurrentVersion = 0;
for (unsigned i = 0, e = DFBlocks.size(); i != e; ++i)
QueuePhiNode(DFBlocks[i].second, AllocaNum, CurrentVersion);
}
