/// \brief Clones a loop \p OrigLoop. Returns the loop and the blocks in \p
/// Blocks.
///
/// Updates LoopInfo and DominatorTree assuming the loop is dominated by block
/// \p LoopDomBB. Insert the new blocks before block specified in \p Before.
Loop *llvm::cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB,
Loop *OrigLoop, ValueToValueMapTy &VMap,
const Twine &NameSuffix, LoopInfo *LI,
DominatorTree *DT,
SmallVectorImpl<BasicBlock *> &Blocks) {
assert(OrigLoop->getSubLoops().empty() &&
"Loop to be cloned cannot have inner loop");
Function *F = OrigLoop->getHeader()->getParent();
Loop *ParentLoop = OrigLoop->getParentLoop();
Loop *NewLoop = new Loop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
LI->addTopLevelLoop(NewLoop);
BasicBlock *OrigPH = OrigLoop->getLoopPreheader();
assert(OrigPH && "No preheader");
BasicBlock *NewPH = CloneBasicBlock(OrigPH, VMap, NameSuffix, F);
// To rename the loop PHIs.
VMap[OrigPH] = NewPH;
Blocks.push_back(NewPH);
// Update LoopInfo.
if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewPH, *LI);
// Update DominatorTree.
DT->addNewBlock(NewPH, LoopDomBB);
for (BasicBlock *BB : OrigLoop->getBlocks()) {
BasicBlock *NewBB = CloneBasicBlock(BB, VMap, NameSuffix, F);
VMap[BB] = NewBB;
// Update LoopInfo.
NewLoop->addBasicBlockToLoop(NewBB, *LI);
// Add DominatorTree node. After seeing all blocks, update to correct IDom.
DT->addNewBlock(NewBB, NewPH);
Blocks.push_back(NewBB);
}
for (BasicBlock *BB : OrigLoop->getBlocks()) {
// Update DominatorTree.
BasicBlock *IDomBB = DT->getNode(BB)->getIDom()->getBlock();
DT->changeImmediateDominator(cast<BasicBlock>(VMap[BB]),
cast<BasicBlock>(VMap[IDomBB]));
}
// Move them physically from the end of the block list.
F->getBasicBlockList().splice(Before->getIterator(), F->getBasicBlockList(),
NewPH);
F->getBasicBlockList().splice(Before->getIterator(), F->getBasicBlockList(),
NewLoop->getHeader()->getIterator(), F->end());
return NewLoop;
}
/// CloneLoop - Recursively clone the specified loop and all of its children,
/// mapping the blocks with the specified map.
static Loop *CloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM,
LoopInfo *LI, LPPassManager *LPM) {
Loop *New = new Loop();
LPM->insertLoop(New, PL);
// Add all of the blocks in L to the new loop.
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
I != E; ++I)
if (LI->getLoopFor(*I) == L)
New->addBasicBlockToLoop(cast<BasicBlock>(VM[*I]), LI->getBase());
// Add all of the subloops to the new loop.
for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
CloneLoop(*I, New, VM, LI, LPM);
return New;
}
void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) {
isl_ast_expr *Cond = isl_ast_node_if_get_cond(If);
Function *F = Builder.GetInsertBlock()->getParent();
LLVMContext &Context = F->getContext();
BasicBlock *CondBB =
SplitBlock(Builder.GetInsertBlock(), Builder.GetInsertPoint(), P);
CondBB->setName("polly.cond");
BasicBlock *MergeBB = SplitBlock(CondBB, CondBB->begin(), P);
MergeBB->setName("polly.merge");
BasicBlock *ThenBB = BasicBlock::Create(Context, "polly.then", F);
BasicBlock *ElseBB = BasicBlock::Create(Context, "polly.else", F);
DT.addNewBlock(ThenBB, CondBB);
DT.addNewBlock(ElseBB, CondBB);
DT.changeImmediateDominator(MergeBB, CondBB);
Loop *L = LI.getLoopFor(CondBB);
if (L) {
L->addBasicBlockToLoop(ThenBB, LI.getBase());
L->addBasicBlockToLoop(ElseBB, LI.getBase());
}
CondBB->getTerminator()->eraseFromParent();
Builder.SetInsertPoint(CondBB);
Value *Predicate = ExprBuilder.create(Cond);
Builder.CreateCondBr(Predicate, ThenBB, ElseBB);
Builder.SetInsertPoint(ThenBB);
Builder.CreateBr(MergeBB);
Builder.SetInsertPoint(ElseBB);
Builder.CreateBr(MergeBB);
Builder.SetInsertPoint(ThenBB->begin());
create(isl_ast_node_if_get_then(If));
Builder.SetInsertPoint(ElseBB->begin());
if (isl_ast_node_if_has_else(If))
create(isl_ast_node_if_get_else(If));
Builder.SetInsertPoint(MergeBB->begin());
isl_ast_node_free(If);
}
/// UpdateAnalysisInformation - Update DominatorTree, LoopInfo, and LCCSA
/// analysis information.
static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
ArrayRef<BasicBlock *> Preds,
Pass *P, bool &HasLoopExit) {
if (!P) return;
LoopInfo *LI = P->getAnalysisIfAvailable<LoopInfo>();
Loop *L = LI ? LI->getLoopFor(OldBB) : 0;
// If we need to preserve loop analyses, collect some information about how
// this split will affect loops.
bool IsLoopEntry = !!L;
bool SplitMakesNewLoopHeader = false;
if (LI) {
bool PreserveLCSSA = P->mustPreserveAnalysisID(LCSSAID);
for (ArrayRef<BasicBlock*>::iterator
i = Preds.begin(), e = Preds.end(); i != e; ++i) {
BasicBlock *Pred = *i;
// If we need to preserve LCSSA, determine if any of the preds is a loop
// exit.
if (PreserveLCSSA)
if (Loop *PL = LI->getLoopFor(Pred))
if (!PL->contains(OldBB))
HasLoopExit = true;
// If we need to preserve LoopInfo, note whether any of the preds crosses
// an interesting loop boundary.
if (!L) continue;
if (L->contains(Pred))
IsLoopEntry = false;
else
SplitMakesNewLoopHeader = true;
}
}
// Update dominator tree if available.
DominatorTree *DT = P->getAnalysisIfAvailable<DominatorTree>();
if (DT)
DT->splitBlock(NewBB);
if (!L) return;
if (IsLoopEntry) {
// Add the new block to the nearest enclosing loop (and not an adjacent
// loop). To find this, examine each of the predecessors and determine which
// loops enclose them, and select the most-nested loop which contains the
// loop containing the block being split.
Loop *InnermostPredLoop = 0;
for (ArrayRef<BasicBlock*>::iterator
i = Preds.begin(), e = Preds.end(); i != e; ++i) {
BasicBlock *Pred = *i;
if (Loop *PredLoop = LI->getLoopFor(Pred)) {
// Seek a loop which actually contains the block being split (to avoid
// adjacent loops).
while (PredLoop && !PredLoop->contains(OldBB))
PredLoop = PredLoop->getParentLoop();
// Select the most-nested of these loops which contains the block.
if (PredLoop && PredLoop->contains(OldBB) &&
(!InnermostPredLoop ||
InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
InnermostPredLoop = PredLoop;
}
}
if (InnermostPredLoop)
InnermostPredLoop->addBasicBlockToLoop(NewBB, LI->getBase());
} else {
L->addBasicBlockToLoop(NewBB, LI->getBase());
if (SplitMakesNewLoopHeader)
L->moveToHeader(NewBB);
}
}
// We generate a loop of either of the following structures:
//
// BeforeBB BeforeBB
// | |
// v v
// GuardBB PreHeaderBB
// / | | _____
// __ PreHeaderBB | v \/ |
// / \ / | HeaderBB latch
// latch HeaderBB | |\ |
// \ / \ / | \------/
// < \ / |
// \ / v
// ExitBB ExitBB
//
// depending on whether or not we know that it is executed at least once. If
// not, GuardBB checks if the loop is executed at least once. If this is the
// case we branch to PreHeaderBB and subsequently to the HeaderBB, which
// contains the loop iv 'polly.indvar', the incremented loop iv
// 'polly.indvar_next' as well as the condition to check if we execute another
// iteration of the loop. After the loop has finished, we branch to ExitBB.
Value *polly::createLoop(Value *LB, Value *UB, Value *Stride,
PollyIRBuilder &Builder, Pass *P, LoopInfo &LI,
DominatorTree &DT, BasicBlock *&ExitBB,
ICmpInst::Predicate Predicate,
ScopAnnotator *Annotator, bool Parallel,
bool UseGuard) {
Function *F = Builder.GetInsertBlock()->getParent();
LLVMContext &Context = F->getContext();
assert(LB->getType() == UB->getType() && "Types of loop bounds do not match");
IntegerType *LoopIVType = dyn_cast<IntegerType>(UB->getType());
assert(LoopIVType && "UB is not integer?");
BasicBlock *BeforeBB = Builder.GetInsertBlock();
BasicBlock *GuardBB =
UseGuard ? BasicBlock::Create(Context, "polly.loop_if", F) : nullptr;
BasicBlock *HeaderBB = BasicBlock::Create(Context, "polly.loop_header", F);
BasicBlock *PreHeaderBB =
BasicBlock::Create(Context, "polly.loop_preheader", F);
// Update LoopInfo
Loop *OuterLoop = LI.getLoopFor(BeforeBB);
Loop *NewLoop = new Loop();
if (OuterLoop)
OuterLoop->addChildLoop(NewLoop);
else
LI.addTopLevelLoop(NewLoop);
if (OuterLoop) {
if (GuardBB)
OuterLoop->addBasicBlockToLoop(GuardBB, LI);
OuterLoop->addBasicBlockToLoop(PreHeaderBB, LI);
}
NewLoop->addBasicBlockToLoop(HeaderBB, LI);
// Notify the annotator (if present) that we have a new loop, but only
// after the header block is set.
if (Annotator)
Annotator->pushLoop(NewLoop, Parallel);
// ExitBB
ExitBB = SplitBlock(BeforeBB, &*Builder.GetInsertPoint(), &DT, &LI);
ExitBB->setName("polly.loop_exit");
// BeforeBB
if (GuardBB) {
BeforeBB->getTerminator()->setSuccessor(0, GuardBB);
DT.addNewBlock(GuardBB, BeforeBB);
// GuardBB
Builder.SetInsertPoint(GuardBB);
Value *LoopGuard;
LoopGuard = Builder.CreateICmp(Predicate, LB, UB);
LoopGuard->setName("polly.loop_guard");
Builder.CreateCondBr(LoopGuard, PreHeaderBB, ExitBB);
DT.addNewBlock(PreHeaderBB, GuardBB);
} else {
BeforeBB->getTerminator()->setSuccessor(0, PreHeaderBB);
DT.addNewBlock(PreHeaderBB, BeforeBB);
}
// PreHeaderBB
Builder.SetInsertPoint(PreHeaderBB);
Builder.CreateBr(HeaderBB);
// HeaderBB
DT.addNewBlock(HeaderBB, PreHeaderBB);
Builder.SetInsertPoint(HeaderBB);
PHINode *IV = Builder.CreatePHI(LoopIVType, 2, "polly.indvar");
IV->addIncoming(LB, PreHeaderBB);
Stride = Builder.CreateZExtOrBitCast(Stride, LoopIVType);
Value *IncrementedIV = Builder.CreateNSWAdd(IV, Stride, "polly.indvar_next");
Value *LoopCondition;
UB = Builder.CreateSub(UB, Stride, "polly.adjust_ub");
LoopCondition = Builder.CreateICmp(Predicate, IV, UB);
LoopCondition->setName("polly.loop_cond");
// Create the loop latch and annotate it as such.
BranchInst *B = Builder.CreateCondBr(LoopCondition, HeaderBB, ExitBB);
if (Annotator)
Annotator->annotateLoopLatch(B, NewLoop, Parallel);
IV->addIncoming(IncrementedIV, HeaderBB);
if (GuardBB)
DT.changeImmediateDominator(ExitBB, GuardBB);
else
DT.changeImmediateDominator(ExitBB, HeaderBB);
// The loop body should be added here.
Builder.SetInsertPoint(HeaderBB->getFirstNonPHI());
return IV;
}
/// UnswitchNontrivialCondition - We determined that the loop is profitable
/// to unswitch when LIC equal Val. Split it into loop versions and test the
/// condition outside of either loop. Return the loops created as Out1/Out2.
void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
Loop *L) {
Function *F = loopHeader->getParent();
DEBUG(dbgs() << "loop-unswitch: Unswitching loop %"
<< loopHeader->getName() << " [" << L->getBlocks().size()
<< " blocks] in Function " << F->getName()
<< " when '" << *Val << "' == " << *LIC << "\n");
if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>())
SE->forgetLoop(L);
LoopBlocks.clear();
NewBlocks.clear();
// First step, split the preheader and exit blocks, and add these blocks to
// the LoopBlocks list.
BasicBlock *NewPreheader = SplitEdge(loopPreheader, loopHeader, this);
LoopBlocks.push_back(NewPreheader);
// We want the loop to come after the preheader, but before the exit blocks.
LoopBlocks.insert(LoopBlocks.end(), L->block_begin(), L->block_end());
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getUniqueExitBlocks(ExitBlocks);
// Split all of the edges from inside the loop to their exit blocks. Update
// the appropriate Phi nodes as we do so.
SplitExitEdges(L, ExitBlocks);
// The exit blocks may have been changed due to edge splitting, recompute.
ExitBlocks.clear();
L->getUniqueExitBlocks(ExitBlocks);
// Add exit blocks to the loop blocks.
LoopBlocks.insert(LoopBlocks.end(), ExitBlocks.begin(), ExitBlocks.end());
// Next step, clone all of the basic blocks that make up the loop (including
// the loop preheader and exit blocks), keeping track of the mapping between
// the instructions and blocks.
NewBlocks.reserve(LoopBlocks.size());
ValueToValueMapTy VMap;
for (unsigned i = 0, e = LoopBlocks.size(); i != e; ++i) {
BasicBlock *NewBB = CloneBasicBlock(LoopBlocks[i], VMap, ".us", F);
NewBlocks.push_back(NewBB);
VMap[LoopBlocks[i]] = NewBB; // Keep the BB mapping.
LPM->cloneBasicBlockSimpleAnalysis(LoopBlocks[i], NewBB, L);
}
// Splice the newly inserted blocks into the function right before the
// original preheader.
F->getBasicBlockList().splice(NewPreheader, F->getBasicBlockList(),
NewBlocks[0], F->end());
// Now we create the new Loop object for the versioned loop.
Loop *NewLoop = CloneLoop(L, L->getParentLoop(), VMap, LI, LPM);
Loop *ParentLoop = L->getParentLoop();
if (ParentLoop) {
// Make sure to add the cloned preheader and exit blocks to the parent loop
// as well.
ParentLoop->addBasicBlockToLoop(NewBlocks[0], LI->getBase());
}
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
BasicBlock *NewExit = cast<BasicBlock>(VMap[ExitBlocks[i]]);
// The new exit block should be in the same loop as the old one.
if (Loop *ExitBBLoop = LI->getLoopFor(ExitBlocks[i]))
ExitBBLoop->addBasicBlockToLoop(NewExit, LI->getBase());
assert(NewExit->getTerminator()->getNumSuccessors() == 1 &&
"Exit block should have been split to have one successor!");
BasicBlock *ExitSucc = NewExit->getTerminator()->getSuccessor(0);
// If the successor of the exit block had PHI nodes, add an entry for
// NewExit.
PHINode *PN;
for (BasicBlock::iterator I = ExitSucc->begin(); isa<PHINode>(I); ++I) {
PN = cast<PHINode>(I);
Value *V = PN->getIncomingValueForBlock(ExitBlocks[i]);
ValueToValueMapTy::iterator It = VMap.find(V);
if (It != VMap.end()) V = It->second;
PN->addIncoming(V, NewExit);
}
}
// Rewrite the code to refer to itself.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i)
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I)
RemapInstruction(I, VMap,RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
// Rewrite the original preheader to select between versions of the loop.
BranchInst *OldBR = cast<BranchInst>(loopPreheader->getTerminator());
assert(OldBR->isUnconditional() && OldBR->getSuccessor(0) == LoopBlocks[0] &&
"Preheader splitting did not work correctly!");
// Emit the new branch that selects between the two versions of this loop.
EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR);
LPM->deleteSimpleAnalysisValue(OldBR, L);
OldBR->eraseFromParent();
LoopProcessWorklist.push_back(NewLoop);
redoLoop = true;
// Keep a WeakVH holding onto LIC. If the first call to RewriteLoopBody
// deletes the instruction (for example by simplifying a PHI that feeds into
// the condition that we're unswitching on), we don't rewrite the second
// iteration.
WeakVH LICHandle(LIC);
// Now we rewrite the original code to know that the condition is true and the
// new code to know that the condition is false.
RewriteLoopBodyWithConditionConstant(L, LIC, Val, false);
// It's possible that simplifying one loop could cause the other to be
// changed to another value or a constant. If its a constant, don't simplify
// it.
if (!LoopProcessWorklist.empty() && LoopProcessWorklist.back() == NewLoop &&
LICHandle && !isa<Constant>(LICHandle))
RewriteLoopBodyWithConditionConstant(NewLoop, LICHandle, Val, true);
}
/// \brief Clones the body of the loop L, putting it between \p InsertTop and \p
/// InsertBot.
/// \param IterNumber The serial number of the iteration currently being
/// peeled off.
/// \param Exit The exit block of the original loop.
/// \param[out] NewBlocks A list of the the blocks in the newly created clone
/// \param[out] VMap The value map between the loop and the new clone.
/// \param LoopBlocks A helper for DFS-traversal of the loop.
/// \param LVMap A value-map that maps instructions from the original loop to
/// instructions in the last peeled-off iteration.
static void cloneLoopBlocks(Loop *L, unsigned IterNumber, BasicBlock *InsertTop,
BasicBlock *InsertBot, BasicBlock *Exit,
SmallVectorImpl<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
ValueToValueMapTy &LVMap, LoopInfo *LI) {
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *PreHeader = L->getLoopPreheader();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *ParentLoop = L->getParentLoop();
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".peel", F);
NewBlocks.push_back(NewBB);
if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewBB, *LI);
VMap[*BB] = NewBB;
}
// Hook-up the control flow for the newly inserted blocks.
// The new header is hooked up directly to the "top", which is either
// the original loop preheader (for the first iteration) or the previous
// iteration's exiting block (for every other iteration)
InsertTop->getTerminator()->setSuccessor(0, cast<BasicBlock>(VMap[Header]));
// Similarly, for the latch:
// The original exiting edge is still hooked up to the loop exit.
// The backedge now goes to the "bottom", which is either the loop's real
// header (for the last peeled iteration) or the copied header of the next
// iteration (for every other iteration)
BranchInst *LatchBR =
cast<BranchInst>(cast<BasicBlock>(VMap[Latch])->getTerminator());
unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1);
LatchBR->setSuccessor(HeaderIdx, InsertBot);
LatchBR->setSuccessor(1 - HeaderIdx, Exit);
// The new copy of the loop body starts with a bunch of PHI nodes
// that pick an incoming value from either the preheader, or the previous
// loop iteration. Since this copy is no longer part of the loop, we
// resolve this statically:
// For the first iteration, we use the value from the preheader directly.
// For any other iteration, we replace the phi with the value generated by
// the immediately preceding clone of the loop body (which represents
// the previous iteration).
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
if (IterNumber == 0) {
VMap[&*I] = NewPHI->getIncomingValueForBlock(PreHeader);
} else {
Value *LatchVal = NewPHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(LatchVal);
if (LatchInst && L->contains(LatchInst))
VMap[&*I] = LVMap[LatchInst];
else
VMap[&*I] = LatchVal;
}
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
}
// Fix up the outgoing values - we need to add a value for the iteration
// we've just created. Note that this must happen *after* the incoming
// values are adjusted, since the value going out of the latch may also be
// a value coming into the header.
for (BasicBlock::iterator I = Exit->begin(); isa<PHINode>(I); ++I) {
PHINode *PHI = cast<PHINode>(I);
Value *LatchVal = PHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(LatchVal);
if (LatchInst && L->contains(LatchInst))
LatchVal = VMap[LatchVal];
PHI->addIncoming(LatchVal, cast<BasicBlock>(VMap[Latch]));
}
// LastValueMap is updated with the values for the current loop
// which are used the next time this function is called.
for (const auto &KV : VMap)
LVMap[KV.first] = KV.second;
}
/// SplitBlockPredecessors - This method transforms BB by introducing a new
/// basic block into the function, and moving some of the predecessors of BB to
/// be predecessors of the new block. The new predecessors are indicated by the
/// Preds array, which has NumPreds elements in it. The new block is given a
/// suffix of 'Suffix'.
///
/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree,
/// DominanceFrontier, LoopInfo, and LCCSA but no other analyses.
/// In particular, it does not preserve LoopSimplify (because it's
/// complicated to handle the case where one of the edges being split
/// is an exit of a loop with other exits).
///
BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
BasicBlock *const *Preds,
unsigned NumPreds, const char *Suffix,
Pass *P) {
// Create new basic block, insert right before the original block.
BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), BB->getName()+Suffix,
BB->getParent(), BB);
// The new block unconditionally branches to the old block.
BranchInst *BI = BranchInst::Create(BB, NewBB);
LoopInfo *LI = P ? P->getAnalysisIfAvailable<LoopInfo>() : 0;
Loop *L = LI ? LI->getLoopFor(BB) : 0;
bool PreserveLCSSA = P->mustPreserveAnalysisID(LCSSAID);
// Move the edges from Preds to point to NewBB instead of BB.
// While here, if we need to preserve loop analyses, collect
// some information about how this split will affect loops.
bool HasLoopExit = false;
bool IsLoopEntry = !!L;
bool SplitMakesNewLoopHeader = false;
for (unsigned i = 0; i != NumPreds; ++i) {
// This is slightly more strict than necessary; the minimum requirement
// is that there be no more than one indirectbr branching to BB. And
// all BlockAddress uses would need to be updated.
assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
"Cannot split an edge from an IndirectBrInst");
Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB);
if (LI) {
// If we need to preserve LCSSA, determine if any of
// the preds is a loop exit.
if (PreserveLCSSA)
if (Loop *PL = LI->getLoopFor(Preds[i]))
if (!PL->contains(BB))
HasLoopExit = true;
// If we need to preserve LoopInfo, note whether any of the
// preds crosses an interesting loop boundary.
if (L) {
if (L->contains(Preds[i]))
IsLoopEntry = false;
else
SplitMakesNewLoopHeader = true;
}
}
}
// Update dominator tree and dominator frontier if available.
DominatorTree *DT = P ? P->getAnalysisIfAvailable<DominatorTree>() : 0;
if (DT)
DT->splitBlock(NewBB);
if (DominanceFrontier *DF = P ? P->getAnalysisIfAvailable<DominanceFrontier>():0)
DF->splitBlock(NewBB);
// Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
// node becomes an incoming value for BB's phi node. However, if the Preds
// list is empty, we need to insert dummy entries into the PHI nodes in BB to
// account for the newly created predecessor.
if (NumPreds == 0) {
// Insert dummy values as the incoming value.
for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
cast<PHINode>(I)->addIncoming(UndefValue::get(I->getType()), NewBB);
return NewBB;
}
AliasAnalysis *AA = P ? P->getAnalysisIfAvailable<AliasAnalysis>() : 0;
if (L) {
if (IsLoopEntry) {
// Add the new block to the nearest enclosing loop (and not an
// adjacent loop). To find this, examine each of the predecessors and
// determine which loops enclose them, and select the most-nested loop
// which contains the loop containing the block being split.
Loop *InnermostPredLoop = 0;
for (unsigned i = 0; i != NumPreds; ++i)
if (Loop *PredLoop = LI->getLoopFor(Preds[i])) {
// Seek a loop which actually contains the block being split (to
// avoid adjacent loops).
while (PredLoop && !PredLoop->contains(BB))
PredLoop = PredLoop->getParentLoop();
// Select the most-nested of these loops which contains the block.
if (PredLoop &&
PredLoop->contains(BB) &&
(!InnermostPredLoop ||
InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
InnermostPredLoop = PredLoop;
}
if (InnermostPredLoop)
InnermostPredLoop->addBasicBlockToLoop(NewBB, LI->getBase());
} else {
L->addBasicBlockToLoop(NewBB, LI->getBase());
if (SplitMakesNewLoopHeader)
L->moveToHeader(NewBB);
}
}
// Otherwise, create a new PHI node in NewBB for each PHI node in BB.
for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ) {
PHINode *PN = cast<PHINode>(I++);
// Check to see if all of the values coming in are the same. If so, we
// don't need to create a new PHI node, unless it's needed for LCSSA.
Value *InVal = 0;
if (!HasLoopExit) {
InVal = PN->getIncomingValueForBlock(Preds[0]);
for (unsigned i = 1; i != NumPreds; ++i)
if (InVal != PN->getIncomingValueForBlock(Preds[i])) {
InVal = 0;
break;
}
}
if (InVal) {
// If all incoming values for the new PHI would be the same, just don't
// make a new PHI. Instead, just remove the incoming values from the old
// PHI.
for (unsigned i = 0; i != NumPreds; ++i)
PN->removeIncomingValue(Preds[i], false);
} else {
// If the values coming into the block are not the same, we need a PHI.
// Create the new PHI node, insert it into NewBB at the end of the block
PHINode *NewPHI =
PHINode::Create(PN->getType(), PN->getName()+".ph", BI);
if (AA) AA->copyValue(PN, NewPHI);
// Move all of the PHI values for 'Preds' to the new PHI.
for (unsigned i = 0; i != NumPreds; ++i) {
Value *V = PN->removeIncomingValue(Preds[i], false);
NewPHI->addIncoming(V, Preds[i]);
}
InVal = NewPHI;
}
// Add an incoming value to the PHI node in the loop for the preheader
// edge.
PN->addIncoming(InVal, NewBB);
}
return NewBB;
}
/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB,
ArrayRef<BasicBlock *> Preds,
DominatorTree *DT, LoopInfo *LI,
bool PreserveLCSSA, bool &HasLoopExit) {
// Update dominator tree if available.
if (DT)
DT->splitBlock(NewBB);
// The rest of the logic is only relevant for updating the loop structures.
if (!LI)
return;
assert(DT && "DT should be available to update LoopInfo!");
Loop *L = LI->getLoopFor(OldBB);
// If we need to preserve loop analyses, collect some information about how
// this split will affect loops.
bool IsLoopEntry = !!L;
bool SplitMakesNewLoopHeader = false;
for (BasicBlock *Pred : Preds) {
// Preds that are not reachable from entry should not be used to identify if
// OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
// are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
// as true and make the NewBB the header of some loop. This breaks LI.
if (!DT->isReachableFromEntry(Pred))
continue;
// If we need to preserve LCSSA, determine if any of the preds is a loop
// exit.
if (PreserveLCSSA)
if (Loop *PL = LI->getLoopFor(Pred))
if (!PL->contains(OldBB))
HasLoopExit = true;
// If we need to preserve LoopInfo, note whether any of the preds crosses
// an interesting loop boundary.
if (!L)
continue;
if (L->contains(Pred))
IsLoopEntry = false;
else
SplitMakesNewLoopHeader = true;
}
// Unless we have a loop for OldBB, nothing else to do here.
if (!L)
return;
if (IsLoopEntry) {
// Add the new block to the nearest enclosing loop (and not an adjacent
// loop). To find this, examine each of the predecessors and determine which
// loops enclose them, and select the most-nested loop which contains the
// loop containing the block being split.
Loop *InnermostPredLoop = nullptr;
for (BasicBlock *Pred : Preds) {
if (Loop *PredLoop = LI->getLoopFor(Pred)) {
// Seek a loop which actually contains the block being split (to avoid
// adjacent loops).
while (PredLoop && !PredLoop->contains(OldBB))
PredLoop = PredLoop->getParentLoop();
// Select the most-nested of these loops which contains the block.
if (PredLoop && PredLoop->contains(OldBB) &&
(!InnermostPredLoop ||
InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
InnermostPredLoop = PredLoop;
}
}
if (InnermostPredLoop)
InnermostPredLoop->addBasicBlockToLoop(NewBB, *LI);
} else {
L->addBasicBlockToLoop(NewBB, *LI);
if (SplitMakesNewLoopHeader)
L->moveToHeader(NewBB);
}
}
/// Create a clone of the blocks in a loop and connect them together.
/// If UnrollProlog is true, loop structure will not be cloned, otherwise a new
/// loop will be created including all cloned blocks, and the iterator of it
/// switches to count NewIter down to 0.
///
static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool UnrollProlog,
BasicBlock *InsertTop, BasicBlock *InsertBot,
std::vector<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
LoopInfo *LI) {
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *NewLoop = 0;
Loop *ParentLoop = L->getParentLoop();
if (!UnrollProlog) {
NewLoop = new Loop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
LI->addTopLevelLoop(NewLoop);
}
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".prol", F);
NewBlocks.push_back(NewBB);
if (NewLoop)
NewLoop->addBasicBlockToLoop(NewBB, *LI);
else if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewBB, *LI);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (Latch == *BB) {
// For the last block, if UnrollProlog is true, create a direct jump to
// InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (UnrollProlog) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, "prol.iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[I]);
if (UnrollProlog) {
VMap[I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (VMap[InVal])
NewPHI->setIncomingValue(idx, VMap[InVal]);
}
}
if (NewLoop) {
// Add unroll disable metadata to disable future unrolling for this loop.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
MDNode *LoopID = NewLoop->getLoopID();
if (LoopID) {
// First remove any existing loop unrolling metadata.
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
}
LLVMContext &Context = NewLoop->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
NewLoop->setLoopID(NewLoopID);
}
}
/*
in addition to the original condition of the loop, insert one cond
on the virtual iterator, given the linf and lsup bounds for the chunk
this cond exits and returns to the decision block to start a new chunk
*/
BasicBlock *insertChunkCond(Loop *&L, LoopInfo *LI, Value *vi, Value *lsup,
BasicBlock *dcb, Value *vi_dcb_val,
PHINode *&phi_vi) {
BasicBlock *H = L->getHeader();
BasicBlock *newCond = BasicBlock::Create(
H->getContext(), Twine("__kernel__" + H->getName().str() + "_viCond"),
H->getParent(), H);
std::vector<BasicBlock *> Lblocks = L->getBlocks();
BasicBlock *exitBlock = BasicBlock::Create(
H->getContext(), Twine(H->getName().str() + "_exitChunk"), H->getParent(),
H);
BranchInst::Create(dcb, exitBlock);
phi_vi = PHINode::Create(Type::getInt64Ty(H->getContext()), 2, "vi_value",
newCond);
phi_vi->addIncoming(vi_dcb_val, dcb);
LoadInst *load_lsup = new LoadInst(lsup, "lsup_value", newCond);
ICmpInst *cmp = new ICmpInst(*newCond, ICmpInst::ICMP_SLT, phi_vi, load_lsup,
vi->getName() + "_cmp");
// Make sure all predecessors now go to our new condition
std::vector<TerminatorInst *> termInstrs;
BasicBlock *lp = L->getLoopPredecessor();
for (auto it = pred_begin(H), end = pred_end(H); it != end; ++it) {
if ((*it) == lp) {
// Original entry should be redirected to dcb
TerminatorInst *tinstr = (*it)->getTerminator();
for (auto it = tinstr->op_begin(), end = tinstr->op_end(); it != end;
++it) {
Use *use = &*it;
if (use->get() == H) {
use->set(dcb);
}
}
} else {
termInstrs.push_back((*it)->getTerminator());
}
}
for (auto &tinstr : termInstrs) {
for (auto it = tinstr->op_begin(), end = tinstr->op_end(); it != end;
++it) {
Use *use = &*it;
if (use->get() == H) {
use->set(newCond);
}
}
}
BranchInst::Create(H, exitBlock, cmp, newCond);
// update loop info
if (L != LI->getLoopFor(newCond)) {
L->addBasicBlockToLoop(newCond, *LI);
}
L->moveToHeader(newCond);
Loop *Lp = L->getParentLoop();
if (Lp)
Lp->addBasicBlockToLoop(exitBlock, *LI);
return newCond;
}
/*
This method performs Unroll and Jam. For a simple loop like:
for (i = ..)
Fore(i)
for (j = ..)
SubLoop(i, j)
Aft(i)
Instead of doing normal inner or outer unrolling, we do:
for (i = .., i+=2)
Fore(i)
Fore(i+1)
for (j = ..)
SubLoop(i, j)
SubLoop(i+1, j)
Aft(i)
Aft(i+1)
So the outer loop is essetially unrolled and then the inner loops are fused
("jammed") together into a single loop. This can increase speed when there
are loads in SubLoop that are invariant to i, as they become shared between
the now jammed inner loops.
We do this by spliting the blocks in the loop into Fore, Subloop and Aft.
Fore blocks are those before the inner loop, Aft are those after. Normal
Unroll code is used to copy each of these sets of blocks and the results are
combined together into the final form above.
isSafeToUnrollAndJam should be used prior to calling this to make sure the
unrolling will be valid. Checking profitablility is also advisable.
*/
LoopUnrollResult
llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
AssumptionCache *AC, OptimizationRemarkEmitter *ORE) {
// When we enter here we should have already checked that it is safe
BasicBlock *Header = L->getHeader();
assert(L->getSubLoops().size() == 1);
Loop *SubLoop = *L->begin();
// Don't enter the unroll code if there is nothing to do.
if (TripCount == 0 && Count < 2) {
LLVM_DEBUG(dbgs() << "Won't unroll; almost nothing to do\n");
return LoopUnrollResult::Unmodified;
}
assert(Count > 0);
assert(TripMultiple > 0);
assert(TripCount == 0 || TripCount % TripMultiple == 0);
// Are we eliminating the loop control altogether?
bool CompletelyUnroll = (Count == TripCount);
// We use the runtime remainder in cases where we don't know trip multiple
if (TripMultiple == 1 || TripMultiple % Count != 0) {
if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
/*UseEpilogRemainder*/ true,
UnrollRemainder, LI, SE, DT, AC, true)) {
LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
"generated when assuming runtime trip count\n");
return LoopUnrollResult::Unmodified;
}
}
// Notify ScalarEvolution that the loop will be substantially changed,
// if not outright eliminated.
if (SE) {
SE->forgetLoop(L);
SE->forgetLoop(SubLoop);
}
using namespace ore;
// Report the unrolling decision.
if (CompletelyUnroll) {
LLVM_DEBUG(dbgs() << "COMPLETELY UNROLL AND JAMMING loop %"
<< Header->getName() << " with trip count " << TripCount
<< "!\n");
ORE->emit(OptimizationRemark(DEBUG_TYPE, "FullyUnrolled", L->getStartLoc(),
L->getHeader())
<< "completely unroll and jammed loop with "
<< NV("UnrollCount", TripCount) << " iterations");
} else {
auto DiagBuilder = [&]() {
OptimizationRemark Diag(DEBUG_TYPE, "PartialUnrolled", L->getStartLoc(),
L->getHeader());
return Diag << "unroll and jammed loop by a factor of "
<< NV("UnrollCount", Count);
};
LLVM_DEBUG(dbgs() << "UNROLL AND JAMMING loop %" << Header->getName()
<< " by " << Count);
if (TripMultiple != 1) {
LLVM_DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
ORE->emit([&]() {
return DiagBuilder() << " with " << NV("TripMultiple", TripMultiple)
<< " trips per branch";
});
} else {
LLVM_DEBUG(dbgs() << " with run-time trip count");
ORE->emit([&]() { return DiagBuilder() << " with run-time trip count"; });
}
LLVM_DEBUG(dbgs() << "!\n");
}
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *LatchBlock = L->getLoopLatch();
BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
assert(Preheader && LatchBlock && Header);
assert(BI && !BI->isUnconditional());
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
bool SubLoopContinueOnTrue = SubLoop->contains(
SubLoop->getLoopLatch()->getTerminator()->getSuccessor(0));
// Partition blocks in an outer/inner loop pair into blocks before and after
// the loop
BasicBlockSet SubLoopBlocks;
BasicBlockSet ForeBlocks;
BasicBlockSet AftBlocks;
partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks, AftBlocks,
DT);
// We keep track of the entering/first and exiting/last block of each of
// Fore/SubLoop/Aft in each iteration. This helps make the stapling up of
// blocks easier.
std::vector<BasicBlock *> ForeBlocksFirst;
std::vector<BasicBlock *> ForeBlocksLast;
std::vector<BasicBlock *> SubLoopBlocksFirst;
std::vector<BasicBlock *> SubLoopBlocksLast;
std::vector<BasicBlock *> AftBlocksFirst;
std::vector<BasicBlock *> AftBlocksLast;
ForeBlocksFirst.push_back(Header);
ForeBlocksLast.push_back(SubLoop->getLoopPreheader());
SubLoopBlocksFirst.push_back(SubLoop->getHeader());
SubLoopBlocksLast.push_back(SubLoop->getExitingBlock());
AftBlocksFirst.push_back(SubLoop->getExitBlock());
AftBlocksLast.push_back(L->getExitingBlock());
// Maps Blocks[0] -> Blocks[It]
ValueToValueMapTy LastValueMap;
// Move any instructions from fore phi operands from AftBlocks into Fore.
moveHeaderPhiOperandsToForeBlocks(
Header, LatchBlock, SubLoop->getLoopPreheader()->getTerminator(),
AftBlocks);
// The current on-the-fly SSA update requires blocks to be processed in
// reverse postorder so that LastValueMap contains the correct value at each
// exit.
LoopBlocksDFS DFS(L);
DFS.perform(LI);
// Stash the DFS iterators before adding blocks to the loop.
LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
if (Header->getParent()->isDebugInfoForProfiling())
for (BasicBlock *BB : L->getBlocks())
for (Instruction &I : *BB)
if (!isa<DbgInfoIntrinsic>(&I))
if (const DILocation *DIL = I.getDebugLoc())
I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
// Copy all blocks
for (unsigned It = 1; It != Count; ++It) {
std::vector<BasicBlock *> NewBlocks;
// Maps Blocks[It] -> Blocks[It-1]
DenseMap<Value *, Value *> PrevItValueMap;
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
ValueToValueMapTy VMap;
BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
Header->getParent()->getBasicBlockList().push_back(New);
if (ForeBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == ForeBlocksFirst[0])
ForeBlocksFirst.push_back(New);
if (*BB == ForeBlocksLast[0])
ForeBlocksLast.push_back(New);
} else if (SubLoopBlocks.count(*BB)) {
SubLoop->addBasicBlockToLoop(New, *LI);
if (*BB == SubLoopBlocksFirst[0])
SubLoopBlocksFirst.push_back(New);
if (*BB == SubLoopBlocksLast[0])
SubLoopBlocksLast.push_back(New);
} else if (AftBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == AftBlocksFirst[0])
AftBlocksFirst.push_back(New);
if (*BB == AftBlocksLast[0])
AftBlocksLast.push_back(New);
} else {
llvm_unreachable("BB being cloned should be in Fore/Sub/Aft");
}
// Update our running maps of newest clones
PrevItValueMap[New] = (It == 1 ? *BB : LastValueMap[*BB]);
LastValueMap[*BB] = New;
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI) {
PrevItValueMap[VI->second] =
const_cast<Value *>(It == 1 ? VI->first : LastValueMap[VI->first]);
LastValueMap[VI->first] = VI->second;
}
NewBlocks.push_back(New);
// Update DomTree:
if (*BB == ForeBlocksFirst[0])
DT->addNewBlock(New, ForeBlocksLast[It - 1]);
else if (*BB == SubLoopBlocksFirst[0])
DT->addNewBlock(New, SubLoopBlocksLast[It - 1]);
else if (*BB == AftBlocksFirst[0])
DT->addNewBlock(New, AftBlocksLast[It - 1]);
else {
// Each set of blocks (Fore/Sub/Aft) will have the same internal domtree
// structure.
auto BBDomNode = DT->getNode(*BB);
auto BBIDom = BBDomNode->getIDom();
BasicBlock *OriginalBBIDom = BBIDom->getBlock();
assert(OriginalBBIDom);
assert(LastValueMap[cast<Value>(OriginalBBIDom)]);
DT->addNewBlock(
New, cast<BasicBlock>(LastValueMap[cast<Value>(OriginalBBIDom)]));
}
}
// Remap all instructions in the most recent iteration
for (BasicBlock *NewBlock : NewBlocks) {
for (Instruction &I : *NewBlock) {
::remapInstruction(&I, LastValueMap);
if (auto *II = dyn_cast<IntrinsicInst>(&I))
if (II->getIntrinsicID() == Intrinsic::assume)
AC->registerAssumption(II);
}
}
// Alter the ForeBlocks phi's, pointing them at the latest version of the
// value from the previous iteration's phis
for (PHINode &Phi : ForeBlocksFirst[It]->phis()) {
Value *OldValue = Phi.getIncomingValueForBlock(AftBlocksLast[It]);
assert(OldValue && "should have incoming edge from Aft[It]");
Value *NewValue = OldValue;
if (Value *PrevValue = PrevItValueMap[OldValue])
NewValue = PrevValue;
assert(Phi.getNumOperands() == 2);
Phi.setIncomingBlock(0, ForeBlocksLast[It - 1]);
Phi.setIncomingValue(0, NewValue);
Phi.removeIncomingValue(1);
}
}
// Now that all the basic blocks for the unrolled iterations are in place,
// finish up connecting the blocks and phi nodes. At this point LastValueMap
// is the last unrolled iterations values.
// Update Phis in BB from OldBB to point to NewBB
auto updatePHIBlocks = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB) {
for (PHINode &Phi : BB->phis()) {
int I = Phi.getBasicBlockIndex(OldBB);
Phi.setIncomingBlock(I, NewBB);
}
};
// Update Phis in BB from OldBB to point to NewBB and use the latest value
// from LastValueMap
auto updatePHIBlocksAndValues = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB,
ValueToValueMapTy &LastValueMap) {
for (PHINode &Phi : BB->phis()) {
for (unsigned b = 0; b < Phi.getNumIncomingValues(); ++b) {
if (Phi.getIncomingBlock(b) == OldBB) {
Value *OldValue = Phi.getIncomingValue(b);
if (Value *LastValue = LastValueMap[OldValue])
Phi.setIncomingValue(b, LastValue);
Phi.setIncomingBlock(b, NewBB);
break;
}
}
}
};
// Move all the phis from Src into Dest
auto movePHIs = [](BasicBlock *Src, BasicBlock *Dest) {
Instruction *insertPoint = Dest->getFirstNonPHI();
while (PHINode *Phi = dyn_cast<PHINode>(Src->begin()))
Phi->moveBefore(insertPoint);
};
// Update the PHI values outside the loop to point to the last block
updatePHIBlocksAndValues(LoopExit, AftBlocksLast[0], AftBlocksLast.back(),
LastValueMap);
// Update ForeBlocks successors and phi nodes
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast.back()->getTerminator());
BasicBlock *Dest = SubLoopBlocksFirst[0];
ForeTerm->setSuccessor(0, Dest);
if (CompletelyUnroll) {
while (PHINode *Phi = dyn_cast<PHINode>(ForeBlocksFirst[0]->begin())) {
Phi->replaceAllUsesWith(Phi->getIncomingValueForBlock(Preheader));
Phi->getParent()->getInstList().erase(Phi);
}
} else {
// Update the PHI values to point to the last aft block
updatePHIBlocksAndValues(ForeBlocksFirst[0], AftBlocksLast[0],
AftBlocksLast.back(), LastValueMap);
}
for (unsigned It = 1; It != Count; It++) {
// Remap ForeBlock successors from previous iteration to this
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast[It - 1]->getTerminator());
BasicBlock *Dest = ForeBlocksFirst[It];
ForeTerm->setSuccessor(0, Dest);
}
// Subloop successors and phis
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast.back()->getTerminator());
SubTerm->setSuccessor(!SubLoopContinueOnTrue, SubLoopBlocksFirst[0]);
SubTerm->setSuccessor(SubLoopContinueOnTrue, AftBlocksFirst[0]);
updatePHIBlocks(SubLoopBlocksFirst[0], ForeBlocksLast[0],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast[It - 1]->getTerminator());
BranchInst::Create(SubLoopBlocksFirst[It], SubTerm);
SubTerm->eraseFromParent();
updatePHIBlocks(SubLoopBlocksFirst[It], ForeBlocksLast[It],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(SubLoopBlocksFirst[It], SubLoopBlocksFirst[0]);
}
// Aft blocks successors and phis
BranchInst *Term = cast<BranchInst>(AftBlocksLast.back()->getTerminator());
if (CompletelyUnroll) {
BranchInst::Create(LoopExit, Term);
Term->eraseFromParent();
} else {
Term->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]);
}
updatePHIBlocks(AftBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *AftTerm =
cast<BranchInst>(AftBlocksLast[It - 1]->getTerminator());
BranchInst::Create(AftBlocksFirst[It], AftTerm);
AftTerm->eraseFromParent();
updatePHIBlocks(AftBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(AftBlocksFirst[It], AftBlocksFirst[0]);
}
// Dominator Tree. Remove the old links between Fore, Sub and Aft, adding the
// new ones required.
if (Count != 1) {
SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, ForeBlocksLast[0],
SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete,
SubLoopBlocksLast[0], AftBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
ForeBlocksLast.back(), SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
SubLoopBlocksLast.back(), AftBlocksFirst[0]);
DT->applyUpdates(DTUpdates);
}
// Merge adjacent basic blocks, if possible.
SmallPtrSet<BasicBlock *, 16> MergeBlocks;
MergeBlocks.insert(ForeBlocksLast.begin(), ForeBlocksLast.end());
MergeBlocks.insert(SubLoopBlocksLast.begin(), SubLoopBlocksLast.end());
MergeBlocks.insert(AftBlocksLast.begin(), AftBlocksLast.end());
while (!MergeBlocks.empty()) {
BasicBlock *BB = *MergeBlocks.begin();
BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
if (Term && Term->isUnconditional() && L->contains(Term->getSuccessor(0))) {
BasicBlock *Dest = Term->getSuccessor(0);
if (BasicBlock *Fold = foldBlockIntoPredecessor(Dest, LI, SE, DT)) {
// Don't remove BB and add Fold as they are the same BB
assert(Fold == BB);
(void)Fold;
MergeBlocks.erase(Dest);
} else
MergeBlocks.erase(BB);
} else
MergeBlocks.erase(BB);
}
// At this point, the code is well formed. We now do a quick sweep over the
// inserted code, doing constant propagation and dead code elimination as we
// go.
simplifyLoopAfterUnroll(SubLoop, true, LI, SE, DT, AC);
simplifyLoopAfterUnroll(L, !CompletelyUnroll && Count > 1, LI, SE, DT, AC);
NumCompletelyUnrolledAndJammed += CompletelyUnroll;
++NumUnrolledAndJammed;
#ifndef NDEBUG
// We shouldn't have done anything to break loop simplify form or LCSSA.
Loop *OuterL = L->getParentLoop();
Loop *OutestLoop = OuterL ? OuterL : (!CompletelyUnroll ? L : SubLoop);
assert(OutestLoop->isRecursivelyLCSSAForm(*DT, *LI));
if (!CompletelyUnroll)
assert(L->isLoopSimplifyForm());
assert(SubLoop->isLoopSimplifyForm());
assert(DT->verify());
#endif
// Update LoopInfo if the loop is completely removed.
if (CompletelyUnroll)
LI->erase(L);
return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled
: LoopUnrollResult::PartiallyUnrolled;
}
/// CloneLoop - Clone Loop. Clone dominator info. Populate ValueMap
/// using old blocks to new blocks mapping.
Loop *llvm::CloneLoop(Loop *OrigL, LPPassManager *LPM, LoopInfo *LI,
DenseMap<const Value *, Value *> &ValueMap, Pass *P) {
DominatorTree *DT = NULL;
DominanceFrontier *DF = NULL;
if (P) {
DT = P->getAnalysisToUpdate<DominatorTree>();
DF = P->getAnalysisToUpdate<DominanceFrontier>();
}
SmallVector<BasicBlock *, 16> NewBlocks;
// Populate loop nest.
SmallVector<Loop *, 8> LoopNest;
LoopNest.push_back(OrigL);
Loop *NewParentLoop = NULL;
while (!LoopNest.empty()) {
Loop *L = LoopNest.back();
LoopNest.pop_back();
Loop *NewLoop = new Loop();
if (!NewParentLoop)
NewParentLoop = NewLoop;
LPM->insertLoop(NewLoop, L->getParentLoop());
// Clone Basic Blocks.
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
I != E; ++I) {
BasicBlock *BB = *I;
BasicBlock *NewBB = CloneBasicBlock(BB, ValueMap, ".clone");
ValueMap[BB] = NewBB;
if (P)
LPM->cloneBasicBlockSimpleAnalysis(BB, NewBB, L);
NewLoop->addBasicBlockToLoop(NewBB, LI->getBase());
NewBlocks.push_back(NewBB);
}
// Clone dominator info.
if (DT)
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
I != E; ++I) {
BasicBlock *BB = *I;
CloneDominatorInfo(BB, ValueMap, DT, DF);
}
// Process sub loops
for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I)
LoopNest.push_back(*I);
}
// Remap instructions to reference operands from ValueMap.
for(SmallVector<BasicBlock *, 16>::iterator NBItr = NewBlocks.begin(),
NBE = NewBlocks.end(); NBItr != NBE; ++NBItr) {
BasicBlock *NB = *NBItr;
for(BasicBlock::iterator BI = NB->begin(), BE = NB->end();
BI != BE; ++BI) {
Instruction *Insn = BI;
for (unsigned index = 0, num_ops = Insn->getNumOperands();
index != num_ops; ++index) {
Value *Op = Insn->getOperand(index);
DenseMap<const Value *, Value *>::iterator OpItr = ValueMap.find(Op);
if (OpItr != ValueMap.end())
Insn->setOperand(index, OpItr->second);
}
}
}
BasicBlock *Latch = OrigL->getLoopLatch();
Function *F = Latch->getParent();
F->getBasicBlockList().insert(OrigL->getHeader(),
NewBlocks.begin(), NewBlocks.end());
return NewParentLoop;
}
