/// Fold the loop tail into the loop exit by speculating the loop tail
/// instructions. Typically, this is a single post-increment. In the case of a
/// simple 2-block loop, hoisting the increment can be much better than
/// duplicating the entire loop header. In the case of loops with early exits,
/// rotation will not work anyway, but simplifyLoopLatch will put the loop in
/// canonical form so downstream passes can handle it.
///
/// I don't believe this invalidates SCEV.
bool LoopRotate::simplifyLoopLatch(Loop *L) {
BasicBlock *Latch = L->getLoopLatch();
if (!Latch || Latch->hasAddressTaken())
return false;
BranchInst *Jmp = dyn_cast<BranchInst>(Latch->getTerminator());
if (!Jmp || !Jmp->isUnconditional())
return false;
BasicBlock *LastExit = Latch->getSinglePredecessor();
if (!LastExit || !L->isLoopExiting(LastExit))
return false;
BranchInst *BI = dyn_cast<BranchInst>(LastExit->getTerminator());
if (!BI)
return false;
if (!shouldSpeculateInstrs(Latch->begin(), Jmp))
return false;
DEBUG(dbgs() << "Folding loop latch " << Latch->getName() << " into "
<< LastExit->getName() << "\n");
// Hoist the instructions from Latch into LastExit.
LastExit->getInstList().splice(BI, Latch->getInstList(), Latch->begin(), Jmp);
unsigned FallThruPath = BI->getSuccessor(0) == Latch ? 0 : 1;
BasicBlock *Header = Jmp->getSuccessor(0);
assert(Header == L->getHeader() && "expected a backward branch");
// Remove Latch from the CFG so that LastExit becomes the new Latch.
BI->setSuccessor(FallThruPath, Header);
Latch->replaceSuccessorsPhiUsesWith(LastExit);
Jmp->eraseFromParent();
// Nuke the Latch block.
assert(Latch->empty() && "unable to evacuate Latch");
LI->removeBlock(Latch);
if (DominatorTreeWrapperPass *DTWP =
getAnalysisIfAvailable<DominatorTreeWrapperPass>())
DTWP->getDomTree().eraseNode(Latch);
Latch->eraseFromParent();
return true;
}
/// Unroll the given loop by Count. The loop must be in LCSSA form. Returns true
/// if unrolling was successful, or false if the loop was unmodified. Unrolling
/// can only fail when the loop's latch block is not terminated by a conditional
/// branch instruction. However, if the trip count (and multiple) are not known,
/// loop unrolling will mostly produce more code that is no faster.
///
/// TripCount is generally defined as the number of times the loop header
/// executes. UnrollLoop relaxes the definition to permit early exits: here
/// TripCount is the iteration on which control exits LatchBlock if no early
/// exits were taken. Note that UnrollLoop assumes that the loop counter test
/// terminates LatchBlock in order to remove unnecesssary instances of the
/// test. In other words, control may exit the loop prior to TripCount
/// iterations via an early branch, but control may not exit the loop from the
/// LatchBlock's terminator prior to TripCount iterations.
///
/// Similarly, TripMultiple divides the number of times that the LatchBlock may
/// execute without exiting the loop.
///
/// The LoopInfo Analysis that is passed will be kept consistent.
///
/// If a LoopPassManager is passed in, and the loop is fully removed, it will be
/// removed from the LoopPassManager as well. LPM can also be NULL.
///
/// This utility preserves LoopInfo. If DominatorTree or ScalarEvolution are
/// available from the Pass it must also preserve those analyses.
bool llvm::UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
bool AllowRuntime, unsigned TripMultiple,
LoopInfo *LI, Pass *PP, LPPassManager *LPM) {
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) {
DEBUG(dbgs() << " Can't unroll; loop preheader-insertion failed.\n");
return false;
}
BasicBlock *LatchBlock = L->getLoopLatch();
if (!LatchBlock) {
DEBUG(dbgs() << " Can't unroll; loop exit-block-insertion failed.\n");
return false;
}
// Loops with indirectbr cannot be cloned.
if (!L->isSafeToClone()) {
DEBUG(dbgs() << " Can't unroll; Loop body cannot be cloned.\n");
return false;
}
BasicBlock *Header = L->getHeader();
BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
if (!BI || BI->isUnconditional()) {
// The loop-rotate pass can be helpful to avoid this in many cases.
DEBUG(dbgs() <<
" Can't unroll; loop not terminated by a conditional branch.\n");
return false;
}
if (Header->hasAddressTaken()) {
// The loop-rotate pass can be helpful to avoid this in many cases.
DEBUG(dbgs() <<
" Won't unroll loop: address of header block is taken.\n");
return false;
}
if (TripCount != 0)
DEBUG(dbgs() << " Trip Count = " << TripCount << "\n");
if (TripMultiple != 1)
DEBUG(dbgs() << " Trip Multiple = " << TripMultiple << "\n");
// Effectively "DCE" unrolled iterations that are beyond the tripcount
// and will never be executed.
if (TripCount != 0 && Count > TripCount)
Count = TripCount;
// Don't enter the unroll code if there is nothing to do. This way we don't
// need to support "partial unrolling by 1".
if (TripCount == 0 && Count < 2)
return false;
assert(Count > 0);
assert(TripMultiple > 0);
assert(TripCount == 0 || TripCount % TripMultiple == 0);
// Are we eliminating the loop control altogether?
bool CompletelyUnroll = Count == TripCount;
// We assume a run-time trip count if the compiler cannot
// figure out the loop trip count and the unroll-runtime
// flag is specified.
bool RuntimeTripCount = (TripCount == 0 && Count > 0 && AllowRuntime);
if (RuntimeTripCount && !UnrollRuntimeLoopProlog(L, Count, LI, LPM))
return false;
// Notify ScalarEvolution that the loop will be substantially changed,
// if not outright eliminated.
if (PP) {
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
if (SE)
SE->forgetLoop(L);
}
// If we know the trip count, we know the multiple...
unsigned BreakoutTrip = 0;
if (TripCount != 0) {
BreakoutTrip = TripCount % Count;
TripMultiple = 0;
} else {
// Figure out what multiple to use.
BreakoutTrip = TripMultiple =
(unsigned)GreatestCommonDivisor64(Count, TripMultiple);
}
// Report the unrolling decision.
DebugLoc LoopLoc = L->getStartLoc();
Function *F = Header->getParent();
LLVMContext &Ctx = F->getContext();
if (CompletelyUnroll) {
DEBUG(dbgs() << "COMPLETELY UNROLLING loop %" << Header->getName()
<< " with trip count " << TripCount << "!\n");
emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc,
Twine("completely unrolled loop with ") +
Twine(TripCount) + " iterations");
} else {
DEBUG(dbgs() << "UNROLLING loop %" << Header->getName()
<< " by " << Count);
Twine DiagMsg("unrolled loop by a factor of " + Twine(Count));
if (TripMultiple == 0 || BreakoutTrip != TripMultiple) {
DEBUG(dbgs() << " with a breakout at trip " << BreakoutTrip);
DiagMsg.concat(" with a breakout at trip " + Twine(BreakoutTrip));
} else if (TripMultiple != 1) {
DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
DiagMsg.concat(" with " + Twine(TripMultiple) + " trips per branch");
} else if (RuntimeTripCount) {
DEBUG(dbgs() << " with run-time trip count");
DiagMsg.concat(" with run-time trip count");
}
DEBUG(dbgs() << "!\n");
emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc, DiagMsg);
}
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
// For the first iteration of the loop, we should use the precloned values for
// PHI nodes. Insert associations now.
ValueToValueMapTy LastValueMap;
std::vector<PHINode*> OrigPHINode;
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
OrigPHINode.push_back(cast<PHINode>(I));
}
std::vector<BasicBlock*> Headers;
std::vector<BasicBlock*> Latches;
Headers.push_back(Header);
Latches.push_back(LatchBlock);
// The current on-the-fly SSA update requires blocks to be processed in
// reverse postorder so that LastValueMap contains the correct value at each
// exit.
LoopBlocksDFS DFS(L);
DFS.perform(LI);
// Stash the DFS iterators before adding blocks to the loop.
LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
for (unsigned It = 1; It != Count; ++It) {
std::vector<BasicBlock*> NewBlocks;
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
ValueToValueMapTy VMap;
BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
Header->getParent()->getBasicBlockList().push_back(New);
// Loop over all of the PHI nodes in the block, changing them to use the
// incoming values from the previous block.
if (*BB == Header)
for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
PHINode *NewPHI = cast<PHINode>(VMap[OrigPHINode[i]]);
Value *InVal = NewPHI->getIncomingValueForBlock(LatchBlock);
if (Instruction *InValI = dyn_cast<Instruction>(InVal))
if (It > 1 && L->contains(InValI))
InVal = LastValueMap[InValI];
VMap[OrigPHINode[i]] = InVal;
New->getInstList().erase(NewPHI);
}
// Update our running map of newest clones
LastValueMap[*BB] = New;
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI)
LastValueMap[VI->first] = VI->second;
L->addBasicBlockToLoop(New, LI->getBase());
// Add phi entries for newly created values to all exit blocks.
for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB);
SI != SE; ++SI) {
if (L->contains(*SI))
continue;
for (BasicBlock::iterator BBI = (*SI)->begin();
PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) {
Value *Incoming = phi->getIncomingValueForBlock(*BB);
ValueToValueMapTy::iterator It = LastValueMap.find(Incoming);
if (It != LastValueMap.end())
Incoming = It->second;
phi->addIncoming(Incoming, New);
}
}
// Keep track of new headers and latches as we create them, so that
// we can insert the proper branches later.
if (*BB == Header)
Headers.push_back(New);
if (*BB == LatchBlock)
Latches.push_back(New);
NewBlocks.push_back(New);
}
// Remap all instructions in the most recent iteration
for (unsigned i = 0; i < NewBlocks.size(); ++i)
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I)
::RemapInstruction(I, LastValueMap);
}
// Loop over the PHI nodes in the original block, setting incoming values.
for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
PHINode *PN = OrigPHINode[i];
if (CompletelyUnroll) {
PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader));
Header->getInstList().erase(PN);
}
else if (Count > 1) {
Value *InVal = PN->removeIncomingValue(LatchBlock, false);
// If this value was defined in the loop, take the value defined by the
// last iteration of the loop.
if (Instruction *InValI = dyn_cast<Instruction>(InVal)) {
if (L->contains(InValI))
InVal = LastValueMap[InVal];
}
assert(Latches.back() == LastValueMap[LatchBlock] && "bad last latch");
PN->addIncoming(InVal, Latches.back());
}
}
// Now that all the basic blocks for the unrolled iterations are in place,
// set up the branches to connect them.
for (unsigned i = 0, e = Latches.size(); i != e; ++i) {
// The original branch was replicated in each unrolled iteration.
BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
// The branch destination.
unsigned j = (i + 1) % e;
BasicBlock *Dest = Headers[j];
bool NeedConditional = true;
if (RuntimeTripCount && j != 0) {
NeedConditional = false;
}
// For a complete unroll, make the last iteration end with a branch
// to the exit block.
if (CompletelyUnroll && j == 0) {
Dest = LoopExit;
NeedConditional = false;
}
// If we know the trip count or a multiple of it, we can safely use an
// unconditional branch for some iterations.
if (j != BreakoutTrip && (TripMultiple == 0 || j % TripMultiple != 0)) {
NeedConditional = false;
}
if (NeedConditional) {
// Update the conditional branch's successor for the following
// iteration.
Term->setSuccessor(!ContinueOnTrue, Dest);
} else {
// Remove phi operands at this loop exit
if (Dest != LoopExit) {
BasicBlock *BB = Latches[i];
for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
SI != SE; ++SI) {
if (*SI == Headers[i])
continue;
for (BasicBlock::iterator BBI = (*SI)->begin();
PHINode *Phi = dyn_cast<PHINode>(BBI); ++BBI) {
Phi->removeIncomingValue(BB, false);
}
}
}
// Replace the conditional branch with an unconditional one.
BranchInst::Create(Dest, Term);
Term->eraseFromParent();
}
}
// Merge adjacent basic blocks, if possible.
for (unsigned i = 0, e = Latches.size(); i != e; ++i) {
BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
if (Term->isUnconditional()) {
BasicBlock *Dest = Term->getSuccessor(0);
if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI, LPM))
std::replace(Latches.begin(), Latches.end(), Dest, Fold);
}
}
DominatorTree *DT = nullptr;
if (PP) {
// FIXME: Reconstruct dom info, because it is not preserved properly.
// Incrementally updating domtree after loop unrolling would be easy.
if (DominatorTreeWrapperPass *DTWP =
PP->getAnalysisIfAvailable<DominatorTreeWrapperPass>()) {
DT = &DTWP->getDomTree();
DT->recalculate(*L->getHeader()->getParent());
}
// Simplify any new induction variables in the partially unrolled loop.
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
if (SE && !CompletelyUnroll) {
SmallVector<WeakVH, 16> DeadInsts;
simplifyLoopIVs(L, SE, LPM, DeadInsts);
// Aggressively clean up dead instructions that simplifyLoopIVs already
// identified. Any remaining should be cleaned up below.
while (!DeadInsts.empty())
if (Instruction *Inst =
dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
RecursivelyDeleteTriviallyDeadInstructions(Inst);
}
}
// At this point, the code is well formed. We now do a quick sweep over the
// inserted code, doing constant propagation and dead code elimination as we
// go.
const std::vector<BasicBlock*> &NewLoopBlocks = L->getBlocks();
for (std::vector<BasicBlock*>::const_iterator BB = NewLoopBlocks.begin(),
BBE = NewLoopBlocks.end(); BB != BBE; ++BB)
for (BasicBlock::iterator I = (*BB)->begin(), E = (*BB)->end(); I != E; ) {
Instruction *Inst = I++;
if (isInstructionTriviallyDead(Inst))
(*BB)->getInstList().erase(Inst);
else if (Value *V = SimplifyInstruction(Inst))
if (LI->replacementPreservesLCSSAForm(Inst, V)) {
Inst->replaceAllUsesWith(V);
(*BB)->getInstList().erase(Inst);
}
}
NumCompletelyUnrolled += CompletelyUnroll;
++NumUnrolled;
Loop *OuterL = L->getParentLoop();
// Remove the loop from the LoopPassManager if it's completely removed.
if (CompletelyUnroll && LPM != nullptr)
LPM->deleteLoopFromQueue(L);
// If we have a pass and a DominatorTree we should re-simplify impacted loops
// to ensure subsequent analyses can rely on this form. We want to simplify
// at least one layer outside of the loop that was unrolled so that any
// changes to the parent loop exposed by the unrolling are considered.
if (PP && DT) {
if (!OuterL && !CompletelyUnroll)
OuterL = L;
if (OuterL) {
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
simplifyLoop(OuterL, DT, LI, PP, /*AliasAnalysis*/ nullptr, SE);
// LCSSA must be performed on the outermost affected loop. The unrolled
// loop's last loop latch is guaranteed to be in the outermost loop after
// deleteLoopFromQueue updates LoopInfo.
Loop *LatchLoop = LI->getLoopFor(Latches.back());
if (!OuterL->contains(LatchLoop))
while (OuterL->getParentLoop() != LatchLoop)
OuterL = OuterL->getParentLoop();
formLCSSARecursively(*OuterL, *DT, SE);
}
}
return true;
}
bool LoopInterchangeTransform::adjustLoopBranches() {
DEBUG(dbgs() << "adjustLoopBranches called\n");
// Adjust the loop preheader
BasicBlock *InnerLoopHeader = InnerLoop->getHeader();
BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
BasicBlock *InnerLoopLatch = InnerLoop->getLoopLatch();
BasicBlock *OuterLoopLatch = OuterLoop->getLoopLatch();
BasicBlock *OuterLoopPreHeader = OuterLoop->getLoopPreheader();
BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
BasicBlock *OuterLoopPredecessor = OuterLoopPreHeader->getUniquePredecessor();
BasicBlock *InnerLoopLatchPredecessor =
InnerLoopLatch->getUniquePredecessor();
BasicBlock *InnerLoopLatchSuccessor;
BasicBlock *OuterLoopLatchSuccessor;
BranchInst *OuterLoopLatchBI =
dyn_cast<BranchInst>(OuterLoopLatch->getTerminator());
BranchInst *InnerLoopLatchBI =
dyn_cast<BranchInst>(InnerLoopLatch->getTerminator());
BranchInst *OuterLoopHeaderBI =
dyn_cast<BranchInst>(OuterLoopHeader->getTerminator());
BranchInst *InnerLoopHeaderBI =
dyn_cast<BranchInst>(InnerLoopHeader->getTerminator());
if (!OuterLoopPredecessor || !InnerLoopLatchPredecessor ||
!OuterLoopLatchBI || !InnerLoopLatchBI || !OuterLoopHeaderBI ||
!InnerLoopHeaderBI)
return false;
BranchInst *InnerLoopLatchPredecessorBI =
dyn_cast<BranchInst>(InnerLoopLatchPredecessor->getTerminator());
BranchInst *OuterLoopPredecessorBI =
dyn_cast<BranchInst>(OuterLoopPredecessor->getTerminator());
if (!OuterLoopPredecessorBI || !InnerLoopLatchPredecessorBI)
return false;
BasicBlock *InnerLoopHeaderSuccessor = InnerLoopHeader->getUniqueSuccessor();
if (!InnerLoopHeaderSuccessor)
return false;
// Adjust Loop Preheader and headers
unsigned NumSucc = OuterLoopPredecessorBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (OuterLoopPredecessorBI->getSuccessor(i) == OuterLoopPreHeader)
OuterLoopPredecessorBI->setSuccessor(i, InnerLoopPreHeader);
}
NumSucc = OuterLoopHeaderBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (OuterLoopHeaderBI->getSuccessor(i) == OuterLoopLatch)
OuterLoopHeaderBI->setSuccessor(i, LoopExit);
else if (OuterLoopHeaderBI->getSuccessor(i) == InnerLoopPreHeader)
OuterLoopHeaderBI->setSuccessor(i, InnerLoopHeaderSuccessor);
}
// Adjust reduction PHI's now that the incoming block has changed.
updateIncomingBlock(InnerLoopHeaderSuccessor, InnerLoopHeader,
OuterLoopHeader);
BranchInst::Create(OuterLoopPreHeader, InnerLoopHeaderBI);
InnerLoopHeaderBI->eraseFromParent();
// -------------Adjust loop latches-----------
if (InnerLoopLatchBI->getSuccessor(0) == InnerLoopHeader)
InnerLoopLatchSuccessor = InnerLoopLatchBI->getSuccessor(1);
else
InnerLoopLatchSuccessor = InnerLoopLatchBI->getSuccessor(0);
NumSucc = InnerLoopLatchPredecessorBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (InnerLoopLatchPredecessorBI->getSuccessor(i) == InnerLoopLatch)
InnerLoopLatchPredecessorBI->setSuccessor(i, InnerLoopLatchSuccessor);
}
// Adjust PHI nodes in InnerLoopLatchSuccessor. Update all uses of PHI with
// the value and remove this PHI node from inner loop.
SmallVector<PHINode *, 8> LcssaVec;
for (auto I = InnerLoopLatchSuccessor->begin(); isa<PHINode>(I); ++I) {
PHINode *LcssaPhi = cast<PHINode>(I);
LcssaVec.push_back(LcssaPhi);
}
for (auto I = LcssaVec.begin(), E = LcssaVec.end(); I != E; ++I) {
PHINode *P = *I;
Value *Incoming = P->getIncomingValueForBlock(InnerLoopLatch);
P->replaceAllUsesWith(Incoming);
P->eraseFromParent();
}
if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopHeader)
OuterLoopLatchSuccessor = OuterLoopLatchBI->getSuccessor(1);
else
OuterLoopLatchSuccessor = OuterLoopLatchBI->getSuccessor(0);
if (InnerLoopLatchBI->getSuccessor(1) == InnerLoopLatchSuccessor)
InnerLoopLatchBI->setSuccessor(1, OuterLoopLatchSuccessor);
else
InnerLoopLatchBI->setSuccessor(0, OuterLoopLatchSuccessor);
updateIncomingBlock(OuterLoopLatchSuccessor, OuterLoopLatch, InnerLoopLatch);
if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopLatchSuccessor) {
OuterLoopLatchBI->setSuccessor(0, InnerLoopLatch);
} else {
OuterLoopLatchBI->setSuccessor(1, InnerLoopLatch);
}
return true;
}
/// UnswitchNontrivialCondition - We determined that the loop is profitable
/// to unswitch when LIC equal Val. Split it into loop versions and test the
/// condition outside of either loop. Return the loops created as Out1/Out2.
void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
Loop *L) {
Function *F = loopHeader->getParent();
DEBUG(dbgs() << "loop-unswitch: Unswitching loop %"
<< loopHeader->getName() << " [" << L->getBlocks().size()
<< " blocks] in Function " << F->getName()
<< " when '" << *Val << "' == " << *LIC << "\n");
if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>())
SE->forgetLoop(L);
LoopBlocks.clear();
NewBlocks.clear();
// First step, split the preheader and exit blocks, and add these blocks to
// the LoopBlocks list.
BasicBlock *NewPreheader = SplitEdge(loopPreheader, loopHeader, this);
LoopBlocks.push_back(NewPreheader);
// We want the loop to come after the preheader, but before the exit blocks.
LoopBlocks.insert(LoopBlocks.end(), L->block_begin(), L->block_end());
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getUniqueExitBlocks(ExitBlocks);
// Split all of the edges from inside the loop to their exit blocks. Update
// the appropriate Phi nodes as we do so.
SplitExitEdges(L, ExitBlocks);
// The exit blocks may have been changed due to edge splitting, recompute.
ExitBlocks.clear();
L->getUniqueExitBlocks(ExitBlocks);
// Add exit blocks to the loop blocks.
LoopBlocks.insert(LoopBlocks.end(), ExitBlocks.begin(), ExitBlocks.end());
// Next step, clone all of the basic blocks that make up the loop (including
// the loop preheader and exit blocks), keeping track of the mapping between
// the instructions and blocks.
NewBlocks.reserve(LoopBlocks.size());
ValueToValueMapTy VMap;
for (unsigned i = 0, e = LoopBlocks.size(); i != e; ++i) {
BasicBlock *NewBB = CloneBasicBlock(LoopBlocks[i], VMap, ".us", F);
NewBlocks.push_back(NewBB);
VMap[LoopBlocks[i]] = NewBB; // Keep the BB mapping.
LPM->cloneBasicBlockSimpleAnalysis(LoopBlocks[i], NewBB, L);
}
// Splice the newly inserted blocks into the function right before the
// original preheader.
F->getBasicBlockList().splice(NewPreheader, F->getBasicBlockList(),
NewBlocks[0], F->end());
// Now we create the new Loop object for the versioned loop.
Loop *NewLoop = CloneLoop(L, L->getParentLoop(), VMap, LI, LPM);
Loop *ParentLoop = L->getParentLoop();
if (ParentLoop) {
// Make sure to add the cloned preheader and exit blocks to the parent loop
// as well.
ParentLoop->addBasicBlockToLoop(NewBlocks[0], LI->getBase());
}
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
BasicBlock *NewExit = cast<BasicBlock>(VMap[ExitBlocks[i]]);
// The new exit block should be in the same loop as the old one.
if (Loop *ExitBBLoop = LI->getLoopFor(ExitBlocks[i]))
ExitBBLoop->addBasicBlockToLoop(NewExit, LI->getBase());
assert(NewExit->getTerminator()->getNumSuccessors() == 1 &&
"Exit block should have been split to have one successor!");
BasicBlock *ExitSucc = NewExit->getTerminator()->getSuccessor(0);
// If the successor of the exit block had PHI nodes, add an entry for
// NewExit.
PHINode *PN;
for (BasicBlock::iterator I = ExitSucc->begin(); isa<PHINode>(I); ++I) {
PN = cast<PHINode>(I);
Value *V = PN->getIncomingValueForBlock(ExitBlocks[i]);
ValueToValueMapTy::iterator It = VMap.find(V);
if (It != VMap.end()) V = It->second;
PN->addIncoming(V, NewExit);
}
}
// Rewrite the code to refer to itself.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i)
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I)
RemapInstruction(I, VMap,RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
// Rewrite the original preheader to select between versions of the loop.
BranchInst *OldBR = cast<BranchInst>(loopPreheader->getTerminator());
assert(OldBR->isUnconditional() && OldBR->getSuccessor(0) == LoopBlocks[0] &&
"Preheader splitting did not work correctly!");
// Emit the new branch that selects between the two versions of this loop.
EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR);
LPM->deleteSimpleAnalysisValue(OldBR, L);
OldBR->eraseFromParent();
LoopProcessWorklist.push_back(NewLoop);
redoLoop = true;
// Keep a WeakVH holding onto LIC. If the first call to RewriteLoopBody
// deletes the instruction (for example by simplifying a PHI that feeds into
// the condition that we're unswitching on), we don't rewrite the second
// iteration.
WeakVH LICHandle(LIC);
// Now we rewrite the original code to know that the condition is true and the
// new code to know that the condition is false.
RewriteLoopBodyWithConditionConstant(L, LIC, Val, false);
// It's possible that simplifying one loop could cause the other to be
// changed to another value or a constant. If its a constant, don't simplify
// it.
if (!LoopProcessWorklist.empty() && LoopProcessWorklist.back() == NewLoop &&
LICHandle && !isa<Constant>(LICHandle))
RewriteLoopBodyWithConditionConstant(NewLoop, LICHandle, Val, true);
}
/// Rotate loop LP. Return true if the loop is rotated.
bool LoopRotate::rotateLoop(Loop *L) {
// If the loop has only one block then there is not much to rotate.
if (L->getBlocks().size() == 1)
return false;
BasicBlock *OrigHeader = L->getHeader();
BranchInst *BI = dyn_cast<BranchInst>(OrigHeader->getTerminator());
if (BI == 0 || BI->isUnconditional())
return false;
// If the loop header is not one of the loop exiting blocks then
// either this loop is already rotated or it is not
// suitable for loop rotation transformations.
if (!L->isLoopExiting(OrigHeader))
return false;
// Updating PHInodes in loops with multiple exits adds complexity.
// Keep it simple, and restrict loop rotation to loops with one exit only.
// In future, lift this restriction and support for multiple exits if
// required.
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getExitBlocks(ExitBlocks);
if (ExitBlocks.size() > 1)
return false;
// Check size of original header and reject loop if it is very big.
{
CodeMetrics Metrics;
Metrics.analyzeBasicBlock(OrigHeader);
if (Metrics.NumInsts > MAX_HEADER_SIZE)
return false;
}
// Now, this loop is suitable for rotation.
BasicBlock *OrigPreheader = L->getLoopPreheader();
BasicBlock *OrigLatch = L->getLoopLatch();
// If the loop could not be converted to canonical form, it must have an
// indirectbr in it, just give up.
if (OrigPreheader == 0 || OrigLatch == 0)
return false;
// Anything ScalarEvolution may know about this loop or the PHI nodes
// in its header will soon be invalidated.
if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>())
SE->forgetLoop(L);
// Find new Loop header. NewHeader is a Header's one and only successor
// that is inside loop. Header's other successor is outside the
// loop. Otherwise loop is not suitable for rotation.
BasicBlock *Exit = BI->getSuccessor(0);
BasicBlock *NewHeader = BI->getSuccessor(1);
if (L->contains(Exit))
std::swap(Exit, NewHeader);
assert(NewHeader && "Unable to determine new loop header");
assert(L->contains(NewHeader) && !L->contains(Exit) &&
"Unable to determine loop header and exit blocks");
// This code assumes that the new header has exactly one predecessor.
// Remove any single-entry PHI nodes in it.
assert(NewHeader->getSinglePredecessor() &&
"New header doesn't have one pred!");
FoldSingleEntryPHINodes(NewHeader);
// Begin by walking OrigHeader and populating ValueMap with an entry for
// each Instruction.
BasicBlock::iterator I = OrigHeader->begin(), E = OrigHeader->end();
ValueToValueMapTy ValueMap;
// For PHI nodes, the value available in OldPreHeader is just the
// incoming value from OldPreHeader.
for (; PHINode *PN = dyn_cast<PHINode>(I); ++I)
ValueMap[PN] = PN->getIncomingValue(PN->getBasicBlockIndex(OrigPreheader));
// For the rest of the instructions, either hoist to the OrigPreheader if
// possible or create a clone in the OldPreHeader if not.
TerminatorInst *LoopEntryBranch = OrigPreheader->getTerminator();
while (I != E) {
Instruction *Inst = I++;
// If the instruction's operands are invariant and it doesn't read or write
// memory, then it is safe to hoist. Doing this doesn't change the order of
// execution in the preheader, but does prevent the instruction from
// executing in each iteration of the loop. This means it is safe to hoist
// something that might trap, but isn't safe to hoist something that reads
// memory (without proving that the loop doesn't write).
if (L->hasLoopInvariantOperands(Inst) &&
!Inst->mayReadFromMemory() && !Inst->mayWriteToMemory() &&
!isa<TerminatorInst>(Inst) && !isa<DbgInfoIntrinsic>(Inst)) {
Inst->moveBefore(LoopEntryBranch);
continue;
}
// Otherwise, create a duplicate of the instruction.
Instruction *C = Inst->clone();
// Eagerly remap the operands of the instruction.
RemapInstruction(C, ValueMap,
RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
// With the operands remapped, see if the instruction constant folds or is
// otherwise simplifyable. This commonly occurs because the entry from PHI
// nodes allows icmps and other instructions to fold.
Value *V = SimplifyInstruction(C);
if (V && LI->replacementPreservesLCSSAForm(C, V)) {
// If so, then delete the temporary instruction and stick the folded value
// in the map.
delete C;
ValueMap[Inst] = V;
} else {
// Otherwise, stick the new instruction into the new block!
C->setName(Inst->getName());
C->insertBefore(LoopEntryBranch);
ValueMap[Inst] = C;
}
}
// Along with all the other instructions, we just cloned OrigHeader's
// terminator into OrigPreHeader. Fix up the PHI nodes in each of OrigHeader's
// successors by duplicating their incoming values for OrigHeader.
TerminatorInst *TI = OrigHeader->getTerminator();
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
for (BasicBlock::iterator BI = TI->getSuccessor(i)->begin();
PHINode *PN = dyn_cast<PHINode>(BI); ++BI)
PN->addIncoming(PN->getIncomingValueForBlock(OrigHeader), OrigPreheader);
// Now that OrigPreHeader has a clone of OrigHeader's terminator, remove
// OrigPreHeader's old terminator (the original branch into the loop), and
// remove the corresponding incoming values from the PHI nodes in OrigHeader.
LoopEntryBranch->eraseFromParent();
// If there were any uses of instructions in the duplicated block outside the
// loop, update them, inserting PHI nodes as required
RewriteUsesOfClonedInstructions(OrigHeader, OrigPreheader, ValueMap);
// NewHeader is now the header of the loop.
L->moveToHeader(NewHeader);
assert(L->getHeader() == NewHeader && "Latch block is our new header");
// At this point, we've finished our major CFG changes. As part of cloning
// the loop into the preheader we've simplified instructions and the
// duplicated conditional branch may now be branching on a constant. If it is
// branching on a constant and if that constant means that we enter the loop,
// then we fold away the cond branch to an uncond branch. This simplifies the
// loop in cases important for nested loops, and it also means we don't have
// to split as many edges.
BranchInst *PHBI = cast<BranchInst>(OrigPreheader->getTerminator());
assert(PHBI->isConditional() && "Should be clone of BI condbr!");
if (!isa<ConstantInt>(PHBI->getCondition()) ||
PHBI->getSuccessor(cast<ConstantInt>(PHBI->getCondition())->isZero())
!= NewHeader) {
// The conditional branch can't be folded, handle the general case.
// Update DominatorTree to reflect the CFG change we just made. Then split
// edges as necessary to preserve LoopSimplify form.
if (DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>()) {
// Since OrigPreheader now has the conditional branch to Exit block, it is
// the dominator of Exit.
DT->changeImmediateDominator(Exit, OrigPreheader);
DT->changeImmediateDominator(NewHeader, OrigPreheader);
// Update OrigHeader to be dominated by the new header block.
DT->changeImmediateDominator(OrigHeader, OrigLatch);
}
// Right now OrigPreHeader has two successors, NewHeader and ExitBlock, and
// thus is not a preheader anymore. Split the edge to form a real preheader.
BasicBlock *NewPH = SplitCriticalEdge(OrigPreheader, NewHeader, this);
NewPH->setName(NewHeader->getName() + ".lr.ph");
// Preserve canonical loop form, which means that 'Exit' should have only one
// predecessor.
BasicBlock *ExitSplit = SplitCriticalEdge(L->getLoopLatch(), Exit, this);
ExitSplit->moveBefore(Exit);
} else {
// We can fold the conditional branch in the preheader, this makes things
// simpler. The first step is to remove the extra edge to the Exit block.
Exit->removePredecessor(OrigPreheader, true /*preserve LCSSA*/);
BranchInst::Create(NewHeader, PHBI);
PHBI->eraseFromParent();
// With our CFG finalized, update DomTree if it is available.
if (DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>()) {
// Update OrigHeader to be dominated by the new header block.
DT->changeImmediateDominator(NewHeader, OrigPreheader);
DT->changeImmediateDominator(OrigHeader, OrigLatch);
}
}
assert(L->getLoopPreheader() && "Invalid loop preheader after loop rotation");
assert(L->getLoopLatch() && "Invalid loop latch after loop rotation");
// Now that the CFG and DomTree are in a consistent state again, try to merge
// the OrigHeader block into OrigLatch. This will succeed if they are
// connected by an unconditional branch. This is just a cleanup so the
// emitted code isn't too gross in this common case.
MergeBlockIntoPredecessor(OrigHeader, this);
++NumRotated;
return true;
}
/// This works like CloneAndPruneFunctionInto, except that it does not clone the
/// entire function. Instead it starts at an instruction provided by the caller
/// and copies (and prunes) only the code reachable from that instruction.
void llvm::CloneAndPruneIntoFromInst(Function *NewFunc, const Function *OldFunc,
const Instruction *StartingInst,
ValueToValueMapTy &VMap,
bool ModuleLevelChanges,
SmallVectorImpl<ReturnInst *> &Returns,
const char *NameSuffix,
ClonedCodeInfo *CodeInfo) {
assert(NameSuffix && "NameSuffix cannot be null!");
ValueMapTypeRemapper *TypeMapper = nullptr;
ValueMaterializer *Materializer = nullptr;
#ifndef NDEBUG
// If the cloning starts at the beginning of the function, verify that
// the function arguments are mapped.
if (!StartingInst)
for (const Argument &II : OldFunc->args())
assert(VMap.count(&II) && "No mapping from source argument specified!");
#endif
PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, ModuleLevelChanges,
NameSuffix, CodeInfo);
const BasicBlock *StartingBB;
if (StartingInst)
StartingBB = StartingInst->getParent();
else {
StartingBB = &OldFunc->getEntryBlock();
StartingInst = &StartingBB->front();
}
// Clone the entry block, and anything recursively reachable from it.
std::vector<const BasicBlock*> CloneWorklist;
PFC.CloneBlock(StartingBB, StartingInst->getIterator(), CloneWorklist);
while (!CloneWorklist.empty()) {
const BasicBlock *BB = CloneWorklist.back();
CloneWorklist.pop_back();
PFC.CloneBlock(BB, BB->begin(), CloneWorklist);
}
// Loop over all of the basic blocks in the old function. If the block was
// reachable, we have cloned it and the old block is now in the value map:
// insert it into the new function in the right order. If not, ignore it.
//
// Defer PHI resolution until rest of function is resolved.
SmallVector<const PHINode*, 16> PHIToResolve;
for (const BasicBlock &BI : *OldFunc) {
Value *V = VMap[&BI];
BasicBlock *NewBB = cast_or_null<BasicBlock>(V);
if (!NewBB) continue; // Dead block.
// Add the new block to the new function.
NewFunc->getBasicBlockList().push_back(NewBB);
// Handle PHI nodes specially, as we have to remove references to dead
// blocks.
for (BasicBlock::const_iterator I = BI.begin(), E = BI.end(); I != E; ++I) {
// PHI nodes may have been remapped to non-PHI nodes by the caller or
// during the cloning process.
if (const PHINode *PN = dyn_cast<PHINode>(I)) {
if (isa<PHINode>(VMap[PN]))
PHIToResolve.push_back(PN);
else
break;
} else {
break;
}
}
// Finally, remap the terminator instructions, as those can't be remapped
// until all BBs are mapped.
RemapInstruction(NewBB->getTerminator(), VMap,
ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges,
TypeMapper, Materializer);
}
// Defer PHI resolution until rest of function is resolved, PHI resolution
// requires the CFG to be up-to-date.
for (unsigned phino = 0, e = PHIToResolve.size(); phino != e; ) {
const PHINode *OPN = PHIToResolve[phino];
unsigned NumPreds = OPN->getNumIncomingValues();
const BasicBlock *OldBB = OPN->getParent();
BasicBlock *NewBB = cast<BasicBlock>(VMap[OldBB]);
// Map operands for blocks that are live and remove operands for blocks
// that are dead.
for (; phino != PHIToResolve.size() &&
PHIToResolve[phino]->getParent() == OldBB; ++phino) {
OPN = PHIToResolve[phino];
PHINode *PN = cast<PHINode>(VMap[OPN]);
for (unsigned pred = 0, e = NumPreds; pred != e; ++pred) {
Value *V = VMap[PN->getIncomingBlock(pred)];
if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(V)) {
Value *InVal = MapValue(PN->getIncomingValue(pred),
VMap,
ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
assert(InVal && "Unknown input value?");
PN->setIncomingValue(pred, InVal);
PN->setIncomingBlock(pred, MappedBlock);
} else {
PN->removeIncomingValue(pred, false);
--pred, --e; // Revisit the next entry.
}
}
}
// The loop above has removed PHI entries for those blocks that are dead
// and has updated others. However, if a block is live (i.e. copied over)
// but its terminator has been changed to not go to this block, then our
// phi nodes will have invalid entries. Update the PHI nodes in this
// case.
PHINode *PN = cast<PHINode>(NewBB->begin());
NumPreds = std::distance(pred_begin(NewBB), pred_end(NewBB));
if (NumPreds != PN->getNumIncomingValues()) {
assert(NumPreds < PN->getNumIncomingValues());
// Count how many times each predecessor comes to this block.
std::map<BasicBlock*, unsigned> PredCount;
for (pred_iterator PI = pred_begin(NewBB), E = pred_end(NewBB);
PI != E; ++PI)
--PredCount[*PI];
// Figure out how many entries to remove from each PHI.
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
++PredCount[PN->getIncomingBlock(i)];
// At this point, the excess predecessor entries are positive in the
// map. Loop over all of the PHIs and remove excess predecessor
// entries.
BasicBlock::iterator I = NewBB->begin();
for (; (PN = dyn_cast<PHINode>(I)); ++I) {
for (std::map<BasicBlock*, unsigned>::iterator PCI =PredCount.begin(),
E = PredCount.end(); PCI != E; ++PCI) {
BasicBlock *Pred = PCI->first;
for (unsigned NumToRemove = PCI->second; NumToRemove; --NumToRemove)
PN->removeIncomingValue(Pred, false);
}
}
}
// If the loops above have made these phi nodes have 0 or 1 operand,
// replace them with undef or the input value. We must do this for
// correctness, because 0-operand phis are not valid.
PN = cast<PHINode>(NewBB->begin());
if (PN->getNumIncomingValues() == 0) {
BasicBlock::iterator I = NewBB->begin();
BasicBlock::const_iterator OldI = OldBB->begin();
while ((PN = dyn_cast<PHINode>(I++))) {
Value *NV = UndefValue::get(PN->getType());
PN->replaceAllUsesWith(NV);
assert(VMap[&*OldI] == PN && "VMap mismatch");
VMap[&*OldI] = NV;
PN->eraseFromParent();
++OldI;
}
}
}
// Make a second pass over the PHINodes now that all of them have been
// remapped into the new function, simplifying the PHINode and performing any
// recursive simplifications exposed. This will transparently update the
// WeakVH in the VMap. Notably, we rely on that so that if we coalesce
// two PHINodes, the iteration over the old PHIs remains valid, and the
// mapping will just map us to the new node (which may not even be a PHI
// node).
for (unsigned Idx = 0, Size = PHIToResolve.size(); Idx != Size; ++Idx)
if (PHINode *PN = dyn_cast<PHINode>(VMap[PHIToResolve[Idx]]))
recursivelySimplifyInstruction(PN);
// Now that the inlined function body has been fully constructed, go through
// and zap unconditional fall-through branches. This happens all the time when
// specializing code: code specialization turns conditional branches into
// uncond branches, and this code folds them.
Function::iterator Begin = cast<BasicBlock>(VMap[StartingBB])->getIterator();
Function::iterator I = Begin;
while (I != NewFunc->end()) {
// Check if this block has become dead during inlining or other
// simplifications. Note that the first block will appear dead, as it has
// not yet been wired up properly.
if (I != Begin && (pred_begin(&*I) == pred_end(&*I) ||
I->getSinglePredecessor() == &*I)) {
BasicBlock *DeadBB = &*I++;
DeleteDeadBlock(DeadBB);
continue;
}
// We need to simplify conditional branches and switches with a constant
// operand. We try to prune these out when cloning, but if the
// simplification required looking through PHI nodes, those are only
// available after forming the full basic block. That may leave some here,
// and we still want to prune the dead code as early as possible.
ConstantFoldTerminator(&*I);
BranchInst *BI = dyn_cast<BranchInst>(I->getTerminator());
if (!BI || BI->isConditional()) { ++I; continue; }
BasicBlock *Dest = BI->getSuccessor(0);
if (!Dest->getSinglePredecessor()) {
++I; continue;
}
// We shouldn't be able to get single-entry PHI nodes here, as instsimplify
// above should have zapped all of them..
assert(!isa<PHINode>(Dest->begin()));
// We know all single-entry PHI nodes in the inlined function have been
// removed, so we just need to splice the blocks.
BI->eraseFromParent();
// Make all PHI nodes that referred to Dest now refer to I as their source.
Dest->replaceAllUsesWith(&*I);
// Move all the instructions in the succ to the pred.
I->getInstList().splice(I->end(), Dest->getInstList());
// Remove the dest block.
Dest->eraseFromParent();
// Do not increment I, iteratively merge all things this block branches to.
}
// Make a final pass over the basic blocks from the old function to gather
// any return instructions which survived folding. We have to do this here
// because we can iteratively remove and merge returns above.
for (Function::iterator I = cast<BasicBlock>(VMap[StartingBB])->getIterator(),
E = NewFunc->end();
I != E; ++I)
if (ReturnInst *RI = dyn_cast<ReturnInst>(I->getTerminator()))
Returns.push_back(RI);
}
/// Insert code in the prolog code when unrolling a loop with a
/// run-time trip-count.
///
/// This method assumes that the loop unroll factor is total number
/// of loop bodes in the loop after unrolling. (Some folks refer
/// to the unroll factor as the number of *extra* copies added).
/// We assume also that the loop unroll factor is a power-of-two. So, after
/// unrolling the loop, the number of loop bodies executed is 2,
/// 4, 8, etc. Note - LLVM converts the if-then-sequence to a switch
/// instruction in SimplifyCFG.cpp. Then, the backend decides how code for
/// the switch instruction is generated.
///
/// extraiters = tripcount % loopfactor
/// if (extraiters == 0) jump Loop:
/// else jump Prol
/// Prol: LoopBody;
/// extraiters -= 1 // Omitted if unroll factor is 2.
/// if (extraiters != 0) jump Prol: // Omitted if unroll factor is 2.
/// if (tripcount < loopfactor) jump End
/// Loop:
/// ...
/// End:
///
bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count,
bool AllowExpensiveTripCount, LoopInfo *LI,
LPPassManager *LPM) {
// for now, only unroll loops that contain a single exit
if (!L->getExitingBlock())
return false;
// Make sure the loop is in canonical form, and there is a single
// exit block only.
if (!L->isLoopSimplifyForm() || !L->getUniqueExitBlock())
return false;
// Use Scalar Evolution to compute the trip count. This allows more
// loops to be unrolled than relying on induction var simplification
if (!LPM)
return false;
ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>();
if (!SE)
return false;
// Only unroll loops with a computable trip count and the trip count needs
// to be an int value (allowing a pointer type is a TODO item)
const SCEV *BECountSC = SE->getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BECountSC) ||
!BECountSC->getType()->isIntegerTy())
return false;
unsigned BEWidth = cast<IntegerType>(BECountSC->getType())->getBitWidth();
// Add 1 since the backedge count doesn't include the first loop iteration
const SCEV *TripCountSC =
SE->getAddExpr(BECountSC, SE->getConstant(BECountSC->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC))
return false;
BasicBlock *Header = L->getHeader();
const DataLayout &DL = Header->getModule()->getDataLayout();
SCEVExpander Expander(*SE, DL, "loop-unroll");
if (!AllowExpensiveTripCount && Expander.isHighCostExpansion(TripCountSC, L))
return false;
// We only handle cases when the unroll factor is a power of 2.
// Count is the loop unroll factor, the number of extra copies added + 1.
if (!isPowerOf2_32(Count))
return false;
// This constraint lets us deal with an overflowing trip count easily; see the
// comment on ModVal below.
if (Log2_32(Count) > BEWidth)
return false;
// If this loop is nested, then the loop unroller changes the code in
// parent loop, so the Scalar Evolution pass needs to be run again
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
// Grab analyses that we preserve.
auto *DTWP = LPM->getAnalysisIfAvailable<DominatorTreeWrapperPass>();
auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;
BasicBlock *PH = L->getLoopPreheader();
BasicBlock *Latch = L->getLoopLatch();
// It helps to splits the original preheader twice, one for the end of the
// prolog code and one for a new loop preheader
BasicBlock *PEnd = SplitEdge(PH, Header, DT, LI);
BasicBlock *NewPH = SplitBlock(PEnd, PEnd->getTerminator(), DT, LI);
BranchInst *PreHeaderBR = cast<BranchInst>(PH->getTerminator());
// Compute the number of extra iterations required, which is:
// extra iterations = run-time trip count % (loop unroll factor + 1)
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Value *BECount = Expander.expandCodeFor(BECountSC, BECountSC->getType(),
PreHeaderBR);
IRBuilder<> B(PreHeaderBR);
Value *ModVal = B.CreateAnd(TripCount, Count - 1, "xtraiter");
// If ModVal is zero, we know that either
// 1. there are no iteration to be run in the prologue loop
// OR
// 2. the addition computing TripCount overflowed
//
// If (2) is true, we know that TripCount really is (1 << BEWidth) and so the
// number of iterations that remain to be run in the original loop is a
// multiple Count == (1 << Log2(Count)) because Log2(Count) <= BEWidth (we
// explicitly check this above).
Value *BranchVal = B.CreateIsNotNull(ModVal, "lcmp.mod");
// Branch to either the extra iterations or the cloned/unrolled loop
// We will fix up the true branch label when adding loop body copies
B.CreateCondBr(BranchVal, PEnd, PEnd);
assert(PreHeaderBR->isUnconditional() &&
PreHeaderBR->getSuccessor(0) == PEnd &&
"CFG edges in Preheader are not correct");
PreHeaderBR->eraseFromParent();
Function *F = Header->getParent();
// Get an ordered list of blocks in the loop to help with the ordering of the
// cloned blocks in the prolog code
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
//
// For each extra loop iteration, create a copy of the loop's basic blocks
// and generate a condition that branches to the copy depending on the
// number of 'left over' iterations.
//
std::vector<BasicBlock *> NewBlocks;
ValueToValueMapTy VMap;
bool UnrollPrologue = Count == 2;
// Clone all the basic blocks in the loop. If Count is 2, we don't clone
// the loop, otherwise we create a cloned loop to execute the extra
// iterations. This function adds the appropriate CFG connections.
CloneLoopBlocks(L, ModVal, UnrollPrologue, PH, PEnd, NewBlocks, LoopBlocks,
VMap, LI);
// Insert the cloned blocks into function just before the original loop
F->getBasicBlockList().splice(PEnd, F->getBasicBlockList(), NewBlocks[0],
F->end());
// Rewrite the cloned instruction operands to use the values
// created when the clone is created.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i) {
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end();
I != E; ++I) {
RemapInstruction(I, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingEntries);
}
}
// Connect the prolog code to the original loop and update the
// PHI functions.
BasicBlock *LastLoopBB = cast<BasicBlock>(VMap[Latch]);
ConnectProlog(L, BECount, Count, LastLoopBB, PEnd, PH, NewPH, VMap,
/*AliasAnalysis*/ nullptr, DT, LI, LPM->getAsPass());
NumRuntimeUnrolled++;
return true;
}
/*
This method performs Unroll and Jam. For a simple loop like:
for (i = ..)
Fore(i)
for (j = ..)
SubLoop(i, j)
Aft(i)
Instead of doing normal inner or outer unrolling, we do:
for (i = .., i+=2)
Fore(i)
Fore(i+1)
for (j = ..)
SubLoop(i, j)
SubLoop(i+1, j)
Aft(i)
Aft(i+1)
So the outer loop is essetially unrolled and then the inner loops are fused
("jammed") together into a single loop. This can increase speed when there
are loads in SubLoop that are invariant to i, as they become shared between
the now jammed inner loops.
We do this by spliting the blocks in the loop into Fore, Subloop and Aft.
Fore blocks are those before the inner loop, Aft are those after. Normal
Unroll code is used to copy each of these sets of blocks and the results are
combined together into the final form above.
isSafeToUnrollAndJam should be used prior to calling this to make sure the
unrolling will be valid. Checking profitablility is also advisable.
*/
LoopUnrollResult
llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
AssumptionCache *AC, OptimizationRemarkEmitter *ORE) {
// When we enter here we should have already checked that it is safe
BasicBlock *Header = L->getHeader();
assert(L->getSubLoops().size() == 1);
Loop *SubLoop = *L->begin();
// Don't enter the unroll code if there is nothing to do.
if (TripCount == 0 && Count < 2) {
LLVM_DEBUG(dbgs() << "Won't unroll; almost nothing to do\n");
return LoopUnrollResult::Unmodified;
}
assert(Count > 0);
assert(TripMultiple > 0);
assert(TripCount == 0 || TripCount % TripMultiple == 0);
// Are we eliminating the loop control altogether?
bool CompletelyUnroll = (Count == TripCount);
// We use the runtime remainder in cases where we don't know trip multiple
if (TripMultiple == 1 || TripMultiple % Count != 0) {
if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
/*UseEpilogRemainder*/ true,
UnrollRemainder, LI, SE, DT, AC, true)) {
LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
"generated when assuming runtime trip count\n");
return LoopUnrollResult::Unmodified;
}
}
// Notify ScalarEvolution that the loop will be substantially changed,
// if not outright eliminated.
if (SE) {
SE->forgetLoop(L);
SE->forgetLoop(SubLoop);
}
using namespace ore;
// Report the unrolling decision.
if (CompletelyUnroll) {
LLVM_DEBUG(dbgs() << "COMPLETELY UNROLL AND JAMMING loop %"
<< Header->getName() << " with trip count " << TripCount
<< "!\n");
ORE->emit(OptimizationRemark(DEBUG_TYPE, "FullyUnrolled", L->getStartLoc(),
L->getHeader())
<< "completely unroll and jammed loop with "
<< NV("UnrollCount", TripCount) << " iterations");
} else {
auto DiagBuilder = [&]() {
OptimizationRemark Diag(DEBUG_TYPE, "PartialUnrolled", L->getStartLoc(),
L->getHeader());
return Diag << "unroll and jammed loop by a factor of "
<< NV("UnrollCount", Count);
};
LLVM_DEBUG(dbgs() << "UNROLL AND JAMMING loop %" << Header->getName()
<< " by " << Count);
if (TripMultiple != 1) {
LLVM_DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
ORE->emit([&]() {
return DiagBuilder() << " with " << NV("TripMultiple", TripMultiple)
<< " trips per branch";
});
} else {
LLVM_DEBUG(dbgs() << " with run-time trip count");
ORE->emit([&]() { return DiagBuilder() << " with run-time trip count"; });
}
LLVM_DEBUG(dbgs() << "!\n");
}
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *LatchBlock = L->getLoopLatch();
BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
assert(Preheader && LatchBlock && Header);
assert(BI && !BI->isUnconditional());
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
bool SubLoopContinueOnTrue = SubLoop->contains(
SubLoop->getLoopLatch()->getTerminator()->getSuccessor(0));
// Partition blocks in an outer/inner loop pair into blocks before and after
// the loop
BasicBlockSet SubLoopBlocks;
BasicBlockSet ForeBlocks;
BasicBlockSet AftBlocks;
partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks, AftBlocks,
DT);
// We keep track of the entering/first and exiting/last block of each of
// Fore/SubLoop/Aft in each iteration. This helps make the stapling up of
// blocks easier.
std::vector<BasicBlock *> ForeBlocksFirst;
std::vector<BasicBlock *> ForeBlocksLast;
std::vector<BasicBlock *> SubLoopBlocksFirst;
std::vector<BasicBlock *> SubLoopBlocksLast;
std::vector<BasicBlock *> AftBlocksFirst;
std::vector<BasicBlock *> AftBlocksLast;
ForeBlocksFirst.push_back(Header);
ForeBlocksLast.push_back(SubLoop->getLoopPreheader());
SubLoopBlocksFirst.push_back(SubLoop->getHeader());
SubLoopBlocksLast.push_back(SubLoop->getExitingBlock());
AftBlocksFirst.push_back(SubLoop->getExitBlock());
AftBlocksLast.push_back(L->getExitingBlock());
// Maps Blocks[0] -> Blocks[It]
ValueToValueMapTy LastValueMap;
// Move any instructions from fore phi operands from AftBlocks into Fore.
moveHeaderPhiOperandsToForeBlocks(
Header, LatchBlock, SubLoop->getLoopPreheader()->getTerminator(),
AftBlocks);
// The current on-the-fly SSA update requires blocks to be processed in
// reverse postorder so that LastValueMap contains the correct value at each
// exit.
LoopBlocksDFS DFS(L);
DFS.perform(LI);
// Stash the DFS iterators before adding blocks to the loop.
LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
if (Header->getParent()->isDebugInfoForProfiling())
for (BasicBlock *BB : L->getBlocks())
for (Instruction &I : *BB)
if (!isa<DbgInfoIntrinsic>(&I))
if (const DILocation *DIL = I.getDebugLoc())
I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
// Copy all blocks
for (unsigned It = 1; It != Count; ++It) {
std::vector<BasicBlock *> NewBlocks;
// Maps Blocks[It] -> Blocks[It-1]
DenseMap<Value *, Value *> PrevItValueMap;
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
ValueToValueMapTy VMap;
BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
Header->getParent()->getBasicBlockList().push_back(New);
if (ForeBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == ForeBlocksFirst[0])
ForeBlocksFirst.push_back(New);
if (*BB == ForeBlocksLast[0])
ForeBlocksLast.push_back(New);
} else if (SubLoopBlocks.count(*BB)) {
SubLoop->addBasicBlockToLoop(New, *LI);
if (*BB == SubLoopBlocksFirst[0])
SubLoopBlocksFirst.push_back(New);
if (*BB == SubLoopBlocksLast[0])
SubLoopBlocksLast.push_back(New);
} else if (AftBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == AftBlocksFirst[0])
AftBlocksFirst.push_back(New);
if (*BB == AftBlocksLast[0])
AftBlocksLast.push_back(New);
} else {
llvm_unreachable("BB being cloned should be in Fore/Sub/Aft");
}
// Update our running maps of newest clones
PrevItValueMap[New] = (It == 1 ? *BB : LastValueMap[*BB]);
LastValueMap[*BB] = New;
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI) {
PrevItValueMap[VI->second] =
const_cast<Value *>(It == 1 ? VI->first : LastValueMap[VI->first]);
LastValueMap[VI->first] = VI->second;
}
NewBlocks.push_back(New);
// Update DomTree:
if (*BB == ForeBlocksFirst[0])
DT->addNewBlock(New, ForeBlocksLast[It - 1]);
else if (*BB == SubLoopBlocksFirst[0])
DT->addNewBlock(New, SubLoopBlocksLast[It - 1]);
else if (*BB == AftBlocksFirst[0])
DT->addNewBlock(New, AftBlocksLast[It - 1]);
else {
// Each set of blocks (Fore/Sub/Aft) will have the same internal domtree
// structure.
auto BBDomNode = DT->getNode(*BB);
auto BBIDom = BBDomNode->getIDom();
BasicBlock *OriginalBBIDom = BBIDom->getBlock();
assert(OriginalBBIDom);
assert(LastValueMap[cast<Value>(OriginalBBIDom)]);
DT->addNewBlock(
New, cast<BasicBlock>(LastValueMap[cast<Value>(OriginalBBIDom)]));
}
}
// Remap all instructions in the most recent iteration
for (BasicBlock *NewBlock : NewBlocks) {
for (Instruction &I : *NewBlock) {
::remapInstruction(&I, LastValueMap);
if (auto *II = dyn_cast<IntrinsicInst>(&I))
if (II->getIntrinsicID() == Intrinsic::assume)
AC->registerAssumption(II);
}
}
// Alter the ForeBlocks phi's, pointing them at the latest version of the
// value from the previous iteration's phis
for (PHINode &Phi : ForeBlocksFirst[It]->phis()) {
Value *OldValue = Phi.getIncomingValueForBlock(AftBlocksLast[It]);
assert(OldValue && "should have incoming edge from Aft[It]");
Value *NewValue = OldValue;
if (Value *PrevValue = PrevItValueMap[OldValue])
NewValue = PrevValue;
assert(Phi.getNumOperands() == 2);
Phi.setIncomingBlock(0, ForeBlocksLast[It - 1]);
Phi.setIncomingValue(0, NewValue);
Phi.removeIncomingValue(1);
}
}
// Now that all the basic blocks for the unrolled iterations are in place,
// finish up connecting the blocks and phi nodes. At this point LastValueMap
// is the last unrolled iterations values.
// Update Phis in BB from OldBB to point to NewBB
auto updatePHIBlocks = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB) {
for (PHINode &Phi : BB->phis()) {
int I = Phi.getBasicBlockIndex(OldBB);
Phi.setIncomingBlock(I, NewBB);
}
};
// Update Phis in BB from OldBB to point to NewBB and use the latest value
// from LastValueMap
auto updatePHIBlocksAndValues = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB,
ValueToValueMapTy &LastValueMap) {
for (PHINode &Phi : BB->phis()) {
for (unsigned b = 0; b < Phi.getNumIncomingValues(); ++b) {
if (Phi.getIncomingBlock(b) == OldBB) {
Value *OldValue = Phi.getIncomingValue(b);
if (Value *LastValue = LastValueMap[OldValue])
Phi.setIncomingValue(b, LastValue);
Phi.setIncomingBlock(b, NewBB);
break;
}
}
}
};
// Move all the phis from Src into Dest
auto movePHIs = [](BasicBlock *Src, BasicBlock *Dest) {
Instruction *insertPoint = Dest->getFirstNonPHI();
while (PHINode *Phi = dyn_cast<PHINode>(Src->begin()))
Phi->moveBefore(insertPoint);
};
// Update the PHI values outside the loop to point to the last block
updatePHIBlocksAndValues(LoopExit, AftBlocksLast[0], AftBlocksLast.back(),
LastValueMap);
// Update ForeBlocks successors and phi nodes
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast.back()->getTerminator());
BasicBlock *Dest = SubLoopBlocksFirst[0];
ForeTerm->setSuccessor(0, Dest);
if (CompletelyUnroll) {
while (PHINode *Phi = dyn_cast<PHINode>(ForeBlocksFirst[0]->begin())) {
Phi->replaceAllUsesWith(Phi->getIncomingValueForBlock(Preheader));
Phi->getParent()->getInstList().erase(Phi);
}
} else {
// Update the PHI values to point to the last aft block
updatePHIBlocksAndValues(ForeBlocksFirst[0], AftBlocksLast[0],
AftBlocksLast.back(), LastValueMap);
}
for (unsigned It = 1; It != Count; It++) {
// Remap ForeBlock successors from previous iteration to this
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast[It - 1]->getTerminator());
BasicBlock *Dest = ForeBlocksFirst[It];
ForeTerm->setSuccessor(0, Dest);
}
// Subloop successors and phis
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast.back()->getTerminator());
SubTerm->setSuccessor(!SubLoopContinueOnTrue, SubLoopBlocksFirst[0]);
SubTerm->setSuccessor(SubLoopContinueOnTrue, AftBlocksFirst[0]);
updatePHIBlocks(SubLoopBlocksFirst[0], ForeBlocksLast[0],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast[It - 1]->getTerminator());
BranchInst::Create(SubLoopBlocksFirst[It], SubTerm);
SubTerm->eraseFromParent();
updatePHIBlocks(SubLoopBlocksFirst[It], ForeBlocksLast[It],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(SubLoopBlocksFirst[It], SubLoopBlocksFirst[0]);
}
// Aft blocks successors and phis
BranchInst *Term = cast<BranchInst>(AftBlocksLast.back()->getTerminator());
if (CompletelyUnroll) {
BranchInst::Create(LoopExit, Term);
Term->eraseFromParent();
} else {
Term->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]);
}
updatePHIBlocks(AftBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *AftTerm =
cast<BranchInst>(AftBlocksLast[It - 1]->getTerminator());
BranchInst::Create(AftBlocksFirst[It], AftTerm);
AftTerm->eraseFromParent();
updatePHIBlocks(AftBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(AftBlocksFirst[It], AftBlocksFirst[0]);
}
// Dominator Tree. Remove the old links between Fore, Sub and Aft, adding the
// new ones required.
if (Count != 1) {
SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, ForeBlocksLast[0],
SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete,
SubLoopBlocksLast[0], AftBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
ForeBlocksLast.back(), SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
SubLoopBlocksLast.back(), AftBlocksFirst[0]);
DT->applyUpdates(DTUpdates);
}
// Merge adjacent basic blocks, if possible.
SmallPtrSet<BasicBlock *, 16> MergeBlocks;
MergeBlocks.insert(ForeBlocksLast.begin(), ForeBlocksLast.end());
MergeBlocks.insert(SubLoopBlocksLast.begin(), SubLoopBlocksLast.end());
MergeBlocks.insert(AftBlocksLast.begin(), AftBlocksLast.end());
while (!MergeBlocks.empty()) {
BasicBlock *BB = *MergeBlocks.begin();
BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
if (Term && Term->isUnconditional() && L->contains(Term->getSuccessor(0))) {
BasicBlock *Dest = Term->getSuccessor(0);
if (BasicBlock *Fold = foldBlockIntoPredecessor(Dest, LI, SE, DT)) {
// Don't remove BB and add Fold as they are the same BB
assert(Fold == BB);
(void)Fold;
MergeBlocks.erase(Dest);
} else
MergeBlocks.erase(BB);
} else
MergeBlocks.erase(BB);
}
// At this point, the code is well formed. We now do a quick sweep over the
// inserted code, doing constant propagation and dead code elimination as we
// go.
simplifyLoopAfterUnroll(SubLoop, true, LI, SE, DT, AC);
simplifyLoopAfterUnroll(L, !CompletelyUnroll && Count > 1, LI, SE, DT, AC);
NumCompletelyUnrolledAndJammed += CompletelyUnroll;
++NumUnrolledAndJammed;
#ifndef NDEBUG
// We shouldn't have done anything to break loop simplify form or LCSSA.
Loop *OuterL = L->getParentLoop();
Loop *OutestLoop = OuterL ? OuterL : (!CompletelyUnroll ? L : SubLoop);
assert(OutestLoop->isRecursivelyLCSSAForm(*DT, *LI));
if (!CompletelyUnroll)
assert(L->isLoopSimplifyForm());
assert(SubLoop->isLoopSimplifyForm());
assert(DT->verify());
#endif
// Update LoopInfo if the loop is completely removed.
if (CompletelyUnroll)
LI->erase(L);
return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled
: LoopUnrollResult::PartiallyUnrolled;
}
bool LoopInterchangeTransform::adjustLoopBranches() {
DEBUG(dbgs() << "adjustLoopBranches called\n");
// Adjust the loop preheader
BasicBlock *InnerLoopHeader = InnerLoop->getHeader();
BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
BasicBlock *InnerLoopLatch = InnerLoop->getLoopLatch();
BasicBlock *OuterLoopLatch = OuterLoop->getLoopLatch();
BasicBlock *OuterLoopPreHeader = OuterLoop->getLoopPreheader();
BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
BasicBlock *OuterLoopPredecessor = OuterLoopPreHeader->getUniquePredecessor();
BasicBlock *InnerLoopLatchPredecessor =
InnerLoopLatch->getUniquePredecessor();
BasicBlock *InnerLoopLatchSuccessor;
BasicBlock *OuterLoopLatchSuccessor;
BranchInst *OuterLoopLatchBI =
dyn_cast<BranchInst>(OuterLoopLatch->getTerminator());
BranchInst *InnerLoopLatchBI =
dyn_cast<BranchInst>(InnerLoopLatch->getTerminator());
BranchInst *OuterLoopHeaderBI =
dyn_cast<BranchInst>(OuterLoopHeader->getTerminator());
BranchInst *InnerLoopHeaderBI =
dyn_cast<BranchInst>(InnerLoopHeader->getTerminator());
if (!OuterLoopPredecessor || !InnerLoopLatchPredecessor ||
!OuterLoopLatchBI || !InnerLoopLatchBI || !OuterLoopHeaderBI ||
!InnerLoopHeaderBI)
return false;
BranchInst *InnerLoopLatchPredecessorBI =
dyn_cast<BranchInst>(InnerLoopLatchPredecessor->getTerminator());
BranchInst *OuterLoopPredecessorBI =
dyn_cast<BranchInst>(OuterLoopPredecessor->getTerminator());
if (!OuterLoopPredecessorBI || !InnerLoopLatchPredecessorBI)
return false;
BasicBlock *InnerLoopHeaderSucessor = InnerLoopHeader->getUniqueSuccessor();
if (!InnerLoopHeaderSucessor)
return false;
// Adjust Loop Preheader and headers
unsigned NumSucc = OuterLoopPredecessorBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (OuterLoopPredecessorBI->getSuccessor(i) == OuterLoopPreHeader)
OuterLoopPredecessorBI->setSuccessor(i, InnerLoopPreHeader);
}
NumSucc = OuterLoopHeaderBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (OuterLoopHeaderBI->getSuccessor(i) == OuterLoopLatch)
OuterLoopHeaderBI->setSuccessor(i, LoopExit);
else if (OuterLoopHeaderBI->getSuccessor(i) == InnerLoopPreHeader)
OuterLoopHeaderBI->setSuccessor(i, InnerLoopHeaderSucessor);
}
BranchInst::Create(OuterLoopPreHeader, InnerLoopHeaderBI);
InnerLoopHeaderBI->eraseFromParent();
// -------------Adjust loop latches-----------
if (InnerLoopLatchBI->getSuccessor(0) == InnerLoopHeader)
InnerLoopLatchSuccessor = InnerLoopLatchBI->getSuccessor(1);
else
InnerLoopLatchSuccessor = InnerLoopLatchBI->getSuccessor(0);
NumSucc = InnerLoopLatchPredecessorBI->getNumSuccessors();
for (unsigned i = 0; i < NumSucc; ++i) {
if (InnerLoopLatchPredecessorBI->getSuccessor(i) == InnerLoopLatch)
InnerLoopLatchPredecessorBI->setSuccessor(i, InnerLoopLatchSuccessor);
}
if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopHeader)
OuterLoopLatchSuccessor = OuterLoopLatchBI->getSuccessor(1);
else
OuterLoopLatchSuccessor = OuterLoopLatchBI->getSuccessor(0);
if (InnerLoopLatchBI->getSuccessor(1) == InnerLoopLatchSuccessor)
InnerLoopLatchBI->setSuccessor(1, OuterLoopLatchSuccessor);
else
InnerLoopLatchBI->setSuccessor(0, OuterLoopLatchSuccessor);
if (OuterLoopLatchBI->getSuccessor(0) == OuterLoopLatchSuccessor) {
OuterLoopLatchBI->setSuccessor(0, InnerLoopLatch);
} else {
OuterLoopLatchBI->setSuccessor(1, InnerLoopLatch);
}
return true;
}
/// Create a clone of the blocks in a loop and connect them together.
/// If UnrollProlog is true, loop structure will not be cloned, otherwise a new
/// loop will be created including all cloned blocks, and the iterator of it
/// switches to count NewIter down to 0.
///
static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool UnrollProlog,
BasicBlock *InsertTop, BasicBlock *InsertBot,
std::vector<BasicBlock *> &NewBlocks,
LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
LoopInfo *LI) {
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *NewLoop = 0;
Loop *ParentLoop = L->getParentLoop();
if (!UnrollProlog) {
NewLoop = new Loop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
LI->addTopLevelLoop(NewLoop);
}
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, ".prol", F);
NewBlocks.push_back(NewBB);
if (NewLoop)
NewLoop->addBasicBlockToLoop(NewBB, *LI);
else if (ParentLoop)
ParentLoop->addBasicBlockToLoop(NewBB, *LI);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (Latch == *BB) {
// For the last block, if UnrollProlog is true, create a direct jump to
// InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (UnrollProlog) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2, "prol.iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[I]);
if (UnrollProlog) {
VMap[I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (VMap[InVal])
NewPHI->setIncomingValue(idx, VMap[InVal]);
}
}
if (NewLoop) {
// Add unroll disable metadata to disable future unrolling for this loop.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
MDNode *LoopID = NewLoop->getLoopID();
if (LoopID) {
// First remove any existing loop unrolling metadata.
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
}
LLVMContext &Context = NewLoop->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
NewLoop->setLoopID(NewLoopID);
}
}
bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
bool AllowExpensiveTripCount,
bool UseEpilogRemainder,
bool UnrollRemainder,
LoopInfo *LI, ScalarEvolution *SE,
DominatorTree *DT, AssumptionCache *AC,
OptimizationRemarkEmitter *ORE,
bool PreserveLCSSA) {
DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
DEBUG(L->dump());
DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n" :
dbgs() << "Using prolog remainder.\n");
// Make sure the loop is in canonical form.
if (!L->isLoopSimplifyForm()) {
DEBUG(dbgs() << "Not in simplify form!\n");
return false;
}
// Guaranteed by LoopSimplifyForm.
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Header = L->getHeader();
BranchInst *LatchBR = cast<BranchInst>(Latch->getTerminator());
unsigned ExitIndex = LatchBR->getSuccessor(0) == Header ? 1 : 0;
BasicBlock *LatchExit = LatchBR->getSuccessor(ExitIndex);
// Cloning the loop basic blocks (`CloneLoopBlocks`) requires that one of the
// targets of the Latch be an exit block out of the loop. This needs
// to be guaranteed by the callers of UnrollRuntimeLoopRemainder.
assert(!L->contains(LatchExit) &&
"one of the loop latch successors should be the exit block!");
// These are exit blocks other than the target of the latch exiting block.
SmallVector<BasicBlock *, 4> OtherExits;
bool isMultiExitUnrollingEnabled =
canSafelyUnrollMultiExitLoop(L, OtherExits, LatchExit, PreserveLCSSA,
UseEpilogRemainder) &&
canProfitablyUnrollMultiExitLoop(L, OtherExits, LatchExit, PreserveLCSSA,
UseEpilogRemainder);
// Support only single exit and exiting block unless multi-exit loop unrolling is enabled.
if (!isMultiExitUnrollingEnabled &&
(!L->getExitingBlock() || OtherExits.size())) {
DEBUG(
dbgs()
<< "Multiple exit/exiting blocks in loop and multi-exit unrolling not "
"enabled!\n");
return false;
}
// Use Scalar Evolution to compute the trip count. This allows more loops to
// be unrolled than relying on induction var simplification.
if (!SE)
return false;
// Only unroll loops with a computable trip count, and the trip count needs
// to be an int value (allowing a pointer type is a TODO item).
// We calculate the backedge count by using getExitCount on the Latch block,
// which is proven to be the only exiting block in this loop. This is same as
// calculating getBackedgeTakenCount on the loop (which computes SCEV for all
// exiting blocks).
const SCEV *BECountSC = SE->getExitCount(L, Latch);
if (isa<SCEVCouldNotCompute>(BECountSC) ||
!BECountSC->getType()->isIntegerTy()) {
DEBUG(dbgs() << "Could not compute exit block SCEV\n");
return false;
}
unsigned BEWidth = cast<IntegerType>(BECountSC->getType())->getBitWidth();
// Add 1 since the backedge count doesn't include the first loop iteration.
const SCEV *TripCountSC =
SE->getAddExpr(BECountSC, SE->getConstant(BECountSC->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC)) {
DEBUG(dbgs() << "Could not compute trip count SCEV.\n");
return false;
}
BasicBlock *PreHeader = L->getLoopPreheader();
BranchInst *PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
const DataLayout &DL = Header->getModule()->getDataLayout();
SCEVExpander Expander(*SE, DL, "loop-unroll");
if (!AllowExpensiveTripCount &&
Expander.isHighCostExpansion(TripCountSC, L, PreHeaderBR)) {
DEBUG(dbgs() << "High cost for expanding trip count scev!\n");
return false;
}
// This constraint lets us deal with an overflowing trip count easily; see the
// comment on ModVal below.
if (Log2_32(Count) > BEWidth) {
DEBUG(dbgs()
<< "Count failed constraint on overflow trip count calculation.\n");
return false;
}
// Loop structure is the following:
//
// PreHeader
// Header
// ...
// Latch
// LatchExit
BasicBlock *NewPreHeader;
BasicBlock *NewExit = nullptr;
BasicBlock *PrologExit = nullptr;
BasicBlock *EpilogPreHeader = nullptr;
BasicBlock *PrologPreHeader = nullptr;
if (UseEpilogRemainder) {
// If epilog remainder
// Split PreHeader to insert a branch around loop for unrolling.
NewPreHeader = SplitBlock(PreHeader, PreHeader->getTerminator(), DT, LI);
NewPreHeader->setName(PreHeader->getName() + ".new");
// Split LatchExit to create phi nodes from branch above.
SmallVector<BasicBlock*, 4> Preds(predecessors(LatchExit));
NewExit = SplitBlockPredecessors(LatchExit, Preds, ".unr-lcssa",
DT, LI, PreserveLCSSA);
// Split NewExit to insert epilog remainder loop.
EpilogPreHeader = SplitBlock(NewExit, NewExit->getTerminator(), DT, LI);
EpilogPreHeader->setName(Header->getName() + ".epil.preheader");
} else {
// If prolog remainder
// Split the original preheader twice to insert prolog remainder loop
PrologPreHeader = SplitEdge(PreHeader, Header, DT, LI);
PrologPreHeader->setName(Header->getName() + ".prol.preheader");
PrologExit = SplitBlock(PrologPreHeader, PrologPreHeader->getTerminator(),
DT, LI);
PrologExit->setName(Header->getName() + ".prol.loopexit");
// Split PrologExit to get NewPreHeader.
NewPreHeader = SplitBlock(PrologExit, PrologExit->getTerminator(), DT, LI);
NewPreHeader->setName(PreHeader->getName() + ".new");
}
// Loop structure should be the following:
// Epilog Prolog
//
// PreHeader PreHeader
// *NewPreHeader *PrologPreHeader
// Header *PrologExit
// ... *NewPreHeader
// Latch Header
// *NewExit ...
// *EpilogPreHeader Latch
// LatchExit LatchExit
// Calculate conditions for branch around loop for unrolling
// in epilog case and around prolog remainder loop in prolog case.
// Compute the number of extra iterations required, which is:
// extra iterations = run-time trip count % loop unroll factor
PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Value *BECount = Expander.expandCodeFor(BECountSC, BECountSC->getType(),
PreHeaderBR);
IRBuilder<> B(PreHeaderBR);
Value *ModVal;
// Calculate ModVal = (BECount + 1) % Count.
// Note that TripCount is BECount + 1.
if (isPowerOf2_32(Count)) {
// When Count is power of 2 we don't BECount for epilog case, however we'll
// need it for a branch around unrolling loop for prolog case.
ModVal = B.CreateAnd(TripCount, Count - 1, "xtraiter");
// 1. There are no iterations to be run in the prolog/epilog loop.
// OR
// 2. The addition computing TripCount overflowed.
//
// If (2) is true, we know that TripCount really is (1 << BEWidth) and so
// the number of iterations that remain to be run in the original loop is a
// multiple Count == (1 << Log2(Count)) because Log2(Count) <= BEWidth (we
// explicitly check this above).
} else {
// As (BECount + 1) can potentially unsigned overflow we count
// (BECount % Count) + 1 which is overflow safe as BECount % Count < Count.
Value *ModValTmp = B.CreateURem(BECount,
ConstantInt::get(BECount->getType(),
Count));
Value *ModValAdd = B.CreateAdd(ModValTmp,
ConstantInt::get(ModValTmp->getType(), 1));
// At that point (BECount % Count) + 1 could be equal to Count.
// To handle this case we need to take mod by Count one more time.
ModVal = B.CreateURem(ModValAdd,
ConstantInt::get(BECount->getType(), Count),
"xtraiter");
}
Value *BranchVal =
UseEpilogRemainder ? B.CreateICmpULT(BECount,
ConstantInt::get(BECount->getType(),
Count - 1)) :
B.CreateIsNotNull(ModVal, "lcmp.mod");
BasicBlock *RemainderLoop = UseEpilogRemainder ? NewExit : PrologPreHeader;
BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit;
// Branch to either remainder (extra iterations) loop or unrolling loop.
B.CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop);
PreHeaderBR->eraseFromParent();
if (DT) {
if (UseEpilogRemainder)
DT->changeImmediateDominator(NewExit, PreHeader);
else
DT->changeImmediateDominator(PrologExit, PreHeader);
}
Function *F = Header->getParent();
// Get an ordered list of blocks in the loop to help with the ordering of the
// cloned blocks in the prolog/epilog code
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
//
// For each extra loop iteration, create a copy of the loop's basic blocks
// and generate a condition that branches to the copy depending on the
// number of 'left over' iterations.
//
std::vector<BasicBlock *> NewBlocks;
ValueToValueMapTy VMap;
// For unroll factor 2 remainder loop will have 1 iterations.
// Do not create 1 iteration loop.
bool CreateRemainderLoop = (Count != 2);
// Clone all the basic blocks in the loop. If Count is 2, we don't clone
// the loop, otherwise we create a cloned loop to execute the extra
// iterations. This function adds the appropriate CFG connections.
BasicBlock *InsertBot = UseEpilogRemainder ? LatchExit : PrologExit;
BasicBlock *InsertTop = UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader;
Loop *remainderLoop = CloneLoopBlocks(
L, ModVal, CreateRemainderLoop, UseEpilogRemainder, UnrollRemainder,
InsertTop, InsertBot,
NewPreHeader, NewBlocks, LoopBlocks, VMap, DT, LI);
// Insert the cloned blocks into the function.
F->getBasicBlockList().splice(InsertBot->getIterator(),
F->getBasicBlockList(),
NewBlocks[0]->getIterator(),
F->end());
// Now the loop blocks are cloned and the other exiting blocks from the
// remainder are connected to the original Loop's exit blocks. The remaining
// work is to update the phi nodes in the original loop, and take in the
// values from the cloned region. Also update the dominator info for
// OtherExits and their immediate successors, since we have new edges into
// OtherExits.
SmallSet<BasicBlock*, 8> ImmediateSuccessorsOfExitBlocks;
for (auto *BB : OtherExits) {
for (auto &II : *BB) {
// Given we preserve LCSSA form, we know that the values used outside the
// loop will be used through these phi nodes at the exit blocks that are
// transformed below.
if (!isa<PHINode>(II))
break;
PHINode *Phi = cast<PHINode>(&II);
unsigned oldNumOperands = Phi->getNumIncomingValues();
// Add the incoming values from the remainder code to the end of the phi
// node.
for (unsigned i =0; i < oldNumOperands; i++){
Value *newVal = VMap[Phi->getIncomingValue(i)];
// newVal can be a constant or derived from values outside the loop, and
// hence need not have a VMap value.
if (!newVal)
newVal = Phi->getIncomingValue(i);
Phi->addIncoming(newVal,
cast<BasicBlock>(VMap[Phi->getIncomingBlock(i)]));
}
}
#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
for (BasicBlock *SuccBB : successors(BB)) {
assert(!(any_of(OtherExits,
[SuccBB](BasicBlock *EB) { return EB == SuccBB; }) ||
SuccBB == LatchExit) &&
"Breaks the definition of dedicated exits!");
}
#endif
// Update the dominator info because the immediate dominator is no longer the
// header of the original Loop. BB has edges both from L and remainder code.
// Since the preheader determines which loop is run (L or directly jump to
// the remainder code), we set the immediate dominator as the preheader.
if (DT) {
DT->changeImmediateDominator(BB, PreHeader);
// Also update the IDom for immediate successors of BB. If the current
// IDom is the header, update the IDom to be the preheader because that is
// the nearest common dominator of all predecessors of SuccBB. We need to
// check for IDom being the header because successors of exit blocks can
// have edges from outside the loop, and we should not incorrectly update
// the IDom in that case.
for (BasicBlock *SuccBB: successors(BB))
if (ImmediateSuccessorsOfExitBlocks.insert(SuccBB).second) {
if (DT->getNode(SuccBB)->getIDom()->getBlock() == Header) {
assert(!SuccBB->getSinglePredecessor() &&
"BB should be the IDom then!");
DT->changeImmediateDominator(SuccBB, PreHeader);
}
}
}
}
// Loop structure should be the following:
// Epilog Prolog
//
// PreHeader PreHeader
// NewPreHeader PrologPreHeader
// Header PrologHeader
// ... ...
// Latch PrologLatch
// NewExit PrologExit
// EpilogPreHeader NewPreHeader
// EpilogHeader Header
// ... ...
// EpilogLatch Latch
// LatchExit LatchExit
// Rewrite the cloned instruction operands to use the values created when the
// clone is created.
for (BasicBlock *BB : NewBlocks) {
for (Instruction &I : *BB) {
RemapInstruction(&I, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
}
}
if (UseEpilogRemainder) {
// Connect the epilog code to the original loop and update the
// PHI functions.
ConnectEpilog(L, ModVal, NewExit, LatchExit, PreHeader,
EpilogPreHeader, NewPreHeader, VMap, DT, LI,
PreserveLCSSA);
// Update counter in loop for unrolling.
// I should be multiply of Count.
IRBuilder<> B2(NewPreHeader->getTerminator());
Value *TestVal = B2.CreateSub(TripCount, ModVal, "unroll_iter");
BranchInst *LatchBR = cast<BranchInst>(Latch->getTerminator());
B2.SetInsertPoint(LatchBR);
PHINode *NewIdx = PHINode::Create(TestVal->getType(), 2, "niter",
Header->getFirstNonPHI());
Value *IdxSub =
B2.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".nsub");
Value *IdxCmp;
if (LatchBR->getSuccessor(0) == Header)
IdxCmp = B2.CreateIsNotNull(IdxSub, NewIdx->getName() + ".ncmp");
else
IdxCmp = B2.CreateIsNull(IdxSub, NewIdx->getName() + ".ncmp");
NewIdx->addIncoming(TestVal, NewPreHeader);
NewIdx->addIncoming(IdxSub, Latch);
LatchBR->setCondition(IdxCmp);
} else {
// Connect the prolog code to the original loop and update the
// PHI functions.
ConnectProlog(L, BECount, Count, PrologExit, LatchExit, PreHeader,
NewPreHeader, VMap, DT, LI, PreserveLCSSA);
}
// If this loop is nested, then the loop unroller changes the code in the
// parent loop, so the Scalar Evolution pass needs to be run again.
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
// Canonicalize to LoopSimplifyForm both original and remainder loops. We
// cannot rely on the LoopUnrollPass to do this because it only does
// canonicalization for parent/subloops and not the sibling loops.
if (OtherExits.size() > 0) {
// Generate dedicated exit blocks for the original loop, to preserve
// LoopSimplifyForm.
formDedicatedExitBlocks(L, DT, LI, PreserveLCSSA);
// Generate dedicated exit blocks for the remainder loop if one exists, to
// preserve LoopSimplifyForm.
if (remainderLoop)
formDedicatedExitBlocks(remainderLoop, DT, LI, PreserveLCSSA);
}
if (remainderLoop && UnrollRemainder) {
DEBUG(dbgs() << "Unrolling remainder loop\n");
UnrollLoop(remainderLoop, /*Count*/Count - 1, /*TripCount*/Count - 1,
/*Force*/false, /*AllowRuntime*/false,
/*AllowExpensiveTripCount*/false, /*PreserveCondBr*/true,
/*PreserveOnlyFirst*/false, /*TripMultiple*/1,
/*PeelCount*/0, /*UnrollRemainder*/false, LI, SE, DT, AC, ORE,
PreserveLCSSA);
}
NumRuntimeUnrolled++;
return true;
}
/// Create a clone of the blocks in a loop and connect them together.
/// If CreateRemainderLoop is false, loop structure will not be cloned,
/// otherwise a new loop will be created including all cloned blocks, and the
/// iterator of it switches to count NewIter down to 0.
/// The cloned blocks should be inserted between InsertTop and InsertBot.
/// If loop structure is cloned InsertTop should be new preheader, InsertBot
/// new loop exit.
/// Return the new cloned loop that is created when CreateRemainderLoop is true.
static Loop *
CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop,
const bool UseEpilogRemainder, const bool UnrollRemainder,
BasicBlock *InsertTop,
BasicBlock *InsertBot, BasicBlock *Preheader,
std::vector<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks,
ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI) {
StringRef suffix = UseEpilogRemainder ? "epil" : "prol";
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *ParentLoop = L->getParentLoop();
NewLoopsMap NewLoops;
NewLoops[ParentLoop] = ParentLoop;
if (!CreateRemainderLoop)
NewLoops[L] = ParentLoop;
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, "." + suffix, F);
NewBlocks.push_back(NewBB);
// If we're unrolling the outermost loop, there's no remainder loop,
// and this block isn't in a nested loop, then the new block is not
// in any loop. Otherwise, add it to loopinfo.
if (CreateRemainderLoop || LI->getLoopFor(*BB) != L || ParentLoop)
addClonedBlockToLoopInfo(*BB, NewBB, LI, NewLoops);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (DT) {
if (Header == *BB) {
// The header is dominated by the preheader.
DT->addNewBlock(NewBB, InsertTop);
} else {
// Copy information from original loop to unrolled loop.
BasicBlock *IDomBB = DT->getNode(*BB)->getIDom()->getBlock();
DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB]));
}
}
if (Latch == *BB) {
// For the last block, if CreateRemainderLoop is false, create a direct
// jump to InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (!CreateRemainderLoop) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2,
suffix + ".iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
if (!CreateRemainderLoop) {
if (UseEpilogRemainder) {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
NewPHI->removeIncomingValue(Latch, false);
} else {
VMap[&*I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
}
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (Value *V = VMap.lookup(InVal))
NewPHI->setIncomingValue(idx, V);
}
}
if (CreateRemainderLoop) {
Loop *NewLoop = NewLoops[L];
assert(NewLoop && "L should have been cloned");
// Only add loop metadata if the loop is not going to be completely
// unrolled.
if (UnrollRemainder)
return NewLoop;
// Add unroll disable metadata to disable future unrolling for this loop.
SmallVector<Metadata *, 4> MDs;
// Reserve first location for self reference to the LoopID metadata node.
MDs.push_back(nullptr);
MDNode *LoopID = NewLoop->getLoopID();
if (LoopID) {
// First remove any existing loop unrolling metadata.
for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
bool IsUnrollMetadata = false;
MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
if (MD) {
const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
}
if (!IsUnrollMetadata)
MDs.push_back(LoopID->getOperand(i));
}
}
LLVMContext &Context = NewLoop->getHeader()->getContext();
SmallVector<Metadata *, 1> DisableOperands;
DisableOperands.push_back(MDString::get(Context,
"llvm.loop.unroll.disable"));
MDNode *DisableNode = MDNode::get(Context, DisableOperands);
MDs.push_back(DisableNode);
MDNode *NewLoopID = MDNode::get(Context, MDs);
// Set operand 0 to refer to the loop id itself.
NewLoopID->replaceOperandWith(0, NewLoopID);
NewLoop->setLoopID(NewLoopID);
return NewLoop;
}
else
return nullptr;
}
bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
bool AllowExpensiveTripCount,
bool UseEpilogRemainder,
LoopInfo *LI, ScalarEvolution *SE,
DominatorTree *DT, bool PreserveLCSSA) {
// for now, only unroll loops that contain a single exit
if (!L->getExitingBlock())
return false;
// Make sure the loop is in canonical form, and there is a single
// exit block only.
if (!L->isLoopSimplifyForm())
return false;
BasicBlock *Exit = L->getUniqueExitBlock(); // successor out of loop
if (!Exit)
return false;
// Use Scalar Evolution to compute the trip count. This allows more loops to
// be unrolled than relying on induction var simplification.
if (!SE)
return false;
// Only unroll loops with a computable trip count, and the trip count needs
// to be an int value (allowing a pointer type is a TODO item).
const SCEV *BECountSC = SE->getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BECountSC) ||
!BECountSC->getType()->isIntegerTy())
return false;
unsigned BEWidth = cast<IntegerType>(BECountSC->getType())->getBitWidth();
// Add 1 since the backedge count doesn't include the first loop iteration.
const SCEV *TripCountSC =
SE->getAddExpr(BECountSC, SE->getConstant(BECountSC->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC))
return false;
BasicBlock *Header = L->getHeader();
BasicBlock *PreHeader = L->getLoopPreheader();
BranchInst *PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
const DataLayout &DL = Header->getModule()->getDataLayout();
SCEVExpander Expander(*SE, DL, "loop-unroll");
if (!AllowExpensiveTripCount &&
Expander.isHighCostExpansion(TripCountSC, L, PreHeaderBR))
return false;
// This constraint lets us deal with an overflowing trip count easily; see the
// comment on ModVal below.
if (Log2_32(Count) > BEWidth)
return false;
BasicBlock *Latch = L->getLoopLatch();
// Loop structure is the following:
//
// PreHeader
// Header
// ...
// Latch
// Exit
BasicBlock *NewPreHeader;
BasicBlock *NewExit = nullptr;
BasicBlock *PrologExit = nullptr;
BasicBlock *EpilogPreHeader = nullptr;
BasicBlock *PrologPreHeader = nullptr;
if (UseEpilogRemainder) {
// If epilog remainder
// Split PreHeader to insert a branch around loop for unrolling.
NewPreHeader = SplitBlock(PreHeader, PreHeader->getTerminator(), DT, LI);
NewPreHeader->setName(PreHeader->getName() + ".new");
// Split Exit to create phi nodes from branch above.
SmallVector<BasicBlock*, 4> Preds(predecessors(Exit));
NewExit = SplitBlockPredecessors(Exit, Preds, ".unr-lcssa",
DT, LI, PreserveLCSSA);
// Split NewExit to insert epilog remainder loop.
EpilogPreHeader = SplitBlock(NewExit, NewExit->getTerminator(), DT, LI);
EpilogPreHeader->setName(Header->getName() + ".epil.preheader");
} else {
// If prolog remainder
// Split the original preheader twice to insert prolog remainder loop
PrologPreHeader = SplitEdge(PreHeader, Header, DT, LI);
PrologPreHeader->setName(Header->getName() + ".prol.preheader");
PrologExit = SplitBlock(PrologPreHeader, PrologPreHeader->getTerminator(),
DT, LI);
PrologExit->setName(Header->getName() + ".prol.loopexit");
// Split PrologExit to get NewPreHeader.
NewPreHeader = SplitBlock(PrologExit, PrologExit->getTerminator(), DT, LI);
NewPreHeader->setName(PreHeader->getName() + ".new");
}
// Loop structure should be the following:
// Epilog Prolog
//
// PreHeader PreHeader
// *NewPreHeader *PrologPreHeader
// Header *PrologExit
// ... *NewPreHeader
// Latch Header
// *NewExit ...
// *EpilogPreHeader Latch
// Exit Exit
// Calculate conditions for branch around loop for unrolling
// in epilog case and around prolog remainder loop in prolog case.
// Compute the number of extra iterations required, which is:
// extra iterations = run-time trip count % loop unroll factor
PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Value *BECount = Expander.expandCodeFor(BECountSC, BECountSC->getType(),
PreHeaderBR);
IRBuilder<> B(PreHeaderBR);
Value *ModVal;
// Calculate ModVal = (BECount + 1) % Count.
// Note that TripCount is BECount + 1.
if (isPowerOf2_32(Count)) {
// When Count is power of 2 we don't BECount for epilog case, however we'll
// need it for a branch around unrolling loop for prolog case.
ModVal = B.CreateAnd(TripCount, Count - 1, "xtraiter");
// 1. There are no iterations to be run in the prolog/epilog loop.
// OR
// 2. The addition computing TripCount overflowed.
//
// If (2) is true, we know that TripCount really is (1 << BEWidth) and so
// the number of iterations that remain to be run in the original loop is a
// multiple Count == (1 << Log2(Count)) because Log2(Count) <= BEWidth (we
// explicitly check this above).
} else {
// As (BECount + 1) can potentially unsigned overflow we count
// (BECount % Count) + 1 which is overflow safe as BECount % Count < Count.
Value *ModValTmp = B.CreateURem(BECount,
ConstantInt::get(BECount->getType(),
Count));
Value *ModValAdd = B.CreateAdd(ModValTmp,
ConstantInt::get(ModValTmp->getType(), 1));
// At that point (BECount % Count) + 1 could be equal to Count.
// To handle this case we need to take mod by Count one more time.
ModVal = B.CreateURem(ModValAdd,
ConstantInt::get(BECount->getType(), Count),
"xtraiter");
}
Value *BranchVal =
UseEpilogRemainder ? B.CreateICmpULT(BECount,
ConstantInt::get(BECount->getType(),
Count - 1)) :
B.CreateIsNotNull(ModVal, "lcmp.mod");
BasicBlock *RemainderLoop = UseEpilogRemainder ? NewExit : PrologPreHeader;
BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit;
// Branch to either remainder (extra iterations) loop or unrolling loop.
B.CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop);
PreHeaderBR->eraseFromParent();
Function *F = Header->getParent();
// Get an ordered list of blocks in the loop to help with the ordering of the
// cloned blocks in the prolog/epilog code
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
//
// For each extra loop iteration, create a copy of the loop's basic blocks
// and generate a condition that branches to the copy depending on the
// number of 'left over' iterations.
//
std::vector<BasicBlock *> NewBlocks;
ValueToValueMapTy VMap;
// For unroll factor 2 remainder loop will have 1 iterations.
// Do not create 1 iteration loop.
bool CreateRemainderLoop = (Count != 2);
// Clone all the basic blocks in the loop. If Count is 2, we don't clone
// the loop, otherwise we create a cloned loop to execute the extra
// iterations. This function adds the appropriate CFG connections.
BasicBlock *InsertBot = UseEpilogRemainder ? Exit : PrologExit;
BasicBlock *InsertTop = UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader;
CloneLoopBlocks(L, ModVal, CreateRemainderLoop, UseEpilogRemainder, InsertTop,
InsertBot, NewPreHeader, NewBlocks, LoopBlocks, VMap, LI);
// Insert the cloned blocks into the function.
F->getBasicBlockList().splice(InsertBot->getIterator(),
F->getBasicBlockList(),
NewBlocks[0]->getIterator(),
F->end());
// Loop structure should be the following:
// Epilog Prolog
//
// PreHeader PreHeader
// NewPreHeader PrologPreHeader
// Header PrologHeader
// ... ...
// Latch PrologLatch
// NewExit PrologExit
// EpilogPreHeader NewPreHeader
// EpilogHeader Header
// ... ...
// EpilogLatch Latch
// Exit Exit
// Rewrite the cloned instruction operands to use the values created when the
// clone is created.
for (BasicBlock *BB : NewBlocks) {
for (Instruction &I : *BB) {
RemapInstruction(&I, VMap,
RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
}
}
if (UseEpilogRemainder) {
// Connect the epilog code to the original loop and update the
// PHI functions.
ConnectEpilog(L, ModVal, NewExit, Exit, PreHeader,
EpilogPreHeader, NewPreHeader, VMap, DT, LI,
PreserveLCSSA);
// Update counter in loop for unrolling.
// I should be multiply of Count.
IRBuilder<> B2(NewPreHeader->getTerminator());
Value *TestVal = B2.CreateSub(TripCount, ModVal, "unroll_iter");
BranchInst *LatchBR = cast<BranchInst>(Latch->getTerminator());
B2.SetInsertPoint(LatchBR);
PHINode *NewIdx = PHINode::Create(TestVal->getType(), 2, "niter",
Header->getFirstNonPHI());
Value *IdxSub =
B2.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".nsub");
Value *IdxCmp;
if (LatchBR->getSuccessor(0) == Header)
IdxCmp = B2.CreateIsNotNull(IdxSub, NewIdx->getName() + ".ncmp");
else
IdxCmp = B2.CreateIsNull(IdxSub, NewIdx->getName() + ".ncmp");
NewIdx->addIncoming(TestVal, NewPreHeader);
NewIdx->addIncoming(IdxSub, Latch);
LatchBR->setCondition(IdxCmp);
} else {
// Connect the prolog code to the original loop and update the
// PHI functions.
ConnectProlog(L, BECount, Count, PrologExit, PreHeader, NewPreHeader,
VMap, DT, LI, PreserveLCSSA);
}
// If this loop is nested, then the loop unroller changes the code in the
// parent loop, so the Scalar Evolution pass needs to be run again.
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
NumRuntimeUnrolled++;
return true;
}
/// CloneAndPruneFunctionInto - This works exactly like CloneFunctionInto,
/// except that it does some simple constant prop and DCE on the fly. The
/// effect of this is to copy significantly less code in cases where (for
/// example) a function call with constant arguments is inlined, and those
/// constant arguments cause a significant amount of code in the callee to be
/// dead. Since this doesn't produce an exact copy of the input, it can't be
/// used for things like CloneFunction or CloneModule.
void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
ValueToValueMapTy &VMap,
bool ModuleLevelChanges,
SmallVectorImpl<ReturnInst*> &Returns,
const char *NameSuffix,
ClonedCodeInfo *CodeInfo,
const TargetData *TD,
Instruction *TheCall) {
assert(NameSuffix && "NameSuffix cannot be null!");
#ifndef NDEBUG
for (Function::const_arg_iterator II = OldFunc->arg_begin(),
E = OldFunc->arg_end(); II != E; ++II)
assert(VMap.count(II) && "No mapping from source argument specified!");
#endif
PruningFunctionCloner PFC(NewFunc, OldFunc, VMap, ModuleLevelChanges,
Returns, NameSuffix, CodeInfo, TD);
// Clone the entry block, and anything recursively reachable from it.
std::vector<const BasicBlock*> CloneWorklist;
CloneWorklist.push_back(&OldFunc->getEntryBlock());
while (!CloneWorklist.empty()) {
const BasicBlock *BB = CloneWorklist.back();
CloneWorklist.pop_back();
PFC.CloneBlock(BB, CloneWorklist);
}
// Loop over all of the basic blocks in the old function. If the block was
// reachable, we have cloned it and the old block is now in the value map:
// insert it into the new function in the right order. If not, ignore it.
//
// Defer PHI resolution until rest of function is resolved.
SmallVector<const PHINode*, 16> PHIToResolve;
for (Function::const_iterator BI = OldFunc->begin(), BE = OldFunc->end();
BI != BE; ++BI) {
Value *V = VMap[BI];
BasicBlock *NewBB = cast_or_null<BasicBlock>(V);
if (NewBB == 0) continue; // Dead block.
// Add the new block to the new function.
NewFunc->getBasicBlockList().push_back(NewBB);
// Loop over all of the instructions in the block, fixing up operand
// references as we go. This uses VMap to do all the hard work.
//
BasicBlock::iterator I = NewBB->begin();
DebugLoc TheCallDL;
if (TheCall)
TheCallDL = TheCall->getDebugLoc();
// Handle PHI nodes specially, as we have to remove references to dead
// blocks.
if (PHINode *PN = dyn_cast<PHINode>(I)) {
// Skip over all PHI nodes, remembering them for later.
BasicBlock::const_iterator OldI = BI->begin();
for (; (PN = dyn_cast<PHINode>(I)); ++I, ++OldI)
PHIToResolve.push_back(cast<PHINode>(OldI));
}
// Otherwise, remap the rest of the instructions normally.
for (; I != NewBB->end(); ++I)
RemapInstruction(I, VMap,
ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
}
// Defer PHI resolution until rest of function is resolved, PHI resolution
// requires the CFG to be up-to-date.
for (unsigned phino = 0, e = PHIToResolve.size(); phino != e; ) {
const PHINode *OPN = PHIToResolve[phino];
unsigned NumPreds = OPN->getNumIncomingValues();
const BasicBlock *OldBB = OPN->getParent();
BasicBlock *NewBB = cast<BasicBlock>(VMap[OldBB]);
// Map operands for blocks that are live and remove operands for blocks
// that are dead.
for (; phino != PHIToResolve.size() &&
PHIToResolve[phino]->getParent() == OldBB; ++phino) {
OPN = PHIToResolve[phino];
PHINode *PN = cast<PHINode>(VMap[OPN]);
for (unsigned pred = 0, e = NumPreds; pred != e; ++pred) {
Value *V = VMap[PN->getIncomingBlock(pred)];
if (BasicBlock *MappedBlock = cast_or_null<BasicBlock>(V)) {
Value *InVal = MapValue(PN->getIncomingValue(pred),
VMap,
ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges);
assert(InVal && "Unknown input value?");
PN->setIncomingValue(pred, InVal);
PN->setIncomingBlock(pred, MappedBlock);
} else {
PN->removeIncomingValue(pred, false);
--pred, --e; // Revisit the next entry.
}
}
}
// The loop above has removed PHI entries for those blocks that are dead
// and has updated others. However, if a block is live (i.e. copied over)
// but its terminator has been changed to not go to this block, then our
// phi nodes will have invalid entries. Update the PHI nodes in this
// case.
PHINode *PN = cast<PHINode>(NewBB->begin());
NumPreds = std::distance(pred_begin(NewBB), pred_end(NewBB));
if (NumPreds != PN->getNumIncomingValues()) {
assert(NumPreds < PN->getNumIncomingValues());
// Count how many times each predecessor comes to this block.
std::map<BasicBlock*, unsigned> PredCount;
for (pred_iterator PI = pred_begin(NewBB), E = pred_end(NewBB);
PI != E; ++PI)
--PredCount[*PI];
// Figure out how many entries to remove from each PHI.
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
++PredCount[PN->getIncomingBlock(i)];
// At this point, the excess predecessor entries are positive in the
// map. Loop over all of the PHIs and remove excess predecessor
// entries.
BasicBlock::iterator I = NewBB->begin();
for (; (PN = dyn_cast<PHINode>(I)); ++I) {
for (std::map<BasicBlock*, unsigned>::iterator PCI =PredCount.begin(),
E = PredCount.end(); PCI != E; ++PCI) {
BasicBlock *Pred = PCI->first;
for (unsigned NumToRemove = PCI->second; NumToRemove; --NumToRemove)
PN->removeIncomingValue(Pred, false);
}
}
}
// If the loops above have made these phi nodes have 0 or 1 operand,
// replace them with undef or the input value. We must do this for
// correctness, because 0-operand phis are not valid.
PN = cast<PHINode>(NewBB->begin());
if (PN->getNumIncomingValues() == 0) {
BasicBlock::iterator I = NewBB->begin();
BasicBlock::const_iterator OldI = OldBB->begin();
while ((PN = dyn_cast<PHINode>(I++))) {
Value *NV = UndefValue::get(PN->getType());
PN->replaceAllUsesWith(NV);
assert(VMap[OldI] == PN && "VMap mismatch");
VMap[OldI] = NV;
PN->eraseFromParent();
++OldI;
}
}
// NOTE: We cannot eliminate single entry phi nodes here, because of
// VMap. Single entry phi nodes can have multiple VMap entries
// pointing at them. Thus, deleting one would require scanning the VMap
// to update any entries in it that would require that. This would be
// really slow.
}
// Now that the inlined function body has been fully constructed, go through
// and zap unconditional fall-through branches. This happen all the time when
// specializing code: code specialization turns conditional branches into
// uncond branches, and this code folds them.
Function::iterator I = cast<BasicBlock>(VMap[&OldFunc->getEntryBlock()]);
while (I != NewFunc->end()) {
BranchInst *BI = dyn_cast<BranchInst>(I->getTerminator());
if (!BI || BI->isConditional()) { ++I; continue; }
// Note that we can't eliminate uncond branches if the destination has
// single-entry PHI nodes. Eliminating the single-entry phi nodes would
// require scanning the VMap to update any entries that point to the phi
// node.
BasicBlock *Dest = BI->getSuccessor(0);
if (!Dest->getSinglePredecessor() || isa<PHINode>(Dest->begin())) {
++I; continue;
}
// We know all single-entry PHI nodes in the inlined function have been
// removed, so we just need to splice the blocks.
BI->eraseFromParent();
// Make all PHI nodes that referred to Dest now refer to I as their source.
Dest->replaceAllUsesWith(I);
// Move all the instructions in the succ to the pred.
I->getInstList().splice(I->end(), Dest->getInstList());
// Remove the dest block.
Dest->eraseFromParent();
// Do not increment I, iteratively merge all things this block branches to.
}
}
///////////////////////////////////
//newOneValueChecker() //
///////////////////////////////////
BasicBlock *InsDuplica::newOneValueChecker(Value *ValuetoCheck, Instruction *synchI, BasicBlock *BBofSynchI, std::string &nameTag)
{
LLVMContext& C = BBofSynchI->getContext();
assert (duplicable(ValuetoCheck) && "checked value must be duplicable");
#ifdef REG_SAFE
//Register Safe SafeReg
//if ValuetoCheck is a load or cast(load), do not check
/* -- We are generalizing the rule.
if (isa<LoadInst>(ValuetoCheck)) return BBofSynchI;
if (CastInst *castI = dyn_cast<CastInst>(ValuetoCheck)) {
if (isa<LoadInst>(castI->getOperand(0)))
return BBofSynchI;
}
*/
if (curSafeRegs->isValueSafe(ValuetoCheck)) {
// Count the checks for reg safe
statRegRemove(synchI);
return BBofSynchI;
}
if (CastInst *castI = dyn_cast<CastInst>(ValuetoCheck)) {
if (curSafeRegs->isValueSafe(castI->getOperand(0))) {
// Count the checks for reg safe
statRegRemove(synchI);
return BBofSynchI;
}
}
#endif
Value *ValuetoCheckDup = ValuetoCheck; //by default
//If ValuetoCheck's dup is itself. Do not check it.
if (valueMap.count(ValuetoCheck) >0)
if (valueMap[ValuetoCheck] == ValuetoCheck)
return BBofSynchI;
//new SetEQ instruction and insert it before synchI
Instruction* newSetEQ = NULL;
Type* ty = ValuetoCheck->getType();
if(ty->isIntOrIntVectorTy()||ty->isPointerTy()) /* newSetEQ */
newSetEQ = new ICmpInst(synchI, ICmpInst::ICMP_EQ, ValuetoCheck, ValuetoCheckDup,
ValuetoCheck->getName()+nameTag);
else
newSetEQ = new FCmpInst(synchI, FCmpInst::FCMP_OEQ, ValuetoCheck, ValuetoCheckDup,
ValuetoCheck->getName()+nameTag);
//FIXME: xiehuc unknow setDUPmethod
//newSetEQ->setDUP(); //set DUP attribute
localnuminsdup++;
NumInsDup++;
#ifdef Jing_DEBUG
std::cerr << "create newSetEQ for " <<ValuetoCheck->getName() <<" at " << BBofSynchI->getName()<<"\n";
#endif
//update ValuetoCheckDup
if (valueMap.count(ValuetoCheck) > 0) {
ValuetoCheckDup = valueMap[ValuetoCheck];
newSetEQ->setOperand(1,ValuetoCheckDup);
} else {
//StoreValue should have a duplica.
//Since we haven't found it, sumbit a request an update request
assert(isa<Instruction>(ValuetoCheck) && "Argu must be already in valueMap");
requestToMap(cast<Instruction>(ValuetoCheck), newSetEQ);
}
//split BBofSynchI to add conditional branch
BasicBlock *newBB = BBofSynchI->splitBasicBlock(synchI, BBofSynchI->getName()+nameTag);
//Now, the end of BBofSynchI is a branch to newBB. We have to replace this branch by a conditional branch based on newSetEQ
BranchInst *BI = dyn_cast<BranchInst>(BBofSynchI->getTerminator());
assert(BI && "After split, the splitted BB must have a Br as its terminator");
//Erase the old branch
BI->eraseFromParent();
Instruction* Term = NULL;
if(ty->isVectorTy()){
/**
* since setcc hasbeen removed, we extract value one by one and 'and' them
* if result is 1 means all value is 1, so it means equal, so we can branch it.
*/
APInt Idx(32, 0);
Instruction* elemx = ExtractElementInst::Create(newSetEQ, ConstantInt::get(C, Idx), "elem", BBofSynchI);
for(int i=1;i<ty->getVectorNumElements();i++){
++Idx;
Value* elemy = ExtractElementInst::Create(newSetEQ, ConstantInt::get(C, Idx), "elem", BBofSynchI);
elemx = BinaryOperator::CreateAnd(elemx, elemy, "aggregation", BBofSynchI);
}
newSetEQ = elemx;
}
Term = BranchInst::Create(newBB,errorBlock,newSetEQ,BBofSynchI);
//FIXME: unknow setDUP
//condBI->setDUP(); //set DUP attribute
localnuminsdup++;
NumInsDup++;
//no need to update PhINodes. splitBasicBlock already does so
#ifdef Jing_DEBUG
std::cerr << "newOneValueChecker creates "<<newBB->getName()<<" inside "<< BBofSynchI->getName()<<"\n";
#endif
#ifdef REG_SAFE
// Now the value is safe.
curSafeRegs->insertValueSafe(ValuetoCheck);
#ifdef Jing_DEBUG
std::cerr << "add_safe("<<ValuetoCheck->getName()<<"),";
#endif
#endif
localnumStorechecker++;
NumStoreChecker++;
return newBB;
}
/// Insert code in the prolog code when unrolling a loop with a
/// run-time trip-count.
///
/// This method assumes that the loop unroll factor is total number
/// of loop bodes in the loop after unrolling. (Some folks refer
/// to the unroll factor as the number of *extra* copies added).
/// We assume also that the loop unroll factor is a power-of-two. So, after
/// unrolling the loop, the number of loop bodies executed is 2,
/// 4, 8, etc. Note - LLVM converts the if-then-sequence to a switch
/// instruction in SimplifyCFG.cpp. Then, the backend decides how code for
/// the switch instruction is generated.
///
/// extraiters = tripcount % loopfactor
/// if (extraiters == 0) jump Loop:
/// if (extraiters == loopfactor) jump L1
/// if (extraiters == loopfactor-1) jump L2
/// ...
/// L1: LoopBody;
/// L2: LoopBody;
/// ...
/// if tripcount < loopfactor jump End
/// Loop:
/// ...
/// End:
///
bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count, LoopInfo *LI,
LPPassManager *LPM) {
// for now, only unroll loops that contain a single exit
if (!L->getExitingBlock())
return false;
// Make sure the loop is in canonical form, and there is a single
// exit block only.
if (!L->isLoopSimplifyForm() || !L->getUniqueExitBlock())
return false;
// Use Scalar Evolution to compute the trip count. This allows more
// loops to be unrolled than relying on induction var simplification
if (!LPM)
return false;
ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>();
if (!SE)
return false;
// Only unroll loops with a computable trip count and the trip count needs
// to be an int value (allowing a pointer type is a TODO item)
const SCEV *BECount = SE->getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BECount) || !BECount->getType()->isIntegerTy())
return false;
// Add 1 since the backedge count doesn't include the first loop iteration
const SCEV *TripCountSC =
SE->getAddExpr(BECount, SE->getConstant(BECount->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC))
return false;
// We only handle cases when the unroll factor is a power of 2.
// Count is the loop unroll factor, the number of extra copies added + 1.
if ((Count & (Count-1)) != 0)
return false;
// If this loop is nested, then the loop unroller changes the code in
// parent loop, so the Scalar Evolution pass needs to be run again
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
BasicBlock *PH = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
// It helps to splits the original preheader twice, one for the end of the
// prolog code and one for a new loop preheader
BasicBlock *PEnd = SplitEdge(PH, Header, LPM->getAsPass());
BasicBlock *NewPH = SplitBlock(PEnd, PEnd->getTerminator(), LPM->getAsPass());
BranchInst *PreHeaderBR = cast<BranchInst>(PH->getTerminator());
// Compute the number of extra iterations required, which is:
// extra iterations = run-time trip count % (loop unroll factor + 1)
SCEVExpander Expander(*SE, "loop-unroll");
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Type *CountTy = TripCount->getType();
BinaryOperator *ModVal =
BinaryOperator::CreateURem(TripCount,
ConstantInt::get(CountTy, Count),
"xtraiter");
ModVal->insertBefore(PreHeaderBR);
// Check if for no extra iterations, then jump to unrolled loop
Value *BranchVal = new ICmpInst(PreHeaderBR,
ICmpInst::ICMP_NE, ModVal,
ConstantInt::get(CountTy, 0), "lcmp");
// Branch to either the extra iterations or the unrolled loop
// We will fix up the true branch label when adding loop body copies
BranchInst::Create(PEnd, PEnd, BranchVal, PreHeaderBR);
assert(PreHeaderBR->isUnconditional() &&
PreHeaderBR->getSuccessor(0) == PEnd &&
"CFG edges in Preheader are not correct");
PreHeaderBR->eraseFromParent();
ValueToValueMapTy LVMap;
Function *F = Header->getParent();
// These variables are used to update the CFG links in each iteration
BasicBlock *CompareBB = nullptr;
BasicBlock *LastLoopBB = PH;
// Get an ordered list of blocks in the loop to help with the ordering of the
// cloned blocks in the prolog code
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
//
// For each extra loop iteration, create a copy of the loop's basic blocks
// and generate a condition that branches to the copy depending on the
// number of 'left over' iterations.
//
for (unsigned leftOverIters = Count-1; leftOverIters > 0; --leftOverIters) {
std::vector<BasicBlock*> NewBlocks;
ValueToValueMapTy VMap;
// Clone all the basic blocks in the loop, but we don't clone the loop
// This function adds the appropriate CFG connections.
CloneLoopBlocks(L, (leftOverIters == Count-1), LastLoopBB, PEnd, NewBlocks,
LoopBlocks, VMap, LVMap, LI);
LastLoopBB = cast<BasicBlock>(VMap[Latch]);
// Insert the cloned blocks into function just before the original loop
F->getBasicBlockList().splice(PEnd, F->getBasicBlockList(),
NewBlocks[0], F->end());
// Generate the code for the comparison which determines if the loop
// prolog code needs to be executed.
if (leftOverIters == Count-1) {
// There is no compare block for the fall-thru case when for the last
// left over iteration
CompareBB = NewBlocks[0];
} else {
// Create a new block for the comparison
BasicBlock *NewBB = BasicBlock::Create(CompareBB->getContext(), "unr.cmp",
F, CompareBB);
if (Loop *ParentLoop = L->getParentLoop()) {
// Add the new block to the parent loop, if needed
ParentLoop->addBasicBlockToLoop(NewBB, LI->getBase());
}
// The comparison w/ the extra iteration value and branch
Value *BranchVal = new ICmpInst(*NewBB, ICmpInst::ICMP_EQ, ModVal,
ConstantInt::get(CountTy, leftOverIters),
"un.tmp");
// Branch to either the extra iterations or the unrolled loop
BranchInst::Create(NewBlocks[0], CompareBB,
BranchVal, NewBB);
CompareBB = NewBB;
PH->getTerminator()->setSuccessor(0, NewBB);
VMap[NewPH] = CompareBB;
}
// Rewrite the cloned instruction operands to use the values
// created when the clone is created.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i) {
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I) {
RemapInstruction(I, VMap,
RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
}
}
}
// Connect the prolog code to the original loop and update the
// PHI functions.
ConnectProlog(L, TripCount, Count, LastLoopBB, PEnd, PH, NewPH, LVMap,
LPM->getAsPass());
NumRuntimeUnrolled++;
return true;
}
/// Create a clone of the blocks in a loop and connect them together.
/// If CreateRemainderLoop is false, loop structure will not be cloned,
/// otherwise a new loop will be created including all cloned blocks, and the
/// iterator of it switches to count NewIter down to 0.
/// The cloned blocks should be inserted between InsertTop and InsertBot.
/// If loop structure is cloned InsertTop should be new preheader, InsertBot
/// new loop exit.
/// Return the new cloned loop that is created when CreateRemainderLoop is true.
static Loop *
CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop,
const bool UseEpilogRemainder, const bool UnrollRemainder,
BasicBlock *InsertTop,
BasicBlock *InsertBot, BasicBlock *Preheader,
std::vector<BasicBlock *> &NewBlocks, LoopBlocksDFS &LoopBlocks,
ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI) {
StringRef suffix = UseEpilogRemainder ? "epil" : "prol";
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
Function *F = Header->getParent();
LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
Loop *ParentLoop = L->getParentLoop();
NewLoopsMap NewLoops;
NewLoops[ParentLoop] = ParentLoop;
if (!CreateRemainderLoop)
NewLoops[L] = ParentLoop;
// For each block in the original loop, create a new copy,
// and update the value map with the newly created values.
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, "." + suffix, F);
NewBlocks.push_back(NewBB);
// If we're unrolling the outermost loop, there's no remainder loop,
// and this block isn't in a nested loop, then the new block is not
// in any loop. Otherwise, add it to loopinfo.
if (CreateRemainderLoop || LI->getLoopFor(*BB) != L || ParentLoop)
addClonedBlockToLoopInfo(*BB, NewBB, LI, NewLoops);
VMap[*BB] = NewBB;
if (Header == *BB) {
// For the first block, add a CFG connection to this newly
// created block.
InsertTop->getTerminator()->setSuccessor(0, NewBB);
}
if (DT) {
if (Header == *BB) {
// The header is dominated by the preheader.
DT->addNewBlock(NewBB, InsertTop);
} else {
// Copy information from original loop to unrolled loop.
BasicBlock *IDomBB = DT->getNode(*BB)->getIDom()->getBlock();
DT->addNewBlock(NewBB, cast<BasicBlock>(VMap[IDomBB]));
}
}
if (Latch == *BB) {
// For the last block, if CreateRemainderLoop is false, create a direct
// jump to InsertBot. If not, create a loop back to cloned head.
VMap.erase((*BB)->getTerminator());
BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
IRBuilder<> Builder(LatchBR);
if (!CreateRemainderLoop) {
Builder.CreateBr(InsertBot);
} else {
PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2,
suffix + ".iter",
FirstLoopBB->getFirstNonPHI());
Value *IdxSub =
Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
NewIdx->getName() + ".sub");
Value *IdxCmp =
Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
NewIdx->addIncoming(NewIter, InsertTop);
NewIdx->addIncoming(IdxSub, NewBB);
}
LatchBR->eraseFromParent();
}
}
// Change the incoming values to the ones defined in the preheader or
// cloned loop.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
if (!CreateRemainderLoop) {
if (UseEpilogRemainder) {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
NewPHI->removeIncomingValue(Latch, false);
} else {
VMap[&*I] = NewPHI->getIncomingValueForBlock(Preheader);
cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
}
} else {
unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
NewPHI->setIncomingBlock(idx, InsertTop);
BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
idx = NewPHI->getBasicBlockIndex(Latch);
Value *InVal = NewPHI->getIncomingValue(idx);
NewPHI->setIncomingBlock(idx, NewLatch);
if (Value *V = VMap.lookup(InVal))
NewPHI->setIncomingValue(idx, V);
}
}
if (CreateRemainderLoop) {
Loop *NewLoop = NewLoops[L];
MDNode *LoopID = NewLoop->getLoopID();
assert(NewLoop && "L should have been cloned");
// Only add loop metadata if the loop is not going to be completely
// unrolled.
if (UnrollRemainder)
return NewLoop;
Optional<MDNode *> NewLoopID = makeFollowupLoopID(
LoopID, {LLVMLoopUnrollFollowupAll, LLVMLoopUnrollFollowupRemainder});
if (NewLoopID.hasValue()) {
NewLoop->setLoopID(NewLoopID.getValue());
// Do not setLoopAlreadyUnrolled if loop attributes have been defined
// explicitly.
return NewLoop;
}
// Add unroll disable metadata to disable future unrolling for this loop.
NewLoop->setLoopAlreadyUnrolled();
return NewLoop;
}
else
return nullptr;
}
