int BranchProbabilities::CheckFloatHeuristic()
{
// Heuristic fails if the last instruction is not a conditional branch
BranchInst *BI = dyn_cast<BranchInst>(_TI);
if ((!BI) || (BI->isUnconditional()))
return -1;
// All float comparisons are done with the fcmp instruction
FCmpInst *fcmp = dyn_cast<FCmpInst>(BI->getCondition());
if (!fcmp)
return -1;
// Choose the prefered branch depending on if this is an eq or neq comp
switch (fcmp->getPredicate())
{
case FCmpInst::FCMP_OEQ:
case FCmpInst::FCMP_UEQ:
return 1;
case FCmpInst::FCMP_ONE:
case FCmpInst::FCMP_UNE:
return 0;
case FCmpInst::FCMP_FALSE:
case FCmpInst::FCMP_TRUE:
assert("true or false predicate should have been folded!");
default:
return -1;
}
}
/// FindReusablePredBB - Check all of the predecessors of the block DestPHI
/// lives in to see if there is a block that we can reuse as a critical edge
/// from TIBB.
static BasicBlock *FindReusablePredBB(PHINode *DestPHI, BasicBlock *TIBB) {
BasicBlock *Dest = DestPHI->getParent();
/// TIPHIValues - This array is lazily computed to determine the values of
/// PHIs in Dest that TI would provide.
SmallVector<Value*, 32> TIPHIValues;
/// TIBBEntryNo - This is a cache to speed up pred queries for TIBB.
unsigned TIBBEntryNo = 0;
// Check to see if Dest has any blocks that can be used as a split edge for
// this terminator.
for (unsigned pi = 0, e = DestPHI->getNumIncomingValues(); pi != e; ++pi) {
BasicBlock *Pred = DestPHI->getIncomingBlock(pi);
// To be usable, the pred has to end with an uncond branch to the dest.
BranchInst *PredBr = dyn_cast<BranchInst>(Pred->getTerminator());
if (!PredBr || !PredBr->isUnconditional())
continue;
// Must be empty other than the branch and debug info.
BasicBlock::iterator I = Pred->begin();
while (isa<DbgInfoIntrinsic>(I))
I++;
if (&*I != PredBr)
continue;
// Cannot be the entry block; its label does not get emitted.
if (Pred == &Dest->getParent()->getEntryBlock())
continue;
// Finally, since we know that Dest has phi nodes in it, we have to make
// sure that jumping to Pred will have the same effect as going to Dest in
// terms of PHI values.
PHINode *PN;
unsigned PHINo = 0;
unsigned PredEntryNo = pi;
bool FoundMatch = true;
for (BasicBlock::iterator I = Dest->begin();
(PN = dyn_cast<PHINode>(I)); ++I, ++PHINo) {
if (PHINo == TIPHIValues.size()) {
if (PN->getIncomingBlock(TIBBEntryNo) != TIBB)
TIBBEntryNo = PN->getBasicBlockIndex(TIBB);
TIPHIValues.push_back(PN->getIncomingValue(TIBBEntryNo));
}
// If the PHI entry doesn't work, we can't use this pred.
if (PN->getIncomingBlock(PredEntryNo) != Pred)
PredEntryNo = PN->getBasicBlockIndex(Pred);
if (TIPHIValues[PHINo] != PN->getIncomingValue(PredEntryNo)) {
FoundMatch = false;
break;
}
}
// If we found a workable predecessor, change TI to branch to Succ.
if (FoundMatch)
return Pred;
}
return 0;
}
void TempScopInfo::buildCondition(BasicBlock *BB, BasicBlock *RegionEntry) {
BBCond Cond;
DomTreeNode *BBNode = DT->getNode(BB), *EntryNode = DT->getNode(RegionEntry);
assert(BBNode && EntryNode && "Get null node while building condition!");
// Walk up the dominance tree until reaching the entry node. Add all
// conditions on the path to BB except if BB postdominates the block
// containing the condition.
while (BBNode != EntryNode) {
BasicBlock *CurBB = BBNode->getBlock();
BBNode = BBNode->getIDom();
assert(BBNode && "BBNode should not reach the root node!");
if (PDT->dominates(CurBB, BBNode->getBlock()))
continue;
BranchInst *Br = dyn_cast<BranchInst>(BBNode->getBlock()->getTerminator());
assert(Br && "A Valid Scop should only contain branch instruction");
if (Br->isUnconditional())
continue;
// Is BB on the ELSE side of the branch?
bool inverted = DT->dominates(Br->getSuccessor(1), BB);
Comparison *Cmp;
buildAffineCondition(*(Br->getCondition()), inverted, &Cmp);
Cond.push_back(*Cmp);
}
if (!Cond.empty())
BBConds[BB] = Cond;
}
bool CFGTree::identifySuccessorRelation(BasicBlock *predBB,
BasicBlock *succBB) {
bool identify = false;
BasicBlock *bb = predBB;
while (true) {
Instruction *inst = &(bb->back());
if (inst->getOpcode() == Instruction::Br) {
BranchInst *bi = dyn_cast<BranchInst>(inst);
if (bi->isUnconditional()) {
bb = bi->getSuccessor(0);
if (bb == succBB) {
identify = true;
break;
}
} else {
identify = foundSameBrInstFromCFGTree(inst, root);
break;
}
} else {
if (inst->getOpcode() == Instruction::Ret)
break;
else
assert(false && "Unsupported instruction!");
}
}
return identify;
}
bool CallAnalyzer::visitBranchInst(BranchInst &BI) {
// We model unconditional branches as essentially free -- they really
// shouldn't exist at all, but handling them makes the behavior of the
// inliner more regular and predictable. Interestingly, conditional branches
// which will fold away are also free.
return BI.isUnconditional() || isa<ConstantInt>(BI.getCondition()) ||
dyn_cast_or_null<ConstantInt>(
SimplifiedValues.lookup(BI.getCondition()));
}
void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) {
if (I.isUnconditional())
return;
Value *Cond = I.getCondition();
if (!DA->isUniform(Cond))
return;
setUniformMetadata(I.getParent()->getTerminator());
}
void Interpreter::visitBranchInst(BranchInst &I) {
ExecutionContext &SF = ECStack.back();
BasicBlock *Dest;
Dest = I.getSuccessor(0); // Uncond branches have a fixed dest...
if (!I.isUnconditional()) {
Value *Cond = I.getCondition();
if (getOperandValue(Cond, SF).BoolVal == 0) // If false cond...
Dest = I.getSuccessor(1);
}
SwitchToNewBasicBlock(Dest, SF);
}
/// Rotate loop LP. Return true if the loop is rotated.
///
/// \param SimplifiedLatch is true if the latch was just folded into the final
/// loop exit. In this case we may want to rotate even though the new latch is
/// now an exiting branch. This rotation would have happened had the latch not
/// been simplified. However, if SimplifiedLatch is false, then we avoid
/// rotating loops in which the latch exits to avoid excessive or endless
/// rotation. LoopRotate should be repeatable and converge to a canonical
/// form. This property is satisfied because simplifying the loop latch can only
/// happen once across multiple invocations of the LoopRotate pass.
bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
// If the loop has only one block then there is not much to rotate.
if (L->getBlocks().size() == 1)
return false;
BasicBlock *OrigHeader = L->getHeader();
BasicBlock *OrigLatch = L->getLoopLatch();
BranchInst *BI = dyn_cast<BranchInst>(OrigHeader->getTerminator());
if (!BI || BI->isUnconditional())
return false;
// If the loop header is not one of the loop exiting blocks then
// either this loop is already rotated or it is not
// suitable for loop rotation transformations.
if (!L->isLoopExiting(OrigHeader))
return false;
// If the loop latch already contains a branch that leaves the loop then the
// loop is already rotated.
if (!OrigLatch)
return false;
// Rotate if either the loop latch does *not* exit the loop, or if the loop
// latch was just simplified. Or if we think it will be profitable.
if (L->isLoopExiting(OrigLatch) && !SimplifiedLatch && IsUtilMode == false &&
!shouldRotateLoopExitingLatch(L))
return false;
// Check size of original header and reject loop if it is very big or we can't
// duplicate blocks inside it.
{
SmallPtrSet<const Value *, 32> EphValues;
CodeMetrics::collectEphemeralValues(L, AC, EphValues);
CodeMetrics Metrics;
Metrics.analyzeBasicBlock(OrigHeader, *TTI, EphValues);
if (Metrics.notDuplicatable) {
LLVM_DEBUG(
dbgs() << "LoopRotation: NOT rotating - contains non-duplicatable"
<< " instructions: ";
L->dump());
return false;
}
if (Metrics.convergent) {
LLVM_DEBUG(dbgs() << "LoopRotation: NOT rotating - contains convergent "
"instructions: ";
L->dump());
return false;
}
// Check if there is PGO data or user annoated branch data:
static bool hasProfileData(Function *F, FunctionOutliningInfo *OI) {
if (F->getEntryCount())
return true;
// Now check if any of the entry block has MD_prof data:
for (auto *E : OI->Entries) {
BranchInst *BR = dyn_cast<BranchInst>(E->getTerminator());
if (!BR || BR->isUnconditional())
continue;
uint64_t T, F;
if (BR->extractProfMetadata(T, F))
return true;
}
return false;
}
// Predict that a comparison in which a register is an operand, the register is
// used before being defined in a successor block, and the successor block
// does not post-dominate will reach the successor block.
int BranchProbabilities::CheckGuardHeuristic()
{
BranchInst *BI = dyn_cast<BranchInst>(_TI);
bool bUses[2] = {false, false};
// If we don't have a conditional branch, abandon
if ((!BI) || (BI->isUnconditional()))
return -1;
// If the condition is not immediately dependent on a comparison, abandon
CmpInst *cmp = dyn_cast<CmpInst>(BI->getCondition());
if (!cmp)
return -1;
for (int i = 0; i < 2; i++)
{
if (_bPostDoms[i])
continue;
// Get the values being compared
Value *v = cmp->getOperand(i);
// For all uses of the first value check if the use post-dominates
for (Value::use_iterator UI = v->use_begin(), UE = v->use_end();
UI != UE; ++UI)
{
// if the use is not an instruction, skip it
Instruction *I = dyn_cast<Instruction>(*UI);
if (!I)
continue;
BasicBlock *UsingBlock = I->getParent();
// Check if the use is in either successor
for (int i = 0; i < 2; i++)
if (UsingBlock == _Succ[i])
bUses[i] = true;
}
}
if (bUses[0] == bUses[1])
return -1;
if (bUses[0])
return 0;
else
return 1;
}
/// Annotate the control flow with intrinsics so the backend can
/// recognize if/then/else and loops.
bool SIAnnotateControlFlow::runOnFunction(Function &F) {
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
DA = &getAnalysis<LegacyDivergenceAnalysis>();
for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()),
E = df_end(&F.getEntryBlock()); I != E; ++I) {
BasicBlock *BB = *I;
BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
if (!Term || Term->isUnconditional()) {
if (isTopOfStack(BB))
closeControlFlow(BB);
continue;
}
if (I.nodeVisited(Term->getSuccessor(1))) {
if (isTopOfStack(BB))
closeControlFlow(BB);
handleLoop(Term);
continue;
}
if (isTopOfStack(BB)) {
PHINode *Phi = dyn_cast<PHINode>(Term->getCondition());
if (Phi && Phi->getParent() == BB && isElse(Phi)) {
insertElse(Term);
eraseIfUnused(Phi);
continue;
}
closeControlFlow(BB);
}
openIf(Term);
}
if (!Stack.empty()) {
// CFG was probably not structured.
report_fatal_error("failed to annotate CFG");
}
return true;
}
/// Fold the loop tail into the loop exit by speculating the loop tail
/// instructions. Typically, this is a single post-increment. In the case of a
/// simple 2-block loop, hoisting the increment can be much better than
/// duplicating the entire loop header. In the case of loops with early exits,
/// rotation will not work anyway, but simplifyLoopLatch will put the loop in
/// canonical form so downstream passes can handle it.
///
/// I don't believe this invalidates SCEV.
bool LoopRotate::simplifyLoopLatch(Loop *L) {
BasicBlock *Latch = L->getLoopLatch();
if (!Latch || Latch->hasAddressTaken())
return false;
BranchInst *Jmp = dyn_cast<BranchInst>(Latch->getTerminator());
if (!Jmp || !Jmp->isUnconditional())
return false;
BasicBlock *LastExit = Latch->getSinglePredecessor();
if (!LastExit || !L->isLoopExiting(LastExit))
return false;
BranchInst *BI = dyn_cast<BranchInst>(LastExit->getTerminator());
if (!BI)
return false;
if (!shouldSpeculateInstrs(Latch->begin(), Jmp))
return false;
DEBUG(dbgs() << "Folding loop latch " << Latch->getName() << " into "
<< LastExit->getName() << "\n");
// Hoist the instructions from Latch into LastExit.
LastExit->getInstList().splice(BI, Latch->getInstList(), Latch->begin(), Jmp);
unsigned FallThruPath = BI->getSuccessor(0) == Latch ? 0 : 1;
BasicBlock *Header = Jmp->getSuccessor(0);
assert(Header == L->getHeader() && "expected a backward branch");
// Remove Latch from the CFG so that LastExit becomes the new Latch.
BI->setSuccessor(FallThruPath, Header);
Latch->replaceSuccessorsPhiUsesWith(LastExit);
Jmp->eraseFromParent();
// Nuke the Latch block.
assert(Latch->empty() && "unable to evacuate Latch");
LI->removeBlock(Latch);
if (DominatorTreeWrapperPass *DTWP =
getAnalysisIfAvailable<DominatorTreeWrapperPass>())
DTWP->getDomTree().eraseNode(Latch);
Latch->eraseFromParent();
return true;
}
bool IRTranslator::translateBr(const BranchInst &BrInst) {
unsigned Succ = 0;
if (!BrInst.isUnconditional()) {
// We want a G_BRCOND to the true BB followed by an unconditional branch.
unsigned Tst = getOrCreateVReg(*BrInst.getCondition());
const BasicBlock &TrueTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ++));
MachineBasicBlock &TrueBB = getOrCreateBB(TrueTgt);
MIRBuilder.buildBrCond(LLT{*BrInst.getCondition()->getType()}, Tst, TrueBB);
}
const BasicBlock &BrTgt = *cast<BasicBlock>(BrInst.getSuccessor(Succ));
MachineBasicBlock &TgtBB = getOrCreateBB(BrTgt);
MIRBuilder.buildBr(TgtBB);
// Link successors.
MachineBasicBlock &CurBB = MIRBuilder.getMBB();
for (const BasicBlock *Succ : BrInst.successors())
CurBB.addSuccessor(&getOrCreateBB(*Succ));
return true;
}
/// Rotate loop LP. Return true if the loop is rotated.
///
/// \param SimplifiedLatch is true if the latch was just folded into the final
/// loop exit. In this case we may want to rotate even though the new latch is
/// now an exiting branch. This rotation would have happened had the latch not
/// been simplified. However, if SimplifiedLatch is false, then we avoid
/// rotating loops in which the latch exits to avoid excessive or endless
/// rotation. LoopRotate should be repeatable and converge to a canonical
/// form. This property is satisfied because simplifying the loop latch can only
/// happen once across multiple invocations of the LoopRotate pass.
bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
// If the loop has only one block then there is not much to rotate.
if (L->getBlocks().size() == 1)
return false;
BasicBlock *OrigHeader = L->getHeader();
BasicBlock *OrigLatch = L->getLoopLatch();
BranchInst *BI = dyn_cast<BranchInst>(OrigHeader->getTerminator());
if (BI == 0 || BI->isUnconditional())
return false;
// If the loop header is not one of the loop exiting blocks then
// either this loop is already rotated or it is not
// suitable for loop rotation transformations.
if (!L->isLoopExiting(OrigHeader))
return false;
// If the loop latch already contains a branch that leaves the loop then the
// loop is already rotated.
if (OrigLatch == 0)
return false;
// Rotate if either the loop latch does *not* exit the loop, or if the loop
// latch was just simplified.
if (L->isLoopExiting(OrigLatch) && !SimplifiedLatch)
return false;
// Check size of original header and reject loop if it is very big or we can't
// duplicate blocks inside it.
{
CodeMetrics Metrics;
Metrics.analyzeBasicBlock(OrigHeader, *TTI);
if (Metrics.notDuplicatable) {
DEBUG(dbgs() << "LoopRotation: NOT rotating - contains non duplicatable"
<< " instructions: "; L->dump());
return false;
}
if (Metrics.NumInsts > MAX_HEADER_SIZE)
return false;
}
/// \brief Annotate the control flow with intrinsics so the backend can
/// recognize if/then/else and loops.
bool SIAnnotateControlFlow::runOnFunction(Function &F) {
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
DA = &getAnalysis<DivergenceAnalysis>();
for (df_iterator<BasicBlock *> I = df_begin(&F.getEntryBlock()),
E = df_end(&F.getEntryBlock()); I != E; ++I) {
BranchInst *Term = dyn_cast<BranchInst>((*I)->getTerminator());
if (!Term || Term->isUnconditional()) {
if (isTopOfStack(*I))
closeControlFlow(*I);
continue;
}
if (I.nodeVisited(Term->getSuccessor(1))) {
if (isTopOfStack(*I))
closeControlFlow(*I);
handleLoop(Term);
continue;
}
if (isTopOfStack(*I)) {
PHINode *Phi = dyn_cast<PHINode>(Term->getCondition());
if (Phi && Phi->getParent() == *I && isElse(Phi)) {
insertElse(Term);
eraseIfUnused(Phi);
continue;
}
closeControlFlow(*I);
}
openIf(Term);
}
assert(Stack.empty());
return true;
}
/// EliminateMostlyEmptyBlocks - eliminate blocks that contain only PHI nodes,
/// debug info directives, and an unconditional branch. Passes before isel
/// (e.g. LSR/loopsimplify) often split edges in ways that are non-optimal for
/// isel. Start by eliminating these blocks so we can split them the way we
/// want them.
bool CodeGenPrepare::EliminateMostlyEmptyBlocks(Function &F) {
bool MadeChange = false;
// Note that this intentionally skips the entry block.
for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ) {
BasicBlock *BB = I++;
// If this block doesn't end with an uncond branch, ignore it.
BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator());
if (!BI || !BI->isUnconditional())
continue;
// If the instruction before the branch (skipping debug info) isn't a phi
// node, then other stuff is happening here.
BasicBlock::iterator BBI = BI;
if (BBI != BB->begin()) {
--BBI;
while (isa<DbgInfoIntrinsic>(BBI)) {
if (BBI == BB->begin())
break;
--BBI;
}
if (!isa<DbgInfoIntrinsic>(BBI) && !isa<PHINode>(BBI))
continue;
}
// Do not break infinite loops.
BasicBlock *DestBB = BI->getSuccessor(0);
if (DestBB == BB)
continue;
if (!CanMergeBlocks(BB, DestBB))
continue;
EliminateMostlyEmptyBlock(BB);
MadeChange = true;
}
return MadeChange;
}
int BranchProbabilities::CheckPointerHeuristic()
{
// Heuristic fails if the last instruction is not a conditional branch
BranchInst *BI = dyn_cast<BranchInst>(_TI);
if ((!BI) || (BI->isUnconditional()))
return -1;
// All pointer comparisons are done with the icmp instruction
ICmpInst *icmp = dyn_cast<ICmpInst>(BI->getCondition());
if (!icmp)
return -1;
Value *v[2];
v[0] = icmp->getOperand(0);
v[1] = icmp->getOperand(1);
// Make sure we're comparing pointers
if (isa<PointerType>(v[0]->getType()))
{
assert(isa<PointerType>(v[1]->getType()) && "v[1] is not a pointer!");
// Choose the prefered branch depending on if this is an eq or neq comp
switch (icmp->getPredicate())
{
case ICmpInst::ICMP_EQ:
return 1;
case ICmpInst::ICMP_NE:
return 0;
default:
assert("non-equality comparison of pointers");
return -1;
}
}
return -1;
}
// RevectorBlockTo - Revector the unconditional branch at the end of FromBB to
// the ToBB block, which is one of the successors of its current successor.
void CondProp::RevectorBlockTo(BasicBlock *FromBB, BasicBlock *ToBB) {
BranchInst *FromBr = cast<BranchInst>(FromBB->getTerminator());
assert(FromBr->isUnconditional() && "FromBB should end with uncond br!");
// Get the old block we are threading through.
BasicBlock *OldSucc = FromBr->getSuccessor(0);
// OldSucc had multiple successors. If ToBB has multiple predecessors, then
// the edge between them would be critical, which we already took care of.
// If ToBB has single operand PHI node then take care of it here.
while (PHINode *PN = dyn_cast<PHINode>(ToBB->begin())) {
assert(PN->getNumIncomingValues() == 1 && "Critical Edge Found!");
PN->replaceAllUsesWith(PN->getIncomingValue(0));
PN->eraseFromParent();
}
// Update PHI nodes in OldSucc to know that FromBB no longer branches to it.
OldSucc->removePredecessor(FromBB);
// Change FromBr to branch to the new destination.
FromBr->setSuccessor(0, ToBB);
MadeChange = true;
}
/// Insert code in the prolog code when unrolling a loop with a
/// run-time trip-count.
///
/// This method assumes that the loop unroll factor is total number
/// of loop bodes in the loop after unrolling. (Some folks refer
/// to the unroll factor as the number of *extra* copies added).
/// We assume also that the loop unroll factor is a power-of-two. So, after
/// unrolling the loop, the number of loop bodies executed is 2,
/// 4, 8, etc. Note - LLVM converts the if-then-sequence to a switch
/// instruction in SimplifyCFG.cpp. Then, the backend decides how code for
/// the switch instruction is generated.
///
/// extraiters = tripcount % loopfactor
/// if (extraiters == 0) jump Loop:
/// if (extraiters == loopfactor) jump L1
/// if (extraiters == loopfactor-1) jump L2
/// ...
/// L1: LoopBody;
/// L2: LoopBody;
/// ...
/// if tripcount < loopfactor jump End
/// Loop:
/// ...
/// End:
///
bool llvm::UnrollRuntimeLoopProlog(Loop *L, unsigned Count, LoopInfo *LI,
LPPassManager *LPM) {
// for now, only unroll loops that contain a single exit
if (!L->getExitingBlock())
return false;
// Make sure the loop is in canonical form, and there is a single
// exit block only.
if (!L->isLoopSimplifyForm() || !L->getUniqueExitBlock())
return false;
// Use Scalar Evolution to compute the trip count. This allows more
// loops to be unrolled than relying on induction var simplification
if (!LPM)
return false;
ScalarEvolution *SE = LPM->getAnalysisIfAvailable<ScalarEvolution>();
if (!SE)
return false;
// Only unroll loops with a computable trip count and the trip count needs
// to be an int value (allowing a pointer type is a TODO item)
const SCEV *BECount = SE->getBackedgeTakenCount(L);
if (isa<SCEVCouldNotCompute>(BECount) || !BECount->getType()->isIntegerTy())
return false;
// Add 1 since the backedge count doesn't include the first loop iteration
const SCEV *TripCountSC =
SE->getAddExpr(BECount, SE->getConstant(BECount->getType(), 1));
if (isa<SCEVCouldNotCompute>(TripCountSC))
return false;
// We only handle cases when the unroll factor is a power of 2.
// Count is the loop unroll factor, the number of extra copies added + 1.
if ((Count & (Count-1)) != 0)
return false;
// If this loop is nested, then the loop unroller changes the code in
// parent loop, so the Scalar Evolution pass needs to be run again
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
BasicBlock *PH = L->getLoopPreheader();
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
// It helps to splits the original preheader twice, one for the end of the
// prolog code and one for a new loop preheader
BasicBlock *PEnd = SplitEdge(PH, Header, LPM->getAsPass());
BasicBlock *NewPH = SplitBlock(PEnd, PEnd->getTerminator(), LPM->getAsPass());
BranchInst *PreHeaderBR = cast<BranchInst>(PH->getTerminator());
// Compute the number of extra iterations required, which is:
// extra iterations = run-time trip count % (loop unroll factor + 1)
SCEVExpander Expander(*SE, "loop-unroll");
Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
PreHeaderBR);
Type *CountTy = TripCount->getType();
BinaryOperator *ModVal =
BinaryOperator::CreateURem(TripCount,
ConstantInt::get(CountTy, Count),
"xtraiter");
ModVal->insertBefore(PreHeaderBR);
// Check if for no extra iterations, then jump to unrolled loop
Value *BranchVal = new ICmpInst(PreHeaderBR,
ICmpInst::ICMP_NE, ModVal,
ConstantInt::get(CountTy, 0), "lcmp");
// Branch to either the extra iterations or the unrolled loop
// We will fix up the true branch label when adding loop body copies
BranchInst::Create(PEnd, PEnd, BranchVal, PreHeaderBR);
assert(PreHeaderBR->isUnconditional() &&
PreHeaderBR->getSuccessor(0) == PEnd &&
"CFG edges in Preheader are not correct");
PreHeaderBR->eraseFromParent();
ValueToValueMapTy LVMap;
Function *F = Header->getParent();
// These variables are used to update the CFG links in each iteration
BasicBlock *CompareBB = nullptr;
BasicBlock *LastLoopBB = PH;
// Get an ordered list of blocks in the loop to help with the ordering of the
// cloned blocks in the prolog code
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
//
// For each extra loop iteration, create a copy of the loop's basic blocks
// and generate a condition that branches to the copy depending on the
// number of 'left over' iterations.
//
for (unsigned leftOverIters = Count-1; leftOverIters > 0; --leftOverIters) {
std::vector<BasicBlock*> NewBlocks;
ValueToValueMapTy VMap;
// Clone all the basic blocks in the loop, but we don't clone the loop
// This function adds the appropriate CFG connections.
CloneLoopBlocks(L, (leftOverIters == Count-1), LastLoopBB, PEnd, NewBlocks,
LoopBlocks, VMap, LVMap, LI);
LastLoopBB = cast<BasicBlock>(VMap[Latch]);
// Insert the cloned blocks into function just before the original loop
F->getBasicBlockList().splice(PEnd, F->getBasicBlockList(),
NewBlocks[0], F->end());
// Generate the code for the comparison which determines if the loop
// prolog code needs to be executed.
if (leftOverIters == Count-1) {
// There is no compare block for the fall-thru case when for the last
// left over iteration
CompareBB = NewBlocks[0];
} else {
// Create a new block for the comparison
BasicBlock *NewBB = BasicBlock::Create(CompareBB->getContext(), "unr.cmp",
F, CompareBB);
if (Loop *ParentLoop = L->getParentLoop()) {
// Add the new block to the parent loop, if needed
ParentLoop->addBasicBlockToLoop(NewBB, LI->getBase());
}
// The comparison w/ the extra iteration value and branch
Value *BranchVal = new ICmpInst(*NewBB, ICmpInst::ICMP_EQ, ModVal,
ConstantInt::get(CountTy, leftOverIters),
"un.tmp");
// Branch to either the extra iterations or the unrolled loop
BranchInst::Create(NewBlocks[0], CompareBB,
BranchVal, NewBB);
CompareBB = NewBB;
PH->getTerminator()->setSuccessor(0, NewBB);
VMap[NewPH] = CompareBB;
}
// Rewrite the cloned instruction operands to use the values
// created when the clone is created.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i) {
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I) {
RemapInstruction(I, VMap,
RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
}
}
}
// Connect the prolog code to the original loop and update the
// PHI functions.
ConnectProlog(L, TripCount, Count, LastLoopBB, PEnd, PH, NewPH, LVMap,
LPM->getAsPass());
NumRuntimeUnrolled++;
return true;
}
/// SimplifyStoreAtEndOfBlock - Turn things like:
/// if () { *P = v1; } else { *P = v2 }
/// into a phi node with a store in the successor.
///
/// Simplify things like:
/// *P = v1; if () { *P = v2; }
/// into a phi node with a store in the successor.
///
bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
BasicBlock *StoreBB = SI.getParent();
// Check to see if the successor block has exactly two incoming edges. If
// so, see if the other predecessor contains a store to the same location.
// if so, insert a PHI node (if needed) and move the stores down.
BasicBlock *DestBB = StoreBB->getTerminator()->getSuccessor(0);
// Determine whether Dest has exactly two predecessors and, if so, compute
// the other predecessor.
pred_iterator PI = pred_begin(DestBB);
BasicBlock *P = *PI;
BasicBlock *OtherBB = nullptr;
if (P != StoreBB)
OtherBB = P;
if (++PI == pred_end(DestBB))
return false;
P = *PI;
if (P != StoreBB) {
if (OtherBB)
return false;
OtherBB = P;
}
if (++PI != pred_end(DestBB))
return false;
// Bail out if all the relevant blocks aren't distinct (this can happen,
// for example, if SI is in an infinite loop)
if (StoreBB == DestBB || OtherBB == DestBB)
return false;
// Verify that the other block ends in a branch and is not otherwise empty.
BasicBlock::iterator BBI(OtherBB->getTerminator());
BranchInst *OtherBr = dyn_cast<BranchInst>(BBI);
if (!OtherBr || BBI == OtherBB->begin())
return false;
// If the other block ends in an unconditional branch, check for the 'if then
// else' case. there is an instruction before the branch.
StoreInst *OtherStore = nullptr;
if (OtherBr->isUnconditional()) {
--BBI;
// Skip over debugging info.
while (isa<DbgInfoIntrinsic>(BBI) ||
(isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy())) {
if (BBI==OtherBB->begin())
return false;
--BBI;
}
// If this isn't a store, isn't a store to the same location, or is not the
// right kind of store, bail out.
OtherStore = dyn_cast<StoreInst>(BBI);
if (!OtherStore || OtherStore->getOperand(1) != SI.getOperand(1) ||
!SI.isSameOperationAs(OtherStore))
return false;
} else {
// Otherwise, the other block ended with a conditional branch. If one of the
// destinations is StoreBB, then we have the if/then case.
if (OtherBr->getSuccessor(0) != StoreBB &&
OtherBr->getSuccessor(1) != StoreBB)
return false;
// Okay, we know that OtherBr now goes to Dest and StoreBB, so this is an
// if/then triangle. See if there is a store to the same ptr as SI that
// lives in OtherBB.
for (;; --BBI) {
// Check to see if we find the matching store.
if ((OtherStore = dyn_cast<StoreInst>(BBI))) {
if (OtherStore->getOperand(1) != SI.getOperand(1) ||
!SI.isSameOperationAs(OtherStore))
return false;
break;
}
// If we find something that may be using or overwriting the stored
// value, or if we run out of instructions, we can't do the xform.
if (BBI->mayReadFromMemory() || BBI->mayWriteToMemory() ||
BBI == OtherBB->begin())
return false;
}
// In order to eliminate the store in OtherBr, we have to
// make sure nothing reads or overwrites the stored value in
// StoreBB.
for (BasicBlock::iterator I = StoreBB->begin(); &*I != &SI; ++I) {
// FIXME: This should really be AA driven.
if (I->mayReadFromMemory() || I->mayWriteToMemory())
return false;
}
}
// Insert a PHI node now if we need it.
Value *MergedVal = OtherStore->getOperand(0);
if (MergedVal != SI.getOperand(0)) {
PHINode *PN = PHINode::Create(MergedVal->getType(), 2, "storemerge");
PN->addIncoming(SI.getOperand(0), SI.getParent());
PN->addIncoming(OtherStore->getOperand(0), OtherBB);
MergedVal = InsertNewInstBefore(PN, DestBB->front());
}
// Advance to a place where it is safe to insert the new store and
// insert it.
BBI = DestBB->getFirstInsertionPt();
StoreInst *NewSI = new StoreInst(MergedVal, SI.getOperand(1),
SI.isVolatile(),
SI.getAlignment(),
SI.getOrdering(),
SI.getSynchScope());
InsertNewInstBefore(NewSI, *BBI);
NewSI->setDebugLoc(OtherStore->getDebugLoc());
// If the two stores had AA tags, merge them.
AAMDNodes AATags;
SI.getAAMetadata(AATags);
if (AATags) {
OtherStore->getAAMetadata(AATags, /* Merge = */ true);
NewSI->setAAMetadata(AATags);
}
// Nuke the old stores.
EraseInstFromFunction(SI);
EraseInstFromFunction(*OtherStore);
return true;
}
/// Unroll the given loop by Count. The loop must be in LCSSA form. Returns true
/// if unrolling was successful, or false if the loop was unmodified. Unrolling
/// can only fail when the loop's latch block is not terminated by a conditional
/// branch instruction. However, if the trip count (and multiple) are not known,
/// loop unrolling will mostly produce more code that is no faster.
///
/// TripCount is generally defined as the number of times the loop header
/// executes. UnrollLoop relaxes the definition to permit early exits: here
/// TripCount is the iteration on which control exits LatchBlock if no early
/// exits were taken. Note that UnrollLoop assumes that the loop counter test
/// terminates LatchBlock in order to remove unnecesssary instances of the
/// test. In other words, control may exit the loop prior to TripCount
/// iterations via an early branch, but control may not exit the loop from the
/// LatchBlock's terminator prior to TripCount iterations.
///
/// Similarly, TripMultiple divides the number of times that the LatchBlock may
/// execute without exiting the loop.
///
/// The LoopInfo Analysis that is passed will be kept consistent.
///
/// If a LoopPassManager is passed in, and the loop is fully removed, it will be
/// removed from the LoopPassManager as well. LPM can also be NULL.
///
/// This utility preserves LoopInfo. If DominatorTree or ScalarEvolution are
/// available from the Pass it must also preserve those analyses.
bool llvm::UnrollLoop(Loop *L, unsigned Count, unsigned TripCount,
bool AllowRuntime, unsigned TripMultiple,
LoopInfo *LI, Pass *PP, LPPassManager *LPM) {
BasicBlock *Preheader = L->getLoopPreheader();
if (!Preheader) {
DEBUG(dbgs() << " Can't unroll; loop preheader-insertion failed.\n");
return false;
}
BasicBlock *LatchBlock = L->getLoopLatch();
if (!LatchBlock) {
DEBUG(dbgs() << " Can't unroll; loop exit-block-insertion failed.\n");
return false;
}
// Loops with indirectbr cannot be cloned.
if (!L->isSafeToClone()) {
DEBUG(dbgs() << " Can't unroll; Loop body cannot be cloned.\n");
return false;
}
BasicBlock *Header = L->getHeader();
BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
if (!BI || BI->isUnconditional()) {
// The loop-rotate pass can be helpful to avoid this in many cases.
DEBUG(dbgs() <<
" Can't unroll; loop not terminated by a conditional branch.\n");
return false;
}
if (Header->hasAddressTaken()) {
// The loop-rotate pass can be helpful to avoid this in many cases.
DEBUG(dbgs() <<
" Won't unroll loop: address of header block is taken.\n");
return false;
}
if (TripCount != 0)
DEBUG(dbgs() << " Trip Count = " << TripCount << "\n");
if (TripMultiple != 1)
DEBUG(dbgs() << " Trip Multiple = " << TripMultiple << "\n");
// Effectively "DCE" unrolled iterations that are beyond the tripcount
// and will never be executed.
if (TripCount != 0 && Count > TripCount)
Count = TripCount;
// Don't enter the unroll code if there is nothing to do. This way we don't
// need to support "partial unrolling by 1".
if (TripCount == 0 && Count < 2)
return false;
assert(Count > 0);
assert(TripMultiple > 0);
assert(TripCount == 0 || TripCount % TripMultiple == 0);
// Are we eliminating the loop control altogether?
bool CompletelyUnroll = Count == TripCount;
// We assume a run-time trip count if the compiler cannot
// figure out the loop trip count and the unroll-runtime
// flag is specified.
bool RuntimeTripCount = (TripCount == 0 && Count > 0 && AllowRuntime);
if (RuntimeTripCount && !UnrollRuntimeLoopProlog(L, Count, LI, LPM))
return false;
// Notify ScalarEvolution that the loop will be substantially changed,
// if not outright eliminated.
if (PP) {
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
if (SE)
SE->forgetLoop(L);
}
// If we know the trip count, we know the multiple...
unsigned BreakoutTrip = 0;
if (TripCount != 0) {
BreakoutTrip = TripCount % Count;
TripMultiple = 0;
} else {
// Figure out what multiple to use.
BreakoutTrip = TripMultiple =
(unsigned)GreatestCommonDivisor64(Count, TripMultiple);
}
// Report the unrolling decision.
DebugLoc LoopLoc = L->getStartLoc();
Function *F = Header->getParent();
LLVMContext &Ctx = F->getContext();
if (CompletelyUnroll) {
DEBUG(dbgs() << "COMPLETELY UNROLLING loop %" << Header->getName()
<< " with trip count " << TripCount << "!\n");
emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc,
Twine("completely unrolled loop with ") +
Twine(TripCount) + " iterations");
} else {
DEBUG(dbgs() << "UNROLLING loop %" << Header->getName()
<< " by " << Count);
Twine DiagMsg("unrolled loop by a factor of " + Twine(Count));
if (TripMultiple == 0 || BreakoutTrip != TripMultiple) {
DEBUG(dbgs() << " with a breakout at trip " << BreakoutTrip);
DiagMsg.concat(" with a breakout at trip " + Twine(BreakoutTrip));
} else if (TripMultiple != 1) {
DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
DiagMsg.concat(" with " + Twine(TripMultiple) + " trips per branch");
} else if (RuntimeTripCount) {
DEBUG(dbgs() << " with run-time trip count");
DiagMsg.concat(" with run-time trip count");
}
DEBUG(dbgs() << "!\n");
emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc, DiagMsg);
}
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
// For the first iteration of the loop, we should use the precloned values for
// PHI nodes. Insert associations now.
ValueToValueMapTy LastValueMap;
std::vector<PHINode*> OrigPHINode;
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
OrigPHINode.push_back(cast<PHINode>(I));
}
std::vector<BasicBlock*> Headers;
std::vector<BasicBlock*> Latches;
Headers.push_back(Header);
Latches.push_back(LatchBlock);
// The current on-the-fly SSA update requires blocks to be processed in
// reverse postorder so that LastValueMap contains the correct value at each
// exit.
LoopBlocksDFS DFS(L);
DFS.perform(LI);
// Stash the DFS iterators before adding blocks to the loop.
LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
for (unsigned It = 1; It != Count; ++It) {
std::vector<BasicBlock*> NewBlocks;
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
ValueToValueMapTy VMap;
BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
Header->getParent()->getBasicBlockList().push_back(New);
// Loop over all of the PHI nodes in the block, changing them to use the
// incoming values from the previous block.
if (*BB == Header)
for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
PHINode *NewPHI = cast<PHINode>(VMap[OrigPHINode[i]]);
Value *InVal = NewPHI->getIncomingValueForBlock(LatchBlock);
if (Instruction *InValI = dyn_cast<Instruction>(InVal))
if (It > 1 && L->contains(InValI))
InVal = LastValueMap[InValI];
VMap[OrigPHINode[i]] = InVal;
New->getInstList().erase(NewPHI);
}
// Update our running map of newest clones
LastValueMap[*BB] = New;
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI)
LastValueMap[VI->first] = VI->second;
L->addBasicBlockToLoop(New, LI->getBase());
// Add phi entries for newly created values to all exit blocks.
for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB);
SI != SE; ++SI) {
if (L->contains(*SI))
continue;
for (BasicBlock::iterator BBI = (*SI)->begin();
PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) {
Value *Incoming = phi->getIncomingValueForBlock(*BB);
ValueToValueMapTy::iterator It = LastValueMap.find(Incoming);
if (It != LastValueMap.end())
Incoming = It->second;
phi->addIncoming(Incoming, New);
}
}
// Keep track of new headers and latches as we create them, so that
// we can insert the proper branches later.
if (*BB == Header)
Headers.push_back(New);
if (*BB == LatchBlock)
Latches.push_back(New);
NewBlocks.push_back(New);
}
// Remap all instructions in the most recent iteration
for (unsigned i = 0; i < NewBlocks.size(); ++i)
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I)
::RemapInstruction(I, LastValueMap);
}
// Loop over the PHI nodes in the original block, setting incoming values.
for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) {
PHINode *PN = OrigPHINode[i];
if (CompletelyUnroll) {
PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader));
Header->getInstList().erase(PN);
}
else if (Count > 1) {
Value *InVal = PN->removeIncomingValue(LatchBlock, false);
// If this value was defined in the loop, take the value defined by the
// last iteration of the loop.
if (Instruction *InValI = dyn_cast<Instruction>(InVal)) {
if (L->contains(InValI))
InVal = LastValueMap[InVal];
}
assert(Latches.back() == LastValueMap[LatchBlock] && "bad last latch");
PN->addIncoming(InVal, Latches.back());
}
}
// Now that all the basic blocks for the unrolled iterations are in place,
// set up the branches to connect them.
for (unsigned i = 0, e = Latches.size(); i != e; ++i) {
// The original branch was replicated in each unrolled iteration.
BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
// The branch destination.
unsigned j = (i + 1) % e;
BasicBlock *Dest = Headers[j];
bool NeedConditional = true;
if (RuntimeTripCount && j != 0) {
NeedConditional = false;
}
// For a complete unroll, make the last iteration end with a branch
// to the exit block.
if (CompletelyUnroll && j == 0) {
Dest = LoopExit;
NeedConditional = false;
}
// If we know the trip count or a multiple of it, we can safely use an
// unconditional branch for some iterations.
if (j != BreakoutTrip && (TripMultiple == 0 || j % TripMultiple != 0)) {
NeedConditional = false;
}
if (NeedConditional) {
// Update the conditional branch's successor for the following
// iteration.
Term->setSuccessor(!ContinueOnTrue, Dest);
} else {
// Remove phi operands at this loop exit
if (Dest != LoopExit) {
BasicBlock *BB = Latches[i];
for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
SI != SE; ++SI) {
if (*SI == Headers[i])
continue;
for (BasicBlock::iterator BBI = (*SI)->begin();
PHINode *Phi = dyn_cast<PHINode>(BBI); ++BBI) {
Phi->removeIncomingValue(BB, false);
}
}
}
// Replace the conditional branch with an unconditional one.
BranchInst::Create(Dest, Term);
Term->eraseFromParent();
}
}
// Merge adjacent basic blocks, if possible.
for (unsigned i = 0, e = Latches.size(); i != e; ++i) {
BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator());
if (Term->isUnconditional()) {
BasicBlock *Dest = Term->getSuccessor(0);
if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI, LPM))
std::replace(Latches.begin(), Latches.end(), Dest, Fold);
}
}
DominatorTree *DT = nullptr;
if (PP) {
// FIXME: Reconstruct dom info, because it is not preserved properly.
// Incrementally updating domtree after loop unrolling would be easy.
if (DominatorTreeWrapperPass *DTWP =
PP->getAnalysisIfAvailable<DominatorTreeWrapperPass>()) {
DT = &DTWP->getDomTree();
DT->recalculate(*L->getHeader()->getParent());
}
// Simplify any new induction variables in the partially unrolled loop.
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
if (SE && !CompletelyUnroll) {
SmallVector<WeakVH, 16> DeadInsts;
simplifyLoopIVs(L, SE, LPM, DeadInsts);
// Aggressively clean up dead instructions that simplifyLoopIVs already
// identified. Any remaining should be cleaned up below.
while (!DeadInsts.empty())
if (Instruction *Inst =
dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
RecursivelyDeleteTriviallyDeadInstructions(Inst);
}
}
// At this point, the code is well formed. We now do a quick sweep over the
// inserted code, doing constant propagation and dead code elimination as we
// go.
const std::vector<BasicBlock*> &NewLoopBlocks = L->getBlocks();
for (std::vector<BasicBlock*>::const_iterator BB = NewLoopBlocks.begin(),
BBE = NewLoopBlocks.end(); BB != BBE; ++BB)
for (BasicBlock::iterator I = (*BB)->begin(), E = (*BB)->end(); I != E; ) {
Instruction *Inst = I++;
if (isInstructionTriviallyDead(Inst))
(*BB)->getInstList().erase(Inst);
else if (Value *V = SimplifyInstruction(Inst))
if (LI->replacementPreservesLCSSAForm(Inst, V)) {
Inst->replaceAllUsesWith(V);
(*BB)->getInstList().erase(Inst);
}
}
NumCompletelyUnrolled += CompletelyUnroll;
++NumUnrolled;
Loop *OuterL = L->getParentLoop();
// Remove the loop from the LoopPassManager if it's completely removed.
if (CompletelyUnroll && LPM != nullptr)
LPM->deleteLoopFromQueue(L);
// If we have a pass and a DominatorTree we should re-simplify impacted loops
// to ensure subsequent analyses can rely on this form. We want to simplify
// at least one layer outside of the loop that was unrolled so that any
// changes to the parent loop exposed by the unrolling are considered.
if (PP && DT) {
if (!OuterL && !CompletelyUnroll)
OuterL = L;
if (OuterL) {
ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>();
simplifyLoop(OuterL, DT, LI, PP, /*AliasAnalysis*/ nullptr, SE);
// LCSSA must be performed on the outermost affected loop. The unrolled
// loop's last loop latch is guaranteed to be in the outermost loop after
// deleteLoopFromQueue updates LoopInfo.
Loop *LatchLoop = LI->getLoopFor(Latches.back());
if (!OuterL->contains(LatchLoop))
while (OuterL->getParentLoop() != LatchLoop)
OuterL = OuterL->getParentLoop();
formLCSSARecursively(*OuterL, *DT, SE);
}
}
return true;
}
bool ScopDetection::isValidCFG(BasicBlock &BB,
DetectionContext &Context) const {
Region &RefRegion = Context.CurRegion;
TerminatorInst *TI = BB.getTerminator();
// Return instructions are only valid if the region is the top level region.
if (isa<ReturnInst>(TI) && !RefRegion.getExit() && TI->getNumOperands() == 0)
return true;
BranchInst *Br = dyn_cast<BranchInst>(TI);
if (!Br)
return invalid<ReportNonBranchTerminator>(Context, /*Assert=*/true, &BB);
if (Br->isUnconditional())
return true;
Value *Condition = Br->getCondition();
// UndefValue is not allowed as condition.
if (isa<UndefValue>(Condition))
return invalid<ReportUndefCond>(Context, /*Assert=*/true, &BB);
// Only Constant and ICmpInst are allowed as condition.
if (!(isa<Constant>(Condition) || isa<ICmpInst>(Condition)))
return invalid<ReportInvalidCond>(Context, /*Assert=*/true, &BB);
// Allow perfectly nested conditions.
assert(Br->getNumSuccessors() == 2 && "Unexpected number of successors");
if (ICmpInst *ICmp = dyn_cast<ICmpInst>(Condition)) {
// Unsigned comparisons are not allowed. They trigger overflow problems
// in the code generation.
//
// TODO: This is not sufficient and just hides bugs. However it does pretty
// well.
if (ICmp->isUnsigned())
return false;
// Are both operands of the ICmp affine?
if (isa<UndefValue>(ICmp->getOperand(0)) ||
isa<UndefValue>(ICmp->getOperand(1)))
return invalid<ReportUndefOperand>(Context, /*Assert=*/true, &BB);
Loop *L = LI->getLoopFor(ICmp->getParent());
const SCEV *LHS = SE->getSCEVAtScope(ICmp->getOperand(0), L);
const SCEV *RHS = SE->getSCEVAtScope(ICmp->getOperand(1), L);
if (!isAffineExpr(&Context.CurRegion, LHS, *SE) ||
!isAffineExpr(&Context.CurRegion, RHS, *SE))
return invalid<ReportNonAffBranch>(Context, /*Assert=*/true, &BB, LHS,
RHS);
}
// Allow loop exit conditions.
Loop *L = LI->getLoopFor(&BB);
if (L && L->getExitingBlock() == &BB)
return true;
// Allow perfectly nested conditions.
Region *R = RI->getRegionFor(&BB);
if (R->getEntry() != &BB)
return invalid<ReportCondition>(Context, /*Assert=*/true, &BB);
return true;
}
Function *PartialInlinerImpl::unswitchFunction(Function *F) {
// First, verify that this function is an unswitching candidate...
BasicBlock *EntryBlock = &F->front();
BranchInst *BR = dyn_cast<BranchInst>(EntryBlock->getTerminator());
if (!BR || BR->isUnconditional())
return nullptr;
BasicBlock *ReturnBlock = nullptr;
BasicBlock *NonReturnBlock = nullptr;
unsigned ReturnCount = 0;
for (BasicBlock *BB : successors(EntryBlock)) {
if (isa<ReturnInst>(BB->getTerminator())) {
ReturnBlock = BB;
ReturnCount++;
} else
NonReturnBlock = BB;
}
if (ReturnCount != 1)
return nullptr;
// Clone the function, so that we can hack away on it.
ValueToValueMapTy VMap;
Function *DuplicateFunction = CloneFunction(F, VMap);
DuplicateFunction->setLinkage(GlobalValue::InternalLinkage);
BasicBlock *NewEntryBlock = cast<BasicBlock>(VMap[EntryBlock]);
BasicBlock *NewReturnBlock = cast<BasicBlock>(VMap[ReturnBlock]);
BasicBlock *NewNonReturnBlock = cast<BasicBlock>(VMap[NonReturnBlock]);
// Go ahead and update all uses to the duplicate, so that we can just
// use the inliner functionality when we're done hacking.
F->replaceAllUsesWith(DuplicateFunction);
// Special hackery is needed with PHI nodes that have inputs from more than
// one extracted block. For simplicity, just split the PHIs into a two-level
// sequence of PHIs, some of which will go in the extracted region, and some
// of which will go outside.
BasicBlock *PreReturn = NewReturnBlock;
NewReturnBlock = NewReturnBlock->splitBasicBlock(
NewReturnBlock->getFirstNonPHI()->getIterator());
BasicBlock::iterator I = PreReturn->begin();
Instruction *Ins = &NewReturnBlock->front();
while (I != PreReturn->end()) {
PHINode *OldPhi = dyn_cast<PHINode>(I);
if (!OldPhi)
break;
PHINode *RetPhi = PHINode::Create(OldPhi->getType(), 2, "", Ins);
OldPhi->replaceAllUsesWith(RetPhi);
Ins = NewReturnBlock->getFirstNonPHI();
RetPhi->addIncoming(&*I, PreReturn);
RetPhi->addIncoming(OldPhi->getIncomingValueForBlock(NewEntryBlock),
NewEntryBlock);
OldPhi->removeIncomingValue(NewEntryBlock);
++I;
}
NewEntryBlock->getTerminator()->replaceUsesOfWith(PreReturn, NewReturnBlock);
// Gather up the blocks that we're going to extract.
std::vector<BasicBlock *> ToExtract;
ToExtract.push_back(NewNonReturnBlock);
for (BasicBlock &BB : *DuplicateFunction)
if (&BB != NewEntryBlock && &BB != NewReturnBlock &&
&BB != NewNonReturnBlock)
ToExtract.push_back(&BB);
// The CodeExtractor needs a dominator tree.
DominatorTree DT;
DT.recalculate(*DuplicateFunction);
// Manually calculate a BlockFrequencyInfo and BranchProbabilityInfo.
LoopInfo LI(DT);
BranchProbabilityInfo BPI(*DuplicateFunction, LI);
BlockFrequencyInfo BFI(*DuplicateFunction, BPI, LI);
// Extract the body of the if.
Function *ExtractedFunction =
CodeExtractor(ToExtract, &DT, /*AggregateArgs*/ false, &BFI, &BPI)
.extractCodeRegion();
// Inline the top-level if test into all callers.
std::vector<User *> Users(DuplicateFunction->user_begin(),
DuplicateFunction->user_end());
for (User *User : Users)
if (CallInst *CI = dyn_cast<CallInst>(User))
InlineFunction(CI, IFI);
else if (InvokeInst *II = dyn_cast<InvokeInst>(User))
InlineFunction(II, IFI);
// Ditch the duplicate, since we're done with it, and rewrite all remaining
// users (function pointers, etc.) back to the original function.
DuplicateFunction->replaceAllUsesWith(F);
DuplicateFunction->eraseFromParent();
++NumPartialInlined;
return ExtractedFunction;
}
bool LoopIndexSplit::splitLoop() {
SplitCondition = NULL;
if (ExitCondition->getPredicate() == ICmpInst::ICMP_NE
|| ExitCondition->getPredicate() == ICmpInst::ICMP_EQ)
return false;
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
BranchInst *SBR = NULL; // Split Condition Branch
BranchInst *EBR = cast<BranchInst>(ExitCondition->getParent()->getTerminator());
// If Exiting block includes loop variant instructions then this
// loop may not be split safely.
BasicBlock *ExitingBlock = ExitCondition->getParent();
if (!cleanBlock(ExitingBlock)) return false;
LLVMContext &Context = Header->getContext();
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
I != E; ++I) {
BranchInst *BR = dyn_cast<BranchInst>((*I)->getTerminator());
if (!BR || BR->isUnconditional()) continue;
ICmpInst *CI = dyn_cast<ICmpInst>(BR->getCondition());
if (!CI || CI == ExitCondition
|| CI->getPredicate() == ICmpInst::ICMP_NE
|| CI->getPredicate() == ICmpInst::ICMP_EQ)
continue;
// Unable to handle triangle loops at the moment.
// In triangle loop, split condition is in header and one of the
// the split destination is loop latch. If split condition is EQ
// then such loops are already handle in processOneIterationLoop().
if (Header == (*I)
&& (Latch == BR->getSuccessor(0) || Latch == BR->getSuccessor(1)))
continue;
// If the block does not dominate the latch then this is not a diamond.
// Such loop may not benefit from index split.
if (!DT->dominates((*I), Latch))
continue;
// If split condition branches heads do not have single predecessor,
// SplitCondBlock, then is not possible to remove inactive branch.
if (!BR->getSuccessor(0)->getSinglePredecessor()
|| !BR->getSuccessor(1)->getSinglePredecessor())
return false;
// If the merge point for BR is not loop latch then skip this condition.
if (BR->getSuccessor(0) != Latch) {
DominanceFrontier::iterator DF0 = DF->find(BR->getSuccessor(0));
assert (DF0 != DF->end() && "Unable to find dominance frontier");
if (!DF0->second.count(Latch))
continue;
}
if (BR->getSuccessor(1) != Latch) {
DominanceFrontier::iterator DF1 = DF->find(BR->getSuccessor(1));
assert (DF1 != DF->end() && "Unable to find dominance frontier");
if (!DF1->second.count(Latch))
continue;
}
SplitCondition = CI;
SBR = BR;
break;
}
if (!SplitCondition)
return false;
// If the predicate sign does not match then skip.
if (ExitCondition->isSigned() != SplitCondition->isSigned())
return false;
unsigned EVOpNum = (ExitCondition->getOperand(1) == IVExitValue);
unsigned SVOpNum = IVBasedValues.count(SplitCondition->getOperand(0));
Value *SplitValue = SplitCondition->getOperand(SVOpNum);
if (!L->isLoopInvariant(SplitValue))
return false;
if (!IVBasedValues.count(SplitCondition->getOperand(!SVOpNum)))
return false;
// Normalize loop conditions so that it is easier to calculate new loop
// bounds.
if (IVisGT(*ExitCondition) || IVisGE(*ExitCondition)) {
ExitCondition->setPredicate(ExitCondition->getInversePredicate());
BasicBlock *T = EBR->getSuccessor(0);
EBR->setSuccessor(0, EBR->getSuccessor(1));
EBR->setSuccessor(1, T);
}
if (IVisGT(*SplitCondition) || IVisGE(*SplitCondition)) {
SplitCondition->setPredicate(SplitCondition->getInversePredicate());
BasicBlock *T = SBR->getSuccessor(0);
SBR->setSuccessor(0, SBR->getSuccessor(1));
SBR->setSuccessor(1, T);
}
//[*] Calculate new loop bounds.
Value *AEV = SplitValue;
Value *BSV = SplitValue;
bool Sign = SplitCondition->isSigned();
Instruction *PHTerm = L->getLoopPreheader()->getTerminator();
if (IVisLT(*ExitCondition)) {
if (IVisLT(*SplitCondition)) {
/* Do nothing */
}
else if (IVisLE(*SplitCondition)) {
AEV = getPlusOne(SplitValue, Sign, PHTerm, Context);
BSV = getPlusOne(SplitValue, Sign, PHTerm, Context);
} else {
assert (0 && "Unexpected split condition!");
}
}
else if (IVisLE(*ExitCondition)) {
if (IVisLT(*SplitCondition)) {
AEV = getMinusOne(SplitValue, Sign, PHTerm, Context);
}
else if (IVisLE(*SplitCondition)) {
BSV = getPlusOne(SplitValue, Sign, PHTerm, Context);
} else {
assert (0 && "Unexpected split condition!");
}
} else {
assert (0 && "Unexpected exit condition!");
}
AEV = getMin(AEV, IVExitValue, Sign, PHTerm);
BSV = getMax(BSV, IVStartValue, Sign, PHTerm);
// [*] Clone Loop
DenseMap<const Value *, Value *> ValueMap;
Loop *BLoop = CloneLoop(L, LPM, LI, ValueMap, this);
Loop *ALoop = L;
// [*] ALoop's exiting edge enters BLoop's header.
// ALoop's original exit block becomes BLoop's exit block.
PHINode *B_IndVar = cast<PHINode>(ValueMap[IndVar]);
BasicBlock *A_ExitingBlock = ExitCondition->getParent();
BranchInst *A_ExitInsn =
dyn_cast<BranchInst>(A_ExitingBlock->getTerminator());
assert (A_ExitInsn && "Unable to find suitable loop exit branch");
BasicBlock *B_ExitBlock = A_ExitInsn->getSuccessor(1);
BasicBlock *B_Header = BLoop->getHeader();
if (ALoop->contains(B_ExitBlock)) {
B_ExitBlock = A_ExitInsn->getSuccessor(0);
A_ExitInsn->setSuccessor(0, B_Header);
} else
A_ExitInsn->setSuccessor(1, B_Header);
// [*] Update ALoop's exit value using new exit value.
ExitCondition->setOperand(EVOpNum, AEV);
// [*] Update BLoop's header phi nodes. Remove incoming PHINode's from
// original loop's preheader. Add incoming PHINode values from
// ALoop's exiting block. Update BLoop header's domiantor info.
// Collect inverse map of Header PHINodes.
DenseMap<Value *, Value *> InverseMap;
for (BasicBlock::iterator BI = ALoop->getHeader()->begin(),
BE = ALoop->getHeader()->end(); BI != BE; ++BI) {
if (PHINode *PN = dyn_cast<PHINode>(BI)) {
PHINode *PNClone = cast<PHINode>(ValueMap[PN]);
InverseMap[PNClone] = PN;
} else
break;
}
BasicBlock *A_Preheader = ALoop->getLoopPreheader();
for (BasicBlock::iterator BI = B_Header->begin(), BE = B_Header->end();
BI != BE; ++BI) {
if (PHINode *PN = dyn_cast<PHINode>(BI)) {
// Remove incoming value from original preheader.
PN->removeIncomingValue(A_Preheader);
// Add incoming value from A_ExitingBlock.
if (PN == B_IndVar)
PN->addIncoming(BSV, A_ExitingBlock);
else {
PHINode *OrigPN = cast<PHINode>(InverseMap[PN]);
Value *V2 = NULL;
// If loop header is also loop exiting block then
// OrigPN is incoming value for B loop header.
if (A_ExitingBlock == ALoop->getHeader())
V2 = OrigPN;
else
V2 = OrigPN->getIncomingValueForBlock(A_ExitingBlock);
PN->addIncoming(V2, A_ExitingBlock);
}
} else
break;
}
DT->changeImmediateDominator(B_Header, A_ExitingBlock);
DF->changeImmediateDominator(B_Header, A_ExitingBlock, DT);
// [*] Update BLoop's exit block. Its new predecessor is BLoop's exit
// block. Remove incoming PHINode values from ALoop's exiting block.
// Add new incoming values from BLoop's incoming exiting value.
// Update BLoop exit block's dominator info..
BasicBlock *B_ExitingBlock = cast<BasicBlock>(ValueMap[A_ExitingBlock]);
for (BasicBlock::iterator BI = B_ExitBlock->begin(), BE = B_ExitBlock->end();
BI != BE; ++BI) {
if (PHINode *PN = dyn_cast<PHINode>(BI)) {
PN->addIncoming(ValueMap[PN->getIncomingValueForBlock(A_ExitingBlock)],
B_ExitingBlock);
PN->removeIncomingValue(A_ExitingBlock);
} else
break;
}
DT->changeImmediateDominator(B_ExitBlock, B_ExitingBlock);
DF->changeImmediateDominator(B_ExitBlock, B_ExitingBlock, DT);
//[*] Split ALoop's exit edge. This creates a new block which
// serves two purposes. First one is to hold PHINode defnitions
// to ensure that ALoop's LCSSA form. Second use it to act
// as a preheader for BLoop.
BasicBlock *A_ExitBlock = SplitEdge(A_ExitingBlock, B_Header, this);
//[*] Preserve ALoop's LCSSA form. Create new forwarding PHINodes
// in A_ExitBlock to redefine outgoing PHI definitions from ALoop.
for(BasicBlock::iterator BI = B_Header->begin(), BE = B_Header->end();
BI != BE; ++BI) {
if (PHINode *PN = dyn_cast<PHINode>(BI)) {
Value *V1 = PN->getIncomingValueForBlock(A_ExitBlock);
PHINode *newPHI = PHINode::Create(PN->getType(), PN->getName());
newPHI->addIncoming(V1, A_ExitingBlock);
A_ExitBlock->getInstList().push_front(newPHI);
PN->removeIncomingValue(A_ExitBlock);
PN->addIncoming(newPHI, A_ExitBlock);
} else
break;
}
//[*] Eliminate split condition's inactive branch from ALoop.
BasicBlock *A_SplitCondBlock = SplitCondition->getParent();
BranchInst *A_BR = cast<BranchInst>(A_SplitCondBlock->getTerminator());
BasicBlock *A_InactiveBranch = NULL;
BasicBlock *A_ActiveBranch = NULL;
A_ActiveBranch = A_BR->getSuccessor(0);
A_InactiveBranch = A_BR->getSuccessor(1);
A_BR->setUnconditionalDest(A_ActiveBranch);
removeBlocks(A_InactiveBranch, L, A_ActiveBranch);
//[*] Eliminate split condition's inactive branch in from BLoop.
BasicBlock *B_SplitCondBlock = cast<BasicBlock>(ValueMap[A_SplitCondBlock]);
BranchInst *B_BR = cast<BranchInst>(B_SplitCondBlock->getTerminator());
BasicBlock *B_InactiveBranch = NULL;
BasicBlock *B_ActiveBranch = NULL;
B_ActiveBranch = B_BR->getSuccessor(1);
B_InactiveBranch = B_BR->getSuccessor(0);
B_BR->setUnconditionalDest(B_ActiveBranch);
removeBlocks(B_InactiveBranch, BLoop, B_ActiveBranch);
BasicBlock *A_Header = ALoop->getHeader();
if (A_ExitingBlock == A_Header)
return true;
//[*] Move exit condition into split condition block to avoid
// executing dead loop iteration.
ICmpInst *B_ExitCondition = cast<ICmpInst>(ValueMap[ExitCondition]);
Instruction *B_IndVarIncrement = cast<Instruction>(ValueMap[IVIncrement]);
ICmpInst *B_SplitCondition = cast<ICmpInst>(ValueMap[SplitCondition]);
moveExitCondition(A_SplitCondBlock, A_ActiveBranch, A_ExitBlock, ExitCondition,
cast<ICmpInst>(SplitCondition), IndVar, IVIncrement,
ALoop, EVOpNum);
moveExitCondition(B_SplitCondBlock, B_ActiveBranch,
B_ExitBlock, B_ExitCondition,
B_SplitCondition, B_IndVar, B_IndVarIncrement,
BLoop, EVOpNum);
NumIndexSplit++;
return true;
}
/// updateLoopIterationSpace -- Update loop's iteration space if loop
/// body is executed for certain IV range only. For example,
///
/// for (i = 0; i < N; ++i) {
/// if ( i > A && i < B) {
/// ...
/// }
/// }
/// is transformed to iterators from A to B, if A > 0 and B < N.
///
bool LoopIndexSplit::updateLoopIterationSpace() {
SplitCondition = NULL;
if (ExitCondition->getPredicate() == ICmpInst::ICMP_NE
|| ExitCondition->getPredicate() == ICmpInst::ICMP_EQ)
return false;
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Header = L->getHeader();
BranchInst *BR = dyn_cast<BranchInst>(Header->getTerminator());
if (!BR) return false;
if (!isa<BranchInst>(Latch->getTerminator())) return false;
if (BR->isUnconditional()) return false;
BinaryOperator *AND = dyn_cast<BinaryOperator>(BR->getCondition());
if (!AND) return false;
if (AND->getOpcode() != Instruction::And) return false;
ICmpInst *Op0 = dyn_cast<ICmpInst>(AND->getOperand(0));
ICmpInst *Op1 = dyn_cast<ICmpInst>(AND->getOperand(1));
if (!Op0 || !Op1)
return false;
IVBasedValues.insert(AND);
IVBasedValues.insert(Op0);
IVBasedValues.insert(Op1);
if (!cleanBlock(Header)) return false;
BasicBlock *ExitingBlock = ExitCondition->getParent();
if (!cleanBlock(ExitingBlock)) return false;
// If the merge point for BR is not loop latch then skip this loop.
if (BR->getSuccessor(0) != Latch) {
DominanceFrontier::iterator DF0 = DF->find(BR->getSuccessor(0));
assert (DF0 != DF->end() && "Unable to find dominance frontier");
if (!DF0->second.count(Latch))
return false;
}
if (BR->getSuccessor(1) != Latch) {
DominanceFrontier::iterator DF1 = DF->find(BR->getSuccessor(1));
assert (DF1 != DF->end() && "Unable to find dominance frontier");
if (!DF1->second.count(Latch))
return false;
}
// Verify that loop exiting block has only two predecessor, where one pred
// is split condition block. The other predecessor will become exiting block's
// dominator after CFG is updated. TODO : Handle CFG's where exiting block has
// more then two predecessors. This requires extra work in updating dominator
// information.
BasicBlock *ExitingBBPred = NULL;
for (pred_iterator PI = pred_begin(ExitingBlock), PE = pred_end(ExitingBlock);
PI != PE; ++PI) {
BasicBlock *BB = *PI;
if (Header == BB)
continue;
if (ExitingBBPred)
return false;
else
ExitingBBPred = BB;
}
if (!restrictLoopBound(*Op0))
return false;
if (!restrictLoopBound(*Op1))
return false;
// Update CFG.
if (BR->getSuccessor(0) == ExitingBlock)
BR->setUnconditionalDest(BR->getSuccessor(1));
else
BR->setUnconditionalDest(BR->getSuccessor(0));
AND->eraseFromParent();
if (Op0->use_empty())
Op0->eraseFromParent();
if (Op1->use_empty())
Op1->eraseFromParent();
// Update domiantor info. Now, ExitingBlock has only one predecessor,
// ExitingBBPred, and it is ExitingBlock's immediate domiantor.
DT->changeImmediateDominator(ExitingBlock, ExitingBBPred);
BasicBlock *ExitBlock = ExitingBlock->getTerminator()->getSuccessor(1);
if (L->contains(ExitBlock))
ExitBlock = ExitingBlock->getTerminator()->getSuccessor(0);
// If ExitingBlock is a member of the loop basic blocks' DF list then
// replace ExitingBlock with header and exit block in the DF list
DominanceFrontier::iterator ExitingBlockDF = DF->find(ExitingBlock);
for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
I != E; ++I) {
BasicBlock *BB = *I;
if (BB == Header || BB == ExitingBlock)
continue;
DominanceFrontier::iterator BBDF = DF->find(BB);
DominanceFrontier::DomSetType::iterator DomSetI = BBDF->second.begin();
DominanceFrontier::DomSetType::iterator DomSetE = BBDF->second.end();
while (DomSetI != DomSetE) {
DominanceFrontier::DomSetType::iterator CurrentItr = DomSetI;
++DomSetI;
BasicBlock *DFBB = *CurrentItr;
if (DFBB == ExitingBlock) {
BBDF->second.erase(DFBB);
for (DominanceFrontier::DomSetType::iterator
EBI = ExitingBlockDF->second.begin(),
EBE = ExitingBlockDF->second.end(); EBI != EBE; ++EBI)
BBDF->second.insert(*EBI);
}
}
}
NumRestrictBounds++;
return true;
}
/// processOneIterationLoop -- Eliminate loop if loop body is executed
/// only once. For example,
/// for (i = 0; i < N; ++i) {
/// if ( i == X) {
/// ...
/// }
/// }
///
bool LoopIndexSplit::processOneIterationLoop() {
SplitCondition = NULL;
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Header = L->getHeader();
BranchInst *BR = dyn_cast<BranchInst>(Header->getTerminator());
if (!BR) return false;
if (!isa<BranchInst>(Latch->getTerminator())) return false;
if (BR->isUnconditional()) return false;
SplitCondition = dyn_cast<ICmpInst>(BR->getCondition());
if (!SplitCondition) return false;
if (SplitCondition == ExitCondition) return false;
if (SplitCondition->getPredicate() != ICmpInst::ICMP_EQ) return false;
if (BR->getOperand(1) != Latch) return false;
if (!IVBasedValues.count(SplitCondition->getOperand(0))
&& !IVBasedValues.count(SplitCondition->getOperand(1)))
return false;
// If IV is used outside the loop then this loop traversal is required.
// FIXME: Calculate and use last IV value.
if (isUsedOutsideLoop(IVIncrement, L))
return false;
// If BR operands are not IV or not loop invariants then skip this loop.
Value *OPV = SplitCondition->getOperand(0);
Value *SplitValue = SplitCondition->getOperand(1);
if (!L->isLoopInvariant(SplitValue))
std::swap(OPV, SplitValue);
if (!L->isLoopInvariant(SplitValue))
return false;
Instruction *OPI = dyn_cast<Instruction>(OPV);
if (!OPI)
return false;
if (OPI->getParent() != Header || isUsedOutsideLoop(OPI, L))
return false;
Value *StartValue = IVStartValue;
Value *ExitValue = IVExitValue;;
if (OPV != IndVar) {
// If BR operand is IV based then use this operand to calculate
// effective conditions for loop body.
BinaryOperator *BOPV = dyn_cast<BinaryOperator>(OPV);
if (!BOPV)
return false;
if (BOPV->getOpcode() != Instruction::Add)
return false;
StartValue = BinaryOperator::CreateAdd(OPV, StartValue, "" , BR);
ExitValue = BinaryOperator::CreateAdd(OPV, ExitValue, "" , BR);
}
if (!cleanBlock(Header))
return false;
if (!cleanBlock(Latch))
return false;
// If the merge point for BR is not loop latch then skip this loop.
if (BR->getSuccessor(0) != Latch) {
DominanceFrontier::iterator DF0 = DF->find(BR->getSuccessor(0));
assert (DF0 != DF->end() && "Unable to find dominance frontier");
if (!DF0->second.count(Latch))
return false;
}
if (BR->getSuccessor(1) != Latch) {
DominanceFrontier::iterator DF1 = DF->find(BR->getSuccessor(1));
assert (DF1 != DF->end() && "Unable to find dominance frontier");
if (!DF1->second.count(Latch))
return false;
}
// Now, Current loop L contains compare instruction
// that compares induction variable, IndVar, against loop invariant. And
// entire (i.e. meaningful) loop body is dominated by this compare
// instruction. In such case eliminate
// loop structure surrounding this loop body. For example,
// for (int i = start; i < end; ++i) {
// if ( i == somevalue) {
// loop_body
// }
// }
// can be transformed into
// if (somevalue >= start && somevalue < end) {
// i = somevalue;
// loop_body
// }
// Replace index variable with split value in loop body. Loop body is executed
// only when index variable is equal to split value.
IndVar->replaceAllUsesWith(SplitValue);
// Replace split condition in header.
// Transform
// SplitCondition : icmp eq i32 IndVar, SplitValue
// into
// c1 = icmp uge i32 SplitValue, StartValue
// c2 = icmp ult i32 SplitValue, ExitValue
// and i32 c1, c2
Instruction *C1 = new ICmpInst(BR, ExitCondition->isSigned() ?
ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE,
SplitValue, StartValue, "lisplit");
CmpInst::Predicate C2P = ExitCondition->getPredicate();
BranchInst *LatchBR = cast<BranchInst>(Latch->getTerminator());
if (LatchBR->getOperand(1) != Header)
C2P = CmpInst::getInversePredicate(C2P);
Instruction *C2 = new ICmpInst(BR, C2P, SplitValue, ExitValue, "lisplit");
Instruction *NSplitCond = BinaryOperator::CreateAnd(C1, C2, "lisplit", BR);
SplitCondition->replaceAllUsesWith(NSplitCond);
SplitCondition->eraseFromParent();
// Remove Latch to Header edge.
BasicBlock *LatchSucc = NULL;
Header->removePredecessor(Latch);
for (succ_iterator SI = succ_begin(Latch), E = succ_end(Latch);
SI != E; ++SI) {
if (Header != *SI)
LatchSucc = *SI;
}
// Clean up latch block.
Value *LatchBRCond = LatchBR->getCondition();
LatchBR->setUnconditionalDest(LatchSucc);
RecursivelyDeleteTriviallyDeadInstructions(LatchBRCond);
LPM->deleteLoopFromQueue(L);
// Update Dominator Info.
// Only CFG change done is to remove Latch to Header edge. This
// does not change dominator tree because Latch did not dominate
// Header.
if (DF) {
DominanceFrontier::iterator HeaderDF = DF->find(Header);
if (HeaderDF != DF->end())
DF->removeFromFrontier(HeaderDF, Header);
DominanceFrontier::iterator LatchDF = DF->find(Latch);
if (LatchDF != DF->end())
DF->removeFromFrontier(LatchDF, Header);
}
++NumIndexSplitRemoved;
return true;
}
/// UnswitchNontrivialCondition - We determined that the loop is profitable
/// to unswitch when LIC equal Val. Split it into loop versions and test the
/// condition outside of either loop. Return the loops created as Out1/Out2.
void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
Loop *L) {
Function *F = loopHeader->getParent();
DEBUG(dbgs() << "loop-unswitch: Unswitching loop %"
<< loopHeader->getName() << " [" << L->getBlocks().size()
<< " blocks] in Function " << F->getName()
<< " when '" << *Val << "' == " << *LIC << "\n");
if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>())
SE->forgetLoop(L);
LoopBlocks.clear();
NewBlocks.clear();
// First step, split the preheader and exit blocks, and add these blocks to
// the LoopBlocks list.
BasicBlock *NewPreheader = SplitEdge(loopPreheader, loopHeader, this);
LoopBlocks.push_back(NewPreheader);
// We want the loop to come after the preheader, but before the exit blocks.
LoopBlocks.insert(LoopBlocks.end(), L->block_begin(), L->block_end());
SmallVector<BasicBlock*, 8> ExitBlocks;
L->getUniqueExitBlocks(ExitBlocks);
// Split all of the edges from inside the loop to their exit blocks. Update
// the appropriate Phi nodes as we do so.
SplitExitEdges(L, ExitBlocks);
// The exit blocks may have been changed due to edge splitting, recompute.
ExitBlocks.clear();
L->getUniqueExitBlocks(ExitBlocks);
// Add exit blocks to the loop blocks.
LoopBlocks.insert(LoopBlocks.end(), ExitBlocks.begin(), ExitBlocks.end());
// Next step, clone all of the basic blocks that make up the loop (including
// the loop preheader and exit blocks), keeping track of the mapping between
// the instructions and blocks.
NewBlocks.reserve(LoopBlocks.size());
ValueToValueMapTy VMap;
for (unsigned i = 0, e = LoopBlocks.size(); i != e; ++i) {
BasicBlock *NewBB = CloneBasicBlock(LoopBlocks[i], VMap, ".us", F);
NewBlocks.push_back(NewBB);
VMap[LoopBlocks[i]] = NewBB; // Keep the BB mapping.
LPM->cloneBasicBlockSimpleAnalysis(LoopBlocks[i], NewBB, L);
}
// Splice the newly inserted blocks into the function right before the
// original preheader.
F->getBasicBlockList().splice(NewPreheader, F->getBasicBlockList(),
NewBlocks[0], F->end());
// Now we create the new Loop object for the versioned loop.
Loop *NewLoop = CloneLoop(L, L->getParentLoop(), VMap, LI, LPM);
Loop *ParentLoop = L->getParentLoop();
if (ParentLoop) {
// Make sure to add the cloned preheader and exit blocks to the parent loop
// as well.
ParentLoop->addBasicBlockToLoop(NewBlocks[0], LI->getBase());
}
for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
BasicBlock *NewExit = cast<BasicBlock>(VMap[ExitBlocks[i]]);
// The new exit block should be in the same loop as the old one.
if (Loop *ExitBBLoop = LI->getLoopFor(ExitBlocks[i]))
ExitBBLoop->addBasicBlockToLoop(NewExit, LI->getBase());
assert(NewExit->getTerminator()->getNumSuccessors() == 1 &&
"Exit block should have been split to have one successor!");
BasicBlock *ExitSucc = NewExit->getTerminator()->getSuccessor(0);
// If the successor of the exit block had PHI nodes, add an entry for
// NewExit.
PHINode *PN;
for (BasicBlock::iterator I = ExitSucc->begin(); isa<PHINode>(I); ++I) {
PN = cast<PHINode>(I);
Value *V = PN->getIncomingValueForBlock(ExitBlocks[i]);
ValueToValueMapTy::iterator It = VMap.find(V);
if (It != VMap.end()) V = It->second;
PN->addIncoming(V, NewExit);
}
}
// Rewrite the code to refer to itself.
for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i)
for (BasicBlock::iterator I = NewBlocks[i]->begin(),
E = NewBlocks[i]->end(); I != E; ++I)
RemapInstruction(I, VMap,RF_NoModuleLevelChanges|RF_IgnoreMissingEntries);
// Rewrite the original preheader to select between versions of the loop.
BranchInst *OldBR = cast<BranchInst>(loopPreheader->getTerminator());
assert(OldBR->isUnconditional() && OldBR->getSuccessor(0) == LoopBlocks[0] &&
"Preheader splitting did not work correctly!");
// Emit the new branch that selects between the two versions of this loop.
EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR);
LPM->deleteSimpleAnalysisValue(OldBR, L);
OldBR->eraseFromParent();
LoopProcessWorklist.push_back(NewLoop);
redoLoop = true;
// Keep a WeakVH holding onto LIC. If the first call to RewriteLoopBody
// deletes the instruction (for example by simplifying a PHI that feeds into
// the condition that we're unswitching on), we don't rewrite the second
// iteration.
WeakVH LICHandle(LIC);
// Now we rewrite the original code to know that the condition is true and the
// new code to know that the condition is false.
RewriteLoopBodyWithConditionConstant(L, LIC, Val, false);
// It's possible that simplifying one loop could cause the other to be
// changed to another value or a constant. If its a constant, don't simplify
// it.
if (!LoopProcessWorklist.empty() && LoopProcessWorklist.back() == NewLoop &&
LICHandle && !isa<Constant>(LICHandle))
RewriteLoopBodyWithConditionConstant(NewLoop, LICHandle, Val, true);
}
/// DupRetToEnableTailCallOpts - Look for opportunities to duplicate return
/// instructions to the predecessor to enable tail call optimizations. The
/// case it is currently looking for is:
/// bb0:
/// %tmp0 = tail call i32 @f0()
/// br label %return
/// bb1:
/// %tmp1 = tail call i32 @f1()
/// br label %return
/// bb2:
/// %tmp2 = tail call i32 @f2()
/// br label %return
/// return:
/// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
/// ret i32 %retval
///
/// =>
///
/// bb0:
/// %tmp0 = tail call i32 @f0()
/// ret i32 %tmp0
/// bb1:
/// %tmp1 = tail call i32 @f1()
/// ret i32 %tmp1
/// bb2:
/// %tmp2 = tail call i32 @f2()
/// ret i32 %tmp2
///
bool CodeGenPrepare::DupRetToEnableTailCallOpts(ReturnInst *RI) {
if (!TLI)
return false;
Value *V = RI->getReturnValue();
PHINode *PN = V ? dyn_cast<PHINode>(V) : NULL;
if (V && !PN)
return false;
BasicBlock *BB = RI->getParent();
if (PN && PN->getParent() != BB)
return false;
// It's not safe to eliminate the sign / zero extension of the return value.
// See llvm::isInTailCallPosition().
const Function *F = BB->getParent();
Attributes CallerRetAttr = F->getAttributes().getRetAttributes();
if ((CallerRetAttr & Attribute::ZExt) || (CallerRetAttr & Attribute::SExt))
return false;
// Make sure there are no instructions between the PHI and return, or that the
// return is the first instruction in the block.
if (PN) {
BasicBlock::iterator BI = BB->begin();
do {
++BI;
}
while (isa<DbgInfoIntrinsic>(BI));
if (&*BI != RI)
return false;
} else {
BasicBlock::iterator BI = BB->begin();
while (isa<DbgInfoIntrinsic>(BI)) ++BI;
if (&*BI != RI)
return false;
}
/// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
/// call.
SmallVector<CallInst*, 4> TailCalls;
if (PN) {
for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) {
CallInst *CI = dyn_cast<CallInst>(PN->getIncomingValue(I));
// Make sure the phi value is indeed produced by the tail call.
if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) &&
TLI->mayBeEmittedAsTailCall(CI))
TailCalls.push_back(CI);
}
} else {
SmallPtrSet<BasicBlock*, 4> VisitedBBs;
for (pred_iterator PI = pred_begin(BB), PE = pred_end(BB); PI != PE; ++PI) {
if (!VisitedBBs.insert(*PI))
continue;
BasicBlock::InstListType &InstList = (*PI)->getInstList();
BasicBlock::InstListType::reverse_iterator RI = InstList.rbegin();
BasicBlock::InstListType::reverse_iterator RE = InstList.rend();
do {
++RI;
}
while (RI != RE && isa<DbgInfoIntrinsic>(&*RI));
if (RI == RE)
continue;
CallInst *CI = dyn_cast<CallInst>(&*RI);
if (CI && CI->use_empty() && TLI->mayBeEmittedAsTailCall(CI))
TailCalls.push_back(CI);
}
}
bool Changed = false;
for (unsigned i = 0, e = TailCalls.size(); i != e; ++i) {
CallInst *CI = TailCalls[i];
CallSite CS(CI);
// Conservatively require the attributes of the call to match those of the
// return. Ignore noalias because it doesn't affect the call sequence.
Attributes CalleeRetAttr = CS.getAttributes().getRetAttributes();
if ((CalleeRetAttr ^ CallerRetAttr) & ~Attribute::NoAlias)
continue;
// Make sure the call instruction is followed by an unconditional branch to
// the return block.
BasicBlock *CallBB = CI->getParent();
BranchInst *BI = dyn_cast<BranchInst>(CallBB->getTerminator());
if (!BI || !BI->isUnconditional() || BI->getSuccessor(0) != BB)
continue;
// Duplicate the return into CallBB.
(void)FoldReturnIntoUncondBranch(RI, BB, CallBB);
ModifiedDT = Changed = true;
++NumRetsDup;
}
// If we eliminated all predecessors of the block, delete the block now.
if (Changed && pred_begin(BB) == pred_end(BB))
BB->eraseFromParent();
return Changed;
}
/// shouldEliminateUnconditionalBranch - Return true if this branch looks
/// attractive to eliminate. We eliminate the branch if the destination basic
/// block has <= 5 instructions in it, not counting PHI nodes. In practice,
/// since one of these is a terminator instruction, this means that we will add
/// up to 4 instructions to the new block.
///
/// We don't count PHI nodes in the count since they will be removed when the
/// contents of the block are copied over.
///
bool TailDup::shouldEliminateUnconditionalBranch(TerminatorInst *TI,
unsigned Threshold) {
BranchInst *BI = dyn_cast<BranchInst>(TI);
if (!BI || !BI->isUnconditional()) return false; // Not an uncond branch!
BasicBlock *Dest = BI->getSuccessor(0);
if (Dest == BI->getParent()) return false; // Do not loop infinitely!
// Do not inline a block if we will just get another branch to the same block!
TerminatorInst *DTI = Dest->getTerminator();
if (BranchInst *DBI = dyn_cast<BranchInst>(DTI))
if (DBI->isUnconditional() && DBI->getSuccessor(0) == Dest)
return false; // Do not loop infinitely!
// FIXME: DemoteRegToStack cannot yet demote invoke instructions to the stack,
// because doing so would require breaking critical edges. This should be
// fixed eventually.
if (!DTI->use_empty())
return false;
// Do not bother with blocks with only a single predecessor: simplify
// CFG will fold these two blocks together!
pred_iterator PI = pred_begin(Dest), PE = pred_end(Dest);
++PI;
if (PI == PE) return false; // Exactly one predecessor!
BasicBlock::iterator I = Dest->getFirstNonPHI();
for (unsigned Size = 0; I != Dest->end(); ++I) {
if (Size == Threshold) return false; // The block is too large.
// Don't tail duplicate call instructions. They are very large compared to
// other instructions.
if (isa<CallInst>(I) || isa<InvokeInst>(I)) return false;
// Also alloca and malloc.
if (isa<AllocaInst>(I)) return false;
// Some vector instructions can expand into a number of instructions.
if (isa<ShuffleVectorInst>(I) || isa<ExtractElementInst>(I) ||
isa<InsertElementInst>(I)) return false;
// Only count instructions that are not debugger intrinsics.
if (!isa<DbgInfoIntrinsic>(I)) ++Size;
}
// Do not tail duplicate a block that has thousands of successors into a block
// with a single successor if the block has many other predecessors. This can
// cause an N^2 explosion in CFG edges (and PHI node entries), as seen in
// cases that have a large number of indirect gotos.
unsigned NumSuccs = DTI->getNumSuccessors();
if (NumSuccs > 8) {
unsigned TooMany = 128;
if (NumSuccs >= TooMany) return false;
TooMany = TooMany/NumSuccs;
for (; PI != PE; ++PI)
if (TooMany-- == 0) return false;
}
// If this unconditional branch is a fall-through, be careful about
// tail duplicating it. In particular, we don't want to taildup it if the
// original block will still be there after taildup is completed: doing so
// would eliminate the fall-through, requiring unconditional branches.
Function::iterator DestI = Dest;
if (&*--DestI == BI->getParent()) {
// The uncond branch is a fall-through. Tail duplication of the block is
// will eliminate the fall-through-ness and end up cloning the terminator
// at the end of the Dest block. Since the original Dest block will
// continue to exist, this means that one or the other will not be able to
// fall through. One typical example that this helps with is code like:
// if (a)
// foo();
// if (b)
// foo();
// Cloning the 'if b' block into the end of the first foo block is messy.
// The messy case is when the fall-through block falls through to other
// blocks. This is what we would be preventing if we cloned the block.
DestI = Dest;
if (++DestI != Dest->getParent()->end()) {
BasicBlock *DestSucc = DestI;
// If any of Dest's successors are fall-throughs, don't do this xform.
for (succ_iterator SI = succ_begin(Dest), SE = succ_end(Dest);
SI != SE; ++SI)
if (*SI == DestSucc)
return false;
}
}
// Finally, check that we haven't redirected to this target block earlier;
// there are cases where we loop forever if we don't check this (PR 2323).
if (!CycleDetector.insert(Dest))
return false;
return true;
}
bool
FastISel::SelectOperator(User *I, unsigned Opcode) {
switch (Opcode) {
case Instruction::Add:
return SelectBinaryOp(I, ISD::ADD);
case Instruction::FAdd:
return SelectBinaryOp(I, ISD::FADD);
case Instruction::Sub:
return SelectBinaryOp(I, ISD::SUB);
case Instruction::FSub:
// FNeg is currently represented in LLVM IR as a special case of FSub.
if (BinaryOperator::isFNeg(I))
return SelectFNeg(I);
return SelectBinaryOp(I, ISD::FSUB);
case Instruction::Mul:
return SelectBinaryOp(I, ISD::MUL);
case Instruction::FMul:
return SelectBinaryOp(I, ISD::FMUL);
case Instruction::SDiv:
return SelectBinaryOp(I, ISD::SDIV);
case Instruction::UDiv:
return SelectBinaryOp(I, ISD::UDIV);
case Instruction::FDiv:
return SelectBinaryOp(I, ISD::FDIV);
case Instruction::SRem:
return SelectBinaryOp(I, ISD::SREM);
case Instruction::URem:
return SelectBinaryOp(I, ISD::UREM);
case Instruction::FRem:
return SelectBinaryOp(I, ISD::FREM);
case Instruction::Shl:
return SelectBinaryOp(I, ISD::SHL);
case Instruction::LShr:
return SelectBinaryOp(I, ISD::SRL);
case Instruction::AShr:
return SelectBinaryOp(I, ISD::SRA);
case Instruction::And:
return SelectBinaryOp(I, ISD::AND);
case Instruction::Or:
return SelectBinaryOp(I, ISD::OR);
case Instruction::Xor:
return SelectBinaryOp(I, ISD::XOR);
case Instruction::GetElementPtr:
return SelectGetElementPtr(I);
case Instruction::Br: {
BranchInst *BI = cast<BranchInst>(I);
if (BI->isUnconditional()) {
BasicBlock *LLVMSucc = BI->getSuccessor(0);
MachineBasicBlock *MSucc = MBBMap[LLVMSucc];
FastEmitBranch(MSucc);
return true;
}
// Conditional branches are not handed yet.
// Halt "fast" selection and bail.
return false;
}
case Instruction::Unreachable:
// Nothing to emit.
return true;
case Instruction::PHI:
// PHI nodes are already emitted.
return true;
case Instruction::Alloca:
// FunctionLowering has the static-sized case covered.
if (StaticAllocaMap.count(cast<AllocaInst>(I)))
return true;
// Dynamic-sized alloca is not handled yet.
return false;
case Instruction::Call:
return SelectCall(I);
case Instruction::BitCast:
return SelectBitCast(I);
case Instruction::FPToSI:
return SelectCast(I, ISD::FP_TO_SINT);
case Instruction::ZExt:
return SelectCast(I, ISD::ZERO_EXTEND);
case Instruction::SExt:
return SelectCast(I, ISD::SIGN_EXTEND);
case Instruction::Trunc:
return SelectCast(I, ISD::TRUNCATE);
case Instruction::SIToFP:
return SelectCast(I, ISD::SINT_TO_FP);
case Instruction::IntToPtr: // Deliberate fall-through.
case Instruction::PtrToInt: {
EVT SrcVT = TLI.getValueType(I->getOperand(0)->getType());
EVT DstVT = TLI.getValueType(I->getType());
if (DstVT.bitsGT(SrcVT))
return SelectCast(I, ISD::ZERO_EXTEND);
if (DstVT.bitsLT(SrcVT))
return SelectCast(I, ISD::TRUNCATE);
unsigned Reg = getRegForValue(I->getOperand(0));
if (Reg == 0) return false;
UpdateValueMap(I, Reg);
return true;
}
default:
// Unhandled instruction. Halt "fast" selection and bail.
return false;
}
}
