// Tries to remove a sanity check; returns true if it worked.
bool AsapPass::optimizeCheckAway(llvm::Instruction *Inst) {
BranchInst *BI = cast<BranchInst>(Inst);
assert(BI->isConditional() && "Sanity check must be conditional branch.");
unsigned int RegularBranch = getRegularBranch(BI, SCI);
bool Changed = false;
if (RegularBranch == 0) {
BI->setCondition(ConstantInt::getTrue(Inst->getContext()));
Changed = true;
} else if (RegularBranch == 1) {
BI->setCondition(ConstantInt::getFalse(Inst->getContext()));
Changed = true;
} else {
// This can happen, e.g., in the following case:
// array[-1] = a + b;
// is transformed into
// if (a + b overflows)
// report_overflow()
// else
// report_index_out_of_bounds();
// In this case, removing the sanity check does not help much, so we
// just do nothing.
// Thanks to Will Dietz for his explanation at
// http://lists.cs.uiuc.edu/pipermail/llvmdev/2014-April/071958.html
dbgs() << "Warning: Sanity check with no regular branch found.\n";
dbgs() << "The sanity check has been kept intact.\n";
}
if (PrintRemovedChecks && Changed) {
DebugLoc DL = getSanityCheckDebugLoc(BI, RegularBranch);
printDebugLoc(DL, BI->getContext(), dbgs());
dbgs() << ": SanityCheck with cost ";
dbgs() << *BI->getMetadata("cost")->getOperand(0);
if (MDNode *IA = DL.getInlinedAt()) {
dbgs() << " (inlined at ";
printDebugLoc(DebugLoc(IA), BI->getContext(), dbgs());
dbgs() << ")";
}
BasicBlock *Succ = BI->getSuccessor(RegularBranch == 0 ? 1 : 0);
if (const CallInst *CI = SCI->findSanityCheckCall(Succ)) {
dbgs() << " " << CI->getCalledFunction()->getName();
}
dbgs() << "\n";
}
return Changed;
}
/// \brief Peel off the first \p PeelCount iterations of loop \p L.
///
/// Note that this does not peel them off as a single straight-line block.
/// Rather, each iteration is peeled off separately, and needs to check the
/// exit condition.
/// For loops that dynamically execute \p PeelCount iterations or less
/// this provides a benefit, since the peeled off iterations, which account
/// for the bulk of dynamic execution, can be further simplified by scalar
/// optimizations.
bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
ScalarEvolution *SE, DominatorTree *DT,
bool PreserveLCSSA) {
if (!canPeel(L))
return false;
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
BasicBlock *Header = L->getHeader();
BasicBlock *PreHeader = L->getLoopPreheader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Exit = L->getUniqueExitBlock();
Function *F = Header->getParent();
// Set up all the necessary basic blocks. It is convenient to split the
// preheader into 3 parts - two blocks to anchor the peeled copy of the loop
// body, and a new preheader for the "real" loop.
// Peeling the first iteration transforms.
//
// PreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
//
// into
//
// InsertTop:
// LoopBody
// If (!cond) goto Exit
// InsertBot:
// NewPreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
//
// Each following iteration will split the current bottom anchor in two,
// and put the new copy of the loop body between these two blocks. That is,
// after peeling another iteration from the example above, we'll split
// InsertBot, and get:
//
// InsertTop:
// LoopBody
// If (!cond) goto Exit
// InsertBot:
// LoopBody
// If (!cond) goto Exit
// InsertBot.next:
// NewPreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
BasicBlock *InsertTop = SplitEdge(PreHeader, Header, DT, LI);
BasicBlock *InsertBot =
SplitBlock(InsertTop, InsertTop->getTerminator(), DT, LI);
BasicBlock *NewPreHeader =
SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI);
InsertTop->setName(Header->getName() + ".peel.begin");
InsertBot->setName(Header->getName() + ".peel.next");
NewPreHeader->setName(PreHeader->getName() + ".peel.newph");
ValueToValueMapTy LVMap;
// If we have branch weight information, we'll want to update it for the
// newly created branches.
BranchInst *LatchBR =
cast<BranchInst>(cast<BasicBlock>(Latch)->getTerminator());
unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1);
uint64_t TrueWeight, FalseWeight;
uint64_t ExitWeight = 0, BackEdgeWeight = 0;
if (LatchBR->extractProfMetadata(TrueWeight, FalseWeight)) {
ExitWeight = HeaderIdx ? TrueWeight : FalseWeight;
BackEdgeWeight = HeaderIdx ? FalseWeight : TrueWeight;
}
// For each peeled-off iteration, make a copy of the loop.
for (unsigned Iter = 0; Iter < PeelCount; ++Iter) {
SmallVector<BasicBlock *, 8> NewBlocks;
ValueToValueMapTy VMap;
// The exit weight of the previous iteration is the header entry weight
// of the current iteration. So this is exactly how many dynamic iterations
// the current peeled-off static iteration uses up.
// FIXME: due to the way the distribution is constructed, we need a
// guard here to make sure we don't end up with non-positive weights.
if (ExitWeight < BackEdgeWeight)
BackEdgeWeight -= ExitWeight;
else
BackEdgeWeight = 1;
cloneLoopBlocks(L, Iter, InsertTop, InsertBot, Exit,
NewBlocks, LoopBlocks, VMap, LVMap, LI);
updateBranchWeights(InsertBot, cast<BranchInst>(VMap[LatchBR]), Iter,
PeelCount, ExitWeight);
InsertTop = InsertBot;
InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI);
InsertBot->setName(Header->getName() + ".peel.next");
F->getBasicBlockList().splice(InsertTop->getIterator(),
F->getBasicBlockList(),
NewBlocks[0]->getIterator(), F->end());
// Remap to use values from the current iteration instead of the
// previous one.
remapInstructionsInBlocks(NewBlocks, VMap);
}
// Now adjust the phi nodes in the loop header to get their initial values
// from the last peeled-off iteration instead of the preheader.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *PHI = cast<PHINode>(I);
Value *NewVal = PHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(NewVal);
if (LatchInst && L->contains(LatchInst))
NewVal = LVMap[LatchInst];
PHI->setIncomingValue(PHI->getBasicBlockIndex(NewPreHeader), NewVal);
}
// Adjust the branch weights on the loop exit.
if (ExitWeight) {
MDBuilder MDB(LatchBR->getContext());
MDNode *WeightNode =
HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight)
: MDB.createBranchWeights(BackEdgeWeight, ExitWeight);
LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode);
}
// If the loop is nested, we changed the parent loop, update SE.
if (Loop *ParentLoop = L->getParentLoop())
SE->forgetLoop(ParentLoop);
NumPeeled++;
return true;
}
/// Peel off the first \p PeelCount iterations of loop \p L.
///
/// Note that this does not peel them off as a single straight-line block.
/// Rather, each iteration is peeled off separately, and needs to check the
/// exit condition.
/// For loops that dynamically execute \p PeelCount iterations or less
/// this provides a benefit, since the peeled off iterations, which account
/// for the bulk of dynamic execution, can be further simplified by scalar
/// optimizations.
bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
ScalarEvolution *SE, DominatorTree *DT,
AssumptionCache *AC, bool PreserveLCSSA) {
assert(PeelCount > 0 && "Attempt to peel out zero iterations?");
assert(canPeel(L) && "Attempt to peel a loop which is not peelable?");
LoopBlocksDFS LoopBlocks(L);
LoopBlocks.perform(LI);
BasicBlock *Header = L->getHeader();
BasicBlock *PreHeader = L->getLoopPreheader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Exit = L->getUniqueExitBlock();
Function *F = Header->getParent();
// Set up all the necessary basic blocks. It is convenient to split the
// preheader into 3 parts - two blocks to anchor the peeled copy of the loop
// body, and a new preheader for the "real" loop.
// Peeling the first iteration transforms.
//
// PreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
//
// into
//
// InsertTop:
// LoopBody
// If (!cond) goto Exit
// InsertBot:
// NewPreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
//
// Each following iteration will split the current bottom anchor in two,
// and put the new copy of the loop body between these two blocks. That is,
// after peeling another iteration from the example above, we'll split
// InsertBot, and get:
//
// InsertTop:
// LoopBody
// If (!cond) goto Exit
// InsertBot:
// LoopBody
// If (!cond) goto Exit
// InsertBot.next:
// NewPreHeader:
// ...
// Header:
// LoopBody
// If (cond) goto Header
// Exit:
BasicBlock *InsertTop = SplitEdge(PreHeader, Header, DT, LI);
BasicBlock *InsertBot =
SplitBlock(InsertTop, InsertTop->getTerminator(), DT, LI);
BasicBlock *NewPreHeader =
SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI);
InsertTop->setName(Header->getName() + ".peel.begin");
InsertBot->setName(Header->getName() + ".peel.next");
NewPreHeader->setName(PreHeader->getName() + ".peel.newph");
ValueToValueMapTy LVMap;
// If we have branch weight information, we'll want to update it for the
// newly created branches.
BranchInst *LatchBR =
cast<BranchInst>(cast<BasicBlock>(Latch)->getTerminator());
unsigned HeaderIdx = (LatchBR->getSuccessor(0) == Header ? 0 : 1);
uint64_t TrueWeight, FalseWeight;
uint64_t ExitWeight = 0, CurHeaderWeight = 0;
if (LatchBR->extractProfMetadata(TrueWeight, FalseWeight)) {
ExitWeight = HeaderIdx ? TrueWeight : FalseWeight;
// The # of times the loop body executes is the sum of the exit block
// weight and the # of times the backedges are taken.
CurHeaderWeight = TrueWeight + FalseWeight;
}
// For each peeled-off iteration, make a copy of the loop.
for (unsigned Iter = 0; Iter < PeelCount; ++Iter) {
SmallVector<BasicBlock *, 8> NewBlocks;
ValueToValueMapTy VMap;
// Subtract the exit weight from the current header weight -- the exit
// weight is exactly the weight of the previous iteration's header.
// FIXME: due to the way the distribution is constructed, we need a
// guard here to make sure we don't end up with non-positive weights.
if (ExitWeight < CurHeaderWeight)
CurHeaderWeight -= ExitWeight;
else
CurHeaderWeight = 1;
cloneLoopBlocks(L, Iter, InsertTop, InsertBot, Exit,
NewBlocks, LoopBlocks, VMap, LVMap, DT, LI);
// Remap to use values from the current iteration instead of the
// previous one.
remapInstructionsInBlocks(NewBlocks, VMap);
if (DT) {
// Latches of the cloned loops dominate over the loop exit, so idom of the
// latter is the first cloned loop body, as original PreHeader dominates
// the original loop body.
if (Iter == 0)
DT->changeImmediateDominator(Exit, cast<BasicBlock>(LVMap[Latch]));
#ifdef EXPENSIVE_CHECKS
assert(DT->verify(DominatorTree::VerificationLevel::Fast));
#endif
}
auto *LatchBRCopy = cast<BranchInst>(VMap[LatchBR]);
updateBranchWeights(InsertBot, LatchBRCopy, Iter,
PeelCount, ExitWeight);
// Remove Loop metadata from the latch branch instruction
// because it is not the Loop's latch branch anymore.
LatchBRCopy->setMetadata(LLVMContext::MD_loop, nullptr);
InsertTop = InsertBot;
InsertBot = SplitBlock(InsertBot, InsertBot->getTerminator(), DT, LI);
InsertBot->setName(Header->getName() + ".peel.next");
F->getBasicBlockList().splice(InsertTop->getIterator(),
F->getBasicBlockList(),
NewBlocks[0]->getIterator(), F->end());
}
// Now adjust the phi nodes in the loop header to get their initial values
// from the last peeled-off iteration instead of the preheader.
for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
PHINode *PHI = cast<PHINode>(I);
Value *NewVal = PHI->getIncomingValueForBlock(Latch);
Instruction *LatchInst = dyn_cast<Instruction>(NewVal);
if (LatchInst && L->contains(LatchInst))
NewVal = LVMap[LatchInst];
PHI->setIncomingValue(PHI->getBasicBlockIndex(NewPreHeader), NewVal);
}
// Adjust the branch weights on the loop exit.
if (ExitWeight) {
// The backedge count is the difference of current header weight and
// current loop exit weight. If the current header weight is smaller than
// the current loop exit weight, we mark the loop backedge weight as 1.
uint64_t BackEdgeWeight = 0;
if (ExitWeight < CurHeaderWeight)
BackEdgeWeight = CurHeaderWeight - ExitWeight;
else
BackEdgeWeight = 1;
MDBuilder MDB(LatchBR->getContext());
MDNode *WeightNode =
HeaderIdx ? MDB.createBranchWeights(ExitWeight, BackEdgeWeight)
: MDB.createBranchWeights(BackEdgeWeight, ExitWeight);
LatchBR->setMetadata(LLVMContext::MD_prof, WeightNode);
}
if (Loop *ParentLoop = L->getParentLoop())
L = ParentLoop;
// We modified the loop, update SE.
SE->forgetTopmostLoop(L);
// FIXME: Incrementally update loop-simplify
simplifyLoop(L, DT, LI, SE, AC, PreserveLCSSA);
NumPeeled++;
return true;
}
