/// canBasicBlockModify - Return true if it is possible for execution of the
/// specified basic block to modify the location Loc.
///
bool AAResults::canBasicBlockModify(const BasicBlock &BB,
const MemoryLocation &Loc) {
return canInstructionRangeModRef(BB.front(), BB.back(), Loc, MRI_Mod);
}
bool DSE::runOnBasicBlock(BasicBlock &BB) {
bool MadeChange = false;
// Do a top-down walk on the BB.
for (BasicBlock::iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE; ) {
Instruction *Inst = BBI++;
// Handle 'free' calls specially.
if (CallInst *F = isFreeCall(Inst)) {
MadeChange |= HandleFree(F);
continue;
}
// If we find something that writes memory, get its memory dependence.
if (!hasMemoryWrite(Inst))
continue;
MemDepResult InstDep = MD->getDependency(Inst);
// Ignore non-local store liveness.
// FIXME: cross-block DSE would be fun. :)
if (InstDep.isNonLocal() ||
// Ignore self dependence, which happens in the entry block of the
// function.
InstDep.getInst() == Inst)
continue;
// If we're storing the same value back to a pointer that we just
// loaded from, then the store can be removed.
if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
if (LoadInst *DepLoad = dyn_cast<LoadInst>(InstDep.getInst())) {
if (SI->getPointerOperand() == DepLoad->getPointerOperand() &&
SI->getOperand(0) == DepLoad && !SI->isVolatile()) {
DEBUG(dbgs() << "DSE: Remove Store Of Load from same pointer:\n "
<< "LOAD: " << *DepLoad << "\n STORE: " << *SI << '\n');
// DeleteDeadInstruction can delete the current instruction. Save BBI
// in case we need it.
WeakVH NextInst(BBI);
DeleteDeadInstruction(SI, *MD);
if (NextInst == 0) // Next instruction deleted.
BBI = BB.begin();
else if (BBI != BB.begin()) // Revisit this instruction if possible.
--BBI;
++NumFastStores;
MadeChange = true;
continue;
}
}
}
// Figure out what location is being stored to.
AliasAnalysis::Location Loc = getLocForWrite(Inst, *AA);
// If we didn't get a useful location, fail.
if (Loc.Ptr == 0)
continue;
while (!InstDep.isNonLocal()) {
// Get the memory clobbered by the instruction we depend on. MemDep will
// skip any instructions that 'Loc' clearly doesn't interact with. If we
// end up depending on a may- or must-aliased load, then we can't optimize
// away the store and we bail out. However, if we depend on on something
// that overwrites the memory location we *can* potentially optimize it.
//
// Find out what memory location the dependant instruction stores.
Instruction *DepWrite = InstDep.getInst();
AliasAnalysis::Location DepLoc = getLocForWrite(DepWrite, *AA);
// If we didn't get a useful location, or if it isn't a size, bail out.
if (DepLoc.Ptr == 0)
break;
// If we find a write that is a) removable (i.e., non-volatile), b) is
// completely obliterated by the store to 'Loc', and c) which we know that
// 'Inst' doesn't load from, then we can remove it.
if (isRemovable(DepWrite) && isCompleteOverwrite(Loc, DepLoc, *AA) &&
!isPossibleSelfRead(Inst, Loc, DepWrite, *AA)) {
DEBUG(dbgs() << "DSE: Remove Dead Store:\n DEAD: "
<< *DepWrite << "\n KILLER: " << *Inst << '\n');
// Delete the store and now-dead instructions that feed it.
DeleteDeadInstruction(DepWrite, *MD);
++NumFastStores;
MadeChange = true;
// DeleteDeadInstruction can delete the current instruction in loop
// cases, reset BBI.
BBI = Inst;
if (BBI != BB.begin())
--BBI;
break;
}
// If this is a may-aliased store that is clobbering the store value, we
// can keep searching past it for another must-aliased pointer that stores
// to the same location. For example, in:
// store -> P
// store -> Q
// store -> P
// we can remove the first store to P even though we don't know if P and Q
// alias.
if (DepWrite == &BB.front()) break;
// Can't look past this instruction if it might read 'Loc'.
if (AA->getModRefInfo(DepWrite, Loc) & AliasAnalysis::Ref)
break;
InstDep = MD->getPointerDependencyFrom(Loc, false, DepWrite, &BB);
}
}
// If this block ends in a return, unwind, or unreachable, all allocas are
// dead at its end, which means stores to them are also dead.
if (BB.getTerminator()->getNumSuccessors() == 0)
MadeChange |= handleEndBlock(BB);
return MadeChange;
}
/// canBasicBlockModify - Return true if it is possible for execution of the
/// specified basic block to modify the value pointed to by Ptr.
///
bool AliasAnalysis::canBasicBlockModify(const BasicBlock &BB,
const Value *Ptr, unsigned Size) {
return canInstructionRangeModify(BB.front(), BB.back(), Ptr, Size);
}
/// @brief Generate LLVM-IR for the SCoP @p S.
bool runOnScop(Scop &S) override {
AI = &getAnalysis<IslAstInfo>();
// Check if we created an isl_ast root node, otherwise exit.
isl_ast_node *AstRoot = AI->getAst();
if (!AstRoot)
return false;
LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
DL = &S.getRegion().getEntry()->getParent()->getParent()->getDataLayout();
RI = &getAnalysis<RegionInfoPass>().getRegionInfo();
Region *R = &S.getRegion();
assert(!R->isTopLevelRegion() && "Top level regions are not supported");
ScopAnnotator Annotator;
Annotator.buildAliasScopes(S);
simplifyRegion(R, DT, LI, RI);
assert(R->isSimple());
BasicBlock *EnteringBB = S.getRegion().getEnteringBlock();
assert(EnteringBB);
PollyIRBuilder Builder = createPollyIRBuilder(EnteringBB, Annotator);
IslNodeBuilder NodeBuilder(Builder, Annotator, this, *DL, *LI, *SE, *DT, S);
// Only build the run-time condition and parameters _after_ having
// introduced the conditional branch. This is important as the conditional
// branch will guard the original scop from new induction variables that
// the SCEVExpander may introduce while code generating the parameters and
// which may introduce scalar dependences that prevent us from correctly
// code generating this scop.
BasicBlock *StartBlock =
executeScopConditionally(S, this, Builder.getTrue());
auto SplitBlock = StartBlock->getSinglePredecessor();
// First generate code for the hoisted invariant loads and transitively the
// parameters they reference. Afterwards, for the remaining parameters that
// might reference the hoisted loads. Finally, build the runtime check
// that might reference both hoisted loads as well as parameters.
// If the hoisting fails we have to bail and execute the original code.
Builder.SetInsertPoint(SplitBlock->getTerminator());
if (!NodeBuilder.preloadInvariantLoads()) {
auto *FalseI1 = Builder.getFalse();
auto *SplitBBTerm = Builder.GetInsertBlock()->getTerminator();
SplitBBTerm->setOperand(0, FalseI1);
auto *StartBBTerm = StartBlock->getTerminator();
Builder.SetInsertPoint(StartBBTerm);
Builder.CreateUnreachable();
StartBBTerm->eraseFromParent();
isl_ast_node_free(AstRoot);
} else {
NodeBuilder.addParameters(S.getContext());
Value *RTC = buildRTC(Builder, NodeBuilder.getExprBuilder());
Builder.GetInsertBlock()->getTerminator()->setOperand(0, RTC);
Builder.SetInsertPoint(&StartBlock->front());
NodeBuilder.create(AstRoot);
NodeBuilder.finalizeSCoP(S);
fixRegionInfo(EnteringBB->getParent(), R->getParent());
}
verifyGeneratedFunction(S, *EnteringBB->getParent());
// Mark the function such that we run additional cleanup passes on this
// function (e.g. mem2reg to rediscover phi nodes).
Function *F = EnteringBB->getParent();
F->addFnAttr("polly-optimized");
return true;
}
/// isSafeToPromoteArgument - As you might guess from the name of this method,
/// it checks to see if it is both safe and useful to promote the argument.
/// This method limits promotion of aggregates to only promote up to three
/// elements of the aggregate in order to avoid exploding the number of
/// arguments passed in.
bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg, bool isByVal) const {
typedef std::set<IndicesVector> GEPIndicesSet;
// Quick exit for unused arguments
if (Arg->use_empty())
return true;
// We can only promote this argument if all of the uses are loads, or are GEP
// instructions (with constant indices) that are subsequently loaded.
//
// Promoting the argument causes it to be loaded in the caller
// unconditionally. This is only safe if we can prove that either the load
// would have happened in the callee anyway (ie, there is a load in the entry
// block) or the pointer passed in at every call site is guaranteed to be
// valid.
// In the former case, invalid loads can happen, but would have happened
// anyway, in the latter case, invalid loads won't happen. This prevents us
// from introducing an invalid load that wouldn't have happened in the
// original code.
//
// This set will contain all sets of indices that are loaded in the entry
// block, and thus are safe to unconditionally load in the caller.
GEPIndicesSet SafeToUnconditionallyLoad;
// This set contains all the sets of indices that we are planning to promote.
// This makes it possible to limit the number of arguments added.
GEPIndicesSet ToPromote;
// If the pointer is always valid, any load with first index 0 is valid.
if (isByVal || AllCallersPassInValidPointerForArgument(Arg))
SafeToUnconditionallyLoad.insert(IndicesVector(1, 0));
// First, iterate the entry block and mark loads of (geps of) arguments as
// safe.
BasicBlock *EntryBlock = Arg->getParent()->begin();
// Declare this here so we can reuse it
IndicesVector Indices;
for (BasicBlock::iterator I = EntryBlock->begin(), E = EntryBlock->end();
I != E; ++I)
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Value *V = LI->getPointerOperand();
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(V)) {
V = GEP->getPointerOperand();
if (V == Arg) {
// This load actually loads (part of) Arg? Check the indices then.
Indices.reserve(GEP->getNumIndices());
for (User::op_iterator II = GEP->idx_begin(), IE = GEP->idx_end();
II != IE; ++II)
if (ConstantInt *CI = dyn_cast<ConstantInt>(*II))
Indices.push_back(CI->getSExtValue());
else
// We found a non-constant GEP index for this argument? Bail out
// right away, can't promote this argument at all.
return false;
// Indices checked out, mark them as safe
MarkIndicesSafe(Indices, SafeToUnconditionallyLoad);
Indices.clear();
}
} else if (V == Arg) {
// Direct loads are equivalent to a GEP with a single 0 index.
MarkIndicesSafe(IndicesVector(1, 0), SafeToUnconditionallyLoad);
}
}
// Now, iterate all uses of the argument to see if there are any uses that are
// not (GEP+)loads, or any (GEP+)loads that are not safe to promote.
SmallVector<LoadInst*, 16> Loads;
IndicesVector Operands;
for (Value::use_iterator UI = Arg->use_begin(), E = Arg->use_end();
UI != E; ++UI) {
User *U = *UI;
Operands.clear();
if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
// Don't hack volatile/atomic loads
if (!LI->isSimple()) return false;
Loads.push_back(LI);
// Direct loads are equivalent to a GEP with a zero index and then a load.
Operands.push_back(0);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(U)) {
if (GEP->use_empty()) {
// Dead GEP's cause trouble later. Just remove them if we run into
// them.
getAnalysis<AliasAnalysis>().deleteValue(GEP);
GEP->eraseFromParent();
// TODO: This runs the above loop over and over again for dead GEPs
// Couldn't we just do increment the UI iterator earlier and erase the
// use?
return isSafeToPromoteArgument(Arg, isByVal);
}
// Ensure that all of the indices are constants.
for (User::op_iterator i = GEP->idx_begin(), e = GEP->idx_end();
i != e; ++i)
if (ConstantInt *C = dyn_cast<ConstantInt>(*i))
Operands.push_back(C->getSExtValue());
else
return false; // Not a constant operand GEP!
// Ensure that the only users of the GEP are load instructions.
for (Value::use_iterator UI = GEP->use_begin(), E = GEP->use_end();
UI != E; ++UI)
if (LoadInst *LI = dyn_cast<LoadInst>(*UI)) {
// Don't hack volatile/atomic loads
if (!LI->isSimple()) return false;
Loads.push_back(LI);
} else {
// Other uses than load?
return false;
}
} else {
return false; // Not a load or a GEP.
}
// Now, see if it is safe to promote this load / loads of this GEP. Loading
// is safe if Operands, or a prefix of Operands, is marked as safe.
if (!PrefixIn(Operands, SafeToUnconditionallyLoad))
return false;
// See if we are already promoting a load with these indices. If not, check
// to make sure that we aren't promoting too many elements. If so, nothing
// to do.
if (ToPromote.find(Operands) == ToPromote.end()) {
if (maxElements > 0 && ToPromote.size() == maxElements) {
DEBUG(dbgs() << "argpromotion not promoting argument '"
<< Arg->getName() << "' because it would require adding more "
<< "than " << maxElements << " arguments to the function.\n");
// We limit aggregate promotion to only promoting up to a fixed number
// of elements of the aggregate.
return false;
}
ToPromote.insert(Operands);
}
}
if (Loads.empty()) return true; // No users, this is a dead argument.
// Okay, now we know that the argument is only used by load instructions and
// it is safe to unconditionally perform all of them. Use alias analysis to
// check to see if the pointer is guaranteed to not be modified from entry of
// the function to each of the load instructions.
// Because there could be several/many load instructions, remember which
// blocks we know to be transparent to the load.
SmallPtrSet<BasicBlock*, 16> TranspBlocks;
AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
for (unsigned i = 0, e = Loads.size(); i != e; ++i) {
// Check to see if the load is invalidated from the start of the block to
// the load itself.
LoadInst *Load = Loads[i];
BasicBlock *BB = Load->getParent();
AliasAnalysis::Location Loc = AA.getLocation(Load);
if (AA.canInstructionRangeModify(BB->front(), *Load, Loc))
return false; // Pointer is invalidated!
// Now check every path from the entry block to the load for transparency.
// To do this, we perform a depth first search on the inverse CFG from the
// loading block.
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
BasicBlock *P = *PI;
for (idf_ext_iterator<BasicBlock*, SmallPtrSet<BasicBlock*, 16> >
I = idf_ext_begin(P, TranspBlocks),
E = idf_ext_end(P, TranspBlocks); I != E; ++I)
if (AA.canBasicBlockModify(**I, Loc))
return false;
}
}
// If the path from the entry of the function to each load is free of
// instructions that potentially invalidate the load, we can make the
// transformation!
return true;
}
/// Given an instruction in the loop, check to see if it has any uses that are
/// outside the current loop. If so, insert LCSSA PHI nodes and rewrite the
/// uses.
static bool processInstruction(Loop &L, Instruction &Inst, DominatorTree &DT,
const SmallVectorImpl<BasicBlock *> &ExitBlocks,
PredIteratorCache &PredCache, LoopInfo *LI) {
SmallVector<Use *, 16> UsesToRewrite;
// Tokens cannot be used in PHI nodes, so we skip over them.
// We can run into tokens which are live out of a loop with catchswitch
// instructions in Windows EH if the catchswitch has one catchpad which
// is inside the loop and another which is not.
if (Inst.getType()->isTokenTy())
return false;
BasicBlock *InstBB = Inst.getParent();
for (Use &U : Inst.uses()) {
Instruction *User = cast<Instruction>(U.getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(U);
if (InstBB != UserBB && !L.contains(UserBB))
UsesToRewrite.push_back(&U);
}
// If there are no uses outside the loop, exit with no change.
if (UsesToRewrite.empty())
return false;
++NumLCSSA; // We are applying the transformation
// Invoke instructions are special in that their result value is not available
// along their unwind edge. The code below tests to see whether DomBB
// dominates the value, so adjust DomBB to the normal destination block,
// which is effectively where the value is first usable.
BasicBlock *DomBB = Inst.getParent();
if (InvokeInst *Inv = dyn_cast<InvokeInst>(&Inst))
DomBB = Inv->getNormalDest();
DomTreeNode *DomNode = DT.getNode(DomBB);
SmallVector<PHINode *, 16> AddedPHIs;
SmallVector<PHINode *, 8> PostProcessPHIs;
SSAUpdater SSAUpdate;
SSAUpdate.Initialize(Inst.getType(), Inst.getName());
// Insert the LCSSA phi's into all of the exit blocks dominated by the
// value, and add them to the Phi's map.
for (BasicBlock *ExitBB : ExitBlocks) {
if (!DT.dominates(DomNode, DT.getNode(ExitBB)))
continue;
// If we already inserted something for this BB, don't reprocess it.
if (SSAUpdate.HasValueForBlock(ExitBB))
continue;
PHINode *PN = PHINode::Create(Inst.getType(), PredCache.size(ExitBB),
Inst.getName() + ".lcssa", &ExitBB->front());
// Add inputs from inside the loop for this PHI.
for (BasicBlock *Pred : PredCache.get(ExitBB)) {
PN->addIncoming(&Inst, Pred);
// If the exit block has a predecessor not within the loop, arrange for
// the incoming value use corresponding to that predecessor to be
// rewritten in terms of a different LCSSA PHI.
if (!L.contains(Pred))
UsesToRewrite.push_back(
&PN->getOperandUse(PN->getOperandNumForIncomingValue(
PN->getNumIncomingValues() - 1)));
}
AddedPHIs.push_back(PN);
// Remember that this phi makes the value alive in this block.
SSAUpdate.AddAvailableValue(ExitBB, PN);
// LoopSimplify might fail to simplify some loops (e.g. when indirect
// branches are involved). In such situations, it might happen that an exit
// for Loop L1 is the header of a disjoint Loop L2. Thus, when we create
// PHIs in such an exit block, we are also inserting PHIs into L2's header.
// This could break LCSSA form for L2 because these inserted PHIs can also
// have uses outside of L2. Remember all PHIs in such situation as to
// revisit than later on. FIXME: Remove this if indirectbr support into
// LoopSimplify gets improved.
if (auto *OtherLoop = LI->getLoopFor(ExitBB))
if (!L.contains(OtherLoop))
PostProcessPHIs.push_back(PN);
}
// Rewrite all uses outside the loop in terms of the new PHIs we just
// inserted.
for (Use *UseToRewrite : UsesToRewrite) {
// If this use is in an exit block, rewrite to use the newly inserted PHI.
// This is required for correctness because SSAUpdate doesn't handle uses in
// the same block. It assumes the PHI we inserted is at the end of the
// block.
Instruction *User = cast<Instruction>(UseToRewrite->getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(*UseToRewrite);
if (isa<PHINode>(UserBB->begin()) && isExitBlock(UserBB, ExitBlocks)) {
// Tell the VHs that the uses changed. This updates SCEV's caches.
if (UseToRewrite->get()->hasValueHandle())
ValueHandleBase::ValueIsRAUWd(*UseToRewrite, &UserBB->front());
UseToRewrite->set(&UserBB->front());
continue;
}
// Otherwise, do full PHI insertion.
SSAUpdate.RewriteUse(*UseToRewrite);
}
// Post process PHI instructions that were inserted into another disjoint loop
// and update their exits properly.
for (auto *I : PostProcessPHIs) {
if (I->use_empty())
continue;
BasicBlock *PHIBB = I->getParent();
Loop *OtherLoop = LI->getLoopFor(PHIBB);
SmallVector<BasicBlock *, 8> EBs;
OtherLoop->getExitBlocks(EBs);
if (EBs.empty())
continue;
// Recurse and re-process each PHI instruction. FIXME: we should really
// convert this entire thing to a worklist approach where we process a
// vector of instructions...
processInstruction(*OtherLoop, *I, DT, EBs, PredCache, LI);
}
// Remove PHI nodes that did not have any uses rewritten.
for (PHINode *PN : AddedPHIs)
if (PN->use_empty())
PN->eraseFromParent();
return true;
}
Function* PartialInliner::unswitchFunction(Function* F) {
// First, verify that this function is an unswitching candidate...
BasicBlock *entryBlock = &F->front();
BranchInst *BR = dyn_cast<BranchInst>(entryBlock->getTerminator());
if (!BR || BR->isUnconditional())
return nullptr;
BasicBlock* returnBlock = nullptr;
BasicBlock* nonReturnBlock = nullptr;
unsigned returnCount = 0;
for (BasicBlock *BB : successors(entryBlock)) {
if (isa<ReturnInst>(BB->getTerminator())) {
returnBlock = BB;
returnCount++;
} else
nonReturnBlock = BB;
}
if (returnCount != 1)
return nullptr;
// Clone the function, so that we can hack away on it.
ValueToValueMapTy VMap;
Function* duplicateFunction = CloneFunction(F, VMap,
/*ModuleLevelChanges=*/false);
duplicateFunction->setLinkage(GlobalValue::InternalLinkage);
F->getParent()->getFunctionList().push_back(duplicateFunction);
BasicBlock* newEntryBlock = cast<BasicBlock>(VMap[entryBlock]);
BasicBlock* newReturnBlock = cast<BasicBlock>(VMap[returnBlock]);
BasicBlock* newNonReturnBlock = cast<BasicBlock>(VMap[nonReturnBlock]);
// Go ahead and update all uses to the duplicate, so that we can just
// use the inliner functionality when we're done hacking.
F->replaceAllUsesWith(duplicateFunction);
// Special hackery is needed with PHI nodes that have inputs from more than
// one extracted block. For simplicity, just split the PHIs into a two-level
// sequence of PHIs, some of which will go in the extracted region, and some
// of which will go outside.
BasicBlock* preReturn = newReturnBlock;
newReturnBlock = newReturnBlock->splitBasicBlock(
newReturnBlock->getFirstNonPHI()->getIterator());
BasicBlock::iterator I = preReturn->begin();
Instruction *Ins = &newReturnBlock->front();
while (I != preReturn->end()) {
PHINode* OldPhi = dyn_cast<PHINode>(I);
if (!OldPhi) break;
PHINode *retPhi = PHINode::Create(OldPhi->getType(), 2, "", Ins);
OldPhi->replaceAllUsesWith(retPhi);
Ins = newReturnBlock->getFirstNonPHI();
retPhi->addIncoming(&*I, preReturn);
retPhi->addIncoming(OldPhi->getIncomingValueForBlock(newEntryBlock),
newEntryBlock);
OldPhi->removeIncomingValue(newEntryBlock);
++I;
}
newEntryBlock->getTerminator()->replaceUsesOfWith(preReturn, newReturnBlock);
// Gather up the blocks that we're going to extract.
std::vector<BasicBlock*> toExtract;
toExtract.push_back(newNonReturnBlock);
for (Function::iterator FI = duplicateFunction->begin(),
FE = duplicateFunction->end(); FI != FE; ++FI)
if (&*FI != newEntryBlock && &*FI != newReturnBlock &&
&*FI != newNonReturnBlock)
toExtract.push_back(&*FI);
// The CodeExtractor needs a dominator tree.
DominatorTree DT;
DT.recalculate(*duplicateFunction);
// Extract the body of the if.
Function* extractedFunction
= CodeExtractor(toExtract, &DT).extractCodeRegion();
InlineFunctionInfo IFI;
// Inline the top-level if test into all callers.
std::vector<User *> Users(duplicateFunction->user_begin(),
duplicateFunction->user_end());
for (std::vector<User*>::iterator UI = Users.begin(), UE = Users.end();
UI != UE; ++UI)
if (CallInst *CI = dyn_cast<CallInst>(*UI))
InlineFunction(CI, IFI);
else if (InvokeInst *II = dyn_cast<InvokeInst>(*UI))
InlineFunction(II, IFI);
// Ditch the duplicate, since we're done with it, and rewrite all remaining
// users (function pointers, etc.) back to the original function.
duplicateFunction->replaceAllUsesWith(F);
duplicateFunction->eraseFromParent();
++NumPartialInlined;
return extractedFunction;
}
/// \brief Recursively handle the condition leading to a loop
Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken,
llvm::Loop *L) {
// Only search through PHI nodes which are inside the loop. If we try this
// with PHI nodes that are outside of the loop, we end up inserting new PHI
// nodes outside of the loop which depend on values defined inside the loop.
// This will break the module with
// 'Instruction does not dominate all users!' errors.
PHINode *Phi = nullptr;
if ((Phi = dyn_cast<PHINode>(Cond)) && L->contains(Phi)) {
BasicBlock *Parent = Phi->getParent();
PHINode *NewPhi = PHINode::Create(Int64, 0, "", &Parent->front());
Value *Ret = NewPhi;
// Handle all non-constant incoming values first
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
BasicBlock *From = Phi->getIncomingBlock(i);
if (isa<ConstantInt>(Incoming)) {
NewPhi->addIncoming(Broken, From);
continue;
}
Phi->setIncomingValue(i, BoolFalse);
Value *PhiArg = handleLoopCondition(Incoming, Broken, L);
NewPhi->addIncoming(PhiArg, From);
}
BasicBlock *IDom = DT->getNode(Parent)->getIDom()->getBlock();
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
if (Incoming != BoolTrue)
continue;
BasicBlock *From = Phi->getIncomingBlock(i);
if (From == IDom) {
CallInst *OldEnd = dyn_cast<CallInst>(Parent->getFirstInsertionPt());
if (OldEnd && OldEnd->getCalledFunction() == EndCf) {
Value *Args[] = { OldEnd->getArgOperand(0), NewPhi };
Ret = CallInst::Create(ElseBreak, Args, "", OldEnd);
continue;
}
}
TerminatorInst *Insert = From->getTerminator();
Value *PhiArg = CallInst::Create(Break, Broken, "", Insert);
NewPhi->setIncomingValue(i, PhiArg);
}
eraseIfUnused(Phi);
return Ret;
} else if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
BasicBlock *Parent = Inst->getParent();
Instruction *Insert;
if (L->contains(Inst)) {
Insert = Parent->getTerminator();
} else {
Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
}
Value *Args[] = { Cond, Broken };
return CallInst::Create(IfBreak, Args, "", Insert);
} else {
llvm_unreachable("Unhandled loop condition!");
}
return 0;
}
/// \brief Recursively handle the condition leading to a loop
Value *SIAnnotateControlFlow::handleLoopCondition(Value *Cond, PHINode *Broken,
llvm::Loop *L, BranchInst *Term) {
// Only search through PHI nodes which are inside the loop. If we try this
// with PHI nodes that are outside of the loop, we end up inserting new PHI
// nodes outside of the loop which depend on values defined inside the loop.
// This will break the module with
// 'Instruction does not dominate all users!' errors.
PHINode *Phi = nullptr;
if ((Phi = dyn_cast<PHINode>(Cond)) && L->contains(Phi)) {
BasicBlock *Parent = Phi->getParent();
PHINode *NewPhi = PHINode::Create(Int64, 0, "", &Parent->front());
Value *Ret = NewPhi;
// Handle all non-constant incoming values first
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
BasicBlock *From = Phi->getIncomingBlock(i);
if (isa<ConstantInt>(Incoming)) {
NewPhi->addIncoming(Broken, From);
continue;
}
Phi->setIncomingValue(i, BoolFalse);
Value *PhiArg = handleLoopCondition(Incoming, Broken, L, Term);
NewPhi->addIncoming(PhiArg, From);
}
BasicBlock *IDom = DT->getNode(Parent)->getIDom()->getBlock();
for (unsigned i = 0, e = Phi->getNumIncomingValues(); i != e; ++i) {
Value *Incoming = Phi->getIncomingValue(i);
if (Incoming != BoolTrue)
continue;
BasicBlock *From = Phi->getIncomingBlock(i);
if (From == IDom) {
// We're in the following situation:
// IDom/From
// | \
// | If-block
// | /
// Parent
// where we want to break out of the loop if the If-block is not taken.
// Due to the depth-first traversal, there should be an end.cf
// intrinsic in Parent, and we insert an else.break before it.
//
// Note that the end.cf need not be the first non-phi instruction
// of parent, particularly when we're dealing with a multi-level
// break, but it should occur within a group of intrinsic calls
// at the beginning of the block.
CallInst *OldEnd = dyn_cast<CallInst>(Parent->getFirstInsertionPt());
while (OldEnd && OldEnd->getCalledFunction() != EndCf)
OldEnd = dyn_cast<CallInst>(OldEnd->getNextNode());
if (OldEnd && OldEnd->getCalledFunction() == EndCf) {
Value *Args[] = { OldEnd->getArgOperand(0), NewPhi };
Ret = CallInst::Create(ElseBreak, Args, "", OldEnd);
continue;
}
}
TerminatorInst *Insert = From->getTerminator();
Value *PhiArg = CallInst::Create(Break, Broken, "", Insert);
NewPhi->setIncomingValue(i, PhiArg);
}
eraseIfUnused(Phi);
return Ret;
} else if (Instruction *Inst = dyn_cast<Instruction>(Cond)) {
BasicBlock *Parent = Inst->getParent();
Instruction *Insert;
if (L->contains(Inst)) {
Insert = Parent->getTerminator();
} else {
Insert = L->getHeader()->getFirstNonPHIOrDbgOrLifetime();
}
Value *Args[] = { Cond, Broken };
return CallInst::Create(IfBreak, Args, "", Insert);
// Insert IfBreak before TERM for constant COND.
} else if (isa<ConstantInt>(Cond)) {
Value *Args[] = { Cond, Broken };
return CallInst::Create(IfBreak, Args, "", Term);
} else {
llvm_unreachable("Unhandled loop condition!");
}
return nullptr;
}
/// [LIBUNWIND] Find the (possibly absent) call to @llvm.eh.selector
/// in the given landing pad. In principle, llvm.eh.exception is
/// required to be in the landing pad; in practice, SplitCriticalEdge
/// can break that invariant, and then inlining can break it further.
/// There's a real need for a reliable solution here, but until that
/// happens, we have some fragile workarounds here.
static EHSelectorInst *findSelectorForLandingPad(BasicBlock *lpad) {
// Look for an exception call in the actual landing pad.
EHExceptionInst *exn = findExceptionInBlock(lpad);
if (exn) return findSelectorForException(exn);
// Okay, if that failed, look for one in an obvious successor. If
// we find one, we'll fix the IR by moving things back to the
// landing pad.
bool dominates = true; // does the lpad dominate the exn call
BasicBlock *nonDominated = 0; // if not, the first non-dominated block
BasicBlock *lastDominated = 0; // and the block which branched to it
BasicBlock *exnBlock = lpad;
// We need to protect against lpads that lead into infinite loops.
SmallPtrSet<BasicBlock*,4> visited;
visited.insert(exnBlock);
do {
// We're not going to apply this hack to anything more complicated
// than a series of unconditional branches, so if the block
// doesn't terminate in an unconditional branch, just fail. More
// complicated cases can arise when, say, sinking a call into a
// split unwind edge and then inlining it; but that can do almost
// *anything* to the CFG, including leaving the selector
// completely unreachable. The only way to fix that properly is
// to (1) prohibit transforms which move the exception or selector
// values away from the landing pad, e.g. by producing them with
// instructions that are pinned to an edge like a phi, or
// producing them with not-really-instructions, and (2) making
// transforms which split edges deal with that.
BranchInst *branch = dyn_cast<BranchInst>(&exnBlock->back());
if (!branch || branch->isConditional()) return 0;
BasicBlock *successor = branch->getSuccessor(0);
// Fail if we found an infinite loop.
if (!visited.insert(successor)) return 0;
// If the successor isn't dominated by exnBlock:
if (!successor->getSinglePredecessor()) {
// We don't want to have to deal with threading the exception
// through multiple levels of phi, so give up if we've already
// followed a non-dominating edge.
if (!dominates) return 0;
// Otherwise, remember this as a non-dominating edge.
dominates = false;
nonDominated = successor;
lastDominated = exnBlock;
}
exnBlock = successor;
// Can we stop here?
exn = findExceptionInBlock(exnBlock);
} while (!exn);
// Look for a selector call for the exception we found.
EHSelectorInst *selector = findSelectorForException(exn);
if (!selector) return 0;
// The easy case is when the landing pad still dominates the
// exception call, in which case we can just move both calls back to
// the landing pad.
if (dominates) {
selector->moveBefore(lpad->getFirstNonPHI());
exn->moveBefore(selector);
return selector;
}
// Otherwise, we have to split at the first non-dominating block.
// The CFG looks basically like this:
// lpad:
// phis_0
// insnsAndBranches_1
// br label %nonDominated
// nonDominated:
// phis_2
// insns_3
// %exn = call i8* @llvm.eh.exception()
// insnsAndBranches_4
// %selector = call @llvm.eh.selector(i8* %exn, ...
// We need to turn this into:
// lpad:
// phis_0
// %exn0 = call i8* @llvm.eh.exception()
// %selector0 = call @llvm.eh.selector(i8* %exn0, ...
// insnsAndBranches_1
// br label %split // from lastDominated
// nonDominated:
// phis_2 (without edge from lastDominated)
// %exn1 = call i8* @llvm.eh.exception()
// %selector1 = call i8* @llvm.eh.selector(i8* %exn1, ...
// br label %split
// split:
// phis_2 (edge from lastDominated, edge from split)
// %exn = phi ...
// %selector = phi ...
// insns_3
// insnsAndBranches_4
assert(nonDominated);
assert(lastDominated);
// First, make clones of the intrinsics to go in lpad.
EHExceptionInst *lpadExn = cast<EHExceptionInst>(exn->clone());
EHSelectorInst *lpadSelector = cast<EHSelectorInst>(selector->clone());
lpadSelector->setArgOperand(0, lpadExn);
lpadSelector->insertBefore(lpad->getFirstNonPHI());
lpadExn->insertBefore(lpadSelector);
// Split the non-dominated block.
BasicBlock *split =
nonDominated->splitBasicBlock(nonDominated->getFirstNonPHI(),
nonDominated->getName() + ".lpad-fix");
// Redirect the last dominated branch there.
cast<BranchInst>(lastDominated->back()).setSuccessor(0, split);
// Move the existing intrinsics to the end of the old block.
selector->moveBefore(&nonDominated->back());
exn->moveBefore(selector);
Instruction *splitIP = &split->front();
// For all the phis in nonDominated, make a new phi in split to join
// that phi with the edge from lastDominated.
for (BasicBlock::iterator
i = nonDominated->begin(), e = nonDominated->end(); i != e; ++i) {
PHINode *phi = dyn_cast<PHINode>(i);
if (!phi) break;
PHINode *splitPhi = PHINode::Create(phi->getType(), 2, phi->getName(),
splitIP);
phi->replaceAllUsesWith(splitPhi);
splitPhi->addIncoming(phi, nonDominated);
splitPhi->addIncoming(phi->removeIncomingValue(lastDominated),
lastDominated);
}
// Make new phis for the exception and selector.
PHINode *exnPhi = PHINode::Create(exn->getType(), 2, "", splitIP);
exn->replaceAllUsesWith(exnPhi);
selector->setArgOperand(0, exn); // except for this use
exnPhi->addIncoming(exn, nonDominated);
exnPhi->addIncoming(lpadExn, lastDominated);
PHINode *selectorPhi = PHINode::Create(selector->getType(), 2, "", splitIP);
selector->replaceAllUsesWith(selectorPhi);
selectorPhi->addIncoming(selector, nonDominated);
selectorPhi->addIncoming(lpadSelector, lastDominated);
return lpadSelector;
}
/// lowerAcrossUnwindEdges - Find all variables which are alive across an unwind
/// edge and spill them.
void SjLjEHPrepare::lowerAcrossUnwindEdges(Function &F,
ArrayRef<InvokeInst *> Invokes) {
// Finally, scan the code looking for instructions with bad live ranges.
for (BasicBlock &BB : F) {
for (Instruction &Inst : BB) {
// Ignore obvious cases we don't have to handle. In particular, most
// instructions either have no uses or only have a single use inside the
// current block. Ignore them quickly.
if (Inst.use_empty())
continue;
if (Inst.hasOneUse() &&
cast<Instruction>(Inst.user_back())->getParent() == &BB &&
!isa<PHINode>(Inst.user_back()))
continue;
// If this is an alloca in the entry block, it's not a real register
// value.
if (auto *AI = dyn_cast<AllocaInst>(&Inst))
if (AI->isStaticAlloca())
continue;
// Avoid iterator invalidation by copying users to a temporary vector.
SmallVector<Instruction *, 16> Users;
for (User *U : Inst.users()) {
Instruction *UI = cast<Instruction>(U);
if (UI->getParent() != &BB || isa<PHINode>(UI))
Users.push_back(UI);
}
// Find all of the blocks that this value is live in.
SmallPtrSet<BasicBlock *, 32> LiveBBs;
LiveBBs.insert(&BB);
while (!Users.empty()) {
Instruction *U = Users.pop_back_val();
if (!isa<PHINode>(U)) {
MarkBlocksLiveIn(U->getParent(), LiveBBs);
} else {
// Uses for a PHI node occur in their predecessor block.
PHINode *PN = cast<PHINode>(U);
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
if (PN->getIncomingValue(i) == &Inst)
MarkBlocksLiveIn(PN->getIncomingBlock(i), LiveBBs);
}
}
// Now that we know all of the blocks that this thing is live in, see if
// it includes any of the unwind locations.
bool NeedsSpill = false;
for (InvokeInst *Invoke : Invokes) {
BasicBlock *UnwindBlock = Invoke->getUnwindDest();
if (UnwindBlock != &BB && LiveBBs.count(UnwindBlock)) {
LLVM_DEBUG(dbgs() << "SJLJ Spill: " << Inst << " around "
<< UnwindBlock->getName() << "\n");
NeedsSpill = true;
break;
}
}
// If we decided we need a spill, do it.
// FIXME: Spilling this way is overkill, as it forces all uses of
// the value to be reloaded from the stack slot, even those that aren't
// in the unwind blocks. We should be more selective.
if (NeedsSpill) {
DemoteRegToStack(Inst, true);
++NumSpilled;
}
}
}
// Go through the landing pads and remove any PHIs there.
for (InvokeInst *Invoke : Invokes) {
BasicBlock *UnwindBlock = Invoke->getUnwindDest();
LandingPadInst *LPI = UnwindBlock->getLandingPadInst();
// Place PHIs into a set to avoid invalidating the iterator.
SmallPtrSet<PHINode *, 8> PHIsToDemote;
for (BasicBlock::iterator PN = UnwindBlock->begin(); isa<PHINode>(PN); ++PN)
PHIsToDemote.insert(cast<PHINode>(PN));
if (PHIsToDemote.empty())
continue;
// Demote the PHIs to the stack.
for (PHINode *PN : PHIsToDemote)
DemotePHIToStack(PN);
// Move the landingpad instruction back to the top of the landing pad block.
LPI->moveBefore(&UnwindBlock->front());
}
}
Function *PartialInlinerImpl::unswitchFunction(Function *F) {
// First, verify that this function is an unswitching candidate...
BasicBlock *EntryBlock = &F->front();
BranchInst *BR = dyn_cast<BranchInst>(EntryBlock->getTerminator());
if (!BR || BR->isUnconditional())
return nullptr;
BasicBlock *ReturnBlock = nullptr;
BasicBlock *NonReturnBlock = nullptr;
unsigned ReturnCount = 0;
for (BasicBlock *BB : successors(EntryBlock)) {
if (isa<ReturnInst>(BB->getTerminator())) {
ReturnBlock = BB;
ReturnCount++;
} else
NonReturnBlock = BB;
}
if (ReturnCount != 1)
return nullptr;
// Clone the function, so that we can hack away on it.
ValueToValueMapTy VMap;
Function *DuplicateFunction = CloneFunction(F, VMap);
DuplicateFunction->setLinkage(GlobalValue::InternalLinkage);
BasicBlock *NewEntryBlock = cast<BasicBlock>(VMap[EntryBlock]);
BasicBlock *NewReturnBlock = cast<BasicBlock>(VMap[ReturnBlock]);
BasicBlock *NewNonReturnBlock = cast<BasicBlock>(VMap[NonReturnBlock]);
// Go ahead and update all uses to the duplicate, so that we can just
// use the inliner functionality when we're done hacking.
F->replaceAllUsesWith(DuplicateFunction);
// Special hackery is needed with PHI nodes that have inputs from more than
// one extracted block. For simplicity, just split the PHIs into a two-level
// sequence of PHIs, some of which will go in the extracted region, and some
// of which will go outside.
BasicBlock *PreReturn = NewReturnBlock;
NewReturnBlock = NewReturnBlock->splitBasicBlock(
NewReturnBlock->getFirstNonPHI()->getIterator());
BasicBlock::iterator I = PreReturn->begin();
Instruction *Ins = &NewReturnBlock->front();
while (I != PreReturn->end()) {
PHINode *OldPhi = dyn_cast<PHINode>(I);
if (!OldPhi)
break;
PHINode *RetPhi = PHINode::Create(OldPhi->getType(), 2, "", Ins);
OldPhi->replaceAllUsesWith(RetPhi);
Ins = NewReturnBlock->getFirstNonPHI();
RetPhi->addIncoming(&*I, PreReturn);
RetPhi->addIncoming(OldPhi->getIncomingValueForBlock(NewEntryBlock),
NewEntryBlock);
OldPhi->removeIncomingValue(NewEntryBlock);
++I;
}
NewEntryBlock->getTerminator()->replaceUsesOfWith(PreReturn, NewReturnBlock);
// Gather up the blocks that we're going to extract.
std::vector<BasicBlock *> ToExtract;
ToExtract.push_back(NewNonReturnBlock);
for (BasicBlock &BB : *DuplicateFunction)
if (&BB != NewEntryBlock && &BB != NewReturnBlock &&
&BB != NewNonReturnBlock)
ToExtract.push_back(&BB);
// The CodeExtractor needs a dominator tree.
DominatorTree DT;
DT.recalculate(*DuplicateFunction);
// Extract the body of the if.
Function *ExtractedFunction =
CodeExtractor(ToExtract, &DT).extractCodeRegion();
// Inline the top-level if test into all callers.
std::vector<User *> Users(DuplicateFunction->user_begin(),
DuplicateFunction->user_end());
for (User *User : Users)
if (CallInst *CI = dyn_cast<CallInst>(User))
InlineFunction(CI, IFI);
else if (InvokeInst *II = dyn_cast<InvokeInst>(User))
InlineFunction(II, IFI);
// Ditch the duplicate, since we're done with it, and rewrite all remaining
// users (function pointers, etc.) back to the original function.
DuplicateFunction->replaceAllUsesWith(F);
DuplicateFunction->eraseFromParent();
++NumPartialInlined;
return ExtractedFunction;
}
/// Check whether \param BB is the merge block of a if-region. If yes, check
/// whether there exists an adjacent if-region upstream, the two if-regions
/// contain identical instructions and can be legally merged. \returns true if
/// the two if-regions are merged.
///
/// From:
/// if (a)
/// statement;
/// if (b)
/// statement;
///
/// To:
/// if (a || b)
/// statement;
bool FlattenCFGOpt::MergeIfRegion(BasicBlock *BB, IRBuilder<> &Builder) {
BasicBlock *IfTrue2, *IfFalse2;
Value *IfCond2 = GetIfCondition(BB, IfTrue2, IfFalse2);
Instruction *CInst2 = dyn_cast_or_null<Instruction>(IfCond2);
if (!CInst2)
return false;
BasicBlock *SecondEntryBlock = CInst2->getParent();
if (SecondEntryBlock->hasAddressTaken())
return false;
BasicBlock *IfTrue1, *IfFalse1;
Value *IfCond1 = GetIfCondition(SecondEntryBlock, IfTrue1, IfFalse1);
Instruction *CInst1 = dyn_cast_or_null<Instruction>(IfCond1);
if (!CInst1)
return false;
BasicBlock *FirstEntryBlock = CInst1->getParent();
// Either then-path or else-path should be empty.
if ((IfTrue1 != FirstEntryBlock) && (IfFalse1 != FirstEntryBlock))
return false;
if ((IfTrue2 != SecondEntryBlock) && (IfFalse2 != SecondEntryBlock))
return false;
TerminatorInst *PTI2 = SecondEntryBlock->getTerminator();
Instruction *PBI2 = &SecondEntryBlock->front();
if (!CompareIfRegionBlock(FirstEntryBlock, SecondEntryBlock, IfTrue1,
IfTrue2))
return false;
if (!CompareIfRegionBlock(FirstEntryBlock, SecondEntryBlock, IfFalse1,
IfFalse2))
return false;
// Check whether \param SecondEntryBlock has side-effect and is safe to
// speculate.
for (BasicBlock::iterator BI(PBI2), BE(PTI2); BI != BE; ++BI) {
Instruction *CI = &*BI;
if (isa<PHINode>(CI) || CI->mayHaveSideEffects() ||
!isSafeToSpeculativelyExecute(CI))
return false;
}
// Merge \param SecondEntryBlock into \param FirstEntryBlock.
FirstEntryBlock->getInstList().pop_back();
FirstEntryBlock->getInstList()
.splice(FirstEntryBlock->end(), SecondEntryBlock->getInstList());
BranchInst *PBI = dyn_cast<BranchInst>(FirstEntryBlock->getTerminator());
Value *CC = PBI->getCondition();
BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
Builder.SetInsertPoint(PBI);
Value *NC = Builder.CreateOr(CInst1, CC);
PBI->replaceUsesOfWith(CC, NC);
Builder.SetInsertPoint(SaveInsertBB, SaveInsertPt);
// Remove IfTrue1
if (IfTrue1 != FirstEntryBlock) {
IfTrue1->dropAllReferences();
IfTrue1->eraseFromParent();
}
// Remove IfFalse1
if (IfFalse1 != FirstEntryBlock) {
IfFalse1->dropAllReferences();
IfFalse1->eraseFromParent();
}
// Remove \param SecondEntryBlock
SecondEntryBlock->dropAllReferences();
SecondEntryBlock->eraseFromParent();
LLVM_DEBUG(dbgs() << "If conditions merged into:\n" << *FirstEntryBlock);
return true;
}
/// mergeEmptyReturnBlocks - If we have more than one empty (other than phi
/// node) return blocks, merge them together to promote recursive block merging.
static bool mergeEmptyReturnBlocks(Function &F) {
bool Changed = false;
BasicBlock *RetBlock = nullptr;
// Scan all the blocks in the function, looking for empty return blocks.
for (Function::iterator BBI = F.begin(), E = F.end(); BBI != E; ) {
BasicBlock &BB = *BBI++;
// Only look at return blocks.
ReturnInst *Ret = dyn_cast<ReturnInst>(BB.getTerminator());
if (!Ret) continue;
// Only look at the block if it is empty or the only other thing in it is a
// single PHI node that is the operand to the return.
if (Ret != &BB.front()) {
// Check for something else in the block.
BasicBlock::iterator I = Ret;
--I;
// Skip over debug info.
while (isa<DbgInfoIntrinsic>(I) && I != BB.begin())
--I;
if (!isa<DbgInfoIntrinsic>(I) &&
(!isa<PHINode>(I) || I != BB.begin() ||
Ret->getNumOperands() == 0 ||
Ret->getOperand(0) != I))
continue;
}
// If this is the first returning block, remember it and keep going.
if (!RetBlock) {
RetBlock = &BB;
continue;
}
// Otherwise, we found a duplicate return block. Merge the two.
Changed = true;
// Case when there is no input to the return or when the returned values
// agree is trivial. Note that they can't agree if there are phis in the
// blocks.
if (Ret->getNumOperands() == 0 ||
Ret->getOperand(0) ==
cast<ReturnInst>(RetBlock->getTerminator())->getOperand(0)) {
BB.replaceAllUsesWith(RetBlock);
BB.eraseFromParent();
continue;
}
// If the canonical return block has no PHI node, create one now.
PHINode *RetBlockPHI = dyn_cast<PHINode>(RetBlock->begin());
if (!RetBlockPHI) {
Value *InVal = cast<ReturnInst>(RetBlock->getTerminator())->getOperand(0);
pred_iterator PB = pred_begin(RetBlock), PE = pred_end(RetBlock);
RetBlockPHI = PHINode::Create(Ret->getOperand(0)->getType(),
std::distance(PB, PE), "merge",
&RetBlock->front());
for (pred_iterator PI = PB; PI != PE; ++PI)
RetBlockPHI->addIncoming(InVal, *PI);
RetBlock->getTerminator()->setOperand(0, RetBlockPHI);
}
// Turn BB into a block that just unconditionally branches to the return
// block. This handles the case when the two return blocks have a common
// predecessor but that return different things.
RetBlockPHI->addIncoming(Ret->getOperand(0), &BB);
BB.getTerminator()->eraseFromParent();
BranchInst::Create(RetBlock, &BB);
}
return Changed;
}
/// For every instruction from the worklist, check to see if it has any uses
/// that are outside the current loop. If so, insert LCSSA PHI nodes and
/// rewrite the uses.
bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
DominatorTree &DT, LoopInfo &LI) {
SmallVector<Use *, 16> UsesToRewrite;
SmallVector<BasicBlock *, 8> ExitBlocks;
SmallSetVector<PHINode *, 16> PHIsToRemove;
PredIteratorCache PredCache;
bool Changed = false;
while (!Worklist.empty()) {
UsesToRewrite.clear();
ExitBlocks.clear();
Instruction *I = Worklist.pop_back_val();
BasicBlock *InstBB = I->getParent();
Loop *L = LI.getLoopFor(InstBB);
L->getExitBlocks(ExitBlocks);
if (ExitBlocks.empty())
continue;
// Tokens cannot be used in PHI nodes, so we skip over them.
// We can run into tokens which are live out of a loop with catchswitch
// instructions in Windows EH if the catchswitch has one catchpad which
// is inside the loop and another which is not.
if (I->getType()->isTokenTy())
continue;
for (Use &U : I->uses()) {
Instruction *User = cast<Instruction>(U.getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(U);
if (InstBB != UserBB && !L->contains(UserBB))
UsesToRewrite.push_back(&U);
}
// If there are no uses outside the loop, exit with no change.
if (UsesToRewrite.empty())
continue;
++NumLCSSA; // We are applying the transformation
// Invoke instructions are special in that their result value is not
// available along their unwind edge. The code below tests to see whether
// DomBB dominates the value, so adjust DomBB to the normal destination
// block, which is effectively where the value is first usable.
BasicBlock *DomBB = InstBB;
if (InvokeInst *Inv = dyn_cast<InvokeInst>(I))
DomBB = Inv->getNormalDest();
DomTreeNode *DomNode = DT.getNode(DomBB);
SmallVector<PHINode *, 16> AddedPHIs;
SmallVector<PHINode *, 8> PostProcessPHIs;
SmallVector<PHINode *, 4> InsertedPHIs;
SSAUpdater SSAUpdate(&InsertedPHIs);
SSAUpdate.Initialize(I->getType(), I->getName());
// Insert the LCSSA phi's into all of the exit blocks dominated by the
// value, and add them to the Phi's map.
for (BasicBlock *ExitBB : ExitBlocks) {
if (!DT.dominates(DomNode, DT.getNode(ExitBB)))
continue;
// If we already inserted something for this BB, don't reprocess it.
if (SSAUpdate.HasValueForBlock(ExitBB))
continue;
PHINode *PN = PHINode::Create(I->getType(), PredCache.size(ExitBB),
I->getName() + ".lcssa", &ExitBB->front());
// Add inputs from inside the loop for this PHI.
for (BasicBlock *Pred : PredCache.get(ExitBB)) {
PN->addIncoming(I, Pred);
// If the exit block has a predecessor not within the loop, arrange for
// the incoming value use corresponding to that predecessor to be
// rewritten in terms of a different LCSSA PHI.
if (!L->contains(Pred))
UsesToRewrite.push_back(
&PN->getOperandUse(PN->getOperandNumForIncomingValue(
PN->getNumIncomingValues() - 1)));
}
AddedPHIs.push_back(PN);
// Remember that this phi makes the value alive in this block.
SSAUpdate.AddAvailableValue(ExitBB, PN);
// LoopSimplify might fail to simplify some loops (e.g. when indirect
// branches are involved). In such situations, it might happen that an
// exit for Loop L1 is the header of a disjoint Loop L2. Thus, when we
// create PHIs in such an exit block, we are also inserting PHIs into L2's
// header. This could break LCSSA form for L2 because these inserted PHIs
// can also have uses outside of L2. Remember all PHIs in such situation
// as to revisit than later on. FIXME: Remove this if indirectbr support
// into LoopSimplify gets improved.
if (auto *OtherLoop = LI.getLoopFor(ExitBB))
if (!L->contains(OtherLoop))
PostProcessPHIs.push_back(PN);
}
// Rewrite all uses outside the loop in terms of the new PHIs we just
// inserted.
for (Use *UseToRewrite : UsesToRewrite) {
// If this use is in an exit block, rewrite to use the newly inserted PHI.
// This is required for correctness because SSAUpdate doesn't handle uses
// in the same block. It assumes the PHI we inserted is at the end of the
// block.
Instruction *User = cast<Instruction>(UseToRewrite->getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(*UseToRewrite);
if (isa<PHINode>(UserBB->begin()) && isExitBlock(UserBB, ExitBlocks)) {
// Tell the VHs that the uses changed. This updates SCEV's caches.
if (UseToRewrite->get()->hasValueHandle())
ValueHandleBase::ValueIsRAUWd(*UseToRewrite, &UserBB->front());
UseToRewrite->set(&UserBB->front());
continue;
}
// Otherwise, do full PHI insertion.
SSAUpdate.RewriteUse(*UseToRewrite);
// SSAUpdater might have inserted phi-nodes inside other loops. We'll need
// to post-process them to keep LCSSA form.
for (PHINode *InsertedPN : InsertedPHIs) {
if (auto *OtherLoop = LI.getLoopFor(InsertedPN->getParent()))
if (!L->contains(OtherLoop))
PostProcessPHIs.push_back(InsertedPN);
}
}
// Post process PHI instructions that were inserted into another disjoint
// loop and update their exits properly.
for (auto *PostProcessPN : PostProcessPHIs) {
if (PostProcessPN->use_empty())
continue;
// Reprocess each PHI instruction.
Worklist.push_back(PostProcessPN);
}
// Keep track of PHI nodes that we want to remove because they did not have
// any uses rewritten.
for (PHINode *PN : AddedPHIs)
if (PN->use_empty())
PHIsToRemove.insert(PN);
Changed = true;
}
// Remove PHI nodes that did not have any uses rewritten.
for (PHINode *PN : PHIsToRemove) {
assert (PN->use_empty() && "Trying to remove a phi with uses.");
PN->eraseFromParent();
}
return Changed;
}
bool DSE::runOnBasicBlock(BasicBlock &BB) {
bool MadeChange = false;
// Do a top-down walk on the BB.
for (BasicBlock::iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE; ) {
Instruction *Inst = BBI++;
// Handle 'free' calls specially.
if (CallInst *F = isFreeCall(Inst)) {
MadeChange |= HandleFree(F);
continue;
}
// If we find something that writes memory, get its memory dependence.
if (!hasMemoryWrite(Inst))
continue;
MemDepResult InstDep = MD->getDependency(Inst);
// Ignore any store where we can't find a local dependence.
// FIXME: cross-block DSE would be fun. :)
if (!InstDep.isDef() && !InstDep.isClobber())
continue;
// If we're storing the same value back to a pointer that we just
// loaded from, then the store can be removed.
if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
if (LoadInst *DepLoad = dyn_cast<LoadInst>(InstDep.getInst())) {
if (SI->getPointerOperand() == DepLoad->getPointerOperand() &&
SI->getOperand(0) == DepLoad && isRemovable(SI)) {
DEBUG(dbgs() << "DSE: Remove Store Of Load from same pointer:\n "
<< "LOAD: " << *DepLoad << "\n STORE: " << *SI << '\n');
// DeleteDeadInstruction can delete the current instruction. Save BBI
// in case we need it.
WeakVH NextInst(BBI);
DeleteDeadInstruction(SI, *MD);
if (NextInst == 0) // Next instruction deleted.
BBI = BB.begin();
else if (BBI != BB.begin()) // Revisit this instruction if possible.
--BBI;
++NumFastStores;
MadeChange = true;
continue;
}
}
}
// Figure out what location is being stored to.
AliasAnalysis::Location Loc = getLocForWrite(Inst, *AA);
// If we didn't get a useful location, fail.
if (Loc.Ptr == 0)
continue;
while (InstDep.isDef() || InstDep.isClobber()) {
// Get the memory clobbered by the instruction we depend on. MemDep will
// skip any instructions that 'Loc' clearly doesn't interact with. If we
// end up depending on a may- or must-aliased load, then we can't optimize
// away the store and we bail out. However, if we depend on on something
// that overwrites the memory location we *can* potentially optimize it.
//
// Find out what memory location the dependent instruction stores.
Instruction *DepWrite = InstDep.getInst();
AliasAnalysis::Location DepLoc = getLocForWrite(DepWrite, *AA);
// If we didn't get a useful location, or if it isn't a size, bail out.
if (DepLoc.Ptr == 0)
break;
// If we find a write that is a) removable (i.e., non-volatile), b) is
// completely obliterated by the store to 'Loc', and c) which we know that
// 'Inst' doesn't load from, then we can remove it.
if (isRemovable(DepWrite) &&
!isPossibleSelfRead(Inst, Loc, DepWrite, *AA)) {
int64_t InstWriteOffset, DepWriteOffset;
OverwriteResult OR = isOverwrite(Loc, DepLoc, *AA,
DepWriteOffset, InstWriteOffset);
if (OR == OverwriteComplete) {
DEBUG(dbgs() << "DSE: Remove Dead Store:\n DEAD: "
<< *DepWrite << "\n KILLER: " << *Inst << '\n');
// Delete the store and now-dead instructions that feed it.
DeleteDeadInstruction(DepWrite, *MD);
++NumFastStores;
MadeChange = true;
// DeleteDeadInstruction can delete the current instruction in loop
// cases, reset BBI.
BBI = Inst;
if (BBI != BB.begin())
--BBI;
break;
} else if (OR == OverwriteEnd && isShortenable(DepWrite)) {
// TODO: base this on the target vector size so that if the earlier
// store was too small to get vector writes anyway then its likely
// a good idea to shorten it
// Power of 2 vector writes are probably always a bad idea to optimize
// as any store/memset/memcpy is likely using vector instructions so
// shortening it to not vector size is likely to be slower
MemIntrinsic* DepIntrinsic = cast<MemIntrinsic>(DepWrite);
unsigned DepWriteAlign = DepIntrinsic->getAlignment();
if (llvm::isPowerOf2_64(InstWriteOffset) ||
((DepWriteAlign != 0) && InstWriteOffset % DepWriteAlign == 0)) {
DEBUG(dbgs() << "DSE: Remove Dead Store:\n OW END: "
<< *DepWrite << "\n KILLER (offset "
<< InstWriteOffset << ", "
<< DepLoc.Size << ")"
<< *Inst << '\n');
Value* DepWriteLength = DepIntrinsic->getLength();
Value* TrimmedLength = ConstantInt::get(DepWriteLength->getType(),
InstWriteOffset -
DepWriteOffset);
DepIntrinsic->setLength(TrimmedLength);
MadeChange = true;
}
}
}
// If this is a may-aliased store that is clobbering the store value, we
// can keep searching past it for another must-aliased pointer that stores
// to the same location. For example, in:
// store -> P
// store -> Q
// store -> P
// we can remove the first store to P even though we don't know if P and Q
// alias.
if (DepWrite == &BB.front()) break;
// Can't look past this instruction if it might read 'Loc'.
if (AA->getModRefInfo(DepWrite, Loc) & AliasAnalysis::Ref)
break;
InstDep = MD->getPointerDependencyFrom(Loc, false, DepWrite, &BB);
}
}
// If this block ends in a return, unwind, or unreachable, all allocas are
// dead at its end, which means stores to them are also dead.
if (BB.getTerminator()->getNumSuccessors() == 0)
MadeChange |= handleEndBlock(BB);
return MadeChange;
}
/// severSplitPHINodes - If a PHI node has multiple inputs from outside of the
/// region, we need to split the entry block of the region so that the PHI node
/// is easier to deal with.
void CodeExtractor::severSplitPHINodes(BasicBlock *&Header) {
unsigned NumPredsFromRegion = 0;
unsigned NumPredsOutsideRegion = 0;
if (Header != &Header->getParent()->getEntryBlock()) {
PHINode *PN = dyn_cast<PHINode>(Header->begin());
if (!PN) return; // No PHI nodes.
// If the header node contains any PHI nodes, check to see if there is more
// than one entry from outside the region. If so, we need to sever the
// header block into two.
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
if (Blocks.count(PN->getIncomingBlock(i)))
++NumPredsFromRegion;
else
++NumPredsOutsideRegion;
// If there is one (or fewer) predecessor from outside the region, we don't
// need to do anything special.
if (NumPredsOutsideRegion <= 1) return;
}
// Otherwise, we need to split the header block into two pieces: one
// containing PHI nodes merging values from outside of the region, and a
// second that contains all of the code for the block and merges back any
// incoming values from inside of the region.
BasicBlock::iterator AfterPHIs = Header->getFirstNonPHI()->getIterator();
BasicBlock *NewBB = Header->splitBasicBlock(AfterPHIs,
Header->getName()+".ce");
// We only want to code extract the second block now, and it becomes the new
// header of the region.
BasicBlock *OldPred = Header;
Blocks.remove(OldPred);
Blocks.insert(NewBB);
Header = NewBB;
// Okay, update dominator sets. The blocks that dominate the new one are the
// blocks that dominate TIBB plus the new block itself.
if (DT)
DT->splitBlock(NewBB);
// Okay, now we need to adjust the PHI nodes and any branches from within the
// region to go to the new header block instead of the old header block.
if (NumPredsFromRegion) {
PHINode *PN = cast<PHINode>(OldPred->begin());
// Loop over all of the predecessors of OldPred that are in the region,
// changing them to branch to NewBB instead.
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
if (Blocks.count(PN->getIncomingBlock(i))) {
TerminatorInst *TI = PN->getIncomingBlock(i)->getTerminator();
TI->replaceUsesOfWith(OldPred, NewBB);
}
// Okay, everything within the region is now branching to the right block, we
// just have to update the PHI nodes now, inserting PHI nodes into NewBB.
for (AfterPHIs = OldPred->begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
PHINode *PN = cast<PHINode>(AfterPHIs);
// Create a new PHI node in the new region, which has an incoming value
// from OldPred of PN.
PHINode *NewPN = PHINode::Create(PN->getType(), 1 + NumPredsFromRegion,
PN->getName() + ".ce", &NewBB->front());
NewPN->addIncoming(PN, OldPred);
// Loop over all of the incoming value in PN, moving them to NewPN if they
// are from the extracted region.
for (unsigned i = 0; i != PN->getNumIncomingValues(); ++i) {
if (Blocks.count(PN->getIncomingBlock(i))) {
NewPN->addIncoming(PN->getIncomingValue(i), PN->getIncomingBlock(i));
PN->removeIncomingValue(i);
--i;
}
}
}
}
}
void PromoteMem2Reg::run() {
Function &F = *DT.getRoot()->getParent();
if (AST)
PointerAllocaValues.resize(Allocas.size());
AllocaDbgDeclares.resize(Allocas.size());
AllocaInfo Info;
LargeBlockInfo LBI;
for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); ++AllocaNum) {
AllocaInst *AI = Allocas[AllocaNum];
assert(isAllocaPromotable(AI) && "Cannot promote non-promotable alloca!");
assert(AI->getParent()->getParent() == &F &&
"All allocas should be in the same function, which is same as DF!");
removeLifetimeIntrinsicUsers(AI);
if (AI->use_empty()) {
// If there are no uses of the alloca, just delete it now.
if (AST)
AST->deleteValue(AI);
AI->eraseFromParent();
// Remove the alloca from the Allocas list, since it has been processed
RemoveFromAllocasList(AllocaNum);
++NumDeadAlloca;
continue;
}
// Calculate the set of read and write-locations for each alloca. This is
// analogous to finding the 'uses' and 'definitions' of each variable.
Info.AnalyzeAlloca(AI);
// If there is only a single store to this value, replace any loads of
// it that are directly dominated by the definition with the value stored.
if (Info.DefiningBlocks.size() == 1) {
if (rewriteSingleStoreAlloca(AI, Info, LBI, DT, AST)) {
// The alloca has been processed, move on.
RemoveFromAllocasList(AllocaNum);
++NumSingleStore;
continue;
}
}
// If the alloca is only read and written in one basic block, just perform a
// linear sweep over the block to eliminate it.
if (Info.OnlyUsedInOneBlock) {
promoteSingleBlockAlloca(AI, Info, LBI, AST);
// The alloca has been processed, move on.
RemoveFromAllocasList(AllocaNum);
continue;
}
// If we haven't computed dominator tree levels, do so now.
if (DomLevels.empty()) {
SmallVector<DomTreeNode *, 32> Worklist;
DomTreeNode *Root = DT.getRootNode();
DomLevels[Root] = 0;
Worklist.push_back(Root);
while (!Worklist.empty()) {
DomTreeNode *Node = Worklist.pop_back_val();
unsigned ChildLevel = DomLevels[Node] + 1;
for (DomTreeNode::iterator CI = Node->begin(), CE = Node->end();
CI != CE; ++CI) {
DomLevels[*CI] = ChildLevel;
Worklist.push_back(*CI);
}
}
}
// If we haven't computed a numbering for the BB's in the function, do so
// now.
if (BBNumbers.empty()) {
unsigned ID = 0;
for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
BBNumbers[I] = ID++;
}
// If we have an AST to keep updated, remember some pointer value that is
// stored into the alloca.
if (AST)
PointerAllocaValues[AllocaNum] = Info.AllocaPointerVal;
// Remember the dbg.declare intrinsic describing this alloca, if any.
if (Info.DbgDeclare)
AllocaDbgDeclares[AllocaNum] = Info.DbgDeclare;
// Keep the reverse mapping of the 'Allocas' array for the rename pass.
AllocaLookup[Allocas[AllocaNum]] = AllocaNum;
// At this point, we're committed to promoting the alloca using IDF's, and
// the standard SSA construction algorithm. Determine which blocks need PHI
// nodes and see if we can optimize out some work by avoiding insertion of
// dead phi nodes.
DetermineInsertionPoint(AI, AllocaNum, Info);
}
if (Allocas.empty())
return; // All of the allocas must have been trivial!
LBI.clear();
// Set the incoming values for the basic block to be null values for all of
// the alloca's. We do this in case there is a load of a value that has not
// been stored yet. In this case, it will get this null value.
//
RenamePassData::ValVector Values(Allocas.size());
for (unsigned i = 0, e = Allocas.size(); i != e; ++i)
Values[i] = UndefValue::get(Allocas[i]->getAllocatedType());
// Walks all basic blocks in the function performing the SSA rename algorithm
// and inserting the phi nodes we marked as necessary
//
std::vector<RenamePassData> RenamePassWorkList;
RenamePassWorkList.push_back(RenamePassData(F.begin(), nullptr, Values));
do {
RenamePassData RPD;
RPD.swap(RenamePassWorkList.back());
RenamePassWorkList.pop_back();
// RenamePass may add new worklist entries.
RenamePass(RPD.BB, RPD.Pred, RPD.Values, RenamePassWorkList);
} while (!RenamePassWorkList.empty());
// The renamer uses the Visited set to avoid infinite loops. Clear it now.
Visited.clear();
// Remove the allocas themselves from the function.
for (unsigned i = 0, e = Allocas.size(); i != e; ++i) {
Instruction *A = Allocas[i];
// If there are any uses of the alloca instructions left, they must be in
// unreachable basic blocks that were not processed by walking the dominator
// tree. Just delete the users now.
if (!A->use_empty())
A->replaceAllUsesWith(UndefValue::get(A->getType()));
if (AST)
AST->deleteValue(A);
A->eraseFromParent();
}
// Remove alloca's dbg.declare instrinsics from the function.
for (unsigned i = 0, e = AllocaDbgDeclares.size(); i != e; ++i)
if (DbgDeclareInst *DDI = AllocaDbgDeclares[i])
DDI->eraseFromParent();
// Loop over all of the PHI nodes and see if there are any that we can get
// rid of because they merge all of the same incoming values. This can
// happen due to undef values coming into the PHI nodes. This process is
// iterative, because eliminating one PHI node can cause others to be removed.
bool EliminatedAPHI = true;
while (EliminatedAPHI) {
EliminatedAPHI = false;
// Iterating over NewPhiNodes is deterministic, so it is safe to try to
// simplify and RAUW them as we go. If it was not, we could add uses to
// the values we replace with in a non-deterministic order, thus creating
// non-deterministic def->use chains.
for (DenseMap<std::pair<unsigned, unsigned>, PHINode *>::iterator
I = NewPhiNodes.begin(),
E = NewPhiNodes.end();
I != E;) {
PHINode *PN = I->second;
// If this PHI node merges one value and/or undefs, get the value.
if (Value *V = SimplifyInstruction(PN, nullptr, nullptr, &DT, AT)) {
if (AST && PN->getType()->isPointerTy())
AST->deleteValue(PN);
PN->replaceAllUsesWith(V);
PN->eraseFromParent();
NewPhiNodes.erase(I++);
EliminatedAPHI = true;
continue;
}
++I;
}
}
// At this point, the renamer has added entries to PHI nodes for all reachable
// code. Unfortunately, there may be unreachable blocks which the renamer
// hasn't traversed. If this is the case, the PHI nodes may not
// have incoming values for all predecessors. Loop over all PHI nodes we have
// created, inserting undef values if they are missing any incoming values.
//
for (DenseMap<std::pair<unsigned, unsigned>, PHINode *>::iterator
I = NewPhiNodes.begin(),
E = NewPhiNodes.end();
I != E; ++I) {
// We want to do this once per basic block. As such, only process a block
// when we find the PHI that is the first entry in the block.
PHINode *SomePHI = I->second;
BasicBlock *BB = SomePHI->getParent();
if (&BB->front() != SomePHI)
continue;
// Only do work here if there the PHI nodes are missing incoming values. We
// know that all PHI nodes that were inserted in a block will have the same
// number of incoming values, so we can just check any of them.
if (SomePHI->getNumIncomingValues() == getNumPreds(BB))
continue;
// Get the preds for BB.
SmallVector<BasicBlock *, 16> Preds(pred_begin(BB), pred_end(BB));
// Ok, now we know that all of the PHI nodes are missing entries for some
// basic blocks. Start by sorting the incoming predecessors for efficient
// access.
std::sort(Preds.begin(), Preds.end());
// Now we loop through all BB's which have entries in SomePHI and remove
// them from the Preds list.
for (unsigned i = 0, e = SomePHI->getNumIncomingValues(); i != e; ++i) {
// Do a log(n) search of the Preds list for the entry we want.
SmallVectorImpl<BasicBlock *>::iterator EntIt = std::lower_bound(
Preds.begin(), Preds.end(), SomePHI->getIncomingBlock(i));
assert(EntIt != Preds.end() && *EntIt == SomePHI->getIncomingBlock(i) &&
"PHI node has entry for a block which is not a predecessor!");
// Remove the entry
Preds.erase(EntIt);
}
// At this point, the blocks left in the preds list must have dummy
// entries inserted into every PHI nodes for the block. Update all the phi
// nodes in this block that we are inserting (there could be phis before
// mem2reg runs).
unsigned NumBadPreds = SomePHI->getNumIncomingValues();
BasicBlock::iterator BBI = BB->begin();
while ((SomePHI = dyn_cast<PHINode>(BBI++)) &&
SomePHI->getNumIncomingValues() == NumBadPreds) {
Value *UndefVal = UndefValue::get(SomePHI->getType());
for (unsigned pred = 0, e = Preds.size(); pred != e; ++pred)
SomePHI->addIncoming(UndefVal, Preds[pred]);
}
}
NewPhiNodes.clear();
}
/// canBasicBlockModify - Return true if it is possible for execution of the
/// specified basic block to modify the value pointed to by Ptr.
///
bool AliasAnalysis::canBasicBlockModify(const BasicBlock &BB,
const Location &Loc) {
return canInstructionRangeModify(BB.front(), BB.back(), Loc);
}
/// SimplifyStoreAtEndOfBlock - Turn things like:
/// if () { *P = v1; } else { *P = v2 }
/// into a phi node with a store in the successor.
///
/// Simplify things like:
/// *P = v1; if () { *P = v2; }
/// into a phi node with a store in the successor.
///
bool InstCombiner::SimplifyStoreAtEndOfBlock(StoreInst &SI) {
BasicBlock *StoreBB = SI.getParent();
// Check to see if the successor block has exactly two incoming edges. If
// so, see if the other predecessor contains a store to the same location.
// if so, insert a PHI node (if needed) and move the stores down.
BasicBlock *DestBB = StoreBB->getTerminator()->getSuccessor(0);
// Determine whether Dest has exactly two predecessors and, if so, compute
// the other predecessor.
pred_iterator PI = pred_begin(DestBB);
BasicBlock *P = *PI;
BasicBlock *OtherBB = nullptr;
if (P != StoreBB)
OtherBB = P;
if (++PI == pred_end(DestBB))
return false;
P = *PI;
if (P != StoreBB) {
if (OtherBB)
return false;
OtherBB = P;
}
if (++PI != pred_end(DestBB))
return false;
// Bail out if all the relevant blocks aren't distinct (this can happen,
// for example, if SI is in an infinite loop)
if (StoreBB == DestBB || OtherBB == DestBB)
return false;
// Verify that the other block ends in a branch and is not otherwise empty.
BasicBlock::iterator BBI(OtherBB->getTerminator());
BranchInst *OtherBr = dyn_cast<BranchInst>(BBI);
if (!OtherBr || BBI == OtherBB->begin())
return false;
// If the other block ends in an unconditional branch, check for the 'if then
// else' case. there is an instruction before the branch.
StoreInst *OtherStore = nullptr;
if (OtherBr->isUnconditional()) {
--BBI;
// Skip over debugging info.
while (isa<DbgInfoIntrinsic>(BBI) ||
(isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy())) {
if (BBI==OtherBB->begin())
return false;
--BBI;
}
// If this isn't a store, isn't a store to the same location, or is not the
// right kind of store, bail out.
OtherStore = dyn_cast<StoreInst>(BBI);
if (!OtherStore || OtherStore->getOperand(1) != SI.getOperand(1) ||
!SI.isSameOperationAs(OtherStore))
return false;
} else {
// Otherwise, the other block ended with a conditional branch. If one of the
// destinations is StoreBB, then we have the if/then case.
if (OtherBr->getSuccessor(0) != StoreBB &&
OtherBr->getSuccessor(1) != StoreBB)
return false;
// Okay, we know that OtherBr now goes to Dest and StoreBB, so this is an
// if/then triangle. See if there is a store to the same ptr as SI that
// lives in OtherBB.
for (;; --BBI) {
// Check to see if we find the matching store.
if ((OtherStore = dyn_cast<StoreInst>(BBI))) {
if (OtherStore->getOperand(1) != SI.getOperand(1) ||
!SI.isSameOperationAs(OtherStore))
return false;
break;
}
// If we find something that may be using or overwriting the stored
// value, or if we run out of instructions, we can't do the xform.
if (BBI->mayReadFromMemory() || BBI->mayWriteToMemory() ||
BBI == OtherBB->begin())
return false;
}
// In order to eliminate the store in OtherBr, we have to
// make sure nothing reads or overwrites the stored value in
// StoreBB.
for (BasicBlock::iterator I = StoreBB->begin(); &*I != &SI; ++I) {
// FIXME: This should really be AA driven.
if (I->mayReadFromMemory() || I->mayWriteToMemory())
return false;
}
}
// Insert a PHI node now if we need it.
Value *MergedVal = OtherStore->getOperand(0);
if (MergedVal != SI.getOperand(0)) {
PHINode *PN = PHINode::Create(MergedVal->getType(), 2, "storemerge");
PN->addIncoming(SI.getOperand(0), SI.getParent());
PN->addIncoming(OtherStore->getOperand(0), OtherBB);
MergedVal = InsertNewInstBefore(PN, DestBB->front());
}
// Advance to a place where it is safe to insert the new store and
// insert it.
BBI = DestBB->getFirstInsertionPt();
StoreInst *NewSI = new StoreInst(MergedVal, SI.getOperand(1),
SI.isVolatile(),
SI.getAlignment(),
SI.getOrdering(),
SI.getSynchScope());
InsertNewInstBefore(NewSI, *BBI);
NewSI->setDebugLoc(OtherStore->getDebugLoc());
// If the two stores had AA tags, merge them.
AAMDNodes AATags;
SI.getAAMetadata(AATags);
if (AATags) {
OtherStore->getAAMetadata(AATags, /* Merge = */ true);
NewSI->setAAMetadata(AATags);
}
// Nuke the old stores.
EraseInstFromFunction(SI);
EraseInstFromFunction(*OtherStore);
return true;
}
bool TailCallElim::ProcessReturningBlock(ReturnInst *Ret, BasicBlock *&OldEntry,
bool &TailCallsAreMarkedTail,
SmallVector<PHINode*, 8> &ArgumentPHIs,
bool CannotTailCallElimCallsMarkedTail) {
BasicBlock *BB = Ret->getParent();
Function *F = BB->getParent();
if (&BB->front() == Ret) // Make sure there is something before the ret...
return false;
// If the return is in the entry block, then making this transformation would
// turn infinite recursion into an infinite loop. This transformation is ok
// in theory, but breaks some code like:
// double fabs(double f) { return __builtin_fabs(f); } // a 'fabs' call
// disable this xform in this case, because the code generator will lower the
// call to fabs into inline code.
if (BB == &F->getEntryBlock())
return false;
// Scan backwards from the return, checking to see if there is a tail call in
// this block. If so, set CI to it.
CallInst *CI;
BasicBlock::iterator BBI = Ret;
while (1) {
CI = dyn_cast<CallInst>(BBI);
if (CI && CI->getCalledFunction() == F)
break;
if (BBI == BB->begin())
return false; // Didn't find a potential tail call.
--BBI;
}
// If this call is marked as a tail call, and if there are dynamic allocas in
// the function, we cannot perform this optimization.
if (CI->isTailCall() && CannotTailCallElimCallsMarkedTail)
return false;
// If we are introducing accumulator recursion to eliminate associative
// operations after the call instruction, this variable contains the initial
// value for the accumulator. If this value is set, we actually perform
// accumulator recursion elimination instead of simple tail recursion
// elimination.
Value *AccumulatorRecursionEliminationInitVal = 0;
Instruction *AccumulatorRecursionInstr = 0;
// Ok, we found a potential tail call. We can currently only transform the
// tail call if all of the instructions between the call and the return are
// movable to above the call itself, leaving the call next to the return.
// Check that this is the case now.
for (BBI = CI, ++BBI; &*BBI != Ret; ++BBI)
if (!CanMoveAboveCall(BBI, CI)) {
// If we can't move the instruction above the call, it might be because it
// is an associative operation that could be tranformed using accumulator
// recursion elimination. Check to see if this is the case, and if so,
// remember the initial accumulator value for later.
if ((AccumulatorRecursionEliminationInitVal =
CanTransformAccumulatorRecursion(BBI, CI))) {
// Yes, this is accumulator recursion. Remember which instruction
// accumulates.
AccumulatorRecursionInstr = BBI;
} else {
return false; // Otherwise, we cannot eliminate the tail recursion!
}
}
// We can only transform call/return pairs that either ignore the return value
// of the call and return void, ignore the value of the call and return a
// constant, return the value returned by the tail call, or that are being
// accumulator recursion variable eliminated.
if (Ret->getNumOperands() == 1 && Ret->getReturnValue() != CI &&
!isa<UndefValue>(Ret->getReturnValue()) &&
AccumulatorRecursionEliminationInitVal == 0 &&
!getCommonReturnValue(Ret, CI))
return false;
// OK! We can transform this tail call. If this is the first one found,
// create the new entry block, allowing us to branch back to the old entry.
if (OldEntry == 0) {
OldEntry = &F->getEntryBlock();
BasicBlock *NewEntry = BasicBlock::Create(F->getContext(), "", F, OldEntry);
NewEntry->takeName(OldEntry);
OldEntry->setName("tailrecurse");
BranchInst::Create(OldEntry, NewEntry);
// If this tail call is marked 'tail' and if there are any allocas in the
// entry block, move them up to the new entry block.
TailCallsAreMarkedTail = CI->isTailCall();
if (TailCallsAreMarkedTail)
// Move all fixed sized allocas from OldEntry to NewEntry.
for (BasicBlock::iterator OEBI = OldEntry->begin(), E = OldEntry->end(),
NEBI = NewEntry->begin(); OEBI != E; )
if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++))
if (isa<ConstantInt>(AI->getArraySize()))
AI->moveBefore(NEBI);
// Now that we have created a new block, which jumps to the entry
// block, insert a PHI node for each argument of the function.
// For now, we initialize each PHI to only have the real arguments
// which are passed in.
Instruction *InsertPos = OldEntry->begin();
for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end();
I != E; ++I) {
PHINode *PN = PHINode::Create(I->getType(),
I->getName() + ".tr", InsertPos);
I->replaceAllUsesWith(PN); // Everyone use the PHI node now!
PN->addIncoming(I, NewEntry);
ArgumentPHIs.push_back(PN);
}
}
// If this function has self recursive calls in the tail position where some
// are marked tail and some are not, only transform one flavor or another. We
// have to choose whether we move allocas in the entry block to the new entry
// block or not, so we can't make a good choice for both. NOTE: We could do
// slightly better here in the case that the function has no entry block
// allocas.
if (TailCallsAreMarkedTail && !CI->isTailCall())
return false;
// Ok, now that we know we have a pseudo-entry block WITH all of the
// required PHI nodes, add entries into the PHI node for the actual
// parameters passed into the tail-recursive call.
for (unsigned i = 0, e = CI->getNumOperands()-1; i != e; ++i)
ArgumentPHIs[i]->addIncoming(CI->getOperand(i+1), BB);
// If we are introducing an accumulator variable to eliminate the recursion,
// do so now. Note that we _know_ that no subsequent tail recursion
// eliminations will happen on this function because of the way the
// accumulator recursion predicate is set up.
//
if (AccumulatorRecursionEliminationInitVal) {
Instruction *AccRecInstr = AccumulatorRecursionInstr;
// Start by inserting a new PHI node for the accumulator.
PHINode *AccPN = PHINode::Create(AccRecInstr->getType(), "accumulator.tr",
OldEntry->begin());
// Loop over all of the predecessors of the tail recursion block. For the
// real entry into the function we seed the PHI with the initial value,
// computed earlier. For any other existing branches to this block (due to
// other tail recursions eliminated) the accumulator is not modified.
// Because we haven't added the branch in the current block to OldEntry yet,
// it will not show up as a predecessor.
for (pred_iterator PI = pred_begin(OldEntry), PE = pred_end(OldEntry);
PI != PE; ++PI) {
if (*PI == &F->getEntryBlock())
AccPN->addIncoming(AccumulatorRecursionEliminationInitVal, *PI);
else
AccPN->addIncoming(AccPN, *PI);
}
// Add an incoming argument for the current block, which is computed by our
// associative accumulator instruction.
AccPN->addIncoming(AccRecInstr, BB);
// Next, rewrite the accumulator recursion instruction so that it does not
// use the result of the call anymore, instead, use the PHI node we just
// inserted.
AccRecInstr->setOperand(AccRecInstr->getOperand(0) != CI, AccPN);
// Finally, rewrite any return instructions in the program to return the PHI
// node instead of the "initval" that they do currently. This loop will
// actually rewrite the return value we are destroying, but that's ok.
for (Function::iterator BBI = F->begin(), E = F->end(); BBI != E; ++BBI)
if (ReturnInst *RI = dyn_cast<ReturnInst>(BBI->getTerminator()))
RI->setOperand(0, AccPN);
++NumAccumAdded;
}
// Now that all of the PHI nodes are in place, remove the call and
// ret instructions, replacing them with an unconditional branch.
BranchInst::Create(OldEntry, Ret);
BB->getInstList().erase(Ret); // Remove return.
BB->getInstList().erase(CI); // Remove call.
++NumEliminated;
return true;
}
