SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) {
// create 2 PHIs: one for size and another for offset
PHINode *SizePHI = Builder.CreatePHI(IntTy, PHI.getNumIncomingValues());
PHINode *OffsetPHI = Builder.CreatePHI(IntTy, PHI.getNumIncomingValues());
// insert right away in the cache to handle recursive PHIs
CacheMap[&PHI] = std::make_pair(SizePHI, OffsetPHI);
// compute offset/size for each PHI incoming pointer
for (unsigned i = 0, e = PHI.getNumIncomingValues(); i != e; ++i) {
Builder.SetInsertPoint(PHI.getIncomingBlock(i)->getFirstInsertionPt());
SizeOffsetEvalType EdgeData = compute_(PHI.getIncomingValue(i));
if (!bothKnown(EdgeData)) {
OffsetPHI->replaceAllUsesWith(UndefValue::get(IntTy));
OffsetPHI->eraseFromParent();
SizePHI->replaceAllUsesWith(UndefValue::get(IntTy));
SizePHI->eraseFromParent();
return unknown();
}
SizePHI->addIncoming(EdgeData.first, PHI.getIncomingBlock(i));
OffsetPHI->addIncoming(EdgeData.second, PHI.getIncomingBlock(i));
}
Value *Size = SizePHI, *Offset = OffsetPHI, *Tmp;
if ((Tmp = SizePHI->hasConstantValue())) {
Size = Tmp;
SizePHI->replaceAllUsesWith(Size);
SizePHI->eraseFromParent();
}
if ((Tmp = OffsetPHI->hasConstantValue())) {
Offset = Tmp;
OffsetPHI->replaceAllUsesWith(Offset);
OffsetPHI->eraseFromParent();
}
return std::make_pair(Size, Offset);
}
void PromoteMem2Reg::run() {
Function &F = *DF.getRoot()->getParent();
if (AST) PointerAllocaValues.resize(Allocas.size());
AllocaDbgDeclares.resize(Allocas.size());
AllocaInfo Info;
LargeBlockInfo LBI;
for (unsigned AllocaNum = 0; AllocaNum != Allocas.size(); ++AllocaNum) {
AllocaInst *AI = Allocas[AllocaNum];
assert(isAllocaPromotable(AI) &&
"Cannot promote non-promotable alloca!");
assert(AI->getParent()->getParent() == &F &&
"All allocas should be in the same function, which is same as DF!");
if (AI->use_empty()) {
// If there are no uses of the alloca, just delete it now.
if (AST) AST->deleteValue(AI);
AI->eraseFromParent();
// Remove the alloca from the Allocas list, since it has been processed
RemoveFromAllocasList(AllocaNum);
++NumDeadAlloca;
continue;
}
// Calculate the set of read and write-locations for each alloca. This is
// analogous to finding the 'uses' and 'definitions' of each variable.
Info.AnalyzeAlloca(AI);
// If there is only a single store to this value, replace any loads of
// it that are directly dominated by the definition with the value stored.
if (Info.DefiningBlocks.size() == 1) {
RewriteSingleStoreAlloca(AI, Info, LBI);
// Finally, after the scan, check to see if the store is all that is left.
if (Info.UsingBlocks.empty()) {
// Record debuginfo for the store and remove the declaration's debuginfo.
if (DbgDeclareInst *DDI = Info.DbgDeclare) {
ConvertDebugDeclareToDebugValue(DDI, Info.OnlyStore);
DDI->eraseFromParent();
}
// Remove the (now dead) store and alloca.
Info.OnlyStore->eraseFromParent();
LBI.deleteValue(Info.OnlyStore);
if (AST) AST->deleteValue(AI);
AI->eraseFromParent();
LBI.deleteValue(AI);
// The alloca has been processed, move on.
RemoveFromAllocasList(AllocaNum);
++NumSingleStore;
continue;
}
}
// If the alloca is only read and written in one basic block, just perform a
// linear sweep over the block to eliminate it.
if (Info.OnlyUsedInOneBlock) {
PromoteSingleBlockAlloca(AI, Info, LBI);
// Finally, after the scan, check to see if the stores are all that is
// left.
if (Info.UsingBlocks.empty()) {
// Remove the (now dead) stores and alloca.
while (!AI->use_empty()) {
StoreInst *SI = cast<StoreInst>(AI->use_back());
// Record debuginfo for the store before removing it.
if (DbgDeclareInst *DDI = Info.DbgDeclare)
ConvertDebugDeclareToDebugValue(DDI, SI);
SI->eraseFromParent();
LBI.deleteValue(SI);
}
if (AST) AST->deleteValue(AI);
AI->eraseFromParent();
LBI.deleteValue(AI);
// The alloca has been processed, move on.
RemoveFromAllocasList(AllocaNum);
// The alloca's debuginfo can be removed as well.
if (DbgDeclareInst *DDI = Info.DbgDeclare)
DDI->eraseFromParent();
++NumLocalPromoted;
continue;
}
}
// If we haven't computed a numbering for the BB's in the function, do so
// now.
if (BBNumbers.empty()) {
unsigned ID = 0;
for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
BBNumbers[I] = ID++;
}
// If we have an AST to keep updated, remember some pointer value that is
// stored into the alloca.
if (AST)
PointerAllocaValues[AllocaNum] = Info.AllocaPointerVal;
// Remember the dbg.declare intrinsic describing this alloca, if any.
if (Info.DbgDeclare) AllocaDbgDeclares[AllocaNum] = Info.DbgDeclare;
// Keep the reverse mapping of the 'Allocas' array for the rename pass.
AllocaLookup[Allocas[AllocaNum]] = AllocaNum;
// At this point, we're committed to promoting the alloca using IDF's, and
// the standard SSA construction algorithm. Determine which blocks need PHI
// nodes and see if we can optimize out some work by avoiding insertion of
// dead phi nodes.
DetermineInsertionPoint(AI, AllocaNum, Info);
}
if (Allocas.empty())
return; // All of the allocas must have been trivial!
LBI.clear();
// Set the incoming values for the basic block to be null values for all of
// the alloca's. We do this in case there is a load of a value that has not
// been stored yet. In this case, it will get this null value.
//
RenamePassData::ValVector Values(Allocas.size());
for (unsigned i = 0, e = Allocas.size(); i != e; ++i)
Values[i] = UndefValue::get(Allocas[i]->getAllocatedType());
// Walks all basic blocks in the function performing the SSA rename algorithm
// and inserting the phi nodes we marked as necessary
//
std::vector<RenamePassData> RenamePassWorkList;
RenamePassWorkList.push_back(RenamePassData(F.begin(), 0, Values));
do {
RenamePassData RPD;
RPD.swap(RenamePassWorkList.back());
RenamePassWorkList.pop_back();
// RenamePass may add new worklist entries.
RenamePass(RPD.BB, RPD.Pred, RPD.Values, RenamePassWorkList);
} while (!RenamePassWorkList.empty());
// The renamer uses the Visited set to avoid infinite loops. Clear it now.
Visited.clear();
// Remove the allocas themselves from the function.
for (unsigned i = 0, e = Allocas.size(); i != e; ++i) {
Instruction *A = Allocas[i];
// If there are any uses of the alloca instructions left, they must be in
// sections of dead code that were not processed on the dominance frontier.
// Just delete the users now.
//
if (!A->use_empty())
A->replaceAllUsesWith(UndefValue::get(A->getType()));
if (AST) AST->deleteValue(A);
A->eraseFromParent();
}
// Remove alloca's dbg.declare instrinsics from the function.
for (unsigned i = 0, e = AllocaDbgDeclares.size(); i != e; ++i)
if (DbgDeclareInst *DDI = AllocaDbgDeclares[i])
DDI->eraseFromParent();
// Loop over all of the PHI nodes and see if there are any that we can get
// rid of because they merge all of the same incoming values. This can
// happen due to undef values coming into the PHI nodes. This process is
// iterative, because eliminating one PHI node can cause others to be removed.
bool EliminatedAPHI = true;
while (EliminatedAPHI) {
EliminatedAPHI = false;
for (DenseMap<std::pair<BasicBlock*, unsigned>, PHINode*>::iterator I =
NewPhiNodes.begin(), E = NewPhiNodes.end(); I != E;) {
PHINode *PN = I->second;
// If this PHI node merges one value and/or undefs, get the value.
if (Value *V = PN->hasConstantValue(&DT)) {
if (AST && PN->getType()->isPointerTy())
AST->deleteValue(PN);
PN->replaceAllUsesWith(V);
PN->eraseFromParent();
NewPhiNodes.erase(I++);
EliminatedAPHI = true;
continue;
}
++I;
}
}
// At this point, the renamer has added entries to PHI nodes for all reachable
// code. Unfortunately, there may be unreachable blocks which the renamer
// hasn't traversed. If this is the case, the PHI nodes may not
// have incoming values for all predecessors. Loop over all PHI nodes we have
// created, inserting undef values if they are missing any incoming values.
//
for (DenseMap<std::pair<BasicBlock*, unsigned>, PHINode*>::iterator I =
NewPhiNodes.begin(), E = NewPhiNodes.end(); I != E; ++I) {
// We want to do this once per basic block. As such, only process a block
// when we find the PHI that is the first entry in the block.
PHINode *SomePHI = I->second;
BasicBlock *BB = SomePHI->getParent();
if (&BB->front() != SomePHI)
continue;
// Only do work here if there the PHI nodes are missing incoming values. We
// know that all PHI nodes that were inserted in a block will have the same
// number of incoming values, so we can just check any of them.
if (SomePHI->getNumIncomingValues() == getNumPreds(BB))
continue;
// Get the preds for BB.
SmallVector<BasicBlock*, 16> Preds(pred_begin(BB), pred_end(BB));
// Ok, now we know that all of the PHI nodes are missing entries for some
// basic blocks. Start by sorting the incoming predecessors for efficient
// access.
std::sort(Preds.begin(), Preds.end());
// Now we loop through all BB's which have entries in SomePHI and remove
// them from the Preds list.
for (unsigned i = 0, e = SomePHI->getNumIncomingValues(); i != e; ++i) {
// Do a log(n) search of the Preds list for the entry we want.
SmallVector<BasicBlock*, 16>::iterator EntIt =
std::lower_bound(Preds.begin(), Preds.end(),
SomePHI->getIncomingBlock(i));
assert(EntIt != Preds.end() && *EntIt == SomePHI->getIncomingBlock(i)&&
"PHI node has entry for a block which is not a predecessor!");
// Remove the entry
Preds.erase(EntIt);
}
// At this point, the blocks left in the preds list must have dummy
// entries inserted into every PHI nodes for the block. Update all the phi
// nodes in this block that we are inserting (there could be phis before
// mem2reg runs).
unsigned NumBadPreds = SomePHI->getNumIncomingValues();
BasicBlock::iterator BBI = BB->begin();
while ((SomePHI = dyn_cast<PHINode>(BBI++)) &&
SomePHI->getNumIncomingValues() == NumBadPreds) {
Value *UndefVal = UndefValue::get(SomePHI->getType());
for (unsigned pred = 0, e = Preds.size(); pred != e; ++pred)
SomePHI->addIncoming(UndefVal, Preds[pred]);
}
}
NewPhiNodes.clear();
}
/// removePredecessor - This method is used to notify a BasicBlock that the
/// specified Predecessor of the block is no longer able to reach it. This is
/// actually not used to update the Predecessor list, but is actually used to
/// update the PHI nodes that reside in the block. Note that this should be
/// called while the predecessor still refers to this block.
///
void BasicBlock::removePredecessor(BasicBlock *Pred,
bool DontDeleteUselessPHIs) {
assert((hasNUsesOrMore(16)||// Reduce cost of this assertion for complex CFGs.
find(pred_begin(this), pred_end(this), Pred) != pred_end(this)) &&
"removePredecessor: BB is not a predecessor!");
if (InstList.empty()) return;
PHINode *APN = dyn_cast<PHINode>(&front());
if (!APN) return; // Quick exit.
// If there are exactly two predecessors, then we want to nuke the PHI nodes
// altogether. However, we cannot do this, if this in this case:
//
// Loop:
// %x = phi [X, Loop]
// %x2 = add %x, 1 ;; This would become %x2 = add %x2, 1
// br Loop ;; %x2 does not dominate all uses
//
// This is because the PHI node input is actually taken from the predecessor
// basic block. The only case this can happen is with a self loop, so we
// check for this case explicitly now.
//
unsigned max_idx = APN->getNumIncomingValues();
assert(max_idx != 0 && "PHI Node in block with 0 predecessors!?!?!");
if (max_idx == 2) {
BasicBlock *Other = APN->getIncomingBlock(APN->getIncomingBlock(0) == Pred);
// Disable PHI elimination!
if (this == Other) max_idx = 3;
}
// <= Two predecessors BEFORE I remove one?
if (max_idx <= 2 && !DontDeleteUselessPHIs) {
// Yup, loop through and nuke the PHI nodes
while (PHINode *PN = dyn_cast<PHINode>(&front())) {
// Remove the predecessor first.
PN->removeIncomingValue(Pred, !DontDeleteUselessPHIs);
// If the PHI _HAD_ two uses, replace PHI node with its now *single* value
if (max_idx == 2) {
if (PN->getIncomingValue(0) != PN)
PN->replaceAllUsesWith(PN->getIncomingValue(0));
else
// We are left with an infinite loop with no entries: kill the PHI.
PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
getInstList().pop_front(); // Remove the PHI node
}
// If the PHI node already only had one entry, it got deleted by
// removeIncomingValue.
}
} else {
// Okay, now we know that we need to remove predecessor #pred_idx from all
// PHI nodes. Iterate over each PHI node fixing them up
PHINode *PN;
for (iterator II = begin(); (PN = dyn_cast<PHINode>(II)); ) {
++II;
PN->removeIncomingValue(Pred, false);
// If all incoming values to the Phi are the same, we can replace the Phi
// with that value.
Value* PNV = 0;
if (!DontDeleteUselessPHIs && (PNV = PN->hasConstantValue()))
if (PNV != PN) {
PN->replaceAllUsesWith(PNV);
PN->eraseFromParent();
}
}
}
}
/// GetValueInMiddleOfBlock - Construct SSA form, materializing a value that
/// is live in the middle of the specified block.
///
/// GetValueInMiddleOfBlock is the same as GetValueAtEndOfBlock except in one
/// important case: if there is a definition of the rewritten value after the
/// 'use' in BB. Consider code like this:
///
/// X1 = ...
/// SomeBB:
/// use(X)
/// X2 = ...
/// br Cond, SomeBB, OutBB
///
/// In this case, there are two values (X1 and X2) added to the AvailableVals
/// set by the client of the rewriter, and those values are both live out of
/// their respective blocks. However, the use of X happens in the *middle* of
/// a block. Because of this, we need to insert a new PHI node in SomeBB to
/// merge the appropriate values, and this value isn't live out of the block.
///
Value *SSAUpdater::GetValueInMiddleOfBlock(BasicBlock *BB) {
// If there is no definition of the renamed variable in this block, just use
// GetValueAtEndOfBlock to do our work.
if (!HasValueForBlock(BB))
return GetValueAtEndOfBlock(BB);
// Otherwise, we have the hard case. Get the live-in values for each
// predecessor.
SmallVector<std::pair<BasicBlock*, Value*>, 8> PredValues;
Value *SingularValue = 0;
// We can get our predecessor info by walking the pred_iterator list, but it
// is relatively slow. If we already have PHI nodes in this block, walk one
// of them to get the predecessor list instead.
if (PHINode *SomePhi = dyn_cast<PHINode>(BB->begin())) {
for (unsigned i = 0, e = SomePhi->getNumIncomingValues(); i != e; ++i) {
BasicBlock *PredBB = SomePhi->getIncomingBlock(i);
Value *PredVal = GetValueAtEndOfBlock(PredBB);
PredValues.push_back(std::make_pair(PredBB, PredVal));
// Compute SingularValue.
if (i == 0)
SingularValue = PredVal;
else if (PredVal != SingularValue)
SingularValue = 0;
}
} else {
bool isFirstPred = true;
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
BasicBlock *PredBB = *PI;
Value *PredVal = GetValueAtEndOfBlock(PredBB);
PredValues.push_back(std::make_pair(PredBB, PredVal));
// Compute SingularValue.
if (isFirstPred) {
SingularValue = PredVal;
isFirstPred = false;
} else if (PredVal != SingularValue)
SingularValue = 0;
}
}
// If there are no predecessors, just return undef.
if (PredValues.empty())
return UndefValue::get(PrototypeValue->getType());
// Otherwise, if all the merged values are the same, just use it.
if (SingularValue != 0)
return SingularValue;
// Otherwise, we do need a PHI: check to see if we already have one available
// in this block that produces the right value.
if (isa<PHINode>(BB->begin())) {
DenseMap<BasicBlock*, Value*> ValueMapping(PredValues.begin(),
PredValues.end());
PHINode *SomePHI;
for (BasicBlock::iterator It = BB->begin();
(SomePHI = dyn_cast<PHINode>(It)); ++It) {
if (IsEquivalentPHI(SomePHI, ValueMapping))
return SomePHI;
}
}
// Ok, we have no way out, insert a new one now.
PHINode *InsertedPHI = PHINode::Create(PrototypeValue->getType(),
PrototypeValue->getName(),
&BB->front());
InsertedPHI->reserveOperandSpace(PredValues.size());
// Fill in all the predecessors of the PHI.
for (unsigned i = 0, e = PredValues.size(); i != e; ++i)
InsertedPHI->addIncoming(PredValues[i].second, PredValues[i].first);
// See if the PHI node can be merged to a single value. This can happen in
// loop cases when we get a PHI of itself and one other value.
if (Value *ConstVal = InsertedPHI->hasConstantValue()) {
InsertedPHI->eraseFromParent();
return ConstVal;
}
// If the client wants to know about all new instructions, tell it.
if (InsertedPHIs) InsertedPHIs->push_back(InsertedPHI);
DEBUG(dbgs() << " Inserted PHI: " << *InsertedPHI << "\n");
return InsertedPHI;
}
/// SplitBlockPredecessors - This method transforms BB by introducing a new
/// basic block into the function, and moving some of the predecessors of BB to
/// be predecessors of the new block. The new predecessors are indicated by the
/// Preds array, which has NumPreds elements in it. The new block is given a
/// suffix of 'Suffix'.
///
/// This currently updates the LLVM IR, AliasAnalysis, DominatorTree and
/// DominanceFrontier, but no other analyses.
BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
BasicBlock *const *Preds,
unsigned NumPreds, const char *Suffix,
Pass *P) {
// Create new basic block, insert right before the original block.
BasicBlock *NewBB =
BasicBlock::Create(BB->getName()+Suffix, BB->getParent(), BB);
// The new block unconditionally branches to the old block.
BranchInst *BI = BranchInst::Create(BB, NewBB);
// Move the edges from Preds to point to NewBB instead of BB.
for (unsigned i = 0; i != NumPreds; ++i)
Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB);
// Update dominator tree and dominator frontier if available.
DominatorTree *DT = P ? P->getAnalysisIfAvailable<DominatorTree>() : 0;
if (DT)
DT->splitBlock(NewBB);
if (DominanceFrontier *DF = P ? P->getAnalysisIfAvailable<DominanceFrontier>():0)
DF->splitBlock(NewBB);
AliasAnalysis *AA = P ? P->getAnalysisIfAvailable<AliasAnalysis>() : 0;
// Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
// node becomes an incoming value for BB's phi node. However, if the Preds
// list is empty, we need to insert dummy entries into the PHI nodes in BB to
// account for the newly created predecessor.
if (NumPreds == 0) {
// Insert dummy values as the incoming value.
for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
cast<PHINode>(I)->addIncoming(UndefValue::get(I->getType()), NewBB);
return NewBB;
}
// Otherwise, create a new PHI node in NewBB for each PHI node in BB.
for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ) {
PHINode *PN = cast<PHINode>(I++);
// Check to see if all of the values coming in are the same. If so, we
// don't need to create a new PHI node.
Value *InVal = PN->getIncomingValueForBlock(Preds[0]);
for (unsigned i = 1; i != NumPreds; ++i)
if (InVal != PN->getIncomingValueForBlock(Preds[i])) {
InVal = 0;
break;
}
if (InVal) {
// If all incoming values for the new PHI would be the same, just don't
// make a new PHI. Instead, just remove the incoming values from the old
// PHI.
for (unsigned i = 0; i != NumPreds; ++i)
PN->removeIncomingValue(Preds[i], false);
} else {
// If the values coming into the block are not the same, we need a PHI.
// Create the new PHI node, insert it into NewBB at the end of the block
PHINode *NewPHI =
PHINode::Create(PN->getType(), PN->getName()+".ph", BI);
if (AA) AA->copyValue(PN, NewPHI);
// Move all of the PHI values for 'Preds' to the new PHI.
for (unsigned i = 0; i != NumPreds; ++i) {
Value *V = PN->removeIncomingValue(Preds[i], false);
NewPHI->addIncoming(V, Preds[i]);
}
InVal = NewPHI;
}
// Add an incoming value to the PHI node in the loop for the preheader
// edge.
PN->addIncoming(InVal, NewBB);
// Check to see if we can eliminate this phi node.
if (Value *V = PN->hasConstantValue(DT != 0)) {
Instruction *I = dyn_cast<Instruction>(V);
if (!I || DT == 0 || DT->dominates(I, PN)) {
PN->replaceAllUsesWith(V);
if (AA) AA->deleteValue(PN);
PN->eraseFromParent();
}
}
}
return NewBB;
}
// InlineFunction - This function inlines the called function into the basic
// block of the caller. This returns false if it is not possible to inline this
// call. The program is still in a well defined state if this occurs though.
//
// Note that this only does one level of inlining. For example, if the
// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now
// exists in the instruction stream. Similiarly this will inline a recursive
// function by one level.
//
bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI) {
Instruction *TheCall = CS.getInstruction();
LLVMContext &Context = TheCall->getContext();
assert(TheCall->getParent() && TheCall->getParent()->getParent() &&
"Instruction not in function!");
// If IFI has any state in it, zap it before we fill it in.
IFI.reset();
const Function *CalledFunc = CS.getCalledFunction();
if (CalledFunc == 0 || // Can't inline external function or indirect
CalledFunc->isDeclaration() || // call, or call to a vararg function!
CalledFunc->getFunctionType()->isVarArg()) return false;
// If the call to the callee is not a tail call, we must clear the 'tail'
// flags on any calls that we inline.
bool MustClearTailCallFlags =
!(isa<CallInst>(TheCall) && cast<CallInst>(TheCall)->isTailCall());
// If the call to the callee cannot throw, set the 'nounwind' flag on any
// calls that we inline.
bool MarkNoUnwind = CS.doesNotThrow();
BasicBlock *OrigBB = TheCall->getParent();
Function *Caller = OrigBB->getParent();
// GC poses two hazards to inlining, which only occur when the callee has GC:
// 1. If the caller has no GC, then the callee's GC must be propagated to the
// caller.
// 2. If the caller has a differing GC, it is invalid to inline.
if (CalledFunc->hasGC()) {
if (!Caller->hasGC())
Caller->setGC(CalledFunc->getGC());
else if (CalledFunc->getGC() != Caller->getGC())
return false;
}
// Get an iterator to the last basic block in the function, which will have
// the new function inlined after it.
//
Function::iterator LastBlock = &Caller->back();
// Make sure to capture all of the return instructions from the cloned
// function.
SmallVector<ReturnInst*, 8> Returns;
ClonedCodeInfo InlinedFunctionInfo;
Function::iterator FirstNewBlock;
{ // Scope to destroy VMap after cloning.
ValueMap<const Value*, Value*> VMap;
assert(CalledFunc->arg_size() == CS.arg_size() &&
"No varargs calls can be inlined!");
// Calculate the vector of arguments to pass into the function cloner, which
// matches up the formal to the actual argument values.
CallSite::arg_iterator AI = CS.arg_begin();
unsigned ArgNo = 0;
for (Function::const_arg_iterator I = CalledFunc->arg_begin(),
E = CalledFunc->arg_end(); I != E; ++I, ++AI, ++ArgNo) {
Value *ActualArg = *AI;
// When byval arguments actually inlined, we need to make the copy implied
// by them explicit. However, we don't do this if the callee is readonly
// or readnone, because the copy would be unneeded: the callee doesn't
// modify the struct.
if (CalledFunc->paramHasAttr(ArgNo+1, Attribute::ByVal) &&
!CalledFunc->onlyReadsMemory()) {
const Type *AggTy = cast<PointerType>(I->getType())->getElementType();
const Type *VoidPtrTy =
Type::getInt8PtrTy(Context);
// Create the alloca. If we have TargetData, use nice alignment.
unsigned Align = 1;
if (IFI.TD) Align = IFI.TD->getPrefTypeAlignment(AggTy);
Value *NewAlloca = new AllocaInst(AggTy, 0, Align,
I->getName(),
&*Caller->begin()->begin());
// Emit a memcpy.
const Type *Tys[3] = {VoidPtrTy, VoidPtrTy, Type::getInt64Ty(Context)};
Function *MemCpyFn = Intrinsic::getDeclaration(Caller->getParent(),
Intrinsic::memcpy,
Tys, 3);
Value *DestCast = new BitCastInst(NewAlloca, VoidPtrTy, "tmp", TheCall);
Value *SrcCast = new BitCastInst(*AI, VoidPtrTy, "tmp", TheCall);
Value *Size;
if (IFI.TD == 0)
Size = ConstantExpr::getSizeOf(AggTy);
else
Size = ConstantInt::get(Type::getInt64Ty(Context),
IFI.TD->getTypeStoreSize(AggTy));
// Always generate a memcpy of alignment 1 here because we don't know
// the alignment of the src pointer. Other optimizations can infer
// better alignment.
Value *CallArgs[] = {
DestCast, SrcCast, Size,
ConstantInt::get(Type::getInt32Ty(Context), 1),
ConstantInt::get(Type::getInt1Ty(Context), 0)
};
CallInst *TheMemCpy =
CallInst::Create(MemCpyFn, CallArgs, CallArgs+5, "", TheCall);
// If we have a call graph, update it.
if (CallGraph *CG = IFI.CG) {
CallGraphNode *MemCpyCGN = CG->getOrInsertFunction(MemCpyFn);
CallGraphNode *CallerNode = (*CG)[Caller];
CallerNode->addCalledFunction(TheMemCpy, MemCpyCGN);
}
// Uses of the argument in the function should use our new alloca
// instead.
ActualArg = NewAlloca;
// Calls that we inline may use the new alloca, so we need to clear
// their 'tail' flags.
MustClearTailCallFlags = true;
}
VMap[I] = ActualArg;
}
// We want the inliner to prune the code as it copies. We would LOVE to
// have no dead or constant instructions leftover after inlining occurs
// (which can happen, e.g., because an argument was constant), but we'll be
// happy with whatever the cloner can do.
CloneAndPruneFunctionInto(Caller, CalledFunc, VMap, Returns, ".i",
&InlinedFunctionInfo, IFI.TD, TheCall);
// Remember the first block that is newly cloned over.
FirstNewBlock = LastBlock; ++FirstNewBlock;
// Update the callgraph if requested.
if (IFI.CG)
UpdateCallGraphAfterInlining(CS, FirstNewBlock, VMap, IFI);
}
// If there are any alloca instructions in the block that used to be the entry
// block for the callee, move them to the entry block of the caller. First
// calculate which instruction they should be inserted before. We insert the
// instructions at the end of the current alloca list.
//
{
BasicBlock::iterator InsertPoint = Caller->begin()->begin();
for (BasicBlock::iterator I = FirstNewBlock->begin(),
E = FirstNewBlock->end(); I != E; ) {
AllocaInst *AI = dyn_cast<AllocaInst>(I++);
if (AI == 0) continue;
// If the alloca is now dead, remove it. This often occurs due to code
// specialization.
if (AI->use_empty()) {
AI->eraseFromParent();
continue;
}
if (!isa<Constant>(AI->getArraySize()))
continue;
// Keep track of the static allocas that we inline into the caller if the
// StaticAllocas pointer is non-null.
IFI.StaticAllocas.push_back(AI);
// Scan for the block of allocas that we can move over, and move them
// all at once.
while (isa<AllocaInst>(I) &&
isa<Constant>(cast<AllocaInst>(I)->getArraySize())) {
IFI.StaticAllocas.push_back(cast<AllocaInst>(I));
++I;
}
// Transfer all of the allocas over in a block. Using splice means
// that the instructions aren't removed from the symbol table, then
// reinserted.
Caller->getEntryBlock().getInstList().splice(InsertPoint,
FirstNewBlock->getInstList(),
AI, I);
}
}
// If the inlined code contained dynamic alloca instructions, wrap the inlined
// code with llvm.stacksave/llvm.stackrestore intrinsics.
if (InlinedFunctionInfo.ContainsDynamicAllocas) {
Module *M = Caller->getParent();
// Get the two intrinsics we care about.
Function *StackSave = Intrinsic::getDeclaration(M, Intrinsic::stacksave);
Function *StackRestore=Intrinsic::getDeclaration(M,Intrinsic::stackrestore);
// If we are preserving the callgraph, add edges to the stacksave/restore
// functions for the calls we insert.
CallGraphNode *StackSaveCGN = 0, *StackRestoreCGN = 0, *CallerNode = 0;
if (CallGraph *CG = IFI.CG) {
StackSaveCGN = CG->getOrInsertFunction(StackSave);
StackRestoreCGN = CG->getOrInsertFunction(StackRestore);
CallerNode = (*CG)[Caller];
}
// Insert the llvm.stacksave.
CallInst *SavedPtr = CallInst::Create(StackSave, "savedstack",
FirstNewBlock->begin());
if (IFI.CG) CallerNode->addCalledFunction(SavedPtr, StackSaveCGN);
// Insert a call to llvm.stackrestore before any return instructions in the
// inlined function.
for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
CallInst *CI = CallInst::Create(StackRestore, SavedPtr, "", Returns[i]);
if (IFI.CG) CallerNode->addCalledFunction(CI, StackRestoreCGN);
}
// Count the number of StackRestore calls we insert.
unsigned NumStackRestores = Returns.size();
// If we are inlining an invoke instruction, insert restores before each
// unwind. These unwinds will be rewritten into branches later.
if (InlinedFunctionInfo.ContainsUnwinds && isa<InvokeInst>(TheCall)) {
for (Function::iterator BB = FirstNewBlock, E = Caller->end();
BB != E; ++BB)
if (UnwindInst *UI = dyn_cast<UnwindInst>(BB->getTerminator())) {
CallInst *CI = CallInst::Create(StackRestore, SavedPtr, "", UI);
if (IFI.CG) CallerNode->addCalledFunction(CI, StackRestoreCGN);
++NumStackRestores;
}
}
}
// If we are inlining tail call instruction through a call site that isn't
// marked 'tail', we must remove the tail marker for any calls in the inlined
// code. Also, calls inlined through a 'nounwind' call site should be marked
// 'nounwind'.
if (InlinedFunctionInfo.ContainsCalls &&
(MustClearTailCallFlags || MarkNoUnwind)) {
for (Function::iterator BB = FirstNewBlock, E = Caller->end();
BB != E; ++BB)
for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
if (CallInst *CI = dyn_cast<CallInst>(I)) {
if (MustClearTailCallFlags)
CI->setTailCall(false);
if (MarkNoUnwind)
CI->setDoesNotThrow();
}
}
// If we are inlining through a 'nounwind' call site then any inlined 'unwind'
// instructions are unreachable.
if (InlinedFunctionInfo.ContainsUnwinds && MarkNoUnwind)
for (Function::iterator BB = FirstNewBlock, E = Caller->end();
BB != E; ++BB) {
TerminatorInst *Term = BB->getTerminator();
if (isa<UnwindInst>(Term)) {
new UnreachableInst(Context, Term);
BB->getInstList().erase(Term);
}
}
// If we are inlining for an invoke instruction, we must make sure to rewrite
// any inlined 'unwind' instructions into branches to the invoke exception
// destination, and call instructions into invoke instructions.
if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall))
HandleInlinedInvoke(II, FirstNewBlock, InlinedFunctionInfo);
// If we cloned in _exactly one_ basic block, and if that block ends in a
// return instruction, we splice the body of the inlined callee directly into
// the calling basic block.
if (Returns.size() == 1 && std::distance(FirstNewBlock, Caller->end()) == 1) {
// Move all of the instructions right before the call.
OrigBB->getInstList().splice(TheCall, FirstNewBlock->getInstList(),
FirstNewBlock->begin(), FirstNewBlock->end());
// Remove the cloned basic block.
Caller->getBasicBlockList().pop_back();
// If the call site was an invoke instruction, add a branch to the normal
// destination.
if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall))
BranchInst::Create(II->getNormalDest(), TheCall);
// If the return instruction returned a value, replace uses of the call with
// uses of the returned value.
if (!TheCall->use_empty()) {
ReturnInst *R = Returns[0];
if (TheCall == R->getReturnValue())
TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
else
TheCall->replaceAllUsesWith(R->getReturnValue());
}
// Since we are now done with the Call/Invoke, we can delete it.
TheCall->eraseFromParent();
// Since we are now done with the return instruction, delete it also.
Returns[0]->eraseFromParent();
// We are now done with the inlining.
return true;
}
// Otherwise, we have the normal case, of more than one block to inline or
// multiple return sites.
// We want to clone the entire callee function into the hole between the
// "starter" and "ender" blocks. How we accomplish this depends on whether
// this is an invoke instruction or a call instruction.
BasicBlock *AfterCallBB;
if (InvokeInst *II = dyn_cast<InvokeInst>(TheCall)) {
// Add an unconditional branch to make this look like the CallInst case...
BranchInst *NewBr = BranchInst::Create(II->getNormalDest(), TheCall);
// Split the basic block. This guarantees that no PHI nodes will have to be
// updated due to new incoming edges, and make the invoke case more
// symmetric to the call case.
AfterCallBB = OrigBB->splitBasicBlock(NewBr,
CalledFunc->getName()+".exit");
} else { // It's a call
// If this is a call instruction, we need to split the basic block that
// the call lives in.
//
AfterCallBB = OrigBB->splitBasicBlock(TheCall,
CalledFunc->getName()+".exit");
}
// Change the branch that used to go to AfterCallBB to branch to the first
// basic block of the inlined function.
//
TerminatorInst *Br = OrigBB->getTerminator();
assert(Br && Br->getOpcode() == Instruction::Br &&
"splitBasicBlock broken!");
Br->setOperand(0, FirstNewBlock);
// Now that the function is correct, make it a little bit nicer. In
// particular, move the basic blocks inserted from the end of the function
// into the space made by splitting the source basic block.
Caller->getBasicBlockList().splice(AfterCallBB, Caller->getBasicBlockList(),
FirstNewBlock, Caller->end());
// Handle all of the return instructions that we just cloned in, and eliminate
// any users of the original call/invoke instruction.
const Type *RTy = CalledFunc->getReturnType();
if (Returns.size() > 1) {
// The PHI node should go at the front of the new basic block to merge all
// possible incoming values.
PHINode *PHI = 0;
if (!TheCall->use_empty()) {
PHI = PHINode::Create(RTy, TheCall->getName(),
AfterCallBB->begin());
// Anything that used the result of the function call should now use the
// PHI node as their operand.
TheCall->replaceAllUsesWith(PHI);
}
// Loop over all of the return instructions adding entries to the PHI node
// as appropriate.
if (PHI) {
for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
ReturnInst *RI = Returns[i];
assert(RI->getReturnValue()->getType() == PHI->getType() &&
"Ret value not consistent in function!");
PHI->addIncoming(RI->getReturnValue(), RI->getParent());
}
// Now that we inserted the PHI, check to see if it has a single value
// (e.g. all the entries are the same or undef). If so, remove the PHI so
// it doesn't block other optimizations.
if (Value *V = PHI->hasConstantValue()) {
PHI->replaceAllUsesWith(V);
PHI->eraseFromParent();
}
}
// Add a branch to the merge points and remove return instructions.
for (unsigned i = 0, e = Returns.size(); i != e; ++i) {
ReturnInst *RI = Returns[i];
BranchInst::Create(AfterCallBB, RI);
RI->eraseFromParent();
}
} else if (!Returns.empty()) {
// Otherwise, if there is exactly one return value, just replace anything
// using the return value of the call with the computed value.
if (!TheCall->use_empty()) {
if (TheCall == Returns[0]->getReturnValue())
TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
else
TheCall->replaceAllUsesWith(Returns[0]->getReturnValue());
}
// Splice the code from the return block into the block that it will return
// to, which contains the code that was after the call.
BasicBlock *ReturnBB = Returns[0]->getParent();
AfterCallBB->getInstList().splice(AfterCallBB->begin(),
ReturnBB->getInstList());
// Update PHI nodes that use the ReturnBB to use the AfterCallBB.
ReturnBB->replaceAllUsesWith(AfterCallBB);
// Delete the return instruction now and empty ReturnBB now.
Returns[0]->eraseFromParent();
ReturnBB->eraseFromParent();
} else if (!TheCall->use_empty()) {
// No returns, but something is using the return value of the call. Just
// nuke the result.
TheCall->replaceAllUsesWith(UndefValue::get(TheCall->getType()));
}
// Since we are now done with the Call/Invoke, we can delete it.
TheCall->eraseFromParent();
// We should always be able to fold the entry block of the function into the
// single predecessor of the block...
assert(cast<BranchInst>(Br)->isUnconditional() && "splitBasicBlock broken!");
BasicBlock *CalleeEntry = cast<BranchInst>(Br)->getSuccessor(0);
// Splice the code entry block into calling block, right before the
// unconditional branch.
OrigBB->getInstList().splice(Br, CalleeEntry->getInstList());
CalleeEntry->replaceAllUsesWith(OrigBB); // Update PHI nodes
// Remove the unconditional branch.
OrigBB->getInstList().erase(Br);
// Now we can remove the CalleeEntry block, which is now empty.
Caller->getBasicBlockList().erase(CalleeEntry);
return true;
}
bool TailCallElim::runOnFunction(Function &F) {
// If this function is a varargs function, we won't be able to PHI the args
// right, so don't even try to convert it...
if (F.getFunctionType()->isVarArg()) return false;
BasicBlock *OldEntry = 0;
bool TailCallsAreMarkedTail = false;
SmallVector<PHINode*, 8> ArgumentPHIs;
bool MadeChange = false;
bool FunctionContainsEscapingAllocas = false;
// CannotTCETailMarkedCall - If true, we cannot perform TCE on tail calls
// marked with the 'tail' attribute, because doing so would cause the stack
// size to increase (real TCE would deallocate variable sized allocas, TCE
// doesn't).
bool CannotTCETailMarkedCall = false;
// Loop over the function, looking for any returning blocks, and keeping track
// of whether this function has any non-trivially used allocas.
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
if (FunctionContainsEscapingAllocas && CannotTCETailMarkedCall)
break;
FunctionContainsEscapingAllocas |=
CheckForEscapingAllocas(BB, CannotTCETailMarkedCall);
}
/// FIXME: The code generator produces really bad code when an 'escaping
/// alloca' is changed from being a static alloca to being a dynamic alloca.
/// Until this is resolved, disable this transformation if that would ever
/// happen. This bug is PR962.
if (FunctionContainsEscapingAllocas)
return false;
// Second pass, change any tail calls to loops.
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB->getTerminator()))
MadeChange |= ProcessReturningBlock(Ret, OldEntry, TailCallsAreMarkedTail,
ArgumentPHIs,CannotTCETailMarkedCall);
// If we eliminated any tail recursions, it's possible that we inserted some
// silly PHI nodes which just merge an initial value (the incoming operand)
// with themselves. Check to see if we did and clean up our mess if so. This
// occurs when a function passes an argument straight through to its tail
// call.
if (!ArgumentPHIs.empty()) {
for (unsigned i = 0, e = ArgumentPHIs.size(); i != e; ++i) {
PHINode *PN = ArgumentPHIs[i];
// If the PHI Node is a dynamic constant, replace it with the value it is.
if (Value *PNV = PN->hasConstantValue()) {
PN->replaceAllUsesWith(PNV);
PN->eraseFromParent();
}
}
}
// Finally, if this function contains no non-escaping allocas, mark all calls
// in the function as eligible for tail calls (there is no stack memory for
// them to access).
if (!FunctionContainsEscapingAllocas)
for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB)
for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
if (CallInst *CI = dyn_cast<CallInst>(I)) {
CI->setTailCall();
MadeChange = true;
}
return MadeChange;
}
