/// isSafeToPromoteArgument - As you might guess from the name of this method,
/// it checks to see if it is both safe and useful to promote the argument.
/// This method limits promotion of aggregates to only promote up to three
/// elements of the aggregate in order to avoid exploding the number of
/// arguments passed in.
bool ArgPromotion::isSafeToPromoteArgument(Argument *Arg, bool isByVal) const {
typedef std::set<IndicesVector> GEPIndicesSet;
// Quick exit for unused arguments
if (Arg->use_empty())
return true;
// We can only promote this argument if all of the uses are loads, or are GEP
// instructions (with constant indices) that are subsequently loaded.
//
// Promoting the argument causes it to be loaded in the caller
// unconditionally. This is only safe if we can prove that either the load
// would have happened in the callee anyway (ie, there is a load in the entry
// block) or the pointer passed in at every call site is guaranteed to be
// valid.
// In the former case, invalid loads can happen, but would have happened
// anyway, in the latter case, invalid loads won't happen. This prevents us
// from introducing an invalid load that wouldn't have happened in the
// original code.
//
// This set will contain all sets of indices that are loaded in the entry
// block, and thus are safe to unconditionally load in the caller.
GEPIndicesSet SafeToUnconditionallyLoad;
// This set contains all the sets of indices that we are planning to promote.
// This makes it possible to limit the number of arguments added.
GEPIndicesSet ToPromote;
// If the pointer is always valid, any load with first index 0 is valid.
if (isByVal || AllCalleesPassInValidPointerForArgument(Arg))
SafeToUnconditionallyLoad.insert(IndicesVector(1, 0));
// First, iterate the entry block and mark loads of (geps of) arguments as
// safe.
BasicBlock *EntryBlock = Arg->getParent()->begin();
// Declare this here so we can reuse it
IndicesVector Indices;
for (BasicBlock::iterator I = EntryBlock->begin(), E = EntryBlock->end();
I != E; ++I)
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
Value *V = LI->getPointerOperand();
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(V)) {
V = GEP->getPointerOperand();
if (V == Arg) {
// This load actually loads (part of) Arg? Check the indices then.
Indices.reserve(GEP->getNumIndices());
for (User::op_iterator II = GEP->idx_begin(), IE = GEP->idx_end();
II != IE; ++II)
if (ConstantInt *CI = dyn_cast<ConstantInt>(*II))
Indices.push_back(CI->getSExtValue());
else
// We found a non-constant GEP index for this argument? Bail out
// right away, can't promote this argument at all.
return false;
// Indices checked out, mark them as safe
MarkIndicesSafe(Indices, SafeToUnconditionallyLoad);
Indices.clear();
}
} else if (V == Arg) {
// Direct loads are equivalent to a GEP with a single 0 index.
MarkIndicesSafe(IndicesVector(1, 0), SafeToUnconditionallyLoad);
}
}
// Now, iterate all uses of the argument to see if there are any uses that are
// not (GEP+)loads, or any (GEP+)loads that are not safe to promote.
SmallVector<LoadInst*, 16> Loads;
IndicesVector Operands;
for (Value::use_iterator UI = Arg->use_begin(), E = Arg->use_end();
UI != E; ++UI) {
User *U = *UI;
Operands.clear();
if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
if (LI->isVolatile()) return false; // Don't hack volatile loads
Loads.push_back(LI);
// Direct loads are equivalent to a GEP with a zero index and then a load.
Operands.push_back(0);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(U)) {
if (GEP->use_empty()) {
// Dead GEP's cause trouble later. Just remove them if we run into
// them.
getAnalysis<AliasAnalysis>().deleteValue(GEP);
GEP->eraseFromParent();
// TODO: This runs the above loop over and over again for dead GEPs
// Couldn't we just do increment the UI iterator earlier and erase the
// use?
return isSafeToPromoteArgument(Arg, isByVal);
}
// Ensure that all of the indices are constants.
for (User::op_iterator i = GEP->idx_begin(), e = GEP->idx_end();
i != e; ++i)
if (ConstantInt *C = dyn_cast<ConstantInt>(*i))
Operands.push_back(C->getSExtValue());
else
return false; // Not a constant operand GEP!
// Ensure that the only users of the GEP are load instructions.
for (Value::use_iterator UI = GEP->use_begin(), E = GEP->use_end();
UI != E; ++UI)
if (LoadInst *LI = dyn_cast<LoadInst>(*UI)) {
if (LI->isVolatile()) return false; // Don't hack volatile loads
Loads.push_back(LI);
} else {
// Other uses than load?
return false;
}
} else {
return false; // Not a load or a GEP.
}
// Now, see if it is safe to promote this load / loads of this GEP. Loading
// is safe if Operands, or a prefix of Operands, is marked as safe.
if (!PrefixIn(Operands, SafeToUnconditionallyLoad))
return false;
// See if we are already promoting a load with these indices. If not, check
// to make sure that we aren't promoting too many elements. If so, nothing
// to do.
if (ToPromote.find(Operands) == ToPromote.end()) {
if (maxElements > 0 && ToPromote.size() == maxElements) {
DEBUG(dbgs() << "argpromotion not promoting argument '"
<< Arg->getName() << "' because it would require adding more "
<< "than " << maxElements << " arguments to the function.\n");
// We limit aggregate promotion to only promoting up to a fixed number
// of elements of the aggregate.
return false;
}
ToPromote.insert(Operands);
}
}
if (Loads.empty()) return true; // No users, this is a dead argument.
// Okay, now we know that the argument is only used by load instructions and
// it is safe to unconditionally perform all of them. Use alias analysis to
// check to see if the pointer is guaranteed to not be modified from entry of
// the function to each of the load instructions.
// Because there could be several/many load instructions, remember which
// blocks we know to be transparent to the load.
SmallPtrSet<BasicBlock*, 16> TranspBlocks;
AliasAnalysis &AA = getAnalysis<AliasAnalysis>();
TargetData *TD = getAnalysisIfAvailable<TargetData>();
if (!TD) return false; // Without TargetData, assume the worst.
for (unsigned i = 0, e = Loads.size(); i != e; ++i) {
// Check to see if the load is invalidated from the start of the block to
// the load itself.
LoadInst *Load = Loads[i];
BasicBlock *BB = Load->getParent();
const PointerType *LoadTy =
cast<PointerType>(Load->getPointerOperand()->getType());
unsigned LoadSize =(unsigned)TD->getTypeStoreSize(LoadTy->getElementType());
if (AA.canInstructionRangeModify(BB->front(), *Load, Arg, LoadSize))
return false; // Pointer is invalidated!
// Now check every path from the entry block to the load for transparency.
// To do this, we perform a depth first search on the inverse CFG from the
// loading block.
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
BasicBlock *P = *PI;
for (idf_ext_iterator<BasicBlock*, SmallPtrSet<BasicBlock*, 16> >
I = idf_ext_begin(P, TranspBlocks),
E = idf_ext_end(P, TranspBlocks); I != E; ++I)
if (AA.canBasicBlockModify(**I, Arg, LoadSize))
return false;
}
}
// If the path from the entry of the function to each load is free of
// instructions that potentially invalidate the load, we can make the
// transformation!
return true;
}
/// isSafeToPromoteArgument - As you might guess from the name of this method,
/// it checks to see if it is both safe and useful to promote the argument.
/// This method limits promotion of aggregates to only promote up to three
/// elements of the aggregate in order to avoid exploding the number of
/// arguments passed in.
static bool isSafeToPromoteArgument(Argument *Arg, bool isByValOrInAlloca,
AAResults &AAR, unsigned MaxElements) {
typedef std::set<IndicesVector> GEPIndicesSet;
// Quick exit for unused arguments
if (Arg->use_empty())
return true;
// We can only promote this argument if all of the uses are loads, or are GEP
// instructions (with constant indices) that are subsequently loaded.
//
// Promoting the argument causes it to be loaded in the caller
// unconditionally. This is only safe if we can prove that either the load
// would have happened in the callee anyway (ie, there is a load in the entry
// block) or the pointer passed in at every call site is guaranteed to be
// valid.
// In the former case, invalid loads can happen, but would have happened
// anyway, in the latter case, invalid loads won't happen. This prevents us
// from introducing an invalid load that wouldn't have happened in the
// original code.
//
// This set will contain all sets of indices that are loaded in the entry
// block, and thus are safe to unconditionally load in the caller.
//
// This optimization is also safe for InAlloca parameters, because it verifies
// that the address isn't captured.
GEPIndicesSet SafeToUnconditionallyLoad;
// This set contains all the sets of indices that we are planning to promote.
// This makes it possible to limit the number of arguments added.
GEPIndicesSet ToPromote;
// If the pointer is always valid, any load with first index 0 is valid.
if (isByValOrInAlloca || AllCallersPassInValidPointerForArgument(Arg))
SafeToUnconditionallyLoad.insert(IndicesVector(1, 0));
// First, iterate the entry block and mark loads of (geps of) arguments as
// safe.
BasicBlock &EntryBlock = Arg->getParent()->front();
// Declare this here so we can reuse it
IndicesVector Indices;
for (Instruction &I : EntryBlock)
if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
Value *V = LI->getPointerOperand();
if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(V)) {
V = GEP->getPointerOperand();
if (V == Arg) {
// This load actually loads (part of) Arg? Check the indices then.
Indices.reserve(GEP->getNumIndices());
for (User::op_iterator II = GEP->idx_begin(), IE = GEP->idx_end();
II != IE; ++II)
if (ConstantInt *CI = dyn_cast<ConstantInt>(*II))
Indices.push_back(CI->getSExtValue());
else
// We found a non-constant GEP index for this argument? Bail out
// right away, can't promote this argument at all.
return false;
// Indices checked out, mark them as safe
MarkIndicesSafe(Indices, SafeToUnconditionallyLoad);
Indices.clear();
}
} else if (V == Arg) {
// Direct loads are equivalent to a GEP with a single 0 index.
MarkIndicesSafe(IndicesVector(1, 0), SafeToUnconditionallyLoad);
}
}
// Now, iterate all uses of the argument to see if there are any uses that are
// not (GEP+)loads, or any (GEP+)loads that are not safe to promote.
SmallVector<LoadInst*, 16> Loads;
IndicesVector Operands;
for (Use &U : Arg->uses()) {
User *UR = U.getUser();
Operands.clear();
if (LoadInst *LI = dyn_cast<LoadInst>(UR)) {
// Don't hack volatile/atomic loads
if (!LI->isSimple()) return false;
Loads.push_back(LI);
// Direct loads are equivalent to a GEP with a zero index and then a load.
Operands.push_back(0);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(UR)) {
if (GEP->use_empty()) {
// Dead GEP's cause trouble later. Just remove them if we run into
// them.
GEP->eraseFromParent();
// TODO: This runs the above loop over and over again for dead GEPs
// Couldn't we just do increment the UI iterator earlier and erase the
// use?
return isSafeToPromoteArgument(Arg, isByValOrInAlloca, AAR,
MaxElements);
}
// Ensure that all of the indices are constants.
for (User::op_iterator i = GEP->idx_begin(), e = GEP->idx_end();
i != e; ++i)
if (ConstantInt *C = dyn_cast<ConstantInt>(*i))
Operands.push_back(C->getSExtValue());
else
return false; // Not a constant operand GEP!
// Ensure that the only users of the GEP are load instructions.
for (User *GEPU : GEP->users())
if (LoadInst *LI = dyn_cast<LoadInst>(GEPU)) {
// Don't hack volatile/atomic loads
if (!LI->isSimple()) return false;
Loads.push_back(LI);
} else {
// Other uses than load?
return false;
}
} else {
return false; // Not a load or a GEP.
}
// Now, see if it is safe to promote this load / loads of this GEP. Loading
// is safe if Operands, or a prefix of Operands, is marked as safe.
if (!PrefixIn(Operands, SafeToUnconditionallyLoad))
return false;
// See if we are already promoting a load with these indices. If not, check
// to make sure that we aren't promoting too many elements. If so, nothing
// to do.
if (ToPromote.find(Operands) == ToPromote.end()) {
if (MaxElements > 0 && ToPromote.size() == MaxElements) {
DEBUG(dbgs() << "argpromotion not promoting argument '"
<< Arg->getName() << "' because it would require adding more "
<< "than " << MaxElements << " arguments to the function.\n");
// We limit aggregate promotion to only promoting up to a fixed number
// of elements of the aggregate.
return false;
}
ToPromote.insert(std::move(Operands));
}
}
if (Loads.empty()) return true; // No users, this is a dead argument.
// Okay, now we know that the argument is only used by load instructions and
// it is safe to unconditionally perform all of them. Use alias analysis to
// check to see if the pointer is guaranteed to not be modified from entry of
// the function to each of the load instructions.
// Because there could be several/many load instructions, remember which
// blocks we know to be transparent to the load.
df_iterator_default_set<BasicBlock*, 16> TranspBlocks;
for (LoadInst *Load : Loads) {
// Check to see if the load is invalidated from the start of the block to
// the load itself.
BasicBlock *BB = Load->getParent();
MemoryLocation Loc = MemoryLocation::get(Load);
if (AAR.canInstructionRangeModRef(BB->front(), *Load, Loc, MRI_Mod))
return false; // Pointer is invalidated!
// Now check every path from the entry block to the load for transparency.
// To do this, we perform a depth first search on the inverse CFG from the
// loading block.
for (BasicBlock *P : predecessors(BB)) {
for (BasicBlock *TranspBB : inverse_depth_first_ext(P, TranspBlocks))
if (AAR.canBasicBlockModify(*TranspBB, Loc))
return false;
}
}
// If the path from the entry of the function to each load is free of
// instructions that potentially invalidate the load, we can make the
// transformation!
return true;
}
