static bool addNoRecurseAttrs(const SCCNodeSet &SCCNodes) {
// Try and identify functions that do not recurse.
// If the SCC contains multiple nodes we know for sure there is recursion.
if (SCCNodes.size() != 1)
return false;
Function *F = *SCCNodes.begin();
if (!F || F->isDeclaration() || F->doesNotRecurse())
return false;
// If all of the calls in F are identifiable and are to norecurse functions, F
// is norecurse. This check also detects self-recursion as F is not currently
// marked norecurse, so any called from F to F will not be marked norecurse.
for (Instruction &I : instructions(*F))
if (auto CS = CallSite(&I)) {
Function *Callee = CS.getCalledFunction();
if (!Callee || Callee == F || !Callee->doesNotRecurse())
// Function calls a potentially recursive function.
return false;
}
// Every call was to a non-recursive function other than this function, and
// we have no indirect recursion as the SCC size is one. This function cannot
// recurse.
return setDoesNotRecurse(*F);
}
PreservedAnalyses PostOrderFunctionAttrsPass::run(LazyCallGraph::SCC &C,
CGSCCAnalysisManager &AM,
LazyCallGraph &CG,
CGSCCUpdateResult &) {
FunctionAnalysisManager &FAM =
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
// We pass a lambda into functions to wire them up to the analysis manager
// for getting function analyses.
auto AARGetter = [&](Function &F) -> AAResults & {
return FAM.getResult<AAManager>(F);
};
// Fill SCCNodes with the elements of the SCC. Also track whether there are
// any external or opt-none nodes that will prevent us from optimizing any
// part of the SCC.
SCCNodeSet SCCNodes;
bool HasUnknownCall = false;
for (LazyCallGraph::Node &N : C) {
Function &F = N.getFunction();
if (F.hasFnAttribute(Attribute::OptimizeNone)) {
// Treat any function we're trying not to optimize as if it were an
// indirect call and omit it from the node set used below.
HasUnknownCall = true;
continue;
}
// Track whether any functions in this SCC have an unknown call edge.
// Note: if this is ever a performance hit, we can common it with
// subsequent routines which also do scans over the instructions of the
// function.
if (!HasUnknownCall)
for (Instruction &I : instructions(F))
if (auto CS = CallSite(&I))
if (!CS.getCalledFunction()) {
HasUnknownCall = true;
break;
}
SCCNodes.insert(&F);
}
bool Changed = false;
Changed |= addArgumentReturnedAttrs(SCCNodes);
Changed |= addReadAttrs(SCCNodes, AARGetter);
Changed |= addArgumentAttrs(SCCNodes);
// If we have no external nodes participating in the SCC, we can deduce some
// more precise attributes as well.
if (!HasUnknownCall) {
Changed |= addNoAliasAttrs(SCCNodes);
Changed |= addNonNullAttrs(SCCNodes);
Changed |= removeConvergentAttrs(SCCNodes);
Changed |= addNoRecurseAttrs(SCCNodes);
}
return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
}
bool PostOrderFunctionAttrsLegacyPass::runOnSCC(CallGraphSCC &SCC) {
TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
bool Changed = false;
// We compute dedicated AA results for each function in the SCC as needed. We
// use a lambda referencing external objects so that they live long enough to
// be queried, but we re-use them each time.
Optional<BasicAAResult> BAR;
Optional<AAResults> AAR;
auto AARGetter = [&](Function &F) -> AAResults & {
BAR.emplace(createLegacyPMBasicAAResult(*this, F));
AAR.emplace(createLegacyPMAAResults(*this, F, *BAR));
return *AAR;
};
// Fill SCCNodes with the elements of the SCC. Used for quickly looking up
// whether a given CallGraphNode is in this SCC. Also track whether there are
// any external or opt-none nodes that will prevent us from optimizing any
// part of the SCC.
SCCNodeSet SCCNodes;
bool ExternalNode = false;
for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
Function *F = (*I)->getFunction();
if (!F || F->hasFnAttribute(Attribute::OptimizeNone)) {
// External node or function we're trying not to optimize - we both avoid
// transform them and avoid leveraging information they provide.
ExternalNode = true;
continue;
}
SCCNodes.insert(F);
}
Changed |= addReadAttrs(SCCNodes, AARGetter);
Changed |= addArgumentAttrs(SCCNodes);
// If we have no external nodes participating in the SCC, we can deduce some
// more precise attributes as well.
if (!ExternalNode) {
Changed |= addNoAliasAttrs(SCCNodes);
Changed |= addNonNullAttrs(SCCNodes, *TLI);
Changed |= removeConvergentAttrs(SCCNodes);
Changed |= addNoRecurseAttrs(SCCNodes);
}
return Changed;
}
/// Removes convergent attributes where we can prove that none of the SCC's
/// callees are themselves convergent. Returns true if successful at removing
/// the attribute.
static bool removeConvergentAttrs(const CallGraphSCC &SCC,
const SCCNodeSet &SCCNodes) {
// Determines whether a function can be made non-convergent, ignoring all
// other functions in SCC. (A function can *actually* be made non-convergent
// only if all functions in its SCC can be made convergent.)
auto CanRemoveConvergent = [&] (CallGraphNode *CGN) {
Function *F = CGN->getFunction();
if (!F) return false;
if (!F->isConvergent()) return true;
// Can't remove convergent from declarations.
if (F->isDeclaration()) return false;
// Don't remove convergent from optnone functions.
if (F->hasFnAttribute(Attribute::OptimizeNone))
return false;
// Can't remove convergent if any of F's callees -- ignoring functions in the
// SCC itself -- are convergent.
if (llvm::any_of(*CGN, [&](const CallGraphNode::CallRecord &CR) {
Function *F = CR.second->getFunction();
return SCCNodes.count(F) == 0 && (!F || F->isConvergent());
}))
return false;
// CGN doesn't contain calls to intrinsics, so iterate over all of F's
// callsites, looking for any calls to convergent intrinsics. If we find one,
// F must remain marked as convergent.
auto IsConvergentIntrinsicCall = [](Instruction &I) {
CallSite CS(cast<Value>(&I));
if (!CS)
return false;
Function *Callee = CS.getCalledFunction();
return Callee && Callee->isIntrinsic() && Callee->isConvergent();
};
return !llvm::any_of(*F, [=](BasicBlock &BB) {
return llvm::any_of(BB, IsConvergentIntrinsicCall);
});
};
// We can remove the convergent attr from functions in the SCC if they all can
// be made non-convergent (because they call only non-convergent functions,
// other than each other).
if (!llvm::all_of(SCC, CanRemoveConvergent)) return false;
// If we got here, all of the SCC's callees are non-convergent, and none of
// the optnone functions in the SCC are marked as convergent. Therefore all
// of the SCC's functions can be marked as non-convergent.
for (CallGraphNode *CGN : SCC)
if (Function *F = CGN->getFunction()) {
if (F->isConvergent())
DEBUG(dbgs() << "Removing convergent attr from " << F->getName()
<< "\n");
F->setNotConvergent();
}
return true;
}
static bool runImpl(CallGraphSCC &SCC, AARGetterT AARGetter) {
bool Changed = false;
// Fill SCCNodes with the elements of the SCC. Used for quickly looking up
// whether a given CallGraphNode is in this SCC. Also track whether there are
// any external or opt-none nodes that will prevent us from optimizing any
// part of the SCC.
SCCNodeSet SCCNodes;
bool ExternalNode = false;
for (CallGraphNode *I : SCC) {
Function *F = I->getFunction();
if (!F || F->hasFnAttribute(Attribute::OptimizeNone)) {
// External node or function we're trying not to optimize - we both avoid
// transform them and avoid leveraging information they provide.
ExternalNode = true;
continue;
}
SCCNodes.insert(F);
}
Changed |= addArgumentReturnedAttrs(SCCNodes);
Changed |= addReadAttrs(SCCNodes, AARGetter);
Changed |= addArgumentAttrs(SCCNodes);
// If we have no external nodes participating in the SCC, we can deduce some
// more precise attributes as well.
if (!ExternalNode) {
Changed |= addNoAliasAttrs(SCCNodes);
Changed |= addNonNullAttrs(SCCNodes);
Changed |= removeConvergentAttrs(SCCNodes);
Changed |= addNoRecurseAttrs(SCCNodes);
}
return Changed;
}
/// Removes convergent attributes where we can prove that none of the SCC's
/// callees are themselves convergent. Returns true if successful at removing
/// the attribute.
static bool removeConvergentAttrs(const SCCNodeSet &SCCNodes) {
// Determines whether a function can be made non-convergent, ignoring all
// other functions in SCC. (A function can *actually* be made non-convergent
// only if all functions in its SCC can be made convergent.)
auto CanRemoveConvergent = [&](Function *F) {
if (!F->isConvergent())
return true;
// Can't remove convergent from declarations.
if (F->isDeclaration())
return false;
for (Instruction &I : instructions(*F))
if (auto CS = CallSite(&I)) {
// Can't remove convergent if any of F's callees -- ignoring functions
// in the SCC itself -- are convergent. This needs to consider both
// function calls and intrinsic calls. We also assume indirect calls
// might call a convergent function.
// FIXME: We should revisit this when we put convergent onto calls
// instead of functions so that indirect calls which should be
// convergent are required to be marked as such.
Function *Callee = CS.getCalledFunction();
if (!Callee || (SCCNodes.count(Callee) == 0 && Callee->isConvergent()))
return false;
}
return true;
};
// We can remove the convergent attr from functions in the SCC if they all
// can be made non-convergent (because they call only non-convergent
// functions, other than each other).
if (!llvm::all_of(SCCNodes, CanRemoveConvergent))
return false;
// If we got here, all of the SCC's callees are non-convergent. Therefore all
// of the SCC's functions can be marked as non-convergent.
for (Function *F : SCCNodes) {
if (F->isConvergent())
DEBUG(dbgs() << "Removing convergent attr from " << F->getName() << "\n");
F->setNotConvergent();
}
return true;
}
/// Remove the convergent attribute from all functions in the SCC if every
/// callsite within the SCC is not convergent (except for calls to functions
/// within the SCC). Returns true if changes were made.
static bool removeConvergentAttrs(const SCCNodeSet &SCCNodes) {
// For every function in SCC, ensure that either
// * it is not convergent, or
// * we can remove its convergent attribute.
bool HasConvergentFn = false;
for (Function *F : SCCNodes) {
if (!F->isConvergent()) continue;
HasConvergentFn = true;
// Can't remove convergent from function declarations.
if (F->isDeclaration()) return false;
// Can't remove convergent if any of our functions has a convergent call to a
// function not in the SCC.
for (Instruction &I : instructions(*F)) {
CallSite CS(&I);
// Bail if CS is a convergent call to a function not in the SCC.
if (CS && CS.isConvergent() &&
SCCNodes.count(CS.getCalledFunction()) == 0)
return false;
}
}
// If the SCC doesn't have any convergent functions, we have nothing to do.
if (!HasConvergentFn) return false;
// If we got here, all of the calls the SCC makes to functions not in the SCC
// are non-convergent. Therefore all of the SCC's functions can also be made
// non-convergent. We'll remove the attr from the callsites in
// InstCombineCalls.
for (Function *F : SCCNodes) {
if (!F->isConvergent()) continue;
DEBUG(dbgs() << "Removing convergent attr from fn " << F->getName()
<< "\n");
F->setNotConvergent();
}
return true;
}
/// Tests whether this function is known to not return null.
///
/// Requires that the function returns a pointer.
///
/// Returns true if it believes the function will not return a null, and sets
/// \p Speculative based on whether the returned conclusion is a speculative
/// conclusion due to SCC calls.
static bool isReturnNonNull(Function *F, const SCCNodeSet &SCCNodes,
bool &Speculative) {
assert(F->getReturnType()->isPointerTy() &&
"nonnull only meaningful on pointer types");
Speculative = false;
SmallSetVector<Value *, 8> FlowsToReturn;
for (BasicBlock &BB : *F)
if (auto *Ret = dyn_cast<ReturnInst>(BB.getTerminator()))
FlowsToReturn.insert(Ret->getReturnValue());
for (unsigned i = 0; i != FlowsToReturn.size(); ++i) {
Value *RetVal = FlowsToReturn[i];
// If this value is locally known to be non-null, we're good
if (isKnownNonNull(RetVal))
continue;
// Otherwise, we need to look upwards since we can't make any local
// conclusions.
Instruction *RVI = dyn_cast<Instruction>(RetVal);
if (!RVI)
return false;
switch (RVI->getOpcode()) {
// Extend the analysis by looking upwards.
case Instruction::BitCast:
case Instruction::GetElementPtr:
case Instruction::AddrSpaceCast:
FlowsToReturn.insert(RVI->getOperand(0));
continue;
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(RVI);
FlowsToReturn.insert(SI->getTrueValue());
FlowsToReturn.insert(SI->getFalseValue());
continue;
}
case Instruction::PHI: {
PHINode *PN = cast<PHINode>(RVI);
for (int i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
FlowsToReturn.insert(PN->getIncomingValue(i));
continue;
}
case Instruction::Call:
case Instruction::Invoke: {
CallSite CS(RVI);
Function *Callee = CS.getCalledFunction();
// A call to a node within the SCC is assumed to return null until
// proven otherwise
if (Callee && SCCNodes.count(Callee)) {
Speculative = true;
continue;
}
return false;
}
default:
return false; // Unknown source, may be null
};
llvm_unreachable("should have either continued or returned");
}
return true;
}
/// Tests whether a function is "malloc-like".
///
/// A function is "malloc-like" if it returns either null or a pointer that
/// doesn't alias any other pointer visible to the caller.
static bool isFunctionMallocLike(Function *F, const SCCNodeSet &SCCNodes) {
SmallSetVector<Value *, 8> FlowsToReturn;
for (BasicBlock &BB : *F)
if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB.getTerminator()))
FlowsToReturn.insert(Ret->getReturnValue());
for (unsigned i = 0; i != FlowsToReturn.size(); ++i) {
Value *RetVal = FlowsToReturn[i];
if (Constant *C = dyn_cast<Constant>(RetVal)) {
if (!C->isNullValue() && !isa<UndefValue>(C))
return false;
continue;
}
if (isa<Argument>(RetVal))
return false;
if (Instruction *RVI = dyn_cast<Instruction>(RetVal))
switch (RVI->getOpcode()) {
// Extend the analysis by looking upwards.
case Instruction::BitCast:
case Instruction::GetElementPtr:
case Instruction::AddrSpaceCast:
FlowsToReturn.insert(RVI->getOperand(0));
continue;
case Instruction::Select: {
SelectInst *SI = cast<SelectInst>(RVI);
FlowsToReturn.insert(SI->getTrueValue());
FlowsToReturn.insert(SI->getFalseValue());
continue;
}
case Instruction::PHI: {
PHINode *PN = cast<PHINode>(RVI);
for (Value *IncValue : PN->incoming_values())
FlowsToReturn.insert(IncValue);
continue;
}
// Check whether the pointer came from an allocation.
case Instruction::Alloca:
break;
case Instruction::Call:
case Instruction::Invoke: {
CallSite CS(RVI);
if (CS.hasRetAttr(Attribute::NoAlias))
break;
if (CS.getCalledFunction() && SCCNodes.count(CS.getCalledFunction()))
break;
LLVM_FALLTHROUGH;
}
default:
return false; // Did not come from an allocation.
}
if (PointerMayBeCaptured(RetVal, false, /*StoreCaptures=*/false))
return false;
}
return true;
}
/// Returns the memory access attribute for function F using AAR for AA results,
/// where SCCNodes is the current SCC.
///
/// If ThisBody is true, this function may examine the function body and will
/// return a result pertaining to this copy of the function. If it is false, the
/// result will be based only on AA results for the function declaration; it
/// will be assumed that some other (perhaps less optimized) version of the
/// function may be selected at link time.
static MemoryAccessKind checkFunctionMemoryAccess(Function &F, bool ThisBody,
AAResults &AAR,
const SCCNodeSet &SCCNodes) {
FunctionModRefBehavior MRB = AAR.getModRefBehavior(&F);
if (MRB == FMRB_DoesNotAccessMemory)
// Already perfect!
return MAK_ReadNone;
if (!ThisBody) {
if (AliasAnalysis::onlyReadsMemory(MRB))
return MAK_ReadOnly;
// Conservatively assume it writes to memory.
return MAK_MayWrite;
}
// Scan the function body for instructions that may read or write memory.
bool ReadsMemory = false;
for (inst_iterator II = inst_begin(F), E = inst_end(F); II != E; ++II) {
Instruction *I = &*II;
// Some instructions can be ignored even if they read or write memory.
// Detect these now, skipping to the next instruction if one is found.
CallSite CS(cast<Value>(I));
if (CS) {
// Ignore calls to functions in the same SCC, as long as the call sites
// don't have operand bundles. Calls with operand bundles are allowed to
// have memory effects not described by the memory effects of the call
// target.
if (!CS.hasOperandBundles() && CS.getCalledFunction() &&
SCCNodes.count(CS.getCalledFunction()))
continue;
FunctionModRefBehavior MRB = AAR.getModRefBehavior(CS);
// If the call doesn't access memory, we're done.
if (!(MRB & MRI_ModRef))
continue;
if (!AliasAnalysis::onlyAccessesArgPointees(MRB)) {
// The call could access any memory. If that includes writes, give up.
if (MRB & MRI_Mod)
return MAK_MayWrite;
// If it reads, note it.
if (MRB & MRI_Ref)
ReadsMemory = true;
continue;
}
// Check whether all pointer arguments point to local memory, and
// ignore calls that only access local memory.
for (CallSite::arg_iterator CI = CS.arg_begin(), CE = CS.arg_end();
CI != CE; ++CI) {
Value *Arg = *CI;
if (!Arg->getType()->isPtrOrPtrVectorTy())
continue;
AAMDNodes AAInfo;
I->getAAMetadata(AAInfo);
MemoryLocation Loc(Arg, MemoryLocation::UnknownSize, AAInfo);
// Skip accesses to local or constant memory as they don't impact the
// externally visible mod/ref behavior.
if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
continue;
if (MRB & MRI_Mod)
// Writes non-local memory. Give up.
return MAK_MayWrite;
if (MRB & MRI_Ref)
// Ok, it reads non-local memory.
ReadsMemory = true;
}
continue;
} else if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
// Ignore non-volatile loads from local memory. (Atomic is okay here.)
if (!LI->isVolatile()) {
MemoryLocation Loc = MemoryLocation::get(LI);
if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
continue;
}
} else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
// Ignore non-volatile stores to local memory. (Atomic is okay here.)
if (!SI->isVolatile()) {
MemoryLocation Loc = MemoryLocation::get(SI);
if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
continue;
}
} else if (VAArgInst *VI = dyn_cast<VAArgInst>(I)) {
// Ignore vaargs on local memory.
MemoryLocation Loc = MemoryLocation::get(VI);
if (AAR.pointsToConstantMemory(Loc, /*OrLocal=*/true))
continue;
}
// Any remaining instructions need to be taken seriously! Check if they
// read or write memory.
if (I->mayWriteToMemory())
// Writes memory. Just give up.
return MAK_MayWrite;
// If this instruction may read memory, remember that.
ReadsMemory |= I->mayReadFromMemory();
}
return ReadsMemory ? MAK_ReadOnly : MAK_ReadNone;
}