static void
computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI, bool HasLocalsInUsed,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
// Summary not currently supported for anonymous functions, they should
// have been named.
assert(F.hasName());
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
SetVector<ValueInfo> RefEdges;
SetVector<GlobalValue::GUID> TypeTests;
ICallPromotionAnalysis ICallAnalysis;
bool HasInlineAsmMaybeReferencingInternal = false;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
const auto *CI = dyn_cast<CallInst>(&I);
// Since we don't know exactly which local values are referenced in inline
// assembly, conservatively mark the function as possibly referencing
// a local value from inline assembly to ensure we don't export a
// reference (which would require renaming and promotion of the
// referenced value).
if (HasLocalsInUsed && CI && CI->isInlineAsm())
HasInlineAsmMaybeReferencingInternal = true;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function or a known
// intrinsic, or an indirect call with profile data.
if (CalledFunction) {
if (CalledFunction->isIntrinsic()) {
if (CalledFunction->getIntrinsicID() != Intrinsic::type_test)
continue;
// Produce a summary from type.test intrinsics. We only summarize
// type.test intrinsics that are used other than by an llvm.assume
// intrinsic. Intrinsics that are assumed are relevant only to the
// devirtualization pass, not the type test lowering pass.
bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
if (!AssumeCI)
return true;
Function *F = AssumeCI->getCalledFunction();
return !F || F->getIntrinsicID() != Intrinsic::assume;
});
if (HasNonAssumeUses) {
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
if (auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata()))
TypeTests.insert(GlobalValue::getGUID(TypeId->getString()));
}
}
// We should have named any anonymous globals
assert(CalledFunction->hasName());
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
CallGraphEdges[cast<GlobalValue>(CalledValue)].updateHotness(Hotness);
} else {
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
CallGraphEdges[Candidate.Value].updateHotness(
getHotness(Candidate.Count, PSI));
}
}
bool NonRenamableLocal = isNonRenamableLocal(F);
bool NotEligibleForImport =
NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
// Inliner doesn't handle variadic functions.
// FIXME: refactor this to use the same code that inliner is using.
F.isVarArg();
GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
/* LiveRoot = */ false);
auto FuncSummary = llvm::make_unique<FunctionSummary>(
Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(),
TypeTests.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(F.getGUID());
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI) {
// Summary not currently supported for anonymous functions, they must
// be renamed.
if (!F.hasName())
return;
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
DenseMap<const Value *, CalleeInfo> CallGraphEdges;
DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges;
DenseSet<const Value *> RefEdges;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
auto *CalledFunction = CS.getCalledFunction();
// Check if this is a direct call to a known function.
if (CalledFunction) {
// Skip nameless and intrinsics.
if (!CalledFunction->hasName() || CalledFunction->isIntrinsic())
continue;
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
auto *CalleeId =
M.getValueSymbolTable().lookup(CalledFunction->getName());
CallGraphEdges[CalleeId] += (ScaledCount ? ScaledCount.getValue() : 0);
} else {
const auto *CI = dyn_cast<CallInst>(&I);
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
IndirectCallEdges[Candidate.Value] += Candidate.Count;
}
}
GlobalValueSummary::GVFlags Flags(F);
std::unique_ptr<FunctionSummary> FuncSummary =
llvm::make_unique<FunctionSummary>(Flags, NumInsts);
FuncSummary->addCallGraphEdges(CallGraphEdges);
FuncSummary->addCallGraphEdges(IndirectCallEdges);
FuncSummary->addRefEdges(RefEdges);
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
static void
computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
// Summary not currently supported for anonymous functions, they should
// have been named.
assert(F.hasName());
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
SetVector<ValueInfo> RefEdges;
SetVector<GlobalValue::GUID> TypeTests;
SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
TypeCheckedLoadVCalls;
SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
TypeCheckedLoadConstVCalls;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
// Add personality function, prefix data and prologue data to function's ref
// list.
findRefEdges(Index, &F, RefEdges, Visited);
bool HasInlineAsmMaybeReferencingInternal = false;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(Index, &I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
const auto *CI = dyn_cast<CallInst>(&I);
// Since we don't know exactly which local values are referenced in inline
// assembly, conservatively mark the function as possibly referencing
// a local value from inline assembly to ensure we don't export a
// reference (which would require renaming and promotion of the
// referenced value).
if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
HasInlineAsmMaybeReferencingInternal = true;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
if (CalledValue && !CalledFunction) {
CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
// Stripping pointer casts can reveal a called function.
CalledFunction = dyn_cast<Function>(CalledValue);
}
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function or a known
// intrinsic, or an indirect call with profile data.
if (CalledFunction) {
if (CI && CalledFunction->isIntrinsic()) {
addIntrinsicToSummary(
CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls);
continue;
}
// We should have named any anonymous globals
assert(CalledFunction->hasName());
auto ScaledCount = PSI->getProfileCount(&I, BFI);
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
Hotness = CalleeInfo::HotnessType::Cold;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
cast<GlobalValue>(CalledValue))];
ValueInfo.updateHotness(Hotness);
// Add the relative block frequency to CalleeInfo if there is no profile
// information.
if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
uint64_t EntryFreq = BFI->getEntryFreq();
ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
}
} else {
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CalledValue || isa<Constant>(CalledValue))
continue;
// Check if the instruction has a callees metadata. If so, add callees
// to CallGraphEdges to reflect the references from the metadata, and
// to enable importing for subsequent indirect call promotion and
// inlining.
if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
for (auto &Op : MD->operands()) {
Function *Callee = mdconst::extract_or_null<Function>(Op);
if (Callee)
CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
}
}
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
.updateHotness(getHotness(Candidate.Count, PSI));
}
}
// Explicit add hot edges to enforce importing for designated GUIDs for
// sample PGO, to enable the same inlines as the profiled optimized binary.
for (auto &I : F.getImportGUIDs())
CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
ForceSummaryEdgesCold == FunctionSummary::FSHT_All
? CalleeInfo::HotnessType::Cold
: CalleeInfo::HotnessType::Critical);
bool NonRenamableLocal = isNonRenamableLocal(F);
bool NotEligibleForImport =
NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
// Inliner doesn't handle variadic functions.
// FIXME: refactor this to use the same code that inliner is using.
F.isVarArg() ||
// Don't try to import functions with noinline attribute.
F.getAttributes().hasFnAttribute(Attribute::NoInline);
GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
/* Live = */ false, F.isDSOLocal());
FunctionSummary::FFlags FunFlags{
F.hasFnAttribute(Attribute::ReadNone),
F.hasFnAttribute(Attribute::ReadOnly),
F.hasFnAttribute(Attribute::NoRecurse),
F.returnDoesNotAlias(),
};
auto FuncSummary = llvm::make_unique<FunctionSummary>(
Flags, NumInsts, FunFlags, RefEdges.takeVector(),
CallGraphEdges.takeVector(), TypeTests.takeVector(),
TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
TypeTestAssumeConstVCalls.takeVector(),
TypeCheckedLoadConstVCalls.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(F.getGUID());
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI) {
// Summary not currently supported for anonymous functions, they must
// be renamed.
if (!F.hasName())
return;
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
DenseMap<const Value *, CalleeInfo> CallGraphEdges;
DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges;
DenseSet<const Value *> RefEdges;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function.
if (CalledFunction) {
// Skip nameless and intrinsics.
if (!CalledFunction->hasName() || CalledFunction->isIntrinsic())
continue;
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
auto *CalleeId =
M.getValueSymbolTable().lookup(CalledValue->getName());
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
CallGraphEdges[CalleeId].updateHotness(Hotness);
} else {
const auto *CI = dyn_cast<CallInst>(&I);
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
IndirectCallEdges[Candidate.Value].updateHotness(
getHotness(Candidate.Count, PSI));
}
}
GlobalValueSummary::GVFlags Flags(F);
std::unique_ptr<FunctionSummary> FuncSummary =
llvm::make_unique<FunctionSummary>(Flags, NumInsts);
FuncSummary->addCallGraphEdges(CallGraphEdges);
FuncSummary->addCallGraphEdges(IndirectCallEdges);
FuncSummary->addRefEdges(RefEdges);
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
