static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, bool HasLocalsInUsed, DenseSet<GlobalValue::GUID> &CantBePromoted) { // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector<ValueInfo, CalleeInfo> CallGraphEdges; SetVector<ValueInfo> RefEdges; SetVector<GlobalValue::GUID> TypeTests; ICallPromotionAnalysis ICallAnalysis; bool HasInlineAsmMaybeReferencingInternal = false; SmallPtrSet<const User *, 8> Visited; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(&I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; const auto *CI = dyn_cast<CallInst>(&I); // Since we don't know exactly which local values are referenced in inline // assembly, conservatively mark the function as possibly referencing // a local value from inline assembly to ensure we don't export a // reference (which would require renaming and promotion of the // referenced value). if (HasLocalsInUsed && CI && CI->isInlineAsm()) HasInlineAsmMaybeReferencingInternal = true; auto *CalledValue = CS.getCalledValue(); auto *CalledFunction = CS.getCalledFunction(); // Check if this is an alias to a function. If so, get the // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast<Function>(GA->getBaseObject()); } // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { if (CalledFunction->isIntrinsic()) { if (CalledFunction->getIntrinsicID() != Intrinsic::type_test) continue; // Produce a summary from type.test intrinsics. We only summarize // type.test intrinsics that are used other than by an llvm.assume // intrinsic. Intrinsics that are assumed are relevant only to the // devirtualization pass, not the type test lowering pass. bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) { auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser()); if (!AssumeCI) return true; Function *F = AssumeCI->getCalledFunction(); return !F || F->getIntrinsicID() != Intrinsic::assume; }); if (HasNonAssumeUses) { auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1)); if (auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata())) TypeTests.insert(GlobalValue::getGUID(TypeId->getString())); } } // We should have named any anonymous globals assert(CalledFunction->hasName()); auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None; auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) : CalleeInfo::HotnessType::Unknown; // Use the original CalledValue, in case it was an alias. We want // to record the call edge to the alias in that case. Eventually // an alias summary will be created to associate the alias and // aliasee. CallGraphEdges[cast<GlobalValue>(CalledValue)].updateHotness(Hotness); } else { // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue())) continue; uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) CallGraphEdges[Candidate.Value].updateHotness( getHotness(Candidate.Count, PSI)); } } bool NonRenamableLocal = isNonRenamableLocal(F); bool NotEligibleForImport = NonRenamableLocal || HasInlineAsmMaybeReferencingInternal || // Inliner doesn't handle variadic functions. // FIXME: refactor this to use the same code that inliner is using. F.isVarArg(); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, /* LiveRoot = */ false); auto FuncSummary = llvm::make_unique<FunctionSummary>( Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(F.getGUID()); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted) { // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector<ValueInfo, CalleeInfo> CallGraphEdges; SetVector<ValueInfo> RefEdges; SetVector<GlobalValue::GUID> TypeTests; SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls; SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls; ICallPromotionAnalysis ICallAnalysis; SmallPtrSet<const User *, 8> Visited; // Add personality function, prefix data and prologue data to function's ref // list. findRefEdges(Index, &F, RefEdges, Visited); bool HasInlineAsmMaybeReferencingInternal = false; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(Index, &I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; const auto *CI = dyn_cast<CallInst>(&I); // Since we don't know exactly which local values are referenced in inline // assembly, conservatively mark the function as possibly referencing // a local value from inline assembly to ensure we don't export a // reference (which would require renaming and promotion of the // referenced value). if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm()) HasInlineAsmMaybeReferencingInternal = true; auto *CalledValue = CS.getCalledValue(); auto *CalledFunction = CS.getCalledFunction(); if (CalledValue && !CalledFunction) { CalledValue = CalledValue->stripPointerCastsNoFollowAliases(); // Stripping pointer casts can reveal a called function. CalledFunction = dyn_cast<Function>(CalledValue); } // Check if this is an alias to a function. If so, get the // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast<Function>(GA->getBaseObject()); } // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { if (CI && CalledFunction->isIntrinsic()) { addIntrinsicToSummary( CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls, TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls); continue; } // We should have named any anonymous globals assert(CalledFunction->hasName()); auto ScaledCount = PSI->getProfileCount(&I, BFI); auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) : CalleeInfo::HotnessType::Unknown; if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None) Hotness = CalleeInfo::HotnessType::Cold; // Use the original CalledValue, in case it was an alias. We want // to record the call edge to the alias in that case. Eventually // an alias summary will be created to associate the alias and // aliasee. auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo( cast<GlobalValue>(CalledValue))]; ValueInfo.updateHotness(Hotness); // Add the relative block frequency to CalleeInfo if there is no profile // information. if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) { uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency(); uint64_t EntryFreq = BFI->getEntryFreq(); ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq); } } else { // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CalledValue || isa<Constant>(CalledValue)) continue; // Check if the instruction has a callees metadata. If so, add callees // to CallGraphEdges to reflect the references from the metadata, and // to enable importing for subsequent indirect call promotion and // inlining. if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) { for (auto &Op : MD->operands()) { Function *Callee = mdconst::extract_or_null<Function>(Op); if (Callee) CallGraphEdges[Index.getOrInsertValueInfo(Callee)]; } } uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)] .updateHotness(getHotness(Candidate.Count, PSI)); } } // Explicit add hot edges to enforce importing for designated GUIDs for // sample PGO, to enable the same inlines as the profiled optimized binary. for (auto &I : F.getImportGUIDs()) CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness( ForceSummaryEdgesCold == FunctionSummary::FSHT_All ? CalleeInfo::HotnessType::Cold : CalleeInfo::HotnessType::Critical); bool NonRenamableLocal = isNonRenamableLocal(F); bool NotEligibleForImport = NonRenamableLocal || HasInlineAsmMaybeReferencingInternal || // Inliner doesn't handle variadic functions. // FIXME: refactor this to use the same code that inliner is using. F.isVarArg() || // Don't try to import functions with noinline attribute. F.getAttributes().hasFnAttribute(Attribute::NoInline); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, /* Live = */ false, F.isDSOLocal()); FunctionSummary::FFlags FunFlags{ F.hasFnAttribute(Attribute::ReadNone), F.hasFnAttribute(Attribute::ReadOnly), F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(), }; auto FuncSummary = llvm::make_unique<FunctionSummary>( Flags, NumInsts, FunFlags, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(), TypeTestAssumeConstVCalls.takeVector(), TypeCheckedLoadConstVCalls.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(F.getGUID()); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }