// Set LiveRoot flag on entries matching the given value name. static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { auto SummaryList = Index.findGlobalValueSummaryList(GlobalValue::getGUID(Name)); if (SummaryList == Index.end()) return; for (auto &Summary : SummaryList->second) Summary->setLiveRoot(); }
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet<GlobalValue::GUID> &CantBePromoted) { bool NonRenamableLocal = isNonRenamableLocal(A); GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal, /* LiveRoot = */ false); auto AS = llvm::make_unique<AliasSummary>(Flags, ArrayRef<ValueInfo>{}); auto *Aliasee = A.getBaseObject(); auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee); assert(AliaseeSummary && "Alias expects aliasee summary to be parsed"); AS->setAliasee(AliaseeSummary); if (NonRenamableLocal) CantBePromoted.insert(A.getGUID()); Index.addGlobalValueSummary(A.getName(), std::move(AS)); }
// Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector<ValueInfo> &RefEdges, SmallPtrSet<const User *, 8> &Visited) { SmallVector<const User *, 32> Worklist; Worklist.push_back(CurUser); while (!Worklist.empty()) { const User *U = Worklist.pop_back_val(); if (!Visited.insert(U).second) continue; ImmutableCallSite CS(U); for (const auto &OI : U->operands()) { const User *Operand = dyn_cast<User>(OI); if (!Operand) continue; if (isa<BlockAddress>(Operand)) continue; if (auto *GV = dyn_cast<GlobalValue>(Operand)) { // We have a reference to a global value. This should be added to // the reference set unless it is a callee. Callees are handled // specially by WriteFunction and are added to a separate list. if (!(CS && CS.isCallee(&OI))) RefEdges.insert(Index.getOrInsertValueInfo(GV)); continue; } Worklist.push_back(Operand); } } }
/** * Perform cross-module importing for the module identified by ModuleIdentifier. */ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule, ModuleSummaryIndex &Index) { auto ModuleMap = generateModuleMap(Modules); auto ModuleCount = Index.modulePaths().size(); // Collect for each module the list of function it defines (GUID -> Summary). StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); }
/** * Compute the list of summaries needed for importing into module. */ void ThinLTOCodeGenerator::gatherImportedSummariesForModule( StringRef ModulePath, ModuleSummaryIndex &Index, std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) { auto ModuleCount = Index.modulePaths().size(); // Collect for each module the list of function it defines (GUID -> Summary). StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries, ImportLists[ModulePath], ModuleToSummariesForIndex); }
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V) { DenseSet<const Value *> RefEdges; SmallPtrSet<const User *, 8> Visited; findRefEdges(&V, RefEdges, Visited); GlobalValueSummary::GVFlags Flags(V); std::unique_ptr<GlobalVarSummary> GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags); GVarSummary->addRefEdges(RefEdges); Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary)); }
/** * Emit the list of files needed for importing into module. */ void ThinLTOCodeGenerator::emitImports(StringRef ModulePath, StringRef OutputName, ModuleSummaryIndex &Index) { auto ModuleCount = Index.modulePaths().size(); // Collect for each module the list of function it defines (GUID -> Summary). StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); std::error_code EC; if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists[ModulePath]))) report_fatal_error(Twine("Failed to open ") + OutputName + " to save imports lists\n"); }
/** * Perform cross-module importing for the module identified by ModuleIdentifier. */ void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule, ModuleSummaryIndex &Index) { auto ModuleMap = generateModuleMap(Modules); // Generate import/export list auto ModuleCount = Index.modulePaths().size(); StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ImportLists, ExportLists); auto &ImportList = ImportLists[TheModule.getModuleIdentifier()]; crossImportIntoModule(TheModule, Index, ModuleMap, ImportList); }
/** * Perform promotion and renaming of exported internal functions. * Index is updated to reflect linkage changes from weak resolution. */ void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index) { auto ModuleCount = Index.modulePaths().size(); auto ModuleIdentifier = TheModule.getModuleIdentifier(); // Collect for each module the list of function it defines (GUID -> Summary). StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries; Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); // Resolve LinkOnce/Weak symbols. StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; resolveWeakForLinkerInIndex(Index, ResolvedODR); thinLTOResolveWeakForLinkerModule( TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]); // Convert the preserved symbols set from string to GUID auto GUIDPreservedSymbols = computeGUIDPreservedSymbols( PreservedSymbols, Triple(TheModule.getTargetTriple())); // Promote the exported values in the index, so that they are promoted // in the module. auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { const auto &ExportList = ExportLists.find(ModuleIdentifier); return (ExportList != ExportLists.end() && ExportList->second.count(GUID)) || GUIDPreservedSymbols.count(GUID); }; thinLTOInternalizeAndPromoteInIndex(Index, isExported); promoteModule(TheModule, Index); }
/** * Perform internalization. Index is updated to reflect linkage changes. */ void ThinLTOCodeGenerator::internalize(Module &TheModule, ModuleSummaryIndex &Index) { initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple())); auto ModuleCount = Index.modulePaths().size(); auto ModuleIdentifier = TheModule.getModuleIdentifier(); // Convert the preserved symbols set from string to GUID auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple); // Collect for each module the list of function it defines (GUID -> Summary). StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount); Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries); // Generate import/export list StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount); ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); auto &ExportList = ExportLists[ModuleIdentifier]; // Be friendly and don't nuke totally the module when the client didn't // supply anything to preserve. if (ExportList.empty() && GUIDPreservedSymbols.empty()) return; // Internalization auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { const auto &ExportList = ExportLists.find(ModuleIdentifier); return (ExportList != ExportLists.end() && ExportList->second.count(GUID)) || GUIDPreservedSymbols.count(GUID); }; thinLTOInternalizeAndPromoteInIndex(Index, isExported); thinLTOInternalizeModule(TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]); }
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet<GlobalValue::GUID> &CantBePromoted) { SetVector<ValueInfo> RefEdges; SmallPtrSet<const User *, 8> Visited; findRefEdges(&V, RefEdges, Visited); bool NonRenamableLocal = isNonRenamableLocal(V); GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal, /* LiveRoot = */ false); auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(V.getGUID()); Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary)); }
/// Compute all the imports for the given module in the Index. void llvm::ComputeCrossModuleImportForModule( StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList) { // Collect the list of functions this module defines. // GUID -> Summary GVSummaryMapTy FunctionSummaryMap; Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap); // Compute the import list for this module. DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n"); ComputeImportForModule(FunctionSummaryMap, Index, ImportList); #ifndef NDEBUG DEBUG(dbgs() << "* Module " << ModulePath << " imports from " << ImportList.size() << " modules.\n"); for (auto &Src : ImportList) { auto SrcModName = Src.first(); DEBUG(dbgs() << " - " << Src.second.size() << " functions imported from " << SrcModName << "\n"); } #endif }
// Resolve Weak and LinkOnce values in the \p Index. // // We'd like to drop these functions if they are no longer referenced in the // current module. However there is a chance that another module is still // referencing them because of the import. We make sure we always emit at least // one copy. void thinLTOResolveWeakForLinkerInIndex( ModuleSummaryIndex &Index, std::function<bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, std::function<bool(StringRef, GlobalValue::GUID)> isExported, std::function<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage) { if (Index.modulePaths().size() == 1) // Nothing to do if we don't have multiple modules return; // We won't optimize the globals that are referenced by an alias for now // Ideally we should turn the alias into a global and duplicate the definition // when needed. DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias; for (auto &I : Index) for (auto &S : I.second) if (auto AS = dyn_cast<AliasSummary>(S.get())) GlobalInvolvedWithAlias.insert(&AS->getAliasee()); for (auto &I : Index) thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias, isPrevailing, isExported, recordNewLinkage); }
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI) { // Summary not currently supported for anonymous functions, they must // be renamed. if (!F.hasName()) return; unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. DenseMap<const Value *, CalleeInfo> CallGraphEdges; DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges; DenseSet<const Value *> RefEdges; ICallPromotionAnalysis ICallAnalysis; SmallPtrSet<const User *, 8> Visited; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(&I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; auto *CalledValue = CS.getCalledValue(); auto *CalledFunction = CS.getCalledFunction(); // Check if this is an alias to a function. If so, get the // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast<Function>(GA->getBaseObject()); } // Check if this is a direct call to a known function. if (CalledFunction) { // Skip nameless and intrinsics. if (!CalledFunction->hasName() || CalledFunction->isIntrinsic()) continue; auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None; // Use the original CalledValue, in case it was an alias. We want // to record the call edge to the alias in that case. Eventually // an alias summary will be created to associate the alias and // aliasee. auto *CalleeId = M.getValueSymbolTable().lookup(CalledValue->getName()); auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) : CalleeInfo::HotnessType::Unknown; CallGraphEdges[CalleeId].updateHotness(Hotness); } else { const auto *CI = dyn_cast<CallInst>(&I); // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue())) continue; uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) IndirectCallEdges[Candidate.Value].updateHotness( getHotness(Candidate.Count, PSI)); } } GlobalValueSummary::GVFlags Flags(F); std::unique_ptr<FunctionSummary> FuncSummary = llvm::make_unique<FunctionSummary>(Flags, NumInsts); FuncSummary->addCallGraphEdges(CallGraphEdges); FuncSummary->addCallGraphEdges(IndirectCallEdges); FuncSummary->addRefEdges(RefEdges); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }
// Set LiveRoot flag on entries matching the given value name. static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) { if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name))) for (auto &Summary : VI.getSummaryList()) Summary->setLive(true); }
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, bool HasLocalsInUsed, DenseSet<GlobalValue::GUID> &CantBePromoted) { // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector<ValueInfo, CalleeInfo> CallGraphEdges; SetVector<ValueInfo> RefEdges; SetVector<GlobalValue::GUID> TypeTests; ICallPromotionAnalysis ICallAnalysis; bool HasInlineAsmMaybeReferencingInternal = false; SmallPtrSet<const User *, 8> Visited; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(&I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; const auto *CI = dyn_cast<CallInst>(&I); // Since we don't know exactly which local values are referenced in inline // assembly, conservatively mark the function as possibly referencing // a local value from inline assembly to ensure we don't export a // reference (which would require renaming and promotion of the // referenced value). if (HasLocalsInUsed && CI && CI->isInlineAsm()) HasInlineAsmMaybeReferencingInternal = true; auto *CalledValue = CS.getCalledValue(); auto *CalledFunction = CS.getCalledFunction(); // Check if this is an alias to a function. If so, get the // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast<Function>(GA->getBaseObject()); } // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { if (CalledFunction->isIntrinsic()) { if (CalledFunction->getIntrinsicID() != Intrinsic::type_test) continue; // Produce a summary from type.test intrinsics. We only summarize // type.test intrinsics that are used other than by an llvm.assume // intrinsic. Intrinsics that are assumed are relevant only to the // devirtualization pass, not the type test lowering pass. bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) { auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser()); if (!AssumeCI) return true; Function *F = AssumeCI->getCalledFunction(); return !F || F->getIntrinsicID() != Intrinsic::assume; }); if (HasNonAssumeUses) { auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1)); if (auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata())) TypeTests.insert(GlobalValue::getGUID(TypeId->getString())); } } // We should have named any anonymous globals assert(CalledFunction->hasName()); auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None; auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) : CalleeInfo::HotnessType::Unknown; // Use the original CalledValue, in case it was an alias. We want // to record the call edge to the alias in that case. Eventually // an alias summary will be created to associate the alias and // aliasee. CallGraphEdges[cast<GlobalValue>(CalledValue)].updateHotness(Hotness); } else { // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue())) continue; uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) CallGraphEdges[Candidate.Value].updateHotness( getHotness(Candidate.Count, PSI)); } } bool NonRenamableLocal = isNonRenamableLocal(F); bool NotEligibleForImport = NonRenamableLocal || HasInlineAsmMaybeReferencingInternal || // Inliner doesn't handle variadic functions. // FIXME: refactor this to use the same code that inliner is using. F.isVarArg(); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, /* LiveRoot = */ false); auto FuncSummary = llvm::make_unique<FunctionSummary>( Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(F.getGUID()); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }
ModuleSummaryIndex llvm::buildModuleSummaryIndex( const Module &M, std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI) { ModuleSummaryIndex Index; // Identify the local values in the llvm.used and llvm.compiler.used sets, // which should not be exported as they would then require renaming and // promotion, but we may have opaque uses e.g. in inline asm. We collect them // here because we use this information to mark functions containing inline // assembly calls as not importable. SmallPtrSet<GlobalValue *, 8> LocalsUsed; SmallPtrSet<GlobalValue *, 8> Used; // First collect those in the llvm.used set. collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false); // Next collect those in the llvm.compiler.used set. collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true); DenseSet<GlobalValue::GUID> CantBePromoted; for (auto *V : Used) { if (V->hasLocalLinkage()) { LocalsUsed.insert(V); CantBePromoted.insert(V->getGUID()); } } // Compute summaries for all functions defined in module, and save in the // index. for (auto &F : M) { if (F.isDeclaration()) continue; BlockFrequencyInfo *BFI = nullptr; std::unique_ptr<BlockFrequencyInfo> BFIPtr; if (GetBFICallback) BFI = GetBFICallback(F); else if (F.getEntryCount().hasValue()) { LoopInfo LI{DominatorTree(const_cast<Function &>(F))}; BranchProbabilityInfo BPI{F, LI}; BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI); BFI = BFIPtr.get(); } computeFunctionSummary(Index, M, F, BFI, PSI, !LocalsUsed.empty(), CantBePromoted); } // Compute summaries for all variables defined in module, and save in the // index. for (const GlobalVariable &G : M.globals()) { if (G.isDeclaration()) continue; computeVariableSummary(Index, G, CantBePromoted); } // Compute summaries for all aliases defined in module, and save in the // index. for (const GlobalAlias &A : M.aliases()) computeAliasSummary(Index, A, CantBePromoted); for (auto *V : LocalsUsed) { auto *Summary = Index.getGlobalValueSummary(*V); assert(Summary && "Missing summary for global value"); Summary->setNotEligibleToImport(); } // The linker doesn't know about these LLVM produced values, so we need // to flag them as live in the index to ensure index-based dead value // analysis treats them as live roots of the analysis. setLiveRoot(Index, "llvm.used"); setLiveRoot(Index, "llvm.compiler.used"); setLiveRoot(Index, "llvm.global_ctors"); setLiveRoot(Index, "llvm.global_dtors"); setLiveRoot(Index, "llvm.global.annotations"); if (!M.getModuleInlineAsm().empty()) { // Collect the local values defined by module level asm, and set up // summaries for these symbols so that they can be marked as NoRename, // to prevent export of any use of them in regular IR that would require // renaming within the module level asm. Note we don't need to create a // summary for weak or global defs, as they don't need to be flagged as // NoRename, and defs in module level asm can't be imported anyway. // Also, any values used but not defined within module level asm should // be listed on the llvm.used or llvm.compiler.used global and marked as // referenced from there. ModuleSymbolTable::CollectAsmSymbols( Triple(M.getTargetTriple()), M.getModuleInlineAsm(), [&M, &Index, &CantBePromoted](StringRef Name, object::BasicSymbolRef::Flags Flags) { // Symbols not marked as Weak or Global are local definitions. if (Flags & (object::BasicSymbolRef::SF_Weak | object::BasicSymbolRef::SF_Global)) return; GlobalValue *GV = M.getNamedValue(Name); if (!GV) return; assert(GV->isDeclaration() && "Def in module asm already has definition"); GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage, /* NotEligibleToImport */ true, /* LiveRoot */ true); CantBePromoted.insert(GlobalValue::getGUID(Name)); // Create the appropriate summary type. if (isa<Function>(GV)) { std::unique_ptr<FunctionSummary> Summary = llvm::make_unique<FunctionSummary>( GVFlags, 0, ArrayRef<ValueInfo>{}, ArrayRef<FunctionSummary::EdgeTy>{}, ArrayRef<GlobalValue::GUID>{}); Index.addGlobalValueSummary(Name, std::move(Summary)); } else { std::unique_ptr<GlobalVarSummary> Summary = llvm::make_unique<GlobalVarSummary>(GVFlags, ArrayRef<ValueInfo>{}); Index.addGlobalValueSummary(Name, std::move(Summary)); } }); } for (auto &GlobalList : Index) { assert(GlobalList.second.size() == 1 && "Expected module's index to have one summary per GUID"); auto &Summary = GlobalList.second[0]; bool AllRefsCanBeExternallyReferenced = llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) { return !CantBePromoted.count(VI.getValue()->getGUID()); }); if (!AllRefsCanBeExternallyReferenced) { Summary->setNotEligibleToImport(); continue; } if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) { bool AllCallsCanBeExternallyReferenced = llvm::all_of( FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) { auto GUID = Edge.first.isGUID() ? Edge.first.getGUID() : Edge.first.getValue()->getGUID(); return !CantBePromoted.count(GUID); }); if (!AllCallsCanBeExternallyReferenced) Summary->setNotEligibleToImport(); } } return Index; }
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI) { // Summary not currently supported for anonymous functions, they must // be renamed. if (!F.hasName()) return; unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. DenseMap<const Value *, CalleeInfo> CallGraphEdges; DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges; DenseSet<const Value *> RefEdges; ICallPromotionAnalysis ICallAnalysis; SmallPtrSet<const User *, 8> Visited; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(&I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; auto *CalledFunction = CS.getCalledFunction(); // Check if this is a direct call to a known function. if (CalledFunction) { // Skip nameless and intrinsics. if (!CalledFunction->hasName() || CalledFunction->isIntrinsic()) continue; auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None; auto *CalleeId = M.getValueSymbolTable().lookup(CalledFunction->getName()); CallGraphEdges[CalleeId] += (ScaledCount ? ScaledCount.getValue() : 0); } else { const auto *CI = dyn_cast<CallInst>(&I); // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue())) continue; uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) IndirectCallEdges[Candidate.Value] += Candidate.Count; } } GlobalValueSummary::GVFlags Flags(F); std::unique_ptr<FunctionSummary> FuncSummary = llvm::make_unique<FunctionSummary>(Flags, NumInsts); FuncSummary->addCallGraphEdges(CallGraphEdges); FuncSummary->addCallGraphEdges(IndirectCallEdges); FuncSummary->addRefEdges(RefEdges); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted) { // Summary not currently supported for anonymous functions, they should // have been named. assert(F.hasName()); unsigned NumInsts = 0; // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector<ValueInfo, CalleeInfo> CallGraphEdges; SetVector<ValueInfo> RefEdges; SetVector<GlobalValue::GUID> TypeTests; SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls; SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls; ICallPromotionAnalysis ICallAnalysis; SmallPtrSet<const User *, 8> Visited; // Add personality function, prefix data and prologue data to function's ref // list. findRefEdges(Index, &F, RefEdges, Visited); bool HasInlineAsmMaybeReferencingInternal = false; for (const BasicBlock &BB : F) for (const Instruction &I : BB) { if (isa<DbgInfoIntrinsic>(I)) continue; ++NumInsts; findRefEdges(Index, &I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); if (!CS) continue; const auto *CI = dyn_cast<CallInst>(&I); // Since we don't know exactly which local values are referenced in inline // assembly, conservatively mark the function as possibly referencing // a local value from inline assembly to ensure we don't export a // reference (which would require renaming and promotion of the // referenced value). if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm()) HasInlineAsmMaybeReferencingInternal = true; auto *CalledValue = CS.getCalledValue(); auto *CalledFunction = CS.getCalledFunction(); if (CalledValue && !CalledFunction) { CalledValue = CalledValue->stripPointerCastsNoFollowAliases(); // Stripping pointer casts can reveal a called function. CalledFunction = dyn_cast<Function>(CalledValue); } // Check if this is an alias to a function. If so, get the // called aliasee for the checks below. if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) { assert(!CalledFunction && "Expected null called function in callsite for alias"); CalledFunction = dyn_cast<Function>(GA->getBaseObject()); } // Check if this is a direct call to a known function or a known // intrinsic, or an indirect call with profile data. if (CalledFunction) { if (CI && CalledFunction->isIntrinsic()) { addIntrinsicToSummary( CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls, TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls); continue; } // We should have named any anonymous globals assert(CalledFunction->hasName()); auto ScaledCount = PSI->getProfileCount(&I, BFI); auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI) : CalleeInfo::HotnessType::Unknown; if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None) Hotness = CalleeInfo::HotnessType::Cold; // Use the original CalledValue, in case it was an alias. We want // to record the call edge to the alias in that case. Eventually // an alias summary will be created to associate the alias and // aliasee. auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo( cast<GlobalValue>(CalledValue))]; ValueInfo.updateHotness(Hotness); // Add the relative block frequency to CalleeInfo if there is no profile // information. if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) { uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency(); uint64_t EntryFreq = BFI->getEntryFreq(); ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq); } } else { // Skip inline assembly calls. if (CI && CI->isInlineAsm()) continue; // Skip direct calls. if (!CalledValue || isa<Constant>(CalledValue)) continue; // Check if the instruction has a callees metadata. If so, add callees // to CallGraphEdges to reflect the references from the metadata, and // to enable importing for subsequent indirect call promotion and // inlining. if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) { for (auto &Op : MD->operands()) { Function *Callee = mdconst::extract_or_null<Function>(Op); if (Callee) CallGraphEdges[Index.getOrInsertValueInfo(Callee)]; } } uint32_t NumVals, NumCandidates; uint64_t TotalCount; auto CandidateProfileData = ICallAnalysis.getPromotionCandidatesForInstruction( &I, NumVals, TotalCount, NumCandidates); for (auto &Candidate : CandidateProfileData) CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)] .updateHotness(getHotness(Candidate.Count, PSI)); } } // Explicit add hot edges to enforce importing for designated GUIDs for // sample PGO, to enable the same inlines as the profiled optimized binary. for (auto &I : F.getImportGUIDs()) CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness( ForceSummaryEdgesCold == FunctionSummary::FSHT_All ? CalleeInfo::HotnessType::Cold : CalleeInfo::HotnessType::Critical); bool NonRenamableLocal = isNonRenamableLocal(F); bool NotEligibleForImport = NonRenamableLocal || HasInlineAsmMaybeReferencingInternal || // Inliner doesn't handle variadic functions. // FIXME: refactor this to use the same code that inliner is using. F.isVarArg() || // Don't try to import functions with noinline attribute. F.getAttributes().hasFnAttribute(Attribute::NoInline); GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, /* Live = */ false, F.isDSOLocal()); FunctionSummary::FFlags FunFlags{ F.hasFnAttribute(Attribute::ReadNone), F.hasFnAttribute(Attribute::ReadOnly), F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(), }; auto FuncSummary = llvm::make_unique<FunctionSummary>( Flags, NumInsts, FunFlags, RefEdges.takeVector(), CallGraphEdges.takeVector(), TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(), TypeTestAssumeConstVCalls.takeVector(), TypeCheckedLoadConstVCalls.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(F.getGUID()); Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary)); }
/// gold informs us that all symbols have been read. At this point, we use /// get_symbols to see if any of our definitions have been overridden by a /// native object file. Then, perform optimization and codegen. static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) { if (Modules.empty()) return LDPS_OK; if (unsigned NumOpts = options::extra.size()) cl::ParseCommandLineOptions(NumOpts, &options::extra[0]); // If we are doing ThinLTO compilation, simply build the combined // module index/summary and emit it. We don't need to parse the modules // and link them in this case. if (options::thinlto) { ModuleSummaryIndex CombinedIndex; uint64_t NextModuleId = 0; for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); std::unique_ptr<ModuleSummaryIndex> Index = getModuleSummaryIndexForFile(F, InputFile.file()); // Skip files without a module summary. if (Index) CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId); } std::error_code EC; raw_fd_ostream OS(output_name + ".thinlto.bc", EC, sys::fs::OpenFlags::F_None); if (EC) message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s", output_name.data(), EC.message().c_str()); WriteIndexToFile(CombinedIndex, OS); OS.close(); if (options::thinlto_index_only) { cleanup_hook(); exit(0); } thinLTOBackends(ApiFile, CombinedIndex); return LDPS_OK; } LLVMContext Context; Context.setDiscardValueNames(options::DiscardValueNames); Context.enableDebugTypeODRUniquing(); // Merge debug info types. Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true); std::unique_ptr<Module> Combined(new Module("ld-temp.o", Context)); IRMover L(*Combined); StringSet<> Internalize; StringSet<> Maybe; for (claimed_file &F : Modules) { PluginInputFile InputFile(F.handle); const void *View = getSymbolsAndView(F); if (!View) continue; if (linkInModule(Context, L, F, View, InputFile.file(), ApiFile, Internalize, Maybe)) message(LDPL_FATAL, "Failed to link module"); } for (const auto &Name : Internalize) { GlobalValue *GV = Combined->getNamedValue(Name.first()); if (GV) internalize(*GV); } for (const auto &Name : Maybe) { GlobalValue *GV = Combined->getNamedValue(Name.first()); if (!GV) continue; GV->setLinkage(GlobalValue::LinkOnceODRLinkage); if (canBeOmittedFromSymbolTable(GV)) internalize(*GV); } if (options::TheOutputType == options::OT_DISABLE) return LDPS_OK; if (options::TheOutputType != options::OT_NORMAL) { std::string path; if (options::TheOutputType == options::OT_BC_ONLY) path = output_name; else path = output_name + ".bc"; saveBCFile(path, *Combined); if (options::TheOutputType == options::OT_BC_ONLY) return LDPS_OK; } CodeGen codeGen(std::move(Combined)); codeGen.runAll(); if (!options::extra_library_path.empty() && set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK) message(LDPL_FATAL, "Unable to set the extra library path."); return LDPS_OK; }