// Set LiveRoot flag on entries matching the given value name.
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
auto SummaryList =
Index.findGlobalValueSummaryList(GlobalValue::getGUID(Name));
if (SummaryList == Index.end())
return;
for (auto &Summary : SummaryList->second)
Summary->setLiveRoot();
}
static void
computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
bool NonRenamableLocal = isNonRenamableLocal(A);
GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal,
/* LiveRoot = */ false);
auto AS = llvm::make_unique<AliasSummary>(Flags, ArrayRef<ValueInfo>{});
auto *Aliasee = A.getBaseObject();
auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee);
assert(AliaseeSummary && "Alias expects aliasee summary to be parsed");
AS->setAliasee(AliaseeSummary);
if (NonRenamableLocal)
CantBePromoted.insert(A.getGUID());
Index.addGlobalValueSummary(A.getName(), std::move(AS));
}
// Walk through the operands of a given User via worklist iteration and populate
// the set of GlobalValue references encountered. Invoked either on an
// Instruction or a GlobalVariable (which walks its initializer).
static void findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
SetVector<ValueInfo> &RefEdges,
SmallPtrSet<const User *, 8> &Visited) {
SmallVector<const User *, 32> Worklist;
Worklist.push_back(CurUser);
while (!Worklist.empty()) {
const User *U = Worklist.pop_back_val();
if (!Visited.insert(U).second)
continue;
ImmutableCallSite CS(U);
for (const auto &OI : U->operands()) {
const User *Operand = dyn_cast<User>(OI);
if (!Operand)
continue;
if (isa<BlockAddress>(Operand))
continue;
if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
// We have a reference to a global value. This should be added to
// the reference set unless it is a callee. Callees are handled
// specially by WriteFunction and are added to a separate list.
if (!(CS && CS.isCallee(&OI)))
RefEdges.insert(Index.getOrInsertValueInfo(GV));
continue;
}
Worklist.push_back(Operand);
}
}
}
/**
* Perform cross-module importing for the module identified by ModuleIdentifier.
*/
void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
ModuleSummaryIndex &Index) {
auto ModuleMap = generateModuleMap(Modules);
auto ModuleCount = Index.modulePaths().size();
// Collect for each module the list of function it defines (GUID -> Summary).
StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
// Generate import/export list
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
}
/**
* Compute the list of summaries needed for importing into module.
*/
void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
StringRef ModulePath, ModuleSummaryIndex &Index,
std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
auto ModuleCount = Index.modulePaths().size();
// Collect for each module the list of function it defines (GUID -> Summary).
StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
// Generate import/export list
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
ImportLists[ModulePath],
ModuleToSummariesForIndex);
}
static void computeVariableSummary(ModuleSummaryIndex &Index,
const GlobalVariable &V) {
DenseSet<const Value *> RefEdges;
SmallPtrSet<const User *, 8> Visited;
findRefEdges(&V, RefEdges, Visited);
GlobalValueSummary::GVFlags Flags(V);
std::unique_ptr<GlobalVarSummary> GVarSummary =
llvm::make_unique<GlobalVarSummary>(Flags);
GVarSummary->addRefEdges(RefEdges);
Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary));
}
/**
* Emit the list of files needed for importing into module.
*/
void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
StringRef OutputName,
ModuleSummaryIndex &Index) {
auto ModuleCount = Index.modulePaths().size();
// Collect for each module the list of function it defines (GUID -> Summary).
StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
// Generate import/export list
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
std::error_code EC;
if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists[ModulePath])))
report_fatal_error(Twine("Failed to open ") + OutputName +
" to save imports lists\n");
}
/**
* Perform cross-module importing for the module identified by ModuleIdentifier.
*/
void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
ModuleSummaryIndex &Index) {
auto ModuleMap = generateModuleMap(Modules);
// Generate import/export list
auto ModuleCount = Index.modulePaths().size();
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ImportLists, ExportLists);
auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
}
/**
* Perform promotion and renaming of exported internal functions.
* Index is updated to reflect linkage changes from weak resolution.
*/
void ThinLTOCodeGenerator::promote(Module &TheModule,
ModuleSummaryIndex &Index) {
auto ModuleCount = Index.modulePaths().size();
auto ModuleIdentifier = TheModule.getModuleIdentifier();
// Collect for each module the list of function it defines (GUID -> Summary).
StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
// Generate import/export list
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
// Resolve LinkOnce/Weak symbols.
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
resolveWeakForLinkerInIndex(Index, ResolvedODR);
thinLTOResolveWeakForLinkerModule(
TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
// Convert the preserved symbols set from string to GUID
auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
PreservedSymbols, Triple(TheModule.getTargetTriple()));
// Promote the exported values in the index, so that they are promoted
// in the module.
auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
const auto &ExportList = ExportLists.find(ModuleIdentifier);
return (ExportList != ExportLists.end() &&
ExportList->second.count(GUID)) ||
GUIDPreservedSymbols.count(GUID);
};
thinLTOInternalizeAndPromoteInIndex(Index, isExported);
promoteModule(TheModule, Index);
}
/**
* Perform internalization. Index is updated to reflect linkage changes.
*/
void ThinLTOCodeGenerator::internalize(Module &TheModule,
ModuleSummaryIndex &Index) {
initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
auto ModuleCount = Index.modulePaths().size();
auto ModuleIdentifier = TheModule.getModuleIdentifier();
// Convert the preserved symbols set from string to GUID
auto GUIDPreservedSymbols =
computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
// Collect for each module the list of function it defines (GUID -> Summary).
StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
// Generate import/export list
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
auto &ExportList = ExportLists[ModuleIdentifier];
// Be friendly and don't nuke totally the module when the client didn't
// supply anything to preserve.
if (ExportList.empty() && GUIDPreservedSymbols.empty())
return;
// Internalization
auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
const auto &ExportList = ExportLists.find(ModuleIdentifier);
return (ExportList != ExportLists.end() &&
ExportList->second.count(GUID)) ||
GUIDPreservedSymbols.count(GUID);
};
thinLTOInternalizeAndPromoteInIndex(Index, isExported);
thinLTOInternalizeModule(TheModule,
ModuleToDefinedGVSummaries[ModuleIdentifier]);
}
static void
computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
SetVector<ValueInfo> RefEdges;
SmallPtrSet<const User *, 8> Visited;
findRefEdges(&V, RefEdges, Visited);
bool NonRenamableLocal = isNonRenamableLocal(V);
GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
/* LiveRoot = */ false);
auto GVarSummary =
llvm::make_unique<GlobalVarSummary>(Flags, RefEdges.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(V.getGUID());
Index.addGlobalValueSummary(V.getName(), std::move(GVarSummary));
}
/// Compute all the imports for the given module in the Index.
void llvm::ComputeCrossModuleImportForModule(
StringRef ModulePath, const ModuleSummaryIndex &Index,
FunctionImporter::ImportMapTy &ImportList) {
// Collect the list of functions this module defines.
// GUID -> Summary
GVSummaryMapTy FunctionSummaryMap;
Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);
// Compute the import list for this module.
DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");
ComputeImportForModule(FunctionSummaryMap, Index, ImportList);
#ifndef NDEBUG
DEBUG(dbgs() << "* Module " << ModulePath << " imports from "
<< ImportList.size() << " modules.\n");
for (auto &Src : ImportList) {
auto SrcModName = Src.first();
DEBUG(dbgs() << " - " << Src.second.size() << " functions imported from "
<< SrcModName << "\n");
}
#endif
}
// Resolve Weak and LinkOnce values in the \p Index.
//
// We'd like to drop these functions if they are no longer referenced in the
// current module. However there is a chance that another module is still
// referencing them because of the import. We make sure we always emit at least
// one copy.
void thinLTOResolveWeakForLinkerInIndex(
ModuleSummaryIndex &Index,
std::function<bool(GlobalValue::GUID, const GlobalValueSummary *)>
isPrevailing,
std::function<bool(StringRef, GlobalValue::GUID)> isExported,
std::function<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
recordNewLinkage) {
if (Index.modulePaths().size() == 1)
// Nothing to do if we don't have multiple modules
return;
// We won't optimize the globals that are referenced by an alias for now
// Ideally we should turn the alias into a global and duplicate the definition
// when needed.
DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
for (auto &I : Index)
for (auto &S : I.second)
if (auto AS = dyn_cast<AliasSummary>(S.get()))
GlobalInvolvedWithAlias.insert(&AS->getAliasee());
for (auto &I : Index)
thinLTOResolveWeakForLinkerGUID(I.second, I.first, GlobalInvolvedWithAlias,
isPrevailing, isExported, recordNewLinkage);
}
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI) {
// Summary not currently supported for anonymous functions, they must
// be renamed.
if (!F.hasName())
return;
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
DenseMap<const Value *, CalleeInfo> CallGraphEdges;
DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges;
DenseSet<const Value *> RefEdges;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function.
if (CalledFunction) {
// Skip nameless and intrinsics.
if (!CalledFunction->hasName() || CalledFunction->isIntrinsic())
continue;
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
auto *CalleeId =
M.getValueSymbolTable().lookup(CalledValue->getName());
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
CallGraphEdges[CalleeId].updateHotness(Hotness);
} else {
const auto *CI = dyn_cast<CallInst>(&I);
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
IndirectCallEdges[Candidate.Value].updateHotness(
getHotness(Candidate.Count, PSI));
}
}
GlobalValueSummary::GVFlags Flags(F);
std::unique_ptr<FunctionSummary> FuncSummary =
llvm::make_unique<FunctionSummary>(Flags, NumInsts);
FuncSummary->addCallGraphEdges(CallGraphEdges);
FuncSummary->addCallGraphEdges(IndirectCallEdges);
FuncSummary->addRefEdges(RefEdges);
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
// Set LiveRoot flag on entries matching the given value name.
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
for (auto &Summary : VI.getSummaryList())
Summary->setLive(true);
}
static void
computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI, bool HasLocalsInUsed,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
// Summary not currently supported for anonymous functions, they should
// have been named.
assert(F.hasName());
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
SetVector<ValueInfo> RefEdges;
SetVector<GlobalValue::GUID> TypeTests;
ICallPromotionAnalysis ICallAnalysis;
bool HasInlineAsmMaybeReferencingInternal = false;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
const auto *CI = dyn_cast<CallInst>(&I);
// Since we don't know exactly which local values are referenced in inline
// assembly, conservatively mark the function as possibly referencing
// a local value from inline assembly to ensure we don't export a
// reference (which would require renaming and promotion of the
// referenced value).
if (HasLocalsInUsed && CI && CI->isInlineAsm())
HasInlineAsmMaybeReferencingInternal = true;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function or a known
// intrinsic, or an indirect call with profile data.
if (CalledFunction) {
if (CalledFunction->isIntrinsic()) {
if (CalledFunction->getIntrinsicID() != Intrinsic::type_test)
continue;
// Produce a summary from type.test intrinsics. We only summarize
// type.test intrinsics that are used other than by an llvm.assume
// intrinsic. Intrinsics that are assumed are relevant only to the
// devirtualization pass, not the type test lowering pass.
bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
if (!AssumeCI)
return true;
Function *F = AssumeCI->getCalledFunction();
return !F || F->getIntrinsicID() != Intrinsic::assume;
});
if (HasNonAssumeUses) {
auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
if (auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata()))
TypeTests.insert(GlobalValue::getGUID(TypeId->getString()));
}
}
// We should have named any anonymous globals
assert(CalledFunction->hasName());
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
CallGraphEdges[cast<GlobalValue>(CalledValue)].updateHotness(Hotness);
} else {
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
CallGraphEdges[Candidate.Value].updateHotness(
getHotness(Candidate.Count, PSI));
}
}
bool NonRenamableLocal = isNonRenamableLocal(F);
bool NotEligibleForImport =
NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
// Inliner doesn't handle variadic functions.
// FIXME: refactor this to use the same code that inliner is using.
F.isVarArg();
GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
/* LiveRoot = */ false);
auto FuncSummary = llvm::make_unique<FunctionSummary>(
Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(),
TypeTests.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(F.getGUID());
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
ModuleSummaryIndex llvm::buildModuleSummaryIndex(
const Module &M,
std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
ProfileSummaryInfo *PSI) {
ModuleSummaryIndex Index;
// Identify the local values in the llvm.used and llvm.compiler.used sets,
// which should not be exported as they would then require renaming and
// promotion, but we may have opaque uses e.g. in inline asm. We collect them
// here because we use this information to mark functions containing inline
// assembly calls as not importable.
SmallPtrSet<GlobalValue *, 8> LocalsUsed;
SmallPtrSet<GlobalValue *, 8> Used;
// First collect those in the llvm.used set.
collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
// Next collect those in the llvm.compiler.used set.
collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true);
DenseSet<GlobalValue::GUID> CantBePromoted;
for (auto *V : Used) {
if (V->hasLocalLinkage()) {
LocalsUsed.insert(V);
CantBePromoted.insert(V->getGUID());
}
}
// Compute summaries for all functions defined in module, and save in the
// index.
for (auto &F : M) {
if (F.isDeclaration())
continue;
BlockFrequencyInfo *BFI = nullptr;
std::unique_ptr<BlockFrequencyInfo> BFIPtr;
if (GetBFICallback)
BFI = GetBFICallback(F);
else if (F.getEntryCount().hasValue()) {
LoopInfo LI{DominatorTree(const_cast<Function &>(F))};
BranchProbabilityInfo BPI{F, LI};
BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI);
BFI = BFIPtr.get();
}
computeFunctionSummary(Index, M, F, BFI, PSI, !LocalsUsed.empty(),
CantBePromoted);
}
// Compute summaries for all variables defined in module, and save in the
// index.
for (const GlobalVariable &G : M.globals()) {
if (G.isDeclaration())
continue;
computeVariableSummary(Index, G, CantBePromoted);
}
// Compute summaries for all aliases defined in module, and save in the
// index.
for (const GlobalAlias &A : M.aliases())
computeAliasSummary(Index, A, CantBePromoted);
for (auto *V : LocalsUsed) {
auto *Summary = Index.getGlobalValueSummary(*V);
assert(Summary && "Missing summary for global value");
Summary->setNotEligibleToImport();
}
// The linker doesn't know about these LLVM produced values, so we need
// to flag them as live in the index to ensure index-based dead value
// analysis treats them as live roots of the analysis.
setLiveRoot(Index, "llvm.used");
setLiveRoot(Index, "llvm.compiler.used");
setLiveRoot(Index, "llvm.global_ctors");
setLiveRoot(Index, "llvm.global_dtors");
setLiveRoot(Index, "llvm.global.annotations");
if (!M.getModuleInlineAsm().empty()) {
// Collect the local values defined by module level asm, and set up
// summaries for these symbols so that they can be marked as NoRename,
// to prevent export of any use of them in regular IR that would require
// renaming within the module level asm. Note we don't need to create a
// summary for weak or global defs, as they don't need to be flagged as
// NoRename, and defs in module level asm can't be imported anyway.
// Also, any values used but not defined within module level asm should
// be listed on the llvm.used or llvm.compiler.used global and marked as
// referenced from there.
ModuleSymbolTable::CollectAsmSymbols(
Triple(M.getTargetTriple()), M.getModuleInlineAsm(),
[&M, &Index, &CantBePromoted](StringRef Name,
object::BasicSymbolRef::Flags Flags) {
// Symbols not marked as Weak or Global are local definitions.
if (Flags & (object::BasicSymbolRef::SF_Weak |
object::BasicSymbolRef::SF_Global))
return;
GlobalValue *GV = M.getNamedValue(Name);
if (!GV)
return;
assert(GV->isDeclaration() && "Def in module asm already has definition");
GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage,
/* NotEligibleToImport */ true,
/* LiveRoot */ true);
CantBePromoted.insert(GlobalValue::getGUID(Name));
// Create the appropriate summary type.
if (isa<Function>(GV)) {
std::unique_ptr<FunctionSummary> Summary =
llvm::make_unique<FunctionSummary>(
GVFlags, 0, ArrayRef<ValueInfo>{},
ArrayRef<FunctionSummary::EdgeTy>{},
ArrayRef<GlobalValue::GUID>{});
Index.addGlobalValueSummary(Name, std::move(Summary));
} else {
std::unique_ptr<GlobalVarSummary> Summary =
llvm::make_unique<GlobalVarSummary>(GVFlags,
ArrayRef<ValueInfo>{});
Index.addGlobalValueSummary(Name, std::move(Summary));
}
});
}
for (auto &GlobalList : Index) {
assert(GlobalList.second.size() == 1 &&
"Expected module's index to have one summary per GUID");
auto &Summary = GlobalList.second[0];
bool AllRefsCanBeExternallyReferenced =
llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
return !CantBePromoted.count(VI.getValue()->getGUID());
});
if (!AllRefsCanBeExternallyReferenced) {
Summary->setNotEligibleToImport();
continue;
}
if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
bool AllCallsCanBeExternallyReferenced = llvm::all_of(
FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
auto GUID = Edge.first.isGUID() ? Edge.first.getGUID()
: Edge.first.getValue()->getGUID();
return !CantBePromoted.count(GUID);
});
if (!AllCallsCanBeExternallyReferenced)
Summary->setNotEligibleToImport();
}
}
return Index;
}
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI) {
// Summary not currently supported for anonymous functions, they must
// be renamed.
if (!F.hasName())
return;
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
DenseMap<const Value *, CalleeInfo> CallGraphEdges;
DenseMap<GlobalValue::GUID, CalleeInfo> IndirectCallEdges;
DenseSet<const Value *> RefEdges;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(&I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
auto *CalledFunction = CS.getCalledFunction();
// Check if this is a direct call to a known function.
if (CalledFunction) {
// Skip nameless and intrinsics.
if (!CalledFunction->hasName() || CalledFunction->isIntrinsic())
continue;
auto ScaledCount = BFI ? BFI->getBlockProfileCount(&BB) : None;
auto *CalleeId =
M.getValueSymbolTable().lookup(CalledFunction->getName());
CallGraphEdges[CalleeId] += (ScaledCount ? ScaledCount.getValue() : 0);
} else {
const auto *CI = dyn_cast<CallInst>(&I);
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CS.getCalledValue() || isa<Constant>(CS.getCalledValue()))
continue;
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
IndirectCallEdges[Candidate.Value] += Candidate.Count;
}
}
GlobalValueSummary::GVFlags Flags(F);
std::unique_ptr<FunctionSummary> FuncSummary =
llvm::make_unique<FunctionSummary>(Flags, NumInsts);
FuncSummary->addCallGraphEdges(CallGraphEdges);
FuncSummary->addCallGraphEdges(IndirectCallEdges);
FuncSummary->addRefEdges(RefEdges);
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
static void
computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
ProfileSummaryInfo *PSI, bool HasLocalsInUsedOrAsm,
DenseSet<GlobalValue::GUID> &CantBePromoted) {
// Summary not currently supported for anonymous functions, they should
// have been named.
assert(F.hasName());
unsigned NumInsts = 0;
// Map from callee ValueId to profile count. Used to accumulate profile
// counts for all static calls to a given callee.
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
SetVector<ValueInfo> RefEdges;
SetVector<GlobalValue::GUID> TypeTests;
SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
TypeCheckedLoadVCalls;
SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
TypeCheckedLoadConstVCalls;
ICallPromotionAnalysis ICallAnalysis;
SmallPtrSet<const User *, 8> Visited;
// Add personality function, prefix data and prologue data to function's ref
// list.
findRefEdges(Index, &F, RefEdges, Visited);
bool HasInlineAsmMaybeReferencingInternal = false;
for (const BasicBlock &BB : F)
for (const Instruction &I : BB) {
if (isa<DbgInfoIntrinsic>(I))
continue;
++NumInsts;
findRefEdges(Index, &I, RefEdges, Visited);
auto CS = ImmutableCallSite(&I);
if (!CS)
continue;
const auto *CI = dyn_cast<CallInst>(&I);
// Since we don't know exactly which local values are referenced in inline
// assembly, conservatively mark the function as possibly referencing
// a local value from inline assembly to ensure we don't export a
// reference (which would require renaming and promotion of the
// referenced value).
if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
HasInlineAsmMaybeReferencingInternal = true;
auto *CalledValue = CS.getCalledValue();
auto *CalledFunction = CS.getCalledFunction();
if (CalledValue && !CalledFunction) {
CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
// Stripping pointer casts can reveal a called function.
CalledFunction = dyn_cast<Function>(CalledValue);
}
// Check if this is an alias to a function. If so, get the
// called aliasee for the checks below.
if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
assert(!CalledFunction && "Expected null called function in callsite for alias");
CalledFunction = dyn_cast<Function>(GA->getBaseObject());
}
// Check if this is a direct call to a known function or a known
// intrinsic, or an indirect call with profile data.
if (CalledFunction) {
if (CI && CalledFunction->isIntrinsic()) {
addIntrinsicToSummary(
CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls);
continue;
}
// We should have named any anonymous globals
assert(CalledFunction->hasName());
auto ScaledCount = PSI->getProfileCount(&I, BFI);
auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
: CalleeInfo::HotnessType::Unknown;
if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
Hotness = CalleeInfo::HotnessType::Cold;
// Use the original CalledValue, in case it was an alias. We want
// to record the call edge to the alias in that case. Eventually
// an alias summary will be created to associate the alias and
// aliasee.
auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
cast<GlobalValue>(CalledValue))];
ValueInfo.updateHotness(Hotness);
// Add the relative block frequency to CalleeInfo if there is no profile
// information.
if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
uint64_t EntryFreq = BFI->getEntryFreq();
ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
}
} else {
// Skip inline assembly calls.
if (CI && CI->isInlineAsm())
continue;
// Skip direct calls.
if (!CalledValue || isa<Constant>(CalledValue))
continue;
// Check if the instruction has a callees metadata. If so, add callees
// to CallGraphEdges to reflect the references from the metadata, and
// to enable importing for subsequent indirect call promotion and
// inlining.
if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
for (auto &Op : MD->operands()) {
Function *Callee = mdconst::extract_or_null<Function>(Op);
if (Callee)
CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
}
}
uint32_t NumVals, NumCandidates;
uint64_t TotalCount;
auto CandidateProfileData =
ICallAnalysis.getPromotionCandidatesForInstruction(
&I, NumVals, TotalCount, NumCandidates);
for (auto &Candidate : CandidateProfileData)
CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
.updateHotness(getHotness(Candidate.Count, PSI));
}
}
// Explicit add hot edges to enforce importing for designated GUIDs for
// sample PGO, to enable the same inlines as the profiled optimized binary.
for (auto &I : F.getImportGUIDs())
CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
ForceSummaryEdgesCold == FunctionSummary::FSHT_All
? CalleeInfo::HotnessType::Cold
: CalleeInfo::HotnessType::Critical);
bool NonRenamableLocal = isNonRenamableLocal(F);
bool NotEligibleForImport =
NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
// Inliner doesn't handle variadic functions.
// FIXME: refactor this to use the same code that inliner is using.
F.isVarArg() ||
// Don't try to import functions with noinline attribute.
F.getAttributes().hasFnAttribute(Attribute::NoInline);
GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
/* Live = */ false, F.isDSOLocal());
FunctionSummary::FFlags FunFlags{
F.hasFnAttribute(Attribute::ReadNone),
F.hasFnAttribute(Attribute::ReadOnly),
F.hasFnAttribute(Attribute::NoRecurse),
F.returnDoesNotAlias(),
};
auto FuncSummary = llvm::make_unique<FunctionSummary>(
Flags, NumInsts, FunFlags, RefEdges.takeVector(),
CallGraphEdges.takeVector(), TypeTests.takeVector(),
TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
TypeTestAssumeConstVCalls.takeVector(),
TypeCheckedLoadConstVCalls.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(F.getGUID());
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
}
/// gold informs us that all symbols have been read. At this point, we use
/// get_symbols to see if any of our definitions have been overridden by a
/// native object file. Then, perform optimization and codegen.
static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
if (Modules.empty())
return LDPS_OK;
if (unsigned NumOpts = options::extra.size())
cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
// If we are doing ThinLTO compilation, simply build the combined
// module index/summary and emit it. We don't need to parse the modules
// and link them in this case.
if (options::thinlto) {
ModuleSummaryIndex CombinedIndex;
uint64_t NextModuleId = 0;
for (claimed_file &F : Modules) {
PluginInputFile InputFile(F.handle);
std::unique_ptr<ModuleSummaryIndex> Index =
getModuleSummaryIndexForFile(F, InputFile.file());
// Skip files without a module summary.
if (Index)
CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
}
std::error_code EC;
raw_fd_ostream OS(output_name + ".thinlto.bc", EC,
sys::fs::OpenFlags::F_None);
if (EC)
message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
output_name.data(), EC.message().c_str());
WriteIndexToFile(CombinedIndex, OS);
OS.close();
if (options::thinlto_index_only) {
cleanup_hook();
exit(0);
}
thinLTOBackends(ApiFile, CombinedIndex);
return LDPS_OK;
}
LLVMContext Context;
Context.setDiscardValueNames(options::DiscardValueNames);
Context.enableDebugTypeODRUniquing(); // Merge debug info types.
Context.setDiagnosticHandler(diagnosticHandlerForContext, nullptr, true);
std::unique_ptr<Module> Combined(new Module("ld-temp.o", Context));
IRMover L(*Combined);
StringSet<> Internalize;
StringSet<> Maybe;
for (claimed_file &F : Modules) {
PluginInputFile InputFile(F.handle);
const void *View = getSymbolsAndView(F);
if (!View)
continue;
if (linkInModule(Context, L, F, View, InputFile.file(), ApiFile,
Internalize, Maybe))
message(LDPL_FATAL, "Failed to link module");
}
for (const auto &Name : Internalize) {
GlobalValue *GV = Combined->getNamedValue(Name.first());
if (GV)
internalize(*GV);
}
for (const auto &Name : Maybe) {
GlobalValue *GV = Combined->getNamedValue(Name.first());
if (!GV)
continue;
GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
if (canBeOmittedFromSymbolTable(GV))
internalize(*GV);
}
if (options::TheOutputType == options::OT_DISABLE)
return LDPS_OK;
if (options::TheOutputType != options::OT_NORMAL) {
std::string path;
if (options::TheOutputType == options::OT_BC_ONLY)
path = output_name;
else
path = output_name + ".bc";
saveBCFile(path, *Combined);
if (options::TheOutputType == options::OT_BC_ONLY)
return LDPS_OK;
}
CodeGen codeGen(std::move(Combined));
codeGen.runAll();
if (!options::extra_library_path.empty() &&
set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK)
message(LDPL_FATAL, "Unable to set the extra library path.");
return LDPS_OK;
}
