bool Devirtualizer::devirtualizeAppliesInFunction(SILFunction &F,
ClassHierarchyAnalysis *CHA) {
bool Changed = false;
llvm::SmallVector<ApplySite, 8> NewApplies;
OptRemark::Emitter ORE(DEBUG_TYPE, F.getModule());
SmallVector<ApplySite, 16> Applies;
for (auto &BB : F) {
for (auto It = BB.begin(), End = BB.end(); It != End;) {
auto &I = *It++;
// Skip non-apply instructions.
auto Apply = ApplySite::isa(&I);
if (!Apply)
continue;
Applies.push_back(Apply);
}
}
for (auto Apply : Applies) {
auto NewInst = tryDevirtualizeApply(Apply, CHA, &ORE);
if (!NewInst)
continue;
Changed = true;
deleteDevirtualizedApply(Apply);
NewApplies.push_back(NewInst);
}
// For each new apply, attempt to link in function bodies if we do
// not already have them, then notify the pass manager of the new
// functions.
//
// We do this after deleting the old applies because otherwise we
// hit verification errors in the linking code due to having
// non-cond_br critical edges.
while (!NewApplies.empty()) {
auto Apply = NewApplies.pop_back_val();
auto *CalleeFn = Apply.getReferencedFunction();
assert(CalleeFn && "Expected devirtualized callee!");
// We need to ensure that we link after devirtualizing in order to pull in
// everything we reference from another module, which may expose optimization
// opportunities and is also needed for correctness if we reference functions
// with non-public linkage. See lib/SIL/Linker.cpp for details.
if (!CalleeFn->isDefinition())
F.getModule().linkFunction(CalleeFn, SILModule::LinkingMode::LinkAll);
// We may not have optimized these functions yet, and it could
// be beneficial to rerun some earlier passes on the current
// function now that we've made these direct references visible.
if (CalleeFn->isDefinition() && CalleeFn->shouldOptimize())
addFunctionToPassManagerWorklist(CalleeFn, nullptr);
}
return Changed;
}
static bool runSROAOnFunction(SILFunction &Fn) {
std::vector<AllocStackInst *> Worklist;
bool Changed = false;
// For each basic block BB in Fn...
for (auto &BB : Fn)
// For each instruction in BB...
for (auto &I : BB)
// If the instruction is an alloc stack inst, add it to the worklist.
if (auto *AI = dyn_cast<AllocStackInst>(&I))
if (shouldExpand(Fn.getModule(), AI->getElementType()))
Worklist.push_back(AI);
while (!Worklist.empty()) {
AllocStackInst *AI = Worklist.back();
Worklist.pop_back();
SROAMemoryUseAnalyzer Analyzer(AI);
if (!Analyzer.analyze())
continue;
Changed = true;
Analyzer.chopUpAlloca(Worklist);
}
return Changed;
}
SILFunction *SILModule::findFunction(StringRef Name, SILLinkage Linkage) {
assert((Linkage == SILLinkage::Public ||
Linkage == SILLinkage::PublicExternal) &&
"Only a lookup of public functions is supported currently");
SILFunction *F = nullptr;
// First, check if there is a function with a required name in the
// current module.
SILFunction *CurF = lookUpFunction(Name);
// Nothing to do if the current module has a required function
// with a proper linkage already.
if (CurF && CurF->getLinkage() == Linkage) {
F = CurF;
} else {
assert((!CurF || CurF->getLinkage() != Linkage) &&
"hasFunction should be only called for functions that are not "
"contained in the SILModule yet or do not have a required linkage");
}
if (!F) {
if (CurF) {
// Perform this lookup only if a function with a given
// name is present in the current module.
// This is done to reduce the amount of IO from the
// swift module file.
if (!getSILLoader()->hasSILFunction(Name, Linkage))
return nullptr;
// The function in the current module will be changed.
F = CurF;
}
// If function with a given name wasn't seen anywhere yet
// or if it is known to exist, perform a lookup.
if (!F) {
// Try to load the function from other modules.
F = getSILLoader()->lookupSILFunction(Name, /*declarationOnly*/ true,
Linkage);
// Bail if nothing was found and we are not sure if
// this function exists elsewhere.
if (!F)
return nullptr;
assert(F && "SILFunction should be present in one of the modules");
assert(F->getLinkage() == Linkage && "SILFunction has a wrong linkage");
}
}
// If a function exists already and it is a non-optimizing
// compilation, simply convert it into an external declaration,
// so that a compiled version from the shared library is used.
if (F->isDefinition() &&
!F->getModule().getOptions().shouldOptimize()) {
F->convertToDeclaration();
}
if (F->isExternalDeclaration())
F->setSerialized(IsSerialized_t::IsNotSerialized);
F->setLinkage(Linkage);
return F;
}
/// Run dead argument elimination of partially applied functions.
/// After this optimization CapturePropagation can replace the partial_apply
/// by a direct reference to the specialized function.
bool removeDeadArgs(int minPartialAppliedArgs) {
if (minPartialAppliedArgs < 1)
return false;
if (!DeadArgumentAnalyzeParameters())
return false;
// Check if at least the minimum number of partially applied arguments
// are dead. Otherwise no partial_apply can be removed anyway.
for (unsigned Idx = 0, Num = ArgumentDescList.size(); Idx < Num; ++Idx) {
if (Idx < Num - minPartialAppliedArgs) {
// Don't remove arguments other than the partial applied ones, even if
// they are dead.
ArgumentDescList[Idx].IsEntirelyDead = false;
} else {
// Is the partially applied argument dead?
if (!ArgumentDescList[Idx].IsEntirelyDead)
return false;
// Currently we require that all dead parameters have trivial types.
// The reason is that it's very hard to find places where we can release
// those parameters (as a replacement for the removed partial_apply).
// TODO: maybe we can skip this restriction when we have semantic ARC.
if (!ArgumentDescList[Idx].Arg->getType().isTrivial(F->getModule()))
return false;
}
}
DEBUG(llvm::dbgs() << " remove dead arguments for partial_apply\n");
DeadArgumentTransformFunction();
createFunctionSignatureOptimizedFunction();
return true;
}
SILFunctionArgument *SILBasicBlock::createFunctionArgument(SILType Ty,
const ValueDecl *D) {
assert(isEntry() && "Function Arguments can only be in the entry block");
SILFunction *Parent = getParent();
auto OwnershipKind = ValueOwnershipKind(
Parent->getModule(), Ty,
Parent->getConventions().getSILArgumentConvention(getNumArguments()));
return new (getModule()) SILFunctionArgument(this, Ty, OwnershipKind, D);
}
SILType SILType::wrapAnyOptionalType(SILFunction &F) const {
SILModule &M = F.getModule();
EnumDecl *OptionalDecl = M.getASTContext().getOptionalDecl(OTK_Optional);
BoundGenericType *BoundEnumDecl =
BoundGenericType::get(OptionalDecl, Type(), {getSwiftRValueType()});
AbstractionPattern Pattern(F.getLoweredFunctionType()->getGenericSignature(),
BoundEnumDecl->getCanonicalType());
return M.Types.getLoweredType(Pattern, BoundEnumDecl);
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary to take into consideration the removed parameters.
void
PromotedParamCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block
SILBasicBlock *OrigEntryBB = &*Orig->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
unsigned ArgNo = 0;
auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end();
while (I != E) {
if (count(PromotedArgIndices, ArgNo)) {
// Create a new argument with the promoted type.
auto boxTy = (*I)->getType().castTo<SILBoxType>();
assert(boxTy->getLayout()->getFields().size() == 1
&& "promoting multi-field boxes not implemented yet");
auto promotedTy = boxTy->getFieldType(Cloned->getModule(), 0);
auto *promotedArg =
ClonedEntryBB->createFunctionArgument(promotedTy, (*I)->getDecl());
PromotedParameters.insert(*I);
// Map any projections of the box to the promoted argument.
for (auto use : (*I)->getUses()) {
if (auto project = dyn_cast<ProjectBoxInst>(use->getUser())) {
ValueMap.insert(std::make_pair(project, promotedArg));
}
}
} else {
// Create a new argument which copies the original argument.
SILValue MappedValue = ClonedEntryBB->createFunctionArgument(
(*I)->getType(), (*I)->getDecl());
ValueMap.insert(std::make_pair(*I, MappedValue));
}
++ArgNo;
++I;
}
getBuilder().setInsertionPoint(ClonedEntryBB);
BBMap.insert(std::make_pair(OrigEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(OrigEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
getBuilder().setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
}
std::string FunctionSignatureTransform::createOptimizedSILFunctionName() {
// Handle arguments' changes.
for (unsigned i : indices(ArgumentDescList)) {
const ArgumentDescriptor &Arg = ArgumentDescList[i];
if (Arg.IsEntirelyDead) {
OldFM.setArgumentDead(i);
NewFM.setArgumentDead(i);
// No point setting other attribute if argument is dead.
continue;
}
// If we have an @owned argument and found a callee release for it,
// convert the argument to guaranteed.
if (Arg.OwnedToGuaranteed) {
OldFM.setArgumentOwnedToGuaranteed(i);
NewFM.setArgumentOwnedToGuaranteed(i);
}
// If this argument is not dead and we can explode it, add 's' to the
// mangling.
if (Arg.Explode) {
OldFM.setArgumentSROA(i);
NewFM.setArgumentSROA(i);
}
}
// Handle return value's change.
// FIXME: handle multiple direct results here
if (ResultDescList.size() == 1 && !ResultDescList[0].CalleeRetain.empty()) {
OldFM.setReturnValueOwnedToUnowned();
NewFM.setReturnValueOwnedToUnowned();
}
OldFM.mangle();
SILModule &M = F->getModule();
std::string Old = getUniqueName(OldFM.getMangler().finalize(), M);
int UniqueID = 0;
std::string New;
do {
New = NewFM.mangle(UniqueID);
++UniqueID;
} while (M.lookUpFunction(New));
return NewMangling::selectMangling(Old, New);
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary to take into consideration the removed parameters.
void
PromotedParamCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block
SILBasicBlock *OrigEntryBB = &*Orig->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
SmallVector<SILValue, 4> entryArgs;
entryArgs.reserve(OrigEntryBB->getArguments().size());
// Initialize all NewPromotedArgs slots to an invalid value.
NewPromotedArgs.resize(OrigEntryBB->getArguments().size());
unsigned ArgNo = 0;
auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end();
while (I != E) {
if (count(PromotedArgIndices, ArgNo)) {
// Create a new argument with the promoted type.
auto boxTy = (*I)->getType().castTo<SILBoxType>();
assert(boxTy->getLayout()->getFields().size() == 1
&& "promoting multi-field boxes not implemented yet");
auto promotedTy = boxTy->getFieldType(Cloned->getModule(), 0);
auto *promotedArg =
ClonedEntryBB->createFunctionArgument(promotedTy, (*I)->getDecl());
OrigPromotedParameters.insert(*I);
NewPromotedArgs[ArgNo] = promotedArg;
// All uses of the promoted box should either be projections, which are
// folded when visited, or copy/destroy operations which are ignored.
entryArgs.push_back(SILValue());
} else {
// Create a new argument which copies the original argument.
entryArgs.push_back(ClonedEntryBB->createFunctionArgument(
(*I)->getType(), (*I)->getDecl()));
}
++ArgNo;
++I;
}
// Visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions and terminators.
cloneFunctionBody(Orig, ClonedEntryBB, entryArgs);
}
std::string
FunctionSignatureTransformDescriptor::createOptimizedSILFunctionName() {
SILFunction *F = OriginalFunction;
auto P = Demangle::SpecializationPass::FunctionSignatureOpts;
Mangle::FunctionSignatureSpecializationMangler Mangler(P, F->isSerialized(),
F);
// Handle arguments' changes.
for (unsigned i : indices(ArgumentDescList)) {
const ArgumentDescriptor &Arg = ArgumentDescList[i];
if (Arg.IsEntirelyDead) {
Mangler.setArgumentDead(i);
// No point setting other attribute if argument is dead.
continue;
}
// If we have an @owned argument and found a callee release for it,
// convert the argument to guaranteed.
if (Arg.OwnedToGuaranteed) {
Mangler.setArgumentOwnedToGuaranteed(i);
}
// If this argument is not dead and we can explode it, add 's' to the
// mangling.
if (Arg.Explode) {
Mangler.setArgumentSROA(i);
}
}
// Handle return value's change.
// FIXME: handle multiple direct results here
if (ResultDescList.size() == 1 && !ResultDescList[0].CalleeRetain.empty()) {
Mangler.setReturnValueOwnedToUnowned();
}
auto MangledName = Mangler.mangle();
assert(!F->getModule().hasFunction(MangledName));
return MangledName;
}
CanSILFunctionType FunctionSignatureTransform::createOptimizedSILFunctionType() {
CanSILFunctionType FTy = F->getLoweredFunctionType();
// The only way that we modify the arity of function parameters is here for
// dead arguments. Doing anything else is unsafe since by definition non-dead
// arguments will have SSA uses in the function. We would need to be smarter
// in our moving to handle such cases.
llvm::SmallVector<SILParameterInfo, 8> InterfaceParams;
for (auto &ArgDesc : ArgumentDescList) {
computeOptimizedArgInterface(ArgDesc, InterfaceParams);
}
// ResultDescs only covers the direct results; we currently can't ever
// change an indirect result. Piece the modified direct result information
// back into the all-results list.
llvm::SmallVector<SILResultInfo, 8> InterfaceResults;
auto &ResultDescs = ResultDescList;
for (SILResultInfo InterfaceResult : FTy->getResults()) {
if (InterfaceResult.isFormalDirect()) {
auto &RV = ResultDescs[0];
if (!RV.CalleeRetain.empty()) {
++NumOwnedConvertedToNotOwnedResult;
InterfaceResults.push_back(SILResultInfo(InterfaceResult.getType(),
ResultConvention::Unowned));
continue;
}
}
InterfaceResults.push_back(InterfaceResult);
}
// Don't use a method representation if we modified self.
auto ExtInfo = FTy->getExtInfo();
if (shouldModifySelfArgument) {
ExtInfo = ExtInfo.withRepresentation(SILFunctionTypeRepresentation::Thin);
}
return SILFunctionType::get(FTy->getGenericSignature(), ExtInfo,
FTy->getCalleeConvention(), InterfaceParams,
InterfaceResults, FTy->getOptionalErrorResult(),
F->getModule().getASTContext());
}
// Returns the callee of an apply_inst if it is basically inlinable.
SILFunction *swift::getEligibleFunction(FullApplySite AI,
InlineSelection WhatToInline) {
SILFunction *Callee = AI.getReferencedFunction();
if (!Callee) {
return nullptr;
}
// Not all apply sites can be inlined, even if they're direct.
if (!SILInliner::canInline(AI))
return nullptr;
ModuleDecl *SwiftModule = Callee->getModule().getSwiftModule();
bool IsInStdlib = (SwiftModule->isStdlibModule() ||
SwiftModule->isOnoneSupportModule());
// Don't inline functions that are marked with the @_semantics or @_effects
// attribute if the inliner is asked not to inline them.
if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
if (shouldSkipApplyDuringEarlyInlining(AI))
return nullptr;
if (Callee->hasSemanticsAttr("inline_late"))
return nullptr;
}
// The "availability" semantics attribute is treated like global-init.
if (Callee->hasSemanticsAttrs() &&
WhatToInline != InlineSelection::Everything &&
(Callee->hasSemanticsAttrThatStartsWith("availability") ||
(Callee->hasSemanticsAttrThatStartsWith("inline_late")))) {
return nullptr;
}
if (Callee->hasSemanticsAttrs() &&
WhatToInline == InlineSelection::Everything) {
if (Callee->hasSemanticsAttrThatStartsWith("inline_late") && IsInStdlib) {
return nullptr;
}
}
} else if (Callee->isGlobalInit()) {
if (WhatToInline != InlineSelection::Everything) {
return nullptr;
}
}
// We can't inline external declarations.
if (Callee->empty() || Callee->isExternalDeclaration()) {
return nullptr;
}
// Explicitly disabled inlining.
if (Callee->getInlineStrategy() == NoInline) {
return nullptr;
}
if (!Callee->shouldOptimize()) {
return nullptr;
}
SILFunction *Caller = AI.getFunction();
// We don't support inlining a function that binds dynamic self because we
// have no mechanism to preserve the original function's local self metadata.
if (mayBindDynamicSelf(Callee)) {
// Check if passed Self is the same as the Self of the caller.
// In this case, it is safe to inline because both functions
// use the same Self.
if (AI.hasSelfArgument() && Caller->hasSelfParam()) {
auto CalleeSelf = stripCasts(AI.getSelfArgument());
auto CallerSelf = Caller->getSelfArgument();
if (CalleeSelf != SILValue(CallerSelf))
return nullptr;
} else
return nullptr;
}
// Detect self-recursive calls.
if (Caller == Callee) {
return nullptr;
}
// A non-fragile function may not be inlined into a fragile function.
if (Caller->isSerialized() &&
!Callee->hasValidLinkageForFragileInline()) {
if (!Callee->hasValidLinkageForFragileRef()) {
llvm::errs() << "caller: " << Caller->getName() << "\n";
llvm::errs() << "callee: " << Callee->getName() << "\n";
llvm_unreachable("Should never be inlining a resilient function into "
"a fragile function");
}
return nullptr;
}
// Inlining self-recursive functions into other functions can result
// in excessive code duplication since we run the inliner multiple
// times in our pipeline
if (calleeIsSelfRecursive(Callee)) {
return nullptr;
}
if (!EnableSILInliningOfGenerics && AI.hasSubstitutions()) {
// Inlining of generics is not allowed unless it is an @inline(__always)
// or transparent function.
if (Callee->getInlineStrategy() != AlwaysInline && !Callee->isTransparent())
return nullptr;
}
// We cannot inline function with layout constraints on its generic types
// if the corresponding substitution type does not have the same constraints.
// The reason for this restriction is that we'd need to be able to express
// in SIL something like casting a value of generic type T into a value of
// generic type T: _LayoutConstraint, which is impossible currently.
if (EnableSILInliningOfGenerics && AI.hasSubstitutions()) {
if (!isCallerAndCalleeLayoutConstraintsCompatible(AI))
return nullptr;
}
// IRGen cannot handle partial_applies containing opened_existentials
// in its substitutions list.
if (calleeHasPartialApplyWithOpenedExistentials(AI)) {
return nullptr;
}
return Callee;
}
/// \brief Make sure that all parameters are passed with a reference count
/// neutral parameter convention except for self.
bool swift::ArraySemanticsCall::isValidSignature() {
assert(SemanticsCall && getKind() != ArrayCallKind::kNone &&
"Need an array semantic call");
FunctionRefInst *FRI = cast<FunctionRefInst>(SemanticsCall->getCallee());
SILFunction *F = FRI->getReferencedFunction();
auto FnTy = F->getLoweredFunctionType();
auto &Mod = F->getModule();
// Check whether we have a valid signature for semantic calls that we hoist.
switch (getKind()) {
// All other calls can be consider valid.
default: break;
case ArrayCallKind::kArrayPropsIsNativeTypeChecked: {
// @guaranteed/@owned Self
if (SemanticsCall->getNumArguments() != 1)
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kCheckIndex: {
// Int, @guaranteed/@owned Self
if (SemanticsCall->getNumArguments() != 2 ||
!SemanticsCall->getArgument(0)->getType().isTrivial(Mod))
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kCheckSubscript: {
// Int, Bool, Self
if (SemanticsCall->getNumArguments() != 3 ||
!SemanticsCall->getArgument(0)->getType().isTrivial(Mod))
return false;
if (!SemanticsCall->getArgument(1)->getType().isTrivial(Mod))
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kMakeMutable: {
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Indirect_Inout;
}
case ArrayCallKind::kArrayUninitialized: {
// Make sure that if we are a _adoptStorage call that our storage is
// uniquely referenced by us.
SILValue Arg0 = SemanticsCall->getArgument(0);
if (Arg0->getType().isExistentialType()) {
auto *AllocBufferAI = dyn_cast<ApplyInst>(Arg0);
if (!AllocBufferAI)
return false;
auto *AllocFn = AllocBufferAI->getReferencedFunction();
if (!AllocFn || AllocFn->getName() != "swift_bufferAllocate" ||
!hasOneNonDebugUse(AllocBufferAI))
return false;
}
return true;
}
case ArrayCallKind::kWithUnsafeMutableBufferPointer: {
SILFunctionConventions origConv(SemanticsCall->getOrigCalleeType(), Mod);
if (origConv.getNumIndirectSILResults() != 1
|| SemanticsCall->getNumArguments() != 3)
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Indirect_Inout;
}
}
return true;
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILModule &M = Cloned->getModule();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = new (M) SILBasicBlock(Cloned);
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->bbarg_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getBBArg(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
new (M) SILArgument(ClonedEntryBB, Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue =
new (M) SILArgument(ClonedEntryBB, PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
/// In this function we create the actual cloned function and its proper cloned
/// type. But we do not create any body. This implies that the creation of the
/// actual arguments in the function is in populateCloned.
///
/// \arg PAUser The function that is being passed the partial apply.
/// \arg PAI The partial apply that is being passed to PAUser.
/// \arg ClosureIndex The index of the partial apply in PAUser's function
/// signature.
/// \arg ClonedName The name of the cloned function that we will create.
SILFunction *
ClosureSpecCloner::initCloned(SILOptFunctionBuilder &FunctionBuilder,
const CallSiteDescriptor &CallSiteDesc,
StringRef ClonedName) {
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// This is the list of new interface parameters of the cloned function.
llvm::SmallVector<SILParameterInfo, 4> NewParameterInfoList;
// First add to NewParameterInfoList all of the SILParameterInfo in the
// original function except for the closure.
CanSILFunctionType ClosureUserFunTy = ClosureUser->getLoweredFunctionType();
auto ClosureUserConv = ClosureUser->getConventions();
unsigned Index = ClosureUserConv.getSILArgIndexOfFirstParam();
for (auto ¶m : ClosureUserConv.getParameters()) {
if (Index != CallSiteDesc.getClosureIndex())
NewParameterInfoList.push_back(param);
++Index;
}
// Then add any arguments that are captured in the closure to the function's
// argument type. Since they are captured, we need to pass them directly into
// the new specialized function.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
auto ClosedOverFunConv = ClosedOverFun->getConventions();
SILModule &M = ClosureUser->getModule();
// Captured parameters are always appended to the function signature. If the
// type of the captured argument is:
// - direct and trivial, pass the argument as Direct_Unowned.
// - direct and non-trivial, pass the argument as Direct_Owned.
// - indirect, pass the argument using the same parameter convention as in the
// original closure.
//
// We use the type of the closure here since we allow for the closure to be an
// external declaration.
unsigned NumTotalParams = ClosedOverFunConv.getNumParameters();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) {
ParameterConvention ParamConv;
if (PInfo.isFormalIndirect()) {
ParamConv = PInfo.getConvention();
assert(!SILModuleConventions(M).useLoweredAddresses()
|| ParamConv == ParameterConvention::Indirect_Inout
|| ParamConv == ParameterConvention::Indirect_InoutAliasable);
} else {
ParamConv = ClosedOverFunConv.getSILType(PInfo).isTrivial(M)
? ParameterConvention::Direct_Unowned
: ParameterConvention::Direct_Owned;
}
SILParameterInfo NewPInfo(PInfo.getType(), ParamConv);
NewParameterInfoList.push_back(NewPInfo);
}
// The specialized function is always a thin function. This is important
// because we may add additional parameters after the Self parameter of
// witness methods. In this case the new function is not a method anymore.
auto ExtInfo = ClosureUserFunTy->getExtInfo();
ExtInfo = ExtInfo.withRepresentation(SILFunctionTypeRepresentation::Thin);
auto ClonedTy = SILFunctionType::get(
ClosureUserFunTy->getGenericSignature(), ExtInfo,
ClosureUserFunTy->getCoroutineKind(),
ClosureUserFunTy->getCalleeConvention(), NewParameterInfoList,
ClosureUserFunTy->getYields(), ClosureUserFunTy->getResults(),
ClosureUserFunTy->getOptionalErrorResult(), M.getASTContext());
// We make this function bare so we don't have to worry about decls in the
// SILArgument.
auto *Fn = FunctionBuilder.createFunction(
// It's important to use a shared linkage for the specialized function
// and not the original linkage.
// Otherwise the new function could have an external linkage (in case the
// original function was de-serialized) and would not be code-gen'd.
// It's also important to disconnect this specialized function from any
// classes (the classSubclassScope), because that may incorrectly
// influence the linkage.
getSpecializedLinkage(ClosureUser, ClosureUser->getLinkage()), ClonedName,
ClonedTy, ClosureUser->getGenericEnvironment(),
ClosureUser->getLocation(), IsBare, ClosureUser->isTransparent(),
CallSiteDesc.isSerialized(), IsNotDynamic, ClosureUser->getEntryCount(),
ClosureUser->isThunk(),
/*classSubclassScope=*/SubclassScope::NotApplicable,
ClosureUser->getInlineStrategy(), ClosureUser->getEffectsKind(),
ClosureUser, ClosureUser->getDebugScope());
if (!ClosureUser->hasQualifiedOwnership()) {
Fn->setUnqualifiedOwnership();
}
for (auto &Attr : ClosureUser->getSemanticsAttrs())
Fn->addSemanticsAttr(Attr);
return Fn;
}
/// In this function we create the actual cloned function and its proper cloned
/// type. But we do not create any body. This implies that the creation of the
/// actual arguments in the function is in populateCloned.
///
/// \arg PAUser The function that is being passed the partial apply.
/// \arg PAI The partial apply that is being passed to PAUser.
/// \arg ClosureIndex The index of the partial apply in PAUser's function
/// signature.
/// \arg ClonedName The name of the cloned function that we will create.
SILFunction *
ClosureSpecCloner::initCloned(const CallSiteDescriptor &CallSiteDesc,
StringRef ClonedName) {
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// This is the list of new interface parameters of the cloned function.
llvm::SmallVector<SILParameterInfo, 4> NewParameterInfoList;
// First add to NewParameterInfoList all of the SILParameterInfo in the
// original function except for the closure.
CanSILFunctionType ClosureUserFunTy = ClosureUser->getLoweredFunctionType();
unsigned Index = ClosureUserFunTy->getNumIndirectResults();
for (auto ¶m : ClosureUserFunTy->getParameters()) {
if (Index != CallSiteDesc.getClosureIndex())
NewParameterInfoList.push_back(param);
++Index;
}
// Then add any arguments that are captured in the closure to the function's
// argument type. Since they are captured, we need to pass them directly into
// the new specialized function.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
SILModule &M = ClosureUser->getModule();
// Captured parameters are always appended to the function signature. If the
// type of the captured argument is trivial, pass the argument as
// Direct_Unowned. Otherwise pass it as Direct_Owned.
//
// We use the type of the closure here since we allow for the closure to be an
// external declaration.
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
if (PInfo.getSILType().isTrivial(M)) {
SILParameterInfo NewPInfo(PInfo.getType(),
ParameterConvention::Direct_Unowned);
NewParameterInfoList.push_back(NewPInfo);
continue;
}
SILParameterInfo NewPInfo(PInfo.getType(),
ParameterConvention::Direct_Owned);
NewParameterInfoList.push_back(NewPInfo);
}
// The specialized function is always a thin function. This is important
// because we may add additional parameters after the Self parameter of
// witness methods. In this case the new function is not a method anymore.
auto ExtInfo = ClosureUserFunTy->getExtInfo();
ExtInfo = ExtInfo.withRepresentation(SILFunctionTypeRepresentation::Thin);
auto ClonedTy = SILFunctionType::get(
ClosureUserFunTy->getGenericSignature(), ExtInfo,
ClosureUserFunTy->getCalleeConvention(), NewParameterInfoList,
ClosureUserFunTy->getAllResults(),
ClosureUserFunTy->getOptionalErrorResult(),
M.getASTContext());
// We make this function bare so we don't have to worry about decls in the
// SILArgument.
auto *Fn = M.createFunction(
// It's important to use a shared linkage for the specialized function
// and not the original linkage.
// Otherwise the new function could have an external linkage (in case the
// original function was de-serialized) and would not be code-gen'd.
getSpecializedLinkage(ClosureUser, ClosureUser->getLinkage()),
ClonedName, ClonedTy,
ClosureUser->getGenericEnvironment(), ClosureUser->getLocation(),
IsBare, ClosureUser->isTransparent(), CallSiteDesc.isFragile(),
ClosureUser->isThunk(), ClosureUser->getClassVisibility(),
ClosureUser->getInlineStrategy(), ClosureUser->getEffectsKind(),
ClosureUser, ClosureUser->getDebugScope());
Fn->setDeclCtx(ClosureUser->getDeclContext());
if (ClosureUser->hasUnqualifiedOwnership()) {
Fn->setUnqualifiedOwnership();
}
for (auto &Attr : ClosureUser->getSemanticsAttrs())
Fn->addSemanticsAttr(Attr);
return Fn;
}
void FunctionSignatureTransform::createFunctionSignatureOptimizedFunction() {
// Create the optimized function !
SILModule &M = F->getModule();
std::string Name = createOptimizedSILFunctionName();
SILLinkage linkage = F->getLinkage();
if (isAvailableExternally(linkage))
linkage = SILLinkage::Shared;
DEBUG(llvm::dbgs() << " -> create specialized function " << Name << "\n");
NewF = M.createFunction(linkage, Name, createOptimizedSILFunctionType(),
F->getGenericEnvironment(), F->getLocation(),
F->isBare(), F->isTransparent(), F->isSerialized(),
F->isThunk(), F->getClassVisibility(),
F->getInlineStrategy(), F->getEffectsKind(), nullptr,
F->getDebugScope());
if (F->hasUnqualifiedOwnership()) {
NewF->setUnqualifiedOwnership();
}
// Then we transfer the body of F to NewF.
NewF->spliceBody(F);
// Array semantic clients rely on the signature being as in the original
// version.
for (auto &Attr : F->getSemanticsAttrs()) {
if (!StringRef(Attr).startswith("array."))
NewF->addSemanticsAttr(Attr);
}
// Do the last bit of work to the newly created optimized function.
ArgumentExplosionFinalizeOptimizedFunction();
DeadArgumentFinalizeOptimizedFunction();
// Create the thunk body !
F->setThunk(IsThunk);
// The thunk now carries the information on how the signature is
// optimized. If we inline the thunk, we will get the benefit of calling
// the signature optimized function without additional setup on the
// caller side.
F->setInlineStrategy(AlwaysInline);
SILBasicBlock *ThunkBody = F->createBasicBlock();
for (auto &ArgDesc : ArgumentDescList) {
ThunkBody->createFunctionArgument(ArgDesc.Arg->getType(), ArgDesc.Decl);
}
SILLocation Loc = ThunkBody->getParent()->getLocation();
SILBuilder Builder(ThunkBody);
Builder.setCurrentDebugScope(ThunkBody->getParent()->getDebugScope());
FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF);
// Create the args for the thunk's apply, ignoring any dead arguments.
llvm::SmallVector<SILValue, 8> ThunkArgs;
for (auto &ArgDesc : ArgumentDescList) {
addThunkArgument(ArgDesc, Builder, ThunkBody, ThunkArgs);
}
// We are ignoring generic functions and functions with out parameters for
// now.
SILValue ReturnValue;
SILType LoweredType = NewF->getLoweredType();
SILType ResultType = NewF->getConventions().getSILResultType();
auto FunctionTy = LoweredType.castTo<SILFunctionType>();
if (FunctionTy->hasErrorResult()) {
// We need a try_apply to call a function with an error result.
SILFunction *Thunk = ThunkBody->getParent();
SILBasicBlock *NormalBlock = Thunk->createBasicBlock();
ReturnValue =
NormalBlock->createPHIArgument(ResultType, ValueOwnershipKind::Owned);
SILBasicBlock *ErrorBlock = Thunk->createBasicBlock();
SILType Error =
SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType());
auto *ErrorArg =
ErrorBlock->createPHIArgument(Error, ValueOwnershipKind::Owned);
Builder.createTryApply(Loc, FRI, LoweredType, SubstitutionList(),
ThunkArgs, NormalBlock, ErrorBlock);
Builder.setInsertionPoint(ErrorBlock);
Builder.createThrow(Loc, ErrorArg);
Builder.setInsertionPoint(NormalBlock);
} else {
ReturnValue = Builder.createApply(Loc, FRI, LoweredType, ResultType,
SubstitutionList(), ThunkArgs,
false);
}
// Set up the return results.
if (NewF->isNoReturnFunction()) {
Builder.createUnreachable(Loc);
} else {
Builder.createReturn(Loc, ReturnValue);
}
// Do the last bit work to finalize the thunk.
OwnedToGuaranteedFinalizeThunkFunction(Builder, F);
assert(F->getDebugScope()->Parent != NewF->getDebugScope()->Parent);
}
bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI,
Weight CallerWeight,
ConstantTracker &callerTracker,
int &NumCallerBlocks,
bool IsGeneric) {
SILFunction *Callee = AI.getReferencedFunction();
SILLoopInfo *LI = LA->get(Callee);
ShortestPathAnalysis *SPA = getSPA(Callee, LI);
assert(SPA->isValid());
ConstantTracker constTracker(Callee, &callerTracker, AI);
DominanceInfo *DT = DA->get(Callee);
SILBasicBlock *CalleeEntry = &Callee->front();
DominanceOrder domOrder(CalleeEntry, DT, Callee->size());
// Calculate the inlining cost of the callee.
int CalleeCost = 0;
int Benefit = 0;
// Start with a base benefit.
int BaseBenefit = RemovedCallBenefit;
const SILOptions &Opts = Callee->getModule().getOptions();
// For some reason -Ounchecked can accept a higher base benefit without
// increasing the code size too much.
if (Opts.Optimization == SILOptions::SILOptMode::OptimizeUnchecked)
BaseBenefit *= 2;
CallerWeight.updateBenefit(Benefit, BaseBenefit);
// Go through all blocks of the function, accumulate the cost and find
// benefits.
while (SILBasicBlock *block = domOrder.getNext()) {
constTracker.beginBlock();
Weight BlockW = SPA->getWeight(block, CallerWeight);
for (SILInstruction &I : *block) {
constTracker.trackInst(&I);
CalleeCost += (int)instructionInlineCost(I);
if (FullApplySite AI = FullApplySite::isa(&I)) {
// Check if the callee is passed as an argument. If so, increase the
// threshold, because inlining will (probably) eliminate the closure.
SILInstruction *def = constTracker.getDefInCaller(AI.getCallee());
if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def)))
BlockW.updateBenefit(Benefit, RemovedClosureBenefit);
} else if (auto *LI = dyn_cast<LoadInst>(&I)) {
// Check if it's a load from a stack location in the caller. Such a load
// might be optimized away if inlined.
if (constTracker.isStackAddrInCaller(LI->getOperand()))
BlockW.updateBenefit(Benefit, RemovedLoadBenefit);
} else if (auto *SI = dyn_cast<StoreInst>(&I)) {
// Check if it's a store to a stack location in the caller. Such a load
// might be optimized away if inlined.
if (constTracker.isStackAddrInCaller(SI->getDest()))
BlockW.updateBenefit(Benefit, RemovedStoreBenefit);
} else if (isa<StrongReleaseInst>(&I) || isa<ReleaseValueInst>(&I)) {
SILValue Op = stripCasts(I.getOperand(0));
if (SILArgument *Arg = dyn_cast<SILArgument>(Op)) {
if (Arg->isFunctionArg() && Arg->getArgumentConvention() ==
SILArgumentConvention::Direct_Guaranteed) {
BlockW.updateBenefit(Benefit, RefCountBenefit);
}
}
} else if (auto *BI = dyn_cast<BuiltinInst>(&I)) {
if (BI->getBuiltinInfo().ID == BuiltinValueKind::OnFastPath)
BlockW.updateBenefit(Benefit, FastPathBuiltinBenefit);
}
}
// Don't count costs in blocks which are dead after inlining.
SILBasicBlock *takenBlock = constTracker.getTakenBlock(block->getTerminator());
if (takenBlock) {
BlockW.updateBenefit(Benefit, RemovedTerminatorBenefit);
domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) {
return child->getSinglePredecessor() != block || child == takenBlock;
});
} else {
domOrder.pushChildren(block);
}
}
if (AI.getFunction()->isThunk()) {
// Only inline trivial functions into thunks (which will not increase the
// code size).
if (CalleeCost > TrivialFunctionThreshold)
return false;
DEBUG(
dumpCaller(AI.getFunction());
llvm::dbgs() << " decision {" << CalleeCost << " into thunk} " <<
Callee->getName() << '\n';
);
return true;
}
bool Devirtualizer::devirtualizeAppliesInFunction(SILFunction &F,
ClassHierarchyAnalysis *CHA) {
bool Changed = false;
llvm::SmallVector<SILInstruction *, 8> DeadApplies;
llvm::SmallVector<ApplySite, 8> NewApplies;
SmallVector<ApplySite, 16> Applies;
for (auto &BB : F) {
for (auto It = BB.begin(), End = BB.end(); It != End;) {
auto &I = *It++;
// Skip non-apply instructions.
auto Apply = ApplySite::isa(&I);
if (!Apply)
continue;
Applies.push_back(Apply);
}
}
for (auto Apply : Applies) {
auto NewInstPair = tryDevirtualizeApply(Apply, CHA);
if (!NewInstPair.second)
continue;
Changed = true;
auto *AI = Apply.getInstruction();
if (!isa<TryApplyInst>(AI))
AI->replaceAllUsesWith(NewInstPair.first);
DeadApplies.push_back(AI);
NewApplies.push_back(NewInstPair.second);
}
// Remove all the now-dead applies.
while (!DeadApplies.empty()) {
auto *AI = DeadApplies.pop_back_val();
recursivelyDeleteTriviallyDeadInstructions(AI, true);
}
// For each new apply, attempt to link in function bodies if we do
// not already have them, then notify the pass manager of the new
// functions.
//
// We do this after deleting the old applies because otherwise we
// hit verification errors in the linking code due to having
// non-cond_br critical edges.
while (!NewApplies.empty()) {
auto Apply = NewApplies.pop_back_val();
auto *CalleeFn = Apply.getReferencedFunction();
assert(CalleeFn && "Expected devirtualized callee!");
// We need to ensure that we link after devirtualizing in order to pull in
// everything we reference from another module. This is especially important
// for transparent functions, because if transparent functions are not
// inlined for some reason, we need to generate code for them.
// Note that functions, which are only referenced from witness/vtables, are
// not linked upfront by the SILLinker.
if (!CalleeFn->isDefinition())
F.getModule().linkFunction(CalleeFn, SILModule::LinkingMode::LinkAll);
// We may not have optimized these functions yet, and it could
// be beneficial to rerun some earlier passes on the current
// function now that we've made these direct references visible.
if (CalleeFn->isDefinition() && CalleeFn->shouldOptimize())
notifyPassManagerOfFunction(CalleeFn, nullptr);
}
return Changed;
}
