/// \brief Inlines all mandatory inlined functions into the body of a function,
/// first recursively inlining all mandatory apply instructions in those
/// functions into their bodies if necessary.
///
/// \param F the function to be processed
/// \param AI nullptr if this is being called from the top level; the relevant
/// ApplyInst requiring the recursive call when non-null
/// \param FullyInlinedSet the set of all functions already known to be fully
/// processed, to avoid processing them over again
/// \param SetFactory an instance of ImmutableFunctionSet::Factory
/// \param CurrentInliningSet the set of functions currently being inlined in
/// the current call stack of recursive calls
///
/// \returns true if successful, false if failed due to circular inlining.
static bool
runOnFunctionRecursively(SILFunction *F, FullApplySite AI,
SILModule::LinkingMode Mode,
DenseFunctionSet &FullyInlinedSet,
ImmutableFunctionSet::Factory &SetFactory,
ImmutableFunctionSet CurrentInliningSet,
ClassHierarchyAnalysis *CHA) {
// Avoid reprocessing functions needlessly.
if (FullyInlinedSet.count(F))
return true;
// Prevent attempt to circularly inline.
if (CurrentInliningSet.contains(F)) {
// This cannot happen on a top-level call, so AI should be non-null.
assert(AI && "Cannot have circular inline without apply");
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::circular_transparent);
return false;
}
// Add to the current inlining set (immutably, so we only affect the set
// during this call and recursive subcalls).
CurrentInliningSet = SetFactory.add(CurrentInliningSet, F);
SmallVector<SILValue, 16> CaptureArgs;
SmallVector<SILValue, 32> FullArgs;
for (auto FI = F->begin(), FE = F->end(); FI != FE; ++FI) {
for (auto I = FI->begin(), E = FI->end(); I != E; ++I) {
FullApplySite InnerAI = FullApplySite::isa(&*I);
if (!InnerAI)
continue;
auto *ApplyBlock = InnerAI.getParent();
auto NewInstPair = tryDevirtualizeApply(InnerAI, CHA);
if (auto *NewInst = NewInstPair.first) {
replaceDeadApply(InnerAI, NewInst);
if (auto *II = dyn_cast<SILInstruction>(NewInst))
I = II->getIterator();
else
I = NewInst->getParentBlock()->begin();
auto NewAI = FullApplySite::isa(NewInstPair.second.getInstruction());
if (!NewAI)
continue;
InnerAI = NewAI;
}
SILLocation Loc = InnerAI.getLoc();
SILValue CalleeValue = InnerAI.getCallee();
bool IsThick;
PartialApplyInst *PAI;
SILFunction *CalleeFunction = getCalleeFunction(InnerAI, IsThick,
CaptureArgs, FullArgs,
PAI,
Mode);
if (!CalleeFunction ||
CalleeFunction->isTransparent() == IsNotTransparent)
continue;
if (F->isFragile() &&
!CalleeFunction->hasValidLinkageForFragileRef()) {
if (!CalleeFunction->hasValidLinkageForFragileInline()) {
llvm::errs() << "caller: " << F->getName() << "\n";
llvm::errs() << "callee: " << CalleeFunction->getName() << "\n";
llvm_unreachable("Should never be inlining a resilient function into "
"a fragile function");
}
continue;
}
// Then recursively process it first before trying to inline it.
if (!runOnFunctionRecursively(CalleeFunction, InnerAI, Mode,
FullyInlinedSet, SetFactory,
CurrentInliningSet, CHA)) {
// If we failed due to circular inlining, then emit some notes to
// trace back the failure if we have more information.
// FIXME: possibly it could be worth recovering and attempting other
// inlines within this same recursive call rather than simply
// propagating the failure.
if (AI) {
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::note_while_inlining);
}
return false;
}
// Inline function at I, which also changes I to refer to the first
// instruction inlined in the case that it succeeds. We purposely
// process the inlined body after inlining, because the inlining may
// have exposed new inlining opportunities beyond those present in
// the inlined function when processed independently.
DEBUG(llvm::errs() << "Inlining @" << CalleeFunction->getName()
<< " into @" << InnerAI.getFunction()->getName()
<< "\n");
// If we intend to inline a thick function, then we need to balance the
// reference counts for correctness.
if (IsThick && I != ApplyBlock->begin()) {
// We need to find an appropriate location for our fix up code
// We used to do this after inlining Without any modifications
// This caused us to add a release in a wrong place:
// It would release a value *before* retaining it!
// It is really problematic to do this after inlining -
// Finding a valid insertion point is tricky:
// Inlining might add new basic blocks and/or remove the apply
// We want to add the fix up *just before* where the current apply is!
// Unfortunately, we *can't* add the fix up code here:
// Inlining might fail for any reason -
// If that occurred we'd need to undo our fix up code.
// Instead, we split the current basic block -
// Making sure we have a basic block that starts with our apply.
SILBuilderWithScope B(I);
ApplyBlock = splitBasicBlockAndBranch(B, &*I, nullptr, nullptr);
I = ApplyBlock->begin();
}
// Decrement our iterator (carefully, to avoid going off the front) so it
// is valid after inlining is done. Inlining deletes the apply, and can
// introduce multiple new basic blocks.
if (I != ApplyBlock->begin())
--I;
else
I = ApplyBlock->end();
std::vector<Substitution> ApplySubs(InnerAI.getSubstitutions());
if (PAI) {
auto PAISubs = PAI->getSubstitutions();
ApplySubs.insert(ApplySubs.end(), PAISubs.begin(), PAISubs.end());
}
SILOpenedArchetypesTracker OpenedArchetypesTracker(*F);
F->getModule().registerDeleteNotificationHandler(
&OpenedArchetypesTracker);
// The callee only needs to know about opened archetypes used in
// the substitution list.
OpenedArchetypesTracker.registerUsedOpenedArchetypes(InnerAI.getInstruction());
if (PAI) {
OpenedArchetypesTracker.registerUsedOpenedArchetypes(PAI);
}
SILInliner Inliner(*F, *CalleeFunction,
SILInliner::InlineKind::MandatoryInline,
ApplySubs, OpenedArchetypesTracker);
if (!Inliner.inlineFunction(InnerAI, FullArgs)) {
I = InnerAI.getInstruction()->getIterator();
continue;
}
// Inlining was successful. Remove the apply.
InnerAI.getInstruction()->eraseFromParent();
// Reestablish our iterator if it wrapped.
if (I == ApplyBlock->end())
I = ApplyBlock->begin();
// Update the iterator when instructions are removed.
DeleteInstructionsHandler DeletionHandler(I);
// If the inlined apply was a thick function, then we need to balance the
// reference counts for correctness.
if (IsThick)
fixupReferenceCounts(I, Loc, CalleeValue, CaptureArgs);
// Now that the IR is correct, see if we can remove dead callee
// computations (e.g. dead partial_apply closures).
cleanupCalleeValue(CalleeValue, CaptureArgs, FullArgs);
// Reposition iterators possibly invalidated by mutation.
FI = SILFunction::iterator(ApplyBlock);
E = ApplyBlock->end();
assert(FI == SILFunction::iterator(I->getParent()) &&
"Mismatch between the instruction and basic block");
++NumMandatoryInlines;
}
}
// Keep track of full inlined functions so we don't waste time recursively
// reprocessing them.
FullyInlinedSet.insert(F);
return true;
}
void MaterializeForSetEmitter::emit(SILGenFunction &gen, ManagedValue self,
SILValue resultBuffer,
SILValue callbackBuffer,
ArrayRef<ManagedValue> indices) {
SILLocation loc = Witness;
loc.markAutoGenerated();
// If there's an abstraction difference, we always need to use the
// get/set pattern.
AccessStrategy strategy;
if (WitnessStorage->getType()->is<ReferenceStorageType>() ||
(Conformance && RequirementStorageType != WitnessStorageType)) {
strategy = AccessStrategy::DispatchToAccessor;
} else {
strategy = WitnessStorage->getAccessStrategy(TheAccessSemantics,
AccessKind::ReadWrite);
}
// Handle the indices.
RValue indicesRV;
if (isa<SubscriptDecl>(WitnessStorage)) {
indicesRV = collectIndicesFromParameters(gen, loc, indices);
} else {
assert(indices.empty() && "indices for a non-subscript?");
}
// As above, assume that we don't need to reabstract 'self'.
// Choose the right implementation.
SILValue address;
SILFunction *callbackFn = nullptr;
switch (strategy) {
case AccessStrategy::Storage:
address = emitUsingStorage(gen, loc, self, std::move(indicesRV));
break;
case AccessStrategy::Addressor:
address = emitUsingAddressor(gen, loc, self, std::move(indicesRV),
callbackBuffer, callbackFn);
break;
case AccessStrategy::DirectToAccessor:
case AccessStrategy::DispatchToAccessor:
address = emitUsingGetterSetter(gen, loc, self, std::move(indicesRV),
resultBuffer, callbackBuffer, callbackFn);
break;
}
// Return the address as a Builtin.RawPointer.
SILType rawPointerTy = SILType::getRawPointerType(gen.getASTContext());
address = gen.B.createAddressToPointer(loc, address, rawPointerTy);
SILType resultTupleTy = gen.F.mapTypeIntoContext(
gen.F.getLoweredFunctionType()->getResult().getSILType());
SILType optCallbackTy = resultTupleTy.getTupleElementType(1);
// Form the callback.
SILValue callback;
if (callbackFn) {
// Make a reference to the function.
callback = gen.B.createFunctionRef(loc, callbackFn);
// If it's polymorphic, cast to RawPointer and then back to the
// right monomorphic type. The safety of this cast relies on some
// assumptions about what exactly IRGen can reconstruct from the
// callback's thick type argument.
if (callbackFn->getLoweredFunctionType()->isPolymorphic()) {
callback = gen.B.createThinFunctionToPointer(loc, callback, rawPointerTy);
OptionalTypeKind optKind;
auto callbackTy = optCallbackTy.getAnyOptionalObjectType(SGM.M, optKind);
callback = gen.B.createPointerToThinFunction(loc, callback, callbackTy);
}
callback = gen.B.createOptionalSome(loc, callback, optCallbackTy);
} else {
callback = gen.B.createOptionalNone(loc, optCallbackTy);
}
// Form the result and return.
auto result = gen.B.createTuple(loc, resultTupleTy, { address, callback });
gen.Cleanups.emitCleanupsForReturn(CleanupLocation::get(loc));
gen.B.createReturn(loc, result);
}
/// \brief Make sure that all parameters are passed with a reference count
/// neutral parameter convention except for self.
bool swift::ArraySemanticsCall::isValidSignature() {
assert(SemanticsCall && getKind() != ArrayCallKind::kNone &&
"Need an array semantic call");
FunctionRefInst *FRI = cast<FunctionRefInst>(SemanticsCall->getCallee());
SILFunction *F = FRI->getReferencedFunction();
auto FnTy = F->getLoweredFunctionType();
auto &Mod = F->getModule();
// Check whether we have a valid signature for semantic calls that we hoist.
switch (getKind()) {
// All other calls can be consider valid.
default: break;
case ArrayCallKind::kArrayPropsIsNativeTypeChecked: {
// @guaranteed/@owned Self
if (SemanticsCall->getNumArguments() != 1)
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kCheckIndex: {
// Int, @guaranteed/@owned Self
if (SemanticsCall->getNumArguments() != 2 ||
!SemanticsCall->getArgument(0).getType().isTrivial(Mod))
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kCheckSubscript: {
// Int, Bool, Self
if (SemanticsCall->getNumArguments() != 3 ||
!SemanticsCall->getArgument(0).getType().isTrivial(Mod))
return false;
if (!SemanticsCall->getArgument(1).getType().isTrivial(Mod))
return false;
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Direct_Guaranteed ||
SelfConvention == ParameterConvention::Direct_Owned;
}
case ArrayCallKind::kMakeMutable: {
auto SelfConvention = FnTy->getSelfParameter().getConvention();
return SelfConvention == ParameterConvention::Indirect_Inout;
}
case ArrayCallKind::kArrayUninitialized: {
// Make sure that if we are a _adoptStorage call that our storage is
// uniquely referenced by us.
SILValue Arg0 = SemanticsCall->getArgument(0);
if (Arg0.getType().isExistentialType()) {
auto *AllocBufferAI = dyn_cast<ApplyInst>(Arg0);
if (!AllocBufferAI)
return false;
auto *AllocFn = AllocBufferAI->getCalleeFunction();
if (!AllocFn)
return false;
StringRef AllocFuncName = AllocFn->getName();
if (AllocFuncName != "swift_bufferAllocate" &&
AllocFuncName != "swift_bufferAllocateOnStack")
return false;
if (!hasOneNonDebugUse(*AllocBufferAI))
return false;
}
return true;
}
}
return true;
}
bool Devirtualizer::devirtualizeAppliesInFunction(SILFunction &F,
ClassHierarchyAnalysis *CHA) {
bool Changed = false;
llvm::SmallVector<SILInstruction *, 8> DeadApplies;
llvm::SmallVector<ApplySite, 8> NewApplies;
for (auto &BB : F) {
for (auto It = BB.begin(), End = BB.end(); It != End;) {
auto &I = *It++;
// Skip non-apply instructions.
auto Apply = FullApplySite::isa(&I);
if (!Apply)
continue;
auto NewInstPair = tryDevirtualizeApply(Apply, CHA);
if (!NewInstPair.second)
continue;
Changed = true;
auto *AI = Apply.getInstruction();
if (!isa<TryApplyInst>(AI))
AI->replaceAllUsesWith(NewInstPair.first);
DeadApplies.push_back(AI);
NewApplies.push_back(NewInstPair.second);
}
}
// Remove all the now-dead applies.
while (!DeadApplies.empty()) {
auto *AI = DeadApplies.pop_back_val();
recursivelyDeleteTriviallyDeadInstructions(AI, true);
}
// For each new apply, attempt to link in function bodies if we do
// not already have them, then notify the pass manager of the new
// functions.
//
// We do this after deleting the old applies because otherwise we
// hit verification errors in the linking code due to having
// non-cond_br critical edges.
while (!NewApplies.empty()) {
auto Apply = NewApplies.pop_back_val();
auto *CalleeFn = Apply.getReferencedFunction();
assert(CalleeFn && "Expected devirtualized callee!");
// FIXME: Until we link everything in up front we need to ensure
// that we link after devirtualizing in order to pull in
// everything we reference from the stdlib. After we do that we
// can move the notification code below back into the main loop
// above.
if (!CalleeFn->isDefinition())
F.getModule().linkFunction(CalleeFn, SILModule::LinkingMode::LinkAll);
// We may not have optimized these functions yet, and it could
// be beneficial to rerun some earlier passes on the current
// function now that we've made these direct references visible.
if (CalleeFn->isDefinition() && CalleeFn->shouldOptimize())
notifyPassManagerOfFunction(CalleeFn, nullptr);
}
return Changed;
}
SILFunction *SILModule::findFunction(StringRef Name, SILLinkage Linkage) {
assert((Linkage == SILLinkage::Public ||
Linkage == SILLinkage::PublicExternal) &&
"Only a lookup of public functions is supported currently");
SILFunction *F = nullptr;
// First, check if there is a function with a required name in the
// current module.
SILFunction *CurF = lookUpFunction(Name);
// Nothing to do if the current module has a required function
// with a proper linkage already.
if (CurF && CurF->getLinkage() == Linkage) {
F = CurF;
} else {
assert((!CurF || CurF->getLinkage() != Linkage) &&
"hasFunction should be only called for functions that are not "
"contained in the SILModule yet or do not have a required linkage");
}
if (!F) {
SILLinkerVisitor Visitor(*this, getSILLoader(),
SILModule::LinkingMode::LinkNormal);
if (CurF) {
// Perform this lookup only if a function with a given
// name is present in the current module.
// This is done to reduce the amount of IO from the
// swift module file.
if (!Visitor.hasFunction(Name, Linkage))
return nullptr;
// The function in the current module will be changed.
F = CurF;
}
// If function with a given name wasn't seen anywhere yet
// or if it is known to exist, perform a lookup.
if (!F) {
// Try to load the function from other modules.
F = Visitor.lookupFunction(Name, Linkage);
// Bail if nothing was found and we are not sure if
// this function exists elsewhere.
if (!F)
return nullptr;
assert(F && "SILFunction should be present in one of the modules");
assert(F->getLinkage() == Linkage && "SILFunction has a wrong linkage");
}
}
// If a function exists already and it is a non-optimizing
// compilation, simply convert it into an external declaration,
// so that a compiled version from the shared library is used.
if (F->isDefinition() &&
F->getModule().getOptions().Optimization <
SILOptions::SILOptMode::Optimize) {
F->convertToDeclaration();
}
if (F->isExternalDeclaration())
F->setSerialized(IsSerialized_t::IsNotSerialized);
F->setLinkage(Linkage);
return F;
}
void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() {
SILFunction *NewF = TransformDescriptor.OptimizedFunction.get();
SILBasicBlock *BB = &*NewF->begin();
SILBuilder Builder(BB->begin());
Builder.setCurrentDebugScope(BB->getParent()->getDebugScope());
unsigned TotalArgIndex = 0;
for (ArgumentDescriptor &AD : TransformDescriptor.ArgumentDescList) {
// If this argument descriptor was dead and we removed it, just skip it. Do
// not increment the argument index.
if (AD.WasErased) {
continue;
}
// Simply continue if do not explode.
if (!AD.Explode) {
TransformDescriptor.AIM[TotalArgIndex] = AD.Index;
++TotalArgIndex;
continue;
}
assert(!AD.IsEntirelyDead &&
"Should never see completely dead values here");
// OK, we need to explode this argument.
unsigned ArgOffset = ++TotalArgIndex;
unsigned OldArgIndex = ArgOffset - 1;
llvm::SmallVector<SILValue, 8> LeafValues;
// We do this in the same order as leaf types since ProjTree expects that
// the order of leaf values matches the order of leaf types.
llvm::SmallVector<const ProjectionTreeNode *, 8> LeafNodes;
AD.ProjTree.getLiveLeafNodes(LeafNodes);
for (auto *Node : LeafNodes) {
auto OwnershipKind = *AD.getTransformedOwnershipKind(Node->getType());
LeafValues.push_back(
BB->insertFunctionArgument(ArgOffset, Node->getType(), OwnershipKind,
BB->getArgument(OldArgIndex)->getDecl()));
TransformDescriptor.AIM[TotalArgIndex - 1] = AD.Index;
++ArgOffset;
++TotalArgIndex;
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
AD.ProjTree.replaceValueUsesWithLeafUses(
Builder, BB->getParent()->getLocation(), LeafValues);
// We ignored debugvalue uses when we constructed the new arguments, in
// order to preserve as much information as possible, we construct a new
// value for OrigArg from the leaf values and use that in place of the
// OrigArg.
SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue(
Builder, BB->getParent()->getLocation(), LeafValues);
// Replace all uses of the original arg with the new value.
SILArgument *OrigArg = BB->getArgument(OldArgIndex);
OrigArg->replaceAllUsesWith(NewOrigArgValue);
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseArgument(OldArgIndex);
--TotalArgIndex;
}
}
// Returns the callee of an apply_inst if it is basically inlineable.
SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) {
SILFunction *Callee = AI.getReferencedFunction();
if (!Callee) {
return nullptr;
}
// Don't inline functions that are marked with the @_semantics or @effects
// attribute if the inliner is asked not to inline them.
if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
return nullptr;
}
// The "availability" semantics attribute is treated like global-init.
if (Callee->hasSemanticsAttrs() &&
WhatToInline != InlineSelection::Everything &&
Callee->hasSemanticsAttrThatStartsWith("availability")) {
return nullptr;
}
} else if (Callee->isGlobalInit()) {
if (WhatToInline != InlineSelection::Everything) {
return nullptr;
}
}
// We can't inline external declarations.
if (Callee->empty() || Callee->isExternalDeclaration()) {
return nullptr;
}
// Explicitly disabled inlining.
if (Callee->getInlineStrategy() == NoInline) {
return nullptr;
}
if (!Callee->shouldOptimize()) {
return nullptr;
}
// We don't support this yet.
if (AI.hasSubstitutions()) {
return nullptr;
}
// We don't support inlining a function that binds dynamic self because we
// have no mechanism to preserve the original function's local self metadata.
if (computeMayBindDynamicSelf(Callee)) {
return nullptr;
}
SILFunction *Caller = AI.getFunction();
// Detect self-recursive calls.
if (Caller == Callee) {
return nullptr;
}
// A non-fragile function may not be inlined into a fragile function.
if (Caller->isFragile() && !Callee->isFragile()) {
return nullptr;
}
// Inlining self-recursive functions into other functions can result
// in excessive code duplication since we run the inliner multiple
// times in our pipeline
if (calleeIsSelfRecursive(Callee)) {
return nullptr;
}
return Callee;
}
/// Remove retain/release pairs around builtin "unsafeGuaranteed" instruction
/// sequences.
static bool removeGuaranteedRetainReleasePairs(SILFunction &F,
RCIdentityFunctionInfo &RCIA) {
DEBUG(llvm::dbgs() << "Running on function " << F.getName() << "\n");
bool Changed = false;
for (auto &BB : F) {
auto It = BB.begin(), End = BB.end();
llvm::DenseMap<SILValue, SILInstruction *> LastRetain;
while (It != End) {
auto *CurInst = &*It;
++It;
// Memorize the last retain.
if (isa<StrongRetainInst>(CurInst) || isa<RetainValueInst>(CurInst)) {
LastRetain[RCIA.getRCIdentityRoot(CurInst->getOperand(0))] = CurInst;
continue;
}
// Look for a builtin "unsafeGuaranteed" instruction.
auto *UnsafeGuaranteedI = dyn_cast<BuiltinInst>(CurInst);
if (!UnsafeGuaranteedI || !UnsafeGuaranteedI->getBuiltinKind() ||
*UnsafeGuaranteedI->getBuiltinKind() !=
BuiltinValueKind::UnsafeGuaranteed)
continue;
auto Opd = UnsafeGuaranteedI->getOperand(0);
auto RCIdOpd = RCIA.getRCIdentityRoot(UnsafeGuaranteedI->getOperand(0));
if (!LastRetain.count(RCIdOpd)) {
DEBUG(llvm::dbgs() << "LastRetain failed\n");
continue;
}
// This code is very conservative. Check that there is a matching retain
// before the unsafeGuaranteed builtin with only retains inbetween.
auto *LastRetainInst = LastRetain[RCIdOpd];
auto NextInstIter = std::next(SILBasicBlock::iterator(LastRetainInst));
while (NextInstIter != BB.end() && &*NextInstIter != CurInst &&
(isa<RetainValueInst>(*NextInstIter) ||
isa<StrongRetainInst>(*NextInstIter) ||
!NextInstIter->mayHaveSideEffects() ||
isa<DebugValueInst>(*NextInstIter) ||
isa<DebugValueAddrInst>(*NextInstIter)))
++NextInstIter;
if (&*NextInstIter != CurInst) {
DEBUG(llvm::dbgs() << "Last retain right before match failed\n");
continue;
}
DEBUG(llvm::dbgs() << "Saw " << *UnsafeGuaranteedI);
DEBUG(llvm::dbgs() << " with operand " << *Opd);
// Match the reference and token result.
// %4 = builtin "unsafeGuaranteed"<Foo>(%0 : $Foo)
// %5 = tuple_extract %4 : $(Foo, Builtin.Int8), 0
// %6 = tuple_extract %4 : $(Foo, Builtin.Int8), 1
SILInstruction *UnsafeGuaranteedValue;
SILInstruction *UnsafeGuaranteedToken;
std::tie(UnsafeGuaranteedValue, UnsafeGuaranteedToken) =
getSingleUnsafeGuaranteedValueResult(UnsafeGuaranteedI);
if (!UnsafeGuaranteedValue) {
DEBUG(llvm::dbgs() << " no single unsafeGuaranteed value use\n");
continue;
}
// Look for a builtin "unsafeGuaranteedEnd" instruction that uses the
// token.
// builtin "unsafeGuaranteedEnd"(%6 : $Builtin.Int8) : $()
BuiltinInst *UnsafeGuaranteedEndI = nullptr;
for (auto *Operand : getNonDebugUses(UnsafeGuaranteedToken)) {
if (UnsafeGuaranteedEndI) {
DEBUG(llvm::dbgs() << " multiple unsafeGuaranteedEnd users\n");
UnsafeGuaranteedEndI = nullptr;
break;
}
auto *BI = dyn_cast<BuiltinInst>(Operand->getUser());
if (!BI || !BI->getBuiltinKind() ||
*BI->getBuiltinKind() != BuiltinValueKind::UnsafeGuaranteedEnd) {
DEBUG(llvm::dbgs() << " wrong unsafeGuaranteed token user "
<< *Operand->getUser());
break;
}
UnsafeGuaranteedEndI = BI;
}
if (!UnsafeGuaranteedEndI) {
DEBUG(llvm::dbgs() << " no single unsafeGuaranteedEnd use found\n");
continue;
}
if (SILBasicBlock::iterator(UnsafeGuaranteedEndI) ==
UnsafeGuaranteedEndI->getParent()->end())
continue;
// Find the release to match with the unsafeGuaranteedValue.
auto &UnsafeGuaranteedEndBB = *UnsafeGuaranteedEndI->getParent();
auto LastRelease = findReleaseToMatchUnsafeGuaranteedValue(
UnsafeGuaranteedEndI, UnsafeGuaranteedI, UnsafeGuaranteedValue,
UnsafeGuaranteedEndBB, RCIA);
if (!LastRelease) {
DEBUG(llvm::dbgs() << " no release before/after unsafeGuaranteedEnd found\n");
continue;
}
SILInstruction *LastRelease = &*LastReleaseIt;
// Restart iteration before the earliest instruction we remove.
bool RestartAtBeginningOfBlock = false;
auto LastRetainIt = SILBasicBlock::iterator(LastRetainInst);
if (LastRetainIt != BB.begin()) {
It = std::prev(LastRetainIt);
} else RestartAtBeginningOfBlock = true;
// Okay we found a post dominating release. Let's remove the
// retain/unsafeGuaranteed/release combo.
//
LastRetainInst->eraseFromParent();
LastRelease->eraseFromParent();
UnsafeGuaranteedEndI->eraseFromParent();
deleteAllDebugUses(UnsafeGuaranteedValue);
deleteAllDebugUses(UnsafeGuaranteedToken);
deleteAllDebugUses(UnsafeGuaranteedI);
UnsafeGuaranteedValue->replaceAllUsesWith(Opd);
UnsafeGuaranteedValue->eraseFromParent();
UnsafeGuaranteedToken->eraseFromParent();
UnsafeGuaranteedI->replaceAllUsesWith(Opd);
UnsafeGuaranteedI->eraseFromParent();
if (RestartAtBeginningOfBlock)
++It = BB.begin();
Changed = true;
}
}
return Changed;
}
/// Bridge argument types and adjust retain count conventions for an ObjC thunk.
static SILFunctionType *emitObjCThunkArguments(SILGenFunction &gen,
SILLocation loc,
SILDeclRef thunk,
SmallVectorImpl<SILValue> &args,
SILValue &foreignErrorSlot,
Optional<ForeignErrorConvention> &foreignError) {
SILDeclRef native = thunk.asForeign(false);
auto mod = gen.SGM.M.getSwiftModule();
auto subs = gen.F.getForwardingSubstitutions();
auto objcInfo = gen.SGM.Types.getConstantInfo(thunk);
auto objcFnTy = objcInfo.SILFnType->substGenericArgs(gen.SGM.M, mod, subs);
auto swiftInfo = gen.SGM.Types.getConstantInfo(native);
auto swiftFnTy = swiftInfo.SILFnType->substGenericArgs(gen.SGM.M, mod, subs);
// We must have the same context archetypes as the unthunked function.
assert(objcInfo.ContextGenericParams == swiftInfo.ContextGenericParams);
SmallVector<ManagedValue, 8> bridgedArgs;
bridgedArgs.reserve(objcFnTy->getParameters().size());
SILFunction *orig = gen.SGM.getFunction(native, NotForDefinition);
// Find the foreign error convention if we have one.
if (orig->getLoweredFunctionType()->hasErrorResult()) {
auto func = cast<AbstractFunctionDecl>(thunk.getDecl());
foreignError = func->getForeignErrorConvention();
assert(foreignError && "couldn't find foreign error convention!");
}
// Emit the indirect result arguments, if any.
// FIXME: we're just assuming that these match up exactly?
for (auto indirectResult : objcFnTy->getIndirectResults()) {
SILType argTy = gen.F.mapTypeIntoContext(indirectResult.getSILType());
auto arg = new (gen.F.getModule()) SILArgument(gen.F.begin(), argTy);
args.push_back(arg);
}
// Emit the other arguments, taking ownership of arguments if necessary.
auto inputs = objcFnTy->getParameters();
auto nativeInputs = swiftFnTy->getParameters();
assert(inputs.size() ==
nativeInputs.size() + unsigned(foreignError.hasValue()));
for (unsigned i = 0, e = inputs.size(); i < e; ++i) {
SILType argTy = gen.F.mapTypeIntoContext(inputs[i].getSILType());
SILValue arg = new(gen.F.getModule()) SILArgument(gen.F.begin(), argTy);
// If this parameter is the foreign error slot, pull it out.
// It does not correspond to a native argument.
if (foreignError && i == foreignError->getErrorParameterIndex()) {
foreignErrorSlot = arg;
continue;
}
// If this parameter is deallocating, emit an unmanaged rvalue and
// continue. The object has the deallocating bit set so retain, release is
// irrelevant.
if (inputs[i].isDeallocating()) {
bridgedArgs.push_back(ManagedValue::forUnmanaged(arg));
continue;
}
// If the argument is a block, copy it.
if (argTy.isBlockPointerCompatible()) {
auto copy = gen.B.createCopyBlock(loc, arg);
// If the argument is consumed, we're still responsible for releasing the
// original.
if (inputs[i].isConsumed())
gen.emitManagedRValueWithCleanup(arg);
arg = copy;
}
// Convert the argument to +1 if necessary.
else if (!inputs[i].isConsumed()) {
arg = emitObjCUnconsumedArgument(gen, loc, arg);
}
auto managedArg = gen.emitManagedRValueWithCleanup(arg);
bridgedArgs.push_back(managedArg);
}
assert(bridgedArgs.size() + unsigned(foreignError.hasValue())
== objcFnTy->getParameters().size() &&
"objc inputs don't match number of arguments?!");
assert(bridgedArgs.size() == swiftFnTy->getNumSILArguments() &&
"swift inputs don't match number of arguments?!");
assert((foreignErrorSlot || !foreignError) &&
"didn't find foreign error slot");
// Bridge the input types.
Scope scope(gen.Cleanups, CleanupLocation::get(loc));
assert(bridgedArgs.size() == nativeInputs.size());
for (unsigned i = 0, size = bridgedArgs.size(); i < size; ++i) {
SILType argTy = gen.F.mapTypeIntoContext(
swiftFnTy->getParameters()[i].getSILType());
ManagedValue native =
gen.emitBridgedToNativeValue(loc,
bridgedArgs[i],
SILFunctionTypeRepresentation::ObjCMethod,
argTy.getSwiftType());
SILValue argValue;
if (nativeInputs[i].isConsumed())
argValue = native.forward(gen);
else
argValue = native.getValue();
args.push_back(argValue);
}
return objcFnTy;
}
// Returns the callee of an apply_inst if it is basically inlineable.
SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) {
SILFunction *Callee = AI.getReferencedFunction();
if (!Callee) {
return nullptr;
}
// Don't inline functions that are marked with the @_semantics or @effects
// attribute if the inliner is asked not to inline them.
if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
return nullptr;
}
// The "availability" semantics attribute is treated like global-init.
if (Callee->hasSemanticsAttrs() &&
WhatToInline != InlineSelection::Everything &&
Callee->hasSemanticsAttrThatStartsWith("availability")) {
return nullptr;
}
} else if (Callee->isGlobalInit()) {
if (WhatToInline != InlineSelection::Everything) {
return nullptr;
}
}
// We can't inline external declarations.
if (Callee->empty() || Callee->isExternalDeclaration()) {
return nullptr;
}
// Explicitly disabled inlining.
if (Callee->getInlineStrategy() == NoInline) {
return nullptr;
}
if (!Callee->shouldOptimize()) {
return nullptr;
}
// We don't support this yet.
if (AI.hasSubstitutions())
return nullptr;
SILFunction *Caller = AI.getFunction();
// We don't support inlining a function that binds dynamic self because we
// have no mechanism to preserve the original function's local self metadata.
if (mayBindDynamicSelf(Callee)) {
// Check if passed Self is the same as the Self of the caller.
// In this case, it is safe to inline because both functions
// use the same Self.
if (AI.hasSelfArgument() && Caller->hasSelfParam()) {
auto CalleeSelf = stripCasts(AI.getSelfArgument());
auto CallerSelf = Caller->getSelfArgument();
if (CalleeSelf != SILValue(CallerSelf))
return nullptr;
} else
return nullptr;
}
// Detect self-recursive calls.
if (Caller == Callee) {
return nullptr;
}
// A non-fragile function may not be inlined into a fragile function.
if (Caller->isFragile() &&
!Callee->hasValidLinkageForFragileInline()) {
if (!Callee->hasValidLinkageForFragileRef()) {
llvm::errs() << "caller: " << Caller->getName() << "\n";
llvm::errs() << "callee: " << Callee->getName() << "\n";
llvm_unreachable("Should never be inlining a resilient function into "
"a fragile function");
}
return nullptr;
}
// Inlining self-recursive functions into other functions can result
// in excessive code duplication since we run the inliner multiple
// times in our pipeline
if (calleeIsSelfRecursive(Callee)) {
return nullptr;
}
return Callee;
}
bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI,
Weight CallerWeight,
ConstantTracker &callerTracker,
int &NumCallerBlocks,
bool IsGeneric) {
SILFunction *Callee = AI.getReferencedFunction();
SILLoopInfo *LI = LA->get(Callee);
ShortestPathAnalysis *SPA = getSPA(Callee, LI);
assert(SPA->isValid());
ConstantTracker constTracker(Callee, &callerTracker, AI);
DominanceInfo *DT = DA->get(Callee);
SILBasicBlock *CalleeEntry = &Callee->front();
DominanceOrder domOrder(CalleeEntry, DT, Callee->size());
// Calculate the inlining cost of the callee.
int CalleeCost = 0;
int Benefit = 0;
// Start with a base benefit.
int BaseBenefit = RemovedCallBenefit;
const SILOptions &Opts = Callee->getModule().getOptions();
// For some reason -Ounchecked can accept a higher base benefit without
// increasing the code size too much.
if (Opts.Optimization == SILOptions::SILOptMode::OptimizeUnchecked)
BaseBenefit *= 2;
CallerWeight.updateBenefit(Benefit, BaseBenefit);
// Go through all blocks of the function, accumulate the cost and find
// benefits.
while (SILBasicBlock *block = domOrder.getNext()) {
constTracker.beginBlock();
Weight BlockW = SPA->getWeight(block, CallerWeight);
for (SILInstruction &I : *block) {
constTracker.trackInst(&I);
CalleeCost += (int)instructionInlineCost(I);
if (FullApplySite AI = FullApplySite::isa(&I)) {
// Check if the callee is passed as an argument. If so, increase the
// threshold, because inlining will (probably) eliminate the closure.
SILInstruction *def = constTracker.getDefInCaller(AI.getCallee());
if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def)))
BlockW.updateBenefit(Benefit, RemovedClosureBenefit);
} else if (auto *LI = dyn_cast<LoadInst>(&I)) {
// Check if it's a load from a stack location in the caller. Such a load
// might be optimized away if inlined.
if (constTracker.isStackAddrInCaller(LI->getOperand()))
BlockW.updateBenefit(Benefit, RemovedLoadBenefit);
} else if (auto *SI = dyn_cast<StoreInst>(&I)) {
// Check if it's a store to a stack location in the caller. Such a load
// might be optimized away if inlined.
if (constTracker.isStackAddrInCaller(SI->getDest()))
BlockW.updateBenefit(Benefit, RemovedStoreBenefit);
} else if (isa<StrongReleaseInst>(&I) || isa<ReleaseValueInst>(&I)) {
SILValue Op = stripCasts(I.getOperand(0));
if (SILArgument *Arg = dyn_cast<SILArgument>(Op)) {
if (Arg->isFunctionArg() && Arg->getArgumentConvention() ==
SILArgumentConvention::Direct_Guaranteed) {
BlockW.updateBenefit(Benefit, RefCountBenefit);
}
}
} else if (auto *BI = dyn_cast<BuiltinInst>(&I)) {
if (BI->getBuiltinInfo().ID == BuiltinValueKind::OnFastPath)
BlockW.updateBenefit(Benefit, FastPathBuiltinBenefit);
}
}
// Don't count costs in blocks which are dead after inlining.
SILBasicBlock *takenBlock = constTracker.getTakenBlock(block->getTerminator());
if (takenBlock) {
BlockW.updateBenefit(Benefit, RemovedTerminatorBenefit);
domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) {
return child->getSinglePredecessor() != block || child == takenBlock;
});
} else {
domOrder.pushChildren(block);
}
}
if (AI.getFunction()->isThunk()) {
// Only inline trivial functions into thunks (which will not increase the
// code size).
if (CalleeCost > TrivialFunctionThreshold)
return false;
DEBUG(
dumpCaller(AI.getFunction());
llvm::dbgs() << " decision {" << CalleeCost << " into thunk} " <<
Callee->getName() << '\n';
);
return true;
}
/// Specialize a partial_apply by promoting the parameters indicated by
/// indices. We expect these parameters to be replaced by stack address
/// references.
static PartialApplyInst *
specializePartialApply(PartialApplyInst *PartialApply,
ParamIndexList &PromotedParamIndices,
bool &CFGChanged) {
auto *FRI = cast<FunctionRefInst>(PartialApply->getCallee());
assert(FRI && "Expected a direct partial_apply!");
auto *F = FRI->getReferencedFunction();
assert(F && "Expected a referenced function!");
IsFragile_t Fragile = IsNotFragile;
if (PartialApply->getFunction()->isFragile() && F->isFragile())
Fragile = IsFragile;
std::string ClonedName = getClonedName(F, Fragile, PromotedParamIndices);
auto &M = PartialApply->getModule();
SILFunction *ClonedFn;
if (auto *PrevFn = M.lookUpFunction(ClonedName)) {
assert(PrevFn->isFragile() == Fragile);
ClonedFn = PrevFn;
} else {
// Clone the function the existing partial_apply references.
PromotedParamCloner Cloner(F, Fragile, PromotedParamIndices, ClonedName);
Cloner.populateCloned();
ClonedFn = Cloner.getCloned();
}
// Now create the new partial_apply using the cloned function.
llvm::SmallVector<SILValue, 16> Args;
ValueLifetimeAnalysis::Frontier PAFrontier;
// Promote the arguments that need promotion.
for (auto &O : PartialApply->getArgumentOperands()) {
auto ParamIndex = getParameterIndexForOperand(&O);
if (!count(PromotedParamIndices, ParamIndex)) {
Args.push_back(O.get());
continue;
}
// If this argument is promoted, it is a box that we're
// turning into an address because we've proven we can
// keep this value on the stack. The partial_apply had ownership
// of this box so we must now release it explicitly when the
// partial_apply is released.
auto box = cast<AllocBoxInst>(O.get());
// If the box address has a MUI, route accesses through it so DI still
// works.
SILInstruction *promoted = nullptr;
int numAddrUses = 0;
for (Operand *BoxUse : box->getUses()) {
if (auto *PBI = dyn_cast<ProjectBoxInst>(BoxUse->getUser())) {
for (auto PBIUse : PBI->getUses()) {
numAddrUses++;
if (auto MUI = dyn_cast<MarkUninitializedInst>(PBIUse->getUser()))
promoted = MUI;
}
}
}
assert((!promoted || numAddrUses == 1) &&
"box value used by mark_uninitialized but not exclusively!");
// We only reuse an existing project_box if it directly follows the
// alloc_box. This makes sure that the project_box dominates the
// partial_apply.
if (!promoted)
promoted = getOrCreateProjectBox(box);
Args.push_back(promoted);
if (PAFrontier.empty()) {
ValueLifetimeAnalysis VLA(PartialApply);
CFGChanged |= !VLA.computeFrontier(PAFrontier,
ValueLifetimeAnalysis::AllowToModifyCFG);
assert(!PAFrontier.empty() && "partial_apply must have at least one use "
"to release the returned function");
}
// Insert releases after each point where the partial_apply becomes dead.
for (SILInstruction *FrontierInst : PAFrontier) {
SILBuilderWithScope Builder(FrontierInst);
Builder.emitStrongReleaseAndFold(PartialApply->getLoc(), O.get());
}
}
SILBuilderWithScope Builder(PartialApply);
// Build the function_ref and partial_apply.
SILValue FunctionRef = Builder.createFunctionRef(PartialApply->getLoc(),
ClonedFn);
CanSILFunctionType CanFnTy = ClonedFn->getLoweredFunctionType();
auto const &Subs = PartialApply->getSubstitutions();
CanSILFunctionType SubstCalleeTy = CanFnTy->substGenericArgs(M,
M.getSwiftModule(),
Subs);
return Builder.createPartialApply(PartialApply->getLoc(), FunctionRef,
SILType::getPrimitiveObjectType(SubstCalleeTy),
PartialApply->getSubstitutions(), Args,
PartialApply->getType());
}
/// Analyze the destructor for the class of ARI to see if any instructions in it
/// could have side effects on the program outside the destructor. If it does
/// not, then we can eliminate the destructor.
static bool doesDestructorHaveSideEffects(AllocRefInst *ARI) {
SILFunction *Fn = getDestructor(ARI);
// If we can't find a constructor then assume it has side effects.
if (!Fn)
return true;
// A destructor only has one argument, self.
assert(Fn->begin()->getNumArguments() == 1 &&
"Destructor should have only one argument, self.");
SILArgument *Self = Fn->begin()->getArgument(0);
LLVM_DEBUG(llvm::dbgs() << " Analyzing destructor.\n");
// For each BB in the destructor...
for (auto &BB : *Fn)
// For each instruction I in BB...
for (auto &I : BB) {
LLVM_DEBUG(llvm::dbgs() << " Visiting: " << I);
// If I has no side effects, we can ignore it.
if (!I.mayHaveSideEffects()) {
LLVM_DEBUG(llvm::dbgs() << " SAFE! Instruction has no side "
"effects.\n");
continue;
}
// RefCounting operations on Self are ok since we are already in the
// destructor. RefCountingOperations on other instructions could have side
// effects though.
if (auto *RefInst = dyn_cast<RefCountingInst>(&I)) {
if (stripCasts(RefInst->getOperand(0)) == Self) {
// For now all ref counting insts have 1 operand. Put in an assert
// just in case.
assert(RefInst->getNumOperands() == 1 &&
"Make sure RefInst only has one argument.");
LLVM_DEBUG(llvm::dbgs() << " SAFE! Ref count operation on "
"Self.\n");
continue;
} else {
LLVM_DEBUG(llvm::dbgs() << " UNSAFE! Ref count operation "
"not on self.\n");
return true;
}
}
// dealloc_stack can be ignored.
if (isa<DeallocStackInst>(I)) {
LLVM_DEBUG(llvm::dbgs() << " SAFE! dealloc_stack can be "
"ignored.\n");
continue;
}
// dealloc_ref on self can be ignored, but dealloc_ref on anything else
// cannot be eliminated.
if (auto *DeallocRef = dyn_cast<DeallocRefInst>(&I)) {
if (stripCasts(DeallocRef->getOperand()) == Self) {
LLVM_DEBUG(llvm::dbgs() <<" SAFE! dealloc_ref on self.\n");
continue;
} else {
LLVM_DEBUG(llvm::dbgs() << " UNSAFE! dealloc_ref on value "
"besides self.\n");
return true;
}
}
// Storing into the object can be ignored.
if (auto *SI = dyn_cast<StoreInst>(&I))
if (stripAddressProjections(SI->getDest()) == Self) {
LLVM_DEBUG(llvm::dbgs() << " SAFE! Instruction is a store "
"into self.\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " UNSAFE! Unknown instruction.\n");
// Otherwise, we can't remove the deallocation completely.
return true;
}
// We didn't find any side effects.
return false;
}
/// Specialize a partial_apply by promoting the parameters indicated by
/// indices. We expect these parameters to be replaced by stack address
/// references.
static PartialApplyInst *
specializePartialApply(PartialApplyInst *PartialApply,
ParamIndexList &PromotedParamIndices) {
auto *FRI = cast<FunctionRefInst>(PartialApply->getCallee());
assert(FRI && "Expected a direct partial_apply!");
auto *F = FRI->getReferencedFunction();
assert(F && "Expected a referenced function!");
std::string ClonedName = getClonedName(F, PromotedParamIndices);
auto &M = PartialApply->getModule();
SILFunction *ClonedFn;
if (auto *PrevFn = M.lookUpFunction(ClonedName)) {
ClonedFn = PrevFn;
} else {
// Clone the function the existing partial_apply references.
PromotedParamCloner Cloner(F, PromotedParamIndices, ClonedName);
Cloner.populateCloned();
ClonedFn = Cloner.getCloned();
}
// Now create the new partial_apply using the cloned function.
llvm::SmallVector<SILValue, 16> Args;
LifetimeTracker Lifetime(PartialApply);
// Promote the arguments that need promotion.
for (auto &O : PartialApply->getArgumentOperands()) {
auto ParamIndex = getParameterIndexForOperand(&O);
if (!std::count(PromotedParamIndices.begin(), PromotedParamIndices.end(),
ParamIndex)) {
Args.push_back(O.get());
continue;
}
auto Endpoints = Lifetime.getEndpoints();
// If this argument is promoted, it is a box that we're
// turning into an address because we've proven we can
// keep this value on the stack. The partial_apply had ownership
// of this box so we must now release it explicitly when the
// partial_apply is released.
auto box = cast<AllocBoxInst>(O.get());
// If the box address has a MUI, route accesses through it so DI still
// works.
SILInstruction *promoted = nullptr;
int numAddrUses = 0;
for (Operand *BoxUse : box->getUses()) {
if (auto *PBI = dyn_cast<ProjectBoxInst>(BoxUse->getUser())) {
for (auto PBIUse : PBI->getUses()) {
numAddrUses++;
if (auto MUI = dyn_cast<MarkUninitializedInst>(PBIUse->getUser()))
promoted = MUI;
}
}
}
assert((!promoted || numAddrUses == 1) &&
"box value used by mark_uninitialized but not exclusively!");
// We only reuse an existing project_box if it directly follows the
// alloc_box. This makes sure that the project_box dominates the
// partial_apply.
if (!promoted)
promoted = getOrCreateProjectBox(box);
Args.push_back(promoted);
// If the partial_apply is dead, insert a release after it.
if (Endpoints.begin() == Endpoints.end()) {
emitStrongReleaseAfter(O.get(), PartialApply);
continue;
}
// Otherwise insert releases after each point where the
// partial_apply becomes dead.
for (auto *User : Endpoints) {
assert((isa<StrongReleaseInst>(User) || isa<ApplyInst>(User)) &&
"Unexpected end of lifetime for partial_apply!");
emitStrongReleaseAfter(O.get(), User);
}
}
SILBuilderWithScope Builder(PartialApply);
// Build the function_ref and partial_apply.
SILValue FunctionRef = Builder.createFunctionRef(PartialApply->getLoc(),
ClonedFn);
CanSILFunctionType CanFnTy = ClonedFn->getLoweredFunctionType();
auto const &Subs = PartialApply->getSubstitutions();
CanSILFunctionType SubstCalleeTy = CanFnTy->substGenericArgs(M,
M.getSwiftModule(),
Subs);
return Builder.createPartialApply(PartialApply->getLoc(), FunctionRef,
SILType::getPrimitiveObjectType(SubstCalleeTy),
PartialApply->getSubstitutions(), Args,
PartialApply->getType());
}
void ClosureSpecializer::gatherCallSites(
SILFunction *Caller,
llvm::SmallVectorImpl<ClosureInfo*> &ClosureCandidates,
llvm::DenseSet<FullApplySite> &MultipleClosureAI) {
// A set of apply inst that we have associated with a closure. We use this to
// make sure that we do not handle call sites with multiple closure arguments.
llvm::DenseSet<FullApplySite> VisitedAI;
// For each basic block BB in Caller...
for (auto &BB : *Caller) {
// For each instruction II in BB...
for (auto &II : BB) {
// If II is not a closure that we support specializing, skip it...
if (!isSupportedClosure(&II))
continue;
ClosureInfo *CInfo = nullptr;
// Go through all uses of our closure.
for (auto *Use : II.getUses()) {
// If this use is not an apply inst or an apply inst with
// substitutions, there is nothing interesting for us to do, so
// continue...
auto AI = FullApplySite::isa(Use->getUser());
if (!AI || AI.hasSubstitutions())
continue;
// Check if we have already associated this apply inst with a closure to
// be specialized. We do not handle applies that take in multiple
// closures at this time.
if (!VisitedAI.insert(AI).second) {
MultipleClosureAI.insert(AI);
continue;
}
// If AI does not have a function_ref definition as its callee, we can
// not do anything here... so continue...
SILFunction *ApplyCallee = AI.getReferencedFunction();
if (!ApplyCallee || ApplyCallee->isExternalDeclaration())
continue;
// Ok, we know that we can perform the optimization but not whether or
// not the optimization is profitable. Find the index of the argument
// corresponding to our partial apply.
Optional<unsigned> ClosureIndex;
for (unsigned i = 0, e = AI.getNumArguments(); i != e; ++i) {
if (AI.getArgument(i) != SILValue(&II))
continue;
ClosureIndex = i;
DEBUG(llvm::dbgs() << " Found callsite with closure argument at "
<< i << ": " << *AI.getInstruction());
break;
}
// If we did not find an index, there is nothing further to do,
// continue.
if (!ClosureIndex.hasValue())
continue;
// Make sure that the Closure is invoked in the Apply's callee. We only
// want to perform closure specialization if we know that we will be
// able to change a partial_apply into an apply.
//
// TODO: Maybe just call the function directly instead of moving the
// partial apply?
SILValue Arg = ApplyCallee->getArgument(ClosureIndex.getValue());
if (std::none_of(Arg->use_begin(), Arg->use_end(),
[&Arg](Operand *Op) -> bool {
auto UserAI = FullApplySite::isa(Op->getUser());
return UserAI && UserAI.getCallee() == Arg;
})) {
continue;
}
auto NumIndirectResults =
AI.getSubstCalleeType()->getNumIndirectResults();
assert(ClosureIndex.getValue() >= NumIndirectResults);
auto ClosureParamIndex = ClosureIndex.getValue() - NumIndirectResults;
auto ParamInfo = AI.getSubstCalleeType()->getParameters();
SILParameterInfo ClosureParamInfo = ParamInfo[ClosureParamIndex];
// Get all non-failure exit BBs in the Apply Callee if our partial apply
// is guaranteed. If we do not understand one of the exit BBs, bail.
//
// We need this to make sure that we insert a release in the appropriate
// locations to balance the +1 from the creation of the partial apply.
llvm::TinyPtrVector<SILBasicBlock *> NonFailureExitBBs;
if (ClosureParamInfo.isGuaranteed() &&
!findAllNonFailureExitBBs(ApplyCallee, NonFailureExitBBs)) {
continue;
}
// Compute the final release points of the closure. We will insert
// release of the captured arguments here.
if (!CInfo) {
CInfo = new ClosureInfo(&II);
ValueLifetimeAnalysis VLA(CInfo->Closure);
VLA.computeFrontier(CInfo->LifetimeFrontier,
ValueLifetimeAnalysis::AllowToModifyCFG);
}
// Now we know that CSDesc is profitable to specialize. Add it to our
// call site list.
CInfo->CallSites.push_back(
CallSiteDescriptor(CInfo, AI, ClosureIndex.getValue(),
ClosureParamInfo, std::move(NonFailureExitBBs)));
}
if (CInfo)
ClosureCandidates.push_back(CInfo);
}
}
}
/// \brief Attempt to inline all calls smaller than our threshold.
/// returns True if a function was inlined.
bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller,
DominanceAnalysis *DA,
SILLoopAnalysis *LA,
llvm::SmallVectorImpl<FullApplySite> &NewApplies) {
// Don't optimize functions that are marked with the opt.never attribute.
if (!Caller->shouldOptimize())
return false;
// Construct a log of all of the names of the functions that we've inlined
// in the current iteration.
SmallVector<StringRef, 16> InlinedFunctionNames;
StringRef CallerName = Caller->getName();
DEBUG(llvm::dbgs() << "Visiting Function: " << CallerName << "\n");
assert(NewApplies.empty() && "Expected empty vector to store results in!");
// First step: collect all the functions we want to inline. We
// don't change anything yet so that the dominator information
// remains valid.
SmallVector<FullApplySite, 8> AppliesToInline;
collectAppliesToInline(Caller, AppliesToInline, DA, LA);
if (AppliesToInline.empty())
return false;
// Second step: do the actual inlining.
for (auto AI : AppliesToInline) {
SILFunction *Callee = AI.getCalleeFunction();
assert(Callee && "apply_inst does not have a direct callee anymore");
DEBUG(llvm::dbgs() << " Inline:" << *AI.getInstruction());
if (!Callee->shouldOptimize()) {
DEBUG(llvm::dbgs() << " Cannot inline function " << Callee->getName()
<< " marked to be excluded from optimizations.\n");
continue;
}
SmallVector<SILValue, 8> Args;
for (const auto &Arg : AI.getArguments())
Args.push_back(Arg);
// As we inline and clone we need to collect new applies.
auto Filter = [](SILInstruction *I) -> bool {
return bool(FullApplySite::isa(I));
};
CloneCollector Collector(Filter);
// Notice that we will skip all of the newly inlined ApplyInsts. That's
// okay because we will visit them in our next invocation of the inliner.
TypeSubstitutionMap ContextSubs;
SILInliner Inliner(*Caller, *Callee,
SILInliner::InlineKind::PerformanceInline,
ContextSubs, AI.getSubstitutions(),
Collector.getCallback());
// Record the name of the inlined function (for cycle detection).
InlinedFunctionNames.push_back(Callee->getName());
auto Success = Inliner.inlineFunction(AI, Args);
(void) Success;
// We've already determined we should be able to inline this, so
// we expect it to have happened.
assert(Success && "Expected inliner to inline this function!");
llvm::SmallVector<FullApplySite, 4> AppliesFromInlinee;
for (auto &P : Collector.getInstructionPairs())
AppliesFromInlinee.push_back(FullApplySite(P.first));
recursivelyDeleteTriviallyDeadInstructions(AI.getInstruction(), true);
NewApplies.insert(NewApplies.end(), AppliesFromInlinee.begin(),
AppliesFromInlinee.end());
DA->invalidate(Caller, SILAnalysis::InvalidationKind::Everything);
NumFunctionsInlined++;
}
// Record the names of the functions that we inlined.
// We'll use this list to detect cycles in future iterations of
// the inliner.
for (auto CalleeName : InlinedFunctionNames) {
InlinedFunctions.insert(std::make_pair(CallerName, CalleeName));
}
DEBUG(llvm::dbgs() << "\n");
return true;
}
static void createThunkBody(SILBasicBlock *BB, SILFunction *NewF,
FunctionAnalyzer &Analyzer) {
// TODO: What is the proper location to use here?
SILLocation Loc = BB->getParent()->getLocation();
SILBuilder Builder(BB);
Builder.setCurrentDebugScope(BB->getParent()->getDebugScope());
FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF);
// Create the args for the thunk's apply, ignoring any dead arguments.
llvm::SmallVector<SILValue, 8> ThunkArgs;
ArrayRef<ArgumentDescriptor> ArgDescs = Analyzer.getArgDescList();
for (auto &ArgDesc : ArgDescs) {
ArgDesc.addThunkArgs(Builder, BB, ThunkArgs);
}
// We are ignoring generic functions and functions with out parameters for
// now.
SILType LoweredType = NewF->getLoweredType();
SILType ResultType = LoweredType.getFunctionInterfaceResultType();
SILValue ReturnValue;
auto FunctionTy = LoweredType.castTo<SILFunctionType>();
if (FunctionTy->hasErrorResult()) {
// We need a try_apply to call a function with an error result.
SILFunction *Thunk = BB->getParent();
SILBasicBlock *NormalBlock = Thunk->createBasicBlock();
ReturnValue = NormalBlock->createBBArg(ResultType, 0);
SILBasicBlock *ErrorBlock = Thunk->createBasicBlock();
SILType ErrorType =
SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType());
auto *ErrorArg = ErrorBlock->createBBArg(ErrorType, 0);
Builder.createTryApply(Loc, FRI, LoweredType, ArrayRef<Substitution>(),
ThunkArgs, NormalBlock, ErrorBlock);
// If we have any arguments that were consumed but are now guaranteed,
// insert a release_value in the error block.
Builder.setInsertionPoint(ErrorBlock);
for (auto &ArgDesc : ArgDescs) {
if (!ArgDesc.CalleeRelease)
continue;
Builder.createReleaseValue(Loc, BB->getBBArg(ArgDesc.Index));
}
Builder.createThrow(Loc, ErrorArg);
// Also insert release_value in the normal block (done below).
Builder.setInsertionPoint(NormalBlock);
} else {
ReturnValue =
Builder.createApply(Loc, FRI, LoweredType, ResultType,
ArrayRef<Substitution>(), ThunkArgs, false);
}
// If we have any arguments that were consumed but are now guaranteed,
// insert a release_value.
for (auto &ArgDesc : ArgDescs) {
if (!ArgDesc.CalleeRelease)
continue;
Builder.createReleaseValue(Loc, BB->getBBArg(ArgDesc.Index));
}
// Function that are marked as @NoReturn must be followed by an 'unreachable'
// instruction.
if (NewF->getLoweredFunctionType()->isNoReturn()) {
Builder.createUnreachable(Loc);
return;
}
Builder.createReturn(Loc, ReturnValue);
}
void SILPerformanceInliner::inlineDevirtualizeAndSpecialize(
SILFunction *Caller,
SILModuleTransform *MT,
DominanceAnalysis *DA,
SILLoopAnalysis *LA,
ClassHierarchyAnalysis *CHA) {
assert(Caller->isDefinition() && "Expected only functions with bodies!");
llvm::SmallVector<SILFunction *, 4> WorkList;
WorkList.push_back(Caller);
while (!WorkList.empty()) {
llvm::SmallVector<ApplySite, 4> WorkItemApplies;
SILFunction *CurrentCaller = WorkList.back();
if (CurrentCaller->shouldOptimize())
collectAllAppliesInFunction(CurrentCaller, WorkItemApplies);
// Devirtualize and specialize any applies we've collected,
// and collect new functions we should inline into as we do
// so.
llvm::SmallVector<SILFunction *, 4> NewFuncs;
if (devirtualizeAndSpecializeApplies(WorkItemApplies, MT, CHA, NewFuncs)) {
WorkList.insert(WorkList.end(), NewFuncs.begin(), NewFuncs.end());
NewFuncs.clear();
}
assert(WorkItemApplies.empty() && "Expected all applies to be processed!");
// We want to inline into each function on the worklist, starting
// with any new ones that were exposed as a result of
// devirtualization (to insure we're inlining into callees first).
//
// After inlining, we may have new opportunities for
// devirtualization, e.g. as a result of exposing the dynamic type
// of an object. When those opportunities arise we want to attempt
// devirtualization and then again attempt to inline into the
// newly exposed functions, etc. until we're back to the function
// we began with.
auto *Initial = WorkList.back();
// In practice we rarely exceed 5, but in a perf test we iterate 51 times.
const unsigned MaxLaps = 1500;
unsigned Lap = 0;
while (1) {
auto *WorkItem = WorkList.back();
assert(WorkItem->isDefinition() &&
"Expected function definition on work list!");
// Devirtualization and specialization might have exposed new
// function references. We want to inline within those functions
// before inlining within our original function.
//
// Inlining in turn might result in new applies that we should
// consider for devirtualization and specialization.
llvm::SmallVector<FullApplySite, 4> NewApplies;
bool Inlined = inlineCallsIntoFunction(WorkItem, DA, LA, NewApplies);
if (Inlined) {
MT->invalidateAnalysis(WorkItem,
SILAnalysis::InvalidationKind::FunctionBody);
// FIXME: Update inlineCallsIntoFunction to collect all
// remaining applies after inlining, not just those
// resulting from inlining code.
llvm::SmallVector<ApplySite, 4> WorkItemApplies;
collectAllAppliesInFunction(WorkItem, WorkItemApplies);
bool Modified =
devirtualizeAndSpecializeApplies(WorkItemApplies, MT,
CHA, NewFuncs);
if (Modified) {
WorkList.insert(WorkList.end(), NewFuncs.begin(), NewFuncs.end());
NewFuncs.clear();
assert(WorkItemApplies.empty() &&
"Expected all applies to be processed!");
} else if (WorkItem == Initial) {
// We did not specialize generics or devirtualize calls and we
// did not create new opportunities so we can bail out now.
break;
} else {
WorkList.pop_back();
}
} else if (WorkItem == Initial) {
// We did not inline any calls and did not create new opportunities
// so we can bail out now.
break;
} else {
WorkList.pop_back();
}
Lap++;
// It's possible to construct real code where this will hit, but
// it's more likely that there is an issue tracking recursive
// inlining, in which case we want to know about it in internal
// builds, and not hang on bots or user machines.
assert(Lap <= MaxLaps && "Possible bug tracking recursion!");
// Give up and move along.
if (Lap > MaxLaps) {
while (WorkList.back() != Initial)
WorkList.pop_back();
break;
}
}
assert(WorkList.back() == Initial &&
"Expected to exit with same element on top of stack!" );
WorkList.pop_back();
}
}
static llvm::Function *
getAccessorForComputedComponent(IRGenModule &IGM,
const KeyPathPatternComponent &component,
KeyPathAccessor whichAccessor,
GenericEnvironment *genericEnv,
ArrayRef<GenericRequirement> requirements) {
SILFunction *accessor;
switch (whichAccessor) {
case Getter:
accessor = component.getComputedPropertyGetter();
break;
case Setter:
accessor = component.getComputedPropertySetter();
break;
case Equals:
accessor = component.getSubscriptIndexEquals();
break;
case Hash:
accessor = component.getSubscriptIndexHash();
break;
}
auto accessorFn = IGM.getAddrOfSILFunction(accessor, NotForDefinition);
// If the accessor is not generic, we can use it as is.
if (requirements.empty()) {
return accessorFn;
}
auto accessorFnTy = cast<llvm::FunctionType>(
accessorFn->getType()->getPointerElementType());;
// Otherwise, we need a thunk to unmarshal the generic environment from the
// argument area. It'd be nice to have a good way to represent this
// directly in SIL, of course...
const char *thunkName;
unsigned numArgsToForward;
switch (whichAccessor) {
case Getter:
thunkName = "keypath_get";
numArgsToForward = 2;
break;
case Setter:
thunkName = "keypath_set";
numArgsToForward = 2;
break;
case Equals:
thunkName = "keypath_equals";
numArgsToForward = 2;
break;
case Hash:
thunkName = "keypath_hash";
numArgsToForward = 1;
break;
}
SmallVector<llvm::Type *, 4> thunkParams;
for (unsigned i = 0; i < numArgsToForward; ++i)
thunkParams.push_back(accessorFnTy->getParamType(i));
switch (whichAccessor) {
case Getter:
case Setter:
thunkParams.push_back(IGM.Int8PtrTy);
break;
case Equals:
case Hash:
break;
}
thunkParams.push_back(IGM.SizeTy);
auto thunkType = llvm::FunctionType::get(accessorFnTy->getReturnType(),
thunkParams,
/*vararg*/ false);
auto accessorThunk = llvm::Function::Create(thunkType,
llvm::GlobalValue::PrivateLinkage, thunkName, IGM.getModule());
accessorThunk->setAttributes(IGM.constructInitialAttributes());
accessorThunk->setCallingConv(IGM.SwiftCC);
switch (whichAccessor) {
case Getter:
// Original accessor's args should be @in or @out, meaning they won't be
// captured or aliased.
accessorThunk->addAttribute(1, llvm::Attribute::NoCapture);
accessorThunk->addAttribute(1, llvm::Attribute::NoAlias);
accessorThunk->addAttribute(2, llvm::Attribute::NoCapture);
accessorThunk->addAttribute(2, llvm::Attribute::NoAlias);
// Output is sret.
accessorThunk->addAttribute(1, llvm::Attribute::StructRet);
break;
case Setter:
// Original accessor's args should be @in or @out, meaning they won't be
// captured or aliased.
accessorThunk->addAttribute(1, llvm::Attribute::NoCapture);
accessorThunk->addAttribute(1, llvm::Attribute::NoAlias);
accessorThunk->addAttribute(2, llvm::Attribute::NoCapture);
accessorThunk->addAttribute(2, llvm::Attribute::NoAlias);
break;
case Equals:
case Hash:
break;
}
{
IRGenFunction IGF(IGM, accessorThunk);
if (IGM.DebugInfo)
IGM.DebugInfo->emitArtificialFunction(IGF, accessorThunk);
auto params = IGF.collectParameters();
Explosion forwardedArgs;
forwardedArgs.add(params.claim(numArgsToForward));
llvm::Value *componentArgsBuf;
switch (whichAccessor) {
case Getter:
case Setter:
// The component arguments are passed alongside the base being projected.
componentArgsBuf = params.claimNext();
// Pass the argument pointer down to the underlying function.
if (!component.getSubscriptIndices().empty()) {
forwardedArgs.add(componentArgsBuf);
}
break;
case Equals:
case Hash:
// We're operating directly on the component argument buffer.
componentArgsBuf = forwardedArgs.getAll()[0];
break;
}
auto componentArgsBufSize = params.claimNext();
bindPolymorphicArgumentsFromComponentIndices(IGF, component,
genericEnv, requirements,
componentArgsBuf,
componentArgsBufSize);
// Use the bound generic metadata to form a call to the original generic
// accessor.
WitnessMetadata ignoreWitnessMetadata;
auto forwardingSubs = genericEnv->getGenericSignature()->getSubstitutionMap(
genericEnv->getForwardingSubstitutions());
emitPolymorphicArguments(IGF, accessor->getLoweredFunctionType(),
forwardingSubs,
&ignoreWitnessMetadata,
forwardedArgs);
auto fnPtr = FunctionPointer::forDirect(IGM, accessorFn,
accessor->getLoweredFunctionType());
auto call = IGF.Builder.CreateCall(fnPtr, forwardedArgs.claimAll());
if (call->getType()->isVoidTy())
IGF.Builder.CreateRetVoid();
else
IGF.Builder.CreateRet(call);
}
return accessorThunk;
}
// Returns the callee of an apply_inst if it is basically inlineable.
SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) {
SILFunction *Callee = AI.getCalleeFunction();
if (!Callee) {
DEBUG(llvm::dbgs() << " FAIL: Cannot find inlineable callee.\n");
return nullptr;
}
// Don't inline functions that are marked with the @_semantics or @effects
// attribute if the inliner is asked not to inline them.
if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) {
if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) {
DEBUG(llvm::dbgs() << " FAIL: Function " << Callee->getName()
<< " has special semantics or effects attribute.\n");
return nullptr;
}
// The "availability" semantics attribute is treated like global-init.
if (Callee->hasSemanticsAttrs() &&
WhatToInline != InlineSelection::Everything &&
Callee->hasSemanticsAttrThatStartsWith("availability")) {
return nullptr;
}
} else if (Callee->isGlobalInit()) {
if (WhatToInline != InlineSelection::Everything) {
DEBUG(llvm::dbgs() << " FAIL: Function " << Callee->getName()
<< " has the global-init attribute.\n");
return nullptr;
}
}
// We can't inline external declarations.
if (Callee->empty() || Callee->isExternalDeclaration()) {
DEBUG(llvm::dbgs() << " FAIL: Cannot inline external " <<
Callee->getName() << ".\n");
return nullptr;
}
// Explicitly disabled inlining.
if (Callee->getInlineStrategy() == NoInline) {
DEBUG(llvm::dbgs() << " FAIL: noinline attribute on " <<
Callee->getName() << ".\n");
return nullptr;
}
if (!Callee->shouldOptimize()) {
DEBUG(llvm::dbgs() << " FAIL: optimizations disabled on " <<
Callee->getName() << ".\n");
return nullptr;
}
// We don't support this yet.
if (AI.hasSubstitutions()) {
DEBUG(llvm::dbgs() << " FAIL: Generic substitutions on " <<
Callee->getName() << ".\n");
return nullptr;
}
// We don't support inlining a function that binds dynamic self because we
// have no mechanism to preserve the original function's local self metadata.
if (computeMayBindDynamicSelf(Callee)) {
DEBUG(llvm::dbgs() << " FAIL: Binding dynamic Self in " <<
Callee->getName() << ".\n");
return nullptr;
}
SILFunction *Caller = AI.getFunction();
// Detect inlining cycles.
if (hasInliningCycle(Caller, Callee)) {
DEBUG(llvm::dbgs() << " FAIL: Detected a recursion inlining " <<
Callee->getName() << ".\n");
return nullptr;
}
// A non-fragile function may not be inlined into a fragile function.
if (Caller->isFragile() && !Callee->isFragile()) {
DEBUG(llvm::dbgs() << " FAIL: Can't inline fragile " <<
Callee->getName() << ".\n");
return nullptr;
}
// Inlining self-recursive functions into other functions can result
// in excessive code duplication since we run the inliner multiple
// times in our pipeline, so we only do it for callees with few
// self-recursive calls.
if (calleeHasMinimalSelfRecursion(Callee)) {
DEBUG(llvm::dbgs() << " FAIL: Callee is self-recursive in "
<< Callee->getName() << ".\n");
return nullptr;
}
DEBUG(llvm::dbgs() << " Eligible callee: " <<
Callee->getName() << "\n");
return Callee;
}
/// Return true if inlining this call site is profitable.
bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI,
unsigned loopDepthOfAI,
DominanceAnalysis *DA,
SILLoopAnalysis *LA,
ConstantTracker &callerTracker,
unsigned &NumCallerBlocks) {
SILFunction *Callee = AI.getReferencedFunction();
if (Callee->getInlineStrategy() == AlwaysInline)
return true;
ConstantTracker constTracker(Callee, &callerTracker, AI);
DominanceInfo *DT = DA->get(Callee);
SILLoopInfo *LI = LA->get(Callee);
DominanceOrder domOrder(&Callee->front(), DT, Callee->size());
// Calculate the inlining cost of the callee.
unsigned CalleeCost = 0;
unsigned Benefit = InlineCostThreshold > 0 ? InlineCostThreshold :
RemovedCallBenefit;
Benefit += loopDepthOfAI * LoopBenefitFactor;
int testThreshold = TestThreshold;
while (SILBasicBlock *block = domOrder.getNext()) {
constTracker.beginBlock();
for (SILInstruction &I : *block) {
constTracker.trackInst(&I);
if (testThreshold >= 0) {
// We are in test-mode: use a simplified cost model.
CalleeCost += testCost(&I);
} else {
// Use the regular cost model.
CalleeCost += unsigned(instructionInlineCost(I));
}
if (ApplyInst *AI = dyn_cast<ApplyInst>(&I)) {
// Check if the callee is passed as an argument. If so, increase the
// threshold, because inlining will (probably) eliminate the closure.
SILInstruction *def = constTracker.getDefInCaller(AI->getCallee());
if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def))) {
unsigned loopDepth = LI->getLoopDepth(block);
Benefit += ConstCalleeBenefit + loopDepth * LoopBenefitFactor;
testThreshold *= 2;
}
}
}
// Don't count costs in blocks which are dead after inlining.
SILBasicBlock *takenBlock = getTakenBlock(block->getTerminator(),
constTracker);
if (takenBlock) {
Benefit += ConstTerminatorBenefit;
domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) {
return child->getSinglePredecessor() != block || child == takenBlock;
});
} else {
domOrder.pushChildren(block);
}
}
unsigned Threshold = Benefit; // The default.
if (testThreshold >= 0) {
// We are in testing mode.
Threshold = testThreshold;
} else if (AI.getFunction()->isThunk()) {
// Only inline trivial functions into thunks (which will not increase the
// code size).
Threshold = TrivialFunctionThreshold;
} else {
// The default case.
// We reduce the benefit if the caller is too large. For this we use a
// cubic function on the number of caller blocks. This starts to prevent
// inlining at about 800 - 1000 caller blocks.
unsigned blockMinus =
(NumCallerBlocks * NumCallerBlocks) / BlockLimitDenominator *
NumCallerBlocks / BlockLimitDenominator;
if (Threshold > blockMinus + TrivialFunctionThreshold)
Threshold -= blockMinus;
else
Threshold = TrivialFunctionThreshold;
}
if (CalleeCost > Threshold) {
return false;
}
NumCallerBlocks += Callee->size();
DEBUG(
dumpCaller(AI.getFunction());
llvm::dbgs() << " decision {" << CalleeCost << " < " << Threshold <<
", ld=" << loopDepthOfAI << ", bb=" << NumCallerBlocks << "} " <<
Callee->getName() << '\n';
);
/// Return true if inlining this call site is profitable.
bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI,
unsigned loopDepthOfAI,
DominanceAnalysis *DA,
SILLoopAnalysis *LA,
ConstantTracker &callerTracker) {
SILFunction *Callee = AI.getCalleeFunction();
if (Callee->getInlineStrategy() == AlwaysInline)
return true;
ConstantTracker constTracker(Callee, &callerTracker, AI);
DominanceInfo *DT = DA->get(Callee);
SILLoopInfo *LI = LA->get(Callee);
DominanceOrder domOrder(&Callee->front(), DT, Callee->size());
// Calculate the inlining cost of the callee.
unsigned CalleeCost = 0;
unsigned Benefit = InlineCostThreshold > 0 ? InlineCostThreshold :
RemovedCallBenefit;
Benefit += loopDepthOfAI * LoopBenefitFactor;
int testThreshold = TestThreshold;
while (SILBasicBlock *block = domOrder.getNext()) {
constTracker.beginBlock();
unsigned loopDepth = LI->getLoopDepth(block);
for (SILInstruction &I : *block) {
constTracker.trackInst(&I);
auto ICost = instructionInlineCost(I);
if (testThreshold >= 0) {
// We are in test-mode: use a simplified cost model.
CalleeCost += testCost(&I);
} else {
// Use the regular cost model.
CalleeCost += unsigned(ICost);
}
if (ApplyInst *AI = dyn_cast<ApplyInst>(&I)) {
// Check if the callee is passed as an argument. If so, increase the
// threshold, because inlining will (probably) eliminate the closure.
SILInstruction *def = constTracker.getDefInCaller(AI->getCallee());
if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def))) {
DEBUG(llvm::dbgs() << " Boost: apply const function at"
<< *AI);
Benefit += ConstCalleeBenefit + loopDepth * LoopBenefitFactor;
testThreshold *= 2;
}
}
}
// Don't count costs in blocks which are dead after inlining.
SILBasicBlock *takenBlock = getTakenBlock(block->getTerminator(),
constTracker);
if (takenBlock) {
Benefit += ConstTerminatorBenefit + TestOpt;
DEBUG(llvm::dbgs() << " Take bb" << takenBlock->getDebugID() <<
" of" << *block->getTerminator());
domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) {
return child->getSinglePredecessor() != block || child == takenBlock;
});
} else {
domOrder.pushChildren(block);
}
}
unsigned Threshold = Benefit; // The default.
if (testThreshold >= 0) {
// We are in testing mode.
Threshold = testThreshold;
} else if (AI.getFunction()->isThunk()) {
// Only inline trivial functions into thunks (which will not increase the
// code size).
Threshold = TrivialFunctionThreshold;
}
if (CalleeCost > Threshold) {
DEBUG(llvm::dbgs() << " NO: Function too big to inline, "
"cost: " << CalleeCost << ", threshold: " << Threshold << "\n");
return false;
}
DEBUG(llvm::dbgs() << " YES: ready to inline, "
"cost: " << CalleeCost << ", threshold: " << Threshold << "\n");
return true;
}
static bool removeUnreachableBlocks(SILFunction &F, SILModule &M,
UnreachableUserCodeReportingState *State) {
if (F.empty())
return false;
SILBasicBlockSet Reachable;
SmallVector<SILBasicBlock*, 128> Worklist;
Worklist.push_back(&F.front());
Reachable.insert(&F.front());
// Collect all reachable blocks by walking the successors.
do {
SILBasicBlock *BB = Worklist.pop_back_val();
for (auto SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) {
if (Reachable.insert(*SI).second)
Worklist.push_back(*SI);
}
} while (!Worklist.empty());
assert(Reachable.size() <= F.size());
// If everything is reachable, we are done.
if (Reachable.size() == F.size())
return false;
// Diagnose user written unreachable code.
if (State) {
for (auto BI = State->PossiblyUnreachableBlocks.begin(),
BE = State->PossiblyUnreachableBlocks.end(); BI != BE; ++BI) {
const SILBasicBlock *BB = *BI;
if (!Reachable.count(BB)) {
llvm::SmallPtrSet<const SILBasicBlock *, 1> visited;
diagnoseUnreachableBlock(**BI, M, Reachable, State, BB, visited);
}
}
}
// Remove references from the dead blocks.
for (auto I = F.begin(), E = F.end(); I != E; ++I) {
SILBasicBlock *BB = &*I;
if (Reachable.count(BB))
continue;
// Drop references to other blocks.
recursivelyDeleteTriviallyDeadInstructions(BB->getTerminator(), true);
NumInstructionsRemoved++;
}
// Delete dead instructions and everything that could become dead after
// their deletion.
llvm::SmallVector<SILInstruction*, 32> ToBeDeleted;
for (auto BI = F.begin(), BE = F.end(); BI != BE; ++BI)
if (!Reachable.count(&*BI))
for (auto I = BI->begin(), E = BI->end(); I != E; ++I)
ToBeDeleted.push_back(&*I);
recursivelyDeleteTriviallyDeadInstructions(ToBeDeleted, true);
NumInstructionsRemoved += ToBeDeleted.size();
// Delete the dead blocks.
for (auto I = F.begin(), E = F.end(); I != E;)
if (!Reachable.count(&*I)) {
I = F.getBlocks().erase(I);
NumBlocksRemoved++;
} else
++I;
return true;
}
void FunctionSignatureTransform::createFunctionSignatureOptimizedFunction() {
// Create the optimized function !
SILModule &M = F->getModule();
std::string Name = getUniqueName(createOptimizedSILFunctionName(), M);
NewF = M.createFunction(
F->getLinkage(), Name,
createOptimizedSILFunctionType(), nullptr, F->getLocation(), F->isBare(),
F->isTransparent(), F->isFragile(), F->isThunk(), F->getClassVisibility(),
F->getInlineStrategy(), F->getEffectsKind(), 0, F->getDebugScope(),
F->getDeclContext());
// Then we transfer the body of F to NewF.
NewF->spliceBody(F);
NewF->setDeclCtx(F->getDeclContext());
// Array semantic clients rely on the signature being as in the original
// version.
for (auto &Attr : F->getSemanticsAttrs()) {
if (!StringRef(Attr).startswith("array."))
NewF->addSemanticsAttr(Attr);
}
// Do the last bit of work to the newly created optimized function.
ArgumentExplosionFinalizeOptimizedFunction();
DeadArgumentFinalizeOptimizedFunction();
// Create the thunk body !
F->setThunk(IsThunk);
// The thunk now carries the information on how the signature is
// optimized. If we inline the thunk, we will get the benefit of calling
// the signature optimized function without additional setup on the
// caller side.
F->setInlineStrategy(AlwaysInline);
SILBasicBlock *ThunkBody = F->createBasicBlock();
for (auto &ArgDesc : ArgumentDescList) {
ThunkBody->createBBArg(ArgDesc.Arg->getType(), ArgDesc.Decl);
}
SILLocation Loc = ThunkBody->getParent()->getLocation();
SILBuilder Builder(ThunkBody);
Builder.setCurrentDebugScope(ThunkBody->getParent()->getDebugScope());
FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF);
// Create the args for the thunk's apply, ignoring any dead arguments.
llvm::SmallVector<SILValue, 8> ThunkArgs;
for (auto &ArgDesc : ArgumentDescList) {
addThunkArgument(ArgDesc, Builder, ThunkBody, ThunkArgs);
}
// We are ignoring generic functions and functions with out parameters for
// now.
SILValue ReturnValue;
SILType LoweredType = NewF->getLoweredType();
SILType ResultType = LoweredType.getFunctionInterfaceResultType();
auto FunctionTy = LoweredType.castTo<SILFunctionType>();
if (FunctionTy->hasErrorResult()) {
// We need a try_apply to call a function with an error result.
SILFunction *Thunk = ThunkBody->getParent();
SILBasicBlock *NormalBlock = Thunk->createBasicBlock();
ReturnValue = NormalBlock->createBBArg(ResultType, 0);
SILBasicBlock *ErrorBlock = Thunk->createBasicBlock();
SILType ErrorProtocol =
SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType());
auto *ErrorArg = ErrorBlock->createBBArg(ErrorProtocol, 0);
Builder.createTryApply(Loc, FRI, LoweredType, ArrayRef<Substitution>(),
ThunkArgs, NormalBlock, ErrorBlock);
Builder.setInsertionPoint(ErrorBlock);
Builder.createThrow(Loc, ErrorArg);
Builder.setInsertionPoint(NormalBlock);
} else {
ReturnValue = Builder.createApply(Loc, FRI, LoweredType, ResultType,
ArrayRef<Substitution>(), ThunkArgs,
false);
}
// Set up the return results.
if (NewF->getLoweredFunctionType()->isNoReturn()) {
Builder.createUnreachable(Loc);
} else {
Builder.createReturn(Loc, ReturnValue);
}
// Do the last bit work to finalize the thunk.
OwnedToGuaranteedFinalizeThunkFunction(Builder, F);
assert(F->getDebugScope()->Parent != NewF->getDebugScope()->Parent);
}
/// Insert monomorphic inline caches for a specific class or metatype
/// type \p SubClassTy.
static FullApplySite speculateMonomorphicTarget(FullApplySite AI,
SILType SubType,
CheckedCastBranchInst *&CCBI) {
CCBI = nullptr;
// Bail if this class_method cannot be devirtualized.
if (!canDevirtualizeClassMethod(AI, SubType))
return FullApplySite();
if (SubType.getSwiftRValueType()->hasDynamicSelfType())
return FullApplySite();
// Create a diamond shaped control flow and a checked_cast_branch
// instruction that checks the exact type of the object.
// This cast selects between two paths: one that calls the slow dynamic
// dispatch and one that calls the specific method.
auto It = AI.getInstruction()->getIterator();
SILFunction *F = AI.getFunction();
SILBasicBlock *Entry = AI.getParent();
// Iden is the basic block containing the direct call.
SILBasicBlock *Iden = F->createBasicBlock();
// Virt is the block containing the slow virtual call.
SILBasicBlock *Virt = F->createBasicBlock();
Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned);
SILBasicBlock *Continue = Entry->split(It);
SILBuilderWithScope Builder(Entry, AI.getInstruction());
// Create the checked_cast_branch instruction that checks at runtime if the
// class instance is identical to the SILType.
ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee());
CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true,
CMI->getOperand(), SubType, Iden,
Virt);
It = CCBI->getIterator();
SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction());
SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction());
// This is the class reference downcasted into subclass SubType.
SILValue DownCastedClassInstance = Iden->getArgument(0);
// Copy the two apply instructions into the two blocks.
FullApplySite IdenAI = CloneApply(AI, IdenBuilder);
FullApplySite VirtAI = CloneApply(AI, VirtBuilder);
// See if Continue has a release on self as the instruction right after the
// apply. If it exists, move it into position in the diamond.
SILBasicBlock::iterator next =
next_or_end(Continue->begin(), Continue->end());
auto *Release =
(next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next);
if (Release && Release->getOperand() == CMI->getOperand()) {
VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(),
Release->getAtomicity());
IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance,
Release->getAtomicity());
Release->eraseFromParent();
}
// Create a PHInode for returning the return value from both apply
// instructions.
SILArgument *Arg =
Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned);
if (!isa<TryApplyInst>(AI)) {
if (AI.getSubstCalleeType()->isNoReturnFunction()) {
IdenBuilder.createUnreachable(AI.getLoc());
VirtBuilder.createUnreachable(AI.getLoc());
} else {
IdenBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(IdenAI) });
VirtBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(VirtAI) });
}
}
// Remove the old Apply instruction.
assert(AI.getInstruction() == &Continue->front() &&
"AI should be the first instruction in the split Continue block");
if (isa<TryApplyInst>(AI)) {
AI.getInstruction()->eraseFromParent();
assert(Continue->empty() &&
"There should not be an instruction after try_apply");
Continue->eraseFromParent();
} else {
auto apply = cast<ApplyInst>(AI);
apply->replaceAllUsesWith(Arg);
apply->eraseFromParent();
assert(!Continue->empty() &&
"There should be at least a terminator after AI");
}
// Update the stats.
NumTargetsPredicted++;
// Devirtualize the apply instruction on the identical path.
auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance);
assert(NewInstPair.first && "Expected to be able to devirtualize apply!");
replaceDeadApply(IdenAI, NewInstPair.first);
// Split critical edges resulting from VirtAI.
if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) {
auto *ErrorBB = TAI->getFunction()->createBasicBlock();
ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(ErrorBB);
Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getArgument(0)});
auto *NormalBB = TAI->getFunction()->createBasicBlock();
NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(NormalBB);
Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(),
{NormalBB->getArgument(0)});
Builder.setInsertionPoint(VirtAI.getInstruction());
SmallVector<SILValue, 4> Args;
for (auto Arg : VirtAI.getArguments()) {
Args.push_back(Arg);
}
FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(),
VirtAI.getSubstitutions(),
Args, NormalBB, ErrorBB);
VirtAI.getInstruction()->eraseFromParent();
VirtAI = NewVirtAI;
}
return VirtAI;
}
/// In this function we create the actual cloned function and its proper cloned
/// type. But we do not create any body. This implies that the creation of the
/// actual arguments in the function is in populateCloned.
///
/// \arg PAUser The function that is being passed the partial apply.
/// \arg PAI The partial apply that is being passed to PAUser.
/// \arg ClosureIndex The index of the partial apply in PAUser's function
/// signature.
/// \arg ClonedName The name of the cloned function that we will create.
SILFunction *
ClosureSpecCloner::initCloned(const CallSiteDescriptor &CallSiteDesc,
StringRef ClonedName) {
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// This is the list of new interface parameters of the cloned function.
llvm::SmallVector<SILParameterInfo, 4> NewParameterInfoList;
// First add to NewParameterInfoList all of the SILParameterInfo in the
// original function except for the closure.
CanSILFunctionType ClosureUserFunTy = ClosureUser->getLoweredFunctionType();
unsigned Index = ClosureUserFunTy->getNumIndirectResults();
for (auto ¶m : ClosureUserFunTy->getParameters()) {
if (Index != CallSiteDesc.getClosureIndex())
NewParameterInfoList.push_back(param);
++Index;
}
// Then add any arguments that are captured in the closure to the function's
// argument type. Since they are captured, we need to pass them directly into
// the new specialized function.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
SILModule &M = ClosureUser->getModule();
// Captured parameters are always appended to the function signature. If the
// type of the captured argument is trivial, pass the argument as
// Direct_Unowned. Otherwise pass it as Direct_Owned.
//
// We use the type of the closure here since we allow for the closure to be an
// external declaration.
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
if (PInfo.getSILType().isTrivial(M)) {
SILParameterInfo NewPInfo(PInfo.getType(),
ParameterConvention::Direct_Unowned);
NewParameterInfoList.push_back(NewPInfo);
continue;
}
SILParameterInfo NewPInfo(PInfo.getType(),
ParameterConvention::Direct_Owned);
NewParameterInfoList.push_back(NewPInfo);
}
// The specialized function is always a thin function. This is important
// because we may add additional parameters after the Self parameter of
// witness methods. In this case the new function is not a method anymore.
auto ExtInfo = ClosureUserFunTy->getExtInfo();
ExtInfo = ExtInfo.withRepresentation(SILFunctionTypeRepresentation::Thin);
auto ClonedTy = SILFunctionType::get(
ClosureUserFunTy->getGenericSignature(), ExtInfo,
ClosureUserFunTy->getCalleeConvention(), NewParameterInfoList,
ClosureUserFunTy->getAllResults(),
ClosureUserFunTy->getOptionalErrorResult(),
M.getASTContext());
// We make this function bare so we don't have to worry about decls in the
// SILArgument.
auto *Fn = M.createFunction(
// It's important to use a shared linkage for the specialized function
// and not the original linkage.
// Otherwise the new function could have an external linkage (in case the
// original function was de-serialized) and would not be code-gen'd.
getSpecializedLinkage(ClosureUser, ClosureUser->getLinkage()),
ClonedName, ClonedTy,
ClosureUser->getGenericEnvironment(), ClosureUser->getLocation(),
IsBare, ClosureUser->isTransparent(), CallSiteDesc.isFragile(),
ClosureUser->isThunk(), ClosureUser->getClassVisibility(),
ClosureUser->getInlineStrategy(), ClosureUser->getEffectsKind(),
ClosureUser, ClosureUser->getDebugScope());
Fn->setDeclCtx(ClosureUser->getDeclContext());
if (ClosureUser->hasUnqualifiedOwnership()) {
Fn->setUnqualifiedOwnership();
}
for (auto &Attr : ClosureUser->getSemanticsAttrs())
Fn->addSemanticsAttr(Attr);
return Fn;
}
bool SILCombiner::doOneIteration(SILFunction &F, unsigned Iteration) {
MadeChange = false;
DEBUG(llvm::dbgs() << "\n\nSILCOMBINE ITERATION #" << Iteration << " on "
<< F.getName() << "\n");
// Add reachable instructions to our worklist.
addReachableCodeToWorklist(&*F.begin());
// Process until we run out of items in our worklist.
while (!Worklist.isEmpty()) {
SILInstruction *I = Worklist.removeOne();
// When we erase an instruction, we use the map in the worklist to check if
// the instruction is in the worklist. If it is, we replace it with null
// instead of shifting all members of the worklist towards the front. This
// check makes sure that if we run into any such residual null pointers, we
// skip them.
if (I == nullptr)
continue;
// Check to see if we can DCE the instruction.
if (isInstructionTriviallyDead(I)) {
DEBUG(llvm::dbgs() << "SC: DCE: " << *I << '\n');
eraseInstFromFunction(*I);
++NumDeadInst;
MadeChange = true;
continue;
}
// Check to see if we can instsimplify the instruction.
if (SILValue Result = simplifyInstruction(I)) {
++NumSimplified;
DEBUG(llvm::dbgs() << "SC: Simplify Old = " << *I << '\n'
<< " New = " << *Result << '\n');
// Everything uses the new instruction now.
replaceInstUsesWith(*cast<SingleValueInstruction>(I), Result);
// Push the new instruction and any users onto the worklist.
Worklist.addUsersToWorklist(Result);
eraseInstFromFunction(*I);
MadeChange = true;
continue;
}
// If we have reached this point, all attempts to do simple simplifications
// have failed. Prepare to SILCombine.
Builder.setInsertionPoint(I);
#ifndef NDEBUG
std::string OrigI;
#endif
DEBUG(llvm::raw_string_ostream SS(OrigI); I->print(SS); OrigI = SS.str(););
DEBUG(llvm::dbgs() << "SC: Visiting: " << OrigI << '\n');
if (SILInstruction *Result = visit(I)) {
++NumCombined;
// Should we replace the old instruction with a new one?
if (Result != I) {
assert(&*std::prev(SILBasicBlock::iterator(I)) == Result &&
"Expected new instruction inserted before existing instruction!");
DEBUG(llvm::dbgs() << "SC: Old = " << *I << '\n'
<< " New = " << *Result << '\n');
// Everything uses the new instruction now.
replaceInstUsesPairwiseWith(I, Result);
// Push the new instruction and any users onto the worklist.
Worklist.add(Result);
Worklist.addUsersOfAllResultsToWorklist(Result);
eraseInstFromFunction(*I);
} else {
DEBUG(llvm::dbgs() << "SC: Mod = " << OrigI << '\n'
<< " New = " << *I << '\n');
// If the instruction was modified, it's possible that it is now dead.
// if so, remove it.
if (isInstructionTriviallyDead(I)) {
eraseInstFromFunction(*I);
} else {
Worklist.add(I);
Worklist.addUsersOfAllResultsToWorklist(I);
}
}
MadeChange = true;
}
// Our tracking list has been accumulating instructions created by the
// SILBuilder during this iteration. Go through the tracking list and add
// its contents to the worklist and then clear said list in preparation for
// the next iteration.
auto &TrackingList = *Builder.getTrackingList();
for (SILInstruction *I : TrackingList) {
DEBUG(llvm::dbgs() << "SC: add " << *I <<
" from tracking list to worklist\n");
Worklist.add(I);
}
TrackingList.clear();
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createArgument(Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue = ClonedEntryBB->createArgument(PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
// This implicitly asserts that a function binding dynamic self has a
// self metadata argument or object from which self metadata can be obtained.
bool isArgumentABIRequired(SILArgument *Arg) {
return MayDynamicBindSelf && (F->getSelfMetadataArgument() == Arg);
}
/// \brief Returns the callee SILFunction called at a call site, in the case
/// that the call is transparent (as in, both that the call is marked
/// with the transparent flag and that callee function is actually transparently
/// determinable from the SIL) or nullptr otherwise. This assumes that the SIL
/// is already in SSA form.
///
/// In the case that a non-null value is returned, FullArgs contains effective
/// argument operands for the callee function.
static SILFunction *
getCalleeFunction(FullApplySite AI, bool &IsThick,
SmallVectorImpl<SILValue>& CaptureArgs,
SmallVectorImpl<SILValue>& FullArgs,
PartialApplyInst *&PartialApply,
SILModule::LinkingMode Mode) {
IsThick = false;
PartialApply = nullptr;
CaptureArgs.clear();
FullArgs.clear();
for (const auto &Arg : AI.getArguments())
FullArgs.push_back(Arg);
SILValue CalleeValue = AI.getCallee();
if (LoadInst *LI = dyn_cast<LoadInst>(CalleeValue)) {
// Conservatively only see through alloc_box; we assume this pass is run
// immediately after SILGen
auto *PBI = dyn_cast<ProjectBoxInst>(LI->getOperand());
if (!PBI)
return nullptr;
auto *ABI = dyn_cast<AllocBoxInst>(PBI->getOperand());
if (!ABI)
return nullptr;
// Ensure there are no other uses of alloc_box than the project_box and
// retains, releases.
for (Operand *ABIUse : ABI->getUses())
if (ABIUse->getUser() != PBI &&
!isa<StrongRetainInst>(ABIUse->getUser()) &&
!isa<StrongReleaseInst>(ABIUse->getUser()))
return nullptr;
// Scan forward from the alloc box to find the first store, which
// (conservatively) must be in the same basic block as the alloc box
StoreInst *SI = nullptr;
for (auto I = SILBasicBlock::iterator(ABI), E = I->getParent()->end();
I != E; ++I) {
// If we find the load instruction first, then the load is loading from
// a non-initialized alloc; this shouldn't really happen but I'm not
// making any assumptions
if (&*I == LI)
return nullptr;
if ((SI = dyn_cast<StoreInst>(I)) && SI->getDest() == PBI) {
// We found a store that we know dominates the load; now ensure there
// are no other uses of the project_box except loads.
for (Operand *PBIUse : PBI->getUses())
if (PBIUse->getUser() != SI && !isa<LoadInst>(PBIUse->getUser()))
return nullptr;
// We can conservatively see through the store
break;
}
}
if (!SI)
return nullptr;
CalleeValue = SI->getSrc();
}
// We are allowed to see through exactly one "partial apply" instruction or
// one "thin to thick function" instructions, since those are the patterns
// generated when using auto closures.
if (PartialApplyInst *PAI =
dyn_cast<PartialApplyInst>(CalleeValue)) {
for (const auto &Arg : PAI->getArguments()) {
CaptureArgs.push_back(Arg);
FullArgs.push_back(Arg);
}
CalleeValue = PAI->getCallee();
IsThick = true;
PartialApply = PAI;
} else if (ThinToThickFunctionInst *TTTFI =
dyn_cast<ThinToThickFunctionInst>(CalleeValue)) {
CalleeValue = TTTFI->getOperand();
IsThick = true;
}
FunctionRefInst *FRI = dyn_cast<FunctionRefInst>(CalleeValue);
if (!FRI)
return nullptr;
SILFunction *CalleeFunction = FRI->getReferencedFunction();
switch (CalleeFunction->getRepresentation()) {
case SILFunctionTypeRepresentation::Thick:
case SILFunctionTypeRepresentation::Thin:
case SILFunctionTypeRepresentation::Method:
case SILFunctionTypeRepresentation::Closure:
case SILFunctionTypeRepresentation::WitnessMethod:
break;
case SILFunctionTypeRepresentation::CFunctionPointer:
case SILFunctionTypeRepresentation::ObjCMethod:
case SILFunctionTypeRepresentation::Block:
return nullptr;
}
// If CalleeFunction is a declaration, see if we can load it. If we fail to
// load it, bail.
if (CalleeFunction->empty()
&& !AI.getModule().linkFunction(CalleeFunction, Mode))
return nullptr;
return CalleeFunction;
}
