void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() {
SILBasicBlock *BB = &*NewF->begin();
SILBuilder Builder(BB->begin());
Builder.setCurrentDebugScope(BB->getParent()->getDebugScope());
unsigned TotalArgIndex = 0;
for (ArgumentDescriptor &AD : ArgumentDescList) {
// Simply continue if do not explode.
if (!AD.Explode) {
AIM[TotalArgIndex] = AD.Index;
TotalArgIndex ++;
continue;
}
// OK, we need to explode this argument.
unsigned ArgOffset = ++TotalArgIndex;
unsigned OldArgIndex = ArgOffset - 1;
llvm::SmallVector<SILValue, 8> LeafValues;
// We do this in the same order as leaf types since ProjTree expects that the
// order of leaf values matches the order of leaf types.
llvm::SmallVector<const ProjectionTreeNode*, 8> LeafNodes;
AD.ProjTree.getLeafNodes(LeafNodes);
for (auto Node : LeafNodes) {
LeafValues.push_back(BB->insertBBArg(ArgOffset++, Node->getType(),
BB->getBBArg(OldArgIndex)->getDecl()));
AIM[TotalArgIndex - 1] = AD.Index;
TotalArgIndex ++;
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
AD.ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(),
LeafValues);
// We ignored debugvalue uses when we constructed the new arguments, in order
// to preserve as much information as possible, we construct a new value for
// OrigArg from the leaf values and use that in place of the OrigArg.
SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue(Builder,
BB->getParent()->getLocation(),
LeafValues);
// Replace all uses of the original arg with the new value.
SILArgument *OrigArg = BB->getBBArg(OldArgIndex);
OrigArg->replaceAllUsesWith(NewOrigArgValue);
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseBBArg(OldArgIndex);
TotalArgIndex --;
}
}
bool SILCombiner::
isTryApplyResultNotUsed(UserListTy &AcceptedUses, TryApplyInst *TAI) {
SILBasicBlock *NormalBB = TAI->getNormalBB();
SILBasicBlock *ErrorBB = TAI->getErrorBB();
// The results of a try_apply are not only the normal and error return values,
// but also the decision whether it throws or not. Therefore we have to check
// if both, the normal and the error block, are empty and lead to a common
// destination block.
// Check if the normal and error blocks have a common single successor.
auto *NormalBr = dyn_cast<BranchInst>(NormalBB->getTerminator());
if (!NormalBr)
return false;
auto *ErrorBr = dyn_cast<BranchInst>(ErrorBB->getTerminator());
if (!ErrorBr || ErrorBr->getDestBB() != NormalBr->getDestBB())
return false;
assert(NormalBr->getNumArgs() == ErrorBr->getNumArgs() &&
"mismatching number of arguments for the same destination block");
// Check if both blocks pass the same arguments to the common destination.
for (unsigned Idx = 0, End = NormalBr->getNumArgs(); Idx < End; Idx++) {
if (NormalBr->getArg(Idx) != ErrorBr->getArg(Idx))
return false;
}
// Check if the normal and error results only have ARC operations as uses.
if (!recursivelyCollectARCUsers(AcceptedUses, NormalBB->getBBArg(0)))
return false;
if (!recursivelyCollectARCUsers(AcceptedUses, ErrorBB->getBBArg(0)))
return false;
SmallPtrSet<SILInstruction *, 8> UsesSet;
for (auto *I : AcceptedUses)
UsesSet.insert(I);
// Check if the normal and error blocks are empty, except the ARC uses.
for (auto &I : *NormalBB) {
if (!UsesSet.count(&I) && !isa<TermInst>(&I))
return false;
}
for (auto &I : *ErrorBB) {
if (!UsesSet.count(&I) && !isa<TermInst>(&I))
return false;
}
return true;
}
SILValue
StackAllocationPromoter::getLiveOutValue(BlockSet &PhiBlocks,
SILBasicBlock *StartBB) {
DEBUG(llvm::dbgs() << "*** Searching for a value definition.\n");
// Walk the Dom tree in search of a defining value:
for (DomTreeNode *Node = DT->getNode(StartBB); Node; Node = Node->getIDom()) {
SILBasicBlock *BB = Node->getBlock();
// If there is a store (that must come after the phi), use its value.
BlockToInstMap::iterator it = LastStoreInBlock.find(BB);
if (it != LastStoreInBlock.end())
if (StoreInst *St = dyn_cast_or_null<StoreInst>(it->second)) {
DEBUG(llvm::dbgs() << "*** Found Store def " << *St->getSrc());
return St->getSrc();
}
// If there is a Phi definition in this block:
if (PhiBlocks.count(BB)) {
// Return the dummy instruction that represents the new value that we will
// add to the basic block.
SILValue Phi = BB->getBBArg(BB->getNumBBArg()-1);
DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *Phi);
return Phi;
}
// Move to the next dominating block.
DEBUG(llvm::dbgs() << "*** Walking up the iDOM.\n");
}
DEBUG(llvm::dbgs() << "*** Could not find a Def. Using Undef.\n");
return SILUndef::get(ASI->getElementType(), ASI->getModule());
}
void FunctionSignatureTransform::DeadArgumentTransformFunction() {
SILBasicBlock *BB = &*F->begin();
for (const ArgumentDescriptor &AD : ArgumentDescList) {
if (!AD.IsEntirelyDead)
continue;
eraseUsesOfValue(BB->getBBArg(AD.Index));
}
}
/// At least one value feeding the specified SILArgument is a Struct. Attempt to
/// replace the Argument with a new Struct in the same block.
///
/// When we handle more types of casts, this can become a template.
///
/// ArgValues are the values feeding the specified Argument from each
/// predecessor. They must be listed in order of Arg->getParent()->getPreds().
static StructInst *replaceBBArgWithStruct(
SILArgument *Arg,
SmallVectorImpl<SILValue> &ArgValues) {
SILBasicBlock *PhiBB = Arg->getParent();
auto *FirstSI = dyn_cast<StructInst>(ArgValues[0]);
if (!FirstSI)
return nullptr;
// Collect the BBArg index of each struct oper.
// e.g.
// struct(A, B)
// br (B, A)
// : ArgIdxForOper => {1, 0}
SmallVector<unsigned, 4> ArgIdxForOper;
for (unsigned OperIdx : indices(FirstSI->getElements())) {
bool FoundMatchingArgIdx = false;
for (unsigned ArgIdx : indices(PhiBB->getBBArgs())) {
SmallVectorImpl<SILValue>::const_iterator AVIter = ArgValues.begin();
bool TryNextArgIdx = false;
for (SILBasicBlock *PredBB : PhiBB->getPreds()) {
// All argument values must be StructInst.
auto *PredSI = dyn_cast<StructInst>(*AVIter++);
if (!PredSI)
return nullptr;
OperandValueArrayRef EdgeValues =
getEdgeValuesForTerminator(PredBB->getTerminator(), PhiBB);
if (EdgeValues[ArgIdx] != PredSI->getElements()[OperIdx]) {
TryNextArgIdx = true;
break;
}
}
if (!TryNextArgIdx) {
assert(AVIter == ArgValues.end() && "# ArgValues does not match # BB preds");
FoundMatchingArgIdx = true;
ArgIdxForOper.push_back(ArgIdx);
break;
}
}
if (!FoundMatchingArgIdx)
return nullptr;
}
SmallVector<SILValue, 4> StructArgs;
for (auto ArgIdx : ArgIdxForOper)
StructArgs.push_back(PhiBB->getBBArg(ArgIdx));
SILBuilder Builder(PhiBB, PhiBB->begin());
return Builder.createStruct(cast<StructInst>(ArgValues[0])->getLoc(),
Arg->getType(), StructArgs);
}
void
FunctionSignatureTransform::
OwnedToGuaranteedAddResultRelease(ResultDescriptor &RD, SILBuilder &Builder,
SILFunction *F) {
// If we have any result that were consumed but are now guaranteed,
// insert a release_value.
if (!RD.OwnedToGuaranteed) {
return;
}
SILInstruction *Call = findOnlyApply(F);
if (isa<ApplyInst>(Call)) {
Builder.setInsertionPoint(&*std::next(SILBasicBlock::iterator(Call)));
Builder.createRetainValue(RegularLocation(SourceLoc()), Call,
Atomicity::Atomic);
} else {
SILBasicBlock *NormalBB = dyn_cast<TryApplyInst>(Call)->getNormalBB();
Builder.setInsertionPoint(&*NormalBB->begin());
Builder.createRetainValue(RegularLocation(SourceLoc()),
NormalBB->getBBArg(0), Atomicity::Atomic);
}
}
/// \brief Devirtualize an apply of a class method.
///
/// \p AI is the apply to devirtualize.
/// \p ClassOrMetatype is a class value or metatype value that is the
/// self argument of the apply we will devirtualize.
/// return the result value of the new ApplyInst if created one or null.
DevirtualizationResult swift::devirtualizeClassMethod(FullApplySite AI,
SILValue ClassOrMetatype) {
DEBUG(llvm::dbgs() << " Trying to devirtualize : " << *AI.getInstruction());
SILModule &Mod = AI.getModule();
auto *CMI = cast<ClassMethodInst>(AI.getCallee());
auto ClassOrMetatypeType = ClassOrMetatype.getType();
auto *F = getTargetClassMethod(Mod, ClassOrMetatypeType, CMI->getMember());
CanSILFunctionType GenCalleeType = F->getLoweredFunctionType();
auto Subs = getSubstitutionsForCallee(Mod, GenCalleeType,
ClassOrMetatypeType, AI);
CanSILFunctionType SubstCalleeType = GenCalleeType;
if (GenCalleeType->isPolymorphic())
SubstCalleeType = GenCalleeType->substGenericArgs(Mod, Mod.getSwiftModule(), Subs);
SILBuilderWithScope B(AI.getInstruction());
FunctionRefInst *FRI = B.createFunctionRef(AI.getLoc(), F);
// Create the argument list for the new apply, casting when needed
// in order to handle covariant indirect return types and
// contravariant argument types.
llvm::SmallVector<SILValue, 8> NewArgs;
auto Args = AI.getArguments();
auto ParamTypes = SubstCalleeType->getParameterSILTypes();
for (unsigned i = 0, e = Args.size() - 1; i != e; ++i)
NewArgs.push_back(castValueToABICompatibleType(&B, AI.getLoc(), Args[i],
Args[i].getType(),
ParamTypes[i]).getValue());
// Add the self argument, upcasting if required because we're
// calling a base class's method.
auto SelfParamTy = SubstCalleeType->getSelfParameter().getSILType();
NewArgs.push_back(castValueToABICompatibleType(&B, AI.getLoc(),
ClassOrMetatype,
ClassOrMetatypeType,
SelfParamTy).getValue());
// If we have a direct return type, make sure we use the subst callee return
// type. If we have an indirect return type, AI's return type of the empty
// tuple should be ok.
SILType ResultTy = AI.getType();
if (!SubstCalleeType->hasIndirectResult()) {
ResultTy = SubstCalleeType->getSILResult();
}
SILType SubstCalleeSILType =
SILType::getPrimitiveObjectType(SubstCalleeType);
FullApplySite NewAI;
SILBasicBlock *ResultBB = nullptr;
SILBasicBlock *NormalBB = nullptr;
SILValue ResultValue;
bool ResultCastRequired = false;
SmallVector<Operand *, 4> OriginalResultUses;
if (!isa<TryApplyInst>(AI)) {
NewAI = B.createApply(AI.getLoc(), FRI, SubstCalleeSILType, ResultTy,
Subs, NewArgs, cast<ApplyInst>(AI)->isNonThrowing());
ResultValue = SILValue(NewAI.getInstruction(), 0);
} else {
auto *TAI = cast<TryApplyInst>(AI);
// Create new normal and error BBs only if:
// - re-using a BB would create a critical edge
// - or, the result of the new apply would be of different
// type than the argument of the original normal BB.
if (TAI->getNormalBB()->getSinglePredecessor())
ResultBB = TAI->getNormalBB();
else {
ResultBB = B.getFunction().createBasicBlock();
ResultBB->createBBArg(ResultTy);
}
NormalBB = TAI->getNormalBB();
SILBasicBlock *ErrorBB = nullptr;
if (TAI->getErrorBB()->getSinglePredecessor())
ErrorBB = TAI->getErrorBB();
else {
ErrorBB = B.getFunction().createBasicBlock();
ErrorBB->createBBArg(TAI->getErrorBB()->getBBArg(0)->getType());
}
NewAI = B.createTryApply(AI.getLoc(), FRI, SubstCalleeSILType,
Subs, NewArgs,
ResultBB, ErrorBB);
if (ErrorBB != TAI->getErrorBB()) {
B.setInsertionPoint(ErrorBB);
B.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getBBArg(0)});
}
// Does the result value need to be casted?
ResultCastRequired = ResultTy != NormalBB->getBBArg(0)->getType();
if (ResultBB != NormalBB)
B.setInsertionPoint(ResultBB);
else if (ResultCastRequired) {
B.setInsertionPoint(NormalBB->begin());
// Collect all uses, before casting.
for (auto *Use : NormalBB->getBBArg(0)->getUses()) {
OriginalResultUses.push_back(Use);
}
NormalBB->getBBArg(0)->replaceAllUsesWith(SILUndef::get(AI.getType(), Mod));
NormalBB->replaceBBArg(0, ResultTy, nullptr);
}
// The result value is passed as a parameter to the normal block.
ResultValue = ResultBB->getBBArg(0);
}
// Check if any casting is required for the return value.
ResultValue = castValueToABICompatibleType(&B, NewAI.getLoc(), ResultValue,
ResultTy, AI.getType()).getValue();
DEBUG(llvm::dbgs() << " SUCCESS: " << F->getName() << "\n");
NumClassDevirt++;
if (NormalBB) {
if (NormalBB != ResultBB) {
// If artificial normal BB was introduced, branch
// to the original normal BB.
B.createBranch(NewAI.getLoc(), NormalBB, { ResultValue });
} else if (ResultCastRequired) {
// Update all original uses by the new value.
for(auto *Use: OriginalResultUses) {
Use->set(ResultValue);
}
}
return std::make_pair(NewAI.getInstruction(), NewAI);
}
// We need to return a pair of values here:
// - the first one is the actual result of the devirtualized call, possibly
// casted into an appropriate type. This SILValue may be a BB arg, if it
// was a cast between optional types.
// - the second one is the new apply site.
return std::make_pair(ResultValue.getDef(), NewAI);
}
std::pair<Optional<SILValue>, SILLocation>
SILGenFunction::emitEpilogBB(SILLocation TopLevel) {
assert(ReturnDest.getBlock() && "no epilog bb prepared?!");
SILBasicBlock *epilogBB = ReturnDest.getBlock();
SILLocation ImplicitReturnFromTopLevel =
ImplicitReturnLocation::getImplicitReturnLoc(TopLevel);
SmallVector<SILValue, 4> directResults;
Optional<SILLocation> returnLoc = None;
// If the current BB isn't terminated, and we require a return, then we
// are not allowed to fall off the end of the function and can't reach here.
if (NeedsReturn && B.hasValidInsertionPoint())
B.createUnreachable(ImplicitReturnFromTopLevel);
if (epilogBB->pred_empty()) {
// If the epilog was not branched to at all, kill the BB and
// just emit the epilog into the current BB.
while (!epilogBB->empty())
epilogBB->back().eraseFromParent();
eraseBasicBlock(epilogBB);
// If the current bb is terminated then the epilog is just unreachable.
if (!B.hasValidInsertionPoint())
return { None, TopLevel };
// We emit the epilog at the current insertion point.
returnLoc = ImplicitReturnFromTopLevel;
} else if (std::next(epilogBB->pred_begin()) == epilogBB->pred_end()
&& !B.hasValidInsertionPoint()) {
// If the epilog has a single predecessor and there's no current insertion
// point to fall through from, then we can weld the epilog to that
// predecessor BB.
// Steal the branch argument as the return value if present.
SILBasicBlock *pred = *epilogBB->pred_begin();
BranchInst *predBranch = cast<BranchInst>(pred->getTerminator());
assert(predBranch->getArgs().size() == epilogBB->bbarg_size() &&
"epilog predecessor arguments does not match block params");
for (auto index : indices(predBranch->getArgs())) {
SILValue result = predBranch->getArgs()[index];
directResults.push_back(result);
epilogBB->getBBArg(index)->replaceAllUsesWith(result);
}
// If we are optimizing, we should use the return location from the single,
// previously processed, return statement if any.
if (predBranch->getLoc().is<ReturnLocation>()) {
returnLoc = predBranch->getLoc();
} else {
returnLoc = ImplicitReturnFromTopLevel;
}
// Kill the branch to the now-dead epilog BB.
pred->erase(predBranch);
// Move any instructions from the EpilogBB to the end of the 'pred' block.
pred->spliceAtEnd(epilogBB);
// Finally we can erase the epilog BB.
eraseBasicBlock(epilogBB);
// Emit the epilog into its former predecessor.
B.setInsertionPoint(pred);
} else {
// Move the epilog block to the end of the ordinary section.
auto endOfOrdinarySection = StartOfPostmatter;
B.moveBlockTo(epilogBB, endOfOrdinarySection);
// Emit the epilog into the epilog bb. Its arguments are the
// direct results.
directResults.append(epilogBB->bbarg_begin(), epilogBB->bbarg_end());
// If we are falling through from the current block, the return is implicit.
B.emitBlock(epilogBB, ImplicitReturnFromTopLevel);
}
// Emit top-level cleanups into the epilog block.
assert(!Cleanups.hasAnyActiveCleanups(getCleanupsDepth(),
ReturnDest.getDepth()) &&
"emitting epilog in wrong scope");
auto cleanupLoc = CleanupLocation::get(TopLevel);
Cleanups.emitCleanupsForReturn(cleanupLoc);
// If the return location is known to be that of an already
// processed return, use it. (This will get triggered when the
// epilog logic is simplified.)
//
// Otherwise make the ret instruction part of the cleanups.
if (!returnLoc) returnLoc = cleanupLoc;
// Build the return value. We don't do this if there are no direct
// results; this can happen for void functions, but also happens when
// prepareEpilog was asked to not add result arguments to the epilog
// block.
SILValue returnValue;
if (!directResults.empty()) {
assert(directResults.size()
== F.getLoweredFunctionType()->getNumDirectResults());
returnValue = buildReturnValue(*this, TopLevel, directResults);
}
return { returnValue, *returnLoc };
}
/// Insert monomorphic inline caches for a specific class or metatype
/// type \p SubClassTy.
static FullApplySite speculateMonomorphicTarget(FullApplySite AI,
SILType SubType,
CheckedCastBranchInst *&CCBI) {
CCBI = nullptr;
// Bail if this class_method cannot be devirtualized.
if (!canDevirtualizeClassMethod(AI, SubType))
return FullApplySite();
// Create a diamond shaped control flow and a checked_cast_branch
// instruction that checks the exact type of the object.
// This cast selects between two paths: one that calls the slow dynamic
// dispatch and one that calls the specific method.
auto It = AI.getInstruction()->getIterator();
SILFunction *F = AI.getFunction();
SILBasicBlock *Entry = AI.getParent();
// Iden is the basic block containing the direct call.
SILBasicBlock *Iden = F->createBasicBlock();
// Virt is the block containing the slow virtual call.
SILBasicBlock *Virt = F->createBasicBlock();
Iden->createBBArg(SubType);
SILBasicBlock *Continue = Entry->splitBasicBlock(It);
SILBuilderWithScope Builder(Entry, AI.getInstruction());
// Create the checked_cast_branch instruction that checks at runtime if the
// class instance is identical to the SILType.
ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee());
CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true,
CMI->getOperand(), SubType, Iden,
Virt);
It = CCBI->getIterator();
SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction());
SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction());
// This is the class reference downcasted into subclass SubType.
SILValue DownCastedClassInstance = Iden->getBBArg(0);
// Copy the two apply instructions into the two blocks.
FullApplySite IdenAI = CloneApply(AI, IdenBuilder);
FullApplySite VirtAI = CloneApply(AI, VirtBuilder);
// See if Continue has a release on self as the instruction right after the
// apply. If it exists, move it into position in the diamond.
if (auto *Release =
dyn_cast<StrongReleaseInst>(std::next(Continue->begin()))) {
if (Release->getOperand() == CMI->getOperand()) {
VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand());
IdenBuilder.createStrongRelease(Release->getLoc(),
DownCastedClassInstance);
Release->eraseFromParent();
}
}
// Create a PHInode for returning the return value from both apply
// instructions.
SILArgument *Arg = Continue->createBBArg(AI.getType());
if (!isa<TryApplyInst>(AI)) {
IdenBuilder.createBranch(AI.getLoc(), Continue,
ArrayRef<SILValue>(IdenAI.getInstruction()));
VirtBuilder.createBranch(AI.getLoc(), Continue,
ArrayRef<SILValue>(VirtAI.getInstruction()));
}
// Remove the old Apply instruction.
if (!isa<TryApplyInst>(AI))
AI.getInstruction()->replaceAllUsesWith(Arg);
auto *OriginalBB = AI.getParent();
AI.getInstruction()->eraseFromParent();
if (OriginalBB->empty())
OriginalBB->removeFromParent();
// Update the stats.
NumTargetsPredicted++;
// Devirtualize the apply instruction on the identical path.
auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance);
assert(NewInstPair.first && "Expected to be able to devirtualize apply!");
replaceDeadApply(IdenAI, NewInstPair.first);
// Split critical edges resulting from VirtAI.
if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) {
auto *ErrorBB = TAI->getFunction()->createBasicBlock();
ErrorBB->createBBArg(TAI->getErrorBB()->getBBArg(0)->getType());
Builder.setInsertionPoint(ErrorBB);
Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getBBArg(0)});
auto *NormalBB = TAI->getFunction()->createBasicBlock();
NormalBB->createBBArg(TAI->getNormalBB()->getBBArg(0)->getType());
Builder.setInsertionPoint(NormalBB);
Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(),
{NormalBB->getBBArg(0) });
Builder.setInsertionPoint(VirtAI.getInstruction());
SmallVector<SILValue, 4> Args;
for (auto Arg : VirtAI.getArguments()) {
Args.push_back(Arg);
}
FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(),
VirtAI.getSubstCalleeSILType(), VirtAI.getSubstitutions(),
Args, NormalBB, ErrorBB);
VirtAI.getInstruction()->eraseFromParent();
VirtAI = NewVirtAI;
}
return VirtAI;
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILModule &M = Cloned->getModule();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = new (M) SILBasicBlock(Cloned);
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->bbarg_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getBBArg(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
new (M) SILArgument(ClonedEntryBB, Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue =
new (M) SILArgument(ClonedEntryBB, PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
