/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary to take into consideration the removed parameters.
void
PromotedParamCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block
SILBasicBlock *OrigEntryBB = &*Orig->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
unsigned ArgNo = 0;
auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end();
while (I != E) {
if (count(PromotedArgIndices, ArgNo)) {
// Create a new argument with the promoted type.
auto boxTy = (*I)->getType().castTo<SILBoxType>();
assert(boxTy->getLayout()->getFields().size() == 1
&& "promoting multi-field boxes not implemented yet");
auto promotedTy = boxTy->getFieldType(Cloned->getModule(), 0);
auto *promotedArg =
ClonedEntryBB->createFunctionArgument(promotedTy, (*I)->getDecl());
PromotedParameters.insert(*I);
// Map any projections of the box to the promoted argument.
for (auto use : (*I)->getUses()) {
if (auto project = dyn_cast<ProjectBoxInst>(use->getUser())) {
ValueMap.insert(std::make_pair(project, promotedArg));
}
}
} else {
// Create a new argument which copies the original argument.
SILValue MappedValue = ClonedEntryBB->createFunctionArgument(
(*I)->getType(), (*I)->getDecl());
ValueMap.insert(std::make_pair(*I, MappedValue));
}
++ArgNo;
++I;
}
getBuilder().setInsertionPoint(ClonedEntryBB);
BBMap.insert(std::make_pair(OrigEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(OrigEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
getBuilder().setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary to take into consideration the removed parameters.
void
PromotedParamCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block
SILBasicBlock *OrigEntryBB = &*Orig->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
SmallVector<SILValue, 4> entryArgs;
entryArgs.reserve(OrigEntryBB->getArguments().size());
// Initialize all NewPromotedArgs slots to an invalid value.
NewPromotedArgs.resize(OrigEntryBB->getArguments().size());
unsigned ArgNo = 0;
auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end();
while (I != E) {
if (count(PromotedArgIndices, ArgNo)) {
// Create a new argument with the promoted type.
auto boxTy = (*I)->getType().castTo<SILBoxType>();
assert(boxTy->getLayout()->getFields().size() == 1
&& "promoting multi-field boxes not implemented yet");
auto promotedTy = boxTy->getFieldType(Cloned->getModule(), 0);
auto *promotedArg =
ClonedEntryBB->createFunctionArgument(promotedTy, (*I)->getDecl());
OrigPromotedParameters.insert(*I);
NewPromotedArgs[ArgNo] = promotedArg;
// All uses of the promoted box should either be projections, which are
// folded when visited, or copy/destroy operations which are ignored.
entryArgs.push_back(SILValue());
} else {
// Create a new argument which copies the original argument.
entryArgs.push_back(ClonedEntryBB->createFunctionArgument(
(*I)->getType(), (*I)->getDecl()));
}
++ArgNo;
++I;
}
// Visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions and terminators.
cloneFunctionBody(Orig, ClonedEntryBB, entryArgs);
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
SmallVector<SILValue, 4> entryArgs;
entryArgs.reserve(ClosureUserEntryBB->getArguments().size());
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
entryArgs.push_back(SILValue());
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
entryArgs.push_back(MappedValue);
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
auto ClosedOverFunConv = ClosedOverFun->getConventions();
unsigned NumTotalParams = ClosedOverFunConv.getNumParameters();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) {
auto paramTy = ClosedOverFunConv.getSILType(PInfo);
SILValue MappedValue = ClonedEntryBB->createFunctionArgument(paramTy);
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
// Clone a chain of ConvertFunctionInsts. This can create further
// reabstraction partial_apply instructions.
SmallVector<PartialApplyInst*, 4> NeedsRelease;
SILValue ConvertedCallee = cloneCalleeConversion(
CallSiteDesc.getClosureCallerArg(), NewClosure, Builder, NeedsRelease);
// Make sure that we actually emit the releases for reabstraction thunks. We
// have guaranteed earlier that we only allow reabstraction thunks if the
// closure was passed trivial.
assert(NeedsRelease.empty() || CallSiteDesc.isTrivialNoEscapeParameter());
entryArgs[CallSiteDesc.getClosureIndex()] = ConvertedCallee;
// Visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions and terminators.
cloneFunctionBody(ClosureUser, ClonedEntryBB, entryArgs);
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure(s).
bool ClosureHasRefSemantics = CallSiteDesc.closureHasRefSemanticContext();
if ((CallSiteDesc.isClosureGuaranteed() ||
CallSiteDesc.isTrivialNoEscapeParameter()) &&
(ClosureHasRefSemantics || !NeedsRelease.empty())) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = getOpBasicBlock(BB);
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have an exit, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (TI->isFunctionExiting()) {
Builder.setInsertionPoint(TI);
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
}
}
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
void FunctionSignatureTransform::createFunctionSignatureOptimizedFunction() {
// Create the optimized function !
SILModule &M = F->getModule();
std::string Name = createOptimizedSILFunctionName();
SILLinkage linkage = F->getLinkage();
if (isAvailableExternally(linkage))
linkage = SILLinkage::Shared;
DEBUG(llvm::dbgs() << " -> create specialized function " << Name << "\n");
NewF = M.createFunction(linkage, Name, createOptimizedSILFunctionType(),
F->getGenericEnvironment(), F->getLocation(),
F->isBare(), F->isTransparent(), F->isSerialized(),
F->isThunk(), F->getClassVisibility(),
F->getInlineStrategy(), F->getEffectsKind(), nullptr,
F->getDebugScope());
if (F->hasUnqualifiedOwnership()) {
NewF->setUnqualifiedOwnership();
}
// Then we transfer the body of F to NewF.
NewF->spliceBody(F);
// Array semantic clients rely on the signature being as in the original
// version.
for (auto &Attr : F->getSemanticsAttrs()) {
if (!StringRef(Attr).startswith("array."))
NewF->addSemanticsAttr(Attr);
}
// Do the last bit of work to the newly created optimized function.
ArgumentExplosionFinalizeOptimizedFunction();
DeadArgumentFinalizeOptimizedFunction();
// Create the thunk body !
F->setThunk(IsThunk);
// The thunk now carries the information on how the signature is
// optimized. If we inline the thunk, we will get the benefit of calling
// the signature optimized function without additional setup on the
// caller side.
F->setInlineStrategy(AlwaysInline);
SILBasicBlock *ThunkBody = F->createBasicBlock();
for (auto &ArgDesc : ArgumentDescList) {
ThunkBody->createFunctionArgument(ArgDesc.Arg->getType(), ArgDesc.Decl);
}
SILLocation Loc = ThunkBody->getParent()->getLocation();
SILBuilder Builder(ThunkBody);
Builder.setCurrentDebugScope(ThunkBody->getParent()->getDebugScope());
FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF);
// Create the args for the thunk's apply, ignoring any dead arguments.
llvm::SmallVector<SILValue, 8> ThunkArgs;
for (auto &ArgDesc : ArgumentDescList) {
addThunkArgument(ArgDesc, Builder, ThunkBody, ThunkArgs);
}
// We are ignoring generic functions and functions with out parameters for
// now.
SILValue ReturnValue;
SILType LoweredType = NewF->getLoweredType();
SILType ResultType = NewF->getConventions().getSILResultType();
auto FunctionTy = LoweredType.castTo<SILFunctionType>();
if (FunctionTy->hasErrorResult()) {
// We need a try_apply to call a function with an error result.
SILFunction *Thunk = ThunkBody->getParent();
SILBasicBlock *NormalBlock = Thunk->createBasicBlock();
ReturnValue =
NormalBlock->createPHIArgument(ResultType, ValueOwnershipKind::Owned);
SILBasicBlock *ErrorBlock = Thunk->createBasicBlock();
SILType Error =
SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType());
auto *ErrorArg =
ErrorBlock->createPHIArgument(Error, ValueOwnershipKind::Owned);
Builder.createTryApply(Loc, FRI, LoweredType, SubstitutionList(),
ThunkArgs, NormalBlock, ErrorBlock);
Builder.setInsertionPoint(ErrorBlock);
Builder.createThrow(Loc, ErrorArg);
Builder.setInsertionPoint(NormalBlock);
} else {
ReturnValue = Builder.createApply(Loc, FRI, LoweredType, ResultType,
SubstitutionList(), ThunkArgs,
false);
}
// Set up the return results.
if (NewF->isNoReturnFunction()) {
Builder.createUnreachable(Loc);
} else {
Builder.createReturn(Loc, ReturnValue);
}
// Do the last bit work to finalize the thunk.
OwnedToGuaranteedFinalizeThunkFunction(Builder, F);
assert(F->getDebugScope()->Parent != NewF->getDebugScope()->Parent);
}
void GenericCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block.
SILBasicBlock *OrigEntryBB = &*Original.begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
getBuilder().setInsertionPoint(ClonedEntryBB);
llvm::SmallVector<AllocStackInst *, 8> AllocStacks;
AllocStackInst *ReturnValueAddr = nullptr;
// Create the entry basic block with the function arguments.
auto origConv = Original.getConventions();
unsigned ArgIdx = 0;
for (auto &OrigArg : OrigEntryBB->getArguments()) {
RegularLocation Loc((Decl *)OrigArg->getDecl());
AllocStackInst *ASI = nullptr;
SILType mappedType = remapType(OrigArg->getType());
auto createAllocStack = [&]() {
// We need an alloc_stack as a replacement for the indirect parameter.
assert(mappedType.isAddress());
mappedType = mappedType.getObjectType();
ASI = getBuilder().createAllocStack(Loc, mappedType);
ValueMap[OrigArg] = ASI;
AllocStacks.push_back(ASI);
};
auto handleConversion = [&]() {
if (!origConv.useLoweredAddresses())
return false;
if (ArgIdx < origConv.getSILArgIndexOfFirstParam()) {
// Handle result arguments.
unsigned formalIdx =
origConv.getIndirectFormalResultIndexForSILArg(ArgIdx);
if (ReInfo.isFormalResultConverted(formalIdx)) {
// This result is converted from indirect to direct. The return inst
// needs to load the value from the alloc_stack. See below.
createAllocStack();
assert(!ReturnValueAddr);
ReturnValueAddr = ASI;
return true;
}
} else {
// Handle arguments for formal parameters.
unsigned paramIdx = ArgIdx - origConv.getSILArgIndexOfFirstParam();
if (ReInfo.isParamConverted(paramIdx)) {
// Store the new direct parameter to the alloc_stack.
createAllocStack();
auto *NewArg = ClonedEntryBB->createFunctionArgument(
mappedType, OrigArg->getDecl());
getBuilder().createStore(Loc, NewArg, ASI,
StoreOwnershipQualifier::Unqualified);
// Try to create a new debug_value from an existing debug_value_addr.
for (Operand *ArgUse : OrigArg->getUses()) {
if (auto *DVAI = dyn_cast<DebugValueAddrInst>(ArgUse->getUser())) {
getBuilder().setCurrentDebugScope(remapScope(DVAI->getDebugScope()));
getBuilder().createDebugValue(DVAI->getLoc(), NewArg,
DVAI->getVarInfo());
getBuilder().setCurrentDebugScope(nullptr);
break;
}
}
return true;
}
}
return false; // No conversion.
};
if (!handleConversion()) {
auto *NewArg =
ClonedEntryBB->createFunctionArgument(mappedType, OrigArg->getDecl());
ValueMap[OrigArg] = NewArg;
}
++ArgIdx;
}
BBMap.insert(std::make_pair(OrigEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(OrigEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
getBuilder().setInsertionPoint(BI->second);
TermInst *OrigTermInst = BI->first->getTerminator();
if (auto *RI = dyn_cast<ReturnInst>(OrigTermInst)) {
SILValue ReturnValue;
if (ReturnValueAddr) {
// The result is converted from indirect to direct. We have to load the
// returned value from the alloc_stack.
ReturnValue =
getBuilder().createLoad(ReturnValueAddr->getLoc(), ReturnValueAddr,
LoadOwnershipQualifier::Unqualified);
}
for (AllocStackInst *ASI : reverse(AllocStacks)) {
getBuilder().createDeallocStack(ASI->getLoc(), ASI);
}
if (ReturnValue) {
getBuilder().createReturn(RI->getLoc(), ReturnValue);
continue;
}
} else if (isa<ThrowInst>(OrigTermInst)) {
for (AllocStackInst *ASI : reverse(AllocStacks)) {
getBuilder().createDeallocStack(ASI->getLoc(), ASI);
}
}
visit(BI->first->getTerminator());
}
}
