void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() {
SILBasicBlock *BB = &*NewF->begin();
SILBuilder Builder(BB->begin());
Builder.setCurrentDebugScope(BB->getParent()->getDebugScope());
unsigned TotalArgIndex = 0;
for (ArgumentDescriptor &AD : ArgumentDescList) {
// Simply continue if do not explode.
if (!AD.Explode) {
AIM[TotalArgIndex] = AD.Index;
TotalArgIndex ++;
continue;
}
// OK, we need to explode this argument.
unsigned ArgOffset = ++TotalArgIndex;
unsigned OldArgIndex = ArgOffset - 1;
llvm::SmallVector<SILValue, 8> LeafValues;
// We do this in the same order as leaf types since ProjTree expects that the
// order of leaf values matches the order of leaf types.
llvm::SmallVector<const ProjectionTreeNode*, 8> LeafNodes;
AD.ProjTree.getLeafNodes(LeafNodes);
for (auto *Node : LeafNodes) {
auto OwnershipKind = *AD.getTransformedOwnershipKind(Node->getType());
LeafValues.push_back(BB->insertFunctionArgument(
ArgOffset++, Node->getType(), OwnershipKind,
BB->getArgument(OldArgIndex)->getDecl()));
AIM[TotalArgIndex - 1] = AD.Index;
TotalArgIndex ++;
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
AD.ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(),
LeafValues);
// We ignored debugvalue uses when we constructed the new arguments, in order
// to preserve as much information as possible, we construct a new value for
// OrigArg from the leaf values and use that in place of the OrigArg.
SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue(Builder,
BB->getParent()->getLocation(),
LeafValues);
// Replace all uses of the original arg with the new value.
SILArgument *OrigArg = BB->getArgument(OldArgIndex);
OrigArg->replaceAllUsesWith(NewOrigArgValue);
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseArgument(OldArgIndex);
TotalArgIndex --;
}
}
SILValue
StackAllocationPromoter::getLiveOutValue(BlockSet &PhiBlocks,
SILBasicBlock *StartBB) {
DEBUG(llvm::dbgs() << "*** Searching for a value definition.\n");
// Walk the Dom tree in search of a defining value:
for (DomTreeNode *Node = DT->getNode(StartBB); Node; Node = Node->getIDom()) {
SILBasicBlock *BB = Node->getBlock();
// If there is a store (that must come after the phi), use its value.
BlockToInstMap::iterator it = LastStoreInBlock.find(BB);
if (it != LastStoreInBlock.end())
if (auto *St = dyn_cast_or_null<StoreInst>(it->second)) {
DEBUG(llvm::dbgs() << "*** Found Store def " << *St->getSrc());
return St->getSrc();
}
// If there is a Phi definition in this block:
if (PhiBlocks.count(BB)) {
// Return the dummy instruction that represents the new value that we will
// add to the basic block.
SILValue Phi = BB->getArgument(BB->getNumArguments() - 1);
DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *Phi);
return Phi;
}
// Move to the next dominating block.
DEBUG(llvm::dbgs() << "*** Walking up the iDOM.\n");
}
DEBUG(llvm::dbgs() << "*** Could not find a Def. Using Undef.\n");
return SILUndef::get(ASI->getElementType(), ASI->getModule());
}
static TryApplyInst *replaceTryApplyInst(SILBuilder &B, SILLocation Loc,
TryApplyInst *OldTAI,
SILValue NewFn,
SubstitutionMap NewSubs,
ArrayRef<SILValue> NewArgs,
SILFunctionConventions Conv) {
SILBasicBlock *NormalBB = OldTAI->getNormalBB();
SILBasicBlock *ResultBB = nullptr;
SILType NewResultTy = Conv.getSILResultType();
// Does the result value need to be casted?
auto OldResultTy = NormalBB->getArgument(0)->getType();
bool ResultCastRequired = NewResultTy != OldResultTy;
// Create a new normal BB only if the result of the new apply differs
// in type from the argument of the original normal BB.
if (!ResultCastRequired) {
ResultBB = NormalBB;
} else {
ResultBB = B.getFunction().createBasicBlockBefore(NormalBB);
ResultBB->createPHIArgument(NewResultTy, ValueOwnershipKind::Owned);
}
// We can always just use the original error BB because we'll be
// deleting the edge to it from the old TAI.
SILBasicBlock *ErrorBB = OldTAI->getErrorBB();
// Insert a try_apply here.
// Note that this makes this block temporarily double-terminated!
// We won't fix that until deleteDevirtualizedApply.
auto NewTAI = B.createTryApply(Loc, NewFn, NewSubs, NewArgs,
ResultBB, ErrorBB);
if (ResultCastRequired) {
B.setInsertionPoint(ResultBB);
SILValue ResultValue = ResultBB->getArgument(0);
ResultValue = castValueToABICompatibleType(&B, Loc, ResultValue,
NewResultTy, OldResultTy);
B.createBranch(Loc, NormalBB, { ResultValue });
}
B.setInsertionPoint(NormalBB->begin());
return NewTAI;
}
void FunctionSignatureTransform::DeadArgumentTransformFunction() {
SILBasicBlock *BB = &*F->begin();
for (const ArgumentDescriptor &AD : ArgumentDescList) {
if (!AD.IsEntirelyDead)
continue;
eraseUsesOfValue(BB->getArgument(AD.Index));
}
}
/// At least one value feeding the specified SILArgument is a Struct. Attempt to
/// replace the Argument with a new Struct in the same block.
///
/// When we handle more types of casts, this can become a template.
///
/// ArgValues are the values feeding the specified Argument from each
/// predecessor. They must be listed in order of Arg->getParent()->getPreds().
static StructInst *
replaceBBArgWithStruct(SILPhiArgument *Arg,
SmallVectorImpl<SILValue> &ArgValues) {
SILBasicBlock *PhiBB = Arg->getParent();
auto *FirstSI = dyn_cast<StructInst>(ArgValues[0]);
if (!FirstSI)
return nullptr;
// Collect the BBArg index of each struct oper.
// e.g.
// struct(A, B)
// br (B, A)
// : ArgIdxForOper => {1, 0}
SmallVector<unsigned, 4> ArgIdxForOper;
for (unsigned OperIdx : indices(FirstSI->getElements())) {
bool FoundMatchingArgIdx = false;
for (unsigned ArgIdx : indices(PhiBB->getArguments())) {
SmallVectorImpl<SILValue>::const_iterator AVIter = ArgValues.begin();
bool TryNextArgIdx = false;
for (SILBasicBlock *PredBB : PhiBB->getPredecessorBlocks()) {
// All argument values must be StructInst.
auto *PredSI = dyn_cast<StructInst>(*AVIter++);
if (!PredSI)
return nullptr;
OperandValueArrayRef EdgeValues =
getEdgeValuesForTerminator(PredBB->getTerminator(), PhiBB);
if (EdgeValues[ArgIdx] != PredSI->getElements()[OperIdx]) {
TryNextArgIdx = true;
break;
}
}
if (!TryNextArgIdx) {
assert(AVIter == ArgValues.end() && "# ArgValues does not match # BB preds");
FoundMatchingArgIdx = true;
ArgIdxForOper.push_back(ArgIdx);
break;
}
}
if (!FoundMatchingArgIdx)
return nullptr;
}
SmallVector<SILValue, 4> StructArgs;
for (auto ArgIdx : ArgIdxForOper)
StructArgs.push_back(PhiBB->getArgument(ArgIdx));
SILBuilder Builder(PhiBB, PhiBB->begin());
return Builder.createStruct(cast<StructInst>(ArgValues[0])->getLoc(),
Arg->getType(), StructArgs);
}
bool OwnershipModelEliminatorVisitor::visitCheckedCastBranchInst(
CheckedCastBranchInst *CBI) {
// In ownership qualified SIL, checked_cast_br must pass its argument to the
// fail case so we can clean it up. In non-ownership qualified SIL, we expect
// no argument from the checked_cast_br in the default case. The way that we
// handle this transformation is that:
//
// 1. We replace all uses of the argument to the false block with a use of the
// checked cast branch's operand.
// 2. We delete the argument from the false block.
SILBasicBlock *FailureBlock = CBI->getFailureBB();
if (FailureBlock->getNumArguments() == 0)
return false;
FailureBlock->getArgument(0)->replaceAllUsesWith(CBI->getOperand());
FailureBlock->eraseArgument(0);
return true;
}
bool OwnershipModelEliminatorVisitor::visitSwitchEnumInst(
SwitchEnumInst *SWEI) {
// In ownership qualified SIL, switch_enum must pass its argument to the fail
// case so we can clean it up. In non-ownership qualified SIL, we expect no
// argument from the switch_enum in the default case. The way that we handle
// this transformation is that:
//
// 1. We replace all uses of the argument to the false block with a use of the
// checked cast branch's operand.
// 2. We delete the argument from the false block.
if (!SWEI->hasDefault())
return false;
SILBasicBlock *DefaultBlock = SWEI->getDefaultBB();
if (DefaultBlock->getNumArguments() == 0)
return false;
DefaultBlock->getArgument(0)->replaceAllUsesWith(SWEI->getOperand());
DefaultBlock->eraseArgument(0);
return true;
}
void
FunctionSignatureTransform::
OwnedToGuaranteedAddResultRelease(ResultDescriptor &RD, SILBuilder &Builder,
SILFunction *F) {
// If we have any result that were consumed but are now guaranteed,
// insert a release_value.
if (!RD.OwnedToGuaranteed) {
return;
}
SILInstruction *Call = findOnlyApply(F);
if (isa<ApplyInst>(Call)) {
Builder.setInsertionPoint(&*std::next(SILBasicBlock::iterator(Call)));
Builder.createRetainValue(RegularLocation(SourceLoc()), Call,
Builder.getDefaultAtomicity());
} else {
SILBasicBlock *NormalBB = dyn_cast<TryApplyInst>(Call)->getNormalBB();
Builder.setInsertionPoint(&*NormalBB->begin());
Builder.createRetainValue(RegularLocation(SourceLoc()),
NormalBB->getArgument(0), Builder.getDefaultAtomicity());
}
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
SmallVector<SILValue, 4> entryArgs;
entryArgs.reserve(ClosureUserEntryBB->getArguments().size());
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
entryArgs.push_back(SILValue());
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
entryArgs.push_back(MappedValue);
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
auto ClosedOverFunConv = ClosedOverFun->getConventions();
unsigned NumTotalParams = ClosedOverFunConv.getNumParameters();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) {
auto paramTy = ClosedOverFunConv.getSILType(PInfo);
SILValue MappedValue = ClonedEntryBB->createFunctionArgument(paramTy);
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
// Clone a chain of ConvertFunctionInsts. This can create further
// reabstraction partial_apply instructions.
SmallVector<PartialApplyInst*, 4> NeedsRelease;
SILValue ConvertedCallee = cloneCalleeConversion(
CallSiteDesc.getClosureCallerArg(), NewClosure, Builder, NeedsRelease);
// Make sure that we actually emit the releases for reabstraction thunks. We
// have guaranteed earlier that we only allow reabstraction thunks if the
// closure was passed trivial.
assert(NeedsRelease.empty() || CallSiteDesc.isTrivialNoEscapeParameter());
entryArgs[CallSiteDesc.getClosureIndex()] = ConvertedCallee;
// Visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions and terminators.
cloneFunctionBody(ClosureUser, ClonedEntryBB, entryArgs);
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure(s).
bool ClosureHasRefSemantics = CallSiteDesc.closureHasRefSemanticContext();
if ((CallSiteDesc.isClosureGuaranteed() ||
CallSiteDesc.isTrivialNoEscapeParameter()) &&
(ClosureHasRefSemantics || !NeedsRelease.empty())) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = getOpBasicBlock(BB);
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have an exit, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (TI->isFunctionExiting()) {
Builder.setInsertionPoint(TI);
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
}
}
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
static bool matchSwitch(SwitchInfo &SI, SILInstruction *Inst,
SILValue SwitchOperand) {
auto *SwitchEnum = dyn_cast<SwitchEnumInst>(Inst);
if (!SwitchEnum || SwitchEnum->getNumCases() != 2 ||
SwitchEnum->getOperand() != SwitchOperand)
return false;
auto *SwitchBB = SwitchEnum->getParent();
SILBasicBlock *SomeBB = SwitchEnum->getCase(0).second;
SILBasicBlock *NoneBB = SwitchEnum->getCase(1).second;
if (NoneBB->getSinglePredecessorBlock() != SwitchBB)
return false;
if (SomeBB->getSinglePredecessorBlock() != SwitchBB)
return false;
if (NoneBB->args_size() == 1)
std::swap(NoneBB, SomeBB);
if (SomeBB->args_size() != 1 || NoneBB->args_size() != 0)
return false;
// bb9:
// %43 = enum $Optional<String>, #Optional.none!enumelt
auto It = NoneBB->begin();
auto *NoneEnum = dyn_cast<EnumInst>(It);
if (!NoneEnum || NoneEnum->hasOperand() || !NoneEnum->hasOneUse())
return false;
// br bb10(%43 : $Optional<String>)
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *Br1 = dyn_cast<BranchInst>(It);
if (!Br1 || Br1->getNumArgs() != 1 || Br1->getArg(0) != NoneEnum)
return false;
auto *MergeBB = Br1->getDestBB();
// bb8(%36 : $NSString):
It = SomeBB->begin();
auto *SomeBBArg = SomeBB->getArgument(0);
if (!SomeBBArg->hasOneUse())
return false;
// %37 = function_ref @$SSS10FoundationE36_unconditionallyBridgeFromObjectiveCSSSo8NSStringCSgFZ : $@convention(method) (@owned Optional<NSString>, @thin String.Type) -> @owned String
auto *FunRef = dyn_cast<FunctionRefInst>(It);
if (!FunRef || !FunRef->hasOneUse())
return false;
// %38 = enum $Optional<NSString>, #Optional.some!enumelt.1, %36 : $NSString
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *SomeEnum = dyn_cast<EnumInst>(It);
if (!SomeEnum || !SomeEnum->hasOperand() || SomeEnum->getOperand() != SomeBBArg)
return false;
size_t numSomeEnumUses = std::distance(SomeEnum->use_begin(), SomeEnum->use_end());
if (numSomeEnumUses > 2)
return false;
// %39 = metatype $@thin String.Type
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *Metatype = dyn_cast<MetatypeInst>(It);
if (!Metatype || !Metatype->hasOneUse())
return false;
// %40 = apply %37(%38, %39) : $@convention(method) (@owned Optional<NSString>, @thin String.Type) -> @owned String
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *Apply = dyn_cast<ApplyInst>(It);
if (!Apply || !Apply->hasOneUse() || Apply->getCallee() != FunRef ||
Apply->getNumArguments() != 2 || Apply->getArgument(0) != SomeEnum ||
Apply->getArgument(1) != Metatype ||
Apply->getSubstCalleeType()->getNumResults() != 1)
return false;
if (Apply->getSubstCalleeType()->getSingleResult().getConvention() !=
ResultConvention::Owned)
return false;
// Check that we call the _unconditionallyBridgeFromObjectiveC witness.
auto NativeType = Apply->getType().getASTType();
auto *BridgeFun = FunRef->getReferencedFunction();
auto *SwiftModule = BridgeFun->getModule().getSwiftModule();
auto bridgeWitness = getBridgeFromObjectiveC(NativeType, SwiftModule);
if (BridgeFun->getName() != bridgeWitness.mangle())
return false;
// %41 = enum $Optional<String>, #Optional.some!enumelt.1, %40 : $String
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *Enum3 = dyn_cast<EnumInst>(It);
if (!Enum3 || !Enum3->hasOneUse() || !Enum3->hasOperand() ||
Enum3->getOperand() != Apply)
return false;
if (numSomeEnumUses == 2) {
// release_value %38 : $Optional<NSString>
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *RVI = dyn_cast<ReleaseValueInst>(It);
if (!RVI || RVI->getOperand() != SomeEnum)
return false;
}
// br bb10(%41 : $Optional<String>)
ADVANCE_ITERATOR_OR_RETURN_FALSE(It);
auto *Br = dyn_cast<BranchInst>(It);
if (!Br || Br->getDestBB() != MergeBB || Br->getNumArgs() != 1 ||
Br->getArg(0) != Enum3)
return false;
SI.SwitchEnum = SwitchEnum;
SI.SomeBB = SomeBB;
SI.NoneBB = NoneBB;
SI.Br = Br;
return true;
}
/// Insert monomorphic inline caches for a specific class or metatype
/// type \p SubClassTy.
static FullApplySite speculateMonomorphicTarget(FullApplySite AI,
SILType SubType,
CheckedCastBranchInst *&CCBI) {
CCBI = nullptr;
// Bail if this class_method cannot be devirtualized.
if (!canDevirtualizeClassMethod(AI, SubType))
return FullApplySite();
if (SubType.getSwiftRValueType()->hasDynamicSelfType())
return FullApplySite();
// Create a diamond shaped control flow and a checked_cast_branch
// instruction that checks the exact type of the object.
// This cast selects between two paths: one that calls the slow dynamic
// dispatch and one that calls the specific method.
auto It = AI.getInstruction()->getIterator();
SILFunction *F = AI.getFunction();
SILBasicBlock *Entry = AI.getParent();
// Iden is the basic block containing the direct call.
SILBasicBlock *Iden = F->createBasicBlock();
// Virt is the block containing the slow virtual call.
SILBasicBlock *Virt = F->createBasicBlock();
Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned);
SILBasicBlock *Continue = Entry->split(It);
SILBuilderWithScope Builder(Entry, AI.getInstruction());
// Create the checked_cast_branch instruction that checks at runtime if the
// class instance is identical to the SILType.
ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee());
CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true,
CMI->getOperand(), SubType, Iden,
Virt);
It = CCBI->getIterator();
SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction());
SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction());
// This is the class reference downcasted into subclass SubType.
SILValue DownCastedClassInstance = Iden->getArgument(0);
// Copy the two apply instructions into the two blocks.
FullApplySite IdenAI = CloneApply(AI, IdenBuilder);
FullApplySite VirtAI = CloneApply(AI, VirtBuilder);
// See if Continue has a release on self as the instruction right after the
// apply. If it exists, move it into position in the diamond.
SILBasicBlock::iterator next =
next_or_end(Continue->begin(), Continue->end());
auto *Release =
(next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next);
if (Release && Release->getOperand() == CMI->getOperand()) {
VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(),
Release->getAtomicity());
IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance,
Release->getAtomicity());
Release->eraseFromParent();
}
// Create a PHInode for returning the return value from both apply
// instructions.
SILArgument *Arg =
Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned);
if (!isa<TryApplyInst>(AI)) {
if (AI.getSubstCalleeType()->isNoReturnFunction()) {
IdenBuilder.createUnreachable(AI.getLoc());
VirtBuilder.createUnreachable(AI.getLoc());
} else {
IdenBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(IdenAI) });
VirtBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(VirtAI) });
}
}
// Remove the old Apply instruction.
assert(AI.getInstruction() == &Continue->front() &&
"AI should be the first instruction in the split Continue block");
if (isa<TryApplyInst>(AI)) {
AI.getInstruction()->eraseFromParent();
assert(Continue->empty() &&
"There should not be an instruction after try_apply");
Continue->eraseFromParent();
} else {
auto apply = cast<ApplyInst>(AI);
apply->replaceAllUsesWith(Arg);
apply->eraseFromParent();
assert(!Continue->empty() &&
"There should be at least a terminator after AI");
}
// Update the stats.
NumTargetsPredicted++;
// Devirtualize the apply instruction on the identical path.
auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance);
assert(NewInstPair.first && "Expected to be able to devirtualize apply!");
replaceDeadApply(IdenAI, NewInstPair.first);
// Split critical edges resulting from VirtAI.
if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) {
auto *ErrorBB = TAI->getFunction()->createBasicBlock();
ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(ErrorBB);
Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getArgument(0)});
auto *NormalBB = TAI->getFunction()->createBasicBlock();
NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(NormalBB);
Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(),
{NormalBB->getArgument(0)});
Builder.setInsertionPoint(VirtAI.getInstruction());
SmallVector<SILValue, 4> Args;
for (auto Arg : VirtAI.getArguments()) {
Args.push_back(Arg);
}
FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(),
VirtAI.getSubstitutions(),
Args, NormalBB, ErrorBB);
VirtAI.getInstruction()->eraseFromParent();
VirtAI = NewVirtAI;
}
return VirtAI;
}
std::pair<Optional<SILValue>, SILLocation>
SILGenFunction::emitEpilogBB(SILLocation TopLevel) {
assert(ReturnDest.getBlock() && "no epilog bb prepared?!");
SILBasicBlock *epilogBB = ReturnDest.getBlock();
SILLocation ImplicitReturnFromTopLevel =
ImplicitReturnLocation::getImplicitReturnLoc(TopLevel);
SmallVector<SILValue, 4> directResults;
Optional<SILLocation> returnLoc = None;
// If the current BB isn't terminated, and we require a return, then we
// are not allowed to fall off the end of the function and can't reach here.
if (NeedsReturn && B.hasValidInsertionPoint())
B.createUnreachable(ImplicitReturnFromTopLevel);
if (epilogBB->pred_empty()) {
// If the epilog was not branched to at all, kill the BB and
// just emit the epilog into the current BB.
while (!epilogBB->empty())
epilogBB->back().eraseFromParent();
eraseBasicBlock(epilogBB);
// If the current bb is terminated then the epilog is just unreachable.
if (!B.hasValidInsertionPoint())
return { None, TopLevel };
// We emit the epilog at the current insertion point.
returnLoc = ImplicitReturnFromTopLevel;
} else if (std::next(epilogBB->pred_begin()) == epilogBB->pred_end()
&& !B.hasValidInsertionPoint()) {
// If the epilog has a single predecessor and there's no current insertion
// point to fall through from, then we can weld the epilog to that
// predecessor BB.
// Steal the branch argument as the return value if present.
SILBasicBlock *pred = *epilogBB->pred_begin();
BranchInst *predBranch = cast<BranchInst>(pred->getTerminator());
assert(predBranch->getArgs().size() == epilogBB->args_size() &&
"epilog predecessor arguments does not match block params");
for (auto index : indices(predBranch->getArgs())) {
SILValue result = predBranch->getArgs()[index];
directResults.push_back(result);
epilogBB->getArgument(index)->replaceAllUsesWith(result);
}
// If we are optimizing, we should use the return location from the single,
// previously processed, return statement if any.
if (predBranch->getLoc().is<ReturnLocation>()) {
returnLoc = predBranch->getLoc();
} else {
returnLoc = ImplicitReturnFromTopLevel;
}
// Kill the branch to the now-dead epilog BB.
pred->erase(predBranch);
// Move any instructions from the EpilogBB to the end of the 'pred' block.
pred->spliceAtEnd(epilogBB);
// Finally we can erase the epilog BB.
eraseBasicBlock(epilogBB);
// Emit the epilog into its former predecessor.
B.setInsertionPoint(pred);
} else {
// Move the epilog block to the end of the ordinary section.
auto endOfOrdinarySection = StartOfPostmatter;
B.moveBlockTo(epilogBB, endOfOrdinarySection);
// Emit the epilog into the epilog bb. Its arguments are the
// direct results.
directResults.append(epilogBB->args_begin(), epilogBB->args_end());
// If we are falling through from the current block, the return is implicit.
B.emitBlock(epilogBB, ImplicitReturnFromTopLevel);
}
// Emit top-level cleanups into the epilog block.
assert(!Cleanups.hasAnyActiveCleanups(getCleanupsDepth(),
ReturnDest.getDepth()) &&
"emitting epilog in wrong scope");
auto cleanupLoc = CleanupLocation::get(TopLevel);
Cleanups.emitCleanupsForReturn(cleanupLoc);
// If the return location is known to be that of an already
// processed return, use it. (This will get triggered when the
// epilog logic is simplified.)
//
// Otherwise make the ret instruction part of the cleanups.
if (!returnLoc) returnLoc = cleanupLoc;
// Build the return value. We don't do this if there are no direct
// results; this can happen for void functions, but also happens when
// prepareEpilog was asked to not add result arguments to the epilog
// block.
SILValue returnValue;
if (!directResults.empty()) {
assert(directResults.size() == F.getConventions().getNumDirectSILResults());
returnValue = buildReturnValue(*this, TopLevel, directResults);
}
return { returnValue, *returnLoc };
}
/// \brief Devirtualize an apply of a class method.
///
/// \p AI is the apply to devirtualize.
/// \p ClassOrMetatype is a class value or metatype value that is the
/// self argument of the apply we will devirtualize.
/// return the result value of the new ApplyInst if created one or null.
DevirtualizationResult swift::devirtualizeClassMethod(FullApplySite AI,
SILValue ClassOrMetatype) {
DEBUG(llvm::dbgs() << " Trying to devirtualize : " << *AI.getInstruction());
SILModule &Mod = AI.getModule();
auto *MI = cast<MethodInst>(AI.getCallee());
auto ClassOrMetatypeType = ClassOrMetatype->getType();
auto *F = getTargetClassMethod(Mod, ClassOrMetatypeType, MI);
CanSILFunctionType GenCalleeType = F->getLoweredFunctionType();
SmallVector<Substitution, 4> Subs;
getSubstitutionsForCallee(Mod, GenCalleeType,
ClassOrMetatypeType.getSwiftRValueType(),
AI, Subs);
CanSILFunctionType SubstCalleeType = GenCalleeType;
if (GenCalleeType->isPolymorphic())
SubstCalleeType = GenCalleeType->substGenericArgs(Mod, Subs);
SILFunctionConventions substConv(SubstCalleeType, Mod);
SILBuilderWithScope B(AI.getInstruction());
FunctionRefInst *FRI = B.createFunctionRef(AI.getLoc(), F);
// Create the argument list for the new apply, casting when needed
// in order to handle covariant indirect return types and
// contravariant argument types.
llvm::SmallVector<SILValue, 8> NewArgs;
auto IndirectResultArgIter = AI.getIndirectSILResults().begin();
for (auto ResultTy : substConv.getIndirectSILResultTypes()) {
NewArgs.push_back(
castValueToABICompatibleType(&B, AI.getLoc(), *IndirectResultArgIter,
IndirectResultArgIter->getType(), ResultTy));
++IndirectResultArgIter;
}
auto ParamArgIter = AI.getArgumentsWithoutIndirectResults().begin();
// Skip the last parameter, which is `self`. Add it below.
for (auto param : substConv.getParameters().drop_back()) {
auto paramType = substConv.getSILType(param);
NewArgs.push_back(
castValueToABICompatibleType(&B, AI.getLoc(), *ParamArgIter,
ParamArgIter->getType(), paramType));
++ParamArgIter;
}
// Add the self argument, upcasting if required because we're
// calling a base class's method.
auto SelfParamTy = substConv.getSILType(SubstCalleeType->getSelfParameter());
NewArgs.push_back(castValueToABICompatibleType(&B, AI.getLoc(),
ClassOrMetatype,
ClassOrMetatypeType,
SelfParamTy));
SILType ResultTy = substConv.getSILResultType();
SILType SubstCalleeSILType =
SILType::getPrimitiveObjectType(SubstCalleeType);
FullApplySite NewAI;
SILBasicBlock *ResultBB = nullptr;
SILBasicBlock *NormalBB = nullptr;
SILValue ResultValue;
bool ResultCastRequired = false;
SmallVector<Operand *, 4> OriginalResultUses;
if (!isa<TryApplyInst>(AI)) {
NewAI = B.createApply(AI.getLoc(), FRI, SubstCalleeSILType, ResultTy,
Subs, NewArgs, cast<ApplyInst>(AI)->isNonThrowing());
ResultValue = NewAI.getInstruction();
} else {
auto *TAI = cast<TryApplyInst>(AI);
// Create new normal and error BBs only if:
// - re-using a BB would create a critical edge
// - or, the result of the new apply would be of different
// type than the argument of the original normal BB.
if (TAI->getNormalBB()->getSinglePredecessorBlock())
ResultBB = TAI->getNormalBB();
else {
ResultBB = B.getFunction().createBasicBlock();
ResultBB->createPHIArgument(ResultTy, ValueOwnershipKind::Owned);
}
NormalBB = TAI->getNormalBB();
SILBasicBlock *ErrorBB = nullptr;
if (TAI->getErrorBB()->getSinglePredecessorBlock())
ErrorBB = TAI->getErrorBB();
else {
ErrorBB = B.getFunction().createBasicBlock();
ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
}
NewAI = B.createTryApply(AI.getLoc(), FRI, SubstCalleeSILType,
Subs, NewArgs,
ResultBB, ErrorBB);
if (ErrorBB != TAI->getErrorBB()) {
B.setInsertionPoint(ErrorBB);
B.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getArgument(0)});
}
// Does the result value need to be casted?
ResultCastRequired = ResultTy != NormalBB->getArgument(0)->getType();
if (ResultBB != NormalBB)
B.setInsertionPoint(ResultBB);
else if (ResultCastRequired) {
B.setInsertionPoint(NormalBB->begin());
// Collect all uses, before casting.
for (auto *Use : NormalBB->getArgument(0)->getUses()) {
OriginalResultUses.push_back(Use);
}
NormalBB->getArgument(0)->replaceAllUsesWith(
SILUndef::get(AI.getType(), Mod));
NormalBB->replacePHIArgument(0, ResultTy, ValueOwnershipKind::Owned);
}
// The result value is passed as a parameter to the normal block.
ResultValue = ResultBB->getArgument(0);
}
// Check if any casting is required for the return value.
ResultValue = castValueToABICompatibleType(&B, NewAI.getLoc(), ResultValue,
ResultTy, AI.getType());
DEBUG(llvm::dbgs() << " SUCCESS: " << F->getName() << "\n");
NumClassDevirt++;
if (NormalBB) {
if (NormalBB != ResultBB) {
// If artificial normal BB was introduced, branch
// to the original normal BB.
B.createBranch(NewAI.getLoc(), NormalBB, { ResultValue });
} else if (ResultCastRequired) {
// Update all original uses by the new value.
for (auto *Use: OriginalResultUses) {
Use->set(ResultValue);
}
}
return std::make_pair(NewAI.getInstruction(), NewAI);
}
// We need to return a pair of values here:
// - the first one is the actual result of the devirtualized call, possibly
// casted into an appropriate type. This SILValue may be a BB arg, if it
// was a cast between optional types.
// - the second one is the new apply site.
return std::make_pair(ResultValue, NewAI);
}
