/// Insert monomorphic inline caches for a specific class or metatype
/// type \p SubClassTy.
static FullApplySite speculateMonomorphicTarget(FullApplySite AI,
SILType SubType,
CheckedCastBranchInst *&CCBI) {
CCBI = nullptr;
// Bail if this class_method cannot be devirtualized.
if (!canDevirtualizeClassMethod(AI, SubType))
return FullApplySite();
if (SubType.getSwiftRValueType()->hasDynamicSelfType())
return FullApplySite();
// Create a diamond shaped control flow and a checked_cast_branch
// instruction that checks the exact type of the object.
// This cast selects between two paths: one that calls the slow dynamic
// dispatch and one that calls the specific method.
auto It = AI.getInstruction()->getIterator();
SILFunction *F = AI.getFunction();
SILBasicBlock *Entry = AI.getParent();
// Iden is the basic block containing the direct call.
SILBasicBlock *Iden = F->createBasicBlock();
// Virt is the block containing the slow virtual call.
SILBasicBlock *Virt = F->createBasicBlock();
Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned);
SILBasicBlock *Continue = Entry->split(It);
SILBuilderWithScope Builder(Entry, AI.getInstruction());
// Create the checked_cast_branch instruction that checks at runtime if the
// class instance is identical to the SILType.
ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee());
CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true,
CMI->getOperand(), SubType, Iden,
Virt);
It = CCBI->getIterator();
SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction());
SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction());
// This is the class reference downcasted into subclass SubType.
SILValue DownCastedClassInstance = Iden->getArgument(0);
// Copy the two apply instructions into the two blocks.
FullApplySite IdenAI = CloneApply(AI, IdenBuilder);
FullApplySite VirtAI = CloneApply(AI, VirtBuilder);
// See if Continue has a release on self as the instruction right after the
// apply. If it exists, move it into position in the diamond.
SILBasicBlock::iterator next =
next_or_end(Continue->begin(), Continue->end());
auto *Release =
(next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next);
if (Release && Release->getOperand() == CMI->getOperand()) {
VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(),
Release->getAtomicity());
IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance,
Release->getAtomicity());
Release->eraseFromParent();
}
// Create a PHInode for returning the return value from both apply
// instructions.
SILArgument *Arg =
Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned);
if (!isa<TryApplyInst>(AI)) {
if (AI.getSubstCalleeType()->isNoReturnFunction()) {
IdenBuilder.createUnreachable(AI.getLoc());
VirtBuilder.createUnreachable(AI.getLoc());
} else {
IdenBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(IdenAI) });
VirtBuilder.createBranch(AI.getLoc(), Continue,
{ cast<ApplyInst>(VirtAI) });
}
}
// Remove the old Apply instruction.
assert(AI.getInstruction() == &Continue->front() &&
"AI should be the first instruction in the split Continue block");
if (isa<TryApplyInst>(AI)) {
AI.getInstruction()->eraseFromParent();
assert(Continue->empty() &&
"There should not be an instruction after try_apply");
Continue->eraseFromParent();
} else {
auto apply = cast<ApplyInst>(AI);
apply->replaceAllUsesWith(Arg);
apply->eraseFromParent();
assert(!Continue->empty() &&
"There should be at least a terminator after AI");
}
// Update the stats.
NumTargetsPredicted++;
// Devirtualize the apply instruction on the identical path.
auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance);
assert(NewInstPair.first && "Expected to be able to devirtualize apply!");
replaceDeadApply(IdenAI, NewInstPair.first);
// Split critical edges resulting from VirtAI.
if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) {
auto *ErrorBB = TAI->getFunction()->createBasicBlock();
ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(ErrorBB);
Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(),
{ErrorBB->getArgument(0)});
auto *NormalBB = TAI->getFunction()->createBasicBlock();
NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(),
ValueOwnershipKind::Owned);
Builder.setInsertionPoint(NormalBB);
Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(),
{NormalBB->getArgument(0)});
Builder.setInsertionPoint(VirtAI.getInstruction());
SmallVector<SILValue, 4> Args;
for (auto Arg : VirtAI.getArguments()) {
Args.push_back(Arg);
}
FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(),
VirtAI.getSubstitutions(),
Args, NormalBB, ErrorBB);
VirtAI.getInstruction()->eraseFromParent();
VirtAI = NewVirtAI;
}
return VirtAI;
}
StackNesting::Changes StackNesting::adaptDeallocs() {
bool InstChanged = false;
bool CFGChanged = false;
BitVector Bits(StackLocs.size());
// Visit all blocks. Actuallly the order doesn't matter, but let's to it in
// the same order as in solve().
for (const BlockInfo &BI : reversed(BlockInfos)) {
// Collect the alive-bits (at the block exit) from the successor blocks.
Bits.reset();
for (BlockInfo *SuccBI : BI.Successors) {
Bits |= SuccBI->AliveStackLocsAtEntry;
}
// Insert deallocations at block boundaries.
// This can be necessary for unreachable blocks. Example:
//
// %1 = alloc_stack
// %2 = alloc_stack
// cond_br %c, bb2, bb3
// bb2: <--- need to insert a dealloc_stack %2 at the begin of bb2
// dealloc_stack %1
// unreachable
// bb3:
// dealloc_stack %2
// dealloc_stack %1
//
for (unsigned SuccIdx = 0, NumSuccs = BI.Successors.size();
SuccIdx < NumSuccs; ++ SuccIdx) {
BlockInfo *SuccBI = BI.Successors[SuccIdx];
// It's acceptable to not deallocate alive locations in unreachable
// blocks - as long as the nesting is not violated. So if there are no
// alive locations at the unreachable successor block, we can ignore it.
if (!SuccBI->ExitReachable && !SuccBI->AliveStackLocsAtEntry.any())
continue;
if (SuccBI->AliveStackLocsAtEntry == Bits)
continue;
// Insert dellocations for all locations which are alive at the end of
// the current block, but not at the begin of the successor block.
SILBasicBlock *InsertionBlock = SuccBI->Block;
if (!InsertionBlock->getSinglePredecessorBlock()) {
// If the current block is not the only predecessor of the successor
// block, we have to insert a new block where we can add the
// deallocations.
InsertionBlock = splitEdge(BI.Block->getTerminator(), SuccIdx);
CFGChanged = true;
}
InstChanged |= insertDeallocs(Bits, SuccBI->AliveStackLocsAtEntry,
&InsertionBlock->front());
}
// Insert/remove deallocations inside blocks.
for (SILInstruction *StackInst : reversed(BI.StackInsts)) {
if (StackInst->isAllocatingStack()) {
// For allocations we just update the bit-set.
int BitNr = StackLoc2BitNumbers.lookup(StackInst);
assert(Bits == StackLocs[BitNr].AliveLocs &&
"dataflow didn't converge");
Bits.reset(BitNr);
} else if (StackInst->isDeallocatingStack()) {
// Handle deallocations.
auto *AllocInst = cast<SILInstruction>(StackInst->getOperand(0));
int BitNr = StackLoc2BitNumbers.lookup(AllocInst);
SILInstruction *InsertionPoint = &*std::next(StackInst->getIterator());
if (Bits.test(BitNr)) {
// The location of StackInst is alive after StackInst. So we have to
// remove this deallocation.
StackInst->eraseFromParent();
InstChanged = true;
} else {
// Avoid inserting another deallocation for BitNr (which is already
// StackInst).
Bits.set(BitNr);
}
// Insert deallocations for all locations which are not alive after
// StackInst but _are_ alive at the StackInst.
InstChanged |= insertDeallocs(StackLocs[BitNr].AliveLocs, Bits,
InsertionPoint);
Bits |= StackLocs[BitNr].AliveLocs;
}
}
assert(Bits == BI.AliveStackLocsAtEntry && "dataflow didn't converge");
}
if (CFGChanged)
return Changes::CFG;
if (InstChanged)
return Changes::Instructions;
return Changes::None;
}
