/// Insert monomorphic inline caches for a specific class or metatype /// type \p SubClassTy. static FullApplySite speculateMonomorphicTarget(FullApplySite AI, SILType SubType, CheckedCastBranchInst *&CCBI) { CCBI = nullptr; // Bail if this class_method cannot be devirtualized. if (!canDevirtualizeClassMethod(AI, SubType)) return FullApplySite(); if (SubType.getSwiftRValueType()->hasDynamicSelfType()) return FullApplySite(); // Create a diamond shaped control flow and a checked_cast_branch // instruction that checks the exact type of the object. // This cast selects between two paths: one that calls the slow dynamic // dispatch and one that calls the specific method. auto It = AI.getInstruction()->getIterator(); SILFunction *F = AI.getFunction(); SILBasicBlock *Entry = AI.getParent(); // Iden is the basic block containing the direct call. SILBasicBlock *Iden = F->createBasicBlock(); // Virt is the block containing the slow virtual call. SILBasicBlock *Virt = F->createBasicBlock(); Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned); SILBasicBlock *Continue = Entry->split(It); SILBuilderWithScope Builder(Entry, AI.getInstruction()); // Create the checked_cast_branch instruction that checks at runtime if the // class instance is identical to the SILType. ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee()); CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true, CMI->getOperand(), SubType, Iden, Virt); It = CCBI->getIterator(); SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction()); SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction()); // This is the class reference downcasted into subclass SubType. SILValue DownCastedClassInstance = Iden->getArgument(0); // Copy the two apply instructions into the two blocks. FullApplySite IdenAI = CloneApply(AI, IdenBuilder); FullApplySite VirtAI = CloneApply(AI, VirtBuilder); // See if Continue has a release on self as the instruction right after the // apply. If it exists, move it into position in the diamond. SILBasicBlock::iterator next = next_or_end(Continue->begin(), Continue->end()); auto *Release = (next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next); if (Release && Release->getOperand() == CMI->getOperand()) { VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(), Release->getAtomicity()); IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance, Release->getAtomicity()); Release->eraseFromParent(); } // Create a PHInode for returning the return value from both apply // instructions. SILArgument *Arg = Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned); if (!isa<TryApplyInst>(AI)) { if (AI.getSubstCalleeType()->isNoReturnFunction()) { IdenBuilder.createUnreachable(AI.getLoc()); VirtBuilder.createUnreachable(AI.getLoc()); } else { IdenBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(IdenAI) }); VirtBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(VirtAI) }); } } // Remove the old Apply instruction. assert(AI.getInstruction() == &Continue->front() && "AI should be the first instruction in the split Continue block"); if (isa<TryApplyInst>(AI)) { AI.getInstruction()->eraseFromParent(); assert(Continue->empty() && "There should not be an instruction after try_apply"); Continue->eraseFromParent(); } else { auto apply = cast<ApplyInst>(AI); apply->replaceAllUsesWith(Arg); apply->eraseFromParent(); assert(!Continue->empty() && "There should be at least a terminator after AI"); } // Update the stats. NumTargetsPredicted++; // Devirtualize the apply instruction on the identical path. auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance); assert(NewInstPair.first && "Expected to be able to devirtualize apply!"); replaceDeadApply(IdenAI, NewInstPair.first); // Split critical edges resulting from VirtAI. if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) { auto *ErrorBB = TAI->getFunction()->createBasicBlock(); ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(ErrorBB); Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(), {ErrorBB->getArgument(0)}); auto *NormalBB = TAI->getFunction()->createBasicBlock(); NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(NormalBB); Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(), {NormalBB->getArgument(0)}); Builder.setInsertionPoint(VirtAI.getInstruction()); SmallVector<SILValue, 4> Args; for (auto Arg : VirtAI.getArguments()) { Args.push_back(Arg); } FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(), VirtAI.getSubstitutions(), Args, NormalBB, ErrorBB); VirtAI.getInstruction()->eraseFromParent(); VirtAI = NewVirtAI; } return VirtAI; }
StackNesting::Changes StackNesting::adaptDeallocs() { bool InstChanged = false; bool CFGChanged = false; BitVector Bits(StackLocs.size()); // Visit all blocks. Actuallly the order doesn't matter, but let's to it in // the same order as in solve(). for (const BlockInfo &BI : reversed(BlockInfos)) { // Collect the alive-bits (at the block exit) from the successor blocks. Bits.reset(); for (BlockInfo *SuccBI : BI.Successors) { Bits |= SuccBI->AliveStackLocsAtEntry; } // Insert deallocations at block boundaries. // This can be necessary for unreachable blocks. Example: // // %1 = alloc_stack // %2 = alloc_stack // cond_br %c, bb2, bb3 // bb2: <--- need to insert a dealloc_stack %2 at the begin of bb2 // dealloc_stack %1 // unreachable // bb3: // dealloc_stack %2 // dealloc_stack %1 // for (unsigned SuccIdx = 0, NumSuccs = BI.Successors.size(); SuccIdx < NumSuccs; ++ SuccIdx) { BlockInfo *SuccBI = BI.Successors[SuccIdx]; // It's acceptable to not deallocate alive locations in unreachable // blocks - as long as the nesting is not violated. So if there are no // alive locations at the unreachable successor block, we can ignore it. if (!SuccBI->ExitReachable && !SuccBI->AliveStackLocsAtEntry.any()) continue; if (SuccBI->AliveStackLocsAtEntry == Bits) continue; // Insert dellocations for all locations which are alive at the end of // the current block, but not at the begin of the successor block. SILBasicBlock *InsertionBlock = SuccBI->Block; if (!InsertionBlock->getSinglePredecessorBlock()) { // If the current block is not the only predecessor of the successor // block, we have to insert a new block where we can add the // deallocations. InsertionBlock = splitEdge(BI.Block->getTerminator(), SuccIdx); CFGChanged = true; } InstChanged |= insertDeallocs(Bits, SuccBI->AliveStackLocsAtEntry, &InsertionBlock->front()); } // Insert/remove deallocations inside blocks. for (SILInstruction *StackInst : reversed(BI.StackInsts)) { if (StackInst->isAllocatingStack()) { // For allocations we just update the bit-set. int BitNr = StackLoc2BitNumbers.lookup(StackInst); assert(Bits == StackLocs[BitNr].AliveLocs && "dataflow didn't converge"); Bits.reset(BitNr); } else if (StackInst->isDeallocatingStack()) { // Handle deallocations. auto *AllocInst = cast<SILInstruction>(StackInst->getOperand(0)); int BitNr = StackLoc2BitNumbers.lookup(AllocInst); SILInstruction *InsertionPoint = &*std::next(StackInst->getIterator()); if (Bits.test(BitNr)) { // The location of StackInst is alive after StackInst. So we have to // remove this deallocation. StackInst->eraseFromParent(); InstChanged = true; } else { // Avoid inserting another deallocation for BitNr (which is already // StackInst). Bits.set(BitNr); } // Insert deallocations for all locations which are not alive after // StackInst but _are_ alive at the StackInst. InstChanged |= insertDeallocs(StackLocs[BitNr].AliveLocs, Bits, InsertionPoint); Bits |= StackLocs[BitNr].AliveLocs; } } assert(Bits == BI.AliveStackLocsAtEntry && "dataflow didn't converge"); } if (CFGChanged) return Changes::CFG; if (InstChanged) return Changes::Instructions; return Changes::None; }