void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() { SILBasicBlock *BB = &*NewF->begin(); SILBuilder Builder(BB->begin()); Builder.setCurrentDebugScope(BB->getParent()->getDebugScope()); unsigned TotalArgIndex = 0; for (ArgumentDescriptor &AD : ArgumentDescList) { // Simply continue if do not explode. if (!AD.Explode) { AIM[TotalArgIndex] = AD.Index; TotalArgIndex ++; continue; } // OK, we need to explode this argument. unsigned ArgOffset = ++TotalArgIndex; unsigned OldArgIndex = ArgOffset - 1; llvm::SmallVector<SILValue, 8> LeafValues; // We do this in the same order as leaf types since ProjTree expects that the // order of leaf values matches the order of leaf types. llvm::SmallVector<const ProjectionTreeNode*, 8> LeafNodes; AD.ProjTree.getLeafNodes(LeafNodes); for (auto *Node : LeafNodes) { auto OwnershipKind = *AD.getTransformedOwnershipKind(Node->getType()); LeafValues.push_back(BB->insertFunctionArgument( ArgOffset++, Node->getType(), OwnershipKind, BB->getArgument(OldArgIndex)->getDecl())); AIM[TotalArgIndex - 1] = AD.Index; TotalArgIndex ++; } // Then go through the projection tree constructing aggregates and replacing // uses. AD.ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(), LeafValues); // We ignored debugvalue uses when we constructed the new arguments, in order // to preserve as much information as possible, we construct a new value for // OrigArg from the leaf values and use that in place of the OrigArg. SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue(Builder, BB->getParent()->getLocation(), LeafValues); // Replace all uses of the original arg with the new value. SILArgument *OrigArg = BB->getArgument(OldArgIndex); OrigArg->replaceAllUsesWith(NewOrigArgValue); // Now erase the old argument since it does not have any uses. We also // decrement ArgOffset since we have one less argument now. BB->eraseArgument(OldArgIndex); TotalArgIndex --; } }
SILValue StackAllocationPromoter::getLiveOutValue(BlockSet &PhiBlocks, SILBasicBlock *StartBB) { DEBUG(llvm::dbgs() << "*** Searching for a value definition.\n"); // Walk the Dom tree in search of a defining value: for (DomTreeNode *Node = DT->getNode(StartBB); Node; Node = Node->getIDom()) { SILBasicBlock *BB = Node->getBlock(); // If there is a store (that must come after the phi), use its value. BlockToInstMap::iterator it = LastStoreInBlock.find(BB); if (it != LastStoreInBlock.end()) if (auto *St = dyn_cast_or_null<StoreInst>(it->second)) { DEBUG(llvm::dbgs() << "*** Found Store def " << *St->getSrc()); return St->getSrc(); } // If there is a Phi definition in this block: if (PhiBlocks.count(BB)) { // Return the dummy instruction that represents the new value that we will // add to the basic block. SILValue Phi = BB->getArgument(BB->getNumArguments() - 1); DEBUG(llvm::dbgs() << "*** Found a dummy Phi def " << *Phi); return Phi; } // Move to the next dominating block. DEBUG(llvm::dbgs() << "*** Walking up the iDOM.\n"); } DEBUG(llvm::dbgs() << "*** Could not find a Def. Using Undef.\n"); return SILUndef::get(ASI->getElementType(), ASI->getModule()); }
static TryApplyInst *replaceTryApplyInst(SILBuilder &B, SILLocation Loc, TryApplyInst *OldTAI, SILValue NewFn, SubstitutionMap NewSubs, ArrayRef<SILValue> NewArgs, SILFunctionConventions Conv) { SILBasicBlock *NormalBB = OldTAI->getNormalBB(); SILBasicBlock *ResultBB = nullptr; SILType NewResultTy = Conv.getSILResultType(); // Does the result value need to be casted? auto OldResultTy = NormalBB->getArgument(0)->getType(); bool ResultCastRequired = NewResultTy != OldResultTy; // Create a new normal BB only if the result of the new apply differs // in type from the argument of the original normal BB. if (!ResultCastRequired) { ResultBB = NormalBB; } else { ResultBB = B.getFunction().createBasicBlockBefore(NormalBB); ResultBB->createPHIArgument(NewResultTy, ValueOwnershipKind::Owned); } // We can always just use the original error BB because we'll be // deleting the edge to it from the old TAI. SILBasicBlock *ErrorBB = OldTAI->getErrorBB(); // Insert a try_apply here. // Note that this makes this block temporarily double-terminated! // We won't fix that until deleteDevirtualizedApply. auto NewTAI = B.createTryApply(Loc, NewFn, NewSubs, NewArgs, ResultBB, ErrorBB); if (ResultCastRequired) { B.setInsertionPoint(ResultBB); SILValue ResultValue = ResultBB->getArgument(0); ResultValue = castValueToABICompatibleType(&B, Loc, ResultValue, NewResultTy, OldResultTy); B.createBranch(Loc, NormalBB, { ResultValue }); } B.setInsertionPoint(NormalBB->begin()); return NewTAI; }
void FunctionSignatureTransform::DeadArgumentTransformFunction() { SILBasicBlock *BB = &*F->begin(); for (const ArgumentDescriptor &AD : ArgumentDescList) { if (!AD.IsEntirelyDead) continue; eraseUsesOfValue(BB->getArgument(AD.Index)); } }
/// At least one value feeding the specified SILArgument is a Struct. Attempt to /// replace the Argument with a new Struct in the same block. /// /// When we handle more types of casts, this can become a template. /// /// ArgValues are the values feeding the specified Argument from each /// predecessor. They must be listed in order of Arg->getParent()->getPreds(). static StructInst * replaceBBArgWithStruct(SILPhiArgument *Arg, SmallVectorImpl<SILValue> &ArgValues) { SILBasicBlock *PhiBB = Arg->getParent(); auto *FirstSI = dyn_cast<StructInst>(ArgValues[0]); if (!FirstSI) return nullptr; // Collect the BBArg index of each struct oper. // e.g. // struct(A, B) // br (B, A) // : ArgIdxForOper => {1, 0} SmallVector<unsigned, 4> ArgIdxForOper; for (unsigned OperIdx : indices(FirstSI->getElements())) { bool FoundMatchingArgIdx = false; for (unsigned ArgIdx : indices(PhiBB->getArguments())) { SmallVectorImpl<SILValue>::const_iterator AVIter = ArgValues.begin(); bool TryNextArgIdx = false; for (SILBasicBlock *PredBB : PhiBB->getPredecessorBlocks()) { // All argument values must be StructInst. auto *PredSI = dyn_cast<StructInst>(*AVIter++); if (!PredSI) return nullptr; OperandValueArrayRef EdgeValues = getEdgeValuesForTerminator(PredBB->getTerminator(), PhiBB); if (EdgeValues[ArgIdx] != PredSI->getElements()[OperIdx]) { TryNextArgIdx = true; break; } } if (!TryNextArgIdx) { assert(AVIter == ArgValues.end() && "# ArgValues does not match # BB preds"); FoundMatchingArgIdx = true; ArgIdxForOper.push_back(ArgIdx); break; } } if (!FoundMatchingArgIdx) return nullptr; } SmallVector<SILValue, 4> StructArgs; for (auto ArgIdx : ArgIdxForOper) StructArgs.push_back(PhiBB->getArgument(ArgIdx)); SILBuilder Builder(PhiBB, PhiBB->begin()); return Builder.createStruct(cast<StructInst>(ArgValues[0])->getLoc(), Arg->getType(), StructArgs); }
bool OwnershipModelEliminatorVisitor::visitCheckedCastBranchInst( CheckedCastBranchInst *CBI) { // In ownership qualified SIL, checked_cast_br must pass its argument to the // fail case so we can clean it up. In non-ownership qualified SIL, we expect // no argument from the checked_cast_br in the default case. The way that we // handle this transformation is that: // // 1. We replace all uses of the argument to the false block with a use of the // checked cast branch's operand. // 2. We delete the argument from the false block. SILBasicBlock *FailureBlock = CBI->getFailureBB(); if (FailureBlock->getNumArguments() == 0) return false; FailureBlock->getArgument(0)->replaceAllUsesWith(CBI->getOperand()); FailureBlock->eraseArgument(0); return true; }
bool OwnershipModelEliminatorVisitor::visitSwitchEnumInst( SwitchEnumInst *SWEI) { // In ownership qualified SIL, switch_enum must pass its argument to the fail // case so we can clean it up. In non-ownership qualified SIL, we expect no // argument from the switch_enum in the default case. The way that we handle // this transformation is that: // // 1. We replace all uses of the argument to the false block with a use of the // checked cast branch's operand. // 2. We delete the argument from the false block. if (!SWEI->hasDefault()) return false; SILBasicBlock *DefaultBlock = SWEI->getDefaultBB(); if (DefaultBlock->getNumArguments() == 0) return false; DefaultBlock->getArgument(0)->replaceAllUsesWith(SWEI->getOperand()); DefaultBlock->eraseArgument(0); return true; }
void FunctionSignatureTransform:: OwnedToGuaranteedAddResultRelease(ResultDescriptor &RD, SILBuilder &Builder, SILFunction *F) { // If we have any result that were consumed but are now guaranteed, // insert a release_value. if (!RD.OwnedToGuaranteed) { return; } SILInstruction *Call = findOnlyApply(F); if (isa<ApplyInst>(Call)) { Builder.setInsertionPoint(&*std::next(SILBasicBlock::iterator(Call))); Builder.createRetainValue(RegularLocation(SourceLoc()), Call, Builder.getDefaultAtomicity()); } else { SILBasicBlock *NormalBB = dyn_cast<TryApplyInst>(Call)->getNormalBB(); Builder.setInsertionPoint(&*NormalBB->begin()); Builder.createRetainValue(RegularLocation(SourceLoc()), NormalBB->getArgument(0), Builder.getDefaultAtomicity()); } }
/// \brief Populate the body of the cloned closure, modifying instructions as /// necessary. This is where we create the actual specialized BB Arguments. void ClosureSpecCloner::populateCloned() { SILFunction *Cloned = getCloned(); SILFunction *ClosureUser = CallSiteDesc.getApplyCallee(); // Create arguments for the entry block. SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); SmallVector<SILValue, 4> entryArgs; entryArgs.reserve(ClosureUserEntryBB->getArguments().size()); // Remove the closure argument. SILArgument *ClosureArg = nullptr; for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) { SILArgument *Arg = ClosureUserEntryBB->getArgument(i); if (i == CallSiteDesc.getClosureIndex()) { ClosureArg = Arg; entryArgs.push_back(SILValue()); continue; } // Otherwise, create a new argument which copies the original argument SILValue MappedValue = ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl()); entryArgs.push_back(MappedValue); } // Next we need to add in any arguments that are not captured as arguments to // the cloned function. // // We do not insert the new mapped arguments into the value map since there by // definition is nothing in the partial apply user function that references // such arguments. After this pass is done the only thing that will reference // the arguments is the partial apply that we will create. SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee(); auto ClosedOverFunConv = ClosedOverFun->getConventions(); unsigned NumTotalParams = ClosedOverFunConv.getNumParameters(); unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments(); llvm::SmallVector<SILValue, 4> NewPAIArgs; for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) { auto paramTy = ClosedOverFunConv.getSILType(PInfo); SILValue MappedValue = ClonedEntryBB->createFunctionArgument(paramTy); NewPAIArgs.push_back(MappedValue); } SILBuilder &Builder = getBuilder(); Builder.setInsertionPoint(ClonedEntryBB); // Clone FRI and PAI, and replace usage of the removed closure argument // with result of cloned PAI. SILValue FnVal = Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun); auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs); // Clone a chain of ConvertFunctionInsts. This can create further // reabstraction partial_apply instructions. SmallVector<PartialApplyInst*, 4> NeedsRelease; SILValue ConvertedCallee = cloneCalleeConversion( CallSiteDesc.getClosureCallerArg(), NewClosure, Builder, NeedsRelease); // Make sure that we actually emit the releases for reabstraction thunks. We // have guaranteed earlier that we only allow reabstraction thunks if the // closure was passed trivial. assert(NeedsRelease.empty() || CallSiteDesc.isTrivialNoEscapeParameter()); entryArgs[CallSiteDesc.getClosureIndex()] = ConvertedCallee; // Visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions and terminators. cloneFunctionBody(ClosureUser, ClonedEntryBB, entryArgs); // Then insert a release in all non failure exit BBs if our partial apply was // guaranteed. This is b/c it was passed at +0 originally and we need to // balance the initial increment of the newly created closure(s). bool ClosureHasRefSemantics = CallSiteDesc.closureHasRefSemanticContext(); if ((CallSiteDesc.isClosureGuaranteed() || CallSiteDesc.isTrivialNoEscapeParameter()) && (ClosureHasRefSemantics || !NeedsRelease.empty())) { for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) { SILBasicBlock *OpBB = getOpBasicBlock(BB); TermInst *TI = OpBB->getTerminator(); auto Loc = CleanupLocation::get(NewClosure->getLoc()); // If we have an exit, we place the release right before it so we know // that it will be executed at the end of the epilogue. if (TI->isFunctionExiting()) { Builder.setInsertionPoint(TI); if (ClosureHasRefSemantics) Builder.createReleaseValue(Loc, SILValue(NewClosure), Builder.getDefaultAtomicity()); for (auto PAI : NeedsRelease) Builder.createReleaseValue(Loc, SILValue(PAI), Builder.getDefaultAtomicity()); continue; } // We use casts where findAllNonFailureExitBBs should have made sure that // this is true. This will ensure that the code is updated when we hit the // cast failure in debug builds. auto *Unreachable = cast<UnreachableInst>(TI); auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable)); auto NoReturnApply = FullApplySite::isa(&*PrevIter); // We insert the release value right before the no return apply so that if // the partial apply is passed into the no-return function as an @owned // value, we will retain the partial apply before we release it and // potentially eliminate it. Builder.setInsertionPoint(NoReturnApply.getInstruction()); if (ClosureHasRefSemantics) Builder.createReleaseValue(Loc, SILValue(NewClosure), Builder.getDefaultAtomicity()); for (auto PAI : NeedsRelease) Builder.createReleaseValue(Loc, SILValue(PAI), Builder.getDefaultAtomicity()); } } }
/// \brief Populate the body of the cloned closure, modifying instructions as /// necessary. This is where we create the actual specialized BB Arguments. void ClosureSpecCloner::populateCloned() { SILFunction *Cloned = getCloned(); SILFunction *ClosureUser = CallSiteDesc.getApplyCallee(); // Create arguments for the entry block. SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); // Remove the closure argument. SILArgument *ClosureArg = nullptr; for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) { SILArgument *Arg = ClosureUserEntryBB->getArgument(i); if (i == CallSiteDesc.getClosureIndex()) { ClosureArg = Arg; continue; } // Otherwise, create a new argument which copies the original argument SILValue MappedValue = ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl()); ValueMap.insert(std::make_pair(Arg, MappedValue)); } // Next we need to add in any arguments that are not captured as arguments to // the cloned function. // // We do not insert the new mapped arguments into the value map since there by // definition is nothing in the partial apply user function that references // such arguments. After this pass is done the only thing that will reference // the arguments is the partial apply that we will create. SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee(); CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType(); unsigned NumTotalParams = ClosedOverFunTy->getParameters().size(); unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments(); llvm::SmallVector<SILValue, 4> NewPAIArgs; for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) { SILValue MappedValue = ClonedEntryBB->createFunctionArgument(PInfo.getSILType()); NewPAIArgs.push_back(MappedValue); } SILBuilder &Builder = getBuilder(); Builder.setInsertionPoint(ClonedEntryBB); // Clone FRI and PAI, and replace usage of the removed closure argument // with result of cloned PAI. SILValue FnVal = Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun); auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs); ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure))); BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB)); // Recursively visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions other than terminators. visitSILBasicBlock(ClosureUserEntryBB); // Now iterate over the BBs and fix up the terminators. for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) { Builder.setInsertionPoint(BI->second); visit(BI->first->getTerminator()); } // Then insert a release in all non failure exit BBs if our partial apply was // guaranteed. This is b/c it was passed at +0 originally and we need to // balance the initial increment of the newly created closure. if (CallSiteDesc.isClosureGuaranteed() && CallSiteDesc.closureHasRefSemanticContext()) { for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) { SILBasicBlock *OpBB = BBMap[BB]; TermInst *TI = OpBB->getTerminator(); auto Loc = CleanupLocation::get(NewClosure->getLoc()); // If we have a return, we place the release right before it so we know // that it will be executed at the end of the epilogue. if (isa<ReturnInst>(TI)) { Builder.setInsertionPoint(TI); Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic); continue; } // We use casts where findAllNonFailureExitBBs should have made sure that // this is true. This will ensure that the code is updated when we hit the // cast failure in debug builds. auto *Unreachable = cast<UnreachableInst>(TI); auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable)); auto NoReturnApply = FullApplySite::isa(&*PrevIter); // We insert the release value right before the no return apply so that if // the partial apply is passed into the no-return function as an @owned // value, we will retain the partial apply before we release it and // potentially eliminate it. Builder.setInsertionPoint(NoReturnApply.getInstruction()); Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic); } } }
static bool matchSwitch(SwitchInfo &SI, SILInstruction *Inst, SILValue SwitchOperand) { auto *SwitchEnum = dyn_cast<SwitchEnumInst>(Inst); if (!SwitchEnum || SwitchEnum->getNumCases() != 2 || SwitchEnum->getOperand() != SwitchOperand) return false; auto *SwitchBB = SwitchEnum->getParent(); SILBasicBlock *SomeBB = SwitchEnum->getCase(0).second; SILBasicBlock *NoneBB = SwitchEnum->getCase(1).second; if (NoneBB->getSinglePredecessorBlock() != SwitchBB) return false; if (SomeBB->getSinglePredecessorBlock() != SwitchBB) return false; if (NoneBB->args_size() == 1) std::swap(NoneBB, SomeBB); if (SomeBB->args_size() != 1 || NoneBB->args_size() != 0) return false; // bb9: // %43 = enum $Optional<String>, #Optional.none!enumelt auto It = NoneBB->begin(); auto *NoneEnum = dyn_cast<EnumInst>(It); if (!NoneEnum || NoneEnum->hasOperand() || !NoneEnum->hasOneUse()) return false; // br bb10(%43 : $Optional<String>) ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *Br1 = dyn_cast<BranchInst>(It); if (!Br1 || Br1->getNumArgs() != 1 || Br1->getArg(0) != NoneEnum) return false; auto *MergeBB = Br1->getDestBB(); // bb8(%36 : $NSString): It = SomeBB->begin(); auto *SomeBBArg = SomeBB->getArgument(0); if (!SomeBBArg->hasOneUse()) return false; // %37 = function_ref @$SSS10FoundationE36_unconditionallyBridgeFromObjectiveCSSSo8NSStringCSgFZ : $@convention(method) (@owned Optional<NSString>, @thin String.Type) -> @owned String auto *FunRef = dyn_cast<FunctionRefInst>(It); if (!FunRef || !FunRef->hasOneUse()) return false; // %38 = enum $Optional<NSString>, #Optional.some!enumelt.1, %36 : $NSString ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *SomeEnum = dyn_cast<EnumInst>(It); if (!SomeEnum || !SomeEnum->hasOperand() || SomeEnum->getOperand() != SomeBBArg) return false; size_t numSomeEnumUses = std::distance(SomeEnum->use_begin(), SomeEnum->use_end()); if (numSomeEnumUses > 2) return false; // %39 = metatype $@thin String.Type ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *Metatype = dyn_cast<MetatypeInst>(It); if (!Metatype || !Metatype->hasOneUse()) return false; // %40 = apply %37(%38, %39) : $@convention(method) (@owned Optional<NSString>, @thin String.Type) -> @owned String ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *Apply = dyn_cast<ApplyInst>(It); if (!Apply || !Apply->hasOneUse() || Apply->getCallee() != FunRef || Apply->getNumArguments() != 2 || Apply->getArgument(0) != SomeEnum || Apply->getArgument(1) != Metatype || Apply->getSubstCalleeType()->getNumResults() != 1) return false; if (Apply->getSubstCalleeType()->getSingleResult().getConvention() != ResultConvention::Owned) return false; // Check that we call the _unconditionallyBridgeFromObjectiveC witness. auto NativeType = Apply->getType().getASTType(); auto *BridgeFun = FunRef->getReferencedFunction(); auto *SwiftModule = BridgeFun->getModule().getSwiftModule(); auto bridgeWitness = getBridgeFromObjectiveC(NativeType, SwiftModule); if (BridgeFun->getName() != bridgeWitness.mangle()) return false; // %41 = enum $Optional<String>, #Optional.some!enumelt.1, %40 : $String ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *Enum3 = dyn_cast<EnumInst>(It); if (!Enum3 || !Enum3->hasOneUse() || !Enum3->hasOperand() || Enum3->getOperand() != Apply) return false; if (numSomeEnumUses == 2) { // release_value %38 : $Optional<NSString> ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *RVI = dyn_cast<ReleaseValueInst>(It); if (!RVI || RVI->getOperand() != SomeEnum) return false; } // br bb10(%41 : $Optional<String>) ADVANCE_ITERATOR_OR_RETURN_FALSE(It); auto *Br = dyn_cast<BranchInst>(It); if (!Br || Br->getDestBB() != MergeBB || Br->getNumArgs() != 1 || Br->getArg(0) != Enum3) return false; SI.SwitchEnum = SwitchEnum; SI.SomeBB = SomeBB; SI.NoneBB = NoneBB; SI.Br = Br; return true; }
/// Insert monomorphic inline caches for a specific class or metatype /// type \p SubClassTy. static FullApplySite speculateMonomorphicTarget(FullApplySite AI, SILType SubType, CheckedCastBranchInst *&CCBI) { CCBI = nullptr; // Bail if this class_method cannot be devirtualized. if (!canDevirtualizeClassMethod(AI, SubType)) return FullApplySite(); if (SubType.getSwiftRValueType()->hasDynamicSelfType()) return FullApplySite(); // Create a diamond shaped control flow and a checked_cast_branch // instruction that checks the exact type of the object. // This cast selects between two paths: one that calls the slow dynamic // dispatch and one that calls the specific method. auto It = AI.getInstruction()->getIterator(); SILFunction *F = AI.getFunction(); SILBasicBlock *Entry = AI.getParent(); // Iden is the basic block containing the direct call. SILBasicBlock *Iden = F->createBasicBlock(); // Virt is the block containing the slow virtual call. SILBasicBlock *Virt = F->createBasicBlock(); Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned); SILBasicBlock *Continue = Entry->split(It); SILBuilderWithScope Builder(Entry, AI.getInstruction()); // Create the checked_cast_branch instruction that checks at runtime if the // class instance is identical to the SILType. ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee()); CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true, CMI->getOperand(), SubType, Iden, Virt); It = CCBI->getIterator(); SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction()); SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction()); // This is the class reference downcasted into subclass SubType. SILValue DownCastedClassInstance = Iden->getArgument(0); // Copy the two apply instructions into the two blocks. FullApplySite IdenAI = CloneApply(AI, IdenBuilder); FullApplySite VirtAI = CloneApply(AI, VirtBuilder); // See if Continue has a release on self as the instruction right after the // apply. If it exists, move it into position in the diamond. SILBasicBlock::iterator next = next_or_end(Continue->begin(), Continue->end()); auto *Release = (next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next); if (Release && Release->getOperand() == CMI->getOperand()) { VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(), Release->getAtomicity()); IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance, Release->getAtomicity()); Release->eraseFromParent(); } // Create a PHInode for returning the return value from both apply // instructions. SILArgument *Arg = Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned); if (!isa<TryApplyInst>(AI)) { if (AI.getSubstCalleeType()->isNoReturnFunction()) { IdenBuilder.createUnreachable(AI.getLoc()); VirtBuilder.createUnreachable(AI.getLoc()); } else { IdenBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(IdenAI) }); VirtBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(VirtAI) }); } } // Remove the old Apply instruction. assert(AI.getInstruction() == &Continue->front() && "AI should be the first instruction in the split Continue block"); if (isa<TryApplyInst>(AI)) { AI.getInstruction()->eraseFromParent(); assert(Continue->empty() && "There should not be an instruction after try_apply"); Continue->eraseFromParent(); } else { auto apply = cast<ApplyInst>(AI); apply->replaceAllUsesWith(Arg); apply->eraseFromParent(); assert(!Continue->empty() && "There should be at least a terminator after AI"); } // Update the stats. NumTargetsPredicted++; // Devirtualize the apply instruction on the identical path. auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance); assert(NewInstPair.first && "Expected to be able to devirtualize apply!"); replaceDeadApply(IdenAI, NewInstPair.first); // Split critical edges resulting from VirtAI. if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) { auto *ErrorBB = TAI->getFunction()->createBasicBlock(); ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(ErrorBB); Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(), {ErrorBB->getArgument(0)}); auto *NormalBB = TAI->getFunction()->createBasicBlock(); NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(NormalBB); Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(), {NormalBB->getArgument(0)}); Builder.setInsertionPoint(VirtAI.getInstruction()); SmallVector<SILValue, 4> Args; for (auto Arg : VirtAI.getArguments()) { Args.push_back(Arg); } FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(), VirtAI.getSubstitutions(), Args, NormalBB, ErrorBB); VirtAI.getInstruction()->eraseFromParent(); VirtAI = NewVirtAI; } return VirtAI; }
std::pair<Optional<SILValue>, SILLocation> SILGenFunction::emitEpilogBB(SILLocation TopLevel) { assert(ReturnDest.getBlock() && "no epilog bb prepared?!"); SILBasicBlock *epilogBB = ReturnDest.getBlock(); SILLocation ImplicitReturnFromTopLevel = ImplicitReturnLocation::getImplicitReturnLoc(TopLevel); SmallVector<SILValue, 4> directResults; Optional<SILLocation> returnLoc = None; // If the current BB isn't terminated, and we require a return, then we // are not allowed to fall off the end of the function and can't reach here. if (NeedsReturn && B.hasValidInsertionPoint()) B.createUnreachable(ImplicitReturnFromTopLevel); if (epilogBB->pred_empty()) { // If the epilog was not branched to at all, kill the BB and // just emit the epilog into the current BB. while (!epilogBB->empty()) epilogBB->back().eraseFromParent(); eraseBasicBlock(epilogBB); // If the current bb is terminated then the epilog is just unreachable. if (!B.hasValidInsertionPoint()) return { None, TopLevel }; // We emit the epilog at the current insertion point. returnLoc = ImplicitReturnFromTopLevel; } else if (std::next(epilogBB->pred_begin()) == epilogBB->pred_end() && !B.hasValidInsertionPoint()) { // If the epilog has a single predecessor and there's no current insertion // point to fall through from, then we can weld the epilog to that // predecessor BB. // Steal the branch argument as the return value if present. SILBasicBlock *pred = *epilogBB->pred_begin(); BranchInst *predBranch = cast<BranchInst>(pred->getTerminator()); assert(predBranch->getArgs().size() == epilogBB->args_size() && "epilog predecessor arguments does not match block params"); for (auto index : indices(predBranch->getArgs())) { SILValue result = predBranch->getArgs()[index]; directResults.push_back(result); epilogBB->getArgument(index)->replaceAllUsesWith(result); } // If we are optimizing, we should use the return location from the single, // previously processed, return statement if any. if (predBranch->getLoc().is<ReturnLocation>()) { returnLoc = predBranch->getLoc(); } else { returnLoc = ImplicitReturnFromTopLevel; } // Kill the branch to the now-dead epilog BB. pred->erase(predBranch); // Move any instructions from the EpilogBB to the end of the 'pred' block. pred->spliceAtEnd(epilogBB); // Finally we can erase the epilog BB. eraseBasicBlock(epilogBB); // Emit the epilog into its former predecessor. B.setInsertionPoint(pred); } else { // Move the epilog block to the end of the ordinary section. auto endOfOrdinarySection = StartOfPostmatter; B.moveBlockTo(epilogBB, endOfOrdinarySection); // Emit the epilog into the epilog bb. Its arguments are the // direct results. directResults.append(epilogBB->args_begin(), epilogBB->args_end()); // If we are falling through from the current block, the return is implicit. B.emitBlock(epilogBB, ImplicitReturnFromTopLevel); } // Emit top-level cleanups into the epilog block. assert(!Cleanups.hasAnyActiveCleanups(getCleanupsDepth(), ReturnDest.getDepth()) && "emitting epilog in wrong scope"); auto cleanupLoc = CleanupLocation::get(TopLevel); Cleanups.emitCleanupsForReturn(cleanupLoc); // If the return location is known to be that of an already // processed return, use it. (This will get triggered when the // epilog logic is simplified.) // // Otherwise make the ret instruction part of the cleanups. if (!returnLoc) returnLoc = cleanupLoc; // Build the return value. We don't do this if there are no direct // results; this can happen for void functions, but also happens when // prepareEpilog was asked to not add result arguments to the epilog // block. SILValue returnValue; if (!directResults.empty()) { assert(directResults.size() == F.getConventions().getNumDirectSILResults()); returnValue = buildReturnValue(*this, TopLevel, directResults); } return { returnValue, *returnLoc }; }
/// \brief Devirtualize an apply of a class method. /// /// \p AI is the apply to devirtualize. /// \p ClassOrMetatype is a class value or metatype value that is the /// self argument of the apply we will devirtualize. /// return the result value of the new ApplyInst if created one or null. DevirtualizationResult swift::devirtualizeClassMethod(FullApplySite AI, SILValue ClassOrMetatype) { DEBUG(llvm::dbgs() << " Trying to devirtualize : " << *AI.getInstruction()); SILModule &Mod = AI.getModule(); auto *MI = cast<MethodInst>(AI.getCallee()); auto ClassOrMetatypeType = ClassOrMetatype->getType(); auto *F = getTargetClassMethod(Mod, ClassOrMetatypeType, MI); CanSILFunctionType GenCalleeType = F->getLoweredFunctionType(); SmallVector<Substitution, 4> Subs; getSubstitutionsForCallee(Mod, GenCalleeType, ClassOrMetatypeType.getSwiftRValueType(), AI, Subs); CanSILFunctionType SubstCalleeType = GenCalleeType; if (GenCalleeType->isPolymorphic()) SubstCalleeType = GenCalleeType->substGenericArgs(Mod, Subs); SILFunctionConventions substConv(SubstCalleeType, Mod); SILBuilderWithScope B(AI.getInstruction()); FunctionRefInst *FRI = B.createFunctionRef(AI.getLoc(), F); // Create the argument list for the new apply, casting when needed // in order to handle covariant indirect return types and // contravariant argument types. llvm::SmallVector<SILValue, 8> NewArgs; auto IndirectResultArgIter = AI.getIndirectSILResults().begin(); for (auto ResultTy : substConv.getIndirectSILResultTypes()) { NewArgs.push_back( castValueToABICompatibleType(&B, AI.getLoc(), *IndirectResultArgIter, IndirectResultArgIter->getType(), ResultTy)); ++IndirectResultArgIter; } auto ParamArgIter = AI.getArgumentsWithoutIndirectResults().begin(); // Skip the last parameter, which is `self`. Add it below. for (auto param : substConv.getParameters().drop_back()) { auto paramType = substConv.getSILType(param); NewArgs.push_back( castValueToABICompatibleType(&B, AI.getLoc(), *ParamArgIter, ParamArgIter->getType(), paramType)); ++ParamArgIter; } // Add the self argument, upcasting if required because we're // calling a base class's method. auto SelfParamTy = substConv.getSILType(SubstCalleeType->getSelfParameter()); NewArgs.push_back(castValueToABICompatibleType(&B, AI.getLoc(), ClassOrMetatype, ClassOrMetatypeType, SelfParamTy)); SILType ResultTy = substConv.getSILResultType(); SILType SubstCalleeSILType = SILType::getPrimitiveObjectType(SubstCalleeType); FullApplySite NewAI; SILBasicBlock *ResultBB = nullptr; SILBasicBlock *NormalBB = nullptr; SILValue ResultValue; bool ResultCastRequired = false; SmallVector<Operand *, 4> OriginalResultUses; if (!isa<TryApplyInst>(AI)) { NewAI = B.createApply(AI.getLoc(), FRI, SubstCalleeSILType, ResultTy, Subs, NewArgs, cast<ApplyInst>(AI)->isNonThrowing()); ResultValue = NewAI.getInstruction(); } else { auto *TAI = cast<TryApplyInst>(AI); // Create new normal and error BBs only if: // - re-using a BB would create a critical edge // - or, the result of the new apply would be of different // type than the argument of the original normal BB. if (TAI->getNormalBB()->getSinglePredecessorBlock()) ResultBB = TAI->getNormalBB(); else { ResultBB = B.getFunction().createBasicBlock(); ResultBB->createPHIArgument(ResultTy, ValueOwnershipKind::Owned); } NormalBB = TAI->getNormalBB(); SILBasicBlock *ErrorBB = nullptr; if (TAI->getErrorBB()->getSinglePredecessorBlock()) ErrorBB = TAI->getErrorBB(); else { ErrorBB = B.getFunction().createBasicBlock(); ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); } NewAI = B.createTryApply(AI.getLoc(), FRI, SubstCalleeSILType, Subs, NewArgs, ResultBB, ErrorBB); if (ErrorBB != TAI->getErrorBB()) { B.setInsertionPoint(ErrorBB); B.createBranch(TAI->getLoc(), TAI->getErrorBB(), {ErrorBB->getArgument(0)}); } // Does the result value need to be casted? ResultCastRequired = ResultTy != NormalBB->getArgument(0)->getType(); if (ResultBB != NormalBB) B.setInsertionPoint(ResultBB); else if (ResultCastRequired) { B.setInsertionPoint(NormalBB->begin()); // Collect all uses, before casting. for (auto *Use : NormalBB->getArgument(0)->getUses()) { OriginalResultUses.push_back(Use); } NormalBB->getArgument(0)->replaceAllUsesWith( SILUndef::get(AI.getType(), Mod)); NormalBB->replacePHIArgument(0, ResultTy, ValueOwnershipKind::Owned); } // The result value is passed as a parameter to the normal block. ResultValue = ResultBB->getArgument(0); } // Check if any casting is required for the return value. ResultValue = castValueToABICompatibleType(&B, NewAI.getLoc(), ResultValue, ResultTy, AI.getType()); DEBUG(llvm::dbgs() << " SUCCESS: " << F->getName() << "\n"); NumClassDevirt++; if (NormalBB) { if (NormalBB != ResultBB) { // If artificial normal BB was introduced, branch // to the original normal BB. B.createBranch(NewAI.getLoc(), NormalBB, { ResultValue }); } else if (ResultCastRequired) { // Update all original uses by the new value. for (auto *Use: OriginalResultUses) { Use->set(ResultValue); } } return std::make_pair(NewAI.getInstruction(), NewAI); } // We need to return a pair of values here: // - the first one is the actual result of the devirtualized call, possibly // casted into an appropriate type. This SILValue may be a BB arg, if it // was a cast between optional types. // - the second one is the new apply site. return std::make_pair(ResultValue, NewAI); }