/// \brief Populate the body of the cloned closure, modifying instructions as /// necessary to take into consideration the removed parameters. void PromotedParamCloner::populateCloned() { SILFunction *Cloned = getCloned(); // Create arguments for the entry block SILBasicBlock *OrigEntryBB = &*Orig->begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); SmallVector<SILValue, 4> entryArgs; entryArgs.reserve(OrigEntryBB->getArguments().size()); // Initialize all NewPromotedArgs slots to an invalid value. NewPromotedArgs.resize(OrigEntryBB->getArguments().size()); unsigned ArgNo = 0; auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end(); while (I != E) { if (count(PromotedArgIndices, ArgNo)) { // Create a new argument with the promoted type. auto boxTy = (*I)->getType().castTo<SILBoxType>(); assert(boxTy->getLayout()->getFields().size() == 1 && "promoting multi-field boxes not implemented yet"); auto promotedTy = boxTy->getFieldType(Cloned->getModule(), 0); auto *promotedArg = ClonedEntryBB->createFunctionArgument(promotedTy, (*I)->getDecl()); OrigPromotedParameters.insert(*I); NewPromotedArgs[ArgNo] = promotedArg; // All uses of the promoted box should either be projections, which are // folded when visited, or copy/destroy operations which are ignored. entryArgs.push_back(SILValue()); } else { // Create a new argument which copies the original argument. entryArgs.push_back(ClonedEntryBB->createFunctionArgument( (*I)->getType(), (*I)->getDecl())); } ++ArgNo; ++I; } // Visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions and terminators. cloneFunctionBody(Orig, ClonedEntryBB, entryArgs); }
/// At least one value feeding the specified SILArgument is a Struct. Attempt to /// replace the Argument with a new Struct in the same block. /// /// When we handle more types of casts, this can become a template. /// /// ArgValues are the values feeding the specified Argument from each /// predecessor. They must be listed in order of Arg->getParent()->getPreds(). static StructInst * replaceBBArgWithStruct(SILPhiArgument *Arg, SmallVectorImpl<SILValue> &ArgValues) { SILBasicBlock *PhiBB = Arg->getParent(); auto *FirstSI = dyn_cast<StructInst>(ArgValues[0]); if (!FirstSI) return nullptr; // Collect the BBArg index of each struct oper. // e.g. // struct(A, B) // br (B, A) // : ArgIdxForOper => {1, 0} SmallVector<unsigned, 4> ArgIdxForOper; for (unsigned OperIdx : indices(FirstSI->getElements())) { bool FoundMatchingArgIdx = false; for (unsigned ArgIdx : indices(PhiBB->getArguments())) { SmallVectorImpl<SILValue>::const_iterator AVIter = ArgValues.begin(); bool TryNextArgIdx = false; for (SILBasicBlock *PredBB : PhiBB->getPredecessorBlocks()) { // All argument values must be StructInst. auto *PredSI = dyn_cast<StructInst>(*AVIter++); if (!PredSI) return nullptr; OperandValueArrayRef EdgeValues = getEdgeValuesForTerminator(PredBB->getTerminator(), PhiBB); if (EdgeValues[ArgIdx] != PredSI->getElements()[OperIdx]) { TryNextArgIdx = true; break; } } if (!TryNextArgIdx) { assert(AVIter == ArgValues.end() && "# ArgValues does not match # BB preds"); FoundMatchingArgIdx = true; ArgIdxForOper.push_back(ArgIdx); break; } } if (!FoundMatchingArgIdx) return nullptr; } SmallVector<SILValue, 4> StructArgs; for (auto ArgIdx : ArgIdxForOper) StructArgs.push_back(PhiBB->getArgument(ArgIdx)); SILBuilder Builder(PhiBB, PhiBB->begin()); return Builder.createStruct(cast<StructInst>(ArgValues[0])->getLoc(), Arg->getType(), StructArgs); }
/// \brief Populate the body of the cloned closure, modifying instructions as /// necessary. This is where we create the actual specialized BB Arguments. void ClosureSpecCloner::populateCloned() { SILFunction *Cloned = getCloned(); SILFunction *ClosureUser = CallSiteDesc.getApplyCallee(); // Create arguments for the entry block. SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); SmallVector<SILValue, 4> entryArgs; entryArgs.reserve(ClosureUserEntryBB->getArguments().size()); // Remove the closure argument. SILArgument *ClosureArg = nullptr; for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) { SILArgument *Arg = ClosureUserEntryBB->getArgument(i); if (i == CallSiteDesc.getClosureIndex()) { ClosureArg = Arg; entryArgs.push_back(SILValue()); continue; } // Otherwise, create a new argument which copies the original argument SILValue MappedValue = ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl()); entryArgs.push_back(MappedValue); } // Next we need to add in any arguments that are not captured as arguments to // the cloned function. // // We do not insert the new mapped arguments into the value map since there by // definition is nothing in the partial apply user function that references // such arguments. After this pass is done the only thing that will reference // the arguments is the partial apply that we will create. SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee(); auto ClosedOverFunConv = ClosedOverFun->getConventions(); unsigned NumTotalParams = ClosedOverFunConv.getNumParameters(); unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments(); llvm::SmallVector<SILValue, 4> NewPAIArgs; for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) { auto paramTy = ClosedOverFunConv.getSILType(PInfo); SILValue MappedValue = ClonedEntryBB->createFunctionArgument(paramTy); NewPAIArgs.push_back(MappedValue); } SILBuilder &Builder = getBuilder(); Builder.setInsertionPoint(ClonedEntryBB); // Clone FRI and PAI, and replace usage of the removed closure argument // with result of cloned PAI. SILValue FnVal = Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun); auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs); // Clone a chain of ConvertFunctionInsts. This can create further // reabstraction partial_apply instructions. SmallVector<PartialApplyInst*, 4> NeedsRelease; SILValue ConvertedCallee = cloneCalleeConversion( CallSiteDesc.getClosureCallerArg(), NewClosure, Builder, NeedsRelease); // Make sure that we actually emit the releases for reabstraction thunks. We // have guaranteed earlier that we only allow reabstraction thunks if the // closure was passed trivial. assert(NeedsRelease.empty() || CallSiteDesc.isTrivialNoEscapeParameter()); entryArgs[CallSiteDesc.getClosureIndex()] = ConvertedCallee; // Visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions and terminators. cloneFunctionBody(ClosureUser, ClonedEntryBB, entryArgs); // Then insert a release in all non failure exit BBs if our partial apply was // guaranteed. This is b/c it was passed at +0 originally and we need to // balance the initial increment of the newly created closure(s). bool ClosureHasRefSemantics = CallSiteDesc.closureHasRefSemanticContext(); if ((CallSiteDesc.isClosureGuaranteed() || CallSiteDesc.isTrivialNoEscapeParameter()) && (ClosureHasRefSemantics || !NeedsRelease.empty())) { for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) { SILBasicBlock *OpBB = getOpBasicBlock(BB); TermInst *TI = OpBB->getTerminator(); auto Loc = CleanupLocation::get(NewClosure->getLoc()); // If we have an exit, we place the release right before it so we know // that it will be executed at the end of the epilogue. if (TI->isFunctionExiting()) { Builder.setInsertionPoint(TI); if (ClosureHasRefSemantics) Builder.createReleaseValue(Loc, SILValue(NewClosure), Builder.getDefaultAtomicity()); for (auto PAI : NeedsRelease) Builder.createReleaseValue(Loc, SILValue(PAI), Builder.getDefaultAtomicity()); continue; } // We use casts where findAllNonFailureExitBBs should have made sure that // this is true. This will ensure that the code is updated when we hit the // cast failure in debug builds. auto *Unreachable = cast<UnreachableInst>(TI); auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable)); auto NoReturnApply = FullApplySite::isa(&*PrevIter); // We insert the release value right before the no return apply so that if // the partial apply is passed into the no-return function as an @owned // value, we will retain the partial apply before we release it and // potentially eliminate it. Builder.setInsertionPoint(NoReturnApply.getInstruction()); if (ClosureHasRefSemantics) Builder.createReleaseValue(Loc, SILValue(NewClosure), Builder.getDefaultAtomicity()); for (auto PAI : NeedsRelease) Builder.createReleaseValue(Loc, SILValue(PAI), Builder.getDefaultAtomicity()); } } }
SILInstruction *StackPromoter::findDeallocPoint(SILInstruction *StartInst, SILInstruction *&RestartPoint, EscapeAnalysis::CGNode *Node, int NumUsePointsToFind) { // In the following we check two requirements for stack promotion: // 1) Are all uses in the same control region as the alloc? E.g. if the // allocation is in a loop then there may not be any uses of the object // outside the loop. // 2) We need to find an insertion place for the deallocation so that it // preserves a properly nested stack allocation-deallocation structure. SILBasicBlock *StartBlock = StartInst->getParent(); // The block where we assume we can insert the deallocation. SILBasicBlock *EndBlock = StartBlock; // We visit all instructions starting at the allocation instruction. WorkListType WorkList; // It's important that the EndBlock is at the head of the WorkList so that // we handle it after all other blocks. WorkList.insert(EndBlock, -1); WorkList.insert(StartBlock, 0); for (;;) { SILBasicBlock *BB = WorkList.pop_back_val(); int StackDepth = 0; SILBasicBlock::iterator Iter; if (BB == StartBlock) { // In the first block we start at the allocation instruction and not at // the begin of the block. Iter = StartInst->getIterator(); } else { // Track all uses in the block arguments. for (SILArgument *BBArg : BB->getArguments()) { if (ConGraph->isUsePoint(BBArg, Node)) NumUsePointsToFind--; } // Make sure that the EndBlock is not inside a loop (which does not // contain the StartBlock). // E.g.: // %obj = alloc_ref // the allocation // br loop // loop: // the_only_use_of_obj(%obj) // cond_br ..., loop, exit // exit: // ... // this is the new EndBlock EndBlock = updateEndBlock(BB, EndBlock, WorkList); if (!EndBlock) return nullptr; Iter = BB->begin(); StackDepth = WorkList.getStackDepth(BB); } // Visit all instructions of the current block. while (Iter != BB->end()) { SILInstruction &I = *Iter++; if (BB == EndBlock && StackDepth == 0 && NumUsePointsToFind == 0) { // We found a place to insert the stack deallocation. return &I; } if (I.isAllocatingStack()) { StackDepth++; } else if (I.isDeallocatingStack()) { if (StackDepth == 0) { // The allocation is inside a stack alloc-dealloc region and we are // now leaving this region without having found a place for the // deallocation. E.g. // E.g.: // %1 = alloc_stack // %obj = alloc_ref // the allocation // dealloc_stack %1 // use_of_obj(%obj) // // In this case we can move the alloc_ref before the alloc_stack // to fix the nesting. auto *Alloc = dyn_cast<SILInstruction>(I.getOperand(0)); if (!Alloc) return nullptr; // This should always be the case, but let's be on the safe side. if (!postDominates(StartBlock, Alloc->getParent())) return nullptr; // Trigger another iteration with a new start point; RestartPoint = Alloc; return nullptr; } StackDepth--; } // Track a use. if (ConGraph->isUsePoint(&I, Node) != 0) NumUsePointsToFind--; } if (WorkList.empty()) { if (EndBlock == BB) { // We reached the EndBlock but didn't find a place for the deallocation // so far (because we didn't find all uses yet or we entered another // stack alloc-dealloc region). Let's extend our lifetime region. // E.g.: // %obj = alloc_ref // the allocation // %1 = alloc_stack // use_of_obj(%obj) // can't insert the deallocation in this block // cond_br ..., bb1, bb2 // bb1: // ... // br bb2 // bb2: // dealloc_stack %1 // this is the new EndBlock EndBlock = getImmediatePostDom(EndBlock); if (!EndBlock) return nullptr; } // Again, it's important that the EndBlock is the first in the WorkList. WorkList.insert(EndBlock, -1); } // Push the successor blocks to the WorkList. for (SILBasicBlock *Succ : BB->getSuccessors()) { if (!strictlyDominates(StartBlock, Succ)) { // The StartBlock is inside a loop but we couldn't find a deallocation // place in this loop, e.g. because there are uses outside the loop. // E.g.: // %container = alloc_ref // br loop // loop: // %obj = alloc_ref // the allocation // store %obj to %some_field_in_container // cond_br ..., loop, exit // exit: // use(%container) return nullptr; } WorkList.insert(Succ, StackDepth); } } }
void GenericCloner::populateCloned() { SILFunction *Cloned = getCloned(); // Create arguments for the entry block. SILBasicBlock *OrigEntryBB = &*Original.begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); getBuilder().setInsertionPoint(ClonedEntryBB); llvm::SmallVector<AllocStackInst *, 8> AllocStacks; AllocStackInst *ReturnValueAddr = nullptr; // Create the entry basic block with the function arguments. auto origConv = Original.getConventions(); unsigned ArgIdx = 0; for (auto &OrigArg : OrigEntryBB->getArguments()) { RegularLocation Loc((Decl *)OrigArg->getDecl()); AllocStackInst *ASI = nullptr; SILType mappedType = remapType(OrigArg->getType()); auto createAllocStack = [&]() { // We need an alloc_stack as a replacement for the indirect parameter. assert(mappedType.isAddress()); mappedType = mappedType.getObjectType(); ASI = getBuilder().createAllocStack(Loc, mappedType); ValueMap[OrigArg] = ASI; AllocStacks.push_back(ASI); }; auto handleConversion = [&]() { if (!origConv.useLoweredAddresses()) return false; if (ArgIdx < origConv.getSILArgIndexOfFirstParam()) { // Handle result arguments. unsigned formalIdx = origConv.getIndirectFormalResultIndexForSILArg(ArgIdx); if (ReInfo.isFormalResultConverted(formalIdx)) { // This result is converted from indirect to direct. The return inst // needs to load the value from the alloc_stack. See below. createAllocStack(); assert(!ReturnValueAddr); ReturnValueAddr = ASI; return true; } } else { // Handle arguments for formal parameters. unsigned paramIdx = ArgIdx - origConv.getSILArgIndexOfFirstParam(); if (ReInfo.isParamConverted(paramIdx)) { // Store the new direct parameter to the alloc_stack. createAllocStack(); auto *NewArg = ClonedEntryBB->createFunctionArgument( mappedType, OrigArg->getDecl()); getBuilder().createStore(Loc, NewArg, ASI, StoreOwnershipQualifier::Unqualified); // Try to create a new debug_value from an existing debug_value_addr. for (Operand *ArgUse : OrigArg->getUses()) { if (auto *DVAI = dyn_cast<DebugValueAddrInst>(ArgUse->getUser())) { getBuilder().setCurrentDebugScope(remapScope(DVAI->getDebugScope())); getBuilder().createDebugValue(DVAI->getLoc(), NewArg, DVAI->getVarInfo()); getBuilder().setCurrentDebugScope(nullptr); break; } } return true; } } return false; // No conversion. }; if (!handleConversion()) { auto *NewArg = ClonedEntryBB->createFunctionArgument(mappedType, OrigArg->getDecl()); ValueMap[OrigArg] = NewArg; } ++ArgIdx; } BBMap.insert(std::make_pair(OrigEntryBB, ClonedEntryBB)); // Recursively visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions other than terminators. visitSILBasicBlock(OrigEntryBB); // Now iterate over the BBs and fix up the terminators. for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) { getBuilder().setInsertionPoint(BI->second); TermInst *OrigTermInst = BI->first->getTerminator(); if (auto *RI = dyn_cast<ReturnInst>(OrigTermInst)) { SILValue ReturnValue; if (ReturnValueAddr) { // The result is converted from indirect to direct. We have to load the // returned value from the alloc_stack. ReturnValue = getBuilder().createLoad(ReturnValueAddr->getLoc(), ReturnValueAddr, LoadOwnershipQualifier::Unqualified); } for (AllocStackInst *ASI : reverse(AllocStacks)) { getBuilder().createDeallocStack(ASI->getLoc(), ASI); } if (ReturnValue) { getBuilder().createReturn(RI->getLoc(), ReturnValue); continue; } } else if (isa<ThrowInst>(OrigTermInst)) { for (AllocStackInst *ASI : reverse(AllocStacks)) { getBuilder().createDeallocStack(ASI->getLoc(), ASI); } } visit(BI->first->getTerminator()); } }