/// \brief Inlines all mandatory inlined functions into the body of a function, /// first recursively inlining all mandatory apply instructions in those /// functions into their bodies if necessary. /// /// \param F the function to be processed /// \param AI nullptr if this is being called from the top level; the relevant /// ApplyInst requiring the recursive call when non-null /// \param FullyInlinedSet the set of all functions already known to be fully /// processed, to avoid processing them over again /// \param SetFactory an instance of ImmutableFunctionSet::Factory /// \param CurrentInliningSet the set of functions currently being inlined in /// the current call stack of recursive calls /// /// \returns true if successful, false if failed due to circular inlining. static bool runOnFunctionRecursively(SILFunction *F, FullApplySite AI, SILModule::LinkingMode Mode, DenseFunctionSet &FullyInlinedSet, ImmutableFunctionSet::Factory &SetFactory, ImmutableFunctionSet CurrentInliningSet, ClassHierarchyAnalysis *CHA) { // Avoid reprocessing functions needlessly. if (FullyInlinedSet.count(F)) return true; // Prevent attempt to circularly inline. if (CurrentInliningSet.contains(F)) { // This cannot happen on a top-level call, so AI should be non-null. assert(AI && "Cannot have circular inline without apply"); SILLocation L = AI.getLoc(); assert(L && "Must have location for transparent inline apply"); diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(), diag::circular_transparent); return false; } // Add to the current inlining set (immutably, so we only affect the set // during this call and recursive subcalls). CurrentInliningSet = SetFactory.add(CurrentInliningSet, F); SmallVector<SILValue, 16> CaptureArgs; SmallVector<SILValue, 32> FullArgs; for (auto FI = F->begin(), FE = F->end(); FI != FE; ++FI) { for (auto I = FI->begin(), E = FI->end(); I != E; ++I) { FullApplySite InnerAI = FullApplySite::isa(&*I); if (!InnerAI) continue; auto *ApplyBlock = InnerAI.getParent(); auto NewInstPair = tryDevirtualizeApply(InnerAI, CHA); if (auto *NewInst = NewInstPair.first) { replaceDeadApply(InnerAI, NewInst); if (auto *II = dyn_cast<SILInstruction>(NewInst)) I = II->getIterator(); else I = NewInst->getParentBlock()->begin(); auto NewAI = FullApplySite::isa(NewInstPair.second.getInstruction()); if (!NewAI) continue; InnerAI = NewAI; } SILLocation Loc = InnerAI.getLoc(); SILValue CalleeValue = InnerAI.getCallee(); bool IsThick; PartialApplyInst *PAI; SILFunction *CalleeFunction = getCalleeFunction(InnerAI, IsThick, CaptureArgs, FullArgs, PAI, Mode); if (!CalleeFunction || CalleeFunction->isTransparent() == IsNotTransparent) continue; if (F->isFragile() && !CalleeFunction->hasValidLinkageForFragileRef()) { if (!CalleeFunction->hasValidLinkageForFragileInline()) { llvm::errs() << "caller: " << F->getName() << "\n"; llvm::errs() << "callee: " << CalleeFunction->getName() << "\n"; llvm_unreachable("Should never be inlining a resilient function into " "a fragile function"); } continue; } // Then recursively process it first before trying to inline it. if (!runOnFunctionRecursively(CalleeFunction, InnerAI, Mode, FullyInlinedSet, SetFactory, CurrentInliningSet, CHA)) { // If we failed due to circular inlining, then emit some notes to // trace back the failure if we have more information. // FIXME: possibly it could be worth recovering and attempting other // inlines within this same recursive call rather than simply // propagating the failure. if (AI) { SILLocation L = AI.getLoc(); assert(L && "Must have location for transparent inline apply"); diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(), diag::note_while_inlining); } return false; } // Inline function at I, which also changes I to refer to the first // instruction inlined in the case that it succeeds. We purposely // process the inlined body after inlining, because the inlining may // have exposed new inlining opportunities beyond those present in // the inlined function when processed independently. DEBUG(llvm::errs() << "Inlining @" << CalleeFunction->getName() << " into @" << InnerAI.getFunction()->getName() << "\n"); // If we intend to inline a thick function, then we need to balance the // reference counts for correctness. if (IsThick && I != ApplyBlock->begin()) { // We need to find an appropriate location for our fix up code // We used to do this after inlining Without any modifications // This caused us to add a release in a wrong place: // It would release a value *before* retaining it! // It is really problematic to do this after inlining - // Finding a valid insertion point is tricky: // Inlining might add new basic blocks and/or remove the apply // We want to add the fix up *just before* where the current apply is! // Unfortunately, we *can't* add the fix up code here: // Inlining might fail for any reason - // If that occurred we'd need to undo our fix up code. // Instead, we split the current basic block - // Making sure we have a basic block that starts with our apply. SILBuilderWithScope B(I); ApplyBlock = splitBasicBlockAndBranch(B, &*I, nullptr, nullptr); I = ApplyBlock->begin(); } // Decrement our iterator (carefully, to avoid going off the front) so it // is valid after inlining is done. Inlining deletes the apply, and can // introduce multiple new basic blocks. if (I != ApplyBlock->begin()) --I; else I = ApplyBlock->end(); std::vector<Substitution> ApplySubs(InnerAI.getSubstitutions()); if (PAI) { auto PAISubs = PAI->getSubstitutions(); ApplySubs.insert(ApplySubs.end(), PAISubs.begin(), PAISubs.end()); } SILOpenedArchetypesTracker OpenedArchetypesTracker(*F); F->getModule().registerDeleteNotificationHandler( &OpenedArchetypesTracker); // The callee only needs to know about opened archetypes used in // the substitution list. OpenedArchetypesTracker.registerUsedOpenedArchetypes(InnerAI.getInstruction()); if (PAI) { OpenedArchetypesTracker.registerUsedOpenedArchetypes(PAI); } SILInliner Inliner(*F, *CalleeFunction, SILInliner::InlineKind::MandatoryInline, ApplySubs, OpenedArchetypesTracker); if (!Inliner.inlineFunction(InnerAI, FullArgs)) { I = InnerAI.getInstruction()->getIterator(); continue; } // Inlining was successful. Remove the apply. InnerAI.getInstruction()->eraseFromParent(); // Reestablish our iterator if it wrapped. if (I == ApplyBlock->end()) I = ApplyBlock->begin(); // Update the iterator when instructions are removed. DeleteInstructionsHandler DeletionHandler(I); // If the inlined apply was a thick function, then we need to balance the // reference counts for correctness. if (IsThick) fixupReferenceCounts(I, Loc, CalleeValue, CaptureArgs); // Now that the IR is correct, see if we can remove dead callee // computations (e.g. dead partial_apply closures). cleanupCalleeValue(CalleeValue, CaptureArgs, FullArgs); // Reposition iterators possibly invalidated by mutation. FI = SILFunction::iterator(ApplyBlock); E = ApplyBlock->end(); assert(FI == SILFunction::iterator(I->getParent()) && "Mismatch between the instruction and basic block"); ++NumMandatoryInlines; } } // Keep track of full inlined functions so we don't waste time recursively // reprocessing them. FullyInlinedSet.insert(F); return true; }
void MaterializeForSetEmitter::emit(SILGenFunction &gen, ManagedValue self, SILValue resultBuffer, SILValue callbackBuffer, ArrayRef<ManagedValue> indices) { SILLocation loc = Witness; loc.markAutoGenerated(); // If there's an abstraction difference, we always need to use the // get/set pattern. AccessStrategy strategy; if (WitnessStorage->getType()->is<ReferenceStorageType>() || (Conformance && RequirementStorageType != WitnessStorageType)) { strategy = AccessStrategy::DispatchToAccessor; } else { strategy = WitnessStorage->getAccessStrategy(TheAccessSemantics, AccessKind::ReadWrite); } // Handle the indices. RValue indicesRV; if (isa<SubscriptDecl>(WitnessStorage)) { indicesRV = collectIndicesFromParameters(gen, loc, indices); } else { assert(indices.empty() && "indices for a non-subscript?"); } // As above, assume that we don't need to reabstract 'self'. // Choose the right implementation. SILValue address; SILFunction *callbackFn = nullptr; switch (strategy) { case AccessStrategy::Storage: address = emitUsingStorage(gen, loc, self, std::move(indicesRV)); break; case AccessStrategy::Addressor: address = emitUsingAddressor(gen, loc, self, std::move(indicesRV), callbackBuffer, callbackFn); break; case AccessStrategy::DirectToAccessor: case AccessStrategy::DispatchToAccessor: address = emitUsingGetterSetter(gen, loc, self, std::move(indicesRV), resultBuffer, callbackBuffer, callbackFn); break; } // Return the address as a Builtin.RawPointer. SILType rawPointerTy = SILType::getRawPointerType(gen.getASTContext()); address = gen.B.createAddressToPointer(loc, address, rawPointerTy); SILType resultTupleTy = gen.F.mapTypeIntoContext( gen.F.getLoweredFunctionType()->getResult().getSILType()); SILType optCallbackTy = resultTupleTy.getTupleElementType(1); // Form the callback. SILValue callback; if (callbackFn) { // Make a reference to the function. callback = gen.B.createFunctionRef(loc, callbackFn); // If it's polymorphic, cast to RawPointer and then back to the // right monomorphic type. The safety of this cast relies on some // assumptions about what exactly IRGen can reconstruct from the // callback's thick type argument. if (callbackFn->getLoweredFunctionType()->isPolymorphic()) { callback = gen.B.createThinFunctionToPointer(loc, callback, rawPointerTy); OptionalTypeKind optKind; auto callbackTy = optCallbackTy.getAnyOptionalObjectType(SGM.M, optKind); callback = gen.B.createPointerToThinFunction(loc, callback, callbackTy); } callback = gen.B.createOptionalSome(loc, callback, optCallbackTy); } else { callback = gen.B.createOptionalNone(loc, optCallbackTy); } // Form the result and return. auto result = gen.B.createTuple(loc, resultTupleTy, { address, callback }); gen.Cleanups.emitCleanupsForReturn(CleanupLocation::get(loc)); gen.B.createReturn(loc, result); }
/// \brief Make sure that all parameters are passed with a reference count /// neutral parameter convention except for self. bool swift::ArraySemanticsCall::isValidSignature() { assert(SemanticsCall && getKind() != ArrayCallKind::kNone && "Need an array semantic call"); FunctionRefInst *FRI = cast<FunctionRefInst>(SemanticsCall->getCallee()); SILFunction *F = FRI->getReferencedFunction(); auto FnTy = F->getLoweredFunctionType(); auto &Mod = F->getModule(); // Check whether we have a valid signature for semantic calls that we hoist. switch (getKind()) { // All other calls can be consider valid. default: break; case ArrayCallKind::kArrayPropsIsNativeTypeChecked: { // @guaranteed/@owned Self if (SemanticsCall->getNumArguments() != 1) return false; auto SelfConvention = FnTy->getSelfParameter().getConvention(); return SelfConvention == ParameterConvention::Direct_Guaranteed || SelfConvention == ParameterConvention::Direct_Owned; } case ArrayCallKind::kCheckIndex: { // Int, @guaranteed/@owned Self if (SemanticsCall->getNumArguments() != 2 || !SemanticsCall->getArgument(0).getType().isTrivial(Mod)) return false; auto SelfConvention = FnTy->getSelfParameter().getConvention(); return SelfConvention == ParameterConvention::Direct_Guaranteed || SelfConvention == ParameterConvention::Direct_Owned; } case ArrayCallKind::kCheckSubscript: { // Int, Bool, Self if (SemanticsCall->getNumArguments() != 3 || !SemanticsCall->getArgument(0).getType().isTrivial(Mod)) return false; if (!SemanticsCall->getArgument(1).getType().isTrivial(Mod)) return false; auto SelfConvention = FnTy->getSelfParameter().getConvention(); return SelfConvention == ParameterConvention::Direct_Guaranteed || SelfConvention == ParameterConvention::Direct_Owned; } case ArrayCallKind::kMakeMutable: { auto SelfConvention = FnTy->getSelfParameter().getConvention(); return SelfConvention == ParameterConvention::Indirect_Inout; } case ArrayCallKind::kArrayUninitialized: { // Make sure that if we are a _adoptStorage call that our storage is // uniquely referenced by us. SILValue Arg0 = SemanticsCall->getArgument(0); if (Arg0.getType().isExistentialType()) { auto *AllocBufferAI = dyn_cast<ApplyInst>(Arg0); if (!AllocBufferAI) return false; auto *AllocFn = AllocBufferAI->getCalleeFunction(); if (!AllocFn) return false; StringRef AllocFuncName = AllocFn->getName(); if (AllocFuncName != "swift_bufferAllocate" && AllocFuncName != "swift_bufferAllocateOnStack") return false; if (!hasOneNonDebugUse(*AllocBufferAI)) return false; } return true; } } return true; }
bool Devirtualizer::devirtualizeAppliesInFunction(SILFunction &F, ClassHierarchyAnalysis *CHA) { bool Changed = false; llvm::SmallVector<SILInstruction *, 8> DeadApplies; llvm::SmallVector<ApplySite, 8> NewApplies; for (auto &BB : F) { for (auto It = BB.begin(), End = BB.end(); It != End;) { auto &I = *It++; // Skip non-apply instructions. auto Apply = FullApplySite::isa(&I); if (!Apply) continue; auto NewInstPair = tryDevirtualizeApply(Apply, CHA); if (!NewInstPair.second) continue; Changed = true; auto *AI = Apply.getInstruction(); if (!isa<TryApplyInst>(AI)) AI->replaceAllUsesWith(NewInstPair.first); DeadApplies.push_back(AI); NewApplies.push_back(NewInstPair.second); } } // Remove all the now-dead applies. while (!DeadApplies.empty()) { auto *AI = DeadApplies.pop_back_val(); recursivelyDeleteTriviallyDeadInstructions(AI, true); } // For each new apply, attempt to link in function bodies if we do // not already have them, then notify the pass manager of the new // functions. // // We do this after deleting the old applies because otherwise we // hit verification errors in the linking code due to having // non-cond_br critical edges. while (!NewApplies.empty()) { auto Apply = NewApplies.pop_back_val(); auto *CalleeFn = Apply.getReferencedFunction(); assert(CalleeFn && "Expected devirtualized callee!"); // FIXME: Until we link everything in up front we need to ensure // that we link after devirtualizing in order to pull in // everything we reference from the stdlib. After we do that we // can move the notification code below back into the main loop // above. if (!CalleeFn->isDefinition()) F.getModule().linkFunction(CalleeFn, SILModule::LinkingMode::LinkAll); // We may not have optimized these functions yet, and it could // be beneficial to rerun some earlier passes on the current // function now that we've made these direct references visible. if (CalleeFn->isDefinition() && CalleeFn->shouldOptimize()) notifyPassManagerOfFunction(CalleeFn, nullptr); } return Changed; }
SILFunction *SILModule::findFunction(StringRef Name, SILLinkage Linkage) { assert((Linkage == SILLinkage::Public || Linkage == SILLinkage::PublicExternal) && "Only a lookup of public functions is supported currently"); SILFunction *F = nullptr; // First, check if there is a function with a required name in the // current module. SILFunction *CurF = lookUpFunction(Name); // Nothing to do if the current module has a required function // with a proper linkage already. if (CurF && CurF->getLinkage() == Linkage) { F = CurF; } else { assert((!CurF || CurF->getLinkage() != Linkage) && "hasFunction should be only called for functions that are not " "contained in the SILModule yet or do not have a required linkage"); } if (!F) { SILLinkerVisitor Visitor(*this, getSILLoader(), SILModule::LinkingMode::LinkNormal); if (CurF) { // Perform this lookup only if a function with a given // name is present in the current module. // This is done to reduce the amount of IO from the // swift module file. if (!Visitor.hasFunction(Name, Linkage)) return nullptr; // The function in the current module will be changed. F = CurF; } // If function with a given name wasn't seen anywhere yet // or if it is known to exist, perform a lookup. if (!F) { // Try to load the function from other modules. F = Visitor.lookupFunction(Name, Linkage); // Bail if nothing was found and we are not sure if // this function exists elsewhere. if (!F) return nullptr; assert(F && "SILFunction should be present in one of the modules"); assert(F->getLinkage() == Linkage && "SILFunction has a wrong linkage"); } } // If a function exists already and it is a non-optimizing // compilation, simply convert it into an external declaration, // so that a compiled version from the shared library is used. if (F->isDefinition() && F->getModule().getOptions().Optimization < SILOptions::SILOptMode::Optimize) { F->convertToDeclaration(); } if (F->isExternalDeclaration()) F->setSerialized(IsSerialized_t::IsNotSerialized); F->setLinkage(Linkage); return F; }
void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() { SILFunction *NewF = TransformDescriptor.OptimizedFunction.get(); SILBasicBlock *BB = &*NewF->begin(); SILBuilder Builder(BB->begin()); Builder.setCurrentDebugScope(BB->getParent()->getDebugScope()); unsigned TotalArgIndex = 0; for (ArgumentDescriptor &AD : TransformDescriptor.ArgumentDescList) { // If this argument descriptor was dead and we removed it, just skip it. Do // not increment the argument index. if (AD.WasErased) { continue; } // Simply continue if do not explode. if (!AD.Explode) { TransformDescriptor.AIM[TotalArgIndex] = AD.Index; ++TotalArgIndex; continue; } assert(!AD.IsEntirelyDead && "Should never see completely dead values here"); // OK, we need to explode this argument. unsigned ArgOffset = ++TotalArgIndex; unsigned OldArgIndex = ArgOffset - 1; llvm::SmallVector<SILValue, 8> LeafValues; // We do this in the same order as leaf types since ProjTree expects that // the order of leaf values matches the order of leaf types. llvm::SmallVector<const ProjectionTreeNode *, 8> LeafNodes; AD.ProjTree.getLiveLeafNodes(LeafNodes); for (auto *Node : LeafNodes) { auto OwnershipKind = *AD.getTransformedOwnershipKind(Node->getType()); LeafValues.push_back( BB->insertFunctionArgument(ArgOffset, Node->getType(), OwnershipKind, BB->getArgument(OldArgIndex)->getDecl())); TransformDescriptor.AIM[TotalArgIndex - 1] = AD.Index; ++ArgOffset; ++TotalArgIndex; } // Then go through the projection tree constructing aggregates and replacing // uses. AD.ProjTree.replaceValueUsesWithLeafUses( Builder, BB->getParent()->getLocation(), LeafValues); // We ignored debugvalue uses when we constructed the new arguments, in // order to preserve as much information as possible, we construct a new // value for OrigArg from the leaf values and use that in place of the // OrigArg. SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue( Builder, BB->getParent()->getLocation(), LeafValues); // Replace all uses of the original arg with the new value. SILArgument *OrigArg = BB->getArgument(OldArgIndex); OrigArg->replaceAllUsesWith(NewOrigArgValue); // Now erase the old argument since it does not have any uses. We also // decrement ArgOffset since we have one less argument now. BB->eraseArgument(OldArgIndex); --TotalArgIndex; } }
// Returns the callee of an apply_inst if it is basically inlineable. SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) { SILFunction *Callee = AI.getReferencedFunction(); if (!Callee) { return nullptr; } // Don't inline functions that are marked with the @_semantics or @effects // attribute if the inliner is asked not to inline them. if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) { if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) { return nullptr; } // The "availability" semantics attribute is treated like global-init. if (Callee->hasSemanticsAttrs() && WhatToInline != InlineSelection::Everything && Callee->hasSemanticsAttrThatStartsWith("availability")) { return nullptr; } } else if (Callee->isGlobalInit()) { if (WhatToInline != InlineSelection::Everything) { return nullptr; } } // We can't inline external declarations. if (Callee->empty() || Callee->isExternalDeclaration()) { return nullptr; } // Explicitly disabled inlining. if (Callee->getInlineStrategy() == NoInline) { return nullptr; } if (!Callee->shouldOptimize()) { return nullptr; } // We don't support this yet. if (AI.hasSubstitutions()) { return nullptr; } // We don't support inlining a function that binds dynamic self because we // have no mechanism to preserve the original function's local self metadata. if (computeMayBindDynamicSelf(Callee)) { return nullptr; } SILFunction *Caller = AI.getFunction(); // Detect self-recursive calls. if (Caller == Callee) { return nullptr; } // A non-fragile function may not be inlined into a fragile function. if (Caller->isFragile() && !Callee->isFragile()) { return nullptr; } // Inlining self-recursive functions into other functions can result // in excessive code duplication since we run the inliner multiple // times in our pipeline if (calleeIsSelfRecursive(Callee)) { return nullptr; } return Callee; }
/// Remove retain/release pairs around builtin "unsafeGuaranteed" instruction /// sequences. static bool removeGuaranteedRetainReleasePairs(SILFunction &F, RCIdentityFunctionInfo &RCIA) { DEBUG(llvm::dbgs() << "Running on function " << F.getName() << "\n"); bool Changed = false; for (auto &BB : F) { auto It = BB.begin(), End = BB.end(); llvm::DenseMap<SILValue, SILInstruction *> LastRetain; while (It != End) { auto *CurInst = &*It; ++It; // Memorize the last retain. if (isa<StrongRetainInst>(CurInst) || isa<RetainValueInst>(CurInst)) { LastRetain[RCIA.getRCIdentityRoot(CurInst->getOperand(0))] = CurInst; continue; } // Look for a builtin "unsafeGuaranteed" instruction. auto *UnsafeGuaranteedI = dyn_cast<BuiltinInst>(CurInst); if (!UnsafeGuaranteedI || !UnsafeGuaranteedI->getBuiltinKind() || *UnsafeGuaranteedI->getBuiltinKind() != BuiltinValueKind::UnsafeGuaranteed) continue; auto Opd = UnsafeGuaranteedI->getOperand(0); auto RCIdOpd = RCIA.getRCIdentityRoot(UnsafeGuaranteedI->getOperand(0)); if (!LastRetain.count(RCIdOpd)) { DEBUG(llvm::dbgs() << "LastRetain failed\n"); continue; } // This code is very conservative. Check that there is a matching retain // before the unsafeGuaranteed builtin with only retains inbetween. auto *LastRetainInst = LastRetain[RCIdOpd]; auto NextInstIter = std::next(SILBasicBlock::iterator(LastRetainInst)); while (NextInstIter != BB.end() && &*NextInstIter != CurInst && (isa<RetainValueInst>(*NextInstIter) || isa<StrongRetainInst>(*NextInstIter) || !NextInstIter->mayHaveSideEffects() || isa<DebugValueInst>(*NextInstIter) || isa<DebugValueAddrInst>(*NextInstIter))) ++NextInstIter; if (&*NextInstIter != CurInst) { DEBUG(llvm::dbgs() << "Last retain right before match failed\n"); continue; } DEBUG(llvm::dbgs() << "Saw " << *UnsafeGuaranteedI); DEBUG(llvm::dbgs() << " with operand " << *Opd); // Match the reference and token result. // %4 = builtin "unsafeGuaranteed"<Foo>(%0 : $Foo) // %5 = tuple_extract %4 : $(Foo, Builtin.Int8), 0 // %6 = tuple_extract %4 : $(Foo, Builtin.Int8), 1 SILInstruction *UnsafeGuaranteedValue; SILInstruction *UnsafeGuaranteedToken; std::tie(UnsafeGuaranteedValue, UnsafeGuaranteedToken) = getSingleUnsafeGuaranteedValueResult(UnsafeGuaranteedI); if (!UnsafeGuaranteedValue) { DEBUG(llvm::dbgs() << " no single unsafeGuaranteed value use\n"); continue; } // Look for a builtin "unsafeGuaranteedEnd" instruction that uses the // token. // builtin "unsafeGuaranteedEnd"(%6 : $Builtin.Int8) : $() BuiltinInst *UnsafeGuaranteedEndI = nullptr; for (auto *Operand : getNonDebugUses(UnsafeGuaranteedToken)) { if (UnsafeGuaranteedEndI) { DEBUG(llvm::dbgs() << " multiple unsafeGuaranteedEnd users\n"); UnsafeGuaranteedEndI = nullptr; break; } auto *BI = dyn_cast<BuiltinInst>(Operand->getUser()); if (!BI || !BI->getBuiltinKind() || *BI->getBuiltinKind() != BuiltinValueKind::UnsafeGuaranteedEnd) { DEBUG(llvm::dbgs() << " wrong unsafeGuaranteed token user " << *Operand->getUser()); break; } UnsafeGuaranteedEndI = BI; } if (!UnsafeGuaranteedEndI) { DEBUG(llvm::dbgs() << " no single unsafeGuaranteedEnd use found\n"); continue; } if (SILBasicBlock::iterator(UnsafeGuaranteedEndI) == UnsafeGuaranteedEndI->getParent()->end()) continue; // Find the release to match with the unsafeGuaranteedValue. auto &UnsafeGuaranteedEndBB = *UnsafeGuaranteedEndI->getParent(); auto LastRelease = findReleaseToMatchUnsafeGuaranteedValue( UnsafeGuaranteedEndI, UnsafeGuaranteedI, UnsafeGuaranteedValue, UnsafeGuaranteedEndBB, RCIA); if (!LastRelease) { DEBUG(llvm::dbgs() << " no release before/after unsafeGuaranteedEnd found\n"); continue; } SILInstruction *LastRelease = &*LastReleaseIt; // Restart iteration before the earliest instruction we remove. bool RestartAtBeginningOfBlock = false; auto LastRetainIt = SILBasicBlock::iterator(LastRetainInst); if (LastRetainIt != BB.begin()) { It = std::prev(LastRetainIt); } else RestartAtBeginningOfBlock = true; // Okay we found a post dominating release. Let's remove the // retain/unsafeGuaranteed/release combo. // LastRetainInst->eraseFromParent(); LastRelease->eraseFromParent(); UnsafeGuaranteedEndI->eraseFromParent(); deleteAllDebugUses(UnsafeGuaranteedValue); deleteAllDebugUses(UnsafeGuaranteedToken); deleteAllDebugUses(UnsafeGuaranteedI); UnsafeGuaranteedValue->replaceAllUsesWith(Opd); UnsafeGuaranteedValue->eraseFromParent(); UnsafeGuaranteedToken->eraseFromParent(); UnsafeGuaranteedI->replaceAllUsesWith(Opd); UnsafeGuaranteedI->eraseFromParent(); if (RestartAtBeginningOfBlock) ++It = BB.begin(); Changed = true; } } return Changed; }
/// Bridge argument types and adjust retain count conventions for an ObjC thunk. static SILFunctionType *emitObjCThunkArguments(SILGenFunction &gen, SILLocation loc, SILDeclRef thunk, SmallVectorImpl<SILValue> &args, SILValue &foreignErrorSlot, Optional<ForeignErrorConvention> &foreignError) { SILDeclRef native = thunk.asForeign(false); auto mod = gen.SGM.M.getSwiftModule(); auto subs = gen.F.getForwardingSubstitutions(); auto objcInfo = gen.SGM.Types.getConstantInfo(thunk); auto objcFnTy = objcInfo.SILFnType->substGenericArgs(gen.SGM.M, mod, subs); auto swiftInfo = gen.SGM.Types.getConstantInfo(native); auto swiftFnTy = swiftInfo.SILFnType->substGenericArgs(gen.SGM.M, mod, subs); // We must have the same context archetypes as the unthunked function. assert(objcInfo.ContextGenericParams == swiftInfo.ContextGenericParams); SmallVector<ManagedValue, 8> bridgedArgs; bridgedArgs.reserve(objcFnTy->getParameters().size()); SILFunction *orig = gen.SGM.getFunction(native, NotForDefinition); // Find the foreign error convention if we have one. if (orig->getLoweredFunctionType()->hasErrorResult()) { auto func = cast<AbstractFunctionDecl>(thunk.getDecl()); foreignError = func->getForeignErrorConvention(); assert(foreignError && "couldn't find foreign error convention!"); } // Emit the indirect result arguments, if any. // FIXME: we're just assuming that these match up exactly? for (auto indirectResult : objcFnTy->getIndirectResults()) { SILType argTy = gen.F.mapTypeIntoContext(indirectResult.getSILType()); auto arg = new (gen.F.getModule()) SILArgument(gen.F.begin(), argTy); args.push_back(arg); } // Emit the other arguments, taking ownership of arguments if necessary. auto inputs = objcFnTy->getParameters(); auto nativeInputs = swiftFnTy->getParameters(); assert(inputs.size() == nativeInputs.size() + unsigned(foreignError.hasValue())); for (unsigned i = 0, e = inputs.size(); i < e; ++i) { SILType argTy = gen.F.mapTypeIntoContext(inputs[i].getSILType()); SILValue arg = new(gen.F.getModule()) SILArgument(gen.F.begin(), argTy); // If this parameter is the foreign error slot, pull it out. // It does not correspond to a native argument. if (foreignError && i == foreignError->getErrorParameterIndex()) { foreignErrorSlot = arg; continue; } // If this parameter is deallocating, emit an unmanaged rvalue and // continue. The object has the deallocating bit set so retain, release is // irrelevant. if (inputs[i].isDeallocating()) { bridgedArgs.push_back(ManagedValue::forUnmanaged(arg)); continue; } // If the argument is a block, copy it. if (argTy.isBlockPointerCompatible()) { auto copy = gen.B.createCopyBlock(loc, arg); // If the argument is consumed, we're still responsible for releasing the // original. if (inputs[i].isConsumed()) gen.emitManagedRValueWithCleanup(arg); arg = copy; } // Convert the argument to +1 if necessary. else if (!inputs[i].isConsumed()) { arg = emitObjCUnconsumedArgument(gen, loc, arg); } auto managedArg = gen.emitManagedRValueWithCleanup(arg); bridgedArgs.push_back(managedArg); } assert(bridgedArgs.size() + unsigned(foreignError.hasValue()) == objcFnTy->getParameters().size() && "objc inputs don't match number of arguments?!"); assert(bridgedArgs.size() == swiftFnTy->getNumSILArguments() && "swift inputs don't match number of arguments?!"); assert((foreignErrorSlot || !foreignError) && "didn't find foreign error slot"); // Bridge the input types. Scope scope(gen.Cleanups, CleanupLocation::get(loc)); assert(bridgedArgs.size() == nativeInputs.size()); for (unsigned i = 0, size = bridgedArgs.size(); i < size; ++i) { SILType argTy = gen.F.mapTypeIntoContext( swiftFnTy->getParameters()[i].getSILType()); ManagedValue native = gen.emitBridgedToNativeValue(loc, bridgedArgs[i], SILFunctionTypeRepresentation::ObjCMethod, argTy.getSwiftType()); SILValue argValue; if (nativeInputs[i].isConsumed()) argValue = native.forward(gen); else argValue = native.getValue(); args.push_back(argValue); } return objcFnTy; }
// Returns the callee of an apply_inst if it is basically inlineable. SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) { SILFunction *Callee = AI.getReferencedFunction(); if (!Callee) { return nullptr; } // Don't inline functions that are marked with the @_semantics or @effects // attribute if the inliner is asked not to inline them. if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) { if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) { return nullptr; } // The "availability" semantics attribute is treated like global-init. if (Callee->hasSemanticsAttrs() && WhatToInline != InlineSelection::Everything && Callee->hasSemanticsAttrThatStartsWith("availability")) { return nullptr; } } else if (Callee->isGlobalInit()) { if (WhatToInline != InlineSelection::Everything) { return nullptr; } } // We can't inline external declarations. if (Callee->empty() || Callee->isExternalDeclaration()) { return nullptr; } // Explicitly disabled inlining. if (Callee->getInlineStrategy() == NoInline) { return nullptr; } if (!Callee->shouldOptimize()) { return nullptr; } // We don't support this yet. if (AI.hasSubstitutions()) return nullptr; SILFunction *Caller = AI.getFunction(); // We don't support inlining a function that binds dynamic self because we // have no mechanism to preserve the original function's local self metadata. if (mayBindDynamicSelf(Callee)) { // Check if passed Self is the same as the Self of the caller. // In this case, it is safe to inline because both functions // use the same Self. if (AI.hasSelfArgument() && Caller->hasSelfParam()) { auto CalleeSelf = stripCasts(AI.getSelfArgument()); auto CallerSelf = Caller->getSelfArgument(); if (CalleeSelf != SILValue(CallerSelf)) return nullptr; } else return nullptr; } // Detect self-recursive calls. if (Caller == Callee) { return nullptr; } // A non-fragile function may not be inlined into a fragile function. if (Caller->isFragile() && !Callee->hasValidLinkageForFragileInline()) { if (!Callee->hasValidLinkageForFragileRef()) { llvm::errs() << "caller: " << Caller->getName() << "\n"; llvm::errs() << "callee: " << Callee->getName() << "\n"; llvm_unreachable("Should never be inlining a resilient function into " "a fragile function"); } return nullptr; } // Inlining self-recursive functions into other functions can result // in excessive code duplication since we run the inliner multiple // times in our pipeline if (calleeIsSelfRecursive(Callee)) { return nullptr; } return Callee; }
bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI, Weight CallerWeight, ConstantTracker &callerTracker, int &NumCallerBlocks, bool IsGeneric) { SILFunction *Callee = AI.getReferencedFunction(); SILLoopInfo *LI = LA->get(Callee); ShortestPathAnalysis *SPA = getSPA(Callee, LI); assert(SPA->isValid()); ConstantTracker constTracker(Callee, &callerTracker, AI); DominanceInfo *DT = DA->get(Callee); SILBasicBlock *CalleeEntry = &Callee->front(); DominanceOrder domOrder(CalleeEntry, DT, Callee->size()); // Calculate the inlining cost of the callee. int CalleeCost = 0; int Benefit = 0; // Start with a base benefit. int BaseBenefit = RemovedCallBenefit; const SILOptions &Opts = Callee->getModule().getOptions(); // For some reason -Ounchecked can accept a higher base benefit without // increasing the code size too much. if (Opts.Optimization == SILOptions::SILOptMode::OptimizeUnchecked) BaseBenefit *= 2; CallerWeight.updateBenefit(Benefit, BaseBenefit); // Go through all blocks of the function, accumulate the cost and find // benefits. while (SILBasicBlock *block = domOrder.getNext()) { constTracker.beginBlock(); Weight BlockW = SPA->getWeight(block, CallerWeight); for (SILInstruction &I : *block) { constTracker.trackInst(&I); CalleeCost += (int)instructionInlineCost(I); if (FullApplySite AI = FullApplySite::isa(&I)) { // Check if the callee is passed as an argument. If so, increase the // threshold, because inlining will (probably) eliminate the closure. SILInstruction *def = constTracker.getDefInCaller(AI.getCallee()); if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def))) BlockW.updateBenefit(Benefit, RemovedClosureBenefit); } else if (auto *LI = dyn_cast<LoadInst>(&I)) { // Check if it's a load from a stack location in the caller. Such a load // might be optimized away if inlined. if (constTracker.isStackAddrInCaller(LI->getOperand())) BlockW.updateBenefit(Benefit, RemovedLoadBenefit); } else if (auto *SI = dyn_cast<StoreInst>(&I)) { // Check if it's a store to a stack location in the caller. Such a load // might be optimized away if inlined. if (constTracker.isStackAddrInCaller(SI->getDest())) BlockW.updateBenefit(Benefit, RemovedStoreBenefit); } else if (isa<StrongReleaseInst>(&I) || isa<ReleaseValueInst>(&I)) { SILValue Op = stripCasts(I.getOperand(0)); if (SILArgument *Arg = dyn_cast<SILArgument>(Op)) { if (Arg->isFunctionArg() && Arg->getArgumentConvention() == SILArgumentConvention::Direct_Guaranteed) { BlockW.updateBenefit(Benefit, RefCountBenefit); } } } else if (auto *BI = dyn_cast<BuiltinInst>(&I)) { if (BI->getBuiltinInfo().ID == BuiltinValueKind::OnFastPath) BlockW.updateBenefit(Benefit, FastPathBuiltinBenefit); } } // Don't count costs in blocks which are dead after inlining. SILBasicBlock *takenBlock = constTracker.getTakenBlock(block->getTerminator()); if (takenBlock) { BlockW.updateBenefit(Benefit, RemovedTerminatorBenefit); domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) { return child->getSinglePredecessor() != block || child == takenBlock; }); } else { domOrder.pushChildren(block); } } if (AI.getFunction()->isThunk()) { // Only inline trivial functions into thunks (which will not increase the // code size). if (CalleeCost > TrivialFunctionThreshold) return false; DEBUG( dumpCaller(AI.getFunction()); llvm::dbgs() << " decision {" << CalleeCost << " into thunk} " << Callee->getName() << '\n'; ); return true; }
/// Specialize a partial_apply by promoting the parameters indicated by /// indices. We expect these parameters to be replaced by stack address /// references. static PartialApplyInst * specializePartialApply(PartialApplyInst *PartialApply, ParamIndexList &PromotedParamIndices, bool &CFGChanged) { auto *FRI = cast<FunctionRefInst>(PartialApply->getCallee()); assert(FRI && "Expected a direct partial_apply!"); auto *F = FRI->getReferencedFunction(); assert(F && "Expected a referenced function!"); IsFragile_t Fragile = IsNotFragile; if (PartialApply->getFunction()->isFragile() && F->isFragile()) Fragile = IsFragile; std::string ClonedName = getClonedName(F, Fragile, PromotedParamIndices); auto &M = PartialApply->getModule(); SILFunction *ClonedFn; if (auto *PrevFn = M.lookUpFunction(ClonedName)) { assert(PrevFn->isFragile() == Fragile); ClonedFn = PrevFn; } else { // Clone the function the existing partial_apply references. PromotedParamCloner Cloner(F, Fragile, PromotedParamIndices, ClonedName); Cloner.populateCloned(); ClonedFn = Cloner.getCloned(); } // Now create the new partial_apply using the cloned function. llvm::SmallVector<SILValue, 16> Args; ValueLifetimeAnalysis::Frontier PAFrontier; // Promote the arguments that need promotion. for (auto &O : PartialApply->getArgumentOperands()) { auto ParamIndex = getParameterIndexForOperand(&O); if (!count(PromotedParamIndices, ParamIndex)) { Args.push_back(O.get()); continue; } // If this argument is promoted, it is a box that we're // turning into an address because we've proven we can // keep this value on the stack. The partial_apply had ownership // of this box so we must now release it explicitly when the // partial_apply is released. auto box = cast<AllocBoxInst>(O.get()); // If the box address has a MUI, route accesses through it so DI still // works. SILInstruction *promoted = nullptr; int numAddrUses = 0; for (Operand *BoxUse : box->getUses()) { if (auto *PBI = dyn_cast<ProjectBoxInst>(BoxUse->getUser())) { for (auto PBIUse : PBI->getUses()) { numAddrUses++; if (auto MUI = dyn_cast<MarkUninitializedInst>(PBIUse->getUser())) promoted = MUI; } } } assert((!promoted || numAddrUses == 1) && "box value used by mark_uninitialized but not exclusively!"); // We only reuse an existing project_box if it directly follows the // alloc_box. This makes sure that the project_box dominates the // partial_apply. if (!promoted) promoted = getOrCreateProjectBox(box); Args.push_back(promoted); if (PAFrontier.empty()) { ValueLifetimeAnalysis VLA(PartialApply); CFGChanged |= !VLA.computeFrontier(PAFrontier, ValueLifetimeAnalysis::AllowToModifyCFG); assert(!PAFrontier.empty() && "partial_apply must have at least one use " "to release the returned function"); } // Insert releases after each point where the partial_apply becomes dead. for (SILInstruction *FrontierInst : PAFrontier) { SILBuilderWithScope Builder(FrontierInst); Builder.emitStrongReleaseAndFold(PartialApply->getLoc(), O.get()); } } SILBuilderWithScope Builder(PartialApply); // Build the function_ref and partial_apply. SILValue FunctionRef = Builder.createFunctionRef(PartialApply->getLoc(), ClonedFn); CanSILFunctionType CanFnTy = ClonedFn->getLoweredFunctionType(); auto const &Subs = PartialApply->getSubstitutions(); CanSILFunctionType SubstCalleeTy = CanFnTy->substGenericArgs(M, M.getSwiftModule(), Subs); return Builder.createPartialApply(PartialApply->getLoc(), FunctionRef, SILType::getPrimitiveObjectType(SubstCalleeTy), PartialApply->getSubstitutions(), Args, PartialApply->getType()); }
/// Analyze the destructor for the class of ARI to see if any instructions in it /// could have side effects on the program outside the destructor. If it does /// not, then we can eliminate the destructor. static bool doesDestructorHaveSideEffects(AllocRefInst *ARI) { SILFunction *Fn = getDestructor(ARI); // If we can't find a constructor then assume it has side effects. if (!Fn) return true; // A destructor only has one argument, self. assert(Fn->begin()->getNumArguments() == 1 && "Destructor should have only one argument, self."); SILArgument *Self = Fn->begin()->getArgument(0); LLVM_DEBUG(llvm::dbgs() << " Analyzing destructor.\n"); // For each BB in the destructor... for (auto &BB : *Fn) // For each instruction I in BB... for (auto &I : BB) { LLVM_DEBUG(llvm::dbgs() << " Visiting: " << I); // If I has no side effects, we can ignore it. if (!I.mayHaveSideEffects()) { LLVM_DEBUG(llvm::dbgs() << " SAFE! Instruction has no side " "effects.\n"); continue; } // RefCounting operations on Self are ok since we are already in the // destructor. RefCountingOperations on other instructions could have side // effects though. if (auto *RefInst = dyn_cast<RefCountingInst>(&I)) { if (stripCasts(RefInst->getOperand(0)) == Self) { // For now all ref counting insts have 1 operand. Put in an assert // just in case. assert(RefInst->getNumOperands() == 1 && "Make sure RefInst only has one argument."); LLVM_DEBUG(llvm::dbgs() << " SAFE! Ref count operation on " "Self.\n"); continue; } else { LLVM_DEBUG(llvm::dbgs() << " UNSAFE! Ref count operation " "not on self.\n"); return true; } } // dealloc_stack can be ignored. if (isa<DeallocStackInst>(I)) { LLVM_DEBUG(llvm::dbgs() << " SAFE! dealloc_stack can be " "ignored.\n"); continue; } // dealloc_ref on self can be ignored, but dealloc_ref on anything else // cannot be eliminated. if (auto *DeallocRef = dyn_cast<DeallocRefInst>(&I)) { if (stripCasts(DeallocRef->getOperand()) == Self) { LLVM_DEBUG(llvm::dbgs() <<" SAFE! dealloc_ref on self.\n"); continue; } else { LLVM_DEBUG(llvm::dbgs() << " UNSAFE! dealloc_ref on value " "besides self.\n"); return true; } } // Storing into the object can be ignored. if (auto *SI = dyn_cast<StoreInst>(&I)) if (stripAddressProjections(SI->getDest()) == Self) { LLVM_DEBUG(llvm::dbgs() << " SAFE! Instruction is a store " "into self.\n"); continue; } LLVM_DEBUG(llvm::dbgs() << " UNSAFE! Unknown instruction.\n"); // Otherwise, we can't remove the deallocation completely. return true; } // We didn't find any side effects. return false; }
/// Specialize a partial_apply by promoting the parameters indicated by /// indices. We expect these parameters to be replaced by stack address /// references. static PartialApplyInst * specializePartialApply(PartialApplyInst *PartialApply, ParamIndexList &PromotedParamIndices) { auto *FRI = cast<FunctionRefInst>(PartialApply->getCallee()); assert(FRI && "Expected a direct partial_apply!"); auto *F = FRI->getReferencedFunction(); assert(F && "Expected a referenced function!"); std::string ClonedName = getClonedName(F, PromotedParamIndices); auto &M = PartialApply->getModule(); SILFunction *ClonedFn; if (auto *PrevFn = M.lookUpFunction(ClonedName)) { ClonedFn = PrevFn; } else { // Clone the function the existing partial_apply references. PromotedParamCloner Cloner(F, PromotedParamIndices, ClonedName); Cloner.populateCloned(); ClonedFn = Cloner.getCloned(); } // Now create the new partial_apply using the cloned function. llvm::SmallVector<SILValue, 16> Args; LifetimeTracker Lifetime(PartialApply); // Promote the arguments that need promotion. for (auto &O : PartialApply->getArgumentOperands()) { auto ParamIndex = getParameterIndexForOperand(&O); if (!std::count(PromotedParamIndices.begin(), PromotedParamIndices.end(), ParamIndex)) { Args.push_back(O.get()); continue; } auto Endpoints = Lifetime.getEndpoints(); // If this argument is promoted, it is a box that we're // turning into an address because we've proven we can // keep this value on the stack. The partial_apply had ownership // of this box so we must now release it explicitly when the // partial_apply is released. auto box = cast<AllocBoxInst>(O.get()); // If the box address has a MUI, route accesses through it so DI still // works. SILInstruction *promoted = nullptr; int numAddrUses = 0; for (Operand *BoxUse : box->getUses()) { if (auto *PBI = dyn_cast<ProjectBoxInst>(BoxUse->getUser())) { for (auto PBIUse : PBI->getUses()) { numAddrUses++; if (auto MUI = dyn_cast<MarkUninitializedInst>(PBIUse->getUser())) promoted = MUI; } } } assert((!promoted || numAddrUses == 1) && "box value used by mark_uninitialized but not exclusively!"); // We only reuse an existing project_box if it directly follows the // alloc_box. This makes sure that the project_box dominates the // partial_apply. if (!promoted) promoted = getOrCreateProjectBox(box); Args.push_back(promoted); // If the partial_apply is dead, insert a release after it. if (Endpoints.begin() == Endpoints.end()) { emitStrongReleaseAfter(O.get(), PartialApply); continue; } // Otherwise insert releases after each point where the // partial_apply becomes dead. for (auto *User : Endpoints) { assert((isa<StrongReleaseInst>(User) || isa<ApplyInst>(User)) && "Unexpected end of lifetime for partial_apply!"); emitStrongReleaseAfter(O.get(), User); } } SILBuilderWithScope Builder(PartialApply); // Build the function_ref and partial_apply. SILValue FunctionRef = Builder.createFunctionRef(PartialApply->getLoc(), ClonedFn); CanSILFunctionType CanFnTy = ClonedFn->getLoweredFunctionType(); auto const &Subs = PartialApply->getSubstitutions(); CanSILFunctionType SubstCalleeTy = CanFnTy->substGenericArgs(M, M.getSwiftModule(), Subs); return Builder.createPartialApply(PartialApply->getLoc(), FunctionRef, SILType::getPrimitiveObjectType(SubstCalleeTy), PartialApply->getSubstitutions(), Args, PartialApply->getType()); }
void ClosureSpecializer::gatherCallSites( SILFunction *Caller, llvm::SmallVectorImpl<ClosureInfo*> &ClosureCandidates, llvm::DenseSet<FullApplySite> &MultipleClosureAI) { // A set of apply inst that we have associated with a closure. We use this to // make sure that we do not handle call sites with multiple closure arguments. llvm::DenseSet<FullApplySite> VisitedAI; // For each basic block BB in Caller... for (auto &BB : *Caller) { // For each instruction II in BB... for (auto &II : BB) { // If II is not a closure that we support specializing, skip it... if (!isSupportedClosure(&II)) continue; ClosureInfo *CInfo = nullptr; // Go through all uses of our closure. for (auto *Use : II.getUses()) { // If this use is not an apply inst or an apply inst with // substitutions, there is nothing interesting for us to do, so // continue... auto AI = FullApplySite::isa(Use->getUser()); if (!AI || AI.hasSubstitutions()) continue; // Check if we have already associated this apply inst with a closure to // be specialized. We do not handle applies that take in multiple // closures at this time. if (!VisitedAI.insert(AI).second) { MultipleClosureAI.insert(AI); continue; } // If AI does not have a function_ref definition as its callee, we can // not do anything here... so continue... SILFunction *ApplyCallee = AI.getReferencedFunction(); if (!ApplyCallee || ApplyCallee->isExternalDeclaration()) continue; // Ok, we know that we can perform the optimization but not whether or // not the optimization is profitable. Find the index of the argument // corresponding to our partial apply. Optional<unsigned> ClosureIndex; for (unsigned i = 0, e = AI.getNumArguments(); i != e; ++i) { if (AI.getArgument(i) != SILValue(&II)) continue; ClosureIndex = i; DEBUG(llvm::dbgs() << " Found callsite with closure argument at " << i << ": " << *AI.getInstruction()); break; } // If we did not find an index, there is nothing further to do, // continue. if (!ClosureIndex.hasValue()) continue; // Make sure that the Closure is invoked in the Apply's callee. We only // want to perform closure specialization if we know that we will be // able to change a partial_apply into an apply. // // TODO: Maybe just call the function directly instead of moving the // partial apply? SILValue Arg = ApplyCallee->getArgument(ClosureIndex.getValue()); if (std::none_of(Arg->use_begin(), Arg->use_end(), [&Arg](Operand *Op) -> bool { auto UserAI = FullApplySite::isa(Op->getUser()); return UserAI && UserAI.getCallee() == Arg; })) { continue; } auto NumIndirectResults = AI.getSubstCalleeType()->getNumIndirectResults(); assert(ClosureIndex.getValue() >= NumIndirectResults); auto ClosureParamIndex = ClosureIndex.getValue() - NumIndirectResults; auto ParamInfo = AI.getSubstCalleeType()->getParameters(); SILParameterInfo ClosureParamInfo = ParamInfo[ClosureParamIndex]; // Get all non-failure exit BBs in the Apply Callee if our partial apply // is guaranteed. If we do not understand one of the exit BBs, bail. // // We need this to make sure that we insert a release in the appropriate // locations to balance the +1 from the creation of the partial apply. llvm::TinyPtrVector<SILBasicBlock *> NonFailureExitBBs; if (ClosureParamInfo.isGuaranteed() && !findAllNonFailureExitBBs(ApplyCallee, NonFailureExitBBs)) { continue; } // Compute the final release points of the closure. We will insert // release of the captured arguments here. if (!CInfo) { CInfo = new ClosureInfo(&II); ValueLifetimeAnalysis VLA(CInfo->Closure); VLA.computeFrontier(CInfo->LifetimeFrontier, ValueLifetimeAnalysis::AllowToModifyCFG); } // Now we know that CSDesc is profitable to specialize. Add it to our // call site list. CInfo->CallSites.push_back( CallSiteDescriptor(CInfo, AI, ClosureIndex.getValue(), ClosureParamInfo, std::move(NonFailureExitBBs))); } if (CInfo) ClosureCandidates.push_back(CInfo); } } }
/// \brief Attempt to inline all calls smaller than our threshold. /// returns True if a function was inlined. bool SILPerformanceInliner::inlineCallsIntoFunction(SILFunction *Caller, DominanceAnalysis *DA, SILLoopAnalysis *LA, llvm::SmallVectorImpl<FullApplySite> &NewApplies) { // Don't optimize functions that are marked with the opt.never attribute. if (!Caller->shouldOptimize()) return false; // Construct a log of all of the names of the functions that we've inlined // in the current iteration. SmallVector<StringRef, 16> InlinedFunctionNames; StringRef CallerName = Caller->getName(); DEBUG(llvm::dbgs() << "Visiting Function: " << CallerName << "\n"); assert(NewApplies.empty() && "Expected empty vector to store results in!"); // First step: collect all the functions we want to inline. We // don't change anything yet so that the dominator information // remains valid. SmallVector<FullApplySite, 8> AppliesToInline; collectAppliesToInline(Caller, AppliesToInline, DA, LA); if (AppliesToInline.empty()) return false; // Second step: do the actual inlining. for (auto AI : AppliesToInline) { SILFunction *Callee = AI.getCalleeFunction(); assert(Callee && "apply_inst does not have a direct callee anymore"); DEBUG(llvm::dbgs() << " Inline:" << *AI.getInstruction()); if (!Callee->shouldOptimize()) { DEBUG(llvm::dbgs() << " Cannot inline function " << Callee->getName() << " marked to be excluded from optimizations.\n"); continue; } SmallVector<SILValue, 8> Args; for (const auto &Arg : AI.getArguments()) Args.push_back(Arg); // As we inline and clone we need to collect new applies. auto Filter = [](SILInstruction *I) -> bool { return bool(FullApplySite::isa(I)); }; CloneCollector Collector(Filter); // Notice that we will skip all of the newly inlined ApplyInsts. That's // okay because we will visit them in our next invocation of the inliner. TypeSubstitutionMap ContextSubs; SILInliner Inliner(*Caller, *Callee, SILInliner::InlineKind::PerformanceInline, ContextSubs, AI.getSubstitutions(), Collector.getCallback()); // Record the name of the inlined function (for cycle detection). InlinedFunctionNames.push_back(Callee->getName()); auto Success = Inliner.inlineFunction(AI, Args); (void) Success; // We've already determined we should be able to inline this, so // we expect it to have happened. assert(Success && "Expected inliner to inline this function!"); llvm::SmallVector<FullApplySite, 4> AppliesFromInlinee; for (auto &P : Collector.getInstructionPairs()) AppliesFromInlinee.push_back(FullApplySite(P.first)); recursivelyDeleteTriviallyDeadInstructions(AI.getInstruction(), true); NewApplies.insert(NewApplies.end(), AppliesFromInlinee.begin(), AppliesFromInlinee.end()); DA->invalidate(Caller, SILAnalysis::InvalidationKind::Everything); NumFunctionsInlined++; } // Record the names of the functions that we inlined. // We'll use this list to detect cycles in future iterations of // the inliner. for (auto CalleeName : InlinedFunctionNames) { InlinedFunctions.insert(std::make_pair(CallerName, CalleeName)); } DEBUG(llvm::dbgs() << "\n"); return true; }
static void createThunkBody(SILBasicBlock *BB, SILFunction *NewF, FunctionAnalyzer &Analyzer) { // TODO: What is the proper location to use here? SILLocation Loc = BB->getParent()->getLocation(); SILBuilder Builder(BB); Builder.setCurrentDebugScope(BB->getParent()->getDebugScope()); FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF); // Create the args for the thunk's apply, ignoring any dead arguments. llvm::SmallVector<SILValue, 8> ThunkArgs; ArrayRef<ArgumentDescriptor> ArgDescs = Analyzer.getArgDescList(); for (auto &ArgDesc : ArgDescs) { ArgDesc.addThunkArgs(Builder, BB, ThunkArgs); } // We are ignoring generic functions and functions with out parameters for // now. SILType LoweredType = NewF->getLoweredType(); SILType ResultType = LoweredType.getFunctionInterfaceResultType(); SILValue ReturnValue; auto FunctionTy = LoweredType.castTo<SILFunctionType>(); if (FunctionTy->hasErrorResult()) { // We need a try_apply to call a function with an error result. SILFunction *Thunk = BB->getParent(); SILBasicBlock *NormalBlock = Thunk->createBasicBlock(); ReturnValue = NormalBlock->createBBArg(ResultType, 0); SILBasicBlock *ErrorBlock = Thunk->createBasicBlock(); SILType ErrorType = SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType()); auto *ErrorArg = ErrorBlock->createBBArg(ErrorType, 0); Builder.createTryApply(Loc, FRI, LoweredType, ArrayRef<Substitution>(), ThunkArgs, NormalBlock, ErrorBlock); // If we have any arguments that were consumed but are now guaranteed, // insert a release_value in the error block. Builder.setInsertionPoint(ErrorBlock); for (auto &ArgDesc : ArgDescs) { if (!ArgDesc.CalleeRelease) continue; Builder.createReleaseValue(Loc, BB->getBBArg(ArgDesc.Index)); } Builder.createThrow(Loc, ErrorArg); // Also insert release_value in the normal block (done below). Builder.setInsertionPoint(NormalBlock); } else { ReturnValue = Builder.createApply(Loc, FRI, LoweredType, ResultType, ArrayRef<Substitution>(), ThunkArgs, false); } // If we have any arguments that were consumed but are now guaranteed, // insert a release_value. for (auto &ArgDesc : ArgDescs) { if (!ArgDesc.CalleeRelease) continue; Builder.createReleaseValue(Loc, BB->getBBArg(ArgDesc.Index)); } // Function that are marked as @NoReturn must be followed by an 'unreachable' // instruction. if (NewF->getLoweredFunctionType()->isNoReturn()) { Builder.createUnreachable(Loc); return; } Builder.createReturn(Loc, ReturnValue); }
void SILPerformanceInliner::inlineDevirtualizeAndSpecialize( SILFunction *Caller, SILModuleTransform *MT, DominanceAnalysis *DA, SILLoopAnalysis *LA, ClassHierarchyAnalysis *CHA) { assert(Caller->isDefinition() && "Expected only functions with bodies!"); llvm::SmallVector<SILFunction *, 4> WorkList; WorkList.push_back(Caller); while (!WorkList.empty()) { llvm::SmallVector<ApplySite, 4> WorkItemApplies; SILFunction *CurrentCaller = WorkList.back(); if (CurrentCaller->shouldOptimize()) collectAllAppliesInFunction(CurrentCaller, WorkItemApplies); // Devirtualize and specialize any applies we've collected, // and collect new functions we should inline into as we do // so. llvm::SmallVector<SILFunction *, 4> NewFuncs; if (devirtualizeAndSpecializeApplies(WorkItemApplies, MT, CHA, NewFuncs)) { WorkList.insert(WorkList.end(), NewFuncs.begin(), NewFuncs.end()); NewFuncs.clear(); } assert(WorkItemApplies.empty() && "Expected all applies to be processed!"); // We want to inline into each function on the worklist, starting // with any new ones that were exposed as a result of // devirtualization (to insure we're inlining into callees first). // // After inlining, we may have new opportunities for // devirtualization, e.g. as a result of exposing the dynamic type // of an object. When those opportunities arise we want to attempt // devirtualization and then again attempt to inline into the // newly exposed functions, etc. until we're back to the function // we began with. auto *Initial = WorkList.back(); // In practice we rarely exceed 5, but in a perf test we iterate 51 times. const unsigned MaxLaps = 1500; unsigned Lap = 0; while (1) { auto *WorkItem = WorkList.back(); assert(WorkItem->isDefinition() && "Expected function definition on work list!"); // Devirtualization and specialization might have exposed new // function references. We want to inline within those functions // before inlining within our original function. // // Inlining in turn might result in new applies that we should // consider for devirtualization and specialization. llvm::SmallVector<FullApplySite, 4> NewApplies; bool Inlined = inlineCallsIntoFunction(WorkItem, DA, LA, NewApplies); if (Inlined) { MT->invalidateAnalysis(WorkItem, SILAnalysis::InvalidationKind::FunctionBody); // FIXME: Update inlineCallsIntoFunction to collect all // remaining applies after inlining, not just those // resulting from inlining code. llvm::SmallVector<ApplySite, 4> WorkItemApplies; collectAllAppliesInFunction(WorkItem, WorkItemApplies); bool Modified = devirtualizeAndSpecializeApplies(WorkItemApplies, MT, CHA, NewFuncs); if (Modified) { WorkList.insert(WorkList.end(), NewFuncs.begin(), NewFuncs.end()); NewFuncs.clear(); assert(WorkItemApplies.empty() && "Expected all applies to be processed!"); } else if (WorkItem == Initial) { // We did not specialize generics or devirtualize calls and we // did not create new opportunities so we can bail out now. break; } else { WorkList.pop_back(); } } else if (WorkItem == Initial) { // We did not inline any calls and did not create new opportunities // so we can bail out now. break; } else { WorkList.pop_back(); } Lap++; // It's possible to construct real code where this will hit, but // it's more likely that there is an issue tracking recursive // inlining, in which case we want to know about it in internal // builds, and not hang on bots or user machines. assert(Lap <= MaxLaps && "Possible bug tracking recursion!"); // Give up and move along. if (Lap > MaxLaps) { while (WorkList.back() != Initial) WorkList.pop_back(); break; } } assert(WorkList.back() == Initial && "Expected to exit with same element on top of stack!" ); WorkList.pop_back(); } }
static llvm::Function * getAccessorForComputedComponent(IRGenModule &IGM, const KeyPathPatternComponent &component, KeyPathAccessor whichAccessor, GenericEnvironment *genericEnv, ArrayRef<GenericRequirement> requirements) { SILFunction *accessor; switch (whichAccessor) { case Getter: accessor = component.getComputedPropertyGetter(); break; case Setter: accessor = component.getComputedPropertySetter(); break; case Equals: accessor = component.getSubscriptIndexEquals(); break; case Hash: accessor = component.getSubscriptIndexHash(); break; } auto accessorFn = IGM.getAddrOfSILFunction(accessor, NotForDefinition); // If the accessor is not generic, we can use it as is. if (requirements.empty()) { return accessorFn; } auto accessorFnTy = cast<llvm::FunctionType>( accessorFn->getType()->getPointerElementType());; // Otherwise, we need a thunk to unmarshal the generic environment from the // argument area. It'd be nice to have a good way to represent this // directly in SIL, of course... const char *thunkName; unsigned numArgsToForward; switch (whichAccessor) { case Getter: thunkName = "keypath_get"; numArgsToForward = 2; break; case Setter: thunkName = "keypath_set"; numArgsToForward = 2; break; case Equals: thunkName = "keypath_equals"; numArgsToForward = 2; break; case Hash: thunkName = "keypath_hash"; numArgsToForward = 1; break; } SmallVector<llvm::Type *, 4> thunkParams; for (unsigned i = 0; i < numArgsToForward; ++i) thunkParams.push_back(accessorFnTy->getParamType(i)); switch (whichAccessor) { case Getter: case Setter: thunkParams.push_back(IGM.Int8PtrTy); break; case Equals: case Hash: break; } thunkParams.push_back(IGM.SizeTy); auto thunkType = llvm::FunctionType::get(accessorFnTy->getReturnType(), thunkParams, /*vararg*/ false); auto accessorThunk = llvm::Function::Create(thunkType, llvm::GlobalValue::PrivateLinkage, thunkName, IGM.getModule()); accessorThunk->setAttributes(IGM.constructInitialAttributes()); accessorThunk->setCallingConv(IGM.SwiftCC); switch (whichAccessor) { case Getter: // Original accessor's args should be @in or @out, meaning they won't be // captured or aliased. accessorThunk->addAttribute(1, llvm::Attribute::NoCapture); accessorThunk->addAttribute(1, llvm::Attribute::NoAlias); accessorThunk->addAttribute(2, llvm::Attribute::NoCapture); accessorThunk->addAttribute(2, llvm::Attribute::NoAlias); // Output is sret. accessorThunk->addAttribute(1, llvm::Attribute::StructRet); break; case Setter: // Original accessor's args should be @in or @out, meaning they won't be // captured or aliased. accessorThunk->addAttribute(1, llvm::Attribute::NoCapture); accessorThunk->addAttribute(1, llvm::Attribute::NoAlias); accessorThunk->addAttribute(2, llvm::Attribute::NoCapture); accessorThunk->addAttribute(2, llvm::Attribute::NoAlias); break; case Equals: case Hash: break; } { IRGenFunction IGF(IGM, accessorThunk); if (IGM.DebugInfo) IGM.DebugInfo->emitArtificialFunction(IGF, accessorThunk); auto params = IGF.collectParameters(); Explosion forwardedArgs; forwardedArgs.add(params.claim(numArgsToForward)); llvm::Value *componentArgsBuf; switch (whichAccessor) { case Getter: case Setter: // The component arguments are passed alongside the base being projected. componentArgsBuf = params.claimNext(); // Pass the argument pointer down to the underlying function. if (!component.getSubscriptIndices().empty()) { forwardedArgs.add(componentArgsBuf); } break; case Equals: case Hash: // We're operating directly on the component argument buffer. componentArgsBuf = forwardedArgs.getAll()[0]; break; } auto componentArgsBufSize = params.claimNext(); bindPolymorphicArgumentsFromComponentIndices(IGF, component, genericEnv, requirements, componentArgsBuf, componentArgsBufSize); // Use the bound generic metadata to form a call to the original generic // accessor. WitnessMetadata ignoreWitnessMetadata; auto forwardingSubs = genericEnv->getGenericSignature()->getSubstitutionMap( genericEnv->getForwardingSubstitutions()); emitPolymorphicArguments(IGF, accessor->getLoweredFunctionType(), forwardingSubs, &ignoreWitnessMetadata, forwardedArgs); auto fnPtr = FunctionPointer::forDirect(IGM, accessorFn, accessor->getLoweredFunctionType()); auto call = IGF.Builder.CreateCall(fnPtr, forwardedArgs.claimAll()); if (call->getType()->isVoidTy()) IGF.Builder.CreateRetVoid(); else IGF.Builder.CreateRet(call); } return accessorThunk; }
// Returns the callee of an apply_inst if it is basically inlineable. SILFunction *SILPerformanceInliner::getEligibleFunction(FullApplySite AI) { SILFunction *Callee = AI.getCalleeFunction(); if (!Callee) { DEBUG(llvm::dbgs() << " FAIL: Cannot find inlineable callee.\n"); return nullptr; } // Don't inline functions that are marked with the @_semantics or @effects // attribute if the inliner is asked not to inline them. if (Callee->hasSemanticsAttrs() || Callee->hasEffectsKind()) { if (WhatToInline == InlineSelection::NoSemanticsAndGlobalInit) { DEBUG(llvm::dbgs() << " FAIL: Function " << Callee->getName() << " has special semantics or effects attribute.\n"); return nullptr; } // The "availability" semantics attribute is treated like global-init. if (Callee->hasSemanticsAttrs() && WhatToInline != InlineSelection::Everything && Callee->hasSemanticsAttrThatStartsWith("availability")) { return nullptr; } } else if (Callee->isGlobalInit()) { if (WhatToInline != InlineSelection::Everything) { DEBUG(llvm::dbgs() << " FAIL: Function " << Callee->getName() << " has the global-init attribute.\n"); return nullptr; } } // We can't inline external declarations. if (Callee->empty() || Callee->isExternalDeclaration()) { DEBUG(llvm::dbgs() << " FAIL: Cannot inline external " << Callee->getName() << ".\n"); return nullptr; } // Explicitly disabled inlining. if (Callee->getInlineStrategy() == NoInline) { DEBUG(llvm::dbgs() << " FAIL: noinline attribute on " << Callee->getName() << ".\n"); return nullptr; } if (!Callee->shouldOptimize()) { DEBUG(llvm::dbgs() << " FAIL: optimizations disabled on " << Callee->getName() << ".\n"); return nullptr; } // We don't support this yet. if (AI.hasSubstitutions()) { DEBUG(llvm::dbgs() << " FAIL: Generic substitutions on " << Callee->getName() << ".\n"); return nullptr; } // We don't support inlining a function that binds dynamic self because we // have no mechanism to preserve the original function's local self metadata. if (computeMayBindDynamicSelf(Callee)) { DEBUG(llvm::dbgs() << " FAIL: Binding dynamic Self in " << Callee->getName() << ".\n"); return nullptr; } SILFunction *Caller = AI.getFunction(); // Detect inlining cycles. if (hasInliningCycle(Caller, Callee)) { DEBUG(llvm::dbgs() << " FAIL: Detected a recursion inlining " << Callee->getName() << ".\n"); return nullptr; } // A non-fragile function may not be inlined into a fragile function. if (Caller->isFragile() && !Callee->isFragile()) { DEBUG(llvm::dbgs() << " FAIL: Can't inline fragile " << Callee->getName() << ".\n"); return nullptr; } // Inlining self-recursive functions into other functions can result // in excessive code duplication since we run the inliner multiple // times in our pipeline, so we only do it for callees with few // self-recursive calls. if (calleeHasMinimalSelfRecursion(Callee)) { DEBUG(llvm::dbgs() << " FAIL: Callee is self-recursive in " << Callee->getName() << ".\n"); return nullptr; } DEBUG(llvm::dbgs() << " Eligible callee: " << Callee->getName() << "\n"); return Callee; }
/// Return true if inlining this call site is profitable. bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI, unsigned loopDepthOfAI, DominanceAnalysis *DA, SILLoopAnalysis *LA, ConstantTracker &callerTracker, unsigned &NumCallerBlocks) { SILFunction *Callee = AI.getReferencedFunction(); if (Callee->getInlineStrategy() == AlwaysInline) return true; ConstantTracker constTracker(Callee, &callerTracker, AI); DominanceInfo *DT = DA->get(Callee); SILLoopInfo *LI = LA->get(Callee); DominanceOrder domOrder(&Callee->front(), DT, Callee->size()); // Calculate the inlining cost of the callee. unsigned CalleeCost = 0; unsigned Benefit = InlineCostThreshold > 0 ? InlineCostThreshold : RemovedCallBenefit; Benefit += loopDepthOfAI * LoopBenefitFactor; int testThreshold = TestThreshold; while (SILBasicBlock *block = domOrder.getNext()) { constTracker.beginBlock(); for (SILInstruction &I : *block) { constTracker.trackInst(&I); if (testThreshold >= 0) { // We are in test-mode: use a simplified cost model. CalleeCost += testCost(&I); } else { // Use the regular cost model. CalleeCost += unsigned(instructionInlineCost(I)); } if (ApplyInst *AI = dyn_cast<ApplyInst>(&I)) { // Check if the callee is passed as an argument. If so, increase the // threshold, because inlining will (probably) eliminate the closure. SILInstruction *def = constTracker.getDefInCaller(AI->getCallee()); if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def))) { unsigned loopDepth = LI->getLoopDepth(block); Benefit += ConstCalleeBenefit + loopDepth * LoopBenefitFactor; testThreshold *= 2; } } } // Don't count costs in blocks which are dead after inlining. SILBasicBlock *takenBlock = getTakenBlock(block->getTerminator(), constTracker); if (takenBlock) { Benefit += ConstTerminatorBenefit; domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) { return child->getSinglePredecessor() != block || child == takenBlock; }); } else { domOrder.pushChildren(block); } } unsigned Threshold = Benefit; // The default. if (testThreshold >= 0) { // We are in testing mode. Threshold = testThreshold; } else if (AI.getFunction()->isThunk()) { // Only inline trivial functions into thunks (which will not increase the // code size). Threshold = TrivialFunctionThreshold; } else { // The default case. // We reduce the benefit if the caller is too large. For this we use a // cubic function on the number of caller blocks. This starts to prevent // inlining at about 800 - 1000 caller blocks. unsigned blockMinus = (NumCallerBlocks * NumCallerBlocks) / BlockLimitDenominator * NumCallerBlocks / BlockLimitDenominator; if (Threshold > blockMinus + TrivialFunctionThreshold) Threshold -= blockMinus; else Threshold = TrivialFunctionThreshold; } if (CalleeCost > Threshold) { return false; } NumCallerBlocks += Callee->size(); DEBUG( dumpCaller(AI.getFunction()); llvm::dbgs() << " decision {" << CalleeCost << " < " << Threshold << ", ld=" << loopDepthOfAI << ", bb=" << NumCallerBlocks << "} " << Callee->getName() << '\n'; );
/// Return true if inlining this call site is profitable. bool SILPerformanceInliner::isProfitableToInline(FullApplySite AI, unsigned loopDepthOfAI, DominanceAnalysis *DA, SILLoopAnalysis *LA, ConstantTracker &callerTracker) { SILFunction *Callee = AI.getCalleeFunction(); if (Callee->getInlineStrategy() == AlwaysInline) return true; ConstantTracker constTracker(Callee, &callerTracker, AI); DominanceInfo *DT = DA->get(Callee); SILLoopInfo *LI = LA->get(Callee); DominanceOrder domOrder(&Callee->front(), DT, Callee->size()); // Calculate the inlining cost of the callee. unsigned CalleeCost = 0; unsigned Benefit = InlineCostThreshold > 0 ? InlineCostThreshold : RemovedCallBenefit; Benefit += loopDepthOfAI * LoopBenefitFactor; int testThreshold = TestThreshold; while (SILBasicBlock *block = domOrder.getNext()) { constTracker.beginBlock(); unsigned loopDepth = LI->getLoopDepth(block); for (SILInstruction &I : *block) { constTracker.trackInst(&I); auto ICost = instructionInlineCost(I); if (testThreshold >= 0) { // We are in test-mode: use a simplified cost model. CalleeCost += testCost(&I); } else { // Use the regular cost model. CalleeCost += unsigned(ICost); } if (ApplyInst *AI = dyn_cast<ApplyInst>(&I)) { // Check if the callee is passed as an argument. If so, increase the // threshold, because inlining will (probably) eliminate the closure. SILInstruction *def = constTracker.getDefInCaller(AI->getCallee()); if (def && (isa<FunctionRefInst>(def) || isa<PartialApplyInst>(def))) { DEBUG(llvm::dbgs() << " Boost: apply const function at" << *AI); Benefit += ConstCalleeBenefit + loopDepth * LoopBenefitFactor; testThreshold *= 2; } } } // Don't count costs in blocks which are dead after inlining. SILBasicBlock *takenBlock = getTakenBlock(block->getTerminator(), constTracker); if (takenBlock) { Benefit += ConstTerminatorBenefit + TestOpt; DEBUG(llvm::dbgs() << " Take bb" << takenBlock->getDebugID() << " of" << *block->getTerminator()); domOrder.pushChildrenIf(block, [=] (SILBasicBlock *child) { return child->getSinglePredecessor() != block || child == takenBlock; }); } else { domOrder.pushChildren(block); } } unsigned Threshold = Benefit; // The default. if (testThreshold >= 0) { // We are in testing mode. Threshold = testThreshold; } else if (AI.getFunction()->isThunk()) { // Only inline trivial functions into thunks (which will not increase the // code size). Threshold = TrivialFunctionThreshold; } if (CalleeCost > Threshold) { DEBUG(llvm::dbgs() << " NO: Function too big to inline, " "cost: " << CalleeCost << ", threshold: " << Threshold << "\n"); return false; } DEBUG(llvm::dbgs() << " YES: ready to inline, " "cost: " << CalleeCost << ", threshold: " << Threshold << "\n"); return true; }
static bool removeUnreachableBlocks(SILFunction &F, SILModule &M, UnreachableUserCodeReportingState *State) { if (F.empty()) return false; SILBasicBlockSet Reachable; SmallVector<SILBasicBlock*, 128> Worklist; Worklist.push_back(&F.front()); Reachable.insert(&F.front()); // Collect all reachable blocks by walking the successors. do { SILBasicBlock *BB = Worklist.pop_back_val(); for (auto SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) { if (Reachable.insert(*SI).second) Worklist.push_back(*SI); } } while (!Worklist.empty()); assert(Reachable.size() <= F.size()); // If everything is reachable, we are done. if (Reachable.size() == F.size()) return false; // Diagnose user written unreachable code. if (State) { for (auto BI = State->PossiblyUnreachableBlocks.begin(), BE = State->PossiblyUnreachableBlocks.end(); BI != BE; ++BI) { const SILBasicBlock *BB = *BI; if (!Reachable.count(BB)) { llvm::SmallPtrSet<const SILBasicBlock *, 1> visited; diagnoseUnreachableBlock(**BI, M, Reachable, State, BB, visited); } } } // Remove references from the dead blocks. for (auto I = F.begin(), E = F.end(); I != E; ++I) { SILBasicBlock *BB = &*I; if (Reachable.count(BB)) continue; // Drop references to other blocks. recursivelyDeleteTriviallyDeadInstructions(BB->getTerminator(), true); NumInstructionsRemoved++; } // Delete dead instructions and everything that could become dead after // their deletion. llvm::SmallVector<SILInstruction*, 32> ToBeDeleted; for (auto BI = F.begin(), BE = F.end(); BI != BE; ++BI) if (!Reachable.count(&*BI)) for (auto I = BI->begin(), E = BI->end(); I != E; ++I) ToBeDeleted.push_back(&*I); recursivelyDeleteTriviallyDeadInstructions(ToBeDeleted, true); NumInstructionsRemoved += ToBeDeleted.size(); // Delete the dead blocks. for (auto I = F.begin(), E = F.end(); I != E;) if (!Reachable.count(&*I)) { I = F.getBlocks().erase(I); NumBlocksRemoved++; } else ++I; return true; }
void FunctionSignatureTransform::createFunctionSignatureOptimizedFunction() { // Create the optimized function ! SILModule &M = F->getModule(); std::string Name = getUniqueName(createOptimizedSILFunctionName(), M); NewF = M.createFunction( F->getLinkage(), Name, createOptimizedSILFunctionType(), nullptr, F->getLocation(), F->isBare(), F->isTransparent(), F->isFragile(), F->isThunk(), F->getClassVisibility(), F->getInlineStrategy(), F->getEffectsKind(), 0, F->getDebugScope(), F->getDeclContext()); // Then we transfer the body of F to NewF. NewF->spliceBody(F); NewF->setDeclCtx(F->getDeclContext()); // Array semantic clients rely on the signature being as in the original // version. for (auto &Attr : F->getSemanticsAttrs()) { if (!StringRef(Attr).startswith("array.")) NewF->addSemanticsAttr(Attr); } // Do the last bit of work to the newly created optimized function. ArgumentExplosionFinalizeOptimizedFunction(); DeadArgumentFinalizeOptimizedFunction(); // Create the thunk body ! F->setThunk(IsThunk); // The thunk now carries the information on how the signature is // optimized. If we inline the thunk, we will get the benefit of calling // the signature optimized function without additional setup on the // caller side. F->setInlineStrategy(AlwaysInline); SILBasicBlock *ThunkBody = F->createBasicBlock(); for (auto &ArgDesc : ArgumentDescList) { ThunkBody->createBBArg(ArgDesc.Arg->getType(), ArgDesc.Decl); } SILLocation Loc = ThunkBody->getParent()->getLocation(); SILBuilder Builder(ThunkBody); Builder.setCurrentDebugScope(ThunkBody->getParent()->getDebugScope()); FunctionRefInst *FRI = Builder.createFunctionRef(Loc, NewF); // Create the args for the thunk's apply, ignoring any dead arguments. llvm::SmallVector<SILValue, 8> ThunkArgs; for (auto &ArgDesc : ArgumentDescList) { addThunkArgument(ArgDesc, Builder, ThunkBody, ThunkArgs); } // We are ignoring generic functions and functions with out parameters for // now. SILValue ReturnValue; SILType LoweredType = NewF->getLoweredType(); SILType ResultType = LoweredType.getFunctionInterfaceResultType(); auto FunctionTy = LoweredType.castTo<SILFunctionType>(); if (FunctionTy->hasErrorResult()) { // We need a try_apply to call a function with an error result. SILFunction *Thunk = ThunkBody->getParent(); SILBasicBlock *NormalBlock = Thunk->createBasicBlock(); ReturnValue = NormalBlock->createBBArg(ResultType, 0); SILBasicBlock *ErrorBlock = Thunk->createBasicBlock(); SILType ErrorProtocol = SILType::getPrimitiveObjectType(FunctionTy->getErrorResult().getType()); auto *ErrorArg = ErrorBlock->createBBArg(ErrorProtocol, 0); Builder.createTryApply(Loc, FRI, LoweredType, ArrayRef<Substitution>(), ThunkArgs, NormalBlock, ErrorBlock); Builder.setInsertionPoint(ErrorBlock); Builder.createThrow(Loc, ErrorArg); Builder.setInsertionPoint(NormalBlock); } else { ReturnValue = Builder.createApply(Loc, FRI, LoweredType, ResultType, ArrayRef<Substitution>(), ThunkArgs, false); } // Set up the return results. if (NewF->getLoweredFunctionType()->isNoReturn()) { Builder.createUnreachable(Loc); } else { Builder.createReturn(Loc, ReturnValue); } // Do the last bit work to finalize the thunk. OwnedToGuaranteedFinalizeThunkFunction(Builder, F); assert(F->getDebugScope()->Parent != NewF->getDebugScope()->Parent); }
/// Insert monomorphic inline caches for a specific class or metatype /// type \p SubClassTy. static FullApplySite speculateMonomorphicTarget(FullApplySite AI, SILType SubType, CheckedCastBranchInst *&CCBI) { CCBI = nullptr; // Bail if this class_method cannot be devirtualized. if (!canDevirtualizeClassMethod(AI, SubType)) return FullApplySite(); if (SubType.getSwiftRValueType()->hasDynamicSelfType()) return FullApplySite(); // Create a diamond shaped control flow and a checked_cast_branch // instruction that checks the exact type of the object. // This cast selects between two paths: one that calls the slow dynamic // dispatch and one that calls the specific method. auto It = AI.getInstruction()->getIterator(); SILFunction *F = AI.getFunction(); SILBasicBlock *Entry = AI.getParent(); // Iden is the basic block containing the direct call. SILBasicBlock *Iden = F->createBasicBlock(); // Virt is the block containing the slow virtual call. SILBasicBlock *Virt = F->createBasicBlock(); Iden->createPHIArgument(SubType, ValueOwnershipKind::Owned); SILBasicBlock *Continue = Entry->split(It); SILBuilderWithScope Builder(Entry, AI.getInstruction()); // Create the checked_cast_branch instruction that checks at runtime if the // class instance is identical to the SILType. ClassMethodInst *CMI = cast<ClassMethodInst>(AI.getCallee()); CCBI = Builder.createCheckedCastBranch(AI.getLoc(), /*exact*/ true, CMI->getOperand(), SubType, Iden, Virt); It = CCBI->getIterator(); SILBuilderWithScope VirtBuilder(Virt, AI.getInstruction()); SILBuilderWithScope IdenBuilder(Iden, AI.getInstruction()); // This is the class reference downcasted into subclass SubType. SILValue DownCastedClassInstance = Iden->getArgument(0); // Copy the two apply instructions into the two blocks. FullApplySite IdenAI = CloneApply(AI, IdenBuilder); FullApplySite VirtAI = CloneApply(AI, VirtBuilder); // See if Continue has a release on self as the instruction right after the // apply. If it exists, move it into position in the diamond. SILBasicBlock::iterator next = next_or_end(Continue->begin(), Continue->end()); auto *Release = (next == Continue->end()) ? nullptr : dyn_cast<StrongReleaseInst>(next); if (Release && Release->getOperand() == CMI->getOperand()) { VirtBuilder.createStrongRelease(Release->getLoc(), CMI->getOperand(), Release->getAtomicity()); IdenBuilder.createStrongRelease(Release->getLoc(), DownCastedClassInstance, Release->getAtomicity()); Release->eraseFromParent(); } // Create a PHInode for returning the return value from both apply // instructions. SILArgument *Arg = Continue->createPHIArgument(AI.getType(), ValueOwnershipKind::Owned); if (!isa<TryApplyInst>(AI)) { if (AI.getSubstCalleeType()->isNoReturnFunction()) { IdenBuilder.createUnreachable(AI.getLoc()); VirtBuilder.createUnreachable(AI.getLoc()); } else { IdenBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(IdenAI) }); VirtBuilder.createBranch(AI.getLoc(), Continue, { cast<ApplyInst>(VirtAI) }); } } // Remove the old Apply instruction. assert(AI.getInstruction() == &Continue->front() && "AI should be the first instruction in the split Continue block"); if (isa<TryApplyInst>(AI)) { AI.getInstruction()->eraseFromParent(); assert(Continue->empty() && "There should not be an instruction after try_apply"); Continue->eraseFromParent(); } else { auto apply = cast<ApplyInst>(AI); apply->replaceAllUsesWith(Arg); apply->eraseFromParent(); assert(!Continue->empty() && "There should be at least a terminator after AI"); } // Update the stats. NumTargetsPredicted++; // Devirtualize the apply instruction on the identical path. auto NewInstPair = devirtualizeClassMethod(IdenAI, DownCastedClassInstance); assert(NewInstPair.first && "Expected to be able to devirtualize apply!"); replaceDeadApply(IdenAI, NewInstPair.first); // Split critical edges resulting from VirtAI. if (auto *TAI = dyn_cast<TryApplyInst>(VirtAI)) { auto *ErrorBB = TAI->getFunction()->createBasicBlock(); ErrorBB->createPHIArgument(TAI->getErrorBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(ErrorBB); Builder.createBranch(TAI->getLoc(), TAI->getErrorBB(), {ErrorBB->getArgument(0)}); auto *NormalBB = TAI->getFunction()->createBasicBlock(); NormalBB->createPHIArgument(TAI->getNormalBB()->getArgument(0)->getType(), ValueOwnershipKind::Owned); Builder.setInsertionPoint(NormalBB); Builder.createBranch(TAI->getLoc(), TAI->getNormalBB(), {NormalBB->getArgument(0)}); Builder.setInsertionPoint(VirtAI.getInstruction()); SmallVector<SILValue, 4> Args; for (auto Arg : VirtAI.getArguments()) { Args.push_back(Arg); } FullApplySite NewVirtAI = Builder.createTryApply(VirtAI.getLoc(), VirtAI.getCallee(), VirtAI.getSubstitutions(), Args, NormalBB, ErrorBB); VirtAI.getInstruction()->eraseFromParent(); VirtAI = NewVirtAI; } return VirtAI; }
/// In this function we create the actual cloned function and its proper cloned /// type. But we do not create any body. This implies that the creation of the /// actual arguments in the function is in populateCloned. /// /// \arg PAUser The function that is being passed the partial apply. /// \arg PAI The partial apply that is being passed to PAUser. /// \arg ClosureIndex The index of the partial apply in PAUser's function /// signature. /// \arg ClonedName The name of the cloned function that we will create. SILFunction * ClosureSpecCloner::initCloned(const CallSiteDescriptor &CallSiteDesc, StringRef ClonedName) { SILFunction *ClosureUser = CallSiteDesc.getApplyCallee(); // This is the list of new interface parameters of the cloned function. llvm::SmallVector<SILParameterInfo, 4> NewParameterInfoList; // First add to NewParameterInfoList all of the SILParameterInfo in the // original function except for the closure. CanSILFunctionType ClosureUserFunTy = ClosureUser->getLoweredFunctionType(); unsigned Index = ClosureUserFunTy->getNumIndirectResults(); for (auto ¶m : ClosureUserFunTy->getParameters()) { if (Index != CallSiteDesc.getClosureIndex()) NewParameterInfoList.push_back(param); ++Index; } // Then add any arguments that are captured in the closure to the function's // argument type. Since they are captured, we need to pass them directly into // the new specialized function. SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee(); CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType(); SILModule &M = ClosureUser->getModule(); // Captured parameters are always appended to the function signature. If the // type of the captured argument is trivial, pass the argument as // Direct_Unowned. Otherwise pass it as Direct_Owned. // // We use the type of the closure here since we allow for the closure to be an // external declaration. unsigned NumTotalParams = ClosedOverFunTy->getParameters().size(); unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments(); for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) { if (PInfo.getSILType().isTrivial(M)) { SILParameterInfo NewPInfo(PInfo.getType(), ParameterConvention::Direct_Unowned); NewParameterInfoList.push_back(NewPInfo); continue; } SILParameterInfo NewPInfo(PInfo.getType(), ParameterConvention::Direct_Owned); NewParameterInfoList.push_back(NewPInfo); } // The specialized function is always a thin function. This is important // because we may add additional parameters after the Self parameter of // witness methods. In this case the new function is not a method anymore. auto ExtInfo = ClosureUserFunTy->getExtInfo(); ExtInfo = ExtInfo.withRepresentation(SILFunctionTypeRepresentation::Thin); auto ClonedTy = SILFunctionType::get( ClosureUserFunTy->getGenericSignature(), ExtInfo, ClosureUserFunTy->getCalleeConvention(), NewParameterInfoList, ClosureUserFunTy->getAllResults(), ClosureUserFunTy->getOptionalErrorResult(), M.getASTContext()); // We make this function bare so we don't have to worry about decls in the // SILArgument. auto *Fn = M.createFunction( // It's important to use a shared linkage for the specialized function // and not the original linkage. // Otherwise the new function could have an external linkage (in case the // original function was de-serialized) and would not be code-gen'd. getSpecializedLinkage(ClosureUser, ClosureUser->getLinkage()), ClonedName, ClonedTy, ClosureUser->getGenericEnvironment(), ClosureUser->getLocation(), IsBare, ClosureUser->isTransparent(), CallSiteDesc.isFragile(), ClosureUser->isThunk(), ClosureUser->getClassVisibility(), ClosureUser->getInlineStrategy(), ClosureUser->getEffectsKind(), ClosureUser, ClosureUser->getDebugScope()); Fn->setDeclCtx(ClosureUser->getDeclContext()); if (ClosureUser->hasUnqualifiedOwnership()) { Fn->setUnqualifiedOwnership(); } for (auto &Attr : ClosureUser->getSemanticsAttrs()) Fn->addSemanticsAttr(Attr); return Fn; }
bool SILCombiner::doOneIteration(SILFunction &F, unsigned Iteration) { MadeChange = false; DEBUG(llvm::dbgs() << "\n\nSILCOMBINE ITERATION #" << Iteration << " on " << F.getName() << "\n"); // Add reachable instructions to our worklist. addReachableCodeToWorklist(&*F.begin()); // Process until we run out of items in our worklist. while (!Worklist.isEmpty()) { SILInstruction *I = Worklist.removeOne(); // When we erase an instruction, we use the map in the worklist to check if // the instruction is in the worklist. If it is, we replace it with null // instead of shifting all members of the worklist towards the front. This // check makes sure that if we run into any such residual null pointers, we // skip them. if (I == nullptr) continue; // Check to see if we can DCE the instruction. if (isInstructionTriviallyDead(I)) { DEBUG(llvm::dbgs() << "SC: DCE: " << *I << '\n'); eraseInstFromFunction(*I); ++NumDeadInst; MadeChange = true; continue; } // Check to see if we can instsimplify the instruction. if (SILValue Result = simplifyInstruction(I)) { ++NumSimplified; DEBUG(llvm::dbgs() << "SC: Simplify Old = " << *I << '\n' << " New = " << *Result << '\n'); // Everything uses the new instruction now. replaceInstUsesWith(*cast<SingleValueInstruction>(I), Result); // Push the new instruction and any users onto the worklist. Worklist.addUsersToWorklist(Result); eraseInstFromFunction(*I); MadeChange = true; continue; } // If we have reached this point, all attempts to do simple simplifications // have failed. Prepare to SILCombine. Builder.setInsertionPoint(I); #ifndef NDEBUG std::string OrigI; #endif DEBUG(llvm::raw_string_ostream SS(OrigI); I->print(SS); OrigI = SS.str();); DEBUG(llvm::dbgs() << "SC: Visiting: " << OrigI << '\n'); if (SILInstruction *Result = visit(I)) { ++NumCombined; // Should we replace the old instruction with a new one? if (Result != I) { assert(&*std::prev(SILBasicBlock::iterator(I)) == Result && "Expected new instruction inserted before existing instruction!"); DEBUG(llvm::dbgs() << "SC: Old = " << *I << '\n' << " New = " << *Result << '\n'); // Everything uses the new instruction now. replaceInstUsesPairwiseWith(I, Result); // Push the new instruction and any users onto the worklist. Worklist.add(Result); Worklist.addUsersOfAllResultsToWorklist(Result); eraseInstFromFunction(*I); } else { DEBUG(llvm::dbgs() << "SC: Mod = " << OrigI << '\n' << " New = " << *I << '\n'); // If the instruction was modified, it's possible that it is now dead. // if so, remove it. if (isInstructionTriviallyDead(I)) { eraseInstFromFunction(*I); } else { Worklist.add(I); Worklist.addUsersOfAllResultsToWorklist(I); } } MadeChange = true; } // Our tracking list has been accumulating instructions created by the // SILBuilder during this iteration. Go through the tracking list and add // its contents to the worklist and then clear said list in preparation for // the next iteration. auto &TrackingList = *Builder.getTrackingList(); for (SILInstruction *I : TrackingList) { DEBUG(llvm::dbgs() << "SC: add " << *I << " from tracking list to worklist\n"); Worklist.add(I); } TrackingList.clear(); }
/// \brief Populate the body of the cloned closure, modifying instructions as /// necessary. This is where we create the actual specialized BB Arguments. void ClosureSpecCloner::populateCloned() { SILFunction *Cloned = getCloned(); SILFunction *ClosureUser = CallSiteDesc.getApplyCallee(); // Create arguments for the entry block. SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin(); SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock(); // Remove the closure argument. SILArgument *ClosureArg = nullptr; for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) { SILArgument *Arg = ClosureUserEntryBB->getArgument(i); if (i == CallSiteDesc.getClosureIndex()) { ClosureArg = Arg; continue; } // Otherwise, create a new argument which copies the original argument SILValue MappedValue = ClonedEntryBB->createArgument(Arg->getType(), Arg->getDecl()); ValueMap.insert(std::make_pair(Arg, MappedValue)); } // Next we need to add in any arguments that are not captured as arguments to // the cloned function. // // We do not insert the new mapped arguments into the value map since there by // definition is nothing in the partial apply user function that references // such arguments. After this pass is done the only thing that will reference // the arguments is the partial apply that we will create. SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee(); CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType(); unsigned NumTotalParams = ClosedOverFunTy->getParameters().size(); unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments(); llvm::SmallVector<SILValue, 4> NewPAIArgs; for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) { SILValue MappedValue = ClonedEntryBB->createArgument(PInfo.getSILType()); NewPAIArgs.push_back(MappedValue); } SILBuilder &Builder = getBuilder(); Builder.setInsertionPoint(ClonedEntryBB); // Clone FRI and PAI, and replace usage of the removed closure argument // with result of cloned PAI. SILValue FnVal = Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun); auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs); ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure))); BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB)); // Recursively visit original BBs in depth-first preorder, starting with the // entry block, cloning all instructions other than terminators. visitSILBasicBlock(ClosureUserEntryBB); // Now iterate over the BBs and fix up the terminators. for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) { Builder.setInsertionPoint(BI->second); visit(BI->first->getTerminator()); } // Then insert a release in all non failure exit BBs if our partial apply was // guaranteed. This is b/c it was passed at +0 originally and we need to // balance the initial increment of the newly created closure. if (CallSiteDesc.isClosureGuaranteed() && CallSiteDesc.closureHasRefSemanticContext()) { for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) { SILBasicBlock *OpBB = BBMap[BB]; TermInst *TI = OpBB->getTerminator(); auto Loc = CleanupLocation::get(NewClosure->getLoc()); // If we have a return, we place the release right before it so we know // that it will be executed at the end of the epilogue. if (isa<ReturnInst>(TI)) { Builder.setInsertionPoint(TI); Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic); continue; } // We use casts where findAllNonFailureExitBBs should have made sure that // this is true. This will ensure that the code is updated when we hit the // cast failure in debug builds. auto *Unreachable = cast<UnreachableInst>(TI); auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable)); auto NoReturnApply = FullApplySite::isa(&*PrevIter); // We insert the release value right before the no return apply so that if // the partial apply is passed into the no-return function as an @owned // value, we will retain the partial apply before we release it and // potentially eliminate it. Builder.setInsertionPoint(NoReturnApply.getInstruction()); Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic); } } }
// This implicitly asserts that a function binding dynamic self has a // self metadata argument or object from which self metadata can be obtained. bool isArgumentABIRequired(SILArgument *Arg) { return MayDynamicBindSelf && (F->getSelfMetadataArgument() == Arg); }
/// \brief Returns the callee SILFunction called at a call site, in the case /// that the call is transparent (as in, both that the call is marked /// with the transparent flag and that callee function is actually transparently /// determinable from the SIL) or nullptr otherwise. This assumes that the SIL /// is already in SSA form. /// /// In the case that a non-null value is returned, FullArgs contains effective /// argument operands for the callee function. static SILFunction * getCalleeFunction(FullApplySite AI, bool &IsThick, SmallVectorImpl<SILValue>& CaptureArgs, SmallVectorImpl<SILValue>& FullArgs, PartialApplyInst *&PartialApply, SILModule::LinkingMode Mode) { IsThick = false; PartialApply = nullptr; CaptureArgs.clear(); FullArgs.clear(); for (const auto &Arg : AI.getArguments()) FullArgs.push_back(Arg); SILValue CalleeValue = AI.getCallee(); if (LoadInst *LI = dyn_cast<LoadInst>(CalleeValue)) { // Conservatively only see through alloc_box; we assume this pass is run // immediately after SILGen auto *PBI = dyn_cast<ProjectBoxInst>(LI->getOperand()); if (!PBI) return nullptr; auto *ABI = dyn_cast<AllocBoxInst>(PBI->getOperand()); if (!ABI) return nullptr; // Ensure there are no other uses of alloc_box than the project_box and // retains, releases. for (Operand *ABIUse : ABI->getUses()) if (ABIUse->getUser() != PBI && !isa<StrongRetainInst>(ABIUse->getUser()) && !isa<StrongReleaseInst>(ABIUse->getUser())) return nullptr; // Scan forward from the alloc box to find the first store, which // (conservatively) must be in the same basic block as the alloc box StoreInst *SI = nullptr; for (auto I = SILBasicBlock::iterator(ABI), E = I->getParent()->end(); I != E; ++I) { // If we find the load instruction first, then the load is loading from // a non-initialized alloc; this shouldn't really happen but I'm not // making any assumptions if (&*I == LI) return nullptr; if ((SI = dyn_cast<StoreInst>(I)) && SI->getDest() == PBI) { // We found a store that we know dominates the load; now ensure there // are no other uses of the project_box except loads. for (Operand *PBIUse : PBI->getUses()) if (PBIUse->getUser() != SI && !isa<LoadInst>(PBIUse->getUser())) return nullptr; // We can conservatively see through the store break; } } if (!SI) return nullptr; CalleeValue = SI->getSrc(); } // We are allowed to see through exactly one "partial apply" instruction or // one "thin to thick function" instructions, since those are the patterns // generated when using auto closures. if (PartialApplyInst *PAI = dyn_cast<PartialApplyInst>(CalleeValue)) { for (const auto &Arg : PAI->getArguments()) { CaptureArgs.push_back(Arg); FullArgs.push_back(Arg); } CalleeValue = PAI->getCallee(); IsThick = true; PartialApply = PAI; } else if (ThinToThickFunctionInst *TTTFI = dyn_cast<ThinToThickFunctionInst>(CalleeValue)) { CalleeValue = TTTFI->getOperand(); IsThick = true; } FunctionRefInst *FRI = dyn_cast<FunctionRefInst>(CalleeValue); if (!FRI) return nullptr; SILFunction *CalleeFunction = FRI->getReferencedFunction(); switch (CalleeFunction->getRepresentation()) { case SILFunctionTypeRepresentation::Thick: case SILFunctionTypeRepresentation::Thin: case SILFunctionTypeRepresentation::Method: case SILFunctionTypeRepresentation::Closure: case SILFunctionTypeRepresentation::WitnessMethod: break; case SILFunctionTypeRepresentation::CFunctionPointer: case SILFunctionTypeRepresentation::ObjCMethod: case SILFunctionTypeRepresentation::Block: return nullptr; } // If CalleeFunction is a declaration, see if we can load it. If we fail to // load it, bail. if (CalleeFunction->empty() && !AI.getModule().linkFunction(CalleeFunction, Mode)) return nullptr; return CalleeFunction; }