/// Inlines all mandatory inlined functions into the body of a function,
/// first recursively inlining all mandatory apply instructions in those
/// functions into their bodies if necessary.
///
/// \param F the function to be processed
/// \param AI nullptr if this is being called from the top level; the relevant
/// ApplyInst requiring the recursive call when non-null
/// \param FullyInlinedSet the set of all functions already known to be fully
/// processed, to avoid processing them over again
/// \param SetFactory an instance of ImmutableFunctionSet::Factory
/// \param CurrentInliningSet the set of functions currently being inlined in
/// the current call stack of recursive calls
///
/// \returns true if successful, false if failed due to circular inlining.
static bool
runOnFunctionRecursively(SILOptFunctionBuilder &FuncBuilder,
SILFunction *F, FullApplySite AI,
DenseFunctionSet &FullyInlinedSet,
ImmutableFunctionSet::Factory &SetFactory,
ImmutableFunctionSet CurrentInliningSet,
ClassHierarchyAnalysis *CHA) {
// Avoid reprocessing functions needlessly.
if (FullyInlinedSet.count(F))
return true;
// Prevent attempt to circularly inline.
if (CurrentInliningSet.contains(F)) {
// This cannot happen on a top-level call, so AI should be non-null.
assert(AI && "Cannot have circular inline without apply");
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::circular_transparent);
return false;
}
// Add to the current inlining set (immutably, so we only affect the set
// during this call and recursive subcalls).
CurrentInliningSet = SetFactory.add(CurrentInliningSet, F);
SmallVector<std::pair<SILValue, ParameterConvention>, 16> CaptureArgs;
SmallVector<SILValue, 32> FullArgs;
// Visiting blocks in reverse order avoids revisiting instructions after block
// splitting, which would be quadratic.
for (auto BI = F->rbegin(), BE = F->rend(), nextBB = BI; BI != BE;
BI = nextBB) {
// After inlining, the block iterator will be adjusted to point to the last
// block containing inlined instructions. This way, the inlined function
// body will be reprocessed within the caller's context without revisiting
// any original instructions.
nextBB = std::next(BI);
// While iterating over this block, instructions are inserted and deleted.
// To avoid quadratic block splitting, instructions must be processed in
// reverse order (block splitting reassigned the parent pointer of all
// instructions below the split point).
for (auto II = BI->rbegin(); II != BI->rend(); ++II) {
FullApplySite InnerAI = FullApplySite::isa(&*II);
if (!InnerAI)
continue;
// *NOTE* If devirtualization succeeds, devirtInst may not be InnerAI,
// but a casted result of InnerAI or even a block argument due to
// abstraction changes when calling the witness or class method.
auto *devirtInst = tryDevirtualizeApplyHelper(InnerAI, CHA);
// Restore II to the current apply site.
II = devirtInst->getReverseIterator();
// If the devirtualized call result is no longer a invalid FullApplySite,
// then it has succeeded, but the result is not immediately inlinable.
InnerAI = FullApplySite::isa(devirtInst);
if (!InnerAI)
continue;
SILValue CalleeValue = InnerAI.getCallee();
bool IsThick;
PartialApplyInst *PAI;
SILFunction *CalleeFunction = getCalleeFunction(
F, InnerAI, IsThick, CaptureArgs, FullArgs, PAI);
if (!CalleeFunction)
continue;
// Then recursively process it first before trying to inline it.
if (!runOnFunctionRecursively(FuncBuilder, CalleeFunction, InnerAI,
FullyInlinedSet, SetFactory,
CurrentInliningSet, CHA)) {
// If we failed due to circular inlining, then emit some notes to
// trace back the failure if we have more information.
// FIXME: possibly it could be worth recovering and attempting other
// inlines within this same recursive call rather than simply
// propagating the failure.
if (AI) {
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::note_while_inlining);
}
return false;
}
// Get our list of substitutions.
auto Subs = (PAI
? PAI->getSubstitutionMap()
: InnerAI.getSubstitutionMap());
SILOpenedArchetypesTracker OpenedArchetypesTracker(F);
F->getModule().registerDeleteNotificationHandler(
&OpenedArchetypesTracker);
// The callee only needs to know about opened archetypes used in
// the substitution list.
OpenedArchetypesTracker.registerUsedOpenedArchetypes(
InnerAI.getInstruction());
if (PAI) {
OpenedArchetypesTracker.registerUsedOpenedArchetypes(PAI);
}
SILInliner Inliner(FuncBuilder, SILInliner::InlineKind::MandatoryInline,
Subs, OpenedArchetypesTracker);
if (!Inliner.canInlineApplySite(InnerAI))
continue;
// Inline function at I, which also changes I to refer to the first
// instruction inlined in the case that it succeeds. We purposely
// process the inlined body after inlining, because the inlining may
// have exposed new inlining opportunities beyond those present in
// the inlined function when processed independently.
LLVM_DEBUG(llvm::errs() << "Inlining @" << CalleeFunction->getName()
<< " into @" << InnerAI.getFunction()->getName()
<< "\n");
// If we intend to inline a thick function, then we need to balance the
// reference counts for correctness.
if (IsThick) {
bool IsCalleeGuaranteed =
PAI &&
PAI->getType().castTo<SILFunctionType>()->isCalleeGuaranteed();
fixupReferenceCounts(InnerAI.getInstruction(), CalleeValue, CaptureArgs,
IsCalleeGuaranteed);
}
// Register a callback to record potentially unused function values after
// inlining.
ClosureCleanup closureCleanup;
Inliner.setDeletionCallback([&closureCleanup](SILInstruction *I) {
closureCleanup.recordDeadFunction(I);
});
// Inlining deletes the apply, and can introduce multiple new basic
// blocks. After this, CalleeValue and other instructions may be invalid.
// nextBB will point to the last inlined block
auto firstInlinedInstAndLastBB =
Inliner.inlineFunction(CalleeFunction, InnerAI, FullArgs);
nextBB = firstInlinedInstAndLastBB.second->getReverseIterator();
++NumMandatoryInlines;
// The IR is now valid, and trivial dead arguments are removed. However,
// we may be able to remove dead callee computations (e.g. dead
// partial_apply closures).
closureCleanup.cleanupDeadClosures(F);
// Resume inlining within nextBB, which contains only the inlined
// instructions and possibly instructions in the original call block that
// have not yet been visited.
break;
}
}
// Keep track of full inlined functions so we don't waste time recursively
// reprocessing them.
FullyInlinedSet.insert(F);
return true;
}
/// \brief Inlines all mandatory inlined functions into the body of a function,
/// first recursively inlining all mandatory apply instructions in those
/// functions into their bodies if necessary.
///
/// \param F the function to be processed
/// \param AI nullptr if this is being called from the top level; the relevant
/// ApplyInst requiring the recursive call when non-null
/// \param FullyInlinedSet the set of all functions already known to be fully
/// processed, to avoid processing them over again
/// \param SetFactory an instance of ImmutableFunctionSet::Factory
/// \param CurrentInliningSet the set of functions currently being inlined in
/// the current call stack of recursive calls
///
/// \returns true if successful, false if failed due to circular inlining.
static bool
runOnFunctionRecursively(SILFunction *F, FullApplySite AI,
DenseFunctionSet &FullyInlinedSet,
ImmutableFunctionSet::Factory &SetFactory,
ImmutableFunctionSet CurrentInliningSet,
ClassHierarchyAnalysis *CHA) {
// Avoid reprocessing functions needlessly.
if (FullyInlinedSet.count(F))
return true;
// Prevent attempt to circularly inline.
if (CurrentInliningSet.contains(F)) {
// This cannot happen on a top-level call, so AI should be non-null.
assert(AI && "Cannot have circular inline without apply");
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::circular_transparent);
return false;
}
// Add to the current inlining set (immutably, so we only affect the set
// during this call and recursive subcalls).
CurrentInliningSet = SetFactory.add(CurrentInliningSet, F);
SmallVector<std::pair<SILValue, ParameterConvention>, 16> CaptureArgs;
SmallVector<SILValue, 32> FullArgs;
for (auto BI = F->begin(), BE = F->end(); BI != BE; ++BI) {
for (auto II = BI->begin(), IE = BI->end(); II != IE; ++II) {
FullApplySite InnerAI = FullApplySite::isa(&*II);
if (!InnerAI)
continue;
auto *ApplyBlock = InnerAI.getParent();
// *NOTE* If devirtualization succeeds, sometimes II will not be InnerAI,
// but a casted result of InnerAI or even a block argument due to
// abstraction changes when calling the witness or class method. We still
// know that InnerAI dominates II though.
std::tie(InnerAI, II) = tryDevirtualizeApplyHelper(InnerAI, II, CHA);
if (!InnerAI)
continue;
SILValue CalleeValue = InnerAI.getCallee();
bool IsThick;
PartialApplyInst *PAI;
SILFunction *CalleeFunction = getCalleeFunction(
F, InnerAI, IsThick, CaptureArgs, FullArgs, PAI);
if (!CalleeFunction)
continue;
// Then recursively process it first before trying to inline it.
if (!runOnFunctionRecursively(CalleeFunction, InnerAI,
FullyInlinedSet, SetFactory,
CurrentInliningSet, CHA)) {
// If we failed due to circular inlining, then emit some notes to
// trace back the failure if we have more information.
// FIXME: possibly it could be worth recovering and attempting other
// inlines within this same recursive call rather than simply
// propagating the failure.
if (AI) {
SILLocation L = AI.getLoc();
assert(L && "Must have location for transparent inline apply");
diagnose(F->getModule().getASTContext(), L.getStartSourceLoc(),
diag::note_while_inlining);
}
return false;
}
// Get our list of substitutions.
auto Subs = (PAI
? PAI->getSubstitutionMap()
: InnerAI.getSubstitutionMap());
SILOpenedArchetypesTracker OpenedArchetypesTracker(F);
F->getModule().registerDeleteNotificationHandler(
&OpenedArchetypesTracker);
// The callee only needs to know about opened archetypes used in
// the substitution list.
OpenedArchetypesTracker.registerUsedOpenedArchetypes(
InnerAI.getInstruction());
if (PAI) {
OpenedArchetypesTracker.registerUsedOpenedArchetypes(PAI);
}
SILInliner Inliner(*F, *CalleeFunction,
SILInliner::InlineKind::MandatoryInline, Subs,
OpenedArchetypesTracker);
if (!Inliner.canInlineFunction(InnerAI)) {
// See comment above about casting when devirtualizing and how this
// sometimes causes II and InnerAI to be different and even in different
// blocks.
II = InnerAI.getInstruction()->getIterator();
continue;
}
// Inline function at I, which also changes I to refer to the first
// instruction inlined in the case that it succeeds. We purposely
// process the inlined body after inlining, because the inlining may
// have exposed new inlining opportunities beyond those present in
// the inlined function when processed independently.
LLVM_DEBUG(llvm::errs() << "Inlining @" << CalleeFunction->getName()
<< " into @" << InnerAI.getFunction()->getName()
<< "\n");
// If we intend to inline a thick function, then we need to balance the
// reference counts for correctness.
if (IsThick) {
bool IsCalleeGuaranteed =
PAI &&
PAI->getType().castTo<SILFunctionType>()->isCalleeGuaranteed();
fixupReferenceCounts(II, CalleeValue, CaptureArgs, IsCalleeGuaranteed);
}
// Decrement our iterator (carefully, to avoid going off the front) so it
// is valid after inlining is done. Inlining deletes the apply, and can
// introduce multiple new basic blocks.
II = prev_or_default(II, ApplyBlock->begin(), ApplyBlock->end());
Inliner.inlineFunction(InnerAI, FullArgs);
// We were able to inline successfully. Remove the apply.
InnerAI.getInstruction()->eraseFromParent();
// Reestablish our iterator if it wrapped.
if (II == ApplyBlock->end())
II = ApplyBlock->begin();
// Update the iterator when instructions are removed.
DeleteInstructionsHandler DeletionHandler(II);
// Now that the IR is correct, see if we can remove dead callee
// computations (e.g. dead partial_apply closures).
cleanupCalleeValue(CalleeValue, FullArgs);
// Reposition iterators possibly invalidated by mutation.
BI = SILFunction::iterator(ApplyBlock);
IE = ApplyBlock->end();
assert(BI == SILFunction::iterator(II->getParent()) &&
"Mismatch between the instruction and basic block");
++NumMandatoryInlines;
}
}
// Keep track of full inlined functions so we don't waste time recursively
// reprocessing them.
FullyInlinedSet.insert(F);
return true;
}