void EpilogueARCContext::initializeDataflow() {
for (auto &B : *F) {
// Find the exit blocks.
if (isInterestedFunctionExitingBlock(&B)) {
ExitBlocks.insert(&B);
}
// Allocate the storage.
EpilogueARCBlockStates[&B] =
new (BPA.Allocate()) EpilogueARCBlockState();
}
// Split the SILargument into local arguments to each specific basic block.
llvm::SmallVector<SILValue, 4> ToProcess;
llvm::DenseSet<SILValue> Processed;
ToProcess.push_back(Arg);
while (!ToProcess.empty()) {
SILValue CArg = ToProcess.pop_back_val();
if (!CArg)
continue;
if (Processed.find(CArg) != Processed.end())
continue;
Processed.insert(CArg);
SILArgument *A = dyn_cast<SILArgument>(CArg);
if (A && !A->isFunctionArg()) {
// Find predecessor and break the SILArgument to predecessors.
for (auto X : A->getParent()->getPreds()) {
// Try to find the predecessor edge-value.
SILValue IA = A->getIncomingValue(X);
EpilogueARCBlockStates[X]->LocalArg = IA;
// Maybe the edge value is another SILArgument.
ToProcess.push_back(IA);
}
}
}
}
/// Determine the number of iterations the loop is at most executed. The loop
/// might contain early exits so this is the maximum if no early exits are
/// taken.
static Optional<uint64_t> getMaxLoopTripCount(SILLoop *Loop,
SILBasicBlock *Preheader,
SILBasicBlock *Header,
SILBasicBlock *Latch) {
// Skip a split backedge.
SILBasicBlock *OrigLatch = Latch;
if (!Loop->isLoopExiting(Latch) && !(Latch = Latch->getSinglePredecessor()))
return None;
if (!Loop->isLoopExiting(Latch))
return None;
// Get the loop exit condition.
auto *CondBr = dyn_cast<CondBranchInst>(Latch->getTerminator());
if (!CondBr)
return None;
// Match an add 1 recurrence.
SILArgument *RecArg;
IntegerLiteralInst *End;
SILValue RecNext;
if (!match(CondBr->getCondition(),
m_BuiltinInst(BuiltinValueKind::ICMP_EQ, m_SILValue(RecNext),
m_IntegerLiteralInst(End))))
return None;
if (!match(RecNext,
m_TupleExtractInst(m_ApplyInst(BuiltinValueKind::SAddOver,
m_SILArgument(RecArg), m_One()),
0)))
return None;
if (RecArg->getParent() != Header)
return None;
auto *Start = dyn_cast_or_null<IntegerLiteralInst>(
RecArg->getIncomingValue(Preheader).getDef());
if (!Start)
return None;
if (RecNext != RecArg->getIncomingValue(OrigLatch))
return None;
auto StartVal = Start->getValue();
auto EndVal = End->getValue();
if (StartVal.sgt(EndVal))
return None;
auto Dist = EndVal - StartVal;
if (Dist.getBitWidth() > 64)
return None;
if (Dist == 0)
return None;
return Dist.getZExtValue();
}
void FunctionSignatureTransform::ArgumentExplosionFinalizeOptimizedFunction() {
SILBasicBlock *BB = &*NewF->begin();
SILBuilder Builder(BB->begin());
Builder.setCurrentDebugScope(BB->getParent()->getDebugScope());
unsigned TotalArgIndex = 0;
for (ArgumentDescriptor &AD : ArgumentDescList) {
// Simply continue if do not explode.
if (!AD.Explode) {
AIM[TotalArgIndex] = AD.Index;
TotalArgIndex ++;
continue;
}
// OK, we need to explode this argument.
unsigned ArgOffset = ++TotalArgIndex;
unsigned OldArgIndex = ArgOffset - 1;
llvm::SmallVector<SILValue, 8> LeafValues;
// We do this in the same order as leaf types since ProjTree expects that the
// order of leaf values matches the order of leaf types.
llvm::SmallVector<const ProjectionTreeNode*, 8> LeafNodes;
AD.ProjTree.getLeafNodes(LeafNodes);
for (auto *Node : LeafNodes) {
auto OwnershipKind = *AD.getTransformedOwnershipKind(Node->getType());
LeafValues.push_back(BB->insertFunctionArgument(
ArgOffset++, Node->getType(), OwnershipKind,
BB->getArgument(OldArgIndex)->getDecl()));
AIM[TotalArgIndex - 1] = AD.Index;
TotalArgIndex ++;
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
AD.ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(),
LeafValues);
// We ignored debugvalue uses when we constructed the new arguments, in order
// to preserve as much information as possible, we construct a new value for
// OrigArg from the leaf values and use that in place of the OrigArg.
SILValue NewOrigArgValue = AD.ProjTree.computeExplodedArgumentValue(Builder,
BB->getParent()->getLocation(),
LeafValues);
// Replace all uses of the original arg with the new value.
SILArgument *OrigArg = BB->getArgument(OldArgIndex);
OrigArg->replaceAllUsesWith(NewOrigArgValue);
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseArgument(OldArgIndex);
TotalArgIndex --;
}
}
void AccessSummaryAnalysis::FunctionSummary::print(raw_ostream &os,
SILFunction *fn) const {
unsigned argCount = getArgumentCount();
os << "(";
for (unsigned i = 0; i < argCount; i++) {
if (i > 0) {
os << ", ";
}
SILArgument *arg = fn->getArgument(i);
SILModule &m = fn->getModule();
os << getAccessForArgument(i).getDescription(arg->getType(), m);
}
os << ")";
}
void ReleaseCodeMotionContext::computeCodeMotionGenKillSet() {
for (SILBasicBlock *BB : PO->getPostOrder()) {
auto *State = BlockStates[BB];
bool InterestBlock = false;
for (auto I = BB->rbegin(), E = BB->rend(); I != E; ++I) {
// Check whether this instruction blocks any RC root code motion.
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
if (!State->BBMaxSet.test(i) || !mayBlockCodeMotion(&*I, RCRootVault[i]))
continue;
// This instruction blocks this RC root.
InterestBlock = true;
State->BBKillSet.set(i);
State->BBGenSet.reset(i);
}
// If this is an epilogue release and we are freezing epilogue release
// simply continue.
if (FreezeEpilogueReleases && ERM.isEpilogueRelease(&*I))
continue;
// If this is a release instruction, it also generates.
if (isReleaseInstruction(&*I)) {
unsigned idx = RCRootIndex[getRCRoot(&*I)];
State->BBGenSet.set(idx);
assert(State->BBKillSet.test(idx) && "Killset computed incorrectly");
State->BBKillSet.reset(idx);
InterestBlock = true;
}
}
// Handle SILArgument, SILArgument can invalidate.
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
SILArgument *A = dyn_cast<SILArgument>(RCRootVault[i]);
if (!A || A->getParent() != BB)
continue;
InterestBlock = true;
State->BBKillSet.set(i);
State->BBGenSet.reset(i);
}
// Is this interesting to the last iteration of the data flow.
if (!InterestBlock)
continue;
InterestBlocks.insert(BB);
}
}
/// Like ValueIsPHI but also check if the PHI has no source
/// operands, i.e., it was just added.
static SILArgument *ValueIsNewPHI(SILValue Val, SILSSAUpdater *Updater) {
SILArgument *PHI = ValueIsPHI(Val, Updater);
if (PHI) {
auto *PhiBB = PHI->getParent();
size_t PhiIdx = PHI->getIndex();
// If all predecessor edges are 'not set' this is a new phi.
for (auto *PredBB : PhiBB->getPreds()) {
OperandValueArrayRef Edges =
getEdgeValuesForTerminator(PredBB->getTerminator(), PhiBB);
assert(PhiIdx < Edges.size() && "Not enough edges!");
SILValue V = Edges[PhiIdx];
// Check for the 'not set' sentinel.
if (V.getDef() != Updater->PHISentinel.get())
return nullptr;
}
return PHI;
}
return nullptr;
}
Effects *FunctionEffects::getEffectsOn(SILValue Addr) {
SILValue BaseAddr = skipValueProjections(skipAddrProjections(Addr));
switch (BaseAddr->getKind()) {
case swift::ValueKind::SILArgument: {
// Can we associate the address to a function parameter?
SILArgument *Arg = cast<SILArgument>(BaseAddr);
if (Arg->isFunctionArg()) {
return &ParamEffects[Arg->getIndex()];
}
break;
}
case ValueKind::AllocStackInst:
case ValueKind::AllocRefInst:
case ValueKind::AllocRefDynamicInst:
case ValueKind::AllocBoxInst:
// Effects on locally allocated storage.
return &LocalEffects;
default:
break;
}
// Everything else.
return &GlobalEffects;
}
/// \brief Propagate/remove basic block input values when all predecessors
/// supply the same arguments.
static void propagateBasicBlockArgs(SILBasicBlock &BB) {
// This functions would simplify the code as following:
//
// bb0:
// br bb2(%1 : $Builtin.Int1, %2 : $Builtin.Int1)
// bb1:
// br bb2(%1 : $Builtin.Int1, %2 : $Builtin.Int1)
// bb2(%3 : $Builtin.Int1, %4 : $Builtin.Int1):
// use(%3 : $Builtin.Int1)
// use(%4 : $Builtin.Int1)
// =>
// bb0:
// br bb2
// bb1:
// br bb2
// bb2:
// use(%1 : $Builtin.Int1)
// use(%2 : $Builtin.Int1)
// If there are no predecessors or no arguments, there is nothing to do.
if (BB.pred_empty() || BB.bbarg_empty())
return;
// Check if all the predecessors supply the same arguments to the BB.
SmallVector<SILValue, 4> Args;
bool checkArgs = false;
for (SILBasicBlock::pred_iterator PI = BB.pred_begin(), PE = BB.pred_end();
PI != PE; ++PI) {
SILBasicBlock *PredB = *PI;
// We are only simplifying cases where all predecessors are
// unconditional branch instructions.
if (!isa<BranchInst>(PredB->getTerminator()))
return;
BranchInst *BI = cast<BranchInst>(PredB->getTerminator());
unsigned Idx = 0;
assert(!BI->getArgs().empty());
for (OperandValueArrayRef::iterator AI = BI->getArgs().begin(),
AE = BI->getArgs().end();
AI != AE; ++AI, ++Idx) {
// When processing the first predecessor, record the arguments.
if (!checkArgs)
Args.push_back(*AI);
else
// On each subsequent predecessor, check the arguments.
if (Args[Idx] != *AI)
return;
}
// After the first branch is processed, the arguments vector is populated.
assert(Args.size() > 0);
checkArgs = true;
}
// If we've reached this point, the optimization is valid, so optimize.
// We know that the incoming arguments from all predecessors are the same,
// so just use them directly and remove the basic block parameters.
// Drop the arguments from the branch instructions by creating a new branch
// instruction and deleting the old one.
llvm::SmallVector<SILInstruction*, 32> ToBeDeleted;
for (SILBasicBlock::pred_iterator PI = BB.pred_begin(), PE = BB.pred_end();
PI != PE; ++PI) {
SILBasicBlock *PredB = *PI;
BranchInst *BI = cast<BranchInst>(PredB->getTerminator());
SILBuilderWithScope Bldr(PredB, BI);
Bldr.createBranch(BI->getLoc(), BI->getDestBB());
ToBeDeleted.push_back(BI);
}
// Drop the parameters from basic blocks and replace all uses with the passed
// in arguments.
unsigned Idx = 0;
for (SILBasicBlock::bbarg_iterator AI = BB.bbarg_begin(),
AE = BB.bbarg_end();
AI != AE; ++AI, ++Idx) {
// FIXME: These could be further propagatable now, we might want to move
// this to CCP and trigger another round of copy propagation.
SILArgument *Arg = *AI;
// We were able to fold, so all users should use the new folded value.
assert(Arg->getTypes().size() == 1 &&
"Currently, we only support single result instructions.");
SILValue(Arg).replaceAllUsesWith(Args[Idx]);
NumBasicBlockArgsPropagated++;
}
// Remove args from the block.
BB.dropAllBBArgs();
// The old branch instructions are no longer used, erase them.
recursivelyDeleteTriviallyDeadInstructions(ToBeDeleted, true);
NumInstructionsRemoved += ToBeDeleted.size();
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
SmallVector<SILValue, 4> entryArgs;
entryArgs.reserve(ClosureUserEntryBB->getArguments().size());
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
entryArgs.push_back(SILValue());
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
entryArgs.push_back(MappedValue);
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
auto ClosedOverFunConv = ClosedOverFun->getConventions();
unsigned NumTotalParams = ClosedOverFunConv.getNumParameters();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunConv.getParameters().slice(NumNotCaptured)) {
auto paramTy = ClosedOverFunConv.getSILType(PInfo);
SILValue MappedValue = ClonedEntryBB->createFunctionArgument(paramTy);
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
// Clone a chain of ConvertFunctionInsts. This can create further
// reabstraction partial_apply instructions.
SmallVector<PartialApplyInst*, 4> NeedsRelease;
SILValue ConvertedCallee = cloneCalleeConversion(
CallSiteDesc.getClosureCallerArg(), NewClosure, Builder, NeedsRelease);
// Make sure that we actually emit the releases for reabstraction thunks. We
// have guaranteed earlier that we only allow reabstraction thunks if the
// closure was passed trivial.
assert(NeedsRelease.empty() || CallSiteDesc.isTrivialNoEscapeParameter());
entryArgs[CallSiteDesc.getClosureIndex()] = ConvertedCallee;
// Visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions and terminators.
cloneFunctionBody(ClosureUser, ClonedEntryBB, entryArgs);
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure(s).
bool ClosureHasRefSemantics = CallSiteDesc.closureHasRefSemanticContext();
if ((CallSiteDesc.isClosureGuaranteed() ||
CallSiteDesc.isTrivialNoEscapeParameter()) &&
(ClosureHasRefSemantics || !NeedsRelease.empty())) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = getOpBasicBlock(BB);
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have an exit, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (TI->isFunctionExiting()) {
Builder.setInsertionPoint(TI);
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
if (ClosureHasRefSemantics)
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Builder.getDefaultAtomicity());
for (auto PAI : NeedsRelease)
Builder.createReleaseValue(Loc, SILValue(PAI),
Builder.getDefaultAtomicity());
}
}
}
/// \brief Populate the body of the cloned closure, modifying instructions as
/// necessary. This is where we create the actual specialized BB Arguments.
void ClosureSpecCloner::populateCloned() {
SILFunction *Cloned = getCloned();
SILFunction *ClosureUser = CallSiteDesc.getApplyCallee();
// Create arguments for the entry block.
SILBasicBlock *ClosureUserEntryBB = &*ClosureUser->begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
// Remove the closure argument.
SILArgument *ClosureArg = nullptr;
for (size_t i = 0, e = ClosureUserEntryBB->args_size(); i != e; ++i) {
SILArgument *Arg = ClosureUserEntryBB->getArgument(i);
if (i == CallSiteDesc.getClosureIndex()) {
ClosureArg = Arg;
continue;
}
// Otherwise, create a new argument which copies the original argument
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(Arg->getType(), Arg->getDecl());
ValueMap.insert(std::make_pair(Arg, MappedValue));
}
// Next we need to add in any arguments that are not captured as arguments to
// the cloned function.
//
// We do not insert the new mapped arguments into the value map since there by
// definition is nothing in the partial apply user function that references
// such arguments. After this pass is done the only thing that will reference
// the arguments is the partial apply that we will create.
SILFunction *ClosedOverFun = CallSiteDesc.getClosureCallee();
CanSILFunctionType ClosedOverFunTy = ClosedOverFun->getLoweredFunctionType();
unsigned NumTotalParams = ClosedOverFunTy->getParameters().size();
unsigned NumNotCaptured = NumTotalParams - CallSiteDesc.getNumArguments();
llvm::SmallVector<SILValue, 4> NewPAIArgs;
for (auto &PInfo : ClosedOverFunTy->getParameters().slice(NumNotCaptured)) {
SILValue MappedValue =
ClonedEntryBB->createFunctionArgument(PInfo.getSILType());
NewPAIArgs.push_back(MappedValue);
}
SILBuilder &Builder = getBuilder();
Builder.setInsertionPoint(ClonedEntryBB);
// Clone FRI and PAI, and replace usage of the removed closure argument
// with result of cloned PAI.
SILValue FnVal =
Builder.createFunctionRef(CallSiteDesc.getLoc(), ClosedOverFun);
auto *NewClosure = CallSiteDesc.createNewClosure(Builder, FnVal, NewPAIArgs);
ValueMap.insert(std::make_pair(ClosureArg, SILValue(NewClosure)));
BBMap.insert(std::make_pair(ClosureUserEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(ClosureUserEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
Builder.setInsertionPoint(BI->second);
visit(BI->first->getTerminator());
}
// Then insert a release in all non failure exit BBs if our partial apply was
// guaranteed. This is b/c it was passed at +0 originally and we need to
// balance the initial increment of the newly created closure.
if (CallSiteDesc.isClosureGuaranteed() &&
CallSiteDesc.closureHasRefSemanticContext()) {
for (SILBasicBlock *BB : CallSiteDesc.getNonFailureExitBBs()) {
SILBasicBlock *OpBB = BBMap[BB];
TermInst *TI = OpBB->getTerminator();
auto Loc = CleanupLocation::get(NewClosure->getLoc());
// If we have a return, we place the release right before it so we know
// that it will be executed at the end of the epilogue.
if (isa<ReturnInst>(TI)) {
Builder.setInsertionPoint(TI);
Builder.createReleaseValue(Loc, SILValue(NewClosure),
Atomicity::Atomic);
continue;
}
// We use casts where findAllNonFailureExitBBs should have made sure that
// this is true. This will ensure that the code is updated when we hit the
// cast failure in debug builds.
auto *Unreachable = cast<UnreachableInst>(TI);
auto PrevIter = std::prev(SILBasicBlock::iterator(Unreachable));
auto NoReturnApply = FullApplySite::isa(&*PrevIter);
// We insert the release value right before the no return apply so that if
// the partial apply is passed into the no-return function as an @owned
// value, we will retain the partial apply before we release it and
// potentially eliminate it.
Builder.setInsertionPoint(NoReturnApply.getInstruction());
Builder.createReleaseValue(Loc, SILValue(NewClosure), Atomicity::Atomic);
}
}
}
unsigned ArgumentDescriptor::updateOptimizedBBArgs(SILBuilder &Builder,
SILBasicBlock *BB,
unsigned ArgOffset) {
// If this argument is completely dead, delete this argument and return
// ArgOffset.
if (IsDead) {
// If we have a callee release and we are dead, set the callee release's
// operand to undef. We do not need it to have the argument anymore, but we
// do need the instruction to be non-null.
//
// TODO: This should not be necessary.
for (auto &X : CalleeRelease) {
SILType CalleeReleaseTy = X->getOperand(0)->getType();
X->setOperand(
0, SILUndef::get(CalleeReleaseTy, Builder.getModule()));
}
// We should be able to recursively delete all of the remaining
// instructions.
SILArgument *Arg = BB->getBBArg(ArgOffset);
eraseUsesOfValue(Arg);
BB->eraseBBArg(ArgOffset);
return ArgOffset;
}
// If this argument is not dead and we did not perform SROA, increment the
// offset and return.
if (!shouldExplode()) {
return ArgOffset + 1;
}
// Create values for the leaf types.
llvm::SmallVector<SILValue, 8> LeafValues;
// Create a reference to the old arg offset and increment arg offset so we can
// create the new arguments.
unsigned OldArgOffset = ArgOffset++;
// We do this in the same order as leaf types since ProjTree expects that the
// order of leaf values matches the order of leaf types.
{
llvm::SmallVector<SILType, 8> LeafTypes;
ProjTree.getLeafTypes(LeafTypes);
for (auto Ty : LeafTypes) {
LeafValues.push_back(BB->insertBBArg(
ArgOffset++, Ty, BB->getBBArg(OldArgOffset)->getDecl()));
}
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
//
// TODO: What is the right location to use here?
ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(),
LeafValues);
// We ignored debugvalue uses when we constructed the new arguments, in order
// to preserve as much information as possible, we construct a new value for
// OrigArg from the leaf values and use that in place of the OrigArg.
SILValue NewOrigArgValue = ProjTree.computeExplodedArgumentValue(Builder,
BB->getParent()->getLocation(),
LeafValues);
// Replace all uses of the original arg with the new value.
SILArgument *OrigArg = BB->getBBArg(OldArgOffset);
OrigArg->replaceAllUsesWith(NewOrigArgValue);
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseBBArg(OldArgOffset);
--ArgOffset;
return ArgOffset;
}
void
ConsumedResultToEpilogueRetainMatcher::
findMatchingRetains(SILBasicBlock *BB) {
// Iterate over the instructions post-order and find retains associated with
// return value.
SILValue RV = SILValue();
for (auto II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
if (ReturnInst *RI = dyn_cast<ReturnInst>(&*II)) {
RV = RI->getOperand();
break;
}
}
// Somehow, we managed not to find a return value.
if (!RV)
return;
// OK. we've found the return value, now iterate on the CFG to find all the
// post-dominating retains.
//
// The ConsumedArgToEpilogueReleaseMatcher finds the final releases
// in the following way.
//
// 1. If an instruction, which is not releaseinst nor releasevalue, that
// could decrement reference count is found. bail out.
//
// 2. If a release is found and the release that can not be mapped to any
// @owned argument. bail as this release may well be the final release of
// an @owned argument, but somehow rc-identity fails to prove that.
//
// 3. A release that is mapped to an argument which already has a release
// that overlaps with this release. This release for sure is not the final
// release.
constexpr unsigned WorkListMaxSize = 4;
llvm::DenseSet<SILBasicBlock *> RetainFrees;
llvm::SmallVector<BasicBlockRetainValue, 4> WorkList;
llvm::DenseSet<SILBasicBlock *> HandledBBs;
WorkList.push_back(std::make_pair(BB, RV));
HandledBBs.insert(BB);
while (!WorkList.empty()) {
// Too many blocks ?.
if (WorkList.size() > WorkListMaxSize) {
EpilogueRetainInsts.clear();
return;
}
// Try to find a retain %value in this basic block.
auto R = WorkList.pop_back_val();
RetainKindValue Kind = findMatchingRetainsInBasicBlock(R.first, R.second);
// We've found a retain on this path.
if (Kind.first == FindRetainKind::Found) {
EpilogueRetainInsts.push_back(Kind.second);
continue;
}
// There is a MayDecrement instruction.
if (Kind.first == FindRetainKind::Blocked) {
EpilogueRetainInsts.clear();
return;
}
// There is a self-recursion. Use the apply instruction as the retain.
if (Kind.first == FindRetainKind::Recursion) {
EpilogueRetainInsts.push_back(Kind.second);
continue;
}
// Did not find a retain in this block, try to go to its predecessors.
if (Kind.first == FindRetainKind::None) {
// We can not find a retain in a block with no predecessors.
if (R.first->getPreds().begin() == R.first->getPreds().end()) {
EpilogueRetainInsts.clear();
return;
}
// This block does not have a retain.
RetainFrees.insert(R.first);
// If this is a SILArgument of current basic block, we can split it up to
// values in the predecessors.
SILArgument *SA = dyn_cast<SILArgument>(R.second);
if (SA && SA->getParent() != R.first)
SA = nullptr;
for (auto X : R.first->getPreds()) {
if (HandledBBs.find(X) != HandledBBs.end())
continue;
// Try to use the predecessor edge-value.
if (SA && SA->getIncomingValue(X)) {
WorkList.push_back(std::make_pair(X, SA->getIncomingValue(X)));
}
else
WorkList.push_back(std::make_pair(X, R.second));
HandledBBs.insert(X);
}
}
}
// Lastly, check whether all the successor blocks are retain-free.
if (!isTransitiveSuccessorsRetainFree(RetainFrees))
EpilogueRetainInsts.clear();
// At this point, we've either failed to find any epilogue retains or
// all the post-dominating epilogue retains.
}
unsigned ArgumentDescriptor::updateOptimizedBBArgs(SILBuilder &Builder,
SILBasicBlock *BB,
unsigned ArgOffset) {
// If this argument is completely dead, delete this argument and return
// ArgOffset.
if (IsDead) {
// If we have a callee release and we are dead, set the callee release's
// operand to undef. We do not need it to have the argument anymore, but we
// do need the instruction to be non-null.
//
// TODO: This should not be necessary.
if (CalleeRelease) {
SILType CalleeReleaseTy = CalleeRelease->getOperand(0)->getType();
CalleeRelease->setOperand(
0, SILUndef::get(CalleeReleaseTy, Builder.getModule()));
// TODO: Currently we cannot mark arguments as dead if they are released
// in a throw block. But as soon as we can do this, we have to handle
// CalleeReleaseInThrowBlock as well.
assert(!CalleeReleaseInThrowBlock &&
"released arg in throw block cannot be dead");
}
// We should be able to recursively delete all of the remaining
// instructions.
SILArgument *Arg = BB->getBBArg(ArgOffset);
eraseUsesOfValue(Arg);
BB->eraseBBArg(ArgOffset);
return ArgOffset;
}
// If this argument is not dead and we did not perform SROA, increment the
// offset and return.
if (!shouldExplode()) {
return ArgOffset + 1;
}
// Create values for the leaf types.
llvm::SmallVector<SILValue, 8> LeafValues;
// Create a reference to the old arg offset and increment arg offset so we can
// create the new arguments.
unsigned OldArgOffset = ArgOffset++;
// We do this in the same order as leaf types since ProjTree expects that the
// order of leaf values matches the order of leaf types.
{
llvm::SmallVector<SILType, 8> LeafTypes;
ProjTree.getLeafTypes(LeafTypes);
for (auto Ty : LeafTypes) {
LeafValues.push_back(BB->insertBBArg(
ArgOffset++, Ty, BB->getBBArg(OldArgOffset)->getDecl()));
}
}
// Then go through the projection tree constructing aggregates and replacing
// uses.
//
// TODO: What is the right location to use here?
ProjTree.replaceValueUsesWithLeafUses(Builder, BB->getParent()->getLocation(),
LeafValues);
// Replace all uses of the original arg with undef so it does not have any
// uses.
SILArgument *OrigArg = BB->getBBArg(OldArgOffset);
OrigArg->replaceAllUsesWith(SILUndef::get(OrigArg->getType(), BB->getModule()));
// Now erase the old argument since it does not have any uses. We also
// decrement ArgOffset since we have one less argument now.
BB->eraseBBArg(OldArgOffset);
--ArgOffset;
return ArgOffset;
}
void ReleaseCodeMotionContext::computeCodeMotionInsertPoints() {
// The BBSetIns have converged, run last iteration and figure out insertion
// point for each RC root.
for (SILBasicBlock *BB : PO->getPostOrder()) {
// Intersect in the successor BBSetIns.
mergeBBDataFlowStates(BB);
ReleaseBlockState *S = BlockStates[BB];
// Compute insertion point generated by the edge value transition.
// If there is a transition from 1 to 0, that means we have a partial
// merge, which means the release can NOT be hoisted to the current block.
// place it at the successors.
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
if (S->BBSetOut[i])
continue;
for (auto &Succ : BB->getSuccessors()) {
BlockState *SBB = BlockStates[Succ];
if (!SBB->BBSetIn[i])
continue;
InsertPoints[RCRootVault[i]].push_back(&*(*Succ).begin());
}
}
// Is this block interesting ?
if (MultiIteration && !InterestBlocks.count(BB))
continue;
// Compute insertion point generated by MayUse terminator inst.
// If terminator instruction can block the RC root. We will have no
// choice but to anchor the release instructions in the successor blocks.
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
SILInstruction *Term = BB->getTerminator();
if (!S->BBSetOut[i] || !mayBlockCodeMotion(Term, RCRootVault[i]))
continue;
for (auto &Succ : BB->getSuccessors()) {
BlockState *SBB = BlockStates[Succ];
if (!SBB->BBSetIn[i])
continue;
InsertPoints[RCRootVault[i]].push_back(&*(*Succ).begin());
}
S->BBSetOut.reset(i);
}
// Compute insertion point generated within the basic block. Process
// instructions in post-order fashion.
for (auto I = std::next(BB->rbegin()), E = BB->rend(); I != E; ++I) {
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
if (!S->BBSetOut[i] || !mayBlockCodeMotion(&*I, RCRootVault[i]))
continue;
auto *InsertPt = &*std::next(SILBasicBlock::iterator(&*I));
InsertPoints[RCRootVault[i]].push_back(InsertPt);
S->BBSetOut.reset(i);
}
// If we are freezing this epilogue release. Simply continue.
if (FreezeEpilogueReleases && ERM.isEpilogueRelease(&*I))
continue;
// This release generates.
if (isReleaseInstruction(&*I)) {
S->BBSetOut.set(RCRootIndex[getRCRoot(&*I)]);
}
}
// Compute insertion point generated by SILArgument. SILArgument blocks if
// it defines the released value.
for (unsigned i = 0; i < RCRootVault.size(); ++i) {
if (!S->BBSetOut[i])
continue;
SILArgument *A = dyn_cast<SILArgument>(RCRootVault[i]);
if (!A || A->getParent() != BB)
continue;
InsertPoints[RCRootVault[i]].push_back(&*BB->begin());
S->BBSetOut.reset(i);
}
// Lastly update the BBSetIn, only necessary when we are running a single
// iteration dataflow.
if (!MultiIteration) {
S->updateBBSetIn(S->BBSetOut);
}
}
}
void GenericCloner::populateCloned() {
SILFunction *Cloned = getCloned();
// Create arguments for the entry block.
SILBasicBlock *OrigEntryBB = &*Original.begin();
SILBasicBlock *ClonedEntryBB = Cloned->createBasicBlock();
getBuilder().setInsertionPoint(ClonedEntryBB);
llvm::SmallVector<AllocStackInst *, 8> AllocStacks;
AllocStackInst *ReturnValueAddr = nullptr;
// Create the entry basic block with the function arguments.
auto I = OrigEntryBB->args_begin(), E = OrigEntryBB->args_end();
int ArgIdx = 0;
while (I != E) {
SILArgument *OrigArg = *I;
RegularLocation Loc((Decl *)OrigArg->getDecl());
AllocStackInst *ASI = nullptr;
SILType mappedType = remapType(OrigArg->getType());
if (ReInfo.isArgConverted(ArgIdx)) {
// We need an alloc_stack as a replacement for the indirect parameter.
assert(mappedType.isAddress());
mappedType = mappedType.getObjectType();
ASI = getBuilder().createAllocStack(Loc, mappedType);
ValueMap[OrigArg] = ASI;
AllocStacks.push_back(ASI);
if (ReInfo.isResultIndex(ArgIdx)) {
// This result is converted from indirect to direct. The return inst
// needs to load the value from the alloc_stack. See below.
assert(!ReturnValueAddr);
ReturnValueAddr = ASI;
} else {
// Store the new direct parameter to the alloc_stack.
auto *NewArg =
ClonedEntryBB->createArgument(mappedType, OrigArg->getDecl());
getBuilder().createStore(Loc, NewArg, ASI,
StoreOwnershipQualifier::Unqualified);
// Try to create a new debug_value from an existing debug_value_addr.
for (Operand *ArgUse : OrigArg->getUses()) {
if (auto *DVAI = dyn_cast<DebugValueAddrInst>(ArgUse->getUser())) {
getBuilder().createDebugValue(DVAI->getLoc(), NewArg,
DVAI->getVarInfo());
break;
}
}
}
} else {
auto *NewArg =
ClonedEntryBB->createArgument(mappedType, OrigArg->getDecl());
ValueMap[OrigArg] = NewArg;
}
++I;
++ArgIdx;
}
BBMap.insert(std::make_pair(OrigEntryBB, ClonedEntryBB));
// Recursively visit original BBs in depth-first preorder, starting with the
// entry block, cloning all instructions other than terminators.
visitSILBasicBlock(OrigEntryBB);
// Now iterate over the BBs and fix up the terminators.
for (auto BI = BBMap.begin(), BE = BBMap.end(); BI != BE; ++BI) {
getBuilder().setInsertionPoint(BI->second);
TermInst *OrigTermInst = BI->first->getTerminator();
if (auto *RI = dyn_cast<ReturnInst>(OrigTermInst)) {
SILValue ReturnValue;
if (ReturnValueAddr) {
// The result is converted from indirect to direct. We have to load the
// returned value from the alloc_stack.
ReturnValue =
getBuilder().createLoad(ReturnValueAddr->getLoc(), ReturnValueAddr,
LoadOwnershipQualifier::Unqualified);
}
for (AllocStackInst *ASI : reverse(AllocStacks)) {
getBuilder().createDeallocStack(ASI->getLoc(), ASI);
}
if (ReturnValue) {
getBuilder().createReturn(RI->getLoc(), ReturnValue);
continue;
}
} else if (isa<ThrowInst>(OrigTermInst)) {
for (AllocStackInst *ASI : reverse(AllocStacks)) {
getBuilder().createDeallocStack(ASI->getLoc(), ASI);
}
}
visit(BI->first->getTerminator());
}
}