void LoopTreeOptimization::analyzeCurrentLoop(
std::unique_ptr<LoopNestSummary> &CurrSummary, ReadSet &SafeReads) {
WriteSet &MayWrites = CurrSummary->MayWrites;
SILLoop *Loop = CurrSummary->Loop;
DEBUG(llvm::dbgs() << " Analyzing accesses.\n");
// Contains function calls in the loop, which only read from memory.
SmallVector<ApplyInst *, 8> ReadOnlyApplies;
for (auto *BB : Loop->getBlocks()) {
for (auto &Inst : *BB) {
// Ignore fix_lifetime instructions.
if (isa<FixLifetimeInst>(&Inst))
continue;
// Collect loads.
auto LI = dyn_cast<LoadInst>(&Inst);
if (LI) {
if (!mayWriteTo(AA, MayWrites, LI))
SafeReads.insert(LI);
continue;
}
if (auto *AI = dyn_cast<ApplyInst>(&Inst)) {
// In contrast to load instructions, we first collect all read-only
// function calls and add them later to SafeReads.
SideEffectAnalysis::FunctionEffects E;
SEA->getEffects(E, AI);
auto MB = E.getMemBehavior(RetainObserveKind::ObserveRetains);
if (MB <= SILInstruction::MemoryBehavior::MayRead)
ReadOnlyApplies.push_back(AI);
}
if (Inst.mayHaveSideEffects()) {
MayWrites.push_back(&Inst);
// Remove clobbered loads we have seen before.
removeWrittenTo(AA, SafeReads, &Inst);
}
}
}
for (auto *AI : ReadOnlyApplies) {
if (!mayWriteTo(AA, SEA, MayWrites, AI))
SafeReads.insert(AI);
}
}
SILBasicBlock *swift::splitEdge(TermInst *T, unsigned EdgeIdx,
DominanceInfo *DT, SILLoopInfo *LI) {
auto *SrcBB = T->getParent();
auto *Fn = SrcBB->getParent();
SILBasicBlock *DestBB = T->getSuccessors()[EdgeIdx];
// Create a new basic block in the edge, and insert it after the SrcBB.
auto *EdgeBB = new (Fn->getModule()) SILBasicBlock(Fn, SrcBB);
SmallVector<SILValue, 16> Args;
getEdgeArgs(T, EdgeIdx, EdgeBB, Args);
SILBuilder(EdgeBB).createBranch(T->getLoc(), DestBB, Args);
// Strip the arguments and rewire the branch in the source block.
changeBranchTarget(T, EdgeIdx, EdgeBB, /*PreserveArgs=*/false);
if (!DT && !LI)
return EdgeBB;
// Update the dominator tree.
if (DT) {
auto *SrcBBNode = DT->getNode(SrcBB);
// Unreachable code could result in a null return here.
if (SrcBBNode) {
// The new block is dominated by the SrcBB.
auto *EdgeBBNode = DT->addNewBlock(EdgeBB, SrcBB);
// Are all predecessors of DestBB dominated by DestBB?
auto *DestBBNode = DT->getNode(DestBB);
bool OldSrcBBDominatesAllPreds = std::all_of(
DestBB->pred_begin(), DestBB->pred_end(), [=](SILBasicBlock *B) {
if (B == EdgeBB)
return true;
auto *PredNode = DT->getNode(B);
if (!PredNode)
return true;
if (DT->dominates(DestBBNode, PredNode))
return true;
return false;
});
// If so, the new bb dominates DestBB now.
if (OldSrcBBDominatesAllPreds)
DT->changeImmediateDominator(DestBBNode, EdgeBBNode);
}
}
if (!LI)
return EdgeBB;
// Update loop info. Both blocks must be in a loop otherwise the split block
// is outside the loop.
SILLoop *SrcBBLoop = LI->getLoopFor(SrcBB);
if (!SrcBBLoop)
return EdgeBB;
SILLoop *DstBBLoop = LI->getLoopFor(DestBB);
if (!DstBBLoop)
return EdgeBB;
// Same loop.
if (DstBBLoop == SrcBBLoop) {
DstBBLoop->addBasicBlockToLoop(EdgeBB, LI->getBase());
return EdgeBB;
}
// Edge from inner to outer loop.
if (DstBBLoop->contains(SrcBBLoop)) {
DstBBLoop->addBasicBlockToLoop(EdgeBB, LI->getBase());
return EdgeBB;
}
// Edge from outer to inner loop.
if (SrcBBLoop->contains(DstBBLoop)) {
SrcBBLoop->addBasicBlockToLoop(EdgeBB, LI->getBase());
return EdgeBB;
}
// Neither loop contains the other. The destination must be the header of its
// loop. Otherwise, we would be creating irreducible control flow.
assert(DstBBLoop->getHeader() == DestBB &&
"Creating irreducible control flow?");
// Add to outer loop if there is one.
if (auto *Parent = DstBBLoop->getParentLoop())
Parent->addBasicBlockToLoop(EdgeBB, LI->getBase());
return EdgeBB;
}
ShortestPathAnalysis::Weight ShortestPathAnalysis::
getWeight(SILBasicBlock *BB, Weight CallerWeight) {
assert(BB->getParent() == F);
SILLoop *Loop = LI->getLoopFor(BB);
if (!Loop) {
// We are not in a loop. So just account the length of our function scope
// in to the length of the CallerWeight.
return Weight(CallerWeight.ScopeLength + getScopeLength(BB, 0),
CallerWeight.LoopWeight);
}
int LoopDepth = Loop->getLoopDepth();
// Deal with the corner case of having more than 4 nested loops.
while (LoopDepth >= MaxNumLoopLevels) {
--LoopDepth;
Loop = Loop->getParentLoop();
}
Weight W(getScopeLength(BB, LoopDepth), SingleLoopWeight);
// Add weights for all the loops BB is in.
while (Loop) {
assert(LoopDepth > 0);
BlockInfo *HeaderInfo = getBlockInfo(Loop->getHeader());
int InnerLoopLength = HeaderInfo->getScopeLength(LoopDepth) *
ShortestPathAnalysis::LoopCount;
int OuterLoopWeight = SingleLoopWeight;
int OuterScopeLength = HeaderInfo->getScopeLength(LoopDepth - 1);
// Reaching the outermost loop, we use the CallerWeight to get the outer
// length+loopweight.
if (LoopDepth == 1) {
// If the apply in the caller is not in a significant loop, just stop with
// what we have now.
if (CallerWeight.LoopWeight < 4)
return W;
// If this function is part of the caller's scope length take the caller's
// scope length. Note: this is not the case e.g. if the apply is in a
// then-branch of an if-then-else in the caller and the else-branch is
// the short path.
if (CallerWeight.ScopeLength > OuterScopeLength)
OuterScopeLength = CallerWeight.ScopeLength;
OuterLoopWeight = CallerWeight.LoopWeight;
}
assert(OuterScopeLength >= InnerLoopLength);
// If the current loop is only a small part of its outer loop, we don't
// take the outer loop that much into account. Only if the current loop is
// actually the "main part" in the outer loop we add the full loop weight
// for the outer loop.
if (OuterScopeLength < InnerLoopLength * 2) {
W.LoopWeight += OuterLoopWeight - 1;
} else if (OuterScopeLength < InnerLoopLength * 3) {
W.LoopWeight += OuterLoopWeight - 2;
} else if (OuterScopeLength < InnerLoopLength * 4) {
W.LoopWeight += OuterLoopWeight - 3;
} else {
return W;
}
--LoopDepth;
Loop = Loop->getParentLoop();
}
assert(LoopDepth == 0);
return W;
}
// Analyzes current loop for hosting/sinking potential:
// Computes set of instructions we may be able to move out of the loop
// Important Note:
// We can't bail out of this method! we have to run it on all loops.
// We *need* to discover all MayWrites -
// even if the loop is otherwise skipped!
// This is because outer loops will depend on the inner loop's writes.
void LoopTreeOptimization::analyzeCurrentLoop(
std::unique_ptr<LoopNestSummary> &CurrSummary) {
WriteSet &MayWrites = CurrSummary->MayWrites;
SILLoop *Loop = CurrSummary->Loop;
LLVM_DEBUG(llvm::dbgs() << " Analyzing accesses.\n");
// Contains function calls in the loop, which only read from memory.
SmallVector<ApplyInst *, 8> ReadOnlyApplies;
// Contains Loads inside the loop.
SmallVector<LoadInst *, 8> Loads;
// Contains fix_lifetime, we might be able to sink them.
SmallVector<FixLifetimeInst *, 8> FixLifetimes;
// Contains begin_access, we might be able to hoist them.
SmallVector<BeginAccessInst *, 8> BeginAccesses;
// Contains all applies - used for begin_access
SmallVector<FullApplySite, 8> fullApplies;
for (auto *BB : Loop->getBlocks()) {
for (auto &Inst : *BB) {
switch (Inst.getKind()) {
case SILInstructionKind::FixLifetimeInst: {
auto *FL = dyn_cast<FixLifetimeInst>(&Inst);
assert(FL && "Expected a FixLifetime instruction");
FixLifetimes.push_back(FL);
// We can ignore the side effects of FixLifetimes
break;
}
case SILInstructionKind::LoadInst: {
auto *LI = dyn_cast<LoadInst>(&Inst);
assert(LI && "Expected a Load instruction");
Loads.push_back(LI);
break;
}
case SILInstructionKind::BeginAccessInst: {
auto *BI = dyn_cast<BeginAccessInst>(&Inst);
assert(BI && "Expected a Begin Access");
BeginAccesses.push_back(BI);
checkSideEffects(Inst, MayWrites);
break;
}
case SILInstructionKind::RefElementAddrInst: {
auto *REA = static_cast<RefElementAddrInst *>(&Inst);
SpecialHoist.push_back(REA);
break;
}
case swift::SILInstructionKind::CondFailInst: {
// We can (and must) hoist cond_fail instructions if the operand is
// invariant. We must hoist them so that we preserve memory safety. A
// cond_fail that would have protected (executed before) a memory access
// must - after hoisting - also be executed before said access.
HoistUp.insert(&Inst);
checkSideEffects(Inst, MayWrites);
break;
}
case SILInstructionKind::ApplyInst: {
auto *AI = dyn_cast<ApplyInst>(&Inst);
assert(AI && "Expected an Apply Instruction");
if (isSafeReadOnlyApply(SEA, AI)) {
ReadOnlyApplies.push_back(AI);
}
// check for array semantics and side effects - same as default
LLVM_FALLTHROUGH;
}
default: {
if (auto fullApply = FullApplySite::isa(&Inst)) {
fullApplies.push_back(fullApply);
}
checkSideEffects(Inst, MayWrites);
if (canHoistUpDefault(&Inst, Loop, DomTree, RunsOnHighLevelSIL)) {
HoistUp.insert(&Inst);
}
break;
}
}
}
}
auto *Preheader = Loop->getLoopPreheader();
if (!Preheader) {
// Can't hoist/sink instructions
return;
}
for (auto *AI : ReadOnlyApplies) {
if (!mayWriteTo(AA, SEA, MayWrites, AI)) {
HoistUp.insert(AI);
}
}
for (auto *LI : Loads) {
if (!mayWriteTo(AA, MayWrites, LI)) {
HoistUp.insert(LI);
}
}
bool mayWritesMayRelease =
std::any_of(MayWrites.begin(), MayWrites.end(),
[&](SILInstruction *W) { return W->mayRelease(); });
for (auto *FL : FixLifetimes) {
if (!DomTree->dominates(FL->getOperand()->getParentBlock(), Preheader)) {
continue;
}
if (!mayWriteTo(AA, MayWrites, FL) || !mayWritesMayRelease) {
SinkDown.push_back(FL);
}
}
for (auto *BI : BeginAccesses) {
if (!handledEndAccesses(BI, Loop)) {
LLVM_DEBUG(llvm::dbgs() << "Skipping: " << *BI);
LLVM_DEBUG(llvm::dbgs() << "Some end accesses can't be handled\n");
continue;
}
if (analyzeBeginAccess(BI, BeginAccesses, fullApplies, MayWrites, ASA,
DomTree)) {
SpecialHoist.push_back(BI);
}
}
}
// Attempt to insert a new access in the loop preheader. If successful, insert
// the new access in DominatedAccessAnalysis so it can be used to dominate other
// accesses. Also convert the current access to static and update the current
// storageToDomMap since the access may already have been recorded (when it was
// still dynamic).
//
// This function cannot add or remove instructions in the current block, but
// may add instructions to the current loop's preheader.
//
// The required conditions for inserting a new dominating access are:
//
// 1. The new preheader access is not enclosed in another scope that doesn't
// also enclose the current scope.
//
// This is inferred from the loop structure; any scope that encloses the
// preheader must also enclose the entire loop.
//
// 2. The current access is not enclosed in another scope that doesn't also
// enclose the preheader.
//
// As before, it is sufficient to check this access' isInner flags in
// DominatedAccessAnalysis; if this access isn't enclosed by any scope within
// the function, then it can't be enclosed within a scope inside the loop.
//
// 3. The current header has no nested conflict within its scope.
//
// 4. The access' source operand is available in the loop preheader.
void DominatedAccessRemoval::tryInsertLoopPreheaderAccess(
BeginAccessInst *BAI, DomAccessedStorage currAccessInfo) {
// 2. the current access may be enclosed.
if (currAccessInfo.isInner())
return;
// 3. the current access must be instantaneous.
if (!BAI->hasNoNestedConflict())
return;
SILLoop *currLoop = loopInfo->getLoopFor(BAI->getParent());
if (!currLoop)
return;
SILBasicBlock *preheader = currLoop->getLoopPreheader();
if (!preheader)
return;
// 4. The source operand must be available in the preheader.
auto sourceOperand = BAI->getOperand();
auto *sourceBB = sourceOperand->getParentBlock();
if (!domInfo->dominates(sourceBB, preheader))
return;
// Insert a new access scope immediately before the
// preheader's terminator.
TermInst *preheaderTerm = preheader->getTerminator();
SILBuilderWithScope scopeBuilder(preheaderTerm);
BeginAccessInst *newBegin = scopeBuilder.createBeginAccess(
preheaderTerm->getLoc(), sourceOperand, BAI->getAccessKind(),
SILAccessEnforcement::Dynamic, true /*no nested conflict*/,
BAI->isFromBuiltin());
scopeBuilder.createEndAccess(preheaderTerm->getLoc(), newBegin, false);
LLVM_DEBUG(llvm::dbgs() << "Created loop preheader access: " << *newBegin
<< "\n"
<< "dominating: " << *BAI << "\n");
BAI->setEnforcement(SILAccessEnforcement::Static);
hasChanged = true;
// Insert the new dominating instruction in both DominatedAccessAnalysis and
// storageToDomMap if it has uniquely identifiable storage.
if (!currAccessInfo.isUniquelyIdentifiedOrClass())
return;
AccessedStorage storage = static_cast<AccessedStorage>(currAccessInfo);
storage.resetSubclassData();
// Create a DomAccessedStorage for the new access with no flags set.
DAA.accessMap.try_emplace(newBegin, DomAccessedStorage(storage));
// Track the new access as long as no other accesses from the same storage are
// already tracked. This also necessarily replaces the current access, which
// was just made static.
DominatingAccess newDomAccess(newBegin, domInfo->getNode(preheader));
auto iterAndInserted = storageToDomMap.try_emplace(storage, newDomAccess);
if (!iterAndInserted.second) {
DominatingAccess &curDomAccess = iterAndInserted.first->second;
if (curDomAccess.beginAccess == BAI)
curDomAccess = newDomAccess;
}
}