/// Check that all users of the destination address of the copy are dominated by
/// the copy. There is no path around copy that could initialize %dest with a
/// different value.
bool CopyForwarding::areCopyDestUsersDominatedBy(
CopyAddrInst *Copy, SmallVectorImpl<Operand *> &DestUses) {
SILValue CopyDest = Copy->getDest();
DominanceInfo *DT = nullptr;
for (auto *Use : CopyDest.getUses()) {
auto *UserInst = Use->getUser();
if (UserInst == Copy)
continue;
// Initialize the dominator tree info.
if (!DT)
DT = DomAnalysis->get(Copy->getFunction());
// Check dominance of the parent blocks.
if (!DT->dominates(Copy->getParent(), UserInst->getParent()))
return false;
bool CheckDominanceInBlock = Copy->getParent() == UserInst->getParent();
// Check whether Copy is before UserInst.
if (CheckDominanceInBlock) {
auto SI = Copy->getIterator(), SE = Copy->getParent()->end();
for (++SI; SI != SE; ++SI)
if (&*SI == UserInst)
break;
if (SI == SE)
return false;
}
// We can forward to this use.
DestUses.push_back(Use);
}
return true;
}
/// \return true if the given block is dominated by a _slowPath branch hint.
///
/// Cache all blocks visited to avoid introducing quadratic behavior.
bool ColdBlockInfo::isCold(const SILBasicBlock *BB, int recursionDepth) {
auto I = ColdBlockMap.find(BB);
if (I != ColdBlockMap.end())
return I->second;
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DominanceInfo *DT = DA->get(const_cast<SILFunction*>(BB->getParent()));
DomTreeNode *Node = DT->getNode(const_cast<SILBasicBlock*>(BB));
// Always consider unreachable code cold.
if (!Node)
return true;
std::vector<const SILBasicBlock*> DomChain;
DomChain.push_back(BB);
bool IsCold = false;
Node = Node->getIDom();
while (Node) {
if (isSlowPath(Node->getBlock(), DomChain.back(), recursionDepth)) {
IsCold = true;
break;
}
auto I = ColdBlockMap.find(Node->getBlock());
if (I != ColdBlockMap.end()) {
IsCold = I->second;
break;
}
DomChain.push_back(Node->getBlock());
Node = Node->getIDom();
}
for (auto *ChainBB : DomChain)
ColdBlockMap[ChainBB] = IsCold;
return IsCold;
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst *> Calls) {
LLVM_DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< Demangle::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction *, ApplyInst *> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
ApplyInst *HoistAI =
getHoistedApplyForInitializer(AI, DT, InitF, ParentF, ParentFuncs);
// If we were unable to find anything, just go onto the next apply.
if (!HoistAI) {
continue;
}
// Otherwise, move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
LLVM_DEBUG(llvm::dbgs() << " don't hoist above availability check "
"at bb"
<< DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
LLVM_DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName()
<< "\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI << " in "
<< HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
bool StackPromoter::tryPromoteAlloc(AllocRefInst *ARI) {
SILInstruction *AllocInsertionPoint = nullptr;
SILInstruction *DeallocInsertionPoint = nullptr;
if (!canPromoteAlloc(ARI, AllocInsertionPoint, DeallocInsertionPoint))
return false;
if (AllocInsertionPoint) {
// Check if any operands of the alloc_ref prevents us from moving the
// instruction.
for (const Operand &Op : ARI->getAllOperands()) {
if (!DT->properlyDominates(Op.get(), AllocInsertionPoint))
return false;
}
}
DEBUG(llvm::dbgs() << "Promoted " << *ARI);
DEBUG(llvm::dbgs() << " in " << ARI->getFunction()->getName() << '\n');
NumStackPromoted++;
SILBuilder B(DeallocInsertionPoint);
// It's an object allocation. We set the [stack] attribute in the alloc_ref.
ARI->setStackAllocatable();
if (AllocInsertionPoint)
ARI->moveBefore(AllocInsertionPoint);
/// And create a dealloc_ref [stack] at the end of the object's lifetime.
B.createDeallocRef(ARI->getLoc(), ARI, true);
return true;
}
bool strictlyDominates(SILBasicBlock *A, SILBasicBlock *B) {
return A != B && DT->dominates(A, B);
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst*> Calls) {
DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< demangle_wrappers::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction*, ApplyInst*> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
auto PFI = ParentFuncs.find(ParentF);
ApplyInst *HoistAI = nullptr;
if (PFI != ParentFuncs.end()) {
// Found a replacement for this init call.
// Ensure the replacement dominates the original call site.
ApplyInst *CommonAI = PFI->second;
assert(cast<FunctionRefInst>(CommonAI->getCallee())
->getReferencedFunction() == InitF &&
"ill-formed global init call");
SILBasicBlock *DomBB =
DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
// We must not move initializers around availability-checks.
if (!isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT)) {
if (DomBB != CommonAI->getParent()) {
CommonAI->moveBefore(&*DomBB->begin());
placeFuncRef(CommonAI, DT);
// Try to hoist the existing AI again if we move it to another block,
// e.g. from a loop exit into the loop.
HoistAI = CommonAI;
}
AI->replaceAllUsesWith(CommonAI);
AI->eraseFromParent();
HasChanged = true;
}
} else {
ParentFuncs[ParentF] = AI;
// It's the first time we found a call to InitF in this function, so we
// try to hoist it out of any loop.
HoistAI = AI;
}
if (HoistAI) {
// Move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
DEBUG(llvm::dbgs() << " don't hoist above availability check at bb" <<
DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
}
else {
DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
}
}
bool RCIdentityFunctionInfo::
findDominatingNonPayloadedEdge(SILBasicBlock *IncomingEdgeBB,
SILValue RCIdentity) {
// First grab the NonPayloadedEnumBB and RCIdentityBB. If we cannot find
// either of them, return false.
SILBasicBlock *RCIdentityBB = RCIdentity->getParentBlock();
if (!RCIdentityBB)
return false;
// Make sure that the incoming edge bb is not the RCIdentityBB. We are not
// trying to handle this case here, so simplify by just bailing if we detect
// it.
//
// I think the only way this can happen is if we have a switch_enum of some
// sort with multiple incoming values going into the destination BB. We are
// not interested in handling that case anyways.
//
// FIXME: If we ever split all critical edges, this should be relooked at.
if (IncomingEdgeBB == RCIdentityBB)
return false;
// Now we know that RCIdentityBB and IncomingEdgeBB are different. Prove that
// RCIdentityBB dominates IncomingEdgeBB.
SILFunction *F = RCIdentityBB->getParent();
// First make sure that IncomingEdgeBB dominates NonPayloadedEnumBB. If not,
// return false.
DominanceInfo *DI = DA->get(F);
if (!DI->dominates(RCIdentityBB, IncomingEdgeBB))
return false;
// Now walk up the dominator tree from IncomingEdgeBB to RCIdentityBB and see
// if we can find a use of RCIdentity that dominates IncomingEdgeBB and
// enables us to know that RCIdentity must be a no-payload enum along
// IncomingEdge. We don't care if the case or enum of RCIdentity match the
// case or enum along RCIdentityBB since a pairing of retain, release of two
// non-payloaded enums can always be eliminated (since we can always eliminate
// ref count operations on non-payloaded enums).
// RCIdentityBB must have a valid dominator tree node.
auto *EndDomNode = DI->getNode(RCIdentityBB);
if (!EndDomNode)
return false;
for (auto *Node = DI->getNode(IncomingEdgeBB); Node; Node = Node->getIDom()) {
// Search for uses of RCIdentity in Node->getBlock() that will enable us to
// know that it has a non-payloaded enum case.
SILBasicBlock *DominatingBB = Node->getBlock();
llvm::Optional<bool> Result =
proveNonPayloadedEnumCase(DominatingBB, RCIdentity);
// If we found either a signal of a payloaded or a non-payloaded enum,
// return that value.
if (Result.hasValue())
return Result.getValue();
// If we didn't reach RCIdentityBB, keep processing up the DomTree.
if (DominatingBB != RCIdentityBB)
continue;
// Otherwise, we failed to find any interesting information, return false.
return false;
}
return false;
}
