/// Hoists the necessary check for beginning and end of the induction
/// encapsulated by this access function to the header.
void hoistCheckToPreheader(ArraySemanticsCall CheckToHoist,
SILBasicBlock *Preheader,
DominanceInfo *DT) {
ApplyInst *AI = CheckToHoist;
SILLocation Loc = AI->getLoc();
SILBuilderWithScope Builder(Preheader->getTerminator(), AI);
// Get the first induction value.
auto FirstVal = Ind->getFirstValue();
// Clone the struct for the start index.
auto Start = cast<SILInstruction>(CheckToHoist.getIndex())
->clone(Preheader->getTerminator());
// Set the new start index to the first value of the induction.
Start->setOperand(0, FirstVal);
// Clone and fixup the load, retain sequence to the header.
auto NewCheck = CheckToHoist.copyTo(Preheader->getTerminator(), DT);
NewCheck->setOperand(1, Start);
// Get the last induction value.
auto LastVal = Ind->getLastValue(Loc, Builder);
// Clone the struct for the end index.
auto End = cast<SILInstruction>(CheckToHoist.getIndex())
->clone(Preheader->getTerminator());
// Set the new end index to the last value of the induction.
End->setOperand(0, LastVal);
NewCheck = CheckToHoist.copyTo(Preheader->getTerminator(), DT);
NewCheck->setOperand(1, End);
}
/// Returns true if the block \p BB is terminated with a cond_br based on an
/// availability check.
static bool isAvailabilityCheck(SILBasicBlock *BB) {
CondBranchInst *CBR = dyn_cast<CondBranchInst>(BB->getTerminator());
if (!CBR)
return false;
ApplyInst *AI = dyn_cast<ApplyInst>(CBR->getCondition());
if (!AI)
return false;
SILFunction *F = AI->getCalleeFunction();
if (!F || !F->hasSemanticsAttrs())
return false;
return F->hasSemanticsAttrsThatStartsWith("availability");
}
/// Returns the Apply and WitnessMethod instructions that use the
/// open_existential_addr instructions, or null if at least one of the
/// instructions is missing.
static ApplyWitnessPair getOpenExistentialUsers(OpenExistentialAddrInst *OE) {
ApplyInst *AI = nullptr;
WitnessMethodInst *WMI = nullptr;
ApplyWitnessPair Empty = std::make_pair(nullptr, nullptr);
for (auto *UI : getNonDebugUses(OE)) {
auto *User = UI->getUser();
if (!isa<WitnessMethodInst>(User) &&
User->isTypeDependentOperand(UI->getOperandNumber()))
continue;
// Check that we have a single Apply user.
if (auto *AA = dyn_cast<ApplyInst>(User)) {
if (AI)
return Empty;
AI = AA;
continue;
}
// Check that we have a single WMI user.
if (auto *W = dyn_cast<WitnessMethodInst>(User)) {
if (WMI)
return Empty;
WMI = W;
continue;
}
// Unknown instruction.
return Empty;
}
// Both instructions need to exist.
if (!WMI || !AI)
return Empty;
// Make sure that the WMI and AI match.
if (AI->getCallee() != WMI)
return Empty;
// We have exactly the pattern that we expected.
return std::make_pair(AI, WMI);
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst *> Calls) {
LLVM_DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< Demangle::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction *, ApplyInst *> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
ApplyInst *HoistAI =
getHoistedApplyForInitializer(AI, DT, InitF, ParentF, ParentFuncs);
// If we were unable to find anything, just go onto the next apply.
if (!HoistAI) {
continue;
}
// Otherwise, move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
LLVM_DEBUG(llvm::dbgs() << " don't hoist above availability check "
"at bb"
<< DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
LLVM_DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName()
<< "\n");
continue;
}
LLVM_DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI << " in "
<< HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
ApplyInst *SILGlobalOpt::getHoistedApplyForInitializer(
ApplyInst *AI, DominanceInfo *DT, SILFunction *InitF, SILFunction *ParentF,
llvm::DenseMap<SILFunction *, ApplyInst *> &ParentFuncs) {
auto PFI = ParentFuncs.find(ParentF);
if (PFI == ParentFuncs.end()) {
ParentFuncs[ParentF] = AI;
// It's the first time we found a call to InitF in this function, so we
// try to hoist it out of any loop.
return AI;
}
// Found a replacement for this init call. Ensure the replacement dominates
// the original call site.
ApplyInst *CommonAI = PFI->second;
assert(
cast<FunctionRefInst>(CommonAI->getCallee())->getReferencedFunction() ==
InitF &&
"ill-formed global init call");
SILBasicBlock *DomBB =
DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
// We must not move initializers around availability-checks.
if (isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT))
return nullptr;
ApplyInst *Result = nullptr;
if (DomBB != CommonAI->getParent()) {
CommonAI->moveBefore(&*DomBB->begin());
placeFuncRef(CommonAI, DT);
// Try to hoist the existing AI again if we move it to another block,
// e.g. from a loop exit into the loop.
Result = CommonAI;
}
AI->replaceAllUsesWith(CommonAI);
AI->eraseFromParent();
HasChanged = true;
return Result;
}
/// Optimize placement of initializer calls given a list of calls to the
/// same initializer. All original initialization points must be dominated by
/// the final initialization calls.
///
/// The current heuristic hoists all initialization points within a function to
/// a single dominating call in the outer loop preheader.
void SILGlobalOpt::placeInitializers(SILFunction *InitF,
ArrayRef<ApplyInst*> Calls) {
DEBUG(llvm::dbgs() << "GlobalOpt: calls to "
<< demangle_wrappers::demangleSymbolAsString(InitF->getName())
<< " : " << Calls.size() << "\n");
// Map each initializer-containing function to its final initializer call.
llvm::DenseMap<SILFunction*, ApplyInst*> ParentFuncs;
for (auto *AI : Calls) {
assert(AI->getNumArguments() == 0 && "ill-formed global init call");
assert(cast<FunctionRefInst>(AI->getCallee())->getReferencedFunction()
== InitF && "wrong init call");
SILFunction *ParentF = AI->getFunction();
DominanceInfo *DT = DA->get(ParentF);
auto PFI = ParentFuncs.find(ParentF);
ApplyInst *HoistAI = nullptr;
if (PFI != ParentFuncs.end()) {
// Found a replacement for this init call.
// Ensure the replacement dominates the original call site.
ApplyInst *CommonAI = PFI->second;
assert(cast<FunctionRefInst>(CommonAI->getCallee())
->getReferencedFunction() == InitF &&
"ill-formed global init call");
SILBasicBlock *DomBB =
DT->findNearestCommonDominator(AI->getParent(), CommonAI->getParent());
// We must not move initializers around availability-checks.
if (!isAvailabilityCheckOnDomPath(DomBB, CommonAI->getParent(), DT)) {
if (DomBB != CommonAI->getParent()) {
CommonAI->moveBefore(&*DomBB->begin());
placeFuncRef(CommonAI, DT);
// Try to hoist the existing AI again if we move it to another block,
// e.g. from a loop exit into the loop.
HoistAI = CommonAI;
}
AI->replaceAllUsesWith(CommonAI);
AI->eraseFromParent();
HasChanged = true;
}
} else {
ParentFuncs[ParentF] = AI;
// It's the first time we found a call to InitF in this function, so we
// try to hoist it out of any loop.
HoistAI = AI;
}
if (HoistAI) {
// Move this call to the outermost loop preheader.
SILBasicBlock *BB = HoistAI->getParent();
typedef llvm::DomTreeNodeBase<SILBasicBlock> DomTreeNode;
DomTreeNode *Node = DT->getNode(BB);
while (Node) {
SILBasicBlock *DomParentBB = Node->getBlock();
if (isAvailabilityCheck(DomParentBB)) {
DEBUG(llvm::dbgs() << " don't hoist above availability check at bb" <<
DomParentBB->getDebugID() << "\n");
break;
}
BB = DomParentBB;
if (!isInLoop(BB))
break;
Node = Node->getIDom();
}
if (BB == HoistAI->getParent()) {
// BB is either unreachable or not in a loop.
DEBUG(llvm::dbgs() << " skipping (not in a loop): " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
}
else {
DEBUG(llvm::dbgs() << " hoisting: " << *HoistAI
<< " in " << HoistAI->getFunction()->getName() << "\n");
HoistAI->moveBefore(&*BB->begin());
placeFuncRef(HoistAI, DT);
HasChanged = true;
}
}
}
}
/// Try to CSE the users of \p From to the users of \p To.
/// The original users of \p To are passed in ToApplyWitnessUsers.
/// Returns true on success.
static bool tryToCSEOpenExtCall(OpenExistentialAddrInst *From,
OpenExistentialAddrInst *To,
ApplyWitnessPair ToApplyWitnessUsers,
DominanceInfo *DA) {
assert(From != To && "Can't replace instruction with itself");
ApplyInst *FromAI = nullptr;
ApplyInst *ToAI = nullptr;
WitnessMethodInst *FromWMI = nullptr;
WitnessMethodInst *ToWMI = nullptr;
std::tie(FromAI, FromWMI) = getOpenExistentialUsers(From);
std::tie(ToAI, ToWMI) = ToApplyWitnessUsers;
// Make sure that the OEA instruction has exactly two expected users.
if (!FromAI || !ToAI || !FromWMI || !ToWMI)
return false;
// Make sure we are calling the same method.
if (FromWMI->getMember() != ToWMI->getMember())
return false;
// We are going to reuse the TO-WMI, so make sure it dominates the call site.
if (!DA->properlyDominates(ToWMI, FromWMI))
return false;
SILBuilder Builder(FromAI);
// Make archetypes used by the ToAI available to the builder.
SILOpenedArchetypesTracker OpenedArchetypesTracker(*FromAI->getFunction());
OpenedArchetypesTracker.registerUsedOpenedArchetypes(ToAI);
Builder.setOpenedArchetypesTracker(&OpenedArchetypesTracker);
assert(FromAI->getArguments().size() == ToAI->getArguments().size() &&
"Invalid number of arguments");
// Don't handle any apply instructions that involve substitutions.
if (ToAI->getSubstitutions().size() != 1) return false;
// Prepare the Apply args.
SmallVector<SILValue, 8> Args;
for (auto Op : FromAI->getArguments()) {
Args.push_back(Op == From ? To : Op);
}
auto FnTy = ToAI->getSubstCalleeSILType();
auto ResTy = FnTy.castTo<SILFunctionType>()->getSILResult();
ApplyInst *NAI = Builder.createApply(ToAI->getLoc(), ToWMI, FnTy, ResTy,
ToAI->getSubstitutions(), Args,
ToAI->isNonThrowing());
FromAI->replaceAllUsesWith(NAI);
FromAI->eraseFromParent();
NumOpenExtRemoved++;
return true;
}
/// Simplify the following two frontend patterns:
///
/// %payload_addr = init_enum_data_addr %payload_allocation
/// store %payload to %payload_addr
/// inject_enum_addr %payload_allocation, $EnumType.case
///
/// inject_enum_add %nopayload_allocation, $EnumType.case
///
/// for a concrete enum type $EnumType.case to:
///
/// %1 = enum $EnumType, $EnumType.case, %payload
/// store %1 to %payload_addr
///
/// %1 = enum $EnumType, $EnumType.case
/// store %1 to %nopayload_addr
///
/// We leave the cleaning up to mem2reg.
SILInstruction *
SILCombiner::visitInjectEnumAddrInst(InjectEnumAddrInst *IEAI) {
// Given an inject_enum_addr of a concrete type without payload, promote it to
// a store of an enum. Mem2reg/load forwarding will clean things up for us. We
// can't handle the payload case here due to the flow problems caused by the
// dependency in between the enum and its data.
assert(IEAI->getOperand()->getType().isAddress() && "Must be an address");
Builder.setCurrentDebugScope(IEAI->getDebugScope());
if (IEAI->getOperand()->getType().isAddressOnly(IEAI->getModule())) {
// Check for the following pattern inside the current basic block:
// inject_enum_addr %payload_allocation, $EnumType.case1
// ... no insns storing anything into %payload_allocation
// select_enum_addr %payload_allocation,
// case $EnumType.case1: %Result1,
// case case $EnumType.case2: %bResult2
// ...
//
// Replace the select_enum_addr by %Result1
auto *Term = IEAI->getParent()->getTerminator();
if (isa<CondBranchInst>(Term) || isa<SwitchValueInst>(Term)) {
auto BeforeTerm = std::prev(std::prev(IEAI->getParent()->end()));
auto *SEAI = dyn_cast<SelectEnumAddrInst>(BeforeTerm);
if (!SEAI)
return nullptr;
if (SEAI->getOperand() != IEAI->getOperand())
return nullptr;
SILBasicBlock::iterator II = IEAI->getIterator();
StoreInst *SI = nullptr;
for (;;) {
SILInstruction *CI = &*II;
if (CI == SEAI)
break;
++II;
SI = dyn_cast<StoreInst>(CI);
if (SI) {
if (SI->getDest() == IEAI->getOperand())
return nullptr;
}
// Allow all instructions in between, which don't have any dependency to
// the store.
if (AA->mayWriteToMemory(&*II, IEAI->getOperand()))
return nullptr;
}
auto *InjectedEnumElement = IEAI->getElement();
auto Result = SEAI->getCaseResult(InjectedEnumElement);
// Replace select_enum_addr by the result
replaceInstUsesWith(*SEAI, Result);
return nullptr;
}
// Check for the following pattern inside the current basic block:
// inject_enum_addr %payload_allocation, $EnumType.case1
// ... no insns storing anything into %payload_allocation
// switch_enum_addr %payload_allocation,
// case $EnumType.case1: %bbX,
// case case $EnumType.case2: %bbY
// ...
//
// Replace the switch_enum_addr by select_enum_addr, switch_value.
if (auto *SEI = dyn_cast<SwitchEnumAddrInst>(Term)) {
if (SEI->getOperand() != IEAI->getOperand())
return nullptr;
SILBasicBlock::iterator II = IEAI->getIterator();
StoreInst *SI = nullptr;
for (;;) {
SILInstruction *CI = &*II;
if (CI == SEI)
break;
++II;
SI = dyn_cast<StoreInst>(CI);
if (SI) {
if (SI->getDest() == IEAI->getOperand())
return nullptr;
}
// Allow all instructions in between, which don't have any dependency to
// the store.
if (AA->mayWriteToMemory(&*II, IEAI->getOperand()))
return nullptr;
}
// Replace switch_enum_addr by a branch instruction.
SILBuilderWithScope B(SEI);
SmallVector<std::pair<EnumElementDecl *, SILValue>, 8> CaseValues;
SmallVector<std::pair<SILValue, SILBasicBlock *>, 8> CaseBBs;
auto IntTy = SILType::getBuiltinIntegerType(32, B.getASTContext());
for (int i = 0, e = SEI->getNumCases(); i < e; ++i) {
auto Pair = SEI->getCase(i);
auto *IL = B.createIntegerLiteral(SEI->getLoc(), IntTy, APInt(32, i, false));
SILValue ILValue = SILValue(IL);
CaseValues.push_back(std::make_pair(Pair.first, ILValue));
CaseBBs.push_back(std::make_pair(ILValue, Pair.second));
}
SILValue DefaultValue;
SILBasicBlock *DefaultBB = nullptr;
if (SEI->hasDefault()) {
auto *IL = B.createIntegerLiteral(
SEI->getLoc(), IntTy,
APInt(32, static_cast<uint64_t>(SEI->getNumCases()), false));
DefaultValue = SILValue(IL);
DefaultBB = SEI->getDefaultBB();
}
auto *SEAI = B.createSelectEnumAddr(SEI->getLoc(), SEI->getOperand(), IntTy, DefaultValue, CaseValues);
B.createSwitchValue(SEI->getLoc(), SILValue(SEAI), DefaultBB, CaseBBs);
return eraseInstFromFunction(*SEI);
}
return nullptr;
}
// If the enum does not have a payload create the enum/store since we don't
// need to worry about payloads.
if (!IEAI->getElement()->hasArgumentType()) {
EnumInst *E =
Builder.createEnum(IEAI->getLoc(), SILValue(), IEAI->getElement(),
IEAI->getOperand()->getType().getObjectType());
Builder.createStore(IEAI->getLoc(), E, IEAI->getOperand(),
StoreOwnershipQualifier::Unqualified);
return eraseInstFromFunction(*IEAI);
}
// Ok, we have a payload enum, make sure that we have a store previous to
// us...
SILValue ASO = IEAI->getOperand();
if (!isa<AllocStackInst>(ASO)) {
return nullptr;
}
InitEnumDataAddrInst *DataAddrInst = nullptr;
InjectEnumAddrInst *EnumAddrIns = nullptr;
llvm::SmallPtrSet<SILInstruction *, 32> WriteSet;
for (auto UsersIt : ASO->getUses()) {
SILInstruction *CurrUser = UsersIt->getUser();
if (CurrUser->isDeallocatingStack()) {
// we don't care about the dealloc stack instructions
continue;
}
if (isDebugInst(CurrUser) || isa<LoadInst>(CurrUser)) {
// These Instructions are a non-risky use we can ignore
continue;
}
if (auto *CurrInst = dyn_cast<InitEnumDataAddrInst>(CurrUser)) {
if (DataAddrInst) {
return nullptr;
}
DataAddrInst = CurrInst;
continue;
}
if (auto *CurrInst = dyn_cast<InjectEnumAddrInst>(CurrUser)) {
if (EnumAddrIns) {
return nullptr;
}
EnumAddrIns = CurrInst;
continue;
}
if (isa<StoreInst>(CurrUser)) {
// The only MayWrite Instruction we can safely handle
WriteSet.insert(CurrUser);
continue;
}
// It is too risky to continue if it is any other instruction.
return nullptr;
}
if (!DataAddrInst || !EnumAddrIns) {
return nullptr;
}
assert((EnumAddrIns == IEAI) &&
"Found InitEnumDataAddrInst differs from IEAI");
// Found the DataAddrInst to this enum payload. Check if it has only use.
if (!hasOneNonDebugUse(DataAddrInst))
return nullptr;
StoreInst *SI = dyn_cast<StoreInst>(getSingleNonDebugUser(DataAddrInst));
ApplyInst *AI = dyn_cast<ApplyInst>(getSingleNonDebugUser(DataAddrInst));
if (!SI && !AI) {
return nullptr;
}
// Make sure the enum pattern instructions are the only ones which write to
// this location
if (!WriteSet.empty()) {
// Analyze the instructions (implicit dominator analysis)
// If we find any of MayWriteSet, return nullptr
SILBasicBlock *InitEnumBB = DataAddrInst->getParent();
assert(InitEnumBB && "DataAddrInst is not in a valid Basic Block");
llvm::SmallVector<SILInstruction *, 64> Worklist;
Worklist.push_back(IEAI);
llvm::SmallPtrSet<SILBasicBlock *, 16> Preds;
Preds.insert(IEAI->getParent());
while (!Worklist.empty()) {
SILInstruction *CurrIns = Worklist.pop_back_val();
SILBasicBlock *CurrBB = CurrIns->getParent();
if (CurrBB->isEntry() && CurrBB != InitEnumBB) {
// reached prologue without encountering the init bb
return nullptr;
}
for (auto InsIt = ++CurrIns->getIterator().getReverse();
InsIt != CurrBB->rend(); ++InsIt) {
SILInstruction *Ins = &*InsIt;
if (Ins == DataAddrInst) {
// don't care about what comes before init enum in the basic block
break;
}
if (WriteSet.count(Ins) != 0) {
return nullptr;
}
}
if (CurrBB == InitEnumBB) {
continue;
}
// Go to predecessors and do all that again
for (SILBasicBlock *Pred : CurrBB->getPredecessorBlocks()) {
// If it's already in the set, then we've already queued and/or
// processed the predecessors.
if (Preds.insert(Pred).second) {
Worklist.push_back(&*Pred->rbegin());
}
}
}
}
if (SI) {
assert((SI->getDest() == DataAddrInst) &&
"Can't find StoreInst with DataAddrInst as its destination");
// In that case, create the payload enum/store.
EnumInst *E = Builder.createEnum(
DataAddrInst->getLoc(), SI->getSrc(), DataAddrInst->getElement(),
DataAddrInst->getOperand()->getType().getObjectType());
Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(),
StoreOwnershipQualifier::Unqualified);
// Cleanup.
eraseInstFromFunction(*SI);
eraseInstFromFunction(*DataAddrInst);
return eraseInstFromFunction(*IEAI);
}
// Check whether we have an apply initializing the enum.
// %iedai = init_enum_data_addr %enum_addr
// = apply(%iedai,...)
// inject_enum_addr %enum_addr
//
// We can localize the store to an alloc_stack.
// Allowing us to perform the same optimization as for the store.
//
// %alloca = alloc_stack
// apply(%alloca,...)
// %load = load %alloca
// %1 = enum $EnumType, $EnumType.case, %load
// store %1 to %nopayload_addr
//
assert(AI && "Must have an apply");
unsigned ArgIdx = 0;
Operand *EnumInitOperand = nullptr;
for (auto &Opd : AI->getArgumentOperands()) {
// Found an apply that initializes the enum. We can optimize this by
// localizing the initialization to an alloc_stack and loading from it.
DataAddrInst = dyn_cast<InitEnumDataAddrInst>(Opd.get());
if (DataAddrInst && DataAddrInst->getOperand() == IEAI->getOperand() &&
ArgIdx < AI->getSubstCalleeType()->getNumIndirectResults()) {
EnumInitOperand = &Opd;
break;
}
++ArgIdx;
}
if (!EnumInitOperand) {
return nullptr;
}
// Localize the address access.
Builder.setInsertionPoint(AI);
auto *AllocStack = Builder.createAllocStack(DataAddrInst->getLoc(),
EnumInitOperand->get()->getType());
EnumInitOperand->set(AllocStack);
Builder.setInsertionPoint(std::next(SILBasicBlock::iterator(AI)));
SILValue Load(Builder.createLoad(DataAddrInst->getLoc(), AllocStack,
LoadOwnershipQualifier::Unqualified));
EnumInst *E = Builder.createEnum(
DataAddrInst->getLoc(), Load, DataAddrInst->getElement(),
DataAddrInst->getOperand()->getType().getObjectType());
Builder.createStore(DataAddrInst->getLoc(), E, DataAddrInst->getOperand(),
StoreOwnershipQualifier::Unqualified);
Builder.createDeallocStack(DataAddrInst->getLoc(), AllocStack);
eraseInstFromFunction(*DataAddrInst);
return eraseInstFromFunction(*IEAI);
}
