/// Collect all blocks from \p CurLoop which lie on all possible paths from
/// the header of \p CurLoop (inclusive) to BB (exclusive) into the set
/// \p Predecessors. If \p BB is the header, \p Predecessors will be empty.
static void collectTransitivePredecessors(
const Loop *CurLoop, const BasicBlock *BB,
SmallPtrSetImpl<const BasicBlock *> &Predecessors) {
assert(Predecessors.empty() && "Garbage in predecessors set?");
assert(CurLoop->contains(BB) && "Should only be called for loop blocks!");
if (BB == CurLoop->getHeader())
return;
SmallVector<const BasicBlock *, 4> WorkList;
for (auto *Pred : predecessors(BB)) {
Predecessors.insert(Pred);
WorkList.push_back(Pred);
}
while (!WorkList.empty()) {
auto *Pred = WorkList.pop_back_val();
assert(CurLoop->contains(Pred) && "Should only reach loop blocks!");
// We are not interested in backedges and we don't want to leave loop.
if (Pred == CurLoop->getHeader())
continue;
// TODO: If BB lies in an inner loop of CurLoop, this will traverse over all
// blocks of this inner loop, even those that are always executed AFTER the
// BB. It may make our analysis more conservative than it could be, see test
// @nested and @nested_no_throw in test/Analysis/MustExecute/loop-header.ll.
// We can ignore backedge of all loops containing BB to get a sligtly more
// optimistic result.
for (auto *PredPred : predecessors(Pred))
if (Predecessors.insert(PredPred).second)
WorkList.push_back(PredPred);
}
}
bool RecurrenceDescriptor::getSourceExtensionKind(
Instruction *Start, Instruction *Exit, Type *RT, bool &IsSigned,
SmallPtrSetImpl<Instruction *> &Visited,
SmallPtrSetImpl<Instruction *> &CI) {
SmallVector<Instruction *, 8> Worklist;
bool FoundOneOperand = false;
unsigned DstSize = RT->getPrimitiveSizeInBits();
Worklist.push_back(Exit);
// Traverse the instructions in the reduction expression, beginning with the
// exit value.
while (!Worklist.empty()) {
Instruction *I = Worklist.pop_back_val();
for (Use &U : I->operands()) {
// Terminate the traversal if the operand is not an instruction, or we
// reach the starting value.
Instruction *J = dyn_cast<Instruction>(U.get());
if (!J || J == Start)
continue;
// Otherwise, investigate the operation if it is also in the expression.
if (Visited.count(J)) {
Worklist.push_back(J);
continue;
}
// If the operand is not in Visited, it is not a reduction operation, but
// it does feed into one. Make sure it is either a single-use sign- or
// zero-extend instruction.
CastInst *Cast = dyn_cast<CastInst>(J);
bool IsSExtInst = isa<SExtInst>(J);
if (!Cast || !Cast->hasOneUse() || !(isa<ZExtInst>(J) || IsSExtInst))
return false;
// Ensure the source type of the extend is no larger than the reduction
// type. It is not necessary for the types to be identical.
unsigned SrcSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
if (SrcSize > DstSize)
return false;
// Furthermore, ensure that all such extends are of the same kind.
if (FoundOneOperand) {
if (IsSigned != IsSExtInst)
return false;
} else {
FoundOneOperand = true;
IsSigned = IsSExtInst;
}
// Lastly, if the source type of the extend matches the reduction type,
// add the extend to CI so that we can avoid accounting for it in the
// cost model.
if (SrcSize == DstSize)
CI.insert(Cast);
}
}
return true;
}
/// CopyFrom - implement operator= from a smallptrset that has the same pointer
/// type, but may have a different small size.
void SmallPtrSetImpl::CopyFrom(const SmallPtrSetImpl &RHS) {
if (isSmall() && RHS.isSmall())
assert(CurArraySize == RHS.CurArraySize &&
"Cannot assign sets with different small sizes");
// If we're becoming small, prepare to insert into our stack space
if (RHS.isSmall()) {
if (!isSmall())
free(CurArray);
CurArray = SmallArray;
// Otherwise, allocate new heap space (unless we were the same size)
} else if (CurArraySize != RHS.CurArraySize) {
if (isSmall())
CurArray = (const void**)malloc(sizeof(void*) * RHS.CurArraySize);
else
CurArray = (const void**)realloc(CurArray, sizeof(void*)*RHS.CurArraySize);
assert(CurArray && "Failed to allocate memory?");
}
// Copy over the new array size
CurArraySize = RHS.CurArraySize;
// Copy over the contents from the other set
memcpy(CurArray, RHS.CurArray, sizeof(void*)*CurArraySize);
NumElements = RHS.NumElements;
NumTombstones = RHS.NumTombstones;
}
void LTOCodeGenerator::
applyRestriction(GlobalValue &GV,
ArrayRef<StringRef> Libcalls,
std::vector<const char*> &MustPreserveList,
SmallPtrSetImpl<GlobalValue*> &AsmUsed,
Mangler &Mangler) {
// There are no restrictions to apply to declarations.
if (GV.isDeclaration())
return;
// There is nothing more restrictive than private linkage.
if (GV.hasPrivateLinkage())
return;
SmallString<64> Buffer;
TargetMach->getNameWithPrefix(Buffer, &GV, Mangler);
if (MustPreserveSymbols.count(Buffer))
MustPreserveList.push_back(GV.getName().data());
if (AsmUndefinedRefs.count(Buffer))
AsmUsed.insert(&GV);
// Conservatively append user-supplied runtime library functions to
// llvm.compiler.used. These could be internalized and deleted by
// optimizations like -globalopt, causing problems when later optimizations
// add new library calls (e.g., llvm.memset => memset and printf => puts).
// Leave it to the linker to remove any dead code (e.g. with -dead_strip).
if (isa<Function>(GV) &&
std::binary_search(Libcalls.begin(), Libcalls.end(), GV.getName()))
AsmUsed.insert(&GV);
}
/// MarkBlocksLiveIn - Insert BB and all of its predecessors into LiveBBs until
/// we reach blocks we've already seen.
static void MarkBlocksLiveIn(BasicBlock *BB,
SmallPtrSetImpl<BasicBlock *> &LiveBBs) {
if (!LiveBBs.insert(BB).second)
return; // already been here.
df_iterator_default_set<BasicBlock*> Visited;
for (BasicBlock *B : inverse_depth_first_ext(BB, Visited))
LiveBBs.insert(B);
}
/// Find values that are marked as llvm.used.
static void findUsedValues(GlobalVariable *LLVMUsed,
SmallPtrSetImpl<const GlobalValue*> &UsedValues) {
if (!LLVMUsed) return;
UsedValues.insert(LLVMUsed);
ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer());
for (unsigned i = 0, e = Inits->getNumOperands(); i != e; ++i)
if (GlobalValue *GV =
dyn_cast<GlobalValue>(Inits->getOperand(i)->stripPointerCasts()))
UsedValues.insert(GV);
}
// Breadth-first walk of the use-def graph; determine the set of nodes
// we care about and eagerly determine if some of them are poisonous.
void Float2IntPass::walkBackwards(const SmallPtrSetImpl<Instruction*> &Roots) {
std::deque<Instruction*> Worklist(Roots.begin(), Roots.end());
while (!Worklist.empty()) {
Instruction *I = Worklist.back();
Worklist.pop_back();
if (SeenInsts.find(I) != SeenInsts.end())
// Seen already.
continue;
switch (I->getOpcode()) {
// FIXME: Handle select and phi nodes.
default:
// Path terminated uncleanly.
seen(I, badRange());
break;
case Instruction::UIToFP:
case Instruction::SIToFP: {
// Path terminated cleanly - use the type of the integer input to seed
// the analysis.
unsigned BW = I->getOperand(0)->getType()->getPrimitiveSizeInBits();
auto Input = ConstantRange(BW, true);
auto CastOp = (Instruction::CastOps)I->getOpcode();
seen(I, validateRange(Input.castOp(CastOp, MaxIntegerBW+1)));
continue;
}
case Instruction::FAdd:
case Instruction::FSub:
case Instruction::FMul:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::FCmp:
seen(I, unknownRange());
break;
}
for (Value *O : I->operands()) {
if (Instruction *OI = dyn_cast<Instruction>(O)) {
// Unify def-use chains if they interfere.
ECs.unionSets(I, OI);
if (SeenInsts.find(I)->second != badRange())
Worklist.push_back(OI);
} else if (!isa<ConstantFP>(O)) {
// Not an instruction or ConstantFP? we can't do anything.
seen(I, badRange());
}
}
}
}
/// Merge an autorelease with a retain into a fused call.
bool ObjCARCContract::contractAutorelease(
Function &F, Instruction *Autorelease, ARCInstKind Class,
SmallPtrSetImpl<Instruction *> &DependingInstructions,
SmallPtrSetImpl<const BasicBlock *> &Visited) {
const Value *Arg = GetArgRCIdentityRoot(Autorelease);
// Check that there are no instructions between the retain and the autorelease
// (such as an autorelease_pop) which may change the count.
CallInst *Retain = nullptr;
if (Class == ARCInstKind::AutoreleaseRV)
FindDependencies(RetainAutoreleaseRVDep, Arg,
Autorelease->getParent(), Autorelease,
DependingInstructions, Visited, PA);
else
FindDependencies(RetainAutoreleaseDep, Arg,
Autorelease->getParent(), Autorelease,
DependingInstructions, Visited, PA);
Visited.clear();
if (DependingInstructions.size() != 1) {
DependingInstructions.clear();
return false;
}
Retain = dyn_cast_or_null<CallInst>(*DependingInstructions.begin());
DependingInstructions.clear();
if (!Retain || GetBasicARCInstKind(Retain) != ARCInstKind::Retain ||
GetArgRCIdentityRoot(Retain) != Arg)
return false;
Changed = true;
++NumPeeps;
LLVM_DEBUG(dbgs() << " Fusing retain/autorelease!\n"
" Autorelease:"
<< *Autorelease
<< "\n"
" Retain: "
<< *Retain << "\n");
Function *Decl = EP.get(Class == ARCInstKind::AutoreleaseRV
? ARCRuntimeEntryPointKind::RetainAutoreleaseRV
: ARCRuntimeEntryPointKind::RetainAutorelease);
Retain->setCalledFunction(Decl);
LLVM_DEBUG(dbgs() << " New RetainAutorelease: " << *Retain << "\n");
EraseInstruction(Autorelease);
return true;
}
void SmallPtrSetImpl::swap(SmallPtrSetImpl &RHS) {
if (this == &RHS) return;
// We can only avoid copying elements if neither set is small.
if (!this->isSmall() && !RHS.isSmall()) {
std::swap(this->CurArray, RHS.CurArray);
std::swap(this->CurArraySize, RHS.CurArraySize);
std::swap(this->NumElements, RHS.NumElements);
std::swap(this->NumTombstones, RHS.NumTombstones);
return;
}
// FIXME: From here on we assume that both sets have the same small size.
// If only RHS is small, copy the small elements into LHS and move the pointer
// from LHS to RHS.
if (!this->isSmall() && RHS.isSmall()) {
std::copy(RHS.SmallArray, RHS.SmallArray+RHS.CurArraySize,
this->SmallArray);
std::swap(this->NumElements, RHS.NumElements);
std::swap(this->CurArraySize, RHS.CurArraySize);
RHS.CurArray = this->CurArray;
RHS.NumTombstones = this->NumTombstones;
this->CurArray = this->SmallArray;
this->NumTombstones = 0;
return;
}
// If only LHS is small, copy the small elements into RHS and move the pointer
// from RHS to LHS.
if (this->isSmall() && !RHS.isSmall()) {
std::copy(this->SmallArray, this->SmallArray+this->CurArraySize,
RHS.SmallArray);
std::swap(RHS.NumElements, this->NumElements);
std::swap(RHS.CurArraySize, this->CurArraySize);
this->CurArray = RHS.CurArray;
this->NumTombstones = RHS.NumTombstones;
RHS.CurArray = RHS.SmallArray;
RHS.NumTombstones = 0;
return;
}
// Both a small, just swap the small elements.
assert(this->isSmall() && RHS.isSmall());
assert(this->CurArraySize == RHS.CurArraySize);
std::swap_ranges(this->SmallArray, this->SmallArray+this->CurArraySize,
RHS.SmallArray);
std::swap(this->NumElements, RHS.NumElements);
}
static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA,
SmallPtrSetImpl<Instruction *> &Stores) {
if (Inst->mayWriteToMemory()) {
Stores.insert(Inst);
return false;
}
if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
MemoryLocation Loc = MemoryLocation::get(L);
for (Instruction *S : Stores)
if (AA->getModRefInfo(S, Loc) & MRI_Mod)
return false;
}
if (isa<TerminatorInst>(Inst) || isa<PHINode>(Inst))
return false;
// Convergent operations cannot be made control-dependent on additional
// values.
if (auto CS = CallSite(Inst)) {
if (CS.hasFnAttr(Attribute::Convergent))
return false;
}
return true;
}
bool RecurrenceDescriptor::areAllUsesIn(Instruction *I,
SmallPtrSetImpl<Instruction *> &Set) {
for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use)
if (!Set.count(dyn_cast<Instruction>(*Use)))
return false;
return true;
}
static void collectMDInDomain(const MDNode *List, const MDNode *Domain,
SmallPtrSetImpl<const MDNode *> &Nodes) {
for (const MDOperand &MDOp : List->operands())
if (const MDNode *MD = dyn_cast<MDNode>(MDOp))
if (AliasScopeNode(MD).getDomain() == Domain)
Nodes.insert(MD);
}
void AMDGPUAlwaysInline::recursivelyVisitUsers(
GlobalValue &GV,
SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) {
SmallVector<User *, 16> Stack;
SmallPtrSet<const Value *, 8> Visited;
for (User *U : GV.users())
Stack.push_back(U);
while (!Stack.empty()) {
User *U = Stack.pop_back_val();
if (!Visited.insert(U).second)
continue;
if (Instruction *I = dyn_cast<Instruction>(U)) {
Function *F = I->getParent()->getParent();
if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) {
FuncsToAlwaysInline.insert(F);
Stack.push_back(F);
}
// No need to look at further users, but we do need to inline any callers.
continue;
}
for (User *UU : U->users())
Stack.push_back(UU);
}
}
static bool isSafeToMove(Instruction *Inst, AliasAnalysis &AA,
SmallPtrSetImpl<Instruction *> &Stores) {
if (Inst->mayWriteToMemory()) {
Stores.insert(Inst);
return false;
}
if (LoadInst *L = dyn_cast<LoadInst>(Inst)) {
MemoryLocation Loc = MemoryLocation::get(L);
for (Instruction *S : Stores)
if (isModSet(AA.getModRefInfo(S, Loc)))
return false;
}
if (Inst->isTerminator() || isa<PHINode>(Inst) || Inst->isEHPad() ||
Inst->mayThrow())
return false;
if (auto *Call = dyn_cast<CallBase>(Inst)) {
// Convergent operations cannot be made control-dependent on additional
// values.
if (Call->hasFnAttr(Attribute::Convergent))
return false;
for (Instruction *S : Stores)
if (isModSet(AA.getModRefInfo(S, Call)))
return false;
}
return true;
}
void MemorySSAUpdater::removeBlocks(
const SmallPtrSetImpl<BasicBlock *> &DeadBlocks) {
// First delete all uses of BB in MemoryPhis.
for (BasicBlock *BB : DeadBlocks) {
TerminatorInst *TI = BB->getTerminator();
assert(TI && "Basic block expected to have a terminator instruction");
for (BasicBlock *Succ : successors(TI))
if (!DeadBlocks.count(Succ))
if (MemoryPhi *MP = MSSA->getMemoryAccess(Succ)) {
MP->unorderedDeleteIncomingBlock(BB);
if (MP->getNumIncomingValues() == 1)
removeMemoryAccess(MP);
}
// Drop all references of all accesses in BB
if (MemorySSA::AccessList *Acc = MSSA->getWritableBlockAccesses(BB))
for (MemoryAccess &MA : *Acc)
MA.dropAllReferences();
}
// Next, delete all memory accesses in each block
for (BasicBlock *BB : DeadBlocks) {
MemorySSA::AccessList *Acc = MSSA->getWritableBlockAccesses(BB);
if (!Acc)
continue;
for (auto AB = Acc->begin(), AE = Acc->end(); AB != AE;) {
MemoryAccess *MA = &*AB;
++AB;
MSSA->removeFromLookups(MA);
MSSA->removeFromLists(MA);
}
}
}
SmallPtrSetImpl::SmallPtrSetImpl(const SmallPtrSetImpl& that) {
NumElements = that.NumElements;
NumTombstones = 0;
if (that.isSmall()) {
CurArraySize = that.CurArraySize;
CurArray = &SmallArray[0];
memcpy(CurArray, that.CurArray, sizeof(void*)*CurArraySize);
} else {
CurArraySize = that.NumElements < 64 ? 128 : that.NumElements*2;
CurArray = new void*[CurArraySize+1];
memset(CurArray, -1, CurArraySize*sizeof(void*));
// The end pointer, always valid, is set to a valid element to help the
// iterator.
CurArray[CurArraySize] = 0;
// Copy over all valid entries.
for (void **BucketPtr = that.CurArray, **E = that.CurArray+CurArraySize;
BucketPtr != E; ++BucketPtr) {
// Copy over the element if it is valid.
void *Elt = *BucketPtr;
if (Elt != getTombstoneMarker() && Elt != getEmptyMarker())
*const_cast<void**>(FindBucketFor(Elt)) = Elt;
}
}
}
/// Collect cast instructions that can be ignored in the vectorizer's cost
/// model, given a reduction exit value and the minimal type in which the
/// reduction can be represented.
static void collectCastsToIgnore(Loop *TheLoop, Instruction *Exit,
Type *RecurrenceType,
SmallPtrSetImpl<Instruction *> &Casts) {
SmallVector<Instruction *, 8> Worklist;
SmallPtrSet<Instruction *, 8> Visited;
Worklist.push_back(Exit);
while (!Worklist.empty()) {
Instruction *Val = Worklist.pop_back_val();
Visited.insert(Val);
if (auto *Cast = dyn_cast<CastInst>(Val))
if (Cast->getSrcTy() == RecurrenceType) {
// If the source type of a cast instruction is equal to the recurrence
// type, it will be eliminated, and should be ignored in the vectorizer
// cost model.
Casts.insert(Cast);
continue;
}
// Add all operands to the work list if they are loop-varying values that
// we haven't yet visited.
for (Value *O : cast<User>(Val)->operands())
if (auto *I = dyn_cast<Instruction>(O))
if (TheLoop->contains(I) && !Visited.count(I))
Worklist.push_back(I);
}
}
void ScopedNoAliasAAResult::collectMDInDomain(
const MDNode *List, const MDNode *Domain,
SmallPtrSetImpl<const MDNode *> &Nodes) const {
for (unsigned i = 0, ie = List->getNumOperands(); i != ie; ++i)
if (const MDNode *MD = dyn_cast<MDNode>(List->getOperand(i)))
if (AliasScopeNode(MD).getDomain() == Domain)
Nodes.insert(MD);
}
/// getMachineBasicBlocks - Populate given set using machine basic blocks which
/// have machine instructions that belong to lexical scope identified by
/// DebugLoc.
void LexicalScopes::getMachineBasicBlocks(
const DILocation *DL, SmallPtrSetImpl<const MachineBasicBlock *> &MBBs) {
MBBs.clear();
LexicalScope *Scope = getOrCreateLexicalScope(DL);
if (!Scope)
return;
if (Scope == CurrentFnLexicalScope) {
for (const auto &MBB : *MF)
MBBs.insert(&MBB);
return;
}
SmallVectorImpl<InsnRange> &InsnRanges = Scope->getRanges();
for (auto &R : InsnRanges)
MBBs.insert(R.first->getParent());
}
/// MarkBlocksLiveIn - Insert BB and all of its predescessors into LiveBBs until
/// we reach blocks we've already seen.
static void MarkBlocksLiveIn(BasicBlock *BB,
SmallPtrSetImpl<BasicBlock *> &LiveBBs) {
if (!LiveBBs.insert(BB).second)
return; // already been here.
for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
MarkBlocksLiveIn(*PI, LiveBBs);
}
Action walkToTypePre(Type ty) override {
if (ty->is<DependentMemberType>())
return Action::SkipChildren;
if (auto typeVar = ty->getAs<TypeVariableType>())
typeVars.insert(typeVar);
return Action::Continue;
}
/// \brief Analyze a basic block for its contribution to the inline cost.
///
/// This method walks the analyzer over every instruction in the given basic
/// block and accounts for their cost during inlining at this callsite. It
/// aborts early if the threshold has been exceeded or an impossible to inline
/// construct has been detected. It returns false if inlining is no longer
/// viable, and true if inlining remains viable.
bool CallAnalyzer::analyzeBlock(BasicBlock *BB,
SmallPtrSetImpl<const Value *> &EphValues) {
for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) {
// FIXME: Currently, the number of instructions in a function regardless of
// our ability to simplify them during inline to constants or dead code,
// are actually used by the vector bonus heuristic. As long as that's true,
// we have to special case debug intrinsics here to prevent differences in
// inlining due to debug symbols. Eventually, the number of unsimplified
// instructions shouldn't factor into the cost computation, but until then,
// hack around it here.
if (isa<DbgInfoIntrinsic>(I))
continue;
// Skip ephemeral values.
if (EphValues.count(I))
continue;
++NumInstructions;
if (isa<ExtractElementInst>(I) || I->getType()->isVectorTy())
++NumVectorInstructions;
// If the instruction simplified to a constant, there is no cost to this
// instruction. Visit the instructions using our InstVisitor to account for
// all of the per-instruction logic. The visit tree returns true if we
// consumed the instruction in any way, and false if the instruction's base
// cost should count against inlining.
if (Base::visit(I))
++NumInstructionsSimplified;
else
Cost += InlineConstants::InstrCost;
// If the visit this instruction detected an uninlinable pattern, abort.
if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca ||
HasIndirectBr)
return false;
// If the caller is a recursive function then we don't want to inline
// functions which allocate a lot of stack space because it would increase
// the caller stack usage dramatically.
if (IsCallerRecursive &&
AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller)
return false;
if (NumVectorInstructions > NumInstructions/2)
VectorBonus = FiftyPercentVectorBonus;
else if (NumVectorInstructions > NumInstructions/10)
VectorBonus = TenPercentVectorBonus;
else
VectorBonus = 0;
// Check if we've past the threshold so we don't spin in huge basic
// blocks that will never inline.
if (Cost > (Threshold + VectorBonus))
return false;
}
return true;
}
void checkEraseAndIterators(SmallPtrSetImpl<int*> &S) {
int buf[3];
S.insert(&buf[0]);
S.insert(&buf[1]);
S.insert(&buf[2]);
// Iterators must still be valid after erase() calls;
auto B = S.begin();
auto M = std::next(B);
auto E = S.end();
EXPECT_TRUE(*B == &buf[0] || *B == &buf[1] || *B == &buf[2]);
EXPECT_TRUE(*M == &buf[0] || *M == &buf[1] || *M == &buf[2]);
EXPECT_TRUE(*B != *M);
int *Removable = *std::next(M);
// No iterator points to Removable now.
EXPECT_TRUE(Removable == &buf[0] || Removable == &buf[1] ||
Removable == &buf[2]);
EXPECT_TRUE(Removable != *B && Removable != *M);
S.erase(Removable);
// B,M,E iterators should still be valid
EXPECT_EQ(B, S.begin());
EXPECT_EQ(M, std::next(B));
EXPECT_EQ(E, S.end());
EXPECT_EQ(std::next(M), E);
}
/// Return adjusted total frequency of \p BBs.
///
/// * If there is only one BB, sinking instruction will not introduce code
/// size increase. Thus there is no need to adjust the frequency.
/// * If there are more than one BB, sinking would lead to code size increase.
/// In this case, we add some "tax" to the total frequency to make it harder
/// to sink. E.g.
/// Freq(Preheader) = 100
/// Freq(BBs) = sum(50, 49) = 99
/// Even if Freq(BBs) < Freq(Preheader), we will not sink from Preheade to
/// BBs as the difference is too small to justify the code size increase.
/// To model this, The adjusted Freq(BBs) will be:
/// AdjustedFreq(BBs) = 99 / SinkFrequencyPercentThreshold%
static BlockFrequency adjustedSumFreq(SmallPtrSetImpl<BasicBlock *> &BBs,
BlockFrequencyInfo &BFI) {
BlockFrequency T = 0;
for (BasicBlock *B : BBs)
T += BFI.getBlockFreq(B);
if (BBs.size() > 1)
T /= BranchProbability(SinkFrequencyPercentThreshold, 100);
return T;
}
/// Walk up the CFG from StartPos (which is in StartBB) and find local and
/// non-local dependencies on Arg.
///
/// TODO: Cache results?
void
llvm::objcarc::FindDependencies(DependenceKind Flavor,
const Value *Arg,
BasicBlock *StartBB, Instruction *StartInst,
SmallPtrSetImpl<Instruction *> &DependingInsts,
SmallPtrSetImpl<const BasicBlock *> &Visited,
ProvenanceAnalysis &PA) {
BasicBlock::iterator StartPos = StartInst;
SmallVector<std::pair<BasicBlock *, BasicBlock::iterator>, 4> Worklist;
Worklist.push_back(std::make_pair(StartBB, StartPos));
do {
std::pair<BasicBlock *, BasicBlock::iterator> Pair =
Worklist.pop_back_val();
BasicBlock *LocalStartBB = Pair.first;
BasicBlock::iterator LocalStartPos = Pair.second;
BasicBlock::iterator StartBBBegin = LocalStartBB->begin();
for (;;) {
if (LocalStartPos == StartBBBegin) {
pred_iterator PI(LocalStartBB), PE(LocalStartBB, false);
if (PI == PE)
// If we've reached the function entry, produce a null dependence.
DependingInsts.insert(nullptr);
else
// Add the predecessors to the worklist.
do {
BasicBlock *PredBB = *PI;
if (Visited.insert(PredBB))
Worklist.push_back(std::make_pair(PredBB, PredBB->end()));
} while (++PI != PE);
break;
}
Instruction *Inst = --LocalStartPos;
if (Depends(Flavor, Inst, Arg, PA)) {
DependingInsts.insert(Inst);
break;
}
}
} while (!Worklist.empty());
// Determine whether the original StartBB post-dominates all of the blocks we
// visited. If not, insert a sentinal indicating that most optimizations are
// not safe.
for (SmallPtrSet<const BasicBlock *, 4>::const_iterator I = Visited.begin(),
E = Visited.end(); I != E; ++I) {
const BasicBlock *BB = *I;
if (BB == StartBB)
continue;
const TerminatorInst *TI = cast<TerminatorInst>(&BB->back());
for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI) {
const BasicBlock *Succ = *SI;
if (Succ != StartBB && !Visited.count(Succ)) {
DependingInsts.insert(reinterpret_cast<Instruction *>(-1));
return;
}
}
}
}
static inline bool
isSimpleEnoughValueToCommit(Constant *C,
SmallPtrSetImpl<Constant *> &SimpleConstants,
const DataLayout &DL) {
// If we already checked this constant, we win.
if (!SimpleConstants.insert(C).second)
return true;
// Check the constant.
return isSimpleEnoughValueToCommitHelper(C, SimpleConstants, DL);
}
/// Return a set of basic blocks to insert sinked instructions.
///
/// The returned set of basic blocks (BBsToSinkInto) should satisfy:
///
/// * Inside the loop \p L
/// * For each UseBB in \p UseBBs, there is at least one BB in BBsToSinkInto
/// that domintates the UseBB
/// * Has minimum total frequency that is no greater than preheader frequency
///
/// The purpose of the function is to find the optimal sinking points to
/// minimize execution cost, which is defined as "sum of frequency of
/// BBsToSinkInto".
/// As a result, the returned BBsToSinkInto needs to have minimum total
/// frequency.
/// Additionally, if the total frequency of BBsToSinkInto exceeds preheader
/// frequency, the optimal solution is not sinking (return empty set).
///
/// \p ColdLoopBBs is used to help find the optimal sinking locations.
/// It stores a list of BBs that is:
///
/// * Inside the loop \p L
/// * Has a frequency no larger than the loop's preheader
/// * Sorted by BB frequency
///
/// The complexity of the function is O(UseBBs.size() * ColdLoopBBs.size()).
/// To avoid expensive computation, we cap the maximum UseBBs.size() in its
/// caller.
static SmallPtrSet<BasicBlock *, 2>
findBBsToSinkInto(const Loop &L, const SmallPtrSetImpl<BasicBlock *> &UseBBs,
const SmallVectorImpl<BasicBlock *> &ColdLoopBBs,
DominatorTree &DT, BlockFrequencyInfo &BFI) {
SmallPtrSet<BasicBlock *, 2> BBsToSinkInto;
if (UseBBs.size() == 0)
return BBsToSinkInto;
BBsToSinkInto.insert(UseBBs.begin(), UseBBs.end());
SmallPtrSet<BasicBlock *, 2> BBsDominatedByColdestBB;
// For every iteration:
// * Pick the ColdestBB from ColdLoopBBs
// * Find the set BBsDominatedByColdestBB that satisfy:
// - BBsDominatedByColdestBB is a subset of BBsToSinkInto
// - Every BB in BBsDominatedByColdestBB is dominated by ColdestBB
// * If Freq(ColdestBB) < Freq(BBsDominatedByColdestBB), remove
// BBsDominatedByColdestBB from BBsToSinkInto, add ColdestBB to
// BBsToSinkInto
for (BasicBlock *ColdestBB : ColdLoopBBs) {
BBsDominatedByColdestBB.clear();
for (BasicBlock *SinkedBB : BBsToSinkInto)
if (DT.dominates(ColdestBB, SinkedBB))
BBsDominatedByColdestBB.insert(SinkedBB);
if (BBsDominatedByColdestBB.size() == 0)
continue;
if (adjustedSumFreq(BBsDominatedByColdestBB, BFI) >
BFI.getBlockFreq(ColdestBB)) {
for (BasicBlock *DominatedBB : BBsDominatedByColdestBB) {
BBsToSinkInto.erase(DominatedBB);
}
BBsToSinkInto.insert(ColdestBB);
}
}
// If the total frequency of BBsToSinkInto is larger than preheader frequency,
// do not sink.
if (adjustedSumFreq(BBsToSinkInto, BFI) >
BFI.getBlockFreq(L.getLoopPreheader()))
BBsToSinkInto.clear();
return BBsToSinkInto;
}
/// Visit each register class belonging to the given register bank.
///
/// A class belongs to the bank iff any of these apply:
/// * It is explicitly specified
/// * It is a subclass of a class that is a member.
/// * It is a class containing subregisters of the registers of a class that
/// is a member. This is known as a subreg-class.
///
/// This function must be called for each explicitly specified register class.
///
/// \param RC The register class to search.
/// \param Kind A debug string containing the path the visitor took to reach RC.
/// \param VisitFn The action to take for each class visited. It may be called
/// multiple times for a given class if there are multiple paths
/// to the class.
static void visitRegisterBankClasses(
CodeGenRegBank &RegisterClassHierarchy, const CodeGenRegisterClass *RC,
const Twine Kind,
std::function<void(const CodeGenRegisterClass *, StringRef)> VisitFn,
SmallPtrSetImpl<const CodeGenRegisterClass *> &VisitedRCs) {
// Make sure we only visit each class once to avoid infinite loops.
if (VisitedRCs.count(RC))
return;
VisitedRCs.insert(RC);
// Visit each explicitly named class.
VisitFn(RC, Kind.str());
for (const auto &PossibleSubclass : RegisterClassHierarchy.getRegClasses()) {
std::string TmpKind =
(Twine(Kind) + " (" + PossibleSubclass.getName() + ")").str();
// Visit each subclass of an explicitly named class.
if (RC != &PossibleSubclass && RC->hasSubClass(&PossibleSubclass))
visitRegisterBankClasses(RegisterClassHierarchy, &PossibleSubclass,
TmpKind + " " + RC->getName() + " subclass",
VisitFn, VisitedRCs);
// Visit each class that contains only subregisters of RC with a common
// subregister-index.
//
// More precisely, PossibleSubclass is a subreg-class iff Reg:SubIdx is in
// PossibleSubclass for all registers Reg from RC using any
// subregister-index SubReg
for (const auto &SubIdx : RegisterClassHierarchy.getSubRegIndices()) {
BitVector BV(RegisterClassHierarchy.getRegClasses().size());
PossibleSubclass.getSuperRegClasses(&SubIdx, BV);
if (BV.test(RC->EnumValue)) {
std::string TmpKind2 = (Twine(TmpKind) + " " + RC->getName() +
" class-with-subregs: " + RC->getName())
.str();
VisitFn(&PossibleSubclass, TmpKind2);
}
}
}
}
bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc,
SmallPtrSetImpl<Instruction *> &Visited) {
auto *Inst = dyn_cast<Instruction>(V);
if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst))
return true;
if (!isSafeToSpeculativelyExecute(Inst, Loc, &DT) ||
Inst->mayReadFromMemory())
return false;
Visited.insert(Inst);
// We only want to go _up_ the dominance chain when recursing.
assert(!isa<PHINode>(Loc) &&
"PHIs should return false for isSafeToSpeculativelyExecute");
assert(DT.isReachableFromEntry(Inst->getParent()) &&
"We did a DFS from the block entry!");
return all_of(Inst->operands(),
[&](Value *Op) { return isAvailableAt(Op, Loc, Visited); });
}
void VPlanHCFGTransforms::VPInstructionsToVPRecipes(
VPlanPtr &Plan,
LoopVectorizationLegality::InductionList *Inductions,
SmallPtrSetImpl<Instruction *> &DeadInstructions) {
VPRegionBlock *TopRegion = dyn_cast<VPRegionBlock>(Plan->getEntry());
ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
for (VPBlockBase *Base : RPOT) {
// Do not widen instructions in pre-header and exit blocks.
if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
continue;
VPBasicBlock *VPBB = Base->getEntryBasicBlock();
VPRecipeBase *LastRecipe = nullptr;
// Introduce each ingredient into VPlan.
for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
VPRecipeBase *Ingredient = &*I++;
// Can only handle VPInstructions.
VPInstruction *VPInst = cast<VPInstruction>(Ingredient);
Instruction *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
if (DeadInstructions.count(Inst)) {
Ingredient->eraseFromParent();
continue;
}
VPRecipeBase *NewRecipe = nullptr;
// Create VPWidenMemoryInstructionRecipe for loads and stores.
if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst))
NewRecipe = new VPWidenMemoryInstructionRecipe(*Inst, nullptr /*Mask*/);
else if (PHINode *Phi = dyn_cast<PHINode>(Inst)) {
InductionDescriptor II = Inductions->lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
II.getKind() == InductionDescriptor::IK_FpInduction) {
NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi);
} else
NewRecipe = new VPWidenPHIRecipe(Phi);
} else {
// If the last recipe is a VPWidenRecipe, add Inst to it instead of
// creating a new recipe.
if (VPWidenRecipe *WidenRecipe =
dyn_cast_or_null<VPWidenRecipe>(LastRecipe)) {
WidenRecipe->appendInstruction(Inst);
Ingredient->eraseFromParent();
continue;
}
NewRecipe = new VPWidenRecipe(Inst);
}
NewRecipe->insertBefore(Ingredient);
LastRecipe = NewRecipe;
Ingredient->eraseFromParent();
}
}
}
