/// Collect all blocks from \p CurLoop which lie on all possible paths from /// the header of \p CurLoop (inclusive) to BB (exclusive) into the set /// \p Predecessors. If \p BB is the header, \p Predecessors will be empty. static void collectTransitivePredecessors( const Loop *CurLoop, const BasicBlock *BB, SmallPtrSetImpl<const BasicBlock *> &Predecessors) { assert(Predecessors.empty() && "Garbage in predecessors set?"); assert(CurLoop->contains(BB) && "Should only be called for loop blocks!"); if (BB == CurLoop->getHeader()) return; SmallVector<const BasicBlock *, 4> WorkList; for (auto *Pred : predecessors(BB)) { Predecessors.insert(Pred); WorkList.push_back(Pred); } while (!WorkList.empty()) { auto *Pred = WorkList.pop_back_val(); assert(CurLoop->contains(Pred) && "Should only reach loop blocks!"); // We are not interested in backedges and we don't want to leave loop. if (Pred == CurLoop->getHeader()) continue; // TODO: If BB lies in an inner loop of CurLoop, this will traverse over all // blocks of this inner loop, even those that are always executed AFTER the // BB. It may make our analysis more conservative than it could be, see test // @nested and @nested_no_throw in test/Analysis/MustExecute/loop-header.ll. // We can ignore backedge of all loops containing BB to get a sligtly more // optimistic result. for (auto *PredPred : predecessors(Pred)) if (Predecessors.insert(PredPred).second) WorkList.push_back(PredPred); } }
bool RecurrenceDescriptor::getSourceExtensionKind( Instruction *Start, Instruction *Exit, Type *RT, bool &IsSigned, SmallPtrSetImpl<Instruction *> &Visited, SmallPtrSetImpl<Instruction *> &CI) { SmallVector<Instruction *, 8> Worklist; bool FoundOneOperand = false; unsigned DstSize = RT->getPrimitiveSizeInBits(); Worklist.push_back(Exit); // Traverse the instructions in the reduction expression, beginning with the // exit value. while (!Worklist.empty()) { Instruction *I = Worklist.pop_back_val(); for (Use &U : I->operands()) { // Terminate the traversal if the operand is not an instruction, or we // reach the starting value. Instruction *J = dyn_cast<Instruction>(U.get()); if (!J || J == Start) continue; // Otherwise, investigate the operation if it is also in the expression. if (Visited.count(J)) { Worklist.push_back(J); continue; } // If the operand is not in Visited, it is not a reduction operation, but // it does feed into one. Make sure it is either a single-use sign- or // zero-extend instruction. CastInst *Cast = dyn_cast<CastInst>(J); bool IsSExtInst = isa<SExtInst>(J); if (!Cast || !Cast->hasOneUse() || !(isa<ZExtInst>(J) || IsSExtInst)) return false; // Ensure the source type of the extend is no larger than the reduction // type. It is not necessary for the types to be identical. unsigned SrcSize = Cast->getSrcTy()->getPrimitiveSizeInBits(); if (SrcSize > DstSize) return false; // Furthermore, ensure that all such extends are of the same kind. if (FoundOneOperand) { if (IsSigned != IsSExtInst) return false; } else { FoundOneOperand = true; IsSigned = IsSExtInst; } // Lastly, if the source type of the extend matches the reduction type, // add the extend to CI so that we can avoid accounting for it in the // cost model. if (SrcSize == DstSize) CI.insert(Cast); } } return true; }
/// CopyFrom - implement operator= from a smallptrset that has the same pointer /// type, but may have a different small size. void SmallPtrSetImpl::CopyFrom(const SmallPtrSetImpl &RHS) { if (isSmall() && RHS.isSmall()) assert(CurArraySize == RHS.CurArraySize && "Cannot assign sets with different small sizes"); // If we're becoming small, prepare to insert into our stack space if (RHS.isSmall()) { if (!isSmall()) free(CurArray); CurArray = SmallArray; // Otherwise, allocate new heap space (unless we were the same size) } else if (CurArraySize != RHS.CurArraySize) { if (isSmall()) CurArray = (const void**)malloc(sizeof(void*) * RHS.CurArraySize); else CurArray = (const void**)realloc(CurArray, sizeof(void*)*RHS.CurArraySize); assert(CurArray && "Failed to allocate memory?"); } // Copy over the new array size CurArraySize = RHS.CurArraySize; // Copy over the contents from the other set memcpy(CurArray, RHS.CurArray, sizeof(void*)*CurArraySize); NumElements = RHS.NumElements; NumTombstones = RHS.NumTombstones; }
void LTOCodeGenerator:: applyRestriction(GlobalValue &GV, ArrayRef<StringRef> Libcalls, std::vector<const char*> &MustPreserveList, SmallPtrSetImpl<GlobalValue*> &AsmUsed, Mangler &Mangler) { // There are no restrictions to apply to declarations. if (GV.isDeclaration()) return; // There is nothing more restrictive than private linkage. if (GV.hasPrivateLinkage()) return; SmallString<64> Buffer; TargetMach->getNameWithPrefix(Buffer, &GV, Mangler); if (MustPreserveSymbols.count(Buffer)) MustPreserveList.push_back(GV.getName().data()); if (AsmUndefinedRefs.count(Buffer)) AsmUsed.insert(&GV); // Conservatively append user-supplied runtime library functions to // llvm.compiler.used. These could be internalized and deleted by // optimizations like -globalopt, causing problems when later optimizations // add new library calls (e.g., llvm.memset => memset and printf => puts). // Leave it to the linker to remove any dead code (e.g. with -dead_strip). if (isa<Function>(GV) && std::binary_search(Libcalls.begin(), Libcalls.end(), GV.getName())) AsmUsed.insert(&GV); }
/// MarkBlocksLiveIn - Insert BB and all of its predecessors into LiveBBs until /// we reach blocks we've already seen. static void MarkBlocksLiveIn(BasicBlock *BB, SmallPtrSetImpl<BasicBlock *> &LiveBBs) { if (!LiveBBs.insert(BB).second) return; // already been here. df_iterator_default_set<BasicBlock*> Visited; for (BasicBlock *B : inverse_depth_first_ext(BB, Visited)) LiveBBs.insert(B); }
/// Find values that are marked as llvm.used. static void findUsedValues(GlobalVariable *LLVMUsed, SmallPtrSetImpl<const GlobalValue*> &UsedValues) { if (!LLVMUsed) return; UsedValues.insert(LLVMUsed); ConstantArray *Inits = cast<ConstantArray>(LLVMUsed->getInitializer()); for (unsigned i = 0, e = Inits->getNumOperands(); i != e; ++i) if (GlobalValue *GV = dyn_cast<GlobalValue>(Inits->getOperand(i)->stripPointerCasts())) UsedValues.insert(GV); }
// Breadth-first walk of the use-def graph; determine the set of nodes // we care about and eagerly determine if some of them are poisonous. void Float2IntPass::walkBackwards(const SmallPtrSetImpl<Instruction*> &Roots) { std::deque<Instruction*> Worklist(Roots.begin(), Roots.end()); while (!Worklist.empty()) { Instruction *I = Worklist.back(); Worklist.pop_back(); if (SeenInsts.find(I) != SeenInsts.end()) // Seen already. continue; switch (I->getOpcode()) { // FIXME: Handle select and phi nodes. default: // Path terminated uncleanly. seen(I, badRange()); break; case Instruction::UIToFP: case Instruction::SIToFP: { // Path terminated cleanly - use the type of the integer input to seed // the analysis. unsigned BW = I->getOperand(0)->getType()->getPrimitiveSizeInBits(); auto Input = ConstantRange(BW, true); auto CastOp = (Instruction::CastOps)I->getOpcode(); seen(I, validateRange(Input.castOp(CastOp, MaxIntegerBW+1))); continue; } case Instruction::FAdd: case Instruction::FSub: case Instruction::FMul: case Instruction::FPToUI: case Instruction::FPToSI: case Instruction::FCmp: seen(I, unknownRange()); break; } for (Value *O : I->operands()) { if (Instruction *OI = dyn_cast<Instruction>(O)) { // Unify def-use chains if they interfere. ECs.unionSets(I, OI); if (SeenInsts.find(I)->second != badRange()) Worklist.push_back(OI); } else if (!isa<ConstantFP>(O)) { // Not an instruction or ConstantFP? we can't do anything. seen(I, badRange()); } } } }
/// Merge an autorelease with a retain into a fused call. bool ObjCARCContract::contractAutorelease( Function &F, Instruction *Autorelease, ARCInstKind Class, SmallPtrSetImpl<Instruction *> &DependingInstructions, SmallPtrSetImpl<const BasicBlock *> &Visited) { const Value *Arg = GetArgRCIdentityRoot(Autorelease); // Check that there are no instructions between the retain and the autorelease // (such as an autorelease_pop) which may change the count. CallInst *Retain = nullptr; if (Class == ARCInstKind::AutoreleaseRV) FindDependencies(RetainAutoreleaseRVDep, Arg, Autorelease->getParent(), Autorelease, DependingInstructions, Visited, PA); else FindDependencies(RetainAutoreleaseDep, Arg, Autorelease->getParent(), Autorelease, DependingInstructions, Visited, PA); Visited.clear(); if (DependingInstructions.size() != 1) { DependingInstructions.clear(); return false; } Retain = dyn_cast_or_null<CallInst>(*DependingInstructions.begin()); DependingInstructions.clear(); if (!Retain || GetBasicARCInstKind(Retain) != ARCInstKind::Retain || GetArgRCIdentityRoot(Retain) != Arg) return false; Changed = true; ++NumPeeps; LLVM_DEBUG(dbgs() << " Fusing retain/autorelease!\n" " Autorelease:" << *Autorelease << "\n" " Retain: " << *Retain << "\n"); Function *Decl = EP.get(Class == ARCInstKind::AutoreleaseRV ? ARCRuntimeEntryPointKind::RetainAutoreleaseRV : ARCRuntimeEntryPointKind::RetainAutorelease); Retain->setCalledFunction(Decl); LLVM_DEBUG(dbgs() << " New RetainAutorelease: " << *Retain << "\n"); EraseInstruction(Autorelease); return true; }
void SmallPtrSetImpl::swap(SmallPtrSetImpl &RHS) { if (this == &RHS) return; // We can only avoid copying elements if neither set is small. if (!this->isSmall() && !RHS.isSmall()) { std::swap(this->CurArray, RHS.CurArray); std::swap(this->CurArraySize, RHS.CurArraySize); std::swap(this->NumElements, RHS.NumElements); std::swap(this->NumTombstones, RHS.NumTombstones); return; } // FIXME: From here on we assume that both sets have the same small size. // If only RHS is small, copy the small elements into LHS and move the pointer // from LHS to RHS. if (!this->isSmall() && RHS.isSmall()) { std::copy(RHS.SmallArray, RHS.SmallArray+RHS.CurArraySize, this->SmallArray); std::swap(this->NumElements, RHS.NumElements); std::swap(this->CurArraySize, RHS.CurArraySize); RHS.CurArray = this->CurArray; RHS.NumTombstones = this->NumTombstones; this->CurArray = this->SmallArray; this->NumTombstones = 0; return; } // If only LHS is small, copy the small elements into RHS and move the pointer // from RHS to LHS. if (this->isSmall() && !RHS.isSmall()) { std::copy(this->SmallArray, this->SmallArray+this->CurArraySize, RHS.SmallArray); std::swap(RHS.NumElements, this->NumElements); std::swap(RHS.CurArraySize, this->CurArraySize); this->CurArray = RHS.CurArray; this->NumTombstones = RHS.NumTombstones; RHS.CurArray = RHS.SmallArray; RHS.NumTombstones = 0; return; } // Both a small, just swap the small elements. assert(this->isSmall() && RHS.isSmall()); assert(this->CurArraySize == RHS.CurArraySize); std::swap_ranges(this->SmallArray, this->SmallArray+this->CurArraySize, RHS.SmallArray); std::swap(this->NumElements, RHS.NumElements); }
static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA, SmallPtrSetImpl<Instruction *> &Stores) { if (Inst->mayWriteToMemory()) { Stores.insert(Inst); return false; } if (LoadInst *L = dyn_cast<LoadInst>(Inst)) { MemoryLocation Loc = MemoryLocation::get(L); for (Instruction *S : Stores) if (AA->getModRefInfo(S, Loc) & MRI_Mod) return false; } if (isa<TerminatorInst>(Inst) || isa<PHINode>(Inst)) return false; // Convergent operations cannot be made control-dependent on additional // values. if (auto CS = CallSite(Inst)) { if (CS.hasFnAttr(Attribute::Convergent)) return false; } return true; }
bool RecurrenceDescriptor::areAllUsesIn(Instruction *I, SmallPtrSetImpl<Instruction *> &Set) { for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use) if (!Set.count(dyn_cast<Instruction>(*Use))) return false; return true; }
static void collectMDInDomain(const MDNode *List, const MDNode *Domain, SmallPtrSetImpl<const MDNode *> &Nodes) { for (const MDOperand &MDOp : List->operands()) if (const MDNode *MD = dyn_cast<MDNode>(MDOp)) if (AliasScopeNode(MD).getDomain() == Domain) Nodes.insert(MD); }
void AMDGPUAlwaysInline::recursivelyVisitUsers( GlobalValue &GV, SmallPtrSetImpl<Function *> &FuncsToAlwaysInline) { SmallVector<User *, 16> Stack; SmallPtrSet<const Value *, 8> Visited; for (User *U : GV.users()) Stack.push_back(U); while (!Stack.empty()) { User *U = Stack.pop_back_val(); if (!Visited.insert(U).second) continue; if (Instruction *I = dyn_cast<Instruction>(U)) { Function *F = I->getParent()->getParent(); if (!AMDGPU::isEntryFunctionCC(F->getCallingConv())) { FuncsToAlwaysInline.insert(F); Stack.push_back(F); } // No need to look at further users, but we do need to inline any callers. continue; } for (User *UU : U->users()) Stack.push_back(UU); } }
static bool isSafeToMove(Instruction *Inst, AliasAnalysis &AA, SmallPtrSetImpl<Instruction *> &Stores) { if (Inst->mayWriteToMemory()) { Stores.insert(Inst); return false; } if (LoadInst *L = dyn_cast<LoadInst>(Inst)) { MemoryLocation Loc = MemoryLocation::get(L); for (Instruction *S : Stores) if (isModSet(AA.getModRefInfo(S, Loc))) return false; } if (Inst->isTerminator() || isa<PHINode>(Inst) || Inst->isEHPad() || Inst->mayThrow()) return false; if (auto *Call = dyn_cast<CallBase>(Inst)) { // Convergent operations cannot be made control-dependent on additional // values. if (Call->hasFnAttr(Attribute::Convergent)) return false; for (Instruction *S : Stores) if (isModSet(AA.getModRefInfo(S, Call))) return false; } return true; }
void MemorySSAUpdater::removeBlocks( const SmallPtrSetImpl<BasicBlock *> &DeadBlocks) { // First delete all uses of BB in MemoryPhis. for (BasicBlock *BB : DeadBlocks) { TerminatorInst *TI = BB->getTerminator(); assert(TI && "Basic block expected to have a terminator instruction"); for (BasicBlock *Succ : successors(TI)) if (!DeadBlocks.count(Succ)) if (MemoryPhi *MP = MSSA->getMemoryAccess(Succ)) { MP->unorderedDeleteIncomingBlock(BB); if (MP->getNumIncomingValues() == 1) removeMemoryAccess(MP); } // Drop all references of all accesses in BB if (MemorySSA::AccessList *Acc = MSSA->getWritableBlockAccesses(BB)) for (MemoryAccess &MA : *Acc) MA.dropAllReferences(); } // Next, delete all memory accesses in each block for (BasicBlock *BB : DeadBlocks) { MemorySSA::AccessList *Acc = MSSA->getWritableBlockAccesses(BB); if (!Acc) continue; for (auto AB = Acc->begin(), AE = Acc->end(); AB != AE;) { MemoryAccess *MA = &*AB; ++AB; MSSA->removeFromLookups(MA); MSSA->removeFromLists(MA); } } }
SmallPtrSetImpl::SmallPtrSetImpl(const SmallPtrSetImpl& that) { NumElements = that.NumElements; NumTombstones = 0; if (that.isSmall()) { CurArraySize = that.CurArraySize; CurArray = &SmallArray[0]; memcpy(CurArray, that.CurArray, sizeof(void*)*CurArraySize); } else { CurArraySize = that.NumElements < 64 ? 128 : that.NumElements*2; CurArray = new void*[CurArraySize+1]; memset(CurArray, -1, CurArraySize*sizeof(void*)); // The end pointer, always valid, is set to a valid element to help the // iterator. CurArray[CurArraySize] = 0; // Copy over all valid entries. for (void **BucketPtr = that.CurArray, **E = that.CurArray+CurArraySize; BucketPtr != E; ++BucketPtr) { // Copy over the element if it is valid. void *Elt = *BucketPtr; if (Elt != getTombstoneMarker() && Elt != getEmptyMarker()) *const_cast<void**>(FindBucketFor(Elt)) = Elt; } } }
/// Collect cast instructions that can be ignored in the vectorizer's cost /// model, given a reduction exit value and the minimal type in which the /// reduction can be represented. static void collectCastsToIgnore(Loop *TheLoop, Instruction *Exit, Type *RecurrenceType, SmallPtrSetImpl<Instruction *> &Casts) { SmallVector<Instruction *, 8> Worklist; SmallPtrSet<Instruction *, 8> Visited; Worklist.push_back(Exit); while (!Worklist.empty()) { Instruction *Val = Worklist.pop_back_val(); Visited.insert(Val); if (auto *Cast = dyn_cast<CastInst>(Val)) if (Cast->getSrcTy() == RecurrenceType) { // If the source type of a cast instruction is equal to the recurrence // type, it will be eliminated, and should be ignored in the vectorizer // cost model. Casts.insert(Cast); continue; } // Add all operands to the work list if they are loop-varying values that // we haven't yet visited. for (Value *O : cast<User>(Val)->operands()) if (auto *I = dyn_cast<Instruction>(O)) if (TheLoop->contains(I) && !Visited.count(I)) Worklist.push_back(I); } }
void ScopedNoAliasAAResult::collectMDInDomain( const MDNode *List, const MDNode *Domain, SmallPtrSetImpl<const MDNode *> &Nodes) const { for (unsigned i = 0, ie = List->getNumOperands(); i != ie; ++i) if (const MDNode *MD = dyn_cast<MDNode>(List->getOperand(i))) if (AliasScopeNode(MD).getDomain() == Domain) Nodes.insert(MD); }
/// getMachineBasicBlocks - Populate given set using machine basic blocks which /// have machine instructions that belong to lexical scope identified by /// DebugLoc. void LexicalScopes::getMachineBasicBlocks( const DILocation *DL, SmallPtrSetImpl<const MachineBasicBlock *> &MBBs) { MBBs.clear(); LexicalScope *Scope = getOrCreateLexicalScope(DL); if (!Scope) return; if (Scope == CurrentFnLexicalScope) { for (const auto &MBB : *MF) MBBs.insert(&MBB); return; } SmallVectorImpl<InsnRange> &InsnRanges = Scope->getRanges(); for (auto &R : InsnRanges) MBBs.insert(R.first->getParent()); }
/// MarkBlocksLiveIn - Insert BB and all of its predescessors into LiveBBs until /// we reach blocks we've already seen. static void MarkBlocksLiveIn(BasicBlock *BB, SmallPtrSetImpl<BasicBlock *> &LiveBBs) { if (!LiveBBs.insert(BB).second) return; // already been here. for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) MarkBlocksLiveIn(*PI, LiveBBs); }
Action walkToTypePre(Type ty) override { if (ty->is<DependentMemberType>()) return Action::SkipChildren; if (auto typeVar = ty->getAs<TypeVariableType>()) typeVars.insert(typeVar); return Action::Continue; }
/// \brief Analyze a basic block for its contribution to the inline cost. /// /// This method walks the analyzer over every instruction in the given basic /// block and accounts for their cost during inlining at this callsite. It /// aborts early if the threshold has been exceeded or an impossible to inline /// construct has been detected. It returns false if inlining is no longer /// viable, and true if inlining remains viable. bool CallAnalyzer::analyzeBlock(BasicBlock *BB, SmallPtrSetImpl<const Value *> &EphValues) { for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) { // FIXME: Currently, the number of instructions in a function regardless of // our ability to simplify them during inline to constants or dead code, // are actually used by the vector bonus heuristic. As long as that's true, // we have to special case debug intrinsics here to prevent differences in // inlining due to debug symbols. Eventually, the number of unsimplified // instructions shouldn't factor into the cost computation, but until then, // hack around it here. if (isa<DbgInfoIntrinsic>(I)) continue; // Skip ephemeral values. if (EphValues.count(I)) continue; ++NumInstructions; if (isa<ExtractElementInst>(I) || I->getType()->isVectorTy()) ++NumVectorInstructions; // If the instruction simplified to a constant, there is no cost to this // instruction. Visit the instructions using our InstVisitor to account for // all of the per-instruction logic. The visit tree returns true if we // consumed the instruction in any way, and false if the instruction's base // cost should count against inlining. if (Base::visit(I)) ++NumInstructionsSimplified; else Cost += InlineConstants::InstrCost; // If the visit this instruction detected an uninlinable pattern, abort. if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca || HasIndirectBr) return false; // If the caller is a recursive function then we don't want to inline // functions which allocate a lot of stack space because it would increase // the caller stack usage dramatically. if (IsCallerRecursive && AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller) return false; if (NumVectorInstructions > NumInstructions/2) VectorBonus = FiftyPercentVectorBonus; else if (NumVectorInstructions > NumInstructions/10) VectorBonus = TenPercentVectorBonus; else VectorBonus = 0; // Check if we've past the threshold so we don't spin in huge basic // blocks that will never inline. if (Cost > (Threshold + VectorBonus)) return false; } return true; }
void checkEraseAndIterators(SmallPtrSetImpl<int*> &S) { int buf[3]; S.insert(&buf[0]); S.insert(&buf[1]); S.insert(&buf[2]); // Iterators must still be valid after erase() calls; auto B = S.begin(); auto M = std::next(B); auto E = S.end(); EXPECT_TRUE(*B == &buf[0] || *B == &buf[1] || *B == &buf[2]); EXPECT_TRUE(*M == &buf[0] || *M == &buf[1] || *M == &buf[2]); EXPECT_TRUE(*B != *M); int *Removable = *std::next(M); // No iterator points to Removable now. EXPECT_TRUE(Removable == &buf[0] || Removable == &buf[1] || Removable == &buf[2]); EXPECT_TRUE(Removable != *B && Removable != *M); S.erase(Removable); // B,M,E iterators should still be valid EXPECT_EQ(B, S.begin()); EXPECT_EQ(M, std::next(B)); EXPECT_EQ(E, S.end()); EXPECT_EQ(std::next(M), E); }
/// Return adjusted total frequency of \p BBs. /// /// * If there is only one BB, sinking instruction will not introduce code /// size increase. Thus there is no need to adjust the frequency. /// * If there are more than one BB, sinking would lead to code size increase. /// In this case, we add some "tax" to the total frequency to make it harder /// to sink. E.g. /// Freq(Preheader) = 100 /// Freq(BBs) = sum(50, 49) = 99 /// Even if Freq(BBs) < Freq(Preheader), we will not sink from Preheade to /// BBs as the difference is too small to justify the code size increase. /// To model this, The adjusted Freq(BBs) will be: /// AdjustedFreq(BBs) = 99 / SinkFrequencyPercentThreshold% static BlockFrequency adjustedSumFreq(SmallPtrSetImpl<BasicBlock *> &BBs, BlockFrequencyInfo &BFI) { BlockFrequency T = 0; for (BasicBlock *B : BBs) T += BFI.getBlockFreq(B); if (BBs.size() > 1) T /= BranchProbability(SinkFrequencyPercentThreshold, 100); return T; }
/// Walk up the CFG from StartPos (which is in StartBB) and find local and /// non-local dependencies on Arg. /// /// TODO: Cache results? void llvm::objcarc::FindDependencies(DependenceKind Flavor, const Value *Arg, BasicBlock *StartBB, Instruction *StartInst, SmallPtrSetImpl<Instruction *> &DependingInsts, SmallPtrSetImpl<const BasicBlock *> &Visited, ProvenanceAnalysis &PA) { BasicBlock::iterator StartPos = StartInst; SmallVector<std::pair<BasicBlock *, BasicBlock::iterator>, 4> Worklist; Worklist.push_back(std::make_pair(StartBB, StartPos)); do { std::pair<BasicBlock *, BasicBlock::iterator> Pair = Worklist.pop_back_val(); BasicBlock *LocalStartBB = Pair.first; BasicBlock::iterator LocalStartPos = Pair.second; BasicBlock::iterator StartBBBegin = LocalStartBB->begin(); for (;;) { if (LocalStartPos == StartBBBegin) { pred_iterator PI(LocalStartBB), PE(LocalStartBB, false); if (PI == PE) // If we've reached the function entry, produce a null dependence. DependingInsts.insert(nullptr); else // Add the predecessors to the worklist. do { BasicBlock *PredBB = *PI; if (Visited.insert(PredBB)) Worklist.push_back(std::make_pair(PredBB, PredBB->end())); } while (++PI != PE); break; } Instruction *Inst = --LocalStartPos; if (Depends(Flavor, Inst, Arg, PA)) { DependingInsts.insert(Inst); break; } } } while (!Worklist.empty()); // Determine whether the original StartBB post-dominates all of the blocks we // visited. If not, insert a sentinal indicating that most optimizations are // not safe. for (SmallPtrSet<const BasicBlock *, 4>::const_iterator I = Visited.begin(), E = Visited.end(); I != E; ++I) { const BasicBlock *BB = *I; if (BB == StartBB) continue; const TerminatorInst *TI = cast<TerminatorInst>(&BB->back()); for (succ_const_iterator SI(TI), SE(TI, false); SI != SE; ++SI) { const BasicBlock *Succ = *SI; if (Succ != StartBB && !Visited.count(Succ)) { DependingInsts.insert(reinterpret_cast<Instruction *>(-1)); return; } } } }
static inline bool isSimpleEnoughValueToCommit(Constant *C, SmallPtrSetImpl<Constant *> &SimpleConstants, const DataLayout &DL) { // If we already checked this constant, we win. if (!SimpleConstants.insert(C).second) return true; // Check the constant. return isSimpleEnoughValueToCommitHelper(C, SimpleConstants, DL); }
/// Return a set of basic blocks to insert sinked instructions. /// /// The returned set of basic blocks (BBsToSinkInto) should satisfy: /// /// * Inside the loop \p L /// * For each UseBB in \p UseBBs, there is at least one BB in BBsToSinkInto /// that domintates the UseBB /// * Has minimum total frequency that is no greater than preheader frequency /// /// The purpose of the function is to find the optimal sinking points to /// minimize execution cost, which is defined as "sum of frequency of /// BBsToSinkInto". /// As a result, the returned BBsToSinkInto needs to have minimum total /// frequency. /// Additionally, if the total frequency of BBsToSinkInto exceeds preheader /// frequency, the optimal solution is not sinking (return empty set). /// /// \p ColdLoopBBs is used to help find the optimal sinking locations. /// It stores a list of BBs that is: /// /// * Inside the loop \p L /// * Has a frequency no larger than the loop's preheader /// * Sorted by BB frequency /// /// The complexity of the function is O(UseBBs.size() * ColdLoopBBs.size()). /// To avoid expensive computation, we cap the maximum UseBBs.size() in its /// caller. static SmallPtrSet<BasicBlock *, 2> findBBsToSinkInto(const Loop &L, const SmallPtrSetImpl<BasicBlock *> &UseBBs, const SmallVectorImpl<BasicBlock *> &ColdLoopBBs, DominatorTree &DT, BlockFrequencyInfo &BFI) { SmallPtrSet<BasicBlock *, 2> BBsToSinkInto; if (UseBBs.size() == 0) return BBsToSinkInto; BBsToSinkInto.insert(UseBBs.begin(), UseBBs.end()); SmallPtrSet<BasicBlock *, 2> BBsDominatedByColdestBB; // For every iteration: // * Pick the ColdestBB from ColdLoopBBs // * Find the set BBsDominatedByColdestBB that satisfy: // - BBsDominatedByColdestBB is a subset of BBsToSinkInto // - Every BB in BBsDominatedByColdestBB is dominated by ColdestBB // * If Freq(ColdestBB) < Freq(BBsDominatedByColdestBB), remove // BBsDominatedByColdestBB from BBsToSinkInto, add ColdestBB to // BBsToSinkInto for (BasicBlock *ColdestBB : ColdLoopBBs) { BBsDominatedByColdestBB.clear(); for (BasicBlock *SinkedBB : BBsToSinkInto) if (DT.dominates(ColdestBB, SinkedBB)) BBsDominatedByColdestBB.insert(SinkedBB); if (BBsDominatedByColdestBB.size() == 0) continue; if (adjustedSumFreq(BBsDominatedByColdestBB, BFI) > BFI.getBlockFreq(ColdestBB)) { for (BasicBlock *DominatedBB : BBsDominatedByColdestBB) { BBsToSinkInto.erase(DominatedBB); } BBsToSinkInto.insert(ColdestBB); } } // If the total frequency of BBsToSinkInto is larger than preheader frequency, // do not sink. if (adjustedSumFreq(BBsToSinkInto, BFI) > BFI.getBlockFreq(L.getLoopPreheader())) BBsToSinkInto.clear(); return BBsToSinkInto; }
/// Visit each register class belonging to the given register bank. /// /// A class belongs to the bank iff any of these apply: /// * It is explicitly specified /// * It is a subclass of a class that is a member. /// * It is a class containing subregisters of the registers of a class that /// is a member. This is known as a subreg-class. /// /// This function must be called for each explicitly specified register class. /// /// \param RC The register class to search. /// \param Kind A debug string containing the path the visitor took to reach RC. /// \param VisitFn The action to take for each class visited. It may be called /// multiple times for a given class if there are multiple paths /// to the class. static void visitRegisterBankClasses( CodeGenRegBank &RegisterClassHierarchy, const CodeGenRegisterClass *RC, const Twine Kind, std::function<void(const CodeGenRegisterClass *, StringRef)> VisitFn, SmallPtrSetImpl<const CodeGenRegisterClass *> &VisitedRCs) { // Make sure we only visit each class once to avoid infinite loops. if (VisitedRCs.count(RC)) return; VisitedRCs.insert(RC); // Visit each explicitly named class. VisitFn(RC, Kind.str()); for (const auto &PossibleSubclass : RegisterClassHierarchy.getRegClasses()) { std::string TmpKind = (Twine(Kind) + " (" + PossibleSubclass.getName() + ")").str(); // Visit each subclass of an explicitly named class. if (RC != &PossibleSubclass && RC->hasSubClass(&PossibleSubclass)) visitRegisterBankClasses(RegisterClassHierarchy, &PossibleSubclass, TmpKind + " " + RC->getName() + " subclass", VisitFn, VisitedRCs); // Visit each class that contains only subregisters of RC with a common // subregister-index. // // More precisely, PossibleSubclass is a subreg-class iff Reg:SubIdx is in // PossibleSubclass for all registers Reg from RC using any // subregister-index SubReg for (const auto &SubIdx : RegisterClassHierarchy.getSubRegIndices()) { BitVector BV(RegisterClassHierarchy.getRegClasses().size()); PossibleSubclass.getSuperRegClasses(&SubIdx, BV); if (BV.test(RC->EnumValue)) { std::string TmpKind2 = (Twine(TmpKind) + " " + RC->getName() + " class-with-subregs: " + RC->getName()) .str(); VisitFn(&PossibleSubclass, TmpKind2); } } } }
bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc, SmallPtrSetImpl<Instruction *> &Visited) { auto *Inst = dyn_cast<Instruction>(V); if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst)) return true; if (!isSafeToSpeculativelyExecute(Inst, Loc, &DT) || Inst->mayReadFromMemory()) return false; Visited.insert(Inst); // We only want to go _up_ the dominance chain when recursing. assert(!isa<PHINode>(Loc) && "PHIs should return false for isSafeToSpeculativelyExecute"); assert(DT.isReachableFromEntry(Inst->getParent()) && "We did a DFS from the block entry!"); return all_of(Inst->operands(), [&](Value *Op) { return isAvailableAt(Op, Loc, Visited); }); }
void VPlanHCFGTransforms::VPInstructionsToVPRecipes( VPlanPtr &Plan, LoopVectorizationLegality::InductionList *Inductions, SmallPtrSetImpl<Instruction *> &DeadInstructions) { VPRegionBlock *TopRegion = dyn_cast<VPRegionBlock>(Plan->getEntry()); ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry()); for (VPBlockBase *Base : RPOT) { // Do not widen instructions in pre-header and exit blocks. if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0) continue; VPBasicBlock *VPBB = Base->getEntryBasicBlock(); VPRecipeBase *LastRecipe = nullptr; // Introduce each ingredient into VPlan. for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) { VPRecipeBase *Ingredient = &*I++; // Can only handle VPInstructions. VPInstruction *VPInst = cast<VPInstruction>(Ingredient); Instruction *Inst = cast<Instruction>(VPInst->getUnderlyingValue()); if (DeadInstructions.count(Inst)) { Ingredient->eraseFromParent(); continue; } VPRecipeBase *NewRecipe = nullptr; // Create VPWidenMemoryInstructionRecipe for loads and stores. if (isa<LoadInst>(Inst) || isa<StoreInst>(Inst)) NewRecipe = new VPWidenMemoryInstructionRecipe(*Inst, nullptr /*Mask*/); else if (PHINode *Phi = dyn_cast<PHINode>(Inst)) { InductionDescriptor II = Inductions->lookup(Phi); if (II.getKind() == InductionDescriptor::IK_IntInduction || II.getKind() == InductionDescriptor::IK_FpInduction) { NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi); } else NewRecipe = new VPWidenPHIRecipe(Phi); } else { // If the last recipe is a VPWidenRecipe, add Inst to it instead of // creating a new recipe. if (VPWidenRecipe *WidenRecipe = dyn_cast_or_null<VPWidenRecipe>(LastRecipe)) { WidenRecipe->appendInstruction(Inst); Ingredient->eraseFromParent(); continue; } NewRecipe = new VPWidenRecipe(Inst); } NewRecipe->insertBefore(Ingredient); LastRecipe = NewRecipe; Ingredient->eraseFromParent(); } } }