void LoopInfo::markAsRemoved(Loop *Unloop) { assert(!Unloop->isInvalid() && "Loop has already been removed"); Unloop->invalidate(); RemovedLoops.push_back(Unloop); // First handle the special case of no parent loop to simplify the algorithm. if (!Unloop->getParentLoop()) { // Since BBLoop had no parent, Unloop blocks are no longer in a loop. for (Loop::block_iterator I = Unloop->block_begin(), E = Unloop->block_end(); I != E; ++I) { // Don't reparent blocks in subloops. if (getLoopFor(*I) != Unloop) continue; // Blocks no longer have a parent but are still referenced by Unloop until // the Unloop object is deleted. changeLoopFor(*I, nullptr); } // Remove the loop from the top-level LoopInfo object. for (iterator I = begin();; ++I) { assert(I != end() && "Couldn't find loop"); if (*I == Unloop) { removeLoop(I); break; } } // Move all of the subloops to the top-level. while (!Unloop->empty()) addTopLevelLoop(Unloop->removeChildLoop(std::prev(Unloop->end()))); return; } // Update the parent loop for all blocks within the loop. Blocks within // subloops will not change parents. UnloopUpdater Updater(Unloop, this); Updater.updateBlockParents(); // Remove blocks from former ancestor loops. Updater.removeBlocksFromAncestors(); // Add direct subloops as children in their new parent loop. Updater.updateSubloopParents(); // Remove unloop from its parent loop. Loop *ParentLoop = Unloop->getParentLoop(); for (Loop::iterator I = ParentLoop->begin();; ++I) { assert(I != ParentLoop->end() && "Couldn't find loop"); if (*I == Unloop) { ParentLoop->removeChildLoop(I); break; } } }
/// FindBackAndExitEdges - Search for back and exit edges for all blocks /// within the function loops, calculated using loop information. void BranchPredictionInfo::FindBackAndExitEdges(Function &F) { SmallPtrSet<const BasicBlock *, 64> LoopsVisited; SmallPtrSet<const BasicBlock *, 64> BlocksVisited; int count = 0; if(F.getName() == "hypre_SMGResidual") count = count + 1; for (LoopInfo::iterator LIT = LI->begin(), LIE = LI->end(); LIT != LIE; ++LIT) { Loop *rootLoop = *LIT; BasicBlock *rootHeader = rootLoop->getHeader(); // Check if we already visited this loop. if (LoopsVisited.count(rootHeader)) continue; // Create a stack to hold loops (inner most on the top). SmallVectorImpl<Loop *> Stack(8); SmallPtrSet<const BasicBlock *, 8> InStack; // Put the current loop into the Stack. Stack.push_back(rootLoop); InStack.insert(rootHeader); do { Loop *loop = Stack.back(); // Search for new inner loops. bool foundNew = false; for (Loop::iterator I = loop->begin(), E = loop->end(); I != E; ++I) { Loop *innerLoop = *I; BasicBlock *innerHeader = innerLoop->getHeader(); // Skip visited inner loops. if (!LoopsVisited.count(innerHeader)) { Stack.push_back(innerLoop); InStack.insert(innerHeader); foundNew = true; break; } } // If a new loop is found, continue. // Otherwise, it is time to expand it, because it is the most inner loop // yet unprocessed. if (foundNew) continue; // The variable "loop" is now the unvisited inner most loop. BasicBlock *header = loop->getHeader(); // Search for all basic blocks on the loop. for (Loop::block_iterator LBI = loop->block_begin(), LBE = loop->block_end(); LBI != LBE; ++LBI) { BasicBlock *lpBB = *LBI; if (!BlocksVisited.insert(lpBB)) continue; // Set the number of back edges to this loop head (lpBB) as zero. BackEdgesCount[lpBB] = 0; // For each loop block successor, check if the block pointing is // outside the loop. TerminatorInst *TI = lpBB->getTerminator(); for (unsigned s = 0; s < TI->getNumSuccessors(); ++s) { BasicBlock *successor = TI->getSuccessor(s); Edge edge = std::make_pair(lpBB, successor); // If the successor matches any loop header on the stack, // then it is a backedge. if (InStack.count(successor)) { listBackEdges.insert(edge); ++BackEdgesCount[lpBB]; } // If the successor is not present in the loop block list, then it is // an exit edge. if (!loop->contains(successor)) listExitEdges.insert(edge); } } // Cleaning the visited loop. LoopsVisited.insert(header); Stack.pop_back(); InStack.erase(header); } while (!InStack.empty()); } }
bool LoopUnroll::visitLoop(Loop *L) { bool Changed = false; // Recurse through all subloops before we process this loop. Copy the loop // list so that the child can update the loop tree if it needs to delete the // loop. std::vector<Loop*> SubLoops(L->begin(), L->end()); for (unsigned i = 0, e = SubLoops.size(); i != e; ++i) Changed |= visitLoop(SubLoops[i]); // We only handle single basic block loops right now. if (L->getBlocks().size() != 1) return Changed; BasicBlock *BB = L->getHeader(); BranchInst *BI = dyn_cast<BranchInst>(BB->getTerminator()); if (BI == 0) return Changed; // Must end in a conditional branch ConstantInt *TripCountC = dyn_cast_or_null<ConstantInt>(L->getTripCount()); if (!TripCountC) return Changed; // Must have constant trip count! unsigned TripCount = TripCountC->getRawValue(); if (TripCount != TripCountC->getRawValue() || TripCount == 0) return Changed; // More than 2^32 iterations??? unsigned LoopSize = ApproximateLoopSize(L); DEBUG(std::cerr << "Loop Unroll: F[" << BB->getParent()->getName() << "] Loop %" << BB->getName() << " Loop Size = " << LoopSize << " Trip Count = " << TripCount << " - "); uint64_t Size = (uint64_t)LoopSize*(uint64_t)TripCount; if (Size > UnrollThreshold) { DEBUG(std::cerr << "TOO LARGE: " << Size << ">" << UnrollThreshold << "\n"); return Changed; } DEBUG(std::cerr << "UNROLLING!\n"); BasicBlock *LoopExit = BI->getSuccessor(L->contains(BI->getSuccessor(0))); // Create a new basic block to temporarily hold all of the cloned code. BasicBlock *NewBlock = new BasicBlock(); // For the first iteration of the loop, we should use the precloned values for // PHI nodes. Insert associations now. std::map<const Value*, Value*> LastValueMap; std::vector<PHINode*> OrigPHINode; for (BasicBlock::iterator I = BB->begin(); PHINode *PN = dyn_cast<PHINode>(I); ++I) { OrigPHINode.push_back(PN); if (Instruction *I =dyn_cast<Instruction>(PN->getIncomingValueForBlock(BB))) if (I->getParent() == BB) LastValueMap[I] = I; } // Remove the exit branch from the loop BB->getInstList().erase(BI); assert(TripCount != 0 && "Trip count of 0 is impossible!"); for (unsigned It = 1; It != TripCount; ++It) { char SuffixBuffer[100]; sprintf(SuffixBuffer, ".%d", It); std::map<const Value*, Value*> ValueMap; BasicBlock *New = CloneBasicBlock(BB, ValueMap, SuffixBuffer); // Loop over all of the PHI nodes in the block, changing them to use the // incoming values from the previous block. for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) { PHINode *NewPHI = cast<PHINode>(ValueMap[OrigPHINode[i]]); Value *InVal = NewPHI->getIncomingValueForBlock(BB); if (Instruction *InValI = dyn_cast<Instruction>(InVal)) if (InValI->getParent() == BB) InVal = LastValueMap[InValI]; ValueMap[OrigPHINode[i]] = InVal; New->getInstList().erase(NewPHI); } for (BasicBlock::iterator I = New->begin(), E = New->end(); I != E; ++I) RemapInstruction(I, ValueMap); // Now that all of the instructions are remapped, splice them into the end // of the NewBlock. NewBlock->getInstList().splice(NewBlock->end(), New->getInstList()); delete New; // LastValue map now contains values from this iteration. std::swap(LastValueMap, ValueMap); } // If there was more than one iteration, replace any uses of values computed // in the loop with values computed during the last iteration of the loop. if (TripCount != 1) { std::set<User*> Users; for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) Users.insert(I->use_begin(), I->use_end()); // We don't want to reprocess entries with PHI nodes in them. For this // reason, we look at each operand of each user exactly once, performing the // stubstitution exactly once. for (std::set<User*>::iterator UI = Users.begin(), E = Users.end(); UI != E; ++UI) { Instruction *I = cast<Instruction>(*UI); if (I->getParent() != BB && I->getParent() != NewBlock) RemapInstruction(I, LastValueMap); } } // Now that we cloned the block as many times as we needed, stitch the new // code into the original block and delete the temporary block. BB->getInstList().splice(BB->end(), NewBlock->getInstList()); delete NewBlock; // Now loop over the PHI nodes in the original block, setting them to their // incoming values. BasicBlock *Preheader = L->getLoopPreheader(); for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) { PHINode *PN = OrigPHINode[i]; PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader)); BB->getInstList().erase(PN); } // Finally, add an unconditional branch to the block to continue into the exit // block. new BranchInst(LoopExit, BB); // At this point, the code is well formed. We now do a quick sweep over the // inserted code, doing constant propagation and dead code elimination as we // go. for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ) { Instruction *Inst = I++; if (isInstructionTriviallyDead(Inst)) BB->getInstList().erase(Inst); else if (Constant *C = ConstantFoldInstruction(Inst)) { Inst->replaceAllUsesWith(C); BB->getInstList().erase(Inst); } } // Update the loop information for this loop. Loop *Parent = L->getParentLoop(); // Move all of the basic blocks in the loop into the parent loop. LI->changeLoopFor(BB, Parent); // Remove the loop from the parent. if (Parent) delete Parent->removeChildLoop(std::find(Parent->begin(), Parent->end(),L)); else delete LI->removeLoop(std::find(LI->begin(), LI->end(), L)); // FIXME: Should update dominator analyses // Now that everything is up-to-date that will be, we fold the loop block into // the preheader and exit block, updating our analyses as we go. LoopExit->getInstList().splice(LoopExit->begin(), BB->getInstList(), BB->getInstList().begin(), prior(BB->getInstList().end())); LoopExit->getInstList().splice(LoopExit->begin(), Preheader->getInstList(), Preheader->getInstList().begin(), prior(Preheader->getInstList().end())); // Make all other blocks in the program branch to LoopExit now instead of // Preheader. Preheader->replaceAllUsesWith(LoopExit); // Remove BB and LoopExit from our analyses. LI->removeBlock(Preheader); LI->removeBlock(BB); // If the preheader was the entry block of this function, move the exit block // to be the new entry of the loop. Function *F = LoopExit->getParent(); if (Preheader == &F->front()) F->getBasicBlockList().splice(F->begin(), F->getBasicBlockList(), LoopExit); // Actually delete the blocks now. F->getBasicBlockList().erase(Preheader); F->getBasicBlockList().erase(BB); ++NumUnrolled; return true; }
bool LoopAnalysisManagerFunctionProxy::Result::invalidate( Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv) { // First compute the sequence of IR units covered by this proxy. We will want // to visit this in postorder, but because this is a tree structure we can do // this by building a preorder sequence and walking it in reverse. SmallVector<Loop *, 4> PreOrderLoops, PreOrderWorklist; // Note that we want to walk the roots in reverse order because we will end // up reversing the preorder sequence. However, it happens that the loop nest // roots are in reverse order within the LoopInfo object. So we just walk // forward here. // FIXME: If we change the order of LoopInfo we will want to add a reverse // here. for (Loop *RootL : *LI) { assert(PreOrderWorklist.empty() && "Must start with an empty preorder walk worklist."); PreOrderWorklist.push_back(RootL); do { Loop *L = PreOrderWorklist.pop_back_val(); PreOrderWorklist.append(L->begin(), L->end()); PreOrderLoops.push_back(L); } while (!PreOrderWorklist.empty()); } // If this proxy or the loop info is going to be invalidated, we also need // to clear all the keys coming from that analysis. We also completely blow // away the loop analyses if any of the standard analyses provided by the // loop pass manager go away so that loop analyses can freely use these // without worrying about declaring dependencies on them etc. // FIXME: It isn't clear if this is the right tradeoff. We could instead make // loop analyses declare any dependencies on these and use the more general // invalidation logic below to act on that. auto PAC = PA.getChecker<LoopAnalysisManagerFunctionProxy>(); if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) || Inv.invalidate<AAManager>(F, PA) || Inv.invalidate<AssumptionAnalysis>(F, PA) || Inv.invalidate<DominatorTreeAnalysis>(F, PA) || Inv.invalidate<LoopAnalysis>(F, PA) || Inv.invalidate<ScalarEvolutionAnalysis>(F, PA)) { // Note that the LoopInfo may be stale at this point, however the loop // objects themselves remain the only viable keys that could be in the // analysis manager's cache. So we just walk the keys and forcibly clear // those results. Note that the order doesn't matter here as this will just // directly destroy the results without calling methods on them. for (Loop *L : PreOrderLoops) InnerAM->clear(*L); // We also need to null out the inner AM so that when the object gets // destroyed as invalid we don't try to clear the inner AM again. At that // point we won't be able to reliably walk the loops for this function and // only clear results associated with those loops the way we do here. // FIXME: Making InnerAM null at this point isn't very nice. Most analyses // try to remain valid during invalidation. Maybe we should add an // `IsClean` flag? InnerAM = nullptr; // Now return true to indicate this *is* invalid and a fresh proxy result // needs to be built. This is especially important given the null InnerAM. return true; } // Directly check if the relevant set is preserved so we can short circuit // invalidating loops. bool AreLoopAnalysesPreserved = PA.allAnalysesInSetPreserved<AllAnalysesOn<Loop>>(); // Since we have a valid LoopInfo we can actually leave the cached results in // the analysis manager associated with the Loop keys, but we need to // propagate any necessary invalidation logic into them. We'd like to // invalidate things in roughly the same order as they were put into the // cache and so we walk the preorder list in reverse to form a valid // postorder. for (Loop *L : reverse(PreOrderLoops)) { Optional<PreservedAnalyses> InnerPA; // Check to see whether the preserved set needs to be adjusted based on // function-level analysis invalidation triggering deferred invalidation // for this loop. if (auto *OuterProxy = InnerAM->getCachedResult<FunctionAnalysisManagerLoopProxy>(*L)) for (const auto &OuterInvalidationPair : OuterProxy->getOuterInvalidations()) { AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first; const auto &InnerAnalysisIDs = OuterInvalidationPair.second; if (Inv.invalidate(OuterAnalysisID, F, PA)) { if (!InnerPA) InnerPA = PA; for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs) InnerPA->abandon(InnerAnalysisID); } } // Check if we needed a custom PA set. If so we'll need to run the inner // invalidation. if (InnerPA) { InnerAM->invalidate(*L, *InnerPA); continue; } // Otherwise we only need to do invalidation if the original PA set didn't // preserve all Loop analyses. if (!AreLoopAnalysesPreserved) InnerAM->invalidate(*L, PA); } // Return false to indicate that this result is still a valid proxy. return false; }
/// CloneLoop - Clone Loop. Clone dominator info. Populate ValueMap /// using old blocks to new blocks mapping. Loop *llvm::CloneLoop(Loop *OrigL, LPPassManager *LPM, LoopInfo *LI, DenseMap<const Value *, Value *> &ValueMap, Pass *P) { DominatorTree *DT = NULL; DominanceFrontier *DF = NULL; if (P) { DT = P->getAnalysisToUpdate<DominatorTree>(); DF = P->getAnalysisToUpdate<DominanceFrontier>(); } SmallVector<BasicBlock *, 16> NewBlocks; // Populate loop nest. SmallVector<Loop *, 8> LoopNest; LoopNest.push_back(OrigL); Loop *NewParentLoop = NULL; while (!LoopNest.empty()) { Loop *L = LoopNest.back(); LoopNest.pop_back(); Loop *NewLoop = new Loop(); if (!NewParentLoop) NewParentLoop = NewLoop; LPM->insertLoop(NewLoop, L->getParentLoop()); // Clone Basic Blocks. for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E; ++I) { BasicBlock *BB = *I; BasicBlock *NewBB = CloneBasicBlock(BB, ValueMap, ".clone"); ValueMap[BB] = NewBB; if (P) LPM->cloneBasicBlockSimpleAnalysis(BB, NewBB, L); NewLoop->addBasicBlockToLoop(NewBB, LI->getBase()); NewBlocks.push_back(NewBB); } // Clone dominator info. if (DT) for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E; ++I) { BasicBlock *BB = *I; CloneDominatorInfo(BB, ValueMap, DT, DF); } // Process sub loops for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) LoopNest.push_back(*I); } // Remap instructions to reference operands from ValueMap. for(SmallVector<BasicBlock *, 16>::iterator NBItr = NewBlocks.begin(), NBE = NewBlocks.end(); NBItr != NBE; ++NBItr) { BasicBlock *NB = *NBItr; for(BasicBlock::iterator BI = NB->begin(), BE = NB->end(); BI != BE; ++BI) { Instruction *Insn = BI; for (unsigned index = 0, num_ops = Insn->getNumOperands(); index != num_ops; ++index) { Value *Op = Insn->getOperand(index); DenseMap<const Value *, Value *>::iterator OpItr = ValueMap.find(Op); if (OpItr != ValueMap.end()) Insn->setOperand(index, OpItr->second); } } } BasicBlock *Latch = OrigL->getLoopLatch(); Function *F = Latch->getParent(); F->getBasicBlockList().insert(OrigL->getHeader(), NewBlocks.begin(), NewBlocks.end()); return NewParentLoop; }