/// Convert the instruction operands from referencing the current values into /// those specified by VMap. static inline void remapInstruction(Instruction *I, ValueToValueMapTy &VMap) { for (unsigned op = 0, E = I->getNumOperands(); op != E; ++op) { Value *Op = I->getOperand(op); // Unwrap arguments of dbg.value intrinsics. bool Wrapped = false; if (auto *V = dyn_cast<MetadataAsValue>(Op)) if (auto *Unwrapped = dyn_cast<ValueAsMetadata>(V->getMetadata())) { Op = Unwrapped->getValue(); Wrapped = true; } auto wrap = [&](Value *V) { auto &C = I->getContext(); return Wrapped ? MetadataAsValue::get(C, ValueAsMetadata::get(V)) : V; }; ValueToValueMapTy::iterator It = VMap.find(Op); if (It != VMap.end()) I->setOperand(op, wrap(It->second)); } if (PHINode *PN = dyn_cast<PHINode>(I)) { for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { ValueToValueMapTy::iterator It = VMap.find(PN->getIncomingBlock(i)); if (It != VMap.end()) PN->setIncomingBlock(i, cast<BasicBlock>(It->second)); } } }
/// Propagate dbg.value intrinsics through the newly inserted Phis. static void insertDebugValues(BasicBlock *OrigHeader, SmallVectorImpl<PHINode*> &InsertedPHIs) { ValueToValueMapTy DbgValueMap; // Map existing PHI nodes to their dbg.values. for (auto &I : *OrigHeader) { if (auto DbgII = dyn_cast<DbgInfoIntrinsic>(&I)) { if (auto *Loc = dyn_cast_or_null<PHINode>(DbgII->getVariableLocation())) DbgValueMap.insert({Loc, DbgII}); } } // Then iterate through the new PHIs and look to see if they use one of the // previously mapped PHIs. If so, insert a new dbg.value intrinsic that will // propagate the info through the new PHI. LLVMContext &C = OrigHeader->getContext(); for (auto PHI : InsertedPHIs) { for (auto VI : PHI->operand_values()) { auto V = DbgValueMap.find(VI); if (V != DbgValueMap.end()) { auto *DbgII = cast<DbgInfoIntrinsic>(V->second); Instruction *NewDbgII = DbgII->clone(); auto PhiMAV = MetadataAsValue::get(C, ValueAsMetadata::get(PHI)); NewDbgII->setOperand(0, PhiMAV); BasicBlock *Parent = PHI->getParent(); NewDbgII->insertBefore(Parent->getFirstNonPHIOrDbgOrLifetime()); } } } }
ValueToValueMapTy* MCJITHelper::generateIdentityMapping(Function* F) { ValueToValueMapTy* VMap = new ValueToValueMapTy(); // arguments for (Function::arg_iterator argIt = F->arg_begin(), argEnd = F->arg_end(); argIt != argEnd; ++argIt) { (*VMap)[argIt] = argIt; } // TODO: metadata (also in the body?) for (Function::iterator bbIt = F->begin(), bbEnd = F->end(); bbIt != bbEnd; ++bbIt) { // basic blocks (*VMap)[bbIt] = bbIt; // TODO: BB.hasAddressTaken() ? for (BasicBlock::iterator insIt = bbIt->begin(), insEnd = bbIt->end(); insIt != insEnd; ++insIt) { // instructions (*VMap)[insIt] = insIt; /** Code adapted from MapValue() called by RemapInstruction() **/ // operators for (User::op_iterator opIt = insIt->op_begin(), opEnd = insIt->op_end(); opIt != opEnd; ++opIt) { Value* v = *opIt; ValueToValueMapTy::iterator I = VMap->find(v); if (I != VMap->end() && I->second) continue; // value already exists in VMap (*VMap)[v] = v; } } } return VMap; }
/// UpdateCallGraphAfterInlining - Once we have cloned code over from a callee /// into the caller, update the specified callgraph to reflect the changes we /// made. Note that it's possible that not all code was copied over, so only /// some edges of the callgraph may remain. static void UpdateCallGraphAfterInlining(CallSite CS, Function::iterator FirstNewBlock, ValueToValueMapTy &VMap, InlineFunctionInfo &IFI) { CallGraph &CG = *IFI.CG; const Function *Caller = CS.getInstruction()->getParent()->getParent(); const Function *Callee = CS.getCalledFunction(); CallGraphNode *CalleeNode = CG[Callee]; CallGraphNode *CallerNode = CG[Caller]; // Since we inlined some uninlined call sites in the callee into the caller, // add edges from the caller to all of the callees of the callee. CallGraphNode::iterator I = CalleeNode->begin(), E = CalleeNode->end(); // Consider the case where CalleeNode == CallerNode. CallGraphNode::CalledFunctionsVector CallCache; if (CalleeNode == CallerNode) { CallCache.assign(I, E); I = CallCache.begin(); E = CallCache.end(); } for (; I != E; ++I) { const Value *OrigCall = I->first; ValueToValueMapTy::iterator VMI = VMap.find(OrigCall); // Only copy the edge if the call was inlined! if (VMI == VMap.end() || VMI->second == 0) continue; // If the call was inlined, but then constant folded, there is no edge to // add. Check for this case. Instruction *NewCall = dyn_cast<Instruction>(VMI->second); if (NewCall == 0) continue; // Remember that this call site got inlined for the client of // InlineFunction. IFI.InlinedCalls.push_back(NewCall); // It's possible that inlining the callsite will cause it to go from an // indirect to a direct call by resolving a function pointer. If this // happens, set the callee of the new call site to a more precise // destination. This can also happen if the call graph node of the caller // was just unnecessarily imprecise. if (I->second->getFunction() == 0) if (Function *F = CallSite(NewCall).getCalledFunction()) { // Indirect call site resolved to direct call. CallerNode->addCalledFunction(CallSite(NewCall), CG[F]); continue; } CallerNode->addCalledFunction(CallSite(NewCall), I->second); } // Update the call graph by deleting the edge from Callee to Caller. We must // do this after the loop above in case Caller and Callee are the same. CallerNode->removeCallEdgeFor(CS); }
/// RemapInstruction - Convert the instruction operands from referencing the /// current values into those specified by VMap. static inline void RemapInstruction(Instruction *I, ValueToValueMapTy &VMap) { for (unsigned op = 0, E = I->getNumOperands(); op != E; ++op) { Value *Op = I->getOperand(op); ValueToValueMapTy::iterator It = VMap.find(Op); if (It != VMap.end()) I->setOperand(op, It->second); } if (PHINode *PN = dyn_cast<PHINode>(I)) { for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { ValueToValueMapTy::iterator It = VMap.find(PN->getIncomingBlock(i)); if (It != VMap.end()) PN->setIncomingBlock(i, cast<BasicBlock>(It->second)); } } }
/// Unroll the given loop by Count. The loop must be in LCSSA form. Returns true /// if unrolling was successful, or false if the loop was unmodified. Unrolling /// can only fail when the loop's latch block is not terminated by a conditional /// branch instruction. However, if the trip count (and multiple) are not known, /// loop unrolling will mostly produce more code that is no faster. /// /// TripCount is generally defined as the number of times the loop header /// executes. UnrollLoop relaxes the definition to permit early exits: here /// TripCount is the iteration on which control exits LatchBlock if no early /// exits were taken. Note that UnrollLoop assumes that the loop counter test /// terminates LatchBlock in order to remove unnecesssary instances of the /// test. In other words, control may exit the loop prior to TripCount /// iterations via an early branch, but control may not exit the loop from the /// LatchBlock's terminator prior to TripCount iterations. /// /// Similarly, TripMultiple divides the number of times that the LatchBlock may /// execute without exiting the loop. /// /// The LoopInfo Analysis that is passed will be kept consistent. /// /// If a LoopPassManager is passed in, and the loop is fully removed, it will be /// removed from the LoopPassManager as well. LPM can also be NULL. /// /// This utility preserves LoopInfo. If DominatorTree or ScalarEvolution are /// available from the Pass it must also preserve those analyses. bool llvm::UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, bool AllowRuntime, unsigned TripMultiple, LoopInfo *LI, Pass *PP, LPPassManager *LPM) { BasicBlock *Preheader = L->getLoopPreheader(); if (!Preheader) { DEBUG(dbgs() << " Can't unroll; loop preheader-insertion failed.\n"); return false; } BasicBlock *LatchBlock = L->getLoopLatch(); if (!LatchBlock) { DEBUG(dbgs() << " Can't unroll; loop exit-block-insertion failed.\n"); return false; } // Loops with indirectbr cannot be cloned. if (!L->isSafeToClone()) { DEBUG(dbgs() << " Can't unroll; Loop body cannot be cloned.\n"); return false; } BasicBlock *Header = L->getHeader(); BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator()); if (!BI || BI->isUnconditional()) { // The loop-rotate pass can be helpful to avoid this in many cases. DEBUG(dbgs() << " Can't unroll; loop not terminated by a conditional branch.\n"); return false; } if (Header->hasAddressTaken()) { // The loop-rotate pass can be helpful to avoid this in many cases. DEBUG(dbgs() << " Won't unroll loop: address of header block is taken.\n"); return false; } if (TripCount != 0) DEBUG(dbgs() << " Trip Count = " << TripCount << "\n"); if (TripMultiple != 1) DEBUG(dbgs() << " Trip Multiple = " << TripMultiple << "\n"); // Effectively "DCE" unrolled iterations that are beyond the tripcount // and will never be executed. if (TripCount != 0 && Count > TripCount) Count = TripCount; // Don't enter the unroll code if there is nothing to do. This way we don't // need to support "partial unrolling by 1". if (TripCount == 0 && Count < 2) return false; assert(Count > 0); assert(TripMultiple > 0); assert(TripCount == 0 || TripCount % TripMultiple == 0); // Are we eliminating the loop control altogether? bool CompletelyUnroll = Count == TripCount; // We assume a run-time trip count if the compiler cannot // figure out the loop trip count and the unroll-runtime // flag is specified. bool RuntimeTripCount = (TripCount == 0 && Count > 0 && AllowRuntime); if (RuntimeTripCount && !UnrollRuntimeLoopProlog(L, Count, LI, LPM)) return false; // Notify ScalarEvolution that the loop will be substantially changed, // if not outright eliminated. if (PP) { ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>(); if (SE) SE->forgetLoop(L); } // If we know the trip count, we know the multiple... unsigned BreakoutTrip = 0; if (TripCount != 0) { BreakoutTrip = TripCount % Count; TripMultiple = 0; } else { // Figure out what multiple to use. BreakoutTrip = TripMultiple = (unsigned)GreatestCommonDivisor64(Count, TripMultiple); } // Report the unrolling decision. DebugLoc LoopLoc = L->getStartLoc(); Function *F = Header->getParent(); LLVMContext &Ctx = F->getContext(); if (CompletelyUnroll) { DEBUG(dbgs() << "COMPLETELY UNROLLING loop %" << Header->getName() << " with trip count " << TripCount << "!\n"); emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc, Twine("completely unrolled loop with ") + Twine(TripCount) + " iterations"); } else { DEBUG(dbgs() << "UNROLLING loop %" << Header->getName() << " by " << Count); Twine DiagMsg("unrolled loop by a factor of " + Twine(Count)); if (TripMultiple == 0 || BreakoutTrip != TripMultiple) { DEBUG(dbgs() << " with a breakout at trip " << BreakoutTrip); DiagMsg.concat(" with a breakout at trip " + Twine(BreakoutTrip)); } else if (TripMultiple != 1) { DEBUG(dbgs() << " with " << TripMultiple << " trips per branch"); DiagMsg.concat(" with " + Twine(TripMultiple) + " trips per branch"); } else if (RuntimeTripCount) { DEBUG(dbgs() << " with run-time trip count"); DiagMsg.concat(" with run-time trip count"); } DEBUG(dbgs() << "!\n"); emitOptimizationRemark(Ctx, DEBUG_TYPE, *F, LoopLoc, DiagMsg); } bool ContinueOnTrue = L->contains(BI->getSuccessor(0)); BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue); // For the first iteration of the loop, we should use the precloned values for // PHI nodes. Insert associations now. ValueToValueMapTy LastValueMap; std::vector<PHINode*> OrigPHINode; for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { OrigPHINode.push_back(cast<PHINode>(I)); } std::vector<BasicBlock*> Headers; std::vector<BasicBlock*> Latches; Headers.push_back(Header); Latches.push_back(LatchBlock); // The current on-the-fly SSA update requires blocks to be processed in // reverse postorder so that LastValueMap contains the correct value at each // exit. LoopBlocksDFS DFS(L); DFS.perform(LI); // Stash the DFS iterators before adding blocks to the loop. LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO(); LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO(); for (unsigned It = 1; It != Count; ++It) { std::vector<BasicBlock*> NewBlocks; for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) { ValueToValueMapTy VMap; BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It)); Header->getParent()->getBasicBlockList().push_back(New); // Loop over all of the PHI nodes in the block, changing them to use the // incoming values from the previous block. if (*BB == Header) for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) { PHINode *NewPHI = cast<PHINode>(VMap[OrigPHINode[i]]); Value *InVal = NewPHI->getIncomingValueForBlock(LatchBlock); if (Instruction *InValI = dyn_cast<Instruction>(InVal)) if (It > 1 && L->contains(InValI)) InVal = LastValueMap[InValI]; VMap[OrigPHINode[i]] = InVal; New->getInstList().erase(NewPHI); } // Update our running map of newest clones LastValueMap[*BB] = New; for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end(); VI != VE; ++VI) LastValueMap[VI->first] = VI->second; L->addBasicBlockToLoop(New, LI->getBase()); // Add phi entries for newly created values to all exit blocks. for (succ_iterator SI = succ_begin(*BB), SE = succ_end(*BB); SI != SE; ++SI) { if (L->contains(*SI)) continue; for (BasicBlock::iterator BBI = (*SI)->begin(); PHINode *phi = dyn_cast<PHINode>(BBI); ++BBI) { Value *Incoming = phi->getIncomingValueForBlock(*BB); ValueToValueMapTy::iterator It = LastValueMap.find(Incoming); if (It != LastValueMap.end()) Incoming = It->second; phi->addIncoming(Incoming, New); } } // Keep track of new headers and latches as we create them, so that // we can insert the proper branches later. if (*BB == Header) Headers.push_back(New); if (*BB == LatchBlock) Latches.push_back(New); NewBlocks.push_back(New); } // Remap all instructions in the most recent iteration for (unsigned i = 0; i < NewBlocks.size(); ++i) for (BasicBlock::iterator I = NewBlocks[i]->begin(), E = NewBlocks[i]->end(); I != E; ++I) ::RemapInstruction(I, LastValueMap); } // Loop over the PHI nodes in the original block, setting incoming values. for (unsigned i = 0, e = OrigPHINode.size(); i != e; ++i) { PHINode *PN = OrigPHINode[i]; if (CompletelyUnroll) { PN->replaceAllUsesWith(PN->getIncomingValueForBlock(Preheader)); Header->getInstList().erase(PN); } else if (Count > 1) { Value *InVal = PN->removeIncomingValue(LatchBlock, false); // If this value was defined in the loop, take the value defined by the // last iteration of the loop. if (Instruction *InValI = dyn_cast<Instruction>(InVal)) { if (L->contains(InValI)) InVal = LastValueMap[InVal]; } assert(Latches.back() == LastValueMap[LatchBlock] && "bad last latch"); PN->addIncoming(InVal, Latches.back()); } } // Now that all the basic blocks for the unrolled iterations are in place, // set up the branches to connect them. for (unsigned i = 0, e = Latches.size(); i != e; ++i) { // The original branch was replicated in each unrolled iteration. BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator()); // The branch destination. unsigned j = (i + 1) % e; BasicBlock *Dest = Headers[j]; bool NeedConditional = true; if (RuntimeTripCount && j != 0) { NeedConditional = false; } // For a complete unroll, make the last iteration end with a branch // to the exit block. if (CompletelyUnroll && j == 0) { Dest = LoopExit; NeedConditional = false; } // If we know the trip count or a multiple of it, we can safely use an // unconditional branch for some iterations. if (j != BreakoutTrip && (TripMultiple == 0 || j % TripMultiple != 0)) { NeedConditional = false; } if (NeedConditional) { // Update the conditional branch's successor for the following // iteration. Term->setSuccessor(!ContinueOnTrue, Dest); } else { // Remove phi operands at this loop exit if (Dest != LoopExit) { BasicBlock *BB = Latches[i]; for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB); SI != SE; ++SI) { if (*SI == Headers[i]) continue; for (BasicBlock::iterator BBI = (*SI)->begin(); PHINode *Phi = dyn_cast<PHINode>(BBI); ++BBI) { Phi->removeIncomingValue(BB, false); } } } // Replace the conditional branch with an unconditional one. BranchInst::Create(Dest, Term); Term->eraseFromParent(); } } // Merge adjacent basic blocks, if possible. for (unsigned i = 0, e = Latches.size(); i != e; ++i) { BranchInst *Term = cast<BranchInst>(Latches[i]->getTerminator()); if (Term->isUnconditional()) { BasicBlock *Dest = Term->getSuccessor(0); if (BasicBlock *Fold = FoldBlockIntoPredecessor(Dest, LI, LPM)) std::replace(Latches.begin(), Latches.end(), Dest, Fold); } } DominatorTree *DT = nullptr; if (PP) { // FIXME: Reconstruct dom info, because it is not preserved properly. // Incrementally updating domtree after loop unrolling would be easy. if (DominatorTreeWrapperPass *DTWP = PP->getAnalysisIfAvailable<DominatorTreeWrapperPass>()) { DT = &DTWP->getDomTree(); DT->recalculate(*L->getHeader()->getParent()); } // Simplify any new induction variables in the partially unrolled loop. ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>(); if (SE && !CompletelyUnroll) { SmallVector<WeakVH, 16> DeadInsts; simplifyLoopIVs(L, SE, LPM, DeadInsts); // Aggressively clean up dead instructions that simplifyLoopIVs already // identified. Any remaining should be cleaned up below. while (!DeadInsts.empty()) if (Instruction *Inst = dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val())) RecursivelyDeleteTriviallyDeadInstructions(Inst); } } // At this point, the code is well formed. We now do a quick sweep over the // inserted code, doing constant propagation and dead code elimination as we // go. const std::vector<BasicBlock*> &NewLoopBlocks = L->getBlocks(); for (std::vector<BasicBlock*>::const_iterator BB = NewLoopBlocks.begin(), BBE = NewLoopBlocks.end(); BB != BBE; ++BB) for (BasicBlock::iterator I = (*BB)->begin(), E = (*BB)->end(); I != E; ) { Instruction *Inst = I++; if (isInstructionTriviallyDead(Inst)) (*BB)->getInstList().erase(Inst); else if (Value *V = SimplifyInstruction(Inst)) if (LI->replacementPreservesLCSSAForm(Inst, V)) { Inst->replaceAllUsesWith(V); (*BB)->getInstList().erase(Inst); } } NumCompletelyUnrolled += CompletelyUnroll; ++NumUnrolled; Loop *OuterL = L->getParentLoop(); // Remove the loop from the LoopPassManager if it's completely removed. if (CompletelyUnroll && LPM != nullptr) LPM->deleteLoopFromQueue(L); // If we have a pass and a DominatorTree we should re-simplify impacted loops // to ensure subsequent analyses can rely on this form. We want to simplify // at least one layer outside of the loop that was unrolled so that any // changes to the parent loop exposed by the unrolling are considered. if (PP && DT) { if (!OuterL && !CompletelyUnroll) OuterL = L; if (OuterL) { ScalarEvolution *SE = PP->getAnalysisIfAvailable<ScalarEvolution>(); simplifyLoop(OuterL, DT, LI, PP, /*AliasAnalysis*/ nullptr, SE); // LCSSA must be performed on the outermost affected loop. The unrolled // loop's last loop latch is guaranteed to be in the outermost loop after // deleteLoopFromQueue updates LoopInfo. Loop *LatchLoop = LI->getLoopFor(Latches.back()); if (!OuterL->contains(LatchLoop)) while (OuterL->getParentLoop() != LatchLoop) OuterL = OuterL->getParentLoop(); formLCSSARecursively(*OuterL, *DT, SE); } } return true; }
/// UnswitchNontrivialCondition - We determined that the loop is profitable /// to unswitch when LIC equal Val. Split it into loop versions and test the /// condition outside of either loop. Return the loops created as Out1/Out2. void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val, Loop *L) { Function *F = loopHeader->getParent(); DEBUG(dbgs() << "loop-unswitch: Unswitching loop %" << loopHeader->getName() << " [" << L->getBlocks().size() << " blocks] in Function " << F->getName() << " when '" << *Val << "' == " << *LIC << "\n"); if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>()) SE->forgetLoop(L); LoopBlocks.clear(); NewBlocks.clear(); // First step, split the preheader and exit blocks, and add these blocks to // the LoopBlocks list. BasicBlock *NewPreheader = SplitEdge(loopPreheader, loopHeader, this); LoopBlocks.push_back(NewPreheader); // We want the loop to come after the preheader, but before the exit blocks. LoopBlocks.insert(LoopBlocks.end(), L->block_begin(), L->block_end()); SmallVector<BasicBlock*, 8> ExitBlocks; L->getUniqueExitBlocks(ExitBlocks); // Split all of the edges from inside the loop to their exit blocks. Update // the appropriate Phi nodes as we do so. SplitExitEdges(L, ExitBlocks); // The exit blocks may have been changed due to edge splitting, recompute. ExitBlocks.clear(); L->getUniqueExitBlocks(ExitBlocks); // Add exit blocks to the loop blocks. LoopBlocks.insert(LoopBlocks.end(), ExitBlocks.begin(), ExitBlocks.end()); // Next step, clone all of the basic blocks that make up the loop (including // the loop preheader and exit blocks), keeping track of the mapping between // the instructions and blocks. NewBlocks.reserve(LoopBlocks.size()); ValueToValueMapTy VMap; for (unsigned i = 0, e = LoopBlocks.size(); i != e; ++i) { BasicBlock *NewBB = CloneBasicBlock(LoopBlocks[i], VMap, ".us", F); NewBlocks.push_back(NewBB); VMap[LoopBlocks[i]] = NewBB; // Keep the BB mapping. LPM->cloneBasicBlockSimpleAnalysis(LoopBlocks[i], NewBB, L); } // Splice the newly inserted blocks into the function right before the // original preheader. F->getBasicBlockList().splice(NewPreheader, F->getBasicBlockList(), NewBlocks[0], F->end()); // Now we create the new Loop object for the versioned loop. Loop *NewLoop = CloneLoop(L, L->getParentLoop(), VMap, LI, LPM); Loop *ParentLoop = L->getParentLoop(); if (ParentLoop) { // Make sure to add the cloned preheader and exit blocks to the parent loop // as well. ParentLoop->addBasicBlockToLoop(NewBlocks[0], LI->getBase()); } for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) { BasicBlock *NewExit = cast<BasicBlock>(VMap[ExitBlocks[i]]); // The new exit block should be in the same loop as the old one. if (Loop *ExitBBLoop = LI->getLoopFor(ExitBlocks[i])) ExitBBLoop->addBasicBlockToLoop(NewExit, LI->getBase()); assert(NewExit->getTerminator()->getNumSuccessors() == 1 && "Exit block should have been split to have one successor!"); BasicBlock *ExitSucc = NewExit->getTerminator()->getSuccessor(0); // If the successor of the exit block had PHI nodes, add an entry for // NewExit. PHINode *PN; for (BasicBlock::iterator I = ExitSucc->begin(); isa<PHINode>(I); ++I) { PN = cast<PHINode>(I); Value *V = PN->getIncomingValueForBlock(ExitBlocks[i]); ValueToValueMapTy::iterator It = VMap.find(V); if (It != VMap.end()) V = It->second; PN->addIncoming(V, NewExit); } } // Rewrite the code to refer to itself. for (unsigned i = 0, e = NewBlocks.size(); i != e; ++i) for (BasicBlock::iterator I = NewBlocks[i]->begin(), E = NewBlocks[i]->end(); I != E; ++I) RemapInstruction(I, VMap,RF_NoModuleLevelChanges|RF_IgnoreMissingEntries); // Rewrite the original preheader to select between versions of the loop. BranchInst *OldBR = cast<BranchInst>(loopPreheader->getTerminator()); assert(OldBR->isUnconditional() && OldBR->getSuccessor(0) == LoopBlocks[0] && "Preheader splitting did not work correctly!"); // Emit the new branch that selects between the two versions of this loop. EmitPreheaderBranchOnCondition(LIC, Val, NewBlocks[0], LoopBlocks[0], OldBR); LPM->deleteSimpleAnalysisValue(OldBR, L); OldBR->eraseFromParent(); LoopProcessWorklist.push_back(NewLoop); redoLoop = true; // Keep a WeakVH holding onto LIC. If the first call to RewriteLoopBody // deletes the instruction (for example by simplifying a PHI that feeds into // the condition that we're unswitching on), we don't rewrite the second // iteration. WeakVH LICHandle(LIC); // Now we rewrite the original code to know that the condition is true and the // new code to know that the condition is false. RewriteLoopBodyWithConditionConstant(L, LIC, Val, false); // It's possible that simplifying one loop could cause the other to be // changed to another value or a constant. If its a constant, don't simplify // it. if (!LoopProcessWorklist.empty() && LoopProcessWorklist.back() == NewLoop && LICHandle && !isa<Constant>(LICHandle)) RewriteLoopBodyWithConditionConstant(NewLoop, LICHandle, Val, true); }
void SuperBlock::fixSideEntrances() { // due to merging of BBs, some superblocks may have 1 BB remaining list<map<BasicBlock*, list<BasicBlock*> >::iterator > delSuperBlocks; for (map<BasicBlock*, list<BasicBlock*> >::iterator sp = superBlocks.begin(), sp_e = superBlocks.end(); sp != sp_e; ++sp) { // we need to keep track of the predecessor of the current basic block // being checked BasicBlock* prev = sp->first; // don't clone basic blocks if the code size threshold is achieved if (currCodeSize/originalCodeSize > CODE_EXPANSION_THRESHOLD) { break; } // the first basic block for a superblock need not be duplicated for (list<BasicBlock*>::iterator bb = sp->second.begin(), bb_e = sp->second.end(); bb != bb_e; ++bb) { // first, collect all predecessors for this BB // (note: we could not just iterate through as the predecessor set may // change list<BasicBlock*> predBBs; for (pred_iterator pred = pred_begin(*bb), pred_e = pred_end(*bb); pred != pred_e; ++pred) { predBBs.push_back(*pred); } // now, walk through all predecessors of this current basic block BasicBlock* clonedBB = NULL; for (list<BasicBlock*>::iterator pred = predBBs.begin(), pred_e = predBBs.end(); pred != pred_e; ++pred) { // if it is not the predecessor of this current basic block present in // the superblock, duplicate! if (*pred != prev) { // there is no need to clone this BB multiple times if (clonedBB == NULL) { ValueToValueMapTy vmap; // clone this basic block, and place the corresponding code after // the last BB of this superblock clonedBB = CloneBasicBlock(*bb, vmap, ".cloned", (*bb)->getParent()); vmap[*bb] = clonedBB; /* errs() << "@@ BEFORE: " << *clonedBB << "\n"; // fix phi nodes in the cloned BB for (BasicBlock::iterator I = clonedBB->begin(); isa<PHINode>(I); ++I) { PHINode* PN = dyn_cast<PHINode>(I); int bbIdx = PN->getBasicBlockIndex(prev); if (bbIdx != -1) { PN->removeIncomingValue(bbIdx, false); } } */ // add size of duplicated BBs to total code size count currCodeSize += clonedBB->size(); // modify operands in this basic block for (BasicBlock::iterator instr = clonedBB->begin(), instr_e = clonedBB->end(); instr != instr_e; ++instr) { for (unsigned idx = 0, num_ops = instr->getNumOperands(); idx < num_ops; ++idx) { Value* op = instr->getOperand(idx); ValueToValueMapTy::iterator op_it = vmap.find(op); if (op_it != vmap.end()) { instr->setOperand(idx, op_it->second); } } } } // remove phi nodes into this BB in the trace /* for (BasicBlock::iterator I = (*bb)->begin(); isa<PHINode>(I); ++I) { PHINode* PN = dyn_cast<PHINode>(I); int bbIdx = PN->getBasicBlockIndex(*pred); if (bbIdx != -1) { PN->removeIncomingValue(bbIdx, false); } } */ // modify the branch instruction of the predecessor not in the superblock to // branch to the cloned basic block Instruction* br_instr = (*pred)->getTerminator(); br_instr->replaceUsesOfWith((Value*)*bb, (Value*)clonedBB); } } // determine if we can merge the BB (definitely can be merged), and its clone // with theirpredecessors if (clonedBB != NULL) { if (MergeBlockIntoPredecessor(*bb, this)) { // since we have merged this BB, delete from our superblock mappings partOfSuperBlock.erase(*bb); bb = sp->second.erase(bb); --bb; if (sp->second.empty()) { delSuperBlocks.push_back(sp); } } MergeBlockIntoPredecessor(clonedBB, this); } prev = *bb; } } // erase some superblocks (which only have 1 BB remaining) for (list<map<BasicBlock*, list<BasicBlock*> >::iterator >::iterator del = delSuperBlocks.begin(), del_e = delSuperBlocks.end(); del != del_e; ++del) { superBlocks.erase(*del); } }
void WinEHPrepare::cloneCommonBlocks(Function &F) { // We need to clone all blocks which belong to multiple funclets. Values are // remapped throughout the funclet to propagate both the new instructions // *and* the new basic blocks themselves. for (auto &Funclets : FuncletBlocks) { BasicBlock *FuncletPadBB = Funclets.first; std::vector<BasicBlock *> &BlocksInFunclet = Funclets.second; Value *FuncletToken; if (FuncletPadBB == &F.getEntryBlock()) FuncletToken = ConstantTokenNone::get(F.getContext()); else FuncletToken = FuncletPadBB->getFirstNonPHI(); std::vector<std::pair<BasicBlock *, BasicBlock *>> Orig2Clone; ValueToValueMapTy VMap; for (BasicBlock *BB : BlocksInFunclet) { ColorVector &ColorsForBB = BlockColors[BB]; // We don't need to do anything if the block is monochromatic. size_t NumColorsForBB = ColorsForBB.size(); if (NumColorsForBB == 1) continue; DEBUG_WITH_TYPE("winehprepare-coloring", dbgs() << " Cloning block \'" << BB->getName() << "\' for funclet \'" << FuncletPadBB->getName() << "\'.\n"); // Create a new basic block and copy instructions into it! BasicBlock *CBB = CloneBasicBlock(BB, VMap, Twine(".for.", FuncletPadBB->getName())); // Insert the clone immediately after the original to ensure determinism // and to keep the same relative ordering of any funclet's blocks. CBB->insertInto(&F, BB->getNextNode()); // Add basic block mapping. VMap[BB] = CBB; // Record delta operations that we need to perform to our color mappings. Orig2Clone.emplace_back(BB, CBB); } // If nothing was cloned, we're done cloning in this funclet. if (Orig2Clone.empty()) continue; // Update our color mappings to reflect that one block has lost a color and // another has gained a color. for (auto &BBMapping : Orig2Clone) { BasicBlock *OldBlock = BBMapping.first; BasicBlock *NewBlock = BBMapping.second; BlocksInFunclet.push_back(NewBlock); ColorVector &NewColors = BlockColors[NewBlock]; assert(NewColors.empty() && "A new block should only have one color!"); NewColors.push_back(FuncletPadBB); DEBUG_WITH_TYPE("winehprepare-coloring", dbgs() << " Assigned color \'" << FuncletPadBB->getName() << "\' to block \'" << NewBlock->getName() << "\'.\n"); BlocksInFunclet.erase( std::remove(BlocksInFunclet.begin(), BlocksInFunclet.end(), OldBlock), BlocksInFunclet.end()); ColorVector &OldColors = BlockColors[OldBlock]; OldColors.erase( std::remove(OldColors.begin(), OldColors.end(), FuncletPadBB), OldColors.end()); DEBUG_WITH_TYPE("winehprepare-coloring", dbgs() << " Removed color \'" << FuncletPadBB->getName() << "\' from block \'" << OldBlock->getName() << "\'.\n"); } // Loop over all of the instructions in this funclet, fixing up operand // references as we go. This uses VMap to do all the hard work. for (BasicBlock *BB : BlocksInFunclet) // Loop over all instructions, fixing each one as we find it... for (Instruction &I : *BB) RemapInstruction(&I, VMap, RF_IgnoreMissingLocals | RF_NoModuleLevelChanges); // Catchrets targeting cloned blocks need to be updated separately from // the loop above because they are not in the current funclet. SmallVector<CatchReturnInst *, 2> FixupCatchrets; for (auto &BBMapping : Orig2Clone) { BasicBlock *OldBlock = BBMapping.first; BasicBlock *NewBlock = BBMapping.second; FixupCatchrets.clear(); for (BasicBlock *Pred : predecessors(OldBlock)) if (auto *CatchRet = dyn_cast<CatchReturnInst>(Pred->getTerminator())) if (CatchRet->getCatchSwitchParentPad() == FuncletToken) FixupCatchrets.push_back(CatchRet); for (CatchReturnInst *CatchRet : FixupCatchrets) CatchRet->setSuccessor(NewBlock); } auto UpdatePHIOnClonedBlock = [&](PHINode *PN, bool IsForOldBlock) { unsigned NumPreds = PN->getNumIncomingValues(); for (unsigned PredIdx = 0, PredEnd = NumPreds; PredIdx != PredEnd; ++PredIdx) { BasicBlock *IncomingBlock = PN->getIncomingBlock(PredIdx); bool EdgeTargetsFunclet; if (auto *CRI = dyn_cast<CatchReturnInst>(IncomingBlock->getTerminator())) { EdgeTargetsFunclet = (CRI->getCatchSwitchParentPad() == FuncletToken); } else { ColorVector &IncomingColors = BlockColors[IncomingBlock]; assert(!IncomingColors.empty() && "Block not colored!"); assert((IncomingColors.size() == 1 || llvm::all_of(IncomingColors, [&](BasicBlock *Color) { return Color != FuncletPadBB; })) && "Cloning should leave this funclet's blocks monochromatic"); EdgeTargetsFunclet = (IncomingColors.front() == FuncletPadBB); } if (IsForOldBlock != EdgeTargetsFunclet) continue; PN->removeIncomingValue(IncomingBlock, /*DeletePHIIfEmpty=*/false); // Revisit the next entry. --PredIdx; --PredEnd; } }; for (auto &BBMapping : Orig2Clone) { BasicBlock *OldBlock = BBMapping.first; BasicBlock *NewBlock = BBMapping.second; for (PHINode &OldPN : OldBlock->phis()) { UpdatePHIOnClonedBlock(&OldPN, /*IsForOldBlock=*/true); } for (PHINode &NewPN : NewBlock->phis()) { UpdatePHIOnClonedBlock(&NewPN, /*IsForOldBlock=*/false); } } // Check to see if SuccBB has PHI nodes. If so, we need to add entries to // the PHI nodes for NewBB now. for (auto &BBMapping : Orig2Clone) { BasicBlock *OldBlock = BBMapping.first; BasicBlock *NewBlock = BBMapping.second; for (BasicBlock *SuccBB : successors(NewBlock)) { for (PHINode &SuccPN : SuccBB->phis()) { // Ok, we have a PHI node. Figure out what the incoming value was for // the OldBlock. int OldBlockIdx = SuccPN.getBasicBlockIndex(OldBlock); if (OldBlockIdx == -1) break; Value *IV = SuccPN.getIncomingValue(OldBlockIdx); // Remap the value if necessary. if (auto *Inst = dyn_cast<Instruction>(IV)) { ValueToValueMapTy::iterator I = VMap.find(Inst); if (I != VMap.end()) IV = I->second; } SuccPN.addIncoming(IV, NewBlock); } } } for (ValueToValueMapTy::value_type VT : VMap) { // If there were values defined in BB that are used outside the funclet, // then we now have to update all uses of the value to use either the // original value, the cloned value, or some PHI derived value. This can // require arbitrary PHI insertion, of which we are prepared to do, clean // these up now. SmallVector<Use *, 16> UsesToRename; auto *OldI = dyn_cast<Instruction>(const_cast<Value *>(VT.first)); if (!OldI) continue; auto *NewI = cast<Instruction>(VT.second); // Scan all uses of this instruction to see if it is used outside of its // funclet, and if so, record them in UsesToRename. for (Use &U : OldI->uses()) { Instruction *UserI = cast<Instruction>(U.getUser()); BasicBlock *UserBB = UserI->getParent(); ColorVector &ColorsForUserBB = BlockColors[UserBB]; assert(!ColorsForUserBB.empty()); if (ColorsForUserBB.size() > 1 || *ColorsForUserBB.begin() != FuncletPadBB) UsesToRename.push_back(&U); } // If there are no uses outside the block, we're done with this // instruction. if (UsesToRename.empty()) continue; // We found a use of OldI outside of the funclet. Rename all uses of OldI // that are outside its funclet to be uses of the appropriate PHI node // etc. SSAUpdater SSAUpdate; SSAUpdate.Initialize(OldI->getType(), OldI->getName()); SSAUpdate.AddAvailableValue(OldI->getParent(), OldI); SSAUpdate.AddAvailableValue(NewI->getParent(), NewI); while (!UsesToRename.empty()) SSAUpdate.RewriteUseAfterInsertions(*UsesToRename.pop_back_val()); } } }
void WorklessInstrument::CloneInnerLoop(Loop * pLoop, vector<BasicBlock *> & vecAdd, ValueToValueMapTy & VMap, set<BasicBlock *> & setCloned) { Function * pFunction = pLoop->getHeader()->getParent(); BasicBlock * pPreHeader = vecAdd[0]; SmallVector<BasicBlock *, 4> ExitBlocks; pLoop->getExitBlocks(ExitBlocks); set<BasicBlock *> setExitBlocks; for(unsigned long i = 0; i < ExitBlocks.size(); i++) { setExitBlocks.insert(ExitBlocks[i]); } for(unsigned long i = 0; i < ExitBlocks.size(); i++ ) { VMap[ExitBlocks[i]] = ExitBlocks[i]; } vector<BasicBlock *> ToClone; vector<BasicBlock *> BeenCloned; //clone loop ToClone.push_back(pLoop->getHeader()); while(ToClone.size()>0) { BasicBlock * pCurrent = ToClone.back(); ToClone.pop_back(); WeakVH & BBEntry = VMap[pCurrent]; if (BBEntry) { continue; } BasicBlock * NewBB; BBEntry = NewBB = BasicBlock::Create(pCurrent->getContext(), "", pFunction); if(pCurrent->hasName()) { NewBB->setName(pCurrent->getName() + ".CPI"); } if(pCurrent->hasAddressTaken()) { errs() << "hasAddressTaken branch\n" ; exit(0); } for(BasicBlock::const_iterator II = pCurrent->begin(); II != pCurrent->end(); ++II ) { Instruction * NewInst = II->clone(); if(II->hasName()) { NewInst->setName(II->getName() + ".CPI"); } VMap[II] = NewInst; NewBB->getInstList().push_back(NewInst); } const TerminatorInst *TI = pCurrent->getTerminator(); for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) { ToClone.push_back(TI->getSuccessor(i)); } setCloned.insert(NewBB); BeenCloned.push_back(NewBB); } //remap value used inside loop vector<BasicBlock *>::iterator itVecBegin = BeenCloned.begin(); vector<BasicBlock *>::iterator itVecEnd = BeenCloned.end(); for(; itVecBegin != itVecEnd; itVecBegin ++) { for(BasicBlock::iterator II = (*itVecBegin)->begin(); II != (*itVecBegin)->end(); II ++ ) { //II->dump(); RemapInstruction(II, VMap); } } //add to the else if body BasicBlock * pElseBody = vecAdd[1]; BasicBlock * pClonedHeader = cast<BasicBlock>(VMap[pLoop->getHeader()]); BranchInst::Create(pClonedHeader, pElseBody); //errs() << pPreHeader->getName() << "\n"; for(BasicBlock::iterator II = pClonedHeader->begin(); II != pClonedHeader->end(); II ++ ) { if(PHINode * pPHI = dyn_cast<PHINode>(II)) { vector<int> vecToRemoved; for (unsigned i = 0, e = pPHI->getNumIncomingValues(); i != e; ++i) { if(pPHI->getIncomingBlock(i) == pPreHeader) { pPHI->setIncomingBlock(i, pElseBody); } } } } set<BasicBlock *> setProcessedBlock; for(unsigned long i = 0; i < ExitBlocks.size(); i++ ) { if(setProcessedBlock.find(ExitBlocks[i]) != setProcessedBlock.end() ) { continue; } else { setProcessedBlock.insert(ExitBlocks[i]); } for(BasicBlock::iterator II = ExitBlocks[i]->begin(); II != ExitBlocks[i]->end(); II ++ ) { if(PHINode * pPHI = dyn_cast<PHINode>(II)) { unsigned numIncomming = pPHI->getNumIncomingValues(); for(unsigned i = 0; i<numIncomming; i++) { BasicBlock * incommingBlock = pPHI->getIncomingBlock(i); if(VMap.find(incommingBlock) != VMap.end() ) { Value * incommingValue = pPHI->getIncomingValue(i); if(VMap.find(incommingValue) != VMap.end() ) { incommingValue = VMap[incommingValue]; } pPHI->addIncoming(incommingValue, cast<BasicBlock>(VMap[incommingBlock])); } } } } } }