bool searchPredecessors(const MachineBasicBlock *MBB, const MachineBasicBlock *CutOff, UnaryPredicate Predicate) { if (MBB == CutOff) return false; DenseSet<const MachineBasicBlock*> Visited; SmallVector<MachineBasicBlock*, 4> Worklist(MBB->pred_begin(), MBB->pred_end()); while (!Worklist.empty()) { MachineBasicBlock *MBB = Worklist.pop_back_val(); if (!Visited.insert(MBB).second) continue; if (MBB == CutOff) continue; if (Predicate(MBB)) return true; Worklist.append(MBB->pred_begin(), MBB->pred_end()); } return false; }
void SplitEditor::extendPHIKillRanges() { // Extend live ranges to be live-out for successor PHI values. for (LiveInterval::const_vni_iterator I = Edit->getParent().vni_begin(), E = Edit->getParent().vni_end(); I != E; ++I) { const VNInfo *PHIVNI = *I; if (PHIVNI->isUnused() || !PHIVNI->isPHIDef()) continue; unsigned RegIdx = RegAssign.lookup(PHIVNI->def); LiveInterval *LI = Edit->get(RegIdx); LiveRangeCalc &LRC = getLRCalc(RegIdx); MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { SlotIndex End = LIS.getMBBEndIdx(*PI); SlotIndex LastUse = End.getPrevSlot(); // The predecessor may not have a live-out value. That is OK, like an // undef PHI operand. if (Edit->getParent().liveAt(LastUse)) { assert(RegAssign.lookup(LastUse) == RegIdx && "Different register assignment in phi predecessor"); LRC.extend(LI, End, LIS.getSlotIndexes(), &MDT, &LIS.getVNInfoAllocator()); } } } }
/// markValueUsed - Remember that VNI failed to rematerialize, so its defining /// instruction cannot be eliminated. See through snippet copies void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { SmallVector<std::pair<LiveInterval*, VNInfo*>, 8> WorkList; WorkList.push_back(std::make_pair(LI, VNI)); do { tie(LI, VNI) = WorkList.pop_back_val(); if (!UsedValues.insert(VNI)) continue; if (VNI->isPHIDef()) { MachineBasicBlock *MBB = LIS.getMBBFromIndex(VNI->def); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { VNInfo *PVNI = LI->getVNInfoBefore(LIS.getMBBEndIdx(*PI)); if (PVNI) WorkList.push_back(std::make_pair(LI, PVNI)); } continue; } // Follow snippet copies. MachineInstr *MI = LIS.getInstructionFromIndex(VNI->def); if (!SnippetCopies.count(MI)) continue; LiveInterval &SnipLI = LIS.getInterval(MI->getOperand(1).getReg()); assert(isRegToSpill(SnipLI.reg) && "Unexpected register in copy"); VNInfo *SnipVNI = SnipLI.getVNInfoAt(VNI->def.getRegSlot(true)); assert(SnipVNI && "Snippet undefined before copy"); WorkList.push_back(std::make_pair(&SnipLI, SnipVNI)); } while (!WorkList.empty()); }
static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) { MachineBasicBlock *MBB = I; SmallSetVector<MachineBasicBlock*, 8> Preds(MBB->pred_begin(), MBB->pred_end()); MachineBasicBlock::iterator MI = MBB->begin(); while (MI != MBB->end()) { if (!MI->isPHI()) break; for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), PE = Preds.end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; bool Found = false; for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) { MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB(); if (PHIBB == PredBB) { Found = true; break; } } if (!Found) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " missing input from predecessor BB#" << PredBB->getNumber() << '\n'; llvm_unreachable(0); } } for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) { MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB(); if (CheckExtra && !Preds.count(PHIBB)) { dbgs() << "Warning: malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " extra input from predecessor BB#" << PHIBB->getNumber() << '\n'; llvm_unreachable(0); } if (PHIBB->getNumber() < 0) { dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n'; llvm_unreachable(0); } } ++MI; } } }
bool Filler::searchSuccBBs(MachineBasicBlock &MBB, Iter Slot) const { if (DisableSuccBBSearch) return false; MachineBasicBlock *SuccBB = selectSuccBB(MBB); if (!SuccBB) return false; RegDefsUses RegDU(*MBB.getParent()->getSubtarget().getRegisterInfo()); bool HasMultipleSuccs = false; BB2BrMap BrMap; std::unique_ptr<InspectMemInstr> IM; Iter Filler; auto *Fn = MBB.getParent(); // Iterate over SuccBB's predecessor list. for (MachineBasicBlock::pred_iterator PI = SuccBB->pred_begin(), PE = SuccBB->pred_end(); PI != PE; ++PI) if (!examinePred(**PI, *SuccBB, RegDU, HasMultipleSuccs, BrMap)) return false; // Do not allow moving instructions which have unallocatable register operands // across basic block boundaries. RegDU.setUnallocatableRegs(*Fn); // Only allow moving loads from stack or constants if any of the SuccBB's // predecessors have multiple successors. if (HasMultipleSuccs) { IM.reset(new LoadFromStackOrConst()); } else { const MachineFrameInfo *MFI = Fn->getFrameInfo(); IM.reset(new MemDefsUses(Fn->getDataLayout(), MFI)); } if (!searchRange(MBB, SuccBB->begin(), SuccBB->end(), RegDU, *IM, Slot, Filler)) return false; insertDelayFiller(Filler, BrMap); addLiveInRegs(Filler, *SuccBB); Filler->eraseFromParent(); return true; }
/// getCanonicalInductionVariable - Check to see if the loop has a canonical /// induction variable. We check for a simple recurrence pattern - an /// integer recurrence that decrements by one each time through the loop and /// ends at zero. If so, return the phi node that corresponds to it. /// /// Based upon the similar code in LoopInfo except this code is specific to /// the machine. /// This method assumes that the IndVarSimplify pass has been run by 'opt'. /// void PPCCTRLoops::getCanonicalInductionVariable(MachineLoop *L, SmallVector<MachineInstr *, 4> &IVars, SmallVector<MachineInstr *, 4> &IOps) const { MachineBasicBlock *TopMBB = L->getTopBlock(); MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(); assert(PI != TopMBB->pred_end() && "Loop must have more than one incoming edge!"); MachineBasicBlock *Backedge = *PI++; if (PI == TopMBB->pred_end()) return; // dead loop MachineBasicBlock *Incoming = *PI++; if (PI != TopMBB->pred_end()) return; // multiple backedges? // make sure there is one incoming and one backedge and determine which // is which. if (L->contains(Incoming)) { if (L->contains(Backedge)) return; std::swap(Incoming, Backedge); } else if (!L->contains(Backedge)) return; // Loop over all of the PHI nodes, looking for a canonical induction variable: // - The PHI node is "reg1 = PHI reg2, BB1, reg3, BB2". // - The recurrence comes from the backedge. // - the definition is an induction operatio.n for (MachineBasicBlock::iterator I = TopMBB->begin(), E = TopMBB->end(); I != E && I->isPHI(); ++I) { MachineInstr *MPhi = &*I; unsigned DefReg = MPhi->getOperand(0).getReg(); for (unsigned i = 1; i != MPhi->getNumOperands(); i += 2) { // Check each operand for the value from the backedge. MachineBasicBlock *MBB = MPhi->getOperand(i+1).getMBB(); if (L->contains(MBB)) { // operands comes from the backedge // Check if the definition is an induction operation. MachineInstr *DI = MRI->getVRegDef(MPhi->getOperand(i).getReg()); if (isInductionOperation(DI, DefReg)) { IOps.push_back(DI); IVars.push_back(MPhi); } } } } return; }
bool TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), PE = BB.pred_end(); PI != PE; ++PI) { MachineBasicBlock *PredBB = *PI; if (PredBB->succ_size() > 1) return false; MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; SmallVector<MachineOperand, 4> PredCond; if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) return false; if (!PredCond.empty()) return false; } return true; }
bool LembergInstrInfo:: ComplexPredecessorPredicates(MachineBasicBlock &MBB) const { for (MachineBasicBlock::pred_iterator PI = MBB.pred_begin(), PE = MBB.pred_end(); PI != PE; ++PI) { MachineBasicBlock *TBB = NULL, *FBB = NULL; SmallVector<MachineOperand, 4> Cond; AnalyzeBranch(**PI, TBB, FBB, Cond, false); if (!Cond.empty() && !(Cond[1].getImm() == LembergCC::FALSE || Cond[1].getImm() == LembergCC::TRUE)) { return true; } } return false; }
bool LoopSplitter::canInsertPreHeader(MachineLoop &loop) { MachineBasicBlock *header = loop.getHeader(); MachineBasicBlock *a = 0, *b = 0; SmallVector<MachineOperand, 4> c; for (MachineBasicBlock::pred_iterator pbItr = header->pred_begin(), pbEnd = header->pred_end(); pbItr != pbEnd; ++pbItr) { MachineBasicBlock *predBlock = *pbItr; if (!!tii->AnalyzeBranch(*predBlock, a, b, c)) { return false; } } MachineFunction::iterator headerItr(header); if (headerItr == mf->begin()) return true; MachineBasicBlock *headerLayoutPred = llvm::prior(headerItr); assert(headerLayoutPred != 0 && "Header should have layout pred."); return (!tii->AnalyzeBranch(*headerLayoutPred, a, b, c)); }
bool Thumb2SizeReduce::ReduceMBB(MachineBasicBlock &MBB) { bool Modified = false; // Yes, CPSR could be livein. bool LiveCPSR = MBB.isLiveIn(ARM::CPSR); MachineInstr *BundleMI = 0; CPSRDef = 0; HighLatencyCPSR = false; // Check predecessors for the latest CPSRDef. for (MachineBasicBlock::pred_iterator I = MBB.pred_begin(), E = MBB.pred_end(); I != E; ++I) { const MBBInfo &PInfo = BlockInfo[(*I)->getNumber()]; if (!PInfo.Visited) { // Since blocks are visited in RPO, this must be a back-edge. continue; } if (PInfo.HighLatencyCPSR) { HighLatencyCPSR = true; break; } } // If this BB loops back to itself, conservatively avoid narrowing the // first instruction that does partial flag update. bool IsSelfLoop = MBB.isSuccessor(&MBB); MachineBasicBlock::instr_iterator MII = MBB.instr_begin(),E = MBB.instr_end(); MachineBasicBlock::instr_iterator NextMII; for (; MII != E; MII = NextMII) { NextMII = llvm::next(MII); MachineInstr *MI = &*MII; if (MI->isBundle()) { BundleMI = MI; continue; } if (MI->isDebugValue()) continue; LiveCPSR = UpdateCPSRUse(*MI, LiveCPSR); // Does NextMII belong to the same bundle as MI? bool NextInSameBundle = NextMII != E && NextMII->isBundledWithPred(); if (ReduceMI(MBB, MI, LiveCPSR, IsSelfLoop)) { Modified = true; MachineBasicBlock::instr_iterator I = prior(NextMII); MI = &*I; // Removing and reinserting the first instruction in a bundle will break // up the bundle. Fix the bundling if it was broken. if (NextInSameBundle && !NextMII->isBundledWithPred()) NextMII->bundleWithPred(); } if (!NextInSameBundle && MI->isInsideBundle()) { // FIXME: Since post-ra scheduler operates on bundles, the CPSR kill // marker is only on the BUNDLE instruction. Process the BUNDLE // instruction as we finish with the bundled instruction to work around // the inconsistency. if (BundleMI->killsRegister(ARM::CPSR)) LiveCPSR = false; MachineOperand *MO = BundleMI->findRegisterDefOperand(ARM::CPSR); if (MO && !MO->isDead()) LiveCPSR = true; } bool DefCPSR = false; LiveCPSR = UpdateCPSRDef(*MI, LiveCPSR, DefCPSR); if (MI->isCall()) { // Calls don't really set CPSR. CPSRDef = 0; HighLatencyCPSR = false; IsSelfLoop = false; } else if (DefCPSR) { // This is the last CPSR defining instruction. CPSRDef = MI; HighLatencyCPSR = isHighLatencyCPSR(CPSRDef); IsSelfLoop = false; } } MBBInfo &Info = BlockInfo[MBB.getNumber()]; Info.HighLatencyCPSR = HighLatencyCPSR; Info.Visited = true; return Modified; }
// This is essentially the same iterative algorithm that SSAUpdater uses, // except we already have a dominator tree, so we don't have to recompute it. void LiveRangeCalc::updateSSA() { assert(Indexes && "Missing SlotIndexes"); assert(DomTree && "Missing dominator tree"); // Interate until convergence. unsigned Changes; do { Changes = 0; // Propagate live-out values down the dominator tree, inserting phi-defs // when necessary. for (SmallVectorImpl<LiveInBlock>::iterator I = LiveIn.begin(), E = LiveIn.end(); I != E; ++I) { MachineDomTreeNode *Node = I->DomNode; // Skip block if the live-in value has already been determined. if (!Node) continue; MachineBasicBlock *MBB = Node->getBlock(); MachineDomTreeNode *IDom = Node->getIDom(); LiveOutPair IDomValue; // We need a live-in value to a block with no immediate dominator? // This is probably an unreachable block that has survived somehow. bool needPHI = !IDom || !Seen.test(IDom->getBlock()->getNumber()); // IDom dominates all of our predecessors, but it may not be their // immediate dominator. Check if any of them have live-out values that are // properly dominated by IDom. If so, we need a phi-def here. if (!needPHI) { IDomValue = LiveOut[IDom->getBlock()]; // Cache the DomTree node that defined the value. if (IDomValue.first && !IDomValue.second) LiveOut[IDom->getBlock()].second = IDomValue.second = DomTree->getNode(Indexes->getMBBFromIndex(IDomValue.first->def)); for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { LiveOutPair &Value = LiveOut[*PI]; if (!Value.first || Value.first == IDomValue.first) continue; // Cache the DomTree node that defined the value. if (!Value.second) Value.second = DomTree->getNode(Indexes->getMBBFromIndex(Value.first->def)); // This predecessor is carrying something other than IDomValue. // It could be because IDomValue hasn't propagated yet, or it could be // because MBB is in the dominance frontier of that value. if (DomTree->dominates(IDom, Value.second)) { needPHI = true; break; } } } // The value may be live-through even if Kill is set, as can happen when // we are called from extendRange. In that case LiveOutSeen is true, and // LiveOut indicates a foreign or missing value. LiveOutPair &LOP = LiveOut[MBB]; // Create a phi-def if required. if (needPHI) { ++Changes; assert(Alloc && "Need VNInfo allocator to create PHI-defs"); SlotIndex Start, End; tie(Start, End) = Indexes->getMBBRange(MBB); VNInfo *VNI = I->LI->getNextValue(Start, *Alloc); I->Value = VNI; // This block is done, we know the final value. I->DomNode = 0; // Add liveness since updateLiveIns now skips this node. if (I->Kill.isValid()) I->LI->addRange(LiveRange(Start, I->Kill, VNI)); else { I->LI->addRange(LiveRange(Start, End, VNI)); LOP = LiveOutPair(VNI, Node); } } else if (IDomValue.first) { // No phi-def here. Remember incoming value. I->Value = IDomValue.first; // If the IDomValue is killed in the block, don't propagate through. if (I->Kill.isValid()) continue; // Propagate IDomValue if it isn't killed: // MBB is live-out and doesn't define its own value. if (LOP.first == IDomValue.first) continue; ++Changes; LOP = IDomValue; } } } while (Changes); }
VNInfo *LiveRangeCalc::findReachingDefs(LiveInterval *LI, MachineBasicBlock *KillMBB, SlotIndex Kill, unsigned PhysReg) { // Blocks where LI should be live-in. SmallVector<MachineBasicBlock*, 16> WorkList(1, KillMBB); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = 0; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = WorkList[i]; #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); errs() << "The register needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = LiveOut[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LI->extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; continue; } // No, we need a live-in value for Pred as well if (Pred != KillMBB) WorkList.push_back(Pred); else // Loopback to KillMBB, so value is really live through. Kill = SlotIndex(); } } // Transfer WorkList to LiveInBlocks in reverse order. // This ordering works best with updateSSA(). LiveIn.clear(); LiveIn.reserve(WorkList.size()); while(!WorkList.empty()) addLiveInBlock(LI, DomTree->getNode(WorkList.pop_back_val())); // The kill block may not be live-through. assert(LiveIn.back().DomNode->getBlock() == KillMBB); LiveIn.back().Kill = Kill; return UniqueVNI ? TheVNI : 0; }
void StackColoring::calculateLocalLiveness() { // Perform a standard reverse dataflow computation to solve for // global liveness. The BEGIN set here is equivalent to KILL in the standard // formulation, and END is equivalent to GEN. The result of this computation // is a map from blocks to bitvectors where the bitvectors represent which // allocas are live in/out of that block. SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(), BasicBlockNumbering.end()); unsigned NumSSMIters = 0; bool changed = true; while (changed) { changed = false; ++NumSSMIters; SmallPtrSet<MachineBasicBlock*, 8> NextBBSet; for (SmallVector<MachineBasicBlock*, 8>::iterator PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end(); PI != PE; ++PI) { MachineBasicBlock *BB = *PI; if (!BBSet.count(BB)) continue; BitVector LocalLiveIn; BitVector LocalLiveOut; // Forward propagation from begins to ends. for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), PE = BB->pred_end(); PI != PE; ++PI) LocalLiveIn |= BlockLiveness[*PI].LiveOut; LocalLiveIn |= BlockLiveness[BB].End; LocalLiveIn.reset(BlockLiveness[BB].Begin); // Reverse propagation from ends to begins. for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) LocalLiveOut |= BlockLiveness[*SI].LiveIn; LocalLiveOut |= BlockLiveness[BB].Begin; LocalLiveOut.reset(BlockLiveness[BB].End); LocalLiveIn |= LocalLiveOut; LocalLiveOut |= LocalLiveIn; // After adopting the live bits, we need to turn-off the bits which // are de-activated in this block. LocalLiveOut.reset(BlockLiveness[BB].End); LocalLiveIn.reset(BlockLiveness[BB].Begin); // If we have both BEGIN and END markers in the same basic block then // we know that the BEGIN marker comes after the END, because we already // handle the case where the BEGIN comes before the END when collecting // the markers (and building the BEGIN/END vectore). // Want to enable the LIVE_IN and LIVE_OUT of slots that have both // BEGIN and END because it means that the value lives before and after // this basic block. BitVector LocalEndBegin = BlockLiveness[BB].End; LocalEndBegin &= BlockLiveness[BB].Begin; LocalLiveIn |= LocalEndBegin; LocalLiveOut |= LocalEndBegin; if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) { changed = true; BlockLiveness[BB].LiveIn |= LocalLiveIn; for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(), PE = BB->pred_end(); PI != PE; ++PI) NextBBSet.insert(*PI); } if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) { changed = true; BlockLiveness[BB].LiveOut |= LocalLiveOut; for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) NextBBSet.insert(*SI); } } BBSet = NextBBSet; }// while changed. }
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) { df_iterator_default_set<MachineBasicBlock*> Reachable; bool ModifiedPHI = false; MMI = getAnalysisIfAvailable<MachineModuleInfo>(); MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>(); MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); // Mark all reachable blocks. for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable)) (void)BB/* Mark all reachable blocks */; // Loop over all dead blocks, remembering them and deleting all instructions // in them. std::vector<MachineBasicBlock*> DeadBlocks; for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = &*I; // Test for deadness. if (!Reachable.count(BB)) { DeadBlocks.push_back(BB); // Update dominator and loop info. if (MLI) MLI->removeBlock(BB); if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB); while (BB->succ_begin() != BB->succ_end()) { MachineBasicBlock* succ = *BB->succ_begin(); MachineBasicBlock::iterator start = succ->begin(); while (start != succ->end() && start->isPHI()) { for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2) if (start->getOperand(i).isMBB() && start->getOperand(i).getMBB() == BB) { start->RemoveOperand(i); start->RemoveOperand(i-1); } start++; } BB->removeSuccessor(BB->succ_begin()); } } } // Actually remove the blocks now. for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) DeadBlocks[i]->eraseFromParent(); // Cleanup PHI nodes. for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = &*I; // Prune unneeded PHI entries. SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(), BB->pred_end()); MachineBasicBlock::iterator phi = BB->begin(); while (phi != BB->end() && phi->isPHI()) { for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2) if (!preds.count(phi->getOperand(i).getMBB())) { phi->RemoveOperand(i); phi->RemoveOperand(i-1); ModifiedPHI = true; } if (phi->getNumOperands() == 3) { const MachineOperand &Input = phi->getOperand(1); const MachineOperand &Output = phi->getOperand(0); unsigned InputReg = Input.getReg(); unsigned OutputReg = Output.getReg(); assert(Output.getSubReg() == 0 && "Cannot have output subregister"); ModifiedPHI = true; if (InputReg != OutputReg) { MachineRegisterInfo &MRI = F.getRegInfo(); unsigned InputSub = Input.getSubReg(); if (InputSub == 0 && MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) { MRI.replaceRegWith(OutputReg, InputReg); } else { // The input register to the PHI has a subregister or it can't be // constrained to the proper register class: // insert a COPY instead of simply replacing the output // with the input. const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo(); BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(), TII->get(TargetOpcode::COPY), OutputReg) .addReg(InputReg, getRegState(Input), InputSub); } phi++->eraseFromParent(); } continue; } ++phi; } } F.RenumberBlocks(); return (!DeadBlocks.empty() || ModifiedPHI); }
bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB, MachineFunction &MF, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII) { SmallPtrSet<MachineBasicBlock *, 2> SinkableBBs; // FIXME: For now, we sink only to a successor which has a single predecessor // so that we can directly sink COPY instructions to the successor without // adding any new block or branch instruction. for (MachineBasicBlock *SI : CurBB.successors()) if (!SI->livein_empty() && SI->pred_size() == 1) SinkableBBs.insert(SI); if (SinkableBBs.empty()) return false; bool Changed = false; // Track which registers have been modified and used between the end of the // block and the current instruction. ModifiedRegUnits.clear(); UsedRegUnits.clear(); for (auto I = CurBB.rbegin(), E = CurBB.rend(); I != E;) { MachineInstr *MI = &*I; ++I; if (MI->isDebugInstr()) continue; // Do not move any instruction across function call. if (MI->isCall()) return false; if (!MI->isCopy() || !MI->getOperand(0).isRenamable()) { LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } // Track the operand index for use in Copy. SmallVector<unsigned, 2> UsedOpsInCopy; // Track the register number defed in Copy. SmallVector<unsigned, 2> DefedRegsInCopy; // Don't sink the COPY if it would violate a register dependency. if (hasRegisterDependency(MI, UsedOpsInCopy, DefedRegsInCopy, ModifiedRegUnits, UsedRegUnits)) { LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } assert((!UsedOpsInCopy.empty() && !DefedRegsInCopy.empty()) && "Unexpect SrcReg or DefReg"); MachineBasicBlock *SuccBB = getSingleLiveInSuccBB(CurBB, SinkableBBs, DefedRegsInCopy, TRI); // Don't sink if we cannot find a single sinkable successor in which Reg // is live-in. if (!SuccBB) { LiveRegUnits::accumulateUsedDefed(*MI, ModifiedRegUnits, UsedRegUnits, TRI); continue; } assert((SuccBB->pred_size() == 1 && *SuccBB->pred_begin() == &CurBB) && "Unexpected predecessor"); // Clear the kill flag if SrcReg is killed between MI and the end of the // block. clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegUnits, TRI); MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI(); performSink(*MI, *SuccBB, InsertPos); updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy); Changed = true; ++NumPostRACopySink; } return Changed; }
/// EliminateUnconditionalJumpsToTop - Move blocks which unconditionally jump /// to the loop top to the top of the loop so that they have a fall through. /// This can introduce a branch on entry to the loop, but it can eliminate a /// branch within the loop. See the @simple case in /// test/CodeGen/X86/loop_blocks.ll for an example of this. bool CodePlacementOpt::EliminateUnconditionalJumpsToTop(MachineFunction &MF, MachineLoop *L) { bool Changed = false; MachineBasicBlock *TopMBB = L->getTopBlock(); bool BotHasFallthrough = HasFallthrough(L->getBottomBlock()); if (TopMBB == MF.begin() || HasAnalyzableTerminator(prior(MachineFunction::iterator(TopMBB)))) { new_top: for (MachineBasicBlock::pred_iterator PI = TopMBB->pred_begin(), PE = TopMBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; if (Pred == TopMBB) continue; if (HasFallthrough(Pred)) continue; if (!L->contains(Pred)) continue; // Verify that we can analyze all the loop entry edges before beginning // any changes which will require us to be able to analyze them. if (Pred == MF.begin()) continue; if (!HasAnalyzableTerminator(Pred)) continue; if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Pred)))) continue; // Move the block. DEBUG(dbgs() << "CGP: Moving blocks starting at BB#" << Pred->getNumber() << " to top of loop.\n"); Changed = true; // Move it and all the blocks that can reach it via fallthrough edges // exclusively, to keep existing fallthrough edges intact. MachineFunction::iterator Begin = Pred; MachineFunction::iterator End = llvm::next(Begin); while (Begin != MF.begin()) { MachineFunction::iterator Prior = prior(Begin); if (Prior == MF.begin()) break; // Stop when a non-fallthrough edge is found. if (!HasFallthrough(Prior)) break; // Stop if a block which could fall-through out of the loop is found. if (Prior->isSuccessor(End)) break; // If we've reached the top, stop scanning. if (Prior == MachineFunction::iterator(TopMBB)) { // We know top currently has a fall through (because we just checked // it) which would be lost if we do the transformation, so it isn't // worthwhile to do the transformation unless it would expose a new // fallthrough edge. if (!Prior->isSuccessor(End)) goto next_pred; // Otherwise we can stop scanning and procede to move the blocks. break; } // If we hit a switch or something complicated, don't move anything // for this predecessor. if (!HasAnalyzableTerminator(prior(MachineFunction::iterator(Prior)))) break; // Ok, the block prior to Begin will be moved along with the rest. // Extend the range to include it. Begin = Prior; ++NumIntraMoved; } // Move the blocks. Splice(MF, TopMBB, Begin, End); // Update TopMBB. TopMBB = L->getTopBlock(); // We have a new loop top. Iterate on it. We shouldn't have to do this // too many times if BranchFolding has done a reasonable job. goto new_top; next_pred:; } } // If the loop previously didn't exit with a fall-through and it now does, // we eliminated a branch. if (Changed && !BotHasFallthrough && HasFallthrough(L->getBottomBlock())) { ++NumIntraElim; } return Changed; }
MachineBasicBlock& LoopSplitter::insertPreHeader(MachineLoop &loop) { assert(loop.getLoopPreheader() == 0 && "Loop already has preheader."); MachineBasicBlock &header = *loop.getHeader(); // Save the preds - we'll need to update them once we insert the preheader. typedef std::set<MachineBasicBlock*> HeaderPreds; HeaderPreds headerPreds; for (MachineBasicBlock::pred_iterator predItr = header.pred_begin(), predEnd = header.pred_end(); predItr != predEnd; ++predItr) { if (!loop.contains(*predItr)) headerPreds.insert(*predItr); } assert(!headerPreds.empty() && "No predecessors for header?"); //dbgs() << fqn << " MBB#" << header.getNumber() << " inserting preheader..."; MachineBasicBlock *preHeader = mf->CreateMachineBasicBlock(header.getBasicBlock()); assert(preHeader != 0 && "Failed to create pre-header."); mf->insert(header, preHeader); for (HeaderPreds::iterator hpItr = headerPreds.begin(), hpEnd = headerPreds.end(); hpItr != hpEnd; ++hpItr) { assert(*hpItr != 0 && "How'd a null predecessor get into this set?"); MachineBasicBlock &hp = **hpItr; hp.ReplaceUsesOfBlockWith(&header, preHeader); } preHeader->addSuccessor(&header); MachineBasicBlock *oldLayoutPred = llvm::prior(MachineFunction::iterator(preHeader)); if (oldLayoutPred != 0) { updateTerminators(*oldLayoutPred); } lis->InsertMBBInMaps(preHeader); if (MachineLoop *parentLoop = loop.getParentLoop()) { assert(parentLoop->getHeader() != loop.getHeader() && "Parent loop has same header?"); parentLoop->addBasicBlockToLoop(preHeader, mli->getBase()); // Invalidate all parent loop ranges. while (parentLoop != 0) { loopRangeMap.erase(parentLoop); parentLoop = parentLoop->getParentLoop(); } } for (LiveIntervals::iterator liItr = lis->begin(), liEnd = lis->end(); liItr != liEnd; ++liItr) { LiveInterval &li = *liItr->second; // Is this safe for physregs? // TargetRegisterInfo::isPhysicalRegister(li.reg) || if (!lis->isLiveInToMBB(li, &header)) continue; if (lis->isLiveInToMBB(li, preHeader)) { assert(lis->isLiveOutOfMBB(li, preHeader) && "Range terminates in newly added preheader?"); continue; } bool insertRange = false; for (MachineBasicBlock::pred_iterator predItr = preHeader->pred_begin(), predEnd = preHeader->pred_end(); predItr != predEnd; ++predItr) { MachineBasicBlock *predMBB = *predItr; if (lis->isLiveOutOfMBB(li, predMBB)) { insertRange = true; break; } } if (!insertRange) continue; SlotIndex newDefIdx = lis->getMBBStartIdx(preHeader); assert(lis->getInstructionFromIndex(newDefIdx) == 0 && "PHI def index points at actual instruction."); VNInfo *newVal = li.getNextValue(newDefIdx, 0, lis->getVNInfoAllocator()); li.addRange(LiveRange(lis->getMBBStartIdx(preHeader), lis->getMBBEndIdx(preHeader), newVal)); } //dbgs() << "Dumping SlotIndexes:\n"; //sis->dump(); //dbgs() << "done. (Added MBB#" << preHeader->getNumber() << ")\n"; return *preHeader; }
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) { SmallPtrSet<MachineBasicBlock*, 8> Reachable; MMI = getAnalysisIfAvailable<MachineModuleInfo>(); MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>(); MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>(); // Mark all reachable blocks. for (df_ext_iterator<MachineFunction*, SmallPtrSet<MachineBasicBlock*, 8> > I = df_ext_begin(&F, Reachable), E = df_ext_end(&F, Reachable); I != E; ++I) /* Mark all reachable blocks */; // Loop over all dead blocks, remembering them and deleting all instructions // in them. std::vector<MachineBasicBlock*> DeadBlocks; for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = I; // Test for deadness. if (!Reachable.count(BB)) { DeadBlocks.push_back(BB); // Update dominator and loop info. if (MLI) MLI->removeBlock(BB); if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB); while (BB->succ_begin() != BB->succ_end()) { MachineBasicBlock* succ = *BB->succ_begin(); MachineBasicBlock::iterator start = succ->begin(); while (start != succ->end() && start->isPHI()) { for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2) if (start->getOperand(i).isMBB() && start->getOperand(i).getMBB() == BB) { start->RemoveOperand(i); start->RemoveOperand(i-1); } start++; } BB->removeSuccessor(BB->succ_begin()); } } } // Actually remove the blocks now. for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) DeadBlocks[i]->eraseFromParent(); // Cleanup PHI nodes. for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) { MachineBasicBlock *BB = I; // Prune unneeded PHI entries. SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(), BB->pred_end()); MachineBasicBlock::iterator phi = BB->begin(); while (phi != BB->end() && phi->isPHI()) { for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2) if (!preds.count(phi->getOperand(i).getMBB())) { phi->RemoveOperand(i); phi->RemoveOperand(i-1); } if (phi->getNumOperands() == 3) { unsigned Input = phi->getOperand(1).getReg(); unsigned Output = phi->getOperand(0).getReg(); MachineInstr* temp = phi; ++phi; temp->eraseFromParent(); if (Input != Output) F.getRegInfo().replaceRegWith(Output, Input); continue; } ++phi; } } F.RenumberBlocks(); return DeadBlocks.size(); }
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB, SlotIndex Use, unsigned PhysReg) { unsigned UseMBBNum = UseMBB.getNumber(); // Block numbers where LR should be live-in. SmallVector<unsigned, 16> WorkList(1, UseMBBNum); // Remember if we have seen more than one value. bool UniqueVNI = true; VNInfo *TheVNI = nullptr; // Using Seen as a visited set, perform a BFS for all reaching defs. for (unsigned i = 0; i != WorkList.size(); ++i) { MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]); #ifndef NDEBUG if (MBB->pred_empty()) { MBB->getParent()->verify(); errs() << "Use of " << PrintReg(PhysReg) << " does not have a corresponding definition on every path:\n"; const MachineInstr *MI = Indexes->getInstructionFromIndex(Use); if (MI != nullptr) errs() << Use << " " << *MI; llvm_unreachable("Use not jointly dominated by defs."); } if (TargetRegisterInfo::isPhysicalRegister(PhysReg) && !MBB->isLiveIn(PhysReg)) { MBB->getParent()->verify(); errs() << "The register " << PrintReg(PhysReg) << " needs to be live in to BB#" << MBB->getNumber() << ", but is missing from the live-in list.\n"; llvm_unreachable("Invalid global physical register"); } #endif for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(), PE = MBB->pred_end(); PI != PE; ++PI) { MachineBasicBlock *Pred = *PI; // Is this a known live-out block? if (Seen.test(Pred->getNumber())) { if (VNInfo *VNI = Map[Pred].first) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; } continue; } SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(Pred); // First time we see Pred. Try to determine the live-out value, but set // it as null if Pred is live-through with an unknown value. VNInfo *VNI = LR.extendInBlock(Start, End); setLiveOutValue(Pred, VNI); if (VNI) { if (TheVNI && TheVNI != VNI) UniqueVNI = false; TheVNI = VNI; continue; } // No, we need a live-in value for Pred as well if (Pred != &UseMBB) WorkList.push_back(Pred->getNumber()); else // Loopback to UseMBB, so value is really live through. Use = SlotIndex(); } } LiveIn.clear(); // Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but // neither require it. Skip the sorting overhead for small updates. if (WorkList.size() > 4) array_pod_sort(WorkList.begin(), WorkList.end()); // If a unique reaching def was found, blit in the live ranges immediately. if (UniqueVNI) { LiveRangeUpdater Updater(&LR); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { SlotIndex Start, End; std::tie(Start, End) = Indexes->getMBBRange(*I); // Trim the live range in UseMBB. if (*I == UseMBBNum && Use.isValid()) End = Use; else Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr); Updater.add(Start, End, TheVNI); } return true; } // Multiple values were found, so transfer the work list to the LiveIn array // where UpdateSSA will use it as a work list. LiveIn.reserve(WorkList.size()); for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(), E = WorkList.end(); I != E; ++I) { MachineBasicBlock *MBB = MF->getBlockNumbered(*I); addLiveInBlock(LR, DomTree->getNode(MBB)); if (MBB == &UseMBB) LiveIn.back().Kill = Use; } return false; }
/// processBasicBlock - Loop over all of the instructions in the basic block, /// inserting vzero upper instructions before function calls. bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF, MachineBasicBlock &BB) { bool Changed = false; unsigned BBNum = BB.getNumber(); MBB = &BB; // Don't process already solved BBs if (BBSolved[BBNum]) return false; // No changes // Check the state of all predecessors unsigned EntryState = ST_INIT; for (MachineBasicBlock::const_pred_iterator PI = BB.pred_begin(), PE = BB.pred_end(); PI != PE; ++PI) { EntryState = computeState(EntryState, BBState[(*PI)->getNumber()]); if (EntryState == ST_DIRTY) break; } // The entry MBB for the function may set the inital state to dirty if // the function receives any YMM incoming arguments if (MBB == MF.begin()) { EntryState = ST_CLEAN; if (FnHasLiveInYmm) EntryState = ST_DIRTY; } // The current state is initialized according to the predecessors unsigned CurState = EntryState; bool BBHasCall = false; for (MachineBasicBlock::iterator I = BB.begin(); I != BB.end(); ++I) { MachineInstr *MI = I; DebugLoc dl = I->getDebugLoc(); bool isControlFlow = MI->isCall() || MI->isReturn(); // Shortcut: don't need to check regular instructions in dirty state. if (!isControlFlow && CurState == ST_DIRTY) continue; if (hasYmmReg(MI)) { // We found a ymm-using instruction; this could be an AVX instruction, // or it could be control flow. CurState = ST_DIRTY; continue; } // Check for control-flow out of the current function (which might // indirectly execute SSE instructions). if (!isControlFlow) continue; BBHasCall = true; // The VZEROUPPER instruction resets the upper 128 bits of all Intel AVX // registers. This instruction has zero latency. In addition, the processor // changes back to Clean state, after which execution of Intel SSE // instructions or Intel AVX instructions has no transition penalty. Add // the VZEROUPPER instruction before any function call/return that might // execute SSE code. // FIXME: In some cases, we may want to move the VZEROUPPER into a // predecessor block. if (CurState == ST_DIRTY) { // Only insert the VZEROUPPER in case the entry state isn't unknown. // When unknown, only compute the information within the block to have // it available in the exit if possible, but don't change the block. if (EntryState != ST_UNKNOWN) { BuildMI(*MBB, I, dl, TII->get(X86::VZEROUPPER)); ++NumVZU; } // After the inserted VZEROUPPER the state becomes clean again, but // other YMM may appear before other subsequent calls or even before // the end of the BB. CurState = ST_CLEAN; } } DEBUG(dbgs() << "MBB #" << BBNum << ", current state: " << CurState << '\n'); // A BB can only be considered solved when we both have done all the // necessary transformations, and have computed the exit state. This happens // in two cases: // 1) We know the entry state: this immediately implies the exit state and // all the necessary transformations. // 2) There are no calls, and and a non-call instruction marks this block: // no transformations are necessary, and we know the exit state. if (EntryState != ST_UNKNOWN || (!BBHasCall && CurState != ST_UNKNOWN)) BBSolved[BBNum] = true; if (CurState != BBState[BBNum]) Changed = true; BBState[BBNum] = CurState; return Changed; }
/// \brief Merge a chain with any viable successor. /// /// This routine walks the predecessors of the current block, looking for /// viable merge candidates. It has strict rules it uses to determine when /// a predecessor can be merged with the current block, which center around /// preserving the CFG structure. It performs the merge if any viable candidate /// is found. void MachineBlockPlacement::mergeSuccessor(MachineBasicBlock *BB, BlockChain *Chain, BlockFilterSet *Filter) { assert(BB); assert(Chain); // If this block is not at the end of its chain, it cannot merge with any // other chain. if (Chain && *llvm::prior(Chain->end()) != BB) return; // Walk through the successors looking for the highest probability edge. MachineBasicBlock *Successor = 0; BranchProbability BestProb = BranchProbability::getZero(); DEBUG(dbgs() << "Attempting merge from: " << getBlockName(BB) << "\n"); for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(), SE = BB->succ_end(); SI != SE; ++SI) { if (BB == *SI || (Filter && !Filter->count(*SI))) continue; BranchProbability SuccProb = MBPI->getEdgeProbability(BB, *SI); DEBUG(dbgs() << " " << getBlockName(*SI) << " -> " << SuccProb << "\n"); if (!Successor || SuccProb > BestProb || (!(SuccProb < BestProb) && BB->isLayoutSuccessor(*SI))) { Successor = *SI; BestProb = SuccProb; } } if (!Successor) return; // Grab a chain if it exists already for this successor and make sure the // successor is at the start of the chain as we can't merge mid-chain. Also, // if the successor chain is the same as our chain, we're already merged. BlockChain *SuccChain = BlockToChain[Successor]; if (SuccChain && (SuccChain == Chain || Successor != *SuccChain->begin())) return; // We only merge chains across a CFG merge when the desired merge path is // significantly hotter than the incoming edge. We define a hot edge more // strictly than the BranchProbabilityInfo does, as the two predecessor // blocks may have dramatically different incoming probabilities we need to // account for. Therefor we use the "global" edge weight which is the // branch's probability times the block frequency of the predecessor. BlockFrequency MergeWeight = MBFI->getBlockFreq(BB); MergeWeight *= MBPI->getEdgeProbability(BB, Successor); // We only want to consider breaking the CFG when the merge weight is much // higher (80% vs. 20%), so multiply it by 1/4. This will require the merged // edge to be 4x more likely before we disrupt the CFG. This number matches // the definition of "hot" in BranchProbabilityAnalysis (80% vs. 20%). MergeWeight *= BranchProbability(1, 4); for (MachineBasicBlock::pred_iterator PI = Successor->pred_begin(), PE = Successor->pred_end(); PI != PE; ++PI) { if (BB == *PI || Successor == *PI) continue; BlockFrequency PredWeight = MBFI->getBlockFreq(*PI); PredWeight *= MBPI->getEdgeProbability(*PI, Successor); // Return on the first predecessor we find which outstrips our merge weight. if (MergeWeight < PredWeight) return; DEBUG(dbgs() << "Breaking CFG edge!\n" << " Edge from " << getBlockNum(BB) << " to " << getBlockNum(Successor) << ": " << MergeWeight << "\n" << " vs. " << getBlockNum(BB) << " to " << getBlockNum(*PI) << ": " << PredWeight << "\n"); } DEBUG(dbgs() << "Merging from " << getBlockNum(BB) << " to " << getBlockNum(Successor) << "\n"); Chain->merge(Successor, SuccChain); }