MachineBasicBlock *Mips16TargetLowering:: emitSel16(unsigned Opc, MachineInstr *MI, MachineBasicBlock *BB) const { if (DontExpandCondPseudos16) return BB; const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); DebugLoc DL = MI->getDebugLoc(); // To "insert" a SELECT_CC instruction, we actually have to insert the // diamond control-flow pattern. The incoming instruction knows the // destination vreg to set, the condition code register to branch on, the // true/false values to select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // thisMBB: // ... // TrueVal = ... // setcc r1, r2, r3 // bNE r1, r0, copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); BuildMI(BB, DL, TII->get(Opc)).addReg(MI->getOperand(3).getReg()) .addMBB(sinkMBB); // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %TrueValue, thisMBB ], [ %FalseValue, copy0MBB ] // ... BB = sinkMBB; BuildMI(*BB, BB->begin(), DL, TII->get(Mips::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()).addMBB(thisMBB) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return BB; }
MachineBasicBlock* SystemZTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const SystemZInstrInfo &TII = *TM.getInstrInfo(); DebugLoc dl = MI->getDebugLoc(); assert((MI->getOpcode() == SystemZ::Select32 || MI->getOpcode() == SystemZ::SelectF32 || MI->getOpcode() == SystemZ::Select64 || MI->getOpcode() == SystemZ::SelectF64) && "Unexpected instr type to insert"); // To "insert" a SELECT instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator I = BB; ++I; // thisMBB: // ... // TrueVal = ... // cmpTY ccX, r1, r2 // jCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); SystemZCC::CondCodes CC = (SystemZCC::CondCodes)MI->getOperand(3).getImm(); BuildMI(BB, dl, TII.getBrCond(CC)).addMBB(copy1MBB); F->insert(I, copy0MBB); F->insert(I, copy1MBB); // Update machine-CFG edges by transferring all successors of the current // block to the new block which will contain the Phi node for the select. copy1MBB->transferSuccessors(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(copy1MBB); // copy0MBB: // %FalseValue = ... // # fallthrough to copy1MBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(copy1MBB); // copy1MBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = copy1MBB; BuildMI(BB, dl, TII.get(SystemZ::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(1).getReg()).addMBB(thisMBB); F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. return BB; }
MachineBasicBlock* ARCompactTargetLowering::EmitInstrWithCustomInserter( MachineInstr *MI, MachineBasicBlock *BB) const { const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo(); DebugLoc dl = MI->getDebugLoc(); assert((MI->getOpcode() == ARC::Select) && "We can only emit SELECT_CC"); // TODO: We can totally use conditional instructions here, can't we? // To "insert" a SELECT instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator I = BB; ++I; // copy0MBB is the fallthrough block, copy1MBB is the branch // block. MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(I, copy0MBB); F->insert(I, copy1MBB); // Update machine-CFG edges by transferring all successors of the current // block to the new block which will contain the Phi node for the select. copy1MBB->splice(copy1MBB->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); copy1MBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(copy1MBB); BuildMI(BB, dl, TII.get(ARC::BCC)).addMBB(copy1MBB) .addImm(MI->getOperand(3).getImm()); // copy0MBB: // %FalseValue = ... // # fallthrough to copy1MBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(copy1MBB); // copy1MBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = copy1MBB; BuildMI(*BB, BB->begin(), dl, TII.get(ARC::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(1).getReg()).addMBB(thisMBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return BB; }
bool MIRParserImpl::initializeMachineFunction(MachineFunction &MF) { auto It = Functions.find(MF.getName()); if (It == Functions.end()) return error(Twine("no machine function information for function '") + MF.getName() + "' in the MIR file"); // TODO: Recreate the machine function. const yaml::MachineFunction &YamlMF = *It->getValue(); if (YamlMF.Alignment) MF.setAlignment(YamlMF.Alignment); MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice); MF.setHasInlineAsm(YamlMF.HasInlineAsm); PerFunctionMIParsingState PFS; if (initializeRegisterInfo(MF, MF.getRegInfo(), YamlMF, PFS.VirtualRegisterSlots)) return true; if (initializeFrameInfo(*MF.getFunction(), *MF.getFrameInfo(), YamlMF, PFS.StackObjectSlots, PFS.FixedStackObjectSlots)) return true; const auto &F = *MF.getFunction(); for (const auto &YamlMBB : YamlMF.BasicBlocks) { const BasicBlock *BB = nullptr; const yaml::StringValue &Name = YamlMBB.Name; if (!Name.Value.empty()) { BB = dyn_cast_or_null<BasicBlock>( F.getValueSymbolTable().lookup(Name.Value)); if (!BB) return error(Name.SourceRange.Start, Twine("basic block '") + Name.Value + "' is not defined in the function '" + MF.getName() + "'"); } auto *MBB = MF.CreateMachineBasicBlock(BB); MF.insert(MF.end(), MBB); bool WasInserted = PFS.MBBSlots.insert(std::make_pair(YamlMBB.ID, MBB)).second; if (!WasInserted) return error(Twine("redefinition of machine basic block with id #") + Twine(YamlMBB.ID)); } if (YamlMF.BasicBlocks.empty()) return error(Twine("machine function '") + Twine(MF.getName()) + "' requires at least one machine basic block in its body"); // Initialize the jump table after creating all the MBBs so that the MBB // references can be resolved. if (!YamlMF.JumpTableInfo.Entries.empty() && initializeJumpTableInfo(MF, YamlMF.JumpTableInfo, PFS)) return true; // Initialize the machine basic blocks after creating them all so that the // machine instructions parser can resolve the MBB references. unsigned I = 0; for (const auto &YamlMBB : YamlMF.BasicBlocks) { if (initializeMachineBasicBlock(MF, *MF.getBlockNumbered(I++), YamlMBB, PFS)) return true; } return false; }
MachineBasicBlock *SIInsertSkips::insertSkipBlock( MachineBasicBlock &MBB, MachineBasicBlock::iterator I) const { MachineFunction *MF = MBB.getParent(); MachineBasicBlock *SkipBB = MF->CreateMachineBasicBlock(); MachineFunction::iterator MBBI(MBB); ++MBBI; MF->insert(MBBI, SkipBB); MBB.addSuccessor(SkipBB); return SkipBB; }
MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A, MachineBasicBlock *B) { assert(A && B && "Missing MBB end point"); MachineFunction *MF = A->getParent(); // We may need to update A's terminator, but we can't do that if AnalyzeBranch // fails. If A uses a jump table, we won't touch it. const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); MachineBasicBlock *TBB = 0, *FBB = 0; SmallVector<MachineOperand, 4> Cond; if (TII->AnalyzeBranch(*A, TBB, FBB, Cond)) return NULL; ++NumSplits; MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); MF->insert(llvm::next(MachineFunction::iterator(A)), NMBB); DEBUG(dbgs() << "PHIElimination splitting critical edge:" " BB#" << A->getNumber() << " -- BB#" << NMBB->getNumber() << " -- BB#" << B->getNumber() << '\n'); A->ReplaceUsesOfBlockWith(B, NMBB); A->updateTerminator(); // Insert unconditional "jump B" instruction in NMBB if necessary. NMBB->addSuccessor(B); if (!NMBB->isLayoutSuccessor(B)) { Cond.clear(); MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, B, NULL, Cond); } // Fix PHI nodes in B so they refer to NMBB instead of A for (MachineBasicBlock::iterator i = B->begin(), e = B->end(); i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == A) i->getOperand(ni+1).setMBB(NMBB); if (LiveVariables *LV=getAnalysisIfAvailable<LiveVariables>()) LV->addNewBlock(NMBB, A, B); if (MachineDominatorTree *MDT=getAnalysisIfAvailable<MachineDominatorTree>()) MDT->addNewBlock(NMBB, A); return NMBB; }
bool MIRParserImpl::initializeMachineFunction(MachineFunction &MF) { auto It = Functions.find(MF.getName()); if (It == Functions.end()) return error(Twine("no machine function information for function '") + MF.getName() + "' in the MIR file"); // TODO: Recreate the machine function. const yaml::MachineFunction &YamlMF = *It->getValue(); if (YamlMF.Alignment) MF.setAlignment(YamlMF.Alignment); MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice); MF.setHasInlineAsm(YamlMF.HasInlineAsm); if (initializeRegisterInfo(MF.getRegInfo(), YamlMF)) return true; const auto &F = *MF.getFunction(); DenseMap<unsigned, MachineBasicBlock *> MBBSlots; for (const auto &YamlMBB : YamlMF.BasicBlocks) { const BasicBlock *BB = nullptr; if (!YamlMBB.Name.empty()) { BB = dyn_cast_or_null<BasicBlock>( F.getValueSymbolTable().lookup(YamlMBB.Name)); if (!BB) return error(Twine("basic block '") + YamlMBB.Name + "' is not defined in the function '" + MF.getName() + "'"); } auto *MBB = MF.CreateMachineBasicBlock(BB); MF.insert(MF.end(), MBB); bool WasInserted = MBBSlots.insert(std::make_pair(YamlMBB.ID, MBB)).second; if (!WasInserted) return error(Twine("redefinition of machine basic block with id #") + Twine(YamlMBB.ID)); } // Initialize the machine basic blocks after creating them all so that the // machine instructions parser can resolve the MBB references. unsigned I = 0; for (const auto &YamlMBB : YamlMF.BasicBlocks) { if (initializeMachineBasicBlock(MF, *MF.getBlockNumbered(I++), YamlMBB, MBBSlots)) return true; } return false; }
MachineBasicBlock * MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { MachineFunction *MF = getParent(); DebugLoc dl; // FIXME: this is nowhere // We may need to update this's terminator, but we can't do that if // AnalyzeBranch fails. If this uses a jump table, we won't touch it. const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); MachineBasicBlock *TBB = 0, *FBB = 0; SmallVector<MachineOperand, 4> Cond; if (TII->AnalyzeBranch(*this, TBB, FBB, Cond)) return NULL; // Avoid bugpoint weirdness: A block may end with a conditional branch but // jumps to the same MBB is either case. We have duplicate CFG edges in that // case that we can't handle. Since this never happens in properly optimized // code, just skip those edges. if (TBB && TBB == FBB) { DEBUG(dbgs() << "Won't split critical edge after degenerate BB#" << getNumber() << '\n'); return NULL; } MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); MF->insert(llvm::next(MachineFunction::iterator(this)), NMBB); DEBUG(dbgs() << "Splitting critical edge:" " BB#" << getNumber() << " -- BB#" << NMBB->getNumber() << " -- BB#" << Succ->getNumber() << '\n'); // On some targets like Mips, branches may kill virtual registers. Make sure // that LiveVariables is properly updated after updateTerminator replaces the // terminators. LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>(); // Collect a list of virtual registers killed by the terminators. SmallVector<unsigned, 4> KilledRegs; if (LV) for (iterator I = getFirstTerminator(), E = end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { if (!OI->isReg() || !OI->isUse() || !OI->isKill() || OI->isUndef()) continue; unsigned Reg = OI->getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg) && LV->getVarInfo(Reg).removeKill(MI)) { KilledRegs.push_back(Reg); DEBUG(dbgs() << "Removing terminator kill: " << *MI); OI->setIsKill(false); } } } ReplaceUsesOfBlockWith(Succ, NMBB); updateTerminator(); // Insert unconditional "jump Succ" instruction in NMBB if necessary. NMBB->addSuccessor(Succ); if (!NMBB->isLayoutSuccessor(Succ)) { Cond.clear(); MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl); } // Fix PHI nodes in Succ so they refer to NMBB instead of this for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end(); i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == this) i->getOperand(ni+1).setMBB(NMBB); // Inherit live-ins from the successor for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(), E = Succ->livein_end(); I != E; ++I) NMBB->addLiveIn(*I); // Update LiveVariables. if (LV) { // Restore kills of virtual registers that were killed by the terminators. while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); for (iterator I = end(), E = begin(); I != E;) { if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false)) continue; LV->getVarInfo(Reg).Kills.push_back(I); DEBUG(dbgs() << "Restored terminator kill: " << *I); break; } } // Update relevant live-through information. LV->addNewBlock(NMBB, this, Succ); } if (MachineDominatorTree *MDT = P->getAnalysisIfAvailable<MachineDominatorTree>()) { // Update dominator information. MachineDomTreeNode *SucccDTNode = MDT->getNode(Succ); bool IsNewIDom = true; for (const_pred_iterator PI = Succ->pred_begin(), E = Succ->pred_end(); PI != E; ++PI) { MachineBasicBlock *PredBB = *PI; if (PredBB == NMBB) continue; if (!MDT->dominates(SucccDTNode, MDT->getNode(PredBB))) { IsNewIDom = false; break; } } // We know "this" dominates the newly created basic block. MachineDomTreeNode *NewDTNode = MDT->addNewBlock(NMBB, this); // If all the other predecessors of "Succ" are dominated by "Succ" itself // then the new block is the new immediate dominator of "Succ". Otherwise, // the new block doesn't dominate anything. if (IsNewIDom) MDT->changeImmediateDominator(SucccDTNode, NewDTNode); } if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>()) if (MachineLoop *TIL = MLI->getLoopFor(this)) { // If one or the other blocks were not in a loop, the new block is not // either, and thus LI doesn't need to be updated. if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) { if (TIL == DestLoop) { // Both in the same loop, the NMBB joins loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (TIL->contains(DestLoop)) { // Edge from an outer loop to an inner loop. Add to the outer loop. TIL->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (DestLoop->contains(TIL)) { // Edge from an inner loop to an outer loop. Add to the outer loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else { // Edge from two loops with no containment relation. Because these // are natural loops, we know that the destination block must be the // header of its loop (adding a branch into a loop elsewhere would // create an irreducible loop). assert(DestLoop->getHeader() == Succ && "Should not create irreducible loops!"); if (MachineLoop *P = DestLoop->getParentLoop()) P->addBasicBlockToLoop(NMBB, MLI->getBase()); } } } return NMBB; }
MachineBasicBlock * AlphaTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); assert((MI->getOpcode() == Alpha::CAS32 || MI->getOpcode() == Alpha::CAS64 || MI->getOpcode() == Alpha::LAS32 || MI->getOpcode() == Alpha::LAS64 || MI->getOpcode() == Alpha::SWAP32 || MI->getOpcode() == Alpha::SWAP64) && "Unexpected instr type to insert"); bool is32 = MI->getOpcode() == Alpha::CAS32 || MI->getOpcode() == Alpha::LAS32 || MI->getOpcode() == Alpha::SWAP32; //Load locked store conditional for atomic ops take on the same form //start: //ll //do stuff (maybe branch to exit) //sc //test sc and maybe branck to start //exit: const BasicBlock *LLVM_BB = BB->getBasicBlock(); DebugLoc dl = MI->getDebugLoc(); MachineFunction::iterator It = BB; ++It; MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *llscMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); sinkMBB->transferSuccessors(thisMBB); F->insert(It, llscMBB); F->insert(It, sinkMBB); BuildMI(thisMBB, dl, TII->get(Alpha::BR)).addMBB(llscMBB); unsigned reg_res = MI->getOperand(0).getReg(), reg_ptr = MI->getOperand(1).getReg(), reg_v2 = MI->getOperand(2).getReg(), reg_store = F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass); BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::LDL_L : Alpha::LDQ_L), reg_res).addImm(0).addReg(reg_ptr); switch (MI->getOpcode()) { case Alpha::CAS32: case Alpha::CAS64: { unsigned reg_cmp = F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass); BuildMI(llscMBB, dl, TII->get(Alpha::CMPEQ), reg_cmp) .addReg(reg_v2).addReg(reg_res); BuildMI(llscMBB, dl, TII->get(Alpha::BEQ)) .addImm(0).addReg(reg_cmp).addMBB(sinkMBB); BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store) .addReg(Alpha::R31).addReg(MI->getOperand(3).getReg()); break; } case Alpha::LAS32: case Alpha::LAS64: { BuildMI(llscMBB, dl,TII->get(is32 ? Alpha::ADDLr : Alpha::ADDQr), reg_store) .addReg(reg_res).addReg(reg_v2); break; } case Alpha::SWAP32: case Alpha::SWAP64: { BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store) .addReg(reg_v2).addReg(reg_v2); break; } } BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::STL_C : Alpha::STQ_C), reg_store) .addReg(reg_store).addImm(0).addReg(reg_ptr); BuildMI(llscMBB, dl, TII->get(Alpha::BEQ)) .addImm(0).addReg(reg_store).addMBB(llscMBB); BuildMI(llscMBB, dl, TII->get(Alpha::BR)).addMBB(sinkMBB); thisMBB->addSuccessor(llscMBB); llscMBB->addSuccessor(llscMBB); llscMBB->addSuccessor(sinkMBB); F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. return sinkMBB; }
//===----------------------------------------------------------------------===// // Lower helper functions //===----------------------------------------------------------------------===// MachineBasicBlock* MBlazeTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); DebugLoc dl = MI->getDebugLoc(); switch (MI->getOpcode()) { default: assert(false && "Unexpected instr type to insert"); case MBlaze::ShiftRL: case MBlaze::ShiftRA: case MBlaze::ShiftL: { // To "insert" a shift left instruction, we actually have to insert a // simple loop. The incoming instruction knows the destination vreg to // set, the source vreg to operate over and the shift amount. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // start: // andi samt, samt, 31 // beqid samt, finish // add dst, src, r0 // loop: // addik samt, samt, -1 // sra dst, dst // bneid samt, loop // nop // finish: MachineFunction *F = BB->getParent(); MachineRegisterInfo &R = F->getRegInfo(); MachineBasicBlock *loop = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *finish = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loop); F->insert(It, finish); // Update machine-CFG edges by transfering adding all successors and // remaining instructions from the current block to the new block which // will contain the Phi node for the select. finish->splice(finish->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); finish->transferSuccessorsAndUpdatePHIs(BB); // Add the true and fallthrough blocks as its successors. BB->addSuccessor(loop); BB->addSuccessor(finish); // Next, add the finish block as a successor of the loop block loop->addSuccessor(finish); loop->addSuccessor(loop); unsigned IAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass); BuildMI(BB, dl, TII->get(MBlaze::ANDI), IAMT) .addReg(MI->getOperand(2).getReg()) .addImm(31); unsigned IVAL = R.createVirtualRegister(MBlaze::GPRRegisterClass); BuildMI(BB, dl, TII->get(MBlaze::ADDI), IVAL) .addReg(MI->getOperand(1).getReg()) .addImm(0); BuildMI(BB, dl, TII->get(MBlaze::BEQID)) .addReg(IAMT) .addMBB(finish); unsigned DST = R.createVirtualRegister(MBlaze::GPRRegisterClass); unsigned NDST = R.createVirtualRegister(MBlaze::GPRRegisterClass); BuildMI(loop, dl, TII->get(MBlaze::PHI), DST) .addReg(IVAL).addMBB(BB) .addReg(NDST).addMBB(loop); unsigned SAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass); unsigned NAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass); BuildMI(loop, dl, TII->get(MBlaze::PHI), SAMT) .addReg(IAMT).addMBB(BB) .addReg(NAMT).addMBB(loop); if (MI->getOpcode() == MBlaze::ShiftL) BuildMI(loop, dl, TII->get(MBlaze::ADD), NDST).addReg(DST).addReg(DST); else if (MI->getOpcode() == MBlaze::ShiftRA) BuildMI(loop, dl, TII->get(MBlaze::SRA), NDST).addReg(DST); else if (MI->getOpcode() == MBlaze::ShiftRL) BuildMI(loop, dl, TII->get(MBlaze::SRL), NDST).addReg(DST); else llvm_unreachable("Cannot lower unknown shift instruction"); BuildMI(loop, dl, TII->get(MBlaze::ADDI), NAMT) .addReg(SAMT) .addImm(-1); BuildMI(loop, dl, TII->get(MBlaze::BNEID)) .addReg(NAMT) .addMBB(loop); BuildMI(*finish, finish->begin(), dl, TII->get(MBlaze::PHI), MI->getOperand(0).getReg()) .addReg(IVAL).addMBB(BB) .addReg(NDST).addMBB(loop); // The pseudo instruction is no longer needed so remove it MI->eraseFromParent(); return finish; } case MBlaze::Select_FCC: case MBlaze::Select_CC: { // To "insert" a SELECT_CC instruction, we actually have to insert the // diamond control-flow pattern. The incoming instruction knows the // destination vreg to set, the condition code register to branch on, the // true/false values to select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // thisMBB: // ... // TrueVal = ... // setcc r1, r2, r3 // bNE r1, r0, copy1MBB // fallthrough --> copy0MBB MachineFunction *F = BB->getParent(); MachineBasicBlock *flsBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *dneBB = F->CreateMachineBasicBlock(LLVM_BB); unsigned Opc; switch (MI->getOperand(4).getImm()) { default: llvm_unreachable("Unknown branch condition"); case MBlazeCC::EQ: Opc = MBlaze::BEQID; break; case MBlazeCC::NE: Opc = MBlaze::BNEID; break; case MBlazeCC::GT: Opc = MBlaze::BGTID; break; case MBlazeCC::LT: Opc = MBlaze::BLTID; break; case MBlazeCC::GE: Opc = MBlaze::BGEID; break; case MBlazeCC::LE: Opc = MBlaze::BLEID; break; } F->insert(It, flsBB); F->insert(It, dneBB); // Transfer the remainder of BB and its successor edges to dneBB. dneBB->splice(dneBB->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); dneBB->transferSuccessorsAndUpdatePHIs(BB); BB->addSuccessor(flsBB); BB->addSuccessor(dneBB); flsBB->addSuccessor(dneBB); BuildMI(BB, dl, TII->get(Opc)) .addReg(MI->getOperand(3).getReg()) .addMBB(dneBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... //BuildMI(dneBB, dl, TII->get(MBlaze::PHI), MI->getOperand(0).getReg()) // .addReg(MI->getOperand(1).getReg()).addMBB(flsBB) // .addReg(MI->getOperand(2).getReg()).addMBB(BB); BuildMI(*dneBB, dneBB->begin(), dl, TII->get(MBlaze::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(flsBB) .addReg(MI->getOperand(1).getReg()).addMBB(BB); MI->eraseFromParent(); // The pseudo instruction is gone now. return dneBB; } } }
MachineBasicBlock * MipsSETargetLowering:: emitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{ // $bb: // bposge32_pseudo $vr0 // => // $bb: // bposge32 $tbb // $fbb: // li $vr2, 0 // b $sink // $tbb: // li $vr1, 1 // $sink: // $vr0 = phi($vr2, $fbb, $vr1, $tbb) MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo(); const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); const TargetRegisterClass *RC = &Mips::CPURegsRegClass; DebugLoc DL = MI->getDebugLoc(); const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB)); MachineFunction *F = BB->getParent(); MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *Sink = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, FBB); F->insert(It, TBB); F->insert(It, Sink); // Transfer the remainder of BB and its successor edges to Sink. Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); Sink->transferSuccessorsAndUpdatePHIs(BB); // Add successors. BB->addSuccessor(FBB); BB->addSuccessor(TBB); FBB->addSuccessor(Sink); TBB->addSuccessor(Sink); // Insert the real bposge32 instruction to $BB. BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB); // Fill $FBB. unsigned VR2 = RegInfo.createVirtualRegister(RC); BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2) .addReg(Mips::ZERO).addImm(0); BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink); // Fill $TBB. unsigned VR1 = RegInfo.createVirtualRegister(RC); BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1) .addReg(Mips::ZERO).addImm(1); // Insert phi function to $Sink. BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI), MI->getOperand(0).getReg()) .addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return Sink; }
void MipsFrameLowering::emitPrologue(MachineFunction &MF) const { MachineBasicBlock &MBB = MF.front(); MachineFrameInfo *MFI = MF.getFrameInfo(); MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>(); const MipsRegisterInfo *RegInfo = static_cast<const MipsRegisterInfo*>(MF.getTarget().getRegisterInfo()); const MipsInstrInfo &TII = *static_cast<const MipsInstrInfo*>(MF.getTarget().getInstrInfo()); MachineBasicBlock::iterator MBBI = MBB.begin(); DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc(); bool isPIC = (MF.getTarget().getRelocationModel() == Reloc::PIC_); unsigned SP = STI.isABI_N64() ? Mips::SP_64 : Mips::SP; unsigned FP = STI.isABI_N64() ? Mips::FP_64 : Mips::FP; unsigned ZERO = STI.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO; unsigned ADDu = STI.isABI_N64() ? Mips::DADDu : Mips::ADDu; unsigned ADDiu = STI.isABI_N64() ? Mips::DADDiu : Mips::ADDiu; // First, compute final stack size. unsigned RegSize = STI.isGP32bit() ? 4 : 8; unsigned StackAlign = getStackAlignment(); unsigned LocalVarAreaOffset = MipsFI->needGPSaveRestore() ? (MFI->getObjectOffset(MipsFI->getGPFI()) + RegSize) : MipsFI->getMaxCallFrameSize(); uint64_t StackSize = RoundUpToAlignment(LocalVarAreaOffset, StackAlign) + RoundUpToAlignment(MFI->getStackSize(), StackAlign); // Update stack size MFI->setStackSize(StackSize); // Emit instructions that set the global base register if the target ABI is // O32. if (isPIC && MipsFI->globalBaseRegSet() && STI.isABI_O32() && !MipsFI->globalBaseRegFixed()) { // See MipsInstrInfo.td for explanation. MachineBasicBlock *NewEntry = MF.CreateMachineBasicBlock(); MF.insert(&MBB, NewEntry); NewEntry->addSuccessor(&MBB); // Copy live in registers. for (MachineBasicBlock::livein_iterator R = MBB.livein_begin(); R != MBB.livein_end(); ++R) NewEntry->addLiveIn(*R); BuildMI(*NewEntry, NewEntry->begin(), dl, TII.get(Mips:: SETGP01), Mips::V0); } // No need to allocate space on the stack. if (StackSize == 0 && !MFI->adjustsStack()) return; MachineModuleInfo &MMI = MF.getMMI(); std::vector<MachineMove> &Moves = MMI.getFrameMoves(); MachineLocation DstML, SrcML; // Adjust stack. if (isInt<16>(-StackSize)) // addi sp, sp, (-stacksize) BuildMI(MBB, MBBI, dl, TII.get(ADDiu), SP).addReg(SP).addImm(-StackSize); else { // Expand immediate that doesn't fit in 16-bit. MipsFI->setEmitNOAT(); expandLargeImm(SP, -StackSize, STI.isABI_N64(), TII, MBB, MBBI, dl); } // emit ".cfi_def_cfa_offset StackSize" MCSymbol *AdjustSPLabel = MMI.getContext().CreateTempSymbol(); BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::PROLOG_LABEL)).addSym(AdjustSPLabel); DstML = MachineLocation(MachineLocation::VirtualFP); SrcML = MachineLocation(MachineLocation::VirtualFP, -StackSize); Moves.push_back(MachineMove(AdjustSPLabel, DstML, SrcML)); const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); if (CSI.size()) { // Find the instruction past the last instruction that saves a callee-saved // register to the stack. for (unsigned i = 0; i < CSI.size(); ++i) ++MBBI; // Iterate over list of callee-saved registers and emit .cfi_offset // directives. MCSymbol *CSLabel = MMI.getContext().CreateTempSymbol(); BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::PROLOG_LABEL)).addSym(CSLabel); for (std::vector<CalleeSavedInfo>::const_iterator I = CSI.begin(), E = CSI.end(); I != E; ++I) { int64_t Offset = MFI->getObjectOffset(I->getFrameIdx()); unsigned Reg = I->getReg(); // If Reg is a double precision register, emit two cfa_offsets, // one for each of the paired single precision registers. if (Mips::AFGR64RegisterClass->contains(Reg)) { const uint16_t *SubRegs = RegInfo->getSubRegisters(Reg); MachineLocation DstML0(MachineLocation::VirtualFP, Offset); MachineLocation DstML1(MachineLocation::VirtualFP, Offset + 4); MachineLocation SrcML0(*SubRegs); MachineLocation SrcML1(*(SubRegs + 1)); if (!STI.isLittle()) std::swap(SrcML0, SrcML1); Moves.push_back(MachineMove(CSLabel, DstML0, SrcML0)); Moves.push_back(MachineMove(CSLabel, DstML1, SrcML1)); } else { // Reg is either in CPURegs or FGR32. DstML = MachineLocation(MachineLocation::VirtualFP, Offset); SrcML = MachineLocation(Reg); Moves.push_back(MachineMove(CSLabel, DstML, SrcML)); } } } // if framepointer enabled, set it to point to the stack pointer. if (hasFP(MF)) { // Insert instruction "move $fp, $sp" at this location. BuildMI(MBB, MBBI, dl, TII.get(ADDu), FP).addReg(SP).addReg(ZERO); // emit ".cfi_def_cfa_register $fp" MCSymbol *SetFPLabel = MMI.getContext().CreateTempSymbol(); BuildMI(MBB, MBBI, dl, TII.get(TargetOpcode::PROLOG_LABEL)).addSym(SetFPLabel); DstML = MachineLocation(FP); SrcML = MachineLocation(MachineLocation::VirtualFP); Moves.push_back(MachineMove(SetFPLabel, DstML, SrcML)); } // Restore GP from the saved stack location if (MipsFI->needGPSaveRestore()) { unsigned Offset = MFI->getObjectOffset(MipsFI->getGPFI()); BuildMI(MBB, MBBI, dl, TII.get(Mips::CPRESTORE)).addImm(Offset) .addReg(Mips::GP); } }
/// Splits a MachineBasicBlock to branch before \p SplitBefore. The original /// branch is \p OrigBranch. The target of the new branch can either be the same /// as the target of the original branch or the fallthrough successor of the /// original block as determined by \p BranchToFallThrough. The branch /// conditions will be inverted according to \p InvertNewBranch and /// \p InvertOrigBranch. If an instruction that previously fed the branch is to /// be deleted, it is provided in \p MIToDelete and \p NewCond will be used as /// the branch condition. The branch probabilities will be set if the /// MachineBranchProbabilityInfo isn't null. static bool splitMBB(BlockSplitInfo &BSI) { assert(BSI.allInstrsInSameMBB() && "All instructions must be in the same block."); MachineBasicBlock *ThisMBB = BSI.OrigBranch->getParent(); MachineFunction *MF = ThisMBB->getParent(); MachineRegisterInfo *MRI = &MF->getRegInfo(); assert(MRI->isSSA() && "Can only do this while the function is in SSA form."); if (ThisMBB->succ_size() != 2) { LLVM_DEBUG( dbgs() << "Don't know how to handle blocks that don't have exactly" << " two successors.\n"); return false; } const PPCInstrInfo *TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo(); unsigned OrigBROpcode = BSI.OrigBranch->getOpcode(); unsigned InvertedOpcode = OrigBROpcode == PPC::BC ? PPC::BCn : OrigBROpcode == PPC::BCn ? PPC::BC : OrigBROpcode == PPC::BCLR ? PPC::BCLRn : PPC::BCLR; unsigned NewBROpcode = BSI.InvertNewBranch ? InvertedOpcode : OrigBROpcode; MachineBasicBlock *OrigTarget = BSI.OrigBranch->getOperand(1).getMBB(); MachineBasicBlock *OrigFallThrough = OrigTarget == *ThisMBB->succ_begin() ? *ThisMBB->succ_rbegin() : *ThisMBB->succ_begin(); MachineBasicBlock *NewBRTarget = BSI.BranchToFallThrough ? OrigFallThrough : OrigTarget; BranchProbability ProbToNewTarget = !BSI.MBPI ? BranchProbability::getUnknown() : BSI.MBPI->getEdgeProbability(ThisMBB, NewBRTarget); // Create a new basic block. MachineBasicBlock::iterator InsertPoint = BSI.SplitBefore; const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock(); MachineFunction::iterator It = ThisMBB->getIterator(); MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(LLVM_BB); MF->insert(++It, NewMBB); // Move everything after SplitBefore into the new block. NewMBB->splice(NewMBB->end(), ThisMBB, InsertPoint, ThisMBB->end()); NewMBB->transferSuccessors(ThisMBB); // Add the two successors to ThisMBB. The probabilities come from the // existing blocks if available. ThisMBB->addSuccessor(NewBRTarget, ProbToNewTarget); ThisMBB->addSuccessor(NewMBB, ProbToNewTarget.getCompl()); // Add the branches to ThisMBB. BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(), TII->get(NewBROpcode)) .addReg(BSI.SplitCond->getOperand(0).getReg()) .addMBB(NewBRTarget); BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(), TII->get(PPC::B)) .addMBB(NewMBB); if (BSI.MIToDelete) BSI.MIToDelete->eraseFromParent(); // Change the condition on the original branch and invert it if requested. auto FirstTerminator = NewMBB->getFirstTerminator(); if (BSI.NewCond) { assert(FirstTerminator->getOperand(0).isReg() && "Can't update condition of unconditional branch."); FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg()); } if (BSI.InvertOrigBranch) FirstTerminator->setDesc(TII->get(InvertedOpcode)); // If any of the PHIs in the successors of NewMBB reference values that // now come from NewMBB, they need to be updated. for (auto *Succ : NewMBB->successors()) { updatePHIs(Succ, ThisMBB, NewMBB, MRI); } addIncomingValuesToPHIs(NewBRTarget, ThisMBB, NewMBB, MRI); LLVM_DEBUG(dbgs() << "After splitting, ThisMBB:\n"; ThisMBB->dump()); LLVM_DEBUG(dbgs() << "NewMBB:\n"; NewMBB->dump()); LLVM_DEBUG(dbgs() << "New branch-to block:\n"; NewBRTarget->dump()); return true; }
void X86CmovConverterPass::convertCmovInstsToBranches( SmallVectorImpl<MachineInstr *> &Group) const { assert(!Group.empty() && "No CMOV instructions to convert"); ++NumOfOptimizedCmovGroups; // If the CMOV group is not packed, e.g., there are debug instructions between // first CMOV and last CMOV, then pack the group and make the CMOV instruction // consecutive by moving the debug instructions to after the last CMOV. packCmovGroup(Group.front(), Group.back()); // To convert a CMOVcc instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. // Before // ----- // MBB: // cond = cmp ... // v1 = CMOVge t1, f1, cond // v2 = CMOVlt t2, f2, cond // v3 = CMOVge v1, f3, cond // // After // ----- // MBB: // cond = cmp ... // jge %SinkMBB // // FalseMBB: // jmp %SinkMBB // // SinkMBB: // %v1 = phi[%f1, %FalseMBB], [%t1, %MBB] // %v2 = phi[%t2, %FalseMBB], [%f2, %MBB] ; For CMOV with OppCC switch // ; true-value with false-value // %v3 = phi[%f3, %FalseMBB], [%t1, %MBB] ; Phi instruction cannot use // ; previous Phi instruction result MachineInstr &MI = *Group.front(); MachineInstr *LastCMOV = Group.back(); DebugLoc DL = MI.getDebugLoc(); X86::CondCode CC = X86::CondCode(X86::getCondFromCMovOpc(MI.getOpcode())); X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC); // Potentially swap the condition codes so that any memory operand to a CMOV // is in the *false* position instead of the *true* position. We can invert // any non-memory operand CMOV instructions to cope with this and we ensure // memory operand CMOVs are only included with a single condition code. if (llvm::any_of(Group, [&](MachineInstr *I) { return I->mayLoad() && X86::getCondFromCMovOpc(I->getOpcode()) == CC; })) std::swap(CC, OppCC); MachineBasicBlock *MBB = MI.getParent(); MachineFunction::iterator It = ++MBB->getIterator(); MachineFunction *F = MBB->getParent(); const BasicBlock *BB = MBB->getBasicBlock(); MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(BB); MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB); F->insert(It, FalseMBB); F->insert(It, SinkMBB); // If the EFLAGS register isn't dead in the terminator, then claim that it's // live into the sink and copy blocks. if (checkEFLAGSLive(LastCMOV)) { FalseMBB->addLiveIn(X86::EFLAGS); SinkMBB->addLiveIn(X86::EFLAGS); } // Transfer the remainder of BB and its successor edges to SinkMBB. SinkMBB->splice(SinkMBB->begin(), MBB, std::next(MachineBasicBlock::iterator(LastCMOV)), MBB->end()); SinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // Add the false and sink blocks as its successors. MBB->addSuccessor(FalseMBB); MBB->addSuccessor(SinkMBB); // Create the conditional branch instruction. BuildMI(MBB, DL, TII->get(X86::GetCondBranchFromCond(CC))).addMBB(SinkMBB); // Add the sink block to the false block successors. FalseMBB->addSuccessor(SinkMBB); MachineInstrBuilder MIB; MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI); MachineBasicBlock::iterator MIItEnd = std::next(MachineBasicBlock::iterator(LastCMOV)); MachineBasicBlock::iterator FalseInsertionPoint = FalseMBB->begin(); MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin(); // First we need to insert an explicit load on the false path for any memory // operand. We also need to potentially do register rewriting here, but it is // simpler as the memory operands are always on the false path so we can // simply take that input, whatever it is. DenseMap<unsigned, unsigned> FalseBBRegRewriteTable; for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd;) { auto &MI = *MIIt++; // Skip any CMOVs in this group which don't load from memory. if (!MI.mayLoad()) { // Remember the false-side register input. unsigned FalseReg = MI.getOperand(X86::getCondFromCMovOpc(MI.getOpcode()) == CC ? 1 : 2) .getReg(); // Walk back through any intermediate cmovs referenced. while (true) { auto FRIt = FalseBBRegRewriteTable.find(FalseReg); if (FRIt == FalseBBRegRewriteTable.end()) break; FalseReg = FRIt->second; } FalseBBRegRewriteTable[MI.getOperand(0).getReg()] = FalseReg; continue; } // The condition must be the *opposite* of the one we've decided to branch // on as the branch will go *around* the load and the load should happen // when the CMOV condition is false. assert(X86::getCondFromCMovOpc(MI.getOpcode()) == OppCC && "Can only handle memory-operand cmov instructions with a condition " "opposite to the selected branch direction."); // The goal is to rewrite the cmov from: // // MBB: // %A = CMOVcc %B (tied), (mem) // // to // // MBB: // %A = CMOVcc %B (tied), %C // FalseMBB: // %C = MOV (mem) // // Which will allow the next loop to rewrite the CMOV in terms of a PHI: // // MBB: // JMP!cc SinkMBB // FalseMBB: // %C = MOV (mem) // SinkMBB: // %A = PHI [ %C, FalseMBB ], [ %B, MBB] // Get a fresh register to use as the destination of the MOV. const TargetRegisterClass *RC = MRI->getRegClass(MI.getOperand(0).getReg()); unsigned TmpReg = MRI->createVirtualRegister(RC); SmallVector<MachineInstr *, 4> NewMIs; bool Unfolded = TII->unfoldMemoryOperand(*MBB->getParent(), MI, TmpReg, /*UnfoldLoad*/ true, /*UnfoldStore*/ false, NewMIs); (void)Unfolded; assert(Unfolded && "Should never fail to unfold a loading cmov!"); // Move the new CMOV to just before the old one and reset any impacted // iterator. auto *NewCMOV = NewMIs.pop_back_val(); assert(X86::getCondFromCMovOpc(NewCMOV->getOpcode()) == OppCC && "Last new instruction isn't the expected CMOV!"); LLVM_DEBUG(dbgs() << "\tRewritten cmov: "; NewCMOV->dump()); MBB->insert(MachineBasicBlock::iterator(MI), NewCMOV); if (&*MIItBegin == &MI) MIItBegin = MachineBasicBlock::iterator(NewCMOV); // Sink whatever instructions were needed to produce the unfolded operand // into the false block. for (auto *NewMI : NewMIs) { LLVM_DEBUG(dbgs() << "\tRewritten load instr: "; NewMI->dump()); FalseMBB->insert(FalseInsertionPoint, NewMI); // Re-map any operands that are from other cmovs to the inputs for this block. for (auto &MOp : NewMI->uses()) { if (!MOp.isReg()) continue; auto It = FalseBBRegRewriteTable.find(MOp.getReg()); if (It == FalseBBRegRewriteTable.end()) continue; MOp.setReg(It->second); // This might have been a kill when it referenced the cmov result, but // it won't necessarily be once rewritten. // FIXME: We could potentially improve this by tracking whether the // operand to the cmov was also a kill, and then skipping the PHI node // construction below. MOp.setIsKill(false); } } MBB->erase(MachineBasicBlock::iterator(MI), std::next(MachineBasicBlock::iterator(MI))); // Add this PHI to the rewrite table. FalseBBRegRewriteTable[NewCMOV->getOperand(0).getReg()] = TmpReg; } // As we are creating the PHIs, we have to be careful if there is more than // one. Later CMOVs may reference the results of earlier CMOVs, but later // PHIs have to reference the individual true/false inputs from earlier PHIs. // That also means that PHI construction must work forward from earlier to // later, and that the code must maintain a mapping from earlier PHI's // destination registers, and the registers that went into the PHI. DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable; for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) { unsigned DestReg = MIIt->getOperand(0).getReg(); unsigned Op1Reg = MIIt->getOperand(1).getReg(); unsigned Op2Reg = MIIt->getOperand(2).getReg(); // If this CMOV we are processing is the opposite condition from the jump we // generated, then we have to swap the operands for the PHI that is going to // be generated. if (X86::getCondFromCMovOpc(MIIt->getOpcode()) == OppCC) std::swap(Op1Reg, Op2Reg); auto Op1Itr = RegRewriteTable.find(Op1Reg); if (Op1Itr != RegRewriteTable.end()) Op1Reg = Op1Itr->second.first; auto Op2Itr = RegRewriteTable.find(Op2Reg); if (Op2Itr != RegRewriteTable.end()) Op2Reg = Op2Itr->second.second; // SinkMBB: // %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, MBB ] // ... MIB = BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(X86::PHI), DestReg) .addReg(Op1Reg) .addMBB(FalseMBB) .addReg(Op2Reg) .addMBB(MBB); (void)MIB; LLVM_DEBUG(dbgs() << "\tFrom: "; MIIt->dump()); LLVM_DEBUG(dbgs() << "\tTo: "; MIB->dump()); // Add this PHI to the rewrite table. RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg); } // Now remove the CMOV(s). MBB->erase(MIItBegin, MIItEnd); }
bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI, unsigned Size) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC, ZERO, BEQ; if (Size == 4) { if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); BEQ = Mips::BEQ; } ZERO = Mips::ZERO; } else { LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; ZERO = Mips::ZERO_64; BEQ = Mips::BEQ64; } unsigned OldVal = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Incr = I->getOperand(2).getReg(); unsigned Scratch = I->getOperand(3).getReg(); unsigned Opcode = 0; unsigned OR = 0; unsigned AND = 0; unsigned NOR = 0; bool IsNand = false; switch (I->getOpcode()) { case Mips::ATOMIC_LOAD_ADD_I32_POSTRA: Opcode = Mips::ADDu; break; case Mips::ATOMIC_LOAD_SUB_I32_POSTRA: Opcode = Mips::SUBu; break; case Mips::ATOMIC_LOAD_AND_I32_POSTRA: Opcode = Mips::AND; break; case Mips::ATOMIC_LOAD_OR_I32_POSTRA: Opcode = Mips::OR; break; case Mips::ATOMIC_LOAD_XOR_I32_POSTRA: Opcode = Mips::XOR; break; case Mips::ATOMIC_LOAD_NAND_I32_POSTRA: IsNand = true; AND = Mips::AND; NOR = Mips::NOR; break; case Mips::ATOMIC_SWAP_I32_POSTRA: OR = Mips::OR; break; case Mips::ATOMIC_LOAD_ADD_I64_POSTRA: Opcode = Mips::DADDu; break; case Mips::ATOMIC_LOAD_SUB_I64_POSTRA: Opcode = Mips::DSUBu; break; case Mips::ATOMIC_LOAD_AND_I64_POSTRA: Opcode = Mips::AND64; break; case Mips::ATOMIC_LOAD_OR_I64_POSTRA: Opcode = Mips::OR64; break; case Mips::ATOMIC_LOAD_XOR_I64_POSTRA: Opcode = Mips::XOR64; break; case Mips::ATOMIC_LOAD_NAND_I64_POSTRA: IsNand = true; AND = Mips::AND64; NOR = Mips::NOR64; break; case Mips::ATOMIC_SWAP_I64_POSTRA: OR = Mips::OR64; break; default: llvm_unreachable("Unknown pseudo atomic!"); } const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); MF->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); loopMBB->addSuccessor(exitMBB); loopMBB->addSuccessor(loopMBB); loopMBB->normalizeSuccProbs(); BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!"); assert((OldVal != Incr) && "Clobbered the wrong reg!"); if (Opcode) { BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr); } else if (IsNand) { assert(AND && NOR && "Unknown nand instruction for atomic pseudo expansion"); BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr); BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch); } else { assert(OR && "Unknown instruction for atomic pseudo expansion!"); BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO); } BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0); BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB); NMBBI = BB.end(); I->eraseFromParent(); LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); return true; }
bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) { const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); TII = ST.getInstrInfo(); TRI = &TII->getRegisterInfo(); SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); bool HaveKill = false; bool NeedFlat = false; unsigned Depth = 0; MachineFunction::iterator NextBB; for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE; BI = NextBB) { NextBB = std::next(BI); MachineBasicBlock &MBB = *BI; MachineBasicBlock *EmptyMBBAtEnd = nullptr; MachineBasicBlock::iterator I, Next; bool ExecModified = false; for (I = MBB.begin(); I != MBB.end(); I = Next) { Next = std::next(I); MachineInstr &MI = *I; // Flat uses m0 in case it needs to access LDS. if (TII->isFLAT(MI)) NeedFlat = true; for (const auto &Def : I->defs()) { if (Def.isReg() && Def.isDef() && Def.getReg() == AMDGPU::EXEC) { ExecModified = true; break; } } switch (MI.getOpcode()) { default: break; case AMDGPU::SI_IF: ++Depth; If(MI); break; case AMDGPU::SI_ELSE: Else(MI, ExecModified); break; case AMDGPU::SI_BREAK: Break(MI); break; case AMDGPU::SI_IF_BREAK: IfBreak(MI); break; case AMDGPU::SI_ELSE_BREAK: ElseBreak(MI); break; case AMDGPU::SI_LOOP: ++Depth; Loop(MI); break; case AMDGPU::SI_END_CF: if (--Depth == 0 && HaveKill) { SkipIfDead(MI); HaveKill = false; } EndCf(MI); break; case AMDGPU::SI_KILL: if (Depth == 0) SkipIfDead(MI); else HaveKill = true; Kill(MI); break; case AMDGPU::S_BRANCH: Branch(MI); break; case AMDGPU::SI_INDIRECT_SRC_V1: case AMDGPU::SI_INDIRECT_SRC_V2: case AMDGPU::SI_INDIRECT_SRC_V4: case AMDGPU::SI_INDIRECT_SRC_V8: case AMDGPU::SI_INDIRECT_SRC_V16: if (indirectSrc(MI)) { // The block was split at this point. We can safely skip the middle // inserted block to the following which contains the rest of this // block's instructions. NextBB = std::next(BI); BE = MF.end(); Next = MBB.end(); } break; case AMDGPU::SI_INDIRECT_DST_V1: case AMDGPU::SI_INDIRECT_DST_V2: case AMDGPU::SI_INDIRECT_DST_V4: case AMDGPU::SI_INDIRECT_DST_V8: case AMDGPU::SI_INDIRECT_DST_V16: if (indirectDst(MI)) { // The block was split at this point. We can safely skip the middle // inserted block to the following which contains the rest of this // block's instructions. NextBB = std::next(BI); BE = MF.end(); Next = MBB.end(); } break; case AMDGPU::S_ENDPGM: { if (MF.getInfo<SIMachineFunctionInfo>()->returnsVoid()) break; // Graphics shaders returning non-void shouldn't contain S_ENDPGM, // because external bytecode will be appended at the end. if (BI != --MF.end() || I != MBB.getFirstTerminator()) { // S_ENDPGM is not the last instruction. Add an empty block at // the end and jump there. if (!EmptyMBBAtEnd) { EmptyMBBAtEnd = MF.CreateMachineBasicBlock(); MF.insert(MF.end(), EmptyMBBAtEnd); } MBB.addSuccessor(EmptyMBBAtEnd); BuildMI(*BI, I, MI.getDebugLoc(), TII->get(AMDGPU::S_BRANCH)) .addMBB(EmptyMBBAtEnd); } I->eraseFromParent(); break; } } } } if (NeedFlat && MFI->IsKernel) { // TODO: What to use with function calls? // We will need to Initialize the flat scratch register pair. if (NeedFlat) MFI->setHasFlatInstructions(true); } return true; }
MachineBasicBlock * SparcTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo(); unsigned BROpcode; unsigned CC; DebugLoc dl = MI->getDebugLoc(); // Figure out the conditional branch opcode to use for this select_cc. switch (MI->getOpcode()) { default: llvm_unreachable("Unknown SELECT_CC!"); case SP::SELECT_CC_Int_ICC: case SP::SELECT_CC_FP_ICC: case SP::SELECT_CC_DFP_ICC: BROpcode = SP::BCOND; break; case SP::SELECT_CC_Int_FCC: case SP::SELECT_CC_FP_FCC: case SP::SELECT_CC_DFP_FCC: BROpcode = SP::FBCOND; break; } CC = (SPCC::CondCodes)MI->getOperand(3).getImm(); // To "insert" a SELECT_CC instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // thisMBB: // ... // TrueVal = ... // [f]bCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); BuildMI(BB, dl, TII.get(BROpcode)).addMBB(sinkMBB).addImm(CC); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Update machine-CFG edges by first adding all successors of the current // block to the new block which will contain the Phi node for the select. for (MachineBasicBlock::succ_iterator I = BB->succ_begin(), E = BB->succ_end(); I != E; ++I) sinkMBB->addSuccessor(*I); // Next, remove all successors of the current block, and add the true // and fallthrough blocks as its successors. while (!BB->succ_empty()) BB->removeSuccessor(BB->succ_begin()); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = sinkMBB; BuildMI(BB, dl, TII.get(SP::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(1).getReg()).addMBB(thisMBB); F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. return BB; }
MachineBasicBlock * MipsTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); bool isFPCmp = false; DebugLoc dl = MI->getDebugLoc(); switch (MI->getOpcode()) { default: assert(false && "Unexpected instr type to insert"); case Mips::Select_FCC: case Mips::Select_FCC_S32: case Mips::Select_FCC_D32: isFPCmp = true; // FALL THROUGH case Mips::Select_CC: case Mips::Select_CC_S32: case Mips::Select_CC_D32: { // To "insert" a SELECT_CC instruction, we actually have to insert the // diamond control-flow pattern. The incoming instruction knows the // destination vreg to set, the condition code register to branch on, the // true/false values to select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // thisMBB: // ... // TrueVal = ... // setcc r1, r2, r3 // bNE r1, r0, copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); // Emit the right instruction according to the type of the operands compared if (isFPCmp) { // Find the condiction code present in the setcc operation. Mips::CondCode CC = (Mips::CondCode)MI->getOperand(4).getImm(); // Get the branch opcode from the branch code. unsigned Opc = FPBranchCodeToOpc(GetFPBranchCodeFromCond(CC)); BuildMI(BB, dl, TII->get(Opc)).addMBB(sinkMBB); } else BuildMI(BB, dl, TII->get(Mips::BNE)).addReg(MI->getOperand(1).getReg()) .addReg(Mips::ZERO).addMBB(sinkMBB); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Update machine-CFG edges by first adding all successors of the current // block to the new block which will contain the Phi node for the select. for(MachineBasicBlock::succ_iterator i = BB->succ_begin(), e = BB->succ_end(); i != e; ++i) sinkMBB->addSuccessor(*i); // Next, remove all successors of the current block, and add the true // and fallthrough blocks as its successors. while(!BB->succ_empty()) BB->removeSuccessor(BB->succ_begin()); BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = sinkMBB; BuildMI(BB, dl, TII->get(Mips::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(3).getReg()).addMBB(thisMBB); F->DeleteMachineInstr(MI); // The pseudo instruction is gone now. return BB; } } }
bool MipsExpandPseudo::expandAtomicCmpSwapSubword( MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC; unsigned ZERO = Mips::ZERO; unsigned BNE = Mips::BNE; unsigned BEQ = Mips::BEQ; unsigned SEOp = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH; if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Mask = I->getOperand(2).getReg(); unsigned ShiftCmpVal = I->getOperand(3).getReg(); unsigned Mask2 = I->getOperand(4).getReg(); unsigned ShiftNewVal = I->getOperand(5).getReg(); unsigned ShiftAmnt = I->getOperand(6).getReg(); unsigned Scratch = I->getOperand(7).getReg(); unsigned Scratch2 = I->getOperand(8).getReg(); // insert new blocks after the current block const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loop1MBB); MF->insert(It, loop2MBB); MF->insert(It, sinkMBB); MF->insert(It, exitMBB); // Transfer the remainder of BB and its successor edges to exitMBB. exitMBB->splice(exitMBB->begin(), &BB, std::next(MachineBasicBlock::iterator(I)), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); // thisMBB: // ... // fallthrough --> loop1MBB BB.addSuccessor(loop1MBB, BranchProbability::getOne()); loop1MBB->addSuccessor(sinkMBB); loop1MBB->addSuccessor(loop2MBB); loop1MBB->normalizeSuccProbs(); loop2MBB->addSuccessor(loop1MBB); loop2MBB->addSuccessor(sinkMBB); loop2MBB->normalizeSuccProbs(); sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); // loop1MBB: // ll dest, 0(ptr) // and Mask', dest, Mask // bne Mask', ShiftCmpVal, exitMBB BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0); BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2) .addReg(Scratch) .addReg(Mask); BuildMI(loop1MBB, DL, TII->get(BNE)) .addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB); // loop2MBB: // and dest, dest, mask2 // or dest, dest, ShiftNewVal // sc dest, dest, 0(ptr) // beq dest, $0, loop1MBB BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch) .addReg(Scratch, RegState::Kill) .addReg(Mask2); BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch) .addReg(Scratch, RegState::Kill) .addReg(ShiftNewVal); BuildMI(loop2MBB, DL, TII->get(SC), Scratch) .addReg(Scratch, RegState::Kill) .addReg(Ptr) .addImm(0); BuildMI(loop2MBB, DL, TII->get(BEQ)) .addReg(Scratch, RegState::Kill) .addReg(ZERO) .addMBB(loop1MBB); // sinkMBB: // srl srlres, Mask', shiftamt // sign_extend dest,srlres BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) .addReg(Scratch2) .addReg(ShiftAmnt); if (STI->hasMips32r2()) { BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); } else { const unsigned ShiftImm = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24; BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); } LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loop1MBB); computeAndAddLiveIns(LiveRegs, *loop2MBB); computeAndAddLiveIns(LiveRegs, *sinkMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }
MachineBasicBlock * AVM2TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { const TargetMachine &TM = getTargetMachine(); const TargetInstrInfo &TII = *TM.getInstrInfo(); unsigned BROpcode; DebugLoc dl = MI->getDebugLoc(); bool useIntrin = TM.getSubtarget<AVM2Subtarget>().useIntrinsics(); // Figure out the conditional branch opcode to use for this select_cc. switch (MI->getOpcode()) { default: llvm_unreachable("Unknown SELECT_CC!"); case AVM2::SL: case AVM2::SLF: BROpcode = useIntrin ? AVM2::inCBR : AVM2::asCBR; break; case AVM2::FSL: case AVM2::FSLF: BROpcode = useIntrin ? AVM2::inFCBR : AVM2::asFCBR; break; } // To "insert" a SELECT_CC instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = BB; ++It; // thisMBB: // ... // TrueVal = ... // [f]bCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); // Add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); BuildMI(BB, dl, TII.get(BROpcode)) .addMBB(sinkMBB) .addImm(MI->getOperand(1).getImm()) // CC .addReg(MI->getOperand(2).getReg()) .addReg(MI->getOperand(3).getReg()); // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = sinkMBB; BuildMI(*BB, BB->begin(), dl, TII.get(AVM2::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(5).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(4).getReg()).addMBB(thisMBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return BB; }
bool MIRParserImpl::initializeMachineFunction(MachineFunction &MF) { auto It = Functions.find(MF.getName()); if (It == Functions.end()) return error(Twine("no machine function information for function '") + MF.getName() + "' in the MIR file"); // TODO: Recreate the machine function. const yaml::MachineFunction &YamlMF = *It->getValue(); if (YamlMF.Alignment) MF.setAlignment(YamlMF.Alignment); MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice); MF.setHasInlineAsm(YamlMF.HasInlineAsm); PerFunctionMIParsingState PFS; if (initializeRegisterInfo(MF, YamlMF, PFS)) return true; if (!YamlMF.Constants.empty()) { auto *ConstantPool = MF.getConstantPool(); assert(ConstantPool && "Constant pool must be created"); if (initializeConstantPool(*ConstantPool, YamlMF, MF, PFS.ConstantPoolSlots)) return true; } const auto &F = *MF.getFunction(); for (const auto &YamlMBB : YamlMF.BasicBlocks) { const BasicBlock *BB = nullptr; const yaml::StringValue &Name = YamlMBB.Name; const yaml::StringValue &IRBlock = YamlMBB.IRBlock; if (!Name.Value.empty()) { BB = dyn_cast_or_null<BasicBlock>( F.getValueSymbolTable().lookup(Name.Value)); if (!BB) return error(Name.SourceRange.Start, Twine("basic block '") + Name.Value + "' is not defined in the function '" + MF.getName() + "'"); } if (!IRBlock.Value.empty()) { // TODO: Report an error when both name and ir block are specified. SMDiagnostic Error; if (parseIRBlockReference(BB, SM, MF, IRBlock.Value, PFS, IRSlots, Error)) return error(Error, IRBlock.SourceRange); } auto *MBB = MF.CreateMachineBasicBlock(BB); MF.insert(MF.end(), MBB); bool WasInserted = PFS.MBBSlots.insert(std::make_pair(YamlMBB.ID, MBB)).second; if (!WasInserted) return error(Twine("redefinition of machine basic block with id #") + Twine(YamlMBB.ID)); } if (YamlMF.BasicBlocks.empty()) return error(Twine("machine function '") + Twine(MF.getName()) + "' requires at least one machine basic block in its body"); // Initialize the frame information after creating all the MBBs so that the // MBB references in the frame information can be resolved. if (initializeFrameInfo(MF, YamlMF, PFS)) return true; // Initialize the jump table after creating all the MBBs so that the MBB // references can be resolved. if (!YamlMF.JumpTableInfo.Entries.empty() && initializeJumpTableInfo(MF, YamlMF.JumpTableInfo, PFS)) return true; // Initialize the machine basic blocks after creating them all so that the // machine instructions parser can resolve the MBB references. unsigned I = 0; for (const auto &YamlMBB : YamlMF.BasicBlocks) { if (initializeMachineBasicBlock(MF, *MF.getBlockNumbered(I++), YamlMBB, PFS)) return true; } // FIXME: This is a temporary workaround until the reserved registers can be // serialized. MF.getRegInfo().freezeReservedRegs(MF); MF.verify(); return false; }
MachineBasicBlock* MSP430TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { unsigned Opc = MI->getOpcode(); if (Opc == MSP430::Shl8 || Opc == MSP430::Shl16 || Opc == MSP430::Sra8 || Opc == MSP430::Sra16 || Opc == MSP430::Srl8 || Opc == MSP430::Srl16) return EmitShiftInstr(MI, BB); const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo(); DebugLoc dl = MI->getDebugLoc(); assert((Opc == MSP430::Select16 || Opc == MSP430::Select8) && "Unexpected instr type to insert"); // To "insert" a SELECT instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator I = BB; ++I; // thisMBB: // ... // TrueVal = ... // cmpTY ccX, r1, r2 // jCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(I, copy0MBB); F->insert(I, copy1MBB); // Update machine-CFG edges by transferring all successors of the current // block to the new block which will contain the Phi node for the select. copy1MBB->splice(copy1MBB->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); copy1MBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(copy1MBB); BuildMI(BB, dl, TII.get(MSP430::JCC)) .addMBB(copy1MBB) .addImm(MI->getOperand(3).getImm()); // copy0MBB: // %FalseValue = ... // # fallthrough to copy1MBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(copy1MBB); // copy1MBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = copy1MBB; BuildMI(*BB, BB->begin(), dl, TII.get(MSP430::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB) .addReg(MI->getOperand(1).getReg()).addMBB(thisMBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return BB; }
MachineBasicBlock * MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) { // Splitting the critical edge to a landing pad block is non-trivial. Don't do // it in this generic function. if (Succ->isLandingPad()) return nullptr; MachineFunction *MF = getParent(); DebugLoc dl; // FIXME: this is nowhere // Performance might be harmed on HW that implements branching using exec mask // where both sides of the branches are always executed. if (MF->getTarget().requiresStructuredCFG()) return nullptr; // We may need to update this's terminator, but we can't do that if // AnalyzeBranch fails. If this uses a jump table, we won't touch it. const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); MachineBasicBlock *TBB = nullptr, *FBB = nullptr; SmallVector<MachineOperand, 4> Cond; if (TII->AnalyzeBranch(*this, TBB, FBB, Cond)) return nullptr; // Avoid bugpoint weirdness: A block may end with a conditional branch but // jumps to the same MBB is either case. We have duplicate CFG edges in that // case that we can't handle. Since this never happens in properly optimized // code, just skip those edges. if (TBB && TBB == FBB) { DEBUG(dbgs() << "Won't split critical edge after degenerate BB#" << getNumber() << '\n'); return nullptr; } MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock(); MF->insert(std::next(MachineFunction::iterator(this)), NMBB); DEBUG(dbgs() << "Splitting critical edge:" " BB#" << getNumber() << " -- BB#" << NMBB->getNumber() << " -- BB#" << Succ->getNumber() << '\n'); LiveIntervals *LIS = P->getAnalysisIfAvailable<LiveIntervals>(); SlotIndexes *Indexes = P->getAnalysisIfAvailable<SlotIndexes>(); if (LIS) LIS->insertMBBInMaps(NMBB); else if (Indexes) Indexes->insertMBBInMaps(NMBB); // On some targets like Mips, branches may kill virtual registers. Make sure // that LiveVariables is properly updated after updateTerminator replaces the // terminators. LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>(); // Collect a list of virtual registers killed by the terminators. SmallVector<unsigned, 4> KilledRegs; if (LV) for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { if (!OI->isReg() || OI->getReg() == 0 || !OI->isUse() || !OI->isKill() || OI->isUndef()) continue; unsigned Reg = OI->getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg) || LV->getVarInfo(Reg).removeKill(MI)) { KilledRegs.push_back(Reg); DEBUG(dbgs() << "Removing terminator kill: " << *MI); OI->setIsKill(false); } } } SmallVector<unsigned, 4> UsedRegs; if (LIS) { for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) { MachineInstr *MI = I; for (MachineInstr::mop_iterator OI = MI->operands_begin(), OE = MI->operands_end(); OI != OE; ++OI) { if (!OI->isReg() || OI->getReg() == 0) continue; unsigned Reg = OI->getReg(); if (std::find(UsedRegs.begin(), UsedRegs.end(), Reg) == UsedRegs.end()) UsedRegs.push_back(Reg); } } } ReplaceUsesOfBlockWith(Succ, NMBB); // If updateTerminator() removes instructions, we need to remove them from // SlotIndexes. SmallVector<MachineInstr*, 4> Terminators; if (Indexes) { for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) Terminators.push_back(I); } updateTerminator(); if (Indexes) { SmallVector<MachineInstr*, 4> NewTerminators; for (instr_iterator I = getFirstInstrTerminator(), E = instr_end(); I != E; ++I) NewTerminators.push_back(I); for (SmallVectorImpl<MachineInstr*>::iterator I = Terminators.begin(), E = Terminators.end(); I != E; ++I) { if (std::find(NewTerminators.begin(), NewTerminators.end(), *I) == NewTerminators.end()) Indexes->removeMachineInstrFromMaps(*I); } } // Insert unconditional "jump Succ" instruction in NMBB if necessary. NMBB->addSuccessor(Succ); if (!NMBB->isLayoutSuccessor(Succ)) { Cond.clear(); MF->getSubtarget().getInstrInfo()->InsertBranch(*NMBB, Succ, nullptr, Cond, dl); if (Indexes) { for (instr_iterator I = NMBB->instr_begin(), E = NMBB->instr_end(); I != E; ++I) { // Some instructions may have been moved to NMBB by updateTerminator(), // so we first remove any instruction that already has an index. if (Indexes->hasIndex(I)) Indexes->removeMachineInstrFromMaps(I); Indexes->insertMachineInstrInMaps(I); } } } // Fix PHI nodes in Succ so they refer to NMBB instead of this for (MachineBasicBlock::instr_iterator i = Succ->instr_begin(),e = Succ->instr_end(); i != e && i->isPHI(); ++i) for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2) if (i->getOperand(ni+1).getMBB() == this) i->getOperand(ni+1).setMBB(NMBB); // Inherit live-ins from the successor for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(), E = Succ->livein_end(); I != E; ++I) NMBB->addLiveIn(*I); // Update LiveVariables. const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); if (LV) { // Restore kills of virtual registers that were killed by the terminators. while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) { if (!(--I)->addRegisterKilled(Reg, TRI, /* addIfNotFound= */ false)) continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) LV->getVarInfo(Reg).Kills.push_back(I); DEBUG(dbgs() << "Restored terminator kill: " << *I); break; } } // Update relevant live-through information. LV->addNewBlock(NMBB, this, Succ); } if (LIS) { // After splitting the edge and updating SlotIndexes, live intervals may be // in one of two situations, depending on whether this block was the last in // the function. If the original block was the last in the function, all live // intervals will end prior to the beginning of the new split block. If the // original block was not at the end of the function, all live intervals will // extend to the end of the new split block. bool isLastMBB = std::next(MachineFunction::iterator(NMBB)) == getParent()->end(); SlotIndex StartIndex = Indexes->getMBBEndIdx(this); SlotIndex PrevIndex = StartIndex.getPrevSlot(); SlotIndex EndIndex = Indexes->getMBBEndIdx(NMBB); // Find the registers used from NMBB in PHIs in Succ. SmallSet<unsigned, 8> PHISrcRegs; for (MachineBasicBlock::instr_iterator I = Succ->instr_begin(), E = Succ->instr_end(); I != E && I->isPHI(); ++I) { for (unsigned ni = 1, ne = I->getNumOperands(); ni != ne; ni += 2) { if (I->getOperand(ni+1).getMBB() == NMBB) { MachineOperand &MO = I->getOperand(ni); unsigned Reg = MO.getReg(); PHISrcRegs.insert(Reg); if (MO.isUndef()) continue; LiveInterval &LI = LIS->getInterval(Reg); VNInfo *VNI = LI.getVNInfoAt(PrevIndex); assert(VNI && "PHI sources should be live out of their predecessors."); LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI)); } } } MachineRegisterInfo *MRI = &getParent()->getRegInfo(); for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) { unsigned Reg = TargetRegisterInfo::index2VirtReg(i); if (PHISrcRegs.count(Reg) || !LIS->hasInterval(Reg)) continue; LiveInterval &LI = LIS->getInterval(Reg); if (!LI.liveAt(PrevIndex)) continue; bool isLiveOut = LI.liveAt(LIS->getMBBStartIdx(Succ)); if (isLiveOut && isLastMBB) { VNInfo *VNI = LI.getVNInfoAt(PrevIndex); assert(VNI && "LiveInterval should have VNInfo where it is live."); LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI)); } else if (!isLiveOut && !isLastMBB) { LI.removeSegment(StartIndex, EndIndex); } } // Update all intervals for registers whose uses may have been modified by // updateTerminator(). LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs); } if (MachineDominatorTree *MDT = P->getAnalysisIfAvailable<MachineDominatorTree>()) MDT->recordSplitCriticalEdge(this, Succ, NMBB); if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>()) if (MachineLoop *TIL = MLI->getLoopFor(this)) { // If one or the other blocks were not in a loop, the new block is not // either, and thus LI doesn't need to be updated. if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) { if (TIL == DestLoop) { // Both in the same loop, the NMBB joins loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (TIL->contains(DestLoop)) { // Edge from an outer loop to an inner loop. Add to the outer loop. TIL->addBasicBlockToLoop(NMBB, MLI->getBase()); } else if (DestLoop->contains(TIL)) { // Edge from an inner loop to an outer loop. Add to the outer loop. DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase()); } else { // Edge from two loops with no containment relation. Because these // are natural loops, we know that the destination block must be the // header of its loop (adding a branch into a loop elsewhere would // create an irreducible loop). assert(DestLoop->getHeader() == Succ && "Should not create irreducible loops!"); if (MachineLoop *P = DestLoop->getParentLoop()) P->addBasicBlockToLoop(NMBB, MLI->getBase()); } } } return NMBB; }
bool MipsExpandPseudo::expandAtomicBinOpSubword( MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC; unsigned BEQ = Mips::BEQ; unsigned SEOp = Mips::SEH; if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); } bool IsSwap = false; bool IsNand = false; unsigned Opcode = 0; switch (I->getOpcode()) { case Mips::ATOMIC_LOAD_NAND_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_NAND_I16_POSTRA: IsNand = true; break; case Mips::ATOMIC_SWAP_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_SWAP_I16_POSTRA: IsSwap = true; break; case Mips::ATOMIC_LOAD_ADD_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_ADD_I16_POSTRA: Opcode = Mips::ADDu; break; case Mips::ATOMIC_LOAD_SUB_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_SUB_I16_POSTRA: Opcode = Mips::SUBu; break; case Mips::ATOMIC_LOAD_AND_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_AND_I16_POSTRA: Opcode = Mips::AND; break; case Mips::ATOMIC_LOAD_OR_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_OR_I16_POSTRA: Opcode = Mips::OR; break; case Mips::ATOMIC_LOAD_XOR_I8_POSTRA: SEOp = Mips::SEB; LLVM_FALLTHROUGH; case Mips::ATOMIC_LOAD_XOR_I16_POSTRA: Opcode = Mips::XOR; break; default: llvm_unreachable("Unknown subword atomic pseudo for expansion!"); } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned Incr = I->getOperand(2).getReg(); unsigned Mask = I->getOperand(3).getReg(); unsigned Mask2 = I->getOperand(4).getReg(); unsigned ShiftAmnt = I->getOperand(5).getReg(); unsigned OldVal = I->getOperand(6).getReg(); unsigned BinOpRes = I->getOperand(7).getReg(); unsigned StoreVal = I->getOperand(8).getReg(); const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loopMBB); MF->insert(It, sinkMBB); MF->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); BB.addSuccessor(loopMBB, BranchProbability::getOne()); loopMBB->addSuccessor(sinkMBB); loopMBB->addSuccessor(loopMBB); loopMBB->normalizeSuccProbs(); BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0); if (IsNand) { // and andres, oldval, incr2 // nor binopres, $0, andres // and newval, binopres, mask BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(OldVal) .addReg(Incr); BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes) .addReg(Mips::ZERO) .addReg(BinOpRes); BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(BinOpRes) .addReg(Mask); } else if (!IsSwap) { // <binop> binopres, oldval, incr2 // and newval, binopres, mask BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes) .addReg(OldVal) .addReg(Incr); BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(BinOpRes) .addReg(Mask); } else { // atomic.swap // and newval, incr2, mask BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes) .addReg(Incr) .addReg(Mask); } // and StoreVal, OlddVal, Mask2 // or StoreVal, StoreVal, BinOpRes // StoreVal<tied1> = sc StoreVal, 0(Ptr) // beq StoreVal, zero, loopMBB BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal) .addReg(OldVal).addReg(Mask2); BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal) .addReg(StoreVal).addReg(BinOpRes); BuildMI(loopMBB, DL, TII->get(SC), StoreVal) .addReg(StoreVal).addReg(Ptr).addImm(0); BuildMI(loopMBB, DL, TII->get(BEQ)) .addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB); // sinkMBB: // and maskedoldval1,oldval,mask // srl srlres,maskedoldval1,shiftamt // sign_extend dest,srlres sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne()); BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest) .addReg(OldVal).addReg(Mask); BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest) .addReg(Dest).addReg(ShiftAmnt); if (STI->hasMips32r2()) { BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest); } else { const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24; BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest) .addReg(Dest, RegState::Kill) .addImm(ShiftImm); } LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loopMBB); computeAndAddLiveIns(LiveRegs, *sinkMBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }
MachineBasicBlock * BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, MachineBasicBlock *BB) const { unsigned Opc = MI->getOpcode(); const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo(); DebugLoc DL = MI->getDebugLoc(); assert(Opc == BPF::Select && "Unexpected instr type to insert"); // To "insert" a SELECT instruction, we actually have to insert the diamond // control-flow pattern. The incoming instruction knows the destination vreg // to set, the condition code register to branch on, the true/false values to // select between, and a branch opcode to use. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator I = BB; ++I; // ThisMBB: // ... // TrueVal = ... // jmp_XX r1, r2 goto Copy1MBB // fallthrough --> Copy0MBB MachineBasicBlock *ThisMBB = BB; MachineFunction *F = BB->getParent(); MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(I, Copy0MBB); F->insert(I, Copy1MBB); // Update machine-CFG edges by transferring all successors of the current // block to the new block which will contain the Phi node for the select. Copy1MBB->splice(Copy1MBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); Copy1MBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(Copy0MBB); BB->addSuccessor(Copy1MBB); // Insert Branch if Flag unsigned LHS = MI->getOperand(1).getReg(); unsigned RHS = MI->getOperand(2).getReg(); int CC = MI->getOperand(3).getImm(); switch (CC) { case ISD::SETGT: BuildMI(BB, DL, TII.get(BPF::JSGT_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; case ISD::SETUGT: BuildMI(BB, DL, TII.get(BPF::JUGT_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; case ISD::SETGE: BuildMI(BB, DL, TII.get(BPF::JSGE_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; case ISD::SETUGE: BuildMI(BB, DL, TII.get(BPF::JUGE_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; case ISD::SETEQ: BuildMI(BB, DL, TII.get(BPF::JEQ_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; case ISD::SETNE: BuildMI(BB, DL, TII.get(BPF::JNE_rr)) .addReg(LHS) .addReg(RHS) .addMBB(Copy1MBB); break; default: report_fatal_error("unimplemented select CondCode " + Twine(CC)); } // Copy0MBB: // %FalseValue = ... // # fallthrough to Copy1MBB BB = Copy0MBB; // Update machine-CFG edges BB->addSuccessor(Copy1MBB); // Copy1MBB: // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ] // ... BB = Copy1MBB; BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI->getOperand(0).getReg()) .addReg(MI->getOperand(5).getReg()) .addMBB(Copy0MBB) .addReg(MI->getOperand(4).getReg()) .addMBB(ThisMBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return BB; }
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop) { MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin(); SetVector<MachineBasicBlock *> RewriteSuccs; // DFS through Loop's body, looking for for irreducible control flow. Loop is // natural, and we stay in its body, and we treat any nested loops // monolithically, so any cycles we encounter indicate irreducibility. SmallPtrSet<MachineBasicBlock *, 8> OnStack; SmallPtrSet<MachineBasicBlock *, 8> Visited; SmallVector<SuccessorList, 4> LoopWorklist; LoopWorklist.push_back(SuccessorList(Header)); OnStack.insert(Header); Visited.insert(Header); while (!LoopWorklist.empty()) { SuccessorList &Top = LoopWorklist.back(); if (Top.HasNext()) { MachineBasicBlock *Next = Top.Next(); if (Next == Header || (Loop && !Loop->contains(Next))) continue; if (LLVM_LIKELY(OnStack.insert(Next).second)) { if (!Visited.insert(Next).second) { OnStack.erase(Next); continue; } MachineLoop *InnerLoop = MLI.getLoopFor(Next); if (InnerLoop != Loop) LoopWorklist.push_back(SuccessorList(InnerLoop)); else LoopWorklist.push_back(SuccessorList(Next)); } else { RewriteSuccs.insert(Top.getBlock()); } continue; } OnStack.erase(Top.getBlock()); LoopWorklist.pop_back(); } // Most likely, we didn't find any irreducible control flow. if (LLVM_LIKELY(RewriteSuccs.empty())) return false; DEBUG(dbgs() << "Irreducible control flow detected!\n"); // Ok. We have irreducible control flow! Create a dispatch block which will // contains a jump table to any block in the problematic set of blocks. MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock(); MF.insert(MF.end(), Dispatch); MLI.changeLoopFor(Dispatch, Loop); // Add the jump table. const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo(); MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(), TII.get(WebAssembly::BR_TABLE_I32)); // Add the register which will be used to tell the jump table which block to // jump to. MachineRegisterInfo &MRI = MF.getRegInfo(); unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); MIB.addReg(Reg); // Collect all the blocks which need to have their successors rewritten, // add the successors to the jump table, and remember their index. DenseMap<MachineBasicBlock *, unsigned> Indices; SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(), RewriteSuccs.end()); while (!SuccWorklist.empty()) { MachineBasicBlock *MBB = SuccWorklist.pop_back_val(); auto Pair = Indices.insert(std::make_pair(MBB, 0)); if (!Pair.second) continue; unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1; DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n"); Pair.first->second = Index; for (auto Pred : MBB->predecessors()) RewriteSuccs.insert(Pred); MIB.addMBB(MBB); Dispatch->addSuccessor(MBB); MetaBlock Meta(MBB); for (auto *Succ : Meta.successors()) if (Succ != Header && (!Loop || Loop->contains(Succ))) SuccWorklist.push_back(Succ); } // Rewrite the problematic successors for every block in RewriteSuccs. // For simplicity, we just introduce a new block for every edge we need to // rewrite. Fancier things are possible. for (MachineBasicBlock *MBB : RewriteSuccs) { DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map; for (auto *Succ : MBB->successors()) { if (!Indices.count(Succ)) continue; MachineBasicBlock *Split = MF.CreateMachineBasicBlock(); MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ) : MF.end(), Split); MLI.changeLoopFor(Split, Loop); // Set the jump table's register of the index of the block we wish to // jump to, and jump to the jump table. BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32), Reg) .addImm(Indices[Succ]); BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR)) .addMBB(Dispatch); Split->addSuccessor(Dispatch); Map[Succ] = Split; } // Remap the terminator operands and the successor list. for (MachineInstr &Term : MBB->terminators()) for (auto &Op : Term.explicit_uses()) if (Op.isMBB() && Indices.count(Op.getMBB())) Op.setMBB(Map[Op.getMBB()]); for (auto Rewrite : Map) MBB->replaceSuccessor(Rewrite.first, Rewrite.second); } // Create a fake default label, because br_table requires one. MIB.addMBB(MIB.getInstr() ->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1) .getMBB()); return true; }
MachineBasicBlock* MSP430TargetLowering::EmitShiftInstr(MachineInstr *MI, MachineBasicBlock *BB) const { MachineFunction *F = BB->getParent(); MachineRegisterInfo &RI = F->getRegInfo(); DebugLoc dl = MI->getDebugLoc(); const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo(); unsigned Opc; const TargetRegisterClass * RC; switch (MI->getOpcode()) { default: assert(0 && "Invalid shift opcode!"); case MSP430::Shl8: Opc = MSP430::SHL8r1; RC = MSP430::GR8RegisterClass; break; case MSP430::Shl16: Opc = MSP430::SHL16r1; RC = MSP430::GR16RegisterClass; break; case MSP430::Sra8: Opc = MSP430::SAR8r1; RC = MSP430::GR8RegisterClass; break; case MSP430::Sra16: Opc = MSP430::SAR16r1; RC = MSP430::GR16RegisterClass; break; case MSP430::Srl8: Opc = MSP430::SAR8r1c; RC = MSP430::GR8RegisterClass; break; case MSP430::Srl16: Opc = MSP430::SAR16r1c; RC = MSP430::GR16RegisterClass; break; } const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator I = BB; ++I; // Create loop block MachineBasicBlock *LoopBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *RemBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(I, LoopBB); F->insert(I, RemBB); // Update machine-CFG edges by transferring all successors of the current // block to the block containing instructions after shift. RemBB->splice(RemBB->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), BB->end()); RemBB->transferSuccessorsAndUpdatePHIs(BB); // Add adges BB => LoopBB => RemBB, BB => RemBB, LoopBB => LoopBB BB->addSuccessor(LoopBB); BB->addSuccessor(RemBB); LoopBB->addSuccessor(RemBB); LoopBB->addSuccessor(LoopBB); unsigned ShiftAmtReg = RI.createVirtualRegister(MSP430::GR8RegisterClass); unsigned ShiftAmtReg2 = RI.createVirtualRegister(MSP430::GR8RegisterClass); unsigned ShiftReg = RI.createVirtualRegister(RC); unsigned ShiftReg2 = RI.createVirtualRegister(RC); unsigned ShiftAmtSrcReg = MI->getOperand(2).getReg(); unsigned SrcReg = MI->getOperand(1).getReg(); unsigned DstReg = MI->getOperand(0).getReg(); // BB: // cmp 0, N // je RemBB BuildMI(BB, dl, TII.get(MSP430::CMP8ri)) .addReg(ShiftAmtSrcReg).addImm(0); BuildMI(BB, dl, TII.get(MSP430::JCC)) .addMBB(RemBB) .addImm(MSP430CC::COND_E); // LoopBB: // ShiftReg = phi [%SrcReg, BB], [%ShiftReg2, LoopBB] // ShiftAmt = phi [%N, BB], [%ShiftAmt2, LoopBB] // ShiftReg2 = shift ShiftReg // ShiftAmt2 = ShiftAmt - 1; BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftReg) .addReg(SrcReg).addMBB(BB) .addReg(ShiftReg2).addMBB(LoopBB); BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftAmtReg) .addReg(ShiftAmtSrcReg).addMBB(BB) .addReg(ShiftAmtReg2).addMBB(LoopBB); BuildMI(LoopBB, dl, TII.get(Opc), ShiftReg2) .addReg(ShiftReg); BuildMI(LoopBB, dl, TII.get(MSP430::SUB8ri), ShiftAmtReg2) .addReg(ShiftAmtReg).addImm(1); BuildMI(LoopBB, dl, TII.get(MSP430::JCC)) .addMBB(LoopBB) .addImm(MSP430CC::COND_NE); // RemBB: // DestReg = phi [%SrcReg, BB], [%ShiftReg, LoopBB] BuildMI(*RemBB, RemBB->begin(), dl, TII.get(MSP430::PHI), DstReg) .addReg(SrcReg).addMBB(BB) .addReg(ShiftReg2).addMBB(LoopBB); MI->eraseFromParent(); // The pseudo instruction is gone now. return RemBB; }
void X86ExpandPseudo::ExpandICallBranchFunnel( MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) { MachineBasicBlock *JTMBB = MBB; MachineInstr *JTInst = &*MBBI; MachineFunction *MF = MBB->getParent(); const BasicBlock *BB = MBB->getBasicBlock(); auto InsPt = MachineFunction::iterator(MBB); ++InsPt; std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs; DebugLoc DL = JTInst->getDebugLoc(); MachineOperand Selector = JTInst->getOperand(0); const GlobalValue *CombinedGlobal = JTInst->getOperand(1).getGlobal(); auto CmpTarget = [&](unsigned Target) { BuildMI(*MBB, MBBI, DL, TII->get(X86::LEA64r), X86::R11) .addReg(X86::RIP) .addImm(1) .addReg(0) .addGlobalAddress(CombinedGlobal, JTInst->getOperand(2 + 2 * Target).getImm()) .addReg(0); BuildMI(*MBB, MBBI, DL, TII->get(X86::CMP64rr)) .add(Selector) .addReg(X86::R11); }; auto CreateMBB = [&]() { auto *NewMBB = MF->CreateMachineBasicBlock(BB); MBB->addSuccessor(NewMBB); return NewMBB; }; auto EmitCondJump = [&](unsigned Opcode, MachineBasicBlock *ThenMBB) { BuildMI(*MBB, MBBI, DL, TII->get(Opcode)).addMBB(ThenMBB); auto *ElseMBB = CreateMBB(); MF->insert(InsPt, ElseMBB); MBB = ElseMBB; MBBI = MBB->end(); }; auto EmitCondJumpTarget = [&](unsigned Opcode, unsigned Target) { auto *ThenMBB = CreateMBB(); TargetMBBs.push_back({ThenMBB, Target}); EmitCondJump(Opcode, ThenMBB); }; auto EmitTailCall = [&](unsigned Target) { BuildMI(*MBB, MBBI, DL, TII->get(X86::TAILJMPd64)) .add(JTInst->getOperand(3 + 2 * Target)); }; std::function<void(unsigned, unsigned)> EmitBranchFunnel = [&](unsigned FirstTarget, unsigned NumTargets) { if (NumTargets == 1) { EmitTailCall(FirstTarget); return; } if (NumTargets == 2) { CmpTarget(FirstTarget + 1); EmitCondJumpTarget(X86::JB_1, FirstTarget); EmitTailCall(FirstTarget + 1); return; } if (NumTargets < 6) { CmpTarget(FirstTarget + 1); EmitCondJumpTarget(X86::JB_1, FirstTarget); EmitCondJumpTarget(X86::JE_1, FirstTarget + 1); EmitBranchFunnel(FirstTarget + 2, NumTargets - 2); return; } auto *ThenMBB = CreateMBB(); CmpTarget(FirstTarget + (NumTargets / 2)); EmitCondJump(X86::JB_1, ThenMBB); EmitCondJumpTarget(X86::JE_1, FirstTarget + (NumTargets / 2)); EmitBranchFunnel(FirstTarget + (NumTargets / 2) + 1, NumTargets - (NumTargets / 2) - 1); MF->insert(InsPt, ThenMBB); MBB = ThenMBB; MBBI = MBB->end(); EmitBranchFunnel(FirstTarget, NumTargets / 2); }; EmitBranchFunnel(0, (JTInst->getNumOperands() - 2) / 2); for (auto P : TargetMBBs) { MF->insert(InsPt, P.first); BuildMI(P.first, DL, TII->get(X86::TAILJMPd64)) .add(JTInst->getOperand(3 + 2 * P.second)); } JTMBB->erase(JTInst); }
bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) { const SISubtarget &ST = MF.getSubtarget<SISubtarget>(); TII = ST.getInstrInfo(); TRI = &TII->getRegisterInfo(); SkipThreshold = SkipThresholdFlag; bool HaveKill = false; bool MadeChange = false; // Track depth of exec mask, divergent branches. SmallVector<MachineBasicBlock *, 16> ExecBranchStack; MachineFunction::iterator NextBB; MachineBasicBlock *EmptyMBBAtEnd = nullptr; for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE; BI = NextBB) { NextBB = std::next(BI); MachineBasicBlock &MBB = *BI; if (!ExecBranchStack.empty() && ExecBranchStack.back() == &MBB) { // Reached convergence point for last divergent branch. ExecBranchStack.pop_back(); } if (HaveKill && ExecBranchStack.empty()) { HaveKill = false; // TODO: Insert skip if exec is 0? } MachineBasicBlock::iterator I, Next; for (I = MBB.begin(); I != MBB.end(); I = Next) { Next = std::next(I); MachineInstr &MI = *I; switch (MI.getOpcode()) { case AMDGPU::SI_MASK_BRANCH: { ExecBranchStack.push_back(MI.getOperand(0).getMBB()); MadeChange |= skipMaskBranch(MI, MBB); break; } case AMDGPU::S_BRANCH: { // Optimize out branches to the next block. // FIXME: Shouldn't this be handled by BranchFolding? if (MBB.isLayoutSuccessor(MI.getOperand(0).getMBB())) MI.eraseFromParent(); break; } case AMDGPU::SI_KILL_TERMINATOR: { MadeChange = true; kill(MI); if (ExecBranchStack.empty()) { if (skipIfDead(MI, *NextBB)) { NextBB = std::next(BI); BE = MF.end(); Next = MBB.end(); } } else { HaveKill = true; } MI.eraseFromParent(); break; } case AMDGPU::SI_RETURN: { // FIXME: Should move somewhere else assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid()); // Graphics shaders returning non-void shouldn't contain S_ENDPGM, // because external bytecode will be appended at the end. if (BI != --MF.end() || I != MBB.getFirstTerminator()) { // SI_RETURN is not the last instruction. Add an empty block at // the end and jump there. if (!EmptyMBBAtEnd) { EmptyMBBAtEnd = MF.CreateMachineBasicBlock(); MF.insert(MF.end(), EmptyMBBAtEnd); } MBB.addSuccessor(EmptyMBBAtEnd); BuildMI(*BI, I, MI.getDebugLoc(), TII->get(AMDGPU::S_BRANCH)) .addMBB(EmptyMBBAtEnd); I->eraseFromParent(); } } default: break; } } } return MadeChange; }
bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB, MachineBasicBlock::iterator I, MachineBasicBlock::iterator &NMBBI) { const unsigned Size = I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8; MachineFunction *MF = BB.getParent(); const bool ArePtrs64bit = STI->getABI().ArePtrs64bit(); DebugLoc DL = I->getDebugLoc(); unsigned LL, SC, ZERO, BNE, BEQ, MOVE; if (Size == 4) { if (STI->inMicroMipsMode()) { LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM; SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM; BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM; BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM; } else { LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6) : (ArePtrs64bit ? Mips::LL64 : Mips::LL); SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6) : (ArePtrs64bit ? Mips::SC64 : Mips::SC); BNE = Mips::BNE; BEQ = Mips::BEQ; } ZERO = Mips::ZERO; MOVE = Mips::OR; } else { LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD; SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD; ZERO = Mips::ZERO_64; BNE = Mips::BNE64; BEQ = Mips::BEQ64; MOVE = Mips::OR64; } unsigned Dest = I->getOperand(0).getReg(); unsigned Ptr = I->getOperand(1).getReg(); unsigned OldVal = I->getOperand(2).getReg(); unsigned NewVal = I->getOperand(3).getReg(); unsigned Scratch = I->getOperand(4).getReg(); // insert new blocks after the current block const BasicBlock *LLVM_BB = BB.getBasicBlock(); MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB); MachineFunction::iterator It = ++BB.getIterator(); MF->insert(It, loop1MBB); MF->insert(It, loop2MBB); MF->insert(It, exitMBB); // Transfer the remainder of BB and its successor edges to exitMBB. exitMBB->splice(exitMBB->begin(), &BB, std::next(MachineBasicBlock::iterator(I)), BB.end()); exitMBB->transferSuccessorsAndUpdatePHIs(&BB); // thisMBB: // ... // fallthrough --> loop1MBB BB.addSuccessor(loop1MBB, BranchProbability::getOne()); loop1MBB->addSuccessor(exitMBB); loop1MBB->addSuccessor(loop2MBB); loop1MBB->normalizeSuccProbs(); loop2MBB->addSuccessor(loop1MBB); loop2MBB->addSuccessor(exitMBB); loop2MBB->normalizeSuccProbs(); // loop1MBB: // ll dest, 0(ptr) // bne dest, oldval, exitMBB BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0); BuildMI(loop1MBB, DL, TII->get(BNE)) .addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB); // loop2MBB: // move scratch, NewVal // sc Scratch, Scratch, 0(ptr) // beq Scratch, $0, loop1MBB BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO); BuildMI(loop2MBB, DL, TII->get(SC), Scratch) .addReg(Scratch).addReg(Ptr).addImm(0); BuildMI(loop2MBB, DL, TII->get(BEQ)) .addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB); LivePhysRegs LiveRegs; computeAndAddLiveIns(LiveRegs, *loop1MBB); computeAndAddLiveIns(LiveRegs, *loop2MBB); computeAndAddLiveIns(LiveRegs, *exitMBB); NMBBI = BB.end(); I->eraseFromParent(); return true; }