/// Insert a conditional branch at the end of \p MBB to \p NewDestBB, using the /// inverse condition of branch \p OldBr. /// \returns The number of bytes added to the block. unsigned AArch64BranchRelaxation::insertInvertedConditionalBranch( MachineBasicBlock &SrcMBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, const MachineInstr &OldBr, MachineBasicBlock &NewDestBB) const { unsigned OppositeCondOpc = getOppositeConditionOpcode(OldBr.getOpcode()); MachineInstrBuilder MIB = BuildMI(SrcMBB, InsPt, DL, TII->get(OppositeCondOpc)) .addOperand(OldBr.getOperand(0)); unsigned Opc = OldBr.getOpcode(); if (Opc == AArch64::TBZW || Opc == AArch64::TBNZW || Opc == AArch64::TBZX || Opc == AArch64::TBNZX) MIB.addOperand(OldBr.getOperand(1)); if (OldBr.getOpcode() == AArch64::Bcc) invertBccCondition(*MIB); MIB.addMBB(&NewDestBB); return TII->getInstSizeInBytes(*MIB); }
// Replace Br with a branch which has the opposite condition code and a // MachineBasicBlock operand MBBOpnd. void MipsLongBranch::replaceBranch(MachineBasicBlock &MBB, Iter Br, const DebugLoc &DL, MachineBasicBlock *MBBOpnd) { const MipsInstrInfo *TII = static_cast<const MipsInstrInfo *>( MBB.getParent()->getSubtarget().getInstrInfo()); unsigned NewOpc = TII->getOppositeBranchOpc(Br->getOpcode()); const MCInstrDesc &NewDesc = TII->get(NewOpc); MachineInstrBuilder MIB = BuildMI(MBB, Br, DL, NewDesc); for (unsigned I = 0, E = Br->getDesc().getNumOperands(); I < E; ++I) { MachineOperand &MO = Br->getOperand(I); if (!MO.isReg()) { assert(MO.isMBB() && "MBB operand expected."); break; } MIB.addReg(MO.getReg()); } MIB.addMBB(MBBOpnd); if (Br->hasDelaySlot()) { // Bundle the instruction in the delay slot to the newly created branch // and erase the original branch. assert(Br->isBundledWithSucc()); MachineBasicBlock::instr_iterator II(Br); MIBundleBuilder(&*MIB).append((++II)->removeFromBundle()); } Br->eraseFromParent(); }
unsigned ARCInstrInfo::insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond, const DebugLoc &dl, int *BytesAdded) const { assert(!BytesAdded && "Code size not handled."); // Shouldn't be a fall through. assert(TBB && "InsertBranch must not be told to insert a fallthrough"); assert((Cond.size() == 3 || Cond.size() == 0) && "ARC branch conditions have two components!"); if (Cond.empty()) { BuildMI(&MBB, dl, get(ARC::BR)).addMBB(TBB); return 1; } int BccOpc = Cond[1].isImm() ? ARC::BRcc_ru6_p : ARC::BRcc_rr_p; MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(BccOpc)); MIB.addMBB(TBB); for (unsigned i = 0; i < 3; i++) { MIB.add(Cond[i]); } // One-way conditional branch. if (!FBB) { return 1; } // Two-way conditional branch. BuildMI(&MBB, dl, get(ARC::BR)).addMBB(FBB); return 2; }
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB, DebugLoc DL, const SmallVectorImpl<MachineOperand>& Cond) const { unsigned Opc = Cond[0].getImm(); const MCInstrDesc &MCID = get(Opc); MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID); for (unsigned i = 1; i < Cond.size(); ++i) MIB.addReg(Cond[i].getReg()); MIB.addMBB(TBB); }
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB, const DebugLoc &DL, ArrayRef<MachineOperand> Cond) const { unsigned Opc = Cond[0].getImm(); const MCInstrDesc &MCID = get(Opc); MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID); for (unsigned i = 1; i < Cond.size(); ++i) { assert((Cond[i].isImm() || Cond[i].isReg()) && "Cannot copy operand for conditional branch!"); MIB.add(Cond[i]); } MIB.addMBB(TBB); }
unsigned AArch64InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const { if (FBB == 0 && Cond.empty()) { BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(TBB); return 1; } else if (FBB == 0) { MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm())); for (int i = 1, e = Cond.size(); i != e; ++i) MIB.addOperand(Cond[i]); MIB.addMBB(TBB); return 1; } MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm())); for (int i = 1, e = Cond.size(); i != e; ++i) MIB.addOperand(Cond[i]); MIB.addMBB(TBB); BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(FBB); return 2; }
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB, const DebugLoc &DL, ArrayRef<MachineOperand> Cond) const { unsigned Opc = Cond[0].getImm(); const MCInstrDesc &MCID = get(Opc); MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID); for (unsigned i = 1; i < Cond.size(); ++i) { if (Cond[i].isReg()) MIB.addReg(Cond[i].getReg()); else if (Cond[i].isImm()) MIB.addImm(Cond[i].getImm()); else assert(false && "Cannot copy operand"); } MIB.addMBB(TBB); }
void CoffeeInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB, DebugLoc DL, const SmallVectorImpl<MachineOperand>& Cond) const { unsigned Opc = Cond[0].getImm(); const MCInstrDesc &MCID = get(Opc); MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID); for (unsigned i = 1; i < Cond.size(); ++i) { if (Cond[i].isReg()) MIB.addReg(Cond[i].getReg()); else if (Cond[i].isImm()) MIB.addImm(Cond[i].getImm()); else assert(true && "Cannot copy operand"); } MIB.addMBB(TBB); }
unsigned SPUInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, const SmallVectorImpl<MachineOperand> &Cond) const { // FIXME this should probably have a DebugLoc argument DebugLoc dl = DebugLoc::getUnknownLoc(); // Shouldn't be a fall through. assert(TBB && "InsertBranch must not be told to insert a fallthrough"); assert((Cond.size() == 2 || Cond.size() == 0) && "SPU branch conditions have two components!"); // One-way branch. if (FBB == 0) { if (Cond.empty()) { // Unconditional branch MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(SPU::BR)); MIB.addMBB(TBB); DEBUG(cerr << "Inserted one-way uncond branch: "); DEBUG((*MIB).dump()); } else { // Conditional branch MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(Cond[0].getImm())); MIB.addReg(Cond[1].getReg()).addMBB(TBB); DEBUG(cerr << "Inserted one-way cond branch: "); DEBUG((*MIB).dump()); } return 1; } else { MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(Cond[0].getImm())); MachineInstrBuilder MIB2 = BuildMI(&MBB, dl, get(SPU::BR)); // Two-way Conditional Branch. MIB.addReg(Cond[1].getReg()).addMBB(TBB); MIB2.addMBB(FBB); DEBUG(cerr << "Inserted conditional branch: "); DEBUG((*MIB).dump()); DEBUG(cerr << "part 2: "); DEBUG((*MIB2).dump()); return 2; } }
// Replace Branch with the compact branch instruction. Iter Filler::replaceWithCompactBranch(MachineBasicBlock &MBB, Iter Branch, DebugLoc DL) { const MipsInstrInfo *TII = MBB.getParent()->getSubtarget<MipsSubtarget>().getInstrInfo(); unsigned NewOpcode = (((unsigned) Branch->getOpcode()) == Mips::BEQ) ? Mips::BEQZC_MM : Mips::BNEZC_MM; const MCInstrDesc &NewDesc = TII->get(NewOpcode); MachineInstrBuilder MIB = BuildMI(MBB, Branch, DL, NewDesc); MIB.addReg(Branch->getOperand(0).getReg()); MIB.addMBB(Branch->getOperand(2).getMBB()); Iter tmpIter = Branch; Branch = std::prev(Branch); MBB.erase(tmpIter); return Branch; }
// Replace Br with a branch which has the opposite condition code and a // MachineBasicBlock operand MBBOpnd. void MipsLongBranch::replaceBranch(MachineBasicBlock &MBB, Iter Br, DebugLoc DL, MachineBasicBlock *MBBOpnd) { const MipsInstrInfo *TII = static_cast<const MipsInstrInfo*>(TM.getInstrInfo()); unsigned NewOpc = TII->getOppositeBranchOpc(Br->getOpcode()); const MCInstrDesc &NewDesc = TII->get(NewOpc); MachineInstrBuilder MIB = BuildMI(MBB, Br, DL, NewDesc); for (unsigned I = 0, E = Br->getDesc().getNumOperands(); I < E; ++I) { MachineOperand &MO = Br->getOperand(I); if (!MO.isReg()) { assert(MO.isMBB() && "MBB operand expected."); break; } MIB.addReg(MO.getReg()); } MIB.addMBB(MBBOpnd); Br->eraseFromParent(); }
unsigned SPUInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, const SmallVectorImpl<MachineOperand> &Cond, DebugLoc DL) const { // Shouldn't be a fall through. assert(TBB && "InsertBranch must not be told to insert a fallthrough"); assert((Cond.size() == 2 || Cond.size() == 0) && "SPU branch conditions have two components!"); MachineInstrBuilder MIB; //TODO: make a more accurate algorithm. bool haveHBR = MBB.size()>8; removeHBR(MBB); MCSymbol *branchLabel = MBB.getParent()->getContext().CreateTempSymbol(); // Add a label just before the branch if (haveHBR) MIB = BuildMI(&MBB, DL, get(SPU::HBR_LABEL)).addSym(branchLabel); // One-way branch. if (FBB == 0) { if (Cond.empty()) { // Unconditional branch MIB = BuildMI(&MBB, DL, get(SPU::BR)); MIB.addMBB(TBB); DEBUG(errs() << "Inserted one-way uncond branch: "); DEBUG((*MIB).dump()); // basic blocks have just one branch so it is safe to add the hint a its if (haveHBR) { MIB = BuildMI( MBB, findHBRPosition(MBB), DL, get(SPU::HBRA)); MIB.addSym(branchLabel); MIB.addMBB(TBB); } } else { // Conditional branch MIB = BuildMI(&MBB, DL, get(Cond[0].getImm())); MIB.addReg(Cond[1].getReg()).addMBB(TBB); if (haveHBR) { MIB = BuildMI(MBB, findHBRPosition(MBB), DL, get(SPU::HBRA)); MIB.addSym(branchLabel); MIB.addMBB(TBB); } DEBUG(errs() << "Inserted one-way cond branch: "); DEBUG((*MIB).dump()); } return 1; } else { MIB = BuildMI(&MBB, DL, get(Cond[0].getImm())); MachineInstrBuilder MIB2 = BuildMI(&MBB, DL, get(SPU::BR)); // Two-way Conditional Branch. MIB.addReg(Cond[1].getReg()).addMBB(TBB); MIB2.addMBB(FBB); if (haveHBR) { MIB = BuildMI( MBB, findHBRPosition(MBB), DL, get(SPU::HBRA)); MIB.addSym(branchLabel); MIB.addMBB(FBB); } DEBUG(errs() << "Inserted conditional branch: "); DEBUG((*MIB).dump()); DEBUG(errs() << "part 2: "); DEBUG((*MIB2).dump()); return 2; } }
/// AddOperand - Add the specified operand to the specified machine instr. II /// specifies the instruction information for the node, and IIOpNum is the /// operand number (in the II) that we are adding. void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, SDValue Op, unsigned IIOpNum, const MCInstrDesc *II, DenseMap<SDValue, unsigned> &VRBaseMap, bool IsDebug, bool IsClone, bool IsCloned) { if (Op.isMachineOpcode()) { AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug, IsClone, IsCloned); } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) { MIB.addImm(C->getSExtValue()); } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) { MIB.addFPImm(F->getConstantFPValue()); } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) { unsigned VReg = R->getReg(); MVT OpVT = Op.getSimpleValueType(); const TargetRegisterClass *OpRC = TLI->isTypeLegal(OpVT) ? TLI->getRegClassFor(OpVT) : nullptr; const TargetRegisterClass *IIRC = II ? TRI->getAllocatableClass(TII->getRegClass(*II, IIOpNum, TRI, *MF)) : nullptr; if (OpRC && IIRC && OpRC != IIRC && TargetRegisterInfo::isVirtualRegister(VReg)) { unsigned NewVReg = MRI->createVirtualRegister(IIRC); BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(), TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg); VReg = NewVReg; } // Turn additional physreg operands into implicit uses on non-variadic // instructions. This is used by call and return instructions passing // arguments in registers. bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic()); MIB.addReg(VReg, getImplRegState(Imp)); } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) { MIB.addRegMask(RM->getRegMask()); } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) { MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(), TGA->getTargetFlags()); } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) { MIB.addMBB(BBNode->getBasicBlock()); } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) { MIB.addFrameIndex(FI->getIndex()); } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) { MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags()); } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) { int Offset = CP->getOffset(); unsigned Align = CP->getAlignment(); Type *Type = CP->getType(); // MachineConstantPool wants an explicit alignment. if (Align == 0) { Align = MF->getDataLayout().getPrefTypeAlignment(Type); if (Align == 0) { // Alignment of vector types. FIXME! Align = MF->getDataLayout().getTypeAllocSize(Type); } } unsigned Idx; MachineConstantPool *MCP = MF->getConstantPool(); if (CP->isMachineConstantPoolEntry()) Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Align); else Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Align); MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags()); } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) { MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags()); } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Op)) { MIB.addSym(SymNode->getMCSymbol()); } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) { MIB.addBlockAddress(BA->getBlockAddress(), BA->getOffset(), BA->getTargetFlags()); } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) { MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags()); } else { assert(Op.getValueType() != MVT::Other && Op.getValueType() != MVT::Glue && "Chain and glue operands should occur at end of operand list!"); AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug, IsClone, IsCloned); } }
bool AMDGPUIndirectAddressingPass::runOnMachineFunction(MachineFunction &MF) { MachineRegisterInfo &MRI = MF.getRegInfo(); int IndirectBegin = TII->getIndirectIndexBegin(MF); int IndirectEnd = TII->getIndirectIndexEnd(MF); if (IndirectBegin == -1) { // No indirect addressing, we can skip this pass assert(IndirectEnd == -1); return false; } // The map keeps track of the indirect address that is represented by // each virtual register. The key is the register and the value is the // indirect address it uses. std::map<unsigned, unsigned> RegisterAddressMap; // First pass - Lower all of the RegisterStore instructions and track which // registers are live. for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end(); BB != BB_E; ++BB) { // This map keeps track of the current live indirect registers. // The key is the address and the value is the register std::map<unsigned, unsigned> LiveAddressRegisterMap; MachineBasicBlock &MBB = *BB; for (MachineBasicBlock::iterator I = MBB.begin(), Next = llvm::next(I); I != MBB.end(); I = Next) { Next = llvm::next(I); MachineInstr &MI = *I; if (!TII->isRegisterStore(MI)) { continue; } // Lower RegisterStore unsigned RegIndex = MI.getOperand(2).getImm(); unsigned Channel = MI.getOperand(3).getImm(); unsigned Address = TII->calculateIndirectAddress(RegIndex, Channel); const TargetRegisterClass *IndirectStoreRegClass = TII->getIndirectAddrStoreRegClass(MI.getOperand(0).getReg()); if (MI.getOperand(1).getReg() == AMDGPU::INDIRECT_BASE_ADDR) { // Direct register access. unsigned DstReg = MRI.createVirtualRegister(IndirectStoreRegClass); BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY), DstReg) .addOperand(MI.getOperand(0)); RegisterAddressMap[DstReg] = Address; LiveAddressRegisterMap[Address] = DstReg; } else { // Indirect register access. MachineInstrBuilder MOV = TII->buildIndirectWrite(BB, I, MI.getOperand(0).getReg(), // Value Address, MI.getOperand(1).getReg()); // Offset for (int i = IndirectBegin; i <= IndirectEnd; ++i) { unsigned Addr = TII->calculateIndirectAddress(i, Channel); unsigned DstReg = MRI.createVirtualRegister(IndirectStoreRegClass); MOV.addReg(DstReg, RegState::Define | RegState::Implicit); RegisterAddressMap[DstReg] = Addr; LiveAddressRegisterMap[Addr] = DstReg; } } MI.eraseFromParent(); } // Update the live-ins of the succesor blocks for (MachineBasicBlock::succ_iterator Succ = MBB.succ_begin(), SuccEnd = MBB.succ_end(); SuccEnd != Succ; ++Succ) { std::map<unsigned, unsigned>::const_iterator Key, KeyEnd; for (Key = LiveAddressRegisterMap.begin(), KeyEnd = LiveAddressRegisterMap.end(); KeyEnd != Key; ++Key) { (*Succ)->addLiveIn(Key->second); } } } // Second pass - Lower the RegisterLoad instructions for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end(); BB != BB_E; ++BB) { // Key is the address and the value is the register std::map<unsigned, unsigned> LiveAddressRegisterMap; MachineBasicBlock &MBB = *BB; MachineBasicBlock::livein_iterator LI = MBB.livein_begin(); while (LI != MBB.livein_end()) { std::vector<unsigned> PhiRegisters; // Make sure this live in is used for indirect addressing if (RegisterAddressMap.find(*LI) == RegisterAddressMap.end()) { ++LI; continue; } unsigned Address = RegisterAddressMap[*LI]; LiveAddressRegisterMap[Address] = *LI; PhiRegisters.push_back(*LI); // Check if there are other live in registers which map to the same // indirect address. for (MachineBasicBlock::livein_iterator LJ = llvm::next(LI), LE = MBB.livein_end(); LJ != LE; ++LJ) { unsigned Reg = *LJ; if (RegisterAddressMap.find(Reg) == RegisterAddressMap.end()) { continue; } if (RegisterAddressMap[Reg] == Address) { PhiRegisters.push_back(Reg); } } if (PhiRegisters.size() == 1) { // We don't need to insert a Phi instruction, so we can just add the // registers to the live list for the block. LiveAddressRegisterMap[Address] = *LI; MBB.removeLiveIn(*LI); } else { // We need to insert a PHI, because we have the same address being // written in multiple predecessor blocks. const TargetRegisterClass *PhiDstClass = TII->getIndirectAddrStoreRegClass(*(PhiRegisters.begin())); unsigned PhiDstReg = MRI.createVirtualRegister(PhiDstClass); MachineInstrBuilder Phi = BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()), TII->get(AMDGPU::PHI), PhiDstReg); for (std::vector<unsigned>::const_iterator RI = PhiRegisters.begin(), RE = PhiRegisters.end(); RI != RE; ++RI) { unsigned Reg = *RI; MachineInstr *DefInst = MRI.getVRegDef(Reg); assert(DefInst); MachineBasicBlock *RegBlock = DefInst->getParent(); Phi.addReg(Reg); Phi.addMBB(RegBlock); MBB.removeLiveIn(Reg); } RegisterAddressMap[PhiDstReg] = Address; LiveAddressRegisterMap[Address] = PhiDstReg; } LI = MBB.livein_begin(); } for (MachineBasicBlock::iterator I = MBB.begin(), Next = llvm::next(I); I != MBB.end(); I = Next) { Next = llvm::next(I); MachineInstr &MI = *I; if (!TII->isRegisterLoad(MI)) { if (MI.getOpcode() == AMDGPU::PHI) { continue; } // Check for indirect register defs for (unsigned OpIdx = 0, NumOperands = MI.getNumOperands(); OpIdx < NumOperands; ++OpIdx) { MachineOperand &MO = MI.getOperand(OpIdx); if (MO.isReg() && MO.isDef() && RegisterAddressMap.find(MO.getReg()) != RegisterAddressMap.end()) { unsigned Reg = MO.getReg(); unsigned LiveAddress = RegisterAddressMap[Reg]; // Chain the live-ins if (LiveAddressRegisterMap.find(LiveAddress) != RegisterAddressMap.end()) { MI.addOperand(MachineOperand::CreateReg( LiveAddressRegisterMap[LiveAddress], false, // isDef true, // isImp true)); // isKill } LiveAddressRegisterMap[LiveAddress] = Reg; } } continue; } const TargetRegisterClass *SuperIndirectRegClass = TII->getSuperIndirectRegClass(); const TargetRegisterClass *IndirectLoadRegClass = TII->getIndirectAddrLoadRegClass(); unsigned IndirectReg = MRI.createVirtualRegister(SuperIndirectRegClass); unsigned RegIndex = MI.getOperand(2).getImm(); unsigned Channel = MI.getOperand(3).getImm(); unsigned Address = TII->calculateIndirectAddress(RegIndex, Channel); if (MI.getOperand(1).getReg() == AMDGPU::INDIRECT_BASE_ADDR) { // Direct register access unsigned Reg = LiveAddressRegisterMap[Address]; unsigned AddrReg = IndirectLoadRegClass->getRegister(Address); if (regHasExplicitDef(MRI, Reg)) { // If the register we are reading from has an explicit def, then that // means it was written via a direct register access (i.e. COPY // or other instruction that doesn't use indirect addressing). In // this case we know where the value has been stored, so we can just // issue a copy. BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY), MI.getOperand(0).getReg()) .addReg(Reg); } else { // If the register we are reading has an implicit def, then that // means it was written by an indirect register access (i.e. An // instruction that uses indirect addressing. BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY), MI.getOperand(0).getReg()) .addReg(AddrReg) .addReg(Reg, RegState::Implicit); } } else { // Indirect register access // Note on REQ_SEQUENCE instructons: You can't actually use the register // it defines unless you have an instruction that takes the defined // register class as an operand. MachineInstrBuilder Sequence = BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::REG_SEQUENCE), IndirectReg); for (int i = IndirectBegin; i <= IndirectEnd; ++i) { unsigned Addr = TII->calculateIndirectAddress(i, Channel); if (LiveAddressRegisterMap.find(Addr) == LiveAddressRegisterMap.end()) { continue; } unsigned Reg = LiveAddressRegisterMap[Addr]; // We only need to use REG_SEQUENCE for explicit defs, since the // register coalescer won't do anything with the implicit defs. if (!regHasExplicitDef(MRI, Reg)) { continue; } // Insert a REQ_SEQUENCE instruction to force the register allocator // to allocate the virtual register to the correct physical register. Sequence.addReg(LiveAddressRegisterMap[Addr]); Sequence.addImm(TII->getRegisterInfo().getIndirectSubReg(Addr)); } MachineInstrBuilder Mov = TII->buildIndirectRead(BB, I, MI.getOperand(0).getReg(), // Value Address, MI.getOperand(1).getReg()); // Offset Mov.addReg(IndirectReg, RegState::Implicit | RegState::Kill); Mov.addReg(LiveAddressRegisterMap[Address], RegState::Implicit); } MI.eraseFromParent(); } } return false; }
void HexagonEarlyIfConversion::convert(const FlowPattern &FP) { MachineBasicBlock *TSB = 0, *FSB = 0; MachineBasicBlock::iterator OldTI = FP.SplitB->getFirstTerminator(); assert(OldTI != FP.SplitB->end()); DebugLoc DL = OldTI->getDebugLoc(); if (FP.TrueB) { TSB = *FP.TrueB->succ_begin(); predicateBlockNB(FP.SplitB, OldTI, FP.TrueB, FP.PredR, true); } if (FP.FalseB) { FSB = *FP.FalseB->succ_begin(); MachineBasicBlock::iterator At = FP.SplitB->getFirstTerminator(); predicateBlockNB(FP.SplitB, At, FP.FalseB, FP.PredR, false); } // Regenerate new terminators in the split block and update the successors. // First, remember any information that may be needed later and remove the // existing terminators/successors from the split block. MachineBasicBlock *SSB = 0; FP.SplitB->erase(OldTI, FP.SplitB->end()); while (FP.SplitB->succ_size() > 0) { MachineBasicBlock *T = *FP.SplitB->succ_begin(); // It's possible that the split block had a successor that is not a pre- // dicated block. This could only happen if there was only one block to // be predicated. Example: // split_b: // if (p) jump true_b // jump unrelated2_b // unrelated1_b: // ... // unrelated2_b: ; can have other predecessors, so it's not "false_b" // jump other_b // true_b: ; only reachable from split_b, can be predicated // ... // // Find this successor (SSB) if it exists. if (T != FP.TrueB && T != FP.FalseB) { assert(!SSB); SSB = T; } FP.SplitB->removeSuccessor(FP.SplitB->succ_begin()); } // Insert new branches and update the successors of the split block. This // may create unconditional branches to the layout successor, etc., but // that will be cleaned up later. For now, make sure that correct code is // generated. if (FP.JoinB) { assert(!SSB || SSB == FP.JoinB); BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jump)) .addMBB(FP.JoinB); FP.SplitB->addSuccessor(FP.JoinB); } else { bool HasBranch = false; if (TSB) { BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jumpt)) .addReg(FP.PredR) .addMBB(TSB); FP.SplitB->addSuccessor(TSB); HasBranch = true; } if (FSB) { const MCInstrDesc &D = HasBranch ? TII->get(Hexagon::J2_jump) : TII->get(Hexagon::J2_jumpf); MachineInstrBuilder MIB = BuildMI(*FP.SplitB, FP.SplitB->end(), DL, D); if (!HasBranch) MIB.addReg(FP.PredR); MIB.addMBB(FSB); FP.SplitB->addSuccessor(FSB); } if (SSB) { // This cannot happen if both TSB and FSB are set. [TF]SB are the // successor blocks of the TrueB and FalseB (or null of the TrueB // or FalseB block is null). SSB is the potential successor block // of the SplitB that is neither TrueB nor FalseB. BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jump)) .addMBB(SSB); FP.SplitB->addSuccessor(SSB); } } // What is left to do is to update the PHI nodes that could have entries // referring to predicated blocks. if (FP.JoinB) { updatePhiNodes(FP.JoinB, FP); } else { if (TSB) updatePhiNodes(TSB, FP); if (FSB) updatePhiNodes(FSB, FP); // Nothing to update in SSB, since SSB's predecessors haven't changed. } }
/// fixupConditionalBranch - Fix up a conditional branch whose destination is /// too far away to fit in its displacement field. It is converted to an inverse /// conditional branch + an unconditional branch to the destination. bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr *MI) { MachineBasicBlock *DestBB = getDestBlock(MI); // Add an unconditional branch to the destination and invert the branch // condition to jump over it: // tbz L1 // => // tbnz L2 // b L1 // L2: // If the branch is at the end of its MBB and that has a fall-through block, // direct the updated conditional branch to the fall-through block. Otherwise, // split the MBB before the next instruction. MachineBasicBlock *MBB = MI->getParent(); MachineInstr *BMI = &MBB->back(); bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB); if (BMI != MI) { if (std::next(MachineBasicBlock::iterator(MI)) == std::prev(MBB->getLastNonDebugInstr()) && BMI->getOpcode() == AArch64::B) { // Last MI in the BB is an unconditional branch. Can we simply invert the // condition and swap destinations: // beq L1 // b L2 // => // bne L2 // b L1 MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB(); if (isBlockInRange(MI, NewDest, getBranchDisplacementBits(MI->getOpcode()))) { DEBUG(dbgs() << " Invert condition and swap its destination with " << *BMI); BMI->getOperand(0).setMBB(DestBB); unsigned OpNum = (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW || MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX) ? 2 : 1; MI->getOperand(OpNum).setMBB(NewDest); MI->setDesc(TII->get(getOppositeConditionOpcode(MI->getOpcode()))); if (MI->getOpcode() == AArch64::Bcc) invertBccCondition(MI); return true; } } } if (NeedSplit) { // Analyze the branch so we know how to update the successor lists. MachineBasicBlock *TBB, *FBB; SmallVector<MachineOperand, 2> Cond; TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, false); MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI); // No need for the branch to the next block. We're adding an unconditional // branch to the destination. int delta = TII->GetInstSizeInBytes(&MBB->back()); BlockInfo[MBB->getNumber()].Size -= delta; MBB->back().eraseFromParent(); // BlockInfo[SplitBB].Offset is wrong temporarily, fixed below // Update the successor lists according to the transformation to follow. // Do it here since if there's no split, no update is needed. MBB->replaceSuccessor(FBB, NewBB); NewBB->addSuccessor(FBB); } MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(MBB)); DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber() << ", invert condition and change dest. to BB#" << NextBB->getNumber() << "\n"); // Insert a new conditional branch and a new unconditional branch. MachineInstrBuilder MIB = BuildMI( MBB, DebugLoc(), TII->get(getOppositeConditionOpcode(MI->getOpcode()))) .addOperand(MI->getOperand(0)); if (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW || MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX) MIB.addOperand(MI->getOperand(1)); if (MI->getOpcode() == AArch64::Bcc) invertBccCondition(MIB); MIB.addMBB(NextBB); BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back()); BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB); BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back()); // Remove the old conditional branch. It may or may not still be in MBB. BlockInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI); MI->eraseFromParent(); // Finally, keep the block offsets up to date. adjustBlockOffsets(*MBB); return true; }
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop) { MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin(); SetVector<MachineBasicBlock *> RewriteSuccs; // DFS through Loop's body, looking for for irreducible control flow. Loop is // natural, and we stay in its body, and we treat any nested loops // monolithically, so any cycles we encounter indicate irreducibility. SmallPtrSet<MachineBasicBlock *, 8> OnStack; SmallPtrSet<MachineBasicBlock *, 8> Visited; SmallVector<SuccessorList, 4> LoopWorklist; LoopWorklist.push_back(SuccessorList(Header)); OnStack.insert(Header); Visited.insert(Header); while (!LoopWorklist.empty()) { SuccessorList &Top = LoopWorklist.back(); if (Top.HasNext()) { MachineBasicBlock *Next = Top.Next(); if (Next == Header || (Loop && !Loop->contains(Next))) continue; if (LLVM_LIKELY(OnStack.insert(Next).second)) { if (!Visited.insert(Next).second) { OnStack.erase(Next); continue; } MachineLoop *InnerLoop = MLI.getLoopFor(Next); if (InnerLoop != Loop) LoopWorklist.push_back(SuccessorList(InnerLoop)); else LoopWorklist.push_back(SuccessorList(Next)); } else { RewriteSuccs.insert(Top.getBlock()); } continue; } OnStack.erase(Top.getBlock()); LoopWorklist.pop_back(); } // Most likely, we didn't find any irreducible control flow. if (LLVM_LIKELY(RewriteSuccs.empty())) return false; DEBUG(dbgs() << "Irreducible control flow detected!\n"); // Ok. We have irreducible control flow! Create a dispatch block which will // contains a jump table to any block in the problematic set of blocks. MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock(); MF.insert(MF.end(), Dispatch); MLI.changeLoopFor(Dispatch, Loop); // Add the jump table. const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo(); MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(), TII.get(WebAssembly::BR_TABLE_I32)); // Add the register which will be used to tell the jump table which block to // jump to. MachineRegisterInfo &MRI = MF.getRegInfo(); unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); MIB.addReg(Reg); // Collect all the blocks which need to have their successors rewritten, // add the successors to the jump table, and remember their index. DenseMap<MachineBasicBlock *, unsigned> Indices; SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(), RewriteSuccs.end()); while (!SuccWorklist.empty()) { MachineBasicBlock *MBB = SuccWorklist.pop_back_val(); auto Pair = Indices.insert(std::make_pair(MBB, 0)); if (!Pair.second) continue; unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1; DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n"); Pair.first->second = Index; for (auto Pred : MBB->predecessors()) RewriteSuccs.insert(Pred); MIB.addMBB(MBB); Dispatch->addSuccessor(MBB); MetaBlock Meta(MBB); for (auto *Succ : Meta.successors()) if (Succ != Header && (!Loop || Loop->contains(Succ))) SuccWorklist.push_back(Succ); } // Rewrite the problematic successors for every block in RewriteSuccs. // For simplicity, we just introduce a new block for every edge we need to // rewrite. Fancier things are possible. for (MachineBasicBlock *MBB : RewriteSuccs) { DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map; for (auto *Succ : MBB->successors()) { if (!Indices.count(Succ)) continue; MachineBasicBlock *Split = MF.CreateMachineBasicBlock(); MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ) : MF.end(), Split); MLI.changeLoopFor(Split, Loop); // Set the jump table's register of the index of the block we wish to // jump to, and jump to the jump table. BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32), Reg) .addImm(Indices[Succ]); BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR)) .addMBB(Dispatch); Split->addSuccessor(Dispatch); Map[Succ] = Split; } // Remap the terminator operands and the successor list. for (MachineInstr &Term : MBB->terminators()) for (auto &Op : Term.explicit_uses()) if (Op.isMBB() && Indices.count(Op.getMBB())) Op.setMBB(Map[Op.getMBB()]); for (auto Rewrite : Map) MBB->replaceSuccessor(Rewrite.first, Rewrite.second); } // Create a fake default label, because br_table requires one. MIB.addMBB(MIB.getInstr() ->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1) .getMBB()); return true; }