/// Insert a conditional branch at the end of \p MBB to \p NewDestBB, using the
/// inverse condition of branch \p OldBr.
/// \returns The number of bytes added to the block.
unsigned AArch64BranchRelaxation::insertInvertedConditionalBranch(
MachineBasicBlock &SrcMBB,
MachineBasicBlock::iterator InsPt,
const DebugLoc &DL,
const MachineInstr &OldBr,
MachineBasicBlock &NewDestBB) const {
unsigned OppositeCondOpc = getOppositeConditionOpcode(OldBr.getOpcode());
MachineInstrBuilder MIB =
BuildMI(SrcMBB, InsPt, DL, TII->get(OppositeCondOpc))
.addOperand(OldBr.getOperand(0));
unsigned Opc = OldBr.getOpcode();
if (Opc == AArch64::TBZW || Opc == AArch64::TBNZW ||
Opc == AArch64::TBZX || Opc == AArch64::TBNZX)
MIB.addOperand(OldBr.getOperand(1));
if (OldBr.getOpcode() == AArch64::Bcc)
invertBccCondition(*MIB);
MIB.addMBB(&NewDestBB);
return TII->getInstSizeInBytes(*MIB);
}
// Replace Br with a branch which has the opposite condition code and a
// MachineBasicBlock operand MBBOpnd.
void MipsLongBranch::replaceBranch(MachineBasicBlock &MBB, Iter Br,
const DebugLoc &DL,
MachineBasicBlock *MBBOpnd) {
const MipsInstrInfo *TII = static_cast<const MipsInstrInfo *>(
MBB.getParent()->getSubtarget().getInstrInfo());
unsigned NewOpc = TII->getOppositeBranchOpc(Br->getOpcode());
const MCInstrDesc &NewDesc = TII->get(NewOpc);
MachineInstrBuilder MIB = BuildMI(MBB, Br, DL, NewDesc);
for (unsigned I = 0, E = Br->getDesc().getNumOperands(); I < E; ++I) {
MachineOperand &MO = Br->getOperand(I);
if (!MO.isReg()) {
assert(MO.isMBB() && "MBB operand expected.");
break;
}
MIB.addReg(MO.getReg());
}
MIB.addMBB(MBBOpnd);
if (Br->hasDelaySlot()) {
// Bundle the instruction in the delay slot to the newly created branch
// and erase the original branch.
assert(Br->isBundledWithSucc());
MachineBasicBlock::instr_iterator II(Br);
MIBundleBuilder(&*MIB).append((++II)->removeFromBundle());
}
Br->eraseFromParent();
}
unsigned ARCInstrInfo::insertBranch(MachineBasicBlock &MBB,
MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
ArrayRef<MachineOperand> Cond,
const DebugLoc &dl, int *BytesAdded) const {
assert(!BytesAdded && "Code size not handled.");
// Shouldn't be a fall through.
assert(TBB && "InsertBranch must not be told to insert a fallthrough");
assert((Cond.size() == 3 || Cond.size() == 0) &&
"ARC branch conditions have two components!");
if (Cond.empty()) {
BuildMI(&MBB, dl, get(ARC::BR)).addMBB(TBB);
return 1;
}
int BccOpc = Cond[1].isImm() ? ARC::BRcc_ru6_p : ARC::BRcc_rr_p;
MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(BccOpc));
MIB.addMBB(TBB);
for (unsigned i = 0; i < 3; i++) {
MIB.add(Cond[i]);
}
// One-way conditional branch.
if (!FBB) {
return 1;
}
// Two-way conditional branch.
BuildMI(&MBB, dl, get(ARC::BR)).addMBB(FBB);
return 2;
}
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB,
MachineBasicBlock *TBB, DebugLoc DL,
const SmallVectorImpl<MachineOperand>& Cond)
const {
unsigned Opc = Cond[0].getImm();
const MCInstrDesc &MCID = get(Opc);
MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
for (unsigned i = 1; i < Cond.size(); ++i)
MIB.addReg(Cond[i].getReg());
MIB.addMBB(TBB);
}
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
const DebugLoc &DL,
ArrayRef<MachineOperand> Cond) const {
unsigned Opc = Cond[0].getImm();
const MCInstrDesc &MCID = get(Opc);
MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
for (unsigned i = 1; i < Cond.size(); ++i) {
assert((Cond[i].isImm() || Cond[i].isReg()) &&
"Cannot copy operand for conditional branch!");
MIB.add(Cond[i]);
}
MIB.addMBB(TBB);
}
unsigned
AArch64InstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
const SmallVectorImpl<MachineOperand> &Cond,
DebugLoc DL) const {
if (FBB == 0 && Cond.empty()) {
BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(TBB);
return 1;
} else if (FBB == 0) {
MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
for (int i = 1, e = Cond.size(); i != e; ++i)
MIB.addOperand(Cond[i]);
MIB.addMBB(TBB);
return 1;
}
MachineInstrBuilder MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
for (int i = 1, e = Cond.size(); i != e; ++i)
MIB.addOperand(Cond[i]);
MIB.addMBB(TBB);
BuildMI(&MBB, DL, get(AArch64::Bimm)).addMBB(FBB);
return 2;
}
void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
const DebugLoc &DL,
ArrayRef<MachineOperand> Cond) const {
unsigned Opc = Cond[0].getImm();
const MCInstrDesc &MCID = get(Opc);
MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
for (unsigned i = 1; i < Cond.size(); ++i) {
if (Cond[i].isReg())
MIB.addReg(Cond[i].getReg());
else if (Cond[i].isImm())
MIB.addImm(Cond[i].getImm());
else
assert(false && "Cannot copy operand");
}
MIB.addMBB(TBB);
}
void CoffeeInstrInfo::BuildCondBr(MachineBasicBlock &MBB,
MachineBasicBlock *TBB, DebugLoc DL,
const SmallVectorImpl<MachineOperand>& Cond)
const {
unsigned Opc = Cond[0].getImm();
const MCInstrDesc &MCID = get(Opc);
MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
for (unsigned i = 1; i < Cond.size(); ++i) {
if (Cond[i].isReg())
MIB.addReg(Cond[i].getReg());
else if (Cond[i].isImm())
MIB.addImm(Cond[i].getImm());
else
assert(true && "Cannot copy operand");
}
MIB.addMBB(TBB);
}
unsigned
SPUInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
const SmallVectorImpl<MachineOperand> &Cond) const {
// FIXME this should probably have a DebugLoc argument
DebugLoc dl = DebugLoc::getUnknownLoc();
// Shouldn't be a fall through.
assert(TBB && "InsertBranch must not be told to insert a fallthrough");
assert((Cond.size() == 2 || Cond.size() == 0) &&
"SPU branch conditions have two components!");
// One-way branch.
if (FBB == 0) {
if (Cond.empty()) {
// Unconditional branch
MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(SPU::BR));
MIB.addMBB(TBB);
DEBUG(cerr << "Inserted one-way uncond branch: ");
DEBUG((*MIB).dump());
} else {
// Conditional branch
MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(Cond[0].getImm()));
MIB.addReg(Cond[1].getReg()).addMBB(TBB);
DEBUG(cerr << "Inserted one-way cond branch: ");
DEBUG((*MIB).dump());
}
return 1;
} else {
MachineInstrBuilder MIB = BuildMI(&MBB, dl, get(Cond[0].getImm()));
MachineInstrBuilder MIB2 = BuildMI(&MBB, dl, get(SPU::BR));
// Two-way Conditional Branch.
MIB.addReg(Cond[1].getReg()).addMBB(TBB);
MIB2.addMBB(FBB);
DEBUG(cerr << "Inserted conditional branch: ");
DEBUG((*MIB).dump());
DEBUG(cerr << "part 2: ");
DEBUG((*MIB2).dump());
return 2;
}
}
// Replace Branch with the compact branch instruction.
Iter Filler::replaceWithCompactBranch(MachineBasicBlock &MBB,
Iter Branch, DebugLoc DL) {
const MipsInstrInfo *TII =
MBB.getParent()->getSubtarget<MipsSubtarget>().getInstrInfo();
unsigned NewOpcode =
(((unsigned) Branch->getOpcode()) == Mips::BEQ) ? Mips::BEQZC_MM
: Mips::BNEZC_MM;
const MCInstrDesc &NewDesc = TII->get(NewOpcode);
MachineInstrBuilder MIB = BuildMI(MBB, Branch, DL, NewDesc);
MIB.addReg(Branch->getOperand(0).getReg());
MIB.addMBB(Branch->getOperand(2).getMBB());
Iter tmpIter = Branch;
Branch = std::prev(Branch);
MBB.erase(tmpIter);
return Branch;
}
// Replace Br with a branch which has the opposite condition code and a
// MachineBasicBlock operand MBBOpnd.
void MipsLongBranch::replaceBranch(MachineBasicBlock &MBB, Iter Br,
DebugLoc DL, MachineBasicBlock *MBBOpnd) {
const MipsInstrInfo *TII =
static_cast<const MipsInstrInfo*>(TM.getInstrInfo());
unsigned NewOpc = TII->getOppositeBranchOpc(Br->getOpcode());
const MCInstrDesc &NewDesc = TII->get(NewOpc);
MachineInstrBuilder MIB = BuildMI(MBB, Br, DL, NewDesc);
for (unsigned I = 0, E = Br->getDesc().getNumOperands(); I < E; ++I) {
MachineOperand &MO = Br->getOperand(I);
if (!MO.isReg()) {
assert(MO.isMBB() && "MBB operand expected.");
break;
}
MIB.addReg(MO.getReg());
}
MIB.addMBB(MBBOpnd);
Br->eraseFromParent();
}
unsigned
SPUInstrInfo::InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
const SmallVectorImpl<MachineOperand> &Cond,
DebugLoc DL) const {
// Shouldn't be a fall through.
assert(TBB && "InsertBranch must not be told to insert a fallthrough");
assert((Cond.size() == 2 || Cond.size() == 0) &&
"SPU branch conditions have two components!");
MachineInstrBuilder MIB;
//TODO: make a more accurate algorithm.
bool haveHBR = MBB.size()>8;
removeHBR(MBB);
MCSymbol *branchLabel = MBB.getParent()->getContext().CreateTempSymbol();
// Add a label just before the branch
if (haveHBR)
MIB = BuildMI(&MBB, DL, get(SPU::HBR_LABEL)).addSym(branchLabel);
// One-way branch.
if (FBB == 0) {
if (Cond.empty()) {
// Unconditional branch
MIB = BuildMI(&MBB, DL, get(SPU::BR));
MIB.addMBB(TBB);
DEBUG(errs() << "Inserted one-way uncond branch: ");
DEBUG((*MIB).dump());
// basic blocks have just one branch so it is safe to add the hint a its
if (haveHBR) {
MIB = BuildMI( MBB, findHBRPosition(MBB), DL, get(SPU::HBRA));
MIB.addSym(branchLabel);
MIB.addMBB(TBB);
}
} else {
// Conditional branch
MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
MIB.addReg(Cond[1].getReg()).addMBB(TBB);
if (haveHBR) {
MIB = BuildMI(MBB, findHBRPosition(MBB), DL, get(SPU::HBRA));
MIB.addSym(branchLabel);
MIB.addMBB(TBB);
}
DEBUG(errs() << "Inserted one-way cond branch: ");
DEBUG((*MIB).dump());
}
return 1;
} else {
MIB = BuildMI(&MBB, DL, get(Cond[0].getImm()));
MachineInstrBuilder MIB2 = BuildMI(&MBB, DL, get(SPU::BR));
// Two-way Conditional Branch.
MIB.addReg(Cond[1].getReg()).addMBB(TBB);
MIB2.addMBB(FBB);
if (haveHBR) {
MIB = BuildMI( MBB, findHBRPosition(MBB), DL, get(SPU::HBRA));
MIB.addSym(branchLabel);
MIB.addMBB(FBB);
}
DEBUG(errs() << "Inserted conditional branch: ");
DEBUG((*MIB).dump());
DEBUG(errs() << "part 2: ");
DEBUG((*MIB2).dump());
return 2;
}
}
/// AddOperand - Add the specified operand to the specified machine instr. II
/// specifies the instruction information for the node, and IIOpNum is the
/// operand number (in the II) that we are adding.
void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
SDValue Op,
unsigned IIOpNum,
const MCInstrDesc *II,
DenseMap<SDValue, unsigned> &VRBaseMap,
bool IsDebug, bool IsClone, bool IsCloned) {
if (Op.isMachineOpcode()) {
AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
IsDebug, IsClone, IsCloned);
} else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
MIB.addImm(C->getSExtValue());
} else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Op)) {
MIB.addFPImm(F->getConstantFPValue());
} else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Op)) {
unsigned VReg = R->getReg();
MVT OpVT = Op.getSimpleValueType();
const TargetRegisterClass *OpRC =
TLI->isTypeLegal(OpVT) ? TLI->getRegClassFor(OpVT) : nullptr;
const TargetRegisterClass *IIRC =
II ? TRI->getAllocatableClass(TII->getRegClass(*II, IIOpNum, TRI, *MF))
: nullptr;
if (OpRC && IIRC && OpRC != IIRC &&
TargetRegisterInfo::isVirtualRegister(VReg)) {
unsigned NewVReg = MRI->createVirtualRegister(IIRC);
BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
TII->get(TargetOpcode::COPY), NewVReg).addReg(VReg);
VReg = NewVReg;
}
// Turn additional physreg operands into implicit uses on non-variadic
// instructions. This is used by call and return instructions passing
// arguments in registers.
bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
MIB.addReg(VReg, getImplRegState(Imp));
} else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Op)) {
MIB.addRegMask(RM->getRegMask());
} else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Op)) {
MIB.addGlobalAddress(TGA->getGlobal(), TGA->getOffset(),
TGA->getTargetFlags());
} else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Op)) {
MIB.addMBB(BBNode->getBasicBlock());
} else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) {
MIB.addFrameIndex(FI->getIndex());
} else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Op)) {
MIB.addJumpTableIndex(JT->getIndex(), JT->getTargetFlags());
} else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Op)) {
int Offset = CP->getOffset();
unsigned Align = CP->getAlignment();
Type *Type = CP->getType();
// MachineConstantPool wants an explicit alignment.
if (Align == 0) {
Align = MF->getDataLayout().getPrefTypeAlignment(Type);
if (Align == 0) {
// Alignment of vector types. FIXME!
Align = MF->getDataLayout().getTypeAllocSize(Type);
}
}
unsigned Idx;
MachineConstantPool *MCP = MF->getConstantPool();
if (CP->isMachineConstantPoolEntry())
Idx = MCP->getConstantPoolIndex(CP->getMachineCPVal(), Align);
else
Idx = MCP->getConstantPoolIndex(CP->getConstVal(), Align);
MIB.addConstantPoolIndex(Idx, Offset, CP->getTargetFlags());
} else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Op)) {
MIB.addExternalSymbol(ES->getSymbol(), ES->getTargetFlags());
} else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Op)) {
MIB.addSym(SymNode->getMCSymbol());
} else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
MIB.addBlockAddress(BA->getBlockAddress(),
BA->getOffset(),
BA->getTargetFlags());
} else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) {
MIB.addTargetIndex(TI->getIndex(), TI->getOffset(), TI->getTargetFlags());
} else {
assert(Op.getValueType() != MVT::Other &&
Op.getValueType() != MVT::Glue &&
"Chain and glue operands should occur at end of operand list!");
AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
IsDebug, IsClone, IsCloned);
}
}
bool AMDGPUIndirectAddressingPass::runOnMachineFunction(MachineFunction &MF) {
MachineRegisterInfo &MRI = MF.getRegInfo();
int IndirectBegin = TII->getIndirectIndexBegin(MF);
int IndirectEnd = TII->getIndirectIndexEnd(MF);
if (IndirectBegin == -1) {
// No indirect addressing, we can skip this pass
assert(IndirectEnd == -1);
return false;
}
// The map keeps track of the indirect address that is represented by
// each virtual register. The key is the register and the value is the
// indirect address it uses.
std::map<unsigned, unsigned> RegisterAddressMap;
// First pass - Lower all of the RegisterStore instructions and track which
// registers are live.
for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end();
BB != BB_E; ++BB) {
// This map keeps track of the current live indirect registers.
// The key is the address and the value is the register
std::map<unsigned, unsigned> LiveAddressRegisterMap;
MachineBasicBlock &MBB = *BB;
for (MachineBasicBlock::iterator I = MBB.begin(), Next = llvm::next(I);
I != MBB.end(); I = Next) {
Next = llvm::next(I);
MachineInstr &MI = *I;
if (!TII->isRegisterStore(MI)) {
continue;
}
// Lower RegisterStore
unsigned RegIndex = MI.getOperand(2).getImm();
unsigned Channel = MI.getOperand(3).getImm();
unsigned Address = TII->calculateIndirectAddress(RegIndex, Channel);
const TargetRegisterClass *IndirectStoreRegClass =
TII->getIndirectAddrStoreRegClass(MI.getOperand(0).getReg());
if (MI.getOperand(1).getReg() == AMDGPU::INDIRECT_BASE_ADDR) {
// Direct register access.
unsigned DstReg = MRI.createVirtualRegister(IndirectStoreRegClass);
BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY), DstReg)
.addOperand(MI.getOperand(0));
RegisterAddressMap[DstReg] = Address;
LiveAddressRegisterMap[Address] = DstReg;
} else {
// Indirect register access.
MachineInstrBuilder MOV = TII->buildIndirectWrite(BB, I,
MI.getOperand(0).getReg(), // Value
Address,
MI.getOperand(1).getReg()); // Offset
for (int i = IndirectBegin; i <= IndirectEnd; ++i) {
unsigned Addr = TII->calculateIndirectAddress(i, Channel);
unsigned DstReg = MRI.createVirtualRegister(IndirectStoreRegClass);
MOV.addReg(DstReg, RegState::Define | RegState::Implicit);
RegisterAddressMap[DstReg] = Addr;
LiveAddressRegisterMap[Addr] = DstReg;
}
}
MI.eraseFromParent();
}
// Update the live-ins of the succesor blocks
for (MachineBasicBlock::succ_iterator Succ = MBB.succ_begin(),
SuccEnd = MBB.succ_end();
SuccEnd != Succ; ++Succ) {
std::map<unsigned, unsigned>::const_iterator Key, KeyEnd;
for (Key = LiveAddressRegisterMap.begin(),
KeyEnd = LiveAddressRegisterMap.end(); KeyEnd != Key; ++Key) {
(*Succ)->addLiveIn(Key->second);
}
}
}
// Second pass - Lower the RegisterLoad instructions
for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end();
BB != BB_E; ++BB) {
// Key is the address and the value is the register
std::map<unsigned, unsigned> LiveAddressRegisterMap;
MachineBasicBlock &MBB = *BB;
MachineBasicBlock::livein_iterator LI = MBB.livein_begin();
while (LI != MBB.livein_end()) {
std::vector<unsigned> PhiRegisters;
// Make sure this live in is used for indirect addressing
if (RegisterAddressMap.find(*LI) == RegisterAddressMap.end()) {
++LI;
continue;
}
unsigned Address = RegisterAddressMap[*LI];
LiveAddressRegisterMap[Address] = *LI;
PhiRegisters.push_back(*LI);
// Check if there are other live in registers which map to the same
// indirect address.
for (MachineBasicBlock::livein_iterator LJ = llvm::next(LI),
LE = MBB.livein_end();
LJ != LE; ++LJ) {
unsigned Reg = *LJ;
if (RegisterAddressMap.find(Reg) == RegisterAddressMap.end()) {
continue;
}
if (RegisterAddressMap[Reg] == Address) {
PhiRegisters.push_back(Reg);
}
}
if (PhiRegisters.size() == 1) {
// We don't need to insert a Phi instruction, so we can just add the
// registers to the live list for the block.
LiveAddressRegisterMap[Address] = *LI;
MBB.removeLiveIn(*LI);
} else {
// We need to insert a PHI, because we have the same address being
// written in multiple predecessor blocks.
const TargetRegisterClass *PhiDstClass =
TII->getIndirectAddrStoreRegClass(*(PhiRegisters.begin()));
unsigned PhiDstReg = MRI.createVirtualRegister(PhiDstClass);
MachineInstrBuilder Phi = BuildMI(MBB, MBB.begin(),
MBB.findDebugLoc(MBB.begin()),
TII->get(AMDGPU::PHI), PhiDstReg);
for (std::vector<unsigned>::const_iterator RI = PhiRegisters.begin(),
RE = PhiRegisters.end();
RI != RE; ++RI) {
unsigned Reg = *RI;
MachineInstr *DefInst = MRI.getVRegDef(Reg);
assert(DefInst);
MachineBasicBlock *RegBlock = DefInst->getParent();
Phi.addReg(Reg);
Phi.addMBB(RegBlock);
MBB.removeLiveIn(Reg);
}
RegisterAddressMap[PhiDstReg] = Address;
LiveAddressRegisterMap[Address] = PhiDstReg;
}
LI = MBB.livein_begin();
}
for (MachineBasicBlock::iterator I = MBB.begin(), Next = llvm::next(I);
I != MBB.end(); I = Next) {
Next = llvm::next(I);
MachineInstr &MI = *I;
if (!TII->isRegisterLoad(MI)) {
if (MI.getOpcode() == AMDGPU::PHI) {
continue;
}
// Check for indirect register defs
for (unsigned OpIdx = 0, NumOperands = MI.getNumOperands();
OpIdx < NumOperands; ++OpIdx) {
MachineOperand &MO = MI.getOperand(OpIdx);
if (MO.isReg() && MO.isDef() &&
RegisterAddressMap.find(MO.getReg()) != RegisterAddressMap.end()) {
unsigned Reg = MO.getReg();
unsigned LiveAddress = RegisterAddressMap[Reg];
// Chain the live-ins
if (LiveAddressRegisterMap.find(LiveAddress) !=
RegisterAddressMap.end()) {
MI.addOperand(MachineOperand::CreateReg(
LiveAddressRegisterMap[LiveAddress],
false, // isDef
true, // isImp
true)); // isKill
}
LiveAddressRegisterMap[LiveAddress] = Reg;
}
}
continue;
}
const TargetRegisterClass *SuperIndirectRegClass =
TII->getSuperIndirectRegClass();
const TargetRegisterClass *IndirectLoadRegClass =
TII->getIndirectAddrLoadRegClass();
unsigned IndirectReg = MRI.createVirtualRegister(SuperIndirectRegClass);
unsigned RegIndex = MI.getOperand(2).getImm();
unsigned Channel = MI.getOperand(3).getImm();
unsigned Address = TII->calculateIndirectAddress(RegIndex, Channel);
if (MI.getOperand(1).getReg() == AMDGPU::INDIRECT_BASE_ADDR) {
// Direct register access
unsigned Reg = LiveAddressRegisterMap[Address];
unsigned AddrReg = IndirectLoadRegClass->getRegister(Address);
if (regHasExplicitDef(MRI, Reg)) {
// If the register we are reading from has an explicit def, then that
// means it was written via a direct register access (i.e. COPY
// or other instruction that doesn't use indirect addressing). In
// this case we know where the value has been stored, so we can just
// issue a copy.
BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY),
MI.getOperand(0).getReg())
.addReg(Reg);
} else {
// If the register we are reading has an implicit def, then that
// means it was written by an indirect register access (i.e. An
// instruction that uses indirect addressing.
BuildMI(MBB, I, MBB.findDebugLoc(I), TII->get(AMDGPU::COPY),
MI.getOperand(0).getReg())
.addReg(AddrReg)
.addReg(Reg, RegState::Implicit);
}
} else {
// Indirect register access
// Note on REQ_SEQUENCE instructons: You can't actually use the register
// it defines unless you have an instruction that takes the defined
// register class as an operand.
MachineInstrBuilder Sequence = BuildMI(MBB, I, MBB.findDebugLoc(I),
TII->get(AMDGPU::REG_SEQUENCE),
IndirectReg);
for (int i = IndirectBegin; i <= IndirectEnd; ++i) {
unsigned Addr = TII->calculateIndirectAddress(i, Channel);
if (LiveAddressRegisterMap.find(Addr) == LiveAddressRegisterMap.end()) {
continue;
}
unsigned Reg = LiveAddressRegisterMap[Addr];
// We only need to use REG_SEQUENCE for explicit defs, since the
// register coalescer won't do anything with the implicit defs.
if (!regHasExplicitDef(MRI, Reg)) {
continue;
}
// Insert a REQ_SEQUENCE instruction to force the register allocator
// to allocate the virtual register to the correct physical register.
Sequence.addReg(LiveAddressRegisterMap[Addr]);
Sequence.addImm(TII->getRegisterInfo().getIndirectSubReg(Addr));
}
MachineInstrBuilder Mov = TII->buildIndirectRead(BB, I,
MI.getOperand(0).getReg(), // Value
Address,
MI.getOperand(1).getReg()); // Offset
Mov.addReg(IndirectReg, RegState::Implicit | RegState::Kill);
Mov.addReg(LiveAddressRegisterMap[Address], RegState::Implicit);
}
MI.eraseFromParent();
}
}
return false;
}
void HexagonEarlyIfConversion::convert(const FlowPattern &FP) {
MachineBasicBlock *TSB = 0, *FSB = 0;
MachineBasicBlock::iterator OldTI = FP.SplitB->getFirstTerminator();
assert(OldTI != FP.SplitB->end());
DebugLoc DL = OldTI->getDebugLoc();
if (FP.TrueB) {
TSB = *FP.TrueB->succ_begin();
predicateBlockNB(FP.SplitB, OldTI, FP.TrueB, FP.PredR, true);
}
if (FP.FalseB) {
FSB = *FP.FalseB->succ_begin();
MachineBasicBlock::iterator At = FP.SplitB->getFirstTerminator();
predicateBlockNB(FP.SplitB, At, FP.FalseB, FP.PredR, false);
}
// Regenerate new terminators in the split block and update the successors.
// First, remember any information that may be needed later and remove the
// existing terminators/successors from the split block.
MachineBasicBlock *SSB = 0;
FP.SplitB->erase(OldTI, FP.SplitB->end());
while (FP.SplitB->succ_size() > 0) {
MachineBasicBlock *T = *FP.SplitB->succ_begin();
// It's possible that the split block had a successor that is not a pre-
// dicated block. This could only happen if there was only one block to
// be predicated. Example:
// split_b:
// if (p) jump true_b
// jump unrelated2_b
// unrelated1_b:
// ...
// unrelated2_b: ; can have other predecessors, so it's not "false_b"
// jump other_b
// true_b: ; only reachable from split_b, can be predicated
// ...
//
// Find this successor (SSB) if it exists.
if (T != FP.TrueB && T != FP.FalseB) {
assert(!SSB);
SSB = T;
}
FP.SplitB->removeSuccessor(FP.SplitB->succ_begin());
}
// Insert new branches and update the successors of the split block. This
// may create unconditional branches to the layout successor, etc., but
// that will be cleaned up later. For now, make sure that correct code is
// generated.
if (FP.JoinB) {
assert(!SSB || SSB == FP.JoinB);
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jump))
.addMBB(FP.JoinB);
FP.SplitB->addSuccessor(FP.JoinB);
} else {
bool HasBranch = false;
if (TSB) {
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jumpt))
.addReg(FP.PredR)
.addMBB(TSB);
FP.SplitB->addSuccessor(TSB);
HasBranch = true;
}
if (FSB) {
const MCInstrDesc &D = HasBranch ? TII->get(Hexagon::J2_jump)
: TII->get(Hexagon::J2_jumpf);
MachineInstrBuilder MIB = BuildMI(*FP.SplitB, FP.SplitB->end(), DL, D);
if (!HasBranch)
MIB.addReg(FP.PredR);
MIB.addMBB(FSB);
FP.SplitB->addSuccessor(FSB);
}
if (SSB) {
// This cannot happen if both TSB and FSB are set. [TF]SB are the
// successor blocks of the TrueB and FalseB (or null of the TrueB
// or FalseB block is null). SSB is the potential successor block
// of the SplitB that is neither TrueB nor FalseB.
BuildMI(*FP.SplitB, FP.SplitB->end(), DL, TII->get(Hexagon::J2_jump))
.addMBB(SSB);
FP.SplitB->addSuccessor(SSB);
}
}
// What is left to do is to update the PHI nodes that could have entries
// referring to predicated blocks.
if (FP.JoinB) {
updatePhiNodes(FP.JoinB, FP);
} else {
if (TSB)
updatePhiNodes(TSB, FP);
if (FSB)
updatePhiNodes(FSB, FP);
// Nothing to update in SSB, since SSB's predecessors haven't changed.
}
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr *MI) {
MachineBasicBlock *DestBB = getDestBlock(MI);
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
// If the branch is at the end of its MBB and that has a fall-through block,
// direct the updated conditional branch to the fall-through block. Otherwise,
// split the MBB before the next instruction.
MachineBasicBlock *MBB = MI->getParent();
MachineInstr *BMI = &MBB->back();
bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB);
if (BMI != MI) {
if (std::next(MachineBasicBlock::iterator(MI)) ==
std::prev(MBB->getLastNonDebugInstr()) &&
BMI->getOpcode() == AArch64::B) {
// Last MI in the BB is an unconditional branch. Can we simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB();
if (isBlockInRange(MI, NewDest,
getBranchDisplacementBits(MI->getOpcode()))) {
DEBUG(dbgs() << " Invert condition and swap its destination with "
<< *BMI);
BMI->getOperand(0).setMBB(DestBB);
unsigned OpNum = (MI->getOpcode() == AArch64::TBZW ||
MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX ||
MI->getOpcode() == AArch64::TBNZX)
? 2
: 1;
MI->getOperand(OpNum).setMBB(NewDest);
MI->setDesc(TII->get(getOppositeConditionOpcode(MI->getOpcode())));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MI);
return true;
}
}
}
if (NeedSplit) {
// Analyze the branch so we know how to update the successor lists.
MachineBasicBlock *TBB, *FBB;
SmallVector<MachineOperand, 2> Cond;
TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, false);
MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI);
// No need for the branch to the next block. We're adding an unconditional
// branch to the destination.
int delta = TII->GetInstSizeInBytes(&MBB->back());
BlockInfo[MBB->getNumber()].Size -= delta;
MBB->back().eraseFromParent();
// BlockInfo[SplitBB].Offset is wrong temporarily, fixed below
// Update the successor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
<< ", invert condition and change dest. to BB#"
<< NextBB->getNumber() << "\n");
// Insert a new conditional branch and a new unconditional branch.
MachineInstrBuilder MIB = BuildMI(
MBB, DebugLoc(), TII->get(getOppositeConditionOpcode(MI->getOpcode())))
.addOperand(MI->getOperand(0));
if (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX)
MIB.addOperand(MI->getOperand(1));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MIB);
MIB.addMBB(NextBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
// Remove the old conditional branch. It may or may not still be in MBB.
BlockInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
MI->eraseFromParent();
// Finally, keep the block offsets up to date.
adjustBlockOffsets(*MBB);
return true;
}
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
MachineLoopInfo &MLI,
MachineLoop *Loop) {
MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
SetVector<MachineBasicBlock *> RewriteSuccs;
// DFS through Loop's body, looking for for irreducible control flow. Loop is
// natural, and we stay in its body, and we treat any nested loops
// monolithically, so any cycles we encounter indicate irreducibility.
SmallPtrSet<MachineBasicBlock *, 8> OnStack;
SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<SuccessorList, 4> LoopWorklist;
LoopWorklist.push_back(SuccessorList(Header));
OnStack.insert(Header);
Visited.insert(Header);
while (!LoopWorklist.empty()) {
SuccessorList &Top = LoopWorklist.back();
if (Top.HasNext()) {
MachineBasicBlock *Next = Top.Next();
if (Next == Header || (Loop && !Loop->contains(Next)))
continue;
if (LLVM_LIKELY(OnStack.insert(Next).second)) {
if (!Visited.insert(Next).second) {
OnStack.erase(Next);
continue;
}
MachineLoop *InnerLoop = MLI.getLoopFor(Next);
if (InnerLoop != Loop)
LoopWorklist.push_back(SuccessorList(InnerLoop));
else
LoopWorklist.push_back(SuccessorList(Next));
} else {
RewriteSuccs.insert(Top.getBlock());
}
continue;
}
OnStack.erase(Top.getBlock());
LoopWorklist.pop_back();
}
// Most likely, we didn't find any irreducible control flow.
if (LLVM_LIKELY(RewriteSuccs.empty()))
return false;
DEBUG(dbgs() << "Irreducible control flow detected!\n");
// Ok. We have irreducible control flow! Create a dispatch block which will
// contains a jump table to any block in the problematic set of blocks.
MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
MF.insert(MF.end(), Dispatch);
MLI.changeLoopFor(Dispatch, Loop);
// Add the jump table.
const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(),
TII.get(WebAssembly::BR_TABLE_I32));
// Add the register which will be used to tell the jump table which block to
// jump to.
MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
MIB.addReg(Reg);
// Collect all the blocks which need to have their successors rewritten,
// add the successors to the jump table, and remember their index.
DenseMap<MachineBasicBlock *, unsigned> Indices;
SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
RewriteSuccs.end());
while (!SuccWorklist.empty()) {
MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
auto Pair = Indices.insert(std::make_pair(MBB, 0));
if (!Pair.second)
continue;
unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n");
Pair.first->second = Index;
for (auto Pred : MBB->predecessors())
RewriteSuccs.insert(Pred);
MIB.addMBB(MBB);
Dispatch->addSuccessor(MBB);
MetaBlock Meta(MBB);
for (auto *Succ : Meta.successors())
if (Succ != Header && (!Loop || Loop->contains(Succ)))
SuccWorklist.push_back(Succ);
}
// Rewrite the problematic successors for every block in RewriteSuccs.
// For simplicity, we just introduce a new block for every edge we need to
// rewrite. Fancier things are possible.
for (MachineBasicBlock *MBB : RewriteSuccs) {
DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
for (auto *Succ : MBB->successors()) {
if (!Indices.count(Succ))
continue;
MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
: MF.end(),
Split);
MLI.changeLoopFor(Split, Loop);
// Set the jump table's register of the index of the block we wish to
// jump to, and jump to the jump table.
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32),
Reg)
.addImm(Indices[Succ]);
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR))
.addMBB(Dispatch);
Split->addSuccessor(Dispatch);
Map[Succ] = Split;
}
// Remap the terminator operands and the successor list.
for (MachineInstr &Term : MBB->terminators())
for (auto &Op : Term.explicit_uses())
if (Op.isMBB() && Indices.count(Op.getMBB()))
Op.setMBB(Map[Op.getMBB()]);
for (auto Rewrite : Map)
MBB->replaceSuccessor(Rewrite.first, Rewrite.second);
}
// Create a fake default label, because br_table requires one.
MIB.addMBB(MIB.getInstr()
->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1)
.getMBB());
return true;
}
