MachineBasicBlock *Mips16TargetLowering::
emitSel16(unsigned Opc, MachineInstr *MI, MachineBasicBlock *BB) const {
if (DontExpandCondPseudos16)
return BB;
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
DebugLoc DL = MI->getDebugLoc();
// To "insert" a SELECT_CC instruction, we actually have to insert the
// diamond control-flow pattern. The incoming instruction knows the
// destination vreg to set, the condition code register to branch on, the
// true/false values to select between, and a branch opcode to use.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// thisMBB:
// ...
// TrueVal = ...
// setcc r1, r2, r3
// bNE r1, r0, copy1MBB
// fallthrough --> copy0MBB
MachineBasicBlock *thisMBB = BB;
MachineFunction *F = BB->getParent();
MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(It, copy0MBB);
F->insert(It, sinkMBB);
// Transfer the remainder of BB and its successor edges to sinkMBB.
sinkMBB->splice(sinkMBB->begin(), BB,
std::next(MachineBasicBlock::iterator(MI)), BB->end());
sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
// Next, add the true and fallthrough blocks as its successors.
BB->addSuccessor(copy0MBB);
BB->addSuccessor(sinkMBB);
BuildMI(BB, DL, TII->get(Opc)).addReg(MI->getOperand(3).getReg())
.addMBB(sinkMBB);
// copy0MBB:
// %FalseValue = ...
// # fallthrough to sinkMBB
BB = copy0MBB;
// Update machine-CFG edges
BB->addSuccessor(sinkMBB);
// sinkMBB:
// %Result = phi [ %TrueValue, thisMBB ], [ %FalseValue, copy0MBB ]
// ...
BB = sinkMBB;
BuildMI(*BB, BB->begin(), DL,
TII->get(Mips::PHI), MI->getOperand(0).getReg())
.addReg(MI->getOperand(1).getReg()).addMBB(thisMBB)
.addReg(MI->getOperand(2).getReg()).addMBB(copy0MBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return BB;
}
MachineBasicBlock*
MSP430TargetLowering::EmitShiftInstr(MachineInstr *MI,
MachineBasicBlock *BB) const {
MachineFunction *F = BB->getParent();
MachineRegisterInfo &RI = F->getRegInfo();
DebugLoc dl = MI->getDebugLoc();
const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo();
unsigned Opc;
const TargetRegisterClass * RC;
switch (MI->getOpcode()) {
default:
assert(0 && "Invalid shift opcode!");
case MSP430::Shl8:
Opc = MSP430::SHL8r1;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Shl16:
Opc = MSP430::SHL16r1;
RC = MSP430::GR16RegisterClass;
break;
case MSP430::Sra8:
Opc = MSP430::SAR8r1;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Sra16:
Opc = MSP430::SAR16r1;
RC = MSP430::GR16RegisterClass;
break;
case MSP430::Srl8:
Opc = MSP430::SAR8r1c;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Srl16:
Opc = MSP430::SAR16r1c;
RC = MSP430::GR16RegisterClass;
break;
}
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator I = BB;
++I;
// Create loop block
MachineBasicBlock *LoopBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *RemBB = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(I, LoopBB);
F->insert(I, RemBB);
// Update machine-CFG edges by transferring all successors of the current
// block to the block containing instructions after shift.
RemBB->splice(RemBB->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
RemBB->transferSuccessorsAndUpdatePHIs(BB);
// Add adges BB => LoopBB => RemBB, BB => RemBB, LoopBB => LoopBB
BB->addSuccessor(LoopBB);
BB->addSuccessor(RemBB);
LoopBB->addSuccessor(RemBB);
LoopBB->addSuccessor(LoopBB);
unsigned ShiftAmtReg = RI.createVirtualRegister(MSP430::GR8RegisterClass);
unsigned ShiftAmtReg2 = RI.createVirtualRegister(MSP430::GR8RegisterClass);
unsigned ShiftReg = RI.createVirtualRegister(RC);
unsigned ShiftReg2 = RI.createVirtualRegister(RC);
unsigned ShiftAmtSrcReg = MI->getOperand(2).getReg();
unsigned SrcReg = MI->getOperand(1).getReg();
unsigned DstReg = MI->getOperand(0).getReg();
// BB:
// cmp 0, N
// je RemBB
BuildMI(BB, dl, TII.get(MSP430::CMP8ri))
.addReg(ShiftAmtSrcReg).addImm(0);
BuildMI(BB, dl, TII.get(MSP430::JCC))
.addMBB(RemBB)
.addImm(MSP430CC::COND_E);
// LoopBB:
// ShiftReg = phi [%SrcReg, BB], [%ShiftReg2, LoopBB]
// ShiftAmt = phi [%N, BB], [%ShiftAmt2, LoopBB]
// ShiftReg2 = shift ShiftReg
// ShiftAmt2 = ShiftAmt - 1;
BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftReg)
.addReg(SrcReg).addMBB(BB)
.addReg(ShiftReg2).addMBB(LoopBB);
BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftAmtReg)
.addReg(ShiftAmtSrcReg).addMBB(BB)
.addReg(ShiftAmtReg2).addMBB(LoopBB);
BuildMI(LoopBB, dl, TII.get(Opc), ShiftReg2)
.addReg(ShiftReg);
BuildMI(LoopBB, dl, TII.get(MSP430::SUB8ri), ShiftAmtReg2)
.addReg(ShiftAmtReg).addImm(1);
BuildMI(LoopBB, dl, TII.get(MSP430::JCC))
.addMBB(LoopBB)
.addImm(MSP430CC::COND_NE);
// RemBB:
// DestReg = phi [%SrcReg, BB], [%ShiftReg, LoopBB]
BuildMI(*RemBB, RemBB->begin(), dl, TII.get(MSP430::PHI), DstReg)
.addReg(SrcReg).addMBB(BB)
.addReg(ShiftReg2).addMBB(LoopBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return RemBB;
}
MachineBasicBlock *
AVM2TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
MachineBasicBlock *BB) const
{
const TargetMachine &TM = getTargetMachine();
const TargetInstrInfo &TII = *TM.getInstrInfo();
unsigned BROpcode;
DebugLoc dl = MI->getDebugLoc();
bool useIntrin = TM.getSubtarget<AVM2Subtarget>().useIntrinsics();
// Figure out the conditional branch opcode to use for this select_cc.
switch (MI->getOpcode()) {
default: llvm_unreachable("Unknown SELECT_CC!");
case AVM2::SL:
case AVM2::SLF:
BROpcode = useIntrin ? AVM2::inCBR : AVM2::asCBR;
break;
case AVM2::FSL:
case AVM2::FSLF:
BROpcode = useIntrin ? AVM2::inFCBR : AVM2::asFCBR;
break;
}
// To "insert" a SELECT_CC instruction, we actually have to insert the diamond
// control-flow pattern. The incoming instruction knows the destination vreg
// to set, the condition code register to branch on, the true/false values to
// select between, and a branch opcode to use.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// thisMBB:
// ...
// TrueVal = ...
// [f]bCC copy1MBB
// fallthrough --> copy0MBB
MachineBasicBlock *thisMBB = BB;
MachineFunction *F = BB->getParent();
MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(It, copy0MBB);
F->insert(It, sinkMBB);
// Transfer the remainder of BB and its successor edges to sinkMBB.
sinkMBB->splice(sinkMBB->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
// Add the true and fallthrough blocks as its successors.
BB->addSuccessor(copy0MBB);
BB->addSuccessor(sinkMBB);
BuildMI(BB, dl, TII.get(BROpcode))
.addMBB(sinkMBB)
.addImm(MI->getOperand(1).getImm()) // CC
.addReg(MI->getOperand(2).getReg())
.addReg(MI->getOperand(3).getReg());
// copy0MBB:
// %FalseValue = ...
// # fallthrough to sinkMBB
BB = copy0MBB;
// Update machine-CFG edges
BB->addSuccessor(sinkMBB);
// sinkMBB:
// %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
// ...
BB = sinkMBB;
BuildMI(*BB, BB->begin(), dl, TII.get(AVM2::PHI), MI->getOperand(0).getReg())
.addReg(MI->getOperand(5).getReg()).addMBB(copy0MBB)
.addReg(MI->getOperand(4).getReg()).addMBB(thisMBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return BB;
}
//===----------------------------------------------------------------------===//
// Lower helper functions
//===----------------------------------------------------------------------===//
MachineBasicBlock*
MBlazeTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
MachineBasicBlock *BB) const {
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
DebugLoc dl = MI->getDebugLoc();
switch (MI->getOpcode()) {
default:
assert(false && "Unexpected instr type to insert");
case MBlaze::ShiftRL:
case MBlaze::ShiftRA:
case MBlaze::ShiftL: {
// To "insert" a shift left instruction, we actually have to insert a
// simple loop. The incoming instruction knows the destination vreg to
// set, the source vreg to operate over and the shift amount.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// start:
// andi samt, samt, 31
// beqid samt, finish
// add dst, src, r0
// loop:
// addik samt, samt, -1
// sra dst, dst
// bneid samt, loop
// nop
// finish:
MachineFunction *F = BB->getParent();
MachineRegisterInfo &R = F->getRegInfo();
MachineBasicBlock *loop = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *finish = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(It, loop);
F->insert(It, finish);
// Update machine-CFG edges by transfering adding all successors and
// remaining instructions from the current block to the new block which
// will contain the Phi node for the select.
finish->splice(finish->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
finish->transferSuccessorsAndUpdatePHIs(BB);
// Add the true and fallthrough blocks as its successors.
BB->addSuccessor(loop);
BB->addSuccessor(finish);
// Next, add the finish block as a successor of the loop block
loop->addSuccessor(finish);
loop->addSuccessor(loop);
unsigned IAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(BB, dl, TII->get(MBlaze::ANDI), IAMT)
.addReg(MI->getOperand(2).getReg())
.addImm(31);
unsigned IVAL = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(BB, dl, TII->get(MBlaze::ADDI), IVAL)
.addReg(MI->getOperand(1).getReg())
.addImm(0);
BuildMI(BB, dl, TII->get(MBlaze::BEQID))
.addReg(IAMT)
.addMBB(finish);
unsigned DST = R.createVirtualRegister(MBlaze::GPRRegisterClass);
unsigned NDST = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(loop, dl, TII->get(MBlaze::PHI), DST)
.addReg(IVAL).addMBB(BB)
.addReg(NDST).addMBB(loop);
unsigned SAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
unsigned NAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(loop, dl, TII->get(MBlaze::PHI), SAMT)
.addReg(IAMT).addMBB(BB)
.addReg(NAMT).addMBB(loop);
if (MI->getOpcode() == MBlaze::ShiftL)
BuildMI(loop, dl, TII->get(MBlaze::ADD), NDST).addReg(DST).addReg(DST);
else if (MI->getOpcode() == MBlaze::ShiftRA)
BuildMI(loop, dl, TII->get(MBlaze::SRA), NDST).addReg(DST);
else if (MI->getOpcode() == MBlaze::ShiftRL)
BuildMI(loop, dl, TII->get(MBlaze::SRL), NDST).addReg(DST);
else
llvm_unreachable("Cannot lower unknown shift instruction");
BuildMI(loop, dl, TII->get(MBlaze::ADDI), NAMT)
.addReg(SAMT)
.addImm(-1);
BuildMI(loop, dl, TII->get(MBlaze::BNEID))
.addReg(NAMT)
.addMBB(loop);
BuildMI(*finish, finish->begin(), dl,
TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
.addReg(IVAL).addMBB(BB)
.addReg(NDST).addMBB(loop);
// The pseudo instruction is no longer needed so remove it
MI->eraseFromParent();
return finish;
}
case MBlaze::Select_FCC:
case MBlaze::Select_CC: {
// To "insert" a SELECT_CC instruction, we actually have to insert the
// diamond control-flow pattern. The incoming instruction knows the
// destination vreg to set, the condition code register to branch on, the
// true/false values to select between, and a branch opcode to use.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// thisMBB:
// ...
// TrueVal = ...
// setcc r1, r2, r3
// bNE r1, r0, copy1MBB
// fallthrough --> copy0MBB
MachineFunction *F = BB->getParent();
MachineBasicBlock *flsBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *dneBB = F->CreateMachineBasicBlock(LLVM_BB);
unsigned Opc;
switch (MI->getOperand(4).getImm()) {
default:
llvm_unreachable("Unknown branch condition");
case MBlazeCC::EQ:
Opc = MBlaze::BEQID;
break;
case MBlazeCC::NE:
Opc = MBlaze::BNEID;
break;
case MBlazeCC::GT:
Opc = MBlaze::BGTID;
break;
case MBlazeCC::LT:
Opc = MBlaze::BLTID;
break;
case MBlazeCC::GE:
Opc = MBlaze::BGEID;
break;
case MBlazeCC::LE:
Opc = MBlaze::BLEID;
break;
}
F->insert(It, flsBB);
F->insert(It, dneBB);
// Transfer the remainder of BB and its successor edges to dneBB.
dneBB->splice(dneBB->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
dneBB->transferSuccessorsAndUpdatePHIs(BB);
BB->addSuccessor(flsBB);
BB->addSuccessor(dneBB);
flsBB->addSuccessor(dneBB);
BuildMI(BB, dl, TII->get(Opc))
.addReg(MI->getOperand(3).getReg())
.addMBB(dneBB);
// sinkMBB:
// %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
// ...
//BuildMI(dneBB, dl, TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
// .addReg(MI->getOperand(1).getReg()).addMBB(flsBB)
// .addReg(MI->getOperand(2).getReg()).addMBB(BB);
BuildMI(*dneBB, dneBB->begin(), dl,
TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
.addReg(MI->getOperand(2).getReg()).addMBB(flsBB)
.addReg(MI->getOperand(1).getReg()).addMBB(BB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return dneBB;
}
}
}
MachineBasicBlock *
AlphaTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
MachineBasicBlock *BB) const {
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
assert((MI->getOpcode() == Alpha::CAS32 ||
MI->getOpcode() == Alpha::CAS64 ||
MI->getOpcode() == Alpha::LAS32 ||
MI->getOpcode() == Alpha::LAS64 ||
MI->getOpcode() == Alpha::SWAP32 ||
MI->getOpcode() == Alpha::SWAP64) &&
"Unexpected instr type to insert");
bool is32 = MI->getOpcode() == Alpha::CAS32 ||
MI->getOpcode() == Alpha::LAS32 ||
MI->getOpcode() == Alpha::SWAP32;
//Load locked store conditional for atomic ops take on the same form
//start:
//ll
//do stuff (maybe branch to exit)
//sc
//test sc and maybe branck to start
//exit:
const BasicBlock *LLVM_BB = BB->getBasicBlock();
DebugLoc dl = MI->getDebugLoc();
MachineFunction::iterator It = BB;
++It;
MachineBasicBlock *thisMBB = BB;
MachineFunction *F = BB->getParent();
MachineBasicBlock *llscMBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
sinkMBB->splice(sinkMBB->begin(), thisMBB,
llvm::next(MachineBasicBlock::iterator(MI)),
thisMBB->end());
sinkMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
F->insert(It, llscMBB);
F->insert(It, sinkMBB);
BuildMI(thisMBB, dl, TII->get(Alpha::BR)).addMBB(llscMBB);
unsigned reg_res = MI->getOperand(0).getReg(),
reg_ptr = MI->getOperand(1).getReg(),
reg_v2 = MI->getOperand(2).getReg(),
reg_store = F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass);
BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::LDL_L : Alpha::LDQ_L),
reg_res).addImm(0).addReg(reg_ptr);
switch (MI->getOpcode()) {
case Alpha::CAS32:
case Alpha::CAS64: {
unsigned reg_cmp
= F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass);
BuildMI(llscMBB, dl, TII->get(Alpha::CMPEQ), reg_cmp)
.addReg(reg_v2).addReg(reg_res);
BuildMI(llscMBB, dl, TII->get(Alpha::BEQ))
.addImm(0).addReg(reg_cmp).addMBB(sinkMBB);
BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store)
.addReg(Alpha::R31).addReg(MI->getOperand(3).getReg());
break;
}
case Alpha::LAS32:
case Alpha::LAS64: {
BuildMI(llscMBB, dl,TII->get(is32 ? Alpha::ADDLr : Alpha::ADDQr), reg_store)
.addReg(reg_res).addReg(reg_v2);
break;
}
case Alpha::SWAP32:
case Alpha::SWAP64: {
BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store)
.addReg(reg_v2).addReg(reg_v2);
break;
}
}
BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::STL_C : Alpha::STQ_C), reg_store)
.addReg(reg_store).addImm(0).addReg(reg_ptr);
BuildMI(llscMBB, dl, TII->get(Alpha::BEQ))
.addImm(0).addReg(reg_store).addMBB(llscMBB);
BuildMI(llscMBB, dl, TII->get(Alpha::BR)).addMBB(sinkMBB);
thisMBB->addSuccessor(llscMBB);
llscMBB->addSuccessor(llscMBB);
llscMBB->addSuccessor(sinkMBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return sinkMBB;
}
void X86CmovConverterPass::convertCmovInstsToBranches(
SmallVectorImpl<MachineInstr *> &Group) const {
assert(!Group.empty() && "No CMOV instructions to convert");
++NumOfOptimizedCmovGroups;
// If the CMOV group is not packed, e.g., there are debug instructions between
// first CMOV and last CMOV, then pack the group and make the CMOV instruction
// consecutive by moving the debug instructions to after the last CMOV.
packCmovGroup(Group.front(), Group.back());
// To convert a CMOVcc instruction, we actually have to insert the diamond
// control-flow pattern. The incoming instruction knows the destination vreg
// to set, the condition code register to branch on, the true/false values to
// select between, and a branch opcode to use.
// Before
// -----
// MBB:
// cond = cmp ...
// v1 = CMOVge t1, f1, cond
// v2 = CMOVlt t2, f2, cond
// v3 = CMOVge v1, f3, cond
//
// After
// -----
// MBB:
// cond = cmp ...
// jge %SinkMBB
//
// FalseMBB:
// jmp %SinkMBB
//
// SinkMBB:
// %v1 = phi[%f1, %FalseMBB], [%t1, %MBB]
// %v2 = phi[%t2, %FalseMBB], [%f2, %MBB] ; For CMOV with OppCC switch
// ; true-value with false-value
// %v3 = phi[%f3, %FalseMBB], [%t1, %MBB] ; Phi instruction cannot use
// ; previous Phi instruction result
MachineInstr &MI = *Group.front();
MachineInstr *LastCMOV = Group.back();
DebugLoc DL = MI.getDebugLoc();
X86::CondCode CC = X86::CondCode(X86::getCondFromCMovOpc(MI.getOpcode()));
X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
// Potentially swap the condition codes so that any memory operand to a CMOV
// is in the *false* position instead of the *true* position. We can invert
// any non-memory operand CMOV instructions to cope with this and we ensure
// memory operand CMOVs are only included with a single condition code.
if (llvm::any_of(Group, [&](MachineInstr *I) {
return I->mayLoad() && X86::getCondFromCMovOpc(I->getOpcode()) == CC;
}))
std::swap(CC, OppCC);
MachineBasicBlock *MBB = MI.getParent();
MachineFunction::iterator It = ++MBB->getIterator();
MachineFunction *F = MBB->getParent();
const BasicBlock *BB = MBB->getBasicBlock();
MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(BB);
MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB);
F->insert(It, FalseMBB);
F->insert(It, SinkMBB);
// If the EFLAGS register isn't dead in the terminator, then claim that it's
// live into the sink and copy blocks.
if (checkEFLAGSLive(LastCMOV)) {
FalseMBB->addLiveIn(X86::EFLAGS);
SinkMBB->addLiveIn(X86::EFLAGS);
}
// Transfer the remainder of BB and its successor edges to SinkMBB.
SinkMBB->splice(SinkMBB->begin(), MBB,
std::next(MachineBasicBlock::iterator(LastCMOV)), MBB->end());
SinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
// Add the false and sink blocks as its successors.
MBB->addSuccessor(FalseMBB);
MBB->addSuccessor(SinkMBB);
// Create the conditional branch instruction.
BuildMI(MBB, DL, TII->get(X86::GetCondBranchFromCond(CC))).addMBB(SinkMBB);
// Add the sink block to the false block successors.
FalseMBB->addSuccessor(SinkMBB);
MachineInstrBuilder MIB;
MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
MachineBasicBlock::iterator MIItEnd =
std::next(MachineBasicBlock::iterator(LastCMOV));
MachineBasicBlock::iterator FalseInsertionPoint = FalseMBB->begin();
MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
// First we need to insert an explicit load on the false path for any memory
// operand. We also need to potentially do register rewriting here, but it is
// simpler as the memory operands are always on the false path so we can
// simply take that input, whatever it is.
DenseMap<unsigned, unsigned> FalseBBRegRewriteTable;
for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd;) {
auto &MI = *MIIt++;
// Skip any CMOVs in this group which don't load from memory.
if (!MI.mayLoad()) {
// Remember the false-side register input.
unsigned FalseReg =
MI.getOperand(X86::getCondFromCMovOpc(MI.getOpcode()) == CC ? 1 : 2)
.getReg();
// Walk back through any intermediate cmovs referenced.
while (true) {
auto FRIt = FalseBBRegRewriteTable.find(FalseReg);
if (FRIt == FalseBBRegRewriteTable.end())
break;
FalseReg = FRIt->second;
}
FalseBBRegRewriteTable[MI.getOperand(0).getReg()] = FalseReg;
continue;
}
// The condition must be the *opposite* of the one we've decided to branch
// on as the branch will go *around* the load and the load should happen
// when the CMOV condition is false.
assert(X86::getCondFromCMovOpc(MI.getOpcode()) == OppCC &&
"Can only handle memory-operand cmov instructions with a condition "
"opposite to the selected branch direction.");
// The goal is to rewrite the cmov from:
//
// MBB:
// %A = CMOVcc %B (tied), (mem)
//
// to
//
// MBB:
// %A = CMOVcc %B (tied), %C
// FalseMBB:
// %C = MOV (mem)
//
// Which will allow the next loop to rewrite the CMOV in terms of a PHI:
//
// MBB:
// JMP!cc SinkMBB
// FalseMBB:
// %C = MOV (mem)
// SinkMBB:
// %A = PHI [ %C, FalseMBB ], [ %B, MBB]
// Get a fresh register to use as the destination of the MOV.
const TargetRegisterClass *RC = MRI->getRegClass(MI.getOperand(0).getReg());
unsigned TmpReg = MRI->createVirtualRegister(RC);
SmallVector<MachineInstr *, 4> NewMIs;
bool Unfolded = TII->unfoldMemoryOperand(*MBB->getParent(), MI, TmpReg,
/*UnfoldLoad*/ true,
/*UnfoldStore*/ false, NewMIs);
(void)Unfolded;
assert(Unfolded && "Should never fail to unfold a loading cmov!");
// Move the new CMOV to just before the old one and reset any impacted
// iterator.
auto *NewCMOV = NewMIs.pop_back_val();
assert(X86::getCondFromCMovOpc(NewCMOV->getOpcode()) == OppCC &&
"Last new instruction isn't the expected CMOV!");
LLVM_DEBUG(dbgs() << "\tRewritten cmov: "; NewCMOV->dump());
MBB->insert(MachineBasicBlock::iterator(MI), NewCMOV);
if (&*MIItBegin == &MI)
MIItBegin = MachineBasicBlock::iterator(NewCMOV);
// Sink whatever instructions were needed to produce the unfolded operand
// into the false block.
for (auto *NewMI : NewMIs) {
LLVM_DEBUG(dbgs() << "\tRewritten load instr: "; NewMI->dump());
FalseMBB->insert(FalseInsertionPoint, NewMI);
// Re-map any operands that are from other cmovs to the inputs for this block.
for (auto &MOp : NewMI->uses()) {
if (!MOp.isReg())
continue;
auto It = FalseBBRegRewriteTable.find(MOp.getReg());
if (It == FalseBBRegRewriteTable.end())
continue;
MOp.setReg(It->second);
// This might have been a kill when it referenced the cmov result, but
// it won't necessarily be once rewritten.
// FIXME: We could potentially improve this by tracking whether the
// operand to the cmov was also a kill, and then skipping the PHI node
// construction below.
MOp.setIsKill(false);
}
}
MBB->erase(MachineBasicBlock::iterator(MI),
std::next(MachineBasicBlock::iterator(MI)));
// Add this PHI to the rewrite table.
FalseBBRegRewriteTable[NewCMOV->getOperand(0).getReg()] = TmpReg;
}
// As we are creating the PHIs, we have to be careful if there is more than
// one. Later CMOVs may reference the results of earlier CMOVs, but later
// PHIs have to reference the individual true/false inputs from earlier PHIs.
// That also means that PHI construction must work forward from earlier to
// later, and that the code must maintain a mapping from earlier PHI's
// destination registers, and the registers that went into the PHI.
DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
unsigned DestReg = MIIt->getOperand(0).getReg();
unsigned Op1Reg = MIIt->getOperand(1).getReg();
unsigned Op2Reg = MIIt->getOperand(2).getReg();
// If this CMOV we are processing is the opposite condition from the jump we
// generated, then we have to swap the operands for the PHI that is going to
// be generated.
if (X86::getCondFromCMovOpc(MIIt->getOpcode()) == OppCC)
std::swap(Op1Reg, Op2Reg);
auto Op1Itr = RegRewriteTable.find(Op1Reg);
if (Op1Itr != RegRewriteTable.end())
Op1Reg = Op1Itr->second.first;
auto Op2Itr = RegRewriteTable.find(Op2Reg);
if (Op2Itr != RegRewriteTable.end())
Op2Reg = Op2Itr->second.second;
// SinkMBB:
// %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, MBB ]
// ...
MIB = BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(X86::PHI), DestReg)
.addReg(Op1Reg)
.addMBB(FalseMBB)
.addReg(Op2Reg)
.addMBB(MBB);
(void)MIB;
LLVM_DEBUG(dbgs() << "\tFrom: "; MIIt->dump());
LLVM_DEBUG(dbgs() << "\tTo: "; MIB->dump());
// Add this PHI to the rewrite table.
RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
}
// Now remove the CMOV(s).
MBB->erase(MIItBegin, MIItEnd);
}
MachineBasicBlock * MipsSETargetLowering::
emitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{
// $bb:
// bposge32_pseudo $vr0
// =>
// $bb:
// bposge32 $tbb
// $fbb:
// li $vr2, 0
// b $sink
// $tbb:
// li $vr1, 1
// $sink:
// $vr0 = phi($vr2, $fbb, $vr1, $tbb)
MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
const TargetRegisterClass *RC = &Mips::CPURegsRegClass;
DebugLoc DL = MI->getDebugLoc();
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
MachineFunction *F = BB->getParent();
MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *Sink = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(It, FBB);
F->insert(It, TBB);
F->insert(It, Sink);
// Transfer the remainder of BB and its successor edges to Sink.
Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
Sink->transferSuccessorsAndUpdatePHIs(BB);
// Add successors.
BB->addSuccessor(FBB);
BB->addSuccessor(TBB);
FBB->addSuccessor(Sink);
TBB->addSuccessor(Sink);
// Insert the real bposge32 instruction to $BB.
BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB);
// Fill $FBB.
unsigned VR2 = RegInfo.createVirtualRegister(RC);
BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2)
.addReg(Mips::ZERO).addImm(0);
BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
// Fill $TBB.
unsigned VR1 = RegInfo.createVirtualRegister(RC);
BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1)
.addReg(Mips::ZERO).addImm(1);
// Insert phi function to $Sink.
BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
MI->getOperand(0).getReg())
.addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return Sink;
}
bool MipsExpandPseudo::expandAtomicCmpSwapSubword(
MachineBasicBlock &BB, MachineBasicBlock::iterator I,
MachineBasicBlock::iterator &NMBBI) {
MachineFunction *MF = BB.getParent();
const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
DebugLoc DL = I->getDebugLoc();
unsigned LL, SC;
unsigned ZERO = Mips::ZERO;
unsigned BNE = Mips::BNE;
unsigned BEQ = Mips::BEQ;
unsigned SEOp =
I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
if (STI->inMicroMipsMode()) {
LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
} else {
LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
: (ArePtrs64bit ? Mips::LL64 : Mips::LL);
SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
: (ArePtrs64bit ? Mips::SC64 : Mips::SC);
}
unsigned Dest = I->getOperand(0).getReg();
unsigned Ptr = I->getOperand(1).getReg();
unsigned Mask = I->getOperand(2).getReg();
unsigned ShiftCmpVal = I->getOperand(3).getReg();
unsigned Mask2 = I->getOperand(4).getReg();
unsigned ShiftNewVal = I->getOperand(5).getReg();
unsigned ShiftAmnt = I->getOperand(6).getReg();
unsigned Scratch = I->getOperand(7).getReg();
unsigned Scratch2 = I->getOperand(8).getReg();
// insert new blocks after the current block
const BasicBlock *LLVM_BB = BB.getBasicBlock();
MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineFunction::iterator It = ++BB.getIterator();
MF->insert(It, loop1MBB);
MF->insert(It, loop2MBB);
MF->insert(It, sinkMBB);
MF->insert(It, exitMBB);
// Transfer the remainder of BB and its successor edges to exitMBB.
exitMBB->splice(exitMBB->begin(), &BB,
std::next(MachineBasicBlock::iterator(I)), BB.end());
exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
// thisMBB:
// ...
// fallthrough --> loop1MBB
BB.addSuccessor(loop1MBB, BranchProbability::getOne());
loop1MBB->addSuccessor(sinkMBB);
loop1MBB->addSuccessor(loop2MBB);
loop1MBB->normalizeSuccProbs();
loop2MBB->addSuccessor(loop1MBB);
loop2MBB->addSuccessor(sinkMBB);
loop2MBB->normalizeSuccProbs();
sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
// loop1MBB:
// ll dest, 0(ptr)
// and Mask', dest, Mask
// bne Mask', ShiftCmpVal, exitMBB
BuildMI(loop1MBB, DL, TII->get(LL), Scratch).addReg(Ptr).addImm(0);
BuildMI(loop1MBB, DL, TII->get(Mips::AND), Scratch2)
.addReg(Scratch)
.addReg(Mask);
BuildMI(loop1MBB, DL, TII->get(BNE))
.addReg(Scratch2).addReg(ShiftCmpVal).addMBB(sinkMBB);
// loop2MBB:
// and dest, dest, mask2
// or dest, dest, ShiftNewVal
// sc dest, dest, 0(ptr)
// beq dest, $0, loop1MBB
BuildMI(loop2MBB, DL, TII->get(Mips::AND), Scratch)
.addReg(Scratch, RegState::Kill)
.addReg(Mask2);
BuildMI(loop2MBB, DL, TII->get(Mips::OR), Scratch)
.addReg(Scratch, RegState::Kill)
.addReg(ShiftNewVal);
BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
.addReg(Scratch, RegState::Kill)
.addReg(Ptr)
.addImm(0);
BuildMI(loop2MBB, DL, TII->get(BEQ))
.addReg(Scratch, RegState::Kill)
.addReg(ZERO)
.addMBB(loop1MBB);
// sinkMBB:
// srl srlres, Mask', shiftamt
// sign_extend dest,srlres
BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
.addReg(Scratch2)
.addReg(ShiftAmnt);
if (STI->hasMips32r2()) {
BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
} else {
const unsigned ShiftImm =
I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
.addReg(Dest, RegState::Kill)
.addImm(ShiftImm);
BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
.addReg(Dest, RegState::Kill)
.addImm(ShiftImm);
}
LivePhysRegs LiveRegs;
computeAndAddLiveIns(LiveRegs, *loop1MBB);
computeAndAddLiveIns(LiveRegs, *loop2MBB);
computeAndAddLiveIns(LiveRegs, *sinkMBB);
computeAndAddLiveIns(LiveRegs, *exitMBB);
NMBBI = BB.end();
I->eraseFromParent();
return true;
}
bool MipsExpandPseudo::expandAtomicBinOp(MachineBasicBlock &BB,
MachineBasicBlock::iterator I,
MachineBasicBlock::iterator &NMBBI,
unsigned Size) {
MachineFunction *MF = BB.getParent();
const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
DebugLoc DL = I->getDebugLoc();
unsigned LL, SC, ZERO, BEQ;
if (Size == 4) {
if (STI->inMicroMipsMode()) {
LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
} else {
LL = STI->hasMips32r6()
? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
: (ArePtrs64bit ? Mips::LL64 : Mips::LL);
SC = STI->hasMips32r6()
? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
: (ArePtrs64bit ? Mips::SC64 : Mips::SC);
BEQ = Mips::BEQ;
}
ZERO = Mips::ZERO;
} else {
LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
ZERO = Mips::ZERO_64;
BEQ = Mips::BEQ64;
}
unsigned OldVal = I->getOperand(0).getReg();
unsigned Ptr = I->getOperand(1).getReg();
unsigned Incr = I->getOperand(2).getReg();
unsigned Scratch = I->getOperand(3).getReg();
unsigned Opcode = 0;
unsigned OR = 0;
unsigned AND = 0;
unsigned NOR = 0;
bool IsNand = false;
switch (I->getOpcode()) {
case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
Opcode = Mips::ADDu;
break;
case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
Opcode = Mips::SUBu;
break;
case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
Opcode = Mips::AND;
break;
case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
Opcode = Mips::OR;
break;
case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
Opcode = Mips::XOR;
break;
case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
IsNand = true;
AND = Mips::AND;
NOR = Mips::NOR;
break;
case Mips::ATOMIC_SWAP_I32_POSTRA:
OR = Mips::OR;
break;
case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
Opcode = Mips::DADDu;
break;
case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
Opcode = Mips::DSUBu;
break;
case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
Opcode = Mips::AND64;
break;
case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
Opcode = Mips::OR64;
break;
case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
Opcode = Mips::XOR64;
break;
case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
IsNand = true;
AND = Mips::AND64;
NOR = Mips::NOR64;
break;
case Mips::ATOMIC_SWAP_I64_POSTRA:
OR = Mips::OR64;
break;
default:
llvm_unreachable("Unknown pseudo atomic!");
}
const BasicBlock *LLVM_BB = BB.getBasicBlock();
MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineFunction::iterator It = ++BB.getIterator();
MF->insert(It, loopMBB);
MF->insert(It, exitMBB);
exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
BB.addSuccessor(loopMBB, BranchProbability::getOne());
loopMBB->addSuccessor(exitMBB);
loopMBB->addSuccessor(loopMBB);
loopMBB->normalizeSuccProbs();
BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
assert((OldVal != Incr) && "Clobbered the wrong reg!");
if (Opcode) {
BuildMI(loopMBB, DL, TII->get(Opcode), Scratch).addReg(OldVal).addReg(Incr);
} else if (IsNand) {
assert(AND && NOR &&
"Unknown nand instruction for atomic pseudo expansion");
BuildMI(loopMBB, DL, TII->get(AND), Scratch).addReg(OldVal).addReg(Incr);
BuildMI(loopMBB, DL, TII->get(NOR), Scratch).addReg(ZERO).addReg(Scratch);
} else {
assert(OR && "Unknown instruction for atomic pseudo expansion!");
BuildMI(loopMBB, DL, TII->get(OR), Scratch).addReg(Incr).addReg(ZERO);
}
BuildMI(loopMBB, DL, TII->get(SC), Scratch).addReg(Scratch).addReg(Ptr).addImm(0);
BuildMI(loopMBB, DL, TII->get(BEQ)).addReg(Scratch).addReg(ZERO).addMBB(loopMBB);
NMBBI = BB.end();
I->eraseFromParent();
LivePhysRegs LiveRegs;
computeAndAddLiveIns(LiveRegs, *loopMBB);
computeAndAddLiveIns(LiveRegs, *exitMBB);
return true;
}
bool MipsExpandPseudo::expandAtomicBinOpSubword(
MachineBasicBlock &BB, MachineBasicBlock::iterator I,
MachineBasicBlock::iterator &NMBBI) {
MachineFunction *MF = BB.getParent();
const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
DebugLoc DL = I->getDebugLoc();
unsigned LL, SC;
unsigned BEQ = Mips::BEQ;
unsigned SEOp = Mips::SEH;
if (STI->inMicroMipsMode()) {
LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
} else {
LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
: (ArePtrs64bit ? Mips::LL64 : Mips::LL);
SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
: (ArePtrs64bit ? Mips::SC64 : Mips::SC);
}
bool IsSwap = false;
bool IsNand = false;
unsigned Opcode = 0;
switch (I->getOpcode()) {
case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
IsNand = true;
break;
case Mips::ATOMIC_SWAP_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_SWAP_I16_POSTRA:
IsSwap = true;
break;
case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
Opcode = Mips::ADDu;
break;
case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
Opcode = Mips::SUBu;
break;
case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
Opcode = Mips::AND;
break;
case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
Opcode = Mips::OR;
break;
case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
SEOp = Mips::SEB;
LLVM_FALLTHROUGH;
case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
Opcode = Mips::XOR;
break;
default:
llvm_unreachable("Unknown subword atomic pseudo for expansion!");
}
unsigned Dest = I->getOperand(0).getReg();
unsigned Ptr = I->getOperand(1).getReg();
unsigned Incr = I->getOperand(2).getReg();
unsigned Mask = I->getOperand(3).getReg();
unsigned Mask2 = I->getOperand(4).getReg();
unsigned ShiftAmnt = I->getOperand(5).getReg();
unsigned OldVal = I->getOperand(6).getReg();
unsigned BinOpRes = I->getOperand(7).getReg();
unsigned StoreVal = I->getOperand(8).getReg();
const BasicBlock *LLVM_BB = BB.getBasicBlock();
MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineFunction::iterator It = ++BB.getIterator();
MF->insert(It, loopMBB);
MF->insert(It, sinkMBB);
MF->insert(It, exitMBB);
exitMBB->splice(exitMBB->begin(), &BB, std::next(I), BB.end());
exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
BB.addSuccessor(loopMBB, BranchProbability::getOne());
loopMBB->addSuccessor(sinkMBB);
loopMBB->addSuccessor(loopMBB);
loopMBB->normalizeSuccProbs();
BuildMI(loopMBB, DL, TII->get(LL), OldVal).addReg(Ptr).addImm(0);
if (IsNand) {
// and andres, oldval, incr2
// nor binopres, $0, andres
// and newval, binopres, mask
BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
.addReg(OldVal)
.addReg(Incr);
BuildMI(loopMBB, DL, TII->get(Mips::NOR), BinOpRes)
.addReg(Mips::ZERO)
.addReg(BinOpRes);
BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
.addReg(BinOpRes)
.addReg(Mask);
} else if (!IsSwap) {
// <binop> binopres, oldval, incr2
// and newval, binopres, mask
BuildMI(loopMBB, DL, TII->get(Opcode), BinOpRes)
.addReg(OldVal)
.addReg(Incr);
BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
.addReg(BinOpRes)
.addReg(Mask);
} else { // atomic.swap
// and newval, incr2, mask
BuildMI(loopMBB, DL, TII->get(Mips::AND), BinOpRes)
.addReg(Incr)
.addReg(Mask);
}
// and StoreVal, OlddVal, Mask2
// or StoreVal, StoreVal, BinOpRes
// StoreVal<tied1> = sc StoreVal, 0(Ptr)
// beq StoreVal, zero, loopMBB
BuildMI(loopMBB, DL, TII->get(Mips::AND), StoreVal)
.addReg(OldVal).addReg(Mask2);
BuildMI(loopMBB, DL, TII->get(Mips::OR), StoreVal)
.addReg(StoreVal).addReg(BinOpRes);
BuildMI(loopMBB, DL, TII->get(SC), StoreVal)
.addReg(StoreVal).addReg(Ptr).addImm(0);
BuildMI(loopMBB, DL, TII->get(BEQ))
.addReg(StoreVal).addReg(Mips::ZERO).addMBB(loopMBB);
// sinkMBB:
// and maskedoldval1,oldval,mask
// srl srlres,maskedoldval1,shiftamt
// sign_extend dest,srlres
sinkMBB->addSuccessor(exitMBB, BranchProbability::getOne());
BuildMI(sinkMBB, DL, TII->get(Mips::AND), Dest)
.addReg(OldVal).addReg(Mask);
BuildMI(sinkMBB, DL, TII->get(Mips::SRLV), Dest)
.addReg(Dest).addReg(ShiftAmnt);
if (STI->hasMips32r2()) {
BuildMI(sinkMBB, DL, TII->get(SEOp), Dest).addReg(Dest);
} else {
const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
BuildMI(sinkMBB, DL, TII->get(Mips::SLL), Dest)
.addReg(Dest, RegState::Kill)
.addImm(ShiftImm);
BuildMI(sinkMBB, DL, TII->get(Mips::SRA), Dest)
.addReg(Dest, RegState::Kill)
.addImm(ShiftImm);
}
LivePhysRegs LiveRegs;
computeAndAddLiveIns(LiveRegs, *loopMBB);
computeAndAddLiveIns(LiveRegs, *sinkMBB);
computeAndAddLiveIns(LiveRegs, *exitMBB);
NMBBI = BB.end();
I->eraseFromParent();
return true;
}
bool MipsExpandPseudo::expandAtomicCmpSwap(MachineBasicBlock &BB,
MachineBasicBlock::iterator I,
MachineBasicBlock::iterator &NMBBI) {
const unsigned Size =
I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
MachineFunction *MF = BB.getParent();
const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
DebugLoc DL = I->getDebugLoc();
unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
if (Size == 4) {
if (STI->inMicroMipsMode()) {
LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
} else {
LL = STI->hasMips32r6()
? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
: (ArePtrs64bit ? Mips::LL64 : Mips::LL);
SC = STI->hasMips32r6()
? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
: (ArePtrs64bit ? Mips::SC64 : Mips::SC);
BNE = Mips::BNE;
BEQ = Mips::BEQ;
}
ZERO = Mips::ZERO;
MOVE = Mips::OR;
} else {
LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
ZERO = Mips::ZERO_64;
BNE = Mips::BNE64;
BEQ = Mips::BEQ64;
MOVE = Mips::OR64;
}
unsigned Dest = I->getOperand(0).getReg();
unsigned Ptr = I->getOperand(1).getReg();
unsigned OldVal = I->getOperand(2).getReg();
unsigned NewVal = I->getOperand(3).getReg();
unsigned Scratch = I->getOperand(4).getReg();
// insert new blocks after the current block
const BasicBlock *LLVM_BB = BB.getBasicBlock();
MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MachineFunction::iterator It = ++BB.getIterator();
MF->insert(It, loop1MBB);
MF->insert(It, loop2MBB);
MF->insert(It, exitMBB);
// Transfer the remainder of BB and its successor edges to exitMBB.
exitMBB->splice(exitMBB->begin(), &BB,
std::next(MachineBasicBlock::iterator(I)), BB.end());
exitMBB->transferSuccessorsAndUpdatePHIs(&BB);
// thisMBB:
// ...
// fallthrough --> loop1MBB
BB.addSuccessor(loop1MBB, BranchProbability::getOne());
loop1MBB->addSuccessor(exitMBB);
loop1MBB->addSuccessor(loop2MBB);
loop1MBB->normalizeSuccProbs();
loop2MBB->addSuccessor(loop1MBB);
loop2MBB->addSuccessor(exitMBB);
loop2MBB->normalizeSuccProbs();
// loop1MBB:
// ll dest, 0(ptr)
// bne dest, oldval, exitMBB
BuildMI(loop1MBB, DL, TII->get(LL), Dest).addReg(Ptr).addImm(0);
BuildMI(loop1MBB, DL, TII->get(BNE))
.addReg(Dest, RegState::Kill).addReg(OldVal).addMBB(exitMBB);
// loop2MBB:
// move scratch, NewVal
// sc Scratch, Scratch, 0(ptr)
// beq Scratch, $0, loop1MBB
BuildMI(loop2MBB, DL, TII->get(MOVE), Scratch).addReg(NewVal).addReg(ZERO);
BuildMI(loop2MBB, DL, TII->get(SC), Scratch)
.addReg(Scratch).addReg(Ptr).addImm(0);
BuildMI(loop2MBB, DL, TII->get(BEQ))
.addReg(Scratch, RegState::Kill).addReg(ZERO).addMBB(loop1MBB);
LivePhysRegs LiveRegs;
computeAndAddLiveIns(LiveRegs, *loop1MBB);
computeAndAddLiveIns(LiveRegs, *loop2MBB);
computeAndAddLiveIns(LiveRegs, *exitMBB);
NMBBI = BB.end();
I->eraseFromParent();
return true;
}
