/// Generate a predicated version of MI (where the condition is given via
/// PredR and Cond) at the point indicated by Where.
void HexagonExpandCondsets::predicateAt(const MachineOperand &DefOp,
MachineInstr &MI,
MachineBasicBlock::iterator Where,
const MachineOperand &PredOp, bool Cond,
std::set<unsigned> &UpdRegs) {
// The problem with updating live intervals is that we can move one def
// past another def. In particular, this can happen when moving an A2_tfrt
// over an A2_tfrf defining the same register. From the point of view of
// live intervals, these two instructions are two separate definitions,
// and each one starts another live segment. LiveIntervals's "handleMove"
// does not allow such moves, so we need to handle it ourselves. To avoid
// invalidating liveness data while we are using it, the move will be
// implemented in 4 steps: (1) add a clone of the instruction MI at the
// target location, (2) update liveness, (3) delete the old instruction,
// and (4) update liveness again.
MachineBasicBlock &B = *MI.getParent();
DebugLoc DL = Where->getDebugLoc(); // "Where" points to an instruction.
unsigned Opc = MI.getOpcode();
unsigned PredOpc = HII->getCondOpcode(Opc, !Cond);
MachineInstrBuilder MB = BuildMI(B, Where, DL, HII->get(PredOpc));
unsigned Ox = 0, NP = MI.getNumOperands();
// Skip all defs from MI first.
while (Ox < NP) {
MachineOperand &MO = MI.getOperand(Ox);
if (!MO.isReg() || !MO.isDef())
break;
Ox++;
}
// Add the new def, then the predicate register, then the rest of the
// operands.
MB.addReg(DefOp.getReg(), getRegState(DefOp), DefOp.getSubReg());
MB.addReg(PredOp.getReg(), PredOp.isUndef() ? RegState::Undef : 0,
PredOp.getSubReg());
while (Ox < NP) {
MachineOperand &MO = MI.getOperand(Ox);
if (!MO.isReg() || !MO.isImplicit())
MB.add(MO);
Ox++;
}
MachineFunction &MF = *B.getParent();
MachineInstr::mmo_iterator I = MI.memoperands_begin();
unsigned NR = std::distance(I, MI.memoperands_end());
MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(NR);
for (unsigned i = 0; i < NR; ++i)
MemRefs[i] = *I++;
MB.setMemRefs(MemRefs, MemRefs+NR);
MachineInstr *NewI = MB;
NewI->clearKillInfo();
LIS->InsertMachineInstrInMaps(*NewI);
for (auto &Op : NewI->operands())
if (Op.isReg())
UpdRegs.insert(Op.getReg());
}
void HexagonEarlyIfConversion::predicateInstr(MachineBasicBlock *ToB,
MachineBasicBlock::iterator At, MachineInstr *MI,
unsigned PredR, bool IfTrue) {
DebugLoc DL;
if (At != ToB->end())
DL = At->getDebugLoc();
else if (!ToB->empty())
DL = ToB->back().getDebugLoc();
unsigned Opc = MI->getOpcode();
if (isPredicableStore(MI)) {
unsigned COpc = getCondStoreOpcode(Opc, IfTrue);
assert(COpc);
MachineInstrBuilder MIB = BuildMI(*ToB, At, DL, HII->get(COpc));
MachineInstr::mop_iterator MOI = MI->operands_begin();
if (HII->isPostIncrement(*MI)) {
MIB.addOperand(*MOI);
++MOI;
}
MIB.addReg(PredR);
for (const MachineOperand &MO : make_range(MOI, MI->operands_end()))
MIB.addOperand(MO);
// Set memory references.
MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
MIB.setMemRefs(MMOBegin, MMOEnd);
MI->eraseFromParent();
return;
}
if (Opc == Hexagon::J2_jump) {
MachineBasicBlock *TB = MI->getOperand(0).getMBB();
const MCInstrDesc &D = HII->get(IfTrue ? Hexagon::J2_jumpt
: Hexagon::J2_jumpf);
BuildMI(*ToB, At, DL, D)
.addReg(PredR)
.addMBB(TB);
MI->eraseFromParent();
return;
}
// Print the offending instruction unconditionally as we are about to
// abort.
dbgs() << *MI;
llvm_unreachable("Unexpected instruction");
}
void ARMInstrInfo::expandLoadStackGuard(MachineBasicBlock::iterator MI) const {
MachineFunction &MF = *MI->getParent()->getParent();
const ARMSubtarget &Subtarget = MF.getSubtarget<ARMSubtarget>();
const TargetMachine &TM = MF.getTarget();
if (!Subtarget.useMovt(MF)) {
if (TM.isPositionIndependent())
expandLoadStackGuardBase(MI, ARM::LDRLIT_ga_pcrel, ARM::LDRi12);
else
expandLoadStackGuardBase(MI, ARM::LDRLIT_ga_abs, ARM::LDRi12);
return;
}
if (!TM.isPositionIndependent()) {
expandLoadStackGuardBase(MI, ARM::MOVi32imm, ARM::LDRi12);
return;
}
const GlobalValue *GV =
cast<GlobalValue>((*MI->memoperands_begin())->getValue());
if (!Subtarget.isGVIndirectSymbol(GV)) {
expandLoadStackGuardBase(MI, ARM::MOV_ga_pcrel, ARM::LDRi12);
return;
}
MachineBasicBlock &MBB = *MI->getParent();
DebugLoc DL = MI->getDebugLoc();
unsigned Reg = MI->getOperand(0).getReg();
MachineInstrBuilder MIB;
MIB = BuildMI(MBB, MI, DL, get(ARM::MOV_ga_pcrel_ldr), Reg)
.addGlobalAddress(GV, 0, ARMII::MO_NONLAZY);
auto Flags = MachineMemOperand::MOLoad |
MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant;
MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4);
MIB.addMemOperand(MMO);
MIB = BuildMI(MBB, MI, DL, get(ARM::LDRi12), Reg);
MIB.addReg(Reg, RegState::Kill).addImm(0);
MIB.setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
AddDefaultPred(MIB);
}
MachineInstrBuilder
MipsInstrInfo::genInstrWithNewOpc(unsigned NewOpc,
MachineBasicBlock::iterator I) const {
MachineInstrBuilder MIB;
// Certain branches have two forms: e.g beq $1, $zero, dst vs beqz $1, dest
// Pick the zero form of the branch for readable assembly and for greater
// branch distance in non-microMIPS mode.
// FIXME: Certain atomic sequences on mips64 generate 32bit references to
// Mips::ZERO, which is incorrect. This test should be updated to use
// Subtarget.getABI().GetZeroReg() when those atomic sequences and others
// are fixed.
bool BranchWithZeroOperand =
(I->isBranch() && !I->isPseudo() && I->getOperand(1).isReg() &&
(I->getOperand(1).getReg() == Mips::ZERO ||
I->getOperand(1).getReg() == Mips::ZERO_64));
if (BranchWithZeroOperand) {
switch (NewOpc) {
case Mips::BEQC:
NewOpc = Mips::BEQZC;
break;
case Mips::BNEC:
NewOpc = Mips::BNEZC;
break;
case Mips::BGEC:
NewOpc = Mips::BGEZC;
break;
case Mips::BLTC:
NewOpc = Mips::BLTZC;
break;
}
}
MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), get(NewOpc));
// For MIPSR6 JI*C requires an immediate 0 as an operand, JIALC(64) an
// immediate 0 as an operand and requires the removal of it's %RA<imp-def>
// implicit operand as copying the implicit operations of the instructio we're
// looking at will give us the correct flags.
if (NewOpc == Mips::JIC || NewOpc == Mips::JIALC || NewOpc == Mips::JIC64 ||
NewOpc == Mips::JIALC64) {
if (NewOpc == Mips::JIALC || NewOpc == Mips::JIALC64)
MIB->RemoveOperand(0);
for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) {
MIB.addOperand(I->getOperand(J));
}
MIB.addImm(0);
} else if (BranchWithZeroOperand) {
// For MIPSR6 and microMIPS branches with an explicit zero operand, copy
// everything after the zero.
MIB.addOperand(I->getOperand(0));
for (unsigned J = 2, E = I->getDesc().getNumOperands(); J < E; ++J) {
MIB.addOperand(I->getOperand(J));
}
} else {
// All other cases copy all other operands.
for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) {
MIB.addOperand(I->getOperand(J));
}
}
MIB.copyImplicitOps(*I);
MIB.setMemRefs(I->memoperands_begin(), I->memoperands_end());
return MIB;
}
MachineInstrBuilder
MipsInstrInfo::genInstrWithNewOpc(unsigned NewOpc,
MachineBasicBlock::iterator I) const {
MachineInstrBuilder MIB;
// Certain branches have two forms: e.g beq $1, $zero, dest vs beqz $1, dest
// Pick the zero form of the branch for readable assembly and for greater
// branch distance in non-microMIPS mode.
// Additional MIPSR6 does not permit the use of register $zero for compact
// branches.
// FIXME: Certain atomic sequences on mips64 generate 32bit references to
// Mips::ZERO, which is incorrect. This test should be updated to use
// Subtarget.getABI().GetZeroReg() when those atomic sequences and others
// are fixed.
int ZeroOperandPosition = -1;
bool BranchWithZeroOperand = false;
if (I->isBranch() && !I->isPseudo()) {
auto TRI = I->getParent()->getParent()->getSubtarget().getRegisterInfo();
ZeroOperandPosition = I->findRegisterUseOperandIdx(Mips::ZERO, false, TRI);
BranchWithZeroOperand = ZeroOperandPosition != -1;
}
if (BranchWithZeroOperand) {
switch (NewOpc) {
case Mips::BEQC:
NewOpc = Mips::BEQZC;
break;
case Mips::BNEC:
NewOpc = Mips::BNEZC;
break;
case Mips::BGEC:
NewOpc = Mips::BGEZC;
break;
case Mips::BLTC:
NewOpc = Mips::BLTZC;
break;
case Mips::BEQC64:
NewOpc = Mips::BEQZC64;
break;
case Mips::BNEC64:
NewOpc = Mips::BNEZC64;
break;
}
}
MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), get(NewOpc));
// For MIPSR6 JI*C requires an immediate 0 as an operand, JIALC(64) an
// immediate 0 as an operand and requires the removal of it's %RA<imp-def>
// implicit operand as copying the implicit operations of the instructio we're
// looking at will give us the correct flags.
if (NewOpc == Mips::JIC || NewOpc == Mips::JIALC || NewOpc == Mips::JIC64 ||
NewOpc == Mips::JIALC64) {
if (NewOpc == Mips::JIALC || NewOpc == Mips::JIALC64)
MIB->RemoveOperand(0);
for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) {
MIB.add(I->getOperand(J));
}
MIB.addImm(0);
} else {
for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) {
if (BranchWithZeroOperand && (unsigned)ZeroOperandPosition == J)
continue;
MIB.add(I->getOperand(J));
}
}
MIB.copyImplicitOps(*I);
MIB.setMemRefs(I->memoperands_begin(), I->memoperands_end());
return MIB;
}
/// EmitMachineNode - Generate machine code for a target-specific node and
/// needed dependencies.
///
void InstrEmitter::
EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap) {
unsigned Opc = Node->getMachineOpcode();
// Handle subreg insert/extract specially
if (Opc == TargetOpcode::EXTRACT_SUBREG ||
Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
return;
}
// Handle COPY_TO_REGCLASS specially.
if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
EmitCopyToRegClassNode(Node, VRBaseMap);
return;
}
// Handle REG_SEQUENCE specially.
if (Opc == TargetOpcode::REG_SEQUENCE) {
EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
return;
}
if (Opc == TargetOpcode::IMPLICIT_DEF)
// We want a unique VR for each IMPLICIT_DEF use.
return;
const MCInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NumDefs = II.getNumDefs();
const MCPhysReg *ScratchRegs = nullptr;
// Handle STACKMAP and PATCHPOINT specially and then use the generic code.
if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
// Stackmaps do not have arguments and do not preserve their calling
// convention. However, to simplify runtime support, they clobber the same
// scratch registers as AnyRegCC.
unsigned CC = CallingConv::AnyReg;
if (Opc == TargetOpcode::PATCHPOINT) {
CC = Node->getConstantOperandVal(PatchPointOpers::CCPos);
NumDefs = NumResults;
}
ScratchRegs = TLI->getScratchRegisters((CallingConv::ID) CC);
}
unsigned NumImpUses = 0;
unsigned NodeOperands =
countOperands(Node, II.getNumOperands() - NumDefs, NumImpUses);
bool HasPhysRegOuts = NumResults > NumDefs && II.getImplicitDefs()!=nullptr;
#ifndef NDEBUG
unsigned NumMIOperands = NodeOperands + NumResults;
if (II.isVariadic())
assert(NumMIOperands >= II.getNumOperands() &&
"Too few operands for a variadic node!");
else
assert(NumMIOperands >= II.getNumOperands() &&
NumMIOperands <= II.getNumOperands() + II.getNumImplicitDefs() +
NumImpUses &&
"#operands for dag node doesn't match .td file!");
#endif
// Create the new machine instruction.
MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
// Add result register values for things that are defined by this
// instruction.
if (NumResults) {
CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
// Transfer any IR flags from the SDNode to the MachineInstr
MachineInstr *MI = MIB.getInstr();
const SDNodeFlags Flags = Node->getFlags();
if (Flags.hasNoSignedZeros())
MI->setFlag(MachineInstr::MIFlag::FmNsz);
if (Flags.hasAllowReciprocal())
MI->setFlag(MachineInstr::MIFlag::FmArcp);
if (Flags.hasNoNaNs())
MI->setFlag(MachineInstr::MIFlag::FmNoNans);
if (Flags.hasNoInfs())
MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
if (Flags.hasAllowContract())
MI->setFlag(MachineInstr::MIFlag::FmContract);
if (Flags.hasApproximateFuncs())
MI->setFlag(MachineInstr::MIFlag::FmAfn);
if (Flags.hasAllowReassociation())
MI->setFlag(MachineInstr::MIFlag::FmReassoc);
}
// Emit all of the actual operands of this instruction, adding them to the
// instruction as appropriate.
bool HasOptPRefs = NumDefs > NumResults;
assert((!HasOptPRefs || !HasPhysRegOuts) &&
"Unable to cope with optional defs and phys regs defs!");
unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
for (unsigned i = NumSkip; i != NodeOperands; ++i)
AddOperand(MIB, Node->getOperand(i), i-NumSkip+NumDefs, &II,
VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
// Add scratch registers as implicit def and early clobber
if (ScratchRegs)
for (unsigned i = 0; ScratchRegs[i]; ++i)
MIB.addReg(ScratchRegs[i], RegState::ImplicitDefine |
RegState::EarlyClobber);
// Transfer all of the memory reference descriptions of this instruction.
MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
cast<MachineSDNode>(Node)->memoperands_end());
// Insert the instruction into position in the block. This needs to
// happen before any custom inserter hook is called so that the
// hook knows where in the block to insert the replacement code.
MBB->insert(InsertPos, MIB);
// The MachineInstr may also define physregs instead of virtregs. These
// physreg values can reach other instructions in different ways:
//
// 1. When there is a use of a Node value beyond the explicitly defined
// virtual registers, we emit a CopyFromReg for one of the implicitly
// defined physregs. This only happens when HasPhysRegOuts is true.
//
// 2. A CopyFromReg reading a physreg may be glued to this instruction.
//
// 3. A glued instruction may implicitly use a physreg.
//
// 4. A glued instruction may use a RegisterSDNode operand.
//
// Collect all the used physreg defs, and make sure that any unused physreg
// defs are marked as dead.
SmallVector<unsigned, 8> UsedRegs;
// Additional results must be physical register defs.
if (HasPhysRegOuts) {
for (unsigned i = NumDefs; i < NumResults; ++i) {
unsigned Reg = II.getImplicitDefs()[i - NumDefs];
if (!Node->hasAnyUseOfValue(i))
continue;
// This implicitly defined physreg has a use.
UsedRegs.push_back(Reg);
EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
}
}
// Scan the glue chain for any used physregs.
if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
if (F->getOpcode() == ISD::CopyFromReg) {
UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
continue;
} else if (F->getOpcode() == ISD::CopyToReg) {
// Skip CopyToReg nodes that are internal to the glue chain.
continue;
}
// Collect declared implicit uses.
const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
UsedRegs.append(MCID.getImplicitUses(),
MCID.getImplicitUses() + MCID.getNumImplicitUses());
// In addition to declared implicit uses, we must also check for
// direct RegisterSDNode operands.
for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
unsigned Reg = R->getReg();
if (TargetRegisterInfo::isPhysicalRegister(Reg))
UsedRegs.push_back(Reg);
}
}
}
// Finally mark unused registers as dead.
if (!UsedRegs.empty() || II.getImplicitDefs())
MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
// Run post-isel target hook to adjust this instruction if needed.
if (II.hasPostISelHook())
TLI->AdjustInstrPostInstrSelection(*MIB, Node);
}
bool
Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
const ReduceEntry &Entry) {
if (ReduceLimitLdSt != -1 && ((int)NumLdSts >= ReduceLimitLdSt))
return false;
unsigned Scale = 1;
bool HasImmOffset = false;
bool HasShift = false;
bool HasOffReg = true;
bool isLdStMul = false;
unsigned Opc = Entry.NarrowOpc1;
unsigned OpNum = 3; // First 'rest' of operands.
uint8_t ImmLimit = Entry.Imm1Limit;
switch (Entry.WideOpc) {
default:
llvm_unreachable("Unexpected Thumb2 load / store opcode!");
case ARM::t2LDRi12:
case ARM::t2STRi12:
if (MI->getOperand(1).getReg() == ARM::SP) {
Opc = Entry.NarrowOpc2;
ImmLimit = Entry.Imm2Limit;
HasOffReg = false;
}
Scale = 4;
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRBi12:
case ARM::t2STRBi12:
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRHi12:
case ARM::t2STRHi12:
Scale = 2;
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRs:
case ARM::t2LDRBs:
case ARM::t2LDRHs:
case ARM::t2LDRSBs:
case ARM::t2LDRSHs:
case ARM::t2STRs:
case ARM::t2STRBs:
case ARM::t2STRHs:
HasShift = true;
OpNum = 4;
break;
case ARM::t2LDMIA:
case ARM::t2LDMDB: {
unsigned BaseReg = MI->getOperand(0).getReg();
if (!isARMLowRegister(BaseReg) || Entry.WideOpc != ARM::t2LDMIA)
return false;
// For the non-writeback version (this one), the base register must be
// one of the registers being loaded.
bool isOK = false;
for (unsigned i = 4; i < MI->getNumOperands(); ++i) {
if (MI->getOperand(i).getReg() == BaseReg) {
isOK = true;
break;
}
}
if (!isOK)
return false;
OpNum = 0;
isLdStMul = true;
break;
}
case ARM::t2LDMIA_RET: {
unsigned BaseReg = MI->getOperand(1).getReg();
if (BaseReg != ARM::SP)
return false;
Opc = Entry.NarrowOpc2; // tPOP_RET
OpNum = 2;
isLdStMul = true;
break;
}
case ARM::t2LDMIA_UPD:
case ARM::t2LDMDB_UPD:
case ARM::t2STMIA_UPD:
case ARM::t2STMDB_UPD: {
OpNum = 0;
unsigned BaseReg = MI->getOperand(1).getReg();
if (BaseReg == ARM::SP &&
(Entry.WideOpc == ARM::t2LDMIA_UPD ||
Entry.WideOpc == ARM::t2STMDB_UPD)) {
Opc = Entry.NarrowOpc2; // tPOP or tPUSH
OpNum = 2;
} else if (!isARMLowRegister(BaseReg) ||
(Entry.WideOpc != ARM::t2LDMIA_UPD &&
Entry.WideOpc != ARM::t2STMIA_UPD)) {
return false;
}
isLdStMul = true;
break;
}
}
unsigned OffsetReg = 0;
bool OffsetKill = false;
if (HasShift) {
OffsetReg = MI->getOperand(2).getReg();
OffsetKill = MI->getOperand(2).isKill();
if (MI->getOperand(3).getImm())
// Thumb1 addressing mode doesn't support shift.
return false;
}
unsigned OffsetImm = 0;
if (HasImmOffset) {
OffsetImm = MI->getOperand(2).getImm();
unsigned MaxOffset = ((1 << ImmLimit) - 1) * Scale;
if ((OffsetImm & (Scale - 1)) || OffsetImm > MaxOffset)
// Make sure the immediate field fits.
return false;
}
// Add the 16-bit load / store instruction.
DebugLoc dl = MI->getDebugLoc();
MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, TII->get(Opc));
if (!isLdStMul) {
MIB.addOperand(MI->getOperand(0));
MIB.addOperand(MI->getOperand(1));
if (HasImmOffset)
MIB.addImm(OffsetImm / Scale);
assert((!HasShift || OffsetReg) && "Invalid so_reg load / store address!");
if (HasOffReg)
MIB.addReg(OffsetReg, getKillRegState(OffsetKill));
}
// Transfer the rest of operands.
for (unsigned e = MI->getNumOperands(); OpNum != e; ++OpNum)
MIB.addOperand(MI->getOperand(OpNum));
// Transfer memoperands.
MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
// Transfer MI flags.
MIB.setMIFlags(MI->getFlags());
DEBUG(errs() << "Converted 32-bit: " << *MI << " to 16-bit: " << *MIB);
MBB.erase_instr(MI);
++NumLdSts;
return true;
}
/// EmitMachineNode - Generate machine code for a target-specific node and
/// needed dependencies.
///
void InstrEmitter::
EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap) {
unsigned Opc = Node->getMachineOpcode();
// Handle subreg insert/extract specially
if (Opc == TargetOpcode::EXTRACT_SUBREG ||
Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
return;
}
// Handle COPY_TO_REGCLASS specially.
if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
EmitCopyToRegClassNode(Node, VRBaseMap);
return;
}
// Handle REG_SEQUENCE specially.
if (Opc == TargetOpcode::REG_SEQUENCE) {
EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
return;
}
if (Opc == TargetOpcode::IMPLICIT_DEF)
// We want a unique VR for each IMPLICIT_DEF use.
return;
const MCInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NumImpUses = 0;
unsigned NodeOperands =
countOperands(Node, II.getNumOperands() - II.getNumDefs(), NumImpUses);
bool HasPhysRegOuts = NumResults > II.getNumDefs() && II.getImplicitDefs()!=0;
#ifndef NDEBUG
unsigned NumMIOperands = NodeOperands + NumResults;
if (II.isVariadic())
assert(NumMIOperands >= II.getNumOperands() &&
"Too few operands for a variadic node!");
else
assert(NumMIOperands >= II.getNumOperands() &&
NumMIOperands <= II.getNumOperands() + II.getNumImplicitDefs() +
NumImpUses &&
"#operands for dag node doesn't match .td file!");
#endif
// Create the new machine instruction.
MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), II);
// Add result register values for things that are defined by this
// instruction.
if (NumResults)
CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
// Emit all of the actual operands of this instruction, adding them to the
// instruction as appropriate.
bool HasOptPRefs = II.getNumDefs() > NumResults;
assert((!HasOptPRefs || !HasPhysRegOuts) &&
"Unable to cope with optional defs and phys regs defs!");
unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0;
for (unsigned i = NumSkip; i != NodeOperands; ++i)
AddOperand(MIB, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
// Transfer all of the memory reference descriptions of this instruction.
MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
cast<MachineSDNode>(Node)->memoperands_end());
// Insert the instruction into position in the block. This needs to
// happen before any custom inserter hook is called so that the
// hook knows where in the block to insert the replacement code.
MBB->insert(InsertPos, MIB);
// The MachineInstr may also define physregs instead of virtregs. These
// physreg values can reach other instructions in different ways:
//
// 1. When there is a use of a Node value beyond the explicitly defined
// virtual registers, we emit a CopyFromReg for one of the implicitly
// defined physregs. This only happens when HasPhysRegOuts is true.
//
// 2. A CopyFromReg reading a physreg may be glued to this instruction.
//
// 3. A glued instruction may implicitly use a physreg.
//
// 4. A glued instruction may use a RegisterSDNode operand.
//
// Collect all the used physreg defs, and make sure that any unused physreg
// defs are marked as dead.
SmallVector<unsigned, 8> UsedRegs;
// Additional results must be physical register defs.
if (HasPhysRegOuts) {
for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
if (!Node->hasAnyUseOfValue(i))
continue;
// This implicitly defined physreg has a use.
UsedRegs.push_back(Reg);
EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
}
}
// Scan the glue chain for any used physregs.
if (Node->getValueType(Node->getNumValues()-1) == MVT::Glue) {
for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
if (F->getOpcode() == ISD::CopyFromReg) {
UsedRegs.push_back(cast<RegisterSDNode>(F->getOperand(1))->getReg());
continue;
} else if (F->getOpcode() == ISD::CopyToReg) {
// Skip CopyToReg nodes that are internal to the glue chain.
continue;
}
// Collect declared implicit uses.
const MCInstrDesc &MCID = TII->get(F->getMachineOpcode());
UsedRegs.append(MCID.getImplicitUses(),
MCID.getImplicitUses() + MCID.getNumImplicitUses());
// In addition to declared implicit uses, we must also check for
// direct RegisterSDNode operands.
for (unsigned i = 0, e = F->getNumOperands(); i != e; ++i)
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(F->getOperand(i))) {
unsigned Reg = R->getReg();
if (TargetRegisterInfo::isPhysicalRegister(Reg))
UsedRegs.push_back(Reg);
}
}
}
// Finally mark unused registers as dead.
if (!UsedRegs.empty() || II.getImplicitDefs())
MIB->setPhysRegsDeadExcept(UsedRegs, *TRI);
// Run post-isel target hook to adjust this instruction if needed.
#ifdef NDEBUG
if (II.hasPostISelHook())
#endif
TLI->AdjustInstrPostInstrSelection(MIB, Node);
}
bool
Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
const ReduceEntry &Entry) {
if (ReduceLimitLdSt != -1 && ((int)NumLdSts >= ReduceLimitLdSt))
return false;
unsigned Scale = 1;
bool HasImmOffset = false;
bool HasShift = false;
bool HasOffReg = true;
bool isLdStMul = false;
unsigned Opc = Entry.NarrowOpc1;
unsigned OpNum = 3; // First 'rest' of operands.
uint8_t ImmLimit = Entry.Imm1Limit;
switch (Entry.WideOpc) {
default:
llvm_unreachable("Unexpected Thumb2 load / store opcode!");
case ARM::t2LDRi12:
case ARM::t2STRi12:
if (MI->getOperand(1).getReg() == ARM::SP) {
Opc = Entry.NarrowOpc2;
ImmLimit = Entry.Imm2Limit;
}
Scale = 4;
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRBi12:
case ARM::t2STRBi12:
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRHi12:
case ARM::t2STRHi12:
Scale = 2;
HasImmOffset = true;
HasOffReg = false;
break;
case ARM::t2LDRs:
case ARM::t2LDRBs:
case ARM::t2LDRHs:
case ARM::t2LDRSBs:
case ARM::t2LDRSHs:
case ARM::t2STRs:
case ARM::t2STRBs:
case ARM::t2STRHs:
HasShift = true;
OpNum = 4;
break;
case ARM::t2LDR_POST:
case ARM::t2STR_POST: {
if (!MBB.getParent()->getFunction().optForMinSize())
return false;
if (!MI->hasOneMemOperand() ||
(*MI->memoperands_begin())->getAlignment() < 4)
return false;
// We're creating a completely different type of load/store - LDM from LDR.
// For this reason we can't reuse the logic at the end of this function; we
// have to implement the MI building here.
bool IsStore = Entry.WideOpc == ARM::t2STR_POST;
unsigned Rt = MI->getOperand(IsStore ? 1 : 0).getReg();
unsigned Rn = MI->getOperand(IsStore ? 0 : 1).getReg();
unsigned Offset = MI->getOperand(3).getImm();
unsigned PredImm = MI->getOperand(4).getImm();
unsigned PredReg = MI->getOperand(5).getReg();
assert(isARMLowRegister(Rt));
assert(isARMLowRegister(Rn));
if (Offset != 4)
return false;
// Add the 16-bit load / store instruction.
DebugLoc dl = MI->getDebugLoc();
auto MIB = BuildMI(MBB, MI, dl, TII->get(Entry.NarrowOpc1))
.addReg(Rn, RegState::Define)
.addReg(Rn)
.addImm(PredImm)
.addReg(PredReg)
.addReg(Rt, IsStore ? 0 : RegState::Define);
// Transfer memoperands.
MIB.setMemRefs(MI->memoperands());
// Transfer MI flags.
MIB.setMIFlags(MI->getFlags());
// Kill the old instruction.
MI->eraseFromBundle();
++NumLdSts;
return true;
}
case ARM::t2LDMIA: {
unsigned BaseReg = MI->getOperand(0).getReg();
assert(isARMLowRegister(BaseReg));
// For the non-writeback version (this one), the base register must be
// one of the registers being loaded.
bool isOK = false;
for (unsigned i = 3; i < MI->getNumOperands(); ++i) {
if (MI->getOperand(i).getReg() == BaseReg) {
isOK = true;
break;
}
}
if (!isOK)
return false;
OpNum = 0;
isLdStMul = true;
break;
}
case ARM::t2STMIA:
// If the base register is killed, we don't care what its value is after the
// instruction, so we can use an updating STMIA.
if (!MI->getOperand(0).isKill())
return false;
break;
case ARM::t2LDMIA_RET: {
unsigned BaseReg = MI->getOperand(1).getReg();
if (BaseReg != ARM::SP)
return false;
Opc = Entry.NarrowOpc2; // tPOP_RET
OpNum = 2;
isLdStMul = true;
break;
}
case ARM::t2LDMIA_UPD:
case ARM::t2STMIA_UPD:
case ARM::t2STMDB_UPD: {
OpNum = 0;
unsigned BaseReg = MI->getOperand(1).getReg();
if (BaseReg == ARM::SP &&
(Entry.WideOpc == ARM::t2LDMIA_UPD ||
Entry.WideOpc == ARM::t2STMDB_UPD)) {
Opc = Entry.NarrowOpc2; // tPOP or tPUSH
OpNum = 2;
} else if (!isARMLowRegister(BaseReg) ||
(Entry.WideOpc != ARM::t2LDMIA_UPD &&
Entry.WideOpc != ARM::t2STMIA_UPD)) {
return false;
}
isLdStMul = true;
break;
}
}
unsigned OffsetReg = 0;
bool OffsetKill = false;
bool OffsetInternal = false;
if (HasShift) {
OffsetReg = MI->getOperand(2).getReg();
OffsetKill = MI->getOperand(2).isKill();
OffsetInternal = MI->getOperand(2).isInternalRead();
if (MI->getOperand(3).getImm())
// Thumb1 addressing mode doesn't support shift.
return false;
}
unsigned OffsetImm = 0;
if (HasImmOffset) {
OffsetImm = MI->getOperand(2).getImm();
unsigned MaxOffset = ((1 << ImmLimit) - 1) * Scale;
if ((OffsetImm & (Scale - 1)) || OffsetImm > MaxOffset)
// Make sure the immediate field fits.
return false;
}
// Add the 16-bit load / store instruction.
DebugLoc dl = MI->getDebugLoc();
MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, TII->get(Opc));
// tSTMIA_UPD takes a defining register operand. We've already checked that
// the register is killed, so mark it as dead here.
if (Entry.WideOpc == ARM::t2STMIA)
MIB.addReg(MI->getOperand(0).getReg(), RegState::Define | RegState::Dead);
if (!isLdStMul) {
MIB.add(MI->getOperand(0));
MIB.add(MI->getOperand(1));
if (HasImmOffset)
MIB.addImm(OffsetImm / Scale);
assert((!HasShift || OffsetReg) && "Invalid so_reg load / store address!");
if (HasOffReg)
MIB.addReg(OffsetReg, getKillRegState(OffsetKill) |
getInternalReadRegState(OffsetInternal));
}
// Transfer the rest of operands.
for (unsigned e = MI->getNumOperands(); OpNum != e; ++OpNum)
MIB.add(MI->getOperand(OpNum));
// Transfer memoperands.
MIB.setMemRefs(MI->memoperands());
// Transfer MI flags.
MIB.setMIFlags(MI->getFlags());
LLVM_DEBUG(errs() << "Converted 32-bit: " << *MI
<< " to 16-bit: " << *MIB);
MBB.erase_instr(MI);
++NumLdSts;
return true;
}
// Convert callee-save register save/restore instruction to do stack pointer
// decrement/increment to allocate/deallocate the callee-save stack area by
// converting store/load to use pre/post increment version.
static MachineBasicBlock::iterator convertCalleeSaveRestoreToSPPrePostIncDec(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
const DebugLoc &DL, const TargetInstrInfo *TII, int CSStackSizeInc) {
unsigned NewOpc;
bool NewIsUnscaled = false;
switch (MBBI->getOpcode()) {
default:
llvm_unreachable("Unexpected callee-save save/restore opcode!");
case AArch64::STPXi:
NewOpc = AArch64::STPXpre;
break;
case AArch64::STPDi:
NewOpc = AArch64::STPDpre;
break;
case AArch64::STRXui:
NewOpc = AArch64::STRXpre;
NewIsUnscaled = true;
break;
case AArch64::STRDui:
NewOpc = AArch64::STRDpre;
NewIsUnscaled = true;
break;
case AArch64::LDPXi:
NewOpc = AArch64::LDPXpost;
break;
case AArch64::LDPDi:
NewOpc = AArch64::LDPDpost;
break;
case AArch64::LDRXui:
NewOpc = AArch64::LDRXpost;
NewIsUnscaled = true;
break;
case AArch64::LDRDui:
NewOpc = AArch64::LDRDpost;
NewIsUnscaled = true;
break;
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
MIB.addReg(AArch64::SP, RegState::Define);
// Copy all operands other than the immediate offset.
unsigned OpndIdx = 0;
for (unsigned OpndEnd = MBBI->getNumOperands() - 1; OpndIdx < OpndEnd;
++OpndIdx)
MIB.add(MBBI->getOperand(OpndIdx));
assert(MBBI->getOperand(OpndIdx).getImm() == 0 &&
"Unexpected immediate offset in first/last callee-save save/restore "
"instruction!");
assert(MBBI->getOperand(OpndIdx - 1).getReg() == AArch64::SP &&
"Unexpected base register in callee-save save/restore instruction!");
// Last operand is immediate offset that needs fixing.
assert(CSStackSizeInc % 8 == 0);
int64_t CSStackSizeIncImm = CSStackSizeInc;
if (!NewIsUnscaled)
CSStackSizeIncImm /= 8;
MIB.addImm(CSStackSizeIncImm);
MIB.setMIFlags(MBBI->getFlags());
MIB.setMemRefs(MBBI->memoperands_begin(), MBBI->memoperands_end());
return std::prev(MBB.erase(MBBI));
}