/// HandlePhysRegUse - Turn previous partial def's into read/mod/writes. Add
/// implicit defs to a machine instruction if there was an earlier def of its
/// super-register.
void LiveVariables::HandlePhysRegUse(unsigned Reg, MachineInstr *MI) {
// If there was a previous use or a "full" def all is well.
if (!PhysRegDef[Reg] && !PhysRegUse[Reg]) {
// Otherwise, the last sub-register def implicitly defines this register.
// e.g.
// AH =
// AL = ... <imp-def EAX>, <imp-kill AH>
// = AH
// ...
// = EAX
// All of the sub-registers must have been defined before the use of Reg!
unsigned PartDefReg = 0;
MachineInstr *LastPartialDef = FindLastPartialDef(Reg, PartDefReg);
// If LastPartialDef is NULL, it must be using a livein register.
if (LastPartialDef) {
LastPartialDef->addOperand(MachineOperand::CreateReg(Reg, true/*IsDef*/,
true/*IsImp*/));
PhysRegDef[Reg] = LastPartialDef;
SmallSet<unsigned, 8> Processed;
for (const unsigned *SubRegs = TRI->getSubRegisters(Reg);
unsigned SubReg = *SubRegs; ++SubRegs) {
if (Processed.count(SubReg))
continue;
if (SubReg == PartDefReg || TRI->isSubRegister(PartDefReg, SubReg))
continue;
// This part of Reg was defined before the last partial def. It's killed
// here.
LastPartialDef->addOperand(MachineOperand::CreateReg(SubReg,
false/*IsDef*/,
true/*IsImp*/));
PhysRegDef[SubReg] = LastPartialDef;
for (const unsigned *SS = TRI->getSubRegisters(SubReg); *SS; ++SS)
Processed.insert(*SS);
}
}
}
// There was an earlier def of a super-register. Add implicit def to that MI.
//
// A: EAX = ...
// B: ... = AX
//
// Add implicit def to A if there isn't a use of AX (or EAX) before B.
if (!PhysRegUse[Reg]) {
MachineInstr *Def = PhysRegDef[Reg];
if (Def && !Def->modifiesRegister(Reg))
Def->addOperand(MachineOperand::CreateReg(Reg,
true /*IsDef*/,
true /*IsImp*/));
}
// Remember this use.
PhysRegUse[Reg] = MI;
for (const unsigned *SubRegs = TRI->getSubRegisters(Reg);
unsigned SubReg = *SubRegs; ++SubRegs)
PhysRegUse[SubReg] = MI;
}
/// HandlePhysRegUse - Turn previous partial def's into read/mod/writes. Add
/// implicit defs to a machine instruction if there was an earlier def of its
/// super-register.
void LiveVariables::HandlePhysRegUse(unsigned Reg, MachineInstr *MI) {
MachineInstr *LastDef = PhysRegDef[Reg];
// If there was a previous use or a "full" def all is well.
if (!LastDef && !PhysRegUse[Reg]) {
// Otherwise, the last sub-register def implicitly defines this register.
// e.g.
// AH =
// AL = ... <imp-def EAX>, <imp-kill AH>
// = AH
// ...
// = EAX
// All of the sub-registers must have been defined before the use of Reg!
SmallSet<unsigned, 4> PartDefRegs;
MachineInstr *LastPartialDef = FindLastPartialDef(Reg, PartDefRegs);
// If LastPartialDef is NULL, it must be using a livein register.
if (LastPartialDef) {
LastPartialDef->addOperand(MachineOperand::CreateReg(Reg, true/*IsDef*/,
true/*IsImp*/));
PhysRegDef[Reg] = LastPartialDef;
SmallSet<unsigned, 8> Processed;
for (const unsigned *SubRegs = TRI->getSubRegisters(Reg);
unsigned SubReg = *SubRegs; ++SubRegs) {
if (Processed.count(SubReg))
continue;
if (PartDefRegs.count(SubReg))
continue;
// This part of Reg was defined before the last partial def. It's killed
// here.
LastPartialDef->addOperand(MachineOperand::CreateReg(SubReg,
false/*IsDef*/,
true/*IsImp*/));
PhysRegDef[SubReg] = LastPartialDef;
for (const unsigned *SS = TRI->getSubRegisters(SubReg); *SS; ++SS)
Processed.insert(*SS);
}
}
}
else if (LastDef && !PhysRegUse[Reg] &&
!LastDef->findRegisterDefOperand(Reg))
// Last def defines the super register, add an implicit def of reg.
LastDef->addOperand(MachineOperand::CreateReg(Reg,
true/*IsDef*/, true/*IsImp*/));
// Remember this use.
PhysRegUse[Reg] = MI;
for (const unsigned *SubRegs = TRI->getSubRegisters(Reg);
unsigned SubReg = *SubRegs; ++SubRegs)
PhysRegUse[SubReg] = MI;
}
void HexagonEarlyIfConversion::updatePhiNodes(MachineBasicBlock *WhereB,
const FlowPattern &FP) {
// Visit all PHI nodes in the WhereB block and generate MUX instructions
// in the split block. Update the PHI nodes with the values of the MUX.
auto NonPHI = WhereB->getFirstNonPHI();
for (auto I = WhereB->begin(); I != NonPHI; ++I) {
MachineInstr *PN = &*I;
// Registers and subregisters corresponding to TrueB, FalseB and SplitB.
unsigned TR = 0, TSR = 0, FR = 0, FSR = 0, SR = 0, SSR = 0;
for (int i = PN->getNumOperands()-2; i > 0; i -= 2) {
const MachineOperand &RO = PN->getOperand(i), &BO = PN->getOperand(i+1);
if (BO.getMBB() == FP.SplitB)
SR = RO.getReg(), SSR = RO.getSubReg();
else if (BO.getMBB() == FP.TrueB)
TR = RO.getReg(), TSR = RO.getSubReg();
else if (BO.getMBB() == FP.FalseB)
FR = RO.getReg(), FSR = RO.getSubReg();
else
continue;
PN->RemoveOperand(i+1);
PN->RemoveOperand(i);
}
if (TR == 0)
TR = SR, TSR = SSR;
else if (FR == 0)
FR = SR, FSR = SSR;
assert(TR || FR);
unsigned MuxR = 0, MuxSR = 0;
if (TR && FR) {
unsigned DR = PN->getOperand(0).getReg();
const TargetRegisterClass *RC = MRI->getRegClass(DR);
MuxR = buildMux(FP.SplitB, FP.SplitB->getFirstTerminator(), RC,
FP.PredR, TR, TSR, FR, FSR);
} else if (TR) {
MuxR = TR;
MuxSR = TSR;
} else {
MuxR = FR;
MuxSR = FSR;
}
PN->addOperand(MachineOperand::CreateReg(MuxR, false, false, false, false,
false, false, MuxSR));
PN->addOperand(MachineOperand::CreateMBB(FP.SplitB));
}
}
void HexagonEarlyIfConversion::updatePhiNodes(MachineBasicBlock *WhereB,
const FlowPattern &FP) {
// Visit all PHI nodes in the WhereB block and generate MUX instructions
// in the split block. Update the PHI nodes with the values of the MUX.
auto NonPHI = WhereB->getFirstNonPHI();
for (auto I = WhereB->begin(); I != NonPHI; ++I) {
MachineInstr *PN = &*I;
// Registers and subregisters corresponding to TrueB, FalseB and SplitB.
unsigned TR = 0, TSR = 0, FR = 0, FSR = 0, SR = 0, SSR = 0;
for (int i = PN->getNumOperands()-2; i > 0; i -= 2) {
const MachineOperand &RO = PN->getOperand(i), &BO = PN->getOperand(i+1);
if (BO.getMBB() == FP.SplitB)
SR = RO.getReg(), SSR = RO.getSubReg();
else if (BO.getMBB() == FP.TrueB)
TR = RO.getReg(), TSR = RO.getSubReg();
else if (BO.getMBB() == FP.FalseB)
FR = RO.getReg(), FSR = RO.getSubReg();
else
continue;
PN->RemoveOperand(i+1);
PN->RemoveOperand(i);
}
if (TR == 0)
TR = SR, TSR = SSR;
else if (FR == 0)
FR = SR, FSR = SSR;
assert(TR && FR);
using namespace Hexagon;
unsigned DR = PN->getOperand(0).getReg();
const TargetRegisterClass *RC = MRI->getRegClass(DR);
const MCInstrDesc &D = RC == &IntRegsRegClass ? TII->get(C2_mux)
: TII->get(MUX64_rr);
MachineBasicBlock::iterator MuxAt = FP.SplitB->getFirstTerminator();
DebugLoc DL;
if (MuxAt != FP.SplitB->end())
DL = MuxAt->getDebugLoc();
unsigned MuxR = MRI->createVirtualRegister(RC);
BuildMI(*FP.SplitB, MuxAt, DL, D, MuxR)
.addReg(FP.PredR)
.addReg(TR, 0, TSR)
.addReg(FR, 0, FSR);
PN->addOperand(MachineOperand::CreateReg(MuxR, false));
PN->addOperand(MachineOperand::CreateMBB(FP.SplitB));
}
}
/// Copy implicit register operands from specified instruction to this
/// instruction that are not part of the instruction definition.
static void copyExtraImplicitOps(MachineInstr &NewMI, MachineFunction &MF,
const MachineInstr &MI) {
for (unsigned i = MI.getDesc().getNumOperands() +
MI.getDesc().getNumImplicitUses() +
MI.getDesc().getNumImplicitDefs(), e = MI.getNumOperands();
i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
if ((MO.isReg() && MO.isImplicit()) || MO.isRegMask())
NewMI.addOperand(MF, MO);
}
}
bool SIFixWWMLiveness::runOnWWMInstruction(MachineInstr &WWM) {
MachineBasicBlock *MBB = WWM.getParent();
// Compute the registers that are live out of MI by figuring out which defs
// are reachable from MI.
SparseBitVector<> LiveOut;
for (auto II = MachineBasicBlock::iterator(WWM), IE =
MBB->end(); II != IE; ++II) {
addDefs(*II, LiveOut);
}
for (df_iterator<MachineBasicBlock *> I = ++df_begin(MBB),
E = df_end(MBB);
I != E; ++I) {
for (const MachineInstr &MI : **I) {
addDefs(MI, LiveOut);
}
}
// Compute the registers that reach MI.
SparseBitVector<> Reachable;
for (auto II = ++MachineBasicBlock::reverse_iterator(WWM), IE =
MBB->rend(); II != IE; ++II) {
addDefs(*II, Reachable);
}
for (idf_iterator<MachineBasicBlock *> I = ++idf_begin(MBB),
E = idf_end(MBB);
I != E; ++I) {
for (const MachineInstr &MI : **I) {
addDefs(MI, Reachable);
}
}
// find the intersection, and add implicit uses.
LiveOut &= Reachable;
bool Modified = false;
for (unsigned Reg : LiveOut) {
WWM.addOperand(MachineOperand::CreateReg(Reg, false, /*isImp=*/true));
if (LIS) {
// FIXME: is there a better way to update the live interval?
LIS->removeInterval(Reg);
LIS->createAndComputeVirtRegInterval(Reg);
}
Modified = true;
}
return Modified;
}
/// addRetOperand - ensure that a return instruction has an operand for each
/// value live out of the function.
///
/// Things marked both call and return are tail calls; do not do this for them.
/// The tail callee need not take the same registers as input that it produces
/// as output, and there are dependencies for its input registers elsewhere.
///
/// FIXME: This should be done as part of instruction selection, and this helper
/// should be deleted. Until then, we use custom logic here to create the proper
/// operand under all circumstances. We can't use addRegisterKilled because that
/// doesn't make sense for undefined values. We can't simply avoid calling it
/// for undefined values, because we must ensure that the operand always exists.
void RAFast::addRetOperands(MachineBasicBlock *MBB) {
if (MBB->empty() || !MBB->back().isReturn() || MBB->back().isCall())
return;
MachineInstr *MI = &MBB->back();
for (MachineRegisterInfo::liveout_iterator
I = MBB->getParent()->getRegInfo().liveout_begin(),
E = MBB->getParent()->getRegInfo().liveout_end(); I != E; ++I) {
unsigned Reg = *I;
assert(TargetRegisterInfo::isPhysicalRegister(Reg) &&
"Cannot have a live-out virtual register.");
bool hasDef = PhysRegState[Reg] == regReserved;
// Check if this register already has an operand.
bool Found = false;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || !MO.isUse())
continue;
unsigned OperReg = MO.getReg();
if (!TargetRegisterInfo::isPhysicalRegister(OperReg))
continue;
if (OperReg == Reg || TRI->isSuperRegister(OperReg, Reg)) {
// If the ret already has an operand for this physreg or a superset,
// don't duplicate it. Set the kill flag if the value is defined.
if (hasDef && !MO.isKill())
MO.setIsKill();
Found = true;
break;
}
}
if (!Found)
MI->addOperand(MachineOperand::CreateReg(Reg,
false /*IsDef*/,
true /*IsImp*/,
hasDef/*IsKill*/));
}
}
bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MRI = &mf.getRegInfo();
TRI = MF->getTarget().getRegisterInfo();
ReservedRegisters = TRI->getReservedRegs(mf);
unsigned NumRegs = TRI->getNumRegs();
PhysRegDef = new MachineInstr*[NumRegs];
PhysRegUse = new MachineInstr*[NumRegs];
PHIVarInfo = new SmallVector<unsigned, 4>[MF->getNumBlockIDs()];
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
PHIJoins.clear();
// FIXME: LiveIntervals will be updated to remove its dependence on
// LiveVariables to improve compilation time and eliminate bizarre pass
// dependencies. Until then, we can't change much in -O0.
if (!MRI->isSSA())
report_fatal_error("regalloc=... not currently supported with -O0");
analyzePHINodes(mf);
// Calculate live variable information in depth first order on the CFG of the
// function. This guarantees that we will see the definition of a virtual
// register before its uses due to dominance properties of SSA (except for PHI
// nodes, which are treated as a special case).
MachineBasicBlock *Entry = MF->begin();
SmallPtrSet<MachineBasicBlock*,16> Visited;
for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> >
DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited);
DFI != E; ++DFI) {
MachineBasicBlock *MBB = *DFI;
// Mark live-in registers as live-in.
SmallVector<unsigned, 4> Defs;
for (MachineBasicBlock::livein_iterator II = MBB->livein_begin(),
EE = MBB->livein_end(); II != EE; ++II) {
assert(TargetRegisterInfo::isPhysicalRegister(*II) &&
"Cannot have a live-in virtual register!");
HandlePhysRegDef(*II, 0, Defs);
}
// Loop over all of the instructions, processing them.
DistanceMap.clear();
unsigned Dist = 0;
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
I != E; ++I) {
MachineInstr *MI = I;
if (MI->isDebugValue())
continue;
DistanceMap.insert(std::make_pair(MI, Dist++));
// Process all of the operands of the instruction...
unsigned NumOperandsToProcess = MI->getNumOperands();
// Unless it is a PHI node. In this case, ONLY process the DEF, not any
// of the uses. They will be handled in other basic blocks.
if (MI->isPHI())
NumOperandsToProcess = 1;
// Clear kill and dead markers. LV will recompute them.
SmallVector<unsigned, 4> UseRegs;
SmallVector<unsigned, 4> DefRegs;
SmallVector<unsigned, 1> RegMasks;
for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isRegMask()) {
RegMasks.push_back(i);
continue;
}
if (!MO.isReg() || MO.getReg() == 0)
continue;
unsigned MOReg = MO.getReg();
if (MO.isUse()) {
MO.setIsKill(false);
UseRegs.push_back(MOReg);
} else /*MO.isDef()*/ {
MO.setIsDead(false);
DefRegs.push_back(MOReg);
}
}
// Process all uses.
for (unsigned i = 0, e = UseRegs.size(); i != e; ++i) {
unsigned MOReg = UseRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegUse(MOReg, MBB, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegUse(MOReg, MI);
}
// Process all masked registers. (Call clobbers).
for (unsigned i = 0, e = RegMasks.size(); i != e; ++i)
HandleRegMask(MI->getOperand(RegMasks[i]));
// Process all defs.
for (unsigned i = 0, e = DefRegs.size(); i != e; ++i) {
unsigned MOReg = DefRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegDef(MOReg, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegDef(MOReg, MI, Defs);
}
UpdatePhysRegDefs(MI, Defs);
}
// Handle any virtual assignments from PHI nodes which might be at the
// bottom of this basic block. We check all of our successor blocks to see
// if they have PHI nodes, and if so, we simulate an assignment at the end
// of the current block.
if (!PHIVarInfo[MBB->getNumber()].empty()) {
SmallVector<unsigned, 4>& VarInfoVec = PHIVarInfo[MBB->getNumber()];
for (SmallVector<unsigned, 4>::iterator I = VarInfoVec.begin(),
E = VarInfoVec.end(); I != E; ++I)
// Mark it alive only in the block we are representing.
MarkVirtRegAliveInBlock(getVarInfo(*I),MRI->getVRegDef(*I)->getParent(),
MBB);
}
// Finally, if the last instruction in the block is a return, make sure to
// mark it as using all of the live-out values in the function.
// Things marked both call and return are tail calls; do not do this for
// them. The tail callee need not take the same registers as input
// that it produces as output, and there are dependencies for its input
// registers elsewhere.
if (!MBB->empty() && MBB->back().isReturn()
&& !MBB->back().isCall()) {
MachineInstr *Ret = &MBB->back();
for (MachineRegisterInfo::liveout_iterator
I = MF->getRegInfo().liveout_begin(),
E = MF->getRegInfo().liveout_end(); I != E; ++I) {
assert(TargetRegisterInfo::isPhysicalRegister(*I) &&
"Cannot have a live-out virtual register!");
HandlePhysRegUse(*I, Ret);
// Add live-out registers as implicit uses.
if (!Ret->readsRegister(*I))
Ret->addOperand(MachineOperand::CreateReg(*I, false, true));
}
}
// MachineCSE may CSE instructions which write to non-allocatable physical
// registers across MBBs. Remember if any reserved register is liveout.
SmallSet<unsigned, 4> LiveOuts;
for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
SE = MBB->succ_end(); SI != SE; ++SI) {
MachineBasicBlock *SuccMBB = *SI;
if (SuccMBB->isLandingPad())
continue;
for (MachineBasicBlock::livein_iterator LI = SuccMBB->livein_begin(),
LE = SuccMBB->livein_end(); LI != LE; ++LI) {
unsigned LReg = *LI;
if (!TRI->isInAllocatableClass(LReg))
// Ignore other live-ins, e.g. those that are live into landing pads.
LiveOuts.insert(LReg);
}
}
// Loop over PhysRegDef / PhysRegUse, killing any registers that are
// available at the end of the basic block.
for (unsigned i = 0; i != NumRegs; ++i)
if ((PhysRegDef[i] || PhysRegUse[i]) && !LiveOuts.count(i))
HandlePhysRegDef(i, 0, Defs);
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
}
// Convert and transfer the dead / killed information we have gathered into
// VirtRegInfo onto MI's.
for (unsigned i = 0, e1 = VirtRegInfo.size(); i != e1; ++i) {
const unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
for (unsigned j = 0, e2 = VirtRegInfo[Reg].Kills.size(); j != e2; ++j)
if (VirtRegInfo[Reg].Kills[j] == MRI->getVRegDef(Reg))
VirtRegInfo[Reg].Kills[j]->addRegisterDead(Reg, TRI);
else
VirtRegInfo[Reg].Kills[j]->addRegisterKilled(Reg, TRI);
}
// Check to make sure there are no unreachable blocks in the MC CFG for the
// function. If so, it is due to a bug in the instruction selector or some
// other part of the code generator if this happens.
#ifndef NDEBUG
for(MachineFunction::iterator i = MF->begin(), e = MF->end(); i != e; ++i)
assert(Visited.count(&*i) != 0 && "unreachable basic block found");
#endif
delete[] PhysRegDef;
delete[] PhysRegUse;
delete[] PHIVarInfo;
return false;
}
/// Add the current register live set to the instruction.
void StackMapLiveness::addLiveOutSetToMI(MachineInstr &MI) {
uint32_t *Mask = createRegisterMask();
MachineOperand MO = MachineOperand::CreateRegLiveOut(Mask);
MI.addOperand(*MF, MO);
}
/// EmitNode - Generate machine code for an node and needed dependencies.
///
void ScheduleDAGSDNodes::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap) {
// If machine instruction
if (Node->isMachineOpcode()) {
unsigned Opc = Node->getMachineOpcode();
// Handle subreg insert/extract specially
if (Opc == TargetInstrInfo::EXTRACT_SUBREG ||
Opc == TargetInstrInfo::INSERT_SUBREG ||
Opc == TargetInstrInfo::SUBREG_TO_REG) {
EmitSubregNode(Node, VRBaseMap);
return;
}
// Handle COPY_TO_REGCLASS specially.
if (Opc == TargetInstrInfo::COPY_TO_REGCLASS) {
EmitCopyToRegClassNode(Node, VRBaseMap);
return;
}
if (Opc == TargetInstrInfo::IMPLICIT_DEF)
// We want a unique VR for each IMPLICIT_DEF use.
return;
const TargetInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NodeOperands = CountOperands(Node);
unsigned MemOperandsEnd = ComputeMemOperandsEnd(Node);
bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
II.getImplicitDefs() != 0;
#ifndef NDEBUG
unsigned NumMIOperands = NodeOperands + NumResults;
assert((II.getNumOperands() == NumMIOperands ||
HasPhysRegOuts || II.isVariadic()) &&
"#operands for dag node doesn't match .td file!");
#endif
// Create the new machine instruction.
MachineInstr *MI = BuildMI(MF, Node->getDebugLoc(), II);
// Add result register values for things that are defined by this
// instruction.
if (NumResults)
CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
// Emit all of the actual operands of this instruction, adding them to the
// instruction as appropriate.
for (unsigned i = 0; i != NodeOperands; ++i)
AddOperand(MI, Node->getOperand(i), i+II.getNumDefs(), &II, VRBaseMap);
// Emit all of the memory operands of this instruction
for (unsigned i = NodeOperands; i != MemOperandsEnd; ++i)
AddMemOperand(MI, cast<MemOperandSDNode>(Node->getOperand(i))->MO);
if (II.usesCustomDAGSchedInsertionHook()) {
// Insert this instruction into the basic block using a target
// specific inserter which may returns a new basic block.
BB = TLI->EmitInstrWithCustomInserter(MI, BB);
InsertPos = BB->end();
} else {
BB->insert(InsertPos, MI);
}
// Additional results must be an physical register def.
if (HasPhysRegOuts) {
for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
if (Node->hasAnyUseOfValue(i))
EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
}
}
return;
}
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
Node->dump(DAG);
#endif
assert(0 && "This target-independent node should have been selected!");
break;
case ISD::EntryToken:
assert(0 && "EntryToken should have been excluded from the schedule!");
break;
case ISD::TokenFactor: // fall thru
break;
case ISD::CopyToReg: {
unsigned SrcReg;
SDValue SrcVal = Node->getOperand(2);
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
SrcReg = R->getReg();
else
SrcReg = getVR(SrcVal, VRBaseMap);
unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
break;
const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0;
// Get the register classes of the src/dst.
if (TargetRegisterInfo::isVirtualRegister(SrcReg))
SrcTRC = MRI.getRegClass(SrcReg);
else
SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType());
if (TargetRegisterInfo::isVirtualRegister(DestReg))
DstTRC = MRI.getRegClass(DestReg);
else
DstTRC = TRI->getPhysicalRegisterRegClass(DestReg,
Node->getOperand(1).getValueType());
bool Emitted = TII->copyRegToReg(*BB, InsertPos, DestReg, SrcReg,
DstTRC, SrcTRC);
// If the target didn't handle the copy with different register
// classes and the destination is a subset of the source,
// try a normal same-RC copy.
if (!Emitted && DstTRC->hasSubClass(SrcTRC))
Emitted = TII->copyRegToReg(*BB, InsertPos, DestReg, SrcReg,
DstTRC, DstTRC);
assert(Emitted && "Unable to issue a copy instruction!\n");
break;
}
case ISD::CopyFromReg: {
unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
break;
}
case ISD::INLINEASM: {
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag)
--NumOps; // Ignore the flag operand.
// Create the inline asm machine instruction.
MachineInstr *MI = BuildMI(MF, Node->getDebugLoc(),
TII->get(TargetInstrInfo::INLINEASM));
// Add the asm string as an external symbol operand.
const char *AsmStr =
cast<ExternalSymbolSDNode>(Node->getOperand(1))->getSymbol();
MI->addOperand(MachineOperand::CreateES(AsmStr));
// Add all of the operand registers to the instruction.
for (unsigned i = 2; i != NumOps;) {
unsigned Flags =
cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
MI->addOperand(MachineOperand::CreateImm(Flags));
++i; // Skip the ID value.
switch (Flags & 7) {
default: assert(0 && "Bad flags!");
case 2: // Def of register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true));
}
break;
case 6: // Def of earlyclobber register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false,
false, 0, true));
}
break;
case 1: // Use of register.
case 3: // Immediate.
case 4: // Addressing mode.
// The addressing mode has been selected, just add all of the
// operands to the machine instruction.
for (; NumVals; --NumVals, ++i)
AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap);
break;
}
}
BB->insert(InsertPos, MI);
break;
}
}
}
void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
assert(MBBI->getDesc().isReturn() &&
"Can only insert epilog into returning blocks");
unsigned RetOpcode = MBBI->getOpcode();
DebugLoc dl = MBBI->getDebugLoc();
MachineFrameInfo *MFI = MF.getFrameInfo();
ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
const TargetRegisterInfo *RegInfo = MF.getTarget().getRegisterInfo();
const ARMBaseInstrInfo &TII =
*static_cast<const ARMBaseInstrInfo*>(MF.getTarget().getInstrInfo());
assert(!AFI->isThumb1OnlyFunction() &&
"This emitEpilogue does not support Thumb1!");
bool isARM = !AFI->isThumbFunction();
unsigned VARegSaveSize = AFI->getVarArgsRegSaveSize();
int NumBytes = (int)MFI->getStackSize();
unsigned FramePtr = RegInfo->getFrameRegister(MF);
if (!AFI->hasStackFrame()) {
if (NumBytes != 0)
emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes);
} else {
// Unwind MBBI to point to first LDR / VLDRD.
const unsigned *CSRegs = RegInfo->getCalleeSavedRegs();
if (MBBI != MBB.begin()) {
do
--MBBI;
while (MBBI != MBB.begin() && isCSRestore(MBBI, TII, CSRegs));
if (!isCSRestore(MBBI, TII, CSRegs))
++MBBI;
}
// Move SP to start of FP callee save spill area.
NumBytes -= (AFI->getGPRCalleeSavedArea1Size() +
AFI->getGPRCalleeSavedArea2Size() +
AFI->getDPRCalleeSavedAreaSize());
// Reset SP based on frame pointer only if the stack frame extends beyond
// frame pointer stack slot or target is ELF and the function has FP.
if (AFI->shouldRestoreSPFromFP()) {
NumBytes = AFI->getFramePtrSpillOffset() - NumBytes;
if (NumBytes) {
if (isARM)
emitARMRegPlusImmediate(MBB, MBBI, dl, ARM::SP, FramePtr, -NumBytes,
ARMCC::AL, 0, TII);
else {
// It's not possible to restore SP from FP in a single instruction.
// For Darwin, this looks like:
// mov sp, r7
// sub sp, #24
// This is bad, if an interrupt is taken after the mov, sp is in an
// inconsistent state.
// Use the first callee-saved register as a scratch register.
assert(MF.getRegInfo().isPhysRegUsed(ARM::R4) &&
"No scratch register to restore SP from FP!");
emitT2RegPlusImmediate(MBB, MBBI, dl, ARM::R4, FramePtr, -NumBytes,
ARMCC::AL, 0, TII);
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr),
ARM::SP)
.addReg(ARM::R4));
}
} else {
// Thumb2 or ARM.
if (isARM)
BuildMI(MBB, MBBI, dl, TII.get(ARM::MOVr), ARM::SP)
.addReg(FramePtr).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0);
else
AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tMOVr),
ARM::SP)
.addReg(FramePtr));
}
} else if (NumBytes)
emitSPUpdate(isARM, MBB, MBBI, dl, TII, NumBytes);
// Increment past our save areas.
if (AFI->getDPRCalleeSavedAreaSize()) {
MBBI++;
// Since vpop register list cannot have gaps, there may be multiple vpop
// instructions in the epilogue.
while (MBBI->getOpcode() == ARM::VLDMDIA_UPD)
MBBI++;
}
if (AFI->getGPRCalleeSavedArea2Size()) MBBI++;
if (AFI->getGPRCalleeSavedArea1Size()) MBBI++;
}
if (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNdiND ||
RetOpcode == ARM::TCRETURNri || RetOpcode == ARM::TCRETURNriND) {
// Tail call return: adjust the stack pointer and jump to callee.
MBBI = MBB.getLastNonDebugInstr();
MachineOperand &JumpTarget = MBBI->getOperand(0);
// Jump to label or value in register.
if (RetOpcode == ARM::TCRETURNdi || RetOpcode == ARM::TCRETURNdiND) {
unsigned TCOpcode = (RetOpcode == ARM::TCRETURNdi)
? (STI.isThumb() ? ARM::tTAILJMPd : ARM::TAILJMPd)
: (STI.isThumb() ? ARM::tTAILJMPdND : ARM::TAILJMPdND);
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII.get(TCOpcode));
if (JumpTarget.isGlobal())
MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
JumpTarget.getTargetFlags());
else {
assert(JumpTarget.isSymbol());
MIB.addExternalSymbol(JumpTarget.getSymbolName(),
JumpTarget.getTargetFlags());
}
// Add the default predicate in Thumb mode.
if (STI.isThumb()) MIB.addImm(ARMCC::AL).addReg(0);
} else if (RetOpcode == ARM::TCRETURNri) {
BuildMI(MBB, MBBI, dl,
TII.get(STI.isThumb() ? ARM::tTAILJMPr : ARM::TAILJMPr)).
addReg(JumpTarget.getReg(), RegState::Kill);
} else if (RetOpcode == ARM::TCRETURNriND) {
BuildMI(MBB, MBBI, dl,
TII.get(STI.isThumb() ? ARM::tTAILJMPrND : ARM::TAILJMPrND)).
addReg(JumpTarget.getReg(), RegState::Kill);
}
MachineInstr *NewMI = prior(MBBI);
for (unsigned i = 1, e = MBBI->getNumOperands(); i != e; ++i)
NewMI->addOperand(MBBI->getOperand(i));
// Delete the pseudo instruction TCRETURN.
MBB.erase(MBBI);
MBBI = NewMI;
}
if (VARegSaveSize)
emitSPUpdate(isARM, MBB, MBBI, dl, TII, VARegSaveSize);
}
void X86FrameInfo::emitEpilogue(MachineFunction &MF,
MachineBasicBlock &MBB) const {
const MachineFrameInfo *MFI = MF.getFrameInfo();
X86MachineFunctionInfo *X86FI = MF.getInfo<X86MachineFunctionInfo>();
const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
const X86InstrInfo &TII = *TM.getInstrInfo();
MachineBasicBlock::iterator MBBI = prior(MBB.end());
unsigned RetOpcode = MBBI->getOpcode();
DebugLoc DL = MBBI->getDebugLoc();
bool Is64Bit = STI.is64Bit();
unsigned StackAlign = getStackAlignment();
unsigned SlotSize = RegInfo->getSlotSize();
unsigned FramePtr = RegInfo->getFrameRegister(MF);
unsigned StackPtr = RegInfo->getStackRegister();
switch (RetOpcode) {
default:
llvm_unreachable("Can only insert epilog into returning blocks");
case X86::RET:
case X86::RETI:
case X86::TCRETURNdi:
case X86::TCRETURNri:
case X86::TCRETURNmi:
case X86::TCRETURNdi64:
case X86::TCRETURNri64:
case X86::TCRETURNmi64:
case X86::EH_RETURN:
case X86::EH_RETURN64:
break; // These are ok
}
// Get the number of bytes to allocate from the FrameInfo.
uint64_t StackSize = MFI->getStackSize();
uint64_t MaxAlign = MFI->getMaxAlignment();
unsigned CSSize = X86FI->getCalleeSavedFrameSize();
uint64_t NumBytes = 0;
// If we're forcing a stack realignment we can't rely on just the frame
// info, we need to know the ABI stack alignment as well in case we
// have a call out. Otherwise just make sure we have some alignment - we'll
// go with the minimum.
if (ForceStackAlign) {
if (MFI->hasCalls())
MaxAlign = (StackAlign > MaxAlign) ? StackAlign : MaxAlign;
else
MaxAlign = MaxAlign ? MaxAlign : 4;
}
if (hasFP(MF)) {
// Calculate required stack adjustment.
uint64_t FrameSize = StackSize - SlotSize;
if (RegInfo->needsStackRealignment(MF))
FrameSize = (FrameSize + MaxAlign - 1)/MaxAlign*MaxAlign;
NumBytes = FrameSize - CSSize;
// Pop EBP.
BuildMI(MBB, MBBI, DL,
TII.get(Is64Bit ? X86::POP64r : X86::POP32r), FramePtr);
} else {
NumBytes = StackSize - CSSize;
}
// Skip the callee-saved pop instructions.
MachineBasicBlock::iterator LastCSPop = MBBI;
while (MBBI != MBB.begin()) {
MachineBasicBlock::iterator PI = prior(MBBI);
unsigned Opc = PI->getOpcode();
if (Opc != X86::POP32r && Opc != X86::POP64r &&
!PI->getDesc().isTerminator())
break;
--MBBI;
}
DL = MBBI->getDebugLoc();
// If there is an ADD32ri or SUB32ri of ESP immediately before this
// instruction, merge the two instructions.
if (NumBytes || MFI->hasVarSizedObjects())
mergeSPUpdatesUp(MBB, MBBI, StackPtr, &NumBytes);
// If dynamic alloca is used, then reset esp to point to the last callee-saved
// slot before popping them off! Same applies for the case, when stack was
// realigned.
if (RegInfo->needsStackRealignment(MF)) {
// We cannot use LEA here, because stack pointer was realigned. We need to
// deallocate local frame back.
if (CSSize) {
emitSPUpdate(MBB, MBBI, StackPtr, NumBytes, Is64Bit, TII);
MBBI = prior(LastCSPop);
}
BuildMI(MBB, MBBI, DL,
TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr),
StackPtr).addReg(FramePtr);
} else if (MFI->hasVarSizedObjects()) {
if (CSSize) {
unsigned Opc = Is64Bit ? X86::LEA64r : X86::LEA32r;
MachineInstr *MI =
addRegOffset(BuildMI(MF, DL, TII.get(Opc), StackPtr),
FramePtr, false, -CSSize);
MBB.insert(MBBI, MI);
} else {
BuildMI(MBB, MBBI, DL,
TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr), StackPtr)
.addReg(FramePtr);
}
} else if (NumBytes) {
// Adjust stack pointer back: ESP += numbytes.
emitSPUpdate(MBB, MBBI, StackPtr, NumBytes, Is64Bit, TII);
}
// We're returning from function via eh_return.
if (RetOpcode == X86::EH_RETURN || RetOpcode == X86::EH_RETURN64) {
MBBI = prior(MBB.end());
MachineOperand &DestAddr = MBBI->getOperand(0);
assert(DestAddr.isReg() && "Offset should be in register!");
BuildMI(MBB, MBBI, DL,
TII.get(Is64Bit ? X86::MOV64rr : X86::MOV32rr),
StackPtr).addReg(DestAddr.getReg());
} else if (RetOpcode == X86::TCRETURNri || RetOpcode == X86::TCRETURNdi ||
RetOpcode == X86::TCRETURNmi ||
RetOpcode == X86::TCRETURNri64 || RetOpcode == X86::TCRETURNdi64 ||
RetOpcode == X86::TCRETURNmi64) {
bool isMem = RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64;
// Tail call return: adjust the stack pointer and jump to callee.
MBBI = prior(MBB.end());
MachineOperand &JumpTarget = MBBI->getOperand(0);
MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1);
assert(StackAdjust.isImm() && "Expecting immediate value.");
// Adjust stack pointer.
int StackAdj = StackAdjust.getImm();
int MaxTCDelta = X86FI->getTCReturnAddrDelta();
int Offset = 0;
assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
// Incoporate the retaddr area.
Offset = StackAdj-MaxTCDelta;
assert(Offset >= 0 && "Offset should never be negative");
if (Offset) {
// Check for possible merge with preceeding ADD instruction.
Offset += mergeSPUpdates(MBB, MBBI, StackPtr, true);
emitSPUpdate(MBB, MBBI, StackPtr, Offset, Is64Bit, TII);
}
// Jump to label or value in register.
if (RetOpcode == X86::TCRETURNdi || RetOpcode == X86::TCRETURNdi64) {
MachineInstrBuilder MIB =
BuildMI(MBB, MBBI, DL, TII.get((RetOpcode == X86::TCRETURNdi)
? X86::TAILJMPd : X86::TAILJMPd64));
if (JumpTarget.isGlobal())
MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
JumpTarget.getTargetFlags());
else {
assert(JumpTarget.isSymbol());
MIB.addExternalSymbol(JumpTarget.getSymbolName(),
JumpTarget.getTargetFlags());
}
} else if (RetOpcode == X86::TCRETURNmi || RetOpcode == X86::TCRETURNmi64) {
MachineInstrBuilder MIB =
BuildMI(MBB, MBBI, DL, TII.get((RetOpcode == X86::TCRETURNmi)
? X86::TAILJMPm : X86::TAILJMPm64));
for (unsigned i = 0; i != 5; ++i)
MIB.addOperand(MBBI->getOperand(i));
} else if (RetOpcode == X86::TCRETURNri64) {
BuildMI(MBB, MBBI, DL, TII.get(X86::TAILJMPr64)).
addReg(JumpTarget.getReg(), RegState::Kill);
} else {
BuildMI(MBB, MBBI, DL, TII.get(X86::TAILJMPr)).
addReg(JumpTarget.getReg(), RegState::Kill);
}
MachineInstr *NewMI = prior(MBBI);
for (unsigned i = 2, e = MBBI->getNumOperands(); i != e; ++i)
NewMI->addOperand(MBBI->getOperand(i));
// Delete the pseudo instruction TCRETURN.
MBB.erase(MBBI);
} else if ((RetOpcode == X86::RET || RetOpcode == X86::RETI) &&
(X86FI->getTCReturnAddrDelta() < 0)) {
// Add the return addr area delta back since we are not tail calling.
int delta = -1*X86FI->getTCReturnAddrDelta();
MBBI = prior(MBB.end());
// Check for possible merge with preceeding ADD instruction.
delta += mergeSPUpdates(MBB, MBBI, StackPtr, true);
emitSPUpdate(MBB, MBBI, StackPtr, delta, Is64Bit, TII);
}
}
/// If \p MBBI is a pseudo instruction, this method expands
/// it to the corresponding (sequence of) actual instruction(s).
/// \returns true if \p MBBI has been expanded.
bool X86ExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI) {
MachineInstr &MI = *MBBI;
unsigned Opcode = MI.getOpcode();
DebugLoc DL = MBBI->getDebugLoc();
switch (Opcode) {
default:
return false;
case X86::TCRETURNdi:
case X86::TCRETURNdicc:
case X86::TCRETURNri:
case X86::TCRETURNmi:
case X86::TCRETURNdi64:
case X86::TCRETURNdi64cc:
case X86::TCRETURNri64:
case X86::TCRETURNmi64: {
bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
MachineOperand &JumpTarget = MBBI->getOperand(0);
MachineOperand &StackAdjust = MBBI->getOperand(isMem ? 5 : 1);
assert(StackAdjust.isImm() && "Expecting immediate value.");
// Adjust stack pointer.
int StackAdj = StackAdjust.getImm();
int MaxTCDelta = X86FI->getTCReturnAddrDelta();
int Offset = 0;
assert(MaxTCDelta <= 0 && "MaxTCDelta should never be positive");
// Incoporate the retaddr area.
Offset = StackAdj - MaxTCDelta;
assert(Offset >= 0 && "Offset should never be negative");
if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
assert(Offset == 0 && "Conditional tail call cannot adjust the stack.");
}
if (Offset) {
// Check for possible merge with preceding ADD instruction.
Offset += X86FL->mergeSPUpdates(MBB, MBBI, true);
X86FL->emitSPUpdate(MBB, MBBI, Offset, /*InEpilogue=*/true);
}
// Jump to label or value in register.
bool IsWin64 = STI->isTargetWin64();
if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
unsigned Op;
switch (Opcode) {
case X86::TCRETURNdi:
Op = X86::TAILJMPd;
break;
case X86::TCRETURNdicc:
Op = X86::TAILJMPd_CC;
break;
case X86::TCRETURNdi64cc:
assert(!IsWin64 && "Conditional tail calls confuse the Win64 unwinder.");
// TODO: We could do it for Win64 "leaf" functions though; PR30337.
Op = X86::TAILJMPd64_CC;
break;
default:
// Note: Win64 uses REX prefixes indirect jumps out of functions, but
// not direct ones.
Op = X86::TAILJMPd64;
break;
}
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
if (JumpTarget.isGlobal()) {
MIB.addGlobalAddress(JumpTarget.getGlobal(), JumpTarget.getOffset(),
JumpTarget.getTargetFlags());
} else {
assert(JumpTarget.isSymbol());
MIB.addExternalSymbol(JumpTarget.getSymbolName(),
JumpTarget.getTargetFlags());
}
if (Op == X86::TAILJMPd_CC || Op == X86::TAILJMPd64_CC) {
MIB.addImm(MBBI->getOperand(2).getImm());
}
} else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
unsigned Op = (Opcode == X86::TCRETURNmi)
? X86::TAILJMPm
: (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(Op));
for (unsigned i = 0; i != 5; ++i)
MIB.add(MBBI->getOperand(i));
} else if (Opcode == X86::TCRETURNri64) {
BuildMI(MBB, MBBI, DL,
TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
.addReg(JumpTarget.getReg(), RegState::Kill);
} else {
BuildMI(MBB, MBBI, DL, TII->get(X86::TAILJMPr))
.addReg(JumpTarget.getReg(), RegState::Kill);
}
MachineInstr &NewMI = *std::prev(MBBI);
NewMI.copyImplicitOps(*MBBI->getParent()->getParent(), *MBBI);
// Delete the pseudo instruction TCRETURN.
MBB.erase(MBBI);
return true;
}
case X86::EH_RETURN:
case X86::EH_RETURN64: {
MachineOperand &DestAddr = MBBI->getOperand(0);
assert(DestAddr.isReg() && "Offset should be in register!");
const bool Uses64BitFramePtr =
STI->isTarget64BitLP64() || STI->isTargetNaCl64();
unsigned StackPtr = TRI->getStackRegister();
BuildMI(MBB, MBBI, DL,
TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)
.addReg(DestAddr.getReg());
// The EH_RETURN pseudo is really removed during the MC Lowering.
return true;
}
case X86::IRET: {
// Adjust stack to erase error code
int64_t StackAdj = MBBI->getOperand(0).getImm();
X86FL->emitSPUpdate(MBB, MBBI, StackAdj, true);
// Replace pseudo with machine iret
BuildMI(MBB, MBBI, DL,
TII->get(STI->is64Bit() ? X86::IRET64 : X86::IRET32));
MBB.erase(MBBI);
return true;
}
case X86::RET: {
// Adjust stack to erase error code
int64_t StackAdj = MBBI->getOperand(0).getImm();
MachineInstrBuilder MIB;
if (StackAdj == 0) {
MIB = BuildMI(MBB, MBBI, DL,
TII->get(STI->is64Bit() ? X86::RETQ : X86::RETL));
} else if (isUInt<16>(StackAdj)) {
MIB = BuildMI(MBB, MBBI, DL,
TII->get(STI->is64Bit() ? X86::RETIQ : X86::RETIL))
.addImm(StackAdj);
} else {
assert(!STI->is64Bit() &&
"shouldn't need to do this for x86_64 targets!");
// A ret can only handle immediates as big as 2**16-1. If we need to pop
// off bytes before the return address, we must do it manually.
BuildMI(MBB, MBBI, DL, TII->get(X86::POP32r)).addReg(X86::ECX, RegState::Define);
X86FL->emitSPUpdate(MBB, MBBI, StackAdj, /*InEpilogue=*/true);
BuildMI(MBB, MBBI, DL, TII->get(X86::PUSH32r)).addReg(X86::ECX);
MIB = BuildMI(MBB, MBBI, DL, TII->get(X86::RETL));
}
for (unsigned I = 1, E = MBBI->getNumOperands(); I != E; ++I)
MIB.add(MBBI->getOperand(I));
MBB.erase(MBBI);
return true;
}
case X86::EH_RESTORE: {
// Restore ESP and EBP, and optionally ESI if required.
bool IsSEH = isAsynchronousEHPersonality(classifyEHPersonality(
MBB.getParent()->getFunction()->getPersonalityFn()));
X86FL->restoreWin32EHStackPointers(MBB, MBBI, DL, /*RestoreSP=*/IsSEH);
MBBI->eraseFromParent();
return true;
}
case X86::LCMPXCHG8B_SAVE_EBX:
case X86::LCMPXCHG16B_SAVE_RBX: {
// Perform the following transformation.
// SaveRbx = pseudocmpxchg Addr, <4 opds for the address>, InArg, SaveRbx
// =>
// [E|R]BX = InArg
// actualcmpxchg Addr
// [E|R]BX = SaveRbx
const MachineOperand &InArg = MBBI->getOperand(6);
unsigned SaveRbx = MBBI->getOperand(7).getReg();
unsigned ActualInArg =
Opcode == X86::LCMPXCHG8B_SAVE_EBX ? X86::EBX : X86::RBX;
// Copy the input argument of the pseudo into the argument of the
// actual instruction.
TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, InArg.getReg(),
InArg.isKill());
// Create the actual instruction.
unsigned ActualOpc =
Opcode == X86::LCMPXCHG8B_SAVE_EBX ? X86::LCMPXCHG8B : X86::LCMPXCHG16B;
MachineInstr *NewInstr = BuildMI(MBB, MBBI, DL, TII->get(ActualOpc));
// Copy the operands related to the address.
for (unsigned Idx = 1; Idx < 6; ++Idx)
NewInstr->addOperand(MBBI->getOperand(Idx));
// Finally, restore the value of RBX.
TII->copyPhysReg(MBB, MBBI, DL, ActualInArg, SaveRbx,
/*SrcIsKill*/ true);
// Delete the pseudo.
MBBI->eraseFromParent();
return true;
}
}
llvm_unreachable("Previous switch has a fallthrough?");
}
bool X86CallFrameOptimization::adjustCallSequence(MachineFunction &MF,
MachineBasicBlock &MBB,
MachineBasicBlock::iterator I) {
// Check that this particular call sequence is amenable to the
// transformation.
const X86RegisterInfo &RegInfo = *static_cast<const X86RegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
unsigned StackPtr = RegInfo.getStackRegister();
int FrameDestroyOpcode = TII->getCallFrameDestroyOpcode();
// We expect to enter this at the beginning of a call sequence
assert(I->getOpcode() == TII->getCallFrameSetupOpcode());
MachineBasicBlock::iterator FrameSetup = I++;
// For globals in PIC mode, we can have some LEAs here.
// Ignore them, they don't bother us.
// TODO: Extend this to something that covers more cases.
while (I->getOpcode() == X86::LEA32r)
++I;
// We expect a copy instruction here.
// TODO: The copy instruction is a lowering artifact.
// We should also support a copy-less version, where the stack
// pointer is used directly.
if (!I->isCopy() || !I->getOperand(0).isReg())
return false;
MachineBasicBlock::iterator SPCopy = I++;
StackPtr = SPCopy->getOperand(0).getReg();
// Scan the call setup sequence for the pattern we're looking for.
// We only handle a simple case - a sequence of MOV32mi or MOV32mr
// instructions, that push a sequence of 32-bit values onto the stack, with
// no gaps between them.
SmallVector<MachineInstr*, 4> MovVector(4, nullptr);
unsigned int MaxAdjust = FrameSetup->getOperand(0).getImm() / 4;
if (MaxAdjust > 4)
MovVector.resize(MaxAdjust, nullptr);
do {
int Opcode = I->getOpcode();
if (Opcode != X86::MOV32mi && Opcode != X86::MOV32mr)
break;
// We only want movs of the form:
// movl imm/r32, k(%esp)
// If we run into something else, bail.
// Note that AddrBaseReg may, counter to its name, not be a register,
// but rather a frame index.
// TODO: Support the fi case. This should probably work now that we
// have the infrastructure to track the stack pointer within a call
// sequence.
if (!I->getOperand(X86::AddrBaseReg).isReg() ||
(I->getOperand(X86::AddrBaseReg).getReg() != StackPtr) ||
!I->getOperand(X86::AddrScaleAmt).isImm() ||
(I->getOperand(X86::AddrScaleAmt).getImm() != 1) ||
(I->getOperand(X86::AddrIndexReg).getReg() != X86::NoRegister) ||
(I->getOperand(X86::AddrSegmentReg).getReg() != X86::NoRegister) ||
!I->getOperand(X86::AddrDisp).isImm())
return false;
int64_t StackDisp = I->getOperand(X86::AddrDisp).getImm();
assert(StackDisp >= 0 && "Negative stack displacement when passing parameters");
// We really don't want to consider the unaligned case.
if (StackDisp % 4)
return false;
StackDisp /= 4;
assert((size_t)StackDisp < MovVector.size() &&
"Function call has more parameters than the stack is adjusted for.");
// If the same stack slot is being filled twice, something's fishy.
if (MovVector[StackDisp] != nullptr)
return false;
MovVector[StackDisp] = I;
++I;
} while (I != MBB.end());
// We now expect the end of the sequence - a call and a stack adjust.
if (I == MBB.end())
return false;
// For PCrel calls, we expect an additional COPY of the basereg.
// If we find one, skip it.
if (I->isCopy()) {
if (I->getOperand(1).getReg() ==
MF.getInfo<X86MachineFunctionInfo>()->getGlobalBaseReg())
++I;
else
return false;
}
if (!I->isCall())
return false;
MachineBasicBlock::iterator Call = I;
if ((++I)->getOpcode() != FrameDestroyOpcode)
return false;
// Now, go through the vector, and see that we don't have any gaps,
// but only a series of 32-bit MOVs.
int64_t ExpectedDist = 0;
auto MMI = MovVector.begin(), MME = MovVector.end();
for (; MMI != MME; ++MMI, ExpectedDist += 4)
if (*MMI == nullptr)
break;
// If the call had no parameters, do nothing
if (!ExpectedDist)
return false;
// We are either at the last parameter, or a gap.
// Make sure it's not a gap
for (; MMI != MME; ++MMI)
if (*MMI != nullptr)
return false;
// Ok, we can in fact do the transformation for this call.
// Do not remove the FrameSetup instruction, but adjust the parameters.
// PEI will end up finalizing the handling of this.
FrameSetup->getOperand(1).setImm(ExpectedDist);
DebugLoc DL = I->getDebugLoc();
// Now, iterate through the vector in reverse order, and replace the movs
// with pushes. MOVmi/MOVmr doesn't have any defs, so no need to
// replace uses.
for (int Idx = (ExpectedDist / 4) - 1; Idx >= 0; --Idx) {
MachineBasicBlock::iterator MOV = *MovVector[Idx];
MachineOperand PushOp = MOV->getOperand(X86::AddrNumOperands);
if (MOV->getOpcode() == X86::MOV32mi) {
unsigned PushOpcode = X86::PUSHi32;
// If the operand is a small (8-bit) immediate, we can use a
// PUSH instruction with a shorter encoding.
// Note that isImm() may fail even though this is a MOVmi, because
// the operand can also be a symbol.
if (PushOp.isImm()) {
int64_t Val = PushOp.getImm();
if (isInt<8>(Val))
PushOpcode = X86::PUSH32i8;
}
BuildMI(MBB, Call, DL, TII->get(PushOpcode)).addOperand(PushOp);
} else {
unsigned int Reg = PushOp.getReg();
// If PUSHrmm is not slow on this target, try to fold the source of the
// push into the instruction.
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
bool SlowPUSHrmm = ST.isAtom() || ST.isSLM();
// Check that this is legal to fold. Right now, we're extremely
// conservative about that.
MachineInstr *DefMov = nullptr;
if (!SlowPUSHrmm && (DefMov = canFoldIntoRegPush(FrameSetup, Reg))) {
MachineInstr *Push = BuildMI(MBB, Call, DL, TII->get(X86::PUSH32rmm));
unsigned NumOps = DefMov->getDesc().getNumOperands();
for (unsigned i = NumOps - X86::AddrNumOperands; i != NumOps; ++i)
Push->addOperand(DefMov->getOperand(i));
DefMov->eraseFromParent();
} else {
BuildMI(MBB, Call, DL, TII->get(X86::PUSH32r)).addReg(Reg).getInstr();
}
}
MBB.erase(MOV);
}
// The stack-pointer copy is no longer used in the call sequences.
// There should not be any other users, but we can't commit to that, so:
if (MRI->use_empty(SPCopy->getOperand(0).getReg()))
SPCopy->eraseFromParent();
// Once we've done this, we need to make sure PEI doesn't assume a reserved
// frame.
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
FuncInfo->setHasPushSequences(true);
return true;
}
bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
MF = &mf;
MRI = &mf.getRegInfo();
TRI = MF->getTarget().getRegisterInfo();
ReservedRegisters = TRI->getReservedRegs(mf);
unsigned NumRegs = TRI->getNumRegs();
PhysRegDef = new MachineInstr*[NumRegs];
PhysRegUse = new MachineInstr*[NumRegs];
PHIVarInfo = new SmallVector<unsigned, 4>[MF->getNumBlockIDs()];
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
/// Get some space for a respectable number of registers.
VirtRegInfo.resize(64);
analyzePHINodes(mf);
// Calculate live variable information in depth first order on the CFG of the
// function. This guarantees that we will see the definition of a virtual
// register before its uses due to dominance properties of SSA (except for PHI
// nodes, which are treated as a special case).
MachineBasicBlock *Entry = MF->begin();
SmallPtrSet<MachineBasicBlock*,16> Visited;
for (df_ext_iterator<MachineBasicBlock*, SmallPtrSet<MachineBasicBlock*,16> >
DFI = df_ext_begin(Entry, Visited), E = df_ext_end(Entry, Visited);
DFI != E; ++DFI) {
MachineBasicBlock *MBB = *DFI;
// Mark live-in registers as live-in.
for (MachineBasicBlock::const_livein_iterator II = MBB->livein_begin(),
EE = MBB->livein_end(); II != EE; ++II) {
assert(TargetRegisterInfo::isPhysicalRegister(*II) &&
"Cannot have a live-in virtual register!");
HandlePhysRegDef(*II, 0);
}
// Loop over all of the instructions, processing them.
DistanceMap.clear();
unsigned Dist = 0;
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
I != E; ++I) {
MachineInstr *MI = I;
DistanceMap.insert(std::make_pair(MI, Dist++));
// Process all of the operands of the instruction...
unsigned NumOperandsToProcess = MI->getNumOperands();
// Unless it is a PHI node. In this case, ONLY process the DEF, not any
// of the uses. They will be handled in other basic blocks.
if (MI->getOpcode() == TargetInstrInfo::PHI)
NumOperandsToProcess = 1;
SmallVector<unsigned, 4> UseRegs;
SmallVector<unsigned, 4> DefRegs;
for (unsigned i = 0; i != NumOperandsToProcess; ++i) {
const MachineOperand &MO = MI->getOperand(i);
if (!MO.isReg() || MO.getReg() == 0)
continue;
unsigned MOReg = MO.getReg();
if (MO.isUse())
UseRegs.push_back(MOReg);
if (MO.isDef())
DefRegs.push_back(MOReg);
}
// Process all uses.
for (unsigned i = 0, e = UseRegs.size(); i != e; ++i) {
unsigned MOReg = UseRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegUse(MOReg, MBB, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegUse(MOReg, MI);
}
// Process all defs.
for (unsigned i = 0, e = DefRegs.size(); i != e; ++i) {
unsigned MOReg = DefRegs[i];
if (TargetRegisterInfo::isVirtualRegister(MOReg))
HandleVirtRegDef(MOReg, MI);
else if (!ReservedRegisters[MOReg])
HandlePhysRegDef(MOReg, MI);
}
}
// Handle any virtual assignments from PHI nodes which might be at the
// bottom of this basic block. We check all of our successor blocks to see
// if they have PHI nodes, and if so, we simulate an assignment at the end
// of the current block.
if (!PHIVarInfo[MBB->getNumber()].empty()) {
SmallVector<unsigned, 4>& VarInfoVec = PHIVarInfo[MBB->getNumber()];
for (SmallVector<unsigned, 4>::iterator I = VarInfoVec.begin(),
E = VarInfoVec.end(); I != E; ++I)
// Mark it alive only in the block we are representing.
MarkVirtRegAliveInBlock(getVarInfo(*I),MRI->getVRegDef(*I)->getParent(),
MBB);
}
// Finally, if the last instruction in the block is a return, make sure to
// mark it as using all of the live-out values in the function.
if (!MBB->empty() && MBB->back().getDesc().isReturn()) {
MachineInstr *Ret = &MBB->back();
for (MachineRegisterInfo::liveout_iterator
I = MF->getRegInfo().liveout_begin(),
E = MF->getRegInfo().liveout_end(); I != E; ++I) {
assert(TargetRegisterInfo::isPhysicalRegister(*I) &&
"Cannot have a live-out virtual register!");
HandlePhysRegUse(*I, Ret);
// Add live-out registers as implicit uses.
if (!Ret->readsRegister(*I))
Ret->addOperand(MachineOperand::CreateReg(*I, false, true));
}
}
// Loop over PhysRegDef / PhysRegUse, killing any registers that are
// available at the end of the basic block.
for (unsigned i = 0; i != NumRegs; ++i)
if (PhysRegDef[i] || PhysRegUse[i])
HandlePhysRegDef(i, 0);
std::fill(PhysRegDef, PhysRegDef + NumRegs, (MachineInstr*)0);
std::fill(PhysRegUse, PhysRegUse + NumRegs, (MachineInstr*)0);
}
// Convert and transfer the dead / killed information we have gathered into
// VirtRegInfo onto MI's.
for (unsigned i = 0, e1 = VirtRegInfo.size(); i != e1; ++i)
for (unsigned j = 0, e2 = VirtRegInfo[i].Kills.size(); j != e2; ++j)
if (VirtRegInfo[i].Kills[j] ==
MRI->getVRegDef(i + TargetRegisterInfo::FirstVirtualRegister))
VirtRegInfo[i]
.Kills[j]->addRegisterDead(i +
TargetRegisterInfo::FirstVirtualRegister,
TRI);
else
VirtRegInfo[i]
.Kills[j]->addRegisterKilled(i +
TargetRegisterInfo::FirstVirtualRegister,
TRI);
// Check to make sure there are no unreachable blocks in the MC CFG for the
// function. If so, it is due to a bug in the instruction selector or some
// other part of the code generator if this happens.
#ifndef NDEBUG
for(MachineFunction::iterator i = MF->begin(), e = MF->end(); i != e; ++i)
assert(Visited.count(&*i) != 0 && "unreachable basic block found");
#endif
delete[] PhysRegDef;
delete[] PhysRegUse;
delete[] PHIVarInfo;
return false;
}
/// EmitSubregNode - Generate machine code for subreg nodes.
///
void ScheduleDAGSDNodes::EmitSubregNode(SDNode *Node,
DenseMap<SDValue, unsigned> &VRBaseMap) {
unsigned VRBase = 0;
unsigned Opc = Node->getMachineOpcode();
// If the node is only used by a CopyToReg and the dest reg is a vreg, use
// the CopyToReg'd destination register instead of creating a new vreg.
for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
UI != E; ++UI) {
SDNode *User = *UI;
if (User->getOpcode() == ISD::CopyToReg &&
User->getOperand(2).getNode() == Node) {
unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
break;
}
}
}
if (Opc == TargetInstrInfo::EXTRACT_SUBREG) {
unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
// Create the extract_subreg machine instruction.
MachineInstr *MI = BuildMI(MF, Node->getDebugLoc(),
TII->get(TargetInstrInfo::EXTRACT_SUBREG));
// Figure out the register class to create for the destreg.
unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
const TargetRegisterClass *TRC = MRI.getRegClass(VReg);
const TargetRegisterClass *SRC = getSubRegisterRegClass(TRC, SubIdx);
// Figure out the register class to create for the destreg.
// Note that if we're going to directly use an existing register,
// it must be precisely the required class, and not a subclass
// thereof.
if (VRBase == 0 || SRC != MRI.getRegClass(VRBase)) {
// Create the reg
assert(SRC && "Couldn't find source register class");
VRBase = MRI.createVirtualRegister(SRC);
}
// Add def, source, and subreg index
MI->addOperand(MachineOperand::CreateReg(VRBase, true));
AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap);
MI->addOperand(MachineOperand::CreateImm(SubIdx));
BB->insert(InsertPos, MI);
} else if (Opc == TargetInstrInfo::INSERT_SUBREG ||
Opc == TargetInstrInfo::SUBREG_TO_REG) {
SDValue N0 = Node->getOperand(0);
SDValue N1 = Node->getOperand(1);
SDValue N2 = Node->getOperand(2);
unsigned SubReg = getVR(N1, VRBaseMap);
unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
const TargetRegisterClass *TRC = MRI.getRegClass(SubReg);
const TargetRegisterClass *SRC =
getSuperRegisterRegClass(TRC, SubIdx,
Node->getValueType(0));
// Figure out the register class to create for the destreg.
// Note that if we're going to directly use an existing register,
// it must be precisely the required class, and not a subclass
// thereof.
if (VRBase == 0 || SRC != MRI.getRegClass(VRBase)) {
// Create the reg
assert(SRC && "Couldn't find source register class");
VRBase = MRI.createVirtualRegister(SRC);
}
// Create the insert_subreg or subreg_to_reg machine instruction.
MachineInstr *MI = BuildMI(MF, Node->getDebugLoc(), TII->get(Opc));
MI->addOperand(MachineOperand::CreateReg(VRBase, true));
// If creating a subreg_to_reg, then the first input operand
// is an implicit value immediate, otherwise it's a register
if (Opc == TargetInstrInfo::SUBREG_TO_REG) {
const ConstantSDNode *SD = cast<ConstantSDNode>(N0);
MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue()));
} else
AddOperand(MI, N0, 0, 0, VRBaseMap);
// Add the subregster being inserted
AddOperand(MI, N1, 0, 0, VRBaseMap);
MI->addOperand(MachineOperand::CreateImm(SubIdx));
BB->insert(InsertPos, MI);
} else
assert(0 && "Node is not insert_subreg, extract_subreg, or subreg_to_reg");
SDValue Op(Node, 0);
bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
isNew = isNew; // Silence compiler warning.
assert(isNew && "Node emitted out of order - early");
}
/// EmitSubregNode - Generate machine code for subreg nodes.
///
void InstrEmitter::EmitSubregNode(SDNode *Node,
DenseMap<SDValue, unsigned> &VRBaseMap,
bool IsClone, bool IsCloned) {
unsigned VRBase = 0;
unsigned Opc = Node->getMachineOpcode();
// If the node is only used by a CopyToReg and the dest reg is a vreg, use
// the CopyToReg'd destination register instead of creating a new vreg.
for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
UI != E; ++UI) {
SDNode *User = *UI;
if (User->getOpcode() == ISD::CopyToReg &&
User->getOperand(2).getNode() == Node) {
unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
break;
}
}
}
if (Opc == TargetOpcode::EXTRACT_SUBREG) {
// EXTRACT_SUBREG is lowered as %dst = COPY %src:sub
unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
// Figure out the register class to create for the destreg.
unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
MachineInstr *DefMI = MRI->getVRegDef(VReg);
unsigned SrcReg, DstReg, DefSubIdx;
if (DefMI &&
TII->isCoalescableExtInstr(*DefMI, SrcReg, DstReg, DefSubIdx) &&
SubIdx == DefSubIdx) {
// Optimize these:
// r1025 = s/zext r1024, 4
// r1026 = extract_subreg r1025, 4
// to a copy
// r1026 = copy r1024
const TargetRegisterClass *TRC = MRI->getRegClass(SrcReg);
VRBase = MRI->createVirtualRegister(TRC);
BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg);
} else {
const TargetRegisterClass *TRC = MRI->getRegClass(VReg);
const TargetRegisterClass *SRC = TRC->getSubRegisterRegClass(SubIdx);
assert(SRC && "Invalid subregister index in EXTRACT_SUBREG");
// Figure out the register class to create for the destreg.
// Note that if we're going to directly use an existing register,
// it must be precisely the required class, and not a subclass
// thereof.
if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) {
// Create the reg
assert(SRC && "Couldn't find source register class");
VRBase = MRI->createVirtualRegister(SRC);
}
// Create the extract_subreg machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
TII->get(TargetOpcode::COPY), VRBase);
// Add source, and subreg index
AddOperand(MI, Node->getOperand(0), 0, 0, VRBaseMap, /*IsDebug=*/false,
IsClone, IsCloned);
assert(TargetRegisterInfo::isVirtualRegister(MI->getOperand(1).getReg())&&
"Cannot yet extract from physregs");
MI->getOperand(1).setSubReg(SubIdx);
MBB->insert(InsertPos, MI);
}
} else if (Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
SDValue N0 = Node->getOperand(0);
SDValue N1 = Node->getOperand(1);
SDValue N2 = Node->getOperand(2);
unsigned SubReg = getVR(N1, VRBaseMap);
unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
const TargetRegisterClass *TRC = MRI->getRegClass(SubReg);
const TargetRegisterClass *SRC =
getSuperRegisterRegClass(TRC, SubIdx, Node->getValueType(0));
// Figure out the register class to create for the destreg.
// Note that if we're going to directly use an existing register,
// it must be precisely the required class, and not a subclass
// thereof.
if (VRBase == 0 || SRC != MRI->getRegClass(VRBase)) {
// Create the reg
assert(SRC && "Couldn't find source register class");
VRBase = MRI->createVirtualRegister(SRC);
}
// Create the insert_subreg or subreg_to_reg machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc));
MI->addOperand(MachineOperand::CreateReg(VRBase, true));
// If creating a subreg_to_reg, then the first input operand
// is an implicit value immediate, otherwise it's a register
if (Opc == TargetOpcode::SUBREG_TO_REG) {
const ConstantSDNode *SD = cast<ConstantSDNode>(N0);
MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue()));
} else
AddOperand(MI, N0, 0, 0, VRBaseMap, /*IsDebug=*/false,
IsClone, IsCloned);
// Add the subregster being inserted
AddOperand(MI, N1, 0, 0, VRBaseMap, /*IsDebug=*/false,
IsClone, IsCloned);
MI->addOperand(MachineOperand::CreateImm(SubIdx));
MBB->insert(InsertPos, MI);
} else
llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
SDValue Op(Node, 0);
bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
(void)isNew; // Silence compiler warning.
assert(isNew && "Node emitted out of order - early");
}
bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) {
QII = static_cast<const HexagonInstrInfo *>(MF.getSubtarget().getInstrInfo());
QRI = MF.getTarget().getSubtarget<HexagonSubtarget>().getRegisterInfo();
MRI = &MF.getRegInfo();
DenseMap<unsigned, unsigned> PeepholeMap;
DenseMap<unsigned, std::pair<unsigned, unsigned> > PeepholeDoubleRegsMap;
if (DisableHexagonPeephole) return false;
// Loop over all of the basic blocks.
for (MachineFunction::iterator MBBb = MF.begin(), MBBe = MF.end();
MBBb != MBBe; ++MBBb) {
MachineBasicBlock* MBB = MBBb;
PeepholeMap.clear();
PeepholeDoubleRegsMap.clear();
// Traverse the basic block.
for (MachineBasicBlock::iterator MII = MBB->begin(); MII != MBB->end();
++MII) {
MachineInstr *MI = MII;
// Look for sign extends:
// %vreg170<def> = SXTW %vreg166
if (!DisableOptSZExt && MI->getOpcode() == Hexagon::A2_sxtw) {
assert (MI->getNumOperands() == 2);
MachineOperand &Dst = MI->getOperand(0);
MachineOperand &Src = MI->getOperand(1);
unsigned DstReg = Dst.getReg();
unsigned SrcReg = Src.getReg();
// Just handle virtual registers.
if (TargetRegisterInfo::isVirtualRegister(DstReg) &&
TargetRegisterInfo::isVirtualRegister(SrcReg)) {
// Map the following:
// %vreg170<def> = SXTW %vreg166
// PeepholeMap[170] = vreg166
PeepholeMap[DstReg] = SrcReg;
}
}
// Look for %vreg170<def> = COMBINE_ir_V4 (0, %vreg169)
// %vreg170:DoublRegs, %vreg169:IntRegs
if (!DisableOptExtTo64 &&
MI->getOpcode () == Hexagon::COMBINE_Ir_V4) {
assert (MI->getNumOperands() == 3);
MachineOperand &Dst = MI->getOperand(0);
MachineOperand &Src1 = MI->getOperand(1);
MachineOperand &Src2 = MI->getOperand(2);
if (Src1.getImm() != 0)
continue;
unsigned DstReg = Dst.getReg();
unsigned SrcReg = Src2.getReg();
PeepholeMap[DstReg] = SrcReg;
}
// Look for this sequence below
// %vregDoubleReg1 = LSRd_ri %vregDoubleReg0, 32
// %vregIntReg = COPY %vregDoubleReg1:subreg_loreg.
// and convert into
// %vregIntReg = COPY %vregDoubleReg0:subreg_hireg.
if (MI->getOpcode() == Hexagon::LSRd_ri) {
assert(MI->getNumOperands() == 3);
MachineOperand &Dst = MI->getOperand(0);
MachineOperand &Src1 = MI->getOperand(1);
MachineOperand &Src2 = MI->getOperand(2);
if (Src2.getImm() != 32)
continue;
unsigned DstReg = Dst.getReg();
unsigned SrcReg = Src1.getReg();
PeepholeDoubleRegsMap[DstReg] =
std::make_pair(*&SrcReg, 1/*Hexagon::subreg_hireg*/);
}
// Look for P=NOT(P).
if (!DisablePNotP &&
(MI->getOpcode() == Hexagon::C2_not)) {
assert (MI->getNumOperands() == 2);
MachineOperand &Dst = MI->getOperand(0);
MachineOperand &Src = MI->getOperand(1);
unsigned DstReg = Dst.getReg();
unsigned SrcReg = Src.getReg();
// Just handle virtual registers.
if (TargetRegisterInfo::isVirtualRegister(DstReg) &&
TargetRegisterInfo::isVirtualRegister(SrcReg)) {
// Map the following:
// %vreg170<def> = NOT_xx %vreg166
// PeepholeMap[170] = vreg166
PeepholeMap[DstReg] = SrcReg;
}
}
// Look for copy:
// %vreg176<def> = COPY %vreg170:subreg_loreg
if (!DisableOptSZExt && MI->isCopy()) {
assert (MI->getNumOperands() == 2);
MachineOperand &Dst = MI->getOperand(0);
MachineOperand &Src = MI->getOperand(1);
// Make sure we are copying the lower 32 bits.
if (Src.getSubReg() != Hexagon::subreg_loreg)
continue;
unsigned DstReg = Dst.getReg();
unsigned SrcReg = Src.getReg();
if (TargetRegisterInfo::isVirtualRegister(DstReg) &&
TargetRegisterInfo::isVirtualRegister(SrcReg)) {
// Try to find in the map.
if (unsigned PeepholeSrc = PeepholeMap.lookup(SrcReg)) {
// Change the 1st operand.
MI->RemoveOperand(1);
MI->addOperand(MachineOperand::CreateReg(PeepholeSrc, false));
} else {
DenseMap<unsigned, std::pair<unsigned, unsigned> >::iterator DI =
PeepholeDoubleRegsMap.find(SrcReg);
if (DI != PeepholeDoubleRegsMap.end()) {
std::pair<unsigned,unsigned> PeepholeSrc = DI->second;
MI->RemoveOperand(1);
MI->addOperand(MachineOperand::CreateReg(PeepholeSrc.first,
false /*isDef*/,
false /*isImp*/,
false /*isKill*/,
false /*isDead*/,
false /*isUndef*/,
false /*isEarlyClobber*/,
PeepholeSrc.second));
}
}
}
}
// Look for Predicated instructions.
if (!DisablePNotP) {
bool Done = false;
if (QII->isPredicated(MI)) {
MachineOperand &Op0 = MI->getOperand(0);
unsigned Reg0 = Op0.getReg();
const TargetRegisterClass *RC0 = MRI->getRegClass(Reg0);
if (RC0->getID() == Hexagon::PredRegsRegClassID) {
// Handle instructions that have a prediate register in op0
// (most cases of predicable instructions).
if (TargetRegisterInfo::isVirtualRegister(Reg0)) {
// Try to find in the map.
if (unsigned PeepholeSrc = PeepholeMap.lookup(Reg0)) {
// Change the 1st operand and, flip the opcode.
MI->getOperand(0).setReg(PeepholeSrc);
int NewOp = QII->getInvertedPredicatedOpcode(MI->getOpcode());
MI->setDesc(QII->get(NewOp));
Done = true;
}
}
}
}
if (!Done) {
// Handle special instructions.
unsigned Op = MI->getOpcode();
unsigned NewOp = 0;
unsigned PR = 1, S1 = 2, S2 = 3; // Operand indices.
switch (Op) {
case Hexagon::C2_mux:
case Hexagon::C2_muxii:
case Hexagon::TFR_condset_ii:
NewOp = Op;
break;
case Hexagon::TFR_condset_ri:
NewOp = Hexagon::TFR_condset_ir;
break;
case Hexagon::TFR_condset_ir:
NewOp = Hexagon::TFR_condset_ri;
break;
case Hexagon::C2_muxri:
NewOp = Hexagon::C2_muxir;
break;
case Hexagon::C2_muxir:
NewOp = Hexagon::C2_muxri;
break;
}
if (NewOp) {
unsigned PSrc = MI->getOperand(PR).getReg();
if (unsigned POrig = PeepholeMap.lookup(PSrc)) {
MI->getOperand(PR).setReg(POrig);
MI->setDesc(QII->get(NewOp));
// Swap operands S1 and S2.
MachineOperand Op1 = MI->getOperand(S1);
MachineOperand Op2 = MI->getOperand(S2);
ChangeOpInto(MI->getOperand(S1), Op2);
ChangeOpInto(MI->getOperand(S2), Op1);
}
} // if (NewOp)
} // if (!Done)
} // if (!DisablePNotP)
} // Instruction
} // Basic Block
return true;
}
/// EmitSubregNode - Generate machine code for subreg nodes.
///
void InstrEmitter::EmitSubregNode(SDNode *Node,
DenseMap<SDValue, unsigned> &VRBaseMap,
bool IsClone, bool IsCloned) {
unsigned VRBase = 0;
unsigned Opc = Node->getMachineOpcode();
// If the node is only used by a CopyToReg and the dest reg is a vreg, use
// the CopyToReg'd destination register instead of creating a new vreg.
for (SDNode::use_iterator UI = Node->use_begin(), E = Node->use_end();
UI != E; ++UI) {
SDNode *User = *UI;
if (User->getOpcode() == ISD::CopyToReg &&
User->getOperand(2).getNode() == Node) {
unsigned DestReg = cast<RegisterSDNode>(User->getOperand(1))->getReg();
if (TargetRegisterInfo::isVirtualRegister(DestReg)) {
VRBase = DestReg;
break;
}
}
}
if (Opc == TargetOpcode::EXTRACT_SUBREG) {
// EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no
// constraints on the %dst register, COPY can target all legal register
// classes.
unsigned SubIdx = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
const TargetRegisterClass *TRC = TLI->getRegClassFor(Node->getValueType(0));
unsigned VReg = getVR(Node->getOperand(0), VRBaseMap);
MachineInstr *DefMI = MRI->getVRegDef(VReg);
unsigned SrcReg, DstReg, DefSubIdx;
if (DefMI &&
TII->isCoalescableExtInstr(*DefMI, SrcReg, DstReg, DefSubIdx) &&
SubIdx == DefSubIdx) {
// Optimize these:
// r1025 = s/zext r1024, 4
// r1026 = extract_subreg r1025, 4
// to a copy
// r1026 = copy r1024
VRBase = MRI->createVirtualRegister(TRC);
BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
TII->get(TargetOpcode::COPY), VRBase).addReg(SrcReg);
} else {
// VReg may not support a SubIdx sub-register, and we may need to
// constrain its register class or issue a COPY to a compatible register
// class.
VReg = ConstrainForSubReg(VReg, SubIdx,
Node->getOperand(0).getValueType(),
Node->getDebugLoc());
// Create the destreg if it is missing.
if (VRBase == 0)
VRBase = MRI->createVirtualRegister(TRC);
// Create the extract_subreg machine instruction.
BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
TII->get(TargetOpcode::COPY), VRBase).addReg(VReg, 0, SubIdx);
}
} else if (Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
SDValue N0 = Node->getOperand(0);
SDValue N1 = Node->getOperand(1);
SDValue N2 = Node->getOperand(2);
unsigned SubIdx = cast<ConstantSDNode>(N2)->getZExtValue();
// Figure out the register class to create for the destreg. It should be
// the largest legal register class supporting SubIdx sub-registers.
// RegisterCoalescer will constrain it further if it decides to eliminate
// the INSERT_SUBREG instruction.
//
// %dst = INSERT_SUBREG %src, %sub, SubIdx
//
// is lowered by TwoAddressInstructionPass to:
//
// %dst = COPY %src
// %dst:SubIdx = COPY %sub
//
// There is no constraint on the %src register class.
//
const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getValueType(0));
SRC = TRI->getSubClassWithSubReg(SRC, SubIdx);
assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
if (VRBase == 0 || !SRC->hasSubClassEq(MRI->getRegClass(VRBase)))
VRBase = MRI->createVirtualRegister(SRC);
// Create the insert_subreg or subreg_to_reg machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), TII->get(Opc));
MI->addOperand(MachineOperand::CreateReg(VRBase, true));
// If creating a subreg_to_reg, then the first input operand
// is an implicit value immediate, otherwise it's a register
if (Opc == TargetOpcode::SUBREG_TO_REG) {
const ConstantSDNode *SD = cast<ConstantSDNode>(N0);
MI->addOperand(MachineOperand::CreateImm(SD->getZExtValue()));
} else
AddOperand(MI, N0, 0, 0, VRBaseMap, /*IsDebug=*/false,
IsClone, IsCloned);
// Add the subregster being inserted
AddOperand(MI, N1, 0, 0, VRBaseMap, /*IsDebug=*/false,
IsClone, IsCloned);
MI->addOperand(MachineOperand::CreateImm(SubIdx));
MBB->insert(InsertPos, MI);
} else
llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
SDValue Op(Node, 0);
bool isNew = VRBaseMap.insert(std::make_pair(Op, VRBase)).second;
(void)isNew; // Silence compiler warning.
assert(isNew && "Node emitted out of order - early");
}
bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *MI) {
MachineInstr *LastDef = PhysRegDef[Reg];
MachineInstr *LastUse = PhysRegUse[Reg];
if (!LastDef && !LastUse)
return false;
MachineInstr *LastRefOrPartRef = LastUse ? LastUse : LastDef;
unsigned LastRefOrPartRefDist = DistanceMap[LastRefOrPartRef];
// The whole register is used.
// AL =
// AH =
//
// = AX
// = AL, AX<imp-use, kill>
// AX =
//
// Or whole register is defined, but not used at all.
// AX<dead> =
// ...
// AX =
//
// Or whole register is defined, but only partly used.
// AX<dead> = AL<imp-def>
// = AL<kill>
// AX =
MachineInstr *LastPartDef = nullptr;
unsigned LastPartDefDist = 0;
SmallSet<unsigned, 8> PartUses;
for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) {
unsigned SubReg = *SubRegs;
MachineInstr *Def = PhysRegDef[SubReg];
if (Def && Def != LastDef) {
// There was a def of this sub-register in between. This is a partial
// def, keep track of the last one.
unsigned Dist = DistanceMap[Def];
if (Dist > LastPartDefDist) {
LastPartDefDist = Dist;
LastPartDef = Def;
}
continue;
}
if (MachineInstr *Use = PhysRegUse[SubReg]) {
for (MCSubRegIterator SS(SubReg, TRI, /*IncludeSelf=*/true); SS.isValid();
++SS)
PartUses.insert(*SS);
unsigned Dist = DistanceMap[Use];
if (Dist > LastRefOrPartRefDist) {
LastRefOrPartRefDist = Dist;
LastRefOrPartRef = Use;
}
}
}
if (!PhysRegUse[Reg]) {
// Partial uses. Mark register def dead and add implicit def of
// sub-registers which are used.
// EAX<dead> = op AL<imp-def>
// That is, EAX def is dead but AL def extends pass it.
PhysRegDef[Reg]->addRegisterDead(Reg, TRI, true);
for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs) {
unsigned SubReg = *SubRegs;
if (!PartUses.count(SubReg))
continue;
bool NeedDef = true;
if (PhysRegDef[Reg] == PhysRegDef[SubReg]) {
MachineOperand *MO = PhysRegDef[Reg]->findRegisterDefOperand(SubReg);
if (MO) {
NeedDef = false;
assert(!MO->isDead());
}
}
if (NeedDef)
PhysRegDef[Reg]->addOperand(MachineOperand::CreateReg(SubReg,
true/*IsDef*/, true/*IsImp*/));
MachineInstr *LastSubRef = FindLastRefOrPartRef(SubReg);
if (LastSubRef)
LastSubRef->addRegisterKilled(SubReg, TRI, true);
else {
LastRefOrPartRef->addRegisterKilled(SubReg, TRI, true);
for (MCSubRegIterator SS(SubReg, TRI, /*IncludeSelf=*/true);
SS.isValid(); ++SS)
PhysRegUse[*SS] = LastRefOrPartRef;
}
for (MCSubRegIterator SS(SubReg, TRI); SS.isValid(); ++SS)
PartUses.erase(*SS);
}
} else if (LastRefOrPartRef == PhysRegDef[Reg] && LastRefOrPartRef != MI) {
if (LastPartDef)
// The last partial def kills the register.
LastPartDef->addOperand(MachineOperand::CreateReg(Reg, false/*IsDef*/,
true/*IsImp*/, true/*IsKill*/));
else {
MachineOperand *MO =
LastRefOrPartRef->findRegisterDefOperand(Reg, false, TRI);
bool NeedEC = MO->isEarlyClobber() && MO->getReg() != Reg;
// If the last reference is the last def, then it's not used at all.
// That is, unless we are currently processing the last reference itself.
LastRefOrPartRef->addRegisterDead(Reg, TRI, true);
if (NeedEC) {
// If we are adding a subreg def and the superreg def is marked early
// clobber, add an early clobber marker to the subreg def.
MO = LastRefOrPartRef->findRegisterDefOperand(Reg);
if (MO)
MO->setIsEarlyClobber();
}
}
} else
LastRefOrPartRef->addRegisterKilled(Reg, TRI, true);
return true;
}
/// EmitSpecialNode - Generate machine code for a target-independent node and
/// needed dependencies.
void InstrEmitter::
EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap) {
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
Node->dump();
#endif
llvm_unreachable("This target-independent node should have been selected!");
case ISD::EntryToken:
llvm_unreachable("EntryToken should have been excluded from the schedule!");
case ISD::MERGE_VALUES:
case ISD::TokenFactor: // fall thru
break;
case ISD::CopyToReg: {
unsigned SrcReg;
SDValue SrcVal = Node->getOperand(2);
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
SrcReg = R->getReg();
else
SrcReg = getVR(SrcVal, VRBaseMap);
unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
break;
BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TargetOpcode::COPY),
DestReg).addReg(SrcReg);
break;
}
case ISD::CopyFromReg: {
unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
break;
}
case ISD::EH_LABEL: {
MCSymbol *S = cast<EHLabelSDNode>(Node)->getLabel();
BuildMI(*MBB, InsertPos, Node->getDebugLoc(),
TII->get(TargetOpcode::EH_LABEL)).addSym(S);
break;
}
case ISD::INLINEASM: {
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
--NumOps; // Ignore the glue operand.
// Create the inline asm machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
TII->get(TargetOpcode::INLINEASM));
// Add the asm string as an external symbol operand.
SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol();
MI->addOperand(MachineOperand::CreateES(AsmStr));
// Add the HasSideEffect and isAlignStack bits.
int64_t ExtraInfo =
cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))->
getZExtValue();
MI->addOperand(MachineOperand::CreateImm(ExtraInfo));
// Add all of the operand registers to the instruction.
for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
unsigned Flags =
cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
MI->addOperand(MachineOperand::CreateImm(Flags));
++i; // Skip the ID value.
switch (InlineAsm::getKind(Flags)) {
default:
llvm_unreachable("Bad flags!");
case InlineAsm::Kind_RegDef:
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
// FIXME: Add dead flags for physical and virtual registers defined.
// For now, mark physical register defs as implicit to help fast
// regalloc. This makes inline asm look a lot like calls.
MI->addOperand(MachineOperand::CreateReg(Reg, true,
/*isImp=*/ TargetRegisterInfo::isPhysicalRegister(Reg)));
}
break;
case InlineAsm::Kind_RegDefEarlyClobber:
case InlineAsm::Kind_Clobber:
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, /*isDef=*/ true,
/*isImp=*/ TargetRegisterInfo::isPhysicalRegister(Reg),
/*isKill=*/ false,
/*isDead=*/ false,
/*isUndef=*/false,
/*isEarlyClobber=*/ true));
}
break;
case InlineAsm::Kind_RegUse: // Use of register.
case InlineAsm::Kind_Imm: // Immediate.
case InlineAsm::Kind_Mem: // Addressing mode.
// The addressing mode has been selected, just add all of the
// operands to the machine instruction.
for (; NumVals; --NumVals, ++i)
AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap,
/*IsDebug=*/false, IsClone, IsCloned);
break;
}
}
// Get the mdnode from the asm if it exists and add it to the instruction.
SDValue MDV = Node->getOperand(InlineAsm::Op_MDNode);
const MDNode *MD = cast<MDNodeSDNode>(MDV)->getMD();
if (MD)
MI->addOperand(MachineOperand::CreateMetadata(MD));
MBB->insert(InsertPos, MI);
break;
}
}
}
bool Thumb2ITBlockPass::InsertITInstructions(MachineBasicBlock &MBB) {
bool Modified = false;
SmallSet<unsigned, 4> Defs;
SmallSet<unsigned, 4> Uses;
MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
while (MBBI != E) {
MachineInstr *MI = &*MBBI;
DebugLoc dl = MI->getDebugLoc();
unsigned PredReg = 0;
ARMCC::CondCodes CC = llvm::getITInstrPredicate(MI, PredReg);
if (CC == ARMCC::AL) {
++MBBI;
continue;
}
Defs.clear();
Uses.clear();
TrackDefUses(MI, Defs, Uses, TRI);
// Insert an IT instruction.
MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(ARM::t2IT))
.addImm(CC);
// Add implicit use of ITSTATE to IT block instructions.
MI->addOperand(MachineOperand::CreateReg(ARM::ITSTATE, false/*ifDef*/,
true/*isImp*/, false/*isKill*/));
MachineInstr *LastITMI = MI;
MachineBasicBlock::iterator InsertPos = MIB;
++MBBI;
// Form IT block.
ARMCC::CondCodes OCC = ARMCC::getOppositeCondition(CC);
unsigned Mask = 0, Pos = 3;
// Branches, including tricky ones like LDM_RET, need to end an IT
// block so check the instruction we just put in the block.
for (; MBBI != E && Pos &&
(!MI->isBranch() && !MI->isReturn()) ; ++MBBI) {
if (MBBI->isDebugValue())
continue;
MachineInstr *NMI = &*MBBI;
MI = NMI;
unsigned NPredReg = 0;
ARMCC::CondCodes NCC = llvm::getITInstrPredicate(NMI, NPredReg);
if (NCC == CC || NCC == OCC) {
Mask |= (NCC & 1) << Pos;
// Add implicit use of ITSTATE.
NMI->addOperand(MachineOperand::CreateReg(ARM::ITSTATE, false/*ifDef*/,
true/*isImp*/, false/*isKill*/));
LastITMI = NMI;
} else {
if (NCC == ARMCC::AL &&
MoveCopyOutOfITBlock(NMI, CC, OCC, Defs, Uses)) {
--MBBI;
MBB.remove(NMI);
MBB.insert(InsertPos, NMI);
++NumMovedInsts;
continue;
}
break;
}
TrackDefUses(NMI, Defs, Uses, TRI);
--Pos;
}
// Finalize IT mask.
Mask |= (1 << Pos);
// Tag along (firstcond[0] << 4) with the mask.
Mask |= (CC & 1) << 4;
MIB.addImm(Mask);
// Last instruction in IT block kills ITSTATE.
LastITMI->findRegisterUseOperand(ARM::ITSTATE)->setIsKill();
Modified = true;
++NumITs;
}
return Modified;
}
bool X86CallFrameOptimization::adjustCallSequence(MachineFunction &MF,
MachineBasicBlock::iterator I,
const CallContext &Context) {
// Ok, we can in fact do the transformation for this call.
// Do not remove the FrameSetup instruction, but adjust the parameters.
// PEI will end up finalizing the handling of this.
MachineBasicBlock::iterator FrameSetup = I;
MachineBasicBlock &MBB = *(I->getParent());
FrameSetup->getOperand(1).setImm(Context.ExpectedDist);
DebugLoc DL = I->getDebugLoc();
// Now, iterate through the vector in reverse order, and replace the movs
// with pushes. MOVmi/MOVmr doesn't have any defs, so no need to
// replace uses.
for (int Idx = (Context.ExpectedDist / 4) - 1; Idx >= 0; --Idx) {
MachineBasicBlock::iterator MOV = *Context.MovVector[Idx];
MachineOperand PushOp = MOV->getOperand(X86::AddrNumOperands);
if (MOV->getOpcode() == X86::MOV32mi) {
unsigned PushOpcode = X86::PUSHi32;
// If the operand is a small (8-bit) immediate, we can use a
// PUSH instruction with a shorter encoding.
// Note that isImm() may fail even though this is a MOVmi, because
// the operand can also be a symbol.
if (PushOp.isImm()) {
int64_t Val = PushOp.getImm();
if (isInt<8>(Val))
PushOpcode = X86::PUSH32i8;
}
BuildMI(MBB, Context.Call, DL, TII->get(PushOpcode)).addOperand(PushOp);
} else {
unsigned int Reg = PushOp.getReg();
// If PUSHrmm is not slow on this target, try to fold the source of the
// push into the instruction.
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
bool SlowPUSHrmm = ST.isAtom() || ST.isSLM();
// Check that this is legal to fold. Right now, we're extremely
// conservative about that.
MachineInstr *DefMov = nullptr;
if (!SlowPUSHrmm && (DefMov = canFoldIntoRegPush(FrameSetup, Reg))) {
MachineInstr *Push =
BuildMI(MBB, Context.Call, DL, TII->get(X86::PUSH32rmm));
unsigned NumOps = DefMov->getDesc().getNumOperands();
for (unsigned i = NumOps - X86::AddrNumOperands; i != NumOps; ++i)
Push->addOperand(DefMov->getOperand(i));
DefMov->eraseFromParent();
} else {
BuildMI(MBB, Context.Call, DL, TII->get(X86::PUSH32r))
.addReg(Reg)
.getInstr();
}
}
MBB.erase(MOV);
}
// The stack-pointer copy is no longer used in the call sequences.
// There should not be any other users, but we can't commit to that, so:
if (MRI->use_empty(Context.SPCopy->getOperand(0).getReg()))
Context.SPCopy->eraseFromParent();
// Once we've done this, we need to make sure PEI doesn't assume a reserved
// frame.
X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
FuncInfo->setHasPushSequences(true);
return true;
}
void SIInstrInfo::moveToVALU(MachineInstr &TopInst) const {
SmallVector<MachineInstr *, 128> Worklist;
Worklist.push_back(&TopInst);
while (!Worklist.empty()) {
MachineInstr *Inst = Worklist.pop_back_val();
unsigned NewOpcode = getVALUOp(*Inst);
if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END)
continue;
MachineRegisterInfo &MRI = Inst->getParent()->getParent()->getRegInfo();
// Use the new VALU Opcode.
const MCInstrDesc &NewDesc = get(NewOpcode);
Inst->setDesc(NewDesc);
// Remove any references to SCC. Vector instructions can't read from it, and
// We're just about to add the implicit use / defs of VCC, and we don't want
// both.
for (unsigned i = Inst->getNumOperands() - 1; i > 0; --i) {
MachineOperand &Op = Inst->getOperand(i);
if (Op.isReg() && Op.getReg() == AMDGPU::SCC)
Inst->RemoveOperand(i);
}
// Add the implict and explicit register definitions.
if (NewDesc.ImplicitUses) {
for (unsigned i = 0; NewDesc.ImplicitUses[i]; ++i) {
unsigned Reg = NewDesc.ImplicitUses[i];
Inst->addOperand(MachineOperand::CreateReg(Reg, false, true));
}
}
if (NewDesc.ImplicitDefs) {
for (unsigned i = 0; NewDesc.ImplicitDefs[i]; ++i) {
unsigned Reg = NewDesc.ImplicitDefs[i];
Inst->addOperand(MachineOperand::CreateReg(Reg, true, true));
}
}
legalizeOperands(Inst);
// Update the destination register class.
const TargetRegisterClass *NewDstRC = getOpRegClass(*Inst, 0);
switch (Inst->getOpcode()) {
// For target instructions, getOpRegClass just returns the virtual
// register class associated with the operand, so we need to find an
// equivalent VGPR register class in order to move the instruction to the
// VALU.
case AMDGPU::COPY:
case AMDGPU::PHI:
case AMDGPU::REG_SEQUENCE:
if (RI.hasVGPRs(NewDstRC))
continue;
NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
if (!NewDstRC)
continue;
break;
default:
break;
}
unsigned DstReg = Inst->getOperand(0).getReg();
unsigned NewDstReg = MRI.createVirtualRegister(NewDstRC);
MRI.replaceRegWith(DstReg, NewDstReg);
for (MachineRegisterInfo::use_iterator I = MRI.use_begin(NewDstReg),
E = MRI.use_end(); I != E; ++I) {
MachineInstr &UseMI = *I;
if (!canReadVGPR(UseMI, I.getOperandNo())) {
Worklist.push_back(&UseMI);
}
}
}
}
/// EmitNode - Generate machine code for a node and needed dependencies.
///
void InstrEmitter::EmitNode(SDNode *Node, bool IsClone, bool IsCloned,
DenseMap<SDValue, unsigned> &VRBaseMap,
DenseMap<MachineBasicBlock*, MachineBasicBlock*> *EM) {
// If machine instruction
if (Node->isMachineOpcode()) {
unsigned Opc = Node->getMachineOpcode();
// Handle subreg insert/extract specially
if (Opc == TargetOpcode::EXTRACT_SUBREG ||
Opc == TargetOpcode::INSERT_SUBREG ||
Opc == TargetOpcode::SUBREG_TO_REG) {
EmitSubregNode(Node, VRBaseMap);
return;
}
// Handle COPY_TO_REGCLASS specially.
if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
EmitCopyToRegClassNode(Node, VRBaseMap);
return;
}
if (Opc == TargetOpcode::IMPLICIT_DEF)
// We want a unique VR for each IMPLICIT_DEF use.
return;
const TargetInstrDesc &II = TII->get(Opc);
unsigned NumResults = CountResults(Node);
unsigned NodeOperands = CountOperands(Node);
bool HasPhysRegOuts = (NumResults > II.getNumDefs()) &&
II.getImplicitDefs() != 0;
#ifndef NDEBUG
unsigned NumMIOperands = NodeOperands + NumResults;
assert((II.getNumOperands() == NumMIOperands ||
HasPhysRegOuts || II.isVariadic()) &&
"#operands for dag node doesn't match .td file!");
#endif
// Create the new machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(), II);
// Add result register values for things that are defined by this
// instruction.
if (NumResults)
CreateVirtualRegisters(Node, MI, II, IsClone, IsCloned, VRBaseMap);
// Emit all of the actual operands of this instruction, adding them to the
// instruction as appropriate.
bool HasOptPRefs = II.getNumDefs() > NumResults;
assert((!HasOptPRefs || !HasPhysRegOuts) &&
"Unable to cope with optional defs and phys regs defs!");
unsigned NumSkip = HasOptPRefs ? II.getNumDefs() - NumResults : 0;
for (unsigned i = NumSkip; i != NodeOperands; ++i)
AddOperand(MI, Node->getOperand(i), i-NumSkip+II.getNumDefs(), &II,
VRBaseMap);
// Transfer all of the memory reference descriptions of this instruction.
MI->setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(),
cast<MachineSDNode>(Node)->memoperands_end());
if (II.usesCustomInsertionHook()) {
// Insert this instruction into the basic block using a target
// specific inserter which may returns a new basic block.
MBB = TLI->EmitInstrWithCustomInserter(MI, MBB, EM);
InsertPos = MBB->end();
} else {
MBB->insert(InsertPos, MI);
}
// Additional results must be an physical register def.
if (HasPhysRegOuts) {
for (unsigned i = II.getNumDefs(); i < NumResults; ++i) {
unsigned Reg = II.getImplicitDefs()[i - II.getNumDefs()];
if (Node->hasAnyUseOfValue(i))
EmitCopyFromReg(Node, i, IsClone, IsCloned, Reg, VRBaseMap);
// If there are no uses, mark the register as dead now, so that
// MachineLICM/Sink can see that it's dead. Don't do this if the
// node has a Flag value, for the benefit of targets still using
// Flag for values in physregs.
else if (Node->getValueType(Node->getNumValues()-1) != MVT::Flag)
MI->addRegisterDead(Reg, TRI);
}
}
return;
}
switch (Node->getOpcode()) {
default:
#ifndef NDEBUG
Node->dump();
#endif
llvm_unreachable("This target-independent node should have been selected!");
break;
case ISD::EntryToken:
llvm_unreachable("EntryToken should have been excluded from the schedule!");
break;
case ISD::MERGE_VALUES:
case ISD::TokenFactor: // fall thru
break;
case ISD::CopyToReg: {
unsigned SrcReg;
SDValue SrcVal = Node->getOperand(2);
if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(SrcVal))
SrcReg = R->getReg();
else
SrcReg = getVR(SrcVal, VRBaseMap);
unsigned DestReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
break;
const TargetRegisterClass *SrcTRC = 0, *DstTRC = 0;
// Get the register classes of the src/dst.
if (TargetRegisterInfo::isVirtualRegister(SrcReg))
SrcTRC = MRI->getRegClass(SrcReg);
else
SrcTRC = TRI->getPhysicalRegisterRegClass(SrcReg,SrcVal.getValueType());
if (TargetRegisterInfo::isVirtualRegister(DestReg))
DstTRC = MRI->getRegClass(DestReg);
else
DstTRC = TRI->getPhysicalRegisterRegClass(DestReg,
Node->getOperand(1).getValueType());
bool Emitted = TII->copyRegToReg(*MBB, InsertPos, DestReg, SrcReg,
DstTRC, SrcTRC);
assert(Emitted && "Unable to issue a copy instruction!\n");
(void) Emitted;
break;
}
case ISD::CopyFromReg: {
unsigned SrcReg = cast<RegisterSDNode>(Node->getOperand(1))->getReg();
EmitCopyFromReg(Node, 0, IsClone, IsCloned, SrcReg, VRBaseMap);
break;
}
case ISD::INLINEASM: {
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Flag)
--NumOps; // Ignore the flag operand.
// Create the inline asm machine instruction.
MachineInstr *MI = BuildMI(*MF, Node->getDebugLoc(),
TII->get(TargetOpcode::INLINEASM));
// Add the asm string as an external symbol operand.
const char *AsmStr =
cast<ExternalSymbolSDNode>(Node->getOperand(1))->getSymbol();
MI->addOperand(MachineOperand::CreateES(AsmStr));
// Add all of the operand registers to the instruction.
for (unsigned i = 2; i != NumOps;) {
unsigned Flags =
cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
MI->addOperand(MachineOperand::CreateImm(Flags));
++i; // Skip the ID value.
switch (Flags & 7) {
default: llvm_unreachable("Bad flags!");
case 2: // Def of register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true));
}
break;
case 6: // Def of earlyclobber register.
for (; NumVals; --NumVals, ++i) {
unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
MI->addOperand(MachineOperand::CreateReg(Reg, true, false, false,
false, false, true));
}
break;
case 1: // Use of register.
case 3: // Immediate.
case 4: // Addressing mode.
// The addressing mode has been selected, just add all of the
// operands to the machine instruction.
for (; NumVals; --NumVals, ++i)
AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap);
break;
}
}
MBB->insert(InsertPos, MI);
break;
}
}
}