bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
// Constants are always irrelevant
if (!Op.isReg())
return false;
// Defines are always relevant
if (Op.isDef())
return true;
// For exports all registers are relevant
MachineInstr &MI = *Op.getParent();
if (MI.getOpcode() == AMDGPU::EXP)
return true;
// For stores the stored value is also relevant
if (!MI.getDesc().mayStore())
return false;
for (MachineInstr::mop_iterator I = MI.operands_begin(),
E = MI.operands_end(); I != E; ++I) {
if (I->isReg() && I->isUse())
return Op.isIdenticalTo(*I);
}
return false;
}
static void replaceFrameIndexes(MachineFunction &MF,
SmallVector<std::pair<int,int64_t>, 16> &FR) {
MachineFrameInfo *MFI = MF.getFrameInfo();
MBlazeFunctionInfo *MBlazeFI = MF.getInfo<MBlazeFunctionInfo>();
const SmallVector<std::pair<int,int64_t>, 16>::iterator FRB = FR.begin();
const SmallVector<std::pair<int,int64_t>, 16>::iterator FRE = FR.end();
SmallVector<std::pair<int,int64_t>, 16>::iterator FRI = FRB;
for (; FRI != FRE; ++FRI) {
MFI->RemoveStackObject(FRI->first);
int NFI = MFI->CreateFixedObject(4, FRI->second, true);
MBlazeFI->recordReplacement(FRI->first, NFI);
for (MachineFunction::iterator MB=MF.begin(), ME=MF.end(); MB!=ME; ++MB) {
MachineBasicBlock::iterator MBB = MB->begin();
const MachineBasicBlock::iterator MBE = MB->end();
for (; MBB != MBE; ++MBB) {
MachineInstr::mop_iterator MIB = MBB->operands_begin();
const MachineInstr::mop_iterator MIE = MBB->operands_end();
for (MachineInstr::mop_iterator MII = MIB; MII != MIE; ++MII) {
if (!MII->isFI() || MII->getIndex() != FRI->first) continue;
DEBUG(dbgs() << "FOUND FI#" << MII->getIndex() << "\n");
MII->setIndex(NFI);
}
}
}
}
}
bool SIInsertWaits::isOpRelevant(MachineOperand &Op) {
// Constants are always irrelevant
if (!Op.isReg())
return false;
// Defines are always relevant
if (Op.isDef())
return true;
// For exports all registers are relevant
MachineInstr &MI = *Op.getParent();
if (MI.getOpcode() == AMDGPU::EXP)
return true;
// For stores the stored value is also relevant
if (!MI.getDesc().mayStore())
return false;
// Check if this operand is the value being stored.
// Special case for DS instructions, since the address
// operand comes before the value operand and it may have
// multiple data operands.
if (TII->isDS(MI.getOpcode())) {
MachineOperand *Data = TII->getNamedOperand(MI, AMDGPU::OpName::data);
if (Data && Op.isIdenticalTo(*Data))
return true;
MachineOperand *Data0 = TII->getNamedOperand(MI, AMDGPU::OpName::data0);
if (Data0 && Op.isIdenticalTo(*Data0))
return true;
MachineOperand *Data1 = TII->getNamedOperand(MI, AMDGPU::OpName::data1);
if (Data1 && Op.isIdenticalTo(*Data1))
return true;
return false;
}
// NOTE: This assumes that the value operand is before the
// address operand, and that there is only one value operand.
for (MachineInstr::mop_iterator I = MI.operands_begin(),
E = MI.operands_end(); I != E; ++I) {
if (I->isReg() && I->isUse())
return Op.isIdenticalTo(*I);
}
return false;
}
bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
DEBUG(dbgs() << "********** Fixing up CALL_INDIRECTs **********\n"
<< MF.getName() << '\n');
bool Changed = false;
const WebAssemblyInstrInfo *TII =
MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
if (IsPseudoCallIndirect(MI)) {
DEBUG(dbgs() << "Found call_indirect: " << MI << '\n');
// Rewrite pseudo to non-pseudo
const MCInstrDesc &Desc = TII->get(GetNonPseudoCallIndirectOpcode(MI));
MI.setDesc(Desc);
// Rewrite argument order
auto Uses = MI.explicit_uses();
MachineInstr::mop_iterator it = Uses.begin();
const MachineOperand MO = *it;
// Set up the flags immediate, which currently has no defined flags
// so it's always zero.
it->ChangeToImmediate(0);
MI.addOperand(MF, MO);
DEBUG(dbgs() << " After transform: " << MI);
Changed = true;
}
}
}
DEBUG(dbgs() << "\nDone fixing up CALL_INDIRECTs\n\n");
return Changed;
}
/// addNewBlock - Add a new basic block BB as an empty succcessor to DomBB. All
/// variables that are live out of DomBB will be marked as passing live through
/// BB.
void LiveVariables::addNewBlock(MachineBasicBlock *BB,
MachineBasicBlock *DomBB,
MachineBasicBlock *SuccBB) {
const unsigned NumNew = BB->getNumber();
SmallSet<unsigned, 16> Defs, Kills;
MachineBasicBlock::iterator BBI = SuccBB->begin(), BBE = SuccBB->end();
for (; BBI != BBE && BBI->isPHI(); ++BBI) {
// Record the def of the PHI node.
Defs.insert(BBI->getOperand(0).getReg());
// All registers used by PHI nodes in SuccBB must be live through BB.
for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
if (BBI->getOperand(i+1).getMBB() == BB)
getVarInfo(BBI->getOperand(i).getReg()).AliveBlocks.set(NumNew);
}
// Record all vreg defs and kills of all instructions in SuccBB.
for (; BBI != BBE; ++BBI) {
for (MachineInstr::mop_iterator I = BBI->operands_begin(),
E = BBI->operands_end(); I != E; ++I) {
if (I->isReg() && TargetRegisterInfo::isVirtualRegister(I->getReg())) {
if (I->isDef())
Defs.insert(I->getReg());
else if (I->isKill())
Kills.insert(I->getReg());
}
}
}
// Update info for all live variables
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
// If the Defs is defined in the successor it can't be live in BB.
if (Defs.count(Reg))
continue;
// If the register is either killed in or live through SuccBB it's also live
// through BB.
VarInfo &VI = getVarInfo(Reg);
if (Kills.count(Reg) || VI.AliveBlocks.test(SuccBB->getNumber()))
VI.AliveBlocks.set(NumNew);
}
}
MachineBasicBlock *
MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
// Splitting the critical edge to a landing pad block is non-trivial. Don't do
// it in this generic function.
if (Succ->isLandingPad())
return nullptr;
MachineFunction *MF = getParent();
DebugLoc dl; // FIXME: this is nowhere
// Performance might be harmed on HW that implements branching using exec mask
// where both sides of the branches are always executed.
if (MF->getTarget().requiresStructuredCFG())
return nullptr;
// We may need to update this's terminator, but we can't do that if
// AnalyzeBranch fails. If this uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*this, TBB, FBB, Cond))
return nullptr;
// Avoid bugpoint weirdness: A block may end with a conditional branch but
// jumps to the same MBB is either case. We have duplicate CFG edges in that
// case that we can't handle. Since this never happens in properly optimized
// code, just skip those edges.
if (TBB && TBB == FBB) {
DEBUG(dbgs() << "Won't split critical edge after degenerate BB#"
<< getNumber() << '\n');
return nullptr;
}
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(std::next(MachineFunction::iterator(this)), NMBB);
DEBUG(dbgs() << "Splitting critical edge:"
" BB#" << getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << Succ->getNumber() << '\n');
LiveIntervals *LIS = P->getAnalysisIfAvailable<LiveIntervals>();
SlotIndexes *Indexes = P->getAnalysisIfAvailable<SlotIndexes>();
if (LIS)
LIS->insertMBBInMaps(NMBB);
else if (Indexes)
Indexes->insertMBBInMaps(NMBB);
// On some targets like Mips, branches may kill virtual registers. Make sure
// that LiveVariables is properly updated after updateTerminator replaces the
// terminators.
LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>();
// Collect a list of virtual registers killed by the terminators.
SmallVector<unsigned, 4> KilledRegs;
if (LV)
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || OI->getReg() == 0 ||
!OI->isUse() || !OI->isKill() || OI->isUndef())
continue;
unsigned Reg = OI->getReg();
if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
LV->getVarInfo(Reg).removeKill(MI)) {
KilledRegs.push_back(Reg);
DEBUG(dbgs() << "Removing terminator kill: " << *MI);
OI->setIsKill(false);
}
}
}
SmallVector<unsigned, 4> UsedRegs;
if (LIS) {
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || OI->getReg() == 0)
continue;
unsigned Reg = OI->getReg();
if (std::find(UsedRegs.begin(), UsedRegs.end(), Reg) == UsedRegs.end())
UsedRegs.push_back(Reg);
}
}
}
ReplaceUsesOfBlockWith(Succ, NMBB);
// If updateTerminator() removes instructions, we need to remove them from
// SlotIndexes.
SmallVector<MachineInstr*, 4> Terminators;
if (Indexes) {
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I)
Terminators.push_back(I);
}
updateTerminator();
if (Indexes) {
SmallVector<MachineInstr*, 4> NewTerminators;
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I)
NewTerminators.push_back(I);
for (SmallVectorImpl<MachineInstr*>::iterator I = Terminators.begin(),
E = Terminators.end(); I != E; ++I) {
if (std::find(NewTerminators.begin(), NewTerminators.end(), *I) ==
NewTerminators.end())
Indexes->removeMachineInstrFromMaps(*I);
}
}
// Insert unconditional "jump Succ" instruction in NMBB if necessary.
NMBB->addSuccessor(Succ);
if (!NMBB->isLayoutSuccessor(Succ)) {
Cond.clear();
MF->getSubtarget().getInstrInfo()->InsertBranch(*NMBB, Succ, nullptr, Cond,
dl);
if (Indexes) {
for (instr_iterator I = NMBB->instr_begin(), E = NMBB->instr_end();
I != E; ++I) {
// Some instructions may have been moved to NMBB by updateTerminator(),
// so we first remove any instruction that already has an index.
if (Indexes->hasIndex(I))
Indexes->removeMachineInstrFromMaps(I);
Indexes->insertMachineInstrInMaps(I);
}
}
}
// Fix PHI nodes in Succ so they refer to NMBB instead of this
for (MachineBasicBlock::instr_iterator
i = Succ->instr_begin(),e = Succ->instr_end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == this)
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
E = Succ->livein_end(); I != E; ++I)
NMBB->addLiveIn(*I);
// Update LiveVariables.
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
if (LV) {
// Restore kills of virtual registers that were killed by the terminators.
while (!KilledRegs.empty()) {
unsigned Reg = KilledRegs.pop_back_val();
for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) {
if (!(--I)->addRegisterKilled(Reg, TRI, /* addIfNotFound= */ false))
continue;
if (TargetRegisterInfo::isVirtualRegister(Reg))
LV->getVarInfo(Reg).Kills.push_back(I);
DEBUG(dbgs() << "Restored terminator kill: " << *I);
break;
}
}
// Update relevant live-through information.
LV->addNewBlock(NMBB, this, Succ);
}
if (LIS) {
// After splitting the edge and updating SlotIndexes, live intervals may be
// in one of two situations, depending on whether this block was the last in
// the function. If the original block was the last in the function, all live
// intervals will end prior to the beginning of the new split block. If the
// original block was not at the end of the function, all live intervals will
// extend to the end of the new split block.
bool isLastMBB =
std::next(MachineFunction::iterator(NMBB)) == getParent()->end();
SlotIndex StartIndex = Indexes->getMBBEndIdx(this);
SlotIndex PrevIndex = StartIndex.getPrevSlot();
SlotIndex EndIndex = Indexes->getMBBEndIdx(NMBB);
// Find the registers used from NMBB in PHIs in Succ.
SmallSet<unsigned, 8> PHISrcRegs;
for (MachineBasicBlock::instr_iterator
I = Succ->instr_begin(), E = Succ->instr_end();
I != E && I->isPHI(); ++I) {
for (unsigned ni = 1, ne = I->getNumOperands(); ni != ne; ni += 2) {
if (I->getOperand(ni+1).getMBB() == NMBB) {
MachineOperand &MO = I->getOperand(ni);
unsigned Reg = MO.getReg();
PHISrcRegs.insert(Reg);
if (MO.isUndef())
continue;
LiveInterval &LI = LIS->getInterval(Reg);
VNInfo *VNI = LI.getVNInfoAt(PrevIndex);
assert(VNI && "PHI sources should be live out of their predecessors.");
LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI));
}
}
}
MachineRegisterInfo *MRI = &getParent()->getRegInfo();
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
if (PHISrcRegs.count(Reg) || !LIS->hasInterval(Reg))
continue;
LiveInterval &LI = LIS->getInterval(Reg);
if (!LI.liveAt(PrevIndex))
continue;
bool isLiveOut = LI.liveAt(LIS->getMBBStartIdx(Succ));
if (isLiveOut && isLastMBB) {
VNInfo *VNI = LI.getVNInfoAt(PrevIndex);
assert(VNI && "LiveInterval should have VNInfo where it is live.");
LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI));
} else if (!isLiveOut && !isLastMBB) {
LI.removeSegment(StartIndex, EndIndex);
}
}
// Update all intervals for registers whose uses may have been modified by
// updateTerminator().
LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs);
}
if (MachineDominatorTree *MDT =
P->getAnalysisIfAvailable<MachineDominatorTree>())
MDT->recordSplitCriticalEdge(this, Succ, NMBB);
if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>())
if (MachineLoop *TIL = MLI->getLoopFor(this)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) {
if (TIL == DestLoop) {
// Both in the same loop, the NMBB joins loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (TIL->contains(DestLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
TIL->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (DestLoop->contains(TIL)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(DestLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (MachineLoop *P = DestLoop->getParentLoop())
P->addBasicBlockToLoop(NMBB, MLI->getBase());
}
}
}
return NMBB;
}
std::auto_ptr<PBQPRAProblem> PBQPBuilder::build(MachineFunction *mf,
const LiveIntervals *lis,
const MachineLoopInfo *loopInfo,
const RegSet &vregs) {
typedef std::vector<const LiveInterval*> LIVector;
ArrayRef<SlotIndex> regMaskSlots = lis->getRegMaskSlots();
MachineRegisterInfo *mri = &mf->getRegInfo();
const TargetRegisterInfo *tri = mf->getTarget().getRegisterInfo();
std::auto_ptr<PBQPRAProblem> p(new PBQPRAProblem());
PBQP::Graph &g = p->getGraph();
RegSet pregs;
// Collect the set of preg intervals, record that they're used in the MF.
for (LiveIntervals::const_iterator itr = lis->begin(), end = lis->end();
itr != end; ++itr) {
if (TargetRegisterInfo::isPhysicalRegister(itr->first)) {
pregs.insert(itr->first);
mri->setPhysRegUsed(itr->first);
}
}
BitVector reservedRegs = tri->getReservedRegs(*mf);
// Iterate over vregs.
for (RegSet::const_iterator vregItr = vregs.begin(), vregEnd = vregs.end();
vregItr != vregEnd; ++vregItr) {
unsigned vreg = *vregItr;
const TargetRegisterClass *trc = mri->getRegClass(vreg);
const LiveInterval *vregLI = &lis->getInterval(vreg);
// Compute an initial allowed set for the current vreg.
typedef std::vector<unsigned> VRAllowed;
VRAllowed vrAllowed;
ArrayRef<uint16_t> rawOrder = trc->getRawAllocationOrder(*mf);
for (unsigned i = 0; i != rawOrder.size(); ++i) {
unsigned preg = rawOrder[i];
if (!reservedRegs.test(preg)) {
vrAllowed.push_back(preg);
}
}
RegSet overlappingPRegs;
// Record physical registers whose ranges overlap.
for (RegSet::const_iterator pregItr = pregs.begin(),
pregEnd = pregs.end();
pregItr != pregEnd; ++pregItr) {
unsigned preg = *pregItr;
const LiveInterval *pregLI = &lis->getInterval(preg);
if (pregLI->empty()) {
continue;
}
if (vregLI->overlaps(*pregLI))
overlappingPRegs.insert(preg);
}
// Record any overlaps with regmask operands.
BitVector regMaskOverlaps(tri->getNumRegs());
for (ArrayRef<SlotIndex>::iterator rmItr = regMaskSlots.begin(),
rmEnd = regMaskSlots.end();
rmItr != rmEnd; ++rmItr) {
SlotIndex rmIdx = *rmItr;
if (vregLI->liveAt(rmIdx)) {
MachineInstr *rmMI = lis->getInstructionFromIndex(rmIdx);
const uint32_t* regMask = 0;
for (MachineInstr::mop_iterator mopItr = rmMI->operands_begin(),
mopEnd = rmMI->operands_end();
mopItr != mopEnd; ++mopItr) {
if (mopItr->isRegMask()) {
regMask = mopItr->getRegMask();
break;
}
}
assert(regMask != 0 && "Couldn't find register mask.");
regMaskOverlaps.setBitsNotInMask(regMask);
}
}
for (unsigned preg = 0; preg < tri->getNumRegs(); ++preg) {
if (regMaskOverlaps.test(preg))
overlappingPRegs.insert(preg);
}
for (RegSet::const_iterator pregItr = overlappingPRegs.begin(),
pregEnd = overlappingPRegs.end();
pregItr != pregEnd; ++pregItr) {
unsigned preg = *pregItr;
// Remove the register from the allowed set.
VRAllowed::iterator eraseItr =
std::find(vrAllowed.begin(), vrAllowed.end(), preg);
if (eraseItr != vrAllowed.end()) {
vrAllowed.erase(eraseItr);
}
// Also remove any aliases.
const uint16_t *aliasItr = tri->getAliasSet(preg);
if (aliasItr != 0) {
for (; *aliasItr != 0; ++aliasItr) {
VRAllowed::iterator eraseItr =
std::find(vrAllowed.begin(), vrAllowed.end(), *aliasItr);
if (eraseItr != vrAllowed.end()) {
vrAllowed.erase(eraseItr);
}
}
}
}
// Construct the node.
PBQP::Graph::NodeItr node =
g.addNode(PBQP::Vector(vrAllowed.size() + 1, 0));
// Record the mapping and allowed set in the problem.
p->recordVReg(vreg, node, vrAllowed.begin(), vrAllowed.end());
PBQP::PBQPNum spillCost = (vregLI->weight != 0.0) ?
vregLI->weight : std::numeric_limits<PBQP::PBQPNum>::min();
addSpillCosts(g.getNodeCosts(node), spillCost);
}
for (RegSet::const_iterator vr1Itr = vregs.begin(), vrEnd = vregs.end();
vr1Itr != vrEnd; ++vr1Itr) {
unsigned vr1 = *vr1Itr;
const LiveInterval &l1 = lis->getInterval(vr1);
const PBQPRAProblem::AllowedSet &vr1Allowed = p->getAllowedSet(vr1);
for (RegSet::const_iterator vr2Itr = llvm::next(vr1Itr);
vr2Itr != vrEnd; ++vr2Itr) {
unsigned vr2 = *vr2Itr;
const LiveInterval &l2 = lis->getInterval(vr2);
const PBQPRAProblem::AllowedSet &vr2Allowed = p->getAllowedSet(vr2);
assert(!l2.empty() && "Empty interval in vreg set?");
if (l1.overlaps(l2)) {
PBQP::Graph::EdgeItr edge =
g.addEdge(p->getNodeForVReg(vr1), p->getNodeForVReg(vr2),
PBQP::Matrix(vr1Allowed.size()+1, vr2Allowed.size()+1, 0));
addInterferenceCosts(g.getEdgeCosts(edge), vr1Allowed, vr2Allowed, tri);
}
}
}
return p;
}
MachineBasicBlock *
MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
MachineFunction *MF = getParent();
DebugLoc dl; // FIXME: this is nowhere
// We may need to update this's terminator, but we can't do that if
// AnalyzeBranch fails. If this uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
MachineBasicBlock *TBB = 0, *FBB = 0;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*this, TBB, FBB, Cond))
return NULL;
// Avoid bugpoint weirdness: A block may end with a conditional branch but
// jumps to the same MBB is either case. We have duplicate CFG edges in that
// case that we can't handle. Since this never happens in properly optimized
// code, just skip those edges.
if (TBB && TBB == FBB) {
DEBUG(dbgs() << "Won't split critical edge after degenerate BB#"
<< getNumber() << '\n');
return NULL;
}
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(llvm::next(MachineFunction::iterator(this)), NMBB);
DEBUG(dbgs() << "Splitting critical edge:"
" BB#" << getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << Succ->getNumber() << '\n');
// On some targets like Mips, branches may kill virtual registers. Make sure
// that LiveVariables is properly updated after updateTerminator replaces the
// terminators.
LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>();
// Collect a list of virtual registers killed by the terminators.
SmallVector<unsigned, 4> KilledRegs;
if (LV)
for (iterator I = getFirstTerminator(), E = end(); I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || !OI->isUse() || !OI->isKill() || OI->isUndef())
continue;
unsigned Reg = OI->getReg();
if (TargetRegisterInfo::isVirtualRegister(Reg) &&
LV->getVarInfo(Reg).removeKill(MI)) {
KilledRegs.push_back(Reg);
DEBUG(dbgs() << "Removing terminator kill: " << *MI);
OI->setIsKill(false);
}
}
}
ReplaceUsesOfBlockWith(Succ, NMBB);
updateTerminator();
// Insert unconditional "jump Succ" instruction in NMBB if necessary.
NMBB->addSuccessor(Succ);
if (!NMBB->isLayoutSuccessor(Succ)) {
Cond.clear();
MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl);
}
// Fix PHI nodes in Succ so they refer to NMBB instead of this
for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == this)
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
E = Succ->livein_end(); I != E; ++I)
NMBB->addLiveIn(*I);
// Update LiveVariables.
if (LV) {
// Restore kills of virtual registers that were killed by the terminators.
while (!KilledRegs.empty()) {
unsigned Reg = KilledRegs.pop_back_val();
for (iterator I = end(), E = begin(); I != E;) {
if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false))
continue;
LV->getVarInfo(Reg).Kills.push_back(I);
DEBUG(dbgs() << "Restored terminator kill: " << *I);
break;
}
}
// Update relevant live-through information.
LV->addNewBlock(NMBB, this, Succ);
}
if (MachineDominatorTree *MDT =
P->getAnalysisIfAvailable<MachineDominatorTree>()) {
// Update dominator information.
MachineDomTreeNode *SucccDTNode = MDT->getNode(Succ);
bool IsNewIDom = true;
for (const_pred_iterator PI = Succ->pred_begin(), E = Succ->pred_end();
PI != E; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB == NMBB)
continue;
if (!MDT->dominates(SucccDTNode, MDT->getNode(PredBB))) {
IsNewIDom = false;
break;
}
}
// We know "this" dominates the newly created basic block.
MachineDomTreeNode *NewDTNode = MDT->addNewBlock(NMBB, this);
// If all the other predecessors of "Succ" are dominated by "Succ" itself
// then the new block is the new immediate dominator of "Succ". Otherwise,
// the new block doesn't dominate anything.
if (IsNewIDom)
MDT->changeImmediateDominator(SucccDTNode, NewDTNode);
}
if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>())
if (MachineLoop *TIL = MLI->getLoopFor(this)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) {
if (TIL == DestLoop) {
// Both in the same loop, the NMBB joins loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (TIL->contains(DestLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
TIL->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (DestLoop->contains(TIL)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(DestLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (MachineLoop *P = DestLoop->getParentLoop())
P->addBasicBlockToLoop(NMBB, MLI->getBase());
}
}
}
return NMBB;
}
/// Find all live intervals that need to shrink, then remove the instruction.
void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink) {
assert(MI->allDefsAreDead() && "Def isn't really dead");
SlotIndex Idx = LIS.getInstructionIndex(MI).getRegSlot();
// Never delete a bundled instruction.
if (MI->isBundled()) {
return;
}
// Never delete inline asm.
if (MI->isInlineAsm()) {
DEBUG(dbgs() << "Won't delete: " << Idx << '\t' << *MI);
return;
}
// Use the same criteria as DeadMachineInstructionElim.
bool SawStore = false;
if (!MI->isSafeToMove(&TII, 0, SawStore)) {
DEBUG(dbgs() << "Can't delete: " << Idx << '\t' << *MI);
return;
}
DEBUG(dbgs() << "Deleting dead def " << Idx << '\t' << *MI);
// Collect virtual registers to be erased after MI is gone.
SmallVector<unsigned, 8> RegsToErase;
bool ReadsPhysRegs = false;
// Check for live intervals that may shrink
for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
if (!MOI->isReg())
continue;
unsigned Reg = MOI->getReg();
if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
// Check if MI reads any unreserved physregs.
if (Reg && MOI->readsReg() && !MRI.isReserved(Reg))
ReadsPhysRegs = true;
else if (MOI->isDef()) {
for (MCRegUnitIterator Units(Reg, MRI.getTargetRegisterInfo());
Units.isValid(); ++Units) {
if (LiveRange *LR = LIS.getCachedRegUnit(*Units)) {
if (VNInfo *VNI = LR->getVNInfoAt(Idx))
LR->removeValNo(VNI);
}
}
}
continue;
}
LiveInterval &LI = LIS.getInterval(Reg);
// Shrink read registers, unless it is likely to be expensive and
// unlikely to change anything. We typically don't want to shrink the
// PIC base register that has lots of uses everywhere.
// Always shrink COPY uses that probably come from live range splitting.
if (MI->readsVirtualRegister(Reg) &&
(MI->isCopy() || MOI->isDef() || MRI.hasOneNonDBGUse(Reg) ||
LI.Query(Idx).isKill()))
ToShrink.insert(&LI);
// Remove defined value.
if (MOI->isDef()) {
if (VNInfo *VNI = LI.getVNInfoAt(Idx)) {
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI.reg);
LI.removeValNo(VNI);
if (LI.empty())
RegsToErase.push_back(Reg);
}
}
}
// Currently, we don't support DCE of physreg live ranges. If MI reads
// any unreserved physregs, don't erase the instruction, but turn it into
// a KILL instead. This way, the physreg live ranges don't end up
// dangling.
// FIXME: It would be better to have something like shrinkToUses() for
// physregs. That could potentially enable more DCE and it would free up
// the physreg. It would not happen often, though.
if (ReadsPhysRegs) {
MI->setDesc(TII.get(TargetOpcode::KILL));
// Remove all operands that aren't physregs.
for (unsigned i = MI->getNumOperands(); i; --i) {
const MachineOperand &MO = MI->getOperand(i-1);
if (MO.isReg() && TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
continue;
MI->RemoveOperand(i-1);
}
DEBUG(dbgs() << "Converted physregs to:\t" << *MI);
} else {
if (TheDelegate)
TheDelegate->LRE_WillEraseInstruction(MI);
LIS.RemoveMachineInstrFromMaps(MI);
MI->eraseFromParent();
++NumDCEDeleted;
}
// Erase any virtregs that are now empty and unused. There may be <undef>
// uses around. Keep the empty live range in that case.
for (unsigned i = 0, e = RegsToErase.size(); i != e; ++i) {
unsigned Reg = RegsToErase[i];
if (LIS.hasInterval(Reg) && MRI.reg_nodbg_empty(Reg)) {
ToShrink.remove(&LIS.getInterval(Reg));
eraseVirtReg(Reg);
}
}
}
void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,
ArrayRef<unsigned> RegsBeingSpilled) {
SetVector<LiveInterval*,
SmallVector<LiveInterval*, 8>,
SmallPtrSet<LiveInterval*, 8> > ToShrink;
for (;;) {
// Erase all dead defs.
while (!Dead.empty()) {
MachineInstr *MI = Dead.pop_back_val();
assert(MI->allDefsAreDead() && "Def isn't really dead");
SlotIndex Idx = LIS.getInstructionIndex(MI).getRegSlot();
// Never delete inline asm.
if (MI->isInlineAsm()) {
DEBUG(dbgs() << "Won't delete: " << Idx << '\t' << *MI);
continue;
}
// Use the same criteria as DeadMachineInstructionElim.
bool SawStore = false;
if (!MI->isSafeToMove(&TII, 0, SawStore)) {
DEBUG(dbgs() << "Can't delete: " << Idx << '\t' << *MI);
continue;
}
DEBUG(dbgs() << "Deleting dead def " << Idx << '\t' << *MI);
// Check for live intervals that may shrink
for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
if (!MOI->isReg())
continue;
unsigned Reg = MOI->getReg();
if (!TargetRegisterInfo::isVirtualRegister(Reg))
continue;
LiveInterval &LI = LIS.getInterval(Reg);
// Shrink read registers, unless it is likely to be expensive and
// unlikely to change anything. We typically don't want to shrink the
// PIC base register that has lots of uses everywhere.
// Always shrink COPY uses that probably come from live range splitting.
if (MI->readsVirtualRegister(Reg) &&
(MI->isCopy() || MOI->isDef() || MRI.hasOneNonDBGUse(Reg) ||
LI.killedAt(Idx)))
ToShrink.insert(&LI);
// Remove defined value.
if (MOI->isDef()) {
if (VNInfo *VNI = LI.getVNInfoAt(Idx)) {
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI.reg);
LI.removeValNo(VNI);
if (LI.empty()) {
ToShrink.remove(&LI);
eraseVirtReg(Reg);
}
}
}
}
if (TheDelegate)
TheDelegate->LRE_WillEraseInstruction(MI);
LIS.RemoveMachineInstrFromMaps(MI);
MI->eraseFromParent();
++NumDCEDeleted;
}
if (ToShrink.empty())
break;
// Shrink just one live interval. Then delete new dead defs.
LiveInterval *LI = ToShrink.back();
ToShrink.pop_back();
if (foldAsLoad(LI, Dead))
continue;
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI->reg);
if (!LIS.shrinkToUses(LI, &Dead))
continue;
// Don't create new intervals for a register being spilled.
// The new intervals would have to be spilled anyway so its not worth it.
// Also they currently aren't spilled so creating them and not spilling
// them results in incorrect code.
bool BeingSpilled = false;
for (unsigned i = 0, e = RegsBeingSpilled.size(); i != e; ++i) {
if (LI->reg == RegsBeingSpilled[i]) {
BeingSpilled = true;
break;
}
}
if (BeingSpilled) continue;
// LI may have been separated, create new intervals.
LI->RenumberValues(LIS);
ConnectedVNInfoEqClasses ConEQ(LIS);
unsigned NumComp = ConEQ.Classify(LI);
if (NumComp <= 1)
continue;
++NumFracRanges;
bool IsOriginal = VRM && VRM->getOriginal(LI->reg) == LI->reg;
DEBUG(dbgs() << NumComp << " components: " << *LI << '\n');
SmallVector<LiveInterval*, 8> Dups(1, LI);
for (unsigned i = 1; i != NumComp; ++i) {
Dups.push_back(&createFrom(LI->reg));
// If LI is an original interval that hasn't been split yet, make the new
// intervals their own originals instead of referring to LI. The original
// interval must contain all the split products, and LI doesn't.
if (IsOriginal)
VRM->setIsSplitFromReg(Dups.back()->reg, 0);
if (TheDelegate)
TheDelegate->LRE_DidCloneVirtReg(Dups.back()->reg, LI->reg);
}
ConEQ.Distribute(&Dups[0], MRI);
DEBUG({
for (unsigned i = 0; i != NumComp; ++i)
dbgs() << '\t' << *Dups[i] << '\n';
});
}