/// leaveIntvAfter - Leave openli after the instruction at Idx.
void SplitEditor::leaveIntvAfter(SlotIndex Idx) {
assert(openli_ && "openIntv not called before leaveIntvAfter");
const LiveRange *CurLR = curli_->getLiveRangeContaining(Idx.getDefIndex());
if (!CurLR || CurLR->end <= Idx.getBoundaryIndex()) {
DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": not live\n");
return;
}
// Was this value of curli live through openli?
if (!openli_->liveAt(CurLR->valno->def)) {
DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": using external value\n");
liveThrough_ = true;
return;
}
// We are going to insert a back copy, so we must have a dupli_.
LiveRange *DupLR = getDupLI()->getLiveRangeContaining(Idx.getDefIndex());
assert(DupLR && "dupli not live into black, but curli is?");
// Insert the COPY instruction.
MachineBasicBlock::iterator I = lis_.getInstructionFromIndex(Idx);
MachineInstr *MI = BuildMI(*I->getParent(), llvm::next(I), I->getDebugLoc(),
tii_.get(TargetOpcode::COPY), dupli_->reg)
.addReg(openli_->reg);
SlotIndex CopyIdx = lis_.InsertMachineInstrInMaps(MI).getDefIndex();
openli_->addRange(LiveRange(Idx.getDefIndex(), CopyIdx,
mapValue(CurLR->valno)));
DupLR->valno->def = CopyIdx;
DEBUG(dbgs() << " leaveIntvAfter " << Idx << ": " << *openli_ << '\n');
}
/// rewrite - after all the new live ranges have been created, rewrite
/// instructions using curli to use the new intervals.
void SplitEditor::rewrite() {
assert(!openli_ && "Previous LI not closed before rewrite");
const LiveInterval *curli = sa_.getCurLI();
for (MachineRegisterInfo::reg_iterator RI = mri_.reg_begin(curli->reg),
RE = mri_.reg_end(); RI != RE;) {
MachineOperand &MO = RI.getOperand();
MachineInstr *MI = MO.getParent();
++RI;
if (MI->isDebugValue()) {
DEBUG(dbgs() << "Zapping " << *MI);
// FIXME: We can do much better with debug values.
MO.setReg(0);
continue;
}
SlotIndex Idx = lis_.getInstructionIndex(MI);
Idx = MO.isUse() ? Idx.getUseIndex() : Idx.getDefIndex();
LiveInterval *LI = dupli_;
for (unsigned i = firstInterval, e = intervals_.size(); i != e; ++i) {
LiveInterval *testli = intervals_[i];
if (testli->liveAt(Idx)) {
LI = testli;
break;
}
}
if (LI) {
MO.setReg(LI->reg);
sa_.removeUse(MI);
DEBUG(dbgs() << " rewrite " << Idx << '\t' << *MI);
}
}
// dupli_ goes in last, after rewriting.
if (dupli_) {
if (dupli_->empty()) {
DEBUG(dbgs() << " dupli became empty?\n");
lis_.removeInterval(dupli_->reg);
dupli_ = 0;
} else {
dupli_->RenumberValues(lis_);
intervals_.push_back(dupli_);
}
}
// Calculate spill weight and allocation hints for new intervals.
VirtRegAuxInfo vrai(vrm_.getMachineFunction(), lis_, sa_.loops_);
for (unsigned i = firstInterval, e = intervals_.size(); i != e; ++i) {
LiveInterval &li = *intervals_[i];
vrai.CalculateRegClass(li.reg);
vrai.CalculateWeightAndHint(li);
DEBUG(dbgs() << " new interval " << mri_.getRegClass(li.reg)->getName()
<< ":" << li << '\n');
}
}
void ConnectedVNInfoEqClasses::Distribute(LiveInterval *LIV[],
MachineRegisterInfo &MRI) {
assert(LIV[0] && "LIV[0] must be set");
LiveInterval &LI = *LIV[0];
// Rewrite instructions.
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(LI.reg),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = RI.getOperand();
MachineInstr *MI = MO.getParent();
++RI;
if (MO.isUse() && MO.isUndef())
continue;
// DBG_VALUE instructions should have been eliminated earlier.
SlotIndex Idx = LIS.getInstructionIndex(MI);
Idx = MO.isUse() ? Idx.getUseIndex() : Idx.getDefIndex();
const VNInfo *VNI = LI.getVNInfoAt(Idx);
assert(VNI && "Interval not live at use.");
MO.setReg(LIV[getEqClass(VNI)]->reg);
}
// Move runs to new intervals.
LiveInterval::iterator J = LI.begin(), E = LI.end();
while (J != E && EqClass[J->valno->id] == 0)
++J;
for (LiveInterval::iterator I = J; I != E; ++I) {
if (unsigned eq = EqClass[I->valno->id]) {
assert((LIV[eq]->empty() || LIV[eq]->expiredAt(I->start)) &&
"New intervals should be empty");
LIV[eq]->ranges.push_back(*I);
} else
*J++ = *I;
}
LI.ranges.erase(J, E);
// Transfer VNInfos to their new owners and renumber them.
unsigned j = 0, e = LI.getNumValNums();
while (j != e && EqClass[j] == 0)
++j;
for (unsigned i = j; i != e; ++i) {
VNInfo *VNI = LI.getValNumInfo(i);
if (unsigned eq = EqClass[i]) {
VNI->id = LIV[eq]->getNumValNums();
LIV[eq]->valnos.push_back(VNI);
} else {
VNI->id = j;
LI.valnos[j++] = VNI;
}
}
LI.valnos.resize(j);
}
/// rewriteAssigned - Rewrite all uses of Edit->getReg().
void SplitEditor::rewriteAssigned(bool ExtendRanges) {
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = RI.getOperand();
MachineInstr *MI = MO.getParent();
++RI;
// LiveDebugVariables should have handled all DBG_VALUE instructions.
if (MI->isDebugValue()) {
DEBUG(dbgs() << "Zapping " << *MI);
MO.setReg(0);
continue;
}
// <undef> operands don't really read the register, so it doesn't matter
// which register we choose. When the use operand is tied to a def, we must
// use the same register as the def, so just do that always.
SlotIndex Idx = LIS.getInstructionIndex(MI);
if (MO.isDef() || MO.isUndef())
Idx = MO.isEarlyClobber() ? Idx.getUseIndex() : Idx.getDefIndex();
// Rewrite to the mapped register at Idx.
unsigned RegIdx = RegAssign.lookup(Idx);
LiveInterval *LI = Edit->get(RegIdx);
MO.setReg(LI->reg);
DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t'
<< Idx << ':' << RegIdx << '\t' << *MI);
// Extend liveness to Idx if the instruction reads reg.
if (!ExtendRanges || MO.isUndef())
continue;
// Skip instructions that don't read Reg.
if (MO.isDef()) {
if (!MO.getSubReg() && !MO.isEarlyClobber())
continue;
// We may wan't to extend a live range for a partial redef, or for a use
// tied to an early clobber.
Idx = Idx.getPrevSlot();
if (!Edit->getParent().liveAt(Idx))
continue;
} else
Idx = Idx.getUseIndex();
getLRCalc(RegIdx).extend(LI, Idx.getNextSlot(), LIS.getSlotIndexes(),
&MDT, &LIS.getVNInfoAllocator());
}
}
/// enterIntvBefore - Enter openli before the instruction at Idx. If curli is
/// not live before Idx, a COPY is not inserted.
void SplitEditor::enterIntvBefore(SlotIndex Idx) {
assert(openli_ && "openIntv not called before enterIntvBefore");
// Copy from curli_ if it is live.
if (VNInfo *CurVNI = curli_->getVNInfoAt(Idx.getUseIndex())) {
MachineInstr *MI = lis_.getInstructionFromIndex(Idx);
assert(MI && "enterIntvBefore called with invalid index");
VNInfo *VNI = insertCopy(*openli_, *MI->getParent(), MI);
openli_->addRange(LiveRange(VNI->def, Idx.getDefIndex(), VNI));
// Make sure CurVNI is properly mapped.
VNInfo *&mapVNI = valueMap_[CurVNI];
// We dont have SSA update yet, so only one entry per value is allowed.
assert(!mapVNI && "enterIntvBefore called more than once for the same value");
mapVNI = VNI;
}
DEBUG(dbgs() << " enterIntvBefore " << Idx << ": " << *openli_ << '\n');
}
void
UserValue::addDefsFromCopies(LiveInterval *LI, unsigned LocNo,
const SmallVectorImpl<SlotIndex> &Kills,
SmallVectorImpl<std::pair<SlotIndex, unsigned> > &NewDefs,
MachineRegisterInfo &MRI, LiveIntervals &LIS) {
if (Kills.empty())
return;
// Don't track copies from physregs, there are too many uses.
if (!TargetRegisterInfo::isVirtualRegister(LI->reg))
return;
// Collect all the (vreg, valno) pairs that are copies of LI.
SmallVector<std::pair<LiveInterval*, const VNInfo*>, 8> CopyValues;
for (MachineRegisterInfo::use_nodbg_iterator
UI = MRI.use_nodbg_begin(LI->reg),
UE = MRI.use_nodbg_end(); UI != UE; ++UI) {
// Copies of the full value.
if (UI.getOperand().getSubReg() || !UI->isCopy())
continue;
MachineInstr *MI = &*UI;
unsigned DstReg = MI->getOperand(0).getReg();
// Don't follow copies to physregs. These are usually setting up call
// arguments, and the argument registers are always call clobbered. We are
// better off in the source register which could be a callee-saved register,
// or it could be spilled.
if (!TargetRegisterInfo::isVirtualRegister(DstReg))
continue;
// Is LocNo extended to reach this copy? If not, another def may be blocking
// it, or we are looking at a wrong value of LI.
SlotIndex Idx = LIS.getInstructionIndex(MI);
LocMap::iterator I = locInts.find(Idx.getUseIndex());
if (!I.valid() || I.value() != LocNo)
continue;
if (!LIS.hasInterval(DstReg))
continue;
LiveInterval *DstLI = &LIS.getInterval(DstReg);
const VNInfo *DstVNI = DstLI->getVNInfoAt(Idx.getDefIndex());
assert(DstVNI && DstVNI->def == Idx.getDefIndex() && "Bad copy value");
CopyValues.push_back(std::make_pair(DstLI, DstVNI));
}
if (CopyValues.empty())
return;
DEBUG(dbgs() << "Got " << CopyValues.size() << " copies of " << *LI << '\n');
// Try to add defs of the copied values for each kill point.
for (unsigned i = 0, e = Kills.size(); i != e; ++i) {
SlotIndex Idx = Kills[i];
for (unsigned j = 0, e = CopyValues.size(); j != e; ++j) {
LiveInterval *DstLI = CopyValues[j].first;
const VNInfo *DstVNI = CopyValues[j].second;
if (DstLI->getVNInfoAt(Idx) != DstVNI)
continue;
// Check that there isn't already a def at Idx
LocMap::iterator I = locInts.find(Idx);
if (I.valid() && I.start() <= Idx)
continue;
DEBUG(dbgs() << "Kill at " << Idx << " covered by valno #"
<< DstVNI->id << " in " << *DstLI << '\n');
MachineInstr *CopyMI = LIS.getInstructionFromIndex(DstVNI->def);
assert(CopyMI && CopyMI->isCopy() && "Bad copy value");
unsigned LocNo = getLocationNo(CopyMI->getOperand(0));
I.insert(Idx, Idx.getNextSlot(), LocNo);
NewDefs.push_back(std::make_pair(Idx, LocNo));
break;
}
}
}
void
MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
const MachineInstr *MI = MO->getParent();
const MCInstrDesc &MCID = MI->getDesc();
const MCOperandInfo &MCOI = MCID.OpInfo[MONum];
// The first MCID.NumDefs operands must be explicit register defines
if (MONum < MCID.getNumDefs()) {
if (!MO->isReg())
report("Explicit definition must be a register", MO, MONum);
else if (!MO->isDef())
report("Explicit definition marked as use", MO, MONum);
else if (MO->isImplicit())
report("Explicit definition marked as implicit", MO, MONum);
} else if (MONum < MCID.getNumOperands()) {
// Don't check if it's the last operand in a variadic instruction. See,
// e.g., LDM_RET in the arm back end.
if (MO->isReg() &&
!(MCID.isVariadic() && MONum == MCID.getNumOperands()-1)) {
if (MO->isDef() && !MCOI.isOptionalDef())
report("Explicit operand marked as def", MO, MONum);
if (MO->isImplicit())
report("Explicit operand marked as implicit", MO, MONum);
}
} else {
// ARM adds %reg0 operands to indicate predicates. We'll allow that.
if (MO->isReg() && !MO->isImplicit() && !MCID.isVariadic() && MO->getReg())
report("Extra explicit operand on non-variadic instruction", MO, MONum);
}
switch (MO->getType()) {
case MachineOperand::MO_Register: {
const unsigned Reg = MO->getReg();
if (!Reg)
return;
// Check Live Variables.
if (MI->isDebugValue()) {
// Liveness checks are not valid for debug values.
} else if (MO->isUse() && !MO->isUndef()) {
regsLiveInButUnused.erase(Reg);
bool isKill = false;
unsigned defIdx;
if (MI->isRegTiedToDefOperand(MONum, &defIdx)) {
// A two-addr use counts as a kill if use and def are the same.
unsigned DefReg = MI->getOperand(defIdx).getReg();
if (Reg == DefReg)
isKill = true;
else if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
report("Two-address instruction operands must be identical",
MO, MONum);
}
} else
isKill = MO->isKill();
if (isKill)
addRegWithSubRegs(regsKilled, Reg);
// Check that LiveVars knows this kill.
if (LiveVars && TargetRegisterInfo::isVirtualRegister(Reg) &&
MO->isKill()) {
LiveVariables::VarInfo &VI = LiveVars->getVarInfo(Reg);
if (std::find(VI.Kills.begin(),
VI.Kills.end(), MI) == VI.Kills.end())
report("Kill missing from LiveVariables", MO, MONum);
}
// Check LiveInts liveness and kill.
if (TargetRegisterInfo::isVirtualRegister(Reg) &&
LiveInts && !LiveInts->isNotInMIMap(MI)) {
SlotIndex UseIdx = LiveInts->getInstructionIndex(MI).getUseIndex();
if (LiveInts->hasInterval(Reg)) {
const LiveInterval &LI = LiveInts->getInterval(Reg);
if (!LI.liveAt(UseIdx)) {
report("No live range at use", MO, MONum);
*OS << UseIdx << " is not live in " << LI << '\n';
}
// Check for extra kill flags.
// Note that we allow missing kill flags for now.
if (MO->isKill() && !LI.killedAt(UseIdx.getDefIndex())) {
report("Live range continues after kill flag", MO, MONum);
*OS << "Live range: " << LI << '\n';
}
} else {
report("Virtual register has no Live interval", MO, MONum);
}
}
// Use of a dead register.
if (!regsLive.count(Reg)) {
if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
// Reserved registers may be used even when 'dead'.
if (!isReserved(Reg))
report("Using an undefined physical register", MO, MONum);
} else {
BBInfo &MInfo = MBBInfoMap[MI->getParent()];
// We don't know which virtual registers are live in, so only complain
// if vreg was killed in this MBB. Otherwise keep track of vregs that
// must be live in. PHI instructions are handled separately.
if (MInfo.regsKilled.count(Reg))
report("Using a killed virtual register", MO, MONum);
else if (!MI->isPHI())
MInfo.vregsLiveIn.insert(std::make_pair(Reg, MI));
}
}
} else if (MO->isDef()) {
// Register defined.
// TODO: verify that earlyclobber ops are not used.
if (MO->isDead())
addRegWithSubRegs(regsDead, Reg);
else
addRegWithSubRegs(regsDefined, Reg);
// Verify SSA form.
if (MRI->isSSA() && TargetRegisterInfo::isVirtualRegister(Reg) &&
llvm::next(MRI->def_begin(Reg)) != MRI->def_end())
report("Multiple virtual register defs in SSA form", MO, MONum);
// Check LiveInts for a live range, but only for virtual registers.
if (LiveInts && TargetRegisterInfo::isVirtualRegister(Reg) &&
!LiveInts->isNotInMIMap(MI)) {
SlotIndex DefIdx = LiveInts->getInstructionIndex(MI).getDefIndex();
if (LiveInts->hasInterval(Reg)) {
const LiveInterval &LI = LiveInts->getInterval(Reg);
if (const VNInfo *VNI = LI.getVNInfoAt(DefIdx)) {
assert(VNI && "NULL valno is not allowed");
if (VNI->def != DefIdx && !MO->isEarlyClobber()) {
report("Inconsistent valno->def", MO, MONum);
*OS << "Valno " << VNI->id << " is not defined at "
<< DefIdx << " in " << LI << '\n';
}
} else {
report("No live range at def", MO, MONum);
*OS << DefIdx << " is not live in " << LI << '\n';
}
} else {
report("Virtual register has no Live interval", MO, MONum);
}
}
}
// Check register classes.
if (MONum < MCID.getNumOperands() && !MO->isImplicit()) {
unsigned SubIdx = MO->getSubReg();
if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
if (SubIdx) {
report("Illegal subregister index for physical register", MO, MONum);
return;
}
if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) {
if (!DRC->contains(Reg)) {
report("Illegal physical register for instruction", MO, MONum);
*OS << TRI->getName(Reg) << " is not a "
<< DRC->getName() << " register.\n";
}
}
} else {
// Virtual register.
const TargetRegisterClass *RC = MRI->getRegClass(Reg);
if (SubIdx) {
const TargetRegisterClass *SRC =
TRI->getSubClassWithSubReg(RC, SubIdx);
if (!SRC) {
report("Invalid subregister index for virtual register", MO, MONum);
*OS << "Register class " << RC->getName()
<< " does not support subreg index " << SubIdx << "\n";
return;
}
if (RC != SRC) {
report("Invalid register class for subregister index", MO, MONum);
*OS << "Register class " << RC->getName()
<< " does not fully support subreg index " << SubIdx << "\n";
return;
}
}
if (const TargetRegisterClass *DRC = TII->getRegClass(MCID,MONum,TRI)) {
if (SubIdx) {
const TargetRegisterClass *SuperRC =
TRI->getLargestLegalSuperClass(RC);
if (!SuperRC) {
report("No largest legal super class exists.", MO, MONum);
return;
}
DRC = TRI->getMatchingSuperRegClass(SuperRC, DRC, SubIdx);
if (!DRC) {
report("No matching super-reg register class.", MO, MONum);
return;
}
}
if (!RC->hasSuperClassEq(DRC)) {
report("Illegal virtual register for instruction", MO, MONum);
*OS << "Expected a " << DRC->getName() << " register, but got a "
<< RC->getName() << " register\n";
}
}
}
}
break;
}
case MachineOperand::MO_MachineBasicBlock:
if (MI->isPHI() && !MO->getMBB()->isSuccessor(MI->getParent()))
report("PHI operand is not in the CFG", MO, MONum);
break;
case MachineOperand::MO_FrameIndex:
if (LiveStks && LiveStks->hasInterval(MO->getIndex()) &&
LiveInts && !LiveInts->isNotInMIMap(MI)) {
LiveInterval &LI = LiveStks->getInterval(MO->getIndex());
SlotIndex Idx = LiveInts->getInstructionIndex(MI);
if (MCID.mayLoad() && !LI.liveAt(Idx.getUseIndex())) {
report("Instruction loads from dead spill slot", MO, MONum);
*OS << "Live stack: " << LI << '\n';
}
if (MCID.mayStore() && !LI.liveAt(Idx.getDefIndex())) {
report("Instruction stores to dead spill slot", MO, MONum);
*OS << "Live stack: " << LI << '\n';
}
}
break;
default:
break;
}
}
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &Fn) {
LiveIntervals& LI = getAnalysis<LiveIntervals>();
// Compute DFS numbers of each block
computeDFS(Fn);
// Determine which phi node operands need copies
for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I)
if (!I->empty() && I->begin()->isPHI())
processBlock(I);
// Break interferences where two different phis want to coalesce
// in the same register.
std::set<unsigned> seen;
typedef std::map<unsigned, std::map<unsigned, MachineBasicBlock*> >
RenameSetType;
for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end();
I != E; ++I) {
for (std::map<unsigned, MachineBasicBlock*>::iterator
OI = I->second.begin(), OE = I->second.end(); OI != OE; ) {
if (!seen.count(OI->first)) {
seen.insert(OI->first);
++OI;
} else {
Waiting[OI->second].insert(std::make_pair(OI->first, I->first));
unsigned reg = OI->first;
++OI;
I->second.erase(reg);
DEBUG(dbgs() << "Removing Renaming: " << reg << " -> " << I->first
<< "\n");
}
}
}
// Insert copies
// FIXME: This process should probably preserve LiveIntervals
SmallPtrSet<MachineBasicBlock*, 16> visited;
MachineDominatorTree& MDT = getAnalysis<MachineDominatorTree>();
InsertCopies(MDT.getRootNode(), visited);
// Perform renaming
for (RenameSetType::iterator I = RenameSets.begin(), E = RenameSets.end();
I != E; ++I)
while (I->second.size()) {
std::map<unsigned, MachineBasicBlock*>::iterator SI = I->second.begin();
DEBUG(dbgs() << "Renaming: " << SI->first << " -> " << I->first << "\n");
if (SI->first != I->first) {
if (mergeLiveIntervals(I->first, SI->first)) {
Fn.getRegInfo().replaceRegWith(SI->first, I->first);
if (RenameSets.count(SI->first)) {
I->second.insert(RenameSets[SI->first].begin(),
RenameSets[SI->first].end());
RenameSets.erase(SI->first);
}
} else {
// Insert a last-minute copy if a conflict was detected.
const TargetInstrInfo *TII = Fn.getTarget().getInstrInfo();
const TargetRegisterClass *RC = Fn.getRegInfo().getRegClass(I->first);
TII->copyRegToReg(*SI->second, SI->second->getFirstTerminator(),
I->first, SI->first, RC, RC, DebugLoc());
LI.renumber();
LiveInterval& Int = LI.getOrCreateInterval(I->first);
SlotIndex instrIdx =
LI.getInstructionIndex(--SI->second->getFirstTerminator());
if (Int.liveAt(instrIdx.getDefIndex()))
Int.removeRange(instrIdx.getDefIndex(),
LI.getMBBEndIdx(SI->second).getNextSlot(), true);
LiveRange R = LI.addLiveRangeToEndOfBlock(I->first,
--SI->second->getFirstTerminator());
R.valno->setCopy(--SI->second->getFirstTerminator());
R.valno->def = instrIdx.getDefIndex();
DEBUG(dbgs() << "Renaming failed: " << SI->first << " -> "
<< I->first << "\n");
}
}
LiveInterval& Int = LI.getOrCreateInterval(I->first);
const LiveRange* LR =
Int.getLiveRangeContaining(LI.getMBBEndIdx(SI->second));
LR->valno->setHasPHIKill(true);
I->second.erase(SI->first);
}
// Remove PHIs
std::vector<MachineInstr*> phis;
for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) {
for (MachineBasicBlock::iterator BI = I->begin(), BE = I->end();
BI != BE; ++BI)
if (BI->isPHI())
phis.push_back(BI);
}
for (std::vector<MachineInstr*>::iterator I = phis.begin(), E = phis.end();
I != E; ) {
MachineInstr* PInstr = *(I++);
// If this is a dead PHI node, then remove it from LiveIntervals.
unsigned DestReg = PInstr->getOperand(0).getReg();
LiveInterval& PI = LI.getInterval(DestReg);
if (PInstr->registerDefIsDead(DestReg)) {
if (PI.containsOneValue()) {
LI.removeInterval(DestReg);
} else {
SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex();
PI.removeRange(*PI.getLiveRangeContaining(idx), true);
}
} else {
// Trim live intervals of input registers. They are no longer live into
// this block if they died after the PHI. If they lived after it, don't
// trim them because they might have other legitimate uses.
for (unsigned i = 1; i < PInstr->getNumOperands(); i += 2) {
unsigned reg = PInstr->getOperand(i).getReg();
MachineBasicBlock* MBB = PInstr->getOperand(i+1).getMBB();
LiveInterval& InputI = LI.getInterval(reg);
if (MBB != PInstr->getParent() &&
InputI.liveAt(LI.getMBBStartIdx(PInstr->getParent())) &&
InputI.expiredAt(LI.getInstructionIndex(PInstr).getNextIndex()))
InputI.removeRange(LI.getMBBStartIdx(PInstr->getParent()),
LI.getInstructionIndex(PInstr),
true);
}
// If the PHI is not dead, then the valno defined by the PHI
// now has an unknown def.
SlotIndex idx = LI.getInstructionIndex(PInstr).getDefIndex();
const LiveRange* PLR = PI.getLiveRangeContaining(idx);
PLR->valno->setIsPHIDef(true);
LiveRange R (LI.getMBBStartIdx(PInstr->getParent()),
PLR->start, PLR->valno);
PI.addRange(R);
}
LI.RemoveMachineInstrFromMaps(PInstr);
PInstr->eraseFromParent();
}
LI.renumber();
return true;
}
/// ScheduleCopies - Insert copies into predecessor blocks, scheduling
/// them properly so as to avoid the 'lost copy' and the 'virtual swap'
/// problems.
///
/// Based on "Practical Improvements to the Construction and Destruction
/// of Static Single Assignment Form" by Briggs, et al.
void StrongPHIElimination::ScheduleCopies(MachineBasicBlock* MBB,
std::set<unsigned>& pushed) {
// FIXME: This function needs to update LiveIntervals
std::multimap<unsigned, unsigned>& copy_set= Waiting[MBB];
std::multimap<unsigned, unsigned> worklist;
std::map<unsigned, unsigned> map;
// Setup worklist of initial copies
for (std::multimap<unsigned, unsigned>::iterator I = copy_set.begin(),
E = copy_set.end(); I != E; ) {
map.insert(std::make_pair(I->first, I->first));
map.insert(std::make_pair(I->second, I->second));
if (!UsedByAnother.count(I->second)) {
worklist.insert(*I);
// Avoid iterator invalidation
std::multimap<unsigned, unsigned>::iterator OI = I;
++I;
copy_set.erase(OI);
} else {
++I;
}
}
LiveIntervals& LI = getAnalysis<LiveIntervals>();
MachineFunction* MF = MBB->getParent();
MachineRegisterInfo& MRI = MF->getRegInfo();
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
SmallVector<std::pair<unsigned, MachineInstr*>, 4> InsertedPHIDests;
// Iterate over the worklist, inserting copies
while (!worklist.empty() || !copy_set.empty()) {
while (!worklist.empty()) {
std::multimap<unsigned, unsigned>::iterator WI = worklist.begin();
std::pair<unsigned, unsigned> curr = *WI;
worklist.erase(WI);
const TargetRegisterClass *RC = MF->getRegInfo().getRegClass(curr.first);
if (isLiveOut(curr.second, MBB, LI)) {
// Create a temporary
unsigned t = MF->getRegInfo().createVirtualRegister(RC);
// Insert copy from curr.second to a temporary at
// the Phi defining curr.second
MachineBasicBlock::iterator PI = MRI.getVRegDef(curr.second);
TII->copyRegToReg(*PI->getParent(), PI, t,
curr.second, RC, RC, DebugLoc());
DEBUG(dbgs() << "Inserted copy from " << curr.second << " to " << t
<< "\n");
// Push temporary on Stacks
Stacks[curr.second].push_back(t);
// Insert curr.second in pushed
pushed.insert(curr.second);
// Create a live interval for this temporary
InsertedPHIDests.push_back(std::make_pair(t, --PI));
}
// Insert copy from map[curr.first] to curr.second
TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), curr.second,
map[curr.first], RC, RC, DebugLoc());
map[curr.first] = curr.second;
DEBUG(dbgs() << "Inserted copy from " << curr.first << " to "
<< curr.second << "\n");
// Push this copy onto InsertedPHICopies so we can
// update LiveIntervals with it.
MachineBasicBlock::iterator MI = MBB->getFirstTerminator();
InsertedPHIDests.push_back(std::make_pair(curr.second, --MI));
// If curr.first is a destination in copy_set...
for (std::multimap<unsigned, unsigned>::iterator I = copy_set.begin(),
E = copy_set.end(); I != E; )
if (curr.first == I->second) {
std::pair<unsigned, unsigned> temp = *I;
worklist.insert(temp);
// Avoid iterator invalidation
std::multimap<unsigned, unsigned>::iterator OI = I;
++I;
copy_set.erase(OI);
break;
} else {
++I;
}
}
if (!copy_set.empty()) {
std::multimap<unsigned, unsigned>::iterator CI = copy_set.begin();
std::pair<unsigned, unsigned> curr = *CI;
worklist.insert(curr);
copy_set.erase(CI);
LiveInterval& I = LI.getInterval(curr.second);
MachineBasicBlock::iterator term = MBB->getFirstTerminator();
SlotIndex endIdx = SlotIndex();
if (term != MBB->end())
endIdx = LI.getInstructionIndex(term);
else
endIdx = LI.getMBBEndIdx(MBB);
if (I.liveAt(endIdx)) {
const TargetRegisterClass *RC =
MF->getRegInfo().getRegClass(curr.first);
// Insert a copy from dest to a new temporary t at the end of b
unsigned t = MF->getRegInfo().createVirtualRegister(RC);
TII->copyRegToReg(*MBB, MBB->getFirstTerminator(), t,
curr.second, RC, RC, DebugLoc());
map[curr.second] = t;
MachineBasicBlock::iterator TI = MBB->getFirstTerminator();
InsertedPHIDests.push_back(std::make_pair(t, --TI));
}
}
}
// Renumber the instructions so that we can perform the index computations
// needed to create new live intervals.
LI.renumber();
// For copies that we inserted at the ends of predecessors, we construct
// live intervals. This is pretty easy, since we know that the destination
// register cannot have be in live at that point previously. We just have
// to make sure that, for registers that serve as inputs to more than one
// PHI, we don't create multiple overlapping live intervals.
std::set<unsigned> RegHandled;
for (SmallVector<std::pair<unsigned, MachineInstr*>, 4>::iterator I =
InsertedPHIDests.begin(), E = InsertedPHIDests.end(); I != E; ++I) {
if (RegHandled.insert(I->first).second) {
LiveInterval& Int = LI.getOrCreateInterval(I->first);
SlotIndex instrIdx = LI.getInstructionIndex(I->second);
if (Int.liveAt(instrIdx.getDefIndex()))
Int.removeRange(instrIdx.getDefIndex(),
LI.getMBBEndIdx(I->second->getParent()).getNextSlot(),
true);
LiveRange R = LI.addLiveRangeToEndOfBlock(I->first, I->second);
R.valno->setCopy(I->second);
R.valno->def = LI.getInstructionIndex(I->second).getDefIndex();
}
}
}
void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI,
MachineBasicBlock *MBB) {
assert(PHI->isPHI());
unsigned PHIColor = getPHIColor(PHI);
for (unsigned i = 1; i < PHI->getNumOperands(); i += 2) {
MachineOperand &SrcMO = PHI->getOperand(i);
// If a source is defined by an implicit def, there is no need to insert a
// copy in the predecessor.
if (SrcMO.isUndef())
continue;
unsigned SrcReg = SrcMO.getReg();
assert(TargetRegisterInfo::isVirtualRegister(SrcReg) &&
"Machine PHI Operands must all be virtual registers!");
MachineBasicBlock *PredBB = PHI->getOperand(i + 1).getMBB();
unsigned SrcColor = getRegColor(SrcReg);
// If neither the PHI nor the operand were isolated, then we only need to
// set the phi-kill flag on the VNInfo at this PHI.
if (PHIColor && SrcColor == PHIColor) {
LiveInterval &SrcInterval = LI->getInterval(SrcReg);
SlotIndex PredIndex = LI->getMBBEndIdx(PredBB);
VNInfo *SrcVNI = SrcInterval.getVNInfoAt(PredIndex.getPrevIndex());
assert(SrcVNI);
SrcVNI->setHasPHIKill(true);
continue;
}
unsigned CopyReg = 0;
if (PHIColor) {
SrcCopyMap::const_iterator I
= InsertedSrcCopyMap.find(std::make_pair(PredBB, PHIColor));
CopyReg
= I != InsertedSrcCopyMap.end() ? I->second->getOperand(0).getReg() : 0;
}
if (!CopyReg) {
const TargetRegisterClass *RC = MRI->getRegClass(SrcReg);
CopyReg = MRI->createVirtualRegister(RC);
MachineBasicBlock::iterator
CopyInsertPoint = findPHICopyInsertPoint(PredBB, MBB, SrcReg);
unsigned SrcSubReg = SrcMO.getSubReg();
MachineInstr *CopyInstr = BuildMI(*PredBB,
CopyInsertPoint,
PHI->getDebugLoc(),
TII->get(TargetOpcode::COPY),
CopyReg).addReg(SrcReg, 0, SrcSubReg);
LI->InsertMachineInstrInMaps(CopyInstr);
// addLiveRangeToEndOfBlock() also adds the phikill flag to the VNInfo for
// the newly added range.
LI->addLiveRangeToEndOfBlock(CopyReg, CopyInstr);
InsertedSrcCopySet.insert(std::make_pair(PredBB, SrcReg));
addReg(CopyReg);
if (PHIColor) {
unionRegs(PHIColor, CopyReg);
assert(getRegColor(CopyReg) != CopyReg);
} else {
PHIColor = CopyReg;
assert(getRegColor(CopyReg) == CopyReg);
}
if (!InsertedSrcCopyMap.count(std::make_pair(PredBB, PHIColor)))
InsertedSrcCopyMap[std::make_pair(PredBB, PHIColor)] = CopyInstr;
}
SrcMO.setReg(CopyReg);
// If SrcReg is not live beyond the PHI, trim its interval so that it is no
// longer live-in to MBB. Note that SrcReg may appear in other PHIs that are
// processed later, but this is still correct to do at this point because we
// never rely on LiveIntervals being correct while inserting copies.
// FIXME: Should this just count uses at PHIs like the normal PHIElimination
// pass does?
LiveInterval &SrcLI = LI->getInterval(SrcReg);
SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB);
SlotIndex PHIIndex = LI->getInstructionIndex(PHI);
SlotIndex NextInstrIndex = PHIIndex.getNextIndex();
if (SrcLI.liveAt(MBBStartIndex) && SrcLI.expiredAt(NextInstrIndex))
SrcLI.removeRange(MBBStartIndex, PHIIndex, true);
}
unsigned DestReg = PHI->getOperand(0).getReg();
unsigned DestColor = getRegColor(DestReg);
if (PHIColor && DestColor == PHIColor) {
LiveInterval &DestLI = LI->getInterval(DestReg);
// Set the phi-def flag for the VN at this PHI.
SlotIndex PHIIndex = LI->getInstructionIndex(PHI);
VNInfo *DestVNI = DestLI.getVNInfoAt(PHIIndex.getDefIndex());
assert(DestVNI);
DestVNI->setIsPHIDef(true);
// Prior to PHI elimination, the live ranges of PHIs begin at their defining
// instruction. After PHI elimination, PHI instructions are replaced by VNs
// with the phi-def flag set, and the live ranges of these VNs start at the
// beginning of the basic block.
SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB);
DestVNI->def = MBBStartIndex;
DestLI.addRange(LiveRange(MBBStartIndex,
PHIIndex.getDefIndex(),
DestVNI));
return;
}
const TargetRegisterClass *RC = MRI->getRegClass(DestReg);
unsigned CopyReg = MRI->createVirtualRegister(RC);
MachineInstr *CopyInstr = BuildMI(*MBB,
MBB->SkipPHIsAndLabels(MBB->begin()),
PHI->getDebugLoc(),
TII->get(TargetOpcode::COPY),
DestReg).addReg(CopyReg);
LI->InsertMachineInstrInMaps(CopyInstr);
PHI->getOperand(0).setReg(CopyReg);
// Add the region from the beginning of MBB to the copy instruction to
// CopyReg's live interval, and give the VNInfo the phidef flag.
LiveInterval &CopyLI = LI->getOrCreateInterval(CopyReg);
SlotIndex MBBStartIndex = LI->getMBBStartIdx(MBB);
SlotIndex DestCopyIndex = LI->getInstructionIndex(CopyInstr);
VNInfo *CopyVNI = CopyLI.getNextValue(MBBStartIndex,
CopyInstr,
LI->getVNInfoAllocator());
CopyVNI->setIsPHIDef(true);
CopyLI.addRange(LiveRange(MBBStartIndex,
DestCopyIndex.getDefIndex(),
CopyVNI));
// Adjust DestReg's live interval to adjust for its new definition at
// CopyInstr.
LiveInterval &DestLI = LI->getOrCreateInterval(DestReg);
SlotIndex PHIIndex = LI->getInstructionIndex(PHI);
DestLI.removeRange(PHIIndex.getDefIndex(), DestCopyIndex.getDefIndex());
VNInfo *DestVNI = DestLI.getVNInfoAt(DestCopyIndex.getDefIndex());
assert(DestVNI);
DestVNI->def = DestCopyIndex.getDefIndex();
InsertedDestCopies[CopyReg] = CopyInstr;
}
bool StrongPHIElimination::runOnMachineFunction(MachineFunction &MF) {
MRI = &MF.getRegInfo();
TII = MF.getTarget().getInstrInfo();
DT = &getAnalysis<MachineDominatorTree>();
LI = &getAnalysis<LiveIntervals>();
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
unsigned DestReg = BBI->getOperand(0).getReg();
addReg(DestReg);
PHISrcDefs[I].push_back(BBI);
for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) {
MachineOperand &SrcMO = BBI->getOperand(i);
unsigned SrcReg = SrcMO.getReg();
addReg(SrcReg);
unionRegs(DestReg, SrcReg);
MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
if (DefMI)
PHISrcDefs[DefMI->getParent()].push_back(DefMI);
}
}
}
// Perform a depth-first traversal of the dominator tree, splitting
// interferences amongst PHI-congruence classes.
DenseMap<unsigned, unsigned> CurrentDominatingParent;
DenseMap<unsigned, unsigned> ImmediateDominatingParent;
for (df_iterator<MachineDomTreeNode*> DI = df_begin(DT->getRootNode()),
DE = df_end(DT->getRootNode()); DI != DE; ++DI) {
SplitInterferencesForBasicBlock(*DI->getBlock(),
CurrentDominatingParent,
ImmediateDominatingParent);
}
// Insert copies for all PHI source and destination registers.
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
InsertCopiesForPHI(BBI, I);
}
}
// FIXME: Preserve the equivalence classes during copy insertion and use
// the preversed equivalence classes instead of recomputing them.
RegNodeMap.clear();
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
for (MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
BBI != BBE && BBI->isPHI(); ++BBI) {
unsigned DestReg = BBI->getOperand(0).getReg();
addReg(DestReg);
for (unsigned i = 1; i < BBI->getNumOperands(); i += 2) {
unsigned SrcReg = BBI->getOperand(i).getReg();
addReg(SrcReg);
unionRegs(DestReg, SrcReg);
}
}
}
DenseMap<unsigned, unsigned> RegRenamingMap;
bool Changed = false;
for (MachineFunction::iterator I = MF.begin(), E = MF.end();
I != E; ++I) {
MachineBasicBlock::iterator BBI = I->begin(), BBE = I->end();
while (BBI != BBE && BBI->isPHI()) {
MachineInstr *PHI = BBI;
assert(PHI->getNumOperands() > 0);
unsigned SrcReg = PHI->getOperand(1).getReg();
unsigned SrcColor = getRegColor(SrcReg);
unsigned NewReg = RegRenamingMap[SrcColor];
if (!NewReg) {
NewReg = SrcReg;
RegRenamingMap[SrcColor] = SrcReg;
}
MergeLIsAndRename(SrcReg, NewReg);
unsigned DestReg = PHI->getOperand(0).getReg();
if (!InsertedDestCopies.count(DestReg))
MergeLIsAndRename(DestReg, NewReg);
for (unsigned i = 3; i < PHI->getNumOperands(); i += 2) {
unsigned SrcReg = PHI->getOperand(i).getReg();
MergeLIsAndRename(SrcReg, NewReg);
}
++BBI;
LI->RemoveMachineInstrFromMaps(PHI);
PHI->eraseFromParent();
Changed = true;
}
}
// Due to the insertion of copies to split live ranges, the live intervals are
// guaranteed to not overlap, except in one case: an original PHI source and a
// PHI destination copy. In this case, they have the same value and thus don't
// truly intersect, so we merge them into the value live at that point.
// FIXME: Is there some better way we can handle this?
for (DestCopyMap::iterator I = InsertedDestCopies.begin(),
E = InsertedDestCopies.end(); I != E; ++I) {
unsigned DestReg = I->first;
unsigned DestColor = getRegColor(DestReg);
unsigned NewReg = RegRenamingMap[DestColor];
LiveInterval &DestLI = LI->getInterval(DestReg);
LiveInterval &NewLI = LI->getInterval(NewReg);
assert(DestLI.ranges.size() == 1
&& "PHI destination copy's live interval should be a single live "
"range from the beginning of the BB to the copy instruction.");
LiveRange *DestLR = DestLI.begin();
VNInfo *NewVNI = NewLI.getVNInfoAt(DestLR->start);
if (!NewVNI) {
NewVNI = NewLI.createValueCopy(DestLR->valno, LI->getVNInfoAllocator());
MachineInstr *CopyInstr = I->second;
CopyInstr->getOperand(1).setIsKill(true);
}
LiveRange NewLR(DestLR->start, DestLR->end, NewVNI);
NewLI.addRange(NewLR);
LI->removeInterval(DestReg);
MRI->replaceRegWith(DestReg, NewReg);
}
// Adjust the live intervals of all PHI source registers to handle the case
// where the PHIs in successor blocks were the only later uses of the source
// register.
for (SrcCopySet::iterator I = InsertedSrcCopySet.begin(),
E = InsertedSrcCopySet.end(); I != E; ++I) {
MachineBasicBlock *MBB = I->first;
unsigned SrcReg = I->second;
if (unsigned RenamedRegister = RegRenamingMap[getRegColor(SrcReg)])
SrcReg = RenamedRegister;
LiveInterval &SrcLI = LI->getInterval(SrcReg);
bool isLiveOut = false;
for (MachineBasicBlock::succ_iterator SI = MBB->succ_begin(),
SE = MBB->succ_end(); SI != SE; ++SI) {
if (SrcLI.liveAt(LI->getMBBStartIdx(*SI))) {
isLiveOut = true;
break;
}
}
if (isLiveOut)
continue;
MachineOperand *LastUse = findLastUse(MBB, SrcReg);
assert(LastUse);
SlotIndex LastUseIndex = LI->getInstructionIndex(LastUse->getParent());
SrcLI.removeRange(LastUseIndex.getDefIndex(), LI->getMBBEndIdx(MBB));
LastUse->setIsKill(true);
}
LI->renumber();
Allocator.Reset();
RegNodeMap.clear();
PHISrcDefs.clear();
InsertedSrcCopySet.clear();
InsertedSrcCopyMap.clear();
InsertedDestCopies.clear();
return Changed;
}
/// reMaterializeFor - Attempt to rematerialize li_->reg before MI instead of
/// reloading it.
bool InlineSpiller::reMaterializeFor(MachineBasicBlock::iterator MI) {
SlotIndex UseIdx = lis_.getInstructionIndex(MI).getUseIndex();
VNInfo *OrigVNI = li_->getVNInfoAt(UseIdx);
if (!OrigVNI) {
DEBUG(dbgs() << "\tadding <undef> flags: ");
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg)
MO.setIsUndef();
}
DEBUG(dbgs() << UseIdx << '\t' << *MI);
return true;
}
if (!reMattable_.count(OrigVNI)) {
DEBUG(dbgs() << "\tusing non-remat valno " << OrigVNI->id << ": "
<< UseIdx << '\t' << *MI);
return false;
}
MachineInstr *OrigMI = lis_.getInstructionFromIndex(OrigVNI->def);
if (!allUsesAvailableAt(OrigMI, OrigVNI->def, UseIdx)) {
usedValues_.insert(OrigVNI);
DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << *MI);
return false;
}
// If the instruction also writes li_->reg, it had better not require the same
// register for uses and defs.
bool Reads, Writes;
SmallVector<unsigned, 8> Ops;
tie(Reads, Writes) = MI->readsWritesVirtualRegister(li_->reg, &Ops);
if (Writes) {
for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(Ops[i]);
if (MO.isUse() ? MI->isRegTiedToDefOperand(Ops[i]) : MO.getSubReg()) {
usedValues_.insert(OrigVNI);
DEBUG(dbgs() << "\tcannot remat tied reg: " << UseIdx << '\t' << *MI);
return false;
}
}
}
// Alocate a new register for the remat.
unsigned NewVReg = mri_.createVirtualRegister(rc_);
vrm_.grow();
LiveInterval &NewLI = lis_.getOrCreateInterval(NewVReg);
NewLI.markNotSpillable();
newIntervals_->push_back(&NewLI);
// Finally we can rematerialize OrigMI before MI.
MachineBasicBlock &MBB = *MI->getParent();
tii_.reMaterialize(MBB, MI, NewLI.reg, 0, OrigMI, tri_);
MachineBasicBlock::iterator RematMI = MI;
SlotIndex DefIdx = lis_.InsertMachineInstrInMaps(--RematMI).getDefIndex();
DEBUG(dbgs() << "\tremat: " << DefIdx << '\t' << *RematMI);
// Replace operands
for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(Ops[i]);
if (MO.isReg() && MO.isUse() && MO.getReg() == li_->reg) {
MO.setReg(NewVReg);
MO.setIsKill();
}
}
DEBUG(dbgs() << "\t " << UseIdx << '\t' << *MI);
VNInfo *DefVNI = NewLI.getNextValue(DefIdx, 0, true,
lis_.getVNInfoAllocator());
NewLI.addRange(LiveRange(DefIdx, UseIdx.getDefIndex(), DefVNI));
DEBUG(dbgs() << "\tinterval: " << NewLI << '\n');
return true;
}
