bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C,
MachineBasicBlock &MBB) {
bool Changed = false;
DEBUG(dbgs() << " - colorChain(" << G->str() << ", "
<< ColorNames[(int)C] << ")\n");
// Try and obtain a free register of the right class. Without a register
// to play with we cannot continue.
int Reg = scavengeRegister(G, C, MBB);
if (Reg == -1) {
DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n");
return false;
}
DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n");
std::map<unsigned, unsigned> Substs;
for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd();
I != E; ++I) {
if (!G->contains(I) &&
(&*I != G->getKill() || G->isKillImmutable()))
continue;
// I is a member of G, or I is a mutable instruction that kills G.
std::vector<unsigned> ToErase;
for (auto &U : I->operands()) {
if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) {
unsigned OrigReg = U.getReg();
U.setReg(Substs[OrigReg]);
if (U.isKill())
// Don't erase straight away, because there may be other operands
// that also reference this substitution!
ToErase.push_back(OrigReg);
} else if (U.isRegMask()) {
for (auto J : Substs) {
if (U.clobbersPhysReg(J.first))
ToErase.push_back(J.first);
}
}
}
// Now it's safe to remove the substs identified earlier.
for (auto J : ToErase)
Substs.erase(J);
// Only change the def if this isn't the last instruction.
if (&*I != G->getKill()) {
MachineOperand &MO = I->getOperand(0);
bool Change = TransformAll || getColor(MO.getReg()) != C;
if (G->requiresFixup() && &*I == G->getLast())
Change = false;
if (Change) {
Substs[MO.getReg()] = Reg;
MO.setReg(Reg);
MRI->setPhysRegUsed(Reg);
Changed = true;
}
}
}
assert(Substs.size() == 0 && "No substitutions should be left active!");
if (G->getKill()) {
DEBUG(dbgs() << " - Kill instruction seen.\n");
} else {
// We didn't have a kill instruction, but we didn't seem to need to change
// the destination register anyway.
DEBUG(dbgs() << " - Destination register not changed.\n");
}
return Changed;
}
unsigned RegScavenger::findSurvivorReg(MachineBasicBlock::iterator StartMI,
BitVector &Candidates,
unsigned InstrLimit,
MachineBasicBlock::iterator &UseMI) {
int Survivor = Candidates.find_first();
assert(Survivor > 0 && "No candidates for scavenging");
MachineBasicBlock::iterator ME = MBB->getFirstTerminator();
assert(StartMI != ME && "MI already at terminator");
MachineBasicBlock::iterator RestorePointMI = StartMI;
MachineBasicBlock::iterator MI = StartMI;
bool inVirtLiveRange = false;
for (++MI; InstrLimit > 0 && MI != ME; ++MI, --InstrLimit) {
if (MI->isDebugValue()) {
++InstrLimit; // Don't count debug instructions
continue;
}
bool isVirtKillInsn = false;
bool isVirtDefInsn = false;
// Remove any candidates touched by instruction.
for (const MachineOperand &MO : MI->operands()) {
if (MO.isRegMask())
Candidates.clearBitsNotInMask(MO.getRegMask());
if (!MO.isReg() || MO.isUndef() || !MO.getReg())
continue;
if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) {
if (MO.isDef())
isVirtDefInsn = true;
else if (MO.isKill())
isVirtKillInsn = true;
continue;
}
for (MCRegAliasIterator AI(MO.getReg(), TRI, true); AI.isValid(); ++AI)
Candidates.reset(*AI);
}
// If we're not in a virtual reg's live range, this is a valid
// restore point.
if (!inVirtLiveRange) RestorePointMI = MI;
// Update whether we're in the live range of a virtual register
if (isVirtKillInsn) inVirtLiveRange = false;
if (isVirtDefInsn) inVirtLiveRange = true;
// Was our survivor untouched by this instruction?
if (Candidates.test(Survivor))
continue;
// All candidates gone?
if (Candidates.none())
break;
Survivor = Candidates.find_first();
}
// If we ran off the end, that's where we want to restore.
if (MI == ME) RestorePointMI = ME;
assert(RestorePointMI != StartMI &&
"No available scavenger restore location!");
// We ran out of candidates, so stop the search.
UseMI = RestorePointMI;
return Survivor;
}
/// Walk the specified region of the CFG and hoist loop invariants out to the
/// preheader.
void MachineLICM::HoistRegionPostRA() {
MachineBasicBlock *Preheader = getCurPreheader();
if (!Preheader)
return;
unsigned NumRegs = TRI->getNumRegs();
BitVector PhysRegDefs(NumRegs); // Regs defined once in the loop.
BitVector PhysRegClobbers(NumRegs); // Regs defined more than once.
SmallVector<CandidateInfo, 32> Candidates;
SmallSet<int, 32> StoredFIs;
// Walk the entire region, count number of defs for each register, and
// collect potential LICM candidates.
const std::vector<MachineBasicBlock *> &Blocks = CurLoop->getBlocks();
for (MachineBasicBlock *BB : Blocks) {
// If the header of the loop containing this basic block is a landing pad,
// then don't try to hoist instructions out of this loop.
const MachineLoop *ML = MLI->getLoopFor(BB);
if (ML && ML->getHeader()->isEHPad()) continue;
// Conservatively treat live-in's as an external def.
// FIXME: That means a reload that're reused in successor block(s) will not
// be LICM'ed.
for (const auto &LI : BB->liveins()) {
for (MCRegAliasIterator AI(LI.PhysReg, TRI, true); AI.isValid(); ++AI)
PhysRegDefs.set(*AI);
}
SpeculationState = SpeculateUnknown;
for (MachineInstr &MI : *BB)
ProcessMI(&MI, PhysRegDefs, PhysRegClobbers, StoredFIs, Candidates);
}
// Gather the registers read / clobbered by the terminator.
BitVector TermRegs(NumRegs);
MachineBasicBlock::iterator TI = Preheader->getFirstTerminator();
if (TI != Preheader->end()) {
for (const MachineOperand &MO : TI->operands()) {
if (!MO.isReg())
continue;
unsigned Reg = MO.getReg();
if (!Reg)
continue;
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
TermRegs.set(*AI);
}
}
// Now evaluate whether the potential candidates qualify.
// 1. Check if the candidate defined register is defined by another
// instruction in the loop.
// 2. If the candidate is a load from stack slot (always true for now),
// check if the slot is stored anywhere in the loop.
// 3. Make sure candidate def should not clobber
// registers read by the terminator. Similarly its def should not be
// clobbered by the terminator.
for (CandidateInfo &Candidate : Candidates) {
if (Candidate.FI != INT_MIN &&
StoredFIs.count(Candidate.FI))
continue;
unsigned Def = Candidate.Def;
if (!PhysRegClobbers.test(Def) && !TermRegs.test(Def)) {
bool Safe = true;
MachineInstr *MI = Candidate.MI;
for (const MachineOperand &MO : MI->operands()) {
if (!MO.isReg() || MO.isDef() || !MO.getReg())
continue;
unsigned Reg = MO.getReg();
if (PhysRegDefs.test(Reg) ||
PhysRegClobbers.test(Reg)) {
// If it's using a non-loop-invariant register, then it's obviously
// not safe to hoist.
Safe = false;
break;
}
}
if (Safe)
HoistPostRA(MI, Candidate.Def);
}
}
}
/// canSpeculateInstrs - Returns true if all the instructions in MBB can safely
/// be speculated. The terminators are not considered.
///
/// If instructions use any values that are defined in the head basic block,
/// the defining instructions are added to InsertAfter.
///
/// Any clobbered regunits are added to ClobberedRegUnits.
///
bool SSAIfConv::canSpeculateInstrs(MachineBasicBlock *MBB) {
// Reject any live-in physregs. It's probably CPSR/EFLAGS, and very hard to
// get right.
if (!MBB->livein_empty()) {
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has live-ins.\n");
return false;
}
unsigned InstrCount = 0;
// Check all instructions, except the terminators. It is assumed that
// terminators never have side effects or define any used register values.
for (MachineBasicBlock::iterator I = MBB->begin(),
E = MBB->getFirstTerminator(); I != E; ++I) {
if (I->isDebugValue())
continue;
if (++InstrCount > BlockInstrLimit && !Stress) {
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " has more than "
<< BlockInstrLimit << " instructions.\n");
return false;
}
// There shouldn't normally be any phis in a single-predecessor block.
if (I->isPHI()) {
DEBUG(dbgs() << "Can't hoist: " << *I);
return false;
}
// Don't speculate loads. Note that it may be possible and desirable to
// speculate GOT or constant pool loads that are guaranteed not to trap,
// but we don't support that for now.
if (I->mayLoad()) {
DEBUG(dbgs() << "Won't speculate load: " << *I);
return false;
}
// We never speculate stores, so an AA pointer isn't necessary.
bool DontMoveAcrossStore = true;
if (!I->isSafeToMove(nullptr, DontMoveAcrossStore)) {
DEBUG(dbgs() << "Can't speculate: " << *I);
return false;
}
// Check for any dependencies on Head instructions.
for (const MachineOperand &MO : I->operands()) {
if (MO.isRegMask()) {
DEBUG(dbgs() << "Won't speculate regmask: " << *I);
return false;
}
if (!MO.isReg())
continue;
unsigned Reg = MO.getReg();
// Remember clobbered regunits.
if (MO.isDef() && TargetRegisterInfo::isPhysicalRegister(Reg))
for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units)
ClobberedRegUnits.set(*Units);
if (!MO.readsReg() || !TargetRegisterInfo::isVirtualRegister(Reg))
continue;
MachineInstr *DefMI = MRI->getVRegDef(Reg);
if (!DefMI || DefMI->getParent() != Head)
continue;
if (InsertAfter.insert(DefMI).second)
DEBUG(dbgs() << "BB#" << MBB->getNumber() << " depends on " << *DefMI);
if (DefMI->isTerminator()) {
DEBUG(dbgs() << "Can't insert instructions below terminator.\n");
return false;
}
}
}
return true;
}
bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI,
const TargetRegisterClass *SrcRC,
unsigned SubReg,
const TargetRegisterClass *DstRC,
unsigned DstSubReg,
const TargetRegisterClass *NewRC,
LiveIntervals &LIS) const {
assert (MI->isCopy() && "Only expecting COPY instructions");
// Coalesce anything which is not a COPY involving a subreg to/from GR128.
if (!(NewRC->hasSuperClassEq(&SystemZ::GR128BitRegClass) &&
(getRegSizeInBits(*SrcRC) <= 64 || getRegSizeInBits(*DstRC) <= 64)))
return true;
// Allow coalescing of a GR128 subreg COPY only if the live ranges are small
// and local to one MBB with not too much interferring registers. Otherwise
// regalloc may run out of registers.
unsigned WideOpNo = (getRegSizeInBits(*SrcRC) == 128 ? 1 : 0);
unsigned GR128Reg = MI->getOperand(WideOpNo).getReg();
unsigned GRNarReg = MI->getOperand((WideOpNo == 1) ? 0 : 1).getReg();
LiveInterval &IntGR128 = LIS.getInterval(GR128Reg);
LiveInterval &IntGRNar = LIS.getInterval(GRNarReg);
// Check that the two virtual registers are local to MBB.
MachineBasicBlock *MBB = MI->getParent();
if (LIS.isLiveInToMBB(IntGR128, MBB) || LIS.isLiveOutOfMBB(IntGR128, MBB) ||
LIS.isLiveInToMBB(IntGRNar, MBB) || LIS.isLiveOutOfMBB(IntGRNar, MBB))
return false;
// Find the first and last MIs of the registers.
MachineInstr *FirstMI = nullptr, *LastMI = nullptr;
if (WideOpNo == 1) {
FirstMI = LIS.getInstructionFromIndex(IntGR128.beginIndex());
LastMI = LIS.getInstructionFromIndex(IntGRNar.endIndex());
} else {
FirstMI = LIS.getInstructionFromIndex(IntGRNar.beginIndex());
LastMI = LIS.getInstructionFromIndex(IntGR128.endIndex());
}
assert (FirstMI && LastMI && "No instruction from index?");
// Check if coalescing seems safe by finding the set of clobbered physreg
// pairs in the region.
BitVector PhysClobbered(getNumRegs());
MachineBasicBlock::iterator MII = FirstMI, MEE = LastMI;
MEE++;
for (; MII != MEE; ++MII) {
for (const MachineOperand &MO : MII->operands())
if (MO.isReg() && isPhysicalRegister(MO.getReg())) {
for (MCSuperRegIterator SI(MO.getReg(), this, true/*IncludeSelf*/);
SI.isValid(); ++SI)
if (NewRC->contains(*SI)) {
PhysClobbered.set(*SI);
break;
}
}
}
// Demand an arbitrary margin of free regs.
unsigned const DemandedFreeGR128 = 3;
if (PhysClobbered.count() > (NewRC->getNumRegs() - DemandedFreeGR128))
return false;
return true;
}