DomainValue *ExeDepsFix::alloc(int domain) {
DomainValue *dv = Avail.empty() ?
new(Allocator.Allocate()) DomainValue :
Avail.pop_back_val();
if (domain >= 0)
dv->addDomain(domain);
assert(dv->Refs == 0 && "Reference count wasn't cleared");
assert(!dv->Next && "Chained DomainValue shouldn't have been recycled");
return dv;
}
// A soft instruction can be changed to work in other domains given by mask.
void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
// Bitmask of available domains for this instruction after taking collapsed
// operands into account.
unsigned available = mask;
// Scan the explicit use operands for incoming domains.
SmallVector<int, 4> used;
if (LiveRegs)
for (unsigned i = mi->getDesc().getNumDefs(),
e = mi->getDesc().getNumOperands(); i != e; ++i) {
MachineOperand &mo = mi->getOperand(i);
if (!mo.isReg()) continue;
int rx = regIndex(mo.getReg());
if (rx < 0) continue;
if (DomainValue *dv = LiveRegs[rx].Value) {
// Bitmask of domains that dv and available have in common.
unsigned common = dv->getCommonDomains(available);
// Is it possible to use this collapsed register for free?
if (dv->isCollapsed()) {
// Restrict available domains to the ones in common with the operand.
// If there are no common domains, we must pay the cross-domain
// penalty for this operand.
if (common) available = common;
} else if (common)
// Open DomainValue is compatible, save it for merging.
used.push_back(rx);
else
// Open DomainValue is not compatible with instruction. It is useless
// now.
kill(rx);
}
}
// If the collapsed operands force a single domain, propagate the collapse.
if (isPowerOf2_32(available)) {
unsigned domain = countTrailingZeros(available);
TII->setExecutionDomain(mi, domain);
visitHardInstr(mi, domain);
return;
}
// Kill off any remaining uses that don't match available, and build a list of
// incoming DomainValues that we want to merge.
SmallVector<LiveReg, 4> Regs;
for (SmallVector<int, 4>::iterator i=used.begin(), e=used.end(); i!=e; ++i) {
int rx = *i;
const LiveReg &LR = LiveRegs[rx];
// This useless DomainValue could have been missed above.
if (!LR.Value->getCommonDomains(available)) {
kill(rx);
continue;
}
// Sorted insertion.
bool Inserted = false;
for (SmallVector<LiveReg, 4>::iterator i = Regs.begin(), e = Regs.end();
i != e && !Inserted; ++i) {
if (LR.Def < i->Def) {
Inserted = true;
Regs.insert(i, LR);
}
}
if (!Inserted)
Regs.push_back(LR);
}
// doms are now sorted in order of appearance. Try to merge them all, giving
// priority to the latest ones.
DomainValue *dv = 0;
while (!Regs.empty()) {
if (!dv) {
dv = Regs.pop_back_val().Value;
// Force the first dv to match the current instruction.
dv->AvailableDomains = dv->getCommonDomains(available);
assert(dv->AvailableDomains && "Domain should have been filtered");
continue;
}
DomainValue *Latest = Regs.pop_back_val().Value;
// Skip already merged values.
if (Latest == dv || Latest->Next)
continue;
if (merge(dv, Latest))
continue;
// If latest didn't merge, it is useless now. Kill all registers using it.
for (SmallVector<int,4>::iterator i=used.begin(), e=used.end(); i != e; ++i)
if (LiveRegs[*i].Value == Latest)
kill(*i);
}
// dv is the DomainValue we are going to use for this instruction.
if (!dv) {
dv = alloc();
dv->AvailableDomains = available;
}
dv->Instrs.push_back(mi);
// Finally set all defs and non-collapsed uses to dv. We must iterate through
// all the operators, including imp-def ones.
for (MachineInstr::mop_iterator ii = mi->operands_begin(),
ee = mi->operands_end();
ii != ee; ++ii) {
MachineOperand &mo = *ii;
if (!mo.isReg()) continue;
int rx = regIndex(mo.getReg());
if (rx < 0) continue;
if (!LiveRegs[rx].Value || (mo.isDef() && LiveRegs[rx].Value != dv)) {
kill(rx);
setLiveReg(rx, dv);
}
}
}
// enterBasicBlock - Set up LiveRegs by merging predecessor live-out values.
void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
// Detect back-edges from predecessors we haven't processed yet.
SeenUnknownBackEdge = false;
// Reset instruction counter in each basic block.
CurInstr = 0;
// Set up LiveRegs to represent registers entering MBB.
if (!LiveRegs)
LiveRegs = new LiveReg[NumRegs];
// Default values are 'nothing happened a long time ago'.
for (unsigned rx = 0; rx != NumRegs; ++rx) {
LiveRegs[rx].Value = 0;
LiveRegs[rx].Def = -(1 << 20);
}
// This is the entry block.
if (MBB->pred_empty()) {
for (MachineBasicBlock::livein_iterator i = MBB->livein_begin(),
e = MBB->livein_end(); i != e; ++i) {
int rx = regIndex(*i);
if (rx < 0)
continue;
// Treat function live-ins as if they were defined just before the first
// instruction. Usually, function arguments are set up immediately
// before the call.
LiveRegs[rx].Def = -1;
}
DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": entry\n");
return;
}
// Try to coalesce live-out registers from predecessors.
for (MachineBasicBlock::const_pred_iterator pi = MBB->pred_begin(),
pe = MBB->pred_end(); pi != pe; ++pi) {
LiveOutMap::const_iterator fi = LiveOuts.find(*pi);
if (fi == LiveOuts.end()) {
SeenUnknownBackEdge = true;
continue;
}
assert(fi->second && "Can't have NULL entries");
for (unsigned rx = 0; rx != NumRegs; ++rx) {
// Use the most recent predecessor def for each register.
LiveRegs[rx].Def = std::max(LiveRegs[rx].Def, fi->second[rx].Def);
DomainValue *pdv = resolve(fi->second[rx].Value);
if (!pdv)
continue;
if (!LiveRegs[rx].Value) {
setLiveReg(rx, pdv);
continue;
}
// We have a live DomainValue from more than one predecessor.
if (LiveRegs[rx].Value->isCollapsed()) {
// We are already collapsed, but predecessor is not. Force him.
unsigned Domain = LiveRegs[rx].Value->getFirstDomain();
if (!pdv->isCollapsed() && pdv->hasDomain(Domain))
collapse(pdv, Domain);
continue;
}
// Currently open, merge in predecessor.
if (!pdv->isCollapsed())
merge(LiveRegs[rx].Value, pdv);
else
force(rx, pdv->getFirstDomain());
}
}
DEBUG(dbgs() << "BB#" << MBB->getNumber()
<< (SeenUnknownBackEdge ? ": incomplete\n" : ": all preds known\n"));
}
/// Set up LiveRegs by merging predecessor live-out values.
void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) {
// Reset instruction counter in each basic block.
CurInstr = 0;
// Set up UndefReads to track undefined register reads.
UndefReads.clear();
LiveRegSet.clear();
// Set up LiveRegs to represent registers entering MBB.
if (!LiveRegs)
LiveRegs = new LiveReg[NumRegs];
// Default values are 'nothing happened a long time ago'.
for (unsigned rx = 0; rx != NumRegs; ++rx) {
LiveRegs[rx].Value = nullptr;
LiveRegs[rx].Def = -(1 << 20);
}
// This is the entry block.
if (MBB->pred_empty()) {
for (const auto &LI : MBB->liveins()) {
for (int rx : regIndices(LI.PhysReg)) {
// Treat function live-ins as if they were defined just before the first
// instruction. Usually, function arguments are set up immediately
// before the call.
LiveRegs[rx].Def = -1;
}
}
DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": entry\n");
return;
}
// Try to coalesce live-out registers from predecessors.
for (MachineBasicBlock::const_pred_iterator pi = MBB->pred_begin(),
pe = MBB->pred_end(); pi != pe; ++pi) {
auto fi = MBBInfos.find(*pi);
assert(fi != MBBInfos.end() &&
"Should have pre-allocated MBBInfos for all MBBs");
LiveReg *Incoming = fi->second.OutRegs;
// Incoming is null if this is a backedge from a BB
// we haven't processed yet
if (Incoming == nullptr) {
continue;
}
for (unsigned rx = 0; rx != NumRegs; ++rx) {
// Use the most recent predecessor def for each register.
LiveRegs[rx].Def = std::max(LiveRegs[rx].Def, Incoming[rx].Def);
DomainValue *pdv = resolve(Incoming[rx].Value);
if (!pdv)
continue;
if (!LiveRegs[rx].Value) {
setLiveReg(rx, pdv);
continue;
}
// We have a live DomainValue from more than one predecessor.
if (LiveRegs[rx].Value->isCollapsed()) {
// We are already collapsed, but predecessor is not. Force it.
unsigned Domain = LiveRegs[rx].Value->getFirstDomain();
if (!pdv->isCollapsed() && pdv->hasDomain(Domain))
collapse(pdv, Domain);
continue;
}
// Currently open, merge in predecessor.
if (!pdv->isCollapsed())
merge(LiveRegs[rx].Value, pdv);
else
force(rx, pdv->getFirstDomain());
}
}
DEBUG(
dbgs() << "BB#" << MBB->getNumber()
<< (!isBlockDone(MBB) ? ": incomplete\n" : ": all preds known\n"));
}