DomainValue *ExeDepsFix::alloc(int domain) { DomainValue *dv = Avail.empty() ? new(Allocator.Allocate()) DomainValue : Avail.pop_back_val(); if (domain >= 0) dv->addDomain(domain); assert(dv->Refs == 0 && "Reference count wasn't cleared"); assert(!dv->Next && "Chained DomainValue shouldn't have been recycled"); return dv; }
// A soft instruction can be changed to work in other domains given by mask. void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) { // Bitmask of available domains for this instruction after taking collapsed // operands into account. unsigned available = mask; // Scan the explicit use operands for incoming domains. SmallVector<int, 4> used; if (LiveRegs) for (unsigned i = mi->getDesc().getNumDefs(), e = mi->getDesc().getNumOperands(); i != e; ++i) { MachineOperand &mo = mi->getOperand(i); if (!mo.isReg()) continue; int rx = regIndex(mo.getReg()); if (rx < 0) continue; if (DomainValue *dv = LiveRegs[rx].Value) { // Bitmask of domains that dv and available have in common. unsigned common = dv->getCommonDomains(available); // Is it possible to use this collapsed register for free? if (dv->isCollapsed()) { // Restrict available domains to the ones in common with the operand. // If there are no common domains, we must pay the cross-domain // penalty for this operand. if (common) available = common; } else if (common) // Open DomainValue is compatible, save it for merging. used.push_back(rx); else // Open DomainValue is not compatible with instruction. It is useless // now. kill(rx); } } // If the collapsed operands force a single domain, propagate the collapse. if (isPowerOf2_32(available)) { unsigned domain = countTrailingZeros(available); TII->setExecutionDomain(mi, domain); visitHardInstr(mi, domain); return; } // Kill off any remaining uses that don't match available, and build a list of // incoming DomainValues that we want to merge. SmallVector<LiveReg, 4> Regs; for (SmallVector<int, 4>::iterator i=used.begin(), e=used.end(); i!=e; ++i) { int rx = *i; const LiveReg &LR = LiveRegs[rx]; // This useless DomainValue could have been missed above. if (!LR.Value->getCommonDomains(available)) { kill(rx); continue; } // Sorted insertion. bool Inserted = false; for (SmallVector<LiveReg, 4>::iterator i = Regs.begin(), e = Regs.end(); i != e && !Inserted; ++i) { if (LR.Def < i->Def) { Inserted = true; Regs.insert(i, LR); } } if (!Inserted) Regs.push_back(LR); } // doms are now sorted in order of appearance. Try to merge them all, giving // priority to the latest ones. DomainValue *dv = 0; while (!Regs.empty()) { if (!dv) { dv = Regs.pop_back_val().Value; // Force the first dv to match the current instruction. dv->AvailableDomains = dv->getCommonDomains(available); assert(dv->AvailableDomains && "Domain should have been filtered"); continue; } DomainValue *Latest = Regs.pop_back_val().Value; // Skip already merged values. if (Latest == dv || Latest->Next) continue; if (merge(dv, Latest)) continue; // If latest didn't merge, it is useless now. Kill all registers using it. for (SmallVector<int,4>::iterator i=used.begin(), e=used.end(); i != e; ++i) if (LiveRegs[*i].Value == Latest) kill(*i); } // dv is the DomainValue we are going to use for this instruction. if (!dv) { dv = alloc(); dv->AvailableDomains = available; } dv->Instrs.push_back(mi); // Finally set all defs and non-collapsed uses to dv. We must iterate through // all the operators, including imp-def ones. for (MachineInstr::mop_iterator ii = mi->operands_begin(), ee = mi->operands_end(); ii != ee; ++ii) { MachineOperand &mo = *ii; if (!mo.isReg()) continue; int rx = regIndex(mo.getReg()); if (rx < 0) continue; if (!LiveRegs[rx].Value || (mo.isDef() && LiveRegs[rx].Value != dv)) { kill(rx); setLiveReg(rx, dv); } } }
// enterBasicBlock - Set up LiveRegs by merging predecessor live-out values. void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) { // Detect back-edges from predecessors we haven't processed yet. SeenUnknownBackEdge = false; // Reset instruction counter in each basic block. CurInstr = 0; // Set up LiveRegs to represent registers entering MBB. if (!LiveRegs) LiveRegs = new LiveReg[NumRegs]; // Default values are 'nothing happened a long time ago'. for (unsigned rx = 0; rx != NumRegs; ++rx) { LiveRegs[rx].Value = 0; LiveRegs[rx].Def = -(1 << 20); } // This is the entry block. if (MBB->pred_empty()) { for (MachineBasicBlock::livein_iterator i = MBB->livein_begin(), e = MBB->livein_end(); i != e; ++i) { int rx = regIndex(*i); if (rx < 0) continue; // Treat function live-ins as if they were defined just before the first // instruction. Usually, function arguments are set up immediately // before the call. LiveRegs[rx].Def = -1; } DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": entry\n"); return; } // Try to coalesce live-out registers from predecessors. for (MachineBasicBlock::const_pred_iterator pi = MBB->pred_begin(), pe = MBB->pred_end(); pi != pe; ++pi) { LiveOutMap::const_iterator fi = LiveOuts.find(*pi); if (fi == LiveOuts.end()) { SeenUnknownBackEdge = true; continue; } assert(fi->second && "Can't have NULL entries"); for (unsigned rx = 0; rx != NumRegs; ++rx) { // Use the most recent predecessor def for each register. LiveRegs[rx].Def = std::max(LiveRegs[rx].Def, fi->second[rx].Def); DomainValue *pdv = resolve(fi->second[rx].Value); if (!pdv) continue; if (!LiveRegs[rx].Value) { setLiveReg(rx, pdv); continue; } // We have a live DomainValue from more than one predecessor. if (LiveRegs[rx].Value->isCollapsed()) { // We are already collapsed, but predecessor is not. Force him. unsigned Domain = LiveRegs[rx].Value->getFirstDomain(); if (!pdv->isCollapsed() && pdv->hasDomain(Domain)) collapse(pdv, Domain); continue; } // Currently open, merge in predecessor. if (!pdv->isCollapsed()) merge(LiveRegs[rx].Value, pdv); else force(rx, pdv->getFirstDomain()); } } DEBUG(dbgs() << "BB#" << MBB->getNumber() << (SeenUnknownBackEdge ? ": incomplete\n" : ": all preds known\n")); }
/// Set up LiveRegs by merging predecessor live-out values. void ExeDepsFix::enterBasicBlock(MachineBasicBlock *MBB) { // Reset instruction counter in each basic block. CurInstr = 0; // Set up UndefReads to track undefined register reads. UndefReads.clear(); LiveRegSet.clear(); // Set up LiveRegs to represent registers entering MBB. if (!LiveRegs) LiveRegs = new LiveReg[NumRegs]; // Default values are 'nothing happened a long time ago'. for (unsigned rx = 0; rx != NumRegs; ++rx) { LiveRegs[rx].Value = nullptr; LiveRegs[rx].Def = -(1 << 20); } // This is the entry block. if (MBB->pred_empty()) { for (const auto &LI : MBB->liveins()) { for (int rx : regIndices(LI.PhysReg)) { // Treat function live-ins as if they were defined just before the first // instruction. Usually, function arguments are set up immediately // before the call. LiveRegs[rx].Def = -1; } } DEBUG(dbgs() << "BB#" << MBB->getNumber() << ": entry\n"); return; } // Try to coalesce live-out registers from predecessors. for (MachineBasicBlock::const_pred_iterator pi = MBB->pred_begin(), pe = MBB->pred_end(); pi != pe; ++pi) { auto fi = MBBInfos.find(*pi); assert(fi != MBBInfos.end() && "Should have pre-allocated MBBInfos for all MBBs"); LiveReg *Incoming = fi->second.OutRegs; // Incoming is null if this is a backedge from a BB // we haven't processed yet if (Incoming == nullptr) { continue; } for (unsigned rx = 0; rx != NumRegs; ++rx) { // Use the most recent predecessor def for each register. LiveRegs[rx].Def = std::max(LiveRegs[rx].Def, Incoming[rx].Def); DomainValue *pdv = resolve(Incoming[rx].Value); if (!pdv) continue; if (!LiveRegs[rx].Value) { setLiveReg(rx, pdv); continue; } // We have a live DomainValue from more than one predecessor. if (LiveRegs[rx].Value->isCollapsed()) { // We are already collapsed, but predecessor is not. Force it. unsigned Domain = LiveRegs[rx].Value->getFirstDomain(); if (!pdv->isCollapsed() && pdv->hasDomain(Domain)) collapse(pdv, Domain); continue; } // Currently open, merge in predecessor. if (!pdv->isCollapsed()) merge(LiveRegs[rx].Value, pdv); else force(rx, pdv->getFirstDomain()); } } DEBUG( dbgs() << "BB#" << MBB->getNumber() << (!isBlockDone(MBB) ? ": incomplete\n" : ": all preds known\n")); }