void ScheduleDAGInstrs::ComputeOperandLatency(SUnit *Def, SUnit *Use,
SDep& dep) const {
if (!InstrItins || InstrItins->isEmpty())
return;
// For a data dependency with a known register...
if ((dep.getKind() != SDep::Data) || (dep.getReg() == 0))
return;
const unsigned Reg = dep.getReg();
// ... find the definition of the register in the defining
// instruction
MachineInstr *DefMI = Def->getInstr();
int DefIdx = DefMI->findRegisterDefOperandIdx(Reg);
if (DefIdx != -1) {
const MachineOperand &MO = DefMI->getOperand(DefIdx);
if (MO.isReg() && MO.isImplicit() &&
DefIdx >= (int)DefMI->getDesc().getNumOperands()) {
// This is an implicit def, getOperandLatency() won't return the correct
// latency. e.g.
// %D6<def>, %D7<def> = VLD1q16 %R2<kill>, 0, ..., %Q3<imp-def>
// %Q1<def> = VMULv8i16 %Q1<kill>, %Q3<kill>, ...
// What we want is to compute latency between def of %D6/%D7 and use of
// %Q3 instead.
unsigned Op2 = DefMI->findRegisterDefOperandIdx(Reg, false, true, TRI);
if (DefMI->getOperand(Op2).isReg())
DefIdx = Op2;
}
MachineInstr *UseMI = Use->getInstr();
// For all uses of the register, calculate the maxmimum latency
int Latency = -1;
if (UseMI) {
for (unsigned i = 0, e = UseMI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = UseMI->getOperand(i);
if (!MO.isReg() || !MO.isUse())
continue;
unsigned MOReg = MO.getReg();
if (MOReg != Reg)
continue;
int UseCycle = TII->getOperandLatency(InstrItins, DefMI, DefIdx,
UseMI, i);
Latency = std::max(Latency, UseCycle);
}
} else {
// UseMI is null, then it must be a scheduling barrier.
if (!InstrItins || InstrItins->isEmpty())
return;
unsigned DefClass = DefMI->getDesc().getSchedClass();
Latency = InstrItins->getOperandCycle(DefClass, DefIdx);
}
// If we found a latency, then replace the existing dependence latency.
if (Latency >= 0)
dep.setLatency(Latency);
}
}
/// getDepth - Computes depth of instructions in vector \InsInstr.
///
/// \param InsInstrs is a vector of machine instructions
/// \param InstrIdxForVirtReg is a dense map of virtual register to index
/// of defining machine instruction in \p InsInstrs
/// \param BlockTrace is a trace of machine instructions
///
/// \returns Depth of last instruction in \InsInstrs ("NewRoot")
unsigned
MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs,
DenseMap<unsigned, unsigned> &InstrIdxForVirtReg,
MachineTraceMetrics::Trace BlockTrace) {
SmallVector<unsigned, 16> InstrDepth;
assert(TSchedModel.hasInstrSchedModel() && "Missing machine model\n");
// Foreach instruction in in the new sequence compute the depth based on the
// operands. Use the trace information when possible. For new operands which
// are tracked in the InstrIdxForVirtReg map depth is looked up in InstrDepth
for (auto *InstrPtr : InsInstrs) { // for each Use
unsigned IDepth = 0;
DEBUG(dbgs() << "NEW INSTR "; InstrPtr->dump(); dbgs() << "\n";);
for (unsigned i = 0, e = InstrPtr->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = InstrPtr->getOperand(i);
// Check for virtual register operand.
if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg())))
continue;
if (!MO.isUse())
continue;
unsigned DepthOp = 0;
unsigned LatencyOp = 0;
DenseMap<unsigned, unsigned>::iterator II =
InstrIdxForVirtReg.find(MO.getReg());
if (II != InstrIdxForVirtReg.end()) {
// Operand is new virtual register not in trace
assert(II->second < InstrDepth.size() && "Bad Index");
MachineInstr *DefInstr = InsInstrs[II->second];
assert(DefInstr &&
"There must be a definition for a new virtual register");
DepthOp = InstrDepth[II->second];
LatencyOp = TSchedModel.computeOperandLatency(
DefInstr, DefInstr->findRegisterDefOperandIdx(MO.getReg()),
InstrPtr, InstrPtr->findRegisterUseOperandIdx(MO.getReg()));
} else {
MachineInstr *DefInstr = getOperandDef(MO);
if (DefInstr) {
DepthOp = BlockTrace.getInstrCycles(DefInstr).Depth;
LatencyOp = TSchedModel.computeOperandLatency(
DefInstr, DefInstr->findRegisterDefOperandIdx(MO.getReg()),
InstrPtr, InstrPtr->findRegisterUseOperandIdx(MO.getReg()));
}
}
IDepth = std::max(IDepth, DepthOp + LatencyOp);
}
InstrDepth.push_back(IDepth);
}
/// isPartialRedef - Return true if the specified def at the specific index is
/// partially re-defining the specified live interval. A common case of this is
/// a definition of the sub-register.
bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO,
LiveInterval &interval) {
if (!MO.getSubReg() || MO.isEarlyClobber())
return false;
SlotIndex RedefIndex = MIIdx.getRegSlot();
const LiveRange *OldLR =
interval.getLiveRangeContaining(RedefIndex.getRegSlot(true));
MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def);
if (DefMI != 0) {
return DefMI->findRegisterDefOperandIdx(interval.reg) != -1;
}
return false;
}
/// addVRegUseDeps - Add a register data dependency if the instruction that
/// defines the virtual register used at OperIdx is mapped to an SUnit. Add a
/// register antidependency from this SUnit to instructions that occur later in
/// the same scheduling region if they write the virtual register.
///
/// TODO: Handle ExitSU "uses" properly.
void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
MachineInstr *MI = SU->getInstr();
unsigned Reg = MI->getOperand(OperIdx).getReg();
// Lookup this operand's reaching definition.
assert(LIS && "vreg dependencies requires LiveIntervals");
LiveRangeQuery LRQ(LIS->getInterval(Reg), LIS->getInstructionIndex(MI));
VNInfo *VNI = LRQ.valueIn();
// VNI will be valid because MachineOperand::readsReg() is checked by caller.
assert(VNI && "No value to read by operand");
MachineInstr *Def = LIS->getInstructionFromIndex(VNI->def);
// Phis and other noninstructions (after coalescing) have a NULL Def.
if (Def) {
SUnit *DefSU = getSUnit(Def);
if (DefSU) {
// The reaching Def lives within this scheduling region.
// Create a data dependence.
//
// TODO: Handle "special" address latencies cleanly.
SDep dep(DefSU, SDep::Data, DefSU->Latency, Reg);
if (!UnitLatencies) {
// Adjust the dependence latency using operand def/use information, then
// allow the target to perform its own adjustments.
int DefOp = Def->findRegisterDefOperandIdx(Reg);
unsigned Latency =
TII->computeOperandLatency(InstrItins, Def, DefOp, MI, OperIdx);
dep.setLatency(Latency);
unsigned MinLatency =
TII->computeOperandLatency(InstrItins, Def, DefOp, MI, OperIdx,
/*FindMin=*/true);
dep.setMinLatency(MinLatency);
const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>();
ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep));
}
SU->addPred(dep);
}
}
// Add antidependence to the following def of the vreg it uses.
VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg);
if (DefI != VRegDefs.end() && DefI->SU != SU)
DefI->SU->addPred(SDep(SU, SDep::Anti, 0, Reg));
}
void ScheduleDAGInstrs::ComputeOperandLatency(SUnit *Def, SUnit *Use,
SDep& dep) const {
const InstrItineraryData &InstrItins = TM.getInstrItineraryData();
if (InstrItins.isEmpty())
return;
// For a data dependency with a known register...
if ((dep.getKind() != SDep::Data) || (dep.getReg() == 0))
return;
const unsigned Reg = dep.getReg();
// ... find the definition of the register in the defining
// instruction
MachineInstr *DefMI = Def->getInstr();
int DefIdx = DefMI->findRegisterDefOperandIdx(Reg);
if (DefIdx != -1) {
int DefCycle = InstrItins.getOperandCycle(DefMI->getDesc().getSchedClass(),
DefIdx);
if (DefCycle >= 0) {
MachineInstr *UseMI = Use->getInstr();
const unsigned UseClass = UseMI->getDesc().getSchedClass();
// For all uses of the register, calculate the maxmimum latency
int Latency = -1;
for (unsigned i = 0, e = UseMI->getNumOperands(); i != e; ++i) {
const MachineOperand &MO = UseMI->getOperand(i);
if (!MO.isReg() || !MO.isUse())
continue;
unsigned MOReg = MO.getReg();
if (MOReg != Reg)
continue;
int UseCycle = InstrItins.getOperandCycle(UseClass, i);
if (UseCycle >= 0)
Latency = std::max(Latency, DefCycle - UseCycle + 1);
}
// If we found a latency, then replace the existing dependence latency.
if (Latency >= 0)
dep.setLatency(Latency);
}
}
}
// The CC users in CCUsers are testing the result of a comparison of some
// value X against zero and we know that any CC value produced by MI
// would also reflect the value of X. Try to adjust CCUsers so that
// they test the result of MI directly, returning true on success.
// Leave everything unchanged on failure.
bool SystemZElimCompare::adjustCCMasksForInstr(
MachineInstr &MI, MachineInstr &Compare,
SmallVectorImpl<MachineInstr *> &CCUsers) {
int Opcode = MI.getOpcode();
const MCInstrDesc &Desc = TII->get(Opcode);
unsigned MIFlags = Desc.TSFlags;
// See which compare-style condition codes are available.
unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags);
// For unsigned comparisons with zero, only equality makes sense.
unsigned CompareFlags = Compare.getDesc().TSFlags;
if (CompareFlags & SystemZII::IsLogical)
ReusableCCMask &= SystemZ::CCMASK_CMP_EQ;
if (ReusableCCMask == 0)
return false;
unsigned CCValues = SystemZII::getCCValues(MIFlags);
assert((ReusableCCMask & ~CCValues) == 0 && "Invalid CCValues");
// Now check whether these flags are enough for all users.
SmallVector<MachineOperand *, 4> AlterMasks;
for (unsigned int I = 0, E = CCUsers.size(); I != E; ++I) {
MachineInstr *MI = CCUsers[I];
// Fail if this isn't a use of CC that we understand.
unsigned Flags = MI->getDesc().TSFlags;
unsigned FirstOpNum;
if (Flags & SystemZII::CCMaskFirst)
FirstOpNum = 0;
else if (Flags & SystemZII::CCMaskLast)
FirstOpNum = MI->getNumExplicitOperands() - 2;
else
return false;
// Check whether the instruction predicate treats all CC values
// outside of ReusableCCMask in the same way. In that case it
// doesn't matter what those CC values mean.
unsigned CCValid = MI->getOperand(FirstOpNum).getImm();
unsigned CCMask = MI->getOperand(FirstOpNum + 1).getImm();
unsigned OutValid = ~ReusableCCMask & CCValid;
unsigned OutMask = ~ReusableCCMask & CCMask;
if (OutMask != 0 && OutMask != OutValid)
return false;
AlterMasks.push_back(&MI->getOperand(FirstOpNum));
AlterMasks.push_back(&MI->getOperand(FirstOpNum + 1));
}
// All users are OK. Adjust the masks for MI.
for (unsigned I = 0, E = AlterMasks.size(); I != E; I += 2) {
AlterMasks[I]->setImm(CCValues);
unsigned CCMask = AlterMasks[I + 1]->getImm();
if (CCMask & ~ReusableCCMask)
AlterMasks[I + 1]->setImm((CCMask & ReusableCCMask) |
(CCValues & ~ReusableCCMask));
}
// CC is now live after MI.
int CCDef = MI.findRegisterDefOperandIdx(SystemZ::CC, false, true, TRI);
assert(CCDef >= 0 && "Couldn't find CC set");
MI.getOperand(CCDef).setIsDead(false);
// Clear any intervening kills of CC.
MachineBasicBlock::iterator MBBI = MI, MBBE = Compare;
for (++MBBI; MBBI != MBBE; ++MBBI)
MBBI->clearRegisterKills(SystemZ::CC, TRI);
return true;
}
