void ScheduleDAGInstrs::ComputeOperandLatency(SUnit *Def, SUnit *Use, SDep& dep) const { if (!InstrItins || InstrItins->isEmpty()) return; // For a data dependency with a known register... if ((dep.getKind() != SDep::Data) || (dep.getReg() == 0)) return; const unsigned Reg = dep.getReg(); // ... find the definition of the register in the defining // instruction MachineInstr *DefMI = Def->getInstr(); int DefIdx = DefMI->findRegisterDefOperandIdx(Reg); if (DefIdx != -1) { const MachineOperand &MO = DefMI->getOperand(DefIdx); if (MO.isReg() && MO.isImplicit() && DefIdx >= (int)DefMI->getDesc().getNumOperands()) { // This is an implicit def, getOperandLatency() won't return the correct // latency. e.g. // %D6<def>, %D7<def> = VLD1q16 %R2<kill>, 0, ..., %Q3<imp-def> // %Q1<def> = VMULv8i16 %Q1<kill>, %Q3<kill>, ... // What we want is to compute latency between def of %D6/%D7 and use of // %Q3 instead. unsigned Op2 = DefMI->findRegisterDefOperandIdx(Reg, false, true, TRI); if (DefMI->getOperand(Op2).isReg()) DefIdx = Op2; } MachineInstr *UseMI = Use->getInstr(); // For all uses of the register, calculate the maxmimum latency int Latency = -1; if (UseMI) { for (unsigned i = 0, e = UseMI->getNumOperands(); i != e; ++i) { const MachineOperand &MO = UseMI->getOperand(i); if (!MO.isReg() || !MO.isUse()) continue; unsigned MOReg = MO.getReg(); if (MOReg != Reg) continue; int UseCycle = TII->getOperandLatency(InstrItins, DefMI, DefIdx, UseMI, i); Latency = std::max(Latency, UseCycle); } } else { // UseMI is null, then it must be a scheduling barrier. if (!InstrItins || InstrItins->isEmpty()) return; unsigned DefClass = DefMI->getDesc().getSchedClass(); Latency = InstrItins->getOperandCycle(DefClass, DefIdx); } // If we found a latency, then replace the existing dependence latency. if (Latency >= 0) dep.setLatency(Latency); } }
/// getDepth - Computes depth of instructions in vector \InsInstr. /// /// \param InsInstrs is a vector of machine instructions /// \param InstrIdxForVirtReg is a dense map of virtual register to index /// of defining machine instruction in \p InsInstrs /// \param BlockTrace is a trace of machine instructions /// /// \returns Depth of last instruction in \InsInstrs ("NewRoot") unsigned MachineCombiner::getDepth(SmallVectorImpl<MachineInstr *> &InsInstrs, DenseMap<unsigned, unsigned> &InstrIdxForVirtReg, MachineTraceMetrics::Trace BlockTrace) { SmallVector<unsigned, 16> InstrDepth; assert(TSchedModel.hasInstrSchedModel() && "Missing machine model\n"); // Foreach instruction in in the new sequence compute the depth based on the // operands. Use the trace information when possible. For new operands which // are tracked in the InstrIdxForVirtReg map depth is looked up in InstrDepth for (auto *InstrPtr : InsInstrs) { // for each Use unsigned IDepth = 0; DEBUG(dbgs() << "NEW INSTR "; InstrPtr->dump(); dbgs() << "\n";); for (unsigned i = 0, e = InstrPtr->getNumOperands(); i != e; ++i) { const MachineOperand &MO = InstrPtr->getOperand(i); // Check for virtual register operand. if (!(MO.isReg() && TargetRegisterInfo::isVirtualRegister(MO.getReg()))) continue; if (!MO.isUse()) continue; unsigned DepthOp = 0; unsigned LatencyOp = 0; DenseMap<unsigned, unsigned>::iterator II = InstrIdxForVirtReg.find(MO.getReg()); if (II != InstrIdxForVirtReg.end()) { // Operand is new virtual register not in trace assert(II->second < InstrDepth.size() && "Bad Index"); MachineInstr *DefInstr = InsInstrs[II->second]; assert(DefInstr && "There must be a definition for a new virtual register"); DepthOp = InstrDepth[II->second]; LatencyOp = TSchedModel.computeOperandLatency( DefInstr, DefInstr->findRegisterDefOperandIdx(MO.getReg()), InstrPtr, InstrPtr->findRegisterUseOperandIdx(MO.getReg())); } else { MachineInstr *DefInstr = getOperandDef(MO); if (DefInstr) { DepthOp = BlockTrace.getInstrCycles(DefInstr).Depth; LatencyOp = TSchedModel.computeOperandLatency( DefInstr, DefInstr->findRegisterDefOperandIdx(MO.getReg()), InstrPtr, InstrPtr->findRegisterUseOperandIdx(MO.getReg())); } } IDepth = std::max(IDepth, DepthOp + LatencyOp); } InstrDepth.push_back(IDepth); }
/// isPartialRedef - Return true if the specified def at the specific index is /// partially re-defining the specified live interval. A common case of this is /// a definition of the sub-register. bool LiveIntervals::isPartialRedef(SlotIndex MIIdx, MachineOperand &MO, LiveInterval &interval) { if (!MO.getSubReg() || MO.isEarlyClobber()) return false; SlotIndex RedefIndex = MIIdx.getRegSlot(); const LiveRange *OldLR = interval.getLiveRangeContaining(RedefIndex.getRegSlot(true)); MachineInstr *DefMI = getInstructionFromIndex(OldLR->valno->def); if (DefMI != 0) { return DefMI->findRegisterDefOperandIdx(interval.reg) != -1; } return false; }
/// addVRegUseDeps - Add a register data dependency if the instruction that /// defines the virtual register used at OperIdx is mapped to an SUnit. Add a /// register antidependency from this SUnit to instructions that occur later in /// the same scheduling region if they write the virtual register. /// /// TODO: Handle ExitSU "uses" properly. void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) { MachineInstr *MI = SU->getInstr(); unsigned Reg = MI->getOperand(OperIdx).getReg(); // Lookup this operand's reaching definition. assert(LIS && "vreg dependencies requires LiveIntervals"); LiveRangeQuery LRQ(LIS->getInterval(Reg), LIS->getInstructionIndex(MI)); VNInfo *VNI = LRQ.valueIn(); // VNI will be valid because MachineOperand::readsReg() is checked by caller. assert(VNI && "No value to read by operand"); MachineInstr *Def = LIS->getInstructionFromIndex(VNI->def); // Phis and other noninstructions (after coalescing) have a NULL Def. if (Def) { SUnit *DefSU = getSUnit(Def); if (DefSU) { // The reaching Def lives within this scheduling region. // Create a data dependence. // // TODO: Handle "special" address latencies cleanly. SDep dep(DefSU, SDep::Data, DefSU->Latency, Reg); if (!UnitLatencies) { // Adjust the dependence latency using operand def/use information, then // allow the target to perform its own adjustments. int DefOp = Def->findRegisterDefOperandIdx(Reg); unsigned Latency = TII->computeOperandLatency(InstrItins, Def, DefOp, MI, OperIdx); dep.setLatency(Latency); unsigned MinLatency = TII->computeOperandLatency(InstrItins, Def, DefOp, MI, OperIdx, /*FindMin=*/true); dep.setMinLatency(MinLatency); const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>(); ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep)); } SU->addPred(dep); } } // Add antidependence to the following def of the vreg it uses. VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg); if (DefI != VRegDefs.end() && DefI->SU != SU) DefI->SU->addPred(SDep(SU, SDep::Anti, 0, Reg)); }
void ScheduleDAGInstrs::ComputeOperandLatency(SUnit *Def, SUnit *Use, SDep& dep) const { const InstrItineraryData &InstrItins = TM.getInstrItineraryData(); if (InstrItins.isEmpty()) return; // For a data dependency with a known register... if ((dep.getKind() != SDep::Data) || (dep.getReg() == 0)) return; const unsigned Reg = dep.getReg(); // ... find the definition of the register in the defining // instruction MachineInstr *DefMI = Def->getInstr(); int DefIdx = DefMI->findRegisterDefOperandIdx(Reg); if (DefIdx != -1) { int DefCycle = InstrItins.getOperandCycle(DefMI->getDesc().getSchedClass(), DefIdx); if (DefCycle >= 0) { MachineInstr *UseMI = Use->getInstr(); const unsigned UseClass = UseMI->getDesc().getSchedClass(); // For all uses of the register, calculate the maxmimum latency int Latency = -1; for (unsigned i = 0, e = UseMI->getNumOperands(); i != e; ++i) { const MachineOperand &MO = UseMI->getOperand(i); if (!MO.isReg() || !MO.isUse()) continue; unsigned MOReg = MO.getReg(); if (MOReg != Reg) continue; int UseCycle = InstrItins.getOperandCycle(UseClass, i); if (UseCycle >= 0) Latency = std::max(Latency, DefCycle - UseCycle + 1); } // If we found a latency, then replace the existing dependence latency. if (Latency >= 0) dep.setLatency(Latency); } } }
// The CC users in CCUsers are testing the result of a comparison of some // value X against zero and we know that any CC value produced by MI // would also reflect the value of X. Try to adjust CCUsers so that // they test the result of MI directly, returning true on success. // Leave everything unchanged on failure. bool SystemZElimCompare::adjustCCMasksForInstr( MachineInstr &MI, MachineInstr &Compare, SmallVectorImpl<MachineInstr *> &CCUsers) { int Opcode = MI.getOpcode(); const MCInstrDesc &Desc = TII->get(Opcode); unsigned MIFlags = Desc.TSFlags; // See which compare-style condition codes are available. unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags); // For unsigned comparisons with zero, only equality makes sense. unsigned CompareFlags = Compare.getDesc().TSFlags; if (CompareFlags & SystemZII::IsLogical) ReusableCCMask &= SystemZ::CCMASK_CMP_EQ; if (ReusableCCMask == 0) return false; unsigned CCValues = SystemZII::getCCValues(MIFlags); assert((ReusableCCMask & ~CCValues) == 0 && "Invalid CCValues"); // Now check whether these flags are enough for all users. SmallVector<MachineOperand *, 4> AlterMasks; for (unsigned int I = 0, E = CCUsers.size(); I != E; ++I) { MachineInstr *MI = CCUsers[I]; // Fail if this isn't a use of CC that we understand. unsigned Flags = MI->getDesc().TSFlags; unsigned FirstOpNum; if (Flags & SystemZII::CCMaskFirst) FirstOpNum = 0; else if (Flags & SystemZII::CCMaskLast) FirstOpNum = MI->getNumExplicitOperands() - 2; else return false; // Check whether the instruction predicate treats all CC values // outside of ReusableCCMask in the same way. In that case it // doesn't matter what those CC values mean. unsigned CCValid = MI->getOperand(FirstOpNum).getImm(); unsigned CCMask = MI->getOperand(FirstOpNum + 1).getImm(); unsigned OutValid = ~ReusableCCMask & CCValid; unsigned OutMask = ~ReusableCCMask & CCMask; if (OutMask != 0 && OutMask != OutValid) return false; AlterMasks.push_back(&MI->getOperand(FirstOpNum)); AlterMasks.push_back(&MI->getOperand(FirstOpNum + 1)); } // All users are OK. Adjust the masks for MI. for (unsigned I = 0, E = AlterMasks.size(); I != E; I += 2) { AlterMasks[I]->setImm(CCValues); unsigned CCMask = AlterMasks[I + 1]->getImm(); if (CCMask & ~ReusableCCMask) AlterMasks[I + 1]->setImm((CCMask & ReusableCCMask) | (CCValues & ~ReusableCCMask)); } // CC is now live after MI. int CCDef = MI.findRegisterDefOperandIdx(SystemZ::CC, false, true, TRI); assert(CCDef >= 0 && "Couldn't find CC set"); MI.getOperand(CCDef).setIsDead(false); // Clear any intervening kills of CC. MachineBasicBlock::iterator MBBI = MI, MBBE = Compare; for (++MBBI; MBBI != MBBE; ++MBBI) MBBI->clearRegisterKills(SystemZ::CC, TRI); return true; }