void ScheduleDAGLinearize::ScheduleNode(SDNode *N) { if (N->getNodeId() != 0) llvm_unreachable(nullptr); if (!N->isMachineOpcode() && (N->getOpcode() == ISD::EntryToken || isPassiveNode(N))) // These nodes do not need to be translated into MIs. return; LLVM_DEBUG(dbgs() << "\n*** Scheduling: "); LLVM_DEBUG(N->dump(DAG)); Sequence.push_back(N); unsigned NumOps = N->getNumOperands(); if (unsigned NumLeft = NumOps) { SDNode *GluedOpN = nullptr; do { const SDValue &Op = N->getOperand(NumLeft-1); SDNode *OpN = Op.getNode(); if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) { // Schedule glue operand right above N. GluedOpN = OpN; assert(OpN->getNodeId() != 0 && "Glue operand not ready?"); OpN->setNodeId(0); ScheduleNode(OpN); continue; } if (OpN == GluedOpN) // Glue operand is already scheduled. continue; DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN); if (DI != GluedMap.end() && DI->second != N) // Users of glues are counted against the glued users. OpN = DI->second; unsigned Degree = OpN->getNodeId(); assert(Degree > 0 && "Predecessor over-released!"); OpN->setNodeId(--Degree); if (Degree == 0) ScheduleNode(OpN); } while (--NumLeft); } }
void ScheduleDAGLinearize::Schedule() { LLVM_DEBUG(dbgs() << "********** DAG Linearization **********\n"); SmallVector<SDNode*, 8> Glues; unsigned DAGSize = 0; for (SDNode &Node : DAG->allnodes()) { SDNode *N = &Node; // Use node id to record degree. unsigned Degree = N->use_size(); N->setNodeId(Degree); unsigned NumVals = N->getNumValues(); if (NumVals && N->getValueType(NumVals-1) == MVT::Glue && N->hasAnyUseOfValue(NumVals-1)) { SDNode *User = findGluedUser(N); if (User) { Glues.push_back(N); GluedMap.insert(std::make_pair(N, User)); } } if (N->isMachineOpcode() || (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N))) ++DAGSize; } for (unsigned i = 0, e = Glues.size(); i != e; ++i) { SDNode *Glue = Glues[i]; SDNode *GUser = GluedMap[Glue]; unsigned Degree = Glue->getNodeId(); unsigned UDegree = GUser->getNodeId(); // Glue user must be scheduled together with the glue operand. So other // users of the glue operand must be treated as its users. SDNode *ImmGUser = Glue->getGluedUser(); for (const SDNode *U : Glue->uses()) if (U == ImmGUser) --Degree; GUser->setNodeId(UDegree + Degree); Glue->setNodeId(1); } Sequence.reserve(DAGSize); ScheduleNode(DAG->getRoot().getNode()); }
/// This is the main entry point for the type legalizer. This does a top-down /// traversal of the dag, legalizing types as it goes. Returns "true" if it made /// any changes. bool DAGTypeLegalizer::run() { bool Changed = false; // Create a dummy node (which is not added to allnodes), that adds a reference // to the root node, preventing it from being deleted, and tracking any // changes of the root. HandleSDNode Dummy(DAG.getRoot()); Dummy.setNodeId(Unanalyzed); // The root of the dag may dangle to deleted nodes until the type legalizer is // done. Set it to null to avoid confusion. DAG.setRoot(SDValue()); // Walk all nodes in the graph, assigning them a NodeId of 'ReadyToProcess' // (and remembering them) if they are leaves and assigning 'Unanalyzed' if // non-leaves. for (SDNode &Node : DAG.allnodes()) { if (Node.getNumOperands() == 0) { AddToWorklist(&Node); } else { Node.setNodeId(Unanalyzed); } } // Now that we have a set of nodes to process, handle them all. while (!Worklist.empty()) { #ifndef EXPENSIVE_CHECKS if (EnableExpensiveChecks) #endif PerformExpensiveChecks(); SDNode *N = Worklist.back(); Worklist.pop_back(); assert(N->getNodeId() == ReadyToProcess && "Node should be ready if on worklist!"); if (IgnoreNodeResults(N)) goto ScanOperands; // Scan the values produced by the node, checking to see if any result // types are illegal. for (unsigned i = 0, NumResults = N->getNumValues(); i < NumResults; ++i) { EVT ResultVT = N->getValueType(i); switch (getTypeAction(ResultVT)) { case TargetLowering::TypeLegal: break; // The following calls must take care of *all* of the node's results, // not just the illegal result they were passed (this includes results // with a legal type). Results can be remapped using ReplaceValueWith, // or their promoted/expanded/etc values registered in PromotedIntegers, // ExpandedIntegers etc. case TargetLowering::TypePromoteInteger: PromoteIntegerResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypeExpandInteger: ExpandIntegerResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypeSoftenFloat: Changed = SoftenFloatResult(N, i); if (Changed) goto NodeDone; // If not changed, the result type should be legally in register. assert(isLegalInHWReg(ResultVT) && "Unchanged SoftenFloatResult should be legal in register!"); goto ScanOperands; case TargetLowering::TypeExpandFloat: ExpandFloatResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypeScalarizeVector: ScalarizeVectorResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypeSplitVector: SplitVectorResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypeWidenVector: WidenVectorResult(N, i); Changed = true; goto NodeDone; case TargetLowering::TypePromoteFloat: PromoteFloatResult(N, i); Changed = true; goto NodeDone; } } ScanOperands: // Scan the operand list for the node, handling any nodes with operands that // are illegal. { unsigned NumOperands = N->getNumOperands(); bool NeedsReanalyzing = false; unsigned i; for (i = 0; i != NumOperands; ++i) { if (IgnoreNodeResults(N->getOperand(i).getNode())) continue; EVT OpVT = N->getOperand(i).getValueType(); switch (getTypeAction(OpVT)) { case TargetLowering::TypeLegal: continue; // The following calls must either replace all of the node's results // using ReplaceValueWith, and return "false"; or update the node's // operands in place, and return "true". case TargetLowering::TypePromoteInteger: NeedsReanalyzing = PromoteIntegerOperand(N, i); Changed = true; break; case TargetLowering::TypeExpandInteger: NeedsReanalyzing = ExpandIntegerOperand(N, i); Changed = true; break; case TargetLowering::TypeSoftenFloat: NeedsReanalyzing = SoftenFloatOperand(N, i); Changed = true; break; case TargetLowering::TypeExpandFloat: NeedsReanalyzing = ExpandFloatOperand(N, i); Changed = true; break; case TargetLowering::TypeScalarizeVector: NeedsReanalyzing = ScalarizeVectorOperand(N, i); Changed = true; break; case TargetLowering::TypeSplitVector: NeedsReanalyzing = SplitVectorOperand(N, i); Changed = true; break; case TargetLowering::TypeWidenVector: NeedsReanalyzing = WidenVectorOperand(N, i); Changed = true; break; case TargetLowering::TypePromoteFloat: NeedsReanalyzing = PromoteFloatOperand(N, i); Changed = true; break; } break; } // The sub-method updated N in place. Check to see if any operands are new, // and if so, mark them. If the node needs revisiting, don't add all users // to the worklist etc. if (NeedsReanalyzing) { assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?"); N->setNodeId(NewNode); // Recompute the NodeId and correct processed operands, adding the node to // the worklist if ready. SDNode *M = AnalyzeNewNode(N); if (M == N) // The node didn't morph - nothing special to do, it will be revisited. continue; // The node morphed - this is equivalent to legalizing by replacing every // value of N with the corresponding value of M. So do that now. assert(N->getNumValues() == M->getNumValues() && "Node morphing changed the number of results!"); for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) // Replacing the value takes care of remapping the new value. ReplaceValueWith(SDValue(N, i), SDValue(M, i)); assert(N->getNodeId() == NewNode && "Unexpected node state!"); // The node continues to live on as part of the NewNode fungus that // grows on top of the useful nodes. Nothing more needs to be done // with it - move on to the next node. continue; } if (i == NumOperands) { DEBUG(dbgs() << "Legally typed node: "; N->dump(&DAG); dbgs() << "\n"); } } NodeDone: // If we reach here, the node was processed, potentially creating new nodes. // Mark it as processed and add its users to the worklist as appropriate. assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?"); N->setNodeId(Processed); for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) { SDNode *User = *UI; int NodeId = User->getNodeId(); // This node has two options: it can either be a new node or its Node ID // may be a count of the number of operands it has that are not ready. if (NodeId > 0) { User->setNodeId(NodeId-1); // If this was the last use it was waiting on, add it to the ready list. if (NodeId-1 == ReadyToProcess) Worklist.push_back(User); continue; } // If this is an unreachable new node, then ignore it. If it ever becomes // reachable by being used by a newly created node then it will be handled // by AnalyzeNewNode. if (NodeId == NewNode) continue; // Otherwise, this node is new: this is the first operand of it that // became ready. Its new NodeId is the number of operands it has minus 1 // (as this node is now processed). assert(NodeId == Unanalyzed && "Unknown node ID!"); User->setNodeId(User->getNumOperands() - 1); // If the node only has a single operand, it is now ready. if (User->getNumOperands() == 1) Worklist.push_back(User); } }
/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled /// successors to the newly created node. SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) { if (SU->getNode()->getGluedNode()) return nullptr; SDNode *N = SU->getNode(); if (!N) return nullptr; SUnit *NewSU; bool TryUnfold = false; for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) { MVT VT = N->getSimpleValueType(i); if (VT == MVT::Glue) return nullptr; else if (VT == MVT::Other) TryUnfold = true; } for (const SDValue &Op : N->op_values()) { MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo()); if (VT == MVT::Glue) return nullptr; } if (TryUnfold) { SmallVector<SDNode*, 2> NewNodes; if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes)) return nullptr; LLVM_DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n"); assert(NewNodes.size() == 2 && "Expected a load folding node!"); N = NewNodes[1]; SDNode *LoadNode = NewNodes[0]; unsigned NumVals = N->getNumValues(); unsigned OldNumVals = SU->getNode()->getNumValues(); for (unsigned i = 0; i != NumVals; ++i) DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i)); DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1), SDValue(LoadNode, 1)); SUnit *NewSU = newSUnit(N); assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NewSU->NodeNum); const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { NewSU->isTwoAddress = true; break; } } if (MCID.isCommutable()) NewSU->isCommutable = true; // LoadNode may already exist. This can happen when there is another // load from the same location and producing the same type of value // but it has different alignment or volatileness. bool isNewLoad = true; SUnit *LoadSU; if (LoadNode->getNodeId() != -1) { LoadSU = &SUnits[LoadNode->getNodeId()]; isNewLoad = false; } else { LoadSU = newSUnit(LoadNode); LoadNode->setNodeId(LoadSU->NodeNum); } SDep ChainPred; SmallVector<SDep, 4> ChainSuccs; SmallVector<SDep, 4> LoadPreds; SmallVector<SDep, 4> NodePreds; SmallVector<SDep, 4> NodeSuccs; for (SDep &Pred : SU->Preds) { if (Pred.isCtrl()) ChainPred = Pred; else if (Pred.getSUnit()->getNode() && Pred.getSUnit()->getNode()->isOperandOf(LoadNode)) LoadPreds.push_back(Pred); else NodePreds.push_back(Pred); } for (SDep &Succ : SU->Succs) { if (Succ.isCtrl()) ChainSuccs.push_back(Succ); else NodeSuccs.push_back(Succ); } if (ChainPred.getSUnit()) { RemovePred(SU, ChainPred); if (isNewLoad) AddPred(LoadSU, ChainPred); } for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) { const SDep &Pred = LoadPreds[i]; RemovePred(SU, Pred); if (isNewLoad) { AddPred(LoadSU, Pred); } } for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) { const SDep &Pred = NodePreds[i]; RemovePred(SU, Pred); AddPred(NewSU, Pred); } for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) { SDep D = NodeSuccs[i]; SUnit *SuccDep = D.getSUnit(); D.setSUnit(SU); RemovePred(SuccDep, D); D.setSUnit(NewSU); AddPred(SuccDep, D); } for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) { SDep D = ChainSuccs[i]; SUnit *SuccDep = D.getSUnit(); D.setSUnit(SU); RemovePred(SuccDep, D); if (isNewLoad) { D.setSUnit(LoadSU); AddPred(SuccDep, D); } } if (isNewLoad) { SDep D(LoadSU, SDep::Barrier); D.setLatency(LoadSU->Latency); AddPred(NewSU, D); } ++NumUnfolds; if (NewSU->NumSuccsLeft == 0) { NewSU->isAvailable = true; return NewSU; } SU = NewSU; } LLVM_DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n"); NewSU = Clone(SU); // New SUnit has the exact same predecessors. for (SDep &Pred : SU->Preds) if (!Pred.isArtificial()) AddPred(NewSU, Pred); // Only copy scheduled successors. Cut them from old node's successor // list and move them over. SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps; for (SDep &Succ : SU->Succs) { if (Succ.isArtificial()) continue; SUnit *SuccSU = Succ.getSUnit(); if (SuccSU->isScheduled) { SDep D = Succ; D.setSUnit(NewSU); AddPred(SuccSU, D); D.setSUnit(SU); DelDeps.push_back(std::make_pair(SuccSU, D)); } } for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) RemovePred(DelDeps[i].first, DelDeps[i].second); ++NumDups; return NewSU; }
void ScheduleDAGSDNodes::AddSchedEdges() { const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>(); // Check to see if the scheduler cares about latencies. bool UnitLatencies = forceUnitLatencies(); // Pass 2: add the preds, succs, etc. for (unsigned su = 0, e = SUnits.size(); su != e; ++su) { SUnit *SU = &SUnits[su]; SDNode *MainNode = SU->getNode(); if (MainNode->isMachineOpcode()) { unsigned Opc = MainNode->getMachineOpcode(); const MCInstrDesc &MCID = TII->get(Opc); for (unsigned i = 0; i != MCID.getNumOperands(); ++i) { if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) { SU->isTwoAddress = true; break; } } if (MCID.isCommutable()) SU->isCommutable = true; } // Find all predecessors and successors of the group. for (SDNode *N = SU->getNode(); N; N = N->getGluedNode()) { if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).getImplicitDefs()) { SU->hasPhysRegClobbers = true; unsigned NumUsed = InstrEmitter::CountResults(N); while (NumUsed != 0 && !N->hasAnyUseOfValue(NumUsed - 1)) --NumUsed; // Skip over unused values at the end. if (NumUsed > TII->get(N->getMachineOpcode()).getNumDefs()) SU->hasPhysRegDefs = true; } for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { SDNode *OpN = N->getOperand(i).getNode(); if (isPassiveNode(OpN)) continue; // Not scheduled. SUnit *OpSU = &SUnits[OpN->getNodeId()]; assert(OpSU && "Node has no SUnit!"); if (OpSU == SU) continue; // In the same group. EVT OpVT = N->getOperand(i).getValueType(); assert(OpVT != MVT::Glue && "Glued nodes should be in same sunit!"); bool isChain = OpVT == MVT::Other; unsigned PhysReg = 0; int Cost = 1; // Determine if this is a physical register dependency. CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost); assert((PhysReg == 0 || !isChain) && "Chain dependence via physreg data?"); // FIXME: See ScheduleDAGSDNodes::EmitCopyFromReg. For now, scheduler // emits a copy from the physical register to a virtual register unless // it requires a cross class copy (cost < 0). That means we are only // treating "expensive to copy" register dependency as physical register // dependency. This may change in the future though. if (Cost >= 0 && !StressSched) PhysReg = 0; // If this is a ctrl dep, latency is 1. unsigned OpLatency = isChain ? 1 : OpSU->Latency; // Special-case TokenFactor chains as zero-latency. if(isChain && OpN->getOpcode() == ISD::TokenFactor) OpLatency = 0; const SDep &dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data, OpLatency, PhysReg); if (!isChain && !UnitLatencies) { computeOperandLatency(OpN, N, i, const_cast<SDep &>(dep)); ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep)); } if (!SU->addPred(dep) && !dep.isCtrl() && OpSU->NumRegDefsLeft > 1) { // Multiple register uses are combined in the same SUnit. For example, // we could have a set of glued nodes with all their defs consumed by // another set of glued nodes. Register pressure tracking sees this as // a single use, so to keep pressure balanced we reduce the defs. // // We can't tell (without more book-keeping) if this results from // glued nodes or duplicate operands. As long as we don't reduce // NumRegDefsLeft to zero, we handle the common cases well. --OpSU->NumRegDefsLeft; } } } } }
void ScheduleDAGSDNodes::BuildSchedUnits() { // During scheduling, the NodeId field of SDNode is used to map SDNodes // to their associated SUnits by holding SUnits table indices. A value // of -1 means the SDNode does not yet have an associated SUnit. unsigned NumNodes = 0; for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), E = DAG->allnodes_end(); NI != E; ++NI) { NI->setNodeId(-1); ++NumNodes; } // Reserve entries in the vector for each of the SUnits we are creating. This // ensure that reallocation of the vector won't happen, so SUnit*'s won't get // invalidated. // FIXME: Multiply by 2 because we may clone nodes during scheduling. // This is a temporary workaround. SUnits.reserve(NumNodes * 2); // Add all nodes in depth first order. SmallVector<SDNode*, 64> Worklist; SmallPtrSet<SDNode*, 64> Visited; Worklist.push_back(DAG->getRoot().getNode()); Visited.insert(DAG->getRoot().getNode()); SmallVector<SUnit*, 8> CallSUnits; while (!Worklist.empty()) { SDNode *NI = Worklist.pop_back_val(); // Add all operands to the worklist unless they've already been added. for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i) if (Visited.insert(NI->getOperand(i).getNode())) Worklist.push_back(NI->getOperand(i).getNode()); if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. continue; // If this node has already been processed, stop now. if (NI->getNodeId() != -1) continue; SUnit *NodeSUnit = newSUnit(NI); // See if anything is glued to this node, if so, add them to glued // nodes. Nodes can have at most one glue input and one glue output. Glue // is required to be the last operand and result of a node. // Scan up to find glued preds. SDNode *N = NI; while (N->getNumOperands() && N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue) { N = N->getOperand(N->getNumOperands()-1).getNode(); assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall()) NodeSUnit->isCall = true; } // Scan down to find any glued succs. N = NI; while (N->getValueType(N->getNumValues()-1) == MVT::Glue) { SDValue GlueVal(N, N->getNumValues()-1); // There are either zero or one users of the Glue result. bool HasGlueUse = false; for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) if (GlueVal.isOperandOf(*UI)) { HasGlueUse = true; assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); N = *UI; if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall()) NodeSUnit->isCall = true; break; } if (!HasGlueUse) break; } if (NodeSUnit->isCall) CallSUnits.push_back(NodeSUnit); // Schedule zero-latency TokenFactor below any nodes that may increase the // schedule height. Otherwise, ancestors of the TokenFactor may appear to // have false stalls. if (NI->getOpcode() == ISD::TokenFactor) NodeSUnit->isScheduleLow = true; // If there are glue operands involved, N is now the bottom-most node // of the sequence of nodes that are glued together. // Update the SUnit. NodeSUnit->setNode(N); assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); // Compute NumRegDefsLeft. This must be done before AddSchedEdges. InitNumRegDefsLeft(NodeSUnit); // Assign the Latency field of NodeSUnit using target-provided information. computeLatency(NodeSUnit); } // Find all call operands. while (!CallSUnits.empty()) { SUnit *SU = CallSUnits.pop_back_val(); for (const SDNode *SUNode = SU->getNode(); SUNode; SUNode = SUNode->getGluedNode()) { if (SUNode->getOpcode() != ISD::CopyToReg) continue; SDNode *SrcN = SUNode->getOperand(2).getNode(); if (isPassiveNode(SrcN)) continue; // Not scheduled. SUnit *SrcSU = &SUnits[SrcN->getNodeId()]; SrcSU->isCallOp = true; } } }
void ScheduleDAGSDNodes::AddSchedEdges() { const TargetSubtarget &ST = TM.getSubtarget<TargetSubtarget>(); // Check to see if the scheduler cares about latencies. bool UnitLatencies = ForceUnitLatencies(); // Pass 2: add the preds, succs, etc. for (unsigned su = 0, e = SUnits.size(); su != e; ++su) { SUnit *SU = &SUnits[su]; SDNode *MainNode = SU->getNode(); if (MainNode->isMachineOpcode()) { unsigned Opc = MainNode->getMachineOpcode(); const TargetInstrDesc &TID = TII->get(Opc); for (unsigned i = 0; i != TID.getNumOperands(); ++i) { if (TID.getOperandConstraint(i, TOI::TIED_TO) != -1) { SU->isTwoAddress = true; break; } } if (TID.isCommutable()) SU->isCommutable = true; } // Find all predecessors and successors of the group. for (SDNode *N = SU->getNode(); N; N = N->getGluedNode()) { if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).getImplicitDefs()) { SU->hasPhysRegClobbers = true; unsigned NumUsed = InstrEmitter::CountResults(N); while (NumUsed != 0 && !N->hasAnyUseOfValue(NumUsed - 1)) --NumUsed; // Skip over unused values at the end. if (NumUsed > TII->get(N->getMachineOpcode()).getNumDefs()) SU->hasPhysRegDefs = true; } for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { SDNode *OpN = N->getOperand(i).getNode(); if (isPassiveNode(OpN)) continue; // Not scheduled. SUnit *OpSU = &SUnits[OpN->getNodeId()]; assert(OpSU && "Node has no SUnit!"); if (OpSU == SU) continue; // In the same group. EVT OpVT = N->getOperand(i).getValueType(); assert(OpVT != MVT::Glue && "Glued nodes should be in same sunit!"); bool isChain = OpVT == MVT::Other; unsigned PhysReg = 0; int Cost = 1; // Determine if this is a physical register dependency. CheckForPhysRegDependency(OpN, N, i, TRI, TII, PhysReg, Cost); assert((PhysReg == 0 || !isChain) && "Chain dependence via physreg data?"); // FIXME: See ScheduleDAGSDNodes::EmitCopyFromReg. For now, scheduler // emits a copy from the physical register to a virtual register unless // it requires a cross class copy (cost < 0). That means we are only // treating "expensive to copy" register dependency as physical register // dependency. This may change in the future though. if (Cost >= 0) PhysReg = 0; // If this is a ctrl dep, latency is 1. unsigned OpLatency = isChain ? 1 : OpSU->Latency; const SDep &dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data, OpLatency, PhysReg); if (!isChain && !UnitLatencies) { ComputeOperandLatency(OpN, N, i, const_cast<SDep &>(dep)); ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep)); } SU->addPred(dep); } } } }
void ScheduleDAGSDNodes::BuildSchedUnits() { // During scheduling, the NodeId field of SDNode is used to map SDNodes // to their associated SUnits by holding SUnits table indices. A value // of -1 means the SDNode does not yet have an associated SUnit. unsigned NumNodes = 0; for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(), E = DAG->allnodes_end(); NI != E; ++NI) { NI->setNodeId(-1); ++NumNodes; } // Reserve entries in the vector for each of the SUnits we are creating. This // ensure that reallocation of the vector won't happen, so SUnit*'s won't get // invalidated. // FIXME: Multiply by 2 because we may clone nodes during scheduling. // This is a temporary workaround. SUnits.reserve(NumNodes * 2); // Check to see if the scheduler cares about latencies. bool UnitLatencies = ForceUnitLatencies(); // Add all nodes in depth first order. SmallVector<SDNode*, 64> Worklist; SmallPtrSet<SDNode*, 64> Visited; Worklist.push_back(DAG->getRoot().getNode()); Visited.insert(DAG->getRoot().getNode()); while (!Worklist.empty()) { SDNode *NI = Worklist.pop_back_val(); // Add all operands to the worklist unless they've already been added. for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i) if (Visited.insert(NI->getOperand(i).getNode())) Worklist.push_back(NI->getOperand(i).getNode()); if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate. continue; // If this node has already been processed, stop now. if (NI->getNodeId() != -1) continue; SUnit *NodeSUnit = NewSUnit(NI); // See if anything is flagged to this node, if so, add them to flagged // nodes. Nodes can have at most one flag input and one flag output. Flags // are required to be the last operand and result of a node. // Scan up to find flagged preds. SDNode *N = NI; while (N->getNumOperands() && N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) { N = N->getOperand(N->getNumOperands()-1).getNode(); assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); } // Scan down to find any flagged succs. N = NI; while (N->getValueType(N->getNumValues()-1) == MVT::Flag) { SDValue FlagVal(N, N->getNumValues()-1); // There are either zero or one users of the Flag result. bool HasFlagUse = false; for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) if (FlagVal.isOperandOf(*UI)) { HasFlagUse = true; assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); N = *UI; break; } if (!HasFlagUse) break; } // If there are flag operands involved, N is now the bottom-most node // of the sequence of nodes that are flagged together. // Update the SUnit. NodeSUnit->setNode(N); assert(N->getNodeId() == -1 && "Node already inserted!"); N->setNodeId(NodeSUnit->NodeNum); // Assign the Latency field of NodeSUnit using target-provided information. if (UnitLatencies) NodeSUnit->Latency = 1; else ComputeLatency(NodeSUnit); } }