/// addPred - This adds the specified edge as a pred of the current node if
/// not already. It also adds the current node as a successor of the
/// specified node.
bool SUnit::addPred(const SDep &D) {
// If this node already has this depenence, don't add a redundant one.
for (SmallVector<SDep, 4>::iterator I = Preds.begin(), E = Preds.end();
I != E; ++I) {
if (I->overlaps(D)) {
// Extend the latency if needed. Equivalent to removePred(I) + addPred(D).
if (I->getLatency() < D.getLatency()) {
SUnit *PredSU = I->getSUnit();
// Find the corresponding successor in N.
SDep ForwardD = *I;
ForwardD.setSUnit(this);
for (SmallVector<SDep, 4>::iterator II = PredSU->Succs.begin(),
EE = PredSU->Succs.end(); II != EE; ++II) {
if (*II == ForwardD) {
II->setLatency(D.getLatency());
break;
}
}
I->setLatency(D.getLatency());
}
return false;
}
}
// Now add a corresponding succ to N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
// Update the bookkeeping.
if (D.getKind() == SDep::Data) {
assert(NumPreds < UINT_MAX && "NumPreds will overflow!");
assert(N->NumSuccs < UINT_MAX && "NumSuccs will overflow!");
++NumPreds;
++N->NumSuccs;
}
if (!N->isScheduled) {
assert(NumPredsLeft < UINT_MAX && "NumPredsLeft will overflow!");
++NumPredsLeft;
}
if (!isScheduled) {
assert(N->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!");
++N->NumSuccsLeft;
}
Preds.push_back(D);
N->Succs.push_back(P);
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
return true;
}
/// addPred - This adds the specified edge as a pred of the current node if
/// not already. It also adds the current node as a successor of the
/// specified node.
void SUnit::addPred(const SDep &D) {
// If this node already has this depenence, don't add a redundant one.
for (unsigned i = 0, e = (unsigned)Preds.size(); i != e; ++i)
if (Preds[i] == D)
return;
// Now add a corresponding succ to N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
// Update the bookkeeping.
if (D.getKind() == SDep::Data) {
++NumPreds;
++N->NumSuccs;
}
if (!N->isScheduled)
++NumPredsLeft;
if (!isScheduled)
++N->NumSuccsLeft;
Preds.push_back(D);
N->Succs.push_back(P);
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
}
/// removePred - This removes the specified edge as a pred of the current
/// node if it exists. It also removes the current node as a successor of
/// the specified node.
void SUnit::removePred(const SDep &D) {
// Find the matching predecessor.
for (SmallVector<SDep, 4>::iterator I = Preds.begin(), E = Preds.end();
I != E; ++I)
if (*I == D) {
bool FoundSucc = false;
// Find the corresponding successor in N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
for (SmallVector<SDep, 4>::iterator II = N->Succs.begin(),
EE = N->Succs.end(); II != EE; ++II)
if (*II == P) {
FoundSucc = true;
N->Succs.erase(II);
break;
}
assert(FoundSucc && "Mismatching preds / succs lists!");
Preds.erase(I);
// Update the bookkeeping.
if (P.getKind() == SDep::Data) {
--NumPreds;
--N->NumSuccs;
}
if (!N->isScheduled)
--NumPredsLeft;
if (!isScheduled)
--N->NumSuccsLeft;
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
return;
}
}
/// addPred - This adds the specified edge as a pred of the current node if
/// not already. It also adds the current node as a successor of the
/// specified node.
bool SUnit::addPred(const SDep &D) {
// If this node already has this depenence, don't add a redundant one.
for (SmallVector<SDep, 4>::const_iterator I = Preds.begin(), E = Preds.end();
I != E; ++I)
if (*I == D)
return false;
// Now add a corresponding succ to N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
// Update the bookkeeping.
if (D.getKind() == SDep::Data) {
assert(NumPreds < UINT_MAX && "NumPreds will overflow!");
assert(N->NumSuccs < UINT_MAX && "NumSuccs will overflow!");
++NumPreds;
++N->NumSuccs;
}
if (!N->isScheduled) {
assert(NumPredsLeft < UINT_MAX && "NumPredsLeft will overflow!");
++NumPredsLeft;
}
if (!isScheduled) {
assert(N->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!");
++N->NumSuccsLeft;
}
Preds.push_back(D);
N->Succs.push_back(P);
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
return true;
}
/// removePred - This removes the specified edge as a pred of the current
/// node if it exists. It also removes the current node as a successor of
/// the specified node.
void SUnit::removePred(const SDep &D) {
// Find the matching predecessor.
for (SmallVector<SDep, 4>::iterator I = Preds.begin(), E = Preds.end();
I != E; ++I)
if (*I == D) {
// Find the corresponding successor in N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
SmallVectorImpl<SDep>::iterator Succ = std::find(N->Succs.begin(),
N->Succs.end(), P);
assert(Succ != N->Succs.end() && "Mismatching preds / succs lists!");
N->Succs.erase(Succ);
Preds.erase(I);
// Update the bookkeeping.
if (P.getKind() == SDep::Data) {
assert(NumPreds > 0 && "NumPreds will underflow!");
assert(N->NumSuccs > 0 && "NumSuccs will underflow!");
--NumPreds;
--N->NumSuccs;
}
if (!N->isScheduled) {
if (D.isWeak())
--WeakPredsLeft;
else {
assert(NumPredsLeft > 0 && "NumPredsLeft will underflow!");
--NumPredsLeft;
}
}
if (!isScheduled) {
if (D.isWeak())
--N->WeakSuccsLeft;
else {
assert(N->NumSuccsLeft > 0 && "NumSuccsLeft will underflow!");
--N->NumSuccsLeft;
}
}
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
return;
}
}
/// InsertCopiesAndMoveSuccs - Insert register copies and move all
/// scheduled successors of the given SUnit to the last copy.
void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
const TargetRegisterClass *DestRC,
const TargetRegisterClass *SrcRC,
SmallVectorImpl<SUnit*> &Copies) {
SUnit *CopyFromSU = newSUnit(static_cast<SDNode *>(nullptr));
CopyFromSU->CopySrcRC = SrcRC;
CopyFromSU->CopyDstRC = DestRC;
SUnit *CopyToSU = newSUnit(static_cast<SDNode *>(nullptr));
CopyToSU->CopySrcRC = DestRC;
CopyToSU->CopyDstRC = SrcRC;
// Only copy scheduled successors. Cut them from old node's successor
// list and move them over.
SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
for (SDep &Succ : SU->Succs) {
if (Succ.isArtificial())
continue;
SUnit *SuccSU = Succ.getSUnit();
if (SuccSU->isScheduled) {
SDep D = Succ;
D.setSUnit(CopyToSU);
AddPred(SuccSU, D);
DelDeps.push_back(std::make_pair(SuccSU, Succ));
}
}
for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
RemovePred(DelDeps[i].first, DelDeps[i].second);
}
SDep FromDep(SU, SDep::Data, Reg);
FromDep.setLatency(SU->Latency);
AddPred(CopyFromSU, FromDep);
SDep ToDep(CopyFromSU, SDep::Data, 0);
ToDep.setLatency(CopyFromSU->Latency);
AddPred(CopyToSU, ToDep);
Copies.push_back(CopyFromSU);
Copies.push_back(CopyToSU);
++NumPRCopies;
}
/// CopyAndMoveSuccessors - Clone the specified node and move its scheduled
/// successors to the newly created node.
SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
if (SU->getNode()->getGluedNode())
return nullptr;
SDNode *N = SU->getNode();
if (!N)
return nullptr;
SUnit *NewSU;
bool TryUnfold = false;
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
MVT VT = N->getSimpleValueType(i);
if (VT == MVT::Glue)
return nullptr;
else if (VT == MVT::Other)
TryUnfold = true;
}
for (const SDValue &Op : N->op_values()) {
MVT VT = Op.getNode()->getSimpleValueType(Op.getResNo());
if (VT == MVT::Glue)
return nullptr;
}
if (TryUnfold) {
SmallVector<SDNode*, 2> NewNodes;
if (!TII->unfoldMemoryOperand(*DAG, N, NewNodes))
return nullptr;
LLVM_DEBUG(dbgs() << "Unfolding SU # " << SU->NodeNum << "\n");
assert(NewNodes.size() == 2 && "Expected a load folding node!");
N = NewNodes[1];
SDNode *LoadNode = NewNodes[0];
unsigned NumVals = N->getNumValues();
unsigned OldNumVals = SU->getNode()->getNumValues();
for (unsigned i = 0; i != NumVals; ++i)
DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), i), SDValue(N, i));
DAG->ReplaceAllUsesOfValueWith(SDValue(SU->getNode(), OldNumVals-1),
SDValue(LoadNode, 1));
SUnit *NewSU = newSUnit(N);
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NewSU->NodeNum);
const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
if (MCID.getOperandConstraint(i, MCOI::TIED_TO) != -1) {
NewSU->isTwoAddress = true;
break;
}
}
if (MCID.isCommutable())
NewSU->isCommutable = true;
// LoadNode may already exist. This can happen when there is another
// load from the same location and producing the same type of value
// but it has different alignment or volatileness.
bool isNewLoad = true;
SUnit *LoadSU;
if (LoadNode->getNodeId() != -1) {
LoadSU = &SUnits[LoadNode->getNodeId()];
isNewLoad = false;
} else {
LoadSU = newSUnit(LoadNode);
LoadNode->setNodeId(LoadSU->NodeNum);
}
SDep ChainPred;
SmallVector<SDep, 4> ChainSuccs;
SmallVector<SDep, 4> LoadPreds;
SmallVector<SDep, 4> NodePreds;
SmallVector<SDep, 4> NodeSuccs;
for (SDep &Pred : SU->Preds) {
if (Pred.isCtrl())
ChainPred = Pred;
else if (Pred.getSUnit()->getNode() &&
Pred.getSUnit()->getNode()->isOperandOf(LoadNode))
LoadPreds.push_back(Pred);
else
NodePreds.push_back(Pred);
}
for (SDep &Succ : SU->Succs) {
if (Succ.isCtrl())
ChainSuccs.push_back(Succ);
else
NodeSuccs.push_back(Succ);
}
if (ChainPred.getSUnit()) {
RemovePred(SU, ChainPred);
if (isNewLoad)
AddPred(LoadSU, ChainPred);
}
for (unsigned i = 0, e = LoadPreds.size(); i != e; ++i) {
const SDep &Pred = LoadPreds[i];
RemovePred(SU, Pred);
if (isNewLoad) {
AddPred(LoadSU, Pred);
}
}
for (unsigned i = 0, e = NodePreds.size(); i != e; ++i) {
const SDep &Pred = NodePreds[i];
RemovePred(SU, Pred);
AddPred(NewSU, Pred);
}
for (unsigned i = 0, e = NodeSuccs.size(); i != e; ++i) {
SDep D = NodeSuccs[i];
SUnit *SuccDep = D.getSUnit();
D.setSUnit(SU);
RemovePred(SuccDep, D);
D.setSUnit(NewSU);
AddPred(SuccDep, D);
}
for (unsigned i = 0, e = ChainSuccs.size(); i != e; ++i) {
SDep D = ChainSuccs[i];
SUnit *SuccDep = D.getSUnit();
D.setSUnit(SU);
RemovePred(SuccDep, D);
if (isNewLoad) {
D.setSUnit(LoadSU);
AddPred(SuccDep, D);
}
}
if (isNewLoad) {
SDep D(LoadSU, SDep::Barrier);
D.setLatency(LoadSU->Latency);
AddPred(NewSU, D);
}
++NumUnfolds;
if (NewSU->NumSuccsLeft == 0) {
NewSU->isAvailable = true;
return NewSU;
}
SU = NewSU;
}
LLVM_DEBUG(dbgs() << "Duplicating SU # " << SU->NodeNum << "\n");
NewSU = Clone(SU);
// New SUnit has the exact same predecessors.
for (SDep &Pred : SU->Preds)
if (!Pred.isArtificial())
AddPred(NewSU, Pred);
// Only copy scheduled successors. Cut them from old node's successor
// list and move them over.
SmallVector<std::pair<SUnit *, SDep>, 4> DelDeps;
for (SDep &Succ : SU->Succs) {
if (Succ.isArtificial())
continue;
SUnit *SuccSU = Succ.getSUnit();
if (SuccSU->isScheduled) {
SDep D = Succ;
D.setSUnit(NewSU);
AddPred(SuccSU, D);
D.setSUnit(SU);
DelDeps.push_back(std::make_pair(SuccSU, D));
}
}
for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
RemovePred(DelDeps[i].first, DelDeps[i].second);
++NumDups;
return NewSU;
}
/// addPred - This adds the specified edge as a pred of the current node if
/// not already. It also adds the current node as a successor of the
/// specified node.
bool SUnit::addPred(const SDep &D, bool Required) {
// If this node already has this dependence, don't add a redundant one.
for (SmallVectorImpl<SDep>::iterator I = Preds.begin(), E = Preds.end();
I != E; ++I) {
// Zero-latency weak edges may be added purely for heuristic ordering. Don't
// add them if another kind of edge already exists.
if (!Required && I->getSUnit() == D.getSUnit())
return false;
if (I->overlaps(D)) {
// Extend the latency if needed. Equivalent to removePred(I) + addPred(D).
if (I->getLatency() < D.getLatency()) {
SUnit *PredSU = I->getSUnit();
// Find the corresponding successor in N.
SDep ForwardD = *I;
ForwardD.setSUnit(this);
for (SmallVectorImpl<SDep>::iterator II = PredSU->Succs.begin(),
EE = PredSU->Succs.end(); II != EE; ++II) {
if (*II == ForwardD) {
II->setLatency(D.getLatency());
break;
}
}
I->setLatency(D.getLatency());
}
return false;
}
}
// Now add a corresponding succ to N.
SDep P = D;
P.setSUnit(this);
SUnit *N = D.getSUnit();
// Update the bookkeeping.
if (D.getKind() == SDep::Data) {
assert(NumPreds < UINT_MAX && "NumPreds will overflow!");
assert(N->NumSuccs < UINT_MAX && "NumSuccs will overflow!");
++NumPreds;
++N->NumSuccs;
}
if (!N->isScheduled) {
if (D.isWeak()) {
++WeakPredsLeft;
}
else {
assert(NumPredsLeft < UINT_MAX && "NumPredsLeft will overflow!");
++NumPredsLeft;
}
}
if (!isScheduled) {
if (D.isWeak()) {
++N->WeakSuccsLeft;
}
else {
assert(N->NumSuccsLeft < UINT_MAX && "NumSuccsLeft will overflow!");
++N->NumSuccsLeft;
}
}
Preds.push_back(D);
N->Succs.push_back(P);
if (P.getLatency() != 0) {
this->setDepthDirty();
N->setHeightDirty();
}
return true;
}
