void VDAGToDAGISel::PostprocessISelDAG() {
CurDAG->AssignTopologicalOrder();
HandleSDNode Dummy(CurDAG->getRoot());
for (SelectionDAG::allnodes_iterator NI = CurDAG->allnodes_begin();
NI != CurDAG->allnodes_end(); ++NI) {
if (NI->getOpcode() == ISD::CopyToReg)
CopyToReg(*CurDAG, NI);
}
CurDAG->setRoot(Dummy.getValue());
}
// After instruction selection, insert COPY_TO_REGCLASS nodes to help in
// choosing the proper register classes.
void BlackfinDAGToDAGISel::FixRegisterClasses(SelectionDAG &DAG) {
const BlackfinInstrInfo &TII = getInstrInfo();
const BlackfinRegisterInfo *TRI = getRegisterInfo();
DAG.AssignTopologicalOrder();
HandleSDNode Dummy(DAG.getRoot());
for (SelectionDAG::allnodes_iterator NI = DAG.allnodes_begin();
NI != DAG.allnodes_end(); ++NI) {
if (NI->use_empty() || !NI->isMachineOpcode())
continue;
const TargetInstrDesc &DefTID = TII.get(NI->getMachineOpcode());
for (SDNode::use_iterator UI = NI->use_begin(); !UI.atEnd(); ++UI) {
if (!UI->isMachineOpcode())
continue;
if (UI.getUse().getResNo() >= DefTID.getNumDefs())
continue;
const TargetRegisterClass *DefRC =
DefTID.OpInfo[UI.getUse().getResNo()].getRegClass(TRI);
const TargetInstrDesc &UseTID = TII.get(UI->getMachineOpcode());
if (UseTID.getNumDefs()+UI.getOperandNo() >= UseTID.getNumOperands())
continue;
const TargetRegisterClass *UseRC =
UseTID.OpInfo[UseTID.getNumDefs()+UI.getOperandNo()].getRegClass(TRI);
if (!DefRC || !UseRC)
continue;
// We cannot copy CC <-> !(CC/D)
if ((isCC(DefRC) && !isDCC(UseRC)) || (isCC(UseRC) && !isDCC(DefRC))) {
SDNode *Copy =
DAG.getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
NI->getDebugLoc(),
MVT::i32,
UI.getUse().get(),
DAG.getTargetConstant(BF::DRegClassID, MVT::i32));
UpdateNodeOperand(DAG, *UI, UI.getOperandNo(), SDValue(Copy, 0));
}
}
}
DAG.setRoot(Dummy.getValue());
}
/// PerformExpensiveChecks - Do extensive, expensive, sanity checking.
void DAGTypeLegalizer::PerformExpensiveChecks() {
// If a node is not processed, then none of its values should be mapped by any
// of PromotedIntegers, ExpandedIntegers, ..., ReplacedValues.
// If a node is processed, then each value with an illegal type must be mapped
// by exactly one of PromotedIntegers, ExpandedIntegers, ..., ReplacedValues.
// Values with a legal type may be mapped by ReplacedValues, but not by any of
// the other maps.
// Note that these invariants may not hold momentarily when processing a node:
// the node being processed may be put in a map before being marked Processed.
// Note that it is possible to have nodes marked NewNode in the DAG. This can
// occur in two ways. Firstly, a node may be created during legalization but
// never passed to the legalization core. This is usually due to the implicit
// folding that occurs when using the DAG.getNode operators. Secondly, a new
// node may be passed to the legalization core, but when analyzed may morph
// into a different node, leaving the original node as a NewNode in the DAG.
// A node may morph if one of its operands changes during analysis. Whether
// it actually morphs or not depends on whether, after updating its operands,
// it is equivalent to an existing node: if so, it morphs into that existing
// node (CSE). An operand can change during analysis if the operand is a new
// node that morphs, or it is a processed value that was mapped to some other
// value (as recorded in ReplacedValues) in which case the operand is turned
// into that other value. If a node morphs then the node it morphed into will
// be used instead of it for legalization, however the original node continues
// to live on in the DAG.
// The conclusion is that though there may be nodes marked NewNode in the DAG,
// all uses of such nodes are also marked NewNode: the result is a fungus of
// NewNodes growing on top of the useful nodes, and perhaps using them, but
// not used by them.
// If a value is mapped by ReplacedValues, then it must have no uses, except
// by nodes marked NewNode (see above).
// The final node obtained by mapping by ReplacedValues is not marked NewNode.
// Note that ReplacedValues should be applied iteratively.
// Note that the ReplacedValues map may also map deleted nodes (by iterating
// over the DAG we never dereference deleted nodes). This means that it may
// also map nodes marked NewNode if the deallocated memory was reallocated as
// another node, and that new node was not seen by the LegalizeTypes machinery
// (for example because it was created but not used). In general, we cannot
// distinguish between new nodes and deleted nodes.
SmallVector<SDNode*, 16> NewNodes;
for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
E = DAG.allnodes_end(); I != E; ++I) {
// Remember nodes marked NewNode - they are subject to extra checking below.
if (I->getNodeId() == NewNode)
NewNodes.push_back(I);
for (unsigned i = 0, e = I->getNumValues(); i != e; ++i) {
SDValue Res(I, i);
bool Failed = false;
unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) {
Mapped |= 1;
// Check that remapped values are only used by nodes marked NewNode.
for (SDNode::use_iterator UI = I->use_begin(), UE = I->use_end();
UI != UE; ++UI)
if (UI.getUse().getResNo() == i)
assert(UI->getNodeId() == NewNode &&
"Remapped value has non-trivial use!");
// Check that the final result of applying ReplacedValues is not
// marked NewNode.
SDValue NewVal = ReplacedValues[Res];
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(NewVal);
while (I != ReplacedValues.end()) {
NewVal = I->second;
I = ReplacedValues.find(NewVal);
}
assert(NewVal.getNode()->getNodeId() != NewNode &&
"ReplacedValues maps to a new node!");
}
if (PromotedIntegers.find(Res) != PromotedIntegers.end())
Mapped |= 2;
if (SoftenedFloats.find(Res) != SoftenedFloats.end())
Mapped |= 4;
if (ScalarizedVectors.find(Res) != ScalarizedVectors.end())
Mapped |= 8;
if (ExpandedIntegers.find(Res) != ExpandedIntegers.end())
Mapped |= 16;
if (ExpandedFloats.find(Res) != ExpandedFloats.end())
Mapped |= 32;
if (SplitVectors.find(Res) != SplitVectors.end())
Mapped |= 64;
if (WidenedVectors.find(Res) != WidenedVectors.end())
Mapped |= 128;
if (I->getNodeId() != Processed) {
// Since we allow ReplacedValues to map deleted nodes, it may map nodes
// marked NewNode too, since a deleted node may have been reallocated as
// another node that has not been seen by the LegalizeTypes machinery.
if ((I->getNodeId() == NewNode && Mapped > 1) ||
(I->getNodeId() != NewNode && Mapped != 0)) {
dbgs() << "Unprocessed value in a map!";
Failed = true;
}
} else if (isTypeLegal(Res.getValueType()) || IgnoreNodeResults(I)) {
if (Mapped > 1) {
dbgs() << "Value with legal type was transformed!";
Failed = true;
}
} else {
if (Mapped == 0) {
dbgs() << "Processed value not in any map!";
Failed = true;
} else if (Mapped & (Mapped - 1)) {
dbgs() << "Value in multiple maps!";
Failed = true;
}
}
if (Failed) {
if (Mapped & 1)
dbgs() << " ReplacedValues";
if (Mapped & 2)
dbgs() << " PromotedIntegers";
if (Mapped & 4)
dbgs() << " SoftenedFloats";
if (Mapped & 8)
dbgs() << " ScalarizedVectors";
if (Mapped & 16)
dbgs() << " ExpandedIntegers";
if (Mapped & 32)
dbgs() << " ExpandedFloats";
if (Mapped & 64)
dbgs() << " SplitVectors";
if (Mapped & 128)
dbgs() << " WidenedVectors";
dbgs() << "\n";
llvm_unreachable(nullptr);
}
}
}
// Checked that NewNodes are only used by other NewNodes.
for (unsigned i = 0, e = NewNodes.size(); i != e; ++i) {
SDNode *N = NewNodes[i];
for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
UI != UE; ++UI)
assert(UI->getNodeId() == NewNode && "NewNode used by non-NewNode!");
}
}
/// run - This is the main entry point for the type legalizer. This does a
/// top-down traversal of the dag, legalizing types as it goes. Returns "true"
/// if it made any changes.
bool DAGTypeLegalizer::run() {
bool Changed = false;
// Create a dummy node (which is not added to allnodes), that adds a reference
// to the root node, preventing it from being deleted, and tracking any
// changes of the root.
HandleSDNode Dummy(DAG.getRoot());
Dummy.setNodeId(Unanalyzed);
// The root of the dag may dangle to deleted nodes until the type legalizer is
// done. Set it to null to avoid confusion.
DAG.setRoot(SDValue());
// Walk all nodes in the graph, assigning them a NodeId of 'ReadyToProcess'
// (and remembering them) if they are leaves and assigning 'Unanalyzed' if
// non-leaves.
for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
E = DAG.allnodes_end(); I != E; ++I) {
if (I->getNumOperands() == 0) {
I->setNodeId(ReadyToProcess);
Worklist.push_back(I);
} else {
I->setNodeId(Unanalyzed);
}
}
// Now that we have a set of nodes to process, handle them all.
while (!Worklist.empty()) {
#ifndef XDEBUG
if (EnableExpensiveChecks)
#endif
PerformExpensiveChecks();
SDNode *N = Worklist.back();
Worklist.pop_back();
assert(N->getNodeId() == ReadyToProcess &&
"Node should be ready if on worklist!");
if (IgnoreNodeResults(N))
goto ScanOperands;
// Scan the values produced by the node, checking to see if any result
// types are illegal.
for (unsigned i = 0, NumResults = N->getNumValues(); i < NumResults; ++i) {
EVT ResultVT = N->getValueType(i);
switch (getTypeAction(ResultVT)) {
case TargetLowering::TypeLegal:
break;
// The following calls must take care of *all* of the node's results,
// not just the illegal result they were passed (this includes results
// with a legal type). Results can be remapped using ReplaceValueWith,
// or their promoted/expanded/etc values registered in PromotedIntegers,
// ExpandedIntegers etc.
case TargetLowering::TypePromoteInteger:
PromoteIntegerResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeExpandInteger:
ExpandIntegerResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeSoftenFloat:
SoftenFloatResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeExpandFloat:
ExpandFloatResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeScalarizeVector:
ScalarizeVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeSplitVector:
SplitVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypeWidenVector:
WidenVectorResult(N, i);
Changed = true;
goto NodeDone;
case TargetLowering::TypePromoteFloat:
PromoteFloatResult(N, i);
Changed = true;
goto NodeDone;
}
}
ScanOperands:
// Scan the operand list for the node, handling any nodes with operands that
// are illegal.
{
unsigned NumOperands = N->getNumOperands();
bool NeedsReanalyzing = false;
unsigned i;
for (i = 0; i != NumOperands; ++i) {
if (IgnoreNodeResults(N->getOperand(i).getNode()))
continue;
EVT OpVT = N->getOperand(i).getValueType();
switch (getTypeAction(OpVT)) {
case TargetLowering::TypeLegal:
continue;
// The following calls must either replace all of the node's results
// using ReplaceValueWith, and return "false"; or update the node's
// operands in place, and return "true".
case TargetLowering::TypePromoteInteger:
NeedsReanalyzing = PromoteIntegerOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeExpandInteger:
NeedsReanalyzing = ExpandIntegerOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeSoftenFloat:
NeedsReanalyzing = SoftenFloatOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeExpandFloat:
NeedsReanalyzing = ExpandFloatOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeScalarizeVector:
NeedsReanalyzing = ScalarizeVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeSplitVector:
NeedsReanalyzing = SplitVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypeWidenVector:
NeedsReanalyzing = WidenVectorOperand(N, i);
Changed = true;
break;
case TargetLowering::TypePromoteFloat:
NeedsReanalyzing = PromoteFloatOperand(N, i);
Changed = true;
break;
}
break;
}
// The sub-method updated N in place. Check to see if any operands are new,
// and if so, mark them. If the node needs revisiting, don't add all users
// to the worklist etc.
if (NeedsReanalyzing) {
assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?");
N->setNodeId(NewNode);
// Recompute the NodeId and correct processed operands, adding the node to
// the worklist if ready.
SDNode *M = AnalyzeNewNode(N);
if (M == N)
// The node didn't morph - nothing special to do, it will be revisited.
continue;
// The node morphed - this is equivalent to legalizing by replacing every
// value of N with the corresponding value of M. So do that now.
assert(N->getNumValues() == M->getNumValues() &&
"Node morphing changed the number of results!");
for (unsigned i = 0, e = N->getNumValues(); i != e; ++i)
// Replacing the value takes care of remapping the new value.
ReplaceValueWith(SDValue(N, i), SDValue(M, i));
assert(N->getNodeId() == NewNode && "Unexpected node state!");
// The node continues to live on as part of the NewNode fungus that
// grows on top of the useful nodes. Nothing more needs to be done
// with it - move on to the next node.
continue;
}
if (i == NumOperands) {
DEBUG(dbgs() << "Legally typed node: "; N->dump(&DAG); dbgs() << "\n");
}
}
NodeDone:
// If we reach here, the node was processed, potentially creating new nodes.
// Mark it as processed and add its users to the worklist as appropriate.
assert(N->getNodeId() == ReadyToProcess && "Node ID recalculated?");
N->setNodeId(Processed);
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI) {
SDNode *User = *UI;
int NodeId = User->getNodeId();
// This node has two options: it can either be a new node or its Node ID
// may be a count of the number of operands it has that are not ready.
if (NodeId > 0) {
User->setNodeId(NodeId-1);
// If this was the last use it was waiting on, add it to the ready list.
if (NodeId-1 == ReadyToProcess)
Worklist.push_back(User);
continue;
}
// If this is an unreachable new node, then ignore it. If it ever becomes
// reachable by being used by a newly created node then it will be handled
// by AnalyzeNewNode.
if (NodeId == NewNode)
continue;
// Otherwise, this node is new: this is the first operand of it that
// became ready. Its new NodeId is the number of operands it has minus 1
// (as this node is now processed).
assert(NodeId == Unanalyzed && "Unknown node ID!");
User->setNodeId(User->getNumOperands() - 1);
// If the node only has a single operand, it is now ready.
if (User->getNumOperands() == 1)
Worklist.push_back(User);
}
}
void ScheduleDAGSDNodes::BuildSchedUnits() {
// During scheduling, the NodeId field of SDNode is used to map SDNodes
// to their associated SUnits by holding SUnits table indices. A value
// of -1 means the SDNode does not yet have an associated SUnit.
unsigned NumNodes = 0;
for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
E = DAG->allnodes_end(); NI != E; ++NI) {
NI->setNodeId(-1);
++NumNodes;
}
// Reserve entries in the vector for each of the SUnits we are creating. This
// ensure that reallocation of the vector won't happen, so SUnit*'s won't get
// invalidated.
// FIXME: Multiply by 2 because we may clone nodes during scheduling.
// This is a temporary workaround.
SUnits.reserve(NumNodes * 2);
// Add all nodes in depth first order.
SmallVector<SDNode*, 64> Worklist;
SmallPtrSet<SDNode*, 64> Visited;
Worklist.push_back(DAG->getRoot().getNode());
Visited.insert(DAG->getRoot().getNode());
SmallVector<SUnit*, 8> CallSUnits;
while (!Worklist.empty()) {
SDNode *NI = Worklist.pop_back_val();
// Add all operands to the worklist unless they've already been added.
for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i)
if (Visited.insert(NI->getOperand(i).getNode()))
Worklist.push_back(NI->getOperand(i).getNode());
if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate.
continue;
// If this node has already been processed, stop now.
if (NI->getNodeId() != -1) continue;
SUnit *NodeSUnit = newSUnit(NI);
// See if anything is glued to this node, if so, add them to glued
// nodes. Nodes can have at most one glue input and one glue output. Glue
// is required to be the last operand and result of a node.
// Scan up to find glued preds.
SDNode *N = NI;
while (N->getNumOperands() &&
N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue) {
N = N->getOperand(N->getNumOperands()-1).getNode();
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall())
NodeSUnit->isCall = true;
}
// Scan down to find any glued succs.
N = NI;
while (N->getValueType(N->getNumValues()-1) == MVT::Glue) {
SDValue GlueVal(N, N->getNumValues()-1);
// There are either zero or one users of the Glue result.
bool HasGlueUse = false;
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI)
if (GlueVal.isOperandOf(*UI)) {
HasGlueUse = true;
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
N = *UI;
if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall())
NodeSUnit->isCall = true;
break;
}
if (!HasGlueUse) break;
}
if (NodeSUnit->isCall)
CallSUnits.push_back(NodeSUnit);
// Schedule zero-latency TokenFactor below any nodes that may increase the
// schedule height. Otherwise, ancestors of the TokenFactor may appear to
// have false stalls.
if (NI->getOpcode() == ISD::TokenFactor)
NodeSUnit->isScheduleLow = true;
// If there are glue operands involved, N is now the bottom-most node
// of the sequence of nodes that are glued together.
// Update the SUnit.
NodeSUnit->setNode(N);
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
// Compute NumRegDefsLeft. This must be done before AddSchedEdges.
InitNumRegDefsLeft(NodeSUnit);
// Assign the Latency field of NodeSUnit using target-provided information.
computeLatency(NodeSUnit);
}
// Find all call operands.
while (!CallSUnits.empty()) {
SUnit *SU = CallSUnits.pop_back_val();
for (const SDNode *SUNode = SU->getNode(); SUNode;
SUNode = SUNode->getGluedNode()) {
if (SUNode->getOpcode() != ISD::CopyToReg)
continue;
SDNode *SrcN = SUNode->getOperand(2).getNode();
if (isPassiveNode(SrcN)) continue; // Not scheduled.
SUnit *SrcSU = &SUnits[SrcN->getNodeId()];
SrcSU->isCallOp = true;
}
}
}
void ScheduleDAGSDNodes::BuildSchedUnits() {
// During scheduling, the NodeId field of SDNode is used to map SDNodes
// to their associated SUnits by holding SUnits table indices. A value
// of -1 means the SDNode does not yet have an associated SUnit.
unsigned NumNodes = 0;
for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
E = DAG->allnodes_end(); NI != E; ++NI) {
NI->setNodeId(-1);
++NumNodes;
}
// Reserve entries in the vector for each of the SUnits we are creating. This
// ensure that reallocation of the vector won't happen, so SUnit*'s won't get
// invalidated.
// FIXME: Multiply by 2 because we may clone nodes during scheduling.
// This is a temporary workaround.
SUnits.reserve(NumNodes * 2);
// Check to see if the scheduler cares about latencies.
bool UnitLatencies = ForceUnitLatencies();
// Add all nodes in depth first order.
SmallVector<SDNode*, 64> Worklist;
SmallPtrSet<SDNode*, 64> Visited;
Worklist.push_back(DAG->getRoot().getNode());
Visited.insert(DAG->getRoot().getNode());
while (!Worklist.empty()) {
SDNode *NI = Worklist.pop_back_val();
// Add all operands to the worklist unless they've already been added.
for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i)
if (Visited.insert(NI->getOperand(i).getNode()))
Worklist.push_back(NI->getOperand(i).getNode());
if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate.
continue;
// If this node has already been processed, stop now.
if (NI->getNodeId() != -1) continue;
SUnit *NodeSUnit = NewSUnit(NI);
// See if anything is flagged to this node, if so, add them to flagged
// nodes. Nodes can have at most one flag input and one flag output. Flags
// are required to be the last operand and result of a node.
// Scan up to find flagged preds.
SDNode *N = NI;
while (N->getNumOperands() &&
N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) {
N = N->getOperand(N->getNumOperands()-1).getNode();
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
}
// Scan down to find any flagged succs.
N = NI;
while (N->getValueType(N->getNumValues()-1) == MVT::Flag) {
SDValue FlagVal(N, N->getNumValues()-1);
// There are either zero or one users of the Flag result.
bool HasFlagUse = false;
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI)
if (FlagVal.isOperandOf(*UI)) {
HasFlagUse = true;
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
N = *UI;
break;
}
if (!HasFlagUse) break;
}
// If there are flag operands involved, N is now the bottom-most node
// of the sequence of nodes that are flagged together.
// Update the SUnit.
NodeSUnit->setNode(N);
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
// Assign the Latency field of NodeSUnit using target-provided information.
if (UnitLatencies)
NodeSUnit->Latency = 1;
else
ComputeLatency(NodeSUnit);
}
}