/// Do extensive, expensive, sanity checking.
void DAGTypeLegalizer::PerformExpensiveChecks() {
// If a node is not processed, then none of its values should be mapped by any
// of PromotedIntegers, ExpandedIntegers, ..., ReplacedValues.
// If a node is processed, then each value with an illegal type must be mapped
// by exactly one of PromotedIntegers, ExpandedIntegers, ..., ReplacedValues.
// Values with a legal type may be mapped by ReplacedValues, but not by any of
// the other maps.
// Note that these invariants may not hold momentarily when processing a node:
// the node being processed may be put in a map before being marked Processed.
// Note that it is possible to have nodes marked NewNode in the DAG. This can
// occur in two ways. Firstly, a node may be created during legalization but
// never passed to the legalization core. This is usually due to the implicit
// folding that occurs when using the DAG.getNode operators. Secondly, a new
// node may be passed to the legalization core, but when analyzed may morph
// into a different node, leaving the original node as a NewNode in the DAG.
// A node may morph if one of its operands changes during analysis. Whether
// it actually morphs or not depends on whether, after updating its operands,
// it is equivalent to an existing node: if so, it morphs into that existing
// node (CSE). An operand can change during analysis if the operand is a new
// node that morphs, or it is a processed value that was mapped to some other
// value (as recorded in ReplacedValues) in which case the operand is turned
// into that other value. If a node morphs then the node it morphed into will
// be used instead of it for legalization, however the original node continues
// to live on in the DAG.
// The conclusion is that though there may be nodes marked NewNode in the DAG,
// all uses of such nodes are also marked NewNode: the result is a fungus of
// NewNodes growing on top of the useful nodes, and perhaps using them, but
// not used by them.
// If a value is mapped by ReplacedValues, then it must have no uses, except
// by nodes marked NewNode (see above).
// The final node obtained by mapping by ReplacedValues is not marked NewNode.
// Note that ReplacedValues should be applied iteratively.
// Note that the ReplacedValues map may also map deleted nodes (by iterating
// over the DAG we never dereference deleted nodes). This means that it may
// also map nodes marked NewNode if the deallocated memory was reallocated as
// another node, and that new node was not seen by the LegalizeTypes machinery
// (for example because it was created but not used). In general, we cannot
// distinguish between new nodes and deleted nodes.
SmallVector<SDNode*, 16> NewNodes;
for (SDNode &Node : DAG.allnodes()) {
// Remember nodes marked NewNode - they are subject to extra checking below.
if (Node.getNodeId() == NewNode)
NewNodes.push_back(&Node);
for (unsigned i = 0, e = Node.getNumValues(); i != e; ++i) {
SDValue Res(&Node, i);
EVT VT = Res.getValueType();
bool Failed = false;
unsigned Mapped = 0;
if (ReplacedValues.find(Res) != ReplacedValues.end()) {
Mapped |= 1;
// Check that remapped values are only used by nodes marked NewNode.
for (SDNode::use_iterator UI = Node.use_begin(), UE = Node.use_end();
UI != UE; ++UI)
if (UI.getUse().getResNo() == i)
assert(UI->getNodeId() == NewNode &&
"Remapped value has non-trivial use!");
// Check that the final result of applying ReplacedValues is not
// marked NewNode.
SDValue NewVal = ReplacedValues[Res];
DenseMap<SDValue, SDValue>::iterator I = ReplacedValues.find(NewVal);
while (I != ReplacedValues.end()) {
NewVal = I->second;
I = ReplacedValues.find(NewVal);
}
assert(NewVal.getNode()->getNodeId() != NewNode &&
"ReplacedValues maps to a new node!");
}
if (PromotedIntegers.find(Res) != PromotedIntegers.end())
Mapped |= 2;
if (SoftenedFloats.find(Res) != SoftenedFloats.end())
Mapped |= 4;
if (ScalarizedVectors.find(Res) != ScalarizedVectors.end())
Mapped |= 8;
if (ExpandedIntegers.find(Res) != ExpandedIntegers.end())
Mapped |= 16;
if (ExpandedFloats.find(Res) != ExpandedFloats.end())
Mapped |= 32;
if (SplitVectors.find(Res) != SplitVectors.end())
Mapped |= 64;
if (WidenedVectors.find(Res) != WidenedVectors.end())
Mapped |= 128;
if (PromotedFloats.find(Res) != PromotedFloats.end())
Mapped |= 256;
if (Node.getNodeId() != Processed) {
// Since we allow ReplacedValues to map deleted nodes, it may map nodes
// marked NewNode too, since a deleted node may have been reallocated as
// another node that has not been seen by the LegalizeTypes machinery.
if ((Node.getNodeId() == NewNode && Mapped > 1) ||
(Node.getNodeId() != NewNode && Mapped != 0)) {
dbgs() << "Unprocessed value in a map!";
Failed = true;
}
} else if (isTypeLegal(VT) || IgnoreNodeResults(&Node)) {
if (Mapped > 1) {
dbgs() << "Value with legal type was transformed!";
Failed = true;
}
} else {
// If the value can be kept in HW registers, softening machinery can
// leave it unchanged and don't put it to any map.
if (Mapped == 0 &&
!(getTypeAction(VT) == TargetLowering::TypeSoftenFloat &&
isLegalInHWReg(VT))) {
dbgs() << "Processed value not in any map!";
Failed = true;
} else if (Mapped & (Mapped - 1)) {
dbgs() << "Value in multiple maps!";
Failed = true;
}
}
if (Failed) {
if (Mapped & 1)
dbgs() << " ReplacedValues";
if (Mapped & 2)
dbgs() << " PromotedIntegers";
if (Mapped & 4)
dbgs() << " SoftenedFloats";
if (Mapped & 8)
dbgs() << " ScalarizedVectors";
if (Mapped & 16)
dbgs() << " ExpandedIntegers";
if (Mapped & 32)
dbgs() << " ExpandedFloats";
if (Mapped & 64)
dbgs() << " SplitVectors";
if (Mapped & 128)
dbgs() << " WidenedVectors";
if (Mapped & 256)
dbgs() << " PromotedFloats";
dbgs() << "\n";
llvm_unreachable(nullptr);
}
}
}
// Checked that NewNodes are only used by other NewNodes.
for (unsigned i = 0, e = NewNodes.size(); i != e; ++i) {
SDNode *N = NewNodes[i];
for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
UI != UE; ++UI)
assert(UI->getNodeId() == NewNode && "NewNode used by non-NewNode!");
}
}
/// ClusterNeighboringLoads - Force nearby loads together by "gluing" them.
/// This function finds loads of the same base and different offsets. If the
/// offsets are not far apart (target specific), it add MVT::Glue inputs and
/// outputs to ensure they are scheduled together and in order. This
/// optimization may benefit some targets by improving cache locality.
void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
SDNode *Chain = 0;
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Other)
Chain = Node->getOperand(NumOps-1).getNode();
if (!Chain)
return;
// Look for other loads of the same chain. Find loads that are loading from
// the same base pointer and different offsets.
SmallPtrSet<SDNode*, 16> Visited;
SmallVector<int64_t, 4> Offsets;
DenseMap<long long, SDNode*> O2SMap; // Map from offset to SDNode.
bool Cluster = false;
SDNode *Base = Node;
for (SDNode::use_iterator I = Chain->use_begin(), E = Chain->use_end();
I != E; ++I) {
SDNode *User = *I;
if (User == Node || !Visited.insert(User))
continue;
int64_t Offset1, Offset2;
if (!TII->areLoadsFromSameBasePtr(Base, User, Offset1, Offset2) ||
Offset1 == Offset2)
// FIXME: Should be ok if they addresses are identical. But earlier
// optimizations really should have eliminated one of the loads.
continue;
if (O2SMap.insert(std::make_pair(Offset1, Base)).second)
Offsets.push_back(Offset1);
O2SMap.insert(std::make_pair(Offset2, User));
Offsets.push_back(Offset2);
if (Offset2 < Offset1)
Base = User;
Cluster = true;
}
if (!Cluster)
return;
// Sort them in increasing order.
std::sort(Offsets.begin(), Offsets.end());
// Check if the loads are close enough.
SmallVector<SDNode*, 4> Loads;
unsigned NumLoads = 0;
int64_t BaseOff = Offsets[0];
SDNode *BaseLoad = O2SMap[BaseOff];
Loads.push_back(BaseLoad);
for (unsigned i = 1, e = Offsets.size(); i != e; ++i) {
int64_t Offset = Offsets[i];
SDNode *Load = O2SMap[Offset];
if (!TII->shouldScheduleLoadsNear(BaseLoad, Load, BaseOff, Offset,NumLoads))
break; // Stop right here. Ignore loads that are further away.
Loads.push_back(Load);
++NumLoads;
}
if (NumLoads == 0)
return;
// Cluster loads by adding MVT::Glue outputs and inputs. This also
// ensure they are scheduled in order of increasing addresses.
SDNode *Lead = Loads[0];
AddGlue(Lead, SDValue(0, 0), true, DAG);
SDValue InGlue = SDValue(Lead, Lead->getNumValues() - 1);
for (unsigned I = 1, E = Loads.size(); I != E; ++I) {
bool OutGlue = I < E - 1;
SDNode *Load = Loads[I];
AddGlue(Load, InGlue, OutGlue, DAG);
if (OutGlue)
InGlue = SDValue(Load, Load->getNumValues() - 1);
++LoadsClustered;
}
}
void ScheduleDAGSDNodes::BuildSchedUnits() {
// During scheduling, the NodeId field of SDNode is used to map SDNodes
// to their associated SUnits by holding SUnits table indices. A value
// of -1 means the SDNode does not yet have an associated SUnit.
unsigned NumNodes = 0;
for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
E = DAG->allnodes_end(); NI != E; ++NI) {
NI->setNodeId(-1);
++NumNodes;
}
// Reserve entries in the vector for each of the SUnits we are creating. This
// ensure that reallocation of the vector won't happen, so SUnit*'s won't get
// invalidated.
// FIXME: Multiply by 2 because we may clone nodes during scheduling.
// This is a temporary workaround.
SUnits.reserve(NumNodes * 2);
// Add all nodes in depth first order.
SmallVector<SDNode*, 64> Worklist;
SmallPtrSet<SDNode*, 64> Visited;
Worklist.push_back(DAG->getRoot().getNode());
Visited.insert(DAG->getRoot().getNode());
SmallVector<SUnit*, 8> CallSUnits;
while (!Worklist.empty()) {
SDNode *NI = Worklist.pop_back_val();
// Add all operands to the worklist unless they've already been added.
for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i)
if (Visited.insert(NI->getOperand(i).getNode()))
Worklist.push_back(NI->getOperand(i).getNode());
if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate.
continue;
// If this node has already been processed, stop now.
if (NI->getNodeId() != -1) continue;
SUnit *NodeSUnit = newSUnit(NI);
// See if anything is glued to this node, if so, add them to glued
// nodes. Nodes can have at most one glue input and one glue output. Glue
// is required to be the last operand and result of a node.
// Scan up to find glued preds.
SDNode *N = NI;
while (N->getNumOperands() &&
N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Glue) {
N = N->getOperand(N->getNumOperands()-1).getNode();
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall())
NodeSUnit->isCall = true;
}
// Scan down to find any glued succs.
N = NI;
while (N->getValueType(N->getNumValues()-1) == MVT::Glue) {
SDValue GlueVal(N, N->getNumValues()-1);
// There are either zero or one users of the Glue result.
bool HasGlueUse = false;
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI)
if (GlueVal.isOperandOf(*UI)) {
HasGlueUse = true;
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
N = *UI;
if (N->isMachineOpcode() && TII->get(N->getMachineOpcode()).isCall())
NodeSUnit->isCall = true;
break;
}
if (!HasGlueUse) break;
}
if (NodeSUnit->isCall)
CallSUnits.push_back(NodeSUnit);
// Schedule zero-latency TokenFactor below any nodes that may increase the
// schedule height. Otherwise, ancestors of the TokenFactor may appear to
// have false stalls.
if (NI->getOpcode() == ISD::TokenFactor)
NodeSUnit->isScheduleLow = true;
// If there are glue operands involved, N is now the bottom-most node
// of the sequence of nodes that are glued together.
// Update the SUnit.
NodeSUnit->setNode(N);
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
// Compute NumRegDefsLeft. This must be done before AddSchedEdges.
InitNumRegDefsLeft(NodeSUnit);
// Assign the Latency field of NodeSUnit using target-provided information.
computeLatency(NodeSUnit);
}
// Find all call operands.
while (!CallSUnits.empty()) {
SUnit *SU = CallSUnits.pop_back_val();
for (const SDNode *SUNode = SU->getNode(); SUNode;
SUNode = SUNode->getGluedNode()) {
if (SUNode->getOpcode() != ISD::CopyToReg)
continue;
SDNode *SrcN = SUNode->getOperand(2).getNode();
if (isPassiveNode(SrcN)) continue; // Not scheduled.
SUnit *SrcSU = &SUnits[SrcN->getNodeId()];
SrcSU->isCallOp = true;
}
}
}
/// ClusterNeighboringLoads - Force nearby loads together by "gluing" them.
/// This function finds loads of the same base and different offsets. If the
/// offsets are not far apart (target specific), it add MVT::Glue inputs and
/// outputs to ensure they are scheduled together and in order. This
/// optimization may benefit some targets by improving cache locality.
void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
SDNode *Chain = nullptr;
unsigned NumOps = Node->getNumOperands();
if (Node->getOperand(NumOps-1).getValueType() == MVT::Other)
Chain = Node->getOperand(NumOps-1).getNode();
if (!Chain)
return;
// Look for other loads of the same chain. Find loads that are loading from
// the same base pointer and different offsets.
SmallPtrSet<SDNode*, 16> Visited;
SmallVector<int64_t, 4> Offsets;
DenseMap<long long, SDNode*> O2SMap; // Map from offset to SDNode.
bool Cluster = false;
SDNode *Base = Node;
// This algorithm requires a reasonably low use count before finding a match
// to avoid uselessly blowing up compile time in large blocks.
unsigned UseCount = 0;
for (SDNode::use_iterator I = Chain->use_begin(), E = Chain->use_end();
I != E && UseCount < 100; ++I, ++UseCount) {
SDNode *User = *I;
if (User == Node || !Visited.insert(User).second)
continue;
int64_t Offset1, Offset2;
if (!TII->areLoadsFromSameBasePtr(Base, User, Offset1, Offset2) ||
Offset1 == Offset2)
// FIXME: Should be ok if they addresses are identical. But earlier
// optimizations really should have eliminated one of the loads.
continue;
if (O2SMap.insert(std::make_pair(Offset1, Base)).second)
Offsets.push_back(Offset1);
O2SMap.insert(std::make_pair(Offset2, User));
Offsets.push_back(Offset2);
if (Offset2 < Offset1)
Base = User;
Cluster = true;
// Reset UseCount to allow more matches.
UseCount = 0;
}
if (!Cluster)
return;
// Sort them in increasing order.
std::sort(Offsets.begin(), Offsets.end());
// Check if the loads are close enough.
SmallVector<SDNode*, 4> Loads;
unsigned NumLoads = 0;
int64_t BaseOff = Offsets[0];
SDNode *BaseLoad = O2SMap[BaseOff];
Loads.push_back(BaseLoad);
for (unsigned i = 1, e = Offsets.size(); i != e; ++i) {
int64_t Offset = Offsets[i];
SDNode *Load = O2SMap[Offset];
if (!TII->shouldScheduleLoadsNear(BaseLoad, Load, BaseOff, Offset,NumLoads))
break; // Stop right here. Ignore loads that are further away.
Loads.push_back(Load);
++NumLoads;
}
if (NumLoads == 0)
return;
// Cluster loads by adding MVT::Glue outputs and inputs. This also
// ensure they are scheduled in order of increasing addresses.
SDNode *Lead = Loads[0];
SDValue InGlue = SDValue(nullptr, 0);
if (AddGlue(Lead, InGlue, true, DAG))
InGlue = SDValue(Lead, Lead->getNumValues() - 1);
for (unsigned I = 1, E = Loads.size(); I != E; ++I) {
bool OutGlue = I < E - 1;
SDNode *Load = Loads[I];
// If AddGlue fails, we could leave an unsused glue value. This should not
// cause any
if (AddGlue(Load, InGlue, OutGlue, DAG)) {
if (OutGlue)
InGlue = SDValue(Load, Load->getNumValues() - 1);
++LoadsClustered;
}
else if (!OutGlue && InGlue.getNode())
RemoveUnusedGlue(InGlue.getNode(), DAG);
}
}
// Note: branch conditions, by definition, only have a chain user.
// This is why it should not be saved in a map for recall.
Value* ARMIREmitter::visitBRCOND(const SDNode *N) {
// Get the address
const ConstantSDNode *DestNode = dyn_cast<ConstantSDNode>(N->getOperand(0));
if (!DestNode) {
printError("visitBRCOND: Not a constant integer for branch!");
return NULL;
}
uint64_t DestInt = DestNode->getSExtValue();
uint64_t PC = Dec->getDisassembler()->getDebugOffset(N->getDebugLoc());
// Note: pipeline is 8 bytes
uint64_t Tgt = PC + 8 + DestInt;
Function *F = IRB->GetInsertBlock()->getParent();
BasicBlock *CurBB = IRB->GetInsertBlock();
BasicBlock *BBTgt = Dec->getOrCreateBasicBlock(Tgt, F);
// Parse the branch condition code
const ConstantSDNode *CCNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
if (!CCNode) {
printError("visitBRCOND: Condition code is not a constant integer!");
return NULL;
}
ARMCC::CondCodes ARMcc = ARMCC::CondCodes(CCNode->getZExtValue());
// Unconditional branch
if (ARMcc == ARMCC::AL) {
Instruction *Br = IRB->CreateBr(BBTgt);
Br->setDebugLoc(N->getDebugLoc());
return Br;
}
// If not a conditional branch, find the successor block and look at CC
BasicBlock *NextBB = NULL;
Function::iterator BI = F->begin(), BE= F->end();
while (BI != BE && BI->getName() != CurBB->getName()) ++BI;
++BI;
if (BI == BE) { // NOTE: This should never happen...
NextBB = Dec->getOrCreateBasicBlock("end", F);
} else {
NextBB = &(*BI);
}
SDNode *CPSR = N->getOperand(2)->getOperand(1).getNode();
SDNode *CMPNode = NULL;
for (SDNode::use_iterator I = CPSR->use_begin(), E = CPSR->use_end(); I != E;
++I) {
if (I->getOpcode() == ISD::CopyToReg) {
CMPNode = I->getOperand(2).getNode();
}
}
if (CMPNode == NULL) {
errs() << "ARMIREmitter ERROR: Could not find CMP SDNode for ARMBRCond!\n";
return NULL;
}
Value *Cmp = NULL;
Value *LHS = visit(CMPNode->getOperand(0).getNode());
Value *RHS = visit(CMPNode->getOperand(1).getNode());
// See ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC); in ARMISelLowering.cpp
// TODO: Add support for conditions that handle floating point
switch(ARMcc) {
default:
printError("Unknown condition code");
return NULL;
case ARMCC::EQ:
Cmp = IRB->CreateICmpEQ(LHS, RHS);
break;
case ARMCC::NE:
Cmp = IRB->CreateICmpNE(LHS, RHS);
break;
case ARMCC::HS:
// HS - unsigned higher or same (or carry set)
Cmp = IRB->CreateICmpUGE(LHS, RHS);
break;
case ARMCC::LO:
// LO - unsigned lower (or carry clear)
Cmp = IRB->CreateICmpULT(LHS, RHS);
break;
case ARMCC::MI:
// MI - minus (negative)
printError("Condition code MI is not handled at this time!");
return NULL;
// break;
case ARMCC::PL:
// PL - plus (positive or zero)
printError("Condition code PL is not handled at this time!");
return NULL;
// break;
case ARMCC::VS:
// VS - V Set (signed overflow)
printError("Condition code VS is not handled at this time!");
return NULL;
// break;
case ARMCC::VC:
// VC - V clear (no signed overflow)
printError("Condition code VC is not handled at this time!");
return NULL;
// break;
case ARMCC::HI:
// HI - unsigned higher
Cmp = IRB->CreateICmpUGT(LHS, RHS);
break;
case ARMCC::LS:
// LS - unsigned lower or same
Cmp = IRB->CreateICmpULE(LHS, RHS);
break;
case ARMCC::GE:
// GE - signed greater or equal
Cmp = IRB->CreateICmpSGE(LHS, RHS);
break;
case ARMCC::LT:
// LT - signed less than
Cmp = IRB->CreateICmpSLT(LHS, RHS);
break;
case ARMCC::GT:
// GT - signed greater than
Cmp = IRB->CreateICmpSGT(LHS, RHS);
break;
case ARMCC::LE:
// LE - signed less than or equal
Cmp = IRB->CreateICmpSLE(LHS, RHS);
break;
}
(dyn_cast<Instruction>(Cmp))->setDebugLoc(N->getOperand(2)->getDebugLoc());
// Conditional branch
Instruction *Br = IRB->CreateCondBr(Cmp, BBTgt, NextBB);
Br->setDebugLoc(N->getDebugLoc());
return Br;
}
void ScheduleDAGSDNodes::BuildSchedUnits() {
// During scheduling, the NodeId field of SDNode is used to map SDNodes
// to their associated SUnits by holding SUnits table indices. A value
// of -1 means the SDNode does not yet have an associated SUnit.
unsigned NumNodes = 0;
for (SelectionDAG::allnodes_iterator NI = DAG->allnodes_begin(),
E = DAG->allnodes_end(); NI != E; ++NI) {
NI->setNodeId(-1);
++NumNodes;
}
// Reserve entries in the vector for each of the SUnits we are creating. This
// ensure that reallocation of the vector won't happen, so SUnit*'s won't get
// invalidated.
// FIXME: Multiply by 2 because we may clone nodes during scheduling.
// This is a temporary workaround.
SUnits.reserve(NumNodes * 2);
// Check to see if the scheduler cares about latencies.
bool UnitLatencies = ForceUnitLatencies();
// Add all nodes in depth first order.
SmallVector<SDNode*, 64> Worklist;
SmallPtrSet<SDNode*, 64> Visited;
Worklist.push_back(DAG->getRoot().getNode());
Visited.insert(DAG->getRoot().getNode());
while (!Worklist.empty()) {
SDNode *NI = Worklist.pop_back_val();
// Add all operands to the worklist unless they've already been added.
for (unsigned i = 0, e = NI->getNumOperands(); i != e; ++i)
if (Visited.insert(NI->getOperand(i).getNode()))
Worklist.push_back(NI->getOperand(i).getNode());
if (isPassiveNode(NI)) // Leaf node, e.g. a TargetImmediate.
continue;
// If this node has already been processed, stop now.
if (NI->getNodeId() != -1) continue;
SUnit *NodeSUnit = NewSUnit(NI);
// See if anything is flagged to this node, if so, add them to flagged
// nodes. Nodes can have at most one flag input and one flag output. Flags
// are required to be the last operand and result of a node.
// Scan up to find flagged preds.
SDNode *N = NI;
while (N->getNumOperands() &&
N->getOperand(N->getNumOperands()-1).getValueType() == MVT::Flag) {
N = N->getOperand(N->getNumOperands()-1).getNode();
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
}
// Scan down to find any flagged succs.
N = NI;
while (N->getValueType(N->getNumValues()-1) == MVT::Flag) {
SDValue FlagVal(N, N->getNumValues()-1);
// There are either zero or one users of the Flag result.
bool HasFlagUse = false;
for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
UI != E; ++UI)
if (FlagVal.isOperandOf(*UI)) {
HasFlagUse = true;
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
N = *UI;
break;
}
if (!HasFlagUse) break;
}
// If there are flag operands involved, N is now the bottom-most node
// of the sequence of nodes that are flagged together.
// Update the SUnit.
NodeSUnit->setNode(N);
assert(N->getNodeId() == -1 && "Node already inserted!");
N->setNodeId(NodeSUnit->NodeNum);
// Assign the Latency field of NodeSUnit using target-provided information.
if (UnitLatencies)
NodeSUnit->Latency = 1;
else
ComputeLatency(NodeSUnit);
}
}
SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
return NULL; // Already selected.
}
switch (Opc) {
default: break;
case AMDGPUISD::CONST_ADDRESS: {
for (SDNode::use_iterator I = N->use_begin(), Next = llvm::next(I);
I != SDNode::use_end(); I = Next) {
Next = llvm::next(I);
if (!I->isMachineOpcode()) {
continue;
}
unsigned Opcode = I->getMachineOpcode();
bool HasDst = TII->getOperandIdx(Opcode, AMDGPU::OpName::dst) > -1;
int SrcIdx = I.getOperandNo();
int SelIdx;
// Unlike MachineInstrs, SDNodes do not have results in their operand
// list, so we need to increment the SrcIdx, since
// R600InstrInfo::getOperandIdx is based on the MachineInstr indices.
if (HasDst) {
SrcIdx++;
}
SelIdx = TII->getSelIdx(I->getMachineOpcode(), SrcIdx);
if (SelIdx < 0) {
continue;
}
SDValue CstOffset;
if (N->getValueType(0).isVector() ||
!SelectGlobalValueConstantOffset(N->getOperand(0), CstOffset))
continue;
// Gather constants values
int SrcIndices[] = {
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src2),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_X),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Y),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_Z),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src0_W),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_X),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Y),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_Z),
TII->getOperandIdx(Opcode, AMDGPU::OpName::src1_W)
};
std::vector<unsigned> Consts;
for (unsigned i = 0; i < sizeof(SrcIndices) / sizeof(int); i++) {
int OtherSrcIdx = SrcIndices[i];
int OtherSelIdx = TII->getSelIdx(Opcode, OtherSrcIdx);
if (OtherSrcIdx < 0 || OtherSelIdx < 0) {
continue;
}
if (HasDst) {
OtherSrcIdx--;
OtherSelIdx--;
}
if (RegisterSDNode *Reg =
dyn_cast<RegisterSDNode>(I->getOperand(OtherSrcIdx))) {
if (Reg->getReg() == AMDGPU::ALU_CONST) {
ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(I->getOperand(OtherSelIdx));
Consts.push_back(Cst->getZExtValue());
}
}
}
ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(CstOffset);
Consts.push_back(Cst->getZExtValue());
if (!TII->fitsConstReadLimitations(Consts))
continue;
// Convert back to SDNode indices
if (HasDst) {
SrcIdx--;
SelIdx--;
}
std::vector<SDValue> Ops;
for (int i = 0, e = I->getNumOperands(); i != e; ++i) {
if (i == SrcIdx) {
Ops.push_back(CurDAG->getRegister(AMDGPU::ALU_CONST, MVT::f32));
} else if (i == SelIdx) {
Ops.push_back(CstOffset);
} else {
Ops.push_back(I->getOperand(i));
}
}
CurDAG->UpdateNodeOperands(*I, Ops.data(), Ops.size());
}
break;
}
case ISD::BUILD_VECTOR: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() > AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
unsigned RegClassID;
switch(N->getValueType(0).getVectorNumElements()) {
case 2: RegClassID = AMDGPU::R600_Reg64RegClassID; break;
case 4: RegClassID = AMDGPU::R600_Reg128RegClassID; break;
default: llvm_unreachable("Do not know how to lower this BUILD_VECTOR");
}
// BUILD_VECTOR is usually lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
// that adds a 128 bits reg copy when going through TwoAddressInstructions
// pass. We want to avoid 128 bits copies as much as possible because they
// can't be bundled by our scheduler.
SDValue RegSeqArgs[9] = {
CurDAG->getTargetConstant(RegClassID, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub2, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub3, MVT::i32)
};
bool IsRegSeq = true;
for (unsigned i = 0; i < N->getNumOperands(); i++) {
if (dyn_cast<RegisterSDNode>(N->getOperand(i))) {
IsRegSeq = false;
break;
}
RegSeqArgs[2 * i + 1] = N->getOperand(i);
}
if (!IsRegSeq)
break;
return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
RegSeqArgs, 2 * N->getNumOperands() + 1);
}
case ISD::BUILD_PAIR: {
SDValue RC, SubReg0, SubReg1;
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
if (N->getValueType(0) == MVT::i128) {
RC = CurDAG->getTargetConstant(AMDGPU::SReg_128RegClassID, MVT::i32);
SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0_sub1, MVT::i32);
SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub2_sub3, MVT::i32);
} else if (N->getValueType(0) == MVT::i64) {
RC = CurDAG->getTargetConstant(AMDGPU::VSrc_64RegClassID, MVT::i32);
SubReg0 = CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32);
SubReg1 = CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32);
} else {
llvm_unreachable("Unhandled value type for BUILD_PAIR");
}
const SDValue Ops[] = { RC, N->getOperand(0), SubReg0,
N->getOperand(1), SubReg1 };
return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE,
SDLoc(N), N->getValueType(0), Ops);
}
case ISD::ConstantFP:
case ISD::Constant: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
// XXX: Custom immediate lowering not implemented yet. Instead we use
// pseudo instructions defined in SIInstructions.td
if (ST.getGeneration() > AMDGPUSubtarget::NORTHERN_ISLANDS) {
break;
}
uint64_t ImmValue = 0;
unsigned ImmReg = AMDGPU::ALU_LITERAL_X;
if (N->getOpcode() == ISD::ConstantFP) {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::f64);
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N);
APFloat Value = C->getValueAPF();
float FloatValue = Value.convertToFloat();
if (FloatValue == 0.0) {
ImmReg = AMDGPU::ZERO;
} else if (FloatValue == 0.5) {
ImmReg = AMDGPU::HALF;
} else if (FloatValue == 1.0) {
ImmReg = AMDGPU::ONE;
} else {
ImmValue = Value.bitcastToAPInt().getZExtValue();
}
} else {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::i64);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
if (C->getZExtValue() == 0) {
ImmReg = AMDGPU::ZERO;
} else if (C->getZExtValue() == 1) {
ImmReg = AMDGPU::ONE_INT;
} else {
ImmValue = C->getZExtValue();
}
}
for (SDNode::use_iterator Use = N->use_begin(), Next = llvm::next(Use);
Use != SDNode::use_end(); Use = Next) {
Next = llvm::next(Use);
std::vector<SDValue> Ops;
for (unsigned i = 0; i < Use->getNumOperands(); ++i) {
Ops.push_back(Use->getOperand(i));
}
if (!Use->isMachineOpcode()) {
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
// We can only use literal constants (e.g. AMDGPU::ZERO,
// AMDGPU::ONE, etc) in machine opcodes.
continue;
}
} else {
if (!TII->isALUInstr(Use->getMachineOpcode()) ||
(TII->get(Use->getMachineOpcode()).TSFlags &
R600_InstFlag::VECTOR)) {
continue;
}
int ImmIdx = TII->getOperandIdx(Use->getMachineOpcode(),
AMDGPU::OpName::literal);
if (ImmIdx == -1) {
continue;
}
if (TII->getOperandIdx(Use->getMachineOpcode(),
AMDGPU::OpName::dst) != -1) {
// subtract one from ImmIdx, because the DST operand is usually index
// 0 for MachineInstrs, but we have no DST in the Ops vector.
ImmIdx--;
}
// Check that we aren't already using an immediate.
// XXX: It's possible for an instruction to have more than one
// immediate operand, but this is not supported yet.
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Use->getOperand(ImmIdx));
assert(C);
if (C->getZExtValue() != 0) {
// This instruction is already using an immediate.
continue;
}
// Set the immediate value
Ops[ImmIdx] = CurDAG->getTargetConstant(ImmValue, MVT::i32);
}
}
// Set the immediate register
Ops[Use.getOperandNo()] = CurDAG->getRegister(ImmReg, MVT::i32);
CurDAG->UpdateNodeOperands(*Use, Ops.data(), Use->getNumOperands());
}
break;
}
}
SDNode *Result = SelectCode(N);
// Fold operands of selected node
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.getGeneration() <= AMDGPUSubtarget::NORTHERN_ISLANDS) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
if (Result && Result->isMachineOpcode() && Result->getMachineOpcode() == AMDGPU::DOT_4) {
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldDotOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
}
if (Result && Result->isMachineOpcode() &&
!(TII->get(Result->getMachineOpcode()).TSFlags & R600_InstFlag::VECTOR)
&& TII->hasInstrModifiers(Result->getMachineOpcode())) {
// Fold FNEG/FABS
// TODO: Isel can generate multiple MachineInst, we need to recursively
// parse Result
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
// If node has a single use which is CLAMP_R600, folds it
if (Result->hasOneUse() && Result->isMachineOpcode()) {
SDNode *PotentialClamp = *Result->use_begin();
if (PotentialClamp->isMachineOpcode() &&
PotentialClamp->getMachineOpcode() == AMDGPU::CLAMP_R600) {
unsigned ClampIdx =
TII->getOperandIdx(Result->getMachineOpcode(), AMDGPU::OpName::clamp);
std::vector<SDValue> Ops;
unsigned NumOp = Result->getNumOperands();
for (unsigned i = 0; i < NumOp; ++i) {
Ops.push_back(Result->getOperand(i));
}
Ops[ClampIdx - 1] = CurDAG->getTargetConstant(1, MVT::i32);
Result = CurDAG->SelectNodeTo(PotentialClamp,
Result->getMachineOpcode(), PotentialClamp->getVTList(),
Ops.data(), NumOp);
}
}
}
}
return Result;
}
SDNode *AMDGPUDAGToDAGISel::Select(SDNode *N) {
unsigned int Opc = N->getOpcode();
if (N->isMachineOpcode()) {
return NULL; // Already selected.
}
switch (Opc) {
default: break;
case ISD::BUILD_VECTOR: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.device()->getGeneration() > AMDGPUDeviceInfo::HD6XXX) {
break;
}
// BUILD_VECTOR is usually lowered into an IMPLICIT_DEF + 4 INSERT_SUBREG
// that adds a 128 bits reg copy when going through TwoAddressInstructions
// pass. We want to avoid 128 bits copies as much as possible because they
// can't be bundled by our scheduler.
SDValue RegSeqArgs[9] = {
CurDAG->getTargetConstant(AMDGPU::R600_Reg128RegClassID, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub0, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub1, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub2, MVT::i32),
SDValue(), CurDAG->getTargetConstant(AMDGPU::sub3, MVT::i32)
};
bool IsRegSeq = true;
for (unsigned i = 0; i < N->getNumOperands(); i++) {
if (dyn_cast<RegisterSDNode>(N->getOperand(i))) {
IsRegSeq = false;
break;
}
RegSeqArgs[2 * i + 1] = N->getOperand(i);
}
if (!IsRegSeq)
break;
return CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(),
RegSeqArgs, 2 * N->getNumOperands() + 1);
}
case ISD::ConstantFP:
case ISD::Constant: {
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
// XXX: Custom immediate lowering not implemented yet. Instead we use
// pseudo instructions defined in SIInstructions.td
if (ST.device()->getGeneration() > AMDGPUDeviceInfo::HD6XXX) {
break;
}
const R600InstrInfo *TII = static_cast<const R600InstrInfo*>(TM.getInstrInfo());
uint64_t ImmValue = 0;
unsigned ImmReg = AMDGPU::ALU_LITERAL_X;
if (N->getOpcode() == ISD::ConstantFP) {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::f64);
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N);
APFloat Value = C->getValueAPF();
float FloatValue = Value.convertToFloat();
if (FloatValue == 0.0) {
ImmReg = AMDGPU::ZERO;
} else if (FloatValue == 0.5) {
ImmReg = AMDGPU::HALF;
} else if (FloatValue == 1.0) {
ImmReg = AMDGPU::ONE;
} else {
ImmValue = Value.bitcastToAPInt().getZExtValue();
}
} else {
// XXX: 64-bit Immediates not supported yet
assert(N->getValueType(0) != MVT::i64);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(N);
if (C->getZExtValue() == 0) {
ImmReg = AMDGPU::ZERO;
} else if (C->getZExtValue() == 1) {
ImmReg = AMDGPU::ONE_INT;
} else {
ImmValue = C->getZExtValue();
}
}
for (SDNode::use_iterator Use = N->use_begin(), Next = llvm::next(Use);
Use != SDNode::use_end(); Use = Next) {
Next = llvm::next(Use);
std::vector<SDValue> Ops;
for (unsigned i = 0; i < Use->getNumOperands(); ++i) {
Ops.push_back(Use->getOperand(i));
}
if (!Use->isMachineOpcode()) {
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
// We can only use literal constants (e.g. AMDGPU::ZERO,
// AMDGPU::ONE, etc) in machine opcodes.
continue;
}
} else {
if (!TII->isALUInstr(Use->getMachineOpcode()) ||
(TII->get(Use->getMachineOpcode()).TSFlags &
R600_InstFlag::VECTOR)) {
continue;
}
int ImmIdx = TII->getOperandIdx(Use->getMachineOpcode(), R600Operands::IMM);
assert(ImmIdx != -1);
// subtract one from ImmIdx, because the DST operand is usually index
// 0 for MachineInstrs, but we have no DST in the Ops vector.
ImmIdx--;
// Check that we aren't already using an immediate.
// XXX: It's possible for an instruction to have more than one
// immediate operand, but this is not supported yet.
if (ImmReg == AMDGPU::ALU_LITERAL_X) {
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Use->getOperand(ImmIdx));
assert(C);
if (C->getZExtValue() != 0) {
// This instruction is already using an immediate.
continue;
}
// Set the immediate value
Ops[ImmIdx] = CurDAG->getTargetConstant(ImmValue, MVT::i32);
}
}
// Set the immediate register
Ops[Use.getOperandNo()] = CurDAG->getRegister(ImmReg, MVT::i32);
CurDAG->UpdateNodeOperands(*Use, Ops.data(), Use->getNumOperands());
}
break;
}
}
SDNode *Result = SelectCode(N);
// Fold operands of selected node
const AMDGPUSubtarget &ST = TM.getSubtarget<AMDGPUSubtarget>();
if (ST.device()->getGeneration() <= AMDGPUDeviceInfo::HD6XXX) {
const R600InstrInfo *TII =
static_cast<const R600InstrInfo*>(TM.getInstrInfo());
if (Result && Result->isMachineOpcode() &&
!(TII->get(Result->getMachineOpcode()).TSFlags & R600_InstFlag::VECTOR)
&& TII->isALUInstr(Result->getMachineOpcode())) {
// Fold FNEG/FABS/CONST_ADDRESS
// TODO: Isel can generate multiple MachineInst, we need to recursively
// parse Result
bool IsModified = false;
do {
std::vector<SDValue> Ops;
for(SDNode::op_iterator I = Result->op_begin(), E = Result->op_end();
I != E; ++I)
Ops.push_back(*I);
IsModified = FoldOperands(Result->getMachineOpcode(), TII, Ops);
if (IsModified) {
Result = CurDAG->UpdateNodeOperands(Result, Ops.data(), Ops.size());
}
} while (IsModified);
// If node has a single use which is CLAMP_R600, folds it
if (Result->hasOneUse() && Result->isMachineOpcode()) {
SDNode *PotentialClamp = *Result->use_begin();
if (PotentialClamp->isMachineOpcode() &&
PotentialClamp->getMachineOpcode() == AMDGPU::CLAMP_R600) {
unsigned ClampIdx =
TII->getOperandIdx(Result->getMachineOpcode(), R600Operands::CLAMP);
std::vector<SDValue> Ops;
unsigned NumOp = Result->getNumOperands();
for (unsigned i = 0; i < NumOp; ++i) {
Ops.push_back(Result->getOperand(i));
}
Ops[ClampIdx - 1] = CurDAG->getTargetConstant(1, MVT::i32);
Result = CurDAG->SelectNodeTo(PotentialClamp,
Result->getMachineOpcode(), PotentialClamp->getVTList(),
Ops.data(), NumOp);
}
}
}
}
return Result;
}
