bool PTXMFInfoExtract::runOnMachineFunction(MachineFunction &MF) {
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
MachineRegisterInfo &MRI = MF.getRegInfo();
// Generate list of all virtual registers used in this function
for (unsigned i = 0; i < MRI.getNumVirtRegs(); ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
unsigned RegType;
if (TRC == PTX::RegPredRegisterClass)
RegType = PTXRegisterType::Pred;
else if (TRC == PTX::RegI16RegisterClass)
RegType = PTXRegisterType::B16;
else if (TRC == PTX::RegI32RegisterClass)
RegType = PTXRegisterType::B32;
else if (TRC == PTX::RegI64RegisterClass)
RegType = PTXRegisterType::B64;
else if (TRC == PTX::RegF32RegisterClass)
RegType = PTXRegisterType::F32;
else if (TRC == PTX::RegF64RegisterClass)
RegType = PTXRegisterType::F64;
MFI->addRegister(Reg, RegType, PTXRegisterSpace::Reg);
}
return false;
}
SDValue PTXTargetLowering::
LowerReturn(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
DebugLoc dl,
SelectionDAG &DAG) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention.");
case CallingConv::PTX_Kernel:
assert(Outs.size() == 0 && "Kernel must return void.");
return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
case CallingConv::PTX_Device:
//assert(Outs.size() <= 1 && "Can at most return one value.");
break;
}
MachineFunction& MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
SmallVector<CCValAssign, 16> RVLocs;
CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
getTargetMachine(), RVLocs, *DAG.getContext());
SDValue Flag;
CCInfo.AnalyzeReturn(Outs, RetCC_PTX);
for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
CCValAssign& VA = RVLocs[i];
assert(VA.isRegLoc() && "CCValAssign must be RegLoc");
unsigned Reg = VA.getLocReg();
DAG.getMachineFunction().getRegInfo().addLiveOut(Reg);
Chain = DAG.getCopyToReg(Chain, dl, Reg, OutVals[i], Flag);
// Guarantee that all emitted copies are stuck together,
// avoiding something bad
Flag = Chain.getValue(1);
MFI->addRetReg(Reg);
}
if (Flag.getNode() == 0) {
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain);
}
else {
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain, Flag);
}
}
SDNode *PTXDAGToDAGISel::SelectFrameIndex(SDNode *Node) {
int FI = cast<FrameIndexSDNode>(Node)->getIndex();
//dbgs() << "Selecting FrameIndex at index " << FI << "\n";
//SDValue TFI = CurDAG->getTargetFrameIndex(FI, Node->getValueType(0));
PTXMachineFunctionInfo *MFI = MF->getInfo<PTXMachineFunctionInfo>();
SDValue FrameSymbol = CurDAG->getTargetExternalSymbol(MFI->getFrameSymbol(FI),
Node->getValueType(0));
return FrameSymbol.getNode();
}
bool PTXMFInfoExtract::runOnMachineFunction(MachineFunction &MF) {
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
MachineRegisterInfo &MRI = MF.getRegInfo();
// Generate list of all virtual registers used in this function
for (unsigned i = 0; i < MRI.getNumVirtRegs(); ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
MFI->addVirtualRegister(TRC, Reg);
}
return false;
}
SDValue PTXTargetLowering::
LowerReturn(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
DebugLoc dl,
SelectionDAG &DAG) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention.");
case CallingConv::PTX_Kernel:
assert(Outs.size() == 0 && "Kernel must return void.");
return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
case CallingConv::PTX_Device:
assert(Outs.size() <= 1 && "Can at most return one value.");
break;
}
// PTX_Device
// return void
if (Outs.size() == 0)
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain);
assert(Outs[0].VT == MVT::i32 && "Can return only basic types");
SDValue Flag;
unsigned reg = PTX::R0;
MachineFunction &MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
MFI->setRetReg(reg);
// If this is the first return lowered for this function, add the regs to the
// liveout set for the function
if (DAG.getMachineFunction().getRegInfo().liveout_empty())
DAG.getMachineFunction().getRegInfo().addLiveOut(reg);
// Copy the result values into the output registers
Chain = DAG.getCopyToReg(Chain, dl, reg, OutVals[0], Flag);
// Guarantee that all emitted copies are stuck together,
// avoiding something bad
Flag = Chain.getValue(1);
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain, Flag);
}
bool PTXMFInfoExtract::runOnMachineFunction(MachineFunction &MF) {
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
MachineRegisterInfo &MRI = MF.getRegInfo();
DEBUG(dbgs() << "******** PTX FUNCTION LOCAL VAR REG DEF ********\n");
unsigned retreg = MFI->retReg();
DEBUG(dbgs()
<< "PTX::NoRegister == " << PTX::NoRegister << "\n"
<< "PTX::NUM_TARGET_REGS == " << PTX::NUM_TARGET_REGS << "\n");
DEBUG(for (unsigned reg = PTX::NoRegister + 1;
reg < PTX::NUM_TARGET_REGS; ++reg)
if (MRI.isPhysRegUsed(reg))
dbgs() << "Used Reg: " << reg << "\n";);
SDValue PTXTargetLowering::
LowerFormalArguments(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl,
SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
MachineFunction &MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention");
break;
case CallingConv::PTX_Kernel:
MFI->setKernel(true);
break;
case CallingConv::PTX_Device:
MFI->setKernel(false);
break;
}
// Make sure we don't add argument registers twice
if (MFI->isDoneAddArg())
llvm_unreachable("cannot add argument registers twice");
// Reset argmap before allocation
for (struct argmap_entry *i = argmap, *e = argmap + array_lengthof(argmap);
i != e; ++ i)
i->reset();
for (int i = 0, e = Ins.size(); i != e; ++ i) {
MVT::SimpleValueType VT = Ins[i].VT.SimpleTy;
struct argmap_entry *entry = std::find(argmap,
argmap + array_lengthof(argmap), VT);
if (entry == argmap + array_lengthof(argmap))
llvm_unreachable("Type of argument is not supported");
if (MFI->isKernel() && entry->RC == PTX::PredsRegisterClass)
llvm_unreachable("cannot pass preds to kernel");
MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
unsigned preg = *++(entry->loc); // allocate start from register 1
unsigned vreg = RegInfo.createVirtualRegister(entry->RC);
RegInfo.addLiveIn(preg, vreg);
MFI->addArgReg(preg);
SDValue inval;
if (MFI->isKernel())
inval = DAG.getNode(PTXISD::READ_PARAM, dl, VT, Chain,
DAG.getTargetConstant(i, MVT::i32));
else
inval = DAG.getCopyFromReg(Chain, dl, vreg, VT);
InVals.push_back(inval);
}
MFI->doneAddArg();
return Chain;
}
SDValue PTXTargetLowering::
LowerFormalArguments(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl,
SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
MachineFunction &MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention");
break;
case CallingConv::PTX_Kernel:
MFI->setKernel(true);
break;
case CallingConv::PTX_Device:
MFI->setKernel(false);
break;
}
if (MFI->isKernel()) {
// For kernel functions, we just need to emit the proper READ_PARAM ISDs
for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
assert(Ins[i].VT != MVT::i1 && "Kernels cannot take pred operands");
SDValue ArgValue = DAG.getNode(PTXISD::READ_PARAM, dl, Ins[i].VT, Chain,
DAG.getTargetConstant(i, MVT::i32));
InVals.push_back(ArgValue);
// Instead of storing a physical register in our argument list, we just
// store the total size of the parameter, in bits. The ASM printer
// knows how to process this.
MFI->addArgReg(Ins[i].VT.getStoreSizeInBits());
}
}
else {
// For device functions, we use the PTX calling convention to do register
// assignments then create CopyFromReg ISDs for the allocated registers
SmallVector<CCValAssign, 16> ArgLocs;
CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(), ArgLocs,
*DAG.getContext());
CCInfo.AnalyzeFormalArguments(Ins, CC_PTX);
for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
CCValAssign& VA = ArgLocs[i];
EVT RegVT = VA.getLocVT();
TargetRegisterClass* TRC = 0;
assert(VA.isRegLoc() && "CCValAssign must be RegLoc");
// Determine which register class we need
if (RegVT == MVT::i1) {
TRC = PTX::RegPredRegisterClass;
}
else if (RegVT == MVT::i16) {
TRC = PTX::RegI16RegisterClass;
}
else if (RegVT == MVT::i32) {
TRC = PTX::RegI32RegisterClass;
}
else if (RegVT == MVT::i64) {
TRC = PTX::RegI64RegisterClass;
}
else if (RegVT == MVT::f32) {
TRC = PTX::RegF32RegisterClass;
}
else if (RegVT == MVT::f64) {
TRC = PTX::RegF64RegisterClass;
}
else {
llvm_unreachable("Unknown parameter type");
}
unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
MF.getRegInfo().addLiveIn(VA.getLocReg(), Reg);
SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
InVals.push_back(ArgValue);
MFI->addArgReg(VA.getLocReg());
}
}
return Chain;
}
SDValue
PTXTargetLowering::LowerCall(SDValue Chain, SDValue Callee,
CallingConv::ID CallConv, bool isVarArg,
bool &isTailCall,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const {
MachineFunction& MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
PTXParamManager &PM = MFI->getParamManager();
assert(getTargetMachine().getSubtarget<PTXSubtarget>().callsAreHandled() &&
"Calls are not handled for the target device");
std::vector<SDValue> Ops;
// The layout of the ops will be [Chain, #Ins, Ins, Callee, #Outs, Outs]
Ops.resize(Outs.size() + Ins.size() + 4);
Ops[0] = Chain;
// Identify the callee function
const GlobalValue *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
assert(cast<Function>(GV)->getCallingConv() == CallingConv::PTX_Device &&
"PTX function calls must be to PTX device functions");
Callee = DAG.getTargetGlobalAddress(GV, dl, getPointerTy());
Ops[Ins.size()+2] = Callee;
// Generate STORE_PARAM nodes for each function argument. In PTX, function
// arguments are explicitly stored into .param variables and passed as
// arguments. There is no register/stack-based calling convention in PTX.
Ops[Ins.size()+3] = DAG.getTargetConstant(OutVals.size(), MVT::i32);
for (unsigned i = 0; i != OutVals.size(); ++i) {
unsigned Size = OutVals[i].getValueType().getSizeInBits();
unsigned Param = PM.addLocalParam(Size);
const std::string &ParamName = PM.getParamName(Param);
SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
MVT::Other);
Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
ParamValue, OutVals[i]);
Ops[i+Ins.size()+4] = ParamValue;
}
std::vector<SDValue> InParams;
// Generate list of .param variables to hold the return value(s).
Ops[1] = DAG.getTargetConstant(Ins.size(), MVT::i32);
for (unsigned i = 0; i < Ins.size(); ++i) {
unsigned Size = Ins[i].VT.getStoreSizeInBits();
unsigned Param = PM.addLocalParam(Size);
const std::string &ParamName = PM.getParamName(Param);
SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
MVT::Other);
Ops[i+2] = ParamValue;
InParams.push_back(ParamValue);
}
Ops[0] = Chain;
// Create the CALL node.
Chain = DAG.getNode(PTXISD::CALL, dl, MVT::Other, &Ops[0], Ops.size());
// Create the LOAD_PARAM nodes that retrieve the function return value(s).
for (unsigned i = 0; i < Ins.size(); ++i) {
SDValue Load = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
InParams[i]);
InVals.push_back(Load);
}
return Chain;
}
SDValue PTXTargetLowering::
LowerReturn(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
DebugLoc dl,
SelectionDAG &DAG) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention.");
case CallingConv::PTX_Kernel:
assert(Outs.size() == 0 && "Kernel must return void.");
return DAG.getNode(PTXISD::EXIT, dl, MVT::Other, Chain);
case CallingConv::PTX_Device:
assert(Outs.size() <= 1 && "Can at most return one value.");
break;
}
MachineFunction& MF = DAG.getMachineFunction();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
PTXParamManager &PM = MFI->getParamManager();
SDValue Flag;
const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
if (ST.useParamSpaceForDeviceArgs()) {
assert(Outs.size() < 2 && "Device functions can return at most one value");
if (Outs.size() == 1) {
unsigned ParamSize = OutVals[0].getValueType().getSizeInBits();
unsigned Param = PM.addReturnParam(ParamSize);
const std::string &ParamName = PM.getParamName(Param);
SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
MVT::Other);
Chain = DAG.getNode(PTXISD::STORE_PARAM, dl, MVT::Other, Chain,
ParamValue, OutVals[0]);
}
} else {
for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
EVT RegVT = Outs[i].VT;
TargetRegisterClass* TRC = 0;
// Determine which register class we need
if (RegVT == MVT::i1) {
TRC = PTX::RegPredRegisterClass;
}
else if (RegVT == MVT::i16) {
TRC = PTX::RegI16RegisterClass;
}
else if (RegVT == MVT::i32) {
TRC = PTX::RegI32RegisterClass;
}
else if (RegVT == MVT::i64) {
TRC = PTX::RegI64RegisterClass;
}
else if (RegVT == MVT::f32) {
TRC = PTX::RegF32RegisterClass;
}
else if (RegVT == MVT::f64) {
TRC = PTX::RegF64RegisterClass;
}
else {
llvm_unreachable("Unknown parameter type");
}
unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
SDValue Copy = DAG.getCopyToReg(Chain, dl, Reg, OutVals[i]/*, Flag*/);
SDValue OutReg = DAG.getRegister(Reg, RegVT);
Chain = DAG.getNode(PTXISD::WRITE_PARAM, dl, MVT::Other, Copy, OutReg);
MFI->addRetReg(Reg);
}
}
if (Flag.getNode() == 0) {
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain);
}
else {
return DAG.getNode(PTXISD::RET, dl, MVT::Other, Chain, Flag);
}
}
SDValue PTXTargetLowering::
LowerFormalArguments(SDValue Chain,
CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl,
SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const {
if (isVarArg) llvm_unreachable("PTX does not support varargs");
MachineFunction &MF = DAG.getMachineFunction();
const PTXSubtarget& ST = getTargetMachine().getSubtarget<PTXSubtarget>();
PTXMachineFunctionInfo *MFI = MF.getInfo<PTXMachineFunctionInfo>();
PTXParamManager &PM = MFI->getParamManager();
switch (CallConv) {
default:
llvm_unreachable("Unsupported calling convention");
break;
case CallingConv::PTX_Kernel:
MFI->setKernel(true);
break;
case CallingConv::PTX_Device:
MFI->setKernel(false);
break;
}
// We do one of two things here:
// IsKernel || SM >= 2.0 -> Use param space for arguments
// SM < 2.0 -> Use registers for arguments
if (MFI->isKernel() || ST.useParamSpaceForDeviceArgs()) {
// We just need to emit the proper LOAD_PARAM ISDs
for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
assert((!MFI->isKernel() || Ins[i].VT != MVT::i1) &&
"Kernels cannot take pred operands");
unsigned ParamSize = Ins[i].VT.getStoreSizeInBits();
unsigned Param = PM.addArgumentParam(ParamSize);
const std::string &ParamName = PM.getParamName(Param);
SDValue ParamValue = DAG.getTargetExternalSymbol(ParamName.c_str(),
MVT::Other);
SDValue ArgValue = DAG.getNode(PTXISD::LOAD_PARAM, dl, Ins[i].VT, Chain,
ParamValue);
InVals.push_back(ArgValue);
}
}
else {
for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
EVT RegVT = Ins[i].VT;
TargetRegisterClass* TRC = getRegClassFor(RegVT);
// Use a unique index in the instruction to prevent instruction folding.
// Yes, this is a hack.
SDValue Index = DAG.getTargetConstant(i, MVT::i32);
unsigned Reg = MF.getRegInfo().createVirtualRegister(TRC);
SDValue ArgValue = DAG.getNode(PTXISD::READ_PARAM, dl, RegVT, Chain,
Index);
InVals.push_back(ArgValue);
MFI->addArgReg(Reg);
}
}
return Chain;
}