EVT VectorProcTargetLowering::getSetCCResultType(EVT VT) const
{
if (!VT.isVector())
return MVT::i32;
return VT.changeVectorElementTypeToInteger();
}
SDValue
AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const
{
SDValue Data = Op.getOperand(0);
VTSDNode *BaseType = cast<VTSDNode>(Op.getOperand(1));
DebugLoc DL = Op.getDebugLoc();
EVT DVT = Data.getValueType();
EVT BVT = BaseType->getVT();
unsigned baseBits = BVT.getScalarType().getSizeInBits();
unsigned srcBits = DVT.isSimple() ? DVT.getScalarType().getSizeInBits() : 1;
unsigned shiftBits = srcBits - baseBits;
if (srcBits < 32) {
// If the op is less than 32 bits, then it needs to extend to 32bits
// so it can properly keep the upper bits valid.
EVT IVT = genIntType(32, DVT.isVector() ? DVT.getVectorNumElements() : 1);
Data = DAG.getNode(ISD::ZERO_EXTEND, DL, IVT, Data);
shiftBits = 32 - baseBits;
DVT = IVT;
}
SDValue Shift = DAG.getConstant(shiftBits, DVT);
// Shift left by 'Shift' bits.
Data = DAG.getNode(ISD::SHL, DL, DVT, Data, Shift);
// Signed shift Right by 'Shift' bits.
Data = DAG.getNode(ISD::SRA, DL, DVT, Data, Shift);
if (srcBits < 32) {
// Once the sign extension is done, the op needs to be converted to
// its original type.
Data = DAG.getSExtOrTrunc(Data, DL, Op.getOperand(0).getValueType());
}
return Data;
}
SDValue
NVPTXTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
bool isVarArg,
const SmallVectorImpl<ISD::OutputArg> &Outs,
const SmallVectorImpl<SDValue> &OutVals,
DebugLoc dl, SelectionDAG &DAG) const {
bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
unsigned sizesofar = 0;
unsigned idx = 0;
for (unsigned i=0, e=Outs.size(); i!=e; ++i) {
SDValue theVal = OutVals[i];
EVT theValType = theVal.getValueType();
unsigned numElems = 1;
if (theValType.isVector()) numElems = theValType.getVectorNumElements();
for (unsigned j=0,je=numElems; j!=je; ++j) {
SDValue tmpval = theVal;
if (theValType.isVector())
tmpval = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
theValType.getVectorElementType(),
tmpval, DAG.getIntPtrConstant(j));
Chain = DAG.getNode(isABI ? NVPTXISD::StoreRetval :NVPTXISD::MoveToRetval,
dl, MVT::Other,
Chain,
DAG.getConstant(isABI ? sizesofar : idx, MVT::i32),
tmpval);
if (theValType.isVector())
sizesofar += theValType.getVectorElementType().getStoreSizeInBits()/8;
else
sizesofar += theValType.getStoreSizeInBits()/8;
++idx;
}
}
return DAG.getNode(NVPTXISD::RET_FLAG, dl, MVT::Other, Chain);
}
// There is no native floating point division, but we can convert this to a
// reciprocal/multiply operation. If the first parameter is constant 1.0, then
// just a reciprocal will suffice.
SDValue
VectorProcTargetLowering::LowerFDIV(SDValue Op, SelectionDAG &DAG) const
{
DebugLoc dl = Op.getDebugLoc();
EVT type = Op.getOperand(1).getValueType();
SDValue two = DAG.getConstantFP(2.0, type);
SDValue denom = Op.getOperand(1);
SDValue estimate = DAG.getNode(VectorProcISD::RECIPROCAL_EST, dl, type, denom);
// Perform a series of newton/raphson refinements. Each iteration doubles
// the precision. The initial estimate has 6 bits of precision, so two iteration
// results in 24 bits, which is larger than the significand.
for (int i = 0; i < 2; i++)
{
// trial = x * estimate (ideally, x * 1/x should be 1.0)
// error = 2.0 - trial
// estimate = estimate * error
SDValue trial = DAG.getNode(ISD::FMUL, dl, type, estimate, denom);
SDValue error = DAG.getNode(ISD::FSUB, dl, type, two, trial);
estimate = DAG.getNode(ISD::FMUL, dl, type, estimate, error);
}
// Check if the first parameter is constant 1.0. If so, we don't need
// to multiply.
bool isOne = false;
if (type.isVector())
{
if (isSplatVector(Op.getOperand(0).getNode()))
{
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op.getOperand(0).getOperand(0));
isOne = C && C->isExactlyValue(1.0);
}
}
else
{
ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op.getOperand(0));
isOne = C && C->isExactlyValue(1.0);
}
if (!isOne)
estimate = DAG.getNode(ISD::FMUL, dl, type, Op.getOperand(0), estimate);
return estimate;
}
unsigned ARMTTI::getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src) const {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
// Single to/from double precision conversions.
static const CostTblEntry<MVT::SimpleValueType> NEONFltDblTbl[] = {
// Vector fptrunc/fpext conversions.
{ ISD::FP_ROUND, MVT::v2f64, 2 },
{ ISD::FP_EXTEND, MVT::v2f32, 2 },
{ ISD::FP_EXTEND, MVT::v4f32, 4 }
};
if (Src->isVectorTy() && ST->hasNEON() && (ISD == ISD::FP_ROUND ||
ISD == ISD::FP_EXTEND)) {
std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Src);
int Idx = CostTableLookup(NEONFltDblTbl, ISD, LT.second);
if (Idx != -1)
return LT.first * NEONFltDblTbl[Idx].Cost;
}
EVT SrcTy = TLI->getValueType(Src);
EVT DstTy = TLI->getValueType(Dst);
if (!SrcTy.isSimple() || !DstTy.isSimple())
return TargetTransformInfo::getCastInstrCost(Opcode, Dst, Src);
// Some arithmetic, load and store operations have specific instructions
// to cast up/down their types automatically at no extra cost.
// TODO: Get these tables to know at least what the related operations are.
static const TypeConversionCostTblEntry<MVT::SimpleValueType>
NEONVectorConversionTbl[] = {
{ ISD::SIGN_EXTEND, MVT::v4i32, MVT::v4i16, 0 },
{ ISD::ZERO_EXTEND, MVT::v4i32, MVT::v4i16, 0 },
{ ISD::SIGN_EXTEND, MVT::v2i64, MVT::v2i32, 1 },
{ ISD::ZERO_EXTEND, MVT::v2i64, MVT::v2i32, 1 },
{ ISD::TRUNCATE, MVT::v4i32, MVT::v4i64, 0 },
{ ISD::TRUNCATE, MVT::v4i16, MVT::v4i32, 1 },
// The number of vmovl instructions for the extension.
{ ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i16, 3 },
{ ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i16, 3 },
{ ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i8, 3 },
{ ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i8, 3 },
{ ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i8, 7 },
{ ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i8, 7 },
{ ISD::SIGN_EXTEND, MVT::v8i64, MVT::v8i16, 6 },
{ ISD::ZERO_EXTEND, MVT::v8i64, MVT::v8i16, 6 },
{ ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
{ ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8, 6 },
// Operations that we legalize using splitting.
{ ISD::TRUNCATE, MVT::v16i8, MVT::v16i32, 6 },
{ ISD::TRUNCATE, MVT::v8i8, MVT::v8i32, 3 },
// Vector float <-> i32 conversions.
{ ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
{ ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1 },
{ ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 },
{ ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3 },
{ ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 2 },
{ ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 2 },
{ ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
{ ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i32, 1 },
{ ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i1, 3 },
{ ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i1, 3 },
{ ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 },
{ ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3 },
{ ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
{ ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2 },
{ ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
{ ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4 },
{ ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i32, 2 },
{ ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i32, 2 },
{ ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i16, 8 },
{ ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i16, 8 },
{ ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i32, 4 },
{ ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i32, 4 },
{ ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1 },
{ ISD::FP_TO_UINT, MVT::v4i32, MVT::v4f32, 1 },
{ ISD::FP_TO_SINT, MVT::v4i8, MVT::v4f32, 3 },
{ ISD::FP_TO_UINT, MVT::v4i8, MVT::v4f32, 3 },
{ ISD::FP_TO_SINT, MVT::v4i16, MVT::v4f32, 2 },
{ ISD::FP_TO_UINT, MVT::v4i16, MVT::v4f32, 2 },
// Vector double <-> i32 conversions.
{ ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
{ ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
{ ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 },
{ ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i8, 4 },
{ ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i16, 3 },
{ ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 3 },
{ ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
{ ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2 },
{ ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f64, 2 },
{ ISD::FP_TO_UINT, MVT::v2i32, MVT::v2f64, 2 },
{ ISD::FP_TO_SINT, MVT::v8i16, MVT::v8f32, 4 },
{ ISD::FP_TO_UINT, MVT::v8i16, MVT::v8f32, 4 },
{ ISD::FP_TO_SINT, MVT::v16i16, MVT::v16f32, 8 },
{ ISD::FP_TO_UINT, MVT::v16i16, MVT::v16f32, 8 }
};
if (SrcTy.isVector() && ST->hasNEON()) {
int Idx = ConvertCostTableLookup(NEONVectorConversionTbl, ISD,
DstTy.getSimpleVT(), SrcTy.getSimpleVT());
if (Idx != -1)
return NEONVectorConversionTbl[Idx].Cost;
}
// Scalar float to integer conversions.
static const TypeConversionCostTblEntry<MVT::SimpleValueType>
NEONFloatConversionTbl[] = {
{ ISD::FP_TO_SINT, MVT::i1, MVT::f32, 2 },
{ ISD::FP_TO_UINT, MVT::i1, MVT::f32, 2 },
{ ISD::FP_TO_SINT, MVT::i1, MVT::f64, 2 },
{ ISD::FP_TO_UINT, MVT::i1, MVT::f64, 2 },
{ ISD::FP_TO_SINT, MVT::i8, MVT::f32, 2 },
{ ISD::FP_TO_UINT, MVT::i8, MVT::f32, 2 },
{ ISD::FP_TO_SINT, MVT::i8, MVT::f64, 2 },
{ ISD::FP_TO_UINT, MVT::i8, MVT::f64, 2 },
{ ISD::FP_TO_SINT, MVT::i16, MVT::f32, 2 },
{ ISD::FP_TO_UINT, MVT::i16, MVT::f32, 2 },
{ ISD::FP_TO_SINT, MVT::i16, MVT::f64, 2 },
{ ISD::FP_TO_UINT, MVT::i16, MVT::f64, 2 },
{ ISD::FP_TO_SINT, MVT::i32, MVT::f32, 2 },
{ ISD::FP_TO_UINT, MVT::i32, MVT::f32, 2 },
{ ISD::FP_TO_SINT, MVT::i32, MVT::f64, 2 },
{ ISD::FP_TO_UINT, MVT::i32, MVT::f64, 2 },
{ ISD::FP_TO_SINT, MVT::i64, MVT::f32, 10 },
{ ISD::FP_TO_UINT, MVT::i64, MVT::f32, 10 },
{ ISD::FP_TO_SINT, MVT::i64, MVT::f64, 10 },
{ ISD::FP_TO_UINT, MVT::i64, MVT::f64, 10 }
};
if (SrcTy.isFloatingPoint() && ST->hasNEON()) {
int Idx = ConvertCostTableLookup(NEONFloatConversionTbl, ISD,
DstTy.getSimpleVT(), SrcTy.getSimpleVT());
if (Idx != -1)
return NEONFloatConversionTbl[Idx].Cost;
}
// Scalar integer to float conversions.
static const TypeConversionCostTblEntry<MVT::SimpleValueType>
NEONIntegerConversionTbl[] = {
{ ISD::SINT_TO_FP, MVT::f32, MVT::i1, 2 },
{ ISD::UINT_TO_FP, MVT::f32, MVT::i1, 2 },
{ ISD::SINT_TO_FP, MVT::f64, MVT::i1, 2 },
{ ISD::UINT_TO_FP, MVT::f64, MVT::i1, 2 },
{ ISD::SINT_TO_FP, MVT::f32, MVT::i8, 2 },
{ ISD::UINT_TO_FP, MVT::f32, MVT::i8, 2 },
{ ISD::SINT_TO_FP, MVT::f64, MVT::i8, 2 },
{ ISD::UINT_TO_FP, MVT::f64, MVT::i8, 2 },
{ ISD::SINT_TO_FP, MVT::f32, MVT::i16, 2 },
{ ISD::UINT_TO_FP, MVT::f32, MVT::i16, 2 },
{ ISD::SINT_TO_FP, MVT::f64, MVT::i16, 2 },
{ ISD::UINT_TO_FP, MVT::f64, MVT::i16, 2 },
{ ISD::SINT_TO_FP, MVT::f32, MVT::i32, 2 },
{ ISD::UINT_TO_FP, MVT::f32, MVT::i32, 2 },
{ ISD::SINT_TO_FP, MVT::f64, MVT::i32, 2 },
{ ISD::UINT_TO_FP, MVT::f64, MVT::i32, 2 },
{ ISD::SINT_TO_FP, MVT::f32, MVT::i64, 10 },
{ ISD::UINT_TO_FP, MVT::f32, MVT::i64, 10 },
{ ISD::SINT_TO_FP, MVT::f64, MVT::i64, 10 },
{ ISD::UINT_TO_FP, MVT::f64, MVT::i64, 10 }
};
if (SrcTy.isInteger() && ST->hasNEON()) {
int Idx = ConvertCostTableLookup(NEONIntegerConversionTbl, ISD,
DstTy.getSimpleVT(), SrcTy.getSimpleVT());
if (Idx != -1)
return NEONIntegerConversionTbl[Idx].Cost;
}
// Scalar integer conversion costs.
static const TypeConversionCostTblEntry<MVT::SimpleValueType>
ARMIntegerConversionTbl[] = {
// i16 -> i64 requires two dependent operations.
{ ISD::SIGN_EXTEND, MVT::i64, MVT::i16, 2 },
// Truncates on i64 are assumed to be free.
{ ISD::TRUNCATE, MVT::i32, MVT::i64, 0 },
{ ISD::TRUNCATE, MVT::i16, MVT::i64, 0 },
{ ISD::TRUNCATE, MVT::i8, MVT::i64, 0 },
{ ISD::TRUNCATE, MVT::i1, MVT::i64, 0 }
};
if (SrcTy.isInteger()) {
int Idx = ConvertCostTableLookup(ARMIntegerConversionTbl, ISD,
DstTy.getSimpleVT(), SrcTy.getSimpleVT());
if (Idx != -1)
return ARMIntegerConversionTbl[Idx].Cost;
}
return TargetTransformInfo::getCastInstrCost(Opcode, Dst, Src);
}
SDValue
NVPTXTargetLowering::LowerFormalArguments(SDValue Chain,
CallingConv::ID CallConv, bool isVarArg,
const SmallVectorImpl<ISD::InputArg> &Ins,
DebugLoc dl, SelectionDAG &DAG,
SmallVectorImpl<SDValue> &InVals) const {
MachineFunction &MF = DAG.getMachineFunction();
const DataLayout *TD = getDataLayout();
const Function *F = MF.getFunction();
const AttrListPtr &PAL = F->getAttributes();
SDValue Root = DAG.getRoot();
std::vector<SDValue> OutChains;
bool isKernel = llvm::isKernelFunction(*F);
bool isABI = (nvptxSubtarget.getSmVersion() >= 20);
std::vector<Type *> argTypes;
std::vector<const Argument *> theArgs;
for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end();
I != E; ++I) {
theArgs.push_back(I);
argTypes.push_back(I->getType());
}
assert(argTypes.size() == Ins.size() &&
"Ins types and function types did not match");
int idx = 0;
for (unsigned i=0, e=Ins.size(); i!=e; ++i, ++idx) {
Type *Ty = argTypes[i];
EVT ObjectVT = getValueType(Ty);
assert(ObjectVT == Ins[i].VT &&
"Ins type did not match function type");
// If the kernel argument is image*_t or sampler_t, convert it to
// a i32 constant holding the parameter position. This can later
// matched in the AsmPrinter to output the correct mangled name.
if (isImageOrSamplerVal(theArgs[i],
(theArgs[i]->getParent() ?
theArgs[i]->getParent()->getParent() : 0))) {
assert(isKernel && "Only kernels can have image/sampler params");
InVals.push_back(DAG.getConstant(i+1, MVT::i32));
continue;
}
if (theArgs[i]->use_empty()) {
// argument is dead
InVals.push_back(DAG.getNode(ISD::UNDEF, dl, ObjectVT));
continue;
}
// In the following cases, assign a node order of "idx+1"
// to newly created nodes. The SDNOdes for params have to
// appear in the same order as their order of appearance
// in the original function. "idx+1" holds that order.
if (PAL.getParamAttributes(i+1).hasAttribute(Attributes::ByVal) == false) {
// A plain scalar.
if (isABI || isKernel) {
// If ABI, load from the param symbol
SDValue Arg = getParamSymbol(DAG, idx);
Value *srcValue = new Argument(PointerType::get(ObjectVT.getTypeForEVT(
F->getContext()),
llvm::ADDRESS_SPACE_PARAM));
SDValue p = DAG.getLoad(ObjectVT, dl, Root, Arg,
MachinePointerInfo(srcValue), false, false,
false,
TD->getABITypeAlignment(ObjectVT.getTypeForEVT(
F->getContext())));
if (p.getNode())
DAG.AssignOrdering(p.getNode(), idx+1);
InVals.push_back(p);
}
else {
// If no ABI, just move the param symbol
SDValue Arg = getParamSymbol(DAG, idx, ObjectVT);
SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
if (p.getNode())
DAG.AssignOrdering(p.getNode(), idx+1);
InVals.push_back(p);
}
continue;
}
// Param has ByVal attribute
if (isABI || isKernel) {
// Return MoveParam(param symbol).
// Ideally, the param symbol can be returned directly,
// but when SDNode builder decides to use it in a CopyToReg(),
// machine instruction fails because TargetExternalSymbol
// (not lowered) is target dependent, and CopyToReg assumes
// the source is lowered.
SDValue Arg = getParamSymbol(DAG, idx, getPointerTy());
SDValue p = DAG.getNode(NVPTXISD::MoveParam, dl, ObjectVT, Arg);
if (p.getNode())
DAG.AssignOrdering(p.getNode(), idx+1);
if (isKernel)
InVals.push_back(p);
else {
SDValue p2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ObjectVT,
DAG.getConstant(Intrinsic::nvvm_ptr_local_to_gen, MVT::i32),
p);
InVals.push_back(p2);
}
} else {
// Have to move a set of param symbols to registers and
// store them locally and return the local pointer in InVals
const PointerType *elemPtrType = dyn_cast<PointerType>(argTypes[i]);
assert(elemPtrType &&
"Byval parameter should be a pointer type");
Type *elemType = elemPtrType->getElementType();
// Compute the constituent parts
SmallVector<EVT, 16> vtparts;
SmallVector<uint64_t, 16> offsets;
ComputeValueVTs(*this, elemType, vtparts, &offsets, 0);
unsigned totalsize = 0;
for (unsigned j=0, je=vtparts.size(); j!=je; ++j)
totalsize += vtparts[j].getStoreSizeInBits();
SDValue localcopy = DAG.getFrameIndex(MF.getFrameInfo()->
CreateStackObject(totalsize/8, 16, false),
getPointerTy());
unsigned sizesofar = 0;
std::vector<SDValue> theChains;
for (unsigned j=0, je=vtparts.size(); j!=je; ++j) {
unsigned numElems = 1;
if (vtparts[j].isVector()) numElems = vtparts[j].getVectorNumElements();
for (unsigned k=0, ke=numElems; k!=ke; ++k) {
EVT tmpvt = vtparts[j];
if (tmpvt.isVector()) tmpvt = tmpvt.getVectorElementType();
SDValue arg = DAG.getNode(NVPTXISD::MoveParam, dl, tmpvt,
getParamSymbol(DAG, idx, tmpvt));
SDValue addr = DAG.getNode(ISD::ADD, dl, getPointerTy(), localcopy,
DAG.getConstant(sizesofar, getPointerTy()));
theChains.push_back(DAG.getStore(Chain, dl, arg, addr,
MachinePointerInfo(), false, false, 0));
sizesofar += tmpvt.getStoreSizeInBits()/8;
++idx;
}
}
--idx;
Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &theChains[0],
theChains.size());
InVals.push_back(localcopy);
}
}
// Clang will check explicit VarArg and issue error if any. However, Clang
// will let code with
// implicit var arg like f() pass.
// We treat this case as if the arg list is empty.
//if (F.isVarArg()) {
// assert(0 && "VarArg not supported yet!");
//}
if (!OutChains.empty())
DAG.setRoot(DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
&OutChains[0], OutChains.size()));
return Chain;
}
void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
SDLoc dl(N);
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
case TargetLowering::TypeLegal:
case TargetLowering::TypePromoteInteger:
break;
case TargetLowering::TypePromoteFloat:
llvm_unreachable("Bitcast of a promotion-needing float should never need"
"expansion");
case TargetLowering::TypeSoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeExpandInteger:
case TargetLowering::TypeExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
if (TLI.hasBigEndianPartOrdering(InVT) !=
TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeSplitVector:
GetSplitVector(InOp, Lo, Hi);
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
case TargetLowering::TypeWidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
InOp = GetWidenedVector(InOp);
EVT LoVT, HiVT;
std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
return;
}
}
if (InVT.isVector() && OutVT.isInteger()) {
// Handle cases like i64 = BITCAST v1i64 on x86, where the operand
// is legal but the result is not.
unsigned NumElems = 2;
EVT ElemVT = NOutVT;
EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
// If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
while (!isTypeLegal(NVT)) {
unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
// If the element size is smaller than byte, bail.
if (NewSizeInBits < 8)
break;
NumElems *= 2;
ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
}
if (isTypeLegal(NVT)) {
SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
SmallVector<SDValue, 8> Vals;
for (unsigned i = 0; i < NumElems; ++i)
Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
CastInOp, DAG.getConstant(i, dl,
TLI.getVectorIdxTy())));
// Build Lo, Hi pair by pairing extracted elements if needed.
unsigned Slot = 0;
for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
// Each iteration will BUILD_PAIR two nodes and append the result until
// there are only two nodes left, i.e. Lo and Hi.
SDValue LHS = Vals[Slot];
SDValue RHS = Vals[Slot + 1];
if (TLI.isBigEndian())
std::swap(LHS, RHS);
Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl,
EVT::getIntegerVT(
*DAG.getContext(),
LHS.getValueType().getSizeInBits() << 1),
LHS, RHS));
}
Lo = Vals[Slot++];
Hi = Vals[Slot++];
if (TLI.isBigEndian())
std::swap(Lo, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getDataLayout()->getPrefTypeAlignment(NOutVT.
getTypeForEVT(*DAG.getContext()));
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo,
false, false, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo,
false, false, false, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getConstant(IncrementSize, dl,
StackPtr.getValueType()));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
PtrInfo.getWithOffset(IncrementSize), false,
false, false, MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.hasBigEndianPartOrdering(OutVT))
std::swap(Lo, Hi);
}
SDValue
AMDGPUTargetLowering::LowerSDIV24(SDValue Op, SelectionDAG &DAG) const
{
DebugLoc DL = Op.getDebugLoc();
EVT OVT = Op.getValueType();
SDValue LHS = Op.getOperand(0);
SDValue RHS = Op.getOperand(1);
MVT INTTY;
MVT FLTTY;
if (!OVT.isVector()) {
INTTY = MVT::i32;
FLTTY = MVT::f32;
} else if (OVT.getVectorNumElements() == 2) {
INTTY = MVT::v2i32;
FLTTY = MVT::v2f32;
} else if (OVT.getVectorNumElements() == 4) {
INTTY = MVT::v4i32;
FLTTY = MVT::v4f32;
}
unsigned bitsize = OVT.getScalarType().getSizeInBits();
// char|short jq = ia ^ ib;
SDValue jq = DAG.getNode(ISD::XOR, DL, OVT, LHS, RHS);
// jq = jq >> (bitsize - 2)
jq = DAG.getNode(ISD::SRA, DL, OVT, jq, DAG.getConstant(bitsize - 2, OVT));
// jq = jq | 0x1
jq = DAG.getNode(ISD::OR, DL, OVT, jq, DAG.getConstant(1, OVT));
// jq = (int)jq
jq = DAG.getSExtOrTrunc(jq, DL, INTTY);
// int ia = (int)LHS;
SDValue ia = DAG.getSExtOrTrunc(LHS, DL, INTTY);
// int ib, (int)RHS;
SDValue ib = DAG.getSExtOrTrunc(RHS, DL, INTTY);
// float fa = (float)ia;
SDValue fa = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ia);
// float fb = (float)ib;
SDValue fb = DAG.getNode(ISD::SINT_TO_FP, DL, FLTTY, ib);
// float fq = native_divide(fa, fb);
SDValue fq = DAG.getNode(AMDGPUISD::DIV_INF, DL, FLTTY, fa, fb);
// fq = trunc(fq);
fq = DAG.getNode(ISD::FTRUNC, DL, FLTTY, fq);
// float fqneg = -fq;
SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FLTTY, fq);
// float fr = mad(fqneg, fb, fa);
SDValue fr = DAG.getNode(AMDGPUISD::MAD, DL, FLTTY, fqneg, fb, fa);
// int iq = (int)fq;
SDValue iq = DAG.getNode(ISD::FP_TO_SINT, DL, INTTY, fq);
// fr = fabs(fr);
fr = DAG.getNode(ISD::FABS, DL, FLTTY, fr);
// fb = fabs(fb);
fb = DAG.getNode(ISD::FABS, DL, FLTTY, fb);
// int cv = fr >= fb;
SDValue cv;
if (INTTY == MVT::i32) {
cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
} else {
cv = DAG.getSetCC(DL, INTTY, fr, fb, ISD::SETOGE);
}
// jq = (cv ? jq : 0);
jq = DAG.getNode(ISD::SELECT, DL, OVT, cv, jq,
DAG.getConstant(0, OVT));
// dst = iq + jq;
iq = DAG.getSExtOrTrunc(iq, DL, OVT);
iq = DAG.getNode(ISD::ADD, DL, OVT, iq, jq);
return iq;
}
void DAGTypeLegalizer::ExpandRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
SDValue &Hi) {
EVT OutVT = N->getValueType(0);
EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
SDValue InOp = N->getOperand(0);
EVT InVT = InOp.getValueType();
DebugLoc dl = N->getDebugLoc();
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
default:
assert(false && "Unknown type action!");
case Legal:
case PromoteInteger:
break;
case SoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case ExpandInteger:
case ExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case SplitVector:
GetSplitVector(InOp, Lo, Hi);
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case ScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case WidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BIT_CONVERT");
InOp = GetWidenedVector(InOp);
EVT InNVT = EVT::getVectorVT(*DAG.getContext(), InVT.getVectorElementType(),
InVT.getVectorNumElements()/2);
Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
}
}
if (InVT.isVector() && OutVT.isInteger()) {
// Handle cases like i64 = BIT_CONVERT v1i64 on x86, where the operand
// is legal but the result is not.
EVT NVT = EVT::getVectorVT(*DAG.getContext(), NOutVT, 2);
if (isTypeLegal(NVT)) {
SDValue CastInOp = DAG.getNode(ISD::BIT_CONVERT, dl, NVT, InOp);
Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
DAG.getIntPtrConstant(1));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getTargetData()->getPrefTypeAlignment(NOutVT.getTypeForEVT(*DAG.getContext()));
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, SV, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getIntPtrConstant(IncrementSize));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, IncrementSize, false,
MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.isBigEndian())
std::swap(Lo, Hi);
}
EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy) const {
assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
if (LHSTy.isVector())
return LHSTy;
return getScalarShiftAmountTy(LHSTy);
}
EVT TargetLoweringBase::getSetCCResultType(LLVMContext &, EVT VT) const {
assert(!VT.isVector() && "No default SetCC type for vectors!");
return getPointerTy(0).SimpleTy;
}
bool PPCCTRLoops::mightUseCTR(BasicBlock *BB) {
for (BasicBlock::iterator J = BB->begin(), JE = BB->end();
J != JE; ++J) {
if (CallInst *CI = dyn_cast<CallInst>(J)) {
// Inline ASM is okay, unless it clobbers the ctr register.
if (InlineAsm *IA = dyn_cast<InlineAsm>(CI->getCalledValue())) {
if (asmClobbersCTR(IA))
return true;
continue;
}
if (Function *F = CI->getCalledFunction()) {
// Most intrinsics don't become function calls, but some might.
// sin, cos, exp and log are always calls.
unsigned Opcode = 0;
if (F->getIntrinsicID() != Intrinsic::not_intrinsic) {
switch (F->getIntrinsicID()) {
default: continue;
// If we have a call to ppc_is_decremented_ctr_nonzero, or ppc_mtctr
// we're definitely using CTR.
case Intrinsic::ppc_is_decremented_ctr_nonzero:
case Intrinsic::ppc_mtctr:
return true;
// VisualStudio defines setjmp as _setjmp
#if defined(_MSC_VER) && defined(setjmp) && \
!defined(setjmp_undefined_for_msvc)
# pragma push_macro("setjmp")
# undef setjmp
# define setjmp_undefined_for_msvc
#endif
case Intrinsic::setjmp:
#if defined(_MSC_VER) && defined(setjmp_undefined_for_msvc)
// let's return it to _setjmp state
# pragma pop_macro("setjmp")
# undef setjmp_undefined_for_msvc
#endif
case Intrinsic::longjmp:
// Exclude eh_sjlj_setjmp; we don't need to exclude eh_sjlj_longjmp
// because, although it does clobber the counter register, the
// control can't then return to inside the loop unless there is also
// an eh_sjlj_setjmp.
case Intrinsic::eh_sjlj_setjmp:
case Intrinsic::memcpy:
case Intrinsic::memmove:
case Intrinsic::memset:
case Intrinsic::powi:
case Intrinsic::log:
case Intrinsic::log2:
case Intrinsic::log10:
case Intrinsic::exp:
case Intrinsic::exp2:
case Intrinsic::pow:
case Intrinsic::sin:
case Intrinsic::cos:
return true;
case Intrinsic::copysign:
if (CI->getArgOperand(0)->getType()->getScalarType()->
isPPC_FP128Ty())
return true;
else
continue; // ISD::FCOPYSIGN is never a library call.
case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
case Intrinsic::rint: Opcode = ISD::FRINT; break;
case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
case Intrinsic::round: Opcode = ISD::FROUND; break;
case Intrinsic::minnum: Opcode = ISD::FMINNUM; break;
case Intrinsic::maxnum: Opcode = ISD::FMAXNUM; break;
case Intrinsic::umul_with_overflow: Opcode = ISD::UMULO; break;
case Intrinsic::smul_with_overflow: Opcode = ISD::SMULO; break;
}
}
// PowerPC does not use [US]DIVREM or other library calls for
// operations on regular types which are not otherwise library calls
// (i.e. soft float or atomics). If adapting for targets that do,
// additional care is required here.
LibFunc Func;
if (!F->hasLocalLinkage() && F->hasName() && LibInfo &&
LibInfo->getLibFunc(F->getName(), Func) &&
LibInfo->hasOptimizedCodeGen(Func)) {
// Non-read-only functions are never treated as intrinsics.
if (!CI->onlyReadsMemory())
return true;
// Conversion happens only for FP calls.
if (!CI->getArgOperand(0)->getType()->isFloatingPointTy())
return true;
switch (Func) {
default: return true;
case LibFunc_copysign:
case LibFunc_copysignf:
continue; // ISD::FCOPYSIGN is never a library call.
case LibFunc_copysignl:
return true;
case LibFunc_fabs:
case LibFunc_fabsf:
case LibFunc_fabsl:
continue; // ISD::FABS is never a library call.
case LibFunc_sqrt:
case LibFunc_sqrtf:
case LibFunc_sqrtl:
Opcode = ISD::FSQRT; break;
case LibFunc_floor:
case LibFunc_floorf:
case LibFunc_floorl:
Opcode = ISD::FFLOOR; break;
case LibFunc_nearbyint:
case LibFunc_nearbyintf:
case LibFunc_nearbyintl:
Opcode = ISD::FNEARBYINT; break;
case LibFunc_ceil:
case LibFunc_ceilf:
case LibFunc_ceill:
Opcode = ISD::FCEIL; break;
case LibFunc_rint:
case LibFunc_rintf:
case LibFunc_rintl:
Opcode = ISD::FRINT; break;
case LibFunc_round:
case LibFunc_roundf:
case LibFunc_roundl:
Opcode = ISD::FROUND; break;
case LibFunc_trunc:
case LibFunc_truncf:
case LibFunc_truncl:
Opcode = ISD::FTRUNC; break;
case LibFunc_fmin:
case LibFunc_fminf:
case LibFunc_fminl:
Opcode = ISD::FMINNUM; break;
case LibFunc_fmax:
case LibFunc_fmaxf:
case LibFunc_fmaxl:
Opcode = ISD::FMAXNUM; break;
}
}
if (Opcode) {
EVT EVTy =
TLI->getValueType(*DL, CI->getArgOperand(0)->getType(), true);
if (EVTy == MVT::Other)
return true;
if (TLI->isOperationLegalOrCustom(Opcode, EVTy))
continue;
else if (EVTy.isVector() &&
TLI->isOperationLegalOrCustom(Opcode, EVTy.getScalarType()))
continue;
return true;
}
}
return true;
} else if (isa<BinaryOperator>(J) &&
J->getType()->getScalarType()->isPPC_FP128Ty()) {
// Most operations on ppc_f128 values become calls.
return true;
} else if (isa<UIToFPInst>(J) || isa<SIToFPInst>(J) ||
isa<FPToUIInst>(J) || isa<FPToSIInst>(J)) {
CastInst *CI = cast<CastInst>(J);
if (CI->getSrcTy()->getScalarType()->isPPC_FP128Ty() ||
CI->getDestTy()->getScalarType()->isPPC_FP128Ty() ||
isLargeIntegerTy(!TM->isPPC64(), CI->getSrcTy()->getScalarType()) ||
isLargeIntegerTy(!TM->isPPC64(), CI->getDestTy()->getScalarType()))
return true;
} else if (isLargeIntegerTy(!TM->isPPC64(),
J->getType()->getScalarType()) &&
(J->getOpcode() == Instruction::UDiv ||
J->getOpcode() == Instruction::SDiv ||
J->getOpcode() == Instruction::URem ||
J->getOpcode() == Instruction::SRem)) {
return true;
} else if (!TM->isPPC64() &&
isLargeIntegerTy(false, J->getType()->getScalarType()) &&
(J->getOpcode() == Instruction::Shl ||
J->getOpcode() == Instruction::AShr ||
J->getOpcode() == Instruction::LShr)) {
// Only on PPC32, for 128-bit integers (specifically not 64-bit
// integers), these might be runtime calls.
return true;
} else if (isa<IndirectBrInst>(J) || isa<InvokeInst>(J)) {
// On PowerPC, indirect jumps use the counter register.
return true;
} else if (SwitchInst *SI = dyn_cast<SwitchInst>(J)) {
if (SI->getNumCases() + 1 >= (unsigned)TLI->getMinimumJumpTableEntries())
return true;
}
// FREM is always a call.
if (J->getOpcode() == Instruction::FRem)
return true;
if (STI->useSoftFloat()) {
switch(J->getOpcode()) {
case Instruction::FAdd:
case Instruction::FSub:
case Instruction::FMul:
case Instruction::FDiv:
case Instruction::FPTrunc:
case Instruction::FPExt:
case Instruction::FPToUI:
case Instruction::FPToSI:
case Instruction::UIToFP:
case Instruction::SIToFP:
case Instruction::FCmp:
return true;
}
}
for (Value *Operand : J->operands())
if (memAddrUsesCTR(*TM, Operand))
return true;
}
return false;
}