SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
// The vector type is legal but the element type needs expansion.
MVT VecVT = N->getValueType(0);
unsigned NumElts = VecVT.getVectorNumElements();
DebugLoc dl = N->getDebugLoc();
SDValue Val = N->getOperand(1);
MVT OldEVT = Val.getValueType();
MVT NewEVT = TLI.getTypeToTransformTo(OldEVT);
assert(OldEVT == VecVT.getVectorElementType() &&
"Inserted element type doesn't match vector element type!");
// Bitconvert to a vector of twice the length with elements of the expanded
// type, insert the expanded vector elements, and then convert back.
MVT NewVecVT = MVT::getVectorVT(NewEVT, NumElts*2);
SDValue NewVec = DAG.getNode(ISD::BIT_CONVERT, dl,
NewVecVT, N->getOperand(0));
SDValue Lo, Hi;
GetExpandedOp(Val, Lo, Hi);
if (TLI.isBigEndian())
std::swap(Lo, Hi);
SDValue Idx = N->getOperand(2);
Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
Idx = DAG.getNode(ISD::ADD, dl,
Idx.getValueType(), Idx, DAG.getIntPtrConstant(1));
NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
// Convert the new vector to the old vector type.
return DAG.getNode(ISD::BIT_CONVERT, dl, VecVT, NewVec);
}
SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
// The vector type is legal but the element type needs expansion.
MVT VecVT = N->getValueType(0);
unsigned NumElts = VecVT.getVectorNumElements();
MVT OldVT = N->getOperand(0).getValueType();
MVT NewVT = TLI.getTypeToTransformTo(OldVT);
DebugLoc dl = N->getDebugLoc();
assert(OldVT == VecVT.getVectorElementType() &&
"BUILD_VECTOR operand type doesn't match vector element type!");
// Build a vector of twice the length out of the expanded elements.
// For example <3 x i64> -> <6 x i32>.
std::vector<SDValue> NewElts;
NewElts.reserve(NumElts*2);
for (unsigned i = 0; i < NumElts; ++i) {
SDValue Lo, Hi;
GetExpandedOp(N->getOperand(i), Lo, Hi);
if (TLI.isBigEndian())
std::swap(Lo, Hi);
NewElts.push_back(Lo);
NewElts.push_back(Hi);
}
SDValue NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
MVT::getVectorVT(NewVT, NewElts.size()),
&NewElts[0], NewElts.size());
// Convert the new vector to the old vector type.
return DAG.getNode(ISD::BIT_CONVERT, dl, VecVT, NewVec);
}
// NVPTX suuport vector of legal types of any length in Intrinsics because the
// NVPTX specific type legalizer
// will legalize them to the PTX supported length.
bool
NVPTXTargetLowering::isTypeSupportedInIntrinsic(MVT VT) const {
if (isTypeLegal(VT))
return true;
if (VT.isVector()) {
MVT eVT = VT.getVectorElementType();
if (isTypeLegal(eVT))
return true;
}
return false;
}
SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
DebugLoc dl = N->getDebugLoc();
MVT VT = N->getValueType(0);
assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
"SCALAR_TO_VECTOR operand type doesn't match vector element type!");
unsigned NumElts = VT.getVectorNumElements();
SmallVector<SDValue, 16> Ops(NumElts);
Ops[0] = N->getOperand(0);
SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
for (unsigned i = 1; i < NumElts; ++i)
Ops[i] = UndefVal;
return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
}
static void EmitTypeGenerate(std::ostream &OS, const Record *ArgType,
unsigned &ArgNo) {
MVT::SimpleValueType VT = getValueType(ArgType->getValueAsDef("VT"));
if (ArgType->isSubClassOf("LLVMMatchType")) {
unsigned Number = ArgType->getValueAsInt("Number");
assert(Number < ArgNo && "Invalid matching number!");
if (ArgType->isSubClassOf("LLVMExtendedElementVectorType"))
OS << "VectorType::getExtendedElementVectorType"
<< "(dyn_cast<VectorType>(Tys[" << Number << "]))";
else if (ArgType->isSubClassOf("LLVMTruncatedElementVectorType"))
OS << "VectorType::getTruncatedElementVectorType"
<< "(dyn_cast<VectorType>(Tys[" << Number << "]))";
else
OS << "Tys[" << Number << "]";
} else if (VT == MVT::iAny || VT == MVT::fAny) {
// NOTE: The ArgNo variable here is not the absolute argument number, it is
// the index of the "arbitrary" type in the Tys array passed to the
// Intrinsic::getDeclaration function. Consequently, we only want to
// increment it when we actually hit an overloaded type. Getting this wrong
// leads to very subtle bugs!
OS << "Tys[" << ArgNo++ << "]";
} else if (MVT(VT).isVector()) {
MVT VVT = VT;
OS << "VectorType::get(";
EmitTypeForValueType(OS, VVT.getVectorElementType().getSimpleVT());
OS << ", " << VVT.getVectorNumElements() << ")";
} else if (VT == MVT::iPTR) {
OS << "PointerType::getUnqual(";
EmitTypeGenerate(OS, ArgType->getValueAsDef("ElTy"), ArgNo);
OS << ")";
} else if (VT == MVT::iPTRAny) {
// Make sure the user has passed us an argument type to overload. If not,
// treat it as an ordinary (not overloaded) intrinsic.
OS << "(" << ArgNo << " < numTys) ? Tys[" << ArgNo
<< "] : PointerType::getUnqual(";
EmitTypeGenerate(OS, ArgType->getValueAsDef("ElTy"), ArgNo);
OS << ")";
++ArgNo;
} else if (VT == MVT::isVoid) {
if (ArgNo == 0)
OS << "Type::VoidTy";
else
// MVT::isVoid is used to mean varargs here.
OS << "...";
} else {
EmitTypeForValueType(OS, VT);
}
}
static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
unsigned &NumIntermediates,
MVT &RegisterVT,
TargetLoweringBase *TLI) {
// Figure out the right, legal destination reg to copy into.
unsigned NumElts = VT.getVectorNumElements();
MVT EltTy = VT.getVectorElementType();
unsigned NumVectorRegs = 1;
// FIXME: We don't support non-power-of-2-sized vectors for now. Ideally we
// could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(NumElts)) {
NumVectorRegs = NumElts;
NumElts = 1;
}
// Divide the input until we get to a supported size. This will always
// end with a scalar if the target doesn't support vectors.
while (NumElts > 1 && !TLI->isTypeLegal(MVT::getVectorVT(EltTy, NumElts))) {
NumElts >>= 1;
NumVectorRegs <<= 1;
}
NumIntermediates = NumVectorRegs;
MVT NewVT = MVT::getVectorVT(EltTy, NumElts);
if (!TLI->isTypeLegal(NewVT))
NewVT = EltTy;
IntermediateVT = NewVT;
unsigned NewVTSize = NewVT.getSizeInBits();
// Convert sizes such as i33 to i64.
if (!isPowerOf2_32(NewVTSize))
NewVTSize = NextPowerOf2(NewVTSize);
MVT DestVT = TLI->getRegisterType(NewVT);
RegisterVT = DestVT;
if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
// Otherwise, promotion or legal types use the same number of registers as
// the vector decimated to the appropriate level.
return NumVectorRegs;
}
LLT::LLT(MVT VT) {
if (VT.isVector()) {
SizeInBits = VT.getVectorElementType().getSizeInBits();
ElementsOrAddrSpace = VT.getVectorNumElements();
Kind = ElementsOrAddrSpace == 1 ? Scalar : Vector;
} else if (VT.isValid()) {
// Aggregates are no different from real scalars as far as GlobalISel is
// concerned.
Kind = Scalar;
SizeInBits = VT.getSizeInBits();
ElementsOrAddrSpace = 1;
assert(SizeInBits != 0 && "invalid zero-sized type");
} else {
Kind = Invalid;
SizeInBits = ElementsOrAddrSpace = 0;
}
}
// Changing the scale of the vector type by reducing the number of elements and
// doubling the scalar size.
static MVT scaleVectorType(MVT VT) {
unsigned ScalarSize = VT.getVectorElementType().getScalarSizeInBits() * 2;
return MVT::getVectorVT(MVT::getIntegerVT(ScalarSize),
VT.getVectorNumElements() / 2);
}
/// computeRegisterProperties - Once all of the register classes are added,
/// this allows us to compute derived properties we expose.
void TargetLoweringBase::computeRegisterProperties() {
assert(MVT::LAST_VALUETYPE <= MVT::MAX_ALLOWED_VALUETYPE &&
"Too many value types for ValueTypeActions to hold!");
// Everything defaults to needing one register.
for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) {
NumRegistersForVT[i] = 1;
RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
}
// ...except isVoid, which doesn't need any registers.
NumRegistersForVT[MVT::isVoid] = 0;
// Find the largest integer register class.
unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
for (; RegClassForVT[LargestIntReg] == 0; --LargestIntReg)
assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
// Every integer value type larger than this largest register takes twice as
// many registers to represent as the previous ValueType.
for (unsigned ExpandedReg = LargestIntReg + 1;
ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
TypeExpandInteger);
}
// Inspect all of the ValueType's smaller than the largest integer
// register to see which ones need promotion.
unsigned LegalIntReg = LargestIntReg;
for (unsigned IntReg = LargestIntReg - 1;
IntReg >= (unsigned)MVT::i1; --IntReg) {
MVT IVT = (MVT::SimpleValueType)IntReg;
if (isTypeLegal(IVT)) {
LegalIntReg = IntReg;
} else {
RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
(const MVT::SimpleValueType)LegalIntReg;
ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
}
}
// ppcf128 type is really two f64's.
if (!isTypeLegal(MVT::ppcf128)) {
NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
TransformToType[MVT::ppcf128] = MVT::f64;
ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
}
// Decide how to handle f128. If the target does not have native f128 support,
// expand it to i128 and we will be generating soft float library calls.
if (!isTypeLegal(MVT::f128)) {
NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
TransformToType[MVT::f128] = MVT::i128;
ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
}
// Decide how to handle f64. If the target does not have native f64 support,
// expand it to i64 and we will be generating soft float library calls.
if (!isTypeLegal(MVT::f64)) {
NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
TransformToType[MVT::f64] = MVT::i64;
ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
}
// Decide how to handle f32. If the target does not have native support for
// f32, promote it to f64 if it is legal. Otherwise, expand it to i32.
if (!isTypeLegal(MVT::f32)) {
if (isTypeLegal(MVT::f64)) {
NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::f64];
RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::f64];
TransformToType[MVT::f32] = MVT::f64;
ValueTypeActions.setTypeAction(MVT::f32, TypePromoteInteger);
} else {
NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
TransformToType[MVT::f32] = MVT::i32;
ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
}
}
// Loop over all of the vector value types to see which need transformations.
for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
MVT VT = (MVT::SimpleValueType)i;
if (isTypeLegal(VT)) continue;
// Determine if there is a legal wider type. If so, we should promote to
// that wider vector type.
MVT EltVT = VT.getVectorElementType();
unsigned NElts = VT.getVectorNumElements();
if (NElts != 1 && !shouldSplitVectorElementType(EltVT)) {
bool IsLegalWiderType = false;
// First try to promote the elements of integer vectors. If no legal
// promotion was found, fallback to the widen-vector method.
for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
MVT SVT = (MVT::SimpleValueType)nVT;
// Promote vectors of integers to vectors with the same number
// of elements, with a wider element type.
if (SVT.getVectorElementType().getSizeInBits() > EltVT.getSizeInBits()
&& SVT.getVectorNumElements() == NElts &&
isTypeLegal(SVT) && SVT.getScalarType().isInteger()) {
TransformToType[i] = SVT;
RegisterTypeForVT[i] = SVT;
NumRegistersForVT[i] = 1;
ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
IsLegalWiderType = true;
break;
}
}
if (IsLegalWiderType) continue;
// Try to widen the vector.
for (unsigned nVT = i+1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
MVT SVT = (MVT::SimpleValueType)nVT;
if (SVT.getVectorElementType() == EltVT &&
SVT.getVectorNumElements() > NElts &&
isTypeLegal(SVT)) {
TransformToType[i] = SVT;
RegisterTypeForVT[i] = SVT;
NumRegistersForVT[i] = 1;
ValueTypeActions.setTypeAction(VT, TypeWidenVector);
IsLegalWiderType = true;
break;
}
}
if (IsLegalWiderType) continue;
}
MVT IntermediateVT;
MVT RegisterVT;
unsigned NumIntermediates;
NumRegistersForVT[i] =
getVectorTypeBreakdownMVT(VT, IntermediateVT, NumIntermediates,
RegisterVT, this);
RegisterTypeForVT[i] = RegisterVT;
MVT NVT = VT.getPow2VectorType();
if (NVT == VT) {
// Type is already a power of 2. The default action is to split.
TransformToType[i] = MVT::Other;
unsigned NumElts = VT.getVectorNumElements();
ValueTypeActions.setTypeAction(VT,
NumElts > 1 ? TypeSplitVector : TypeScalarizeVector);
} else {
TransformToType[i] = NVT;
ValueTypeActions.setTypeAction(VT, TypeWidenVector);
}
}
// Determine the 'representative' register class for each value type.
// An representative register class is the largest (meaning one which is
// not a sub-register class / subreg register class) legal register class for
// a group of value types. For example, on i386, i8, i16, and i32
// representative would be GR32; while on x86_64 it's GR64.
for (unsigned i = 0; i != MVT::LAST_VALUETYPE; ++i) {
const TargetRegisterClass* RRC;
uint8_t Cost;
tie(RRC, Cost) = findRepresentativeClass((MVT::SimpleValueType)i);
RepRegClassForVT[i] = RRC;
RepRegClassCostForVT[i] = Cost;
}
}
void DAGTypeLegalizer::ExpandRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
SDValue &Hi) {
MVT OutVT = N->getValueType(0);
MVT NOutVT = TLI.getTypeToTransformTo(OutVT);
SDValue InOp = N->getOperand(0);
MVT InVT = InOp.getValueType();
DebugLoc dl = N->getDebugLoc();
// Handle some special cases efficiently.
switch (getTypeAction(InVT)) {
default:
assert(false && "Unknown type action!");
case Legal:
case PromoteInteger:
break;
case SoftenFloat:
// Convert the integer operand instead.
SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case ExpandInteger:
case ExpandFloat:
// Convert the expanded pieces of the input.
GetExpandedOp(InOp, Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case SplitVector:
// Convert the split parts of the input if it was split in two.
GetSplitVector(InOp, Lo, Hi);
if (Lo.getValueType() == Hi.getValueType()) {
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
}
break;
case ScalarizeVector:
// Convert the element instead.
SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
case WidenVector: {
assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BIT_CONVERT");
InOp = GetWidenedVector(InOp);
MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
InVT.getVectorNumElements()/2);
Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(0));
Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
if (TLI.isBigEndian())
std::swap(Lo, Hi);
Lo = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Lo);
Hi = DAG.getNode(ISD::BIT_CONVERT, dl, NOutVT, Hi);
return;
}
}
// Lower the bit-convert to a store/load from the stack.
assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
// Create the stack frame object. Make sure it is aligned for both
// the source and expanded destination types.
unsigned Alignment =
TLI.getTargetData()->getPrefTypeAlignment(NOutVT.getTypeForMVT());
SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
// Emit a store to the stack slot.
SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, SV, 0);
// Load the first half from the stack slot.
Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, 0);
// Increment the pointer to the other half.
unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
DAG.getIntPtrConstant(IncrementSize));
// Load the second half from the stack slot.
Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr, SV, IncrementSize, false,
MinAlign(Alignment, IncrementSize));
// Handle endianness of the load.
if (TLI.isBigEndian())
std::swap(Lo, Hi);
}
