static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
};
}
std::pair<LegalizerInfo::LegalizeAction, LLT>
LegalizerInfo::findVectorLegalAction(const InstrAspect &Aspect) const {
assert(Aspect.Type.isVector());
// First legalize the vector element size, then legalize the number of
// lanes in the vector.
if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp)
return {NotFound, Aspect.Type};
const unsigned OpcodeIdx = Aspect.Opcode - FirstOp;
const unsigned TypeIdx = Aspect.Idx;
if (TypeIdx >= ScalarInVectorActions[OpcodeIdx].size())
return {NotFound, Aspect.Type};
const SizeAndActionsVec &ElemSizeVec =
ScalarInVectorActions[OpcodeIdx][TypeIdx];
LLT IntermediateType;
auto ElementSizeAndAction =
findAction(ElemSizeVec, Aspect.Type.getScalarSizeInBits());
IntermediateType =
LLT::vector(Aspect.Type.getNumElements(), ElementSizeAndAction.first);
if (ElementSizeAndAction.second != Legal)
return {ElementSizeAndAction.second, IntermediateType};
auto i = NumElements2Actions[OpcodeIdx].find(
IntermediateType.getScalarSizeInBits());
if (i == NumElements2Actions[OpcodeIdx].end()) {
return {NotFound, IntermediateType};
}
const SizeAndActionsVec &NumElementsVec = (*i).second[TypeIdx];
auto NumElementsAndAction =
findAction(NumElementsVec, IntermediateType.getNumElements());
return {NumElementsAndAction.second,
LLT::vector(NumElementsAndAction.first,
IntermediateType.getScalarSizeInBits())};
}
static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
return Ty.isVector() &&
Ty.getNumElements() % 2 != 0 &&
Ty.getElementType().getSizeInBits() < 32;
};
}
static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
unsigned Size = Ty.getSizeInBits();
unsigned Pieces = (Size + 63) / 64;
unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
};
}
LegalizeMutation LegalizeMutations::moreElementsToNextPow2(unsigned TypeIdx,
unsigned Min) {
return [=](const LegalityQuery &Query) {
const LLT VecTy = Query.Types[TypeIdx];
unsigned NewNumElements =
std::max(1u << Log2_32_Ceil(VecTy.getNumElements()), Min);
return std::make_pair(TypeIdx,
LLT::vector(NewNumElements, VecTy.getElementType()));
};
}
void MachineIRBuilder::validateTruncExt(unsigned Dst, unsigned Src,
bool IsExtend) {
#ifndef NDEBUG
LLT SrcTy = MRI->getType(Src);
LLT DstTy = MRI->getType(Dst);
if (DstTy.isVector()) {
assert(SrcTy.isVector() && "mismatched cast between vecot and non-vector");
assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
"different number of elements in a trunc/ext");
} else
assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
if (IsExtend)
assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
"invalid narrowing extend");
else
assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() &&
"invalid widening trunc");
#endif
}
void MachineLegalizer::computeTables() {
for (auto &Op : Actions) {
LLT Ty = Op.first.second;
if (!Ty.isVector())
continue;
auto &Entry =
MaxLegalVectorElts[std::make_pair(Op.first.first, Ty.getElementType())];
Entry = std::max(Entry, Ty.getNumElements());
}
TablesInitialized = true;
}
// FIXME: inefficient implementation for now. Without ComputeValueVTs we're
// probably going to need specialized lookup structures for various types before
// we have any hope of doing well with something like <13 x i3>. Even the common
// cases should do better than what we have now.
std::pair<MachineLegalizer::LegalizeAction, LLT>
MachineLegalizer::getAction(const InstrAspect &Aspect) const {
assert(TablesInitialized && "backend forgot to call computeTables");
// These *have* to be implemented for now, they're the fundamental basis of
// how everything else is transformed.
// FIXME: the long-term plan calls for expansion in terms of load/store (if
// they're not legal).
if (Aspect.Opcode == TargetOpcode::G_SEQUENCE ||
Aspect.Opcode == TargetOpcode::G_EXTRACT)
return std::make_pair(Legal, Aspect.Type);
LegalizeAction Action = findInActions(Aspect);
if (Action != NotFound)
return findLegalAction(Aspect, Action);
unsigned Opcode = Aspect.Opcode;
LLT Ty = Aspect.Type;
if (!Ty.isVector()) {
auto DefaultAction = DefaultActions.find(Aspect.Opcode);
if (DefaultAction != DefaultActions.end() && DefaultAction->second == Legal)
return std::make_pair(Legal, Ty);
assert(DefaultAction->second == NarrowScalar && "unexpected default");
return findLegalAction(Aspect, NarrowScalar);
}
LLT EltTy = Ty.getElementType();
int NumElts = Ty.getNumElements();
auto ScalarAction = ScalarInVectorActions.find(std::make_pair(Opcode, EltTy));
if (ScalarAction != ScalarInVectorActions.end() &&
ScalarAction->second != Legal)
return findLegalAction(Aspect, ScalarAction->second);
// The element type is legal in principle, but the number of elements is
// wrong.
auto MaxLegalElts = MaxLegalVectorElts.lookup(std::make_pair(Opcode, EltTy));
if (MaxLegalElts > NumElts)
return findLegalAction(Aspect, MoreElements);
if (MaxLegalElts == 0) {
// Scalarize if there's no legal vector type, which is just a special case
// of FewerElements.
return std::make_pair(FewerElements, EltTy);
}
return findLegalAction(Aspect, FewerElements);
}
// Make sure the returned mutation makes sense for the match type.
static bool mutationIsSane(const LegalizeRule &Rule,
const LegalityQuery &Q,
std::pair<unsigned, LLT> Mutation) {
const unsigned TypeIdx = Mutation.first;
const LLT OldTy = Q.Types[TypeIdx];
const LLT NewTy = Mutation.second;
switch (Rule.getAction()) {
case FewerElements:
case MoreElements: {
if (!OldTy.isVector())
return false;
if (NewTy.isVector()) {
if (Rule.getAction() == FewerElements) {
// Make sure the element count really decreased.
if (NewTy.getNumElements() >= OldTy.getNumElements())
return false;
} else {
// Make sure the element count really increased.
if (NewTy.getNumElements() <= OldTy.getNumElements())
return false;
}
}
// Make sure the element type didn't change.
return NewTy.getScalarType() == OldTy.getElementType();
}
case NarrowScalar:
case WidenScalar: {
if (OldTy.isVector()) {
// Number of elements should not change.
if (!NewTy.isVector() || OldTy.getNumElements() != NewTy.getNumElements())
return false;
} else {
// Both types must be vectors
if (NewTy.isVector())
return false;
}
if (Rule.getAction() == NarrowScalar) {
// Make sure the size really decreased.
if (NewTy.getScalarSizeInBits() >= OldTy.getScalarSizeInBits())
return false;
} else {
// Make sure the size really increased.
if (NewTy.getScalarSizeInBits() <= OldTy.getScalarSizeInBits())
return false;
}
return true;
}
default:
return true;
}
}
void LegalizerInfo::computeTables() {
for (unsigned Opcode = 0; Opcode <= LastOp - FirstOp; ++Opcode) {
for (unsigned Idx = 0; Idx != Actions[Opcode].size(); ++Idx) {
for (auto &Action : Actions[Opcode][Idx]) {
LLT Ty = Action.first;
if (!Ty.isVector())
continue;
auto &Entry = MaxLegalVectorElts[std::make_pair(Opcode + FirstOp,
Ty.getElementType())];
Entry = std::max(Entry, Ty.getNumElements());
}
}
}
TablesInitialized = true;
}
static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT QueryTy = Query.Types[TypeIdx];
return QueryTy.isVector() && QueryTy.getNumElements() % 2 != 0;
};
}
// FIXME: inefficient implementation for now. Without ComputeValueVTs we're
// probably going to need specialized lookup structures for various types before
// we have any hope of doing well with something like <13 x i3>. Even the common
// cases should do better than what we have now.
std::pair<LegalizerInfo::LegalizeAction, LLT>
LegalizerInfo::getAction(const InstrAspect &Aspect) const {
assert(TablesInitialized && "backend forgot to call computeTables");
// These *have* to be implemented for now, they're the fundamental basis of
// how everything else is transformed.
// FIXME: the long-term plan calls for expansion in terms of load/store (if
// they're not legal).
if (Aspect.Opcode == TargetOpcode::G_MERGE_VALUES ||
Aspect.Opcode == TargetOpcode::G_UNMERGE_VALUES)
return std::make_pair(Legal, Aspect.Type);
LLT Ty = Aspect.Type;
LegalizeAction Action = findInActions(Aspect);
// LegalizerHelper is not able to handle non-power-of-2 types right now, so do
// not try to legalize them unless they are marked as Legal or Custom.
// FIXME: This is a temporary hack until the general non-power-of-2
// legalization works.
if (!isPowerOf2_64(Ty.getSizeInBits()) &&
!(Action == Legal || Action == Custom))
return std::make_pair(Unsupported, LLT());
if (Action != NotFound)
return findLegalAction(Aspect, Action);
unsigned Opcode = Aspect.Opcode;
if (!Ty.isVector()) {
auto DefaultAction = DefaultActions.find(Aspect.Opcode);
if (DefaultAction != DefaultActions.end() && DefaultAction->second == Legal)
return std::make_pair(Legal, Ty);
if (DefaultAction != DefaultActions.end() && DefaultAction->second == Lower)
return std::make_pair(Lower, Ty);
if (DefaultAction == DefaultActions.end() ||
DefaultAction->second != NarrowScalar)
return std::make_pair(Unsupported, LLT());
return findLegalAction(Aspect, NarrowScalar);
}
LLT EltTy = Ty.getElementType();
int NumElts = Ty.getNumElements();
auto ScalarAction = ScalarInVectorActions.find(std::make_pair(Opcode, EltTy));
if (ScalarAction != ScalarInVectorActions.end() &&
ScalarAction->second != Legal)
return findLegalAction(Aspect, ScalarAction->second);
// The element type is legal in principle, but the number of elements is
// wrong.
auto MaxLegalElts = MaxLegalVectorElts.lookup(std::make_pair(Opcode, EltTy));
if (MaxLegalElts > NumElts)
return findLegalAction(Aspect, MoreElements);
if (MaxLegalElts == 0) {
// Scalarize if there's no legal vector type, which is just a special case
// of FewerElements.
return std::make_pair(FewerElements, EltTy);
}
return findLegalAction(Aspect, FewerElements);
}