SDNode *AArch64DAGToDAGISel::SelectVTBL(SDNode *N, unsigned NumVecs,
bool IsExt) {
assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
SDLoc dl(N);
// Check the element of look up table is 64-bit or not
unsigned Vec0Idx = IsExt ? 2 : 1;
assert(!N->getOperand(Vec0Idx + 0).getValueType().is64BitVector() &&
"The element of lookup table for vtbl and vtbx must be 128-bit");
// Check the return value type is 64-bit or not
EVT ResVT = N->getValueType(0);
bool is64BitRes = ResVT.is64BitVector();
// Create new SDValue for vector list
SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Idx,
N->op_begin() + Vec0Idx + NumVecs);
SDValue TblReg = createQTuple(Regs);
unsigned Opc = getTBLOpc(IsExt, is64BitRes, NumVecs);
SmallVector<SDValue, 3> Ops;
if (IsExt)
Ops.push_back(N->getOperand(1));
Ops.push_back(TblReg);
Ops.push_back(N->getOperand(Vec0Idx + NumVecs));
return CurDAG->getMachineNode(Opc, dl, ResVT, Ops);
}
SDNode *AArch64DAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
bool isUpdating,
const uint16_t *Opcodes) {
assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
SDLoc dl(N);
MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
unsigned AddrOpIdx = isUpdating ? 1 : 2;
unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
EVT VT = N->getOperand(Vec0Idx).getValueType();
unsigned OpcodeIndex;
switch (VT.getSimpleVT().SimpleTy) {
default: llvm_unreachable("unhandled vector store type");
case MVT::v8i8: OpcodeIndex = 0; break;
case MVT::v4i16: OpcodeIndex = 1; break;
case MVT::v2f32:
case MVT::v2i32: OpcodeIndex = 2; break;
case MVT::v1f64:
case MVT::v1i64: OpcodeIndex = 3; break;
case MVT::v16i8: OpcodeIndex = 4; break;
case MVT::v8f16:
case MVT::v8i16: OpcodeIndex = 5; break;
case MVT::v4f32:
case MVT::v4i32: OpcodeIndex = 6; break;
case MVT::v2f64:
case MVT::v2i64: OpcodeIndex = 7; break;
}
unsigned Opc = Opcodes[OpcodeIndex];
std::vector<EVT> ResTys;
if (isUpdating)
ResTys.push_back(MVT::i64);
ResTys.push_back(MVT::Other); // Type for the Chain
SmallVector<SDValue, 6> Ops;
Ops.push_back(N->getOperand(AddrOpIdx)); // Push back the Memory Address
if (isUpdating) {
SDValue Inc = N->getOperand(AddrOpIdx + 1);
if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register
Opc = getVLDSTRegisterUpdateOpcode(Opc);
Ops.push_back(Inc);
}
bool is64BitVector = VT.is64BitVector();
SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Idx,
N->op_begin() + Vec0Idx + NumVecs);
SDValue SrcReg = is64BitVector ? createDTuple(Regs) : createQTuple(Regs);
Ops.push_back(SrcReg);
// Push back the Chain
Ops.push_back(N->getOperand(0));
// Transfer memoperands.
SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
return VSt;
}
SDNode *AArch64DAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
bool isUpdating,
const uint16_t *Opcodes) {
assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
EVT VT = N->getValueType(0);
unsigned OpcodeIndex;
switch (VT.getSimpleVT().SimpleTy) {
default: llvm_unreachable("unhandled vector load type");
case MVT::v8i8: OpcodeIndex = 0; break;
case MVT::v4i16: OpcodeIndex = 1; break;
case MVT::v2f32:
case MVT::v2i32: OpcodeIndex = 2; break;
case MVT::v1f64:
case MVT::v1i64: OpcodeIndex = 3; break;
case MVT::v16i8: OpcodeIndex = 4; break;
case MVT::v8f16:
case MVT::v8i16: OpcodeIndex = 5; break;
case MVT::v4f32:
case MVT::v4i32: OpcodeIndex = 6; break;
case MVT::v2f64:
case MVT::v2i64: OpcodeIndex = 7; break;
}
unsigned Opc = Opcodes[OpcodeIndex];
SmallVector<SDValue, 2> Ops;
unsigned AddrOpIdx = isUpdating ? 1 : 2;
Ops.push_back(N->getOperand(AddrOpIdx)); // Push back the Memory Address
if (isUpdating) {
SDValue Inc = N->getOperand(AddrOpIdx + 1);
if (!isa<ConstantSDNode>(Inc.getNode())) // Increment in Register
Opc = getVLDSTRegisterUpdateOpcode(Opc);
Ops.push_back(Inc);
}
Ops.push_back(N->getOperand(0)); // Push back the Chain
std::vector<EVT> ResTys;
bool is64BitVector = VT.is64BitVector();
if (NumVecs == 1)
ResTys.push_back(VT);
else if (NumVecs == 3)
ResTys.push_back(MVT::Untyped);
else {
EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,
is64BitVector ? NumVecs : NumVecs * 2);
ResTys.push_back(ResTy);
}
if (isUpdating)
ResTys.push_back(MVT::i64); // Type of the updated register
ResTys.push_back(MVT::Other); // Type of the Chain
SDLoc dl(N);
SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
// Transfer memoperands.
MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
if (NumVecs == 1)
return VLd;
// If NumVecs > 1, the return result is a super register containing 2-4
// consecutive vector registers.
SDValue SuperReg = SDValue(VLd, 0);
unsigned Sub0 = is64BitVector ? AArch64::dsub_0 : AArch64::qsub_0;
for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
ReplaceUses(SDValue(N, Vec),
CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
// Update users of the Chain
ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
if (isUpdating)
ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
return NULL;
}
