void test_vshlQ_nu16 (void)
{
uint16x8_t out_uint16x8_t;
uint16x8_t arg0_uint16x8_t;
out_uint16x8_t = vshlq_n_u16 (arg0_uint16x8_t, 1);
}
void SkBlitLCD16Row_neon(SkPMColor dst[], const uint16_t src[],
SkColor color, int width, SkPMColor) {
int colA = SkColorGetA(color);
int colR = SkColorGetR(color);
int colG = SkColorGetG(color);
int colB = SkColorGetB(color);
colA = SkAlpha255To256(colA);
uint8x8_t vcolR, vcolG, vcolB;
uint16x8_t vcolA;
if (width >= 8) {
vcolA = vdupq_n_u16(colA);
vcolR = vdup_n_u8(colR);
vcolG = vdup_n_u8(colG);
vcolB = vdup_n_u8(colB);
}
while (width >= 8) {
uint8x8x4_t vdst;
uint16x8_t vmask;
uint16x8_t vmaskR, vmaskG, vmaskB;
vdst = vld4_u8((uint8_t*)dst);
vmask = vld1q_u16(src);
// Get all the color masks on 5 bits
vmaskR = vshrq_n_u16(vmask, SK_R16_SHIFT);
vmaskG = vshrq_n_u16(vshlq_n_u16(vmask, SK_R16_BITS),
SK_B16_BITS + SK_R16_BITS + 1);
vmaskB = vmask & vdupq_n_u16(SK_B16_MASK);
// Upscale to 0..32
vmaskR = vmaskR + vshrq_n_u16(vmaskR, 4);
vmaskG = vmaskG + vshrq_n_u16(vmaskG, 4);
vmaskB = vmaskB + vshrq_n_u16(vmaskB, 4);
vmaskR = vshrq_n_u16(vmaskR * vcolA, 8);
vmaskG = vshrq_n_u16(vmaskG * vcolA, 8);
vmaskB = vshrq_n_u16(vmaskB * vcolA, 8);
vdst.val[NEON_A] = vdup_n_u8(0xFF);
vdst.val[NEON_R] = SkBlend32_neon8(vcolR, vdst.val[NEON_R], vmaskR);
vdst.val[NEON_G] = SkBlend32_neon8(vcolG, vdst.val[NEON_G], vmaskG);
vdst.val[NEON_B] = SkBlend32_neon8(vcolB, vdst.val[NEON_B], vmaskB);
vst4_u8((uint8_t*)dst, vdst);
dst += 8;
src += 8;
width -= 8;
}
for (int i = 0; i < width; i++) {
dst[i] = SkBlendLCD16(colA, colR, colG, colB, dst[i], src[i]);
}
}
void SkBlitLCD16OpaqueRow_neon(SkPMColor dst[], const uint16_t src[],
SkColor color, int width,
SkPMColor opaqueDst) {
int colR = SkColorGetR(color);
int colG = SkColorGetG(color);
int colB = SkColorGetB(color);
uint8x8_t vcolR, vcolG, vcolB;
uint8x8_t vopqDstA, vopqDstR, vopqDstG, vopqDstB;
if (width >= 8) {
vcolR = vdup_n_u8(colR);
vcolG = vdup_n_u8(colG);
vcolB = vdup_n_u8(colB);
vopqDstA = vdup_n_u8(SkGetPackedA32(opaqueDst));
vopqDstR = vdup_n_u8(SkGetPackedR32(opaqueDst));
vopqDstG = vdup_n_u8(SkGetPackedG32(opaqueDst));
vopqDstB = vdup_n_u8(SkGetPackedB32(opaqueDst));
}
while (width >= 8) {
uint8x8x4_t vdst;
uint16x8_t vmask;
uint16x8_t vmaskR, vmaskG, vmaskB;
uint8x8_t vsel_trans, vsel_opq;
vdst = vld4_u8((uint8_t*)dst);
vmask = vld1q_u16(src);
// Prepare compare masks
vsel_trans = vmovn_u16(vceqq_u16(vmask, vdupq_n_u16(0)));
vsel_opq = vmovn_u16(vceqq_u16(vmask, vdupq_n_u16(0xFFFF)));
// Get all the color masks on 5 bits
vmaskR = vshrq_n_u16(vmask, SK_R16_SHIFT);
vmaskG = vshrq_n_u16(vshlq_n_u16(vmask, SK_R16_BITS),
SK_B16_BITS + SK_R16_BITS + 1);
vmaskB = vmask & vdupq_n_u16(SK_B16_MASK);
// Upscale to 0..32
vmaskR = vmaskR + vshrq_n_u16(vmaskR, 4);
vmaskG = vmaskG + vshrq_n_u16(vmaskG, 4);
vmaskB = vmaskB + vshrq_n_u16(vmaskB, 4);
vdst.val[NEON_A] = vbsl_u8(vsel_trans, vdst.val[NEON_A], vdup_n_u8(0xFF));
vdst.val[NEON_A] = vbsl_u8(vsel_opq, vopqDstA, vdst.val[NEON_A]);
vdst.val[NEON_R] = SkBlend32_neon8(vcolR, vdst.val[NEON_R], vmaskR);
vdst.val[NEON_G] = SkBlend32_neon8(vcolG, vdst.val[NEON_G], vmaskG);
vdst.val[NEON_B] = SkBlend32_neon8(vcolB, vdst.val[NEON_B], vmaskB);
vdst.val[NEON_R] = vbsl_u8(vsel_opq, vopqDstR, vdst.val[NEON_R]);
vdst.val[NEON_G] = vbsl_u8(vsel_opq, vopqDstG, vdst.val[NEON_G]);
vdst.val[NEON_B] = vbsl_u8(vsel_opq, vopqDstB, vdst.val[NEON_B]);
vst4_u8((uint8_t*)dst, vdst);
dst += 8;
src += 8;
width -= 8;
}
// Leftovers
for (int i = 0; i < width; i++) {
dst[i] = SkBlendLCD16Opaque(colR, colG, colB, dst[i], src[i],
opaqueDst);
}
}
XnStatus XnPacked12DepthProcessor::Unpack12to16(const XnUInt8* pcInput, const XnUInt32 nInputSize, XnUInt32* pnActualRead)
{
const XnUInt8* pOrigInput = pcInput;
XnUInt32 nElements = nInputSize / XN_INPUT_ELEMENT_SIZE; // floored
XnUInt32 nNeededOutput = nElements * XN_OUTPUT_ELEMENT_SIZE;
*pnActualRead = 0;
XnBuffer* pWriteBuffer = GetWriteBuffer();
if (!CheckDepthBufferForOverflow(nNeededOutput))
{
return XN_STATUS_OUTPUT_BUFFER_OVERFLOW;
}
XnUInt16* pnOutput = GetDepthOutputBuffer();
XnUInt16* pShiftOut = GetShiftsOutputBuffer();
XnUInt16 shift[16];
#ifdef XN_NEON
XnUInt16 depth[16];
uint8x8x3_t inD3;
uint8x8_t rshft4D, lshft4D;
uint16x8_t rshft4Q, lshft4Q;
uint16x8_t depthQ;
uint16x8x2_t shiftQ2;
#endif
// Convert the 11bit packed data into 16bit shorts
for (XnUInt32 nElem = 0; nElem < nElements; ++nElem)
{
#ifndef XN_NEON
// input: 0, 1,2,3, 4,5,6, 7,8,9, 10,11,12, 13,14,15, 16,17,18, 19,20,21, 22,23
// -,---,-,-,---,-,-,---,-,-,---,--,--,---,--,--,---,--,--,---,--,--,---,--
// bits: 8,4,4,8,8,4,4,8,8,4,4,8,8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8
// ---,---,---,---,---,---,---,----,----,----,----,----,----,----,----,----
// output: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
shift[0] = (XN_TAKE_BITS(pcInput[0],8,0) << 4) | XN_TAKE_BITS(pcInput[1],4,4);
shift[1] = (XN_TAKE_BITS(pcInput[1],4,0) << 8) | XN_TAKE_BITS(pcInput[2],8,0);
shift[2] = (XN_TAKE_BITS(pcInput[3],8,0) << 4) | XN_TAKE_BITS(pcInput[4],4,4);
shift[3] = (XN_TAKE_BITS(pcInput[4],4,0) << 8) | XN_TAKE_BITS(pcInput[5],8,0);
shift[4] = (XN_TAKE_BITS(pcInput[6],8,0) << 4) | XN_TAKE_BITS(pcInput[7],4,4);
shift[5] = (XN_TAKE_BITS(pcInput[7],4,0) << 8) | XN_TAKE_BITS(pcInput[8],8,0);
shift[6] = (XN_TAKE_BITS(pcInput[9],8,0) << 4) | XN_TAKE_BITS(pcInput[10],4,4);
shift[7] = (XN_TAKE_BITS(pcInput[10],4,0) << 8) | XN_TAKE_BITS(pcInput[11],8,0);
shift[8] = (XN_TAKE_BITS(pcInput[12],8,0) << 4) | XN_TAKE_BITS(pcInput[13],4,4);
shift[9] = (XN_TAKE_BITS(pcInput[13],4,0) << 8) | XN_TAKE_BITS(pcInput[14],8,0);
shift[10] = (XN_TAKE_BITS(pcInput[15],8,0) << 4) | XN_TAKE_BITS(pcInput[16],4,4);
shift[11] = (XN_TAKE_BITS(pcInput[16],4,0) << 8) | XN_TAKE_BITS(pcInput[17],8,0);
shift[12] = (XN_TAKE_BITS(pcInput[18],8,0) << 4) | XN_TAKE_BITS(pcInput[19],4,4);
shift[13] = (XN_TAKE_BITS(pcInput[19],4,0) << 8) | XN_TAKE_BITS(pcInput[20],8,0);
shift[14] = (XN_TAKE_BITS(pcInput[21],8,0) << 4) | XN_TAKE_BITS(pcInput[22],4,4);
shift[15] = (XN_TAKE_BITS(pcInput[22],4,0) << 8) | XN_TAKE_BITS(pcInput[23],8,0);
pShiftOut[0] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[0]) : 0);
pShiftOut[1] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[1]) : 0);
pShiftOut[2] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[2]) : 0);
pShiftOut[3] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[3]) : 0);
pShiftOut[4] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[4]) : 0);
pShiftOut[5] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[5]) : 0);
pShiftOut[6] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[6]) : 0);
pShiftOut[7] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[7]) : 0);
pShiftOut[8] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[8]) : 0);
pShiftOut[9] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[9]) : 0);
pShiftOut[10] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[10]) : 0);
pShiftOut[11] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[11]) : 0);
pShiftOut[12] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[12]) : 0);
pShiftOut[13] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[13]) : 0);
pShiftOut[14] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[14]) : 0);
pShiftOut[15] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[15]) : 0);
pnOutput[0] = GetOutput(shift[0]);
pnOutput[1] = GetOutput(shift[1]);
pnOutput[2] = GetOutput(shift[2]);
pnOutput[3] = GetOutput(shift[3]);
pnOutput[4] = GetOutput(shift[4]);
pnOutput[5] = GetOutput(shift[5]);
pnOutput[6] = GetOutput(shift[6]);
pnOutput[7] = GetOutput(shift[7]);
pnOutput[8] = GetOutput(shift[8]);
pnOutput[9] = GetOutput(shift[9]);
pnOutput[10] = GetOutput(shift[10]);
pnOutput[11] = GetOutput(shift[11]);
pnOutput[12] = GetOutput(shift[12]);
pnOutput[13] = GetOutput(shift[13]);
pnOutput[14] = GetOutput(shift[14]);
pnOutput[15] = GetOutput(shift[15]);
#else
// input: 0, 1,2 (X8)
// -,---,-
// bits: 8,4,4,8 (X8)
// ---,---
// output: 0, 1 (X8)
// Split 24 bytes into 3 vectors (64 bit each)
inD3 = vld3_u8(pcInput);
// rshft4D0 contains 4 MSB of second vector (placed at offset 0)
rshft4D = vshr_n_u8(inD3.val[1], 4);
// lshft4D0 contains 4 LSB of second vector (placed at offset 4)
lshft4D = vshl_n_u8(inD3.val[1], 4);
// Expand 64 bit vectors to 128 bit (8 values of 16 bits)
shiftQ2.val[0] = vmovl_u8(inD3.val[0]);
shiftQ2.val[1] = vmovl_u8(inD3.val[2]);
rshft4Q = vmovl_u8(rshft4D);
lshft4Q = vmovl_u8(lshft4D);
// Even indexed shift = 8 bits from first vector + 4 MSB bits of second vector
shiftQ2.val[0] = vshlq_n_u16(shiftQ2.val[0], 4);
shiftQ2.val[0] = vorrq_u16(shiftQ2.val[0], rshft4Q);
// Odd indexed shift = 4 LSB bits of second vector + 8 bits from third vector
lshft4Q = vshlq_n_u16(lshft4Q, 4);
shiftQ2.val[1] = vorrq_u16(shiftQ2.val[1], lshft4Q);
// Interleave shift values to a single vector
vst2q_u16(shift, shiftQ2);
shift[0] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[0]) : 0);
shift[1] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[1]) : 0);
shift[2] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[2]) : 0);
shift[3] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[3]) : 0);
shift[4] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[4]) : 0);
shift[5] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[5]) : 0);
shift[6] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[6]) : 0);
shift[7] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[7]) : 0);
shift[8] = (((shift[0]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[8]) : 0);
shift[9] = (((shift[1]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[9]) : 0);
shift[10] = (((shift[2]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[10]) : 0);
shift[11] = (((shift[3]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[11]) : 0);
shift[12] = (((shift[4]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[12]) : 0);
shift[13] = (((shift[5]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[13]) : 0);
shift[14] = (((shift[6]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[14]) : 0);
shift[15] = (((shift[7]) < (XN_DEVICE_SENSOR_MAX_SHIFT_VALUE-1)) ? (shift[15]) : 0);
depth[0] = GetOutput(shift[0]);
depth[1] = GetOutput(shift[1]);
depth[2] = GetOutput(shift[2]);
depth[3] = GetOutput(shift[3]);
depth[4] = GetOutput(shift[4]);
depth[5] = GetOutput(shift[5]);
depth[6] = GetOutput(shift[6]);
depth[7] = GetOutput(shift[7]);
// Load
depthQ = vld1q_u16(depth);
//Store
vst1q_u16(pnOutput, depthQ);
// Load
depthQ = vld1q_u16(shift);
// Store
vst1q_u16(pShiftOut, depthQ);
depth[8] = GetOutput(shift[8]);
depth[9] = GetOutput(shift[9]);
depth[10] = GetOutput(shift[10]);
depth[11] = GetOutput(shift[11]);
depth[12] = GetOutput(shift[12]);
depth[13] = GetOutput(shift[13]);
depth[14] = GetOutput(shift[14]);
depth[15] = GetOutput(shift[15]);
// Load
depthQ = vld1q_u16(depth + 8);
// Store
vst1q_u16(pnOutput + 8, depthQ);
// Load
depthQ = vld1q_u16(shift + 8);
// Store
vst1q_u16(pShiftOut + 8, depthQ);
#endif
pcInput += XN_INPUT_ELEMENT_SIZE;
pnOutput += 16;
pShiftOut += 16;
}
*pnActualRead = (XnUInt32)(pcInput - pOrigInput);
pWriteBuffer->UnsafeUpdateSize(nNeededOutput);
return XN_STATUS_OK;
}
/*
* Notice:
* - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
* - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
* numbers of DD bits
*/
static inline uint16_t
_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts, uint8_t *split_packet)
{
volatile union i40e_rx_desc *rxdp;
struct i40e_rx_entry *sw_ring;
uint16_t nb_pkts_recd;
int pos;
uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
/* mask to shuffle from desc. to mbuf */
uint8x16_t shuf_msk = {
0xFF, 0xFF, /* pkt_type set as unknown */
0xFF, 0xFF, /* pkt_type set as unknown */
14, 15, /* octet 15~14, low 16 bits pkt_len */
0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
14, 15, /* octet 15~14, 16 bits data_len */
2, 3, /* octet 2~3, low 16 bits vlan_macip */
4, 5, 6, 7 /* octet 4~7, 32bits rss */
};
uint8x16_t eop_check = {
0x02, 0x00, 0x02, 0x00,
0x02, 0x00, 0x02, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00
};
uint16x8_t crc_adjust = {
0, 0, /* ignore pkt_type field */
rxq->crc_len, /* sub crc on pkt_len */
0, /* ignore high-16bits of pkt_len */
rxq->crc_len, /* sub crc on data_len */
0, 0, 0 /* ignore non-length fields */
};
/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);
/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);
/* Just the act of getting into the function from the application is
* going to cost about 7 cycles
*/
rxdp = rxq->rx_ring + rxq->rx_tail;
rte_prefetch_non_temporal(rxdp);
/* See if we need to rearm the RX queue - gives the prefetch a bit
* of time to act
*/
if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
i40e_rxq_rearm(rxq);
/* Before we start moving massive data around, check to see if
* there is actually a packet available
*/
if (!(rxdp->wb.qword1.status_error_len &
rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
return 0;
/* Cache is empty -> need to scan the buffer rings, but first move
* the next 'n' mbufs into the cache
*/
sw_ring = &rxq->sw_ring[rxq->rx_tail];
/* A. load 4 packet in one loop
* [A*. mask out 4 unused dirty field in desc]
* B. copy 4 mbuf point from swring to rx_pkts
* C. calc the number of DD bits among the 4 packets
* [C*. extract the end-of-packet bit, if requested]
* D. fill info. from desc to mbuf
*/
for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
pos += RTE_I40E_DESCS_PER_LOOP,
rxdp += RTE_I40E_DESCS_PER_LOOP) {
uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP];
uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
uint16x8x2_t sterr_tmp1, sterr_tmp2;
uint64x2_t mbp1, mbp2;
uint16x8_t staterr;
uint16x8_t tmp;
uint64_t stat;
int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT};
/* B.1 load 1 mbuf point */
mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
/* Read desc statuses backwards to avoid race condition */
/* A.1 load 4 pkts desc */
descs[3] = vld1q_u64((uint64_t *)(rxdp + 3));
rte_rmb();
/* B.2 copy 2 mbuf point into rx_pkts */
vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
/* B.1 load 1 mbuf point */
mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
descs[2] = vld1q_u64((uint64_t *)(rxdp + 2));
/* B.1 load 2 mbuf point */
descs[1] = vld1q_u64((uint64_t *)(rxdp + 1));
descs[0] = vld1q_u64((uint64_t *)(rxdp));
/* B.2 copy 2 mbuf point into rx_pkts */
vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
if (split_packet) {
rte_mbuf_prefetch_part2(rx_pkts[pos]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
}
/* avoid compiler reorder optimization */
rte_compiler_barrier();
/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
len_shl);
descs[3] = vreinterpretq_u64_u32(len3);
uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]),
len_shl);
descs[2] = vreinterpretq_u64_u32(len2);
/* D.1 pkt 3,4 convert format from desc to pktmbuf */
pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
/* C.1 4=>2 filter staterr info only */
sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
vreinterpretq_u16_u64(descs[3]));
/* C.1 4=>2 filter staterr info only */
sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
vreinterpretq_u16_u64(descs[2]));
/* C.2 get 4 pkts staterr value */
staterr = vzipq_u16(sterr_tmp1.val[1],
sterr_tmp2.val[1]).val[0];
desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
pkt_mb4 = vreinterpretq_u8_u16(tmp);
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
pkt_mb3 = vreinterpretq_u8_u16(tmp);
/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]),
len_shl);
descs[1] = vreinterpretq_u64_u32(len1);
uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]),
len_shl);
descs[0] = vreinterpretq_u64_u32(len0);
/* D.1 pkt 1,2 convert format from desc to pktmbuf */
pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
/* D.3 copy final 3,4 data to rx_pkts */
vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
pkt_mb4);
vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
pkt_mb3);
/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
pkt_mb2 = vreinterpretq_u8_u16(tmp);
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
pkt_mb1 = vreinterpretq_u8_u16(tmp);
/* C* extract and record EOP bit */
if (split_packet) {
uint8x16_t eop_shuf_mask = {
0x00, 0x02, 0x04, 0x06,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF};
uint8x16_t eop_bits;
/* and with mask to extract bits, flipping 1-0 */
eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr));
eop_bits = vandq_u8(eop_bits, eop_check);
/* the staterr values are not in order, as the count
* count of dd bits doesn't care. However, for end of
* packet tracking, we do care, so shuffle. This also
* compresses the 32-bit values to 8-bit
*/
eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
/* store the resulting 32-bit value */
vst1q_lane_u32((uint32_t *)split_packet,
vreinterpretq_u32_u8(eop_bits), 0);
split_packet += RTE_I40E_DESCS_PER_LOOP;
/* zero-out next pointers */
rx_pkts[pos]->next = NULL;
rx_pkts[pos + 1]->next = NULL;
rx_pkts[pos + 2]->next = NULL;
rx_pkts[pos + 3]->next = NULL;
}
staterr = vshlq_n_u16(staterr, I40E_UINT16_BIT - 1);
staterr = vreinterpretq_u16_s16(
vshrq_n_s16(vreinterpretq_s16_u16(staterr),
I40E_UINT16_BIT - 1));
stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
/* D.3 copy final 1,2 data to rx_pkts */
vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
pkt_mb2);
vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1,
pkt_mb1);
desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
/* C.4 calc avaialbe number of desc */
if (unlikely(stat == 0)) {
nb_pkts_recd += RTE_I40E_DESCS_PER_LOOP;
} else {
nb_pkts_recd += __builtin_ctzl(stat) / I40E_UINT16_BIT;
break;
}
}
/* Update our internal tail pointer */
rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
return nb_pkts_recd;
}
XnStatus Link12BitS2DParser::Unpack12to16(const XnUInt8* pcInput,XnUInt8* pDest, const XnUInt32 nInputSize, XnUInt32* pnActualRead, XnUInt32* pnActualWritten)
{
const XnUInt8* pOrigInput = (XnUInt8*)pcInput;
XnUInt32 nElements = nInputSize / XN_INPUT_ELEMENT_SIZE; // floored
//XnUInt32 nNeededOutput = nElements * XN_OUTPUT_ELEMENT_SIZE;
*pnActualRead = 0;
XnUInt16 *pnOutput = (XnUInt16*)pDest;
XnUInt16 shift[16];
#ifdef XN_NEON
XnUInt16 depth[16];
uint8x8x3_t inD3;
uint8x8_t rshft4D, lshft4D;
uint16x8_t rshft4Q, lshft4Q;
uint16x8_t depthQ;
uint16x8x2_t shiftQ2;
#endif
// Convert the 11bit packed data into 16bit shorts
for (XnUInt32 nElem = 0; nElem < nElements; ++nElem)
{
#ifndef XN_NEON
// input: 0, 1,2,3, 4,5,6, 7,8,9, 10,11,12, 13,14,15, 16,17,18, 19,20,21, 22,23
// -,---,-,-,---,-,-,---,-,-,---,--,--,---,--,--,---,--,--,---,--,--,---,--
// bits: 8,4,4,8,8,4,4,8,8,4,4,8,8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8, 8,4,4, 8
// ---,---,---,---,---,---,---,----,----,----,----,----,----,----,----,----
// output: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
shift[0] = (XN_TAKE_BITS(pcInput[0],8,0) << 4) | XN_TAKE_BITS(pcInput[1],4,4);
shift[1] = (XN_TAKE_BITS(pcInput[1],4,0) << 8) | XN_TAKE_BITS(pcInput[2],8,0);
shift[2] = (XN_TAKE_BITS(pcInput[3],8,0) << 4) | XN_TAKE_BITS(pcInput[4],4,4);
shift[3] = (XN_TAKE_BITS(pcInput[4],4,0) << 8) | XN_TAKE_BITS(pcInput[5],8,0);
shift[4] = (XN_TAKE_BITS(pcInput[6],8,0) << 4) | XN_TAKE_BITS(pcInput[7],4,4);
shift[5] = (XN_TAKE_BITS(pcInput[7],4,0) << 8) | XN_TAKE_BITS(pcInput[8],8,0);
shift[6] = (XN_TAKE_BITS(pcInput[9],8,0) << 4) | XN_TAKE_BITS(pcInput[10],4,4);
shift[7] = (XN_TAKE_BITS(pcInput[10],4,0) << 8) | XN_TAKE_BITS(pcInput[11],8,0);
shift[8] = (XN_TAKE_BITS(pcInput[12],8,0) << 4) | XN_TAKE_BITS(pcInput[13],4,4);
shift[9] = (XN_TAKE_BITS(pcInput[13],4,0) << 8) | XN_TAKE_BITS(pcInput[14],8,0);
shift[10] = (XN_TAKE_BITS(pcInput[15],8,0) << 4) | XN_TAKE_BITS(pcInput[16],4,4);
shift[11] = (XN_TAKE_BITS(pcInput[16],4,0) << 8) | XN_TAKE_BITS(pcInput[17],8,0);
shift[12] = (XN_TAKE_BITS(pcInput[18],8,0) << 4) | XN_TAKE_BITS(pcInput[19],4,4);
shift[13] = (XN_TAKE_BITS(pcInput[19],4,0) << 8) | XN_TAKE_BITS(pcInput[20],8,0);
shift[14] = (XN_TAKE_BITS(pcInput[21],8,0) << 4) | XN_TAKE_BITS(pcInput[22],4,4);
shift[15] = (XN_TAKE_BITS(pcInput[22],4,0) << 8) | XN_TAKE_BITS(pcInput[23],8,0);
pnOutput[0] = m_pShiftToDepth[(shift[0])];
pnOutput[1] = m_pShiftToDepth[(shift[1])];
pnOutput[2] = m_pShiftToDepth[(shift[2])];
pnOutput[3] = m_pShiftToDepth[(shift[3])];
pnOutput[4] = m_pShiftToDepth[(shift[4])];
pnOutput[5] = m_pShiftToDepth[(shift[5])];
pnOutput[6] = m_pShiftToDepth[(shift[6])];
pnOutput[7] = m_pShiftToDepth[(shift[7])];
pnOutput[8] = m_pShiftToDepth[(shift[8])];
pnOutput[9] = m_pShiftToDepth[(shift[9])];
pnOutput[10] = m_pShiftToDepth[(shift[10])];
pnOutput[11] = m_pShiftToDepth[(shift[11])];
pnOutput[12] = m_pShiftToDepth[(shift[12])];
pnOutput[13] = m_pShiftToDepth[(shift[13])];
pnOutput[14] = m_pShiftToDepth[(shift[14])];
pnOutput[15] = m_pShiftToDepth[(shift[15])];
#else
// input: 0, 1,2 (X8)
// -,---,-
// bits: 8,4,4,8 (X8)
// ---,---
// output: 0, 1 (X8)
// Split 24 bytes into 3 vectors (64 bit each)
inD3 = vld3_u8(pcInput);
// rshft4D0 contains 4 MSB of second vector (placed at offset 0)
rshft4D = vshr_n_u8(inD3.val[1], 4);
// lshft4D0 contains 4 LSB of second vector (placed at offset 4)
lshft4D = vshl_n_u8(inD3.val[1], 4);
// Expand 64 bit vectors to 128 bit (8 values of 16 bits)
shiftQ2.val[0] = vmovl_u8(inD3.val[0]);
shiftQ2.val[1] = vmovl_u8(inD3.val[2]);
rshft4Q = vmovl_u8(rshft4D);
lshft4Q = vmovl_u8(lshft4D);
// Even indexed shift = 8 bits from first vector + 4 MSB bits of second vector
shiftQ2.val[0] = vshlq_n_u16(shiftQ2.val[0], 4);
shiftQ2.val[0] = vorrq_u16(shiftQ2.val[0], rshft4Q);
// Odd indexed shift = 4 LSB bits of second vector + 8 bits from third vector
lshft4Q = vshlq_n_u16(lshft4Q, 4);
shiftQ2.val[1] = vorrq_u16(shiftQ2.val[1], lshft4Q);
// Interleave shift values to a single vector
vst2q_u16(shift, shiftQ2);
depth[0] = m_pShiftToDepth[(shift[0])];
depth[1] = m_pShiftToDepth[(shift[1])];
depth[2] = m_pShiftToDepth[(shift[2])];
depth[3] = m_pShiftToDepth[(shift[3])];
depth[4] = m_pShiftToDepth[(shift[4])];
depth[5] = m_pShiftToDepth[(shift[5])];
depth[6] = m_pShiftToDepth[(shift[6])];
depth[7] = m_pShiftToDepth[(shift[7])];
// Load
depthQ = vld1q_u16(depth);
//Store
vst1q_u16(pnOutput, depthQ);
depth[8] = m_pShiftToDepth[(shift[8])];
depth[9] = m_pShiftToDepth[(shift[9])];
depth[10] = m_pShiftToDepth[(shift[10])];
depth[11] = m_pShiftToDepth[(shift[11])];
depth[12] = m_pShiftToDepth[(shift[12])];
depth[13] = m_pShiftToDepth[(shift[13])];
depth[14] = m_pShiftToDepth[(shift[14])];
depth[15] = m_pShiftToDepth[(shift[15])];
// Load
depthQ = vld1q_u16(depth + 8);
// Store
vst1q_u16(pnOutput + 8, depthQ);
#endif
pcInput += XN_INPUT_ELEMENT_SIZE;
pnOutput += 16;
}
*pnActualRead = (XnUInt32)(pcInput - pOrigInput); // total bytes
*pnActualWritten = (XnUInt32)((XnUInt8*)pnOutput - pDest);
return XN_STATUS_OK;
}
