/*
* Notice:
* - nb_pkts < RTE_I40E_DESCS_PER_LOOP, just return no packet
* - nb_pkts > RTE_I40E_VPMD_RX_BURST, only scan RTE_I40E_VPMD_RX_BURST
* numbers of DD bits
*/
static inline uint16_t
_recv_raw_pkts_vec(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts, uint8_t *split_packet)
{
volatile union i40e_rx_desc *rxdp;
struct i40e_rx_entry *sw_ring;
uint16_t nb_pkts_recd;
int pos;
uint32_t *ptype_tbl = rxq->vsi->adapter->ptype_tbl;
/* mask to shuffle from desc. to mbuf */
uint8x16_t shuf_msk = {
0xFF, 0xFF, /* pkt_type set as unknown */
0xFF, 0xFF, /* pkt_type set as unknown */
14, 15, /* octet 15~14, low 16 bits pkt_len */
0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
14, 15, /* octet 15~14, 16 bits data_len */
2, 3, /* octet 2~3, low 16 bits vlan_macip */
4, 5, 6, 7 /* octet 4~7, 32bits rss */
};
uint8x16_t eop_check = {
0x02, 0x00, 0x02, 0x00,
0x02, 0x00, 0x02, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00
};
uint16x8_t crc_adjust = {
0, 0, /* ignore pkt_type field */
rxq->crc_len, /* sub crc on pkt_len */
0, /* ignore high-16bits of pkt_len */
rxq->crc_len, /* sub crc on data_len */
0, 0, 0 /* ignore non-length fields */
};
/* nb_pkts shall be less equal than RTE_I40E_MAX_RX_BURST */
nb_pkts = RTE_MIN(nb_pkts, RTE_I40E_MAX_RX_BURST);
/* nb_pkts has to be floor-aligned to RTE_I40E_DESCS_PER_LOOP */
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_I40E_DESCS_PER_LOOP);
/* Just the act of getting into the function from the application is
* going to cost about 7 cycles
*/
rxdp = rxq->rx_ring + rxq->rx_tail;
rte_prefetch_non_temporal(rxdp);
/* See if we need to rearm the RX queue - gives the prefetch a bit
* of time to act
*/
if (rxq->rxrearm_nb > RTE_I40E_RXQ_REARM_THRESH)
i40e_rxq_rearm(rxq);
/* Before we start moving massive data around, check to see if
* there is actually a packet available
*/
if (!(rxdp->wb.qword1.status_error_len &
rte_cpu_to_le_32(1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
return 0;
/* Cache is empty -> need to scan the buffer rings, but first move
* the next 'n' mbufs into the cache
*/
sw_ring = &rxq->sw_ring[rxq->rx_tail];
/* A. load 4 packet in one loop
* [A*. mask out 4 unused dirty field in desc]
* B. copy 4 mbuf point from swring to rx_pkts
* C. calc the number of DD bits among the 4 packets
* [C*. extract the end-of-packet bit, if requested]
* D. fill info. from desc to mbuf
*/
for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
pos += RTE_I40E_DESCS_PER_LOOP,
rxdp += RTE_I40E_DESCS_PER_LOOP) {
uint64x2_t descs[RTE_I40E_DESCS_PER_LOOP];
uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
uint16x8x2_t sterr_tmp1, sterr_tmp2;
uint64x2_t mbp1, mbp2;
uint16x8_t staterr;
uint16x8_t tmp;
uint64_t stat;
int32x4_t len_shl = {0, 0, 0, PKTLEN_SHIFT};
/* B.1 load 1 mbuf point */
mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
/* Read desc statuses backwards to avoid race condition */
/* A.1 load 4 pkts desc */
descs[3] = vld1q_u64((uint64_t *)(rxdp + 3));
rte_rmb();
/* B.2 copy 2 mbuf point into rx_pkts */
vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
/* B.1 load 1 mbuf point */
mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
descs[2] = vld1q_u64((uint64_t *)(rxdp + 2));
/* B.1 load 2 mbuf point */
descs[1] = vld1q_u64((uint64_t *)(rxdp + 1));
descs[0] = vld1q_u64((uint64_t *)(rxdp));
/* B.2 copy 2 mbuf point into rx_pkts */
vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
if (split_packet) {
rte_mbuf_prefetch_part2(rx_pkts[pos]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
}
/* avoid compiler reorder optimization */
rte_compiler_barrier();
/* pkt 3,4 shift the pktlen field to be 16-bit aligned*/
uint32x4_t len3 = vshlq_u32(vreinterpretq_u32_u64(descs[3]),
len_shl);
descs[3] = vreinterpretq_u64_u32(len3);
uint32x4_t len2 = vshlq_u32(vreinterpretq_u32_u64(descs[2]),
len_shl);
descs[2] = vreinterpretq_u64_u32(len2);
/* D.1 pkt 3,4 convert format from desc to pktmbuf */
pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
/* C.1 4=>2 filter staterr info only */
sterr_tmp2 = vzipq_u16(vreinterpretq_u16_u64(descs[1]),
vreinterpretq_u16_u64(descs[3]));
/* C.1 4=>2 filter staterr info only */
sterr_tmp1 = vzipq_u16(vreinterpretq_u16_u64(descs[0]),
vreinterpretq_u16_u64(descs[2]));
/* C.2 get 4 pkts staterr value */
staterr = vzipq_u16(sterr_tmp1.val[1],
sterr_tmp2.val[1]).val[0];
desc_to_olflags_v(rxq, descs, &rx_pkts[pos]);
/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
pkt_mb4 = vreinterpretq_u8_u16(tmp);
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
pkt_mb3 = vreinterpretq_u8_u16(tmp);
/* pkt 1,2 shift the pktlen field to be 16-bit aligned*/
uint32x4_t len1 = vshlq_u32(vreinterpretq_u32_u64(descs[1]),
len_shl);
descs[1] = vreinterpretq_u64_u32(len1);
uint32x4_t len0 = vshlq_u32(vreinterpretq_u32_u64(descs[0]),
len_shl);
descs[0] = vreinterpretq_u64_u32(len0);
/* D.1 pkt 1,2 convert format from desc to pktmbuf */
pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
/* D.3 copy final 3,4 data to rx_pkts */
vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
pkt_mb4);
vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
pkt_mb3);
/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
pkt_mb2 = vreinterpretq_u8_u16(tmp);
tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
pkt_mb1 = vreinterpretq_u8_u16(tmp);
/* C* extract and record EOP bit */
if (split_packet) {
uint8x16_t eop_shuf_mask = {
0x00, 0x02, 0x04, 0x06,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF};
uint8x16_t eop_bits;
/* and with mask to extract bits, flipping 1-0 */
eop_bits = vmvnq_u8(vreinterpretq_u8_u16(staterr));
eop_bits = vandq_u8(eop_bits, eop_check);
/* the staterr values are not in order, as the count
* count of dd bits doesn't care. However, for end of
* packet tracking, we do care, so shuffle. This also
* compresses the 32-bit values to 8-bit
*/
eop_bits = vqtbl1q_u8(eop_bits, eop_shuf_mask);
/* store the resulting 32-bit value */
vst1q_lane_u32((uint32_t *)split_packet,
vreinterpretq_u32_u8(eop_bits), 0);
split_packet += RTE_I40E_DESCS_PER_LOOP;
/* zero-out next pointers */
rx_pkts[pos]->next = NULL;
rx_pkts[pos + 1]->next = NULL;
rx_pkts[pos + 2]->next = NULL;
rx_pkts[pos + 3]->next = NULL;
}
staterr = vshlq_n_u16(staterr, I40E_UINT16_BIT - 1);
staterr = vreinterpretq_u16_s16(
vshrq_n_s16(vreinterpretq_s16_u16(staterr),
I40E_UINT16_BIT - 1));
stat = ~vgetq_lane_u64(vreinterpretq_u64_u16(staterr), 0);
rte_prefetch_non_temporal(rxdp + RTE_I40E_DESCS_PER_LOOP);
/* D.3 copy final 1,2 data to rx_pkts */
vst1q_u8((void *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
pkt_mb2);
vst1q_u8((void *)&rx_pkts[pos]->rx_descriptor_fields1,
pkt_mb1);
desc_to_ptype_v(descs, &rx_pkts[pos], ptype_tbl);
/* C.4 calc avaialbe number of desc */
if (unlikely(stat == 0)) {
nb_pkts_recd += RTE_I40E_DESCS_PER_LOOP;
} else {
nb_pkts_recd += __builtin_ctzl(stat) / I40E_UINT16_BIT;
break;
}
}
/* Update our internal tail pointer */
rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
return nb_pkts_recd;
}
static inline void
desc_to_olflags_v(struct i40e_rx_queue *rxq, uint64x2_t descs[4],
struct rte_mbuf **rx_pkts)
{
uint32x4_t vlan0, vlan1, rss, l3_l4e;
const uint64x2_t mbuf_init = {rxq->mbuf_initializer, 0};
uint64x2_t rearm0, rearm1, rearm2, rearm3;
/* mask everything except RSS, flow director and VLAN flags
* bit2 is for VLAN tag, bit11 for flow director indication
* bit13:12 for RSS indication.
*/
const uint32x4_t rss_vlan_msk = {
0x1c03804, 0x1c03804, 0x1c03804, 0x1c03804};
const uint32x4_t cksum_mask = {
PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
PKT_RX_EIP_CKSUM_BAD,
PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
PKT_RX_EIP_CKSUM_BAD,
PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
PKT_RX_EIP_CKSUM_BAD,
PKT_RX_IP_CKSUM_GOOD | PKT_RX_IP_CKSUM_BAD |
PKT_RX_L4_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD |
PKT_RX_EIP_CKSUM_BAD};
/* map rss and vlan type to rss hash and vlan flag */
const uint8x16_t vlan_flags = {
0, 0, 0, 0,
PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0};
const uint8x16_t rss_flags = {
0, PKT_RX_FDIR, 0, 0,
0, 0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH | PKT_RX_FDIR,
0, 0, 0, 0,
0, 0, 0, 0};
const uint8x16_t l3_l4e_flags = {
(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD) >> 1,
PKT_RX_IP_CKSUM_BAD >> 1,
(PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_BAD) >> 1,
(PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
(PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD) >> 1,
(PKT_RX_EIP_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD) >> 1,
(PKT_RX_IP_CKSUM_GOOD | PKT_RX_EIP_CKSUM_BAD |
PKT_RX_L4_CKSUM_BAD) >> 1,
(PKT_RX_EIP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD |
PKT_RX_IP_CKSUM_BAD) >> 1,
0, 0, 0, 0, 0, 0, 0, 0};
vlan0 = vzipq_u32(vreinterpretq_u32_u64(descs[0]),
vreinterpretq_u32_u64(descs[2])).val[1];
vlan1 = vzipq_u32(vreinterpretq_u32_u64(descs[1]),
vreinterpretq_u32_u64(descs[3])).val[1];
vlan0 = vzipq_u32(vlan0, vlan1).val[0];
vlan1 = vandq_u32(vlan0, rss_vlan_msk);
vlan0 = vreinterpretq_u32_u8(vqtbl1q_u8(vlan_flags,
vreinterpretq_u8_u32(vlan1)));
rss = vshrq_n_u32(vlan1, 11);
rss = vreinterpretq_u32_u8(vqtbl1q_u8(rss_flags,
vreinterpretq_u8_u32(rss)));
l3_l4e = vshrq_n_u32(vlan1, 22);
l3_l4e = vreinterpretq_u32_u8(vqtbl1q_u8(l3_l4e_flags,
vreinterpretq_u8_u32(l3_l4e)));
/* then we shift left 1 bit */
l3_l4e = vshlq_n_u32(l3_l4e, 1);
/* we need to mask out the reduntant bits */
l3_l4e = vandq_u32(l3_l4e, cksum_mask);
vlan0 = vorrq_u32(vlan0, rss);
vlan0 = vorrq_u32(vlan0, l3_l4e);
rearm0 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 0), mbuf_init, 1);
rearm1 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 1), mbuf_init, 1);
rearm2 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 2), mbuf_init, 1);
rearm3 = vsetq_lane_u64(vgetq_lane_u32(vlan0, 3), mbuf_init, 1);
vst1q_u64((uint64_t *)&rx_pkts[0]->rearm_data, rearm0);
vst1q_u64((uint64_t *)&rx_pkts[1]->rearm_data, rearm1);
vst1q_u64((uint64_t *)&rx_pkts[2]->rearm_data, rearm2);
vst1q_u64((uint64_t *)&rx_pkts[3]->rearm_data, rearm3);
}
static
void
neon_w32_split_4_32_multiply_region(gf_t *gf, uint32_t *src, uint32_t *dst,
uint32_t *d_end, uint8_t btable[8][4][16],
uint32_t val, int xor, int altmap)
{
int i, j;
#ifdef ARCH_AARCH64
uint8x16_t tables[8][4];
#else
uint8x8x2_t tables[8][4];
#endif
uint32x4_t v0, v1, v2, v3, s0, s1, s2, s3;
uint8x16_t p0, p1, p2, p3, si, mask1;
uint16x8x2_t r0, r1;
uint8x16x2_t q0, q1;
for (i = 0; i < 8; i++) {
for (j = 0; j < 4; j++) {
#ifdef ARCH_AARCH64
tables[i][j] = vld1q_u8(btable[i][j]);
#else
tables[i][j].val[0] = vld1_u8(btable[i][j]);
tables[i][j].val[1] = vld1_u8(btable[i][j] + 8);
#endif
}
}
mask1 = vdupq_n_u8(0xf);
while (dst < d_end) {
v0 = vld1q_u32(src); src += 4;
v1 = vld1q_u32(src); src += 4;
v2 = vld1q_u32(src); src += 4;
v3 = vld1q_u32(src); src += 4;
if (altmap) {
q0.val[0] = vreinterpretq_u8_u32(v0);
q0.val[1] = vreinterpretq_u8_u32(v1);
q1.val[0] = vreinterpretq_u8_u32(v2);
q1.val[1] = vreinterpretq_u8_u32(v3);
} else {
r0 = vtrnq_u16(vreinterpretq_u16_u32(v0), vreinterpretq_u16_u32(v2));
r1 = vtrnq_u16(vreinterpretq_u16_u32(v1), vreinterpretq_u16_u32(v3));
q0 = vtrnq_u8(vreinterpretq_u8_u16(r0.val[0]),
vreinterpretq_u8_u16(r1.val[0]));
q1 = vtrnq_u8(vreinterpretq_u8_u16(r0.val[1]),
vreinterpretq_u8_u16(r1.val[1]));
}
si = vandq_u8(q0.val[0], mask1);
p0 = vqtbl1q_u8(tables[0][0], si);
p1 = vqtbl1q_u8(tables[0][1], si);
p2 = vqtbl1q_u8(tables[0][2], si);
p3 = vqtbl1q_u8(tables[0][3], si);
si = vshrq_n_u8(q0.val[0], 4);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[1][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[1][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[1][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[1][3], si));
si = vandq_u8(q0.val[1], mask1);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[2][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[2][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[2][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[2][3], si));
si = vshrq_n_u8(q0.val[1], 4);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[3][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[3][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[3][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[3][3], si));
si = vandq_u8(q1.val[0], mask1);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[4][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[4][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[4][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[4][3], si));
si = vshrq_n_u8(q1.val[0], 4);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[5][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[5][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[5][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[5][3], si));
si = vandq_u8(q1.val[1], mask1);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[6][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[6][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[6][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[6][3], si));
si = vshrq_n_u8(q1.val[1], 4);
p0 = veorq_u8(p0, vqtbl1q_u8(tables[7][0], si));
p1 = veorq_u8(p1, vqtbl1q_u8(tables[7][1], si));
p2 = veorq_u8(p2, vqtbl1q_u8(tables[7][2], si));
p3 = veorq_u8(p3, vqtbl1q_u8(tables[7][3], si));
if (altmap) {
s0 = vreinterpretq_u32_u8(p0);
s1 = vreinterpretq_u32_u8(p1);
s2 = vreinterpretq_u32_u8(p2);
s3 = vreinterpretq_u32_u8(p3);
} else {
q0 = vtrnq_u8(p0, p1);
q1 = vtrnq_u8(p2, p3);
r0 = vtrnq_u16(vreinterpretq_u16_u8(q0.val[0]),
vreinterpretq_u16_u8(q1.val[0]));
r1 = vtrnq_u16(vreinterpretq_u16_u8(q0.val[1]),
vreinterpretq_u16_u8(q1.val[1]));
s0 = vreinterpretq_u32_u16(r0.val[0]);
s1 = vreinterpretq_u32_u16(r1.val[0]);
s2 = vreinterpretq_u32_u16(r0.val[1]);
s3 = vreinterpretq_u32_u16(r1.val[1]);
}
if (xor) {
v0 = vld1q_u32(dst);
v1 = vld1q_u32(dst + 4);
v2 = vld1q_u32(dst + 8);
v3 = vld1q_u32(dst + 12);
s0 = veorq_u32(s0, v0);
s1 = veorq_u32(s1, v1);
s2 = veorq_u32(s2, v2);
s3 = veorq_u32(s3, v3);
}
vst1q_u32(dst, s0);
vst1q_u32(dst + 4, s1);
vst1q_u32(dst + 8, s2);
vst1q_u32(dst + 12, s3);
dst += 16;
}
}
uint8x16_t
qtblq_testu8_ (uint8x16_t tab, uint8x16_t idx)
{
return vqtbl1q_u8 (tab, idx);
}
/* virtio vPMD receive routine, only accept(nb_pkts >= RTE_VIRTIO_DESC_PER_LOOP)
*
* This routine is for non-mergeable RX, one desc for each guest buffer.
* This routine is based on the RX ring layout optimization. Each entry in the
* avail ring points to the desc with the same index in the desc ring and this
* will never be changed in the driver.
*
* - nb_pkts < RTE_VIRTIO_DESC_PER_LOOP, just return no packet
*/
uint16_t
virtio_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct virtnet_rx *rxvq = rx_queue;
struct virtqueue *vq = rxvq->vq;
uint16_t nb_used;
uint16_t desc_idx;
struct vring_used_elem *rused;
struct rte_mbuf **sw_ring;
struct rte_mbuf **sw_ring_end;
uint16_t nb_pkts_received;
uint8x16_t shuf_msk1 = {
0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
4, 5, 0xFF, 0xFF, /* pkt len */
4, 5, /* dat len */
0xFF, 0xFF, /* vlan tci */
0xFF, 0xFF, 0xFF, 0xFF
};
uint8x16_t shuf_msk2 = {
0xFF, 0xFF, 0xFF, 0xFF, /* packet type */
12, 13, 0xFF, 0xFF, /* pkt len */
12, 13, /* dat len */
0xFF, 0xFF, /* vlan tci */
0xFF, 0xFF, 0xFF, 0xFF
};
/* Subtract the header length.
* In which case do we need the header length in used->len ?
*/
uint16x8_t len_adjust = {
0, 0,
(uint16_t)vq->hw->vtnet_hdr_size, 0,
(uint16_t)vq->hw->vtnet_hdr_size,
0,
0, 0
};
if (unlikely(nb_pkts < RTE_VIRTIO_DESC_PER_LOOP))
return 0;
nb_used = VIRTQUEUE_NUSED(vq);
rte_rmb();
if (unlikely(nb_used == 0))
return 0;
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_VIRTIO_DESC_PER_LOOP);
nb_used = RTE_MIN(nb_used, nb_pkts);
desc_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
rused = &vq->vq_ring.used->ring[desc_idx];
sw_ring = &vq->sw_ring[desc_idx];
sw_ring_end = &vq->sw_ring[vq->vq_nentries];
rte_prefetch_non_temporal(rused);
if (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
virtio_rxq_rearm_vec(rxvq);
if (unlikely(virtqueue_kick_prepare(vq)))
virtqueue_notify(vq);
}
for (nb_pkts_received = 0;
nb_pkts_received < nb_used;) {
uint64x2_t desc[RTE_VIRTIO_DESC_PER_LOOP / 2];
uint64x2_t mbp[RTE_VIRTIO_DESC_PER_LOOP / 2];
uint64x2_t pkt_mb[RTE_VIRTIO_DESC_PER_LOOP];
mbp[0] = vld1q_u64((uint64_t *)(sw_ring + 0));
desc[0] = vld1q_u64((uint64_t *)(rused + 0));
vst1q_u64((uint64_t *)&rx_pkts[0], mbp[0]);
mbp[1] = vld1q_u64((uint64_t *)(sw_ring + 2));
desc[1] = vld1q_u64((uint64_t *)(rused + 2));
vst1q_u64((uint64_t *)&rx_pkts[2], mbp[1]);
mbp[2] = vld1q_u64((uint64_t *)(sw_ring + 4));
desc[2] = vld1q_u64((uint64_t *)(rused + 4));
vst1q_u64((uint64_t *)&rx_pkts[4], mbp[2]);
mbp[3] = vld1q_u64((uint64_t *)(sw_ring + 6));
desc[3] = vld1q_u64((uint64_t *)(rused + 6));
vst1q_u64((uint64_t *)&rx_pkts[6], mbp[3]);
pkt_mb[1] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[0]), shuf_msk2));
pkt_mb[0] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[0]), shuf_msk1));
pkt_mb[1] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[1]), len_adjust));
pkt_mb[0] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[0]), len_adjust));
vst1q_u64((void *)&rx_pkts[1]->rx_descriptor_fields1,
pkt_mb[1]);
vst1q_u64((void *)&rx_pkts[0]->rx_descriptor_fields1,
pkt_mb[0]);
pkt_mb[3] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[1]), shuf_msk2));
pkt_mb[2] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[1]), shuf_msk1));
pkt_mb[3] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[3]), len_adjust));
pkt_mb[2] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[2]), len_adjust));
vst1q_u64((void *)&rx_pkts[3]->rx_descriptor_fields1,
pkt_mb[3]);
vst1q_u64((void *)&rx_pkts[2]->rx_descriptor_fields1,
pkt_mb[2]);
pkt_mb[5] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[2]), shuf_msk2));
pkt_mb[4] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[2]), shuf_msk1));
pkt_mb[5] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[5]), len_adjust));
pkt_mb[4] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[4]), len_adjust));
vst1q_u64((void *)&rx_pkts[5]->rx_descriptor_fields1,
pkt_mb[5]);
vst1q_u64((void *)&rx_pkts[4]->rx_descriptor_fields1,
pkt_mb[4]);
pkt_mb[7] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[3]), shuf_msk2));
pkt_mb[6] = vreinterpretq_u64_u8(vqtbl1q_u8(
vreinterpretq_u8_u64(desc[3]), shuf_msk1));
pkt_mb[7] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[7]), len_adjust));
pkt_mb[6] = vreinterpretq_u64_u16(vsubq_u16(
vreinterpretq_u16_u64(pkt_mb[6]), len_adjust));
vst1q_u64((void *)&rx_pkts[7]->rx_descriptor_fields1,
pkt_mb[7]);
vst1q_u64((void *)&rx_pkts[6]->rx_descriptor_fields1,
pkt_mb[6]);
if (unlikely(nb_used <= RTE_VIRTIO_DESC_PER_LOOP)) {
if (sw_ring + nb_used <= sw_ring_end)
nb_pkts_received += nb_used;
else
nb_pkts_received += sw_ring_end - sw_ring;
break;
} else {
if (unlikely(sw_ring + RTE_VIRTIO_DESC_PER_LOOP >=
sw_ring_end)) {
nb_pkts_received += sw_ring_end - sw_ring;
break;
} else {
nb_pkts_received += RTE_VIRTIO_DESC_PER_LOOP;
rx_pkts += RTE_VIRTIO_DESC_PER_LOOP;
sw_ring += RTE_VIRTIO_DESC_PER_LOOP;
rused += RTE_VIRTIO_DESC_PER_LOOP;
nb_used -= RTE_VIRTIO_DESC_PER_LOOP;
}
}
}
vq->vq_used_cons_idx += nb_pkts_received;
vq->vq_free_cnt += nb_pkts_received;
rxvq->stats.packets += nb_pkts_received;
return nb_pkts_received;
}
uint8x16_t test_vqtbl1q_u8(uint8x16_t a, uint8x16_t b) {
// CHECK: test_vqtbl1q_u8
return vqtbl1q_u8(a, b);
// CHECK: tbl {{v[0-9]+}}.16b, {{{v[0-9]+}}.16b}, {{v[0-9]+}}.16b
}
