static struct rte_mbuf *
rte_rxmbuf_alloc(struct rte_mempool *mp)
{
struct rte_mbuf *m;
m = __rte_mbuf_raw_alloc(mp);
__rte_mbuf_sanity_check_raw(m, 0);
return m;
}
static inline struct rte_mbuf *
tx_mbuf_alloc(struct rte_mempool *mp)
{
struct rte_mbuf *m;
m = __rte_mbuf_raw_alloc(mp);
__rte_mbuf_sanity_check_raw(m, 0);
return (m);
}
static inline struct rte_mbuf *
rte_rxmbuf_alloc(struct rte_mempool *mp)
{
struct rte_mbuf *m;
m = __rte_mbuf_raw_alloc(mp);
__rte_mbuf_sanity_check_raw(m, RTE_MBUF_PKT, 0);
return m;
}
static inline struct rte_mbuf *
bnx2x_rxmbuf_alloc(struct rte_mempool *mp)
{
struct rte_mbuf *m;
m = __rte_mbuf_raw_alloc(mp);
__rte_mbuf_sanity_check(m, 0);
return m;
}
/**
* DPDK callback for RX with scattered packets support.
*
* @param dpdk_rxq
* Generic pointer to RX queue structure.
* @param[out] pkts
* Array to store received packets.
* @param pkts_n
* Maximum number of packets in array.
*
* @return
* Number of packets successfully received (<= pkts_n).
*/
uint16_t
mlx5_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
{
struct rxq *rxq = (struct rxq *)dpdk_rxq;
struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp;
const unsigned int elts_n = rxq->elts_n;
unsigned int elts_head = rxq->elts_head;
unsigned int i;
unsigned int pkts_ret = 0;
int ret;
if (unlikely(!rxq->sp))
return mlx5_rx_burst(dpdk_rxq, pkts, pkts_n);
if (unlikely(elts == NULL)) /* See RTE_DEV_CMD_SET_MTU. */
return 0;
for (i = 0; (i != pkts_n); ++i) {
struct rxq_elt_sp *elt = &(*elts)[elts_head];
unsigned int len;
unsigned int pkt_buf_len;
struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */
struct rte_mbuf **pkt_buf_next = &pkt_buf;
unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM;
unsigned int j = 0;
uint32_t flags;
uint16_t vlan_tci;
/* Sanity checks. */
assert(elts_head < rxq->elts_n);
assert(rxq->elts_head < rxq->elts_n);
ret = rxq->poll(rxq->cq, NULL, NULL, &flags, &vlan_tci);
if (unlikely(ret < 0)) {
struct ibv_wc wc;
int wcs_n;
DEBUG("rxq=%p, poll_length() failed (ret=%d)",
(void *)rxq, ret);
/* ibv_poll_cq() must be used in case of failure. */
wcs_n = ibv_poll_cq(rxq->cq, 1, &wc);
if (unlikely(wcs_n == 0))
break;
if (unlikely(wcs_n < 0)) {
DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)",
(void *)rxq, wcs_n);
break;
}
assert(wcs_n == 1);
if (unlikely(wc.status != IBV_WC_SUCCESS)) {
/* Whatever, just repost the offending WR. */
DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work"
" completion status (%d): %s",
(void *)rxq, wc.wr_id, wc.status,
ibv_wc_status_str(wc.status));
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment dropped packets counter. */
++rxq->stats.idropped;
#endif
goto repost;
}
ret = wc.byte_len;
}
if (ret == 0)
break;
assert(ret >= (rxq->crc_present << 2));
len = ret - (rxq->crc_present << 2);
pkt_buf_len = len;
/*
* Replace spent segments with new ones, concatenate and
* return them as pkt_buf.
*/
while (1) {
struct ibv_sge *sge = &elt->sges[j];
struct rte_mbuf *seg = elt->bufs[j];
struct rte_mbuf *rep;
unsigned int seg_tailroom;
assert(seg != NULL);
/*
* Fetch initial bytes of packet descriptor into a
* cacheline while allocating rep.
*/
rte_prefetch0(seg);
rep = __rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(rep == NULL)) {
/*
* Unable to allocate a replacement mbuf,
* repost WR.
*/
DEBUG("rxq=%p: can't allocate a new mbuf",
(void *)rxq);
if (pkt_buf != NULL) {
*pkt_buf_next = NULL;
rte_pktmbuf_free(pkt_buf);
}
/* Increment out of memory counters. */
++rxq->stats.rx_nombuf;
++rxq->priv->dev->data->rx_mbuf_alloc_failed;
goto repost;
}
#ifndef NDEBUG
/* Poison user-modifiable fields in rep. */
NEXT(rep) = (void *)((uintptr_t)-1);
SET_DATA_OFF(rep, 0xdead);
DATA_LEN(rep) = 0xd00d;
PKT_LEN(rep) = 0xdeadd00d;
NB_SEGS(rep) = 0x2a;
PORT(rep) = 0x2a;
rep->ol_flags = -1;
#endif
assert(rep->buf_len == seg->buf_len);
assert(rep->buf_len == rxq->mb_len);
/* Reconfigure sge to use rep instead of seg. */
assert(sge->lkey == rxq->mr->lkey);
sge->addr = ((uintptr_t)rep->buf_addr + seg_headroom);
elt->bufs[j] = rep;
++j;
/* Update pkt_buf if it's the first segment, or link
* seg to the previous one and update pkt_buf_next. */
*pkt_buf_next = seg;
pkt_buf_next = &NEXT(seg);
/* Update seg information. */
seg_tailroom = (seg->buf_len - seg_headroom);
assert(sge->length == seg_tailroom);
SET_DATA_OFF(seg, seg_headroom);
if (likely(len <= seg_tailroom)) {
/* Last segment. */
DATA_LEN(seg) = len;
PKT_LEN(seg) = len;
/* Sanity check. */
assert(rte_pktmbuf_headroom(seg) ==
seg_headroom);
assert(rte_pktmbuf_tailroom(seg) ==
(seg_tailroom - len));
break;
}
DATA_LEN(seg) = seg_tailroom;
PKT_LEN(seg) = seg_tailroom;
/* Sanity check. */
assert(rte_pktmbuf_headroom(seg) == seg_headroom);
assert(rte_pktmbuf_tailroom(seg) == 0);
/* Fix len and clear headroom for next segments. */
len -= seg_tailroom;
seg_headroom = 0;
}
/* Update head and tail segments. */
*pkt_buf_next = NULL;
assert(pkt_buf != NULL);
assert(j != 0);
NB_SEGS(pkt_buf) = j;
PORT(pkt_buf) = rxq->port_id;
PKT_LEN(pkt_buf) = pkt_buf_len;
if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip) {
pkt_buf->packet_type = rxq_cq_to_pkt_type(flags);
pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags);
#ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
if (flags & IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1) {
pkt_buf->ol_flags |= PKT_RX_VLAN_PKT;
pkt_buf->vlan_tci = vlan_tci;
}
#endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
}
/* Return packet. */
*(pkts++) = pkt_buf;
++pkts_ret;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += pkt_buf_len;
#endif
repost:
ret = rxq->recv(rxq->wq, elt->sges, RTE_DIM(elt->sges));
if (unlikely(ret)) {
/* Inability to repost WRs is fatal. */
DEBUG("%p: recv_sg_list(): failed (ret=%d)",
(void *)rxq->priv,
ret);
abort();
}
if (++elts_head >= elts_n)
elts_head = 0;
continue;
}
if (unlikely(i == 0))
return 0;
rxq->elts_head = elts_head;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment packets counter. */
rxq->stats.ipackets += pkts_ret;
#endif
return pkts_ret;
}
/**
* DPDK callback for RX.
*
* The following function is the same as mlx5_rx_burst_sp(), except it doesn't
* manage scattered packets. Improves performance when MRU is lower than the
* size of the first segment.
*
* @param dpdk_rxq
* Generic pointer to RX queue structure.
* @param[out] pkts
* Array to store received packets.
* @param pkts_n
* Maximum number of packets in array.
*
* @return
* Number of packets successfully received (<= pkts_n).
*/
uint16_t
mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
{
struct rxq *rxq = (struct rxq *)dpdk_rxq;
struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp;
const unsigned int elts_n = rxq->elts_n;
unsigned int elts_head = rxq->elts_head;
struct ibv_sge sges[pkts_n];
unsigned int i;
unsigned int pkts_ret = 0;
int ret;
if (unlikely(rxq->sp))
return mlx5_rx_burst_sp(dpdk_rxq, pkts, pkts_n);
for (i = 0; (i != pkts_n); ++i) {
struct rxq_elt *elt = &(*elts)[elts_head];
unsigned int len;
struct rte_mbuf *seg = elt->buf;
struct rte_mbuf *rep;
uint32_t flags;
uint16_t vlan_tci;
/* Sanity checks. */
assert(seg != NULL);
assert(elts_head < rxq->elts_n);
assert(rxq->elts_head < rxq->elts_n);
/*
* Fetch initial bytes of packet descriptor into a
* cacheline while allocating rep.
*/
rte_prefetch0(seg);
rte_prefetch0(&seg->cacheline1);
ret = rxq->poll(rxq->cq, NULL, NULL, &flags, &vlan_tci);
if (unlikely(ret < 0)) {
struct ibv_wc wc;
int wcs_n;
DEBUG("rxq=%p, poll_length() failed (ret=%d)",
(void *)rxq, ret);
/* ibv_poll_cq() must be used in case of failure. */
wcs_n = ibv_poll_cq(rxq->cq, 1, &wc);
if (unlikely(wcs_n == 0))
break;
if (unlikely(wcs_n < 0)) {
DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)",
(void *)rxq, wcs_n);
break;
}
assert(wcs_n == 1);
if (unlikely(wc.status != IBV_WC_SUCCESS)) {
/* Whatever, just repost the offending WR. */
DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work"
" completion status (%d): %s",
(void *)rxq, wc.wr_id, wc.status,
ibv_wc_status_str(wc.status));
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment dropped packets counter. */
++rxq->stats.idropped;
#endif
/* Add SGE to array for repost. */
sges[i] = elt->sge;
goto repost;
}
ret = wc.byte_len;
}
if (ret == 0)
break;
assert(ret >= (rxq->crc_present << 2));
len = ret - (rxq->crc_present << 2);
rep = __rte_mbuf_raw_alloc(rxq->mp);
if (unlikely(rep == NULL)) {
/*
* Unable to allocate a replacement mbuf,
* repost WR.
*/
DEBUG("rxq=%p: can't allocate a new mbuf",
(void *)rxq);
/* Increment out of memory counters. */
++rxq->stats.rx_nombuf;
++rxq->priv->dev->data->rx_mbuf_alloc_failed;
goto repost;
}
/* Reconfigure sge to use rep instead of seg. */
elt->sge.addr = (uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM;
assert(elt->sge.lkey == rxq->mr->lkey);
elt->buf = rep;
/* Add SGE to array for repost. */
sges[i] = elt->sge;
/* Update seg information. */
SET_DATA_OFF(seg, RTE_PKTMBUF_HEADROOM);
NB_SEGS(seg) = 1;
PORT(seg) = rxq->port_id;
NEXT(seg) = NULL;
PKT_LEN(seg) = len;
DATA_LEN(seg) = len;
if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip) {
seg->packet_type = rxq_cq_to_pkt_type(flags);
seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags);
#ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS
if (flags & IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1) {
seg->ol_flags |= PKT_RX_VLAN_PKT;
seg->vlan_tci = vlan_tci;
}
#endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */
}
/* Return packet. */
*(pkts++) = seg;
++pkts_ret;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment bytes counter. */
rxq->stats.ibytes += len;
#endif
repost:
if (++elts_head >= elts_n)
elts_head = 0;
continue;
}
if (unlikely(i == 0))
return 0;
/* Repost WRs. */
#ifdef DEBUG_RECV
DEBUG("%p: reposting %u WRs", (void *)rxq, i);
#endif
ret = rxq->recv(rxq->wq, sges, i);
if (unlikely(ret)) {
/* Inability to repost WRs is fatal. */
DEBUG("%p: recv_burst(): failed (ret=%d)",
(void *)rxq->priv,
ret);
abort();
}
rxq->elts_head = elts_head;
#ifdef MLX5_PMD_SOFT_COUNTERS
/* Increment packets counter. */
rxq->stats.ipackets += pkts_ret;
#endif
return pkts_ret;
}
