static struct rte_mbuf * rte_rxmbuf_alloc(struct rte_mempool *mp) { struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); __rte_mbuf_sanity_check_raw(m, 0); return m; }
static inline struct rte_mbuf * tx_mbuf_alloc(struct rte_mempool *mp) { struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); __rte_mbuf_sanity_check_raw(m, 0); return (m); }
static inline struct rte_mbuf * rte_rxmbuf_alloc(struct rte_mempool *mp) { struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); __rte_mbuf_sanity_check_raw(m, RTE_MBUF_PKT, 0); return m; }
static inline struct rte_mbuf * bnx2x_rxmbuf_alloc(struct rte_mempool *mp) { struct rte_mbuf *m; m = __rte_mbuf_raw_alloc(mp); __rte_mbuf_sanity_check(m, 0); return m; }
/** * DPDK callback for RX with scattered packets support. * * @param dpdk_rxq * Generic pointer to RX queue structure. * @param[out] pkts * Array to store received packets. * @param pkts_n * Maximum number of packets in array. * * @return * Number of packets successfully received (<= pkts_n). */ uint16_t mlx5_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct rxq *rxq = (struct rxq *)dpdk_rxq; struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp; const unsigned int elts_n = rxq->elts_n; unsigned int elts_head = rxq->elts_head; unsigned int i; unsigned int pkts_ret = 0; int ret; if (unlikely(!rxq->sp)) return mlx5_rx_burst(dpdk_rxq, pkts, pkts_n); if (unlikely(elts == NULL)) /* See RTE_DEV_CMD_SET_MTU. */ return 0; for (i = 0; (i != pkts_n); ++i) { struct rxq_elt_sp *elt = &(*elts)[elts_head]; unsigned int len; unsigned int pkt_buf_len; struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */ struct rte_mbuf **pkt_buf_next = &pkt_buf; unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM; unsigned int j = 0; uint32_t flags; uint16_t vlan_tci; /* Sanity checks. */ assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); ret = rxq->poll(rxq->cq, NULL, NULL, &flags, &vlan_tci); if (unlikely(ret < 0)) { struct ibv_wc wc; int wcs_n; DEBUG("rxq=%p, poll_length() failed (ret=%d)", (void *)rxq, ret); /* ibv_poll_cq() must be used in case of failure. */ wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); if (unlikely(wcs_n == 0)) break; if (unlikely(wcs_n < 0)) { DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", (void *)rxq, wcs_n); break; } assert(wcs_n == 1); if (unlikely(wc.status != IBV_WC_SUCCESS)) { /* Whatever, just repost the offending WR. */ DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" " completion status (%d): %s", (void *)rxq, wc.wr_id, wc.status, ibv_wc_status_str(wc.status)); #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment dropped packets counter. */ ++rxq->stats.idropped; #endif goto repost; } ret = wc.byte_len; } if (ret == 0) break; assert(ret >= (rxq->crc_present << 2)); len = ret - (rxq->crc_present << 2); pkt_buf_len = len; /* * Replace spent segments with new ones, concatenate and * return them as pkt_buf. */ while (1) { struct ibv_sge *sge = &elt->sges[j]; struct rte_mbuf *seg = elt->bufs[j]; struct rte_mbuf *rep; unsigned int seg_tailroom; assert(seg != NULL); /* * Fetch initial bytes of packet descriptor into a * cacheline while allocating rep. */ rte_prefetch0(seg); rep = __rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { /* * Unable to allocate a replacement mbuf, * repost WR. */ DEBUG("rxq=%p: can't allocate a new mbuf", (void *)rxq); if (pkt_buf != NULL) { *pkt_buf_next = NULL; rte_pktmbuf_free(pkt_buf); } /* Increment out of memory counters. */ ++rxq->stats.rx_nombuf; ++rxq->priv->dev->data->rx_mbuf_alloc_failed; goto repost; } #ifndef NDEBUG /* Poison user-modifiable fields in rep. */ NEXT(rep) = (void *)((uintptr_t)-1); SET_DATA_OFF(rep, 0xdead); DATA_LEN(rep) = 0xd00d; PKT_LEN(rep) = 0xdeadd00d; NB_SEGS(rep) = 0x2a; PORT(rep) = 0x2a; rep->ol_flags = -1; #endif assert(rep->buf_len == seg->buf_len); assert(rep->buf_len == rxq->mb_len); /* Reconfigure sge to use rep instead of seg. */ assert(sge->lkey == rxq->mr->lkey); sge->addr = ((uintptr_t)rep->buf_addr + seg_headroom); elt->bufs[j] = rep; ++j; /* Update pkt_buf if it's the first segment, or link * seg to the previous one and update pkt_buf_next. */ *pkt_buf_next = seg; pkt_buf_next = &NEXT(seg); /* Update seg information. */ seg_tailroom = (seg->buf_len - seg_headroom); assert(sge->length == seg_tailroom); SET_DATA_OFF(seg, seg_headroom); if (likely(len <= seg_tailroom)) { /* Last segment. */ DATA_LEN(seg) = len; PKT_LEN(seg) = len; /* Sanity check. */ assert(rte_pktmbuf_headroom(seg) == seg_headroom); assert(rte_pktmbuf_tailroom(seg) == (seg_tailroom - len)); break; } DATA_LEN(seg) = seg_tailroom; PKT_LEN(seg) = seg_tailroom; /* Sanity check. */ assert(rte_pktmbuf_headroom(seg) == seg_headroom); assert(rte_pktmbuf_tailroom(seg) == 0); /* Fix len and clear headroom for next segments. */ len -= seg_tailroom; seg_headroom = 0; } /* Update head and tail segments. */ *pkt_buf_next = NULL; assert(pkt_buf != NULL); assert(j != 0); NB_SEGS(pkt_buf) = j; PORT(pkt_buf) = rxq->port_id; PKT_LEN(pkt_buf) = pkt_buf_len; if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip) { pkt_buf->packet_type = rxq_cq_to_pkt_type(flags); pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags); #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS if (flags & IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1) { pkt_buf->ol_flags |= PKT_RX_VLAN_PKT; pkt_buf->vlan_tci = vlan_tci; } #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */ } /* Return packet. */ *(pkts++) = pkt_buf; ++pkts_ret; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment bytes counter. */ rxq->stats.ibytes += pkt_buf_len; #endif repost: ret = rxq->recv(rxq->wq, elt->sges, RTE_DIM(elt->sges)); if (unlikely(ret)) { /* Inability to repost WRs is fatal. */ DEBUG("%p: recv_sg_list(): failed (ret=%d)", (void *)rxq->priv, ret); abort(); } if (++elts_head >= elts_n) elts_head = 0; continue; } if (unlikely(i == 0)) return 0; rxq->elts_head = elts_head; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment packets counter. */ rxq->stats.ipackets += pkts_ret; #endif return pkts_ret; }
/** * DPDK callback for RX. * * The following function is the same as mlx5_rx_burst_sp(), except it doesn't * manage scattered packets. Improves performance when MRU is lower than the * size of the first segment. * * @param dpdk_rxq * Generic pointer to RX queue structure. * @param[out] pkts * Array to store received packets. * @param pkts_n * Maximum number of packets in array. * * @return * Number of packets successfully received (<= pkts_n). */ uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct rxq *rxq = (struct rxq *)dpdk_rxq; struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp; const unsigned int elts_n = rxq->elts_n; unsigned int elts_head = rxq->elts_head; struct ibv_sge sges[pkts_n]; unsigned int i; unsigned int pkts_ret = 0; int ret; if (unlikely(rxq->sp)) return mlx5_rx_burst_sp(dpdk_rxq, pkts, pkts_n); for (i = 0; (i != pkts_n); ++i) { struct rxq_elt *elt = &(*elts)[elts_head]; unsigned int len; struct rte_mbuf *seg = elt->buf; struct rte_mbuf *rep; uint32_t flags; uint16_t vlan_tci; /* Sanity checks. */ assert(seg != NULL); assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); /* * Fetch initial bytes of packet descriptor into a * cacheline while allocating rep. */ rte_prefetch0(seg); rte_prefetch0(&seg->cacheline1); ret = rxq->poll(rxq->cq, NULL, NULL, &flags, &vlan_tci); if (unlikely(ret < 0)) { struct ibv_wc wc; int wcs_n; DEBUG("rxq=%p, poll_length() failed (ret=%d)", (void *)rxq, ret); /* ibv_poll_cq() must be used in case of failure. */ wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); if (unlikely(wcs_n == 0)) break; if (unlikely(wcs_n < 0)) { DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", (void *)rxq, wcs_n); break; } assert(wcs_n == 1); if (unlikely(wc.status != IBV_WC_SUCCESS)) { /* Whatever, just repost the offending WR. */ DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" " completion status (%d): %s", (void *)rxq, wc.wr_id, wc.status, ibv_wc_status_str(wc.status)); #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment dropped packets counter. */ ++rxq->stats.idropped; #endif /* Add SGE to array for repost. */ sges[i] = elt->sge; goto repost; } ret = wc.byte_len; } if (ret == 0) break; assert(ret >= (rxq->crc_present << 2)); len = ret - (rxq->crc_present << 2); rep = __rte_mbuf_raw_alloc(rxq->mp); if (unlikely(rep == NULL)) { /* * Unable to allocate a replacement mbuf, * repost WR. */ DEBUG("rxq=%p: can't allocate a new mbuf", (void *)rxq); /* Increment out of memory counters. */ ++rxq->stats.rx_nombuf; ++rxq->priv->dev->data->rx_mbuf_alloc_failed; goto repost; } /* Reconfigure sge to use rep instead of seg. */ elt->sge.addr = (uintptr_t)rep->buf_addr + RTE_PKTMBUF_HEADROOM; assert(elt->sge.lkey == rxq->mr->lkey); elt->buf = rep; /* Add SGE to array for repost. */ sges[i] = elt->sge; /* Update seg information. */ SET_DATA_OFF(seg, RTE_PKTMBUF_HEADROOM); NB_SEGS(seg) = 1; PORT(seg) = rxq->port_id; NEXT(seg) = NULL; PKT_LEN(seg) = len; DATA_LEN(seg) = len; if (rxq->csum | rxq->csum_l2tun | rxq->vlan_strip) { seg->packet_type = rxq_cq_to_pkt_type(flags); seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags); #ifdef HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS if (flags & IBV_EXP_CQ_RX_CVLAN_STRIPPED_V1) { seg->ol_flags |= PKT_RX_VLAN_PKT; seg->vlan_tci = vlan_tci; } #endif /* HAVE_EXP_DEVICE_ATTR_VLAN_OFFLOADS */ } /* Return packet. */ *(pkts++) = seg; ++pkts_ret; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment bytes counter. */ rxq->stats.ibytes += len; #endif repost: if (++elts_head >= elts_n) elts_head = 0; continue; } if (unlikely(i == 0)) return 0; /* Repost WRs. */ #ifdef DEBUG_RECV DEBUG("%p: reposting %u WRs", (void *)rxq, i); #endif ret = rxq->recv(rxq->wq, sges, i); if (unlikely(ret)) { /* Inability to repost WRs is fatal. */ DEBUG("%p: recv_burst(): failed (ret=%d)", (void *)rxq->priv, ret); abort(); } rxq->elts_head = elts_head; #ifdef MLX5_PMD_SOFT_COUNTERS /* Increment packets counter. */ rxq->stats.ipackets += pkts_ret; #endif return pkts_ret; }