static void
vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
{
uint32_t avail = 0;
if (rxq == NULL)
return;
PMD_RX_LOG(DEBUG,
"RXQ: cmd0 base : 0x%p cmd1 base : 0x%p comp ring base : 0x%p.",
rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
PMD_RX_LOG(DEBUG,
"RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
(unsigned long)rxq->cmd_ring[0].basePA,
(unsigned long)rxq->cmd_ring[1].basePA,
(unsigned long)rxq->comp_ring.basePA);
avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
PMD_RX_LOG(DEBUG,
"RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
(uint32_t)rxq->cmd_ring[0].size, avail,
rxq->comp_ring.next2proc,
rxq->cmd_ring[0].size - avail);
avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
(uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
rxq->cmd_ring[1].size - avail);
}
static uint16_t
eth_xenvirt_rx(void *q, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
struct virtqueue *rxvq = q;
struct rte_mbuf *rxm, *new_mbuf;
uint16_t nb_used, num;
uint32_t len[VIRTIO_MBUF_BURST_SZ];
uint32_t i;
struct pmd_internals *pi = rxvq->internals;
nb_used = VIRTQUEUE_NUSED(rxvq);
rte_compiler_barrier(); /* rmb */
num = (uint16_t)(likely(nb_used <= nb_pkts) ? nb_used : nb_pkts);
num = (uint16_t)(likely(num <= VIRTIO_MBUF_BURST_SZ) ? num : VIRTIO_MBUF_BURST_SZ);
if (unlikely(num == 0)) return 0;
num = virtqueue_dequeue_burst(rxvq, rx_pkts, len, num);
PMD_RX_LOG(DEBUG, "used:%d dequeue:%d\n", nb_used, num);
for (i = 0; i < num ; i ++) {
rxm = rx_pkts[i];
PMD_RX_LOG(DEBUG, "packet len:%d\n", len[i]);
rxm->next = NULL;
rxm->data_off = RTE_PKTMBUF_HEADROOM;
rxm->data_len = (uint16_t)(len[i] - sizeof(struct virtio_net_hdr));
rxm->nb_segs = 1;
rxm->port = pi->port_id;
rxm->pkt_len = (uint32_t)(len[i] - sizeof(struct virtio_net_hdr));
}
/* allocate new mbuf for the used descriptor */
while (likely(!virtqueue_full(rxvq))) {
new_mbuf = rte_rxmbuf_alloc(rxvq->mpool);
if (unlikely(new_mbuf == NULL)) {
break;
}
if (unlikely(virtqueue_enqueue_recv_refill(rxvq, new_mbuf))) {
rte_pktmbuf_free_seg(new_mbuf);
break;
}
}
pi->eth_stats.ipackets += num;
return num;
}
/*
* Process the Rx Completion Ring of given vmxnet3_rx_queue
* for nb_pkts burst and return the number of packets received
*/
uint16_t
vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
uint16_t nb_rx;
uint32_t nb_rxd, idx;
uint8_t ring_idx;
vmxnet3_rx_queue_t *rxq;
Vmxnet3_RxCompDesc *rcd;
vmxnet3_buf_info_t *rbi;
Vmxnet3_RxDesc *rxd;
struct rte_mbuf *rxm = NULL;
struct vmxnet3_hw *hw;
nb_rx = 0;
ring_idx = 0;
nb_rxd = 0;
idx = 0;
rxq = rx_queue;
hw = rxq->hw;
rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
if (unlikely(rxq->stopped)) {
PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
return 0;
}
while (rcd->gen == rxq->comp_ring.gen) {
if (nb_rx >= nb_pkts)
break;
idx = rcd->rxdIdx;
ring_idx = (uint8_t)((rcd->rqID == rxq->qid1) ? 0 : 1);
rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
if (unlikely(rcd->sop != 1 || rcd->eop != 1)) {
rte_pktmbuf_free_seg(rbi->m);
PMD_RX_LOG(DEBUG, "Packet spread across multiple buffers\n)");
goto rcd_done;
}
PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
VMXNET3_ASSERT(rcd->len <= rxd->len);
VMXNET3_ASSERT(rbi->m);
if (unlikely(rcd->len == 0)) {
PMD_RX_LOG(DEBUG, "Rx buf was skipped. rxring[%d][%d]\n)",
ring_idx, idx);
VMXNET3_ASSERT(rcd->sop && rcd->eop);
rte_pktmbuf_free_seg(rbi->m);
goto rcd_done;
}
/* Assuming a packet is coming in a single packet buffer */
if (unlikely(rxd->btype != VMXNET3_RXD_BTYPE_HEAD)) {
PMD_RX_LOG(DEBUG,
"Alert : Misbehaving device, incorrect "
" buffer type used. iPacket dropped.");
rte_pktmbuf_free_seg(rbi->m);
goto rcd_done;
}
VMXNET3_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
/* Get the packet buffer pointer from buf_info */
rxm = rbi->m;
/* Clear descriptor associated buf_info to be reused */
rbi->m = NULL;
rbi->bufPA = 0;
/* Update the index that we received a packet */
rxq->cmd_ring[ring_idx].next2comp = idx;
/* For RCD with EOP set, check if there is frame error */
if (unlikely(rcd->err)) {
rxq->stats.drop_total++;
rxq->stats.drop_err++;
if (!rcd->fcs) {
rxq->stats.drop_fcs++;
PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
}
PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
(int)(rcd - (struct Vmxnet3_RxCompDesc *)
rxq->comp_ring.base), rcd->rxdIdx);
rte_pktmbuf_free_seg(rxm);
goto rcd_done;
}
/* Initialize newly received packet buffer */
rxm->port = rxq->port_id;
rxm->nb_segs = 1;
rxm->next = NULL;
rxm->pkt_len = (uint16_t)rcd->len;
rxm->data_len = (uint16_t)rcd->len;
rxm->data_off = RTE_PKTMBUF_HEADROOM;
rxm->ol_flags = 0;
rxm->vlan_tci = 0;
vmxnet3_rx_offload(rcd, rxm);
rx_pkts[nb_rx++] = rxm;
rcd_done:
rxq->cmd_ring[ring_idx].next2comp = idx;
VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp, rxq->cmd_ring[ring_idx].size);
/* It's time to allocate some new buf and renew descriptors */
vmxnet3_post_rx_bufs(rxq, ring_idx);
if (unlikely(rxq->shared->ctrl.updateRxProd)) {
VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
rxq->cmd_ring[ring_idx].next2fill);
}
/* Advance to the next descriptor in comp_ring */
vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
nb_rxd++;
if (nb_rxd > rxq->cmd_ring[0].size) {
PMD_RX_LOG(ERR,
"Used up quota of receiving packets,"
" relinquish control.");
break;
}
}
return nb_rx;
}
/*
* Allocates mbufs and clusters. Post rx descriptors with buffer details
* so that device can receive packets in those buffers.
* Ring layout:
* Among the two rings, 1st ring contains buffers of type 0 and type1.
* bufs_per_pkt is set such that for non-LRO cases all the buffers required
* by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
* 2nd ring contains buffers of type 1 alone. Second ring mostly be used
* only for LRO.
*
*/
static int
vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
{
int err = 0;
uint32_t i = 0, val = 0;
struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
if (ring_id == 0) {
/* Usually: One HEAD type buf per packet
* val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
* VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
*/
/* We use single packet buffer so all heads here */
val = VMXNET3_RXD_BTYPE_HEAD;
} else {
/* All BODY type buffers for 2nd ring */
val = VMXNET3_RXD_BTYPE_BODY;
}
while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
struct Vmxnet3_RxDesc *rxd;
struct rte_mbuf *mbuf;
vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
rxd = (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
/* Allocate blank mbuf for the current Rx Descriptor */
mbuf = rte_rxmbuf_alloc(rxq->mp);
if (unlikely(mbuf == NULL)) {
PMD_RX_LOG(ERR, "Error allocating mbuf in %s", __func__);
rxq->stats.rx_buf_alloc_failure++;
err = ENOMEM;
break;
}
/*
* Load mbuf pointer into buf_info[ring_size]
* buf_info structure is equivalent to cookie for virtio-virtqueue
*/
buf_info->m = mbuf;
buf_info->len = (uint16_t)(mbuf->buf_len -
RTE_PKTMBUF_HEADROOM);
buf_info->bufPA =
rte_mbuf_data_dma_addr_default(mbuf);
/* Load Rx Descriptor with the buffer's GPA */
rxd->addr = buf_info->bufPA;
/* After this point rxd->addr MUST not be NULL */
rxd->btype = val;
rxd->len = buf_info->len;
/* Flip gen bit at the end to change ownership */
rxd->gen = ring->gen;
vmxnet3_cmd_ring_adv_next2fill(ring);
i++;
}
/* Return error only if no buffers are posted at present */
if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
return -err;
else
return i;
}
static inline void dump_rxd(union fm10k_rx_desc *rxd)
{
PMD_RX_LOG(DEBUG, "+----------------|----------------+");
PMD_RX_LOG(DEBUG, "| GLORT | PKT HDR & TYPE |");
PMD_RX_LOG(DEBUG, "| 0x%08x | 0x%08x |", rxd->d.glort,
rxd->d.data);
PMD_RX_LOG(DEBUG, "+----------------|----------------+");
PMD_RX_LOG(DEBUG, "| VLAN & LEN | STATUS |");
PMD_RX_LOG(DEBUG, "| 0x%08x | 0x%08x |", rxd->d.vlan_len,
rxd->d.staterr);
PMD_RX_LOG(DEBUG, "+----------------|----------------+");
PMD_RX_LOG(DEBUG, "| RESERVED | RSS_HASH |");
PMD_RX_LOG(DEBUG, "| 0x%08x | 0x%08x |", 0, rxd->d.rss);
PMD_RX_LOG(DEBUG, "+----------------|----------------+");
PMD_RX_LOG(DEBUG, "| TIME TAG |");
PMD_RX_LOG(DEBUG, "| 0x%016lx |", rxd->q.timestamp);
PMD_RX_LOG(DEBUG, "+----------------|----------------+");
}
uint16_t
fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct rte_mbuf *mbuf;
union fm10k_rx_desc desc;
struct fm10k_rx_queue *q = rx_queue;
uint16_t count = 0;
uint16_t nb_rcv, nb_seg;
int alloc = 0;
uint16_t next_dd;
struct rte_mbuf *first_seg = q->pkt_first_seg;
struct rte_mbuf *last_seg = q->pkt_last_seg;
int ret;
next_dd = q->next_dd;
nb_rcv = 0;
nb_seg = RTE_MIN(nb_pkts, q->alloc_thresh);
for (count = 0; count < nb_seg; count++) {
mbuf = q->sw_ring[next_dd];
desc = q->hw_ring[next_dd];
if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
break;
#ifdef RTE_LIBRTE_FM10K_DEBUG_RX
dump_rxd(&desc);
#endif
if (++next_dd == q->nb_desc) {
next_dd = 0;
alloc = 1;
}
/* Prefetch next mbuf while processing current one. */
rte_prefetch0(q->sw_ring[next_dd]);
/*
* When next RX descriptor is on a cache-line boundary,
* prefetch the next 4 RX descriptors and the next 8 pointers
* to mbufs.
*/
if ((next_dd & 0x3) == 0) {
rte_prefetch0(&q->hw_ring[next_dd]);
rte_prefetch0(&q->sw_ring[next_dd]);
}
/* Fill data length */
rte_pktmbuf_data_len(mbuf) = desc.w.length;
/*
* If this is the first buffer of the received packet,
* set the pointer to the first mbuf of the packet and
* initialize its context.
* Otherwise, update the total length and the number of segments
* of the current scattered packet, and update the pointer to
* the last mbuf of the current packet.
*/
if (!first_seg) {
first_seg = mbuf;
first_seg->pkt_len = desc.w.length;
} else {
first_seg->pkt_len =
(uint16_t)(first_seg->pkt_len +
rte_pktmbuf_data_len(mbuf));
first_seg->nb_segs++;
last_seg->next = mbuf;
}
/*
* If this is not the last buffer of the received packet,
* update the pointer to the last mbuf of the current scattered
* packet and continue to parse the RX ring.
*/
if (!(desc.d.staterr & FM10K_RXD_STATUS_EOP)) {
last_seg = mbuf;
continue;
}
first_seg->ol_flags = 0;
#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
rx_desc_to_ol_flags(first_seg, &desc);
#endif
first_seg->hash.rss = desc.d.rss;
/* Prefetch data of first segment, if configured to do so. */
rte_packet_prefetch((char *)first_seg->buf_addr +
first_seg->data_off);
/*
* Store the mbuf address into the next entry of the array
* of returned packets.
*/
rx_pkts[nb_rcv++] = first_seg;
/*
* Setup receipt context for a new packet.
*/
first_seg = NULL;
}
q->next_dd = next_dd;
if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
ret = rte_mempool_get_bulk(q->mp,
(void **)&q->sw_ring[q->next_alloc],
q->alloc_thresh);
if (unlikely(ret != 0)) {
uint8_t port = q->port_id;
PMD_RX_LOG(ERR, "Failed to alloc mbuf");
/*
* Need to restore next_dd if we cannot allocate new
* buffers to replenish the old ones.
*/
q->next_dd = (q->next_dd + q->nb_desc - count) %
q->nb_desc;
rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
return 0;
}
for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
mbuf = q->sw_ring[q->next_alloc];
/* setup static mbuf fields */
fm10k_pktmbuf_reset(mbuf, q->port_id);
/* write descriptor */
desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
q->hw_ring[q->next_alloc] = desc;
}
FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
q->next_trigger += q->alloc_thresh;
if (q->next_trigger >= q->nb_desc) {
q->next_trigger = q->alloc_thresh - 1;
q->next_alloc = 0;
}
}
q->pkt_first_seg = first_seg;
q->pkt_last_seg = last_seg;
return nb_rcv;
}
uint16_t
fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct rte_mbuf *mbuf;
union fm10k_rx_desc desc;
struct fm10k_rx_queue *q = rx_queue;
uint16_t count = 0;
int alloc = 0;
uint16_t next_dd;
int ret;
next_dd = q->next_dd;
nb_pkts = RTE_MIN(nb_pkts, q->alloc_thresh);
for (count = 0; count < nb_pkts; ++count) {
mbuf = q->sw_ring[next_dd];
desc = q->hw_ring[next_dd];
if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
break;
#ifdef RTE_LIBRTE_FM10K_DEBUG_RX
dump_rxd(&desc);
#endif
rte_pktmbuf_pkt_len(mbuf) = desc.w.length;
rte_pktmbuf_data_len(mbuf) = desc.w.length;
mbuf->ol_flags = 0;
#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
rx_desc_to_ol_flags(mbuf, &desc);
#endif
mbuf->hash.rss = desc.d.rss;
rx_pkts[count] = mbuf;
if (++next_dd == q->nb_desc) {
next_dd = 0;
alloc = 1;
}
/* Prefetch next mbuf while processing current one. */
rte_prefetch0(q->sw_ring[next_dd]);
/*
* When next RX descriptor is on a cache-line boundary,
* prefetch the next 4 RX descriptors and the next 8 pointers
* to mbufs.
*/
if ((next_dd & 0x3) == 0) {
rte_prefetch0(&q->hw_ring[next_dd]);
rte_prefetch0(&q->sw_ring[next_dd]);
}
}
q->next_dd = next_dd;
if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
ret = rte_mempool_get_bulk(q->mp,
(void **)&q->sw_ring[q->next_alloc],
q->alloc_thresh);
if (unlikely(ret != 0)) {
uint8_t port = q->port_id;
PMD_RX_LOG(ERR, "Failed to alloc mbuf");
/*
* Need to restore next_dd if we cannot allocate new
* buffers to replenish the old ones.
*/
q->next_dd = (q->next_dd + q->nb_desc - count) %
q->nb_desc;
rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
return 0;
}
for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
mbuf = q->sw_ring[q->next_alloc];
/* setup static mbuf fields */
fm10k_pktmbuf_reset(mbuf, q->port_id);
/* write descriptor */
desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
q->hw_ring[q->next_alloc] = desc;
}
FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
q->next_trigger += q->alloc_thresh;
if (q->next_trigger >= q->nb_desc) {
q->next_trigger = q->alloc_thresh - 1;
q->next_alloc = 0;
}
}
return count;
}
int
bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
uint16_t queue_idx,
uint16_t nb_desc,
unsigned int socket_id,
const struct rte_eth_rxconf *rx_conf,
struct rte_mempool *mp)
{
uint16_t j, idx;
const struct rte_memzone *dma;
struct bnx2x_rx_queue *rxq;
uint32_t dma_size;
struct rte_mbuf *mbuf;
struct bnx2x_softc *sc = dev->data->dev_private;
struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
struct eth_rx_cqe_next_page *nextpg;
phys_addr_t *rx_bd;
phys_addr_t busaddr;
/* First allocate the rx queue data structure */
rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
RTE_CACHE_LINE_SIZE, socket_id);
if (NULL == rxq) {
PMD_INIT_LOG(ERR, "rte_zmalloc for rxq failed!");
return -ENOMEM;
}
rxq->sc = sc;
rxq->mb_pool = mp;
rxq->queue_id = queue_idx;
rxq->port_id = dev->data->port_id;
rxq->crc_len = (uint8_t)((dev->data->dev_conf.rxmode.hw_strip_crc) ? 0 : ETHER_CRC_LEN);
rxq->nb_rx_pages = 1;
while (USABLE_RX_BD(rxq) < nb_desc)
rxq->nb_rx_pages <<= 1;
rxq->nb_rx_desc = TOTAL_RX_BD(rxq);
sc->rx_ring_size = USABLE_RX_BD(rxq);
rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
rxq->rx_free_thresh = rx_conf->rx_free_thresh ?
rx_conf->rx_free_thresh : DEFAULT_RX_FREE_THRESH;
PMD_INIT_LOG(DEBUG, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
"total_bd=%lu, rx_pages=%u, cq_pages=%u",
queue_idx, nb_desc, rxq->rx_free_thresh,
(unsigned long)USABLE_RX_BD(rxq),
(unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
rxq->nb_cq_pages);
/* Allocate RX ring hardware descriptors */
dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
if (NULL == dma) {
PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
bnx2x_rx_queue_release(rxq);
return -ENOMEM;
}
fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
rxq->rx_ring = (uint64_t*)dma->addr;
memset((void *)rxq->rx_ring, 0, dma_size);
/* Link the RX chain pages. */
for (j = 1; j <= rxq->nb_rx_pages; j++) {
rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
*rx_bd = busaddr;
}
/* Allocate software ring */
dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
RTE_CACHE_LINE_SIZE,
socket_id);
if (NULL == rxq->sw_ring) {
PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
bnx2x_rx_queue_release(rxq);
return -ENOMEM;
}
/* Initialize software ring entries */
rxq->rx_mbuf_alloc = 0;
for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
mbuf = bnx2x_rxmbuf_alloc(mp);
if (NULL == mbuf) {
PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
(unsigned)rxq->queue_id, idx);
bnx2x_rx_queue_release(rxq);
return -ENOMEM;
}
rxq->sw_ring[idx] = mbuf;
rxq->rx_ring[idx] = mbuf->buf_physaddr;
rxq->rx_mbuf_alloc++;
}
rxq->pkt_first_seg = NULL;
rxq->pkt_last_seg = NULL;
rxq->rx_bd_head = 0;
rxq->rx_bd_tail = rxq->nb_rx_desc;
/* Allocate CQ chain. */
dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
if (NULL == dma) {
PMD_RX_LOG(ERR, "RCQ alloc failed");
return -ENOMEM;
}
fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
/* Link the CQ chain pages. */
for (j = 1; j <= rxq->nb_cq_pages; j++) {
nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
}
rxq->rx_cq_head = 0;
rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
dev->data->rx_queues[queue_idx] = rxq;
if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
return 0;
}
static uint16_t
bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
struct bnx2x_rx_queue *rxq = p_rxq;
struct bnx2x_softc *sc = rxq->sc;
struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
uint32_t nb_rx = 0;
uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
uint16_t bd_cons, bd_prod;
struct rte_mbuf *new_mb;
uint16_t rx_pref;
struct eth_fast_path_rx_cqe *cqe_fp;
uint16_t len, pad;
struct rte_mbuf *rx_mb = NULL;
hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
USABLE_RCQ_ENTRIES_PER_PAGE) {
++hw_cq_cons;
}
bd_cons = rxq->rx_bd_head;
bd_prod = rxq->rx_bd_tail;
sw_cq_cons = rxq->rx_cq_head;
sw_cq_prod = rxq->rx_cq_tail;
if (sw_cq_cons == hw_cq_cons)
return 0;
while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
bd_prod &= MAX_RX_BD(rxq);
bd_cons &= MAX_RX_BD(rxq);
cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
PMD_RX_LOG(ERR, "slowpath event during traffic processing");
break;
}
if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
cqe_fp->type_error_flags, sw_cq_cons);
goto next_rx;
}
len = cqe_fp->pkt_len_or_gro_seg_len;
pad = cqe_fp->placement_offset;
new_mb = bnx2x_rxmbuf_alloc(rxq->mb_pool);
if (unlikely(!new_mb)) {
PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
goto next_rx;
}
rx_mb = rxq->sw_ring[bd_cons];
rxq->sw_ring[bd_cons] = new_mb;
rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
rte_prefetch0(rxq->sw_ring[rx_pref]);
if ((rx_pref & 0x3) == 0) {
rte_prefetch0(&rxq->rx_ring[rx_pref]);
rte_prefetch0(&rxq->sw_ring[rx_pref]);
}
rx_mb->data_off = pad;
rx_mb->nb_segs = 1;
rx_mb->next = NULL;
rx_mb->pkt_len = rx_mb->data_len = len;
rx_mb->port = rxq->port_id;
rx_mb->buf_len = len + pad;
rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
/*
* If we received a packet with a vlan tag,
* attach that information to the packet.
*/
if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
rx_mb->vlan_tci = cqe_fp->vlan_tag;
rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
}
rx_pkts[nb_rx] = rx_mb;
nb_rx++;
/* limit spinning on the queue */
if (unlikely(nb_rx == sc->rx_budget)) {
PMD_RX_LOG(ERR, "Limit spinning on the queue");
break;
}
next_rx:
bd_cons = NEXT_RX_BD(bd_cons);
bd_prod = NEXT_RX_BD(bd_prod);
sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
}
rxq->rx_bd_head = bd_cons;
rxq->rx_bd_tail = bd_prod;
rxq->rx_cq_head = sw_cq_cons;
rxq->rx_cq_tail = sw_cq_prod;
bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
return nb_rx;
}