uint16_t fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct fm10k_tx_queue *q = tx_queue; struct rte_mbuf *mb; uint16_t count; for (count = 0; count < nb_pkts; ++count) { mb = tx_pkts[count]; /* running low on descriptors? try to free some... */ if (q->nb_free < q->free_trigger) tx_free_descriptors(q); /* make sure there are enough free descriptors to transmit the * entire packet before doing anything */ if (q->nb_free < mb->nb_segs) break; /* sanity check to make sure the mbuf is valid */ if ((mb->nb_segs == 0) || ((mb->nb_segs > 1) && (mb->next == NULL))) break; /* process the packet */ tx_xmit_pkt(q, mb); } /* update the tail pointer if any packets were processed */ if (likely(count > 0)) FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_free); return count; }
fm10k_reset_tx_queue(struct fm10k_tx_queue *txq) { static const struct fm10k_tx_desc zeroed_desc = {0}; struct rte_mbuf **txe = txq->sw_ring; uint16_t i; /* Zero out HW ring memory */ for (i = 0; i < txq->nb_desc; i++) txq->hw_ring[i] = zeroed_desc; /* Initialize SW ring entries */ for (i = 0; i < txq->nb_desc; i++) txe[i] = NULL; txq->next_dd = (uint16_t)(txq->rs_thresh - 1); txq->next_rs = (uint16_t)(txq->rs_thresh - 1); txq->next_free = 0; txq->nb_used = 0; /* Always allow 1 descriptor to be un-allocated to avoid * a H/W race condition */ txq->nb_free = (uint16_t)(txq->nb_desc - 1); FM10K_PCI_REG_WRITE(txq->tail_ptr, 0); }
uint16_t fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct rte_mbuf *mbuf; union fm10k_rx_desc desc; struct fm10k_rx_queue *q = rx_queue; uint16_t count = 0; uint16_t nb_rcv, nb_seg; int alloc = 0; uint16_t next_dd; struct rte_mbuf *first_seg = q->pkt_first_seg; struct rte_mbuf *last_seg = q->pkt_last_seg; int ret; next_dd = q->next_dd; nb_rcv = 0; nb_seg = RTE_MIN(nb_pkts, q->alloc_thresh); for (count = 0; count < nb_seg; count++) { mbuf = q->sw_ring[next_dd]; desc = q->hw_ring[next_dd]; if (!(desc.d.staterr & FM10K_RXD_STATUS_DD)) break; #ifdef RTE_LIBRTE_FM10K_DEBUG_RX dump_rxd(&desc); #endif if (++next_dd == q->nb_desc) { next_dd = 0; alloc = 1; } /* Prefetch next mbuf while processing current one. */ rte_prefetch0(q->sw_ring[next_dd]); /* * When next RX descriptor is on a cache-line boundary, * prefetch the next 4 RX descriptors and the next 8 pointers * to mbufs. */ if ((next_dd & 0x3) == 0) { rte_prefetch0(&q->hw_ring[next_dd]); rte_prefetch0(&q->sw_ring[next_dd]); } /* Fill data length */ rte_pktmbuf_data_len(mbuf) = desc.w.length; /* * If this is the first buffer of the received packet, * set the pointer to the first mbuf of the packet and * initialize its context. * Otherwise, update the total length and the number of segments * of the current scattered packet, and update the pointer to * the last mbuf of the current packet. */ if (!first_seg) { first_seg = mbuf; first_seg->pkt_len = desc.w.length; } else { first_seg->pkt_len = (uint16_t)(first_seg->pkt_len + rte_pktmbuf_data_len(mbuf)); first_seg->nb_segs++; last_seg->next = mbuf; } /* * If this is not the last buffer of the received packet, * update the pointer to the last mbuf of the current scattered * packet and continue to parse the RX ring. */ if (!(desc.d.staterr & FM10K_RXD_STATUS_EOP)) { last_seg = mbuf; continue; } first_seg->ol_flags = 0; #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE rx_desc_to_ol_flags(first_seg, &desc); #endif first_seg->hash.rss = desc.d.rss; /* Prefetch data of first segment, if configured to do so. */ rte_packet_prefetch((char *)first_seg->buf_addr + first_seg->data_off); /* * Store the mbuf address into the next entry of the array * of returned packets. */ rx_pkts[nb_rcv++] = first_seg; /* * Setup receipt context for a new packet. */ first_seg = NULL; } q->next_dd = next_dd; if ((q->next_dd > q->next_trigger) || (alloc == 1)) { ret = rte_mempool_get_bulk(q->mp, (void **)&q->sw_ring[q->next_alloc], q->alloc_thresh); if (unlikely(ret != 0)) { uint8_t port = q->port_id; PMD_RX_LOG(ERR, "Failed to alloc mbuf"); /* * Need to restore next_dd if we cannot allocate new * buffers to replenish the old ones. */ q->next_dd = (q->next_dd + q->nb_desc - count) % q->nb_desc; rte_eth_devices[port].data->rx_mbuf_alloc_failed++; return 0; } for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) { mbuf = q->sw_ring[q->next_alloc]; /* setup static mbuf fields */ fm10k_pktmbuf_reset(mbuf, q->port_id); /* write descriptor */ desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf); desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf); q->hw_ring[q->next_alloc] = desc; } FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger); q->next_trigger += q->alloc_thresh; if (q->next_trigger >= q->nb_desc) { q->next_trigger = q->alloc_thresh - 1; q->next_alloc = 0; } } q->pkt_first_seg = first_seg; q->pkt_last_seg = last_seg; return nb_rcv; }
uint16_t fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct rte_mbuf *mbuf; union fm10k_rx_desc desc; struct fm10k_rx_queue *q = rx_queue; uint16_t count = 0; int alloc = 0; uint16_t next_dd; int ret; next_dd = q->next_dd; nb_pkts = RTE_MIN(nb_pkts, q->alloc_thresh); for (count = 0; count < nb_pkts; ++count) { mbuf = q->sw_ring[next_dd]; desc = q->hw_ring[next_dd]; if (!(desc.d.staterr & FM10K_RXD_STATUS_DD)) break; #ifdef RTE_LIBRTE_FM10K_DEBUG_RX dump_rxd(&desc); #endif rte_pktmbuf_pkt_len(mbuf) = desc.w.length; rte_pktmbuf_data_len(mbuf) = desc.w.length; mbuf->ol_flags = 0; #ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE rx_desc_to_ol_flags(mbuf, &desc); #endif mbuf->hash.rss = desc.d.rss; rx_pkts[count] = mbuf; if (++next_dd == q->nb_desc) { next_dd = 0; alloc = 1; } /* Prefetch next mbuf while processing current one. */ rte_prefetch0(q->sw_ring[next_dd]); /* * When next RX descriptor is on a cache-line boundary, * prefetch the next 4 RX descriptors and the next 8 pointers * to mbufs. */ if ((next_dd & 0x3) == 0) { rte_prefetch0(&q->hw_ring[next_dd]); rte_prefetch0(&q->sw_ring[next_dd]); } } q->next_dd = next_dd; if ((q->next_dd > q->next_trigger) || (alloc == 1)) { ret = rte_mempool_get_bulk(q->mp, (void **)&q->sw_ring[q->next_alloc], q->alloc_thresh); if (unlikely(ret != 0)) { uint8_t port = q->port_id; PMD_RX_LOG(ERR, "Failed to alloc mbuf"); /* * Need to restore next_dd if we cannot allocate new * buffers to replenish the old ones. */ q->next_dd = (q->next_dd + q->nb_desc - count) % q->nb_desc; rte_eth_devices[port].data->rx_mbuf_alloc_failed++; return 0; } for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) { mbuf = q->sw_ring[q->next_alloc]; /* setup static mbuf fields */ fm10k_pktmbuf_reset(mbuf, q->port_id); /* write descriptor */ desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf); desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf); q->hw_ring[q->next_alloc] = desc; } FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger); q->next_trigger += q->alloc_thresh; if (q->next_trigger >= q->nb_desc) { q->next_trigger = q->alloc_thresh - 1; q->next_alloc = 0; } } return count; }
uint16_t fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue; volatile struct fm10k_tx_desc *txdp; struct rte_mbuf **txep; uint16_t n, nb_commit, tx_id; uint64_t flags = FM10K_TXD_FLAG_LAST; uint64_t rs = FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_LAST; int i; /* cross rx_thresh boundary is not allowed */ nb_pkts = RTE_MIN(nb_pkts, txq->rs_thresh); if (txq->nb_free < txq->free_thresh) fm10k_tx_free_bufs(txq); nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_free, nb_pkts); if (unlikely(nb_pkts == 0)) return 0; tx_id = txq->next_free; txdp = &txq->hw_ring[tx_id]; txep = &txq->sw_ring[tx_id]; txq->nb_free = (uint16_t)(txq->nb_free - nb_pkts); n = (uint16_t)(txq->nb_desc - tx_id); if (nb_commit >= n) { tx_backlog_entry(txep, tx_pkts, n); for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp) vtx1(txdp, *tx_pkts, flags); vtx1(txdp, *tx_pkts++, rs); nb_commit = (uint16_t)(nb_commit - n); tx_id = 0; txq->next_rs = (uint16_t)(txq->rs_thresh - 1); /* avoid reach the end of ring */ txdp = &(txq->hw_ring[tx_id]); txep = &txq->sw_ring[tx_id]; } tx_backlog_entry(txep, tx_pkts, nb_commit); vtx(txdp, tx_pkts, nb_commit, flags); tx_id = (uint16_t)(tx_id + nb_commit); if (tx_id > txq->next_rs) { txq->hw_ring[txq->next_rs].flags |= FM10K_TXD_FLAG_RS; txq->next_rs = (uint16_t)(txq->next_rs + txq->rs_thresh); } txq->next_free = tx_id; FM10K_PCI_REG_WRITE(txq->tail_ptr, txq->next_free); return nb_pkts; }
fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq) { uintptr_t p; struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */ mb_def.nb_segs = 1; /* data_off will be ajusted after new mbuf allocated for 512-byte * alignment. */ mb_def.data_off = RTE_PKTMBUF_HEADROOM; mb_def.port = rxq->port_id; rte_mbuf_refcnt_set(&mb_def, 1); /* prevent compiler reordering: rearm_data covers previous fields */ rte_compiler_barrier(); p = (uintptr_t)&mb_def.rearm_data; rxq->mbuf_initializer = *(uint64_t *)p; return 0; } static inline void fm10k_rxq_rearm(struct fm10k_rx_queue *rxq) { int i; uint16_t rx_id; volatile union fm10k_rx_desc *rxdp; struct rte_mbuf **mb_alloc = &rxq->sw_ring[rxq->rxrearm_start]; struct rte_mbuf *mb0, *mb1; __m128i head_off = _mm_set_epi64x( RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1, RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1); __m128i dma_addr0, dma_addr1; /* Rx buffer need to be aligned with 512 byte */ const __m128i hba_msk = _mm_set_epi64x(0, UINT64_MAX - FM10K_RX_DATABUF_ALIGN + 1); rxdp = rxq->hw_ring + rxq->rxrearm_start; /* Pull 'n' more MBUFs into the software ring */ if (rte_mempool_get_bulk(rxq->mp, (void *)mb_alloc, RTE_FM10K_RXQ_REARM_THRESH) < 0) { dma_addr0 = _mm_setzero_si128(); /* Clean up all the HW/SW ring content */ for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i++) { mb_alloc[i] = &rxq->fake_mbuf; _mm_store_si128((__m128i *)&rxdp[i].q, dma_addr0); } rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed += RTE_FM10K_RXQ_REARM_THRESH; return; } /* Initialize the mbufs in vector, process 2 mbufs in one loop */ for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i += 2, mb_alloc += 2) { __m128i vaddr0, vaddr1; uintptr_t p0, p1; mb0 = mb_alloc[0]; mb1 = mb_alloc[1]; /* Flush mbuf with pkt template. * Data to be rearmed is 6 bytes long. * Though, RX will overwrite ol_flags that are coming next * anyway. So overwrite whole 8 bytes with one load: * 6 bytes of rearm_data plus first 2 bytes of ol_flags. */ p0 = (uintptr_t)&mb0->rearm_data; *(uint64_t *)p0 = rxq->mbuf_initializer; p1 = (uintptr_t)&mb1->rearm_data; *(uint64_t *)p1 = rxq->mbuf_initializer; /* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */ vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr); vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr); /* convert pa to dma_addr hdr/data */ dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0); dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1); /* add headroom to pa values */ dma_addr0 = _mm_add_epi64(dma_addr0, head_off); dma_addr1 = _mm_add_epi64(dma_addr1, head_off); /* Do 512 byte alignment to satisfy HW requirement, in the * meanwhile, set Header Buffer Address to zero. */ dma_addr0 = _mm_and_si128(dma_addr0, hba_msk); dma_addr1 = _mm_and_si128(dma_addr1, hba_msk); /* flush desc with pa dma_addr */ _mm_store_si128((__m128i *)&rxdp++->q, dma_addr0); _mm_store_si128((__m128i *)&rxdp++->q, dma_addr1); /* enforce 512B alignment on default Rx virtual addresses */ mb0->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb0->buf_addr + RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN) - (char *)mb0->buf_addr); mb1->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb1->buf_addr + RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN) - (char *)mb1->buf_addr); } rxq->rxrearm_start += RTE_FM10K_RXQ_REARM_THRESH; if (rxq->rxrearm_start >= rxq->nb_desc) rxq->rxrearm_start = 0; rxq->rxrearm_nb -= RTE_FM10K_RXQ_REARM_THRESH; rx_id = (uint16_t)((rxq->rxrearm_start == 0) ? (rxq->nb_desc - 1) : (rxq->rxrearm_start - 1)); /* Update the tail pointer on the NIC */ FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id); }