uint16_t
fm10k_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct fm10k_tx_queue *q = tx_queue;
struct rte_mbuf *mb;
uint16_t count;
for (count = 0; count < nb_pkts; ++count) {
mb = tx_pkts[count];
/* running low on descriptors? try to free some... */
if (q->nb_free < q->free_trigger)
tx_free_descriptors(q);
/* make sure there are enough free descriptors to transmit the
* entire packet before doing anything */
if (q->nb_free < mb->nb_segs)
break;
/* sanity check to make sure the mbuf is valid */
if ((mb->nb_segs == 0) ||
((mb->nb_segs > 1) && (mb->next == NULL)))
break;
/* process the packet */
tx_xmit_pkt(q, mb);
}
/* update the tail pointer if any packets were processed */
if (likely(count > 0))
FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_free);
return count;
}
fm10k_reset_tx_queue(struct fm10k_tx_queue *txq)
{
static const struct fm10k_tx_desc zeroed_desc = {0};
struct rte_mbuf **txe = txq->sw_ring;
uint16_t i;
/* Zero out HW ring memory */
for (i = 0; i < txq->nb_desc; i++)
txq->hw_ring[i] = zeroed_desc;
/* Initialize SW ring entries */
for (i = 0; i < txq->nb_desc; i++)
txe[i] = NULL;
txq->next_dd = (uint16_t)(txq->rs_thresh - 1);
txq->next_rs = (uint16_t)(txq->rs_thresh - 1);
txq->next_free = 0;
txq->nb_used = 0;
/* Always allow 1 descriptor to be un-allocated to avoid
* a H/W race condition
*/
txq->nb_free = (uint16_t)(txq->nb_desc - 1);
FM10K_PCI_REG_WRITE(txq->tail_ptr, 0);
}
uint16_t
fm10k_recv_scattered_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct rte_mbuf *mbuf;
union fm10k_rx_desc desc;
struct fm10k_rx_queue *q = rx_queue;
uint16_t count = 0;
uint16_t nb_rcv, nb_seg;
int alloc = 0;
uint16_t next_dd;
struct rte_mbuf *first_seg = q->pkt_first_seg;
struct rte_mbuf *last_seg = q->pkt_last_seg;
int ret;
next_dd = q->next_dd;
nb_rcv = 0;
nb_seg = RTE_MIN(nb_pkts, q->alloc_thresh);
for (count = 0; count < nb_seg; count++) {
mbuf = q->sw_ring[next_dd];
desc = q->hw_ring[next_dd];
if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
break;
#ifdef RTE_LIBRTE_FM10K_DEBUG_RX
dump_rxd(&desc);
#endif
if (++next_dd == q->nb_desc) {
next_dd = 0;
alloc = 1;
}
/* Prefetch next mbuf while processing current one. */
rte_prefetch0(q->sw_ring[next_dd]);
/*
* When next RX descriptor is on a cache-line boundary,
* prefetch the next 4 RX descriptors and the next 8 pointers
* to mbufs.
*/
if ((next_dd & 0x3) == 0) {
rte_prefetch0(&q->hw_ring[next_dd]);
rte_prefetch0(&q->sw_ring[next_dd]);
}
/* Fill data length */
rte_pktmbuf_data_len(mbuf) = desc.w.length;
/*
* If this is the first buffer of the received packet,
* set the pointer to the first mbuf of the packet and
* initialize its context.
* Otherwise, update the total length and the number of segments
* of the current scattered packet, and update the pointer to
* the last mbuf of the current packet.
*/
if (!first_seg) {
first_seg = mbuf;
first_seg->pkt_len = desc.w.length;
} else {
first_seg->pkt_len =
(uint16_t)(first_seg->pkt_len +
rte_pktmbuf_data_len(mbuf));
first_seg->nb_segs++;
last_seg->next = mbuf;
}
/*
* If this is not the last buffer of the received packet,
* update the pointer to the last mbuf of the current scattered
* packet and continue to parse the RX ring.
*/
if (!(desc.d.staterr & FM10K_RXD_STATUS_EOP)) {
last_seg = mbuf;
continue;
}
first_seg->ol_flags = 0;
#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
rx_desc_to_ol_flags(first_seg, &desc);
#endif
first_seg->hash.rss = desc.d.rss;
/* Prefetch data of first segment, if configured to do so. */
rte_packet_prefetch((char *)first_seg->buf_addr +
first_seg->data_off);
/*
* Store the mbuf address into the next entry of the array
* of returned packets.
*/
rx_pkts[nb_rcv++] = first_seg;
/*
* Setup receipt context for a new packet.
*/
first_seg = NULL;
}
q->next_dd = next_dd;
if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
ret = rte_mempool_get_bulk(q->mp,
(void **)&q->sw_ring[q->next_alloc],
q->alloc_thresh);
if (unlikely(ret != 0)) {
uint8_t port = q->port_id;
PMD_RX_LOG(ERR, "Failed to alloc mbuf");
/*
* Need to restore next_dd if we cannot allocate new
* buffers to replenish the old ones.
*/
q->next_dd = (q->next_dd + q->nb_desc - count) %
q->nb_desc;
rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
return 0;
}
for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
mbuf = q->sw_ring[q->next_alloc];
/* setup static mbuf fields */
fm10k_pktmbuf_reset(mbuf, q->port_id);
/* write descriptor */
desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
q->hw_ring[q->next_alloc] = desc;
}
FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
q->next_trigger += q->alloc_thresh;
if (q->next_trigger >= q->nb_desc) {
q->next_trigger = q->alloc_thresh - 1;
q->next_alloc = 0;
}
}
q->pkt_first_seg = first_seg;
q->pkt_last_seg = last_seg;
return nb_rcv;
}
uint16_t
fm10k_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct rte_mbuf *mbuf;
union fm10k_rx_desc desc;
struct fm10k_rx_queue *q = rx_queue;
uint16_t count = 0;
int alloc = 0;
uint16_t next_dd;
int ret;
next_dd = q->next_dd;
nb_pkts = RTE_MIN(nb_pkts, q->alloc_thresh);
for (count = 0; count < nb_pkts; ++count) {
mbuf = q->sw_ring[next_dd];
desc = q->hw_ring[next_dd];
if (!(desc.d.staterr & FM10K_RXD_STATUS_DD))
break;
#ifdef RTE_LIBRTE_FM10K_DEBUG_RX
dump_rxd(&desc);
#endif
rte_pktmbuf_pkt_len(mbuf) = desc.w.length;
rte_pktmbuf_data_len(mbuf) = desc.w.length;
mbuf->ol_flags = 0;
#ifdef RTE_LIBRTE_FM10K_RX_OLFLAGS_ENABLE
rx_desc_to_ol_flags(mbuf, &desc);
#endif
mbuf->hash.rss = desc.d.rss;
rx_pkts[count] = mbuf;
if (++next_dd == q->nb_desc) {
next_dd = 0;
alloc = 1;
}
/* Prefetch next mbuf while processing current one. */
rte_prefetch0(q->sw_ring[next_dd]);
/*
* When next RX descriptor is on a cache-line boundary,
* prefetch the next 4 RX descriptors and the next 8 pointers
* to mbufs.
*/
if ((next_dd & 0x3) == 0) {
rte_prefetch0(&q->hw_ring[next_dd]);
rte_prefetch0(&q->sw_ring[next_dd]);
}
}
q->next_dd = next_dd;
if ((q->next_dd > q->next_trigger) || (alloc == 1)) {
ret = rte_mempool_get_bulk(q->mp,
(void **)&q->sw_ring[q->next_alloc],
q->alloc_thresh);
if (unlikely(ret != 0)) {
uint8_t port = q->port_id;
PMD_RX_LOG(ERR, "Failed to alloc mbuf");
/*
* Need to restore next_dd if we cannot allocate new
* buffers to replenish the old ones.
*/
q->next_dd = (q->next_dd + q->nb_desc - count) %
q->nb_desc;
rte_eth_devices[port].data->rx_mbuf_alloc_failed++;
return 0;
}
for (; q->next_alloc <= q->next_trigger; ++q->next_alloc) {
mbuf = q->sw_ring[q->next_alloc];
/* setup static mbuf fields */
fm10k_pktmbuf_reset(mbuf, q->port_id);
/* write descriptor */
desc.q.pkt_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
desc.q.hdr_addr = MBUF_DMA_ADDR_DEFAULT(mbuf);
q->hw_ring[q->next_alloc] = desc;
}
FM10K_PCI_REG_WRITE(q->tail_ptr, q->next_trigger);
q->next_trigger += q->alloc_thresh;
if (q->next_trigger >= q->nb_desc) {
q->next_trigger = q->alloc_thresh - 1;
q->next_alloc = 0;
}
}
return count;
}
uint16_t
fm10k_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct fm10k_tx_queue *txq = (struct fm10k_tx_queue *)tx_queue;
volatile struct fm10k_tx_desc *txdp;
struct rte_mbuf **txep;
uint16_t n, nb_commit, tx_id;
uint64_t flags = FM10K_TXD_FLAG_LAST;
uint64_t rs = FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_LAST;
int i;
/* cross rx_thresh boundary is not allowed */
nb_pkts = RTE_MIN(nb_pkts, txq->rs_thresh);
if (txq->nb_free < txq->free_thresh)
fm10k_tx_free_bufs(txq);
nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_free, nb_pkts);
if (unlikely(nb_pkts == 0))
return 0;
tx_id = txq->next_free;
txdp = &txq->hw_ring[tx_id];
txep = &txq->sw_ring[tx_id];
txq->nb_free = (uint16_t)(txq->nb_free - nb_pkts);
n = (uint16_t)(txq->nb_desc - tx_id);
if (nb_commit >= n) {
tx_backlog_entry(txep, tx_pkts, n);
for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
vtx1(txdp, *tx_pkts, flags);
vtx1(txdp, *tx_pkts++, rs);
nb_commit = (uint16_t)(nb_commit - n);
tx_id = 0;
txq->next_rs = (uint16_t)(txq->rs_thresh - 1);
/* avoid reach the end of ring */
txdp = &(txq->hw_ring[tx_id]);
txep = &txq->sw_ring[tx_id];
}
tx_backlog_entry(txep, tx_pkts, nb_commit);
vtx(txdp, tx_pkts, nb_commit, flags);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->next_rs) {
txq->hw_ring[txq->next_rs].flags |= FM10K_TXD_FLAG_RS;
txq->next_rs = (uint16_t)(txq->next_rs + txq->rs_thresh);
}
txq->next_free = tx_id;
FM10K_PCI_REG_WRITE(txq->tail_ptr, txq->next_free);
return nb_pkts;
}
fm10k_rxq_vec_setup(struct fm10k_rx_queue *rxq)
{
uintptr_t p;
struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
mb_def.nb_segs = 1;
/* data_off will be ajusted after new mbuf allocated for 512-byte
* alignment.
*/
mb_def.data_off = RTE_PKTMBUF_HEADROOM;
mb_def.port = rxq->port_id;
rte_mbuf_refcnt_set(&mb_def, 1);
/* prevent compiler reordering: rearm_data covers previous fields */
rte_compiler_barrier();
p = (uintptr_t)&mb_def.rearm_data;
rxq->mbuf_initializer = *(uint64_t *)p;
return 0;
}
static inline void
fm10k_rxq_rearm(struct fm10k_rx_queue *rxq)
{
int i;
uint16_t rx_id;
volatile union fm10k_rx_desc *rxdp;
struct rte_mbuf **mb_alloc = &rxq->sw_ring[rxq->rxrearm_start];
struct rte_mbuf *mb0, *mb1;
__m128i head_off = _mm_set_epi64x(
RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1,
RTE_PKTMBUF_HEADROOM + FM10K_RX_DATABUF_ALIGN - 1);
__m128i dma_addr0, dma_addr1;
/* Rx buffer need to be aligned with 512 byte */
const __m128i hba_msk = _mm_set_epi64x(0,
UINT64_MAX - FM10K_RX_DATABUF_ALIGN + 1);
rxdp = rxq->hw_ring + rxq->rxrearm_start;
/* Pull 'n' more MBUFs into the software ring */
if (rte_mempool_get_bulk(rxq->mp,
(void *)mb_alloc,
RTE_FM10K_RXQ_REARM_THRESH) < 0) {
dma_addr0 = _mm_setzero_si128();
/* Clean up all the HW/SW ring content */
for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i++) {
mb_alloc[i] = &rxq->fake_mbuf;
_mm_store_si128((__m128i *)&rxdp[i].q,
dma_addr0);
}
rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
RTE_FM10K_RXQ_REARM_THRESH;
return;
}
/* Initialize the mbufs in vector, process 2 mbufs in one loop */
for (i = 0; i < RTE_FM10K_RXQ_REARM_THRESH; i += 2, mb_alloc += 2) {
__m128i vaddr0, vaddr1;
uintptr_t p0, p1;
mb0 = mb_alloc[0];
mb1 = mb_alloc[1];
/* Flush mbuf with pkt template.
* Data to be rearmed is 6 bytes long.
* Though, RX will overwrite ol_flags that are coming next
* anyway. So overwrite whole 8 bytes with one load:
* 6 bytes of rearm_data plus first 2 bytes of ol_flags.
*/
p0 = (uintptr_t)&mb0->rearm_data;
*(uint64_t *)p0 = rxq->mbuf_initializer;
p1 = (uintptr_t)&mb1->rearm_data;
*(uint64_t *)p1 = rxq->mbuf_initializer;
/* load buf_addr(lo 64bit) and buf_physaddr(hi 64bit) */
vaddr0 = _mm_loadu_si128((__m128i *)&mb0->buf_addr);
vaddr1 = _mm_loadu_si128((__m128i *)&mb1->buf_addr);
/* convert pa to dma_addr hdr/data */
dma_addr0 = _mm_unpackhi_epi64(vaddr0, vaddr0);
dma_addr1 = _mm_unpackhi_epi64(vaddr1, vaddr1);
/* add headroom to pa values */
dma_addr0 = _mm_add_epi64(dma_addr0, head_off);
dma_addr1 = _mm_add_epi64(dma_addr1, head_off);
/* Do 512 byte alignment to satisfy HW requirement, in the
* meanwhile, set Header Buffer Address to zero.
*/
dma_addr0 = _mm_and_si128(dma_addr0, hba_msk);
dma_addr1 = _mm_and_si128(dma_addr1, hba_msk);
/* flush desc with pa dma_addr */
_mm_store_si128((__m128i *)&rxdp++->q, dma_addr0);
_mm_store_si128((__m128i *)&rxdp++->q, dma_addr1);
/* enforce 512B alignment on default Rx virtual addresses */
mb0->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb0->buf_addr
+ RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN)
- (char *)mb0->buf_addr);
mb1->data_off = (uint16_t)(RTE_PTR_ALIGN((char *)mb1->buf_addr
+ RTE_PKTMBUF_HEADROOM, FM10K_RX_DATABUF_ALIGN)
- (char *)mb1->buf_addr);
}
rxq->rxrearm_start += RTE_FM10K_RXQ_REARM_THRESH;
if (rxq->rxrearm_start >= rxq->nb_desc)
rxq->rxrearm_start = 0;
rxq->rxrearm_nb -= RTE_FM10K_RXQ_REARM_THRESH;
rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
(rxq->nb_desc - 1) : (rxq->rxrearm_start - 1));
/* Update the tail pointer on the NIC */
FM10K_PCI_REG_WRITE(rxq->tail_ptr, rx_id);
}
