uint16_t
vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
uint16_t nb_tx;
vmxnet3_tx_queue_t *txq = tx_queue;
struct vmxnet3_hw *hw = txq->hw;
if (unlikely(txq->stopped)) {
PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
return 0;
}
/* Free up the comp_descriptors aggressively */
vmxnet3_tq_tx_complete(txq);
nb_tx = 0;
while (nb_tx < nb_pkts) {
Vmxnet3_GenericDesc *gdesc;
vmxnet3_buf_info_t *tbi;
uint32_t first2fill, avail, dw2;
struct rte_mbuf *txm = tx_pkts[nb_tx];
struct rte_mbuf *m_seg = txm;
/* Is this packet execessively fragmented, then drop */
if (unlikely(txm->nb_segs > VMXNET3_MAX_TXD_PER_PKT)) {
++txq->stats.drop_too_many_segs;
++txq->stats.drop_total;
rte_pktmbuf_free(txm);
++nb_tx;
continue;
}
/* Is command ring full? */
avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
if (txm->nb_segs > avail) {
++txq->stats.tx_ring_full;
break;
}
/* use the previous gen bit for the SOP desc */
dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
first2fill = txq->cmd_ring.next2fill;
do {
/* Remember the transmit buffer for cleanup */
tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
tbi->m = m_seg;
/* NB: the following assumes that VMXNET3 maximum
transmit buffer size (16K) is greater than
maximum sizeof mbuf segment size. */
gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
gdesc->txd.addr = rte_mbuf_data_dma_addr(m_seg);
gdesc->dword[2] = dw2 | m_seg->data_len;
gdesc->dword[3] = 0;
/* move to the next2fill descriptor */
vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
/* use the right gen for non-SOP desc */
dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
} while ((m_seg = m_seg->next) != NULL);
/* Update the EOP descriptor */
gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
/* Add VLAN tag if present */
gdesc = txq->cmd_ring.base + first2fill;
if (txm->ol_flags & PKT_TX_VLAN_PKT) {
gdesc->txd.ti = 1;
gdesc->txd.tci = txm->vlan_tci;
}
/* TODO: Add transmit checksum offload here */
/* flip the GEN bit on the SOP */
rte_compiler_barrier();
gdesc->dword[2] ^= VMXNET3_TXD_GEN;
txq->shared->ctrl.txNumDeferred++;
nb_tx++;
}
PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", txq->shared->ctrl.txThreshold);
if (txq->shared->ctrl.txNumDeferred >= txq->shared->ctrl.txThreshold) {
txq->shared->ctrl.txNumDeferred = 0;
/* Notify vSwitch that packets are available. */
VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
txq->cmd_ring.next2fill);
}
return nb_tx;
}
static uint16_t
sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
unsigned int added = txq->added;
unsigned int pushed = added;
unsigned int pkts_sent = 0;
efx_desc_t *pend = &txq->pend_desc[0];
const unsigned int hard_max_fill = EFX_TXQ_LIMIT(txq->ptr_mask + 1);
const unsigned int soft_max_fill = hard_max_fill - txq->free_thresh;
unsigned int fill_level = added - txq->completed;
boolean_t reap_done;
int rc __rte_unused;
struct rte_mbuf **pktp;
if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
goto done;
/*
* If insufficient space for a single packet is present,
* we should reap; otherwise, we shouldn't do that all the time
* to avoid latency increase
*/
reap_done = (fill_level > soft_max_fill);
if (reap_done) {
sfc_efx_tx_reap(txq);
/*
* Recalculate fill level since 'txq->completed'
* might have changed on reap
*/
fill_level = added - txq->completed;
}
for (pkts_sent = 0, pktp = &tx_pkts[0];
(pkts_sent < nb_pkts) && (fill_level <= soft_max_fill);
pkts_sent++, pktp++) {
struct rte_mbuf *m_seg = *pktp;
size_t pkt_len = m_seg->pkt_len;
unsigned int pkt_descs = 0;
size_t in_off = 0;
/*
* Here VLAN TCI is expected to be zero in case if no
* DEV_TX_VLAN_OFFLOAD capability is advertised;
* if the calling app ignores the absence of
* DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
* TX_ERROR will occur
*/
pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
/*
* We expect correct 'pkt->l[2, 3, 4]_len' values
* to be set correctly by the caller
*/
if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
&pkt_descs, &pkt_len) != 0) {
/* We may have reached this place for
* one of the following reasons:
*
* 1) Packet header length is greater
* than SFC_TSOH_STD_LEN
* 2) TCP header starts at more then
* 208 bytes into the frame
*
* We will deceive RTE saying that we have sent
* the packet, but we will actually drop it.
* Hence, we should revert 'pend' to the
* previous state (in case we have added
* VLAN descriptor) and start processing
* another one packet. But the original
* mbuf shouldn't be orphaned
*/
pend -= pkt_descs;
rte_pktmbuf_free(*pktp);
continue;
}
/*
* We've only added 2 FATSOv2 option descriptors
* and 1 descriptor for the linearized packet header.
* The outstanding work will be done in the same manner
* as for the usual non-TSO path
*/
}
for (; m_seg != NULL; m_seg = m_seg->next) {
efsys_dma_addr_t next_frag;
size_t seg_len;
seg_len = m_seg->data_len;
next_frag = rte_mbuf_data_dma_addr(m_seg);
/*
* If we've started TSO transaction few steps earlier,
* we'll skip packet header using an offset in the
* current segment (which has been set to the
* first one containing payload)
*/
seg_len -= in_off;
next_frag += in_off;
in_off = 0;
do {
efsys_dma_addr_t frag_addr = next_frag;
size_t frag_len;
/*
* It is assumed here that there is no
* limitation on address boundary
* crossing by DMA descriptor.
*/
frag_len = MIN(seg_len, txq->dma_desc_size_max);
next_frag += frag_len;
seg_len -= frag_len;
pkt_len -= frag_len;
efx_tx_qdesc_dma_create(txq->common,
frag_addr, frag_len,
(pkt_len == 0),
pend++);
pkt_descs++;
} while (seg_len != 0);
}
added += pkt_descs;
fill_level += pkt_descs;
if (unlikely(fill_level > hard_max_fill)) {
/*
* Our estimation for maximum number of descriptors
* required to send a packet seems to be wrong.
* Try to reap (if we haven't yet).
*/
if (!reap_done) {
sfc_efx_tx_reap(txq);
reap_done = B_TRUE;
fill_level = added - txq->completed;
if (fill_level > hard_max_fill) {
pend -= pkt_descs;
break;
}
} else {
pend -= pkt_descs;
break;
}
}
/* Assign mbuf to the last used desc */
txq->sw_ring[(added - 1) & txq->ptr_mask].mbuf = *pktp;
}
if (likely(pkts_sent > 0)) {
rc = efx_tx_qdesc_post(txq->common, txq->pend_desc,
pend - &txq->pend_desc[0],
txq->completed, &txq->added);
SFC_ASSERT(rc == 0);
if (likely(pushed != txq->added))
efx_tx_qpush(txq->common, txq->added, pushed);
}
#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
if (!reap_done)
sfc_efx_tx_reap(txq);
#endif
done:
return pkts_sent;
}
