uint16_t vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { uint16_t nb_tx; vmxnet3_tx_queue_t *txq = tx_queue; struct vmxnet3_hw *hw = txq->hw; if (unlikely(txq->stopped)) { PMD_TX_LOG(DEBUG, "Tx queue is stopped."); return 0; } /* Free up the comp_descriptors aggressively */ vmxnet3_tq_tx_complete(txq); nb_tx = 0; while (nb_tx < nb_pkts) { Vmxnet3_GenericDesc *gdesc; vmxnet3_buf_info_t *tbi; uint32_t first2fill, avail, dw2; struct rte_mbuf *txm = tx_pkts[nb_tx]; struct rte_mbuf *m_seg = txm; /* Is this packet execessively fragmented, then drop */ if (unlikely(txm->nb_segs > VMXNET3_MAX_TXD_PER_PKT)) { ++txq->stats.drop_too_many_segs; ++txq->stats.drop_total; rte_pktmbuf_free(txm); ++nb_tx; continue; } /* Is command ring full? */ avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring); if (txm->nb_segs > avail) { ++txq->stats.tx_ring_full; break; } /* use the previous gen bit for the SOP desc */ dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; first2fill = txq->cmd_ring.next2fill; do { /* Remember the transmit buffer for cleanup */ tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill; tbi->m = m_seg; /* NB: the following assumes that VMXNET3 maximum transmit buffer size (16K) is greater than maximum sizeof mbuf segment size. */ gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill; gdesc->txd.addr = rte_mbuf_data_dma_addr(m_seg); gdesc->dword[2] = dw2 | m_seg->data_len; gdesc->dword[3] = 0; /* move to the next2fill descriptor */ vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring); /* use the right gen for non-SOP desc */ dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT; } while ((m_seg = m_seg->next) != NULL); /* Update the EOP descriptor */ gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ; /* Add VLAN tag if present */ gdesc = txq->cmd_ring.base + first2fill; if (txm->ol_flags & PKT_TX_VLAN_PKT) { gdesc->txd.ti = 1; gdesc->txd.tci = txm->vlan_tci; } /* TODO: Add transmit checksum offload here */ /* flip the GEN bit on the SOP */ rte_compiler_barrier(); gdesc->dword[2] ^= VMXNET3_TXD_GEN; txq->shared->ctrl.txNumDeferred++; nb_tx++; } PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", txq->shared->ctrl.txThreshold); if (txq->shared->ctrl.txNumDeferred >= txq->shared->ctrl.txThreshold) { txq->shared->ctrl.txNumDeferred = 0; /* Notify vSwitch that packets are available. */ VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN), txq->cmd_ring.next2fill); } return nb_tx; }
static uint16_t sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) { struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue; struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq); unsigned int added = txq->added; unsigned int pushed = added; unsigned int pkts_sent = 0; efx_desc_t *pend = &txq->pend_desc[0]; const unsigned int hard_max_fill = EFX_TXQ_LIMIT(txq->ptr_mask + 1); const unsigned int soft_max_fill = hard_max_fill - txq->free_thresh; unsigned int fill_level = added - txq->completed; boolean_t reap_done; int rc __rte_unused; struct rte_mbuf **pktp; if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0)) goto done; /* * If insufficient space for a single packet is present, * we should reap; otherwise, we shouldn't do that all the time * to avoid latency increase */ reap_done = (fill_level > soft_max_fill); if (reap_done) { sfc_efx_tx_reap(txq); /* * Recalculate fill level since 'txq->completed' * might have changed on reap */ fill_level = added - txq->completed; } for (pkts_sent = 0, pktp = &tx_pkts[0]; (pkts_sent < nb_pkts) && (fill_level <= soft_max_fill); pkts_sent++, pktp++) { struct rte_mbuf *m_seg = *pktp; size_t pkt_len = m_seg->pkt_len; unsigned int pkt_descs = 0; size_t in_off = 0; /* * Here VLAN TCI is expected to be zero in case if no * DEV_TX_VLAN_OFFLOAD capability is advertised; * if the calling app ignores the absence of * DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then * TX_ERROR will occur */ pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend); if (m_seg->ol_flags & PKT_TX_TCP_SEG) { /* * We expect correct 'pkt->l[2, 3, 4]_len' values * to be set correctly by the caller */ if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend, &pkt_descs, &pkt_len) != 0) { /* We may have reached this place for * one of the following reasons: * * 1) Packet header length is greater * than SFC_TSOH_STD_LEN * 2) TCP header starts at more then * 208 bytes into the frame * * We will deceive RTE saying that we have sent * the packet, but we will actually drop it. * Hence, we should revert 'pend' to the * previous state (in case we have added * VLAN descriptor) and start processing * another one packet. But the original * mbuf shouldn't be orphaned */ pend -= pkt_descs; rte_pktmbuf_free(*pktp); continue; } /* * We've only added 2 FATSOv2 option descriptors * and 1 descriptor for the linearized packet header. * The outstanding work will be done in the same manner * as for the usual non-TSO path */ } for (; m_seg != NULL; m_seg = m_seg->next) { efsys_dma_addr_t next_frag; size_t seg_len; seg_len = m_seg->data_len; next_frag = rte_mbuf_data_dma_addr(m_seg); /* * If we've started TSO transaction few steps earlier, * we'll skip packet header using an offset in the * current segment (which has been set to the * first one containing payload) */ seg_len -= in_off; next_frag += in_off; in_off = 0; do { efsys_dma_addr_t frag_addr = next_frag; size_t frag_len; /* * It is assumed here that there is no * limitation on address boundary * crossing by DMA descriptor. */ frag_len = MIN(seg_len, txq->dma_desc_size_max); next_frag += frag_len; seg_len -= frag_len; pkt_len -= frag_len; efx_tx_qdesc_dma_create(txq->common, frag_addr, frag_len, (pkt_len == 0), pend++); pkt_descs++; } while (seg_len != 0); } added += pkt_descs; fill_level += pkt_descs; if (unlikely(fill_level > hard_max_fill)) { /* * Our estimation for maximum number of descriptors * required to send a packet seems to be wrong. * Try to reap (if we haven't yet). */ if (!reap_done) { sfc_efx_tx_reap(txq); reap_done = B_TRUE; fill_level = added - txq->completed; if (fill_level > hard_max_fill) { pend -= pkt_descs; break; } } else { pend -= pkt_descs; break; } } /* Assign mbuf to the last used desc */ txq->sw_ring[(added - 1) & txq->ptr_mask].mbuf = *pktp; } if (likely(pkts_sent > 0)) { rc = efx_tx_qdesc_post(txq->common, txq->pend_desc, pend - &txq->pend_desc[0], txq->completed, &txq->added); SFC_ASSERT(rc == 0); if (likely(pushed != txq->added)) efx_tx_qpush(txq->common, txq->added, pushed); } #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE if (!reap_done) sfc_efx_tx_reap(txq); #endif done: return pkts_sent; }