static uint16_t
sfc_efx_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct sfc_dp_txq *dp_txq = (struct sfc_dp_txq *)tx_queue;
struct sfc_efx_txq *txq = sfc_efx_txq_by_dp_txq(dp_txq);
unsigned int added = txq->added;
unsigned int pushed = added;
unsigned int pkts_sent = 0;
efx_desc_t *pend = &txq->pend_desc[0];
const unsigned int hard_max_fill = EFX_TXQ_LIMIT(txq->ptr_mask + 1);
const unsigned int soft_max_fill = hard_max_fill - txq->free_thresh;
unsigned int fill_level = added - txq->completed;
boolean_t reap_done;
int rc __rte_unused;
struct rte_mbuf **pktp;
if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_RUNNING) == 0))
goto done;
/*
* If insufficient space for a single packet is present,
* we should reap; otherwise, we shouldn't do that all the time
* to avoid latency increase
*/
reap_done = (fill_level > soft_max_fill);
if (reap_done) {
sfc_efx_tx_reap(txq);
/*
* Recalculate fill level since 'txq->completed'
* might have changed on reap
*/
fill_level = added - txq->completed;
}
for (pkts_sent = 0, pktp = &tx_pkts[0];
(pkts_sent < nb_pkts) && (fill_level <= soft_max_fill);
pkts_sent++, pktp++) {
struct rte_mbuf *m_seg = *pktp;
size_t pkt_len = m_seg->pkt_len;
unsigned int pkt_descs = 0;
size_t in_off = 0;
/*
* Here VLAN TCI is expected to be zero in case if no
* DEV_TX_VLAN_OFFLOAD capability is advertised;
* if the calling app ignores the absence of
* DEV_TX_VLAN_OFFLOAD and pushes VLAN TCI, then
* TX_ERROR will occur
*/
pkt_descs += sfc_efx_tx_maybe_insert_tag(txq, m_seg, &pend);
if (m_seg->ol_flags & PKT_TX_TCP_SEG) {
/*
* We expect correct 'pkt->l[2, 3, 4]_len' values
* to be set correctly by the caller
*/
if (sfc_efx_tso_do(txq, added, &m_seg, &in_off, &pend,
&pkt_descs, &pkt_len) != 0) {
/* We may have reached this place for
* one of the following reasons:
*
* 1) Packet header length is greater
* than SFC_TSOH_STD_LEN
* 2) TCP header starts at more then
* 208 bytes into the frame
*
* We will deceive RTE saying that we have sent
* the packet, but we will actually drop it.
* Hence, we should revert 'pend' to the
* previous state (in case we have added
* VLAN descriptor) and start processing
* another one packet. But the original
* mbuf shouldn't be orphaned
*/
pend -= pkt_descs;
rte_pktmbuf_free(*pktp);
continue;
}
/*
* We've only added 2 FATSOv2 option descriptors
* and 1 descriptor for the linearized packet header.
* The outstanding work will be done in the same manner
* as for the usual non-TSO path
*/
}
for (; m_seg != NULL; m_seg = m_seg->next) {
efsys_dma_addr_t next_frag;
size_t seg_len;
seg_len = m_seg->data_len;
next_frag = rte_mbuf_data_dma_addr(m_seg);
/*
* If we've started TSO transaction few steps earlier,
* we'll skip packet header using an offset in the
* current segment (which has been set to the
* first one containing payload)
*/
seg_len -= in_off;
next_frag += in_off;
in_off = 0;
do {
efsys_dma_addr_t frag_addr = next_frag;
size_t frag_len;
/*
* It is assumed here that there is no
* limitation on address boundary
* crossing by DMA descriptor.
*/
frag_len = MIN(seg_len, txq->dma_desc_size_max);
next_frag += frag_len;
seg_len -= frag_len;
pkt_len -= frag_len;
efx_tx_qdesc_dma_create(txq->common,
frag_addr, frag_len,
(pkt_len == 0),
pend++);
pkt_descs++;
} while (seg_len != 0);
}
added += pkt_descs;
fill_level += pkt_descs;
if (unlikely(fill_level > hard_max_fill)) {
/*
* Our estimation for maximum number of descriptors
* required to send a packet seems to be wrong.
* Try to reap (if we haven't yet).
*/
if (!reap_done) {
sfc_efx_tx_reap(txq);
reap_done = B_TRUE;
fill_level = added - txq->completed;
if (fill_level > hard_max_fill) {
pend -= pkt_descs;
break;
}
} else {
pend -= pkt_descs;
break;
}
}
/* Assign mbuf to the last used desc */
txq->sw_ring[(added - 1) & txq->ptr_mask].mbuf = *pktp;
}
if (likely(pkts_sent > 0)) {
rc = efx_tx_qdesc_post(txq->common, txq->pend_desc,
pend - &txq->pend_desc[0],
txq->completed, &txq->added);
SFC_ASSERT(rc == 0);
if (likely(pushed != txq->added))
efx_tx_qpush(txq->common, txq->added, pushed);
}
#if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE
if (!reap_done)
sfc_efx_tx_reap(txq);
#endif
done:
return pkts_sent;
}
int
sfc_efx_tso_do(struct sfc_efx_txq *txq, unsigned int idx,
struct rte_mbuf **in_seg, size_t *in_off, efx_desc_t **pend,
unsigned int *pkt_descs, size_t *pkt_len)
{
uint8_t *tsoh;
const struct tcp_hdr *th;
efsys_dma_addr_t header_paddr;
uint16_t packet_id;
uint32_t sent_seq;
struct rte_mbuf *m = *in_seg;
size_t nh_off = m->l2_len; /* IP header offset */
size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */
size_t header_len = m->l2_len + m->l3_len + m->l4_len;
const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic);
idx += SFC_TSO_OPT_DESCS_NUM;
/* Packets which have too big headers should be discarded */
if (unlikely(header_len > SFC_TSOH_STD_LEN))
return EMSGSIZE;
/*
* The TCP header must start at most 208 bytes into the frame.
* If it starts later than this then the NIC won't realise
* it's a TCP packet and TSO edits won't be applied
*/
if (unlikely(tcph_off > encp->enc_tx_tso_tcp_header_offset_limit))
return EMSGSIZE;
header_paddr = rte_pktmbuf_iova(m);
/*
* Sometimes headers may be split across multiple mbufs. In such cases
* we need to glue those pieces and store them in some temporary place.
* Also, packet headers must be contiguous in memory, so that
* they can be referred to with a single DMA descriptor. EF10 has no
* limitations on address boundaries crossing by DMA descriptor data.
*/
if (m->data_len < header_len) {
tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
sfc_tso_prepare_header(tsoh, header_len, in_seg, in_off);
header_paddr = rte_malloc_virt2iova((void *)tsoh);
} else {
if (m->data_len == header_len) {
*in_off = 0;
*in_seg = m->next;
} else {
*in_off = header_len;
}
tsoh = rte_pktmbuf_mtod(m, uint8_t *);
}
/* Handle IP header */
if (m->ol_flags & PKT_TX_IPV4) {
const struct ipv4_hdr *iphe4;
iphe4 = (const struct ipv4_hdr *)(tsoh + nh_off);
rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t));
packet_id = rte_be_to_cpu_16(packet_id);
} else if (m->ol_flags & PKT_TX_IPV6) {
packet_id = 0;
} else {
return EINVAL;
}
/* Handle TCP header */
th = (const struct tcp_hdr *)(tsoh + tcph_off);
rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
sent_seq = rte_be_to_cpu_32(sent_seq);
efx_tx_qdesc_tso2_create(txq->common, packet_id, 0, sent_seq,
m->tso_segsz,
*pend, EFX_TX_FATSOV2_OPT_NDESCS);
*pend += EFX_TX_FATSOV2_OPT_NDESCS;
*pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS;
efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len,
B_FALSE, (*pend)++);
(*pkt_descs)++;
*pkt_len -= header_len;
return 0;
}
