static int
rte_port_ethdev_writer_nodrop_tx_bulk(void *port,
struct rte_mbuf **pkts,
uint64_t pkts_mask)
{
struct rte_port_ethdev_writer_nodrop *p =
(struct rte_port_ethdev_writer_nodrop *) port;
uint64_t bsz_mask = p->bsz_mask;
uint32_t tx_buf_count = p->tx_buf_count;
uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
((pkts_mask & bsz_mask) ^ bsz_mask);
if (expr == 0) {
uint64_t n_pkts = __builtin_popcountll(pkts_mask);
uint32_t n_pkts_ok;
if (tx_buf_count)
send_burst_nodrop(p);
RTE_PORT_ETHDEV_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
n_pkts_ok = rte_eth_tx_burst(p->port_id, p->queue_id, pkts,
n_pkts);
if (n_pkts_ok >= n_pkts)
return 0;
/*
* If we didnt manage to send all packets in single burst, move
* remaining packets to the buffer and call send burst.
*/
for (; n_pkts_ok < n_pkts; n_pkts_ok++) {
struct rte_mbuf *pkt = pkts[n_pkts_ok];
p->tx_buf[p->tx_buf_count++] = pkt;
}
send_burst_nodrop(p);
} else {
for ( ; pkts_mask; ) {
uint32_t pkt_index = __builtin_ctzll(pkts_mask);
uint64_t pkt_mask = 1LLU << pkt_index;
struct rte_mbuf *pkt = pkts[pkt_index];
p->tx_buf[tx_buf_count++] = pkt;
RTE_PORT_ETHDEV_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
pkts_mask &= ~pkt_mask;
}
p->tx_buf_count = tx_buf_count;
if (tx_buf_count >= p->tx_burst_sz)
send_burst_nodrop(p);
}
return 0;
}
static int
rte_port_fd_writer_nodrop_tx_bulk(void *port,
struct rte_mbuf **pkts,
uint64_t pkts_mask)
{
struct rte_port_fd_writer_nodrop *p =
port;
uint32_t tx_buf_count = p->tx_buf_count;
if ((pkts_mask & (pkts_mask + 1)) == 0) {
uint64_t n_pkts = __builtin_popcountll(pkts_mask);
uint32_t i;
for (i = 0; i < n_pkts; i++)
p->tx_buf[tx_buf_count++] = pkts[i];
RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
} else
for ( ; pkts_mask; ) {
uint32_t pkt_index = __builtin_ctzll(pkts_mask);
uint64_t pkt_mask = 1LLU << pkt_index;
struct rte_mbuf *pkt = pkts[pkt_index];
p->tx_buf[tx_buf_count++] = pkt;
RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
pkts_mask &= ~pkt_mask;
}
p->tx_buf_count = tx_buf_count;
if (tx_buf_count >= p->tx_burst_sz)
send_burst_nodrop(p);
return 0;
}
static int
rte_port_ring_writer_nodrop_flush(void *port)
{
struct rte_port_ring_writer_nodrop *p =
(struct rte_port_ring_writer_nodrop *) port;
if (p->tx_buf_count > 0)
send_burst_nodrop(p);
return 0;
}
static int
rte_port_ring_writer_nodrop_tx(void *port, struct rte_mbuf *pkt)
{
struct rte_port_ring_writer_nodrop *p =
(struct rte_port_ring_writer_nodrop *) port;
p->tx_buf[p->tx_buf_count++] = pkt;
RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
if (p->tx_buf_count >= p->tx_burst_sz)
send_burst_nodrop(p);
return 0;
}
rte_port_ring_writer_nodrop_tx_bulk_internal(void *port,
struct rte_mbuf **pkts,
uint64_t pkts_mask,
uint32_t is_multi)
{
struct rte_port_ring_writer_nodrop *p =
(struct rte_port_ring_writer_nodrop *) port;
uint64_t bsz_mask = p->bsz_mask;
uint32_t tx_buf_count = p->tx_buf_count;
uint64_t expr = (pkts_mask & (pkts_mask + 1)) |
((pkts_mask & bsz_mask) ^ bsz_mask);
if (expr == 0) {
uint64_t n_pkts = __builtin_popcountll(pkts_mask);
uint32_t n_pkts_ok;
if (tx_buf_count) {
if (is_multi)
send_burst_mp_nodrop(p);
else
send_burst_nodrop(p);
}
RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts);
if (is_multi)
n_pkts_ok =
rte_ring_mp_enqueue_burst(p->ring, (void **)pkts, n_pkts);
else
n_pkts_ok =
rte_ring_sp_enqueue_burst(p->ring, (void **)pkts, n_pkts);
if (n_pkts_ok >= n_pkts)
return 0;
/*
* If we didn't manage to send all packets in single burst, move
* remaining packets to the buffer and call send burst.
*/
for (; n_pkts_ok < n_pkts; n_pkts_ok++) {
struct rte_mbuf *pkt = pkts[n_pkts_ok];
p->tx_buf[p->tx_buf_count++] = pkt;
}
if (is_multi)
send_burst_mp_nodrop(p);
else
send_burst_nodrop(p);
} else {
for ( ; pkts_mask; ) {
uint32_t pkt_index = __builtin_ctzll(pkts_mask);
uint64_t pkt_mask = 1LLU << pkt_index;
struct rte_mbuf *pkt = pkts[pkt_index];
p->tx_buf[tx_buf_count++] = pkt;
RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1);
pkts_mask &= ~pkt_mask;
}
p->tx_buf_count = tx_buf_count;
if (tx_buf_count >= p->tx_burst_sz) {
if (is_multi)
send_burst_mp_nodrop(p);
else
send_burst_nodrop(p);
}
}
return 0;
}
