static int rte_port_ethdev_writer_nodrop_tx_bulk(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask) { struct rte_port_ethdev_writer_nodrop *p = (struct rte_port_ethdev_writer_nodrop *) port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; uint64_t expr = (pkts_mask & (pkts_mask + 1)) | ((pkts_mask & bsz_mask) ^ bsz_mask); if (expr == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); uint32_t n_pkts_ok; if (tx_buf_count) send_burst_nodrop(p); RTE_PORT_ETHDEV_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); n_pkts_ok = rte_eth_tx_burst(p->port_id, p->queue_id, pkts, n_pkts); if (n_pkts_ok >= n_pkts) return 0; /* * If we didnt manage to send all packets in single burst, move * remaining packets to the buffer and call send burst. */ for (; n_pkts_ok < n_pkts; n_pkts_ok++) { struct rte_mbuf *pkt = pkts[n_pkts_ok]; p->tx_buf[p->tx_buf_count++] = pkt; } send_burst_nodrop(p); } else { for ( ; pkts_mask; ) { uint32_t pkt_index = __builtin_ctzll(pkts_mask); uint64_t pkt_mask = 1LLU << pkt_index; struct rte_mbuf *pkt = pkts[pkt_index]; p->tx_buf[tx_buf_count++] = pkt; RTE_PORT_ETHDEV_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); pkts_mask &= ~pkt_mask; } p->tx_buf_count = tx_buf_count; if (tx_buf_count >= p->tx_burst_sz) send_burst_nodrop(p); } return 0; }
static int rte_port_fd_writer_nodrop_tx_bulk(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask) { struct rte_port_fd_writer_nodrop *p = port; uint32_t tx_buf_count = p->tx_buf_count; if ((pkts_mask & (pkts_mask + 1)) == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); uint32_t i; for (i = 0; i < n_pkts; i++) p->tx_buf[tx_buf_count++] = pkts[i]; RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); } else for ( ; pkts_mask; ) { uint32_t pkt_index = __builtin_ctzll(pkts_mask); uint64_t pkt_mask = 1LLU << pkt_index; struct rte_mbuf *pkt = pkts[pkt_index]; p->tx_buf[tx_buf_count++] = pkt; RTE_PORT_FD_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); pkts_mask &= ~pkt_mask; } p->tx_buf_count = tx_buf_count; if (tx_buf_count >= p->tx_burst_sz) send_burst_nodrop(p); return 0; }
static int rte_port_ring_writer_nodrop_flush(void *port) { struct rte_port_ring_writer_nodrop *p = (struct rte_port_ring_writer_nodrop *) port; if (p->tx_buf_count > 0) send_burst_nodrop(p); return 0; }
static int rte_port_ring_writer_nodrop_tx(void *port, struct rte_mbuf *pkt) { struct rte_port_ring_writer_nodrop *p = (struct rte_port_ring_writer_nodrop *) port; p->tx_buf[p->tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); if (p->tx_buf_count >= p->tx_burst_sz) send_burst_nodrop(p); return 0; }
rte_port_ring_writer_nodrop_tx_bulk_internal(void *port, struct rte_mbuf **pkts, uint64_t pkts_mask, uint32_t is_multi) { struct rte_port_ring_writer_nodrop *p = (struct rte_port_ring_writer_nodrop *) port; uint64_t bsz_mask = p->bsz_mask; uint32_t tx_buf_count = p->tx_buf_count; uint64_t expr = (pkts_mask & (pkts_mask + 1)) | ((pkts_mask & bsz_mask) ^ bsz_mask); if (expr == 0) { uint64_t n_pkts = __builtin_popcountll(pkts_mask); uint32_t n_pkts_ok; if (tx_buf_count) { if (is_multi) send_burst_mp_nodrop(p); else send_burst_nodrop(p); } RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, n_pkts); if (is_multi) n_pkts_ok = rte_ring_mp_enqueue_burst(p->ring, (void **)pkts, n_pkts); else n_pkts_ok = rte_ring_sp_enqueue_burst(p->ring, (void **)pkts, n_pkts); if (n_pkts_ok >= n_pkts) return 0; /* * If we didn't manage to send all packets in single burst, move * remaining packets to the buffer and call send burst. */ for (; n_pkts_ok < n_pkts; n_pkts_ok++) { struct rte_mbuf *pkt = pkts[n_pkts_ok]; p->tx_buf[p->tx_buf_count++] = pkt; } if (is_multi) send_burst_mp_nodrop(p); else send_burst_nodrop(p); } else { for ( ; pkts_mask; ) { uint32_t pkt_index = __builtin_ctzll(pkts_mask); uint64_t pkt_mask = 1LLU << pkt_index; struct rte_mbuf *pkt = pkts[pkt_index]; p->tx_buf[tx_buf_count++] = pkt; RTE_PORT_RING_WRITER_NODROP_STATS_PKTS_IN_ADD(p, 1); pkts_mask &= ~pkt_mask; } p->tx_buf_count = tx_buf_count; if (tx_buf_count >= p->tx_burst_sz) { if (is_multi) send_burst_mp_nodrop(p); else send_burst_nodrop(p); } } return 0; }