/**
* Receive packet from ethernet driver and queueing into worker queue.
* This function is called from I/O (Input) thread.
*/
static inline void
app_lcore_io_rx(
struct app_lcore_params_io *lp,
uint32_t n_workers,
uint32_t bsz_rd,
uint32_t bsz_wr) {
struct rte_mbuf *mbuf_1_0, *mbuf_1_1, *mbuf_2_0, *mbuf_2_1;
uint32_t i, fifoness;
fifoness = app.fifoness;
for (i = 0; i < lp->rx.n_nic_queues; i++) {
uint8_t portid = lp->rx.nic_queues[i].port;
uint8_t queue = lp->rx.nic_queues[i].queue;
uint32_t n_mbufs, j;
if (unlikely(lp->rx.nic_queues[i].enabled != true)) {
continue;
}
n_mbufs = rte_eth_rx_burst(portid,
queue,
lp->rx.mbuf_in.array,
(uint16_t) bsz_rd);
if (unlikely(n_mbufs == 0)) {
continue;
}
#if APP_STATS
lp->rx.nic_queues_iters[i] ++;
lp->rx.nic_queues_count[i] += n_mbufs;
if (unlikely(lp->rx.nic_queues_iters[i] == APP_STATS)) {
struct rte_eth_stats stats;
unsigned lcore = rte_lcore_id();
rte_eth_stats_get(portid, &stats);
printf("I/O RX %u in (NIC port %u): NIC drop ratio = %.2f avg burst size = %.2f\n",
lcore,
(unsigned) portid,
(double) stats.ierrors / (double) (stats.ierrors + stats.ipackets),
((double) lp->rx.nic_queues_count[i]) / ((double)
lp->rx.nic_queues_iters[i]));
lp->rx.nic_queues_iters[i] = 0;
lp->rx.nic_queues_count[i] = 0;
}
#endif
#if APP_IO_RX_DROP_ALL_PACKETS
for (j = 0; j < n_mbufs; j ++) {
struct rte_mbuf *pkt = lp->rx.mbuf_in.array[j];
rte_pktmbuf_free(pkt);
}
continue;
#endif
mbuf_1_0 = lp->rx.mbuf_in.array[0];
mbuf_1_1 = lp->rx.mbuf_in.array[1];
mbuf_2_0 = lp->rx.mbuf_in.array[2];
mbuf_2_1 = lp->rx.mbuf_in.array[3];
APP_IO_RX_PREFETCH0(mbuf_2_0);
APP_IO_RX_PREFETCH0(mbuf_2_1);
for (j = 0; j + 3 < n_mbufs; j += 2) {
struct rte_mbuf *mbuf_0_0, *mbuf_0_1;
uint32_t worker_0, worker_1;
mbuf_0_0 = mbuf_1_0;
mbuf_0_1 = mbuf_1_1;
mbuf_1_0 = mbuf_2_0;
mbuf_1_1 = mbuf_2_1;
mbuf_2_0 = lp->rx.mbuf_in.array[j+4];
mbuf_2_1 = lp->rx.mbuf_in.array[j+5];
APP_IO_RX_PREFETCH0(mbuf_2_0);
APP_IO_RX_PREFETCH0(mbuf_2_1);
switch (fifoness) {
case FIFONESS_FLOW:
#ifdef __SSE4_2__
worker_0 = rte_hash_crc(rte_pktmbuf_mtod(mbuf_0_0, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
worker_1 = rte_hash_crc(rte_pktmbuf_mtod(mbuf_0_1, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
#else
worker_0 = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf_0_0, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
worker_1 = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf_0_1, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
#endif /* __SSE4_2__ */
break;
case FIFONESS_PORT:
worker_0 = worker_1 = portid % n_workers;
break;
case FIFONESS_NONE:
default:
worker_0 = j % n_workers;
worker_1 = (j + 1) % n_workers;
break;
}
app_lcore_io_rx_buffer_to_send(lp, worker_0, mbuf_0_0, bsz_wr);
app_lcore_io_rx_buffer_to_send(lp, worker_1, mbuf_0_1, bsz_wr);
}
/*
* Handle the last 1, 2 (when n_mbufs is even) or
* 3 (when n_mbufs is odd) packets
*/
for ( ; j < n_mbufs; j += 1) {
struct rte_mbuf *mbuf;
uint32_t worker;
mbuf = mbuf_1_0;
mbuf_1_0 = mbuf_1_1;
mbuf_1_1 = mbuf_2_0;
mbuf_2_0 = mbuf_2_1;
APP_IO_RX_PREFETCH0(mbuf_1_0);
switch (fifoness) {
case FIFONESS_FLOW:
#ifdef __SSE4_2__
worker = rte_hash_crc(rte_pktmbuf_mtod(mbuf, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
#else
worker = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf, void *),
sizeof(ETHER_HDR) + 2, portid) % n_workers;
#endif /* __SSE4_2__ */
break;
case FIFONESS_PORT:
worker = portid % n_workers;
break;
case FIFONESS_NONE:
default:
worker = j % n_workers;
break;
}
app_lcore_io_rx_buffer_to_send(lp, worker, mbuf, bsz_wr);
}
}
}
static inline void
app_lcore_io_rx(
struct app_lcore_params_io *lp,
uint32_t n_workers,
uint32_t bsz_rd,
uint32_t bsz_wr,
uint8_t pos_lb)
{
struct rte_mbuf *mbuf_1_0, *mbuf_1_1, *mbuf_2_0, *mbuf_2_1;
uint8_t *data_1_0, *data_1_1 = NULL;
uint32_t i;
for (i = 0; i < lp->rx.n_nic_queues; i ++) {
uint8_t port = lp->rx.nic_queues[i].port;
uint8_t queue = lp->rx.nic_queues[i].queue;
uint32_t n_mbufs, j;
n_mbufs = rte_eth_rx_burst(
port,
queue,
lp->rx.mbuf_in.array,
(uint16_t) bsz_rd);
if (unlikely(n_mbufs == 0)) {
continue;
}
#if APP_STATS
lp->rx.nic_queues_iters[i] ++;
lp->rx.nic_queues_count[i] += n_mbufs;
if (unlikely(lp->rx.nic_queues_iters[i] == APP_STATS)) {
struct rte_eth_stats stats;
unsigned lcore = rte_lcore_id();
rte_eth_stats_get(port, &stats);
printf("I/O RX %u in (NIC port %u): NIC drop ratio = %.2f avg burst size = %.2f\n",
lcore,
(unsigned) port,
(double) stats.imissed / (double) (stats.imissed + stats.ipackets),
((double) lp->rx.nic_queues_count[i]) / ((double) lp->rx.nic_queues_iters[i]));
lp->rx.nic_queues_iters[i] = 0;
lp->rx.nic_queues_count[i] = 0;
}
#endif
#if APP_IO_RX_DROP_ALL_PACKETS
for (j = 0; j < n_mbufs; j ++) {
struct rte_mbuf *pkt = lp->rx.mbuf_in.array[j];
rte_pktmbuf_free(pkt);
}
continue;
#endif
mbuf_1_0 = lp->rx.mbuf_in.array[0];
mbuf_1_1 = lp->rx.mbuf_in.array[1];
data_1_0 = rte_pktmbuf_mtod(mbuf_1_0, uint8_t *);
if (likely(n_mbufs > 1)) {
data_1_1 = rte_pktmbuf_mtod(mbuf_1_1, uint8_t *);
}
mbuf_2_0 = lp->rx.mbuf_in.array[2];
mbuf_2_1 = lp->rx.mbuf_in.array[3];
APP_IO_RX_PREFETCH0(mbuf_2_0);
APP_IO_RX_PREFETCH0(mbuf_2_1);
for (j = 0; j + 3 < n_mbufs; j += 2) {
struct rte_mbuf *mbuf_0_0, *mbuf_0_1;
uint8_t *data_0_0, *data_0_1;
uint32_t worker_0, worker_1;
mbuf_0_0 = mbuf_1_0;
mbuf_0_1 = mbuf_1_1;
data_0_0 = data_1_0;
data_0_1 = data_1_1;
mbuf_1_0 = mbuf_2_0;
mbuf_1_1 = mbuf_2_1;
data_1_0 = rte_pktmbuf_mtod(mbuf_2_0, uint8_t *);
data_1_1 = rte_pktmbuf_mtod(mbuf_2_1, uint8_t *);
APP_IO_RX_PREFETCH0(data_1_0);
APP_IO_RX_PREFETCH0(data_1_1);
mbuf_2_0 = lp->rx.mbuf_in.array[j+4];
mbuf_2_1 = lp->rx.mbuf_in.array[j+5];
APP_IO_RX_PREFETCH0(mbuf_2_0);
APP_IO_RX_PREFETCH0(mbuf_2_1);
worker_0 = data_0_0[pos_lb] & (n_workers - 1);
worker_1 = data_0_1[pos_lb] & (n_workers - 1);
app_lcore_io_rx_buffer_to_send(lp, worker_0, mbuf_0_0, bsz_wr);
app_lcore_io_rx_buffer_to_send(lp, worker_1, mbuf_0_1, bsz_wr);
}
/* Handle the last 1, 2 (when n_mbufs is even) or 3 (when n_mbufs is odd) packets */
for ( ; j < n_mbufs; j += 1) {
struct rte_mbuf *mbuf;
uint8_t *data;
uint32_t worker;
mbuf = mbuf_1_0;
mbuf_1_0 = mbuf_1_1;
mbuf_1_1 = mbuf_2_0;
mbuf_2_0 = mbuf_2_1;
data = rte_pktmbuf_mtod(mbuf, uint8_t *);
APP_IO_RX_PREFETCH0(mbuf_1_0);
worker = data[pos_lb] & (n_workers - 1);
app_lcore_io_rx_buffer_to_send(lp, worker, mbuf, bsz_wr);
}
}
