/** * Receive packet from ethernet driver and queueing into worker queue. * This function is called from I/O (Input) thread. */ static inline void app_lcore_io_rx( struct app_lcore_params_io *lp, uint32_t n_workers, uint32_t bsz_rd, uint32_t bsz_wr) { struct rte_mbuf *mbuf_1_0, *mbuf_1_1, *mbuf_2_0, *mbuf_2_1; uint32_t i, fifoness; fifoness = app.fifoness; for (i = 0; i < lp->rx.n_nic_queues; i++) { uint8_t portid = lp->rx.nic_queues[i].port; uint8_t queue = lp->rx.nic_queues[i].queue; uint32_t n_mbufs, j; if (unlikely(lp->rx.nic_queues[i].enabled != true)) { continue; } n_mbufs = rte_eth_rx_burst(portid, queue, lp->rx.mbuf_in.array, (uint16_t) bsz_rd); if (unlikely(n_mbufs == 0)) { continue; } #if APP_STATS lp->rx.nic_queues_iters[i] ++; lp->rx.nic_queues_count[i] += n_mbufs; if (unlikely(lp->rx.nic_queues_iters[i] == APP_STATS)) { struct rte_eth_stats stats; unsigned lcore = rte_lcore_id(); rte_eth_stats_get(portid, &stats); printf("I/O RX %u in (NIC port %u): NIC drop ratio = %.2f avg burst size = %.2f\n", lcore, (unsigned) portid, (double) stats.ierrors / (double) (stats.ierrors + stats.ipackets), ((double) lp->rx.nic_queues_count[i]) / ((double) lp->rx.nic_queues_iters[i])); lp->rx.nic_queues_iters[i] = 0; lp->rx.nic_queues_count[i] = 0; } #endif #if APP_IO_RX_DROP_ALL_PACKETS for (j = 0; j < n_mbufs; j ++) { struct rte_mbuf *pkt = lp->rx.mbuf_in.array[j]; rte_pktmbuf_free(pkt); } continue; #endif mbuf_1_0 = lp->rx.mbuf_in.array[0]; mbuf_1_1 = lp->rx.mbuf_in.array[1]; mbuf_2_0 = lp->rx.mbuf_in.array[2]; mbuf_2_1 = lp->rx.mbuf_in.array[3]; APP_IO_RX_PREFETCH0(mbuf_2_0); APP_IO_RX_PREFETCH0(mbuf_2_1); for (j = 0; j + 3 < n_mbufs; j += 2) { struct rte_mbuf *mbuf_0_0, *mbuf_0_1; uint32_t worker_0, worker_1; mbuf_0_0 = mbuf_1_0; mbuf_0_1 = mbuf_1_1; mbuf_1_0 = mbuf_2_0; mbuf_1_1 = mbuf_2_1; mbuf_2_0 = lp->rx.mbuf_in.array[j+4]; mbuf_2_1 = lp->rx.mbuf_in.array[j+5]; APP_IO_RX_PREFETCH0(mbuf_2_0); APP_IO_RX_PREFETCH0(mbuf_2_1); switch (fifoness) { case FIFONESS_FLOW: #ifdef __SSE4_2__ worker_0 = rte_hash_crc(rte_pktmbuf_mtod(mbuf_0_0, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; worker_1 = rte_hash_crc(rte_pktmbuf_mtod(mbuf_0_1, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; #else worker_0 = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf_0_0, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; worker_1 = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf_0_1, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; #endif /* __SSE4_2__ */ break; case FIFONESS_PORT: worker_0 = worker_1 = portid % n_workers; break; case FIFONESS_NONE: default: worker_0 = j % n_workers; worker_1 = (j + 1) % n_workers; break; } app_lcore_io_rx_buffer_to_send(lp, worker_0, mbuf_0_0, bsz_wr); app_lcore_io_rx_buffer_to_send(lp, worker_1, mbuf_0_1, bsz_wr); } /* * Handle the last 1, 2 (when n_mbufs is even) or * 3 (when n_mbufs is odd) packets */ for ( ; j < n_mbufs; j += 1) { struct rte_mbuf *mbuf; uint32_t worker; mbuf = mbuf_1_0; mbuf_1_0 = mbuf_1_1; mbuf_1_1 = mbuf_2_0; mbuf_2_0 = mbuf_2_1; APP_IO_RX_PREFETCH0(mbuf_1_0); switch (fifoness) { case FIFONESS_FLOW: #ifdef __SSE4_2__ worker = rte_hash_crc(rte_pktmbuf_mtod(mbuf, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; #else worker = CityHash64WithSeed(rte_pktmbuf_mtod(mbuf, void *), sizeof(ETHER_HDR) + 2, portid) % n_workers; #endif /* __SSE4_2__ */ break; case FIFONESS_PORT: worker = portid % n_workers; break; case FIFONESS_NONE: default: worker = j % n_workers; break; } app_lcore_io_rx_buffer_to_send(lp, worker, mbuf, bsz_wr); } } }
static inline void app_lcore_io_rx( struct app_lcore_params_io *lp, uint32_t n_workers, uint32_t bsz_rd, uint32_t bsz_wr, uint8_t pos_lb) { struct rte_mbuf *mbuf_1_0, *mbuf_1_1, *mbuf_2_0, *mbuf_2_1; uint8_t *data_1_0, *data_1_1 = NULL; uint32_t i; for (i = 0; i < lp->rx.n_nic_queues; i ++) { uint8_t port = lp->rx.nic_queues[i].port; uint8_t queue = lp->rx.nic_queues[i].queue; uint32_t n_mbufs, j; n_mbufs = rte_eth_rx_burst( port, queue, lp->rx.mbuf_in.array, (uint16_t) bsz_rd); if (unlikely(n_mbufs == 0)) { continue; } #if APP_STATS lp->rx.nic_queues_iters[i] ++; lp->rx.nic_queues_count[i] += n_mbufs; if (unlikely(lp->rx.nic_queues_iters[i] == APP_STATS)) { struct rte_eth_stats stats; unsigned lcore = rte_lcore_id(); rte_eth_stats_get(port, &stats); printf("I/O RX %u in (NIC port %u): NIC drop ratio = %.2f avg burst size = %.2f\n", lcore, (unsigned) port, (double) stats.imissed / (double) (stats.imissed + stats.ipackets), ((double) lp->rx.nic_queues_count[i]) / ((double) lp->rx.nic_queues_iters[i])); lp->rx.nic_queues_iters[i] = 0; lp->rx.nic_queues_count[i] = 0; } #endif #if APP_IO_RX_DROP_ALL_PACKETS for (j = 0; j < n_mbufs; j ++) { struct rte_mbuf *pkt = lp->rx.mbuf_in.array[j]; rte_pktmbuf_free(pkt); } continue; #endif mbuf_1_0 = lp->rx.mbuf_in.array[0]; mbuf_1_1 = lp->rx.mbuf_in.array[1]; data_1_0 = rte_pktmbuf_mtod(mbuf_1_0, uint8_t *); if (likely(n_mbufs > 1)) { data_1_1 = rte_pktmbuf_mtod(mbuf_1_1, uint8_t *); } mbuf_2_0 = lp->rx.mbuf_in.array[2]; mbuf_2_1 = lp->rx.mbuf_in.array[3]; APP_IO_RX_PREFETCH0(mbuf_2_0); APP_IO_RX_PREFETCH0(mbuf_2_1); for (j = 0; j + 3 < n_mbufs; j += 2) { struct rte_mbuf *mbuf_0_0, *mbuf_0_1; uint8_t *data_0_0, *data_0_1; uint32_t worker_0, worker_1; mbuf_0_0 = mbuf_1_0; mbuf_0_1 = mbuf_1_1; data_0_0 = data_1_0; data_0_1 = data_1_1; mbuf_1_0 = mbuf_2_0; mbuf_1_1 = mbuf_2_1; data_1_0 = rte_pktmbuf_mtod(mbuf_2_0, uint8_t *); data_1_1 = rte_pktmbuf_mtod(mbuf_2_1, uint8_t *); APP_IO_RX_PREFETCH0(data_1_0); APP_IO_RX_PREFETCH0(data_1_1); mbuf_2_0 = lp->rx.mbuf_in.array[j+4]; mbuf_2_1 = lp->rx.mbuf_in.array[j+5]; APP_IO_RX_PREFETCH0(mbuf_2_0); APP_IO_RX_PREFETCH0(mbuf_2_1); worker_0 = data_0_0[pos_lb] & (n_workers - 1); worker_1 = data_0_1[pos_lb] & (n_workers - 1); app_lcore_io_rx_buffer_to_send(lp, worker_0, mbuf_0_0, bsz_wr); app_lcore_io_rx_buffer_to_send(lp, worker_1, mbuf_0_1, bsz_wr); } /* Handle the last 1, 2 (when n_mbufs is even) or 3 (when n_mbufs is odd) packets */ for ( ; j < n_mbufs; j += 1) { struct rte_mbuf *mbuf; uint8_t *data; uint32_t worker; mbuf = mbuf_1_0; mbuf_1_0 = mbuf_1_1; mbuf_1_1 = mbuf_2_0; mbuf_2_0 = mbuf_2_1; data = rte_pktmbuf_mtod(mbuf, uint8_t *); APP_IO_RX_PREFETCH0(mbuf_1_0); worker = data[pos_lb] & (n_workers - 1); app_lcore_io_rx_buffer_to_send(lp, worker, mbuf, bsz_wr); } }