/*
* This function is a NUMA-aware equivalent of calc_num_pages.
* It takes in the list of hugepage sizes and the
* number of pages thereof, and calculates the best number of
* pages of each size to fulfill the request for <memory> ram
*/
static int
calc_num_pages_per_socket(uint64_t * memory,
struct hugepage_info *hp_info,
struct hugepage_info *hp_used,
unsigned num_hp_info)
{
unsigned socket, j, i = 0;
unsigned requested, available;
int total_num_pages = 0;
uint64_t remaining_mem, cur_mem;
uint64_t total_mem = internal_config.memory;
if (num_hp_info == 0)
return -1;
/* if specific memory amounts per socket weren't requested */
if (internal_config.force_sockets == 0) {
int cpu_per_socket[RTE_MAX_NUMA_NODES];
size_t default_size, total_size;
unsigned lcore_id;
/* Compute number of cores per socket */
memset(cpu_per_socket, 0, sizeof(cpu_per_socket));
RTE_LCORE_FOREACH(lcore_id) {
cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;
}
/*
* Automatically spread requested memory amongst detected sockets according
* to number of cores from cpu mask present on each socket
*/
total_size = internal_config.memory;
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
/* Set memory amount per socket */
default_size = (internal_config.memory * cpu_per_socket[socket])
/ rte_lcore_count();
/* Limit to maximum available memory on socket */
default_size = RTE_MIN(default_size, get_socket_mem_size(socket));
/* Update sizes */
memory[socket] = default_size;
total_size -= default_size;
}
/*
* If some memory is remaining, try to allocate it by getting all
* available memory from sockets, one after the other
*/
for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_size != 0; socket++) {
/* take whatever is available */
default_size = RTE_MIN(get_socket_mem_size(socket) - memory[socket],
total_size);
/* Update sizes */
memory[socket] += default_size;
total_size -= default_size;
}
}
static int
usiw_query_device(struct ibv_context *context,
struct ibv_device_attr *device_attr)
{
struct usiw_context *ourctx;
struct ibv_query_device cmd;
__attribute__((unused)) uint64_t raw_fw_ver;
int ret;
if (!context || !device_attr) {
return -EINVAL;
}
ret = ibv_cmd_query_device(context, device_attr,
&raw_fw_ver, &cmd, sizeof(cmd));
if (ret) {
return ret;
}
ourctx = usiw_get_context(context);
strncpy(device_attr->fw_ver, PACKAGE_VERSION,
sizeof(device_attr->fw_ver));
device_attr->max_mr_size = MAX_MR_SIZE;
device_attr->page_size_cap = 4096;
device_attr->vendor_id = URDMA_VENDOR_ID;
device_attr->vendor_part_id = URDMA_VENDOR_PART_ID;
device_attr->hw_ver = 0;
device_attr->max_qp = ourctx->dev->max_qp;
device_attr->max_qp_wr = RTE_MIN(MAX_SEND_WR, MAX_RECV_WR);
device_attr->device_cap_flags = 0;
device_attr->max_sge = DPDK_VERBS_IOV_LEN_MAX;
device_attr->max_sge_rd = DPDK_VERBS_RDMA_READ_IOV_LEN_MAX;
device_attr->max_cq = INT_MAX;
device_attr->max_cqe = RTE_MIN(INT_MAX, SIZE_POW2_MAX);
device_attr->max_mr = INT_MAX;
device_attr->max_pd = INT_MAX;
device_attr->max_qp_rd_atom = USIW_ORD_MAX;
device_attr->max_ee_rd_atom = 1;
device_attr->max_res_rd_atom = USIW_ORD_MAX;
device_attr->max_qp_init_rd_atom = USIW_IRD_MAX;
device_attr->max_ee_init_rd_atom = USIW_IRD_MAX;
device_attr->atomic_cap = IBV_ATOMIC_NONE;
device_attr->max_ee = 0;
device_attr->max_rdd = 0;
device_attr->max_mw = 0;
device_attr->max_raw_ipv6_qp = 0;
device_attr->max_raw_ethy_qp = 0;
device_attr->max_mcast_grp = 0;
device_attr->max_mcast_qp_attach = 0;
device_attr->max_total_mcast_qp_attach = 0;
device_attr->max_ah = 0;
device_attr->max_fmr = 0;
device_attr->max_srq = 0;
device_attr->max_pkeys = 0;
device_attr->local_ca_ack_delay = 0;
device_attr->phys_port_cnt = 1;
return 0;
} /* usiw_query_device */
static int
avf_init_rss(struct avf_adapter *adapter)
{
struct avf_info *vf = AVF_DEV_PRIVATE_TO_VF(adapter);
struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(adapter);
struct rte_eth_rss_conf *rss_conf;
uint8_t i, j, nb_q;
int ret;
rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues,
AVF_MAX_NUM_QUEUES);
if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) {
PMD_DRV_LOG(DEBUG, "RSS is not supported");
return -ENOTSUP;
}
if (adapter->eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default");
/* set all lut items to default queue */
for (i = 0; i < vf->vf_res->rss_lut_size; i++)
vf->rss_lut[i] = 0;
ret = avf_configure_rss_lut(adapter);
return ret;
}
/* In AVF, RSS enablement is set by PF driver. It is not supported
* to set based on rss_conf->rss_hf.
*/
/* configure RSS key */
if (!rss_conf->rss_key) {
/* Calculate the default hash key */
for (i = 0; i <= vf->vf_res->rss_key_size; i++)
vf->rss_key[i] = (uint8_t)rte_rand();
} else
rte_memcpy(vf->rss_key, rss_conf->rss_key,
RTE_MIN(rss_conf->rss_key_len,
vf->vf_res->rss_key_size));
/* init RSS LUT table */
for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) {
if (j >= nb_q)
j = 0;
vf->rss_lut[i] = j;
}
/* send virtchnnl ops to configure rss*/
ret = avf_configure_rss_lut(adapter);
if (ret)
return ret;
ret = avf_configure_rss_key(adapter);
if (ret)
return ret;
return 0;
}
/* benchmark alloc-build-free of ops */
static inline int
pmd_cyclecount_bench_ops(struct pmd_cyclecount_state *state, uint32_t cur_op,
uint16_t test_burst_size)
{
uint32_t iter_ops_left = state->opts->total_ops - cur_op;
uint32_t iter_ops_needed =
RTE_MIN(state->opts->nb_descriptors, iter_ops_left);
uint32_t cur_iter_op;
uint32_t imix_idx = 0;
for (cur_iter_op = 0; cur_iter_op < iter_ops_needed;
cur_iter_op += test_burst_size) {
uint32_t burst_size = RTE_MIN(state->opts->total_ops - cur_op,
test_burst_size);
struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op];
/* Allocate objects containing crypto operations and mbufs */
if (rte_mempool_get_bulk(state->ctx->pool, (void **)ops,
burst_size) != 0) {
RTE_LOG(ERR, USER1,
"Failed to allocate more crypto operations "
"from the crypto operation pool.\n"
"Consider increasing the pool size "
"with --pool-sz\n");
return -1;
}
/* Setup crypto op, attach mbuf etc */
(state->ctx->populate_ops)(ops,
state->ctx->src_buf_offset,
state->ctx->dst_buf_offset,
burst_size,
state->ctx->sess, state->opts,
state->ctx->test_vector, iv_offset,
&imix_idx);
#ifdef CPERF_LINEARIZATION_ENABLE
/* Check if source mbufs require coalescing */
if (state->linearize) {
uint8_t i;
for (i = 0; i < burst_size; i++) {
struct rte_mbuf *src = ops[i]->sym->m_src;
rte_pktmbuf_linearize(src);
}
}
#endif /* CPERF_LINEARIZATION_ENABLE */
rte_mempool_put_bulk(state->ctx->pool, (void **)ops,
burst_size);
}
return 0;
}
/**
* A call to tx_sync_ring will try to empty a Netmap TX ring by converting its
* buffers into rte_mbufs and sending them out on the rings's dpdk port.
*/
static int
tx_sync_ring(struct netmap_ring *ring, uint8_t port, uint16_t ring_number,
struct rte_mempool *pool, uint16_t max_burst)
{
uint32_t i, n_tx;
uint16_t burst_size;
uint32_t cur_slot, n_used_slots;
struct rte_mbuf *tx_mbufs[COMPAT_NETMAP_MAX_BURST];
n_used_slots = ring->num_slots - ring->avail;
n_used_slots = RTE_MIN(n_used_slots, max_burst);
cur_slot = (ring->cur + ring->avail) & (ring->num_slots - 1);
while (n_used_slots) {
burst_size = (uint16_t)RTE_MIN(n_used_slots, RTE_DIM(tx_mbufs));
for (i = 0; i < burst_size; i++) {
tx_mbufs[i] = rte_pktmbuf_alloc(pool);
if (tx_mbufs[i] == NULL)
goto err;
slot_to_mbuf(ring, cur_slot, tx_mbufs[i]);
cur_slot = NETMAP_RING_NEXT(ring, cur_slot);
}
n_tx = rte_eth_tx_burst(port, ring_number, tx_mbufs,
burst_size);
/* Update the Netmap ring structure to reflect the change */
ring->avail += n_tx;
n_used_slots -= n_tx;
/* Return the mbufs that failed to transmit to their pool */
if (unlikely(n_tx != burst_size)) {
for (i = n_tx; i < burst_size; i++)
rte_pktmbuf_free(tx_mbufs[i]);
break;
}
}
return 0;
err:
for (; i == 0; --i)
rte_pktmbuf_free(tx_mbufs[i]);
RTE_LOG(ERR, USER1,
"Couldn't get mbuf from mempool is the mempool too small?\n");
return -1;
}
/* allocate and build ops (no free) */
static int
pmd_cyclecount_build_ops(struct pmd_cyclecount_state *state,
uint32_t iter_ops_needed, uint16_t test_burst_size)
{
uint32_t cur_iter_op;
uint32_t imix_idx = 0;
for (cur_iter_op = 0; cur_iter_op < iter_ops_needed;
cur_iter_op += test_burst_size) {
uint32_t burst_size = RTE_MIN(
iter_ops_needed - cur_iter_op, test_burst_size);
struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op];
/* Allocate objects containing crypto operations and mbufs */
if (rte_mempool_get_bulk(state->ctx->pool, (void **)ops,
burst_size) != 0) {
RTE_LOG(ERR, USER1,
"Failed to allocate more crypto operations "
"from the crypto operation pool.\n"
"Consider increasing the pool size "
"with --pool-sz\n");
return -1;
}
/* Setup crypto op, attach mbuf etc */
(state->ctx->populate_ops)(ops,
state->ctx->src_buf_offset,
state->ctx->dst_buf_offset,
burst_size,
state->ctx->sess, state->opts,
state->ctx->test_vector, iv_offset,
&imix_idx);
}
return 0;
}
/* free mbufs from death row */
int
#else
void
#endif
rte_ip_frag_free_death_row(struct rte_ip_frag_death_row *dr,
uint32_t prefetch)
{
uint32_t i, k, n;
k = RTE_MIN(prefetch, dr->cnt);
n = dr->cnt;
for (i = 0; i != k; i++)
rte_prefetch0(dr->row[i]);
for (i = 0; i != n - k; i++) {
rte_prefetch0(dr->row[i + k]);
rte_pktmbuf_free(dr->row[i]);
}
for (; i != n; i++)
rte_pktmbuf_free(dr->row[i]);
dr->cnt = 0;
#ifdef RTE_LIBRW_PIOT
return n;
#endif
}
/* benchmark enqueue, returns number of ops enqueued */
static uint32_t
pmd_cyclecount_bench_enq(struct pmd_cyclecount_state *state,
uint32_t iter_ops_needed, uint16_t test_burst_size)
{
/* Enqueue full descriptor ring of ops on crypto device */
uint32_t cur_iter_op = 0;
while (cur_iter_op < iter_ops_needed) {
uint32_t burst_size = RTE_MIN(iter_ops_needed - cur_iter_op,
test_burst_size);
struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op];
uint32_t burst_enqd;
burst_enqd = rte_cryptodev_enqueue_burst(state->ctx->dev_id,
state->ctx->qp_id, ops, burst_size);
/* if we couldn't enqueue anything, the queue is full */
if (!burst_enqd) {
/* don't try to dequeue anything we didn't enqueue */
return cur_iter_op;
}
if (burst_enqd < burst_size)
state->ops_enq_retries++;
state->ops_enqd += burst_enqd;
cur_iter_op += burst_enqd;
}
return iter_ops_needed;
}
/*
* Receive burst of packets from client
*/
static void
receive_from_client(uint16_t client)
{
int j = 0;
uint16_t dq_pkt = PKT_BURST_SIZE;
struct rte_mbuf *buf[PKT_BURST_SIZE] = {0};
struct client *cl = NULL;
struct statistics *s = NULL;
cl = &clients[client];
s = &vport_stats[client];
/* Attempt to dequeue maximum available number of mbufs from ring */
while (dq_pkt > 0 &&
unlikely(rte_ring_sc_dequeue_bulk(
cl->tx_q, (void **)buf, dq_pkt) != 0))
dq_pkt = (uint16_t)RTE_MIN(
rte_ring_count(cl->tx_q), PKT_BURST_SIZE);
/* Update number of packets transmitted by client */
s->tx += dq_pkt;
for (j = 0; j < dq_pkt; j++) {
switch_packet(buf[j], client);
}
}
/* this is really a sanity check */
static int
test_macros(int __rte_unused unused_parm)
{
#define SMALLER 0x1000U
#define BIGGER 0x2000U
#define PTR_DIFF BIGGER - SMALLER
#define FAIL_MACRO(x)\
{printf(#x "() test failed!\n");\
return -1;}
uintptr_t unused = 0;
RTE_SET_USED(unused);
if ((uintptr_t)RTE_PTR_ADD(SMALLER, PTR_DIFF) != BIGGER)
FAIL_MACRO(RTE_PTR_ADD);
if ((uintptr_t)RTE_PTR_SUB(BIGGER, PTR_DIFF) != SMALLER)
FAIL_MACRO(RTE_PTR_SUB);
if (RTE_PTR_DIFF(BIGGER, SMALLER) != PTR_DIFF)
FAIL_MACRO(RTE_PTR_DIFF);
if (RTE_MAX(SMALLER, BIGGER) != BIGGER)
FAIL_MACRO(RTE_MAX);
if (RTE_MIN(SMALLER, BIGGER) != SMALLER)
FAIL_MACRO(RTE_MIN);
if (strncmp(RTE_STR(test), "test", sizeof("test")))
FAIL_MACRO(RTE_STR);
return 0;
}
/*
* Function receives messages from the daemon.
*/
static void
receive_request_from_vswitchd(void)
{
int j = 0;
uint16_t dq_pkt = PKT_BURST_SIZE;
struct client *vswd = NULL;
struct statistics *vswd_stat = NULL;
struct rte_mbuf *buf[PKT_BURST_SIZE] = {0};
vswd = &clients[VSWITCHD];
vswd_stat = &vport_stats[VSWITCHD];
/* Attempt to dequeue maximum available number of mbufs from ring */
while (dq_pkt > 0 &&
unlikely(rte_ring_sc_dequeue_bulk(
vswd->tx_q, (void **)buf, dq_pkt) != 0))
dq_pkt = (uint16_t)RTE_MIN(
rte_ring_count(vswd->tx_q), PKT_BURST_SIZE);
/* Update number of packets transmitted by daemon */
vswd_stat->rx += dq_pkt;
for (j = 0; j < dq_pkt; j++) {
handle_vswitchd_cmd(buf[j]);
}
}
/* Update device info */
static void
dpdk_ethdev_info_update(struct vr_dpdk_ethdev *ethdev)
{
struct rte_eth_dev_info dev_info;
rte_eth_dev_info_get(ethdev->ethdev_port_id, &dev_info);
ethdev->ethdev_nb_rx_queues = RTE_MIN(dev_info.max_rx_queues,
VR_DPDK_MAX_NB_RX_QUEUES);
if (dev_info.max_tx_queues > VR_DPDK_MAX_NB_TX_QUEUES)
dev_info.max_tx_queues = VR_DPDK_MAX_NB_TX_QUEUES;
ethdev->ethdev_nb_tx_queues = dev_info.max_tx_queues;
/* Check if we have dedicated an lcore for SR-IOV VF IO. */
if (vr_dpdk.vf_lcore_id) {
ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_tx_queues = 1;
}
ethdev->ethdev_nb_rss_queues = RTE_MIN(RTE_MIN(ethdev->ethdev_nb_rx_queues,
vr_dpdk.nb_fwd_lcores), VR_DPDK_MAX_NB_RSS_QUEUES);
ethdev->ethdev_reta_size = RTE_MIN(dev_info.reta_size,
VR_DPDK_MAX_RETA_SIZE);
RTE_LOG(DEBUG, VROUTER, "dev_info: driver_name=%s if_index=%u"
" max_rx_queues=%" PRIu16 " max_tx_queues=%" PRIu16
" max_vfs=%" PRIu16 " max_vmdq_pools=%" PRIu16
" rx_offload_capa=%" PRIx32 " tx_offload_capa=%" PRIx32 "\n",
dev_info.driver_name, dev_info.if_index,
dev_info.max_rx_queues, dev_info.max_tx_queues,
dev_info.max_vfs, dev_info.max_vmdq_pools,
dev_info.rx_offload_capa, dev_info.tx_offload_capa);
#if !VR_DPDK_USE_HW_FILTERING
/* use RSS queues only */
ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_rss_queues;
#else
/* we use just RSS queues if the device does not support RETA */
if (ethdev->ethdev_reta_size == 0)
ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_rss_queues;
#endif
return;
}
/**************************************************************************//**
*
* pktgen_pcap_mbuf_ctor - Callback routine to construct PCAP packets.
*
* DESCRIPTION
* Callback routine to construct a set of PCAP packet buffers.
*
* RETURNS: N/A
*
* SEE ALSO:
*/
static void
pktgen_pcap_mbuf_ctor(struct rte_mempool *mp, void *opaque_arg, void *_m, unsigned i)
{
struct rte_mbuf *m = _m;
uint32_t buf_len = mp->elt_size - sizeof(struct rte_mbuf);
pcaprec_hdr_t hdr;
ssize_t len = -1;
char buffer[2048];
pcap_info_t *pcap = (pcap_info_t *)opaque_arg;
RTE_MBUF_ASSERT(mp->elt_size >= sizeof(struct rte_mbuf));
memset(m, 0, mp->elt_size);
/* start of buffer is just after mbuf structure */
m->buf_addr = (char *)m + sizeof(struct rte_mbuf);
m->buf_physaddr = rte_mempool_virt2phy(mp, m->buf_addr);
m->buf_len = (uint16_t)buf_len;
/* keep some headroom between start of buffer and data */
m->data_off = RTE_MIN(RTE_PKTMBUF_HEADROOM, m->buf_len);
/* init some constant fields */
m->pool = mp;
m->nb_segs = 1;
m->port = 0xff;
m->ol_flags = 0;
for (;; ) {
if ( (i & 0x3ff) == 0) {
rte_printf_status("%c\b", "-\\|/"[(i >> 10) & 3]);
i++;
}
if (unlikely(wr_pcap_read(pcap, &hdr, buffer, sizeof(buffer)) <= 0) ) {
wr_pcap_rewind(pcap);
continue;
}
len = hdr.incl_len;
/* Adjust the packet length if not a valid size. */
if (len < (ETHER_MIN_LEN - 4) )
len = (ETHER_MIN_LEN - 4);
else if (len > (ETHER_MAX_LEN - 4) )
len = (ETHER_MAX_LEN - 4);
m->data_len = len;
m->pkt_len = len;
rte_memcpy((uint8_t *)m->buf_addr + m->data_off, buffer, len);
break;
}
static inline int
mbox_wait_response(struct mbox *m, struct octeontx_mbox_hdr *hdr,
void *rxmsg, uint16_t rxsize)
{
int res = 0, wait;
uint16_t len;
struct mbox_ram_hdr rx_hdr;
uint64_t *ram_mbox_hdr = (uint64_t *)m->ram_mbox_base;
uint8_t *ram_mbox_msg = m->ram_mbox_base + sizeof(struct mbox_ram_hdr);
/* Wait for response */
wait = MBOX_WAIT_TIME_SEC * 1000 * 10;
while (wait > 0) {
rte_delay_us(100);
rx_hdr.u64 = rte_read64(ram_mbox_hdr);
if (rx_hdr.chan_state == MBOX_CHAN_STATE_RES)
break;
--wait;
}
hdr->res_code = rx_hdr.res_code;
m->tag_own++;
/* Timeout */
if (wait <= 0) {
res = -ETIMEDOUT;
goto error;
}
/* Tag mismatch */
if (m->tag_own != rx_hdr.tag) {
res = -EINVAL;
goto error;
}
/* PF nacked the msg */
if (rx_hdr.res_code != MBOX_RET_SUCCESS) {
res = -EBADMSG;
goto error;
}
len = RTE_MIN(rx_hdr.len, rxsize);
if (rxmsg)
mbox_msgcpy(rxmsg, ram_mbox_msg, len);
return len;
error:
mbox_log_err("Failed to send mbox(%d/%d) coproc=%d msg=%d ret=(%d,%d)",
m->tag_own, rx_hdr.tag, hdr->coproc, hdr->msg, res,
hdr->res_code);
return res;
}
static int
exec_burst(uint32_t flags, int lcore)
{
unsigned i, portid, nb_tx = 0;
struct lcore_conf *conf;
uint32_t pkt_per_port;
int num, idx = 0;
int diff_tsc;
conf = &lcore_conf[lcore];
pkt_per_port = MAX_TRAFFIC_BURST;
num = pkt_per_port;
rte_atomic64_init(&start);
/* start polling thread, but not actually poll yet */
rte_eal_remote_launch(poll_burst,
(void *)&pkt_per_port, lcore);
/* Only when polling first */
if (flags == SC_BURST_POLL_FIRST)
rte_atomic64_set(&start, 1);
/* start xmit */
while (num) {
nb_tx = RTE_MIN(MAX_PKT_BURST, num);
for (i = 0; i < conf->nb_ports; i++) {
portid = conf->portlist[i];
rte_eth_tx_burst(portid, 0,
&tx_burst[idx], nb_tx);
idx += nb_tx;
}
num -= nb_tx;
}
sleep(5);
/* only when polling second */
if (flags == SC_BURST_XMIT_FIRST)
rte_atomic64_set(&start, 1);
/* wait for polling finished */
diff_tsc = rte_eal_wait_lcore(lcore);
if (diff_tsc < 0) {
printf("exec_burst: Failed to measure cycles per packet\n");
return -1;
}
printf("Result: %d cycles per packet\n", diff_tsc);
return 0;
}
/**
* A call to rx_sync_ring will try to fill a Netmap RX ring with as many
* packets as it can hold coming from its dpdk port.
*/
static inline int
rx_sync_ring(struct netmap_ring *ring, uint8_t port, uint16_t ring_number,
uint16_t max_burst)
{
int32_t i, n_rx;
uint16_t burst_size;
uint32_t cur_slot, n_free_slots;
struct rte_mbuf *rx_mbufs[COMPAT_NETMAP_MAX_BURST];
n_free_slots = ring->num_slots - (ring->avail + ring->reserved);
n_free_slots = RTE_MIN(n_free_slots, max_burst);
cur_slot = (ring->cur + ring->avail) & (ring->num_slots - 1);
while (n_free_slots) {
burst_size = (uint16_t)RTE_MIN(n_free_slots, RTE_DIM(rx_mbufs));
/* receive up to burst_size packets from the NIC's queue */
n_rx = rte_eth_rx_burst(port, ring_number, rx_mbufs,
burst_size);
if (n_rx == 0)
return 0;
if (unlikely(n_rx < 0))
return -1;
/* Put those n_rx packets in the Netmap structures */
for (i = 0; i < n_rx ; i++) {
mbuf_to_slot(rx_mbufs[i], ring, cur_slot);
rte_pktmbuf_free(rx_mbufs[i]);
cur_slot = NETMAP_RING_NEXT(ring, cur_slot);
}
/* Update the Netmap ring structure to reflect the change */
ring->avail += n_rx;
n_free_slots -= n_rx;
}
return 0;
}
/*
* i40e_set_flx_pld_cfg -configure the rule how bytes stream is extracted as flexible payload
* @pf: board private structure
* @cfg: the rule how bytes stream is extracted as flexible payload
*/
static void
i40e_set_flx_pld_cfg(struct i40e_pf *pf,
const struct rte_eth_flex_payload_cfg *cfg)
{
struct i40e_hw *hw = I40E_PF_TO_HW(pf);
struct i40e_fdir_flex_pit flex_pit[I40E_MAX_FLXPLD_FIED];
uint32_t flx_pit;
uint16_t num, min_next_off; /* in words */
uint8_t field_idx = 0;
uint8_t layer_idx = 0;
uint16_t i;
if (cfg->type == RTE_ETH_L2_PAYLOAD)
layer_idx = I40E_FLXPLD_L2_IDX;
else if (cfg->type == RTE_ETH_L3_PAYLOAD)
layer_idx = I40E_FLXPLD_L3_IDX;
else if (cfg->type == RTE_ETH_L4_PAYLOAD)
layer_idx = I40E_FLXPLD_L4_IDX;
memset(flex_pit, 0, sizeof(flex_pit));
num = i40e_srcoff_to_flx_pit(cfg->src_offset, flex_pit);
for (i = 0; i < RTE_MIN(num, RTE_DIM(flex_pit)); i++) {
field_idx = layer_idx * I40E_MAX_FLXPLD_FIED + i;
/* record the info in fdir structure */
pf->fdir.flex_set[field_idx].src_offset =
flex_pit[i].src_offset / sizeof(uint16_t);
pf->fdir.flex_set[field_idx].size =
flex_pit[i].size / sizeof(uint16_t);
pf->fdir.flex_set[field_idx].dst_offset =
flex_pit[i].dst_offset / sizeof(uint16_t);
flx_pit = MK_FLX_PIT(pf->fdir.flex_set[field_idx].src_offset,
pf->fdir.flex_set[field_idx].size,
pf->fdir.flex_set[field_idx].dst_offset);
I40E_WRITE_REG(hw, I40E_PRTQF_FLX_PIT(field_idx), flx_pit);
}
min_next_off = pf->fdir.flex_set[field_idx].src_offset +
pf->fdir.flex_set[field_idx].size;
for (; i < I40E_MAX_FLXPLD_FIED; i++) {
/* set the non-used register obeying register's constrain */
flx_pit = MK_FLX_PIT(min_next_off, NONUSE_FLX_PIT_FSIZE,
NONUSE_FLX_PIT_DEST_OFF);
I40E_WRITE_REG(hw,
I40E_PRTQF_FLX_PIT(layer_idx * I40E_MAX_FLXPLD_FIED + i),
flx_pit);
min_next_off++;
}
}
int
rte_vhost_get_ifname(int vid, char *buf, size_t len)
{
struct virtio_net *dev = get_device(vid);
if (dev == NULL)
return -1;
len = RTE_MIN(len, sizeof(dev->ifname));
strncpy(buf, dev->ifname, len);
buf[len - 1] = '\0';
return 0;
}
static int
add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg,
uint64_t page_size)
{
uint64_t reg_size = reg->size;
uint64_t host_user_addr = reg->host_user_addr;
uint64_t guest_phys_addr = reg->guest_phys_addr;
uint64_t host_phys_addr;
uint64_t size;
host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)host_user_addr);
size = page_size - (guest_phys_addr & (page_size - 1));
size = RTE_MIN(size, reg_size);
if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0)
return -1;
host_user_addr += size;
guest_phys_addr += size;
reg_size -= size;
while (reg_size > 0) {
size = RTE_MIN(reg_size, page_size);
host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t)
host_user_addr);
if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr,
size) < 0)
return -1;
host_user_addr += size;
guest_phys_addr += size;
reg_size -= size;
}
return 0;
}
static int
avf_init_rxq(struct rte_eth_dev *dev, struct avf_rx_queue *rxq)
{
struct avf_hw *hw = AVF_DEV_PRIVATE_TO_HW(dev->data->dev_private);
struct rte_eth_dev_data *dev_data = dev->data;
uint16_t buf_size, max_pkt_len, len;
buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM;
/* Calculate the maximum packet length allowed */
len = rxq->rx_buf_len * AVF_MAX_CHAINED_RX_BUFFERS;
max_pkt_len = RTE_MIN(len, dev->data->dev_conf.rxmode.max_rx_pkt_len);
/* Check if the jumbo frame and maximum packet length are set
* correctly.
*/
if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
if (max_pkt_len <= ETHER_MAX_LEN ||
max_pkt_len > AVF_FRAME_SIZE_MAX) {
PMD_DRV_LOG(ERR, "maximum packet length must be "
"larger than %u and smaller than %u, "
"as jumbo frame is enabled",
(uint32_t)ETHER_MAX_LEN,
(uint32_t)AVF_FRAME_SIZE_MAX);
return -EINVAL;
}
} else {
if (max_pkt_len < ETHER_MIN_LEN ||
max_pkt_len > ETHER_MAX_LEN) {
PMD_DRV_LOG(ERR, "maximum packet length must be "
"larger than %u and smaller than %u, "
"as jumbo frame is disabled",
(uint32_t)ETHER_MIN_LEN,
(uint32_t)ETHER_MAX_LEN);
return -EINVAL;
}
}
rxq->max_pkt_len = max_pkt_len;
if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) ||
(rxq->max_pkt_len + 2 * AVF_VLAN_TAG_SIZE) > buf_size) {
dev_data->scattered_rx = 1;
}
AVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
AVF_WRITE_FLUSH(hw);
return 0;
}
/*
* emit one of:
* mov %<sreg>, <ofs>(%<dreg>)
* mov <imm>, <ofs>(%<dreg>)
*/
static void
emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg, uint32_t imm, int32_t ofs)
{
uint32_t mods, imsz, opsz, opx;
const uint8_t prfx16 = 0x66;
/* 8 bit instruction opcodes */
static const uint8_t op8[] = {0xC6, 0x88};
/* 16/32/64 bit instruction opcodes */
static const uint8_t ops[] = {0xC7, 0x89};
/* is the instruction has immediate value or src reg? */
opx = (BPF_CLASS(op) == BPF_STX);
opsz = BPF_SIZE(op);
if (opsz == BPF_H)
emit_bytes(st, &prfx16, sizeof(prfx16));
emit_rex(st, op, sreg, dreg);
if (opsz == BPF_B)
emit_bytes(st, &op8[opx], sizeof(op8[opx]));
else
emit_bytes(st, &ops[opx], sizeof(ops[opx]));
imsz = imm_size(ofs);
mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_modregrm(st, mods, sreg, dreg);
if (dreg == RSP || dreg == R12)
emit_sib(st, SIB_SCALE_1, dreg, dreg);
emit_imm(st, ofs, imsz);
if (opx == 0) {
imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
emit_imm(st, imm, imsz);
}
}
static uint16_t
bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct bnx2x_tx_queue *txq;
struct bnx2x_softc *sc;
struct bnx2x_fastpath *fp;
uint32_t burst, nb_tx;
struct rte_mbuf **m = tx_pkts;
int ret;
txq = p_txq;
sc = txq->sc;
fp = &sc->fp[txq->queue_id];
nb_tx = nb_pkts;
do {
burst = RTE_MIN(nb_pkts, RTE_PMD_BNX2X_TX_MAX_BURST);
ret = bnx2x_tx_encap(txq, m, burst);
if (unlikely(ret)) {
PMD_TX_LOG(ERR, "tx_encap failed!");
}
bnx2x_update_fp_sb_idx(fp);
if ((txq->nb_tx_desc - txq->nb_tx_avail) > txq->tx_free_thresh) {
bnx2x_txeof(sc, fp);
}
if (unlikely(ret == -ENOMEM)) {
break;
}
m += burst;
nb_pkts -= burst;
} while (nb_pkts);
return nb_tx - nb_pkts;
}
static uint16_t
bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
struct bnx2x_tx_queue *txq;
struct bnx2x_softc *sc;
struct bnx2x_fastpath *fp;
uint16_t nb_tx_pkts;
uint16_t nb_pkt_sent = 0;
uint32_t ret;
txq = p_txq;
sc = txq->sc;
fp = &sc->fp[txq->queue_id];
if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
txq->tx_free_thresh)))
bnx2x_txeof(sc, fp);
nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
if (unlikely(nb_tx_pkts == 0))
return 0;
while (nb_tx_pkts--) {
struct rte_mbuf *m = *tx_pkts++;
assert(m != NULL);
ret = bnx2x_tx_encap(txq, m);
fp->tx_db.data.prod += ret;
nb_pkt_sent++;
}
bnx2x_update_fp_sb_idx(fp);
mb();
DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
mb();
if ((txq->nb_tx_desc - txq->nb_tx_avail) >
txq->tx_free_thresh)
bnx2x_txeof(sc, fp);
return nb_pkt_sent;
}
/*
* Function handles messages from the daemon.
*/
void
handle_request_from_vswitchd(void)
{
int j = 0;
uint16_t dq_pkt = PKT_BURST_SIZE;
struct rte_mbuf *buf[PKT_BURST_SIZE] = {0};
/* Attempt to dequeue maximum available number of mbufs from ring */
while (dq_pkt > 0 &&
unlikely(rte_ring_sc_dequeue_bulk(
vswitchd_message_ring, (void **)buf, dq_pkt) != 0))
dq_pkt = (uint16_t)RTE_MIN(
rte_ring_count(vswitchd_message_ring), PKT_BURST_SIZE);
/* Update number of packets transmitted by daemon */
stats_vport_rx_increment(VSWITCHD, dq_pkt);
for (j = 0; j < dq_pkt; j++) {
handle_vswitchd_cmd(buf[j]);
}
}
/* benchmark dequeue */
static void
pmd_cyclecount_bench_deq(struct pmd_cyclecount_state *state,
uint32_t iter_ops_needed, uint16_t test_burst_size)
{
/* Dequeue full descriptor ring of ops on crypto device */
uint32_t cur_iter_op = 0;
while (cur_iter_op < iter_ops_needed) {
uint32_t burst_size = RTE_MIN(iter_ops_needed - cur_iter_op,
test_burst_size);
struct rte_crypto_op **ops_processed =
&state->ctx->ops[cur_iter_op];
uint32_t burst_deqd;
burst_deqd = rte_cryptodev_dequeue_burst(state->ctx->dev_id,
state->ctx->qp_id, ops_processed, burst_size);
if (burst_deqd < burst_size)
state->ops_deq_retries++;
state->ops_deqd += burst_deqd;
cur_iter_op += burst_deqd;
}
}
/* free mbufs from death row */
void
rte_ip_frag_free_death_row(struct rte_ip_frag_death_row *dr,
uint32_t prefetch)
{
uint32_t i, k, n;
k = RTE_MIN(prefetch, dr->cnt);
n = dr->cnt;
for (i = 0; i != k; i++)
rte_prefetch0(dr->row[i]);
for (i = 0; i != n - k; i++) {
rte_prefetch0(dr->row[i + k]);
rte_pktmbuf_free(dr->row[i]);
}
for (; i != n; i++)
rte_pktmbuf_free(dr->row[i]);
dr->cnt = 0;
}
/*
* vPMD receive routine that reassembles scattered packets
*
* Notice:
* - don't support ol_flags for rss and csum err
* - nb_pkts > RTE_FM10K_MAX_RX_BURST, only scan RTE_FM10K_MAX_RX_BURST
* numbers of DD bit
*/
uint16_t
fm10k_recv_scattered_pkts_vec(void *rx_queue,
struct rte_mbuf **rx_pkts,
uint16_t nb_pkts)
{
struct fm10k_rx_queue *rxq = rx_queue;
uint8_t split_flags[RTE_FM10K_MAX_RX_BURST] = {0};
unsigned i = 0;
/* Split_flags only can support max of RTE_FM10K_MAX_RX_BURST */
nb_pkts = RTE_MIN(nb_pkts, RTE_FM10K_MAX_RX_BURST);
/* get some new buffers */
uint16_t nb_bufs = fm10k_recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
split_flags);
if (nb_bufs == 0)
return 0;
/* happy day case, full burst + no packets to be joined */
const uint64_t *split_fl64 = (uint64_t *)split_flags;
if (rxq->pkt_first_seg == NULL &&
split_fl64[0] == 0 && split_fl64[1] == 0 &&
split_fl64[2] == 0 && split_fl64[3] == 0)
return nb_bufs;
/* reassemble any packets that need reassembly*/
if (rxq->pkt_first_seg == NULL) {
/* find the first split flag, and only reassemble then*/
while (i < nb_bufs && !split_flags[i])
i++;
if (i == nb_bufs)
return nb_bufs;
}
return i + fm10k_reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
&split_flags[i]);
}
uint16_t
ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts)
{
struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
volatile union ixgbe_adv_tx_desc *txdp;
struct ixgbe_tx_entry_v *txep;
uint16_t n, nb_commit, tx_id;
uint64_t flags = DCMD_DTYP_FLAGS;
uint64_t rs = IXGBE_ADVTXD_DCMD_RS|DCMD_DTYP_FLAGS;
int i;
/* cross rx_thresh boundary is not allowed */
nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
if (txq->nb_tx_free < txq->tx_free_thresh)
ixgbe_tx_free_bufs(txq);
nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
if (unlikely(nb_pkts == 0))
return 0;
tx_id = txq->tx_tail;
txdp = &txq->tx_ring[tx_id];
txep = &txq->sw_ring_v[tx_id];
txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
n = (uint16_t)(txq->nb_tx_desc - tx_id);
if (nb_commit >= n) {
tx_backlog_entry(txep, tx_pkts, n);
for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
vtx1(txdp, *tx_pkts, flags);
vtx1(txdp, *tx_pkts++, rs);
nb_commit = (uint16_t)(nb_commit - n);
tx_id = 0;
txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
/* avoid reach the end of ring */
txdp = &(txq->tx_ring[tx_id]);
txep = &txq->sw_ring_v[tx_id];
}
tx_backlog_entry(txep, tx_pkts, nb_commit);
vtx(txdp, tx_pkts, nb_commit, flags);
tx_id = (uint16_t)(tx_id + nb_commit);
if (tx_id > txq->tx_next_rs) {
txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |=
rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
txq->tx_rs_thresh);
}
txq->tx_tail = tx_id;
IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail);
return nb_pkts;
}
/*
* vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
*
* Notice:
* - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
* - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
* numbers of DD bit
* - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
* - don't support ol_flags for rss and csum err
*/
static inline uint16_t
_recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
uint16_t nb_pkts, uint8_t *split_packet)
{
volatile union ixgbe_adv_rx_desc *rxdp;
struct ixgbe_rx_entry *sw_ring;
uint16_t nb_pkts_recd;
int pos;
uint64_t var;
__m128i shuf_msk;
__m128i crc_adjust = _mm_set_epi16(
0, 0, 0, /* ignore non-length fields */
-rxq->crc_len, /* sub crc on data_len */
0, /* ignore high-16bits of pkt_len */
-rxq->crc_len, /* sub crc on pkt_len */
0, 0 /* ignore pkt_type field */
);
__m128i dd_check, eop_check;
/* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */
nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_MAX_RX_BURST);
/* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */
nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP);
/* Just the act of getting into the function from the application is
* going to cost about 7 cycles
*/
rxdp = rxq->rx_ring + rxq->rx_tail;
_mm_prefetch((const void *)rxdp, _MM_HINT_T0);
/* See if we need to rearm the RX queue - gives the prefetch a bit
* of time to act
*/
if (rxq->rxrearm_nb > RTE_IXGBE_RXQ_REARM_THRESH)
ixgbe_rxq_rearm(rxq);
/* Before we start moving massive data around, check to see if
* there is actually a packet available
*/
if (!(rxdp->wb.upper.status_error &
rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
return 0;
/* 4 packets DD mask */
dd_check = _mm_set_epi64x(0x0000000100000001LL, 0x0000000100000001LL);
/* 4 packets EOP mask */
eop_check = _mm_set_epi64x(0x0000000200000002LL, 0x0000000200000002LL);
/* mask to shuffle from desc. to mbuf */
shuf_msk = _mm_set_epi8(
7, 6, 5, 4, /* octet 4~7, 32bits rss */
15, 14, /* octet 14~15, low 16 bits vlan_macip */
13, 12, /* octet 12~13, 16 bits data_len */
0xFF, 0xFF, /* skip high 16 bits pkt_len, zero out */
13, 12, /* octet 12~13, low 16 bits pkt_len */
0xFF, 0xFF, /* skip 32 bit pkt_type */
0xFF, 0xFF
);
/* Cache is empty -> need to scan the buffer rings, but first move
* the next 'n' mbufs into the cache
*/
sw_ring = &rxq->sw_ring[rxq->rx_tail];
/* A. load 4 packet in one loop
* [A*. mask out 4 unused dirty field in desc]
* B. copy 4 mbuf point from swring to rx_pkts
* C. calc the number of DD bits among the 4 packets
* [C*. extract the end-of-packet bit, if requested]
* D. fill info. from desc to mbuf
*/
for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
pos += RTE_IXGBE_DESCS_PER_LOOP,
rxdp += RTE_IXGBE_DESCS_PER_LOOP) {
__m128i descs[RTE_IXGBE_DESCS_PER_LOOP];
__m128i pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
__m128i zero, staterr, sterr_tmp1, sterr_tmp2;
__m128i mbp1, mbp2; /* two mbuf pointer in one XMM reg. */
/* B.1 load 1 mbuf point */
mbp1 = _mm_loadu_si128((__m128i *)&sw_ring[pos]);
/* Read desc statuses backwards to avoid race condition */
/* A.1 load 4 pkts desc */
descs[3] = _mm_loadu_si128((__m128i *)(rxdp + 3));
/* B.2 copy 2 mbuf point into rx_pkts */
_mm_storeu_si128((__m128i *)&rx_pkts[pos], mbp1);
/* B.1 load 1 mbuf point */
mbp2 = _mm_loadu_si128((__m128i *)&sw_ring[pos+2]);
descs[2] = _mm_loadu_si128((__m128i *)(rxdp + 2));
/* B.1 load 2 mbuf point */
descs[1] = _mm_loadu_si128((__m128i *)(rxdp + 1));
descs[0] = _mm_loadu_si128((__m128i *)(rxdp));
/* B.2 copy 2 mbuf point into rx_pkts */
_mm_storeu_si128((__m128i *)&rx_pkts[pos+2], mbp2);
if (split_packet) {
rte_mbuf_prefetch_part2(rx_pkts[pos]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
}
/* avoid compiler reorder optimization */
rte_compiler_barrier();
/* D.1 pkt 3,4 convert format from desc to pktmbuf */
pkt_mb4 = _mm_shuffle_epi8(descs[3], shuf_msk);
pkt_mb3 = _mm_shuffle_epi8(descs[2], shuf_msk);
/* D.1 pkt 1,2 convert format from desc to pktmbuf */
pkt_mb2 = _mm_shuffle_epi8(descs[1], shuf_msk);
pkt_mb1 = _mm_shuffle_epi8(descs[0], shuf_msk);
/* C.1 4=>2 filter staterr info only */
sterr_tmp2 = _mm_unpackhi_epi32(descs[3], descs[2]);
/* C.1 4=>2 filter staterr info only */
sterr_tmp1 = _mm_unpackhi_epi32(descs[1], descs[0]);
/* set ol_flags with vlan packet type */
desc_to_olflags_v(descs, &rx_pkts[pos]);
/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
pkt_mb4 = _mm_add_epi16(pkt_mb4, crc_adjust);
pkt_mb3 = _mm_add_epi16(pkt_mb3, crc_adjust);
/* C.2 get 4 pkts staterr value */
zero = _mm_xor_si128(dd_check, dd_check);
staterr = _mm_unpacklo_epi32(sterr_tmp1, sterr_tmp2);
/* D.3 copy final 3,4 data to rx_pkts */
_mm_storeu_si128((void *)&rx_pkts[pos+3]->rx_descriptor_fields1,
pkt_mb4);
_mm_storeu_si128((void *)&rx_pkts[pos+2]->rx_descriptor_fields1,
pkt_mb3);
/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
pkt_mb2 = _mm_add_epi16(pkt_mb2, crc_adjust);
pkt_mb1 = _mm_add_epi16(pkt_mb1, crc_adjust);
/* C* extract and record EOP bit */
if (split_packet) {
__m128i eop_shuf_mask = _mm_set_epi8(
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0xFF, 0xFF, 0xFF, 0xFF,
0x04, 0x0C, 0x00, 0x08
);
/* and with mask to extract bits, flipping 1-0 */
__m128i eop_bits = _mm_andnot_si128(staterr, eop_check);
/* the staterr values are not in order, as the count
* count of dd bits doesn't care. However, for end of
* packet tracking, we do care, so shuffle. This also
* compresses the 32-bit values to 8-bit
*/
eop_bits = _mm_shuffle_epi8(eop_bits, eop_shuf_mask);
/* store the resulting 32-bit value */
*(int *)split_packet = _mm_cvtsi128_si32(eop_bits);
split_packet += RTE_IXGBE_DESCS_PER_LOOP;
/* zero-out next pointers */
rx_pkts[pos]->next = NULL;
rx_pkts[pos + 1]->next = NULL;
rx_pkts[pos + 2]->next = NULL;
rx_pkts[pos + 3]->next = NULL;
}
/* C.3 calc available number of desc */
staterr = _mm_and_si128(staterr, dd_check);
staterr = _mm_packs_epi32(staterr, zero);
/* D.3 copy final 1,2 data to rx_pkts */
_mm_storeu_si128((void *)&rx_pkts[pos+1]->rx_descriptor_fields1,
pkt_mb2);
_mm_storeu_si128((void *)&rx_pkts[pos]->rx_descriptor_fields1,
pkt_mb1);
/* C.4 calc avaialbe number of desc */
var = __builtin_popcountll(_mm_cvtsi128_si64(staterr));
nb_pkts_recd += var;
if (likely(var != RTE_IXGBE_DESCS_PER_LOOP))
break;
}
/* Update our internal tail pointer */
rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
return nb_pkts_recd;
}
static inline uint32_t
sw_schedule_parallel_to_cq(struct sw_evdev *sw, struct sw_qid * const qid,
uint32_t iq_num, unsigned int count, int keep_order)
{
uint32_t i;
uint32_t cq_idx = qid->cq_next_tx;
/* This is the QID ID. The QID ID is static, hence it can be
* used to identify the stage of processing in history lists etc
*/
uint32_t qid_id = qid->id;
if (count > MAX_PER_IQ_DEQUEUE)
count = MAX_PER_IQ_DEQUEUE;
if (keep_order)
/* only schedule as many as we have reorder buffer entries */
count = RTE_MIN(count,
rte_ring_count(qid->reorder_buffer_freelist));
for (i = 0; i < count; i++) {
const struct rte_event *qe = iq_ring_peek(qid->iq[iq_num]);
uint32_t cq_check_count = 0;
uint32_t cq;
/*
* for parallel, just send to next available CQ in round-robin
* fashion. So scan for an available CQ. If all CQs are full
* just return and move on to next QID
*/
do {
if (++cq_check_count > qid->cq_num_mapped_cqs)
goto exit;
cq = qid->cq_map[cq_idx];
if (++cq_idx == qid->cq_num_mapped_cqs)
cq_idx = 0;
} while (rte_event_ring_free_count(
sw->ports[cq].cq_worker_ring) == 0 ||
sw->ports[cq].inflights == SW_PORT_HIST_LIST);
struct sw_port *p = &sw->ports[cq];
if (sw->cq_ring_space[cq] == 0 ||
p->inflights == SW_PORT_HIST_LIST)
break;
sw->cq_ring_space[cq]--;
qid->stats.tx_pkts++;
const int head = (p->hist_head & (SW_PORT_HIST_LIST-1));
p->hist_list[head].fid = SW_HASH_FLOWID(qe->flow_id);
p->hist_list[head].qid = qid_id;
if (keep_order)
rte_ring_sc_dequeue(qid->reorder_buffer_freelist,
(void *)&p->hist_list[head].rob_entry);
sw->ports[cq].cq_buf[sw->ports[cq].cq_buf_count++] = *qe;
iq_ring_pop(qid->iq[iq_num]);
rte_compiler_barrier();
p->inflights++;
p->stats.tx_pkts++;
p->hist_head++;
}
exit:
qid->cq_next_tx = cq_idx;
return i;
}
