/*
* Create a scheduler on the current lcore
*/
struct lthread_sched *_lthread_sched_create(size_t stack_size)
{
int status;
struct lthread_sched *new_sched;
unsigned lcoreid = rte_lcore_id();
RTE_ASSERT(stack_size <= LTHREAD_MAX_STACK_SIZE);
if (stack_size == 0)
stack_size = LTHREAD_MAX_STACK_SIZE;
new_sched =
rte_calloc_socket(NULL, 1, sizeof(struct lthread_sched),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (new_sched == NULL) {
RTE_LOG(CRIT, LTHREAD,
"Failed to allocate memory for scheduler\n");
return NULL;
}
_lthread_key_pool_init();
new_sched->stack_size = stack_size;
new_sched->birth = rte_rdtsc();
THIS_SCHED = new_sched;
status = _lthread_sched_alloc_resources(new_sched);
if (status != SCHED_ALLOC_OK) {
RTE_LOG(CRIT, LTHREAD,
"Failed to allocate resources for scheduler code = %d\n",
status);
rte_free(new_sched);
return NULL;
}
bzero(&new_sched->ctx, sizeof(struct ctx));
new_sched->lcore_id = lcoreid;
schedcore[lcoreid] = new_sched;
new_sched->run_flag = 1;
DIAG_EVENT(new_sched, LT_DIAG_SCHED_CREATE, rte_lcore_id(), 0);
rte_wmb();
return new_sched;
}
static int
sfc_efx_tx_qcreate(uint16_t port_id, uint16_t queue_id,
const struct rte_pci_addr *pci_addr,
int socket_id,
const struct sfc_dp_tx_qcreate_info *info,
struct sfc_dp_txq **dp_txqp)
{
struct sfc_efx_txq *txq;
struct sfc_txq *ctrl_txq;
int rc;
rc = ENOMEM;
txq = rte_zmalloc_socket("sfc-efx-txq", sizeof(*txq),
RTE_CACHE_LINE_SIZE, socket_id);
if (txq == NULL)
goto fail_txq_alloc;
sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr);
rc = ENOMEM;
txq->pend_desc = rte_calloc_socket("sfc-efx-txq-pend-desc",
EFX_TXQ_LIMIT(info->txq_entries),
sizeof(*txq->pend_desc), 0,
socket_id);
if (txq->pend_desc == NULL)
goto fail_pend_desc_alloc;
rc = ENOMEM;
txq->sw_ring = rte_calloc_socket("sfc-efx-txq-sw_ring",
info->txq_entries,
sizeof(*txq->sw_ring),
RTE_CACHE_LINE_SIZE, socket_id);
if (txq->sw_ring == NULL)
goto fail_sw_ring_alloc;
ctrl_txq = sfc_txq_by_dp_txq(&txq->dp);
if (ctrl_txq->evq->sa->tso) {
rc = sfc_efx_tso_alloc_tsoh_objs(txq->sw_ring,
info->txq_entries, socket_id);
if (rc != 0)
goto fail_alloc_tsoh_objs;
}
txq->evq = ctrl_txq->evq;
txq->ptr_mask = info->txq_entries - 1;
txq->free_thresh = info->free_thresh;
txq->dma_desc_size_max = info->dma_desc_size_max;
*dp_txqp = &txq->dp;
return 0;
fail_alloc_tsoh_objs:
rte_free(txq->sw_ring);
fail_sw_ring_alloc:
rte_free(txq->pend_desc);
fail_pend_desc_alloc:
rte_free(txq);
fail_txq_alloc:
return rc;
}
int
sfc_tx_configure(struct sfc_adapter *sa)
{
const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
const struct rte_eth_conf *dev_conf = &sa->eth_dev->data->dev_conf;
const unsigned int nb_tx_queues = sa->eth_dev->data->nb_tx_queues;
int rc = 0;
sfc_log_init(sa, "nb_tx_queues=%u (old %u)",
nb_tx_queues, sa->txq_count);
/*
* The datapath implementation assumes absence of boundary
* limits on Tx DMA descriptors. Addition of these checks on
* datapath would simply make the datapath slower.
*/
if (encp->enc_tx_dma_desc_boundary != 0) {
rc = ENOTSUP;
goto fail_tx_dma_desc_boundary;
}
rc = sfc_tx_check_mode(sa, &dev_conf->txmode);
if (rc != 0)
goto fail_check_mode;
if (nb_tx_queues == sa->txq_count)
goto done;
if (sa->txq_info == NULL) {
sa->txq_info = rte_calloc_socket("sfc-txqs", nb_tx_queues,
sizeof(sa->txq_info[0]), 0,
sa->socket_id);
if (sa->txq_info == NULL)
goto fail_txqs_alloc;
} else {
struct sfc_txq_info *new_txq_info;
if (nb_tx_queues < sa->txq_count)
sfc_tx_fini_queues(sa, nb_tx_queues);
new_txq_info =
rte_realloc(sa->txq_info,
nb_tx_queues * sizeof(sa->txq_info[0]), 0);
if (new_txq_info == NULL && nb_tx_queues > 0)
goto fail_txqs_realloc;
sa->txq_info = new_txq_info;
if (nb_tx_queues > sa->txq_count)
memset(&sa->txq_info[sa->txq_count], 0,
(nb_tx_queues - sa->txq_count) *
sizeof(sa->txq_info[0]));
}
while (sa->txq_count < nb_tx_queues) {
rc = sfc_tx_qinit_info(sa, sa->txq_count);
if (rc != 0)
goto fail_tx_qinit_info;
sa->txq_count++;
}
done:
return 0;
fail_tx_qinit_info:
fail_txqs_realloc:
fail_txqs_alloc:
sfc_tx_close(sa);
fail_check_mode:
fail_tx_dma_desc_boundary:
sfc_log_init(sa, "failed (rc = %d)", rc);
return rc;
}
/**
* Configure secondary process queues from a private data pointer (primary
* or secondary) and update burst callbacks. Can take place only once.
*
* All queues must have been previously created by the primary process to
* avoid undefined behavior.
*
* @param priv
* Private data pointer from either primary or secondary process.
*
* @return
* Private data pointer from secondary process, NULL in case of error.
*/
struct priv *
mlx5_secondary_data_setup(struct priv *priv)
{
unsigned int port_id = 0;
struct mlx5_secondary_data *sd;
void **tx_queues;
void **rx_queues;
unsigned int nb_tx_queues;
unsigned int nb_rx_queues;
unsigned int i;
/* priv must be valid at this point. */
assert(priv != NULL);
/* priv->dev must also be valid but may point to local memory from
* another process, possibly with the same address and must not
* be dereferenced yet. */
assert(priv->dev != NULL);
/* Determine port ID by finding out where priv comes from. */
while (1) {
sd = &mlx5_secondary_data[port_id];
rte_spinlock_lock(&sd->lock);
/* Primary process? */
if (sd->primary_priv == priv)
break;
/* Secondary process? */
if (sd->data.dev_private == priv)
break;
rte_spinlock_unlock(&sd->lock);
if (++port_id == RTE_DIM(mlx5_secondary_data))
port_id = 0;
}
/* Switch to secondary private structure. If private data has already
* been updated by another thread, there is nothing else to do. */
priv = sd->data.dev_private;
if (priv->dev->data == &sd->data)
goto end;
/* Sanity checks. Secondary private structure is supposed to point
* to local eth_dev, itself still pointing to the shared device data
* structure allocated by the primary process. */
assert(sd->shared_dev_data != &sd->data);
assert(sd->data.nb_tx_queues == 0);
assert(sd->data.tx_queues == NULL);
assert(sd->data.nb_rx_queues == 0);
assert(sd->data.rx_queues == NULL);
assert(priv != sd->primary_priv);
assert(priv->dev->data == sd->shared_dev_data);
assert(priv->txqs_n == 0);
assert(priv->txqs == NULL);
assert(priv->rxqs_n == 0);
assert(priv->rxqs == NULL);
nb_tx_queues = sd->shared_dev_data->nb_tx_queues;
nb_rx_queues = sd->shared_dev_data->nb_rx_queues;
/* Allocate local storage for queues. */
tx_queues = rte_zmalloc("secondary ethdev->tx_queues",
sizeof(sd->data.tx_queues[0]) * nb_tx_queues,
RTE_CACHE_LINE_SIZE);
rx_queues = rte_zmalloc("secondary ethdev->rx_queues",
sizeof(sd->data.rx_queues[0]) * nb_rx_queues,
RTE_CACHE_LINE_SIZE);
if (tx_queues == NULL || rx_queues == NULL)
goto error;
/* Lock to prevent control operations during setup. */
priv_lock(priv);
/* TX queues. */
for (i = 0; i != nb_tx_queues; ++i) {
struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
struct txq *txq;
if (primary_txq == NULL)
continue;
txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
primary_txq->socket);
if (txq != NULL) {
if (txq_setup(priv->dev,
txq,
primary_txq->elts_n * MLX5_PMD_SGE_WR_N,
primary_txq->socket,
NULL) == 0) {
txq->stats.idx = primary_txq->stats.idx;
tx_queues[i] = txq;
continue;
}
rte_free(txq);
}
while (i) {
txq = tx_queues[--i];
txq_cleanup(txq);
rte_free(txq);
}
goto error;
}
/* RX queues. */
for (i = 0; i != nb_rx_queues; ++i) {
struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
if (primary_rxq == NULL)
continue;
/* Not supported yet. */
rx_queues[i] = NULL;
}
/* Update everything. */
priv->txqs = (void *)tx_queues;
priv->txqs_n = nb_tx_queues;
priv->rxqs = (void *)rx_queues;
priv->rxqs_n = nb_rx_queues;
sd->data.rx_queues = rx_queues;
sd->data.tx_queues = tx_queues;
sd->data.nb_rx_queues = nb_rx_queues;
sd->data.nb_tx_queues = nb_tx_queues;
sd->data.dev_link = sd->shared_dev_data->dev_link;
sd->data.mtu = sd->shared_dev_data->mtu;
memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state,
sizeof(sd->data.rx_queue_state));
memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state,
sizeof(sd->data.tx_queue_state));
sd->data.dev_flags = sd->shared_dev_data->dev_flags;
/* Use local data from now on. */
rte_mb();
priv->dev->data = &sd->data;
rte_mb();
priv->dev->tx_pkt_burst = mlx5_tx_burst;
priv->dev->rx_pkt_burst = removed_rx_burst;
priv_unlock(priv);
end:
/* More sanity checks. */
assert(priv->dev->tx_pkt_burst == mlx5_tx_burst);
assert(priv->dev->rx_pkt_burst == removed_rx_burst);
assert(priv->dev->data == &sd->data);
rte_spinlock_unlock(&sd->lock);
return priv;
error:
priv_unlock(priv);
rte_free(tx_queues);
rte_free(rx_queues);
rte_spinlock_unlock(&sd->lock);
return NULL;
}
static int
test_pmd_perf(void)
{
uint16_t nb_ports, num, nb_lcores, slave_id = (uint16_t)-1;
uint16_t nb_rxd = MAX_TRAFFIC_BURST;
uint16_t nb_txd = MAX_TRAFFIC_BURST;
uint16_t portid;
uint16_t nb_rx_queue = 1, nb_tx_queue = 1;
int socketid = -1;
int ret;
printf("Start PMD RXTX cycles cost test.\n");
signal(SIGUSR1, signal_handler);
signal(SIGUSR2, signal_handler);
nb_ports = rte_eth_dev_count();
if (nb_ports < NB_ETHPORTS_USED) {
printf("At least %u port(s) used for perf. test\n",
NB_ETHPORTS_USED);
return -1;
}
if (nb_ports > RTE_MAX_ETHPORTS)
nb_ports = RTE_MAX_ETHPORTS;
nb_lcores = rte_lcore_count();
memset(lcore_conf, 0, sizeof(lcore_conf));
init_lcores();
init_mbufpool(NB_MBUF);
if (sc_flag == SC_CONTINUOUS) {
nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
nb_txd = RTE_TEST_TX_DESC_DEFAULT;
}
printf("CONFIG RXD=%d TXD=%d\n", nb_rxd, nb_txd);
reset_count();
num = 0;
for (portid = 0; portid < nb_ports; portid++) {
if (socketid == -1) {
socketid = rte_eth_dev_socket_id(portid);
slave_id = alloc_lcore(socketid);
if (slave_id == (uint16_t)-1) {
printf("No avail lcore to run test\n");
return -1;
}
printf("Performance test runs on lcore %u socket %u\n",
slave_id, socketid);
}
if (socketid != rte_eth_dev_socket_id(portid)) {
printf("Skip port %d\n", portid);
continue;
}
/* port configure */
ret = rte_eth_dev_configure(portid, nb_rx_queue,
nb_tx_queue, &port_conf);
if (ret < 0)
rte_exit(EXIT_FAILURE,
"Cannot configure device: err=%d, port=%d\n",
ret, portid);
rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
printf("Port %u ", portid);
print_ethaddr("Address:", &ports_eth_addr[portid]);
printf("\n");
/* tx queue setup */
ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
socketid, &tx_conf);
if (ret < 0)
rte_exit(EXIT_FAILURE,
"rte_eth_tx_queue_setup: err=%d, "
"port=%d\n", ret, portid);
/* rx queue steup */
ret = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
socketid, &rx_conf,
mbufpool[socketid]);
if (ret < 0)
rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d,"
"port=%d\n", ret, portid);
/* Start device */
stop = 0;
ret = rte_eth_dev_start(portid);
if (ret < 0)
rte_exit(EXIT_FAILURE,
"rte_eth_dev_start: err=%d, port=%d\n",
ret, portid);
/* always eanble promiscuous */
rte_eth_promiscuous_enable(portid);
lcore_conf[slave_id].portlist[num++] = portid;
lcore_conf[slave_id].nb_ports++;
}
check_all_ports_link_status(nb_ports, RTE_PORT_ALL);
if (tx_burst == NULL) {
tx_burst = (struct rte_mbuf **)
rte_calloc_socket("tx_buff",
MAX_TRAFFIC_BURST * nb_ports,
sizeof(void *),
RTE_CACHE_LINE_SIZE, socketid);
if (!tx_burst)
return -1;
}
init_traffic(mbufpool[socketid],
tx_burst, MAX_TRAFFIC_BURST * nb_ports);
printf("Generate %d packets @socket %d\n",
MAX_TRAFFIC_BURST * nb_ports, socketid);
if (sc_flag == SC_CONTINUOUS) {
/* do both rxtx by default */
if (NULL == do_measure)
do_measure = measure_rxtx;
rte_eal_remote_launch(main_loop, NULL, slave_id);
if (rte_eal_wait_lcore(slave_id) < 0)
return -1;
} else if (sc_flag == SC_BURST_POLL_FIRST ||
sc_flag == SC_BURST_XMIT_FIRST)
if (exec_burst(sc_flag, slave_id) < 0)
return -1;
/* port tear down */
for (portid = 0; portid < nb_ports; portid++) {
if (socketid != rte_eth_dev_socket_id(portid))
continue;
rte_eth_dev_stop(portid);
}
return 0;
}
static inline int
poll_burst(void *args)
{
#define MAX_IDLE (10000)
unsigned lcore_id;
struct rte_mbuf **pkts_burst;
uint64_t diff_tsc, cur_tsc;
uint16_t next[RTE_MAX_ETHPORTS];
struct lcore_conf *conf;
uint32_t pkt_per_port = *((uint32_t *)args);
unsigned i, portid, nb_rx = 0;
uint64_t total;
uint64_t timeout = MAX_IDLE;
lcore_id = rte_lcore_id();
conf = &lcore_conf[lcore_id];
if (conf->status != LCORE_USED)
return 0;
total = pkt_per_port * conf->nb_ports;
printf("start to receive total expect %"PRIu64"\n", total);
pkts_burst = (struct rte_mbuf **)
rte_calloc_socket("poll_burst",
total, sizeof(void *),
RTE_CACHE_LINE_SIZE, conf->socketid);
if (!pkts_burst)
return -1;
for (i = 0; i < conf->nb_ports; i++) {
portid = conf->portlist[i];
next[portid] = i * pkt_per_port;
}
while (!rte_atomic64_read(&start))
;
cur_tsc = rte_rdtsc();
while (total) {
for (i = 0; i < conf->nb_ports; i++) {
portid = conf->portlist[i];
nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
&pkts_burst[next[portid]],
MAX_PKT_BURST);
if (unlikely(nb_rx == 0)) {
timeout--;
if (unlikely(timeout == 0))
goto timeout;
continue;
}
next[portid] += nb_rx;
total -= nb_rx;
}
}
timeout:
diff_tsc = rte_rdtsc() - cur_tsc;
printf("%"PRIu64" packets lost, IDLE %"PRIu64" times\n",
total, MAX_IDLE - timeout);
/* clean up */
total = pkt_per_port * conf->nb_ports - total;
for (i = 0; i < total; i++)
rte_pktmbuf_free(pkts_burst[i]);
rte_free(pkts_burst);
if (total > 0)
return diff_tsc / total;
else
return -1;
}