static void *
rte_port_fd_writer_create(void *params, int socket_id)
{
struct rte_port_fd_writer_params *conf =
params;
struct rte_port_fd_writer *port;
/* Check input parameters */
if ((conf == NULL) ||
(conf->tx_burst_sz == 0) ||
(conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
(!rte_is_power_of_2(conf->tx_burst_sz))) {
RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
return NULL;
}
/* Memory allocation */
port = rte_zmalloc_socket("PORT", sizeof(*port),
RTE_CACHE_LINE_SIZE, socket_id);
if (port == NULL) {
RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
return NULL;
}
/* Initialization */
port->fd = conf->fd;
port->tx_burst_sz = conf->tx_burst_sz;
port->tx_buf_count = 0;
return port;
}
void init_qinq_gre_table(struct task_args *targ, struct qinq_gre_map *qinq_gre_map)
{
struct rte_table_hash* qinq_gre_table;
uint8_t table_part = targ->nb_slave_threads;
if (!rte_is_power_of_2(table_part)) {
table_part = rte_align32pow2(table_part) >> 1;
}
static int
scheduler_option_set(struct rte_cryptodev *dev, uint32_t option_type,
void *option)
{
struct psd_scheduler_ctx *psd_ctx = ((struct scheduler_ctx *)
dev->data->dev_private)->private_ctx;
uint32_t threshold;
if ((enum rte_cryptodev_schedule_option_type)option_type !=
CDEV_SCHED_OPTION_THRESHOLD) {
CS_LOG_ERR("Option not supported");
return -EINVAL;
}
threshold = ((struct rte_cryptodev_scheduler_threshold_option *)
option)->threshold;
if (!rte_is_power_of_2(threshold)) {
CS_LOG_ERR("Threshold is not power of 2");
return -EINVAL;
}
psd_ctx->threshold = ~(threshold - 1);
return 0;
}
static void *
rte_table_array_create(void *params, int socket_id, uint32_t entry_size)
{
struct rte_table_array_params *p = params;
struct rte_table_array *t;
uint32_t total_cl_size, total_size;
/* Check input parameters */
if ((p == NULL) ||
(p->n_entries == 0) ||
(!rte_is_power_of_2(p->n_entries)))
return NULL;
/* Memory allocation */
total_cl_size = (sizeof(struct rte_table_array) +
RTE_CACHE_LINE_SIZE) / RTE_CACHE_LINE_SIZE;
total_cl_size += (p->n_entries * entry_size +
RTE_CACHE_LINE_SIZE) / RTE_CACHE_LINE_SIZE;
total_size = total_cl_size * RTE_CACHE_LINE_SIZE;
t = rte_zmalloc_socket("TABLE", total_size, RTE_CACHE_LINE_SIZE, socket_id);
if (t == NULL) {
RTE_LOG(ERR, TABLE,
"%s: Cannot allocate %u bytes for array table\n",
__func__, total_size);
return NULL;
}
/* Memory initialization */
t->entry_size = entry_size;
t->n_entries = p->n_entries;
t->offset = p->offset;
t->entry_pos_mask = t->n_entries - 1;
return t;
}
static void *
dpdk_knidev_writer_create(void *params, int socket_id)
{
struct dpdk_knidev_writer_params *conf =
(struct dpdk_knidev_writer_params *) params;
struct dpdk_knidev_writer *port;
/* Check input parameters */
if ((conf == NULL) ||
(conf->tx_burst_sz == 0) ||
(conf->tx_burst_sz > VR_DPDK_TX_BURST_SZ) ||
(!rte_is_power_of_2(conf->tx_burst_sz))) {
RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
return NULL;
}
/* Memory allocation */
port = rte_zmalloc_socket("PORT", sizeof(*port),
RTE_CACHE_LINE_SIZE, socket_id);
if (port == NULL) {
RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
return NULL;
}
/* Initialization */
port->kni = conf->kni;
port->tx_burst_sz = conf->tx_burst_sz;
port->tx_buf_count = 0;
port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
return port;
}
static void
check_txqs(struct app_params *app)
{
uint32_t i;
for (i = 0; i < app->n_pktq_hwq_out; i++) {
struct app_pktq_hwq_out_params *p = &app->hwq_out_params[i];
uint32_t n_writers = app_txq_get_writers(app, p);
APP_CHECK((p->size > 0),
"%s size is 0\n", p->name);
APP_CHECK((rte_is_power_of_2(p->size)),
"%s size is not a power of 2\n", p->name);
APP_CHECK((p->burst > 0),
"%s burst size is 0\n", p->name);
APP_CHECK((p->burst <= p->size),
"%s burst size is bigger than its size\n", p->name);
APP_CHECK((n_writers != 0),
"%s has no writer\n", p->name);
APP_CHECK((n_writers == 1),
"%s has more than one writer\n", p->name);
}
}
static int
app_install_port_mask(const char *arg)
{
char *end = NULL;
uint64_t port_mask;
uint32_t i;
if (arg[0] == '\0')
return -1;
port_mask = strtoul(arg, &end, 16);
if ((end == NULL) || (*end != '\0'))
return -2;
if (port_mask == 0)
return -3;
app.n_ports = 0;
for (i = 0; i < 64; i++) {
if ((port_mask & (1LLU << i)) == 0)
continue;
if (app.n_ports >= APP_MAX_PORTS)
return -4;
app.ports[app.n_ports] = i;
app.n_ports++;
}
if (!rte_is_power_of_2(app.n_ports))
return -5;
return 0;
}
void init_cpe6_table(struct task_args *targ)
{
char name[64];
sprintf(name, "core_%u_CPEv6Table", targ->lconf->id);
uint8_t table_part = targ->nb_slave_threads;
if (!rte_is_power_of_2(table_part)) {
table_part = rte_align32pow2(table_part) >> 1;
}
static void init_task_lb_qinq(struct task_base *tbase, struct task_args *targ)
{
struct task_lb_qinq *task = (struct task_lb_qinq *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
task->qinq_tag = targ->qinq_tag;
task->nb_worker_threads = targ->nb_worker_threads;
task->bit_mask = rte_is_power_of_2(targ->nb_worker_threads) ? targ->nb_worker_threads - 1 : 0xff;
/* The load distributor is sending to a set of cores. These
cores are responsible for handling a set of flows
identified by a qinq tag. The load distributor identifies
the flows and forwards them to the appropriate worker. The
mapping from flow to worker is stored within the
work_table. Build the worker_table by asking each worker
which flows are handled. */
task->worker_table = prox_zmalloc(0x1000000, socket_id);
for (int i = 0; i < targ->nb_worker_threads; ++i) {
struct core_task ct = targ->core_task_set[0].core_task[i];
struct task_args *t = core_targ_get(ct.core, ct.task);
PROX_PANIC(t->task_init->flow_iter.beg == NULL,
"Load distributor can't find flows owned by destination worker %d\n", i);
struct flow_iter *it = &t->task_init->flow_iter;
int cnt = 0;
for (it->beg(it, t); !it->is_end(it, t); it->next(it, t)) {
uint16_t svlan = it->get_svlan(it, t);
uint16_t cvlan = it->get_cvlan(it, t);
task->worker_table[PKT_TO_LUTQINQ(svlan, cvlan)] = i;
}
}
/* Check which protocols we are allowed to send to worker tasks */
for (int i = 0; i < MAX_PROTOCOLS; ++i) {
int is_active = !!targ->core_task_set[i].n_elems;
task->protocols_mask |= is_active << i;
}
plog_info("\t\ttask_lb_qinq protocols_mask = 0x%x\n", task->protocols_mask);
if (targ->task_init->flag_features & TASK_FEATURE_LUT_QINQ_RSS)
tbase->flags |= BASE_FLAG_LUT_QINQ_RSS;
if (targ->task_init->flag_features & TASK_FEATURE_LUT_QINQ_HASH)
tbase->flags |= BASE_FLAG_LUT_QINQ_HASH;
plog_info("\t\ttask_lb_qinq flags = 0x%x\n", tbase->flags);
}
/**
* Internal helper to allocate memory once for several disparate objects.
*
* The most restrictive alignment constraint for standard objects is assumed
* to be sizeof(double) and is used as a default value.
*
* C11 code would include stdalign.h and use alignof(max_align_t) however
* we'll stick with C99 for the time being.
*/
static inline size_t
mlx4_mallocv_inline(const char *type, const struct mlx4_malloc_vec *vec,
unsigned int cnt, int zero, int socket)
{
unsigned int i;
size_t size;
size_t least;
uint8_t *data = NULL;
int fill = !vec[0].addr;
fill:
size = 0;
least = 0;
for (i = 0; i < cnt; ++i) {
size_t align = (uintptr_t)vec[i].align;
if (!align) {
align = sizeof(double);
} else if (!rte_is_power_of_2(align)) {
rte_errno = EINVAL;
goto error;
}
if (least < align)
least = align;
align = RTE_ALIGN_CEIL(size, align);
size = align + vec[i].size;
if (fill && vec[i].addr)
*vec[i].addr = data + align;
}
if (fill)
return size;
if (!zero)
data = rte_malloc_socket(type, size, least, socket);
else
data = rte_zmalloc_socket(type, size, least, socket);
if (data) {
fill = 1;
goto fill;
}
rte_errno = ENOMEM;
error:
for (i = 0; i != cnt; ++i)
if (vec[i].addr)
*vec[i].addr = NULL;
return 0;
}
int
sfc_ev_qinit(struct sfc_adapter *sa,
enum sfc_evq_type type, unsigned int type_index,
unsigned int entries, int socket_id, struct sfc_evq **evqp)
{
struct sfc_evq *evq;
int rc;
sfc_log_init(sa, "type=%s type_index=%u",
sfc_evq_type2str(type), type_index);
SFC_ASSERT(rte_is_power_of_2(entries));
rc = ENOMEM;
evq = rte_zmalloc_socket("sfc-evq", sizeof(*evq), RTE_CACHE_LINE_SIZE,
socket_id);
if (evq == NULL)
goto fail_evq_alloc;
evq->sa = sa;
evq->type = type;
evq->entries = entries;
/* Allocate DMA space */
rc = sfc_dma_alloc(sa, sfc_evq_type2str(type), type_index,
EFX_EVQ_SIZE(evq->entries), socket_id, &evq->mem);
if (rc != 0)
goto fail_dma_alloc;
evq->init_state = SFC_EVQ_INITIALIZED;
sa->evq_count++;
*evqp = evq;
return 0;
fail_dma_alloc:
rte_free(evq);
fail_evq_alloc:
sfc_log_init(sa, "failed %d", rc);
return rc;
}
static void
check_mempools(struct app_params *app)
{
uint32_t i;
for (i = 0; i < app->n_mempools; i++) {
struct app_mempool_params *p = &app->mempool_params[i];
APP_CHECK((p->pool_size > 0),
"Mempool %s size is 0\n", p->name);
APP_CHECK((p->cache_size > 0),
"Mempool %s cache size is 0\n", p->name);
APP_CHECK(rte_is_power_of_2(p->cache_size),
"Mempool %s cache size not a power of 2\n", p->name);
}
}
/*
* Allocate memory on specified heap.
*/
void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
/* return NULL if size is 0 or alignment is not power-of-2 */
if (size == 0 || !rte_is_power_of_2(align))
return NULL;
if (socket == SOCKET_ID_ANY)
socket = malloc_get_numa_socket();
/* Check socket parameter */
if (socket >= RTE_MAX_NUMA_NODES)
return NULL;
return malloc_heap_alloc(&mcfg->malloc_heaps[socket], type,
size, align == 0 ? 1 : align);
}
static void
check_swqs(struct app_params *app)
{
uint32_t i;
for (i = 0; i < app->n_pktq_swq; i++) {
struct app_pktq_swq_params *p = &app->swq_params[i];
uint32_t n_readers = app_swq_get_readers(app, p);
uint32_t n_writers = app_swq_get_writers(app, p);
APP_CHECK((p->size > 0),
"%s size is 0\n", p->name);
APP_CHECK((rte_is_power_of_2(p->size)),
"%s size is not a power of 2\n", p->name);
APP_CHECK((p->burst_read > 0),
"%s read burst size is 0\n", p->name);
APP_CHECK((p->burst_read <= p->size),
"%s read burst size is bigger than its size\n",
p->name);
APP_CHECK((p->burst_write > 0),
"%s write burst size is 0\n", p->name);
APP_CHECK((p->burst_write <= p->size),
"%s write burst size is bigger than its size\n",
p->name);
APP_CHECK((n_readers != 0),
"%s has no reader\n", p->name);
APP_CHECK((n_readers == 1),
"%s has more than one reader\n", p->name);
APP_CHECK((n_writers != 0),
"%s has no writer\n", p->name);
APP_CHECK((n_writers == 1),
"%s has more than one writer\n", p->name);
}
}
/* create fragmentation table */
struct rte_ip_frag_tbl *
rte_ip_frag_table_create(uint32_t bucket_num, uint32_t bucket_entries,
uint32_t max_entries, uint64_t max_cycles, int socket_id)
{
struct rte_ip_frag_tbl *tbl;
size_t sz;
uint64_t nb_entries;
nb_entries = rte_align32pow2(bucket_num);
nb_entries *= bucket_entries;
nb_entries *= IP_FRAG_HASH_FNUM;
/* check input parameters. */
if (rte_is_power_of_2(bucket_entries) == 0 ||
nb_entries > UINT32_MAX || nb_entries == 0 ||
nb_entries < max_entries) {
RTE_LOG(ERR, USER1, "%s: invalid input parameter\n", __func__);
return (NULL);
}
sz = sizeof (*tbl) + nb_entries * sizeof (tbl->pkt[0]);
if ((tbl = rte_zmalloc_socket(__func__, sz, CACHE_LINE_SIZE,
socket_id)) == NULL) {
RTE_LOG(ERR, USER1,
"%s: allocation of %zu bytes at socket %d failed do\n",
__func__, sz, socket_id);
return (NULL);
}
RTE_LOG(INFO, USER1, "%s: allocated of %zu bytes at socket %d\n",
__func__, sz, socket_id);
tbl->max_cycles = max_cycles;
tbl->max_entries = max_entries;
tbl->nb_entries = (uint32_t)nb_entries;
tbl->nb_buckets = bucket_num;
tbl->bucket_entries = bucket_entries;
tbl->entry_mask = (tbl->nb_entries - 1) & ~(tbl->bucket_entries - 1);
TAILQ_INIT(&(tbl->lru));
return (tbl);
}
int
ark_mpu_configure(struct ark_mpu_t *mpu, phys_addr_t ring, uint32_t ring_size,
int is_tx)
{
ark_mpu_reset(mpu);
if (!rte_is_power_of_2(ring_size)) {
PMD_DRV_LOG(ERR, "ARK: Invalid ring size for MPU %d\n",
ring_size);
return -1;
}
mpu->cfg.ring_base = ring;
mpu->cfg.ring_size = ring_size;
mpu->cfg.ring_mask = ring_size - 1;
mpu->cfg.min_host_move = is_tx ? 1 : mpu->hw.obj_per_mrr;
mpu->cfg.min_hw_move = mpu->hw.obj_per_mrr;
mpu->cfg.sw_prod_index = 0;
mpu->cfg.hw_cons_index = 0;
return 0;
}
static void *
rte_port_ethdev_writer_nodrop_create(void *params, int socket_id)
{
struct rte_port_ethdev_writer_nodrop_params *conf =
(struct rte_port_ethdev_writer_nodrop_params *) params;
struct rte_port_ethdev_writer_nodrop *port;
/* Check input parameters */
if ((conf == NULL) ||
(conf->tx_burst_sz == 0) ||
(conf->tx_burst_sz > RTE_PORT_IN_BURST_SIZE_MAX) ||
(!rte_is_power_of_2(conf->tx_burst_sz))) {
RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
return NULL;
}
/* Memory allocation */
port = rte_zmalloc_socket("PORT", sizeof(*port),
RTE_CACHE_LINE_SIZE, socket_id);
if (port == NULL) {
RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
return NULL;
}
/* Initialization */
port->port_id = conf->port_id;
port->queue_id = conf->queue_id;
port->tx_burst_sz = conf->tx_burst_sz;
port->tx_buf_count = 0;
port->bsz_mask = 1LLU << (conf->tx_burst_sz - 1);
/*
* When n_retries is 0 it means that we should wait for every packet to
* send no matter how many retries should it take. To limit number of
* branches in fast path, we use UINT64_MAX instead of branching.
*/
port->n_retries = (conf->n_retries == 0) ? UINT64_MAX : conf->n_retries;
return port;
}
/*
* Allocate memory on specified heap.
*/
void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
{
struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
int socket, i;
void *ret;
/* return NULL if size is 0 or alignment is not power-of-2 */
if (size == 0 || !rte_is_power_of_2(align))
return NULL;
if (socket_arg == SOCKET_ID_ANY)
socket = malloc_get_numa_socket();
else
socket = socket_arg;
/* Check socket parameter */
if (socket >= RTE_MAX_NUMA_NODES)
return NULL;
ret = malloc_heap_alloc(&mcfg->malloc_heaps[socket], type,
size, align == 0 ? 1 : align);
if (ret != NULL || socket_arg != SOCKET_ID_ANY)
return ret;
/* try other heaps */
for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
/* we already tried this one */
if (i == socket)
continue;
ret = malloc_heap_alloc(&mcfg->malloc_heaps[i], type,
size, align == 0 ? 1 : align);
if (ret != NULL)
return ret;
}
return NULL;
}
static int
test_align(void)
{
#define FAIL_ALIGN(x, i, p)\
{printf(x "() test failed: %u %u\n", i, p);\
return -1;}
#define ERROR_FLOOR(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((res / pow) != (i / pow)) /* check if correct alignment */
#define ERROR_CEIL(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((i % pow) == 0 ? /* check if ceiling is invoked */ \
val / pow != i / pow : /* if aligned */ \
val / pow != (i / pow) + 1) /* if not aligned, hence +1 */
uint32_t i, p, val;
for (i = 1, p = 1; i <= MAX_NUM; i ++) {
if (rte_align32pow2(i) != p)
FAIL_ALIGN("rte_align32pow2", i, p);
if (i == p)
p <<= 1;
}
for (p = 2; p <= MAX_NUM; p <<= 1) {
if (!rte_is_power_of_2(p))
FAIL("rte_is_power_of_2");
for (i = 1; i <= MAX_NUM; i++) {
/* align floor */
if (RTE_ALIGN_FLOOR((uintptr_t)i, p) % p)
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
val = RTE_PTR_ALIGN_FLOOR((uintptr_t) i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_FLOOR", i, p);
val = RTE_ALIGN_FLOOR(i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
/* align ceiling */
val = RTE_PTR_ALIGN((uintptr_t) i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN", i, p);
val = RTE_ALIGN(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN", i, p);
val = RTE_ALIGN_CEIL(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN_CEIL", i, p);
val = RTE_PTR_ALIGN_CEIL((uintptr_t)i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_CEIL", i, p);
/* by this point we know that val is aligned to p */
if (!rte_is_aligned((void*)(uintptr_t) val, p))
FAIL("rte_is_aligned");
}
}
return 0;
}
static struct _mempool_gntalloc_info
_create_mempool(const char *name, unsigned elt_num, unsigned elt_size,
unsigned cache_size, unsigned private_data_size,
rte_mempool_ctor_t *mp_init, void *mp_init_arg,
rte_mempool_obj_cb_t *obj_init, void *obj_init_arg,
int socket_id, unsigned flags)
{
struct _mempool_gntalloc_info mgi;
struct rte_mempool *mp = NULL;
struct rte_mempool_objsz objsz;
uint32_t pg_num, rpg_num, pg_shift, pg_sz;
char *va, *orig_va, *uv; /* uv: from which, the pages could be freed */
ssize_t sz, usz; /* usz: unused size */
/*
* for each page allocated through xen_gntalloc driver,
* gref_arr:stores grant references,
* pa_arr: stores physical address,
* gnt_arr: stores all meta dat
*/
uint32_t *gref_arr = NULL;
phys_addr_t *pa_arr = NULL;
struct _gntarr *gnt_arr = NULL;
/* start index of the grant referances, used for dealloc*/
uint64_t start_index;
uint32_t i, j;
int rv = 0;
struct ioctl_gntalloc_dealloc_gref arg;
mgi.mp = NULL;
va = orig_va = uv = NULL;
pg_num = rpg_num = 0;
sz = 0;
pg_sz = getpagesize();
if (rte_is_power_of_2(pg_sz) == 0) {
goto out;
}
pg_shift = rte_bsf32(pg_sz);
rte_mempool_calc_obj_size(elt_size, flags, &objsz);
sz = rte_mempool_xmem_size(elt_num, objsz.total_size, pg_shift);
pg_num = sz >> pg_shift;
pa_arr = calloc(pg_num, sizeof(pa_arr[0]));
gref_arr = calloc(pg_num, sizeof(gref_arr[0]));
gnt_arr = calloc(pg_num, sizeof(gnt_arr[0]));
if ((gnt_arr == NULL) || (gref_arr == NULL) || (pa_arr == NULL))
goto out;
/* grant index is continuous in ascending order */
orig_va = gntalloc(sz, gref_arr, &start_index);
if (orig_va == NULL)
goto out;
get_phys_map(orig_va, pa_arr, pg_num, pg_sz);
for (i = 0; i < pg_num; i++) {
gnt_arr[i].index = start_index + i * pg_sz;
gnt_arr[i].gref = gref_arr[i];
gnt_arr[i].pa = pa_arr[i];
gnt_arr[i].va = RTE_PTR_ADD(orig_va, i * pg_sz);
}
qsort(gnt_arr, pg_num, sizeof(struct _gntarr), compare);
va = get_xen_virtual(sz, pg_sz);
if (va == NULL) {
goto out;
}
/*
* map one by one, as index isn't continuous now.
* pg_num VMAs, doesn't linux has a limitation on this?
*/
for (i = 0; i < pg_num; i++) {
/* update gref_arr and pa_arr after sort */
gref_arr[i] = gnt_arr[i].gref;
pa_arr[i] = gnt_arr[i].pa;
gnt_arr[i].va = mmap(va + i * pg_sz, pg_sz, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_FIXED, gntalloc_fd, gnt_arr[i].index);
if ((gnt_arr[i].va == MAP_FAILED) || (gnt_arr[i].va != (va + i * pg_sz))) {
RTE_LOG(ERR, PMD, "failed to map %d pages\n", i);
goto mmap_failed;
}
}
/*
* Check that allocated size is big enough to hold elt_num
* objects and a calcualte how many bytes are actually required.
*/
usz = rte_mempool_xmem_usage(va, elt_num, objsz.total_size, pa_arr, pg_num, pg_shift);
if (usz < 0) {
mp = NULL;
i = pg_num;
goto mmap_failed;
} else {
/* unmap unused pages if any */
uv = RTE_PTR_ADD(va, usz);
if ((usz = va + sz - uv) > 0) {
RTE_LOG(ERR, PMD,
"%s(%s): unmap unused %zu of %zu "
"mmaped bytes @%p orig:%p\n",
__func__, name, usz, sz, uv, va);
munmap(uv, usz);
i = (sz - usz) / pg_sz;
for (; i < pg_num; i++) {
arg.count = 1;
arg.index = gnt_arr[i].index;
rv = ioctl(gntalloc_fd, IOCTL_GNTALLOC_DEALLOC_GREF, &arg);
if (rv) {
/* shouldn't fail here */
RTE_LOG(ERR, PMD, "va=%p pa=%"PRIu64"x index=%"PRIu64" %s\n",
gnt_arr[i].va,
gnt_arr[i].pa,
arg.index, strerror(errno));
rte_panic("gntdealloc failed when freeing pages\n");
}
}
rpg_num = (sz - usz) >> pg_shift;
} else
static const struct rte_memzone *
memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
int socket_id, unsigned flags, unsigned align, unsigned bound)
{
struct rte_memzone *mz;
struct rte_mem_config *mcfg;
size_t requested_len;
int socket, i;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
/* no more room in config */
if (mcfg->memzone_cnt >= RTE_MAX_MEMZONE) {
RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__);
rte_errno = ENOSPC;
return NULL;
}
/* zone already exist */
if ((memzone_lookup_thread_unsafe(name)) != NULL) {
RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
__func__, name);
rte_errno = EEXIST;
return NULL;
}
if (strlen(name) >= sizeof(mz->name) - 1) {
RTE_LOG(DEBUG, EAL, "%s(): memzone <%s>: name too long\n",
__func__, name);
rte_errno = EEXIST;
return NULL;
}
/* if alignment is not a power of two */
if (align && !rte_is_power_of_2(align)) {
RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
align);
rte_errno = EINVAL;
return NULL;
}
/* alignment less than cache size is not allowed */
if (align < RTE_CACHE_LINE_SIZE)
align = RTE_CACHE_LINE_SIZE;
/* align length on cache boundary. Check for overflow before doing so */
if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
rte_errno = EINVAL; /* requested size too big */
return NULL;
}
len += RTE_CACHE_LINE_MASK;
len &= ~((size_t) RTE_CACHE_LINE_MASK);
/* save minimal requested length */
requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
/* check that boundary condition is valid */
if (bound != 0 && (requested_len > bound || !rte_is_power_of_2(bound))) {
rte_errno = EINVAL;
return NULL;
}
if ((socket_id != SOCKET_ID_ANY) && (socket_id >= RTE_MAX_NUMA_NODES)) {
rte_errno = EINVAL;
return NULL;
}
if (!rte_eal_has_hugepages())
socket_id = SOCKET_ID_ANY;
if (len == 0) {
if (bound != 0)
requested_len = bound;
else {
requested_len = find_heap_max_free_elem(&socket_id, align);
if (requested_len == 0) {
rte_errno = ENOMEM;
return NULL;
}
}
}
if (socket_id == SOCKET_ID_ANY)
socket = malloc_get_numa_socket();
else
socket = socket_id;
/* allocate memory on heap */
void *mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[socket], NULL,
requested_len, flags, align, bound);
if ((mz_addr == NULL) && (socket_id == SOCKET_ID_ANY)) {
/* try other heaps */
for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
if (socket == i)
continue;
mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[i],
NULL, requested_len, flags, align, bound);
if (mz_addr != NULL)
break;
}
}
if (mz_addr == NULL) {
rte_errno = ENOMEM;
return NULL;
}
const struct malloc_elem *elem = malloc_elem_from_data(mz_addr);
/* fill the zone in config */
mz = get_next_free_memzone();
if (mz == NULL) {
RTE_LOG(ERR, EAL, "%s(): Cannot find free memzone but there is room "
"in config!\n", __func__);
rte_errno = ENOSPC;
return NULL;
}
mcfg->memzone_cnt++;
snprintf(mz->name, sizeof(mz->name), "%s", name);
mz->phys_addr = rte_malloc_virt2phy(mz_addr);
mz->addr = mz_addr;
mz->len = (requested_len == 0 ? elem->size : requested_len);
mz->hugepage_sz = elem->ms->hugepage_sz;
mz->socket_id = elem->ms->socket_id;
mz->flags = 0;
mz->memseg_id = elem->ms - rte_eal_get_configuration()->mem_config->memseg;
return mz;
}
static void
check_swqs(struct app_params *app)
{
uint32_t i;
for (i = 0; i < app->n_pktq_swq; i++) {
struct app_pktq_swq_params *p = &app->swq_params[i];
uint32_t n_readers = app_swq_get_readers(app, p);
uint32_t n_writers = app_swq_get_writers(app, p);
uint32_t n_flags;
APP_CHECK((p->size > 0),
"%s size is 0\n", p->name);
APP_CHECK((rte_is_power_of_2(p->size)),
"%s size is not a power of 2\n", p->name);
APP_CHECK((p->burst_read > 0),
"%s read burst size is 0\n", p->name);
APP_CHECK((p->burst_read <= p->size),
"%s read burst size is bigger than its size\n",
p->name);
APP_CHECK((p->burst_write > 0),
"%s write burst size is 0\n", p->name);
APP_CHECK((p->burst_write <= p->size),
"%s write burst size is bigger than its size\n",
p->name);
APP_CHECK((n_readers != 0),
"%s has no reader\n", p->name);
if (n_readers > 1)
APP_LOG(app, LOW, "%s has more than one reader", p->name);
APP_CHECK((n_writers != 0),
"%s has no writer\n", p->name);
if (n_writers > 1)
APP_LOG(app, LOW, "%s has more than one writer", p->name);
n_flags = p->ipv4_frag + p->ipv6_frag + p->ipv4_ras + p->ipv6_ras;
APP_CHECK((n_flags < 2),
"%s has more than one fragmentation or reassembly mode enabled\n",
p->name);
APP_CHECK((!((n_readers > 1) && (n_flags == 1))),
"%s has more than one reader when fragmentation or reassembly"
" mode enabled\n",
p->name);
APP_CHECK((!((n_writers > 1) && (n_flags == 1))),
"%s has more than one writer when fragmentation or reassembly"
" mode enabled\n",
p->name);
n_flags = p->ipv4_ras + p->ipv6_ras;
APP_CHECK((!((p->dropless == 1) && (n_flags == 1))),
"%s has dropless when reassembly mode enabled\n", p->name);
n_flags = p->ipv4_frag + p->ipv6_frag;
if (n_flags == 1) {
uint16_t ip_hdr_size = (p->ipv4_frag) ? sizeof(struct ipv4_hdr) :
sizeof(struct ipv6_hdr);
APP_CHECK((p->mtu > ip_hdr_size),
"%s mtu size is smaller than ip header\n", p->name);
APP_CHECK((!((p->mtu - ip_hdr_size) % 8)),
"%s mtu size is incorrect\n", p->name);
}
}
}
int
grant_node_create(uint32_t pg_num, uint32_t *gref_arr, phys_addr_t *pa_arr, char *val_str, size_t str_size)
{
uint64_t start_index;
int pg_size;
uint32_t pg_shift;
void *ptr = NULL;
uint32_t count, entries_per_pg;
uint32_t i, j = 0, k = 0;
uint32_t *gref_tmp;
int first = 1;
char tmp_str[PATH_MAX] = {0};
int rv = -1;
pg_size = getpagesize();
if (rte_is_power_of_2(pg_size) == 0) {
return -1;
}
pg_shift = rte_bsf32(pg_size);
if (pg_size % sizeof(struct grant_node_item)) {
RTE_LOG(ERR, PMD, "pg_size isn't a multiple of grant node item\n");
return -1;
}
entries_per_pg = pg_size / sizeof(struct grant_node_item);
count = (pg_num + entries_per_pg - 1 ) / entries_per_pg;
gref_tmp = malloc(count * sizeof(uint32_t));
if (gref_tmp == NULL)
return -1;
ptr = gntalloc(pg_size * count, gref_tmp, &start_index);
if (ptr == NULL) {
RTE_LOG(ERR, PMD, "%s: gntalloc error of %d pages\n", __func__, count);
free(gref_tmp);
return -1;
}
while (j < pg_num) {
if (first) {
rv = snprintf(val_str, str_size, "%u", gref_tmp[k]);
first = 0;
} else {
snprintf(tmp_str, PATH_MAX, "%s", val_str);
rv = snprintf(val_str, str_size, "%s,%u", tmp_str, gref_tmp[k]);
}
k++;
if (rv == -1)
break;
for (i = 0; i < entries_per_pg && j < pg_num ; i++) {
((struct grant_node_item *)ptr)->gref = gref_arr[j];
((struct grant_node_item *)ptr)->pfn = pa_arr[j] >> pg_shift;
ptr = RTE_PTR_ADD(ptr, sizeof(struct grant_node_item));
j++;
}
}
if (rv == -1) {
gntfree(ptr, pg_size * count, start_index);
} else
rv = 0;
free(gref_tmp);
return rv;
}
static const struct rte_memzone *
memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
int socket_id, unsigned flags, unsigned align, unsigned bound)
{
struct rte_mem_config *mcfg;
unsigned i = 0;
int memseg_idx = -1;
uint64_t addr_offset, seg_offset = 0;
size_t requested_len;
size_t memseg_len = 0;
phys_addr_t memseg_physaddr;
void *memseg_addr;
/* get pointer to global configuration */
mcfg = rte_eal_get_configuration()->mem_config;
/* no more room in config */
if (mcfg->memzone_idx >= RTE_MAX_MEMZONE) {
RTE_LOG(ERR, EAL, "%s(): No more room in config\n", __func__);
rte_errno = ENOSPC;
return NULL;
}
/* zone already exist */
if ((memzone_lookup_thread_unsafe(name)) != NULL) {
RTE_LOG(DEBUG, EAL, "%s(): memzone <%s> already exists\n",
__func__, name);
rte_errno = EEXIST;
return NULL;
}
/* if alignment is not a power of two */
if (align && !rte_is_power_of_2(align)) {
RTE_LOG(ERR, EAL, "%s(): Invalid alignment: %u\n", __func__,
align);
rte_errno = EINVAL;
return NULL;
}
/* alignment less than cache size is not allowed */
if (align < RTE_CACHE_LINE_SIZE)
align = RTE_CACHE_LINE_SIZE;
/* align length on cache boundary. Check for overflow before doing so */
if (len > SIZE_MAX - RTE_CACHE_LINE_MASK) {
rte_errno = EINVAL; /* requested size too big */
return NULL;
}
len += RTE_CACHE_LINE_MASK;
len &= ~((size_t) RTE_CACHE_LINE_MASK);
/* save minimal requested length */
requested_len = RTE_MAX((size_t)RTE_CACHE_LINE_SIZE, len);
/* check that boundary condition is valid */
if (bound != 0 &&
(requested_len > bound || !rte_is_power_of_2(bound))) {
rte_errno = EINVAL;
return NULL;
}
/* find the smallest segment matching requirements */
for (i = 0; i < RTE_MAX_MEMSEG; i++) {
/* last segment */
if (free_memseg[i].addr == NULL)
break;
/* empty segment, skip it */
if (free_memseg[i].len == 0)
continue;
/* bad socket ID */
if (socket_id != SOCKET_ID_ANY &&
free_memseg[i].socket_id != SOCKET_ID_ANY &&
socket_id != free_memseg[i].socket_id)
continue;
/*
* calculate offset to closest alignment that
* meets boundary conditions.
*/
addr_offset = align_phys_boundary(free_memseg + i,
requested_len, align, bound);
/* check len */
if ((requested_len + addr_offset) > free_memseg[i].len)
continue;
/* check flags for hugepage sizes */
if ((flags & RTE_MEMZONE_2MB) &&
free_memseg[i].hugepage_sz == RTE_PGSIZE_1G)
continue;
if ((flags & RTE_MEMZONE_1GB) &&
free_memseg[i].hugepage_sz == RTE_PGSIZE_2M)
continue;
if ((flags & RTE_MEMZONE_16MB) &&
free_memseg[i].hugepage_sz == RTE_PGSIZE_16G)
continue;
if ((flags & RTE_MEMZONE_16GB) &&
free_memseg[i].hugepage_sz == RTE_PGSIZE_16M)
continue;
/* this segment is the best until now */
if (memseg_idx == -1) {
memseg_idx = i;
memseg_len = free_memseg[i].len;
seg_offset = addr_offset;
}
/* find the biggest contiguous zone */
else if (len == 0) {
if (free_memseg[i].len > memseg_len) {
memseg_idx = i;
memseg_len = free_memseg[i].len;
seg_offset = addr_offset;
}
}
/*
* find the smallest (we already checked that current
* zone length is > len
*/
else if (free_memseg[i].len + align < memseg_len ||
(free_memseg[i].len <= memseg_len + align &&
addr_offset < seg_offset)) {
memseg_idx = i;
memseg_len = free_memseg[i].len;
seg_offset = addr_offset;
}
}
/* no segment found */
if (memseg_idx == -1) {
/*
* If RTE_MEMZONE_SIZE_HINT_ONLY flag is specified,
* try allocating again without the size parameter otherwise -fail.
*/
if ((flags & RTE_MEMZONE_SIZE_HINT_ONLY) &&
((flags & RTE_MEMZONE_1GB) || (flags & RTE_MEMZONE_2MB)
|| (flags & RTE_MEMZONE_16MB) || (flags & RTE_MEMZONE_16GB)))
return memzone_reserve_aligned_thread_unsafe(name,
len, socket_id, 0, align, bound);
rte_errno = ENOMEM;
return NULL;
}
/* save aligned physical and virtual addresses */
memseg_physaddr = free_memseg[memseg_idx].phys_addr + seg_offset;
memseg_addr = RTE_PTR_ADD(free_memseg[memseg_idx].addr,
(uintptr_t) seg_offset);
/* if we are looking for a biggest memzone */
if (len == 0) {
if (bound == 0)
requested_len = memseg_len - seg_offset;
else
requested_len = RTE_ALIGN_CEIL(memseg_physaddr + 1,
bound) - memseg_physaddr;
}
/* set length to correct value */
len = (size_t)seg_offset + requested_len;
/* update our internal state */
free_memseg[memseg_idx].len -= len;
free_memseg[memseg_idx].phys_addr += len;
free_memseg[memseg_idx].addr =
(char *)free_memseg[memseg_idx].addr + len;
/* fill the zone in config */
struct rte_memzone *mz = &mcfg->memzone[mcfg->memzone_idx++];
snprintf(mz->name, sizeof(mz->name), "%s", name);
mz->phys_addr = memseg_physaddr;
mz->addr = memseg_addr;
mz->len = requested_len;
mz->hugepage_sz = free_memseg[memseg_idx].hugepage_sz;
mz->socket_id = free_memseg[memseg_idx].socket_id;
mz->flags = 0;
mz->memseg_id = memseg_idx;
return mz;
}
static void
check_msgqs(struct app_params *app)
{
uint32_t i;
for (i = 0; i < app->n_msgq; i++) {
struct app_msgq_params *p = &app->msgq_params[i];
uint32_t n_readers = app_msgq_get_readers(app, p);
uint32_t n_writers = app_msgq_get_writers(app, p);
uint32_t msgq_req_pipeline, msgq_rsp_pipeline;
uint32_t msgq_req_core, msgq_rsp_core;
APP_CHECK((p->size > 0),
"%s size is 0\n", p->name);
APP_CHECK((rte_is_power_of_2(p->size)),
"%s size is not a power of 2\n", p->name);
msgq_req_pipeline = (strncmp(p->name, "MSGQ-REQ-PIPELINE",
strlen("MSGQ-REQ-PIPELINE")) == 0);
msgq_rsp_pipeline = (strncmp(p->name, "MSGQ-RSP-PIPELINE",
strlen("MSGQ-RSP-PIPELINE")) == 0);
msgq_req_core = (strncmp(p->name, "MSGQ-REQ-CORE",
strlen("MSGQ-REQ-CORE")) == 0);
msgq_rsp_core = (strncmp(p->name, "MSGQ-RSP-CORE",
strlen("MSGQ-RSP-CORE")) == 0);
if ((msgq_req_pipeline == 0) &&
(msgq_rsp_pipeline == 0) &&
(msgq_req_core == 0) &&
(msgq_rsp_core == 0)) {
APP_CHECK((n_readers != 0),
"%s has no reader\n", p->name);
APP_CHECK((n_readers == 1),
"%s has more than one reader\n", p->name);
APP_CHECK((n_writers != 0),
"%s has no writer\n", p->name);
APP_CHECK((n_writers == 1),
"%s has more than one writer\n", p->name);
}
if (msgq_req_pipeline) {
struct app_pipeline_params *pipeline;
uint32_t pipeline_id;
APP_PARAM_GET_ID(p, "MSGQ-REQ-PIPELINE", pipeline_id);
APP_PARAM_FIND_BY_ID(app->pipeline_params,
"PIPELINE",
pipeline_id,
pipeline);
APP_CHECK((pipeline != NULL),
"%s is not associated with a valid pipeline\n",
p->name);
}
if (msgq_rsp_pipeline) {
struct app_pipeline_params *pipeline;
uint32_t pipeline_id;
APP_PARAM_GET_ID(p, "MSGQ-RSP-PIPELINE", pipeline_id);
APP_PARAM_FIND_BY_ID(app->pipeline_params,
"PIPELINE",
pipeline_id,
pipeline);
APP_CHECK((pipeline != NULL),
"%s is not associated with a valid pipeline\n",
p->name);
}
}
}
/* Initialize cores and allocate mempools */
static void init_lcores(void)
{
char name[64];
struct lcore_cfg *lconf = 0;
static uint8_t *worker_thread_table[MAX_SOCKETS] = {0};
static uint16_t *user_table[MAX_SOCKETS] = {0};
struct rte_lpm *ipv4_lpm[MAX_SOCKETS] = {0};
struct rte_hash *qinq_to_gre_lookup[MAX_SOCKETS] = {0};
struct next_hop_struct *next_hop[MAX_SOCKETS] = {0};
/* need to allocate mempools as the first thing to use the lowest possible address range */
setup_mempools(lcore_cfg_init);
lcore_cfg = rte_zmalloc_socket("lcore_cfg_hp", RTE_MAX_LCORE * sizeof(struct lcore_cfg), CACHE_LINE_SIZE, rte_socket_id());
TGEN_PANIC(lcore_cfg == NULL, "Could not allocate memory for core control structures\n");
rte_memcpy(lcore_cfg, lcore_cfg_init, RTE_MAX_LCORE * sizeof(struct lcore_cfg));
init_lcore_info();
check_no_mode_core();
mprintf("=== Initializing rings on cores ===\n");
init_rings();
for (uint8_t socket_id = 0; socket_id < MAX_SOCKETS; ++socket_id) {
uint16_t data_structs_flags = data_structs_needed(lconf, socket_id);
if (data_structs_flags & DATA_STRUCTS_NEED_WT_TABLE) {
worker_thread_table[socket_id] = rte_zmalloc_socket(NULL , 0x1000000, CACHE_LINE_SIZE, socket_id);
TGEN_PANIC(worker_thread_table == NULL, "Error creating worker thread table");
}
if (data_structs_flags & DATA_STRUCTS_NEED_GRE_TABLE) {
mprintf("=== user <-> QinQ table configuration ===\n");
qinq_to_gre_lookup[socket_id] = read_gre_table_config(config_path, "gre_table.cfg", worker_thread_table[socket_id], lb_nb_txrings, socket_id);
TGEN_PANIC(NULL == qinq_to_gre_lookup[socket_id], "Failed to allocate qinq to gre lookup table\n");
}
if (data_structs_flags & DATA_STRUCTS_NEED_USER_TABLE) {
mprintf("=== User table configuration ===\n");
user_table[socket_id] = read_user_table_config(config_path, "user_table.cfg", &qinq_to_gre_lookup[socket_id], socket_id);
TGEN_PANIC(NULL == user_table[socket_id], "Failed to allocate user lookup table\n");
}
if (data_structs_flags & DATA_STRUCTS_NEED_NEXT_HOP) {
mprintf("=== Next hop configuration ===\n");
next_hop[socket_id] = read_next_hop_config(config_path, "next_hop.cfg", &tgen_used_port_mask, socket_id);
init_routing_ports();
}
if (data_structs_flags & DATA_STRUCTS_NEED_LPM_V4) {
mprintf("=== IPv4 routing configuration ===\n");
ipv4_lpm[socket_id] = read_lpm_v4_config(config_path, "ipv4.cfg", socket_id);
TGEN_PANIC(NULL == ipv4_lpm[socket_id], "Failed to allocate IPv4 LPM\n");
}
if (data_structs_flags & DATA_STRUCTS_NEED_LPM_V6) {
mprintf("=== IPv6 routing configuration ===\n");
read_lpm_v6_config(config_path, "ipv6.cfg", socket_id);
}
}
check_consistent_cfg();
mprintf("=== Initializing tables, mempools and queue numbers on cores ===\n");
for (uint8_t lcore_id = 0; lcore_id < RTE_MAX_LCORE; ++lcore_id) {
if (!rte_lcore_is_enabled(lcore_id) || lcore_id == tgen_cfg.master) {
continue;
}
lconf = &lcore_cfg[lcore_id];
uint8_t socket = rte_lcore_to_socket_id(lcore_id);
for (uint8_t task_id = 0; task_id < lconf->nb_tasks; ++task_id) {
struct task_startup_cfg *startup_cfg = &lconf->startup_cfg[task_id];
if (QOS == startup_cfg->mode) {
rte_snprintf(name, sizeof(name), "qos_sched_port_%u_%u", lcore_id, task_id);
startup_cfg->qos_conf.port_params.name = name;
startup_cfg->qos_conf.port_params.socket = socket;
startup_cfg->qos_conf.port_params.rate = TEN_GIGABIT;
startup_cfg->sched_port = rte_sched_port_config(&startup_cfg->qos_conf.port_params);
TGEN_PANIC(startup_cfg->sched_port == NULL, "failed to create sched_port");
mprintf("number of pipes: %d\n\n", startup_cfg->qos_conf.port_params.n_pipes_per_subport);
int err = rte_sched_subport_config(startup_cfg->sched_port, 0, startup_cfg->qos_conf.subport_params);
TGEN_PANIC(err != 0, "Failed setting up sched_port subport, error: %d", err);
/* only single subport and single pipe profile is supported */
for (uint32_t pipe = 0; pipe < startup_cfg->qos_conf.port_params.n_pipes_per_subport; ++pipe) {
err = rte_sched_pipe_config(startup_cfg->sched_port, 0 , pipe, 0);
TGEN_PANIC(err != 0, "failed setting up sched port pipe, error: %d", err);
}
}
if (LB_QINQ == startup_cfg->mode) {
startup_cfg->worker_thread_table = worker_thread_table[rte_socket_id()];
}
if (QINQ_DECAP_ARP == startup_cfg->mode || QINQ_DECAP_V4 == startup_cfg->mode) {
startup_cfg->qinq_gre = qinq_to_gre_lookup[rte_socket_id()];
}
if (QOS == startup_cfg->mode || CLASSIFY == startup_cfg->mode || QINQ_DECAP_V6 == startup_cfg->mode) {
startup_cfg->user_table = user_table[rte_socket_id()];
}
if (ROUTING == startup_cfg->mode || FWD == startup_cfg->mode || QINQ_DECAP_V4 == startup_cfg->mode) {
startup_cfg->next_hop = next_hop[rte_socket_id()];
}
if (QINQ_DECAP_V4 == startup_cfg->mode || FWD == startup_cfg->mode || ROUTING == startup_cfg->mode) {
startup_cfg->ipv4_lpm = ipv4_lpm[rte_socket_id()];
}
}
mprintf("\t*** Initializing core %u ***\n", lcore_id);
if (lconf->flags & PCFG_CPETABLEv4) {
sprintf(name, "core_%u_CPEv4Table", lcore_id);
uint8_t table_part = lconf->startup_cfg[0].nb_slave_threads;
if (!rte_is_power_of_2(table_part)) {
table_part = rte_align32pow2(table_part) >> 1;
}
struct rte_hash_parameters hash_params = {
.name = name,
.entries = MAX_GRE / table_part,
.bucket_entries = GRE_BUCKET_ENTRIES,
.key_len = sizeof(struct hash_gre_struct),
.entry_len = sizeof(struct cpe_table_hash_entry),
.hash_func_init_val = 0,
.socket_id = socket
};
lconf->cpe_v4_table = rte_hash_ext_create(&hash_params);
TGEN_PANIC(lconf->cpe_v4_table == NULL, "Unable to allocate memory for IPv4 hash table on core %u\n", lcore_id);
/* set all entries to expire at MAX_TSC (i.e. never) so that we don't waste cycles at startup going through all the empty entries */
setup_arp_entries(lconf->cpe_v4_table);
/* for locality, copy the pointer to the port structure where it is needed at packet handling time */
for (uint8_t task_id = 0; task_id < lconf->nb_tasks; ++task_id) {
if (lconf->startup_cfg[task_id].flags & PORT_STARTUP_CPEv4) {
lconf->startup_cfg[task_id].cpe_table = lconf->cpe_v4_table;
}
}
}
if (lconf->flags & PCFG_CPETABLEv6) {
sprintf(name, "core_%u_CPEv6Table", lcore_id);
uint8_t table_part = lconf->startup_cfg[0].nb_slave_threads;
if (!rte_is_power_of_2(table_part)) {
table_part = rte_align32pow2(table_part) >> 1;
}
struct rte_hash *
rte_hash_create(const struct rte_hash_parameters *params)
{
struct rte_hash *h = NULL;
uint32_t num_buckets, sig_bucket_size, key_size,
hash_tbl_size, sig_tbl_size, key_tbl_size, mem_size;
char hash_name[RTE_HASH_NAMESIZE];
struct rte_hash_list *hash_list;
/* check that we have an initialised tail queue */
if ((hash_list =
RTE_TAILQ_LOOKUP_BY_IDX(RTE_TAILQ_HASH, rte_hash_list)) == NULL) {
rte_errno = E_RTE_NO_TAILQ;
return NULL;
}
/* Check for valid parameters */
if ((params == NULL) ||
(params->entries > RTE_HASH_ENTRIES_MAX) ||
(params->bucket_entries > RTE_HASH_BUCKET_ENTRIES_MAX) ||
(params->entries < params->bucket_entries) ||
!rte_is_power_of_2(params->entries) ||
!rte_is_power_of_2(params->bucket_entries) ||
(params->key_len == 0) ||
(params->key_len > RTE_HASH_KEY_LENGTH_MAX)) {
rte_errno = EINVAL;
RTE_LOG(ERR, HASH, "rte_hash_create has invalid parameters\n");
return NULL;
}
rte_snprintf(hash_name, sizeof(hash_name), "HT_%s", params->name);
/* Calculate hash dimensions */
num_buckets = params->entries / params->bucket_entries;
sig_bucket_size = align_size(params->bucket_entries *
sizeof(hash_sig_t), SIG_BUCKET_ALIGNMENT);
key_size = align_size(params->key_len, KEY_ALIGNMENT);
hash_tbl_size = align_size(sizeof(struct rte_hash), CACHE_LINE_SIZE);
sig_tbl_size = align_size(num_buckets * sig_bucket_size,
CACHE_LINE_SIZE);
key_tbl_size = align_size(num_buckets * key_size *
params->bucket_entries, CACHE_LINE_SIZE);
/* Total memory required for hash context */
mem_size = hash_tbl_size + sig_tbl_size + key_tbl_size;
rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
/* guarantee there's no existing */
TAILQ_FOREACH(h, hash_list, next) {
if (strncmp(params->name, h->name, RTE_HASH_NAMESIZE) == 0)
break;
}
if (h != NULL)
goto exit;
h = (struct rte_hash *)rte_zmalloc_socket(hash_name, mem_size,
CACHE_LINE_SIZE, params->socket_id);
if (h == NULL) {
RTE_LOG(ERR, HASH, "memory allocation failed\n");
goto exit;
}
/* Setup hash context */
rte_snprintf(h->name, sizeof(h->name), "%s", params->name);
h->entries = params->entries;
h->bucket_entries = params->bucket_entries;
h->key_len = params->key_len;
h->hash_func_init_val = params->hash_func_init_val;
h->num_buckets = num_buckets;
h->bucket_bitmask = h->num_buckets - 1;
h->sig_msb = 1 << (sizeof(hash_sig_t) * 8 - 1);
h->sig_tbl = (uint8_t *)h + hash_tbl_size;
h->sig_tbl_bucket_size = sig_bucket_size;
h->key_tbl = h->sig_tbl + sig_tbl_size;
h->key_tbl_key_size = key_size;
h->hash_func = (params->hash_func == NULL) ?
DEFAULT_HASH_FUNC : params->hash_func;
TAILQ_INSERT_TAIL(hash_list, h, next);
exit:
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
return h;
}
/*
* vBF currently implemented as a big array.
* The BFs have a vertical layout. Bits in same location of all bfs will stay
* in the same cache line.
* For example, if we have 32 bloom filters, we use a uint32_t array to
* represent all of them. array[0] represent the first location of all the
* bloom filters, array[1] represents the second location of all the
* bloom filters, etc. The advantage of this layout is to minimize the average
* number of memory accesses to test all bloom filters.
*
* Currently the implementation supports vBF containing 1,2,4,8,16,32 BFs.
*/
int
rte_member_create_vbf(struct rte_member_setsum *ss,
const struct rte_member_parameters *params)
{
if (params->num_set > RTE_MEMBER_MAX_BF ||
!rte_is_power_of_2(params->num_set) ||
params->num_keys == 0 ||
params->false_positive_rate == 0 ||
params->false_positive_rate > 1) {
rte_errno = EINVAL;
RTE_MEMBER_LOG(ERR, "Membership vBF create with invalid parameters\n");
return -EINVAL;
}
/* We assume expected keys evenly distribute to all BFs */
uint32_t num_keys_per_bf = 1 + (params->num_keys - 1) / ss->num_set;
/*
* Note that the false positive rate is for all BFs in the vBF
* such that the single BF's false positive rate needs to be
* calculated.
* Assume each BF's False positive rate is fp_one_bf. The total false
* positive rate is fp = 1-(1-fp_one_bf)^n.
* => fp_one_bf = 1 - (1-fp)^(1/n)
*/
float fp_one_bf = 1 - pow((1 - params->false_positive_rate),
1.0 / ss->num_set);
if (fp_one_bf == 0) {
rte_errno = EINVAL;
RTE_MEMBER_LOG(ERR, "Membership BF false positive rate is too small\n");
return -EINVAL;
}
uint32_t bits = ceil((num_keys_per_bf *
log(fp_one_bf)) /
log(1.0 / (pow(2.0, log(2.0)))));
/* We round to power of 2 for performance during lookup */
ss->bits = rte_align32pow2(bits);
ss->num_hashes = (uint32_t)(log(2.0) * bits / num_keys_per_bf);
ss->bit_mask = ss->bits - 1;
/*
* Since we round the bits to power of 2, the final false positive
* rate will probably not be same as the user specified. We log the
* new value as debug message.
*/
float new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf *
ss->num_hashes)), ss->num_hashes);
new_fp = 1 - pow((1 - new_fp), ss->num_set);
/*
* Reduce hash function count, until we approach the user specified
* false-positive rate. Otherwise it is too conservative
*/
int tmp_num_hash = ss->num_hashes;
while (tmp_num_hash > 1) {
float tmp_fp = new_fp;
tmp_num_hash--;
new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf *
tmp_num_hash)), tmp_num_hash);
new_fp = 1 - pow((1 - new_fp), ss->num_set);
if (new_fp > params->false_positive_rate) {
new_fp = tmp_fp;
tmp_num_hash++;
break;
}
}
ss->num_hashes = tmp_num_hash;
/*
* To avoid multiplication and division:
* mul_shift is used for multiplication shift during bit test
* div_shift is used for division shift, to be divided by number of bits
* represented by a uint32_t variable
*/
ss->mul_shift = __builtin_ctzl(ss->num_set);
ss->div_shift = __builtin_ctzl(32 >> ss->mul_shift);
RTE_MEMBER_LOG(DEBUG, "vector bloom filter created, "
"each bloom filter expects %u keys, needs %u bits, %u hashes, "
"with false positive rate set as %.5f, "
"The new calculated vBF false positive rate is %.5f\n",
num_keys_per_bf, ss->bits, ss->num_hashes, fp_one_bf, new_fp);
ss->table = rte_zmalloc_socket(NULL, ss->num_set * (ss->bits >> 3),
RTE_CACHE_LINE_SIZE, ss->socket_id);
if (ss->table == NULL)
return -ENOMEM;
return 0;
}