int bnxt_init_rx_ring_struct(struct bnxt_rx_queue *rxq, unsigned int socket_id)
{
struct bnxt *bp = rxq->bp;
struct bnxt_cp_ring_info *cpr;
struct bnxt_rx_ring_info *rxr;
struct bnxt_ring *ring;
rxq->rx_buf_use_size = bp->eth_dev->data->mtu +
ETHER_HDR_LEN + ETHER_CRC_LEN +
(2 * VLAN_TAG_SIZE);
rxq->rx_buf_size = rxq->rx_buf_use_size + sizeof(struct rte_mbuf);
rxr = rte_zmalloc_socket("bnxt_rx_ring",
sizeof(struct bnxt_rx_ring_info),
RTE_CACHE_LINE_SIZE, socket_id);
if (rxr == NULL)
return -ENOMEM;
rxq->rx_ring = rxr;
ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
sizeof(struct bnxt_ring),
RTE_CACHE_LINE_SIZE, socket_id);
if (ring == NULL)
return -ENOMEM;
rxr->rx_ring_struct = ring;
ring->ring_size = rte_align32pow2(rxq->nb_rx_desc);
ring->ring_mask = ring->ring_size - 1;
ring->bd = (void *)rxr->rx_desc_ring;
ring->bd_dma = rxr->rx_desc_mapping;
ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_rx_bd);
ring->vmem = (void **)&rxr->rx_buf_ring;
cpr = rte_zmalloc_socket("bnxt_rx_ring",
sizeof(struct bnxt_cp_ring_info),
RTE_CACHE_LINE_SIZE, socket_id);
if (cpr == NULL)
return -ENOMEM;
rxq->cp_ring = cpr;
ring = rte_zmalloc_socket("bnxt_rx_ring_struct",
sizeof(struct bnxt_ring),
RTE_CACHE_LINE_SIZE, socket_id);
if (ring == NULL)
return -ENOMEM;
cpr->cp_ring_struct = ring;
ring->ring_size = rxr->rx_ring_struct->ring_size * 2;
ring->ring_mask = ring->ring_size - 1;
ring->bd = (void *)cpr->cp_desc_ring;
ring->bd_dma = cpr->cp_desc_mapping;
ring->vmem_size = 0;
ring->vmem = NULL;
return 0;
}
int bnxt_init_tx_ring_struct(struct bnxt_tx_queue *txq, unsigned int socket_id)
{
struct bnxt_cp_ring_info *cpr;
struct bnxt_tx_ring_info *txr;
struct bnxt_ring *ring;
txr = rte_zmalloc_socket("bnxt_tx_ring",
sizeof(struct bnxt_tx_ring_info),
RTE_CACHE_LINE_SIZE, socket_id);
if (txr == NULL)
return -ENOMEM;
txq->tx_ring = txr;
ring = rte_zmalloc_socket("bnxt_tx_ring_struct",
sizeof(struct bnxt_ring),
RTE_CACHE_LINE_SIZE, socket_id);
if (ring == NULL)
return -ENOMEM;
txr->tx_ring_struct = ring;
ring->ring_size = rte_align32pow2(txq->nb_tx_desc + 1);
ring->ring_mask = ring->ring_size - 1;
ring->bd = (void *)txr->tx_desc_ring;
ring->bd_dma = txr->tx_desc_mapping;
ring->vmem_size = ring->ring_size * sizeof(struct bnxt_sw_tx_bd);
ring->vmem = (void **)&txr->tx_buf_ring;
cpr = rte_zmalloc_socket("bnxt_tx_ring",
sizeof(struct bnxt_cp_ring_info),
RTE_CACHE_LINE_SIZE, socket_id);
if (cpr == NULL)
return -ENOMEM;
txq->cp_ring = cpr;
ring = rte_zmalloc_socket("bnxt_tx_ring_struct",
sizeof(struct bnxt_ring),
RTE_CACHE_LINE_SIZE, socket_id);
if (ring == NULL)
return -ENOMEM;
cpr->cp_ring_struct = ring;
ring->ring_size = txr->tx_ring_struct->ring_size;
ring->ring_mask = ring->ring_size - 1;
ring->bd = (void *)cpr->cp_desc_ring;
ring->bd_dma = cpr->cp_desc_mapping;
ring->vmem_size = 0;
ring->vmem = NULL;
return 0;
}
static int
update_order_ring(struct rte_cryptodev *dev, uint16_t qp_id)
{
struct scheduler_ctx *sched_ctx = dev->data->dev_private;
struct scheduler_qp_ctx *qp_ctx = dev->data->queue_pairs[qp_id];
if (sched_ctx->reordering_enabled) {
char order_ring_name[RTE_CRYPTODEV_NAME_MAX_LEN];
uint32_t buff_size = rte_align32pow2(
sched_ctx->nb_slaves * PER_SLAVE_BUFF_SIZE);
if (qp_ctx->order_ring) {
rte_ring_free(qp_ctx->order_ring);
qp_ctx->order_ring = NULL;
}
if (!buff_size)
return 0;
if (snprintf(order_ring_name, RTE_CRYPTODEV_NAME_MAX_LEN,
"%s_rb_%u_%u", RTE_STR(CRYPTODEV_NAME_SCHEDULER_PMD),
dev->data->dev_id, qp_id) < 0) {
CS_LOG_ERR("failed to create unique reorder buffer "
"name");
return -ENOMEM;
}
qp_ctx->order_ring = rte_ring_create(order_ring_name,
buff_size, rte_socket_id(),
RING_F_SP_ENQ | RING_F_SC_DEQ);
if (!qp_ctx->order_ring) {
CS_LOG_ERR("failed to create order ring");
return -ENOMEM;
}
} else {
if (qp_ctx->order_ring) {
rte_ring_free(qp_ctx->order_ring);
qp_ctx->order_ring = NULL;
}
}
return 0;
}
/* create fragmentation table */
struct rte_ip_frag_tbl *
rte_ip_frag_table_create(uint32_t bucket_num, uint32_t bucket_entries,
uint32_t max_entries, uint64_t max_cycles, int socket_id)
{
struct rte_ip_frag_tbl *tbl;
size_t sz;
uint64_t nb_entries;
nb_entries = rte_align32pow2(bucket_num);
nb_entries *= bucket_entries;
nb_entries *= IP_FRAG_HASH_FNUM;
/* check input parameters. */
if (rte_is_power_of_2(bucket_entries) == 0 ||
nb_entries > UINT32_MAX || nb_entries == 0 ||
nb_entries < max_entries) {
RTE_LOG(ERR, USER1, "%s: invalid input parameter\n", __func__);
return (NULL);
}
sz = sizeof (*tbl) + nb_entries * sizeof (tbl->pkt[0]);
if ((tbl = rte_zmalloc_socket(__func__, sz, CACHE_LINE_SIZE,
socket_id)) == NULL) {
RTE_LOG(ERR, USER1,
"%s: allocation of %zu bytes at socket %d failed do\n",
__func__, sz, socket_id);
return (NULL);
}
RTE_LOG(INFO, USER1, "%s: allocated of %zu bytes at socket %d\n",
__func__, sz, socket_id);
tbl->max_cycles = max_cycles;
tbl->max_entries = max_entries;
tbl->nb_entries = (uint32_t)nb_entries;
tbl->nb_buckets = bucket_num;
tbl->bucket_entries = bucket_entries;
tbl->entry_mask = (tbl->nb_entries - 1) & ~(tbl->bucket_entries - 1);
TAILQ_INIT(&(tbl->lru));
return (tbl);
}
static int
test_align(void)
{
#define FAIL_ALIGN(x, i, p)\
{printf(x "() test failed: %u %u\n", i, p);\
return -1;}
#define ERROR_FLOOR(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((res / pow) != (i / pow)) /* check if correct alignment */
#define ERROR_CEIL(res, i, pow) \
(res % pow) || /* check if not aligned */ \
((i % pow) == 0 ? /* check if ceiling is invoked */ \
val / pow != i / pow : /* if aligned */ \
val / pow != (i / pow) + 1) /* if not aligned, hence +1 */
uint32_t i, p, val;
for (i = 1, p = 1; i <= MAX_NUM; i ++) {
if (rte_align32pow2(i) != p)
FAIL_ALIGN("rte_align32pow2", i, p);
if (i == p)
p <<= 1;
}
for (p = 2; p <= MAX_NUM; p <<= 1) {
if (!rte_is_power_of_2(p))
FAIL("rte_is_power_of_2");
for (i = 1; i <= MAX_NUM; i++) {
/* align floor */
if (RTE_ALIGN_FLOOR((uintptr_t)i, p) % p)
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
val = RTE_PTR_ALIGN_FLOOR((uintptr_t) i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_FLOOR", i, p);
val = RTE_ALIGN_FLOOR(i, p);
if (ERROR_FLOOR(val, i, p))
FAIL_ALIGN("RTE_ALIGN_FLOOR", i, p);
/* align ceiling */
val = RTE_PTR_ALIGN((uintptr_t) i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN", i, p);
val = RTE_ALIGN(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN", i, p);
val = RTE_ALIGN_CEIL(i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_ALIGN_CEIL", i, p);
val = RTE_PTR_ALIGN_CEIL((uintptr_t)i, p);
if (ERROR_CEIL(val, i, p))
FAIL_ALIGN("RTE_PTR_ALIGN_CEIL", i, p);
/* by this point we know that val is aligned to p */
if (!rte_is_aligned((void*)(uintptr_t) val, p))
FAIL("rte_is_aligned");
}
}
return 0;
}
int
rte_netmap_init_port(uint8_t portid, const struct rte_netmap_port_conf *conf)
{
int32_t ret;
uint16_t i;
uint16_t rx_slots, tx_slots;
if (conf == NULL ||
portid >= RTE_DIM(ports) ||
conf->nr_tx_rings > netmap.conf.max_rings ||
conf->nr_rx_rings > netmap.conf.max_rings) {
RTE_LOG(ERR, USER1, "%s(%hhu): invalid parameters\n",
__func__, portid);
return (-EINVAL);
}
rx_slots = (uint16_t)rte_align32pow2(conf->nr_rx_slots);
tx_slots = (uint16_t)rte_align32pow2(conf->nr_tx_slots);
if (tx_slots > netmap.conf.max_slots ||
rx_slots > netmap.conf.max_slots) {
RTE_LOG(ERR, USER1, "%s(%hhu): invalid parameters\n",
__func__, portid);
return (-EINVAL);
}
ret = rte_eth_dev_configure(portid, conf->nr_rx_rings,
conf->nr_tx_rings, conf->eth_conf);
if (ret < 0) {
RTE_LOG(ERR, USER1, "Couldn't configure port %hhu\n", portid);
return (ret);
}
for (i = 0; i < conf->nr_tx_rings; i++) {
ret = rte_eth_tx_queue_setup(portid, i, tx_slots,
conf->socket_id, conf->tx_conf);
if (ret < 0) {
RTE_LOG(ERR, USER1,
"Couldn't configure TX queue %"PRIu16" of "
"port %"PRIu8"\n",
i, portid);
return (ret);
}
ret = rte_eth_rx_queue_setup(portid, i, rx_slots,
conf->socket_id, conf->rx_conf, conf->pool);
if (ret < 0) {
RTE_LOG(ERR, USER1,
"Couldn't configure RX queue %"PRIu16" of "
"port %"PRIu8"\n",
i, portid);
return (ret);
}
}
/* copy config to the private storage. */
ports[portid].eth_conf = conf->eth_conf[0];
ports[portid].rx_conf = conf->rx_conf[0];
ports[portid].tx_conf = conf->tx_conf[0];
ports[portid].pool = conf->pool;
ports[portid].socket_id = conf->socket_id;
ports[portid].nr_tx_rings = conf->nr_tx_rings;
ports[portid].nr_rx_rings = conf->nr_rx_rings;
ports[portid].nr_tx_slots = tx_slots;
ports[portid].nr_rx_slots = rx_slots;
ports[portid].tx_burst = conf->tx_burst;
ports[portid].rx_burst = conf->rx_burst;
return (0);
}
static void init_task_gen(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
static char name[] = "server_mempool";
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket_id, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_listen = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_listen == 0, "No services specified to listen on\n");
task->bundle_cfgs = prox_zmalloc(n_listen * sizeof(task->bundle_cfgs[0]), socket_id);
plogx_info("n_listen = %d\n", n_listen);
struct hash_set *hs = prox_sh_find_socket(socket_id, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 1, hash_set
will grow if full. */
hs = hash_set_create(n_listen, socket_id);
prox_sh_add_socket(socket_id, "genl4_streams", hs);
}
const struct rte_hash_parameters listen_table = {
.name = name,
.entries = n_listen * 4,
.key_len = sizeof(struct new_tuple),
.hash_func = rte_hash_crc,
.hash_func_init_val = 0,
.socket_id = socket_id,
};
name[0]++;
task->listen_hash = rte_hash_create(&listen_table);
task->listen_entries = prox_zmalloc(listen_table.entries * sizeof(task->listen_entries[0]), socket_id);
int idx = 0;
lua_pushnil(prox_lua());
while (lua_next(prox_lua(), -2)) {
task->bundle_cfgs[idx].n_stream_cfgs = 1;
task->bundle_cfgs[idx].stream_cfgs = prox_zmalloc(sizeof(*task->bundle_cfgs[idx].stream_cfgs), socket_id);
int ret = lua_to_stream_cfg(prox_lua(), STACK, NULL, socket_id, &task->bundle_cfgs[idx].stream_cfgs[0], hs);
PROX_PANIC(ret, "Failed to load stream cfg\n");
struct stream_cfg *stream = task->bundle_cfgs[idx].stream_cfgs[0];
// TODO: check mask and add to hash for each host
struct new_tuple nt = {
.dst_addr = stream->servers.ip,
.proto_id = stream->proto,
.dst_port = stream->servers.port,
.l2_types[0] = 0x0008,
};
ret = rte_hash_add_key(task->listen_hash, &nt);
PROX_PANIC(ret < 0, "Failed to add\n");
task->listen_entries[ret] = &task->bundle_cfgs[idx];
plogx_dbg("Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&nt.dst_addr)), rte_bswap16(nt.dst_port));
++idx;
lua_pop(prox_lua(), 1);
}
static char name2[] = "task_gen_hash2";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn * 2, &task->bundle_ctx_pool, NULL, 0, NULL, socket_id)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn * 2, socket_id);
task->seed = rte_rdtsc();
/* TODO: calculate the CDF of the reply distribution and the
number of replies as the number to cover for 99% of the
replies. For now, assume that this is number is 2. */
uint32_t queue_size = rte_align32pow2(targ->n_concur_conn * 2);
PROX_PANIC(queue_size == 0, "Overflow resulted in queue size 0\n");
task->fqueue = fqueue_create(queue_size, socket_id);
PROX_PANIC(task->fqueue == NULL, "Failed to allocate local queue\n");
uint32_t n_descriptors;
if (targ->nb_txports) {
PROX_PANIC(targ->nb_txports != 1, "Need exactly one TX port for L4 generation\n");
n_descriptors = prox_port_cfg[targ->tx_port_queue[0].port].n_txd;
} else {
PROX_PANIC(targ->nb_txrings != 1, "Need exactly one TX ring for L4 generation\n");
n_descriptors = 256;
}
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = n_descriptors * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void init_task_gen_client(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
static char name[] = "gen_pool";
const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
/* streams contains a lua table. Go through it and read each
stream with associated imix_fraction. */
uint32_t imix;
uint32_t i = 0;
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_bundle_cfgs = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_bundle_cfgs == 0, "No configs specified\n");
plogx_info("loading %d bundle_cfgs\n", n_bundle_cfgs);
struct hash_set *hs = prox_sh_find_socket(socket, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 8, hash_set
will grow if full. */
hs = hash_set_create(n_bundle_cfgs * 8, socket);
prox_sh_add_socket(socket, "genl4_streams", hs);
}
task->bundle_cfgs = prox_zmalloc(n_bundle_cfgs * sizeof(task->bundle_cfgs[0]), socket);
lua_pushnil(prox_lua());
int total_imix = 0;
uint32_t *occur = prox_zmalloc(n_bundle_cfgs * sizeof(*occur), socket);
struct cdf *cdf = cdf_create(n_bundle_cfgs, socket);
while (lua_next(prox_lua(), -2)) {
PROX_PANIC(lua_to_int(prox_lua(), TABLE, "imix_fraction", &imix) ||
lua_to_bundle_cfg(prox_lua(), TABLE, "bundle", socket, &task->bundle_cfgs[i], hs),
"Failed to load bundle cfg:\n%s\n", get_lua_to_errors());
cdf_add(cdf, imix);
occur[i] = imix;
total_imix += imix;
++i;
lua_pop(prox_lua(), 1);
}
lua_pop(prox_lua(), pop);
cdf_setup(cdf);
PROX_PANIC(targ->max_setup_rate == 0, "Max setup rate not set\n");
task->new_conn_cost = rte_get_tsc_hz()/targ->max_setup_rate;
static char name2[] = "task_gen_hash";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn, &task->bundle_ctx_pool, occur, n_bundle_cfgs, task->bundle_cfgs, socket)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn, socket);
task->seed = rte_rdtsc();
/* task->token_time.bytes_max = MAX_PKT_BURST * (ETHER_MAX_LEN + 20); */
/* To avoid overflowing the tx descriptors, the token bucket
size needs to be limited. The descriptors are filled most
quickly with the smallest packets. For that reason, the
token bucket size is given by "number of tx descriptors" *
"smallest Ethernet packet". */
PROX_ASSERT(targ->nb_txports == 1);
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = prox_port_cfg[targ->tx_port_queue[0].port].n_txd * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void start_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
task->new_conn_tokens = 0;
task->new_conn_last_tsc = rte_rdtsc();
}
static void stop_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
struct bundle_ctx *bundle;
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
}
static void start_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
}
static void stop_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
struct bundle_ctx *bundle;
uint8_t out[MAX_PKT_BURST];
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
if (task->cancelled) {
struct rte_mbuf *mbuf = task->mbuf_saved;
out[0] = OUT_DISCARD;
task->cancelled = 0;
task->base.tx_pkt(&task->base, &mbuf, 1, out);
}
do {
if (task->cur_mbufs_beg == task->cur_mbufs_end) {
task->cur_mbufs_end = fqueue_get(task->fqueue, task->cur_mbufs, MAX_PKT_BURST);
task->cur_mbufs_beg = 0;
if (task->cur_mbufs_end == 0)
break;
}
uint16_t n_pkts = task->cur_mbufs_end - task->cur_mbufs_beg;
struct rte_mbuf **mbufs = task->cur_mbufs + task->cur_mbufs_beg;
if (n_pkts) {
for (uint16_t j = 0; j < n_pkts; ++j) {
out[j] = OUT_DISCARD;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
} while (1);
}
static struct task_init task_init_gen1 = {
.mode_str = "genl4",
.sub_mode_str = "server",
.init = init_task_gen,
.handle = handle_gen_bulk,
.start = start_task_gen_server,
.stop = stop_task_gen_server,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_server),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
static struct task_init task_init_gen2 = {
.mode_str = "genl4",
.init = init_task_gen_client,
.handle = handle_gen_bulk_client,
.start = start_task_gen_client,
.stop = stop_task_gen_client,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_client),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
__attribute__((constructor)) static void reg_task_gen(void)
{
reg_task(&task_init_gen1);
reg_task(&task_init_gen2);
}
/* create the mempool */
struct rte_mempool *
rte_mempool_create(const char *name, unsigned n, unsigned elt_size,
unsigned cache_size, unsigned private_data_size,
rte_mempool_ctor_t *mp_init, void *mp_init_arg,
rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
int socket_id, unsigned flags)
{
char mz_name[RTE_MEMZONE_NAMESIZE];
char rg_name[RTE_RING_NAMESIZE];
struct rte_mempool *mp = NULL;
struct rte_ring *r;
const struct rte_memzone *mz;
size_t mempool_size, total_elt_size;
int mz_flags = RTE_MEMZONE_1GB|RTE_MEMZONE_SIZE_HINT_ONLY;
int rg_flags = 0;
uint32_t header_size, trailer_size;
unsigned i;
void *obj;
/* compilation-time checks */
RTE_BUILD_BUG_ON((sizeof(struct rte_mempool) &
CACHE_LINE_MASK) != 0);
#if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_cache) &
CACHE_LINE_MASK) != 0);
RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, local_cache) &
CACHE_LINE_MASK) != 0);
#endif
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
RTE_BUILD_BUG_ON((sizeof(struct rte_mempool_debug_stats) &
CACHE_LINE_MASK) != 0);
RTE_BUILD_BUG_ON((offsetof(struct rte_mempool, stats) &
CACHE_LINE_MASK) != 0);
#endif
/* check that we have an initialised tail queue */
if (RTE_TAILQ_LOOKUP_BY_IDX(RTE_TAILQ_MEMPOOL, rte_mempool_list) == NULL) {
rte_errno = E_RTE_NO_TAILQ;
return NULL;
}
/* asked cache too big */
if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE){
rte_errno = EINVAL;
return NULL;
}
/* "no cache align" imply "no spread" */
if (flags & MEMPOOL_F_NO_CACHE_ALIGN)
flags |= MEMPOOL_F_NO_SPREAD;
/* ring flags */
if (flags & MEMPOOL_F_SP_PUT)
rg_flags |= RING_F_SP_ENQ;
if (flags & MEMPOOL_F_SC_GET)
rg_flags |= RING_F_SC_DEQ;
rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK);
/* allocate the ring that will be used to store objects */
/* Ring functions will return appropriate errors if we are
* running as a secondary process etc., so no checks made
* in this function for that condition */
rte_snprintf(rg_name, sizeof(rg_name), "MP_%s", name);
r = rte_ring_create(rg_name, rte_align32pow2(n+1), socket_id, rg_flags);
if (r == NULL)
goto exit;
/*
* In header, we have at least the pointer to the pool, and
* optionaly a 64 bits cookie.
*/
header_size = 0;
header_size += sizeof(struct rte_mempool *); /* ptr to pool */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
header_size += sizeof(uint64_t); /* cookie */
#endif
if ((flags & MEMPOOL_F_NO_CACHE_ALIGN) == 0)
header_size = (header_size + CACHE_LINE_MASK) & (~CACHE_LINE_MASK);
/* trailer contains the cookie in debug mode */
trailer_size = 0;
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
trailer_size += sizeof(uint64_t); /* cookie */
#endif
/* element size is 8 bytes-aligned at least */
elt_size = (elt_size + 7) & (~7);
/* expand trailer to next cache line */
if ((flags & MEMPOOL_F_NO_CACHE_ALIGN) == 0) {
total_elt_size = header_size + elt_size + trailer_size;
trailer_size += ((CACHE_LINE_SIZE -
(total_elt_size & CACHE_LINE_MASK)) &
CACHE_LINE_MASK);
}
/*
* increase trailer to add padding between objects in order to
* spread them accross memory channels/ranks
*/
if ((flags & MEMPOOL_F_NO_SPREAD) == 0) {
unsigned new_size;
new_size = optimize_object_size(header_size + elt_size +
trailer_size);
trailer_size = new_size - header_size - elt_size;
}
/* this is the size of an object, including header and trailer */
total_elt_size = header_size + elt_size + trailer_size;
/* reserve a memory zone for this mempool: private data is
* cache-aligned */
private_data_size = (private_data_size +
CACHE_LINE_MASK) & (~CACHE_LINE_MASK);
mempool_size = total_elt_size * n +
sizeof(struct rte_mempool) + private_data_size;
rte_snprintf(mz_name, sizeof(mz_name), "MP_%s", name);
mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags);
/*
* no more memory: in this case we loose previously reserved
* space for the as we cannot free it
*/
if (mz == NULL)
goto exit;
/* init the mempool structure */
mp = mz->addr;
memset(mp, 0, sizeof(*mp));
rte_snprintf(mp->name, sizeof(mp->name), "%s", name);
mp->phys_addr = mz->phys_addr;
mp->ring = r;
mp->size = n;
mp->flags = flags;
mp->elt_size = elt_size;
mp->header_size = header_size;
mp->trailer_size = trailer_size;
mp->cache_size = cache_size;
mp->cache_flushthresh = (uint32_t)(cache_size * CACHE_FLUSHTHRESH_MULTIPLIER);
mp->private_data_size = private_data_size;
/* call the initializer */
if (mp_init)
mp_init(mp, mp_init_arg);
/* fill the headers and trailers, and add objects in ring */
obj = (char *)mp + sizeof(struct rte_mempool) + private_data_size;
for (i = 0; i < n; i++) {
struct rte_mempool **mpp;
obj = (char *)obj + header_size;
/* set mempool ptr in header */
mpp = __mempool_from_obj(obj);
*mpp = mp;
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
__mempool_write_header_cookie(obj, 1);
__mempool_write_trailer_cookie(obj);
#endif
/* call the initializer */
if (obj_init)
obj_init(mp, obj_init_arg, obj, i);
/* enqueue in ring */
rte_ring_sp_enqueue(mp->ring, obj);
obj = (char *)obj + elt_size + trailer_size;
}
RTE_EAL_TAILQ_INSERT_TAIL(RTE_TAILQ_MEMPOOL, rte_mempool_list, mp);
exit:
rte_rwlock_write_unlock(RTE_EAL_MEMPOOL_RWLOCK);
return mp;
}
/*
* vBF currently implemented as a big array.
* The BFs have a vertical layout. Bits in same location of all bfs will stay
* in the same cache line.
* For example, if we have 32 bloom filters, we use a uint32_t array to
* represent all of them. array[0] represent the first location of all the
* bloom filters, array[1] represents the second location of all the
* bloom filters, etc. The advantage of this layout is to minimize the average
* number of memory accesses to test all bloom filters.
*
* Currently the implementation supports vBF containing 1,2,4,8,16,32 BFs.
*/
int
rte_member_create_vbf(struct rte_member_setsum *ss,
const struct rte_member_parameters *params)
{
if (params->num_set > RTE_MEMBER_MAX_BF ||
!rte_is_power_of_2(params->num_set) ||
params->num_keys == 0 ||
params->false_positive_rate == 0 ||
params->false_positive_rate > 1) {
rte_errno = EINVAL;
RTE_MEMBER_LOG(ERR, "Membership vBF create with invalid parameters\n");
return -EINVAL;
}
/* We assume expected keys evenly distribute to all BFs */
uint32_t num_keys_per_bf = 1 + (params->num_keys - 1) / ss->num_set;
/*
* Note that the false positive rate is for all BFs in the vBF
* such that the single BF's false positive rate needs to be
* calculated.
* Assume each BF's False positive rate is fp_one_bf. The total false
* positive rate is fp = 1-(1-fp_one_bf)^n.
* => fp_one_bf = 1 - (1-fp)^(1/n)
*/
float fp_one_bf = 1 - pow((1 - params->false_positive_rate),
1.0 / ss->num_set);
if (fp_one_bf == 0) {
rte_errno = EINVAL;
RTE_MEMBER_LOG(ERR, "Membership BF false positive rate is too small\n");
return -EINVAL;
}
uint32_t bits = ceil((num_keys_per_bf *
log(fp_one_bf)) /
log(1.0 / (pow(2.0, log(2.0)))));
/* We round to power of 2 for performance during lookup */
ss->bits = rte_align32pow2(bits);
ss->num_hashes = (uint32_t)(log(2.0) * bits / num_keys_per_bf);
ss->bit_mask = ss->bits - 1;
/*
* Since we round the bits to power of 2, the final false positive
* rate will probably not be same as the user specified. We log the
* new value as debug message.
*/
float new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf *
ss->num_hashes)), ss->num_hashes);
new_fp = 1 - pow((1 - new_fp), ss->num_set);
/*
* Reduce hash function count, until we approach the user specified
* false-positive rate. Otherwise it is too conservative
*/
int tmp_num_hash = ss->num_hashes;
while (tmp_num_hash > 1) {
float tmp_fp = new_fp;
tmp_num_hash--;
new_fp = pow((1 - pow((1 - 1.0 / ss->bits), num_keys_per_bf *
tmp_num_hash)), tmp_num_hash);
new_fp = 1 - pow((1 - new_fp), ss->num_set);
if (new_fp > params->false_positive_rate) {
new_fp = tmp_fp;
tmp_num_hash++;
break;
}
}
ss->num_hashes = tmp_num_hash;
/*
* To avoid multiplication and division:
* mul_shift is used for multiplication shift during bit test
* div_shift is used for division shift, to be divided by number of bits
* represented by a uint32_t variable
*/
ss->mul_shift = __builtin_ctzl(ss->num_set);
ss->div_shift = __builtin_ctzl(32 >> ss->mul_shift);
RTE_MEMBER_LOG(DEBUG, "vector bloom filter created, "
"each bloom filter expects %u keys, needs %u bits, %u hashes, "
"with false positive rate set as %.5f, "
"The new calculated vBF false positive rate is %.5f\n",
num_keys_per_bf, ss->bits, ss->num_hashes, fp_one_bf, new_fp);
ss->table = rte_zmalloc_socket(NULL, ss->num_set * (ss->bits >> 3),
RTE_CACHE_LINE_SIZE, ss->socket_id);
if (ss->table == NULL)
return -ENOMEM;
return 0;
}