/* Create a new flow table made of an rte_hash table and a fixed size
* data array for storing values. Only supports IPv4 5-tuple lookups. */
struct onvm_ft*
onvm_ft_create(int cnt, int entry_size) {
struct rte_hash* hash;
struct onvm_ft* ft;
struct rte_hash_parameters ipv4_hash_params = {
.name = NULL,
.entries = cnt,
.key_len = sizeof(struct onvm_ft_ipv4_5tuple),
.hash_func = NULL,
.hash_func_init_val = 0,
};
char s[64];
/* create ipv4 hash table. use core number and cycle counter to get a unique name. */
ipv4_hash_params.name = s;
ipv4_hash_params.socket_id = rte_socket_id();
snprintf(s, sizeof(s), "onvm_ft_%d-%"PRIu64, rte_lcore_id(), rte_get_tsc_cycles());
hash = rte_hash_create(&ipv4_hash_params);
if (hash == NULL) {
return NULL;
}
ft = (struct onvm_ft*)rte_calloc("table", 1, sizeof(struct onvm_ft), 0);
if (ft == NULL) {
rte_hash_free(hash);
return NULL;
}
ft->hash = hash;
ft->cnt = cnt;
ft->entry_size = entry_size;
/* Create data array for storing values */
ft->data = rte_calloc("entry", cnt, entry_size, 0);
if (ft->data == NULL) {
rte_hash_free(hash);
rte_free(ft);
return NULL;
}
return ft;
}
/* Add an entry in flow table and set data to point to the new value.
Returns:
index in the array on success
-EPROTONOSUPPORT if packet is not ipv4.
-EINVAL if the parameters are invalid.
-ENOSPC if there is no space in the hash for this key.
*/
int
onvm_ft_add_pkt(struct onvm_ft* table, struct rte_mbuf *pkt, char** data) {
int32_t tbl_index;
struct onvm_ft_ipv4_5tuple key;
int err;
err = onvm_ft_fill_key(&key, pkt);
if (err < 0) {
return err;
}
tbl_index = rte_hash_add_key_with_hash(table->hash, (const void *)&key, pkt->hash.rss);
if (tbl_index >= 0) {
*data = &table->data[tbl_index*table->entry_size];
}
return tbl_index;
}
static int
create_table(unsigned with_data, unsigned table_index)
{
char name[RTE_HASH_NAMESIZE];
if (with_data)
/* Table will store 8-byte data */
sprintf(name, "test_hash%d_data", hashtest_key_lens[table_index]);
else
sprintf(name, "test_hash%d", hashtest_key_lens[table_index]);
ut_params.name = name;
ut_params.key_len = hashtest_key_lens[table_index];
ut_params.socket_id = rte_socket_id();
h[table_index] = rte_hash_find_existing(name);
if (h[table_index] != NULL)
/*
* If table was already created, free it to create it again,
* so we force it is empty
*/
rte_hash_free(h[table_index]);
h[table_index] = rte_hash_create(&ut_params);
if (h[table_index] == NULL) {
printf("Error creating table\n");
return -1;
}
return 0;
}
void enic_clsf_destroy(struct enic *enic)
{
u32 index;
struct enic_fdir_node *key;
/* delete classifier entries */
for (index = 0; index < ENICPMD_FDIR_MAX; index++) {
key = enic->fdir.nodes[index];
if (key) {
vnic_dev_classifier(enic->vdev, CLSF_DEL,
&key->fltr_id, NULL);
rte_free(key);
}
}
if (enic->fdir.hash) {
rte_hash_free(enic->fdir.hash);
enic->fdir.hash = NULL;
}
}
/* Clears a flow table and frees associated memory */
void
onvm_ft_free(struct onvm_ft *table)
{
rte_hash_reset(table->hash);
rte_hash_free(table->hash);
}
static void
free_table(unsigned table_index)
{
rte_hash_free(h[table_index]);
}
/*
* Looks for random keys which
* ALL can fit in hash table (no errors)
*/
static int
get_input_keys(unsigned with_pushes, unsigned table_index)
{
unsigned i, j;
unsigned bucket_idx, incr, success = 1;
uint8_t k = 0;
int32_t ret;
const uint32_t bucket_bitmask = NUM_BUCKETS - 1;
/* Reset all arrays */
for (i = 0; i < MAX_ENTRIES; i++)
slot_taken[i] = 0;
for (i = 0; i < NUM_BUCKETS; i++)
buckets[i] = 0;
for (j = 0; j < hashtest_key_lens[table_index]; j++)
keys[0][j] = 0;
/*
* Add only entries that are not duplicated and that fits in the table
* (cannot store more than BUCKET_SIZE entries in a bucket).
* Regardless a key has been added correctly or not (success),
* the next one to try will be increased by 1.
*/
for (i = 0; i < KEYS_TO_ADD;) {
incr = 0;
if (i != 0) {
keys[i][0] = ++k;
/* Overflow, need to increment the next byte */
if (keys[i][0] == 0)
incr = 1;
for (j = 1; j < hashtest_key_lens[table_index]; j++) {
/* Do not increase next byte */
if (incr == 0)
if (success == 1)
keys[i][j] = keys[i - 1][j];
else
keys[i][j] = keys[i][j];
/* Increase next byte by one */
else {
if (success == 1)
keys[i][j] = keys[i-1][j] + 1;
else
keys[i][j] = keys[i][j] + 1;
if (keys[i][j] == 0)
incr = 1;
else
incr = 0;
}
}
}
success = 0;
signatures[i] = rte_hash_hash(h[table_index], keys[i]);
bucket_idx = signatures[i] & bucket_bitmask;
/*
* If we are not inserting keys in secondary location,
* when bucket is full, do not try to insert the key
*/
if (with_pushes == 0)
if (buckets[bucket_idx] == BUCKET_SIZE)
continue;
/* If key can be added, leave in successful key arrays "keys" */
ret = rte_hash_add_key_with_hash(h[table_index], keys[i],
signatures[i]);
if (ret >= 0) {
/* If key is already added, ignore the entry and do not store */
if (slot_taken[ret])
continue;
else {
/* Store the returned position and mark slot as taken */
slot_taken[ret] = 1;
positions[i] = ret;
buckets[bucket_idx]++;
success = 1;
i++;
}
}
}
/* Reset the table, so we can measure the time to add all the entries */
rte_hash_free(h[table_index]);
h[table_index] = rte_hash_create(&ut_params);
return 0;
}
/*
* Initialize a given port using default settings and with the RX buffers
* coming from the mbuf_pool passed as a parameter.
* FIXME: Starting with assumption of one thread/core per port
*/
static inline int uhd_dpdk_port_init(struct uhd_dpdk_port *port,
struct rte_mempool *rx_mbuf_pool,
unsigned int mtu)
{
int retval;
/* Check for a valid port */
if (port->id >= rte_eth_dev_count())
return -ENODEV;
/* Set up Ethernet device with defaults (1 RX ring, 1 TX ring) */
/* FIXME: Check if hw_ip_checksum is possible */
struct rte_eth_conf port_conf = {
.rxmode = {
.max_rx_pkt_len = mtu,
.jumbo_frame = 1,
.hw_ip_checksum = 1,
}
};
retval = rte_eth_dev_configure(port->id, 1, 1, &port_conf);
if (retval != 0)
return retval;
retval = rte_eth_rx_queue_setup(port->id, 0, DEFAULT_RING_SIZE,
rte_eth_dev_socket_id(port->id), NULL, rx_mbuf_pool);
if (retval < 0)
return retval;
retval = rte_eth_tx_queue_setup(port->id, 0, DEFAULT_RING_SIZE,
rte_eth_dev_socket_id(port->id), NULL);
if (retval < 0)
goto port_init_fail;
/* Create the hash table for the RX sockets */
char name[32];
snprintf(name, sizeof(name), "rx_table_%u", port->id);
struct rte_hash_parameters hash_params = {
.name = name,
.entries = UHD_DPDK_MAX_SOCKET_CNT,
.key_len = sizeof(struct uhd_dpdk_ipv4_5tuple),
.hash_func = NULL,
.hash_func_init_val = 0,
};
port->rx_table = rte_hash_create(&hash_params);
if (port->rx_table == NULL) {
retval = rte_errno;
goto port_init_fail;
}
/* Create ARP table */
snprintf(name, sizeof(name), "arp_table_%u", port->id);
hash_params.name = name;
hash_params.entries = UHD_DPDK_MAX_SOCKET_CNT;
hash_params.key_len = sizeof(uint32_t);
hash_params.hash_func = NULL;
hash_params.hash_func_init_val = 0;
port->arp_table = rte_hash_create(&hash_params);
if (port->arp_table == NULL) {
retval = rte_errno;
goto free_rx_table;
}
/* Set up list for TX queues */
LIST_INIT(&port->txq_list);
/* Start the Ethernet port. */
retval = rte_eth_dev_start(port->id);
if (retval < 0) {
goto free_arp_table;
}
/* Display the port MAC address. */
rte_eth_macaddr_get(port->id, &port->mac_addr);
RTE_LOG(INFO, EAL, "Port %u MAC: %02x %02x %02x %02x %02x %02x\n",
(unsigned)port->id,
port->mac_addr.addr_bytes[0], port->mac_addr.addr_bytes[1],
port->mac_addr.addr_bytes[2], port->mac_addr.addr_bytes[3],
port->mac_addr.addr_bytes[4], port->mac_addr.addr_bytes[5]);
struct rte_eth_link link;
rte_eth_link_get(port->id, &link);
RTE_LOG(INFO, EAL, "Port %u UP: %d\n", port->id, link.link_status);
return 0;
free_arp_table:
rte_hash_free(port->arp_table);
free_rx_table:
rte_hash_free(port->rx_table);
port_init_fail:
return rte_errno;
}
static int uhd_dpdk_thread_init(struct uhd_dpdk_thread *thread, unsigned int id)
{
if (!ctx || !thread)
return -EINVAL;
unsigned int socket_id = rte_lcore_to_socket_id(id);
thread->id = id;
thread->rx_pktbuf_pool = ctx->rx_pktbuf_pools[socket_id];
thread->tx_pktbuf_pool = ctx->tx_pktbuf_pools[socket_id];
LIST_INIT(&thread->port_list);
char name[32];
snprintf(name, sizeof(name), "sockreq_ring_%u", id);
thread->sock_req_ring = rte_ring_create(
name,
UHD_DPDK_MAX_PENDING_SOCK_REQS,
socket_id,
RING_F_SC_DEQ
);
if (!thread->sock_req_ring)
return -ENOMEM;
return 0;
}
int uhd_dpdk_init(int argc, char **argv, unsigned int num_ports,
int *port_thread_mapping, int num_mbufs, int mbuf_cache_size,
int mtu)
{
/* Init context only once */
if (ctx)
return 1;
if ((num_ports == 0) || (port_thread_mapping == NULL)) {
return -EINVAL;
}
/* Grabs arguments intended for DPDK's EAL */
int ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");
ctx = (struct uhd_dpdk_ctx *) rte_zmalloc("uhd_dpdk_ctx", sizeof(*ctx), rte_socket_id());
if (!ctx)
return -ENOMEM;
ctx->num_threads = rte_lcore_count();
if (ctx->num_threads <= 1)
rte_exit(EXIT_FAILURE, "Error: No worker threads enabled\n");
/* Check that we have ports to send/receive on */
ctx->num_ports = rte_eth_dev_count();
if (ctx->num_ports < 1)
rte_exit(EXIT_FAILURE, "Error: Found no ports\n");
if (ctx->num_ports < num_ports)
rte_exit(EXIT_FAILURE, "Error: User requested more ports than available\n");
/* Get memory for thread and port data structures */
ctx->threads = rte_zmalloc("uhd_dpdk_thread", RTE_MAX_LCORE*sizeof(struct uhd_dpdk_thread), 0);
if (!ctx->threads)
rte_exit(EXIT_FAILURE, "Error: Could not allocate memory for thread data\n");
ctx->ports = rte_zmalloc("uhd_dpdk_port", ctx->num_ports*sizeof(struct uhd_dpdk_port), 0);
if (!ctx->ports)
rte_exit(EXIT_FAILURE, "Error: Could not allocate memory for port data\n");
/* Initialize the thread data structures */
for (int i = rte_get_next_lcore(-1, 1, 0);
(i < RTE_MAX_LCORE);
i = rte_get_next_lcore(i, 1, 0))
{
/* Do one mempool of RX/TX per socket */
unsigned int socket_id = rte_lcore_to_socket_id(i);
/* FIXME Probably want to take into account actual number of ports per socket */
if (ctx->tx_pktbuf_pools[socket_id] == NULL) {
/* Creates a new mempool in memory to hold the mbufs.
* This is done for each CPU socket
*/
const int mbuf_size = mtu + 2048 + RTE_PKTMBUF_HEADROOM;
char name[32];
snprintf(name, sizeof(name), "rx_mbuf_pool_%u", socket_id);
ctx->rx_pktbuf_pools[socket_id] = rte_pktmbuf_pool_create(
name,
ctx->num_ports*num_mbufs,
mbuf_cache_size,
0,
mbuf_size,
socket_id
);
snprintf(name, sizeof(name), "tx_mbuf_pool_%u", socket_id);
ctx->tx_pktbuf_pools[socket_id] = rte_pktmbuf_pool_create(
name,
ctx->num_ports*num_mbufs,
mbuf_cache_size,
0,
mbuf_size,
socket_id
);
if ((ctx->rx_pktbuf_pools[socket_id]== NULL) ||
(ctx->tx_pktbuf_pools[socket_id]== NULL))
rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
}
if (uhd_dpdk_thread_init(&ctx->threads[i], i) < 0)
rte_exit(EXIT_FAILURE, "Error initializing thread %i\n", i);
}
unsigned master_lcore = rte_get_master_lcore();
/* Assign ports to threads and initialize the port data structures */
for (unsigned int i = 0; i < num_ports; i++) {
int thread_id = port_thread_mapping[i];
if (thread_id < 0)
continue;
if (((unsigned int) thread_id) == master_lcore)
RTE_LOG(WARNING, EAL, "User requested master lcore for port %u\n", i);
if (ctx->threads[thread_id].id != (unsigned int) thread_id)
rte_exit(EXIT_FAILURE, "Requested inactive lcore %u for port %u\n", (unsigned int) thread_id, i);
struct uhd_dpdk_port *port = &ctx->ports[i];
port->id = i;
port->parent = &ctx->threads[thread_id];
ctx->threads[thread_id].num_ports++;
LIST_INSERT_HEAD(&ctx->threads[thread_id].port_list, port, port_entry);
/* Initialize port. */
if (uhd_dpdk_port_init(port, port->parent->rx_pktbuf_pool, mtu) != 0)
rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu8 "\n",
i);
}
RTE_LOG(INFO, EAL, "Init DONE!\n");
/* FIXME: Create functions to do this */
RTE_LOG(INFO, EAL, "Starting I/O threads!\n");
for (int i = rte_get_next_lcore(-1, 1, 0);
(i < RTE_MAX_LCORE);
i = rte_get_next_lcore(i, 1, 0))
{
struct uhd_dpdk_thread *t = &ctx->threads[i];
if (!LIST_EMPTY(&t->port_list)) {
rte_eal_remote_launch(_uhd_dpdk_driver_main, NULL, ctx->threads[i].id);
}
}
return 0;
}
/* FIXME: This will be changed once we have functions to handle the threads */
int uhd_dpdk_destroy(void)
{
if (!ctx)
return -ENODEV;
struct uhd_dpdk_config_req *req = (struct uhd_dpdk_config_req *) rte_zmalloc(NULL, sizeof(*req), 0);
if (!req)
return -ENOMEM;
req->req_type = UHD_DPDK_LCORE_TERM;
for (int i = rte_get_next_lcore(-1, 1, 0);
(i < RTE_MAX_LCORE);
i = rte_get_next_lcore(i, 1, 0))
{
struct uhd_dpdk_thread *t = &ctx->threads[i];
if (LIST_EMPTY(&t->port_list))
continue;
if (rte_eal_get_lcore_state(t->id) == FINISHED)
continue;
pthread_mutex_init(&req->mutex, NULL);
pthread_cond_init(&req->cond, NULL);
pthread_mutex_lock(&req->mutex);
if (rte_ring_enqueue(t->sock_req_ring, req)) {
pthread_mutex_unlock(&req->mutex);
RTE_LOG(ERR, USER2, "Failed to terminate thread %d\n", i);
rte_free(req);
return -ENOSPC;
}
struct timespec timeout = {
.tv_sec = 1,
.tv_nsec = 0
};
pthread_cond_timedwait(&req->cond, &req->mutex, &timeout);
pthread_mutex_unlock(&req->mutex);
}
rte_free(req);
return 0;
}
/*
* Run a hash table performance test based on params.
*/
static int
run_tbl_perf_test(struct tbl_perf_test_params *params)
{
static unsigned calledCount = 5;
struct rte_hash_parameters hash_params = {
.entries = params->entries,
.bucket_entries = params->bucket_entries,
.key_len = params->key_len,
.hash_func = params->hash_func,
.hash_func_init_val = params->hash_func_init_val,
.socket_id = rte_socket_id(),
};
struct rte_hash *handle;
double avg_occupancy = 0, ticks = 0;
uint32_t num_iterations, invalid_pos;
char name[RTE_HASH_NAMESIZE];
char hashname[RTE_HASH_NAMESIZE];
snprintf(name, 32, "test%u", calledCount++);
hash_params.name = name;
handle = rte_hash_create(&hash_params);
RETURN_IF_ERROR(handle == NULL, "hash creation failed");
switch (params->test_type){
case ADD_ON_EMPTY:
ticks = run_single_tbl_perf_test(handle, rte_hash_add_key,
params, &avg_occupancy, &invalid_pos);
break;
case DELETE_ON_EMPTY:
ticks = run_single_tbl_perf_test(handle, rte_hash_del_key,
params, &avg_occupancy, &invalid_pos);
break;
case LOOKUP_ON_EMPTY:
ticks = run_single_tbl_perf_test(handle, rte_hash_lookup,
params, &avg_occupancy, &invalid_pos);
break;
case ADD_UPDATE:
num_iterations = params->num_iterations;
params->num_iterations = params->entries;
run_single_tbl_perf_test(handle, rte_hash_add_key, params,
&avg_occupancy, &invalid_pos);
params->num_iterations = num_iterations;
ticks = run_single_tbl_perf_test(handle, rte_hash_add_key,
params, &avg_occupancy, &invalid_pos);
break;
case DELETE:
num_iterations = params->num_iterations;
params->num_iterations = params->entries;
run_single_tbl_perf_test(handle, rte_hash_add_key, params,
&avg_occupancy, &invalid_pos);
params->num_iterations = num_iterations;
ticks = run_single_tbl_perf_test(handle, rte_hash_del_key,
params, &avg_occupancy, &invalid_pos);
break;
case LOOKUP:
num_iterations = params->num_iterations;
params->num_iterations = params->entries;
run_single_tbl_perf_test(handle, rte_hash_add_key, params,
&avg_occupancy, &invalid_pos);
params->num_iterations = num_iterations;
ticks = run_single_tbl_perf_test(handle, rte_hash_lookup,
params, &avg_occupancy, &invalid_pos);
break;
default: return -1;
}
snprintf(hashname, RTE_HASH_NAMESIZE, "%s", get_hash_name(params->hash_func));
printf("%-12s, %-15s, %-16u, %-7u, %-18u, %-8u, %-19.2f, %.2f\n",
hashname,
get_tbl_perf_test_desc(params->test_type),
(unsigned) params->key_len,
(unsigned) params->entries,
(unsigned) params->bucket_entries,
(unsigned) invalid_pos,
avg_occupancy,
ticks
);
/* Free */
rte_hash_free(handle);
return 0;
}
/*
* Run all hash table performance tests.
*/
static int run_all_tbl_perf_tests(void)
{
unsigned i;
printf(" *** Hash table performance test results ***\n");
printf("Hash Func. , Operation , Key size (bytes), Entries, "
"Entries per bucket, Errors , Avg. bucket entries, Ticks/Op.\n");
/* Loop through every combination of test parameters */
for (i = 0;
i < sizeof(tbl_perf_params) / sizeof(struct tbl_perf_test_params);
i++) {
/* Perform test */
if (run_tbl_perf_test(&tbl_perf_params[i]) < 0)
return -1;
}
return 0;
}