static int file_read_cached(const char *file_name, uint8_t **mem, uint32_t beg, uint32_t len, uint32_t socket, struct hash_set *hs)
{
if (len == 0) {
*mem = 0;
return 0;
}
uint8_t *data_mem;
/* Since the configuration can reference the same file from
multiple places, use prox_shared infrastructure to detect
this and return previously loaded data. */
char name[256];
snprintf(name, sizeof(name), "%u-%u:%s", beg, len, file_name);
*mem = prox_sh_find_socket(socket, name);
if (*mem)
return 0;
/* check if the file has been loaded on the other socket. */
if (socket == 1 && (data_mem = prox_sh_find_socket(0, name))) {
uint8_t *data_find = hash_set_find(hs, data_mem, len);
if (!data_find) {
data_find = prox_zmalloc(len, socket);
PROX_PANIC(data_find == NULL, "Failed to allocate memory (%u bytes) to hold header for peer\n", len);
rte_memcpy(data_find, data_mem, len);
hash_set_add(hs, data_find, len);
}
*mem = data_find;
prox_sh_add_socket(socket, name, *mem);
return 0;
}
/* It is possible that a file with a different name contains
the same data. In that case, search all loaded files and
compare the data to reduce memory utilization.*/
data_mem = malloc(len);
PROX_PANIC(data_mem == NULL, "Failed to allocate temporary memory to hold data\n");
if (file_read_content(file_name, data_mem, beg, len)) {
plog_err("%s\n", file_get_error());
return -1;
}
uint8_t *data_find = hash_set_find(hs, data_mem, len);
if (!data_find) {
data_find = prox_zmalloc(len, socket);
PROX_PANIC(data_find == NULL, "Failed to allocate memory (%u bytes) to hold header for peer\n", len);
rte_memcpy(data_find, data_mem, len);
hash_set_add(hs, data_find, len);
}
free(data_mem);
*mem = data_find;
prox_sh_add_socket(socket, name, *mem);
return 0;
}
static void configure_if_tx_queues(struct task_args *targ, uint8_t socket)
{
uint8_t if_port;
for (uint8_t i = 0; i < targ->nb_txports; ++i) {
if_port = targ->tx_port_queue[i].port;
PROX_PANIC(if_port == OUT_DISCARD, "port misconfigured, exiting\n");
PROX_PANIC(!prox_port_cfg[if_port].active, "\tPort %u not used, skipping...\n", if_port);
int dsocket = prox_port_cfg[if_port].socket;
if (dsocket != -1 && dsocket != socket) {
plog_warn("TX core on socket %d while device on socket %d\n", socket, dsocket);
}
if (prox_port_cfg[if_port].tx_ring[0] == '\0') { // Rings-backed port can use single queue
targ->tx_port_queue[i].queue = prox_port_cfg[if_port].n_txq;
prox_port_cfg[if_port].n_txq++;
} else {
prox_port_cfg[if_port].n_txq = 1;
targ->tx_port_queue[i].queue = 0;
}
/* Set the ETH_TXQ_FLAGS_NOREFCOUNT flag if none of
the tasks up to the task transmitting to the port
does not use refcnt. */
if (!chain_flag_state(targ, TASK_FEATURE_TXQ_FLAGS_REFCOUNT, 1)) {
prox_port_cfg[if_port].tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOREFCOUNT;
plog_info("\t\tEnabling No refcnt on port %d\n", if_port);
}
else {
plog_info("\t\tRefcnt used on port %d\n", if_port);
}
/* By default OFFLOAD is enabled, but if the whole
chain has NOOFFLOADS set all the way until the
first task that receives from a port, it will be
disabled for the destination port. */
if (chain_flag_state(targ, TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS, 1)) {
prox_port_cfg[if_port].tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOOFFLOADS;
plog_info("\t\tDisabling TX offloads on port %d\n", if_port);
} else {
plog_info("\t\tEnabling TX offloads on port %d\n", if_port);
}
/* By default NOMULTSEGS is disabled, as drivers/NIC might split packets on RX
It should only be enabled when we know for sure that the RX does not split packets.
Set the ETH_TXQ_FLAGS_NOMULTSEGS flag if none of the tasks up to the task
transmitting to the port does not use multsegs. */
if (!chain_flag_state(targ, TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS, 0)) {
prox_port_cfg[if_port].tx_conf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
plog_info("\t\tEnabling No MultiSegs on port %d\n", if_port);
}
else {
plog_info("\t\tMultiSegs used on port %d\n", if_port);
}
}
}
struct qinq_gre_map *get_qinq_gre_map(struct task_args *targ)
{
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
struct qinq_gre_map *ret = prox_sh_find_socket(socket_id, "qinq_gre_map");
if (!ret) {
PROX_PANIC(!strcmp(targ->user_table, ""), "No user table defined\n");
int rv = lua_to_qinq_gre_map(prox_lua(), GLOBAL, targ->user_table, socket_id, &ret);
PROX_PANIC(rv, "Error reading mapping between qinq and gre from qinq_gre_map: \n%s\n",
get_lua_to_errors());
prox_sh_add_socket(socket_id, "qinq_gre_map", ret);
}
return ret;
}
static void init_task_qinq_encap4(struct task_base *tbase, struct task_args *targ)
{
struct task_qinq_encap4 *task = (struct task_qinq_encap4 *)(tbase);
int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
task->qinq_tag = targ->qinq_tag;
task->cpe_table = targ->cpe_table;
task->cpe_timeout = rte_get_tsc_hz()/1000*targ->cpe_table_timeout_ms;
if (!strcmp(targ->task_init->sub_mode_str, "pe")) {
PROX_PANIC(!strcmp(targ->cpe_table_name, ""), "CPE table not configured\n");
fill_table(targ, task->cpe_table);
}
#ifdef ENABLE_EXTRA_USER_STATISTICS
task->n_users = targ->n_users;
task->stats_per_user = rte_zmalloc_socket(NULL, targ->n_users * sizeof(uint32_t),
RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(targ->lconf->id));
#endif
if (targ->runtime_flags & TASK_CLASSIFY) {
PROX_PANIC(!strcmp(targ->dscp, ""), "DSCP table not specified\n");
task->dscp = prox_sh_find_socket(socket_id, targ->dscp);
if (!task->dscp) {
int ret = lua_to_dscp(prox_lua(), GLOBAL, targ->dscp, socket_id, &task->dscp);
PROX_PANIC(ret, "Failed to create dscp table from config:\n%s\n",
get_lua_to_errors());
prox_sh_add_socket(socket_id, targ->dscp, task->dscp);
}
}
task->runtime_flags = targ->runtime_flags;
for (uint32_t i = 0; i < 64; ++i) {
task->fake_packets[i] = (struct rte_mbuf*)((uint8_t*)&task->keys[i] - sizeof (struct rte_mbuf));
}
targ->lconf->ctrl_timeout = rte_get_tsc_hz()/targ->ctrl_freq;
targ->lconf->ctrl_func_m[targ->task] = arp_msg;
/* TODO: check if it is not necessary to limit reverse mapping
for the elements that have been changing in mapping? */
for (uint32_t i =0 ; i < sizeof(targ->mapping)/sizeof(targ->mapping[0]); ++i) {
task->src_mac[targ->mapping[i]] = *(uint64_t*)&prox_port_cfg[i].eth_addr;
}
/* task->src_mac[entry->port_idx] = *(uint64_t*)&prox_port_cfg[entry->port_idx].eth_addr; */
}
static void setup_all_task_structs(void)
{
struct lcore_cfg *lconf;
uint32_t lcore_id = -1;
struct task_base *tmaster = NULL;
while(prox_core_next(&lcore_id, 1) == 0) {
lconf = &lcore_cfg[lcore_id];
for (uint8_t task_id = 0; task_id < lconf->n_tasks_all; ++task_id) {
if (task_is_master(&lconf->targs[task_id])) {
plog_info("\tInitializing MASTER struct for core %d task %d\n", lcore_id, task_id);
lconf->tasks_all[task_id] = init_task_struct(&lconf->targs[task_id]);
tmaster = lconf->tasks_all[task_id];
}
}
}
PROX_PANIC(tmaster == NULL, "Can't initialize master task\n");
lcore_id = -1;
while(prox_core_next(&lcore_id, 1) == 0) {
lconf = &lcore_cfg[lcore_id];
plog_info("\tInitializing struct for core %d with %d task\n", lcore_id, lconf->n_tasks_all);
for (uint8_t task_id = 0; task_id < lconf->n_tasks_all; ++task_id) {
if (!task_is_master(&lconf->targs[task_id])) {
plog_info("\tInitializing struct for core %d task %d\n", lcore_id, task_id);
lconf->targs[task_id].tmaster = tmaster;
lconf->tasks_all[task_id] = init_task_struct(&lconf->targs[task_id]);
}
}
}
}
static void configure_if_rx_queues(struct task_args *targ, uint8_t socket)
{
for (int i = 0; i < targ->nb_rxports; i++) {
uint8_t if_port = targ->rx_port_queue[i].port;
if (if_port == OUT_DISCARD) {
return;
}
PROX_PANIC(!prox_port_cfg[if_port].active, "Port %u not used, aborting...\n", if_port);
if(prox_port_cfg[if_port].rx_ring[0] != '\0') {
prox_port_cfg[if_port].n_rxq = 0;
}
targ->rx_port_queue[i].queue = prox_port_cfg[if_port].n_rxq;
prox_port_cfg[if_port].pool[targ->rx_port_queue[i].queue] = targ->pool;
prox_port_cfg[if_port].pool_size[targ->rx_port_queue[i].queue] = targ->nb_mbuf - 1;
prox_port_cfg[if_port].n_rxq++;
int dsocket = prox_port_cfg[if_port].socket;
if (dsocket != -1 && dsocket != socket) {
plog_warn("RX core on socket %d while device on socket %d\n", socket, dsocket);
}
}
}
static void check_mixed_normal_pipeline(void)
{
struct lcore_cfg *lconf = NULL;
uint32_t lcore_id = -1;
while (prox_core_next(&lcore_id, 0) == 0) {
lconf = &lcore_cfg[lcore_id];
int all_thread_nop = 1;
int generic = 0;
int pipeline = 0;
for (uint8_t task_id = 0; task_id < lconf->n_tasks_all; ++task_id) {
struct task_args *targ = &lconf->targs[task_id];
all_thread_nop = all_thread_nop &&
targ->task_init->thread_x == thread_nop;
pipeline = pipeline || targ->task_init->thread_x == thread_pipeline;
generic = generic || targ->task_init->thread_x == thread_generic;
}
PROX_PANIC(generic && pipeline, "Can't run both pipeline and normal thread on same core\n");
if (all_thread_nop)
lconf->thread_x = thread_nop;
else {
lconf->thread_x = thread_generic;
}
}
}
static int lua_to_peer_action(struct lua_State *L, enum lua_place from, const char *name, struct peer_action *action, size_t client_contents_len, size_t server_contents_len)
{
int pop;
if ((pop = lua_getfrom(L, from, name)) < 0)
return -1;
if (!lua_istable(L, -1))
return -1;
uint32_t peer, beg, len;
if (lua_to_int(L, TABLE, "peer", &peer) ||
lua_to_int(L, TABLE, "beg", &beg) ||
lua_to_int(L, TABLE, "len", &len)) {
return -1;
}
size_t data_len = (peer == PEER_CLIENT? client_contents_len : server_contents_len);
if (len == (uint32_t)-1)
len = data_len - beg;
PROX_PANIC(beg + len > data_len, "Accessing data past the end (starting at %u for %u bytes) while total length is %zu\n", beg, len, data_len);
action->peer = peer;
action->beg = beg;
action->len = len;
lua_pop(L, pop);
return 0;
}
static void check_missing_rx(void)
{
struct lcore_cfg *lconf = NULL, *rx_lconf = NULL;
struct task_args *targ, *rx_targ = NULL;
struct prox_port_cfg *port;
uint8_t port_id, rx_port_id, ok;
while (core_targ_next(&lconf, &targ, 0) == 0) {
PROX_PANIC((targ->flags & TASK_ARG_RX_RING) && targ->rx_rings[0] == 0 && !targ->tx_opt_ring_task,
"Configuration Error - Core %u task %u Receiving from ring, but nobody xmitting to this ring\n", lconf->id, targ->id);
if (targ->nb_rxports == 0 && targ->nb_rxrings == 0) {
PROX_PANIC(!task_init_flag_set(targ->task_init, TASK_FEATURE_NO_RX),
"\tCore %u task %u: no rx_ports and no rx_rings configured while required by mode %s\n", lconf->id, targ->id, targ->task_init->mode_str);
}
}
lconf = NULL;
while (core_targ_next(&lconf, &targ, 0) == 0) {
if (strcmp(targ->task_init->sub_mode_str, "l3") != 0)
continue;
port = find_reachable_port(targ);
if (port == NULL)
continue;
port_id = port - prox_port_cfg;
rx_lconf = NULL;
ok = 0;
plog_info("\tCore %d task %d transmitting to port %d in L3 mode\n", lconf->id, targ->id, port_id);
while (core_targ_next(&rx_lconf, &rx_targ, 0) == 0) {
for (uint8_t i = 0; i < rx_targ->nb_rxports; ++i) {
rx_port_id = rx_targ->rx_port_queue[i].port;
if ((rx_port_id == port_id) && (rx_targ->task_init->flag_features & TASK_FEATURE_L3)){
ok = 1;
break;
}
}
if (ok == 1) {
plog_info("\tCore %d task %d has found core %d task %d receiving from port %d\n", lconf->id, targ->id, rx_lconf->id, rx_targ->id, port_id);
break;
}
}
PROX_PANIC(ok == 0, "L3 sub mode for port %d on core %d task %d, but no core/task receiving on that port\n", port_id, lconf->id, targ->id);
}
}
static struct cpe_table_data *read_cpe_table_config(const char *name, uint8_t socket)
{
struct lua_State *L = prox_lua();
struct cpe_table_data *ret = NULL;
lua_getglobal(L, name);
PROX_PANIC(lua_isnil(L, -1), "Coudn't find cpe_table data\n");
return ret;
}
void lcore_cfg_alloc_hp(void)
{
size_t mem_size = RTE_MAX_LCORE * sizeof(struct lcore_cfg);
lcore_cfg = prox_zmalloc(mem_size, rte_socket_id());
PROX_PANIC(lcore_cfg == NULL, "Could not allocate memory for core control structures\n");
rte_memcpy(lcore_cfg, lcore_cfg_init, mem_size);
/* get thread ID for master core */
lcore_cfg[rte_lcore_id()].thread_id = pthread_self();
}
static void check_zero_rx(void)
{
struct lcore_cfg *lconf = NULL;
struct task_args *targ;
while (core_targ_next(&lconf, &targ, 0) == 0) {
if (targ->nb_rxports != 0) {
PROX_PANIC(task_init_flag_set(targ->task_init, TASK_FEATURE_NO_RX),
"\tCore %u task %u: rx_ports configured while mode %s does not use it\n", lconf->id, targ->id, targ->task_init->mode_str);
}
}
}
static void setup_mempools_multiple_per_socket(void)
{
struct lcore_cfg *lconf = NULL;
struct task_args *targ;
while (core_targ_next_early(&lconf, &targ, 0) == 0) {
PROX_PANIC(targ->task_init == NULL, "task_init = NULL, is mode specified for core %d, task %d ?\n", lconf->id, targ->id);
if (targ->rx_port_queue[0].port == OUT_DISCARD)
continue;
setup_mempool_for_rx_task(lconf, targ);
}
}
static void init_task_lb_qinq(struct task_base *tbase, struct task_args *targ)
{
struct task_lb_qinq *task = (struct task_lb_qinq *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
task->qinq_tag = targ->qinq_tag;
task->nb_worker_threads = targ->nb_worker_threads;
task->bit_mask = rte_is_power_of_2(targ->nb_worker_threads) ? targ->nb_worker_threads - 1 : 0xff;
/* The load distributor is sending to a set of cores. These
cores are responsible for handling a set of flows
identified by a qinq tag. The load distributor identifies
the flows and forwards them to the appropriate worker. The
mapping from flow to worker is stored within the
work_table. Build the worker_table by asking each worker
which flows are handled. */
task->worker_table = prox_zmalloc(0x1000000, socket_id);
for (int i = 0; i < targ->nb_worker_threads; ++i) {
struct core_task ct = targ->core_task_set[0].core_task[i];
struct task_args *t = core_targ_get(ct.core, ct.task);
PROX_PANIC(t->task_init->flow_iter.beg == NULL,
"Load distributor can't find flows owned by destination worker %d\n", i);
struct flow_iter *it = &t->task_init->flow_iter;
int cnt = 0;
for (it->beg(it, t); !it->is_end(it, t); it->next(it, t)) {
uint16_t svlan = it->get_svlan(it, t);
uint16_t cvlan = it->get_cvlan(it, t);
task->worker_table[PKT_TO_LUTQINQ(svlan, cvlan)] = i;
}
}
/* Check which protocols we are allowed to send to worker tasks */
for (int i = 0; i < MAX_PROTOCOLS; ++i) {
int is_active = !!targ->core_task_set[i].n_elems;
task->protocols_mask |= is_active << i;
}
plog_info("\t\ttask_lb_qinq protocols_mask = 0x%x\n", task->protocols_mask);
if (targ->task_init->flag_features & TASK_FEATURE_LUT_QINQ_RSS)
tbase->flags |= BASE_FLAG_LUT_QINQ_RSS;
if (targ->task_init->flag_features & TASK_FEATURE_LUT_QINQ_HASH)
tbase->flags |= BASE_FLAG_LUT_QINQ_HASH;
plog_info("\t\ttask_lb_qinq flags = 0x%x\n", tbase->flags);
}
static int setup_prox(int argc, char **argv)
{
if (prox_read_config_file() != 0 ||
prox_setup_rte(argv[0]) != 0) {
return -1;
}
if (prox_cfg.flags & DSF_CHECK_SYNTAX) {
plog_info("=== Configuration file syntax has been checked ===\n\n");
exit(EXIT_SUCCESS);
}
init_port_activate();
plog_info("=== Initializing rte devices ===\n");
if (!(prox_cfg.flags & DSF_USE_DUMMY_DEVICES))
init_rte_ring_dev();
init_rte_dev(prox_cfg.flags & DSF_USE_DUMMY_DEVICES);
plog_info("=== Calibrating TSC overhead ===\n");
clock_init();
plog_info("\tTSC running at %"PRIu64" Hz\n", rte_get_tsc_hz());
init_lcores();
plog_info("=== Initializing ports ===\n");
init_port_all();
if (prox_cfg.logbuf_size) {
prox_cfg.logbuf = prox_zmalloc(prox_cfg.logbuf_size, rte_socket_id());
PROX_PANIC(prox_cfg.logbuf == NULL, "Failed to allocate memory for logbuf with size = %d\n", prox_cfg.logbuf_size);
}
if (prox_cfg.flags & DSF_CHECK_INIT) {
plog_info("=== Initialization sequence completed ===\n\n");
exit(EXIT_SUCCESS);
}
/* Current way that works to disable DPDK logging */
FILE *f = fopen("/dev/null", "r");
rte_openlog_stream(f);
plog_info("=== PROX started ===\n");
return 0;
}
static void fill_table(struct task_args *targ, struct rte_table_hash *table)
{
struct cpe_table_data *cpe_table_data;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
int ret = lua_to_cpe_table_data(prox_lua(), GLOBAL, targ->cpe_table_name, socket_id, &cpe_table_data);
const uint8_t n_slaves = targ->nb_slave_threads;
const uint8_t worker_id = targ->worker_thread_id;
for (uint32_t i = 0; i < cpe_table_data->n_entries; ++i) {
if (rte_bswap32(cpe_table_data->entries[i].ip) % n_slaves != worker_id) {
continue;
}
struct cpe_table_entry *entry = &cpe_table_data->entries[i];
uint32_t port_idx = prox_cfg.cpe_table_ports[entry->port_idx];
PROX_PANIC(targ->mapping[port_idx] == 255, "Error reading cpe table: Mapping for port %d is missing", port_idx);
struct cpe_key key = {
.ip = entry->ip,
.gre_id = entry->gre_id,
};
struct cpe_data data = {
.qinq_svlan = entry->svlan,
.qinq_cvlan = entry->cvlan,
.user = entry->user,
.mac_port = {
.mac = entry->eth_addr,
.out_idx = targ->mapping[port_idx],
},
.tsc = UINT64_MAX,
};
int key_found;
void* entry_in_hash;
rte_table_hash_key8_ext_dosig_ops.f_add(table, &key, &data, &key_found, &entry_in_hash);
}
}
/* Initialize cores and allocate mempools */
static void init_lcores(void)
{
struct lcore_cfg *lconf = 0;
uint32_t lcore_id = -1;
while(prox_core_next(&lcore_id, 0) == 0) {
uint8_t socket = rte_lcore_to_socket_id(lcore_id);
PROX_PANIC(socket + 1 > MAX_SOCKETS, "Can't configure core %u (on socket %u). MAX_SOCKET is set to %d\n", lcore_id, socket, MAX_SOCKETS);
}
/* need to allocate mempools as the first thing to use the lowest possible address range */
plog_info("=== Initializing mempools ===\n");
setup_mempools();
lcore_cfg_alloc_hp();
set_dest_threads();
set_task_lconf();
plog_info("=== Initializing port addresses ===\n");
init_port_addr();
plog_info("=== Initializing queue numbers on cores ===\n");
configure_if_queues();
plog_info("=== Initializing rings on cores ===\n");
init_rings();
plog_info("=== Checking configuration consistency ===\n");
check_cfg_consistent();
plog_all_rings();
setup_all_task_structs_early_init();
plog_info("=== Initializing tasks ===\n");
setup_all_task_structs();
}
/* Replace $... and each occurrence of ${...} with what has been added
through add_var */
static int parse_vars(const char **val, const char *name)
{
static char result[2048];
static char cur_var[2048];
size_t name_len = strlen(name);
enum parse_vars_state {NO_VAR, WHOLE_VAR, INLINE_VAR} state = NO_VAR;
size_t result_len = 0;
size_t start_var = 0;
if (name == result) {
*val = name;
return 0;
}
memset(result, 0, sizeof(result));
PROX_PANIC(name_len > sizeof(result), "\tUnable to parse var %s: too long\n", name);
for (size_t i = 0; i < name_len; ++i) {
switch (state) {
case NO_VAR:
if (name[i] == '$') {
if (i != name_len - 1 && name[i + 1] == '{') {
start_var = i + 2;
state = INLINE_VAR;
i = i + 1;
}
else if (i == 0 && i != name_len - 1) {
state = WHOLE_VAR;
}
else {
set_errf("Invalid variable syntax");
return -1;
}
}
else {
result[result_len++] = name[i];
}
break;
case INLINE_VAR:
if (name[i] == '}') {
const char *parsed;
cur_var[0] = '$';
size_t var_len = i - start_var;
if (var_len == 0) {
set_errf("Empty variable are not allowed");
return -1;
}
strncpy(&cur_var[1], &name[start_var], var_len);
cur_var[1 + var_len] = 0;
if (parse_single_var(&parsed, cur_var)) {
return -1;
}
strcpy(&result[result_len], parsed);
result_len += strlen(parsed);
state = NO_VAR;
}
else if (i == name_len - 1) {
set_errf("Invalid variable syntax, expected '}'.");
return -1;
}
break;
case WHOLE_VAR:
if (i == name_len - 1) {
return parse_single_var(val, name);
}
break;
}
}
*val = result;
return 0;
}
static void init_task_gen(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
static char name[] = "server_mempool";
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket_id, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_listen = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_listen == 0, "No services specified to listen on\n");
task->bundle_cfgs = prox_zmalloc(n_listen * sizeof(task->bundle_cfgs[0]), socket_id);
plogx_info("n_listen = %d\n", n_listen);
struct hash_set *hs = prox_sh_find_socket(socket_id, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 1, hash_set
will grow if full. */
hs = hash_set_create(n_listen, socket_id);
prox_sh_add_socket(socket_id, "genl4_streams", hs);
}
const struct rte_hash_parameters listen_table = {
.name = name,
.entries = n_listen * 4,
.key_len = sizeof(struct new_tuple),
.hash_func = rte_hash_crc,
.hash_func_init_val = 0,
.socket_id = socket_id,
};
name[0]++;
task->listen_hash = rte_hash_create(&listen_table);
task->listen_entries = prox_zmalloc(listen_table.entries * sizeof(task->listen_entries[0]), socket_id);
int idx = 0;
lua_pushnil(prox_lua());
while (lua_next(prox_lua(), -2)) {
task->bundle_cfgs[idx].n_stream_cfgs = 1;
task->bundle_cfgs[idx].stream_cfgs = prox_zmalloc(sizeof(*task->bundle_cfgs[idx].stream_cfgs), socket_id);
int ret = lua_to_stream_cfg(prox_lua(), STACK, NULL, socket_id, &task->bundle_cfgs[idx].stream_cfgs[0], hs);
PROX_PANIC(ret, "Failed to load stream cfg\n");
struct stream_cfg *stream = task->bundle_cfgs[idx].stream_cfgs[0];
// TODO: check mask and add to hash for each host
struct new_tuple nt = {
.dst_addr = stream->servers.ip,
.proto_id = stream->proto,
.dst_port = stream->servers.port,
.l2_types[0] = 0x0008,
};
ret = rte_hash_add_key(task->listen_hash, &nt);
PROX_PANIC(ret < 0, "Failed to add\n");
task->listen_entries[ret] = &task->bundle_cfgs[idx];
plogx_dbg("Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&nt.dst_addr)), rte_bswap16(nt.dst_port));
++idx;
lua_pop(prox_lua(), 1);
}
static char name2[] = "task_gen_hash2";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn * 2, &task->bundle_ctx_pool, NULL, 0, NULL, socket_id)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn * 2, socket_id);
task->seed = rte_rdtsc();
/* TODO: calculate the CDF of the reply distribution and the
number of replies as the number to cover for 99% of the
replies. For now, assume that this is number is 2. */
uint32_t queue_size = rte_align32pow2(targ->n_concur_conn * 2);
PROX_PANIC(queue_size == 0, "Overflow resulted in queue size 0\n");
task->fqueue = fqueue_create(queue_size, socket_id);
PROX_PANIC(task->fqueue == NULL, "Failed to allocate local queue\n");
uint32_t n_descriptors;
if (targ->nb_txports) {
PROX_PANIC(targ->nb_txports != 1, "Need exactly one TX port for L4 generation\n");
n_descriptors = prox_port_cfg[targ->tx_port_queue[0].port].n_txd;
} else {
PROX_PANIC(targ->nb_txrings != 1, "Need exactly one TX ring for L4 generation\n");
n_descriptors = 256;
}
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = n_descriptors * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void init_task_gen_client(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
static char name[] = "gen_pool";
const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
/* streams contains a lua table. Go through it and read each
stream with associated imix_fraction. */
uint32_t imix;
uint32_t i = 0;
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_bundle_cfgs = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_bundle_cfgs == 0, "No configs specified\n");
plogx_info("loading %d bundle_cfgs\n", n_bundle_cfgs);
struct hash_set *hs = prox_sh_find_socket(socket, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 8, hash_set
will grow if full. */
hs = hash_set_create(n_bundle_cfgs * 8, socket);
prox_sh_add_socket(socket, "genl4_streams", hs);
}
task->bundle_cfgs = prox_zmalloc(n_bundle_cfgs * sizeof(task->bundle_cfgs[0]), socket);
lua_pushnil(prox_lua());
int total_imix = 0;
uint32_t *occur = prox_zmalloc(n_bundle_cfgs * sizeof(*occur), socket);
struct cdf *cdf = cdf_create(n_bundle_cfgs, socket);
while (lua_next(prox_lua(), -2)) {
PROX_PANIC(lua_to_int(prox_lua(), TABLE, "imix_fraction", &imix) ||
lua_to_bundle_cfg(prox_lua(), TABLE, "bundle", socket, &task->bundle_cfgs[i], hs),
"Failed to load bundle cfg:\n%s\n", get_lua_to_errors());
cdf_add(cdf, imix);
occur[i] = imix;
total_imix += imix;
++i;
lua_pop(prox_lua(), 1);
}
lua_pop(prox_lua(), pop);
cdf_setup(cdf);
PROX_PANIC(targ->max_setup_rate == 0, "Max setup rate not set\n");
task->new_conn_cost = rte_get_tsc_hz()/targ->max_setup_rate;
static char name2[] = "task_gen_hash";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn, &task->bundle_ctx_pool, occur, n_bundle_cfgs, task->bundle_cfgs, socket)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn, socket);
task->seed = rte_rdtsc();
/* task->token_time.bytes_max = MAX_PKT_BURST * (ETHER_MAX_LEN + 20); */
/* To avoid overflowing the tx descriptors, the token bucket
size needs to be limited. The descriptors are filled most
quickly with the smallest packets. For that reason, the
token bucket size is given by "number of tx descriptors" *
"smallest Ethernet packet". */
PROX_ASSERT(targ->nb_txports == 1);
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = prox_port_cfg[targ->tx_port_queue[0].port].n_txd * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void start_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
task->new_conn_tokens = 0;
task->new_conn_last_tsc = rte_rdtsc();
}
static void stop_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
struct bundle_ctx *bundle;
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
}
static void start_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
}
static void stop_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
struct bundle_ctx *bundle;
uint8_t out[MAX_PKT_BURST];
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
if (task->cancelled) {
struct rte_mbuf *mbuf = task->mbuf_saved;
out[0] = OUT_DISCARD;
task->cancelled = 0;
task->base.tx_pkt(&task->base, &mbuf, 1, out);
}
do {
if (task->cur_mbufs_beg == task->cur_mbufs_end) {
task->cur_mbufs_end = fqueue_get(task->fqueue, task->cur_mbufs, MAX_PKT_BURST);
task->cur_mbufs_beg = 0;
if (task->cur_mbufs_end == 0)
break;
}
uint16_t n_pkts = task->cur_mbufs_end - task->cur_mbufs_beg;
struct rte_mbuf **mbufs = task->cur_mbufs + task->cur_mbufs_beg;
if (n_pkts) {
for (uint16_t j = 0; j < n_pkts; ++j) {
out[j] = OUT_DISCARD;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
} while (1);
}
static struct task_init task_init_gen1 = {
.mode_str = "genl4",
.sub_mode_str = "server",
.init = init_task_gen,
.handle = handle_gen_bulk,
.start = start_task_gen_server,
.stop = stop_task_gen_server,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_server),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
static struct task_init task_init_gen2 = {
.mode_str = "genl4",
.init = init_task_gen_client,
.handle = handle_gen_bulk_client,
.start = start_task_gen_client,
.stop = stop_task_gen_client,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_client),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
__attribute__((constructor)) static void reg_task_gen(void)
{
reg_task(&task_init_gen1);
reg_task(&task_init_gen2);
}
static struct rte_ring *init_ring_between_tasks(struct lcore_cfg *lconf, struct task_args *starg,
const struct core_task ct, uint8_t ring_idx, int idx,
struct ring_init_stats *ris)
{
uint8_t socket;
struct rte_ring *ring = NULL;
struct lcore_cfg *lworker;
struct task_args *dtarg;
PROX_ASSERT(prox_core_active(ct.core, 0));
lworker = &lcore_cfg[ct.core];
/* socket used is the one that the sending core resides on */
socket = rte_lcore_to_socket_id(lconf->id);
plog_info("\t\tCreating ring on socket %u with size %u\n"
"\t\t\tsource core, task and socket = %u, %u, %u\n"
"\t\t\tdestination core, task and socket = %u, %u, %u\n"
"\t\t\tdestination worker id = %u\n",
socket, starg->ring_size,
lconf->id, starg->id, socket,
ct.core, ct.task, rte_lcore_to_socket_id(ct.core),
ring_idx);
if (ct.type) {
struct rte_ring **dring = NULL;
if (ct.type == CTRL_TYPE_MSG)
dring = &lworker->ctrl_rings_m[ct.task];
else if (ct.type == CTRL_TYPE_PKT) {
dring = &lworker->ctrl_rings_p[ct.task];
starg->flags |= TASK_ARG_CTRL_RINGS_P;
}
if (*dring == NULL)
ring = rte_ring_create(gen_ring_name(), starg->ring_size, socket, RING_F_SC_DEQ);
else
ring = *dring;
PROX_PANIC(ring == NULL, "Cannot create ring to connect I/O core %u with worker core %u\n", lconf->id, ct.core);
starg->tx_rings[starg->tot_n_txrings_inited] = ring;
starg->tot_n_txrings_inited++;
*dring = ring;
if (lconf->id == prox_cfg.master) {
ctrl_rings[ct.core*MAX_TASKS_PER_CORE + ct.task] = ring;
} else if (ct.core == prox_cfg.master) {
starg->ctrl_plane_ring = ring;
}
plog_info("\t\tCore %u task %u to -> core %u task %u ctrl_ring %s %p %s\n",
lconf->id, starg->id, ct.core, ct.task, ct.type == CTRL_TYPE_PKT?
"pkt" : "msg", ring, ring->name);
ris->n_ctrl_rings++;
return ring;
}
dtarg = &lworker->targs[ct.task];
lworker->targs[ct.task].worker_thread_id = ring_idx;
PROX_ASSERT(dtarg->flags & TASK_ARG_RX_RING);
PROX_ASSERT(ct.task < lworker->n_tasks_all);
/* If all the following conditions are met, the ring can be
optimized away. */
if (!task_is_master(starg) && !task_is_master(dtarg) && starg->lconf->id == dtarg->lconf->id &&
starg->nb_txrings == 1 && idx == 0 && dtarg->task &&
dtarg->tot_rxrings == 1 && starg->task == dtarg->task - 1) {
plog_info("\t\tOptimizing away ring on core %u from task %u to task %u\n",
dtarg->lconf->id, starg->task, dtarg->task);
/* No need to set up ws_mbuf. */
starg->tx_opt_ring = 1;
/* During init of destination task, the buffer in the
source task will be initialized. */
dtarg->tx_opt_ring_task = starg;
ris->n_opt_rings++;
++dtarg->nb_rxrings;
return NULL;
}
int ring_created = 1;
/* Only create multi-producer rings if configured to do so AND
there is only one task sending to the task */
if ((prox_cfg.flags & DSF_MP_RINGS && count_incoming_tasks(ct.core, ct.task) > 1)
|| (prox_cfg.flags & DSF_ENABLE_BYPASS)) {
ring = get_existing_ring(ct.core, ct.task);
if (ring) {
plog_info("\t\tCore %u task %u creatign MP ring %p to core %u task %u\n",
lconf->id, starg->id, ring, ct.core, ct.task);
ring_created = 0;
}
else {
ring = rte_ring_create(gen_ring_name(), starg->ring_size, socket, RING_F_SC_DEQ);
plog_info("\t\tCore %u task %u using MP ring %p from core %u task %u\n",
lconf->id, starg->id, ring, ct.core, ct.task);
}
}
else
ring = rte_ring_create(gen_ring_name(), starg->ring_size, socket, RING_F_SP_ENQ | RING_F_SC_DEQ);
PROX_PANIC(ring == NULL, "Cannot create ring to connect I/O core %u with worker core %u\n", lconf->id, ct.core);
starg->tx_rings[starg->tot_n_txrings_inited] = ring;
starg->tot_n_txrings_inited++;
if (ring_created) {
PROX_ASSERT(dtarg->nb_rxrings < MAX_RINGS_PER_TASK);
dtarg->rx_rings[dtarg->nb_rxrings] = ring;
++dtarg->nb_rxrings;
}
dtarg->nb_slave_threads = starg->core_task_set[idx].n_elems;
dtarg->lb_friend_core = lconf->id;
dtarg->lb_friend_task = starg->id;
plog_info("\t\tWorker thread %d has core %d, task %d as a lb friend\n", ct.core, lconf->id, starg->id);
plog_info("\t\tCore %u task %u tx_ring[%u] -> core %u task %u rx_ring[%u] %p %s %u WT\n",
lconf->id, starg->id, ring_idx, ct.core, ct.task, dtarg->nb_rxrings, ring, ring->name,
dtarg->nb_slave_threads);
++ris->n_pkt_rings;
return ring;
}
static void init_task_gen(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
static char name[] = "server_mempool";
name[0]++;
task->mempool = rte_mempool_create(name,
targ->nb_mbuf - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket_id, 0);
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_listen = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_listen == 0, "No services specified to listen on\n");
task->bundle_cfgs = rte_zmalloc_socket(NULL, n_listen * sizeof(task->bundle_cfgs[0]), RTE_CACHE_LINE_SIZE, socket_id);
plogx_info("n_listen = %d\n", n_listen);
const struct rte_hash_parameters listen_table = {
.name = name,
.entries = n_listen * 4,
.key_len = sizeof(struct new_tuple),
.hash_func = rte_hash_crc,
.hash_func_init_val = 0,
.socket_id = socket_id,
};
name[0]++;
task->listen_hash = rte_hash_create(&listen_table);
task->listen_entries = rte_zmalloc_socket(NULL, listen_table.entries * sizeof(task->listen_entries[0]), RTE_CACHE_LINE_SIZE, socket_id);
int idx = 0;
lua_pushnil(prox_lua());
while (lua_next(prox_lua(), -2)) {
task->bundle_cfgs[idx].n_stream_cfgs = 1;
int ret = lua_to_stream_cfg(prox_lua(), STACK, NULL, socket_id, &task->bundle_cfgs[idx].stream_cfgs[0]);
PROX_PANIC(ret, "Failed to load stream cfg\n");
struct stream_cfg *stream = task->bundle_cfgs[idx].stream_cfgs[0];
// TODO: check mask and add to hash for each host
struct new_tuple nt = {
.dst_addr = stream->servers.ip,
.proto_id = stream->proto,
.dst_port = stream->servers.port,
.l2_types[0] = 0x0008,
};
ret = rte_hash_add_key(task->listen_hash, &nt);
PROX_PANIC(ret < 0, "Failed to add\n");
task->listen_entries[ret] = &task->bundle_cfgs[idx];
plogx_dbg("Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&nt.dst_addr)), rte_bswap16(nt.dst_port));
++idx;
lua_pop(prox_lua(), 1);
}
static char name2[] = "task_gen_hash2";
name2[0]++;
PROX_PANIC(bundle_ctx_pool_create(name2, targ->n_concur_conn*2, &task->bundle_ctx_pool, socket_id), "Failed to create conn_ctx_pool");
task->heap = heap_create(targ->n_concur_conn*2, socket_id);
task->seed = rte_rdtsc();
}
static void init_task_gen_client(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
static char name[] = "gen_pool";
const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
name[0]++;
task->mempool = rte_mempool_create(name,
targ->nb_mbuf - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket, 0);
/* streams contains a lua table. Go through it and read each
stream with associated imix_fraction. */
uint32_t imix;
int i = 0;
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_bundle_cfgs = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_bundle_cfgs == 0, "No configs specified\n");
plogx_info("loading %d bundle_cfgs\n", n_bundle_cfgs);
task->bundle_cfgs = rte_zmalloc_socket(NULL, n_bundle_cfgs * sizeof(task->bundle_cfgs[0]), RTE_CACHE_LINE_SIZE, socket);
lua_pushnil(prox_lua());
int total_imix = 0;
struct cdf *cdf = cdf_create(n_bundle_cfgs, socket);
while (lua_next(prox_lua(), -2)) {
PROX_PANIC(lua_to_int(prox_lua(), TABLE, "imix_fraction", &imix) ||
lua_to_bundle_cfg(prox_lua(), TABLE, "bundle", socket, &task->bundle_cfgs[i]),
"Failed to load bundle cfg:\n%s\n", get_lua_to_errors());
cdf_add(cdf, imix);
total_imix += imix;
++i;
lua_pop(prox_lua(), 1);
}
lua_pop(prox_lua(), pop);
cdf_setup(cdf);
task->tot_imix = total_imix;
task->cdf = cdf;
static char name2[] = "task_gen_hash";
name2[0]++;
PROX_PANIC(bundle_ctx_pool_create(name2, targ->n_concur_conn, &task->bundle_ctx_pool, socket), "Failed to create conn_ctx_pool");
task->heap = heap_create(targ->n_concur_conn, socket);
task->seed = rte_rdtsc();
}
static struct task_init task_init_gen1 = {
.mode_str = "genl4",
.sub_mode_str = "server",
.init = init_task_gen,
.handle = handle_gen_bulk,
.flag_features = TASK_ZERO_RX,
.size = sizeof(struct task_gen_server),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
static struct task_init task_init_gen2 = {
.mode_str = "genl4",
.init = init_task_gen_client,
.handle = handle_gen_bulk_client,
.flag_features = TASK_ZERO_RX,
.size = sizeof(struct task_gen_client),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
__attribute__((constructor)) static void reg_task_gen(void)
{
reg_task(&task_init_gen1);
reg_task(&task_init_gen2);
}
static int lua_to_peer_data(struct lua_State *L, enum lua_place from, const char *name, uint32_t socket, struct peer_data *peer_data, size_t *cl)
{
uint32_t hdr_len, hdr_beg, content_len, content_beg;
char hdr_file[256], content_file[256];
int pop;
if ((pop = lua_getfrom(L, from, name)) < 0)
return -1;
if (!lua_istable(L, -1))
return -1;
if (lua_getfrom(L, TABLE, "header") < 0)
return -1;
if (lua_to_int(L, TABLE, "len", &hdr_len) < 0)
return -1;
if (lua_to_int(L, TABLE, "beg", &hdr_beg) < 0)
return -1;
if (lua_to_string(L, TABLE, "file_name", hdr_file, sizeof(hdr_file)) < 0)
return -1;
lua_pop(L, 1);
if (lua_getfrom(L, TABLE, "content") < 0)
return -1;
if (lua_to_int(L, TABLE, "len", &content_len) < 0)
return -1;
if (lua_to_int(L, TABLE, "beg", &content_beg) < 0)
return -1;
if (lua_to_string(L, TABLE, "file_name", content_file, sizeof(content_file)) < 0)
return -1;
lua_pop(L, 1);
if (hdr_len == (uint32_t)-1) {
char file_name[PATH_MAX];
snprintf(file_name, sizeof(file_name), "%s/%s", get_cfg_dir(), hdr_file);
FILE *f = fopen(file_name, "r");
long file_beg;
if (!f)
return -1;
fseek(f, hdr_beg, SEEK_SET);
file_beg = ftell(f);
fseek(f, 0, SEEK_END);
hdr_len = ftell(f) - file_beg;
fclose(f);
}
if (content_len == (uint32_t)-1) {
char file_name[PATH_MAX];
snprintf(file_name, sizeof(file_name), "%s/%s", get_cfg_dir(), content_file);
FILE *f = fopen(file_name, "r");
long file_beg;
if (!f)
return -1;
fseek(f, content_beg, SEEK_SET);
file_beg = ftell(f);
fseek(f, 0, SEEK_END);
content_len = ftell(f) - file_beg;
fclose(f);
}
*cl = content_len;
peer_data->mem = rte_zmalloc_socket(NULL, hdr_len + content_len, RTE_CACHE_LINE_SIZE, socket);
PROX_PANIC(peer_data->mem == NULL, "Failed to allocate memory (%u bytes) to hold headers and contents for peer\n", hdr_len + content_len);
peer_data->hdr = peer_data->mem;
peer_data->hdr_len = hdr_len;
peer_data->content = peer_data->mem + hdr_len;
char file_name[PATH_MAX];
snprintf(file_name, sizeof(file_name), "%s/%s", get_cfg_dir(), hdr_file);
FILE *f = fopen(file_name, "r");
if (!f) {
return -1;
}
fseek(f, hdr_beg, SEEK_SET);
size_t ret;
ret = fread(peer_data->hdr, 1, hdr_len, f);
if ((uint32_t)ret != hdr_len) {
plog_err("Failed to read hdr, %zu, %u\n", ret, hdr_len);
return -1;
}
fclose(f);
snprintf(file_name, sizeof(file_name), "%s/%s", get_cfg_dir(), content_file);
f = fopen(file_name, "r");
if (!f)
return -1;
fseek(f, content_beg, SEEK_SET);
ret = fread(peer_data->content, 1, content_len, f);
if ((uint32_t)ret != content_len) {
plog_err("Failed to read content, %zu, %u\n", ret, content_len);
return -1;
}
fclose(f);
lua_pop(L, pop);
return 0;
}
static void setup_mempools_unique_per_socket(void)
{
uint32_t flags = 0;
char name[64];
struct lcore_cfg *lconf = NULL;
struct task_args *targ;
struct rte_mempool *pool[MAX_SOCKETS];
uint32_t mbuf_count[MAX_SOCKETS] = {0};
uint32_t nb_cache_mbuf[MAX_SOCKETS] = {0};
uint32_t mbuf_size[MAX_SOCKETS] = {0};
while (core_targ_next_early(&lconf, &targ, 0) == 0) {
PROX_PANIC(targ->task_init == NULL, "task_init = NULL, is mode specified for core %d, task %d ?\n", lconf->id, targ->id);
uint8_t socket = rte_lcore_to_socket_id(lconf->id);
PROX_ASSERT(socket < MAX_SOCKETS);
if (targ->mbuf_size_set_explicitely)
flags = MEMPOOL_F_NO_SPREAD;
if ((!targ->mbuf_size_set_explicitely) && (targ->task_init->mbuf_size != 0)) {
targ->mbuf_size = targ->task_init->mbuf_size;
}
if (targ->rx_port_queue[0].port != OUT_DISCARD) {
struct prox_port_cfg* port_cfg = &prox_port_cfg[targ->rx_port_queue[0].port];
PROX_ASSERT(targ->nb_mbuf != 0);
mbuf_count[socket] += targ->nb_mbuf;
if (nb_cache_mbuf[socket] == 0)
nb_cache_mbuf[socket] = targ->nb_cache_mbuf;
else {
PROX_PANIC(nb_cache_mbuf[socket] != targ->nb_cache_mbuf,
"all mbuf_cache must have the same size if using a unique mempool per socket\n");
}
if (mbuf_size[socket] == 0)
mbuf_size[socket] = targ->mbuf_size;
else {
PROX_PANIC(mbuf_size[socket] != targ->mbuf_size,
"all mbuf_size must have the same size if using a unique mempool per socket\n");
}
if ((!targ->mbuf_size_set_explicitely) && (strcmp(port_cfg->short_name, "vmxnet3") == 0)) {
if (mbuf_size[socket] < MBUF_SIZE + RTE_PKTMBUF_HEADROOM)
mbuf_size[socket] = MBUF_SIZE + RTE_PKTMBUF_HEADROOM;
}
}
}
for (int i = 0 ; i < MAX_SOCKETS; i++) {
if (mbuf_count[i] != 0) {
sprintf(name, "socket_%u_pool", i);
pool[i] = rte_mempool_create(name,
mbuf_count[i] - 1, mbuf_size[i],
nb_cache_mbuf[i],
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
prox_pktmbuf_init, NULL,
i, flags);
PROX_PANIC(pool[i] == NULL, "\t\tError: cannot create mempool for socket %u\n", i);
plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", pool[i],
mbuf_count[i], mbuf_size[i], nb_cache_mbuf[i], i);
if (prox_cfg.flags & DSF_SHUFFLE) {
shuffle_mempool(pool[i], mbuf_count[i]);
}
}
}
lconf = NULL;
while (core_targ_next_early(&lconf, &targ, 0) == 0) {
uint8_t socket = rte_lcore_to_socket_id(lconf->id);
if (targ->rx_port_queue[0].port != OUT_DISCARD) {
/* use this pool for the interface that the core is receiving from */
/* If one core receives from multiple ports, all the ports use the same mempool */
targ->pool = pool[socket];
/* Set the number of mbuf to the number of the unique mempool, so that the used and free work */
targ->nb_mbuf = mbuf_count[socket];
plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
targ->nb_mbuf, mbuf_size[socket], targ->nb_cache_mbuf, socket);
}
}
}
static void setup_mempool_for_rx_task(struct lcore_cfg *lconf, struct task_args *targ)
{
const uint8_t socket = rte_lcore_to_socket_id(lconf->id);
struct prox_port_cfg *port_cfg = &prox_port_cfg[targ->rx_port_queue[0].port];
const struct rte_memzone *mz;
struct rte_mempool *mp = NULL;
uint32_t flags = 0;
char memzone_name[64];
char name[64];
/* mbuf size can be set
* - from config file (highest priority, overwriting any other config) - should only be used as workaround
* - through each 'mode', overwriting the default mbuf_size
* - defaulted to MBUF_SIZE i.e. 1518 Bytes
* Except is set expliciteky, ensure that size is big enough for vmxnet3 driver
*/
if (targ->mbuf_size_set_explicitely) {
flags = MEMPOOL_F_NO_SPREAD;
/* targ->mbuf_size already set */
}
else if (targ->task_init->mbuf_size != 0) {
/* mbuf_size not set through config file but set through mode */
targ->mbuf_size = targ->task_init->mbuf_size;
}
else if (strcmp(port_cfg->short_name, "vmxnet3") == 0) {
if (targ->mbuf_size < MBUF_SIZE + RTE_PKTMBUF_HEADROOM)
targ->mbuf_size = MBUF_SIZE + RTE_PKTMBUF_HEADROOM;
}
/* allocate memory pool for packets */
PROX_ASSERT(targ->nb_mbuf != 0);
if (targ->pool_name[0] == '\0') {
sprintf(name, "core_%u_port_%u_pool", lconf->id, targ->id);
}
snprintf(memzone_name, sizeof(memzone_name)-1, "MP_%s", targ->pool_name);
mz = rte_memzone_lookup(memzone_name);
if (mz != NULL) {
mp = (struct rte_mempool*)mz->addr;
targ->nb_mbuf = mp->size;
targ->pool = mp;
}
#ifdef RTE_LIBRTE_IVSHMEM_FALSE
if (mz != NULL && mp != NULL && mp->phys_addr != mz->ioremap_addr) {
/* Init mbufs with ioremap_addr for dma */
mp->phys_addr = mz->ioremap_addr;
mp->elt_pa[0] = mp->phys_addr + (mp->elt_va_start - (uintptr_t)mp);
struct prox_pktmbuf_reinit_args init_args;
init_args.mp = mp;
init_args.lconf = lconf;
uint32_t elt_sz = mp->elt_size + mp->header_size + mp->trailer_size;
rte_mempool_obj_iter((void*)mp->elt_va_start, mp->size, elt_sz, 1,
mp->elt_pa, mp->pg_num, mp->pg_shift, prox_pktmbuf_reinit, &init_args);
}
#endif
/* Use this pool for the interface that the core is
receiving from if one core receives from multiple
ports, all the ports use the same mempool */
if (targ->pool == NULL) {
plog_info("\t\tCreating mempool with name '%s'\n", name);
targ->pool = rte_mempool_create(name,
targ->nb_mbuf - 1, targ->mbuf_size,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
prox_pktmbuf_init, lconf,
socket, flags);
}
PROX_PANIC(targ->pool == NULL,
"\t\tError: cannot create mempool for core %u port %u: %s\n", lconf->id, targ->id, rte_strerror(rte_errno));
plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", targ->pool,
targ->nb_mbuf, targ->mbuf_size, targ->nb_cache_mbuf, socket);
if (prox_cfg.flags & DSF_SHUFFLE) {
shuffle_mempool(targ->pool, targ->nb_mbuf);
}
}
