static void init_task_gen(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
static char name[] = "server_mempool";
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket_id, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_listen = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_listen == 0, "No services specified to listen on\n");
task->bundle_cfgs = prox_zmalloc(n_listen * sizeof(task->bundle_cfgs[0]), socket_id);
plogx_info("n_listen = %d\n", n_listen);
struct hash_set *hs = prox_sh_find_socket(socket_id, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 1, hash_set
will grow if full. */
hs = hash_set_create(n_listen, socket_id);
prox_sh_add_socket(socket_id, "genl4_streams", hs);
}
const struct rte_hash_parameters listen_table = {
.name = name,
.entries = n_listen * 4,
.key_len = sizeof(struct new_tuple),
.hash_func = rte_hash_crc,
.hash_func_init_val = 0,
.socket_id = socket_id,
};
name[0]++;
task->listen_hash = rte_hash_create(&listen_table);
task->listen_entries = prox_zmalloc(listen_table.entries * sizeof(task->listen_entries[0]), socket_id);
int idx = 0;
lua_pushnil(prox_lua());
while (lua_next(prox_lua(), -2)) {
task->bundle_cfgs[idx].n_stream_cfgs = 1;
task->bundle_cfgs[idx].stream_cfgs = prox_zmalloc(sizeof(*task->bundle_cfgs[idx].stream_cfgs), socket_id);
int ret = lua_to_stream_cfg(prox_lua(), STACK, NULL, socket_id, &task->bundle_cfgs[idx].stream_cfgs[0], hs);
PROX_PANIC(ret, "Failed to load stream cfg\n");
struct stream_cfg *stream = task->bundle_cfgs[idx].stream_cfgs[0];
// TODO: check mask and add to hash for each host
struct new_tuple nt = {
.dst_addr = stream->servers.ip,
.proto_id = stream->proto,
.dst_port = stream->servers.port,
.l2_types[0] = 0x0008,
};
ret = rte_hash_add_key(task->listen_hash, &nt);
PROX_PANIC(ret < 0, "Failed to add\n");
task->listen_entries[ret] = &task->bundle_cfgs[idx];
plogx_dbg("Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&nt.dst_addr)), rte_bswap16(nt.dst_port));
++idx;
lua_pop(prox_lua(), 1);
}
static char name2[] = "task_gen_hash2";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn * 2, &task->bundle_ctx_pool, NULL, 0, NULL, socket_id)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn * 2, socket_id);
task->seed = rte_rdtsc();
/* TODO: calculate the CDF of the reply distribution and the
number of replies as the number to cover for 99% of the
replies. For now, assume that this is number is 2. */
uint32_t queue_size = rte_align32pow2(targ->n_concur_conn * 2);
PROX_PANIC(queue_size == 0, "Overflow resulted in queue size 0\n");
task->fqueue = fqueue_create(queue_size, socket_id);
PROX_PANIC(task->fqueue == NULL, "Failed to allocate local queue\n");
uint32_t n_descriptors;
if (targ->nb_txports) {
PROX_PANIC(targ->nb_txports != 1, "Need exactly one TX port for L4 generation\n");
n_descriptors = prox_port_cfg[targ->tx_port_queue[0].port].n_txd;
} else {
PROX_PANIC(targ->nb_txrings != 1, "Need exactly one TX ring for L4 generation\n");
n_descriptors = 256;
}
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = n_descriptors * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void init_task_gen_client(struct task_base *tbase, struct task_args *targ)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
static char name[] = "gen_pool";
const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
name[0]++;
task->mempool = rte_mempool_create(name,
4*1024 - 1, MBUF_SIZE,
targ->nb_cache_mbuf,
sizeof(struct rte_pktmbuf_pool_private),
rte_pktmbuf_pool_init, NULL,
rte_pktmbuf_init, 0,
socket, 0);
PROX_PANIC(task->mempool == NULL, "Failed to allocate memory pool with %u elements\n", 4*1024 - 1);
/* streams contains a lua table. Go through it and read each
stream with associated imix_fraction. */
uint32_t imix;
uint32_t i = 0;
int pop = lua_getfrom(prox_lua(), GLOBAL, targ->streams);
PROX_PANIC(pop < 0, "Failed to find '%s' in lua\n", targ->streams);
lua_len(prox_lua(), -1);
uint32_t n_bundle_cfgs = lua_tointeger(prox_lua(), -1);
lua_pop(prox_lua(), 1);
PROX_PANIC(n_bundle_cfgs == 0, "No configs specified\n");
plogx_info("loading %d bundle_cfgs\n", n_bundle_cfgs);
struct hash_set *hs = prox_sh_find_socket(socket, "genl4_streams");
if (hs == NULL) {
/* Expected number of streams per bundle = 8, hash_set
will grow if full. */
hs = hash_set_create(n_bundle_cfgs * 8, socket);
prox_sh_add_socket(socket, "genl4_streams", hs);
}
task->bundle_cfgs = prox_zmalloc(n_bundle_cfgs * sizeof(task->bundle_cfgs[0]), socket);
lua_pushnil(prox_lua());
int total_imix = 0;
uint32_t *occur = prox_zmalloc(n_bundle_cfgs * sizeof(*occur), socket);
struct cdf *cdf = cdf_create(n_bundle_cfgs, socket);
while (lua_next(prox_lua(), -2)) {
PROX_PANIC(lua_to_int(prox_lua(), TABLE, "imix_fraction", &imix) ||
lua_to_bundle_cfg(prox_lua(), TABLE, "bundle", socket, &task->bundle_cfgs[i], hs),
"Failed to load bundle cfg:\n%s\n", get_lua_to_errors());
cdf_add(cdf, imix);
occur[i] = imix;
total_imix += imix;
++i;
lua_pop(prox_lua(), 1);
}
lua_pop(prox_lua(), pop);
cdf_setup(cdf);
PROX_PANIC(targ->max_setup_rate == 0, "Max setup rate not set\n");
task->new_conn_cost = rte_get_tsc_hz()/targ->max_setup_rate;
static char name2[] = "task_gen_hash";
name2[0]++;
plogx_dbg("Creating bundle ctx pool\n");
if (bundle_ctx_pool_create(name2, targ->n_concur_conn, &task->bundle_ctx_pool, occur, n_bundle_cfgs, task->bundle_cfgs, socket)) {
cmd_mem_stats();
PROX_PANIC(1, "Failed to create conn_ctx_pool\n");
}
task->heap = heap_create(targ->n_concur_conn, socket);
task->seed = rte_rdtsc();
/* task->token_time.bytes_max = MAX_PKT_BURST * (ETHER_MAX_LEN + 20); */
/* To avoid overflowing the tx descriptors, the token bucket
size needs to be limited. The descriptors are filled most
quickly with the smallest packets. For that reason, the
token bucket size is given by "number of tx descriptors" *
"smallest Ethernet packet". */
PROX_ASSERT(targ->nb_txports == 1);
struct token_time_cfg tt_cfg = {
.bpp = targ->rate_bps,
.period = rte_get_tsc_hz(),
.bytes_max = prox_port_cfg[targ->tx_port_queue[0].port].n_txd * (ETHER_MIN_LEN + 20),
};
token_time_init(&task->token_time, &tt_cfg);
}
static void start_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
task->new_conn_tokens = 0;
task->new_conn_last_tsc = rte_rdtsc();
}
static void stop_task_gen_client(struct task_base *tbase)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
struct bundle_ctx *bundle;
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
}
static void start_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
token_time_reset(&task->token_time, rte_rdtsc(), 0);
}
static void stop_task_gen_server(struct task_base *tbase)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
struct bundle_ctx *bundle;
uint8_t out[MAX_PKT_BURST];
while (!heap_is_empty(task->heap)) {
bundle = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
bundle_expire(bundle, &task->bundle_ctx_pool, &task->l4_stats);
}
if (task->cancelled) {
struct rte_mbuf *mbuf = task->mbuf_saved;
out[0] = OUT_DISCARD;
task->cancelled = 0;
task->base.tx_pkt(&task->base, &mbuf, 1, out);
}
do {
if (task->cur_mbufs_beg == task->cur_mbufs_end) {
task->cur_mbufs_end = fqueue_get(task->fqueue, task->cur_mbufs, MAX_PKT_BURST);
task->cur_mbufs_beg = 0;
if (task->cur_mbufs_end == 0)
break;
}
uint16_t n_pkts = task->cur_mbufs_end - task->cur_mbufs_beg;
struct rte_mbuf **mbufs = task->cur_mbufs + task->cur_mbufs_beg;
if (n_pkts) {
for (uint16_t j = 0; j < n_pkts; ++j) {
out[j] = OUT_DISCARD;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
} while (1);
}
static struct task_init task_init_gen1 = {
.mode_str = "genl4",
.sub_mode_str = "server",
.init = init_task_gen,
.handle = handle_gen_bulk,
.start = start_task_gen_server,
.stop = stop_task_gen_server,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_server),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
static struct task_init task_init_gen2 = {
.mode_str = "genl4",
.init = init_task_gen_client,
.handle = handle_gen_bulk_client,
.start = start_task_gen_client,
.stop = stop_task_gen_client,
.flag_features = TASK_FEATURE_ZERO_RX,
.size = sizeof(struct task_gen_client),
.mbuf_size = 2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM,
};
__attribute__((constructor)) static void reg_task_gen(void)
{
reg_task(&task_init_gen1);
reg_task(&task_init_gen2);
}
static int handle_gen_scheduled(struct task_gen_server *task)
{
struct bundle_ctx *conn;
uint8_t out[MAX_PKT_BURST];
int ret;
uint16_t n_called_back = 0;
if (task->cancelled) {
struct rte_mbuf *mbuf = task->mbuf_saved;
uint16_t pkt_len = mbuf_wire_size(mbuf);
if (token_time_take(&task->token_time, pkt_len) == 0) {
task->cancelled = 0;
out[0] = 0;
task->base.tx_pkt(&task->base, &mbuf, 1, out);
}
else {
return -1;
}
}
if (0 != refill_mbufs(&task->n_new_mbufs, task->mempool, task->new_mbufs))
return -1;
conn = NULL;
while (heap_top_is_lower(task->heap, rte_rdtsc()) && n_called_back < task->n_new_mbufs) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
struct rte_mbuf *mbuf = task->new_mbufs[n_called_back];
uint16_t pkt_len = mbuf_wire_size(mbuf);
if (token_time_take(&task->token_time, pkt_len) == 0) {
out[n_called_back] = 0;
n_called_back++;
}
else {
struct ether_hdr *eth = rte_pktmbuf_mtod(mbuf, struct ether_hdr *);
struct ipv4_hdr *ip = (struct ipv4_hdr*)(eth + 1);
struct tcp_hdr *tcp = (struct tcp_hdr*)(ip + 1);
task->out_saved = 0;
task->cancelled = 1;
task->mbuf_saved = mbuf;
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
/* The mbuf that is currently been
processed (and which has been
cancelled) is saved in
task->mbuf_saved. It will be
restored as the first mbuf when
this function is called again. */
task->n_new_mbufs -= (n_called_back + 1);
return -1;
}
}
}
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
return 0;
}
static void handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
struct pkt_tuple pkt_tuple[MAX_PKT_BURST];
uint8_t out[MAX_PKT_BURST];
struct l4_meta l4_meta[MAX_PKT_BURST];
struct bundle_ctx *conn;
int ret;
for (uint16_t j = 0; j < n_pkts; ++j) {
if (parse_pkt(mbufs[j], &pkt_tuple[j], &l4_meta[j]))
plogdx_err(mbufs[j], "Unknown packet, parsing failed\n");
}
/* Main proc loop */
for (uint16_t j = 0; j < n_pkts; ++j) {
conn = NULL;
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple[j]);
if (ret >= 0)
conn = task->bundle_ctx_pool.hash_entries[ret];
/* If not part of existing connection, try to create a connection */
if (NULL == conn) {
struct new_tuple nt;
nt.dst_addr = pkt_tuple[j].dst_addr;
nt.proto_id = pkt_tuple[j].proto_id;
nt.dst_port = pkt_tuple[j].dst_port;
rte_memcpy(nt.l2_types, pkt_tuple[j].l2_types, sizeof(nt.l2_types));
const struct bundle_cfg *n;
if (NULL != (n = server_accept(task, &nt))) {
conn = bundle_ctx_pool_get(&task->bundle_ctx_pool);
if (!conn) {
out[j] = NO_PORT_AVAIL;
plogx_err("No more free bundles to accept new connection\n");
continue;
}
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple[j]);
if (ret < 0) {
out[j] = NO_PORT_AVAIL;
bundle_ctx_pool_put(&task->bundle_ctx_pool, conn);
plog_err("Adding key failed while trying to accept connection\n");
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = conn;
bundle_init(conn, n, task->heap, PEER_SERVER, &task->seed);
conn->tuple = pkt_tuple[j];
if (conn->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
}
}
/* bundle contains either an active connection or a
newly created connection. If it is NULL, then not
listening. */
if (NULL != conn) {
int ret = bundle_proc_data(conn, mbufs[j], &l4_meta[j], &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[j] = ret == 0? 0: NO_PORT_AVAIL;
}
else {
plog_err("Packet received for service that does not exist\n");
pkt_tuple_debug(&pkt_tuple[j]);
plogd_dbg(mbufs[j], NULL);
out[j] = NO_PORT_AVAIL;
}
}
conn = NULL;
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
if (!(task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)))
return ;
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
if (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)) {
uint16_t n_called_back = 0;
while (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
}
static int handle_gen_bulk_client(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
uint8_t out[MAX_PKT_BURST] = {0};
struct bundle_ctx *conn;
int ret;
if (n_pkts) {
for (int i = 0; i < n_pkts; ++i) {
struct pkt_tuple pt;
struct l4_meta l4_meta;
if (parse_pkt(mbufs[i], &pt, &l4_meta)) {
plogdx_err(mbufs[i], "Parsing failed\n");
out[i] = OUT_DISCARD;
continue;
}
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pt);
if (ret < 0) {
plogx_dbg("Client: packet RX that does not belong to connection:"
"Client = "IPv4_BYTES_FMT":%d, Server = "IPv4_BYTES_FMT":%d\n",
IPv4_BYTES(((uint8_t*)&pt.dst_addr)),
rte_bswap16(pt.dst_port),
IPv4_BYTES(((uint8_t*)&pt.src_addr)),
rte_bswap16(pt.src_port));
plogdx_dbg(mbufs[i], NULL);
if (pt.proto_id == IPPROTO_TCP) {
stream_tcp_create_rst(mbufs[i], &l4_meta, &pt);
out[i] = 0;
continue;
}
else {
out[i] = OUT_DISCARD;
continue;
}
}
conn = task->bundle_ctx_pool.hash_entries[ret];
ret = bundle_proc_data(conn, mbufs[i], &l4_meta, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: OUT_HANDLED;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
/* If there is at least one callback to handle, handle at most MAX_PKT_BURST */
if (heap_top_is_lower(task->heap, rte_rdtsc())) {
if (0 != refill_mbufs(&task->n_new_mbufs, task->mempool, task->new_mbufs))
return 0;
uint16_t n_called_back = 0;
while (heap_top_is_lower(task->heap, rte_rdtsc()) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
plogx_dbg("During callback, will send %d packets\n", n_called_back);
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
uint32_t n_new = task->bundle_ctx_pool.n_free_bundles;
n_new = n_new > MAX_PKT_BURST? MAX_PKT_BURST : n_new;
uint64_t diff = (rte_rdtsc() - task->new_conn_last_tsc)/task->new_conn_cost;
task->new_conn_last_tsc += diff * task->new_conn_cost;
task->new_conn_tokens += diff;
if (task->new_conn_tokens > 16)
task->new_conn_tokens = 16;
if (n_new > task->new_conn_tokens)
n_new = task->new_conn_tokens;
task->new_conn_tokens -= n_new;
if (n_new == 0)
return 0;
if (0 != refill_mbufs(&task->n_new_mbufs, task->mempool, task->new_mbufs))
return 0;
for (uint32_t i = 0; i < n_new; ++i) {
struct bundle_ctx *bundle_ctx = bundle_ctx_pool_get_w_cfg(&task->bundle_ctx_pool);
PROX_ASSERT(bundle_ctx);
struct pkt_tuple *pt = &bundle_ctx->tuple;
int n_retries = 0;
do {
/* Note that the actual packet sent will
contain swapped addresses and ports
(i.e. pkt.src <=> tuple.dst). The incoming
packet will match this struct. */
bundle_init(bundle_ctx, task->heap, PEER_CLIENT, &task->seed);
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret >= 0) {
if (n_retries++ == 1000) {
plogx_err("Already tried 1K times\n");
}
}
} while (ret >= 0);
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret < 0) {
plogx_err("Failed to add key ret = %d, n_free = %d\n", ret, task->bundle_ctx_pool.n_free_bundles);
bundle_ctx_pool_put(&task->bundle_ctx_pool, bundle_ctx);
pkt_tuple_debug2(pt);
out[i] = OUT_DISCARD;
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = bundle_ctx;
if (bundle_ctx->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
task->l4_stats.bundles_created++;
ret = bundle_proc_data(bundle_ctx, task->new_mbufs[i], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: OUT_HANDLED;
}
int ret2 = task->base.tx_pkt(&task->base, task->new_mbufs, n_new, out);
task->n_new_mbufs -= n_new;
return ret2;
}
static void handle_gen_bulk_client(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
uint8_t out[MAX_PKT_BURST] = {0};
struct bundle_ctx *conn;
int ret;
if (n_pkts) {
for (int i = 0; i < n_pkts; ++i) {
struct pkt_tuple pt;
struct l4_meta l4_meta;
if (parse_pkt(mbufs[i], &pt, &l4_meta)) {
plogdx_err(mbufs[i], "Parsing failed\n");
out[i] = NO_PORT_AVAIL;
continue;
}
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pt);
if (ret < 0) {
plogx_dbg("Client: packet RX that does not belong to connection:"
"Client = "IPv4_BYTES_FMT":%d, Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&pt.dst_addr)), rte_bswap16(pt.dst_port), IPv4_BYTES(((uint8_t*)&pt.src_addr)), rte_bswap16(pt.src_port));
plogdx_dbg(mbufs[i], NULL);
// if tcp, send RST
/* pkt_tuple_debug2(&pt); */
out[i] = NO_PORT_AVAIL;
continue;
}
conn = task->bundle_ctx_pool.hash_entries[ret];
ret = bundle_proc_data(conn, mbufs[i], &l4_meta, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: NO_PORT_AVAIL;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
plogx_err("4Mempool alloc failed %d\n", MAX_PKT_BURST);
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
/* If there is at least one callback to handle, handle at most MAX_PKT_BURST */
if (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)) {
uint16_t n_called_back = 0;
while (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
plogx_dbg("During callback, will send %d packets\n", n_called_back);
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
int n_new = task->bundle_ctx_pool.n_free_bundles;
n_new = n_new > MAX_PKT_BURST? MAX_PKT_BURST : n_new;
if (n_new == 0)
return ;
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
plogx_err("4Mempool alloc failed %d\n", MAX_PKT_BURST);
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
for (int i = 0; i < n_new; ++i) {
int32_t ret = cdf_sample(task->cdf, &task->seed);
/* Select a new bundle_cfg according to imix */
struct bundle_cfg *bundle_cfg = &task->bundle_cfgs[ret];
struct bundle_ctx *bundle_ctx;
bundle_ctx = bundle_ctx_pool_get(&task->bundle_ctx_pool);
/* Should be an assert: */
if (!bundle_ctx) {
plogx_err("No more available bundles\n");
exit(-1);
}
struct pkt_tuple *pt = &bundle_ctx->tuple;
int n_retries = 0;
do {
/* Note that the actual packet sent will
contain swapped addresses and ports
(i.e. pkt.src <=> tuple.dst). The incoming
packet will match this struct. */
bundle_init(bundle_ctx, bundle_cfg, task->heap, PEER_CLIENT, &task->seed);
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)pt);
if (n_retries == 1000) {
plogx_err("Already tried 1K times\n");
}
if (ret >= 0) {
n_retries++;
}
} while (ret >= 0);
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret < 0) {
plogx_err("Failed to add key ret = %d, n_free = %d\n", ret, task->bundle_ctx_pool.n_free_bundles);
bundle_ctx_pool_put(&task->bundle_ctx_pool, bundle_ctx);
pkt_tuple_debug2(pt);
out[i] = NO_PORT_AVAIL;
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = bundle_ctx;
if (bundle_ctx->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
ret = bundle_proc_data(bundle_ctx, task->new_mbufs[i], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: NO_PORT_AVAIL;
}
task->base.tx_pkt(&task->base, task->new_mbufs, n_new, out);
task->n_new_mbufs -= n_new;
}
