static inline int wait_command_handled(struct lcore_cfg *lconf)
{
uint64_t t1 = rte_rdtsc(), t2;
while (lconf_is_req(lconf)) {
t2 = rte_rdtsc();
if (t2 - t1 > 5 * rte_get_tsc_hz()) {
// Failed to handle command ...
for (uint8_t task_id = 0; task_id < lconf->n_tasks_all; ++task_id) {
struct task_args *targs = &lconf->targs[task_id];
if (!(targs->flags & TASK_ARG_DROP)) {
plogx_err("Failed to handle command - task is in NO_DROP and might be stuck...\n");
return - 1;
}
}
plogx_err("Failed to handle command\n");
return -1;
}
}
return 0;
}
static int refill_mbufs(uint32_t *n_new_mbufs, struct rte_mempool *mempool, struct rte_mbuf **mbufs)
{
if (*n_new_mbufs == MAX_PKT_BURST)
return 0;
if (rte_mempool_get_bulk(mempool, (void **)mbufs, MAX_PKT_BURST - *n_new_mbufs) < 0) {
plogx_err("4Mempool alloc failed for %d mbufs\n", MAX_PKT_BURST - *n_new_mbufs);
return -1;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - *n_new_mbufs; ++i) {
init_mbuf_seg(mbufs[i]);
}
*n_new_mbufs = MAX_PKT_BURST;
return 0;
}
/* Encapsulate IPv6 packet in QinQ where the QinQ is derived from the IPv6 address */
static inline uint8_t handle_qinq_encap6(struct rte_mbuf *mbuf, struct task_qinq_encap6 *task)
{
struct qinq_hdr *pqinq = (struct qinq_hdr *)rte_pktmbuf_prepend(mbuf, 2 * sizeof(struct vlan_hdr));
PROX_RUNTIME_ASSERT(pqinq);
struct ipv6_hdr *pip6 = (struct ipv6_hdr *)(pqinq + 1);
if (pip6->hop_limits) {
pip6->hop_limits--;
}
else {
plog_info("TTL = 0 => Dropping\n");
return NO_PORT_AVAIL;
}
// TODO: optimize to use bulk as intended with the rte_table_library
uint64_t pkts_mask = RTE_LEN2MASK(1, uint64_t);
uint64_t lookup_hit_mask;
struct cpe_data* entries[64]; // TODO: use bulk size
rte_table_hash_ext_dosig_ops.f_lookup(task->cpe_table, &mbuf, pkts_mask, &lookup_hit_mask, (void**)entries);
if (lookup_hit_mask == 0x1) {
/* will also overwrite part of the destination addr */
(*(uint64_t *)pqinq) = entries[0]->mac_port_8bytes;
pqinq->svlan.eth_proto = task->qinq_tag;
pqinq->cvlan.eth_proto = ETYPE_VLAN;
pqinq->svlan.vlan_tci = entries[0]->qinq_svlan;
pqinq->cvlan.vlan_tci = entries[0]->qinq_cvlan;
pqinq->ether_type = ETYPE_IPv6;
/* classification can only be done from this point */
if (task->runtime_flags & TASK_CLASSIFY) {
rte_sched_port_pkt_write(mbuf, 0, entries[0]->user, 0, 0, 0);
}
return 0;
}
else {
plogx_err("Unknown IP " IPv6_BYTES_FMT "\n", IPv6_BYTES(pip6->dst_addr));
return NO_PORT_AVAIL;
}
}
static int handle_gen_queued(struct task_gen_server *task)
{
uint8_t out[MAX_PKT_BURST];
struct bundle_ctx *conn;
struct pkt_tuple pkt_tuple;
struct l4_meta l4_meta;
uint16_t j;
uint16_t cancelled = 0;
int ret;
if (task->cur_mbufs_beg == task->cur_mbufs_end) {
task->cur_mbufs_end = fqueue_get(task->fqueue, task->cur_mbufs, MAX_PKT_BURST);
task->cur_mbufs_beg = 0;
}
uint16_t n_pkts = task->cur_mbufs_end - task->cur_mbufs_beg;
struct rte_mbuf **mbufs = task->cur_mbufs + task->cur_mbufs_beg;
j = task->cancelled;
if (task->cancelled) {
uint16_t pkt_len = mbuf_wire_size(mbufs[0]);
if (token_time_take(&task->token_time, pkt_len) != 0)
return -1;
out[0] = task->out_saved;
task->cancelled = 0;
}
/* Main proc loop */
for (; j < n_pkts; ++j) {
if (parse_pkt(mbufs[j], &pkt_tuple, &l4_meta)) {
plogdx_err(mbufs[j], "Unknown packet, parsing failed\n");
out[j] = OUT_DISCARD;
}
conn = NULL;
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple);
if (ret >= 0)
conn = task->bundle_ctx_pool.hash_entries[ret];
else {
/* If not part of existing connection, try to create a connection */
struct new_tuple nt;
nt.dst_addr = pkt_tuple.dst_addr;
nt.proto_id = pkt_tuple.proto_id;
nt.dst_port = pkt_tuple.dst_port;
rte_memcpy(nt.l2_types, pkt_tuple.l2_types, sizeof(nt.l2_types));
const struct bundle_cfg *n;
if (NULL != (n = server_accept(task, &nt))) {
conn = bundle_ctx_pool_get(&task->bundle_ctx_pool);
if (!conn) {
out[j] = OUT_DISCARD;
plogx_err("No more free bundles to accept new connection\n");
continue;
}
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple);
if (ret < 0) {
out[j] = OUT_DISCARD;
bundle_ctx_pool_put(&task->bundle_ctx_pool, conn);
plog_err("Adding key failed while trying to accept connection\n");
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = conn;
bundle_init_w_cfg(conn, n, task->heap, PEER_SERVER, &task->seed);
conn->tuple = pkt_tuple;
if (conn->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
}
else {
plog_err("Packet received for service that does not exist :\n"
"source ip = %0x:%u\n"
"dst ip = %0x:%u\n",
pkt_tuple.src_addr, rte_bswap16(pkt_tuple.src_port),
pkt_tuple.dst_addr, rte_bswap16(pkt_tuple.dst_port));
}
}
/* bundle contains either an active connection or a
newly created connection. If it is NULL, then not
listening. */
if (NULL != conn) {
ret = bundle_proc_data(conn, mbufs[j], &l4_meta, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[j] = ret == 0? 0: OUT_HANDLED;
if (ret == 0) {
uint16_t pkt_len = mbuf_wire_size(mbufs[j]);
if (token_time_take(&task->token_time, pkt_len) != 0) {
task->out_saved = out[j];
task->cancelled = 1;
task->base.tx_pkt(&task->base, mbufs, j, out);
task->cur_mbufs_beg += j;
return -1;
}
}
}
else {
pkt_tuple_debug(&pkt_tuple);
plogd_dbg(mbufs[j], NULL);
out[j] = OUT_DISCARD;
}
}
task->base.tx_pkt(&task->base, mbufs, j, out);
task->cur_mbufs_beg += j;
return 0;
}
static int handle_gen_bulk_client(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
uint8_t out[MAX_PKT_BURST] = {0};
struct bundle_ctx *conn;
int ret;
if (n_pkts) {
for (int i = 0; i < n_pkts; ++i) {
struct pkt_tuple pt;
struct l4_meta l4_meta;
if (parse_pkt(mbufs[i], &pt, &l4_meta)) {
plogdx_err(mbufs[i], "Parsing failed\n");
out[i] = OUT_DISCARD;
continue;
}
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pt);
if (ret < 0) {
plogx_dbg("Client: packet RX that does not belong to connection:"
"Client = "IPv4_BYTES_FMT":%d, Server = "IPv4_BYTES_FMT":%d\n",
IPv4_BYTES(((uint8_t*)&pt.dst_addr)),
rte_bswap16(pt.dst_port),
IPv4_BYTES(((uint8_t*)&pt.src_addr)),
rte_bswap16(pt.src_port));
plogdx_dbg(mbufs[i], NULL);
if (pt.proto_id == IPPROTO_TCP) {
stream_tcp_create_rst(mbufs[i], &l4_meta, &pt);
out[i] = 0;
continue;
}
else {
out[i] = OUT_DISCARD;
continue;
}
}
conn = task->bundle_ctx_pool.hash_entries[ret];
ret = bundle_proc_data(conn, mbufs[i], &l4_meta, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: OUT_HANDLED;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
/* If there is at least one callback to handle, handle at most MAX_PKT_BURST */
if (heap_top_is_lower(task->heap, rte_rdtsc())) {
if (0 != refill_mbufs(&task->n_new_mbufs, task->mempool, task->new_mbufs))
return 0;
uint16_t n_called_back = 0;
while (heap_top_is_lower(task->heap, rte_rdtsc()) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
plogx_dbg("During callback, will send %d packets\n", n_called_back);
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
uint32_t n_new = task->bundle_ctx_pool.n_free_bundles;
n_new = n_new > MAX_PKT_BURST? MAX_PKT_BURST : n_new;
uint64_t diff = (rte_rdtsc() - task->new_conn_last_tsc)/task->new_conn_cost;
task->new_conn_last_tsc += diff * task->new_conn_cost;
task->new_conn_tokens += diff;
if (task->new_conn_tokens > 16)
task->new_conn_tokens = 16;
if (n_new > task->new_conn_tokens)
n_new = task->new_conn_tokens;
task->new_conn_tokens -= n_new;
if (n_new == 0)
return 0;
if (0 != refill_mbufs(&task->n_new_mbufs, task->mempool, task->new_mbufs))
return 0;
for (uint32_t i = 0; i < n_new; ++i) {
struct bundle_ctx *bundle_ctx = bundle_ctx_pool_get_w_cfg(&task->bundle_ctx_pool);
PROX_ASSERT(bundle_ctx);
struct pkt_tuple *pt = &bundle_ctx->tuple;
int n_retries = 0;
do {
/* Note that the actual packet sent will
contain swapped addresses and ports
(i.e. pkt.src <=> tuple.dst). The incoming
packet will match this struct. */
bundle_init(bundle_ctx, task->heap, PEER_CLIENT, &task->seed);
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret >= 0) {
if (n_retries++ == 1000) {
plogx_err("Already tried 1K times\n");
}
}
} while (ret >= 0);
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret < 0) {
plogx_err("Failed to add key ret = %d, n_free = %d\n", ret, task->bundle_ctx_pool.n_free_bundles);
bundle_ctx_pool_put(&task->bundle_ctx_pool, bundle_ctx);
pkt_tuple_debug2(pt);
out[i] = OUT_DISCARD;
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = bundle_ctx;
if (bundle_ctx->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
task->l4_stats.bundles_created++;
ret = bundle_proc_data(bundle_ctx, task->new_mbufs[i], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: OUT_HANDLED;
}
int ret2 = task->base.tx_pkt(&task->base, task->new_mbufs, n_new, out);
task->n_new_mbufs -= n_new;
return ret2;
}
static void handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_server *task = (struct task_gen_server *)tbase;
struct pkt_tuple pkt_tuple[MAX_PKT_BURST];
uint8_t out[MAX_PKT_BURST];
struct l4_meta l4_meta[MAX_PKT_BURST];
struct bundle_ctx *conn;
int ret;
for (uint16_t j = 0; j < n_pkts; ++j) {
if (parse_pkt(mbufs[j], &pkt_tuple[j], &l4_meta[j]))
plogdx_err(mbufs[j], "Unknown packet, parsing failed\n");
}
/* Main proc loop */
for (uint16_t j = 0; j < n_pkts; ++j) {
conn = NULL;
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple[j]);
if (ret >= 0)
conn = task->bundle_ctx_pool.hash_entries[ret];
/* If not part of existing connection, try to create a connection */
if (NULL == conn) {
struct new_tuple nt;
nt.dst_addr = pkt_tuple[j].dst_addr;
nt.proto_id = pkt_tuple[j].proto_id;
nt.dst_port = pkt_tuple[j].dst_port;
rte_memcpy(nt.l2_types, pkt_tuple[j].l2_types, sizeof(nt.l2_types));
const struct bundle_cfg *n;
if (NULL != (n = server_accept(task, &nt))) {
conn = bundle_ctx_pool_get(&task->bundle_ctx_pool);
if (!conn) {
out[j] = NO_PORT_AVAIL;
plogx_err("No more free bundles to accept new connection\n");
continue;
}
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)&pkt_tuple[j]);
if (ret < 0) {
out[j] = NO_PORT_AVAIL;
bundle_ctx_pool_put(&task->bundle_ctx_pool, conn);
plog_err("Adding key failed while trying to accept connection\n");
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = conn;
bundle_init(conn, n, task->heap, PEER_SERVER, &task->seed);
conn->tuple = pkt_tuple[j];
if (conn->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
}
}
/* bundle contains either an active connection or a
newly created connection. If it is NULL, then not
listening. */
if (NULL != conn) {
int ret = bundle_proc_data(conn, mbufs[j], &l4_meta[j], &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[j] = ret == 0? 0: NO_PORT_AVAIL;
}
else {
plog_err("Packet received for service that does not exist\n");
pkt_tuple_debug(&pkt_tuple[j]);
plogd_dbg(mbufs[j], NULL);
out[j] = NO_PORT_AVAIL;
}
}
conn = NULL;
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
if (!(task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)))
return ;
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
if (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)) {
uint16_t n_called_back = 0;
while (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
}
static void handle_gen_bulk_client(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
struct task_gen_client *task = (struct task_gen_client *)tbase;
uint8_t out[MAX_PKT_BURST] = {0};
struct bundle_ctx *conn;
int ret;
if (n_pkts) {
for (int i = 0; i < n_pkts; ++i) {
struct pkt_tuple pt;
struct l4_meta l4_meta;
if (parse_pkt(mbufs[i], &pt, &l4_meta)) {
plogdx_err(mbufs[i], "Parsing failed\n");
out[i] = NO_PORT_AVAIL;
continue;
}
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)&pt);
if (ret < 0) {
plogx_dbg("Client: packet RX that does not belong to connection:"
"Client = "IPv4_BYTES_FMT":%d, Server = "IPv4_BYTES_FMT":%d\n", IPv4_BYTES(((uint8_t*)&pt.dst_addr)), rte_bswap16(pt.dst_port), IPv4_BYTES(((uint8_t*)&pt.src_addr)), rte_bswap16(pt.src_port));
plogdx_dbg(mbufs[i], NULL);
// if tcp, send RST
/* pkt_tuple_debug2(&pt); */
out[i] = NO_PORT_AVAIL;
continue;
}
conn = task->bundle_ctx_pool.hash_entries[ret];
ret = bundle_proc_data(conn, mbufs[i], &l4_meta, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: NO_PORT_AVAIL;
}
task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
}
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
plogx_err("4Mempool alloc failed %d\n", MAX_PKT_BURST);
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
/* If there is at least one callback to handle, handle at most MAX_PKT_BURST */
if (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap)) {
uint16_t n_called_back = 0;
while (task->heap->n_elems && rte_rdtsc() > heap_peek_prio(task->heap) && n_called_back < MAX_PKT_BURST) {
conn = BUNDLE_CTX_UPCAST(heap_pop(task->heap));
/* handle packet TX (retransmit or delayed transmit) */
ret = bundle_proc_data(conn, task->new_mbufs[n_called_back], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
if (ret == 0) {
out[n_called_back] = 0;
n_called_back++;
}
}
plogx_dbg("During callback, will send %d packets\n", n_called_back);
task->base.tx_pkt(&task->base, task->new_mbufs, n_called_back, out);
task->n_new_mbufs -= n_called_back;
}
int n_new = task->bundle_ctx_pool.n_free_bundles;
n_new = n_new > MAX_PKT_BURST? MAX_PKT_BURST : n_new;
if (n_new == 0)
return ;
if (task->n_new_mbufs < MAX_PKT_BURST) {
if (rte_mempool_get_bulk(task->mempool, (void **)task->new_mbufs, MAX_PKT_BURST - task->n_new_mbufs) < 0) {
plogx_err("4Mempool alloc failed %d\n", MAX_PKT_BURST);
return ;
}
for (uint32_t i = 0; i < MAX_PKT_BURST - task->n_new_mbufs; ++i) {
init_mbuf_seg(task->new_mbufs[i]);
}
task->n_new_mbufs = MAX_PKT_BURST;
}
for (int i = 0; i < n_new; ++i) {
int32_t ret = cdf_sample(task->cdf, &task->seed);
/* Select a new bundle_cfg according to imix */
struct bundle_cfg *bundle_cfg = &task->bundle_cfgs[ret];
struct bundle_ctx *bundle_ctx;
bundle_ctx = bundle_ctx_pool_get(&task->bundle_ctx_pool);
/* Should be an assert: */
if (!bundle_ctx) {
plogx_err("No more available bundles\n");
exit(-1);
}
struct pkt_tuple *pt = &bundle_ctx->tuple;
int n_retries = 0;
do {
/* Note that the actual packet sent will
contain swapped addresses and ports
(i.e. pkt.src <=> tuple.dst). The incoming
packet will match this struct. */
bundle_init(bundle_ctx, bundle_cfg, task->heap, PEER_CLIENT, &task->seed);
ret = rte_hash_lookup(task->bundle_ctx_pool.hash, (const void *)pt);
if (n_retries == 1000) {
plogx_err("Already tried 1K times\n");
}
if (ret >= 0) {
n_retries++;
}
} while (ret >= 0);
ret = rte_hash_add_key(task->bundle_ctx_pool.hash, (const void *)pt);
if (ret < 0) {
plogx_err("Failed to add key ret = %d, n_free = %d\n", ret, task->bundle_ctx_pool.n_free_bundles);
bundle_ctx_pool_put(&task->bundle_ctx_pool, bundle_ctx);
pkt_tuple_debug2(pt);
out[i] = NO_PORT_AVAIL;
continue;
}
task->bundle_ctx_pool.hash_entries[ret] = bundle_ctx;
if (bundle_ctx->ctx.stream_cfg->proto == IPPROTO_TCP)
task->l4_stats.tcp_created++;
else
task->l4_stats.udp_created++;
ret = bundle_proc_data(bundle_ctx, task->new_mbufs[i], NULL, &task->bundle_ctx_pool, &task->seed, &task->l4_stats);
out[i] = ret == 0? 0: NO_PORT_AVAIL;
}
task->base.tx_pkt(&task->base, task->new_mbufs, n_new, out);
task->n_new_mbufs -= n_new;
}
