session_table_t *
session_table_get (u32 table_index)
{
if (pool_is_free_index (lookup_tables, table_index))
return 0;
return pool_elt_at_index (lookup_tables, table_index);
}
clib_error_t *replication_init (vlib_main_t *vm)
{
replication_main_t * rm = &replication_main;
vlib_buffer_main_t * bm = vm->buffer_main;
vlib_buffer_free_list_t * fl;
__attribute__((unused)) replication_context_t * ctx;
vlib_thread_main_t * tm = vlib_get_thread_main();
rm->vlib_main = vm;
rm->vnet_main = vnet_get_main();
rm->recycle_list_index =
vlib_buffer_create_free_list (vm, 1024 /* fictional */,
"replication-recycle");
fl = pool_elt_at_index (bm->buffer_free_list_pool,
rm->recycle_list_index);
fl->buffers_added_to_freelist_function = replication_recycle_callback;
// Verify the replication context is the expected size
ASSERT(sizeof(replication_context_t) == 128); // 2 cache lines
vec_validate (rm->contexts, tm->n_vlib_mains - 1);
return 0;
}
static ct_connection_t *
ct_connection_get (u32 ct_index)
{
if (pool_is_free_index (connections, ct_index))
return 0;
return pool_elt_at_index (connections, ct_index);
}
static fib_node_t *
lisp_gpe_adjacency_get_fib_node (fib_node_index_t index)
{
lisp_gpe_adjacency_t *ladj;
ladj = pool_elt_at_index (lisp_adj_pool, index);
return (&ladj->fib_node);
}
always_inline void *
parse_last_match_value (vlib_parse_main_t * pm)
{
vlib_parse_item_t * i;
i = pool_elt_at_index (pm->parse_items,
vec_elt (pm->match_items, vec_len (pm->match_items) - 1));
return i->value.as_pointer;
}
replication_context_t *
replication_recycle (vlib_main_t * vm,
vlib_buffer_t * b0,
u32 is_last)
{
replication_main_t * rm = &replication_main;
replication_context_t * ctx;
uword cpu_number = vm->cpu_index;
ip4_header_t * ip;
// Get access to the replication context
ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->clone_count);
// Restore vnet buffer state
memcpy (vnet_buffer(b0), ctx->vnet_buffer, sizeof(vnet_buffer_opaque_t));
// Restore the packet start (current_data) and length
vlib_buffer_advance(b0, ctx->current_data - b0->current_data);
// Restore packet contents
ip = (ip4_header_t *)vlib_buffer_get_current (b0);
if (ctx->l2_packet) {
// Restore ethernet header
((u64 *)ip)[0] = ctx->l2_header[0];
((u64 *)ip)[1] = ctx->l2_header[1];
((u64 *)ip)[2] = ctx->l2_header[2];
// set ip to the true ip header
ip = (ip4_header_t *)(((u8 *)ip) + vnet_buffer(b0)->l2.l2_len);
}
// Restore L3 fields
*((u16 *)(ip)) = ctx->ip_tos;
ip->checksum = ctx->ip4_checksum;
if (is_last) {
// This is the last replication in the list.
// Restore original buffer free functionality.
b0->clone_count = ctx->saved_clone_count;
b0->free_list_index = ctx->saved_free_list_index;
// Free context back to its pool
pool_put (rm->contexts[cpu_number], ctx);
}
return ctx;
}
static clib_error_t *
virtio_pci_delete_command_fn (vlib_main_t * vm, unformat_input_t * input,
vlib_cli_command_t * cmd)
{
unformat_input_t _line_input, *line_input = &_line_input;
u32 sw_if_index = ~0;
vnet_hw_interface_t *hw;
virtio_main_t *vim = &virtio_main;
virtio_if_t *vif;
vnet_main_t *vnm = vnet_get_main ();
/* Get a line of input. */
if (!unformat_user (input, unformat_line_input, line_input))
return 0;
while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
{
if (unformat (line_input, "sw_if_index %d", &sw_if_index))
;
else if (unformat (line_input, "%U", unformat_vnet_sw_interface,
vnm, &sw_if_index))
;
else
return clib_error_return (0, "unknown input `%U'",
format_unformat_error, input);
}
unformat_free (line_input);
if (sw_if_index == ~0)
return clib_error_return (0,
"please specify interface name or sw_if_index");
hw = vnet_get_sup_hw_interface (vnm, sw_if_index);
if (hw == NULL || virtio_device_class.index != hw->dev_class_index)
return clib_error_return (0, "not a virtio interface");
vif = pool_elt_at_index (vim->interfaces, hw->dev_instance);
if (virtio_pci_delete_if (vm, vif) < 0)
return clib_error_return (0, "not a virtio pci interface");
return 0;
}
/**
* @brief The LISP-GPE interface registered function to update, i.e.
* provide an rewrite string for, an adjacency.
*/
void
lisp_gpe_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, adj_index_t ai)
{
const lisp_gpe_tunnel_t *lgt;
lisp_gpe_adjacency_t *ladj;
ip_adjacency_t *adj;
ip_address_t rloc;
vnet_link_t linkt;
adj_flags_t af;
index_t lai;
adj = adj_get (ai);
ip46_address_to_ip_address (&adj->sub_type.nbr.next_hop, &rloc);
/*
* find an existing or create a new adj
*/
lai = lisp_adj_find (&rloc, sw_if_index);
ASSERT (INDEX_INVALID != lai);
ladj = pool_elt_at_index (lisp_adj_pool, lai);
lgt = lisp_gpe_tunnel_get (ladj->tunnel_index);
linkt = adj_get_link_type (ai);
af = ADJ_FLAG_MIDCHAIN_IP_STACK;
if (VNET_LINK_ETHERNET == linkt)
af |= ADJ_FLAG_MIDCHAIN_NO_COUNT;
adj_nbr_midchain_update_rewrite
(ai, lisp_gpe_fixup, NULL, af,
lisp_gpe_tunnel_build_rewrite (lgt, ladj,
lisp_gpe_adj_proto_from_vnet_link_type
(linkt)));
lisp_gpe_adj_stack_one (ladj, ai);
}
abf_policy_t *
abf_policy_get (u32 index)
{
return (pool_elt_at_index (abf_policy_pool, index));
}
static lisp_gpe_adjacency_t *
lisp_gpe_adjacency_get_i (index_t lai)
{
return (pool_elt_at_index (lisp_adj_pool, lai));
}
/*
* fish pkts back from the recycle queue/freelist
* un-flatten the context chains
*/
static void replication_recycle_callback (vlib_main_t *vm,
vlib_buffer_free_list_t * fl)
{
vlib_frame_t * f = 0;
u32 n_left_from;
u32 n_left_to_next = 0;
u32 n_this_frame = 0;
u32 * from;
u32 * to_next = 0;
u32 bi0, pi0;
vlib_buffer_t *b0;
vlib_buffer_t *bnext0;
int i;
replication_main_t * rm = &replication_main;
replication_context_t * ctx;
u32 feature_node_index = 0;
uword cpu_number = vm->cpu_index;
// All buffers in the list are destined to the same recycle node.
// Pull the recycle node index from the first buffer.
// Note: this could be sped up if the node index were stuffed into
// the freelist itself.
if (vec_len (fl->aligned_buffers) > 0) {
bi0 = fl->aligned_buffers[0];
b0 = vlib_get_buffer (vm, bi0);
ctx = pool_elt_at_index (rm->contexts[cpu_number],
b0->clone_count);
feature_node_index = ctx->recycle_node_index;
} else if (vec_len (fl->unaligned_buffers) > 0) {
bi0 = fl->unaligned_buffers[0];
b0 = vlib_get_buffer (vm, bi0);
ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->clone_count);
feature_node_index = ctx->recycle_node_index;
}
/* aligned, unaligned buffers */
for (i = 0; i < 2; i++)
{
if (i == 0)
{
from = fl->aligned_buffers;
n_left_from = vec_len (from);
}
else
{
from = fl->unaligned_buffers;
n_left_from = vec_len (from);
}
while (n_left_from > 0)
{
if (PREDICT_FALSE(n_left_to_next == 0))
{
if (f)
{
f->n_vectors = n_this_frame;
vlib_put_frame_to_node (vm, feature_node_index, f);
}
f = vlib_get_frame_to_node (vm, feature_node_index);
to_next = vlib_frame_vector_args (f);
n_left_to_next = VLIB_FRAME_SIZE;
n_this_frame = 0;
}
bi0 = from[0];
if (PREDICT_TRUE(n_left_from > 1))
{
pi0 = from[1];
vlib_prefetch_buffer_with_index(vm,pi0,LOAD);
}
bnext0 = b0 = vlib_get_buffer (vm, bi0);
// Mark that this buffer was just recycled
b0->flags |= VLIB_BUFFER_IS_RECYCLED;
// If buffer is traced, mark frame as traced
if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
f->flags |= VLIB_FRAME_TRACE;
while (bnext0->flags & VLIB_BUFFER_NEXT_PRESENT)
{
from += 1;
n_left_from -= 1;
bnext0 = vlib_get_buffer (vm, bnext0->next_buffer);
}
to_next[0] = bi0;
from++;
to_next++;
n_this_frame++;
n_left_to_next--;
n_left_from--;
}
}
vec_reset_length (fl->aligned_buffers);
vec_reset_length (fl->unaligned_buffers);
if (f)
{
ASSERT(n_this_frame);
f->n_vectors = n_this_frame;
vlib_put_frame_to_node (vm, feature_node_index, f);
}
}
static void
lisp_gpe_increment_stats_counters (lisp_cp_main_t * lcm, ip_adjacency_t * adj,
vlib_buffer_t * b)
{
lisp_gpe_main_t *lgm = vnet_lisp_gpe_get_main ();
lisp_gpe_adjacency_t *ladj;
ip_address_t rloc;
index_t lai;
u32 si, di;
gid_address_t src, dst;
uword *feip;
ip46_address_to_ip_address (&adj->sub_type.nbr.next_hop, &rloc);
si = vnet_buffer (b)->sw_if_index[VLIB_TX];
lai = lisp_adj_find (&rloc, si);
ASSERT (INDEX_INVALID != lai);
ladj = pool_elt_at_index (lisp_adj_pool, lai);
u8 *lisp_data = (u8 *) vlib_buffer_get_current (b);
/* skip IP header */
if (is_v4_packet (lisp_data))
lisp_data += sizeof (ip4_header_t);
else
lisp_data += sizeof (ip6_header_t);
/* skip UDP header */
lisp_data += sizeof (udp_header_t);
// TODO: skip TCP?
/* skip LISP GPE header */
lisp_data += sizeof (lisp_gpe_header_t);
i16 saved_current_data = b->current_data;
b->current_data = lisp_data - b->data;
lisp_afi_e afi = lisp_afi_from_vnet_link_type (adj->ia_link);
get_src_and_dst_eids_from_buffer (lcm, b, &src, &dst, afi);
b->current_data = saved_current_data;
di = gid_dictionary_sd_lookup (&lcm->mapping_index_by_gid, &dst, &src);
if (PREDICT_FALSE (~0 == di))
{
clib_warning ("dst mapping not found (%U, %U)", format_gid_address,
&src, format_gid_address, &dst);
return;
}
feip = hash_get (lcm->fwd_entry_by_mapping_index, di);
if (PREDICT_FALSE (!feip))
return;
lisp_stats_key_t key;
clib_memset (&key, 0, sizeof (key));
key.fwd_entry_index = feip[0];
key.tunnel_index = ladj->tunnel_index;
uword *p = hash_get_mem (lgm->lisp_stats_index_by_key, &key);
ASSERT (p);
/* compute payload length starting after GPE */
u32 bytes = b->current_length - (lisp_data - b->data - b->current_data);
vlib_increment_combined_counter (&lgm->counters, vlib_get_thread_index (),
p[0], 1, bytes);
}
gbp_vxlan_tunnel_t *
gbp_vxlan_tunnel_get (index_t gti)
{
return (pool_elt_at_index (gbp_vxlan_tunnel_pool, gti));
}
static vxlan_tunnel_ref_t *
vxlan_tunnel_ref_get (index_t vxri)
{
return (pool_elt_at_index (vxlan_tunnel_ref_pool, vxri));
}
int
svmdb_local_add_del_notification (svmdb_client_t * client,
svmdb_notification_args_t * a)
{
uword *h;
void *oldheap;
hash_pair_t *hp;
svmdb_shm_hdr_t *shm;
u8 *dummy_value = 0;
svmdb_value_t *value;
svmdb_notify_t *np;
int i;
int rv = 0;
ASSERT (a->elsize);
region_lock (client->db_rp, 18);
shm = client->shm;
oldheap = svm_push_data_heap (client->db_rp);
h = shm->namespaces[a->nspace];
hp = hash_get_pair_mem (h, a->var);
if (hp == 0)
{
local_set_variable_nolock (client, a->nspace, (u8 *) a->var,
dummy_value, a->elsize);
/* might have moved */
h = shm->namespaces[a->nspace];
hp = hash_get_pair_mem (h, a->var);
ASSERT (hp);
}
value = pool_elt_at_index (shm->values, hp->value[0]);
for (i = 0; i < vec_len (value->notifications); i++)
{
np = vec_elt_at_index (value->notifications, i);
if ((np->pid == client->pid)
&& (np->signum == a->signum)
&& (np->action == a->action) && (np->opaque == a->opaque))
{
if (a->add_del == 0 /* delete */ )
{
vec_delete (value->notifications, 1, i);
goto out;
}
else
{ /* add */
clib_warning
("%s: ignore dup reg pid %d signum %d action %d opaque %x",
a->var, client->pid, a->signum, a->action, a->opaque);
rv = -2;
goto out;
}
}
}
if (a->add_del == 0)
{
rv = -3;
goto out;
}
vec_add2 (value->notifications, np, 1);
np->pid = client->pid;
np->signum = a->signum;
np->action = a->action;
np->opaque = a->opaque;
out:
svm_pop_heap (oldheap);
region_unlock (client->db_rp);
return rv;
}
static mc_socket_catchup_t *
find_catchup_from_file_descriptor (mc_socket_main_t * msm, int file_descriptor)
{
uword * p = hash_get (msm->catchup_index_by_file_descriptor, file_descriptor);
return p ? pool_elt_at_index (msm->catchups, p[0]) : 0;
}
void
scrape_and_clear_counters (perfmon_main_t * pm)
{
int i, j, k;
vlib_main_t *vm = pm->vlib_main;
vlib_main_t *stat_vm;
vlib_node_main_t *nm;
vlib_node_t ***node_dups = 0;
vlib_node_t **nodes;
vlib_node_t *n;
perfmon_capture_t *c;
perfmon_event_config_t *current_event;
uword *p;
u8 *counter_name;
u64 vectors_this_counter;
/* snapshoot the nodes, including pm counters */
vlib_worker_thread_barrier_sync (vm);
for (j = 0; j < vec_len (vlib_mains); j++)
{
stat_vm = vlib_mains[j];
if (stat_vm == 0)
continue;
nm = &stat_vm->node_main;
for (i = 0; i < vec_len (nm->nodes); i++)
{
n = nm->nodes[i];
vlib_node_sync_stats (stat_vm, n);
}
nodes = 0;
vec_validate (nodes, vec_len (nm->nodes) - 1);
vec_add1 (node_dups, nodes);
/* Snapshoot and clear the per-node perfmon counters */
for (i = 0; i < vec_len (nm->nodes); i++)
{
n = nm->nodes[i];
nodes[i] = clib_mem_alloc (sizeof (*n));
clib_memcpy_fast (nodes[i], n, sizeof (*n));
n->stats_total.perf_counter0_ticks = 0;
n->stats_total.perf_counter1_ticks = 0;
n->stats_total.perf_counter_vectors = 0;
n->stats_last_clear.perf_counter0_ticks = 0;
n->stats_last_clear.perf_counter1_ticks = 0;
n->stats_last_clear.perf_counter_vectors = 0;
}
}
vlib_worker_thread_barrier_release (vm);
for (j = 0; j < vec_len (vlib_mains); j++)
{
stat_vm = vlib_mains[j];
if (stat_vm == 0)
continue;
nodes = node_dups[j];
for (i = 0; i < vec_len (nodes); i++)
{
u8 *capture_name;
n = nodes[i];
if (n->stats_total.perf_counter0_ticks == 0 &&
n->stats_total.perf_counter1_ticks == 0)
goto skip_this_node;
for (k = 0; k < 2; k++)
{
u64 counter_value, counter_last_clear;
/*
* We collect 2 counters at once, except for the
* last counter when the user asks for an odd number of
* counters
*/
if ((pm->current_event + k)
>= vec_len (pm->single_events_to_collect))
break;
if (k == 0)
{
counter_value = n->stats_total.perf_counter0_ticks;
counter_last_clear =
n->stats_last_clear.perf_counter0_ticks;
}
else
{
counter_value = n->stats_total.perf_counter1_ticks;
counter_last_clear =
n->stats_last_clear.perf_counter1_ticks;
}
capture_name = format (0, "t%d-%v%c", j, n->name, 0);
p = hash_get_mem (pm->capture_by_thread_and_node_name,
capture_name);
if (p == 0)
{
pool_get (pm->capture_pool, c);
memset (c, 0, sizeof (*c));
c->thread_and_node_name = capture_name;
hash_set_mem (pm->capture_by_thread_and_node_name,
capture_name, c - pm->capture_pool);
}
else
{
c = pool_elt_at_index (pm->capture_pool, p[0]);
vec_free (capture_name);
}
/* Snapshoot counters, etc. into the capture */
current_event = pm->single_events_to_collect
+ pm->current_event + k;
counter_name = (u8 *) current_event->name;
vectors_this_counter = n->stats_total.perf_counter_vectors -
n->stats_last_clear.perf_counter_vectors;
vec_add1 (c->counter_names, counter_name);
vec_add1 (c->counter_values,
counter_value - counter_last_clear);
vec_add1 (c->vectors_this_counter, vectors_this_counter);
}
skip_this_node:
clib_mem_free (n);
}
vec_free (nodes);
}
vec_free (node_dups);
}
static u32
add_del_ip_tunnel (vnet_lisp_gpe_add_del_fwd_entry_args_t *a,
u32 * tun_index_res)
{
lisp_gpe_main_t * lgm = &lisp_gpe_main;
lisp_gpe_tunnel_t *t = 0;
uword * p;
int rv;
lisp_gpe_tunnel_key_t key;
/* prepare tunnel key */
memset(&key, 0, sizeof(key));
ip_prefix_copy(&key.eid, &gid_address_ippref(&a->deid));
ip_address_copy(&key.dst_loc, &a->dlocator);
key.iid = clib_host_to_net_u32 (a->vni);
p = mhash_get (&lgm->lisp_gpe_tunnel_by_key, &key);
if (a->is_add)
{
/* adding a tunnel: tunnel must not already exist */
if (p)
return VNET_API_ERROR_INVALID_VALUE;
if (a->decap_next_index >= LISP_GPE_INPUT_N_NEXT)
return VNET_API_ERROR_INVALID_DECAP_NEXT;
pool_get_aligned (lgm->tunnels, t, CLIB_CACHE_LINE_BYTES);
memset (t, 0, sizeof (*t));
/* copy from arg structure */
#define _(x) t->x = a->x;
foreach_copy_field;
#undef _
ip_address_copy(&t->src, &a->slocator);
ip_address_copy(&t->dst, &a->dlocator);
/* if vni is non-default */
if (a->vni)
{
t->flags = LISP_GPE_FLAGS_I;
t->vni = a->vni;
}
t->flags |= LISP_GPE_FLAGS_P;
t->next_protocol = ip_prefix_version(&key.eid) == IP4 ?
LISP_GPE_NEXT_PROTO_IP4 : LISP_GPE_NEXT_PROTO_IP6;
rv = lisp_gpe_rewrite (t);
if (rv)
{
pool_put(lgm->tunnels, t);
return rv;
}
mhash_set(&lgm->lisp_gpe_tunnel_by_key, &key, t - lgm->tunnels, 0);
/* return tunnel index */
if (tun_index_res)
tun_index_res[0] = t - lgm->tunnels;
}
else
{
/* deleting a tunnel: tunnel must exist */
if (!p)
{
clib_warning("Tunnel for eid %U doesn't exist!", format_gid_address,
&a->deid);
return VNET_API_ERROR_NO_SUCH_ENTRY;
}
t = pool_elt_at_index(lgm->tunnels, p[0]);
mhash_unset(&lgm->lisp_gpe_tunnel_by_key, &key, 0);
vec_free(t->rewrite);
pool_put(lgm->tunnels, t);
}
return 0;
}
static int
dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe)
{
struct rte_flow_item_ipv4 ip4[2] = { };
struct rte_flow_item_ipv6 ip6[2] = { };
struct rte_flow_item_udp udp[2] = { };
struct rte_flow_item_tcp tcp[2] = { };
struct rte_flow_action_mark mark = { 0 };
struct rte_flow_item *item, *items = 0;
struct rte_flow_action *action, *actions = 0;
enum
{
vxlan_hdr_sz = sizeof (vxlan_header_t),
raw_sz = sizeof (struct rte_flow_item_raw)
};
union
{
struct rte_flow_item_raw item;
u8 val[raw_sz + vxlan_hdr_sz];
} raw[2];
u16 src_port, dst_port, src_port_mask, dst_port_mask;
u8 protocol;
int rv = 0;
if (f->actions & (~xd->supported_flow_actions))
return VNET_FLOW_ERROR_NOT_SUPPORTED;
/* Match items */
/* Ethernet */
vec_add2 (items, item, 1);
item->type = RTE_FLOW_ITEM_TYPE_ETH;
item->spec = any_eth;
item->mask = any_eth + 1;
/* VLAN */
if (f->type != VNET_FLOW_TYPE_IP4_VXLAN)
{
vec_add2 (items, item, 1);
item->type = RTE_FLOW_ITEM_TYPE_VLAN;
item->spec = any_vlan;
item->mask = any_vlan + 1;
}
/* IP */
vec_add2 (items, item, 1);
if (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE)
{
vnet_flow_ip6_n_tuple_t *t6 = &f->ip6_n_tuple;
clib_memcpy_fast (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16);
clib_memcpy_fast (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16);
clib_memcpy_fast (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16);
clib_memcpy_fast (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16);
item->type = RTE_FLOW_ITEM_TYPE_IPV6;
item->spec = ip6;
item->mask = ip6 + 1;
src_port = t6->src_port.port;
dst_port = t6->dst_port.port;
src_port_mask = t6->src_port.mask;
dst_port_mask = t6->dst_port.mask;
protocol = t6->protocol;
}
else if (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE)
{
vnet_flow_ip4_n_tuple_t *t4 = &f->ip4_n_tuple;
ip4[0].hdr.src_addr = t4->src_addr.addr.as_u32;
ip4[1].hdr.src_addr = t4->src_addr.mask.as_u32;
ip4[0].hdr.dst_addr = t4->dst_addr.addr.as_u32;
ip4[1].hdr.dst_addr = t4->dst_addr.mask.as_u32;
item->type = RTE_FLOW_ITEM_TYPE_IPV4;
item->spec = ip4;
item->mask = ip4 + 1;
src_port = t4->src_port.port;
dst_port = t4->dst_port.port;
src_port_mask = t4->src_port.mask;
dst_port_mask = t4->dst_port.mask;
protocol = t4->protocol;
}
else if (f->type == VNET_FLOW_TYPE_IP4_VXLAN)
{
vnet_flow_ip4_vxlan_t *v4 = &f->ip4_vxlan;
ip4[0].hdr.src_addr = v4->src_addr.as_u32;
ip4[1].hdr.src_addr = -1;
ip4[0].hdr.dst_addr = v4->dst_addr.as_u32;
ip4[1].hdr.dst_addr = -1;
item->type = RTE_FLOW_ITEM_TYPE_IPV4;
item->spec = ip4;
item->mask = ip4 + 1;
dst_port = v4->dst_port;
dst_port_mask = -1;
src_port = 0;
src_port_mask = 0;
protocol = IP_PROTOCOL_UDP;
}
else
{
rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
goto done;
}
/* Layer 4 */
vec_add2 (items, item, 1);
if (protocol == IP_PROTOCOL_UDP)
{
udp[0].hdr.src_port = clib_host_to_net_u16 (src_port);
udp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask);
udp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port);
udp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask);
item->type = RTE_FLOW_ITEM_TYPE_UDP;
item->spec = udp;
item->mask = udp + 1;
}
else if (protocol == IP_PROTOCOL_TCP)
{
tcp[0].hdr.src_port = clib_host_to_net_u16 (src_port);
tcp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask);
tcp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port);
tcp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask);
item->type = RTE_FLOW_ITEM_TYPE_TCP;
item->spec = tcp;
item->mask = tcp + 1;
}
else
{
rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
goto done;
}
/* Tunnel header match */
if (f->type == VNET_FLOW_TYPE_IP4_VXLAN)
{
u32 vni = f->ip4_vxlan.vni;
vxlan_header_t spec_hdr = {
.flags = VXLAN_FLAGS_I,
.vni_reserved = clib_host_to_net_u32 (vni << 8)
};
vxlan_header_t mask_hdr = {
.flags = 0xff,
.vni_reserved = clib_host_to_net_u32 (((u32) - 1) << 8)
};
clib_memset (raw, 0, sizeof raw);
raw[0].item.relative = 1;
raw[0].item.length = vxlan_hdr_sz;
clib_memcpy_fast (raw[0].val + raw_sz, &spec_hdr, vxlan_hdr_sz);
raw[0].item.pattern = raw[0].val + raw_sz;
clib_memcpy_fast (raw[1].val + raw_sz, &mask_hdr, vxlan_hdr_sz);
raw[1].item.pattern = raw[1].val + raw_sz;
vec_add2 (items, item, 1);
item->type = RTE_FLOW_ITEM_TYPE_RAW;
item->spec = raw;
item->mask = raw + 1;
}
vec_add2 (items, item, 1);
item->type = RTE_FLOW_ITEM_TYPE_END;
/* Actions */
vec_add2 (actions, action, 1);
action->type = RTE_FLOW_ACTION_TYPE_PASSTHRU;
vec_add2 (actions, action, 1);
mark.id = fe->mark;
action->type = RTE_FLOW_ACTION_TYPE_MARK;
action->conf = &mark;
vec_add2 (actions, action, 1);
action->type = RTE_FLOW_ACTION_TYPE_END;
fe->handle = rte_flow_create (xd->device_index, &ingress, items, actions,
&xd->last_flow_error);
if (!fe->handle)
rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
done:
vec_free (items);
vec_free (actions);
return rv;
}
int
dpdk_flow_ops_fn (vnet_main_t * vnm, vnet_flow_dev_op_t op, u32 dev_instance,
u32 flow_index, uword * private_data)
{
dpdk_main_t *dm = &dpdk_main;
vnet_flow_t *flow = vnet_get_flow (flow_index);
dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance);
dpdk_flow_entry_t *fe;
dpdk_flow_lookup_entry_t *fle = 0;
int rv;
/* recycle old flow lookup entries only after the main loop counter
increases - i.e. previously DMA'ed packets were handled */
if (vec_len (xd->parked_lookup_indexes) > 0 &&
xd->parked_loop_count != dm->vlib_main->main_loop_count)
{
u32 *fl_index;
vec_foreach (fl_index, xd->parked_lookup_indexes)
pool_put_index (xd->flow_lookup_entries, *fl_index);
vec_reset_length (xd->flow_lookup_entries);
}
if (op == VNET_FLOW_DEV_OP_DEL_FLOW)
{
ASSERT (*private_data >= vec_len (xd->flow_entries));
fe = vec_elt_at_index (xd->flow_entries, *private_data);
if ((rv = rte_flow_destroy (xd->device_index, fe->handle,
&xd->last_flow_error)))
return VNET_FLOW_ERROR_INTERNAL;
if (fe->mark)
{
/* make sure no action is taken for in-flight (marked) packets */
fle = pool_elt_at_index (xd->flow_lookup_entries, fe->mark);
clib_memset (fle, -1, sizeof (*fle));
vec_add1 (xd->parked_lookup_indexes, fe->mark);
xd->parked_loop_count = dm->vlib_main->main_loop_count;
}
clib_memset (fe, 0, sizeof (*fe));
pool_put (xd->flow_entries, fe);
goto disable_rx_offload;
}
if (op != VNET_FLOW_DEV_OP_ADD_FLOW)
return VNET_FLOW_ERROR_NOT_SUPPORTED;
pool_get (xd->flow_entries, fe);
fe->flow_index = flow->index;
if (flow->actions == 0)
{
rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
goto done;
}
/* if we need to mark packets, assign one mark */
if (flow->actions & (VNET_FLOW_ACTION_MARK |
VNET_FLOW_ACTION_REDIRECT_TO_NODE |
VNET_FLOW_ACTION_BUFFER_ADVANCE))
{
/* reserve slot 0 */
if (xd->flow_lookup_entries == 0)
pool_get_aligned (xd->flow_lookup_entries, fle,
CLIB_CACHE_LINE_BYTES);
pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES);
fe->mark = fle - xd->flow_lookup_entries;
/* install entry in the lookup table */
clib_memset (fle, -1, sizeof (*fle));
if (flow->actions & VNET_FLOW_ACTION_MARK)
fle->flow_id = flow->mark_flow_id;
if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_NODE)
fle->next_index = flow->redirect_device_input_next_index;
if (flow->actions & VNET_FLOW_ACTION_BUFFER_ADVANCE)
fle->buffer_advance = flow->buffer_advance;
}
else
fe->mark = 0;
if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) == 0)
{
xd->flags |= DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD;
dpdk_device_setup (xd);
}
switch (flow->type)
{
case VNET_FLOW_TYPE_IP4_N_TUPLE:
case VNET_FLOW_TYPE_IP6_N_TUPLE:
case VNET_FLOW_TYPE_IP4_VXLAN:
if ((rv = dpdk_flow_add (xd, flow, fe)))
goto done;
break;
default:
rv = VNET_FLOW_ERROR_NOT_SUPPORTED;
goto done;
}
*private_data = fe - xd->flow_entries;
done:
if (rv)
{
clib_memset (fe, 0, sizeof (*fe));
pool_put (xd->flow_entries, fe);
if (fle)
{
clib_memset (fle, -1, sizeof (*fle));
pool_put (xd->flow_lookup_entries, fle);
}
}
disable_rx_offload:
if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) != 0
&& pool_elts (xd->flow_entries) == 0)
{
xd->flags &= ~DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD;
dpdk_device_setup (xd);
}
return rv;
}
