session_table_t * session_table_get (u32 table_index) { if (pool_is_free_index (lookup_tables, table_index)) return 0; return pool_elt_at_index (lookup_tables, table_index); }
clib_error_t *replication_init (vlib_main_t *vm) { replication_main_t * rm = &replication_main; vlib_buffer_main_t * bm = vm->buffer_main; vlib_buffer_free_list_t * fl; __attribute__((unused)) replication_context_t * ctx; vlib_thread_main_t * tm = vlib_get_thread_main(); rm->vlib_main = vm; rm->vnet_main = vnet_get_main(); rm->recycle_list_index = vlib_buffer_create_free_list (vm, 1024 /* fictional */, "replication-recycle"); fl = pool_elt_at_index (bm->buffer_free_list_pool, rm->recycle_list_index); fl->buffers_added_to_freelist_function = replication_recycle_callback; // Verify the replication context is the expected size ASSERT(sizeof(replication_context_t) == 128); // 2 cache lines vec_validate (rm->contexts, tm->n_vlib_mains - 1); return 0; }
static ct_connection_t * ct_connection_get (u32 ct_index) { if (pool_is_free_index (connections, ct_index)) return 0; return pool_elt_at_index (connections, ct_index); }
static fib_node_t * lisp_gpe_adjacency_get_fib_node (fib_node_index_t index) { lisp_gpe_adjacency_t *ladj; ladj = pool_elt_at_index (lisp_adj_pool, index); return (&ladj->fib_node); }
always_inline void * parse_last_match_value (vlib_parse_main_t * pm) { vlib_parse_item_t * i; i = pool_elt_at_index (pm->parse_items, vec_elt (pm->match_items, vec_len (pm->match_items) - 1)); return i->value.as_pointer; }
replication_context_t * replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) { replication_main_t * rm = &replication_main; replication_context_t * ctx; uword cpu_number = vm->cpu_index; ip4_header_t * ip; // Get access to the replication context ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->clone_count); // Restore vnet buffer state memcpy (vnet_buffer(b0), ctx->vnet_buffer, sizeof(vnet_buffer_opaque_t)); // Restore the packet start (current_data) and length vlib_buffer_advance(b0, ctx->current_data - b0->current_data); // Restore packet contents ip = (ip4_header_t *)vlib_buffer_get_current (b0); if (ctx->l2_packet) { // Restore ethernet header ((u64 *)ip)[0] = ctx->l2_header[0]; ((u64 *)ip)[1] = ctx->l2_header[1]; ((u64 *)ip)[2] = ctx->l2_header[2]; // set ip to the true ip header ip = (ip4_header_t *)(((u8 *)ip) + vnet_buffer(b0)->l2.l2_len); } // Restore L3 fields *((u16 *)(ip)) = ctx->ip_tos; ip->checksum = ctx->ip4_checksum; if (is_last) { // This is the last replication in the list. // Restore original buffer free functionality. b0->clone_count = ctx->saved_clone_count; b0->free_list_index = ctx->saved_free_list_index; // Free context back to its pool pool_put (rm->contexts[cpu_number], ctx); } return ctx; }
static clib_error_t * virtio_pci_delete_command_fn (vlib_main_t * vm, unformat_input_t * input, vlib_cli_command_t * cmd) { unformat_input_t _line_input, *line_input = &_line_input; u32 sw_if_index = ~0; vnet_hw_interface_t *hw; virtio_main_t *vim = &virtio_main; virtio_if_t *vif; vnet_main_t *vnm = vnet_get_main (); /* Get a line of input. */ if (!unformat_user (input, unformat_line_input, line_input)) return 0; while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) { if (unformat (line_input, "sw_if_index %d", &sw_if_index)) ; else if (unformat (line_input, "%U", unformat_vnet_sw_interface, vnm, &sw_if_index)) ; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); } unformat_free (line_input); if (sw_if_index == ~0) return clib_error_return (0, "please specify interface name or sw_if_index"); hw = vnet_get_sup_hw_interface (vnm, sw_if_index); if (hw == NULL || virtio_device_class.index != hw->dev_class_index) return clib_error_return (0, "not a virtio interface"); vif = pool_elt_at_index (vim->interfaces, hw->dev_instance); if (virtio_pci_delete_if (vm, vif) < 0) return clib_error_return (0, "not a virtio pci interface"); return 0; }
/** * @brief The LISP-GPE interface registered function to update, i.e. * provide an rewrite string for, an adjacency. */ void lisp_gpe_update_adjacency (vnet_main_t * vnm, u32 sw_if_index, adj_index_t ai) { const lisp_gpe_tunnel_t *lgt; lisp_gpe_adjacency_t *ladj; ip_adjacency_t *adj; ip_address_t rloc; vnet_link_t linkt; adj_flags_t af; index_t lai; adj = adj_get (ai); ip46_address_to_ip_address (&adj->sub_type.nbr.next_hop, &rloc); /* * find an existing or create a new adj */ lai = lisp_adj_find (&rloc, sw_if_index); ASSERT (INDEX_INVALID != lai); ladj = pool_elt_at_index (lisp_adj_pool, lai); lgt = lisp_gpe_tunnel_get (ladj->tunnel_index); linkt = adj_get_link_type (ai); af = ADJ_FLAG_MIDCHAIN_IP_STACK; if (VNET_LINK_ETHERNET == linkt) af |= ADJ_FLAG_MIDCHAIN_NO_COUNT; adj_nbr_midchain_update_rewrite (ai, lisp_gpe_fixup, NULL, af, lisp_gpe_tunnel_build_rewrite (lgt, ladj, lisp_gpe_adj_proto_from_vnet_link_type (linkt))); lisp_gpe_adj_stack_one (ladj, ai); }
abf_policy_t * abf_policy_get (u32 index) { return (pool_elt_at_index (abf_policy_pool, index)); }
static lisp_gpe_adjacency_t * lisp_gpe_adjacency_get_i (index_t lai) { return (pool_elt_at_index (lisp_adj_pool, lai)); }
/* * fish pkts back from the recycle queue/freelist * un-flatten the context chains */ static void replication_recycle_callback (vlib_main_t *vm, vlib_buffer_free_list_t * fl) { vlib_frame_t * f = 0; u32 n_left_from; u32 n_left_to_next = 0; u32 n_this_frame = 0; u32 * from; u32 * to_next = 0; u32 bi0, pi0; vlib_buffer_t *b0; vlib_buffer_t *bnext0; int i; replication_main_t * rm = &replication_main; replication_context_t * ctx; u32 feature_node_index = 0; uword cpu_number = vm->cpu_index; // All buffers in the list are destined to the same recycle node. // Pull the recycle node index from the first buffer. // Note: this could be sped up if the node index were stuffed into // the freelist itself. if (vec_len (fl->aligned_buffers) > 0) { bi0 = fl->aligned_buffers[0]; b0 = vlib_get_buffer (vm, bi0); ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->clone_count); feature_node_index = ctx->recycle_node_index; } else if (vec_len (fl->unaligned_buffers) > 0) { bi0 = fl->unaligned_buffers[0]; b0 = vlib_get_buffer (vm, bi0); ctx = pool_elt_at_index (rm->contexts[cpu_number], b0->clone_count); feature_node_index = ctx->recycle_node_index; } /* aligned, unaligned buffers */ for (i = 0; i < 2; i++) { if (i == 0) { from = fl->aligned_buffers; n_left_from = vec_len (from); } else { from = fl->unaligned_buffers; n_left_from = vec_len (from); } while (n_left_from > 0) { if (PREDICT_FALSE(n_left_to_next == 0)) { if (f) { f->n_vectors = n_this_frame; vlib_put_frame_to_node (vm, feature_node_index, f); } f = vlib_get_frame_to_node (vm, feature_node_index); to_next = vlib_frame_vector_args (f); n_left_to_next = VLIB_FRAME_SIZE; n_this_frame = 0; } bi0 = from[0]; if (PREDICT_TRUE(n_left_from > 1)) { pi0 = from[1]; vlib_prefetch_buffer_with_index(vm,pi0,LOAD); } bnext0 = b0 = vlib_get_buffer (vm, bi0); // Mark that this buffer was just recycled b0->flags |= VLIB_BUFFER_IS_RECYCLED; // If buffer is traced, mark frame as traced if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) f->flags |= VLIB_FRAME_TRACE; while (bnext0->flags & VLIB_BUFFER_NEXT_PRESENT) { from += 1; n_left_from -= 1; bnext0 = vlib_get_buffer (vm, bnext0->next_buffer); } to_next[0] = bi0; from++; to_next++; n_this_frame++; n_left_to_next--; n_left_from--; } } vec_reset_length (fl->aligned_buffers); vec_reset_length (fl->unaligned_buffers); if (f) { ASSERT(n_this_frame); f->n_vectors = n_this_frame; vlib_put_frame_to_node (vm, feature_node_index, f); } }
static void lisp_gpe_increment_stats_counters (lisp_cp_main_t * lcm, ip_adjacency_t * adj, vlib_buffer_t * b) { lisp_gpe_main_t *lgm = vnet_lisp_gpe_get_main (); lisp_gpe_adjacency_t *ladj; ip_address_t rloc; index_t lai; u32 si, di; gid_address_t src, dst; uword *feip; ip46_address_to_ip_address (&adj->sub_type.nbr.next_hop, &rloc); si = vnet_buffer (b)->sw_if_index[VLIB_TX]; lai = lisp_adj_find (&rloc, si); ASSERT (INDEX_INVALID != lai); ladj = pool_elt_at_index (lisp_adj_pool, lai); u8 *lisp_data = (u8 *) vlib_buffer_get_current (b); /* skip IP header */ if (is_v4_packet (lisp_data)) lisp_data += sizeof (ip4_header_t); else lisp_data += sizeof (ip6_header_t); /* skip UDP header */ lisp_data += sizeof (udp_header_t); // TODO: skip TCP? /* skip LISP GPE header */ lisp_data += sizeof (lisp_gpe_header_t); i16 saved_current_data = b->current_data; b->current_data = lisp_data - b->data; lisp_afi_e afi = lisp_afi_from_vnet_link_type (adj->ia_link); get_src_and_dst_eids_from_buffer (lcm, b, &src, &dst, afi); b->current_data = saved_current_data; di = gid_dictionary_sd_lookup (&lcm->mapping_index_by_gid, &dst, &src); if (PREDICT_FALSE (~0 == di)) { clib_warning ("dst mapping not found (%U, %U)", format_gid_address, &src, format_gid_address, &dst); return; } feip = hash_get (lcm->fwd_entry_by_mapping_index, di); if (PREDICT_FALSE (!feip)) return; lisp_stats_key_t key; clib_memset (&key, 0, sizeof (key)); key.fwd_entry_index = feip[0]; key.tunnel_index = ladj->tunnel_index; uword *p = hash_get_mem (lgm->lisp_stats_index_by_key, &key); ASSERT (p); /* compute payload length starting after GPE */ u32 bytes = b->current_length - (lisp_data - b->data - b->current_data); vlib_increment_combined_counter (&lgm->counters, vlib_get_thread_index (), p[0], 1, bytes); }
gbp_vxlan_tunnel_t * gbp_vxlan_tunnel_get (index_t gti) { return (pool_elt_at_index (gbp_vxlan_tunnel_pool, gti)); }
static vxlan_tunnel_ref_t * vxlan_tunnel_ref_get (index_t vxri) { return (pool_elt_at_index (vxlan_tunnel_ref_pool, vxri)); }
int svmdb_local_add_del_notification (svmdb_client_t * client, svmdb_notification_args_t * a) { uword *h; void *oldheap; hash_pair_t *hp; svmdb_shm_hdr_t *shm; u8 *dummy_value = 0; svmdb_value_t *value; svmdb_notify_t *np; int i; int rv = 0; ASSERT (a->elsize); region_lock (client->db_rp, 18); shm = client->shm; oldheap = svm_push_data_heap (client->db_rp); h = shm->namespaces[a->nspace]; hp = hash_get_pair_mem (h, a->var); if (hp == 0) { local_set_variable_nolock (client, a->nspace, (u8 *) a->var, dummy_value, a->elsize); /* might have moved */ h = shm->namespaces[a->nspace]; hp = hash_get_pair_mem (h, a->var); ASSERT (hp); } value = pool_elt_at_index (shm->values, hp->value[0]); for (i = 0; i < vec_len (value->notifications); i++) { np = vec_elt_at_index (value->notifications, i); if ((np->pid == client->pid) && (np->signum == a->signum) && (np->action == a->action) && (np->opaque == a->opaque)) { if (a->add_del == 0 /* delete */ ) { vec_delete (value->notifications, 1, i); goto out; } else { /* add */ clib_warning ("%s: ignore dup reg pid %d signum %d action %d opaque %x", a->var, client->pid, a->signum, a->action, a->opaque); rv = -2; goto out; } } } if (a->add_del == 0) { rv = -3; goto out; } vec_add2 (value->notifications, np, 1); np->pid = client->pid; np->signum = a->signum; np->action = a->action; np->opaque = a->opaque; out: svm_pop_heap (oldheap); region_unlock (client->db_rp); return rv; }
static mc_socket_catchup_t * find_catchup_from_file_descriptor (mc_socket_main_t * msm, int file_descriptor) { uword * p = hash_get (msm->catchup_index_by_file_descriptor, file_descriptor); return p ? pool_elt_at_index (msm->catchups, p[0]) : 0; }
void scrape_and_clear_counters (perfmon_main_t * pm) { int i, j, k; vlib_main_t *vm = pm->vlib_main; vlib_main_t *stat_vm; vlib_node_main_t *nm; vlib_node_t ***node_dups = 0; vlib_node_t **nodes; vlib_node_t *n; perfmon_capture_t *c; perfmon_event_config_t *current_event; uword *p; u8 *counter_name; u64 vectors_this_counter; /* snapshoot the nodes, including pm counters */ vlib_worker_thread_barrier_sync (vm); for (j = 0; j < vec_len (vlib_mains); j++) { stat_vm = vlib_mains[j]; if (stat_vm == 0) continue; nm = &stat_vm->node_main; for (i = 0; i < vec_len (nm->nodes); i++) { n = nm->nodes[i]; vlib_node_sync_stats (stat_vm, n); } nodes = 0; vec_validate (nodes, vec_len (nm->nodes) - 1); vec_add1 (node_dups, nodes); /* Snapshoot and clear the per-node perfmon counters */ for (i = 0; i < vec_len (nm->nodes); i++) { n = nm->nodes[i]; nodes[i] = clib_mem_alloc (sizeof (*n)); clib_memcpy_fast (nodes[i], n, sizeof (*n)); n->stats_total.perf_counter0_ticks = 0; n->stats_total.perf_counter1_ticks = 0; n->stats_total.perf_counter_vectors = 0; n->stats_last_clear.perf_counter0_ticks = 0; n->stats_last_clear.perf_counter1_ticks = 0; n->stats_last_clear.perf_counter_vectors = 0; } } vlib_worker_thread_barrier_release (vm); for (j = 0; j < vec_len (vlib_mains); j++) { stat_vm = vlib_mains[j]; if (stat_vm == 0) continue; nodes = node_dups[j]; for (i = 0; i < vec_len (nodes); i++) { u8 *capture_name; n = nodes[i]; if (n->stats_total.perf_counter0_ticks == 0 && n->stats_total.perf_counter1_ticks == 0) goto skip_this_node; for (k = 0; k < 2; k++) { u64 counter_value, counter_last_clear; /* * We collect 2 counters at once, except for the * last counter when the user asks for an odd number of * counters */ if ((pm->current_event + k) >= vec_len (pm->single_events_to_collect)) break; if (k == 0) { counter_value = n->stats_total.perf_counter0_ticks; counter_last_clear = n->stats_last_clear.perf_counter0_ticks; } else { counter_value = n->stats_total.perf_counter1_ticks; counter_last_clear = n->stats_last_clear.perf_counter1_ticks; } capture_name = format (0, "t%d-%v%c", j, n->name, 0); p = hash_get_mem (pm->capture_by_thread_and_node_name, capture_name); if (p == 0) { pool_get (pm->capture_pool, c); memset (c, 0, sizeof (*c)); c->thread_and_node_name = capture_name; hash_set_mem (pm->capture_by_thread_and_node_name, capture_name, c - pm->capture_pool); } else { c = pool_elt_at_index (pm->capture_pool, p[0]); vec_free (capture_name); } /* Snapshoot counters, etc. into the capture */ current_event = pm->single_events_to_collect + pm->current_event + k; counter_name = (u8 *) current_event->name; vectors_this_counter = n->stats_total.perf_counter_vectors - n->stats_last_clear.perf_counter_vectors; vec_add1 (c->counter_names, counter_name); vec_add1 (c->counter_values, counter_value - counter_last_clear); vec_add1 (c->vectors_this_counter, vectors_this_counter); } skip_this_node: clib_mem_free (n); } vec_free (nodes); } vec_free (node_dups); }
static u32 add_del_ip_tunnel (vnet_lisp_gpe_add_del_fwd_entry_args_t *a, u32 * tun_index_res) { lisp_gpe_main_t * lgm = &lisp_gpe_main; lisp_gpe_tunnel_t *t = 0; uword * p; int rv; lisp_gpe_tunnel_key_t key; /* prepare tunnel key */ memset(&key, 0, sizeof(key)); ip_prefix_copy(&key.eid, &gid_address_ippref(&a->deid)); ip_address_copy(&key.dst_loc, &a->dlocator); key.iid = clib_host_to_net_u32 (a->vni); p = mhash_get (&lgm->lisp_gpe_tunnel_by_key, &key); if (a->is_add) { /* adding a tunnel: tunnel must not already exist */ if (p) return VNET_API_ERROR_INVALID_VALUE; if (a->decap_next_index >= LISP_GPE_INPUT_N_NEXT) return VNET_API_ERROR_INVALID_DECAP_NEXT; pool_get_aligned (lgm->tunnels, t, CLIB_CACHE_LINE_BYTES); memset (t, 0, sizeof (*t)); /* copy from arg structure */ #define _(x) t->x = a->x; foreach_copy_field; #undef _ ip_address_copy(&t->src, &a->slocator); ip_address_copy(&t->dst, &a->dlocator); /* if vni is non-default */ if (a->vni) { t->flags = LISP_GPE_FLAGS_I; t->vni = a->vni; } t->flags |= LISP_GPE_FLAGS_P; t->next_protocol = ip_prefix_version(&key.eid) == IP4 ? LISP_GPE_NEXT_PROTO_IP4 : LISP_GPE_NEXT_PROTO_IP6; rv = lisp_gpe_rewrite (t); if (rv) { pool_put(lgm->tunnels, t); return rv; } mhash_set(&lgm->lisp_gpe_tunnel_by_key, &key, t - lgm->tunnels, 0); /* return tunnel index */ if (tun_index_res) tun_index_res[0] = t - lgm->tunnels; } else { /* deleting a tunnel: tunnel must exist */ if (!p) { clib_warning("Tunnel for eid %U doesn't exist!", format_gid_address, &a->deid); return VNET_API_ERROR_NO_SUCH_ENTRY; } t = pool_elt_at_index(lgm->tunnels, p[0]); mhash_unset(&lgm->lisp_gpe_tunnel_by_key, &key, 0); vec_free(t->rewrite); pool_put(lgm->tunnels, t); } return 0; }
static int dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe) { struct rte_flow_item_ipv4 ip4[2] = { }; struct rte_flow_item_ipv6 ip6[2] = { }; struct rte_flow_item_udp udp[2] = { }; struct rte_flow_item_tcp tcp[2] = { }; struct rte_flow_action_mark mark = { 0 }; struct rte_flow_item *item, *items = 0; struct rte_flow_action *action, *actions = 0; enum { vxlan_hdr_sz = sizeof (vxlan_header_t), raw_sz = sizeof (struct rte_flow_item_raw) }; union { struct rte_flow_item_raw item; u8 val[raw_sz + vxlan_hdr_sz]; } raw[2]; u16 src_port, dst_port, src_port_mask, dst_port_mask; u8 protocol; int rv = 0; if (f->actions & (~xd->supported_flow_actions)) return VNET_FLOW_ERROR_NOT_SUPPORTED; /* Match items */ /* Ethernet */ vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_ETH; item->spec = any_eth; item->mask = any_eth + 1; /* VLAN */ if (f->type != VNET_FLOW_TYPE_IP4_VXLAN) { vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_VLAN; item->spec = any_vlan; item->mask = any_vlan + 1; } /* IP */ vec_add2 (items, item, 1); if (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE) { vnet_flow_ip6_n_tuple_t *t6 = &f->ip6_n_tuple; clib_memcpy_fast (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16); clib_memcpy_fast (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16); clib_memcpy_fast (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16); clib_memcpy_fast (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16); item->type = RTE_FLOW_ITEM_TYPE_IPV6; item->spec = ip6; item->mask = ip6 + 1; src_port = t6->src_port.port; dst_port = t6->dst_port.port; src_port_mask = t6->src_port.mask; dst_port_mask = t6->dst_port.mask; protocol = t6->protocol; } else if (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE) { vnet_flow_ip4_n_tuple_t *t4 = &f->ip4_n_tuple; ip4[0].hdr.src_addr = t4->src_addr.addr.as_u32; ip4[1].hdr.src_addr = t4->src_addr.mask.as_u32; ip4[0].hdr.dst_addr = t4->dst_addr.addr.as_u32; ip4[1].hdr.dst_addr = t4->dst_addr.mask.as_u32; item->type = RTE_FLOW_ITEM_TYPE_IPV4; item->spec = ip4; item->mask = ip4 + 1; src_port = t4->src_port.port; dst_port = t4->dst_port.port; src_port_mask = t4->src_port.mask; dst_port_mask = t4->dst_port.mask; protocol = t4->protocol; } else if (f->type == VNET_FLOW_TYPE_IP4_VXLAN) { vnet_flow_ip4_vxlan_t *v4 = &f->ip4_vxlan; ip4[0].hdr.src_addr = v4->src_addr.as_u32; ip4[1].hdr.src_addr = -1; ip4[0].hdr.dst_addr = v4->dst_addr.as_u32; ip4[1].hdr.dst_addr = -1; item->type = RTE_FLOW_ITEM_TYPE_IPV4; item->spec = ip4; item->mask = ip4 + 1; dst_port = v4->dst_port; dst_port_mask = -1; src_port = 0; src_port_mask = 0; protocol = IP_PROTOCOL_UDP; } else { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* Layer 4 */ vec_add2 (items, item, 1); if (protocol == IP_PROTOCOL_UDP) { udp[0].hdr.src_port = clib_host_to_net_u16 (src_port); udp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask); udp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port); udp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask); item->type = RTE_FLOW_ITEM_TYPE_UDP; item->spec = udp; item->mask = udp + 1; } else if (protocol == IP_PROTOCOL_TCP) { tcp[0].hdr.src_port = clib_host_to_net_u16 (src_port); tcp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask); tcp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port); tcp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask); item->type = RTE_FLOW_ITEM_TYPE_TCP; item->spec = tcp; item->mask = tcp + 1; } else { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* Tunnel header match */ if (f->type == VNET_FLOW_TYPE_IP4_VXLAN) { u32 vni = f->ip4_vxlan.vni; vxlan_header_t spec_hdr = { .flags = VXLAN_FLAGS_I, .vni_reserved = clib_host_to_net_u32 (vni << 8) }; vxlan_header_t mask_hdr = { .flags = 0xff, .vni_reserved = clib_host_to_net_u32 (((u32) - 1) << 8) }; clib_memset (raw, 0, sizeof raw); raw[0].item.relative = 1; raw[0].item.length = vxlan_hdr_sz; clib_memcpy_fast (raw[0].val + raw_sz, &spec_hdr, vxlan_hdr_sz); raw[0].item.pattern = raw[0].val + raw_sz; clib_memcpy_fast (raw[1].val + raw_sz, &mask_hdr, vxlan_hdr_sz); raw[1].item.pattern = raw[1].val + raw_sz; vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_RAW; item->spec = raw; item->mask = raw + 1; } vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_END; /* Actions */ vec_add2 (actions, action, 1); action->type = RTE_FLOW_ACTION_TYPE_PASSTHRU; vec_add2 (actions, action, 1); mark.id = fe->mark; action->type = RTE_FLOW_ACTION_TYPE_MARK; action->conf = &mark; vec_add2 (actions, action, 1); action->type = RTE_FLOW_ACTION_TYPE_END; fe->handle = rte_flow_create (xd->device_index, &ingress, items, actions, &xd->last_flow_error); if (!fe->handle) rv = VNET_FLOW_ERROR_NOT_SUPPORTED; done: vec_free (items); vec_free (actions); return rv; } int dpdk_flow_ops_fn (vnet_main_t * vnm, vnet_flow_dev_op_t op, u32 dev_instance, u32 flow_index, uword * private_data) { dpdk_main_t *dm = &dpdk_main; vnet_flow_t *flow = vnet_get_flow (flow_index); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance); dpdk_flow_entry_t *fe; dpdk_flow_lookup_entry_t *fle = 0; int rv; /* recycle old flow lookup entries only after the main loop counter increases - i.e. previously DMA'ed packets were handled */ if (vec_len (xd->parked_lookup_indexes) > 0 && xd->parked_loop_count != dm->vlib_main->main_loop_count) { u32 *fl_index; vec_foreach (fl_index, xd->parked_lookup_indexes) pool_put_index (xd->flow_lookup_entries, *fl_index); vec_reset_length (xd->flow_lookup_entries); } if (op == VNET_FLOW_DEV_OP_DEL_FLOW) { ASSERT (*private_data >= vec_len (xd->flow_entries)); fe = vec_elt_at_index (xd->flow_entries, *private_data); if ((rv = rte_flow_destroy (xd->device_index, fe->handle, &xd->last_flow_error))) return VNET_FLOW_ERROR_INTERNAL; if (fe->mark) { /* make sure no action is taken for in-flight (marked) packets */ fle = pool_elt_at_index (xd->flow_lookup_entries, fe->mark); clib_memset (fle, -1, sizeof (*fle)); vec_add1 (xd->parked_lookup_indexes, fe->mark); xd->parked_loop_count = dm->vlib_main->main_loop_count; } clib_memset (fe, 0, sizeof (*fe)); pool_put (xd->flow_entries, fe); goto disable_rx_offload; } if (op != VNET_FLOW_DEV_OP_ADD_FLOW) return VNET_FLOW_ERROR_NOT_SUPPORTED; pool_get (xd->flow_entries, fe); fe->flow_index = flow->index; if (flow->actions == 0) { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* if we need to mark packets, assign one mark */ if (flow->actions & (VNET_FLOW_ACTION_MARK | VNET_FLOW_ACTION_REDIRECT_TO_NODE | VNET_FLOW_ACTION_BUFFER_ADVANCE)) { /* reserve slot 0 */ if (xd->flow_lookup_entries == 0) pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES); pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES); fe->mark = fle - xd->flow_lookup_entries; /* install entry in the lookup table */ clib_memset (fle, -1, sizeof (*fle)); if (flow->actions & VNET_FLOW_ACTION_MARK) fle->flow_id = flow->mark_flow_id; if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_NODE) fle->next_index = flow->redirect_device_input_next_index; if (flow->actions & VNET_FLOW_ACTION_BUFFER_ADVANCE) fle->buffer_advance = flow->buffer_advance; } else fe->mark = 0; if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) == 0) { xd->flags |= DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD; dpdk_device_setup (xd); } switch (flow->type) { case VNET_FLOW_TYPE_IP4_N_TUPLE: case VNET_FLOW_TYPE_IP6_N_TUPLE: case VNET_FLOW_TYPE_IP4_VXLAN: if ((rv = dpdk_flow_add (xd, flow, fe))) goto done; break; default: rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } *private_data = fe - xd->flow_entries; done: if (rv) { clib_memset (fe, 0, sizeof (*fe)); pool_put (xd->flow_entries, fe); if (fle) { clib_memset (fle, -1, sizeof (*fle)); pool_put (xd->flow_lookup_entries, fle); } } disable_rx_offload: if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) != 0 && pool_elts (xd->flow_entries) == 0) { xd->flags &= ~DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD; dpdk_device_setup (xd); } return rv; }