replication_context_t * replication_prep (vlib_main_t * vm, vlib_buffer_t * b0, u32 recycle_node_index, u32 l2_packet) { replication_main_t * rm = &replication_main; replication_context_t * ctx; uword cpu_number = vm->cpu_index; ip4_header_t * ip; u32 ctx_id; // Allocate a context, reserve context 0 if (PREDICT_FALSE(rm->contexts[cpu_number] == 0)) pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); pool_get_aligned (rm->contexts[cpu_number], ctx, CLIB_CACHE_LINE_BYTES); ctx_id = ctx - rm->contexts[cpu_number]; // Save state from vlib buffer ctx->saved_clone_count = b0->clone_count; ctx->saved_free_list_index = b0->free_list_index; ctx->current_data = b0->current_data; // Set up vlib buffer hooks b0->clone_count = ctx_id; b0->free_list_index = rm->recycle_list_index; // Save feature state ctx->recycle_node_index = recycle_node_index; // Save vnet state memcpy (ctx->vnet_buffer, vnet_buffer(b0), sizeof(vnet_buffer_opaque_t)); // Save packet contents ctx->l2_packet = l2_packet; ip = (ip4_header_t *)vlib_buffer_get_current (b0); if (l2_packet) { // Save ethernet header ctx->l2_header[0] = ((u64 *)ip)[0]; ctx->l2_header[1] = ((u64 *)ip)[1]; ctx->l2_header[2] = ((u64 *)ip)[2]; // set ip to the true ip header ip = (ip4_header_t *)(((u8 *)ip) + vnet_buffer(b0)->l2.l2_len); } // Copy L3 fields. // We need to save TOS for ip4 and ip6 packets. Fortunately the TOS field is // in the first two bytes of both the ip4 and ip6 headers. ctx->ip_tos = *((u16 *)(ip)); // Save the ip4 checksum as well. We just blindly save the corresponding two // bytes even for ip6 packets. ctx->ip4_checksum = ip->checksum; return ctx; }
session_table_t * session_table_alloc (void) { session_table_t *slt; pool_get_aligned (lookup_tables, slt, CLIB_CACHE_LINE_BYTES); clib_memset (slt, 0, sizeof (*slt)); return slt; }
session_t * session_alloc (u32 thread_index) { session_worker_t *wrk = &session_main.wrk[thread_index]; session_t *s; u8 will_expand = 0; pool_get_aligned_will_expand (wrk->sessions, will_expand, CLIB_CACHE_LINE_BYTES); /* If we have peekers, let them finish */ if (PREDICT_FALSE (will_expand && vlib_num_workers ())) { clib_rwlock_writer_lock (&wrk->peekers_rw_locks); pool_get_aligned (wrk->sessions, s, CLIB_CACHE_LINE_BYTES); clib_rwlock_writer_unlock (&wrk->peekers_rw_locks); } else { pool_get_aligned (wrk->sessions, s, CLIB_CACHE_LINE_BYTES); } clib_memset (s, 0, sizeof (*s)); s->session_index = s - wrk->sessions; s->thread_index = thread_index; return s; }
static replicate_t * replicate_alloc_i (void) { replicate_t *rep; pool_get_aligned(replicate_pool, rep, CLIB_CACHE_LINE_BYTES); clib_memset(rep, 0, sizeof(*rep)); vlib_validate_combined_counter(&(replicate_main.repm_counters), replicate_get_index(rep)); vlib_zero_combined_counter(&(replicate_main.repm_counters), replicate_get_index(rep)); return (rep); }
index_t mfib_itf_create (fib_node_index_t path_index, mfib_itf_flags_t mfi_flags) { mfib_itf_t *mfib_itf; pool_get_aligned(mfib_itf_pool, mfib_itf, CLIB_CACHE_LINE_BYTES); mfib_itf->mfi_sw_if_index = fib_path_get_resolving_interface(path_index); mfib_itf->mfi_si = INDEX_INVALID; /* * add the path index to the per-path hash */ mfib_itf->mfi_hash = hash_set(mfib_itf->mfi_hash, path_index, mfi_flags); /* * the combined flags from all the paths is from just the one contributor */ mfib_itf->mfi_flags = mfi_flags; return (mfib_itf - mfib_itf_pool); }
clib_error_t * policer_add_del (vlib_main_t *vm, u8 * name, sse2_qos_pol_cfg_params_st * cfg, u32 * policer_index, u8 is_add) { vnet_policer_main_t *pm = &vnet_policer_main; policer_read_response_type_st test_policer; policer_read_response_type_st * policer; uword * p; u32 pi; int rv; p = hash_get_mem (pm->policer_config_by_name, name); if (is_add == 0) { if (p == 0) { vec_free(name); return clib_error_return (0, "No such policer configuration"); } hash_unset_mem (pm->policer_config_by_name, name); hash_unset_mem (pm->policer_index_by_name, name); vec_free(name); return 0; } if (p != 0) { vec_free(name); return clib_error_return (0, "Policer already exists"); } /* Vet the configuration before adding it to the table */ rv = sse2_pol_logical_2_physical (cfg, &test_policer); if (rv == 0) { policer_read_response_type_st *pp; sse2_qos_pol_cfg_params_st *cp; pool_get (pm->configs, cp); pool_get (pm->policer_templates, pp); ASSERT (cp - pm->configs == pp - pm->policer_templates); clib_memcpy (cp, cfg, sizeof (*cp)); clib_memcpy (pp, &test_policer, sizeof (*pp)); hash_set_mem (pm->policer_config_by_name, name, cp - pm->configs); pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES); policer[0] = pp[0]; pi = policer - pm->policers; hash_set_mem (pm->policer_index_by_name, name, pi); *policer_index = pi; } else { vec_free (name); return clib_error_return (0, "Config failed sanity check"); } return 0; }
static u32 add_del_ip_tunnel (vnet_lisp_gpe_add_del_fwd_entry_args_t *a, u32 * tun_index_res) { lisp_gpe_main_t * lgm = &lisp_gpe_main; lisp_gpe_tunnel_t *t = 0; uword * p; int rv; lisp_gpe_tunnel_key_t key; /* prepare tunnel key */ memset(&key, 0, sizeof(key)); ip_prefix_copy(&key.eid, &gid_address_ippref(&a->deid)); ip_address_copy(&key.dst_loc, &a->dlocator); key.iid = clib_host_to_net_u32 (a->vni); p = mhash_get (&lgm->lisp_gpe_tunnel_by_key, &key); if (a->is_add) { /* adding a tunnel: tunnel must not already exist */ if (p) return VNET_API_ERROR_INVALID_VALUE; if (a->decap_next_index >= LISP_GPE_INPUT_N_NEXT) return VNET_API_ERROR_INVALID_DECAP_NEXT; pool_get_aligned (lgm->tunnels, t, CLIB_CACHE_LINE_BYTES); memset (t, 0, sizeof (*t)); /* copy from arg structure */ #define _(x) t->x = a->x; foreach_copy_field; #undef _ ip_address_copy(&t->src, &a->slocator); ip_address_copy(&t->dst, &a->dlocator); /* if vni is non-default */ if (a->vni) { t->flags = LISP_GPE_FLAGS_I; t->vni = a->vni; } t->flags |= LISP_GPE_FLAGS_P; t->next_protocol = ip_prefix_version(&key.eid) == IP4 ? LISP_GPE_NEXT_PROTO_IP4 : LISP_GPE_NEXT_PROTO_IP6; rv = lisp_gpe_rewrite (t); if (rv) { pool_put(lgm->tunnels, t); return rv; } mhash_set(&lgm->lisp_gpe_tunnel_by_key, &key, t - lgm->tunnels, 0); /* return tunnel index */ if (tun_index_res) tun_index_res[0] = t - lgm->tunnels; } else { /* deleting a tunnel: tunnel must exist */ if (!p) { clib_warning("Tunnel for eid %U doesn't exist!", format_gid_address, &a->deid); return VNET_API_ERROR_NO_SUCH_ENTRY; } t = pool_elt_at_index(lgm->tunnels, p[0]); mhash_unset(&lgm->lisp_gpe_tunnel_by_key, &key, 0); vec_free(t->rewrite); pool_put(lgm->tunnels, t); } return 0; }
static int dpdk_flow_add (dpdk_device_t * xd, vnet_flow_t * f, dpdk_flow_entry_t * fe) { struct rte_flow_item_ipv4 ip4[2] = { }; struct rte_flow_item_ipv6 ip6[2] = { }; struct rte_flow_item_udp udp[2] = { }; struct rte_flow_item_tcp tcp[2] = { }; struct rte_flow_action_mark mark = { 0 }; struct rte_flow_item *item, *items = 0; struct rte_flow_action *action, *actions = 0; enum { vxlan_hdr_sz = sizeof (vxlan_header_t), raw_sz = sizeof (struct rte_flow_item_raw) }; union { struct rte_flow_item_raw item; u8 val[raw_sz + vxlan_hdr_sz]; } raw[2]; u16 src_port, dst_port, src_port_mask, dst_port_mask; u8 protocol; int rv = 0; if (f->actions & (~xd->supported_flow_actions)) return VNET_FLOW_ERROR_NOT_SUPPORTED; /* Match items */ /* Ethernet */ vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_ETH; item->spec = any_eth; item->mask = any_eth + 1; /* VLAN */ if (f->type != VNET_FLOW_TYPE_IP4_VXLAN) { vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_VLAN; item->spec = any_vlan; item->mask = any_vlan + 1; } /* IP */ vec_add2 (items, item, 1); if (f->type == VNET_FLOW_TYPE_IP6_N_TUPLE) { vnet_flow_ip6_n_tuple_t *t6 = &f->ip6_n_tuple; clib_memcpy_fast (ip6[0].hdr.src_addr, &t6->src_addr.addr, 16); clib_memcpy_fast (ip6[1].hdr.src_addr, &t6->src_addr.mask, 16); clib_memcpy_fast (ip6[0].hdr.dst_addr, &t6->dst_addr.addr, 16); clib_memcpy_fast (ip6[1].hdr.dst_addr, &t6->dst_addr.mask, 16); item->type = RTE_FLOW_ITEM_TYPE_IPV6; item->spec = ip6; item->mask = ip6 + 1; src_port = t6->src_port.port; dst_port = t6->dst_port.port; src_port_mask = t6->src_port.mask; dst_port_mask = t6->dst_port.mask; protocol = t6->protocol; } else if (f->type == VNET_FLOW_TYPE_IP4_N_TUPLE) { vnet_flow_ip4_n_tuple_t *t4 = &f->ip4_n_tuple; ip4[0].hdr.src_addr = t4->src_addr.addr.as_u32; ip4[1].hdr.src_addr = t4->src_addr.mask.as_u32; ip4[0].hdr.dst_addr = t4->dst_addr.addr.as_u32; ip4[1].hdr.dst_addr = t4->dst_addr.mask.as_u32; item->type = RTE_FLOW_ITEM_TYPE_IPV4; item->spec = ip4; item->mask = ip4 + 1; src_port = t4->src_port.port; dst_port = t4->dst_port.port; src_port_mask = t4->src_port.mask; dst_port_mask = t4->dst_port.mask; protocol = t4->protocol; } else if (f->type == VNET_FLOW_TYPE_IP4_VXLAN) { vnet_flow_ip4_vxlan_t *v4 = &f->ip4_vxlan; ip4[0].hdr.src_addr = v4->src_addr.as_u32; ip4[1].hdr.src_addr = -1; ip4[0].hdr.dst_addr = v4->dst_addr.as_u32; ip4[1].hdr.dst_addr = -1; item->type = RTE_FLOW_ITEM_TYPE_IPV4; item->spec = ip4; item->mask = ip4 + 1; dst_port = v4->dst_port; dst_port_mask = -1; src_port = 0; src_port_mask = 0; protocol = IP_PROTOCOL_UDP; } else { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* Layer 4 */ vec_add2 (items, item, 1); if (protocol == IP_PROTOCOL_UDP) { udp[0].hdr.src_port = clib_host_to_net_u16 (src_port); udp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask); udp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port); udp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask); item->type = RTE_FLOW_ITEM_TYPE_UDP; item->spec = udp; item->mask = udp + 1; } else if (protocol == IP_PROTOCOL_TCP) { tcp[0].hdr.src_port = clib_host_to_net_u16 (src_port); tcp[1].hdr.src_port = clib_host_to_net_u16 (src_port_mask); tcp[0].hdr.dst_port = clib_host_to_net_u16 (dst_port); tcp[1].hdr.dst_port = clib_host_to_net_u16 (dst_port_mask); item->type = RTE_FLOW_ITEM_TYPE_TCP; item->spec = tcp; item->mask = tcp + 1; } else { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* Tunnel header match */ if (f->type == VNET_FLOW_TYPE_IP4_VXLAN) { u32 vni = f->ip4_vxlan.vni; vxlan_header_t spec_hdr = { .flags = VXLAN_FLAGS_I, .vni_reserved = clib_host_to_net_u32 (vni << 8) }; vxlan_header_t mask_hdr = { .flags = 0xff, .vni_reserved = clib_host_to_net_u32 (((u32) - 1) << 8) }; clib_memset (raw, 0, sizeof raw); raw[0].item.relative = 1; raw[0].item.length = vxlan_hdr_sz; clib_memcpy_fast (raw[0].val + raw_sz, &spec_hdr, vxlan_hdr_sz); raw[0].item.pattern = raw[0].val + raw_sz; clib_memcpy_fast (raw[1].val + raw_sz, &mask_hdr, vxlan_hdr_sz); raw[1].item.pattern = raw[1].val + raw_sz; vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_RAW; item->spec = raw; item->mask = raw + 1; } vec_add2 (items, item, 1); item->type = RTE_FLOW_ITEM_TYPE_END; /* Actions */ vec_add2 (actions, action, 1); action->type = RTE_FLOW_ACTION_TYPE_PASSTHRU; vec_add2 (actions, action, 1); mark.id = fe->mark; action->type = RTE_FLOW_ACTION_TYPE_MARK; action->conf = &mark; vec_add2 (actions, action, 1); action->type = RTE_FLOW_ACTION_TYPE_END; fe->handle = rte_flow_create (xd->device_index, &ingress, items, actions, &xd->last_flow_error); if (!fe->handle) rv = VNET_FLOW_ERROR_NOT_SUPPORTED; done: vec_free (items); vec_free (actions); return rv; } int dpdk_flow_ops_fn (vnet_main_t * vnm, vnet_flow_dev_op_t op, u32 dev_instance, u32 flow_index, uword * private_data) { dpdk_main_t *dm = &dpdk_main; vnet_flow_t *flow = vnet_get_flow (flow_index); dpdk_device_t *xd = vec_elt_at_index (dm->devices, dev_instance); dpdk_flow_entry_t *fe; dpdk_flow_lookup_entry_t *fle = 0; int rv; /* recycle old flow lookup entries only after the main loop counter increases - i.e. previously DMA'ed packets were handled */ if (vec_len (xd->parked_lookup_indexes) > 0 && xd->parked_loop_count != dm->vlib_main->main_loop_count) { u32 *fl_index; vec_foreach (fl_index, xd->parked_lookup_indexes) pool_put_index (xd->flow_lookup_entries, *fl_index); vec_reset_length (xd->flow_lookup_entries); } if (op == VNET_FLOW_DEV_OP_DEL_FLOW) { ASSERT (*private_data >= vec_len (xd->flow_entries)); fe = vec_elt_at_index (xd->flow_entries, *private_data); if ((rv = rte_flow_destroy (xd->device_index, fe->handle, &xd->last_flow_error))) return VNET_FLOW_ERROR_INTERNAL; if (fe->mark) { /* make sure no action is taken for in-flight (marked) packets */ fle = pool_elt_at_index (xd->flow_lookup_entries, fe->mark); clib_memset (fle, -1, sizeof (*fle)); vec_add1 (xd->parked_lookup_indexes, fe->mark); xd->parked_loop_count = dm->vlib_main->main_loop_count; } clib_memset (fe, 0, sizeof (*fe)); pool_put (xd->flow_entries, fe); goto disable_rx_offload; } if (op != VNET_FLOW_DEV_OP_ADD_FLOW) return VNET_FLOW_ERROR_NOT_SUPPORTED; pool_get (xd->flow_entries, fe); fe->flow_index = flow->index; if (flow->actions == 0) { rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } /* if we need to mark packets, assign one mark */ if (flow->actions & (VNET_FLOW_ACTION_MARK | VNET_FLOW_ACTION_REDIRECT_TO_NODE | VNET_FLOW_ACTION_BUFFER_ADVANCE)) { /* reserve slot 0 */ if (xd->flow_lookup_entries == 0) pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES); pool_get_aligned (xd->flow_lookup_entries, fle, CLIB_CACHE_LINE_BYTES); fe->mark = fle - xd->flow_lookup_entries; /* install entry in the lookup table */ clib_memset (fle, -1, sizeof (*fle)); if (flow->actions & VNET_FLOW_ACTION_MARK) fle->flow_id = flow->mark_flow_id; if (flow->actions & VNET_FLOW_ACTION_REDIRECT_TO_NODE) fle->next_index = flow->redirect_device_input_next_index; if (flow->actions & VNET_FLOW_ACTION_BUFFER_ADVANCE) fle->buffer_advance = flow->buffer_advance; } else fe->mark = 0; if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) == 0) { xd->flags |= DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD; dpdk_device_setup (xd); } switch (flow->type) { case VNET_FLOW_TYPE_IP4_N_TUPLE: case VNET_FLOW_TYPE_IP6_N_TUPLE: case VNET_FLOW_TYPE_IP4_VXLAN: if ((rv = dpdk_flow_add (xd, flow, fe))) goto done; break; default: rv = VNET_FLOW_ERROR_NOT_SUPPORTED; goto done; } *private_data = fe - xd->flow_entries; done: if (rv) { clib_memset (fe, 0, sizeof (*fe)); pool_put (xd->flow_entries, fe); if (fle) { clib_memset (fle, -1, sizeof (*fle)); pool_put (xd->flow_lookup_entries, fle); } } disable_rx_offload: if ((xd->flags & DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD) != 0 && pool_elts (xd->flow_entries) == 0) { xd->flags &= ~DPDK_DEVICE_FLAG_RX_FLOW_OFFLOAD; dpdk_device_setup (xd); } return rv; }