clib_error_t * foreach_directory_file (char *dir_name, clib_error_t * (*f) (void *arg, u8 * path_name, u8 * file_name), void *arg, int scan_dirs) { DIR *d; struct dirent *e; clib_error_t *error = 0; u8 *s, *t; d = opendir (dir_name); if (!d) { if (errno == ENOENT) return 0; return clib_error_return_unix (0, "open `%s'", dir_name); } s = t = 0; while (1) { e = readdir (d); if (!e) break; if (scan_dirs) { if (e->d_type == DT_DIR && (!strcmp (e->d_name, ".") || !strcmp (e->d_name, ".."))) continue; } else { if (e->d_type == DT_DIR) continue; } s = format (s, "%s/%s", dir_name, e->d_name); t = format (t, "%s", e->d_name); error = f (arg, s, t); _vec_len (s) = 0; _vec_len (t) = 0; if (error) break; } vec_free (s); closedir (d); return error; }
void * _clib_fifo_resize (void *v_old, uword n_new_elts, uword elt_bytes) { void *v_new, *end, *head; uword n_old_elts, header_bytes; uword n_copy_bytes, n_zero_bytes; clib_fifo_header_t *f_new, *f_old; n_old_elts = clib_fifo_elts (v_old); n_new_elts += n_old_elts; if (n_new_elts < 32) n_new_elts = 32; else n_new_elts = max_pow2 (n_new_elts); header_bytes = vec_header_bytes (sizeof (clib_fifo_header_t)); v_new = clib_mem_alloc_no_fail (n_new_elts * elt_bytes + header_bytes); v_new += header_bytes; f_new = clib_fifo_header (v_new); f_new->head_index = 0; f_new->tail_index = n_old_elts; _vec_len (v_new) = n_new_elts; /* Copy old -> new. */ n_copy_bytes = n_old_elts * elt_bytes; if (n_copy_bytes > 0) { f_old = clib_fifo_header (v_old); end = v_old + _vec_len (v_old) * elt_bytes; head = v_old + f_old->head_index * elt_bytes; if (head + n_copy_bytes >= end) { uword n = end - head; clib_memcpy_fast (v_new, head, n); clib_memcpy_fast (v_new + n, v_old, n_copy_bytes - n); } else clib_memcpy_fast (v_new, head, n_copy_bytes); } /* Zero empty space. */ n_zero_bytes = (n_new_elts - n_old_elts) * elt_bytes; clib_memset (v_new + n_copy_bytes, 0, n_zero_bytes); clib_fifo_free (v_old); return v_new; }
static clib_error_t * sendmsg_helper (mc_socket_main_t * msm, int socket, struct sockaddr_in * tx_addr, u32 buffer_index) { vlib_main_t * vm = msm->mc_main.vlib_main; struct msghdr h; word n_bytes, n_bytes_tx, n_retries; memset (&h, 0, sizeof (h)); h.msg_name = tx_addr; h.msg_namelen = sizeof (tx_addr[0]); if (msm->iovecs) _vec_len (msm->iovecs) = 0; n_bytes = append_buffer_index_to_iovec (vm, buffer_index, &msm->iovecs); ASSERT (n_bytes <= msm->mc_main.transport.max_packet_size); if (n_bytes > msm->mc_main.transport.max_packet_size) clib_error ("sending packet larger than interace MTU %d bytes", n_bytes); h.msg_iov = msm->iovecs; h.msg_iovlen = vec_len (msm->iovecs); n_retries = 0; while ((n_bytes_tx = sendmsg (socket, &h, /* flags */ 0)) != n_bytes && errno == EAGAIN) n_retries++; if (n_bytes_tx != n_bytes) { clib_unix_warning ("sendmsg"); return 0; } if (n_retries) { ELOG_TYPE_DECLARE (e) = { .format = "sendmsg-helper: %d retries", .format_args = "i4", }; struct { u32 retries; } * ed = 0; ed = ELOG_DATA (&vm->elog_main, e); ed->retries = n_retries; } return 0; } static clib_error_t * tx_buffer (void * transport, mc_transport_type_t type, u32 buffer_index) { mc_socket_main_t *msm = (mc_socket_main_t *)transport; vlib_main_t * vm = msm->mc_main.vlib_main; mc_multicast_socket_t * ms = &msm->multicast_sockets[type]; clib_error_t * error; error = sendmsg_helper (msm, ms->socket, &ms->tx_addr, buffer_index); if (type != MC_TRANSPORT_USER_REQUEST_TO_RELAY) vlib_buffer_free_one (vm, buffer_index); return error; }
static vnet_config_t * find_config_with_features (vlib_main_t * vm, vnet_config_main_t * cm, vnet_config_feature_t * feature_vector) { u32 last_node_index = ~0; vnet_config_feature_t * f; u32 * config_string; uword * p; vnet_config_t * c; config_string = cm->config_string_temp; cm->config_string_temp = 0; if (config_string) _vec_len (config_string) = 0; vec_foreach (f, feature_vector) { /* Connect node graph. */ f->next_index = add_next (vm, cm, last_node_index, f->node_index); last_node_index = f->node_index; /* Store next index in config string. */ vec_add1 (config_string, f->next_index); /* Store feature config. */ vec_add (config_string, f->feature_config, vec_len (f->feature_config)); }
/* * cdp periodic function */ static uword cdp_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { cdp_main_t * cm = &cdp_main; f64 poll_time_remaining; uword event_type, * event_data = 0; /* So we can send events to the cdp process */ cm->cdp_process_node_index = cdp_process_node.index; /* Dynamically register the cdp input node with the snap classifier */ snap_register_input_protocol (vm, "cdp-input", 0xC /* ieee_oui, Cisco */, 0x2000 /* protocol CDP */, cdp_input_node.index); snap_register_input_protocol (vm, "cdp-input", 0xC /* ieee_oui, Cisco */, 0x2004 /* protocol CDP */, cdp_input_node.index); #if 0 /* retain for reference */ /* with the hdlc classifier */ hdlc_register_input_protocol (vm, HDLC_PROTOCOL_cdp, cdp_input_node.index); #endif /* with ethernet input (for SRP) */ ethernet_register_input_type (vm, ETHERNET_TYPE_CDP /* CDP */, cdp_input_node.index); poll_time_remaining = 10.0 /* seconds */; while (1) { /* sleep until next poll time, or msg serialize event occurs */ poll_time_remaining = vlib_process_wait_for_event_or_clock (vm, poll_time_remaining); event_type = vlib_process_get_events (vm, &event_data); switch (event_type) { case ~0: /* no events => timeout */ break; default: clib_warning ("BUG: event type 0x%wx", event_type); break; } if (event_data) _vec_len (event_data) = 0; /* peer timeout scan, send announcements */ if (vlib_process_suspend_time_is_zero (poll_time_remaining)) { cdp_periodic (vm); poll_time_remaining = 10.0; } } return 0; }
static void elt_delete (heap_header_t * h, heap_elt_t * e) { heap_elt_t * l = vec_end (h->elts) - 1; ASSERT (e >= h->elts && e <= l); /* Update doubly linked pointers. */ { heap_elt_t * p = heap_prev (e); heap_elt_t * n = heap_next (e); if (p == e) { n->prev = 0; h->head = n - h->elts; } else if (n == e) { p->next = 0; h->tail = p - h->elts; } else { p->next = n - p; n->prev = p - n; } } /* Add to index free list or delete from end. */ if (e < l) vec_add1 (h->free_elts, e - h->elts); else _vec_len (h->elts)--; }
void init_trajectory_trace (vlib_buffer_t * b) { if (!clib_mem_is_vec (vnet_buffer2 (b)->trajectory_trace)) { vnet_buffer2 (b)->trajectory_trace = 0; vec_validate (vnet_buffer2 (b)->trajectory_trace, 7); } _vec_len (vnet_buffer2 (b)->trajectory_trace) = 0; }
static clib_error_t * catchup_socket_read_ready (unix_file_t * uf, int is_server) { unix_main_t * um = &unix_main; mc_socket_main_t *msm = (mc_socket_main_t *)uf->private_data; mc_main_t *mcm = &msm->mc_main; mc_socket_catchup_t * c = find_catchup_from_file_descriptor (msm, uf->file_descriptor); word l, n, is_eof; l = vec_len (c->input_vector); vec_resize (c->input_vector, 4096); n = read (uf->file_descriptor, c->input_vector + l, vec_len (c->input_vector) - l); is_eof = n == 0; if (n < 0) { if (errno == EAGAIN) n = 0; else { catchup_cleanup (msm, c, um, uf); return clib_error_return_unix (0, "read"); } } _vec_len (c->input_vector) = l + n; if (is_eof && vec_len (c->input_vector) > 0) { if (is_server) { mc_msg_catchup_request_handler (mcm, (void *) c->input_vector, c - msm->catchups); _vec_len (c->input_vector) = 0; } else { mc_msg_catchup_reply_handler (mcm, (void *) c->input_vector, c - msm->catchups); c->input_vector = 0; /* reply handler is responsible for freeing vector */ catchup_cleanup (msm, c, um, uf); } } return 0 /* no error */; }
/* Arrange for function to be called some time, roughly equal to dt seconds, in the future. */ void timer_call (timer_func_t * func, any arg, f64 dt) { timer_callback_t *t; sigset_t save; /* Install signal handler on first call. */ static word signal_installed = 0; if (!signal_installed) { struct sigaction sa; /* Initialize time_resolution before first call to timer_interrupt */ time_resolution = 0.75 / (f64) HZ; clib_memset (&sa, 0, sizeof (sa)); sa.sa_handler = timer_interrupt; if (sigaction (TIMER_SIGNAL, &sa, 0) < 0) clib_panic ("sigaction"); signal_installed = 1; } timer_block (&save); /* Add new timer. */ vec_add2 (timers, t, 1); t->time = unix_time_now () + dt; t->func = func; t->arg = arg; { word reset_timer = vec_len (timers) == 1; if (_vec_len (timers) > 1) { reset_timer += t->time < (t - 1)->time; sort_timers (timers); } if (reset_timer) timer_interrupt (TIMER_SIGNAL); } timer_unblock (&save); }
/* Interrupt handler. Call functions for all expired timers. Set time for next timer interrupt. */ static void timer_interrupt (int signum) { f64 now = unix_time_now (); f64 dt; timer_callback_t *t; while (1) { if (vec_len (timers) <= 0) return; /* Consider last (earliest) timer in reverse sorted vector of pending timers. */ t = vec_end (timers) - 1; ASSERT (now >= 0 && isfinite (now)); /* Time difference between when timer goes off and now. */ dt = t->time - now; /* If timer is within threshold of going off call user's callback. */ if (dt <= time_resolution && isfinite (dt)) { _vec_len (timers) -= 1; (*t->func) (t->arg, -dt); } else { /* Set timer for to go off in future. */ struct itimerval itv; clib_memset (&itv, 0, sizeof (itv)); f64_to_tv (dt, &itv.it_value); if (setitimer (ITIMER_REAL, &itv, 0) < 0) clib_unix_error ("sititmer"); return; } } }
clib_error_t * vlib_sysfs_read (char *file_name, char *fmt, ...) { unformat_input_t input; u8 *s = 0; int fd; ssize_t sz; uword result; fd = open (file_name, O_RDONLY); if (fd < 0) return clib_error_return_unix (0, "open `%s'", file_name); vec_validate (s, 4095); sz = read (fd, s, vec_len (s)); if (sz < 0) { close (fd); vec_free (s); return clib_error_return_unix (0, "read `%s'", file_name); } _vec_len (s) = sz; unformat_init_vector (&input, s); va_list va; va_start (va, fmt); result = va_unformat (&input, fmt, &va); va_end (va); vec_free (s); close (fd); if (result == 0) return clib_error_return (0, "unformat error"); return 0; }
clib_error_t * unix_proc_file_contents (char *file, u8 ** result) { u8 *rv = 0; uword pos; int bytes, fd; /* Unfortunately, stat(/proc/XXX) returns zero... */ fd = open (file, O_RDONLY); if (fd < 0) return clib_error_return_unix (0, "open `%s'", file); vec_validate (rv, 4095); pos = 0; while (1) { bytes = read (fd, rv + pos, 4096); if (bytes < 0) { close (fd); vec_free (rv); return clib_error_return_unix (0, "read '%s'", file); } if (bytes == 0) { _vec_len (rv) = pos; break; } pos += bytes; vec_validate (rv, pos + 4095); } *result = rv; close (fd); return 0; }
int vat_load_new_plugins (plugin_main_t * pm) { DIR *dp; struct dirent *entry; struct stat statb; uword *p; plugin_info_t *pi; u8 **plugin_path; int i; plugin_path = split_plugin_path (pm); for (i = 0; i < vec_len (plugin_path); i++) { dp = opendir ((char *) plugin_path[i]); if (dp == 0) continue; while ((entry = readdir (dp))) { u8 *plugin_name; u8 *file_name; if (pm->plugin_name_filter) { int j; for (j = 0; j < vec_len (pm->plugin_name_filter); j++) if (entry->d_name[j] != pm->plugin_name_filter[j]) goto next; } file_name = format (0, "%s/%s%c", plugin_path[i], entry->d_name, 0); plugin_name = format (0, "%s%c", entry->d_name, 0); /* unreadable */ if (stat ((char *) file_name, &statb) < 0) { ignore: vec_free (file_name); vec_free (plugin_name); continue; } /* a dir or other things which aren't plugins */ if (!S_ISREG (statb.st_mode)) goto ignore; p = hash_get_mem (pm->plugin_by_name_hash, plugin_name); if (p == 0) { vec_add2 (pm->plugin_info, pi, 1); pi->name = plugin_name; pi->filename = file_name; pi->file_info = statb; if (load_one_vat_plugin (pm, pi)) { vec_free (file_name); vec_free (plugin_name); _vec_len (pm->plugin_info) = vec_len (pm->plugin_info) - 1; continue; } clib_memset (pi, 0, sizeof (*pi)); hash_set_mem (pm->plugin_by_name_hash, plugin_name, pi - pm->plugin_info); } next: ; } closedir (dp); vec_free (plugin_path[i]); } vec_free (plugin_path); return 0; }
always_inline uword geneve_encap_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u32 is_ip4) { u32 n_left_from, next_index, *from, *to_next; geneve_main_t *vxm = &geneve_main; vnet_main_t *vnm = vxm->vnet_main; vnet_interface_main_t *im = &vnm->interface_main; u32 pkts_encapsulated = 0; u16 old_l0 = 0, old_l1 = 0; u32 thread_index = vm->thread_index; u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; u32 sw_if_index0 = ~0, sw_if_index1 = ~0; u32 next0 = 0, next1 = 0; vnet_hw_interface_t *hi0, *hi1; geneve_tunnel_t *t0 = NULL, *t1 = NULL; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; stats_sw_if_index = node->runtime_data[0]; stats_n_packets = stats_n_bytes = 0; while (n_left_from > 0) { u32 n_left_to_next; vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); while (n_left_from >= 4 && n_left_to_next >= 2) { u32 bi0, bi1; vlib_buffer_t *b0, *b1; u32 flow_hash0, flow_hash1; u32 len0, len1; ip4_header_t *ip4_0, *ip4_1; ip6_header_t *ip6_0, *ip6_1; udp_header_t *udp0, *udp1; u64 *copy_src0, *copy_dst0; u64 *copy_src1, *copy_dst1; u32 *copy_src_last0, *copy_dst_last0; u32 *copy_src_last1, *copy_dst_last1; u16 new_l0, new_l1; ip_csum_t sum0, sum1; /* Prefetch next iteration. */ { vlib_buffer_t *p2, *p3; p2 = vlib_get_buffer (vm, from[2]); p3 = vlib_get_buffer (vm, from[3]); vlib_prefetch_buffer_header (p2, LOAD); vlib_prefetch_buffer_header (p3, LOAD); CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD); } bi0 = from[0]; bi1 = from[1]; to_next[0] = bi0; to_next[1] = bi1; from += 2; to_next += 2; n_left_to_next -= 2; n_left_from -= 2; b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); flow_hash0 = vnet_l2_compute_flow_hash (b0); flow_hash1 = vnet_l2_compute_flow_hash (b1); /* Get next node index and adj index from tunnel next_dpo */ if (sw_if_index0 != vnet_buffer (b0)->sw_if_index[VLIB_TX]) { sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); t0 = &vxm->tunnels[hi0->dev_instance]; /* Note: change to always set next0 if it may be set to drop */ next0 = t0->next_dpo.dpoi_next_node; } ASSERT (t0 != NULL); vnet_buffer (b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index; /* Get next node index and adj index from tunnel next_dpo */ if (sw_if_index1 != vnet_buffer (b1)->sw_if_index[VLIB_TX]) { sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_TX]; hi1 = vnet_get_sup_hw_interface (vnm, sw_if_index1); t1 = &vxm->tunnels[hi1->dev_instance]; /* Note: change to always set next1 if it may be set to drop */ next1 = t1->next_dpo.dpoi_next_node; } ASSERT (t1 != NULL); vnet_buffer (b1)->ip.adj_index[VLIB_TX] = t1->next_dpo.dpoi_index; /* Apply the rewrite string. $$$$ vnet_rewrite? */ vlib_buffer_advance (b0, -(word) _vec_len (t0->rewrite)); vlib_buffer_advance (b1, -(word) _vec_len (t1->rewrite)); if (is_ip4) { u8 ip4_geneve_base_header_len = sizeof (ip4_header_t) + sizeof (udp_header_t) + GENEVE_BASE_HEADER_LENGTH; u8 ip4_geneve_header_total_len0 = ip4_geneve_base_header_len; u8 ip4_geneve_header_total_len1 = ip4_geneve_base_header_len; #if SUPPORT_OPTIONS_HEADER==1 ip4_geneve_header_total_len0 += t0->options_len; ip4_geneve_header_total_len1 += t1->options_len; #endif ASSERT (vec_len (t0->rewrite) == ip4_geneve_header_total_len0); ASSERT (vec_len (t1->rewrite) == ip4_geneve_header_total_len1); ip4_0 = vlib_buffer_get_current (b0); ip4_1 = vlib_buffer_get_current (b1); /* Copy the fixed header */ copy_dst0 = (u64 *) ip4_0; copy_src0 = (u64 *) t0->rewrite; copy_dst1 = (u64 *) ip4_1; copy_src1 = (u64 *) t1->rewrite; /* Copy first 32 octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ #define _(offs) copy_dst1[offs] = copy_src1[offs]; foreach_fixed_header4_offset; #undef _ /* Last 4 octets. Hopefully gcc will be our friend */ copy_dst_last0 = (u32 *) (©_dst0[4]); copy_src_last0 = (u32 *) (©_src0[4]); copy_dst_last0[0] = copy_src_last0[0]; copy_dst_last1 = (u32 *) (©_dst1[4]); copy_src_last1 = (u32 *) (©_src1[4]); copy_dst_last1[0] = copy_src_last1[0]; /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; new_l0 = /* old_l0 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)); sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, length /* changed member */ ); ip4_0->checksum = ip_csum_fold (sum0); ip4_0->length = new_l0; sum1 = ip4_1->checksum; new_l1 = /* old_l1 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1)); sum1 = ip_csum_update (sum1, old_l1, new_l1, ip4_header_t, length /* changed member */ ); ip4_1->checksum = ip_csum_fold (sum1); ip4_1->length = new_l1; /* Fix UDP length and set source port */ udp0 = (udp_header_t *) (ip4_0 + 1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip4_0)); udp0->length = new_l0; udp0->src_port = flow_hash0; udp1 = (udp_header_t *) (ip4_1 + 1); new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1) - sizeof (*ip4_1)); udp1->length = new_l1; udp1->src_port = flow_hash1; } else /* ipv6 */ { int bogus = 0; u8 ip6_geneve_base_header_len = sizeof (ip6_header_t) + sizeof (udp_header_t) + GENEVE_BASE_HEADER_LENGTH; u8 ip6_geneve_header_total_len0 = ip6_geneve_base_header_len; u8 ip6_geneve_header_total_len1 = ip6_geneve_base_header_len; #if SUPPORT_OPTIONS_HEADER==1 ip6_geneve_header_total_len0 += t0->options_len; ip6_geneve_header_total_len1 += t1->options_len; #endif ASSERT (vec_len (t0->rewrite) == ip6_geneve_header_total_len0); ASSERT (vec_len (t1->rewrite) == ip6_geneve_header_total_len1); ip6_0 = vlib_buffer_get_current (b0); ip6_1 = vlib_buffer_get_current (b1); /* Copy the fixed header */ copy_dst0 = (u64 *) ip6_0; copy_src0 = (u64 *) t0->rewrite; copy_dst1 = (u64 *) ip6_1; copy_src1 = (u64 *) t1->rewrite; /* Copy first 56 (ip6) octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ #define _(offs) copy_dst1[offs] = copy_src1[offs]; foreach_fixed_header6_offset; #undef _ /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip6_0)); ip6_0->payload_length = new_l0; new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1) - sizeof (*ip6_1)); ip6_1->payload_length = new_l1; /* Fix UDP length and set source port */ udp0 = (udp_header_t *) (ip6_0 + 1); udp0->length = new_l0; udp0->src_port = flow_hash0; udp1 = (udp_header_t *) (ip6_1 + 1); udp1->length = new_l1; udp1->src_port = flow_hash1; /* IPv6 UDP checksum is mandatory */ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0, &bogus); ASSERT (bogus == 0); if (udp0->checksum == 0) udp0->checksum = 0xffff; udp1->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b1, ip6_1, &bogus); ASSERT (bogus == 0); if (udp1->checksum == 0) udp1->checksum = 0xffff; } pkts_encapsulated += 2; len0 = vlib_buffer_length_in_chain (vm, b0); len1 = vlib_buffer_length_in_chain (vm, b1); stats_n_packets += 2; stats_n_bytes += len0 + len1; /* Batch stats increment on the same geneve tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile to check for this rare case and affect normal path performance. */ if (PREDICT_FALSE ((sw_if_index0 != stats_sw_if_index) || (sw_if_index1 != stats_sw_if_index))) { stats_n_packets -= 2; stats_n_bytes -= len0 + len1; if (sw_if_index0 == sw_if_index1) { if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 2; stats_n_bytes = len0 + len1; } else { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index1, 1, len1); } } if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { geneve_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - vxm->tunnels; tr->vni = t0->vni; } if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED)) { geneve_encap_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); tr->tunnel_index = t1 - vxm->tunnels; tr->vni = t1->vni; } vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1); } while (n_left_from > 0 && n_left_to_next > 0) { u32 bi0; vlib_buffer_t *b0; u32 flow_hash0; u32 len0; ip4_header_t *ip4_0; ip6_header_t *ip6_0; udp_header_t *udp0; u64 *copy_src0, *copy_dst0; u32 *copy_src_last0, *copy_dst_last0; u16 new_l0; ip_csum_t sum0; bi0 = from[0]; to_next[0] = bi0; from += 1; to_next += 1; n_left_from -= 1; n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); flow_hash0 = vnet_l2_compute_flow_hash (b0); /* Get next node index and adj index from tunnel next_dpo */ if (sw_if_index0 != vnet_buffer (b0)->sw_if_index[VLIB_TX]) { sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_TX]; hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); t0 = &vxm->tunnels[hi0->dev_instance]; /* Note: change to always set next0 if it may be set to drop */ next0 = t0->next_dpo.dpoi_next_node; } vnet_buffer (b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index; /* Apply the rewrite string. $$$$ vnet_rewrite? */ vlib_buffer_advance (b0, -(word) _vec_len (t0->rewrite)); if (is_ip4) { u8 ip4_geneve_base_header_len = sizeof (ip4_header_t) + sizeof (udp_header_t) + GENEVE_BASE_HEADER_LENGTH; u8 ip4_geneve_header_total_len0 = ip4_geneve_base_header_len; #if SUPPORT_OPTIONS_HEADER==1 ip4_geneve_header_total_len0 += t0->options_len; #endif ASSERT (vec_len (t0->rewrite) == ip4_geneve_header_total_len0); ip4_0 = vlib_buffer_get_current (b0); /* Copy the fixed header */ copy_dst0 = (u64 *) ip4_0; copy_src0 = (u64 *) t0->rewrite; /* Copy first 32 octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ /* Last 4 octets. Hopefully gcc will be our friend */ copy_dst_last0 = (u32 *) (©_dst0[4]); copy_src_last0 = (u32 *) (©_src0[4]); copy_dst_last0[0] = copy_src_last0[0]; /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; new_l0 = /* old_l0 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)); sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, length /* changed member */ ); ip4_0->checksum = ip_csum_fold (sum0); ip4_0->length = new_l0; /* Fix UDP length and set source port */ udp0 = (udp_header_t *) (ip4_0 + 1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip4_0)); udp0->length = new_l0; udp0->src_port = flow_hash0; } else /* ip6 path */ { int bogus = 0; u8 ip6_geneve_base_header_len = sizeof (ip6_header_t) + sizeof (udp_header_t) + GENEVE_BASE_HEADER_LENGTH; u8 ip6_geneve_header_total_len0 = ip6_geneve_base_header_len; #if SUPPORT_OPTIONS_HEADER==1 ip6_geneve_header_total_len0 += t0->options_len; #endif ASSERT (vec_len (t0->rewrite) == ip6_geneve_header_total_len0); ip6_0 = vlib_buffer_get_current (b0); /* Copy the fixed header */ copy_dst0 = (u64 *) ip6_0; copy_src0 = (u64 *) t0->rewrite; /* Copy first 56 (ip6) octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof (*ip6_0)); ip6_0->payload_length = new_l0; /* Fix UDP length and set source port */ udp0 = (udp_header_t *) (ip6_0 + 1); udp0->length = new_l0; udp0->src_port = flow_hash0; /* IPv6 UDP checksum is mandatory */ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b0, ip6_0, &bogus); ASSERT (bogus == 0); if (udp0->checksum == 0) udp0->checksum = 0xffff; } pkts_encapsulated++; len0 = vlib_buffer_length_in_chain (vm, b0); stats_n_packets += 1; stats_n_bytes += len0; /* Batch stats increment on the same geneve tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile to check for this rare case and affect normal path performance. */ if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index)) { stats_n_packets -= 1; stats_n_bytes -= len0; if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; } if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) { geneve_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - vxm->tunnels; tr->vni = t0->vni; } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } /* Do we still need this now that tunnel tx stats is kept? */ vlib_node_increment_counter (vm, node->node_index, GENEVE_ENCAP_ERROR_ENCAPSULATED, pkts_encapsulated); /* Increment any remaining batch stats */ if (stats_n_packets) { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } return from_frame->n_vectors; }
static uword startup_config_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f) { unix_main_t *um = &unix_main; u8 *buf = 0; uword l, n = 1; vlib_process_suspend (vm, 2.0); while (um->unix_config_complete == 0) vlib_process_suspend (vm, 0.1); if (um->startup_config_filename) { unformat_input_t sub_input; int fd; struct stat s; char *fn = (char *) um->startup_config_filename; fd = open (fn, O_RDONLY); if (fd < 0) { clib_warning ("failed to open `%s'", fn); return 0; } if (fstat (fd, &s) < 0) { clib_warning ("failed to stat `%s'", fn); bail: close (fd); return 0; } if (!(S_ISREG (s.st_mode) || S_ISLNK (s.st_mode))) { clib_warning ("not a regular file: `%s'", fn); goto bail; } while (n > 0) { l = vec_len (buf); vec_resize (buf, 4096); n = read (fd, buf + l, 4096); if (n > 0) { _vec_len (buf) = l + n; if (n < 4096) break; } else break; } if (um->log_fd && vec_len (buf)) { u8 *lv = 0; lv = format (lv, "%U: ***** Startup Config *****\n%v", format_timeval, 0 /* current bat-time */ , 0 /* current bat-format */ , buf); { int rv __attribute__ ((unused)) = write (um->log_fd, lv, vec_len (lv)); } vec_reset_length (lv); lv = format (lv, "%U: ***** End Startup Config *****\n", format_timeval, 0 /* current bat-time */ , 0 /* current bat-format */ ); { int rv __attribute__ ((unused)) = write (um->log_fd, lv, vec_len (lv)); } vec_free (lv); } if (vec_len (buf)) { unformat_init_vector (&sub_input, buf); vlib_cli_input (vm, &sub_input, 0, 0); /* frees buf for us */ unformat_free (&sub_input); } close (fd); } return 0; }
always_inline uword gtpu_encap_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame, u32 is_ip4) { u32 n_left_from, next_index, * from, * to_next; gtpu_main_t * gtm = >pu_main; vnet_main_t * vnm = gtm->vnet_main; vnet_interface_main_t * im = &vnm->interface_main; u32 pkts_encapsulated = 0; u16 old_l0 = 0, old_l1 = 0, old_l2 = 0, old_l3 = 0; u32 thread_index = vlib_get_thread_index(); u32 stats_sw_if_index, stats_n_packets, stats_n_bytes; u32 sw_if_index0 = 0, sw_if_index1 = 0, sw_if_index2 = 0, sw_if_index3 = 0; u32 next0 = 0, next1 = 0, next2 = 0, next3 = 0; vnet_hw_interface_t * hi0, * hi1, * hi2, * hi3; gtpu_tunnel_t * t0 = NULL, * t1 = NULL, * t2 = NULL, * t3 = NULL; from = vlib_frame_vector_args (from_frame); n_left_from = from_frame->n_vectors; next_index = node->cached_next_index; stats_sw_if_index = node->runtime_data[0]; stats_n_packets = stats_n_bytes = 0; while (n_left_from > 0) { u32 n_left_to_next; vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); while (n_left_from >= 8 && n_left_to_next >= 4) { u32 bi0, bi1, bi2, bi3; vlib_buffer_t * b0, * b1, * b2, * b3; u32 flow_hash0, flow_hash1, flow_hash2, flow_hash3; u32 len0, len1, len2, len3; ip4_header_t * ip4_0, * ip4_1, * ip4_2, * ip4_3; ip6_header_t * ip6_0, * ip6_1, * ip6_2, * ip6_3; udp_header_t * udp0, * udp1, * udp2, * udp3; gtpu_header_t * gtpu0, * gtpu1, * gtpu2, * gtpu3; u64 * copy_src0, * copy_dst0; u64 * copy_src1, * copy_dst1; u64 * copy_src2, * copy_dst2; u64 * copy_src3, * copy_dst3; u32 * copy_src_last0, * copy_dst_last0; u32 * copy_src_last1, * copy_dst_last1; u32 * copy_src_last2, * copy_dst_last2; u32 * copy_src_last3, * copy_dst_last3; u16 new_l0, new_l1, new_l2, new_l3; ip_csum_t sum0, sum1, sum2, sum3; /* Prefetch next iteration. */ { vlib_buffer_t * p4, * p5, * p6, * p7; p4 = vlib_get_buffer (vm, from[4]); p5 = vlib_get_buffer (vm, from[5]); p6 = vlib_get_buffer (vm, from[6]); p7 = vlib_get_buffer (vm, from[7]); vlib_prefetch_buffer_header (p4, LOAD); vlib_prefetch_buffer_header (p5, LOAD); vlib_prefetch_buffer_header (p6, LOAD); vlib_prefetch_buffer_header (p7, LOAD); CLIB_PREFETCH (p4->data, 2*CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (p5->data, 2*CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (p6->data, 2*CLIB_CACHE_LINE_BYTES, LOAD); CLIB_PREFETCH (p7->data, 2*CLIB_CACHE_LINE_BYTES, LOAD); } bi0 = from[0]; bi1 = from[1]; bi2 = from[2]; bi3 = from[3]; to_next[0] = bi0; to_next[1] = bi1; to_next[2] = bi2; to_next[3] = bi3; from += 4; to_next += 4; n_left_to_next -= 4; n_left_from -= 4; b0 = vlib_get_buffer (vm, bi0); b1 = vlib_get_buffer (vm, bi1); b2 = vlib_get_buffer (vm, bi2); b3 = vlib_get_buffer (vm, bi3); flow_hash0 = vnet_l2_compute_flow_hash (b0); flow_hash1 = vnet_l2_compute_flow_hash (b1); flow_hash2 = vnet_l2_compute_flow_hash (b2); flow_hash3 = vnet_l2_compute_flow_hash (b3); /* Get next node index and adj index from tunnel next_dpo */ sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX]; sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_TX]; sw_if_index2 = vnet_buffer(b2)->sw_if_index[VLIB_TX]; sw_if_index3 = vnet_buffer(b3)->sw_if_index[VLIB_TX]; hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); hi1 = vnet_get_sup_hw_interface (vnm, sw_if_index1); hi2 = vnet_get_sup_hw_interface (vnm, sw_if_index2); hi3 = vnet_get_sup_hw_interface (vnm, sw_if_index3); t0 = >m->tunnels[hi0->dev_instance]; t1 = >m->tunnels[hi1->dev_instance]; t2 = >m->tunnels[hi2->dev_instance]; t3 = >m->tunnels[hi3->dev_instance]; /* Note: change to always set next0 if it may be set to drop */ next0 = t0->next_dpo.dpoi_next_node; vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index; next1 = t1->next_dpo.dpoi_next_node; vnet_buffer(b1)->ip.adj_index[VLIB_TX] = t1->next_dpo.dpoi_index; next2 = t2->next_dpo.dpoi_next_node; vnet_buffer(b2)->ip.adj_index[VLIB_TX] = t2->next_dpo.dpoi_index; next3 = t3->next_dpo.dpoi_next_node; vnet_buffer(b3)->ip.adj_index[VLIB_TX] = t3->next_dpo.dpoi_index; /* Apply the rewrite string. $$$$ vnet_rewrite? */ vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite)); vlib_buffer_advance (b1, -(word)_vec_len(t1->rewrite)); vlib_buffer_advance (b2, -(word)_vec_len(t2->rewrite)); vlib_buffer_advance (b3, -(word)_vec_len(t3->rewrite)); if (is_ip4) { ip4_0 = vlib_buffer_get_current(b0); ip4_1 = vlib_buffer_get_current(b1); ip4_2 = vlib_buffer_get_current(b2); ip4_3 = vlib_buffer_get_current(b3); /* Copy the fixed header */ copy_dst0 = (u64 *) ip4_0; copy_src0 = (u64 *) t0->rewrite; copy_dst1 = (u64 *) ip4_1; copy_src1 = (u64 *) t1->rewrite; copy_dst2 = (u64 *) ip4_2; copy_src2 = (u64 *) t2->rewrite; copy_dst3 = (u64 *) ip4_3; copy_src3 = (u64 *) t3->rewrite; /* Copy first 32 octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ #define _(offs) copy_dst1[offs] = copy_src1[offs]; foreach_fixed_header4_offset; #undef _ #define _(offs) copy_dst2[offs] = copy_src2[offs]; foreach_fixed_header4_offset; #undef _ #define _(offs) copy_dst3[offs] = copy_src3[offs]; foreach_fixed_header4_offset; #undef _ /* Last 4 octets. Hopefully gcc will be our friend */ copy_dst_last0 = (u32 *)(©_dst0[4]); copy_src_last0 = (u32 *)(©_src0[4]); copy_dst_last0[0] = copy_src_last0[0]; copy_dst_last1 = (u32 *)(©_dst1[4]); copy_src_last1 = (u32 *)(©_src1[4]); copy_dst_last1[0] = copy_src_last1[0]; copy_dst_last2 = (u32 *)(©_dst2[4]); copy_src_last2 = (u32 *)(©_src2[4]); copy_dst_last2[0] = copy_src_last2[0]; copy_dst_last3 = (u32 *)(©_dst3[4]); copy_src_last3 = (u32 *)(©_src3[4]); copy_dst_last3[0] = copy_src_last3[0]; /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; new_l0 = /* old_l0 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)); sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, length /* changed member */); ip4_0->checksum = ip_csum_fold (sum0); ip4_0->length = new_l0; sum1 = ip4_1->checksum; new_l1 = /* old_l1 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1)); sum1 = ip_csum_update (sum1, old_l1, new_l1, ip4_header_t, length /* changed member */); ip4_1->checksum = ip_csum_fold (sum1); ip4_1->length = new_l1; sum2 = ip4_2->checksum; new_l2 = /* old_l0 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b2)); sum2 = ip_csum_update (sum2, old_l2, new_l2, ip4_header_t, length /* changed member */); ip4_2->checksum = ip_csum_fold (sum2); ip4_2->length = new_l2; sum3 = ip4_3->checksum; new_l3 = /* old_l1 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b3)); sum3 = ip_csum_update (sum3, old_l3, new_l3, ip4_header_t, length /* changed member */); ip4_3->checksum = ip_csum_fold (sum3); ip4_3->length = new_l3; /* Fix UDP length and set source port */ udp0 = (udp_header_t *)(ip4_0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip4_0)); udp0->length = new_l0; udp0->src_port = flow_hash0; udp1 = (udp_header_t *)(ip4_1+1); new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1) - sizeof (*ip4_1)); udp1->length = new_l1; udp1->src_port = flow_hash1; udp2 = (udp_header_t *)(ip4_2+1); new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2) - sizeof (*ip4_2)); udp2->length = new_l2; udp2->src_port = flow_hash2; udp3 = (udp_header_t *)(ip4_3+1); new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3) - sizeof (*ip4_3)); udp3->length = new_l3; udp3->src_port = flow_hash3; /* Fix GTPU length */ gtpu0 = (gtpu_header_t *)(udp0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip4_0) - sizeof(*udp0) - GTPU_V1_HDR_LEN); gtpu0->length = new_l0; gtpu1 = (gtpu_header_t *)(udp1+1); new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1) - sizeof (*ip4_1) - sizeof(*udp1) - GTPU_V1_HDR_LEN); gtpu1->length = new_l1; gtpu2 = (gtpu_header_t *)(udp2+1); new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2) - sizeof (*ip4_2) - sizeof(*udp2) - GTPU_V1_HDR_LEN); gtpu2->length = new_l2; gtpu3 = (gtpu_header_t *)(udp3+1); new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3) - sizeof (*ip4_3) - sizeof(*udp3) - GTPU_V1_HDR_LEN); gtpu3->length = new_l3; } else /* ipv6 */ { int bogus = 0; ip6_0 = vlib_buffer_get_current(b0); ip6_1 = vlib_buffer_get_current(b1); ip6_2 = vlib_buffer_get_current(b2); ip6_3 = vlib_buffer_get_current(b3); /* Copy the fixed header */ copy_dst0 = (u64 *) ip6_0; copy_src0 = (u64 *) t0->rewrite; copy_dst1 = (u64 *) ip6_1; copy_src1 = (u64 *) t1->rewrite; copy_dst2 = (u64 *) ip6_2; copy_src2 = (u64 *) t2->rewrite; copy_dst3 = (u64 *) ip6_3; copy_src3 = (u64 *) t3->rewrite; /* Copy first 56 (ip6) octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ #define _(offs) copy_dst1[offs] = copy_src1[offs]; foreach_fixed_header6_offset; #undef _ #define _(offs) copy_dst2[offs] = copy_src2[offs]; foreach_fixed_header6_offset; #undef _ #define _(offs) copy_dst3[offs] = copy_src3[offs]; foreach_fixed_header6_offset; #undef _ /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof(*ip6_0)); ip6_0->payload_length = new_l0; new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b1) - sizeof(*ip6_1)); ip6_1->payload_length = new_l1; new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b2) - sizeof(*ip6_2)); ip6_2->payload_length = new_l2; new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b3) - sizeof(*ip6_3)); ip6_3->payload_length = new_l3; /* Fix UDP length and set source port */ udp0 = (udp_header_t *)(ip6_0+1); udp0->length = new_l0; udp0->src_port = flow_hash0; udp1 = (udp_header_t *)(ip6_1+1); udp1->length = new_l1; udp1->src_port = flow_hash1; udp2 = (udp_header_t *)(ip6_2+1); udp2->length = new_l2; udp2->src_port = flow_hash2; udp3 = (udp_header_t *)(ip6_3+1); udp3->length = new_l3; udp3->src_port = flow_hash3; /* Fix GTPU length */ gtpu0 = (gtpu_header_t *)(udp0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip6_0) - sizeof(*udp0) - GTPU_V1_HDR_LEN); gtpu0->length = new_l0; gtpu1 = (gtpu_header_t *)(udp1+1); new_l1 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b1) - sizeof (*ip6_1) - sizeof(*udp1) - GTPU_V1_HDR_LEN); gtpu1->length = new_l1; gtpu2 = (gtpu_header_t *)(udp2+1); new_l2 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b2) - sizeof (*ip6_2) - sizeof(*udp2) - GTPU_V1_HDR_LEN); gtpu2->length = new_l2; gtpu3 = (gtpu_header_t *)(udp3+1); new_l3 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b3) - sizeof (*ip6_3) - sizeof(*udp3) - GTPU_V1_HDR_LEN); gtpu3->length = new_l3; /* IPv6 UDP checksum is mandatory */ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0, ip6_0, &bogus); if (udp0->checksum == 0) udp0->checksum = 0xffff; udp1->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b1, ip6_1, &bogus); if (udp1->checksum == 0) udp1->checksum = 0xffff; udp2->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b2, ip6_2, &bogus); if (udp2->checksum == 0) udp2->checksum = 0xffff; udp3->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b3, ip6_3, &bogus); if (udp3->checksum == 0) udp3->checksum = 0xffff; } pkts_encapsulated += 4; len0 = vlib_buffer_length_in_chain (vm, b0); len1 = vlib_buffer_length_in_chain (vm, b1); len2 = vlib_buffer_length_in_chain (vm, b2); len3 = vlib_buffer_length_in_chain (vm, b3); stats_n_packets += 4; stats_n_bytes += len0 + len1 + len2 + len3; /* Batch stats increment on the same gtpu tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile to check for this rare case and affect normal path performance. */ if (PREDICT_FALSE ((sw_if_index0 != stats_sw_if_index) || (sw_if_index1 != stats_sw_if_index) || (sw_if_index2 != stats_sw_if_index) || (sw_if_index3 != stats_sw_if_index) )) { stats_n_packets -= 4; stats_n_bytes -= len0 + len1 + len2 + len3; if ( (sw_if_index0 == sw_if_index1 ) && (sw_if_index1 == sw_if_index2 ) && (sw_if_index2 == sw_if_index3 ) ) { if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_sw_if_index = sw_if_index0; stats_n_packets = 4; stats_n_bytes = len0 + len1 + len2 + len3; } else { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index0, 1, len0); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index1, 1, len1); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index2, 1, len2); vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, sw_if_index3, 1, len3); } } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - gtm->tunnels; tr->teid = t0->teid; } if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED)) { gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b1, sizeof (*tr)); tr->tunnel_index = t1 - gtm->tunnels; tr->teid = t1->teid; } vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, next0, next1, next2, next3); } while (n_left_from > 0 && n_left_to_next > 0) { u32 bi0; vlib_buffer_t * b0; u32 flow_hash0; u32 len0; ip4_header_t * ip4_0; ip6_header_t * ip6_0; udp_header_t * udp0; gtpu_header_t * gtpu0; u64 * copy_src0, * copy_dst0; u32 * copy_src_last0, * copy_dst_last0; u16 new_l0; ip_csum_t sum0; bi0 = from[0]; to_next[0] = bi0; from += 1; to_next += 1; n_left_from -= 1; n_left_to_next -= 1; b0 = vlib_get_buffer (vm, bi0); flow_hash0 = vnet_l2_compute_flow_hash(b0); /* Get next node index and adj index from tunnel next_dpo */ sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_TX]; hi0 = vnet_get_sup_hw_interface (vnm, sw_if_index0); t0 = >m->tunnels[hi0->dev_instance]; /* Note: change to always set next0 if it may be set to drop */ next0 = t0->next_dpo.dpoi_next_node; vnet_buffer(b0)->ip.adj_index[VLIB_TX] = t0->next_dpo.dpoi_index; /* Apply the rewrite string. $$$$ vnet_rewrite? */ vlib_buffer_advance (b0, -(word)_vec_len(t0->rewrite)); if (is_ip4) { ip4_0 = vlib_buffer_get_current(b0); /* Copy the fixed header */ copy_dst0 = (u64 *) ip4_0; copy_src0 = (u64 *) t0->rewrite; /* Copy first 32 octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header4_offset; #undef _ /* Last 4 octets. Hopefully gcc will be our friend */ copy_dst_last0 = (u32 *)(©_dst0[4]); copy_src_last0 = (u32 *)(©_src0[4]); copy_dst_last0[0] = copy_src_last0[0]; /* Fix the IP4 checksum and length */ sum0 = ip4_0->checksum; new_l0 = /* old_l0 always 0, see the rewrite setup */ clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0)); sum0 = ip_csum_update (sum0, old_l0, new_l0, ip4_header_t, length /* changed member */); ip4_0->checksum = ip_csum_fold (sum0); ip4_0->length = new_l0; /* Fix UDP length and set source port */ udp0 = (udp_header_t *)(ip4_0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip4_0)); udp0->length = new_l0; udp0->src_port = flow_hash0; /* Fix GTPU length */ gtpu0 = (gtpu_header_t *)(udp0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip4_0) - sizeof(*udp0) - GTPU_V1_HDR_LEN); gtpu0->length = new_l0; } else /* ip6 path */ { int bogus = 0; ip6_0 = vlib_buffer_get_current(b0); /* Copy the fixed header */ copy_dst0 = (u64 *) ip6_0; copy_src0 = (u64 *) t0->rewrite; /* Copy first 56 (ip6) octets 8-bytes at a time */ #define _(offs) copy_dst0[offs] = copy_src0[offs]; foreach_fixed_header6_offset; #undef _ /* Fix IP6 payload length */ new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain (vm, b0) - sizeof(*ip6_0)); ip6_0->payload_length = new_l0; /* Fix UDP length and set source port */ udp0 = (udp_header_t *)(ip6_0+1); udp0->length = new_l0; udp0->src_port = flow_hash0; /* Fix GTPU length */ gtpu0 = (gtpu_header_t *)(udp0+1); new_l0 = clib_host_to_net_u16 (vlib_buffer_length_in_chain(vm, b0) - sizeof (*ip4_0) - sizeof(*udp0) - GTPU_V1_HDR_LEN); gtpu0->length = new_l0; /* IPv6 UDP checksum is mandatory */ udp0->checksum = ip6_tcp_udp_icmp_compute_checksum(vm, b0, ip6_0, &bogus); if (udp0->checksum == 0) udp0->checksum = 0xffff; } pkts_encapsulated ++; len0 = vlib_buffer_length_in_chain (vm, b0); stats_n_packets += 1; stats_n_bytes += len0; /* Batch stats increment on the same gtpu tunnel so counter is not incremented per packet. Note stats are still incremented for deleted and admin-down tunnel where packets are dropped. It is not worthwhile to check for this rare case and affect normal path performance. */ if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index)) { stats_n_packets -= 1; stats_n_bytes -= len0; if (stats_n_packets) vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); stats_n_packets = 1; stats_n_bytes = len0; stats_sw_if_index = sw_if_index0; } if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED)) { gtpu_encap_trace_t *tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); tr->tunnel_index = t0 - gtm->tunnels; tr->teid = t0->teid; } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, n_left_to_next, bi0, next0); } vlib_put_next_frame (vm, node, next_index, n_left_to_next); } /* Do we still need this now that tunnel tx stats is kept? */ vlib_node_increment_counter (vm, node->node_index, GTPU_ENCAP_ERROR_ENCAPSULATED, pkts_encapsulated); /* Increment any remaining batch stats */ if (stats_n_packets) { vlib_increment_combined_counter (im->combined_sw_if_counters + VNET_INTERFACE_COUNTER_TX, thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes); node->runtime_data[0] = stats_sw_if_index; } return from_frame->n_vectors; }
int test_socket_main (unformat_input_t * input) { clib_socket_t _s = {0}, * s = &_s; char * config; clib_error_t * error; s->config = "localhost:22"; s->flags = SOCKET_IS_CLIENT; while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) { if (unformat (input, "server %s %=", &config, &s->flags, SOCKET_IS_SERVER)) ; else if (unformat (input, "client %s %=", &config, &s->flags, SOCKET_IS_CLIENT)) ; else { error = clib_error_create ("unknown input `%U'\n", format_unformat_error, input); goto done; } } error = clib_socket_init (s); if (error) goto done; if (0) { struct { int a, b; } * msg; msg = clib_socket_tx_add (s, sizeof (msg[0])); msg->a = 99; msg->b = 100; } else clib_socket_tx_add_formatted (s, "hello there mr server %d\n", 99); error = clib_socket_tx (s); if (error) goto done; while (1) { error = clib_socket_rx (s, 100); if (error) break; if (clib_socket_rx_end_of_file (s)) break; if_verbose ("%v", s->rx_buffer); _vec_len (s->rx_buffer) = 0; } error = clib_socket_close (s); done: if (error) clib_error_report (error); return 0; }
/* Given next hop vector is over-written with normalized one with sorted weights and with weights corresponding to the number of adjacencies for each next hop. Returns number of adjacencies in block. */ static u32 ip_multipath_normalize_next_hops (ip_lookup_main_t * lm, ip_multipath_next_hop_t * raw_next_hops, ip_multipath_next_hop_t ** normalized_next_hops) { ip_multipath_next_hop_t * nhs; uword n_nhs, n_adj, n_adj_left, i; f64 sum_weight, norm, error; n_nhs = vec_len (raw_next_hops); ASSERT (n_nhs > 0); if (n_nhs == 0) return 0; /* Allocate enough space for 2 copies; we'll use second copy to save original weights. */ nhs = *normalized_next_hops; vec_validate (nhs, 2*n_nhs - 1); /* Fast path: 1 next hop in block. */ n_adj = n_nhs; if (n_nhs == 1) { nhs[0] = raw_next_hops[0]; nhs[0].weight = 1; _vec_len (nhs) = 1; goto done; } else if (n_nhs == 2) { int cmp = next_hop_sort_by_weight (&raw_next_hops[0], &raw_next_hops[1]) < 0; /* Fast sort. */ nhs[0] = raw_next_hops[cmp]; nhs[1] = raw_next_hops[cmp ^ 1]; /* Fast path: equal cost multipath with 2 next hops. */ if (nhs[0].weight == nhs[1].weight) { nhs[0].weight = nhs[1].weight = 1; _vec_len (nhs) = 2; goto done; } } else { memcpy (nhs, raw_next_hops, n_nhs * sizeof (raw_next_hops[0])); qsort (nhs, n_nhs, sizeof (nhs[0]), (void *) next_hop_sort_by_weight); } /* Find total weight to normalize weights. */ sum_weight = 0; for (i = 0; i < n_nhs; i++) sum_weight += nhs[i].weight; /* In the unlikely case that all weights are given as 0, set them all to 1. */ if (sum_weight == 0) { for (i = 0; i < n_nhs; i++) nhs[i].weight = 1; sum_weight = n_nhs; } /* Save copies of all next hop weights to avoid being overwritten in loop below. */ for (i = 0; i < n_nhs; i++) nhs[n_nhs + i].weight = nhs[i].weight; /* Try larger and larger power of 2 sized adjacency blocks until we find one where traffic flows to within 1% of specified weights. */ for (n_adj = max_pow2 (n_nhs); ; n_adj *= 2) { error = 0; norm = n_adj / sum_weight; n_adj_left = n_adj; for (i = 0; i < n_nhs; i++) { f64 nf = nhs[n_nhs + i].weight * norm; /* use saved weights */ word n = flt_round_nearest (nf); n = n > n_adj_left ? n_adj_left : n; n_adj_left -= n; error += fabs (nf - n); nhs[i].weight = n; } nhs[0].weight += n_adj_left; /* Less than 5% average error per adjacency with this size adjacency block? */ if (error <= lm->multipath_next_hop_error_tolerance*n_adj) { /* Truncate any next hops with zero weight. */ _vec_len (nhs) = i; break; } } done: /* Save vector for next call. */ *normalized_next_hops = nhs; return n_adj; }
uword ssvm_eth_interface_tx (ssvm_private_t * intfc, char *buf_to_send, int len_to_send) // , // vlib_frame_t * f) { ssvm_eth_main_t * em = &ssvm_eth_main; ssvm_shared_header_t * sh = intfc->sh; unix_shared_memory_queue_t * q; u32 * from; u32 n_left; ssvm_eth_queue_elt_t * elts, * elt, * prev_elt; u32 my_pid = intfc->my_pid; vlib_buffer_t * b0; u32 bi0; u32 size_this_buffer; u32 chunks_this_buffer; u8 i_am_master = intfc->i_am_master; u32 elt_index; int is_ring_full, interface_down; int i; volatile u32 *queue_lock; u32 n_to_alloc = VLIB_FRAME_SIZE; u32 n_allocated, n_present_in_cache, n_available; u32 * elt_indices; if (i_am_master) q = (unix_shared_memory_queue_t *)sh->opaque [TO_SLAVE_Q_INDEX]; else q = (unix_shared_memory_queue_t *)sh->opaque [TO_MASTER_Q_INDEX]; queue_lock = (u32 *) q; // from = vlib_frame_vector_args (f); //n_left = f->n_vectors; n_left = 1; is_ring_full = 0; interface_down = 0; n_present_in_cache = vec_len (em->chunk_cache); #ifdef XXX /* admin / link up/down check */ if (sh->opaque [MASTER_ADMIN_STATE_INDEX] == 0 || sh->opaque [SLAVE_ADMIN_STATE_INDEX] == 0) { interface_down = 1; goto out; } #endif ssvm_lock (sh, my_pid, 1); elts = (ssvm_eth_queue_elt_t *) (sh->opaque [CHUNK_POOL_INDEX]); elt_indices = (u32 *) (sh->opaque [CHUNK_POOL_FREELIST_INDEX]); n_available = (u32) pointer_to_uword(sh->opaque [CHUNK_POOL_NFREE]); printf("AYXX: n_left: %d, n_present_in_cache: %d\n", n_left, n_present_in_cache); if (n_present_in_cache < n_left*2) { vec_validate (em->chunk_cache, n_to_alloc + n_present_in_cache - 1); n_allocated = n_to_alloc < n_available ? n_to_alloc : n_available; printf("AYXX: n_allocated: %d, n_to_alloc: %d, n_available: %d\n", n_allocated, n_to_alloc, n_available); if (PREDICT_TRUE(n_allocated > 0)) { memcpy (&em->chunk_cache[n_present_in_cache], &elt_indices[n_available - n_allocated], sizeof(u32) * n_allocated); } n_present_in_cache += n_allocated; n_available -= n_allocated; sh->opaque [CHUNK_POOL_NFREE] = uword_to_pointer(n_available, void*); _vec_len (em->chunk_cache) = n_present_in_cache; }
static clib_error_t * recvmsg_helper (mc_socket_main_t * msm, int socket, struct sockaddr_in * rx_addr, u32 * buffer_index, u32 drop_message) { vlib_main_t * vm = msm->mc_main.vlib_main; vlib_buffer_t * b; uword n_left, n_alloc, n_mtu, i, i_rx; const uword buffer_size = VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES; word n_bytes_left; /* Make sure we have at least a MTU worth of buffers. */ n_mtu = msm->rx_mtu_n_buffers; n_left = vec_len (msm->rx_buffers); if (n_left < n_mtu) { uword max_alloc = 8 * n_mtu; vec_validate (msm->rx_buffers, max_alloc - 1); n_alloc = vlib_buffer_alloc (vm, msm->rx_buffers + n_left, max_alloc - n_left); _vec_len (msm->rx_buffers) = n_left + n_alloc; } ASSERT (vec_len (msm->rx_buffers) >= n_mtu); vec_validate (msm->iovecs, n_mtu - 1); /* Allocate RX buffers from end of rx_buffers. Turn them into iovecs to pass to readv. */ i_rx = vec_len (msm->rx_buffers) - 1; for (i = 0; i < n_mtu; i++) { b = vlib_get_buffer (vm, msm->rx_buffers[i_rx - i]); msm->iovecs[i].iov_base = b->data; msm->iovecs[i].iov_len = buffer_size; } _vec_len (msm->iovecs) = n_mtu; { struct msghdr h; memset (&h, 0, sizeof (h)); if (rx_addr) { h.msg_name = rx_addr; h.msg_namelen = sizeof (rx_addr[0]); } h.msg_iov = msm->iovecs; h.msg_iovlen = vec_len (msm->iovecs); n_bytes_left = recvmsg (socket, &h, 0); if (n_bytes_left < 0) return clib_error_return_unix (0, "recvmsg"); } if (drop_message) { *buffer_index = ~0; return 0; } *buffer_index = msm->rx_buffers[i_rx]; while (1) { b = vlib_get_buffer (vm, msm->rx_buffers[i_rx]); b->flags = 0; b->current_data = 0; b->current_length = n_bytes_left < buffer_size ? n_bytes_left : buffer_size; n_bytes_left -= buffer_size; if (n_bytes_left <= 0) break; i_rx--; b->flags |= VLIB_BUFFER_NEXT_PRESENT; b->next_buffer = msm->rx_buffers[i_rx]; } _vec_len (msm->rx_buffers) = i_rx; return 0 /* no error */; }
static void vl_msg_api_process_file (vlib_main_t * vm, u8 * filename, u32 first_index, u32 last_index, vl_api_replay_t which) { vl_api_trace_file_header_t *hp; int i, fd; struct stat statb; size_t file_size; u8 *msg; u8 endian_swap_needed = 0; api_main_t *am = &api_main; static u8 *tmpbuf; u32 nitems; void **saved_print_handlers = 0; fd = open ((char *) filename, O_RDONLY); if (fd < 0) { vlib_cli_output (vm, "Couldn't open %s\n", filename); return; } if (fstat (fd, &statb) < 0) { vlib_cli_output (vm, "Couldn't stat %s\n", filename); return; } if (!(statb.st_mode & S_IFREG) || (statb.st_size < sizeof (*hp))) { vlib_cli_output (vm, "File not plausible: %s\n", filename); close(fd); return; } file_size = statb.st_size; file_size = (file_size + 4095) & ~(4096); hp = mmap (0, file_size, PROT_READ, MAP_PRIVATE, fd, 0); if (hp == (vl_api_trace_file_header_t *) MAP_FAILED) { vlib_cli_output (vm, "mmap failed: %s\n", filename); close (fd); return; } close (fd); if ((clib_arch_is_little_endian && hp->endian == VL_API_BIG_ENDIAN) || (clib_arch_is_big_endian && hp->endian == VL_API_LITTLE_ENDIAN)) endian_swap_needed = 1; if (endian_swap_needed) nitems = ntohl (hp->nitems); else nitems = hp->nitems; if (last_index == (u32) ~ 0) { last_index = nitems - 1; } if (first_index >= nitems || last_index >= nitems) { vlib_cli_output (vm, "Range (%d, %d) outside file range (0, %d)\n", first_index, last_index, nitems - 1); munmap (hp, file_size); return; } if (hp->wrapped) vlib_cli_output (vm, "Note: wrapped/incomplete trace, results may vary\n"); if (which == CUSTOM_DUMP) { saved_print_handlers = (void **) vec_dup (am->msg_print_handlers); vl_msg_api_custom_dump_configure (am); } msg = (u8 *) (hp + 1); for (i = 0; i < first_index; i++) { trace_cfg_t *cfgp; int size; u16 msg_id; if (clib_arch_is_little_endian) msg_id = ntohs (*((u16 *) msg)); else msg_id = *((u16 *) msg); cfgp = am->api_trace_cfg + msg_id; if (!cfgp) { vlib_cli_output (vm, "Ugh: msg id %d no trace config\n", msg_id); return; } size = cfgp->size; msg += size; } for (; i <= last_index; i++) { trace_cfg_t *cfgp; u16 *msg_idp; u16 msg_id; int size; if (which == DUMP) vlib_cli_output (vm, "---------- trace %d -----------\n", i); if (clib_arch_is_little_endian) msg_id = ntohs (*((u16 *) msg)); else msg_id = *((u16 *) msg); cfgp = am->api_trace_cfg + msg_id; if (!cfgp) { vlib_cli_output (vm, "Ugh: msg id %d no trace config\n", msg_id); return; } size = cfgp->size; /* Copy the buffer (from the read-only mmap'ed file) */ vec_validate (tmpbuf, size - 1 + sizeof (uword)); clib_memcpy (tmpbuf + sizeof (uword), msg, size); memset (tmpbuf, 0xf, sizeof (uword)); /* * Endian swap if needed. All msg data is supposed to be * in network byte order. All msg handlers are supposed to * know that. The generic message dumpers don't know that. * One could fix apigen, I suppose. */ if ((which == DUMP && clib_arch_is_little_endian) || endian_swap_needed) { void (*endian_fp) (void *); if (msg_id >= vec_len (am->msg_endian_handlers) || (am->msg_endian_handlers[msg_id] == 0)) { vlib_cli_output (vm, "Ugh: msg id %d no endian swap\n", msg_id); return; } endian_fp = am->msg_endian_handlers[msg_id]; (*endian_fp) (tmpbuf + sizeof (uword)); } /* msg_id always in network byte order */ if (clib_arch_is_little_endian) { msg_idp = (u16 *) (tmpbuf + sizeof (uword)); *msg_idp = msg_id; } switch (which) { case CUSTOM_DUMP: case DUMP: if (msg_id < vec_len (am->msg_print_handlers) && am->msg_print_handlers[msg_id]) { u8 *(*print_fp) (void *, void *); print_fp = (void *) am->msg_print_handlers[msg_id]; (*print_fp) (tmpbuf + sizeof (uword), vm); } else { vlib_cli_output (vm, "Skipping msg id %d: no print fcn\n", msg_id); break; } break; case INITIALIZERS: if (msg_id < vec_len (am->msg_print_handlers) && am->msg_print_handlers[msg_id]) { u8 *s; int j; u8 *(*print_fp) (void *, void *); print_fp = (void *) am->msg_print_handlers[msg_id]; vlib_cli_output (vm, "/*"); (*print_fp) (tmpbuf + sizeof (uword), vm); vlib_cli_output (vm, "*/\n"); s = format (0, "static u8 * vl_api_%s_%d[%d] = {", am->msg_names[msg_id], i, am->api_trace_cfg[msg_id].size); for (j = 0; j < am->api_trace_cfg[msg_id].size; j++) { if ((j & 7) == 0) s = format (s, "\n "); s = format (s, "0x%02x,", tmpbuf[sizeof (uword) + j]); } s = format (s, "\n};\n%c", 0); vlib_cli_output (vm, (char *) s); vec_free (s); } break; case REPLAY: if (msg_id < vec_len (am->msg_print_handlers) && am->msg_print_handlers[msg_id] && cfgp->replay_enable) { void (*handler) (void *); handler = (void *) am->msg_handlers[msg_id]; if (!am->is_mp_safe[msg_id]) vl_msg_api_barrier_sync (); (*handler) (tmpbuf + sizeof (uword)); if (!am->is_mp_safe[msg_id]) vl_msg_api_barrier_release (); } else { if (cfgp->replay_enable) vlib_cli_output (vm, "Skipping msg id %d: no handler\n", msg_id); break; } break; } _vec_len (tmpbuf) = 0; msg += size; } if (saved_print_handlers) { clib_memcpy (am->msg_print_handlers, saved_print_handlers, vec_len (am->msg_print_handlers) * sizeof (void *)); vec_free (saved_print_handlers); } munmap (hp, file_size); }
int vl_map_shmem (const char *region_name, int is_vlib) { svm_map_region_args_t _a, *a = &_a; svm_region_t *vlib_rp, *root_rp; api_main_t *am = &api_main; int i; struct timespec ts, tsrem; char *vpe_api_region_suffix = "-vpe-api"; clib_memset (a, 0, sizeof (*a)); if (strstr (region_name, vpe_api_region_suffix)) { u8 *root_path = format (0, "%s", region_name); _vec_len (root_path) = (vec_len (root_path) - strlen (vpe_api_region_suffix)); vec_terminate_c_string (root_path); a->root_path = (const char *) root_path; am->root_path = (const char *) root_path; } if (is_vlib == 0) { int tfd; u8 *api_name; /* * Clients wait for vpp to set up the root / API regioins */ if (am->root_path) api_name = format (0, "/dev/shm/%s-%s%c", am->root_path, region_name + 1, 0); else api_name = format (0, "/dev/shm%s%c", region_name, 0); /* Wait up to 100 seconds... */ for (i = 0; i < 10000; i++) { ts.tv_sec = 0; ts.tv_nsec = 10000 * 1000; /* 10 ms */ while (nanosleep (&ts, &tsrem) < 0) ts = tsrem; tfd = open ((char *) api_name, O_RDWR); if (tfd >= 0) break; } vec_free (api_name); if (tfd < 0) { clib_warning ("region init fail"); return -2; } close (tfd); svm_region_init_chroot_uid_gid (am->root_path, getuid (), getgid ()); } if (a->root_path != NULL) { a->name = "/vpe-api"; } else a->name = region_name; a->size = am->api_size ? am->api_size : (16 << 20); a->flags = SVM_FLAGS_MHEAP; a->uid = am->api_uid; a->gid = am->api_gid; a->pvt_heap_size = am->api_pvt_heap_size; vlib_rp = svm_region_find_or_create (a); if (vlib_rp == 0) return (-2); pthread_mutex_lock (&vlib_rp->mutex); /* Has someone else set up the shared-memory variable table? */ if (vlib_rp->user_ctx) { am->shmem_hdr = (void *) vlib_rp->user_ctx; am->our_pid = getpid (); if (is_vlib) { svm_queue_t *q; uword old_msg; /* * application restart. Reset cached pids, API message * rings, list of clients; otherwise, various things * fail. (e.g. queue non-empty notification) */ /* ghosts keep the region from disappearing properly */ svm_client_scan_this_region_nolock (vlib_rp); am->shmem_hdr->application_restarts++; q = am->shmem_hdr->vl_input_queue; am->shmem_hdr->vl_pid = getpid (); q->consumer_pid = am->shmem_hdr->vl_pid; /* Drain the input queue, freeing msgs */ for (i = 0; i < 10; i++) { if (pthread_mutex_trylock (&q->mutex) == 0) { pthread_mutex_unlock (&q->mutex); goto mutex_ok; } ts.tv_sec = 0; ts.tv_nsec = 10000 * 1000; /* 10 ms */ while (nanosleep (&ts, &tsrem) < 0) ts = tsrem; } /* Mutex buggered, "fix" it */ clib_memset (&q->mutex, 0, sizeof (q->mutex)); clib_warning ("forcibly release main input queue mutex"); mutex_ok: am->vlib_rp = vlib_rp; while (svm_queue_sub (q, (u8 *) & old_msg, SVM_Q_NOWAIT, 0) != -2 /* queue underflow */ ) { vl_msg_api_free_nolock ((void *) old_msg); am->shmem_hdr->restart_reclaims++; } pthread_mutex_unlock (&vlib_rp->mutex); root_rp = svm_get_root_rp (); ASSERT (root_rp); /* Clean up the root region client list */ pthread_mutex_lock (&root_rp->mutex); svm_client_scan_this_region_nolock (root_rp); pthread_mutex_unlock (&root_rp->mutex); } else { pthread_mutex_unlock (&vlib_rp->mutex); } am->vlib_rp = vlib_rp; vec_add1 (am->mapped_shmem_regions, vlib_rp); return 0; } /* Clients simply have to wait... */ if (!is_vlib) { pthread_mutex_unlock (&vlib_rp->mutex); /* Wait up to 100 seconds... */ for (i = 0; i < 10000; i++) { ts.tv_sec = 0; ts.tv_nsec = 10000 * 1000; /* 10 ms */ while (nanosleep (&ts, &tsrem) < 0) ts = tsrem; if (vlib_rp->user_ctx) goto ready; } /* Clean up and leave... */ svm_region_unmap (vlib_rp); clib_warning ("region init fail"); return (-2); ready: am->shmem_hdr = (void *) vlib_rp->user_ctx; am->our_pid = getpid (); am->vlib_rp = vlib_rp; vec_add1 (am->mapped_shmem_regions, vlib_rp); return 0; } /* Nope, it's our problem... */ vl_init_shmem (vlib_rp, 0 /* default config */ , 1 /* is vlib */ , 0 /* is_private_region */ ); vec_add1 (am->mapped_shmem_regions, vlib_rp); return 0; }