int
test1 (void)
{
clib_time_t clib_time;
void *h_mem = clib_mem_alloc (2ULL << 30);
void *h;
uword *objects = 0;
int i;
f64 before, after;
clib_time_init (&clib_time);
vec_validate (objects, 2000000 - 1);
h = mheap_alloc (h_mem, (uword) (2 << 30));
before = clib_time_now (&clib_time);
for (i = 0; i < vec_len (objects); i++)
{
h = mheap_get_aligned (h, 24 /* size */ ,
64 /* align */ ,
16 /* align at offset */ , &objects[i]);
}
after = clib_time_now (&clib_time);
fformat (stdout, "alloc: %u objects in %.2f seconds, %.2f objects/second\n",
vec_len (objects), (after - before),
((f64) vec_len (objects)) / (after - before));
return 0;
}
static clib_error_t *
flowprobe_create_state_tables (u32 active_timer)
{
flowprobe_main_t *fm = &flowprobe_main;
vlib_thread_main_t *tm = &vlib_thread_main;
vlib_main_t *vm = vlib_get_main ();
clib_error_t *error = 0;
u32 num_threads;
int i;
/* Decide how many worker threads we have */
num_threads = 1 /* main thread */ + tm->n_threads;
/* Hash table per worker */
fm->ht_log2len = FLOWPROBE_LOG2_HASHSIZE;
/* Init per worker flow state and timer wheels */
if (active_timer)
{
vec_validate (fm->timers_per_worker, num_threads - 1);
vec_validate (fm->expired_passive_per_worker, num_threads - 1);
vec_validate (fm->hash_per_worker, num_threads - 1);
vec_validate (fm->pool_per_worker, num_threads - 1);
for (i = 0; i < num_threads; i++)
{
int j;
pool_alloc (fm->pool_per_worker[i], 1 << fm->ht_log2len);
vec_resize (fm->hash_per_worker[i], 1 << fm->ht_log2len);
for (j = 0; j < (1 << fm->ht_log2len); j++)
fm->hash_per_worker[i][j] = ~0;
fm->timers_per_worker[i] =
clib_mem_alloc (sizeof (TWT (tw_timer_wheel)));
tw_timer_wheel_init_2t_1w_2048sl (fm->timers_per_worker[i],
flowprobe_expired_timer_callback,
1.0, 1024);
}
fm->disabled = true;
}
else
{
f64 now = vlib_time_now (vm);
vec_validate (fm->stateless_entry, num_threads - 1);
for (i = 0; i < num_threads; i++)
fm->stateless_entry[i].last_exported = now;
fm->disabled = false;
}
fm->initialized = true;
return error;
}
static uword
dhcp6_pd_reply_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
vlib_frame_t * f)
{
/* These cross the longjmp boundary (vlib_process_wait_for_event)
* and need to be volatile - to prevent them from being optimized into
* a register - which could change during suspension */
while (1)
{
vlib_process_wait_for_event (vm);
uword event_type = DHCP6_PD_DP_REPLY_REPORT;
void *event_data = vlib_process_get_event_data (vm, &event_type);
int i;
if (event_type == DHCP6_PD_DP_REPLY_REPORT)
{
prefix_report_t *events = event_data;
for (i = 0; i < vec_len (events); i++)
{
u32 event_size =
sizeof (vl_api_dhcp6_pd_reply_event_t) +
vec_len (events[i].prefixes) *
sizeof (vl_api_dhcp6_pd_prefix_info_t);
vl_api_dhcp6_pd_reply_event_t *event =
clib_mem_alloc (event_size);
clib_memset (event, 0, event_size);
event->sw_if_index = htonl (events[i].body.sw_if_index);
event->server_index = htonl (events[i].body.server_index);
event->msg_type = events[i].body.msg_type;
event->T1 = htonl (events[i].body.T1);
event->T2 = htonl (events[i].body.T2);
event->inner_status_code =
htons (events[i].body.inner_status_code);
event->status_code = htons (events[i].body.status_code);
event->preference = events[i].body.preference;
event->n_prefixes = htonl (vec_len (events[i].prefixes));
vl_api_dhcp6_pd_prefix_info_t *prefix =
(typeof (prefix)) event->prefixes;
u32 j;
for (j = 0; j < vec_len (events[i].prefixes); j++)
{
dhcp6_prefix_info_t *info = &events[i].prefixes[j];
memcpy (prefix->prefix, &info->prefix, 16);
prefix->prefix_length = info->prefix_length;
prefix->valid_time = htonl (info->valid_time);
prefix->preferred_time = htonl (info->preferred_time);
prefix++;
}
vec_free (events[i].prefixes);
dhcp6_pd_client_public_main_t *dpcpm =
&dhcp6_pd_client_public_main;
call_dhcp6_pd_reply_event_callbacks (event, dpcpm->functions);
vpe_client_registration_t *reg;
/* *INDENT-OFF* */
pool_foreach(reg, vpe_api_main.dhcp6_pd_reply_events_registrations,
({
vl_api_registration_t *vl_reg;
vl_reg =
vl_api_client_index_to_registration (reg->client_index);
if (vl_reg && vl_api_can_send_msg (vl_reg))
{
vl_api_dhcp6_pd_reply_event_t *msg =
vl_msg_api_alloc (event_size);
clib_memcpy (msg, event, event_size);
msg->_vl_msg_id = htons (VL_API_DHCP6_PD_REPLY_EVENT);
msg->client_index = reg->client_index;
msg->pid = reg->client_pid;
vl_api_send_msg (vl_reg, (u8 *) msg);
}
}));
/* *INDENT-ON* */
clib_mem_free (event);
}
}
int main (int argc, char * argv[])
{
word i, j, k, n, seed, check_mask;
u32 * h = 0;
uword * objects = 0;
uword * handles = 0;
uword objects_used;
uword align, fixed_size;
n = 10;
seed = getpid ();
check_mask = 0;
fixed_size = 0;
if (argc > 1)
{
n = atoi (argv[1]);
verbose = 1;
}
if (argc > 2)
{
word i = atoi (argv[2]);
if (i)
seed = i;
}
if (argc > 3)
check_mask = atoi (argv[3]);
align = 0;
if (argc > 4)
align = 1 << atoi (argv[4]);
if_verbose ("testing %wd iterations seed %wd\n", n, seed);
srandom (seed);
if (verbose) fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0);
vec_resize (objects, 1000);
memset (objects, ~0, vec_bytes (objects));
vec_resize (handles, vec_len (objects));
objects_used = 0;
if (fixed_size)
{
uword max_len = 1024 * 1024;
void * memory = clib_mem_alloc (max_len * sizeof (h[0]));
h = heap_create_from_memory (memory, max_len, sizeof (h[0]));
}
for (i = 0; i < n; i++)
{
while (1)
{
j = random () % vec_len (objects);
if (objects[j] != ~0 || i + objects_used < n)
break;
}
if (objects[j] != ~0)
{
heap_dealloc (h, handles[j]);
objects_used--;
objects[j] = ~0;
}
else
{
u32 * data;
uword size;
size = 1 + (random () % 100);
objects[j] = heap_alloc_aligned (h, size, align, handles[j]);
objects_used++;
if (align)
ASSERT (0 == (objects[j] & (align - 1)));
ASSERT (objects[j] < vec_len (h));
ASSERT (size <= heap_len (h, handles[j]));
/* Set newly allocated object with test data. */
if (check_mask & 2)
{
data = h + objects[j];
for (k = 0; k < size; k++)
data[k] = objects[j] + k;
}
}
if (check_mask & 1)
heap_validate (h);
if (check_mask & 4)
{
/* Duplicate heap at each iteration. */
u32 * h1 = heap_dup (h);
heap_free (h);
h = h1;
}
/* Verify that all used objects have correct test data. */
if (check_mask & 2)
{
for (j = 0; j < vec_len (objects); j++)
if (objects[j] != ~0)
{
u32 * data = h + objects[j];
for (k = 0; k < heap_len (h, handles[j]); k++)
ASSERT(data[k] == objects[j] + k);
}
}
}
if (verbose) fformat (stderr, "%U\n", format_heap, h, 1);
{
u32 * h1 = heap_dup (h);
if (verbose) fformat (stderr, "%U\n", format_heap, h1, 1);
heap_free (h1);
}
heap_free (h);
if (verbose) fformat (stderr, "%U\n", format_heap, h, 1);
ASSERT (objects_used == 0);
vec_free (objects);
vec_free (handles);
if (fixed_size)
vec_free_h (h, sizeof (heap_header_t));
if (verbose) fformat (stderr, "%U\n", format_clib_mem_usage, /* verbose */ 0);
return 0;
}
int
test_mheap_main (unformat_input_t * input)
{
int i, j, k, n_iterations;
void *h, *h_mem;
uword *objects = 0;
u32 objects_used, really_verbose, n_objects, max_object_size;
u32 check_mask, seed, trace, use_vm;
u32 print_every = 0;
u32 *data;
mheap_t *mh;
/* Validation flags. */
check_mask = 0;
#define CHECK_VALIDITY 1
#define CHECK_DATA 2
#define CHECK_ALIGN 4
#define TEST1 8
n_iterations = 10;
seed = 0;
max_object_size = 100;
n_objects = 1000;
trace = 0;
really_verbose = 0;
use_vm = 0;
while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
{
if (0 == unformat (input, "iter %d", &n_iterations)
&& 0 == unformat (input, "count %d", &n_objects)
&& 0 == unformat (input, "size %d", &max_object_size)
&& 0 == unformat (input, "seed %d", &seed)
&& 0 == unformat (input, "print %d", &print_every)
&& 0 == unformat (input, "validdata %|",
&check_mask, CHECK_DATA | CHECK_VALIDITY)
&& 0 == unformat (input, "valid %|",
&check_mask, CHECK_VALIDITY)
&& 0 == unformat (input, "verbose %=", &really_verbose, 1)
&& 0 == unformat (input, "trace %=", &trace, 1)
&& 0 == unformat (input, "vm %=", &use_vm, 1)
&& 0 == unformat (input, "align %|", &check_mask, CHECK_ALIGN)
&& 0 == unformat (input, "test1 %|", &check_mask, TEST1))
{
clib_warning ("unknown input `%U'", format_unformat_error, input);
return 1;
}
}
/* Zero seed means use default. */
if (!seed)
seed = random_default_seed ();
if (check_mask & TEST1)
{
return test1 ();
}
if_verbose
("testing %d iterations, %d %saligned objects, max. size %d, seed %d",
n_iterations, n_objects, (check_mask & CHECK_ALIGN) ? "randomly " : "un",
max_object_size, seed);
vec_resize (objects, n_objects);
if (vec_bytes (objects) > 0) /* stupid warning be gone */
clib_memset (objects, ~0, vec_bytes (objects));
objects_used = 0;
/* Allocate initial heap. */
{
uword size =
max_pow2 (2 * n_objects * max_object_size * sizeof (data[0]));
h_mem = clib_mem_alloc (size);
if (!h_mem)
return 0;
h = mheap_alloc (h_mem, size);
}
if (trace)
mheap_trace (h, trace);
mh = mheap_header (h);
if (use_vm)
mh->flags &= ~MHEAP_FLAG_DISABLE_VM;
else
mh->flags |= MHEAP_FLAG_DISABLE_VM;
if (check_mask & CHECK_VALIDITY)
mh->flags |= MHEAP_FLAG_VALIDATE;
for (i = 0; i < n_iterations; i++)
{
while (1)
{
j = random_u32 (&seed) % vec_len (objects);
if (objects[j] != ~0 || i + objects_used < n_iterations)
break;
}
if (objects[j] != ~0)
{
mheap_put (h, objects[j]);
objects_used--;
objects[j] = ~0;
}
else
{
uword size, align, align_offset;
size = (random_u32 (&seed) % max_object_size) * sizeof (data[0]);
align = align_offset = 0;
if (check_mask & CHECK_ALIGN)
{
align = 1 << (random_u32 (&seed) % 10);
align_offset = round_pow2 (random_u32 (&seed) & (align - 1),
sizeof (u32));
}
h = mheap_get_aligned (h, size, align, align_offset, &objects[j]);
if (align > 0)
ASSERT (0 == ((objects[j] + align_offset) & (align - 1)));
ASSERT (objects[j] != ~0);
objects_used++;
/* Set newly allocated object with test data. */
if (check_mask & CHECK_DATA)
{
uword len;
data = (void *) h + objects[j];
len = mheap_len (h, data);
ASSERT (size <= mheap_data_bytes (h, objects[j]));
data[0] = len;
for (k = 1; k < len; k++)
data[k] = objects[j] + k;
}
}
/* Verify that all used objects have correct test data. */
if (check_mask & 2)
{
for (j = 0; j < vec_len (objects); j++)
if (objects[j] != ~0)
{
u32 *data = h + objects[j];
uword len = data[0];
for (k = 1; k < len; k++)
ASSERT (data[k] == objects[j] + k);
}
}
if (print_every != 0 && i > 0 && (i % print_every) == 0)
fformat (stderr, "iteration %d: %U\n", i, format_mheap, h,
really_verbose);
}
if (verbose)
fformat (stderr, "%U\n", format_mheap, h, really_verbose);
mheap_free (h);
clib_mem_free (h_mem);
vec_free (objects);
return 0;
}
void
scrape_and_clear_counters (perfmon_main_t * pm)
{
int i, j, k;
vlib_main_t *vm = pm->vlib_main;
vlib_main_t *stat_vm;
vlib_node_main_t *nm;
vlib_node_t ***node_dups = 0;
vlib_node_t **nodes;
vlib_node_t *n;
perfmon_capture_t *c;
perfmon_event_config_t *current_event;
uword *p;
u8 *counter_name;
u64 vectors_this_counter;
/* snapshoot the nodes, including pm counters */
vlib_worker_thread_barrier_sync (vm);
for (j = 0; j < vec_len (vlib_mains); j++)
{
stat_vm = vlib_mains[j];
if (stat_vm == 0)
continue;
nm = &stat_vm->node_main;
for (i = 0; i < vec_len (nm->nodes); i++)
{
n = nm->nodes[i];
vlib_node_sync_stats (stat_vm, n);
}
nodes = 0;
vec_validate (nodes, vec_len (nm->nodes) - 1);
vec_add1 (node_dups, nodes);
/* Snapshoot and clear the per-node perfmon counters */
for (i = 0; i < vec_len (nm->nodes); i++)
{
n = nm->nodes[i];
nodes[i] = clib_mem_alloc (sizeof (*n));
clib_memcpy_fast (nodes[i], n, sizeof (*n));
n->stats_total.perf_counter0_ticks = 0;
n->stats_total.perf_counter1_ticks = 0;
n->stats_total.perf_counter_vectors = 0;
n->stats_last_clear.perf_counter0_ticks = 0;
n->stats_last_clear.perf_counter1_ticks = 0;
n->stats_last_clear.perf_counter_vectors = 0;
}
}
vlib_worker_thread_barrier_release (vm);
for (j = 0; j < vec_len (vlib_mains); j++)
{
stat_vm = vlib_mains[j];
if (stat_vm == 0)
continue;
nodes = node_dups[j];
for (i = 0; i < vec_len (nodes); i++)
{
u8 *capture_name;
n = nodes[i];
if (n->stats_total.perf_counter0_ticks == 0 &&
n->stats_total.perf_counter1_ticks == 0)
goto skip_this_node;
for (k = 0; k < 2; k++)
{
u64 counter_value, counter_last_clear;
/*
* We collect 2 counters at once, except for the
* last counter when the user asks for an odd number of
* counters
*/
if ((pm->current_event + k)
>= vec_len (pm->single_events_to_collect))
break;
if (k == 0)
{
counter_value = n->stats_total.perf_counter0_ticks;
counter_last_clear =
n->stats_last_clear.perf_counter0_ticks;
}
else
{
counter_value = n->stats_total.perf_counter1_ticks;
counter_last_clear =
n->stats_last_clear.perf_counter1_ticks;
}
capture_name = format (0, "t%d-%v%c", j, n->name, 0);
p = hash_get_mem (pm->capture_by_thread_and_node_name,
capture_name);
if (p == 0)
{
pool_get (pm->capture_pool, c);
memset (c, 0, sizeof (*c));
c->thread_and_node_name = capture_name;
hash_set_mem (pm->capture_by_thread_and_node_name,
capture_name, c - pm->capture_pool);
}
else
{
c = pool_elt_at_index (pm->capture_pool, p[0]);
vec_free (capture_name);
}
/* Snapshoot counters, etc. into the capture */
current_event = pm->single_events_to_collect
+ pm->current_event + k;
counter_name = (u8 *) current_event->name;
vectors_this_counter = n->stats_total.perf_counter_vectors -
n->stats_last_clear.perf_counter_vectors;
vec_add1 (c->counter_names, counter_name);
vec_add1 (c->counter_values,
counter_value - counter_last_clear);
vec_add1 (c->vectors_this_counter, vectors_this_counter);
}
skip_this_node:
clib_mem_free (n);
}
vec_free (nodes);
}
vec_free (node_dups);
}
static inline void *
vl_msg_api_alloc_internal (int nbytes, int pool, int may_return_null)
{
int i;
msgbuf_t *rv;
ring_alloc_t *ap;
svm_queue_t *q;
void *oldheap;
vl_shmem_hdr_t *shmem_hdr;
api_main_t *am = &api_main;
shmem_hdr = am->shmem_hdr;
#if DEBUG_MESSAGE_BUFFER_OVERRUN > 0
nbytes += 4;
#endif
ASSERT (pool == 0 || vlib_get_thread_index () == 0);
if (shmem_hdr == 0)
{
clib_warning ("shared memory header NULL");
return 0;
}
/* account for the msgbuf_t header */
nbytes += sizeof (msgbuf_t);
if (shmem_hdr->vl_rings == 0)
{
clib_warning ("vl_rings NULL");
ASSERT (0);
abort ();
}
if (shmem_hdr->client_rings == 0)
{
clib_warning ("client_rings NULL");
ASSERT (0);
abort ();
}
ap = pool ? shmem_hdr->vl_rings : shmem_hdr->client_rings;
for (i = 0; i < vec_len (ap); i++)
{
/* Too big? */
if (nbytes > ap[i].size)
{
continue;
}
q = ap[i].rp;
if (pool == 0)
{
pthread_mutex_lock (&q->mutex);
}
rv = (msgbuf_t *) (&q->data[0] + q->head * q->elsize);
/*
* Is this item still in use?
*/
if (rv->q)
{
u32 now = (u32) time (0);
if (PREDICT_TRUE (rv->gc_mark_timestamp == 0))
rv->gc_mark_timestamp = now;
else
{
if (now - rv->gc_mark_timestamp > 10)
{
if (CLIB_DEBUG > 0)
{
u16 *msg_idp, msg_id;
clib_warning
("garbage collect pool %d ring %d index %d", pool, i,
q->head);
msg_idp = (u16 *) (rv->data);
msg_id = clib_net_to_host_u16 (*msg_idp);
if (msg_id < vec_len (api_main.msg_names))
clib_warning ("msg id %d name %s", (u32) msg_id,
api_main.msg_names[msg_id]);
}
shmem_hdr->garbage_collects++;
goto collected;
}
}
/* yes, loser; try next larger pool */
ap[i].misses++;
if (pool == 0)
pthread_mutex_unlock (&q->mutex);
continue;
}
collected:
/* OK, we have a winner */
ap[i].hits++;
/*
* Remember the source queue, although we
* don't need to know the queue to free the item.
*/
rv->q = q;
rv->gc_mark_timestamp = 0;
q->head++;
if (q->head == q->maxsize)
q->head = 0;
if (pool == 0)
pthread_mutex_unlock (&q->mutex);
goto out;
}
/*
* Request too big, or head element of all size-compatible rings
* still in use. Fall back to shared-memory malloc.
*/
am->ring_misses++;
pthread_mutex_lock (&am->vlib_rp->mutex);
oldheap = svm_push_data_heap (am->vlib_rp);
if (may_return_null)
{
rv = clib_mem_alloc_or_null (nbytes);
if (PREDICT_FALSE (rv == 0))
{
svm_pop_heap (oldheap);
pthread_mutex_unlock (&am->vlib_rp->mutex);
return 0;
}
}
else
rv = clib_mem_alloc (nbytes);
rv->q = 0;
rv->gc_mark_timestamp = 0;
svm_pop_heap (oldheap);
pthread_mutex_unlock (&am->vlib_rp->mutex);
out:
#if DEBUG_MESSAGE_BUFFER_OVERRUN > 0
{
nbytes -= 4;
u32 *overrun;
overrun = (u32 *) (rv->data + nbytes - sizeof (msgbuf_t));
*overrun = 0x1badbabe;
}
#endif
rv->data_len = htonl (nbytes - sizeof (msgbuf_t));
return (rv->data);
}
