/*
* Always called with rcu_registry lock held. Releases this lock between
* iterations and grabs it again. Holds the lock when it returns.
*/
static void wait_for_readers(struct cds_list_head *input_readers,
struct cds_list_head *cur_snap_readers,
struct cds_list_head *qsreaders)
{
unsigned int wait_loops = 0;
struct rcu_reader *index, *tmp;
/*
* Wait for each thread URCU_TLS(rcu_reader).ctr to either
* indicate quiescence (offline), or for them to observe the
* current rcu_gp.ctr value.
*/
for (;;) {
if (wait_loops < RCU_QS_ACTIVE_ATTEMPTS)
wait_loops++;
if (wait_loops >= RCU_QS_ACTIVE_ATTEMPTS) {
uatomic_set(&rcu_gp.futex, -1);
/*
* Write futex before write waiting (the other side
* reads them in the opposite order).
*/
cmm_smp_wmb();
cds_list_for_each_entry(index, input_readers, node) {
_CMM_STORE_SHARED(index->waiting, 1);
}
/* Write futex before read reader_gp */
cmm_smp_mb();
}
void *thr_writer(void *data)
{
unsigned long wtidx = (unsigned long)data;
printf_verbose("thread_begin %s, tid %lu\n",
"writer", urcu_get_thread_id());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
pthread_mutex_lock(&lock);
test_array.a = 0;
test_array.a = 8;
if (caa_unlikely(wduration))
loop_sleep(wduration);
pthread_mutex_unlock(&lock);
URCU_TLS(nr_writes)++;
if (caa_unlikely(!test_duration_write()))
break;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
}
printf_verbose("thread_end %s, tid %lu\n",
"writer", urcu_get_thread_id());
tot_nr_writes[wtidx] = URCU_TLS(nr_writes);
return ((void*)2);
}
int compat_futex_async(int32_t *uaddr, int op, int32_t val,
const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
{
/*
* Check if NULL. Don't let users expect that they are taken into
* account.
*/
assert(!timeout);
assert(!uaddr2);
assert(!val3);
/*
* Ensure previous memory operations on uaddr have completed.
*/
cmm_smp_mb();
switch (op) {
case FUTEX_WAIT:
while (*uaddr == val)
poll(NULL, 0, 10);
break;
case FUTEX_WAKE:
break;
default:
return -EINVAL;
}
return 0;
}
void *thr_writer(void *_count)
{
unsigned long long *count = _count;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"writer", (unsigned long) pthread_self(),
(unsigned long) gettid());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
pthread_rwlock_wrlock(&lock);
test_array.a = 0;
test_array.a = 8;
if (caa_unlikely(wduration))
loop_sleep(wduration);
pthread_rwlock_unlock(&lock);
URCU_TLS(nr_writes)++;
if (caa_unlikely(!test_duration_write()))
break;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
}
printf_verbose("thread_end %s, thread id : %lx, tid %lu\n",
"writer", (unsigned long) pthread_self(),
(unsigned long) gettid());
*count = URCU_TLS(nr_writes);
return ((void*)2);
}
/*
* Prepare futex.
*/
LTTNG_HIDDEN
void futex_nto1_prepare(int32_t *futex)
{
uatomic_set(futex, -1);
cmm_smp_mb();
DBG("Futex n to 1 prepare done");
}
void *test_hash_rw_thr_reader(void *_count)
{
unsigned long long *count = _count;
struct lfht_test_node *node;
struct cds_lfht_iter iter;
printf_verbose("thread_begin %s, tid %lu\n",
"reader", urcu_get_thread_id());
URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL);
set_affinity();
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
rcu_read_lock();
cds_lfht_test_lookup(test_ht,
(void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % lookup_pool_size) + lookup_pool_offset),
sizeof(void *), &iter);
node = cds_lfht_iter_get_test_node(&iter);
if (node == NULL) {
if (validate_lookup) {
printf("[ERROR] Lookup cannot find initial node.\n");
exit(-1);
}
URCU_TLS(lookup_fail)++;
} else {
URCU_TLS(lookup_ok)++;
}
rcu_debug_yield_read();
if (caa_unlikely(rduration))
loop_sleep(rduration);
rcu_read_unlock();
URCU_TLS(nr_reads)++;
if (caa_unlikely(!test_duration_read()))
break;
if (caa_unlikely((URCU_TLS(nr_reads) & ((1 << 10) - 1)) == 0))
rcu_quiescent_state();
}
rcu_unregister_thread();
*count = URCU_TLS(nr_reads);
printf_verbose("thread_end %s, tid %lu\n",
"reader", urcu_get_thread_id());
printf_verbose("read tid : %lx, lookupfail %lu, lookupok %lu\n",
urcu_get_thread_id(),
URCU_TLS(lookup_fail),
URCU_TLS(lookup_ok));
return ((void*)1);
}
/*
* Wait futex.
*/
LTTNG_HIDDEN
void futex_nto1_wait(int32_t *futex)
{
cmm_smp_mb();
if (uatomic_read(futex) == -1) {
futex_async(futex, FUTEX_WAIT, -1, NULL, NULL, 0);
}
DBG("Futex n to 1 wait done");
}
void *thr_dequeuer(void *_count)
{
unsigned long long *count = _count;
int ret;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"dequeuer", pthread_self(), (unsigned long)gettid());
set_affinity();
ret = rcu_defer_register_thread();
if (ret) {
printf("Error in rcu_defer_register_thread\n");
exit(-1);
}
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct cds_lfq_node_rcu *qnode;
struct test *node;
rcu_read_lock();
qnode = cds_lfq_dequeue_rcu(&q);
node = caa_container_of(qnode, struct test, list);
rcu_read_unlock();
if (node) {
call_rcu(&node->rcu, free_node_cb);
nr_successful_dequeues++;
}
nr_dequeues++;
if (caa_unlikely(!test_duration_dequeue()))
break;
if (caa_unlikely(rduration))
loop_sleep(rduration);
}
rcu_unregister_thread();
rcu_defer_unregister_thread();
printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, "
"dequeues %llu, successful_dequeues %llu\n",
pthread_self(), (unsigned long)gettid(), nr_dequeues,
nr_successful_dequeues);
count[0] = nr_dequeues;
count[1] = nr_successful_dequeues;
return ((void*)2);
}
static void *thr_dequeuer(void *_count)
{
unsigned long long *count = _count;
unsigned int counter = 0;
printf_verbose("thread_begin %s, tid %lu\n",
"dequeuer", urcu_get_thread_id());
set_affinity();
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
assert(test_pop || test_pop_all);
for (;;) {
if (test_pop && test_pop_all) {
/* both pop and pop all */
if (counter & 1)
do_test_pop(test_sync);
else
do_test_pop_all(test_sync);
counter++;
} else {
if (test_pop)
do_test_pop(test_sync);
else
do_test_pop_all(test_sync);
}
if (caa_unlikely(!test_duration_dequeue()))
break;
if (caa_unlikely(rduration))
loop_sleep(rduration);
}
rcu_unregister_thread();
printf_verbose("dequeuer thread_end, tid %lu, "
"dequeues %llu, successful_dequeues %llu\n",
urcu_get_thread_id(),
URCU_TLS(nr_dequeues),
URCU_TLS(nr_successful_dequeues));
count[0] = URCU_TLS(nr_dequeues);
count[1] = URCU_TLS(nr_successful_dequeues);
return ((void*)2);
}
static void *thr_enqueuer(void *_count)
{
unsigned long long *count = _count;
bool was_nonempty;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"enqueuer", (unsigned long) pthread_self(),
(unsigned long) gettid());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct cds_wfs_node *node = malloc(sizeof(*node));
if (!node)
goto fail;
cds_wfs_node_init(node);
was_nonempty = cds_wfs_push(&s, node);
URCU_TLS(nr_successful_enqueues)++;
if (!was_nonempty)
URCU_TLS(nr_empty_dest_enqueues)++;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
fail:
URCU_TLS(nr_enqueues)++;
if (caa_unlikely(!test_duration_enqueue()))
break;
}
uatomic_inc(&test_enqueue_stopped);
count[0] = URCU_TLS(nr_enqueues);
count[1] = URCU_TLS(nr_successful_enqueues);
count[2] = URCU_TLS(nr_empty_dest_enqueues);
printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, "
"enqueues %llu successful_enqueues %llu, "
"empty_dest_enqueues %llu\n",
pthread_self(),
(unsigned long) gettid(),
URCU_TLS(nr_enqueues),
URCU_TLS(nr_successful_enqueues),
URCU_TLS(nr_empty_dest_enqueues));
return ((void*)1);
}
void *thr_dequeuer(void *_count)
{
unsigned long long *count = _count;
printf_verbose("thread_begin %s, tid %lu\n",
"dequeuer", urcu_get_thread_id());
set_affinity();
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct cds_lfq_node_rcu *qnode;
rcu_read_lock();
qnode = cds_lfq_dequeue_rcu(&q);
rcu_read_unlock();
if (qnode) {
struct test *node;
node = caa_container_of(qnode, struct test, list);
call_rcu(&node->rcu, free_node_cb);
URCU_TLS(nr_successful_dequeues)++;
}
URCU_TLS(nr_dequeues)++;
if (caa_unlikely(!test_duration_dequeue()))
break;
if (caa_unlikely(rduration))
loop_sleep(rduration);
}
rcu_unregister_thread();
printf_verbose("dequeuer thread_end, tid %lu, "
"dequeues %llu, successful_dequeues %llu\n",
urcu_get_thread_id(),
URCU_TLS(nr_dequeues),
URCU_TLS(nr_successful_dequeues));
count[0] = URCU_TLS(nr_dequeues);
count[1] = URCU_TLS(nr_successful_dequeues);
return ((void*)2);
}
int lttng_ust_enable_trace_clock_override(void)
{
if (CMM_LOAD_SHARED(lttng_trace_clock))
return -EBUSY;
if (!user_tc.read64)
return -EINVAL;
if (!user_tc.freq)
return -EINVAL;
if (!user_tc.name)
return -EINVAL;
if (!user_tc.description)
return -EINVAL;
/* Use default uuid cb when NULL */
cmm_smp_mb(); /* Store callbacks before trace clock */
CMM_STORE_SHARED(lttng_trace_clock, &user_tc);
return 0;
}
void *thr_enqueuer(void *_count)
{
unsigned long long *count = _count;
printf_verbose("thread_begin %s, tid %lu\n",
"enqueuer", urcu_get_thread_id());
set_affinity();
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct test *node = malloc(sizeof(*node));
if (!node)
goto fail;
cds_lfq_node_init_rcu(&node->list);
rcu_read_lock();
cds_lfq_enqueue_rcu(&q, &node->list);
rcu_read_unlock();
URCU_TLS(nr_successful_enqueues)++;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
fail:
URCU_TLS(nr_enqueues)++;
if (caa_unlikely(!test_duration_enqueue()))
break;
}
rcu_unregister_thread();
count[0] = URCU_TLS(nr_enqueues);
count[1] = URCU_TLS(nr_successful_enqueues);
printf_verbose("enqueuer thread_end, tid %lu, "
"enqueues %llu successful_enqueues %llu\n",
urcu_get_thread_id(),
URCU_TLS(nr_enqueues),
URCU_TLS(nr_successful_enqueues));
return ((void*)1);
}
void *thr_writer(void *data)
{
unsigned long wtidx = (unsigned long)data;
long tidx;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"writer", (unsigned long) pthread_self(),
(unsigned long) gettid());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
for (tidx = 0; tidx < nr_readers; tidx++) {
pthread_mutex_lock(&per_thread_lock[tidx].lock);
}
test_array.a = 0;
test_array.a = 8;
if (caa_unlikely(wduration))
loop_sleep(wduration);
for (tidx = (long)nr_readers - 1; tidx >= 0; tidx--) {
pthread_mutex_unlock(&per_thread_lock[tidx].lock);
}
URCU_TLS(nr_writes)++;
if (caa_unlikely(!test_duration_write()))
break;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
}
printf_verbose("thread_end %s, thread id : %lx, tid %lu\n",
"writer", (unsigned long) pthread_self(),
(unsigned long) gettid());
tot_nr_writes[wtidx] = URCU_TLS(nr_writes);
return ((void*)2);
}
void *thr_enqueuer(void *_count)
{
unsigned long long *count = _count;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"enqueuer", pthread_self(), (unsigned long)gettid());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct cds_wfq_node *node = malloc(sizeof(*node));
if (!node)
goto fail;
cds_wfq_node_init(node);
cds_wfq_enqueue(&q, node);
nr_successful_enqueues++;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
fail:
nr_enqueues++;
if (caa_unlikely(!test_duration_enqueue()))
break;
}
count[0] = nr_enqueues;
count[1] = nr_successful_enqueues;
printf_verbose("enqueuer thread_end, thread id : %lx, tid %lu, "
"enqueues %llu successful_enqueues %llu\n",
pthread_self(), (unsigned long)gettid(), nr_enqueues,
nr_successful_enqueues);
return ((void*)1);
}
void *thr_dequeuer(void *_count)
{
unsigned long long *count = _count;
printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n",
"dequeuer", (unsigned long) pthread_self(),
(unsigned long) gettid());
set_affinity();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
struct cds_wfq_node *node = cds_wfq_dequeue_blocking(&q);
if (node) {
free(node);
URCU_TLS(nr_successful_dequeues)++;
}
URCU_TLS(nr_dequeues)++;
if (caa_unlikely(!test_duration_dequeue()))
break;
if (caa_unlikely(rduration))
loop_sleep(rduration);
}
printf_verbose("dequeuer thread_end, thread id : %lx, tid %lu, "
"dequeues %llu, successful_dequeues %llu\n",
pthread_self(),
(unsigned long) gettid(),
URCU_TLS(nr_dequeues), URCU_TLS(nr_successful_dequeues));
count[0] = URCU_TLS(nr_dequeues);
count[1] = URCU_TLS(nr_successful_dequeues);
return ((void*)2);
}
int compat_futex_noasync(int32_t *uaddr, int op, int32_t val,
const struct timespec *timeout, int32_t *uaddr2, int32_t val3)
{
int ret, gret = 0;
/*
* Check if NULL. Don't let users expect that they are taken into
* account.
*/
assert(!timeout);
assert(!uaddr2);
assert(!val3);
/*
* memory barriers to serialize with the previous uaddr modification.
*/
cmm_smp_mb();
ret = pthread_mutex_lock(&__urcu_compat_futex_lock);
assert(!ret);
switch (op) {
case FUTEX_WAIT:
if (*uaddr != val)
goto end;
pthread_cond_wait(&__urcu_compat_futex_cond, &__urcu_compat_futex_lock);
break;
case FUTEX_WAKE:
pthread_cond_broadcast(&__urcu_compat_futex_cond);
break;
default:
gret = -EINVAL;
}
end:
ret = pthread_mutex_unlock(&__urcu_compat_futex_lock);
assert(!ret);
return gret;
}
/*
* Wait futex.
*/
LTTNG_HIDDEN
void futex_nto1_wait(int32_t *futex)
{
cmm_smp_mb();
if (uatomic_read(futex) != -1)
goto end;
while (futex_async(futex, FUTEX_WAIT, -1, NULL, NULL, 0)) {
switch (errno) {
case EWOULDBLOCK:
/* Value already changed. */
goto end;
case EINTR:
/* Retry if interrupted by signal. */
break; /* Get out of switch. */
default:
/* Unexpected error. */
PERROR("futex_async");
abort();
}
}
end:
DBG("Futex n to 1 wait done");
}
int main(int argc, char **argv)
{
int err;
pthread_t *tid_reader, *tid_writer;
void *tret;
unsigned long long *count_reader, *count_writer;
unsigned long long tot_reads = 0, tot_writes = 0;
int i, a;
if (argc < 4) {
show_usage(argc, argv);
return -1;
}
cmm_smp_mb();
err = sscanf(argv[1], "%u", &nr_readers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[2], "%u", &nr_writers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[3], "%lu", &duration);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
for (i = 4; i < argc; i++) {
if (argv[i][0] != '-')
continue;
switch (argv[i][1]) {
#ifdef DEBUG_YIELD
case 'r':
yield_active |= YIELD_READ;
break;
case 'w':
yield_active |= YIELD_WRITE;
break;
#endif
case 'a':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
a = atoi(argv[++i]);
cpu_affinities[next_aff++] = a;
use_affinity = 1;
printf_verbose("Adding CPU %d affinity\n", a);
break;
case 'c':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
rduration = atol(argv[++i]);
break;
case 'd':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
wdelay = atol(argv[++i]);
break;
case 'e':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
wduration = atol(argv[++i]);
break;
case 'v':
verbose_mode = 1;
break;
}
}
printf_verbose("running test for %lu seconds, %u readers, %u writers.\n",
duration, nr_readers, nr_writers);
printf_verbose("Writer delay : %lu loops.\n", wdelay);
printf_verbose("Reader duration : %lu loops.\n", rduration);
printf_verbose("thread %-6s, thread id : %lx, tid %lu\n",
"main", (unsigned long) pthread_self(),
(unsigned long) gettid());
tid_reader = malloc(sizeof(*tid_reader) * nr_readers);
tid_writer = malloc(sizeof(*tid_writer) * nr_writers);
count_reader = malloc(sizeof(*count_reader) * nr_readers);
count_writer = malloc(sizeof(*count_writer) * nr_writers);
next_aff = 0;
for (i = 0; i < nr_readers; i++) {
err = pthread_create(&tid_reader[i], NULL, thr_reader,
&count_reader[i]);
if (err != 0)
exit(1);
}
for (i = 0; i < nr_writers; i++) {
err = pthread_create(&tid_writer[i], NULL, thr_writer,
&count_writer[i]);
if (err != 0)
exit(1);
}
cmm_smp_mb();
test_go = 1;
sleep(duration);
test_stop = 1;
for (i = 0; i < nr_readers; i++) {
err = pthread_join(tid_reader[i], &tret);
if (err != 0)
exit(1);
tot_reads += count_reader[i];
}
for (i = 0; i < nr_writers; i++) {
err = pthread_join(tid_writer[i], &tret);
if (err != 0)
exit(1);
tot_writes += count_writer[i];
}
printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads,
tot_writes);
printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu wdur %6lu "
"nr_writers %3u "
"wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n",
argv[0], duration, nr_readers, rduration, wduration,
nr_writers, wdelay, tot_reads, tot_writes,
tot_reads + tot_writes);
free(tid_reader);
free(tid_writer);
free(count_reader);
free(count_writer);
return 0;
}
int main(int argc, char **argv)
{
int err;
pthread_t *tid_enqueuer, *tid_dequeuer;
void *tret;
unsigned long long *count_enqueuer, *count_dequeuer;
unsigned long long tot_enqueues = 0, tot_dequeues = 0;
unsigned long long tot_successful_enqueues = 0,
tot_successful_dequeues = 0;
unsigned long long end_dequeues = 0;
int i, a;
if (argc < 4) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[1], "%u", &nr_dequeuers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[2], "%u", &nr_enqueuers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[3], "%lu", &duration);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
for (i = 4; i < argc; i++) {
if (argv[i][0] != '-')
continue;
switch (argv[i][1]) {
case 'a':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
a = atoi(argv[++i]);
cpu_affinities[next_aff++] = a;
use_affinity = 1;
printf_verbose("Adding CPU %d affinity\n", a);
break;
case 'c':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
rduration = atol(argv[++i]);
break;
case 'd':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
wdelay = atol(argv[++i]);
break;
case 'v':
verbose_mode = 1;
break;
}
}
printf_verbose("running test for %lu seconds, %u enqueuers, "
"%u dequeuers.\n",
duration, nr_enqueuers, nr_dequeuers);
printf_verbose("Writer delay : %lu loops.\n", rduration);
printf_verbose("Reader duration : %lu loops.\n", wdelay);
printf_verbose("thread %-6s, tid %lu\n",
"main", urcu_get_thread_id());
tid_enqueuer = calloc(nr_enqueuers, sizeof(*tid_enqueuer));
tid_dequeuer = calloc(nr_dequeuers, sizeof(*tid_dequeuer));
count_enqueuer = calloc(nr_enqueuers, 2 * sizeof(*count_enqueuer));
count_dequeuer = calloc(nr_dequeuers, 2 * sizeof(*count_dequeuer));
cds_lfq_init_rcu(&q, call_rcu);
err = create_all_cpu_call_rcu_data(0);
if (err) {
printf("Per-CPU call_rcu() worker threads unavailable. Using default global worker thread.\n");
}
next_aff = 0;
for (i = 0; i < nr_enqueuers; i++) {
err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer,
&count_enqueuer[2 * i]);
if (err != 0)
exit(1);
}
for (i = 0; i < nr_dequeuers; i++) {
err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer,
&count_dequeuer[2 * i]);
if (err != 0)
exit(1);
}
cmm_smp_mb();
test_go = 1;
for (i = 0; i < duration; i++) {
sleep(1);
if (verbose_mode) {
fwrite(".", sizeof(char), 1, stdout);
fflush(stdout);
}
}
test_stop = 1;
for (i = 0; i < nr_enqueuers; i++) {
err = pthread_join(tid_enqueuer[i], &tret);
if (err != 0)
exit(1);
tot_enqueues += count_enqueuer[2 * i];
tot_successful_enqueues += count_enqueuer[2 * i + 1];
}
for (i = 0; i < nr_dequeuers; i++) {
err = pthread_join(tid_dequeuer[i], &tret);
if (err != 0)
exit(1);
tot_dequeues += count_dequeuer[2 * i];
tot_successful_dequeues += count_dequeuer[2 * i + 1];
}
test_end(&q, &end_dequeues);
err = cds_lfq_destroy_rcu(&q);
assert(!err);
printf_verbose("total number of enqueues : %llu, dequeues %llu\n",
tot_enqueues, tot_dequeues);
printf_verbose("total number of successful enqueues : %llu, "
"successful dequeues %llu\n",
tot_successful_enqueues, tot_successful_dequeues);
printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu "
"nr_dequeuers %3u "
"rdur %6lu nr_enqueues %12llu nr_dequeues %12llu "
"successful enqueues %12llu successful dequeues %12llu "
"end_dequeues %llu nr_ops %12llu\n",
argv[0], duration, nr_enqueuers, wdelay,
nr_dequeuers, rduration, tot_enqueues, tot_dequeues,
tot_successful_enqueues,
tot_successful_dequeues, end_dequeues,
tot_enqueues + tot_dequeues);
if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues)
printf("WARNING! Discrepancy between nr succ. enqueues %llu vs "
"succ. dequeues + end dequeues %llu.\n",
tot_successful_enqueues,
tot_successful_dequeues + end_dequeues);
free_all_cpu_call_rcu_data();
free(count_enqueuer);
free(count_dequeuer);
free(tid_enqueuer);
free(tid_dequeuer);
return 0;
}
int main(int argc, char **argv)
{
int err;
pthread_t *tid_enqueuer, *tid_dequeuer;
void *tret;
unsigned long long *count_enqueuer, *count_dequeuer;
unsigned long long tot_enqueues = 0, tot_dequeues = 0;
unsigned long long tot_successful_enqueues = 0,
tot_successful_dequeues = 0;
unsigned long long end_dequeues = 0;
int i, a;
if (argc < 4) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[1], "%u", &nr_dequeuers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[2], "%u", &nr_enqueuers);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
err = sscanf(argv[3], "%lu", &duration);
if (err != 1) {
show_usage(argc, argv);
return -1;
}
for (i = 4; i < argc; i++) {
if (argv[i][0] != '-')
continue;
switch (argv[i][1]) {
case 'a':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
a = atoi(argv[++i]);
cpu_affinities[next_aff++] = a;
use_affinity = 1;
printf_verbose("Adding CPU %d affinity\n", a);
break;
case 'c':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
rduration = atol(argv[++i]);
break;
case 'd':
if (argc < i + 2) {
show_usage(argc, argv);
return -1;
}
wdelay = atol(argv[++i]);
break;
case 'v':
verbose_mode = 1;
break;
}
}
printf_verbose("running test for %lu seconds, %u enqueuers, "
"%u dequeuers.\n",
duration, nr_enqueuers, nr_dequeuers);
printf_verbose("Writer delay : %lu loops.\n", rduration);
printf_verbose("Reader duration : %lu loops.\n", wdelay);
printf_verbose("thread %-6s, thread id : %lx, tid %lu\n",
"main", pthread_self(), (unsigned long)gettid());
tid_enqueuer = malloc(sizeof(*tid_enqueuer) * nr_enqueuers);
tid_dequeuer = malloc(sizeof(*tid_dequeuer) * nr_dequeuers);
count_enqueuer = malloc(2 * sizeof(*count_enqueuer) * nr_enqueuers);
count_dequeuer = malloc(2 * sizeof(*count_dequeuer) * nr_dequeuers);
cds_wfq_init(&q);
next_aff = 0;
for (i = 0; i < nr_enqueuers; i++) {
err = pthread_create(&tid_enqueuer[i], NULL, thr_enqueuer,
&count_enqueuer[2 * i]);
if (err != 0)
exit(1);
}
for (i = 0; i < nr_dequeuers; i++) {
err = pthread_create(&tid_dequeuer[i], NULL, thr_dequeuer,
&count_dequeuer[2 * i]);
if (err != 0)
exit(1);
}
cmm_smp_mb();
test_go = 1;
for (i = 0; i < duration; i++) {
sleep(1);
if (verbose_mode)
write (1, ".", 1);
}
test_stop = 1;
for (i = 0; i < nr_enqueuers; i++) {
err = pthread_join(tid_enqueuer[i], &tret);
if (err != 0)
exit(1);
tot_enqueues += count_enqueuer[2 * i];
tot_successful_enqueues += count_enqueuer[2 * i + 1];
}
for (i = 0; i < nr_dequeuers; i++) {
err = pthread_join(tid_dequeuer[i], &tret);
if (err != 0)
exit(1);
tot_dequeues += count_dequeuer[2 * i];
tot_successful_dequeues += count_dequeuer[2 * i + 1];
}
test_end(&q, &end_dequeues);
printf_verbose("total number of enqueues : %llu, dequeues %llu\n",
tot_enqueues, tot_dequeues);
printf_verbose("total number of successful enqueues : %llu, "
"successful dequeues %llu\n",
tot_successful_enqueues, tot_successful_dequeues);
printf("SUMMARY %-25s testdur %4lu nr_enqueuers %3u wdelay %6lu "
"nr_dequeuers %3u "
"rdur %6lu nr_enqueues %12llu nr_dequeues %12llu "
"successful enqueues %12llu successful dequeues %12llu "
"end_dequeues %llu nr_ops %12llu\n",
argv[0], duration, nr_enqueuers, wdelay,
nr_dequeuers, rduration, tot_enqueues, tot_dequeues,
tot_successful_enqueues,
tot_successful_dequeues, end_dequeues,
tot_enqueues + tot_dequeues);
if (tot_successful_enqueues != tot_successful_dequeues + end_dequeues)
printf("WARNING! Discrepancy between nr succ. enqueues %llu vs "
"succ. dequeues + end dequeues %llu.\n",
tot_successful_enqueues,
tot_successful_dequeues + end_dequeues);
free(count_enqueuer);
free(count_dequeuer);
free(tid_enqueuer);
free(tid_dequeuer);
return 0;
}
void *test_hash_rw_thr_writer(void *_count)
{
struct lfht_test_node *node;
struct cds_lfht_node *ret_node;
struct cds_lfht_iter iter;
struct wr_count *count = _count;
int ret;
printf_verbose("thread_begin %s, tid %lu\n",
"writer", urcu_get_thread_id());
URCU_TLS(rand_lookup) = urcu_get_thread_id() ^ time(NULL);
set_affinity();
rcu_register_thread();
while (!test_go)
{
}
cmm_smp_mb();
for (;;) {
if ((addremove == AR_ADD || add_only)
|| (addremove == AR_RANDOM && rand_r(&URCU_TLS(rand_lookup)) & 1)) {
node = malloc(sizeof(struct lfht_test_node));
lfht_test_node_init(node,
(void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset),
sizeof(void *));
rcu_read_lock();
if (add_unique) {
ret_node = cds_lfht_add_unique(test_ht,
test_hash(node->key, node->key_len, TEST_HASH_SEED),
test_match, node->key, &node->node);
} else {
if (add_replace)
ret_node = cds_lfht_add_replace(test_ht,
test_hash(node->key, node->key_len, TEST_HASH_SEED),
test_match, node->key, &node->node);
else
cds_lfht_add(test_ht,
test_hash(node->key, node->key_len, TEST_HASH_SEED),
&node->node);
}
rcu_read_unlock();
if (add_unique && ret_node != &node->node) {
free(node);
URCU_TLS(nr_addexist)++;
} else {
if (add_replace && ret_node) {
call_rcu(&to_test_node(ret_node)->head,
free_node_cb);
URCU_TLS(nr_addexist)++;
} else {
URCU_TLS(nr_add)++;
}
}
} else {
/* May delete */
rcu_read_lock();
cds_lfht_test_lookup(test_ht,
(void *)(((unsigned long) rand_r(&URCU_TLS(rand_lookup)) % write_pool_size) + write_pool_offset),
sizeof(void *), &iter);
ret = cds_lfht_del(test_ht, cds_lfht_iter_get_node(&iter));
rcu_read_unlock();
if (ret == 0) {
node = cds_lfht_iter_get_test_node(&iter);
call_rcu(&node->head, free_node_cb);
URCU_TLS(nr_del)++;
} else
URCU_TLS(nr_delnoent)++;
}
#if 0
//if (URCU_TLS(nr_writes) % 100000 == 0) {
if (URCU_TLS(nr_writes) % 1000 == 0) {
rcu_read_lock();
if (rand_r(&URCU_TLS(rand_lookup)) & 1) {
ht_resize(test_ht, 1);
} else {
ht_resize(test_ht, -1);
}
rcu_read_unlock();
}
#endif //0
URCU_TLS(nr_writes)++;
if (caa_unlikely(!test_duration_write()))
break;
if (caa_unlikely(wdelay))
loop_sleep(wdelay);
if (caa_unlikely((URCU_TLS(nr_writes) & ((1 << 10) - 1)) == 0))
rcu_quiescent_state();
}
rcu_unregister_thread();
printf_verbose("thread_end %s, tid %lu\n",
"writer", urcu_get_thread_id());
printf_verbose("info tid %lu: nr_add %lu, nr_addexist %lu, nr_del %lu, "
"nr_delnoent %lu\n", urcu_get_thread_id(),
URCU_TLS(nr_add),
URCU_TLS(nr_addexist),
URCU_TLS(nr_del),
URCU_TLS(nr_delnoent));
count->update_ops = URCU_TLS(nr_writes);
count->add = URCU_TLS(nr_add);
count->add_exist = URCU_TLS(nr_addexist);
count->remove = URCU_TLS(nr_del);
return ((void*)2);
}
/*
* main
*/
int main(int argc, char **argv)
{
int ret = 0, retval = 0;
void *status;
if (set_signal_handler()) {
retval = -1;
goto exit_set_signal_handler;
}
/* Parse arguments */
progname = argv[0];
if (parse_args(argc, argv)) {
retval = -1;
goto exit_options;
}
/* Daemonize */
if (opt_daemon) {
int i;
/*
* fork
* child: setsid, close FD 0, 1, 2, chdir /
* parent: exit (if fork is successful)
*/
ret = daemon(0, 0);
if (ret < 0) {
PERROR("daemon");
retval = -1;
goto exit_options;
}
/*
* We are in the child. Make sure all other file
* descriptors are closed, in case we are called with
* more opened file descriptors than the standard ones.
*/
for (i = 3; i < sysconf(_SC_OPEN_MAX); i++) {
(void) close(i);
}
}
/*
* Starting from here, we can create threads. This needs to be after
* lttng_daemonize due to RCU.
*/
health_consumerd = health_app_create(NR_HEALTH_CONSUMERD_TYPES);
if (!health_consumerd) {
retval = -1;
goto exit_health_consumerd_cleanup;
}
/* Set up max poll set size */
if (lttng_poll_set_max_size()) {
retval = -1;
goto exit_init_data;
}
if (*command_sock_path == '\0') {
switch (opt_type) {
case LTTNG_CONSUMER_KERNEL:
ret = snprintf(command_sock_path, PATH_MAX,
DEFAULT_KCONSUMERD_CMD_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
case LTTNG_CONSUMER64_UST:
ret = snprintf(command_sock_path, PATH_MAX,
DEFAULT_USTCONSUMERD64_CMD_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
case LTTNG_CONSUMER32_UST:
ret = snprintf(command_sock_path, PATH_MAX,
DEFAULT_USTCONSUMERD32_CMD_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
default:
ERR("Unknown consumerd type");
retval = -1;
goto exit_init_data;
}
}
/* Init */
if (lttng_consumer_init()) {
retval = -1;
goto exit_init_data;
}
/* Initialize communication library */
lttcomm_init();
/* Initialize TCP timeout values */
lttcomm_inet_init();
if (!getuid()) {
/* Set limit for open files */
set_ulimit();
}
/* create the consumer instance with and assign the callbacks */
ctx = lttng_consumer_create(opt_type, lttng_consumer_read_subbuffer,
NULL, lttng_consumer_on_recv_stream, NULL);
if (!ctx) {
retval = -1;
goto exit_init_data;
}
lttng_consumer_set_command_sock_path(ctx, command_sock_path);
if (*error_sock_path == '\0') {
switch (opt_type) {
case LTTNG_CONSUMER_KERNEL:
ret = snprintf(error_sock_path, PATH_MAX,
DEFAULT_KCONSUMERD_ERR_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
case LTTNG_CONSUMER64_UST:
ret = snprintf(error_sock_path, PATH_MAX,
DEFAULT_USTCONSUMERD64_ERR_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
case LTTNG_CONSUMER32_UST:
ret = snprintf(error_sock_path, PATH_MAX,
DEFAULT_USTCONSUMERD32_ERR_SOCK_PATH,
DEFAULT_LTTNG_RUNDIR);
if (ret < 0) {
retval = -1;
goto exit_init_data;
}
break;
default:
ERR("Unknown consumerd type");
retval = -1;
goto exit_init_data;
}
}
/* Connect to the socket created by lttng-sessiond to report errors */
DBG("Connecting to error socket %s", error_sock_path);
ret = lttcomm_connect_unix_sock(error_sock_path);
/*
* Not a fatal error, but all communication with lttng-sessiond will
* fail.
*/
if (ret < 0) {
WARN("Cannot connect to error socket (is lttng-sessiond started?)");
}
lttng_consumer_set_error_sock(ctx, ret);
/*
* Block RT signals used for UST periodical metadata flush and the live
* timer in main, and create a dedicated thread to handle these signals.
*/
if (consumer_signal_init()) {
retval = -1;
goto exit_init_data;
}
ctx->type = opt_type;
if (utils_create_pipe(health_quit_pipe)) {
retval = -1;
goto exit_health_pipe;
}
/* Create thread to manage the client socket */
ret = pthread_create(&health_thread, NULL,
thread_manage_health, (void *) NULL);
if (ret) {
errno = ret;
PERROR("pthread_create health");
retval = -1;
goto exit_health_thread;
}
/*
* Wait for health thread to be initialized before letting the
* sessiond thread reply to the sessiond that we are ready.
*/
while (uatomic_read(<tng_consumer_ready)) {
usleep(100000);
}
cmm_smp_mb(); /* Read ready before following operations */
/* Create thread to manage channels */
ret = pthread_create(&channel_thread, NULL,
consumer_thread_channel_poll,
(void *) ctx);
if (ret) {
errno = ret;
PERROR("pthread_create");
retval = -1;
goto exit_channel_thread;
}
/* Create thread to manage the polling/writing of trace metadata */
ret = pthread_create(&metadata_thread, NULL,
consumer_thread_metadata_poll,
(void *) ctx);
if (ret) {
errno = ret;
PERROR("pthread_create");
retval = -1;
goto exit_metadata_thread;
}
/* Create thread to manage the polling/writing of trace data */
ret = pthread_create(&data_thread, NULL, consumer_thread_data_poll,
(void *) ctx);
if (ret) {
errno = ret;
PERROR("pthread_create");
retval = -1;
goto exit_data_thread;
}
/* Create the thread to manage the receive of fd */
ret = pthread_create(&sessiond_thread, NULL,
consumer_thread_sessiond_poll,
(void *) ctx);
if (ret) {
errno = ret;
PERROR("pthread_create");
retval = -1;
goto exit_sessiond_thread;
}
/*
* Create the thread to manage the UST metadata periodic timer and
* live timer.
*/
ret = pthread_create(&metadata_timer_thread, NULL,
consumer_timer_thread, (void *) ctx);
if (ret) {
errno = ret;
PERROR("pthread_create");
retval = -1;
goto exit_metadata_timer_thread;
}
ret = pthread_detach(metadata_timer_thread);
if (ret) {
errno = ret;
PERROR("pthread_detach");
retval = -1;
goto exit_metadata_timer_detach;
}
/*
* This is where we start awaiting program completion (e.g. through
* signal that asks threads to teardown.
*/
exit_metadata_timer_detach:
exit_metadata_timer_thread:
ret = pthread_join(sessiond_thread, &status);
if (ret) {
errno = ret;
PERROR("pthread_join sessiond_thread");
retval = -1;
}
exit_sessiond_thread:
ret = pthread_join(data_thread, &status);
if (ret) {
errno = ret;
PERROR("pthread_join data_thread");
retval = -1;
}
exit_data_thread:
ret = pthread_join(metadata_thread, &status);
if (ret) {
errno = ret;
PERROR("pthread_join metadata_thread");
retval = -1;
}
exit_metadata_thread:
ret = pthread_join(channel_thread, &status);
if (ret) {
errno = ret;
PERROR("pthread_join channel_thread");
retval = -1;
}
exit_channel_thread:
ret = pthread_join(health_thread, &status);
if (ret) {
errno = ret;
PERROR("pthread_join health_thread");
retval = -1;
}
exit_health_thread:
utils_close_pipe(health_quit_pipe);
exit_health_pipe:
exit_init_data:
lttng_consumer_destroy(ctx);
lttng_consumer_cleanup();
if (health_consumerd) {
health_app_destroy(health_consumerd);
}
exit_health_consumerd_cleanup:
exit_options:
exit_set_signal_handler:
if (!retval) {
exit(EXIT_SUCCESS);
} else {
exit(EXIT_FAILURE);
}
}