/* Function to compute Fibonacci numbers */
void fibonacci_task(void *arguments)
{
int n, result;
task_args *a1, *a2, *parent, *temp;
ABT_task t1, t2;
task_args *args = (task_args *)arguments;
n = args->n;
parent = args->parent;
/* checking for base cases */
if (n <= 2) {
args->result = 1;
result = 1;
int flag = 1;
while (flag && parent != NULL) {
ABT_mutex_lock(parent->mutex);
parent->result += result;
if (result == parent->result) flag = 0;
ABT_mutex_unlock(parent->mutex);
result = parent->result;
temp = parent->parent;
if (flag && temp) {
ABT_mutex_free(&parent->mutex);
free(parent);
}
parent = temp;
}
ABT_mutex_free(&args->mutex);
if (args->parent) {
free(args);
}
} else {
a1 = (task_args *)malloc(sizeof(task_args));
a1->n = n - 1;
a1->result = 0;
ABT_mutex_create(&a1->mutex);
a1->parent = args;
ABT_task_create(g_pool, fibonacci_task, a1, &t1);
a2 = (task_args *)malloc(sizeof(task_args));
a2->n = n - 2;
a2->result = 0;
ABT_mutex_create(&a2->mutex);
a2->parent = args;
ABT_task_create(g_pool, fibonacci_task, a2, &t2);
}
}
void accalt_tasklet_creation_to(void(*thread_func)(void *), void *arg, ACCALT_tasklet *new_ult, int dest) {
#ifdef ARGOBOTS
ABT_pool pool;
ABT_xstream_get_main_pools(main_team->team[dest], 1, &pool);
ABT_task_create(pool, thread_func, arg, new_ult);
#endif
#ifdef MASSIVETHREADS
accalt_ult_creation(thread_func, arg, new_ult);
#endif
#ifdef QTHREADS
qthread_fork_to((void *) thread_func, arg, new_ult, dest);
#endif
}
void accalt_tasklet_creation(void(*thread_func)(void *), void *arg, ACCALT_tasklet *new_ult) {
#ifdef ARGOBOTS
ABT_xstream xstream;
ABT_xstream_self(&xstream);
ABT_pool pool;
ABT_xstream_get_main_pools(xstream, 1, &pool);
ABT_task_create(pool, thread_func, arg, new_ult);
#endif
#ifdef MASSIVETHREADS
*new_ult = myth_create((void *) thread_func, arg);
#endif
#ifdef QTHREADS
qthread_fork((void *) thread_func, arg, new_ult);
#endif
}
static void gen_work(void *arg)
{
int idx = (size_t)arg;
int num_pools = g_data.num_pools[idx];
ABT_pool *my_pools = g_data.pools[idx];
ABT_bool flag;
int i, ret;
unsigned seed = time(NULL);
ABT_test_printf(1, "[E%d] creating work units\n", idx);
/* Create work units on the current ES */
/* TODO: add work units to pools of other ESs */
for (i = 0; i < num_units; i++) {
unit_arg_t *my_arg = (unit_arg_t *)malloc(sizeof(unit_arg_t));
my_arg->es_id = idx;
my_arg->my_id = i;
my_arg->prio = rand_r(&seed) % num_pools;
if (i & 1) {
ret = ABT_thread_create(my_pools[my_arg->prio],
thread_func, (void *)my_arg,
ABT_THREAD_ATTR_NULL, NULL);
ABT_TEST_ERROR(ret, "ABT_thread_create");
} else {
ret = ABT_task_create(my_pools[my_arg->prio],
task_func, (void *)my_arg,
NULL);
ABT_TEST_ERROR(ret, "ABT_task_create");
}
}
/* Stack the priority scheduler if it is associated with PW pools */
ret = ABT_self_on_primary_xstream(&flag);
ABT_TEST_ERROR(ret, "ABT_self_on_primary_stream");
if (flag == ABT_FALSE) {
if (accesses[idx] == ABT_POOL_ACCESS_PRIV ||
accesses[idx] == ABT_POOL_ACCESS_SPSC ||
accesses[idx] == ABT_POOL_ACCESS_SPMC) {
ABT_pool main_pool;
ret = ABT_xstream_get_main_pools(g_data.xstreams[idx],
1, &main_pool);
ABT_TEST_ERROR(ret, "ABT_xstream_get_main_pools");
ret = ABT_pool_add_sched(main_pool, g_data.scheds[idx]);
ABT_TEST_ERROR(ret, "ABT_pool_add_sched");
}
}
}
void vector_scal_launch(void *arguments)
{
int i, it, j, num_ults, rank, mystart, myend, p;
ABT_task *tasks;
ABT_xstream xstream;
ABT_xstream_self(&xstream);
vector_scal_task_args_t *arg;
arg = (vector_scal_task_args_t *) arguments;
vector_scal_args_t *args;
it = arg->it;
num_ults = arg->nxstreams;
mystart = arg->start;
myend = arg->end;
args = (vector_scal_args_t *)malloc(sizeof(vector_scal_args_t)
* num_ults);
tasks = (ABT_task *)malloc(sizeof(ABT_task) * num_ults);
int bloc = it / (num_ults);
int rest = it % (num_ults);
ABT_xstream_self_rank(&rank);
for (i = mystart; i < myend; i++) {
int start = 0;
int end = 0;
for (j = 0; j < num_ults; j++) {
start = end;
int inc = (j < rest) ? 1 : 0;
end += bloc + inc;
args[j].start = start;
args[j].end = end;
args[j].x = i;
if (j > 0) {
ABT_task_create(g_pools[rank], vector_scal, (void *)&args[j], &tasks[j]);
}
}
vector_scal((void *)&args[0]);
for (p = 1; p < num_ults; p++) {
ABT_task_free(&tasks[p]);
}
}
free(tasks);
free(args);
}
void rt1_launcher(void *arg)
{
int idx = (int)(intptr_t)arg;
ABT_thread cur_thread;
ABT_pool cur_pool;
ABT_sched_config config;
ABT_sched sched;
size_t size;
double t_start, t_end;
ABT_sched_config_var cv_event_freq = {
.idx = 0,
.type = ABT_SCHED_CONFIG_INT
};
ABT_sched_config_var cv_idx = {
.idx = 1,
.type = ABT_SCHED_CONFIG_INT
};
ABT_sched_def sched_def = {
.type = ABT_SCHED_TYPE_ULT,
.init = sched_init,
.run = sched_run,
.free = sched_free,
.get_migr_pool = NULL
};
/* Create a scheduler */
ABT_sched_config_create(&config,
cv_event_freq, 10,
cv_idx, idx,
ABT_sched_config_var_end);
ABT_sched_create(&sched_def, 1, &rt1_data->pool, config, &sched);
/* Push the scheduler to the current pool */
ABT_thread_self(&cur_thread);
ABT_thread_get_last_pool(cur_thread, &cur_pool);
ABT_pool_add_sched(cur_pool, sched);
/* Free */
ABT_sched_config_free(&config);
t_start = ABT_get_wtime();
while (1) {
rt1_app(idx);
ABT_pool_get_total_size(cur_pool, &size);
if (size == 0) {
ABT_sched_free(&sched);
int rank;
ABT_xstream_self_rank(&rank);
printf("ES%d: finished\n", rank);
ABT_mutex_lock(rt1_data->mutex);
rt1_data->xstreams[rank] = ABT_XSTREAM_NULL;
rt1_data->num_xstreams--;
ABT_mutex_unlock(rt1_data->mutex);
break;
}
t_end = ABT_get_wtime();
if ((t_end - t_start) > g_timeout) {
ABT_sched_finish(sched);
}
}
}
static void rt1_app(int eid)
{
int i, num_comps;
size_t size;
ABT_thread cur_thread;
ABT_pool cur_pool;
ABT_thread_self(&cur_thread);
ABT_thread_get_last_pool(cur_thread, &cur_pool);
if (eid == 0) ABT_event_prof_start();
num_comps = rt1_data->num_comps;
for (i = 0; i < num_comps * 2; i += 2) {
ABT_thread_create(rt1_data->pool, rt1_app_compute,
(void *)(intptr_t)(eid * num_comps * 2 + i),
ABT_THREAD_ATTR_NULL, NULL);
ABT_task_create(rt1_data->pool, rt1_app_compute,
(void *)(intptr_t)(eid * num_comps * 2 + i + 1),
NULL);
}
do {
ABT_thread_yield();
/* If the size of cur_pool is zero, it means the stacked scheduler has
* been terminated because of the shrinking event. */
ABT_pool_get_total_size(cur_pool, &size);
if (size == 0) break;
ABT_pool_get_total_size(rt1_data->pool, &size);
} while (size > 0);
if (eid == 0) {
ABT_event_prof_stop();
int cnt = __atomic_exchange_n(&rt1_data->cnt, 0, __ATOMIC_SEQ_CST);
double local_work = (double)(cnt * rt1_data->num_iters);
ABT_event_prof_publish("ops", local_work, local_work);
}
}
static void rt1_app_compute(void *arg)
{
int pos = (int)(intptr_t)arg;
int i;
rt1_data->app_data[pos] = 0;
for (i = 0; i < rt1_data->num_iters; i++) {
rt1_data->app_data[pos] += sin((double)pos);
}
__atomic_fetch_add(&rt1_data->cnt, 1, __ATOMIC_SEQ_CST);
}
/* Main function */
int main(int argc, char *argv[])
{
int n, i, result, expected;
int num_xstreams;
ABT_xstream *xstreams;
ABT_thread thread;
thread_args args_thread;
ABT_task task;
task_args *args_task;
if (argc > 1 && strcmp(argv[1], "-h") == 0) {
printf("Usage: %s [N=10] [num_ES=4]\n", argv[0]);
return EXIT_SUCCESS;
}
n = argc > 1 ? atoi(argv[1]) : N;
num_xstreams = argc > 2 ? atoi(argv[2]) : NUM_XSTREAMS;
printf("# of ESs: %d\n", num_xstreams);
if (n <= 2) {
result = 1;
goto fn_result;
}
/* initialization */
ABT_init(argc, argv);
/* shared pool creation */
ABT_pool_create_basic(ABT_POOL_FIFO, ABT_POOL_ACCESS_MPMC, ABT_TRUE,
&g_pool);
/* ES creation */
xstreams = (ABT_xstream *)malloc(sizeof(ABT_xstream) * num_xstreams);
ABT_xstream_self(&xstreams[0]);
ABT_xstream_set_main_sched_basic(xstreams[0], ABT_SCHED_DEFAULT,
1, &g_pool);
for (i = 1; i < num_xstreams; i++) {
ABT_xstream_create_basic(ABT_SCHED_DEFAULT, 1, &g_pool,
ABT_SCHED_CONFIG_NULL, &xstreams[i]);
ABT_xstream_start(xstreams[i]);
}
/* creating thread */
args_thread.n = n - 1;
args_thread.eventual = ABT_EVENTUAL_NULL;
ABT_thread_create(g_pool, fibonacci_thread, &args_thread, ABT_THREAD_ATTR_NULL, &thread);
/* creating task */
args_task = (task_args *)malloc(sizeof(task_args));
args_task->n = n - 2;
args_task->result = 0;
ABT_mutex_create(&args_task->mutex);
args_task->parent = NULL;
ABT_task_create(g_pool, fibonacci_task, args_task, &task);
/* switch to other user-level threads */
ABT_thread_yield();
/* join other threads */
ABT_thread_join(thread);
ABT_thread_free(&thread);
/* join ESs */
for (i = 1; i < num_xstreams; i++) {
ABT_xstream_join(xstreams[i]);
ABT_xstream_free(&xstreams[i]);
}
result = args_thread.result + args_task->result;
free(args_task);
ABT_finalize();
free(xstreams);
fn_result:
printf("Fib(%d): %d\n", n, result);
expected = verify(n);
if (result != expected) {
fprintf(stderr, "ERROR: expected=%d\n", expected);
exit(EXIT_FAILURE);
}
return EXIT_SUCCESS;
}
void eventual_test(void *arg)
{
int i, t, ret;
size_t pid = (size_t)arg;
int eid = (int)pid;
ABT_thread *threads;
arg_t *thread_args;
int *nbytes;
ABT_eventual *evs1;
ABT_thread *waiters;
arg_t *waiter_args;
ABT_task *tasks;
arg_t *task_args;
ABT_eventual *evs2;
ABT_test_printf(1, "[M%u] start\n", (unsigned)(size_t)arg);
assert(num_tasks * 2 == num_threads);
threads = (ABT_thread *)malloc(num_threads * sizeof(ABT_thread));
thread_args = (arg_t *)malloc(num_threads * sizeof(arg_t));
nbytes = (int *)malloc(num_tasks * sizeof(int));
evs1 = (ABT_eventual *)malloc(num_tasks * sizeof(ABT_eventual));
assert(threads && thread_args && nbytes && evs1);
waiters = (ABT_thread *)malloc(num_tasks * sizeof(ABT_thread));
waiter_args = (arg_t *)malloc(num_tasks * sizeof(arg_t));
tasks = (ABT_task *)malloc(num_tasks * sizeof(ABT_task));
task_args = (arg_t *)malloc(num_tasks * sizeof(arg_t));
evs2 = (ABT_eventual *)malloc(num_tasks * sizeof(ABT_eventual));
assert(waiters && waiter_args && tasks && task_args && evs2);
for (t = 0; t < num_tasks; t++) {
nbytes[t] = (t & 1) ? sizeof(int) : 0;
ret = ABT_eventual_create(nbytes[t], &evs1[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_create");
ret = ABT_eventual_create(nbytes[t], &evs2[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_create");
}
for (i = 0; i < num_iter; i++) {
ABT_test_printf(2, "[M%u] iter=%d\n", (unsigned)(size_t)arg, i);
/* Use eventual between ULTs */
for (t = 0; t < num_threads; t += 2) {
thread_args[t].eid = eid;
thread_args[t].tid = t;
thread_args[t].ev = evs1[t/2];
thread_args[t].nbytes = nbytes[t/2];
thread_args[t].op_type = OP_WAIT;
ret = ABT_thread_create(pools[pid], thread_func, &thread_args[t],
ABT_THREAD_ATTR_NULL, &threads[t]);
ABT_TEST_ERROR(ret, "ABT_thread_create");
pid = (pid + 1) % num_xstreams;
thread_args[t+1].eid = eid;
thread_args[t+1].tid = t + 1;
thread_args[t+1].ev = evs1[t/2];
thread_args[t+1].nbytes = nbytes[t/2];
thread_args[t+1].op_type = OP_SET;
ret = ABT_thread_create(pools[pid], thread_func, &thread_args[t+1],
ABT_THREAD_ATTR_NULL, &threads[t+1]);
ABT_TEST_ERROR(ret, "ABT_thread_create");
pid = (pid + 1) % num_xstreams;
}
/* Use eventual between ULT and tasklet */
for (t = 0; t < num_tasks; t++) {
waiter_args[t].ev = evs2[t];
waiter_args[t].nbytes = nbytes[t];
waiter_args[t].op_type = OP_WAIT;
ret = ABT_thread_create(pools[pid], thread_func, &waiter_args[t],
ABT_THREAD_ATTR_NULL, &waiters[t]);
ABT_TEST_ERROR(ret, "ABT_thread_create");
pid = (pid + 1) % num_xstreams;
task_args[t].ev = evs2[t];
task_args[t].nbytes = nbytes[t];
task_args[t].op_type = OP_SET;
ret = ABT_task_create(pools[pid], task_func, &task_args[t], &tasks[t]);
ABT_TEST_ERROR(ret, "ABT_task_create");
pid = (pid + 1) % num_xstreams;
}
for (t = 0; t < num_threads; t++) {
ret = ABT_thread_free(&threads[t]);
ABT_TEST_ERROR(ret, "ABT_thread_free");
}
for (t = 0; t < num_tasks; t++) {
ret = ABT_thread_free(&waiters[t]);
ABT_TEST_ERROR(ret, "ABT_thread_free");
ret = ABT_task_free(&tasks[t]);
ABT_TEST_ERROR(ret, "ABT_task_free");
}
for (t = 0; t < num_tasks; t++) {
ret = ABT_eventual_reset(evs1[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_reset");
ret = ABT_eventual_reset(evs2[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_reset");
}
}
for (t = 0; t < num_tasks; t++) {
ret = ABT_eventual_free(&evs1[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_free");
ret = ABT_eventual_free(&evs2[t]);
ABT_TEST_ERROR(ret, "ABT_eventual_free");
}
free(threads);
free(thread_args);
free(nbytes);
free(evs1);
free(waiters);
free(waiter_args);
free(tasks);
free(task_args);
free(evs2);
ABT_test_printf(1, "[M%u] end\n", (unsigned)(size_t)arg);
}
int main(int argc, char *argv[])
{
int i, j, ret;
int num_xstreams = DEFAULT_NUM_XSTREAMS;
int num_threads = DEFAULT_NUM_THREADS;
int num_tasks = DEFAULT_NUM_TASKS;
if (argc > 1) num_xstreams = atoi(argv[1]);
assert(num_xstreams >= 0);
if (argc > 2) num_threads = atoi(argv[2]);
assert(num_threads >= 0);
if (argc > 3) num_tasks = atoi(argv[3]);
assert(num_tasks >= 0);
ABT_xstream *xstreams;
ABT_thread **threads;
thread_arg_t **thread_args;
ABT_task *tasks;
task_arg_t *task_args;
xstreams = (ABT_xstream *)malloc(sizeof(ABT_xstream) * num_xstreams);
threads = (ABT_thread **)malloc(sizeof(ABT_thread *) * num_xstreams);
thread_args = (thread_arg_t **)malloc(sizeof(thread_arg_t*) * num_xstreams);
for (i = 0; i < num_xstreams; i++) {
threads[i] = (ABT_thread *)malloc(sizeof(ABT_thread) * num_threads);
for (j = 0; j < num_threads; j++) {
threads[i][j] = ABT_THREAD_NULL;
}
thread_args[i] = (thread_arg_t *)malloc(sizeof(thread_arg_t) *
num_threads);
}
tasks = (ABT_task *)malloc(sizeof(ABT_task) * num_tasks);
task_args = (task_arg_t *)malloc(sizeof(task_arg_t) * num_tasks);
/* Initialize */
ABT_test_init(argc, argv);
/* Create Execution Streams */
ret = ABT_xstream_self(&xstreams[0]);
ABT_TEST_ERROR(ret, "ABT_xstream_self");
for (i = 1; i < num_xstreams; i++) {
ret = ABT_xstream_create(ABT_SCHED_NULL, &xstreams[i]);
ABT_TEST_ERROR(ret, "ABT_xstream_create");
}
/* Get the pools attached to an execution stream */
ABT_pool *pools;
pools = (ABT_pool *)malloc(sizeof(ABT_pool) * num_xstreams);
for (i = 0; i < num_xstreams; i++) {
ret = ABT_xstream_get_main_pools(xstreams[i], 1, pools+i);
ABT_TEST_ERROR(ret, "ABT_xstream_get_main_pools");
}
/* Create threads */
for (i = 0; i < num_xstreams; i++) {
for (j = 0; j < num_threads; j++) {
int tid = i * num_threads + j + 1;
thread_args[i][j].id = tid;
thread_args[i][j].num_threads = num_threads;
thread_args[i][j].threads = &threads[i][0];
ret = ABT_thread_create(pools[i],
thread_func, (void *)&thread_args[i][j],
ABT_THREAD_ATTR_NULL,
&threads[i][j]);
ABT_TEST_ERROR(ret, "ABT_thread_create");
}
}
/* Create tasks with task_func1 */
for (i = 0; i < num_tasks; i++) {
size_t num = 100 + i;
ret = ABT_task_create(pools[i % num_xstreams],
task_func1, (void *)num,
NULL);
ABT_TEST_ERROR(ret, "ABT_task_create");
}
/* Create tasks with task_func2 */
for (i = 0; i < num_tasks; i++) {
task_args[i].num = 100 + i;
ret = ABT_task_create(pools[i % num_xstreams],
task_func2, (void *)&task_args[i],
&tasks[i]);
ABT_TEST_ERROR(ret, "ABT_task_create");
}
/* Switch to other work units */
ABT_thread_yield();
/* Results of task_funcs2 */
for (i = 0; i < num_tasks; i++) {
ABT_task_state state;
do {
ABT_task_get_state(tasks[i], &state);
ABT_thread_yield();
} while (state != ABT_TASK_STATE_TERMINATED);
ABT_test_printf(1, "task_func2: num=%lu result=%llu\n",
task_args[i].num, task_args[i].result);
/* Free named tasks */
ret = ABT_task_free(&tasks[i]);
ABT_TEST_ERROR(ret, "ABT_task_free");
}
/* Join Execution Streams */
for (i = 1; i < num_xstreams; i++) {
ret = ABT_xstream_join(xstreams[i]);
ABT_TEST_ERROR(ret, "ABT_xstream_join");
}
/* Free Execution Streams */
for (i = 0; i < num_xstreams; i++) {
for (j = 0; j < num_threads; j++) {
ret = ABT_thread_free(&threads[i][j]);
ABT_TEST_ERROR(ret, "ABT_thread_free");
}
if (i == 0) continue;
ret = ABT_xstream_free(&xstreams[i]);
ABT_TEST_ERROR(ret, "ABT_xstream_free");
}
/* Finalize */
ret = ABT_test_finalize(0);
for (i = 0; i < num_xstreams; i++) {
free(thread_args[i]);
free(threads[i]);
}
free(thread_args);
free(threads);
free(task_args);
free(tasks);
free(pools);
free(xstreams);
return ret;
}
