void
gomp_ordered_static_next (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_work_share *ws = thr->ts.work_share;
unsigned id = thr->ts.team_id;
if (team == NULL || team->nthreads == 1)
return;
ws->ordered_owner = -1;
/* This thread currently owns the lock. Increment the owner. */
if (++id == team->nthreads)
id = 0;
ws->ordered_team_ids[0] = id;
gomp_sem_post (team->ordered_release[id]);
}
void
gomp_ordered_next (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_work_share *ws = thr->ts.work_share;
unsigned index, next_id;
/* Work share constructs can be orphaned. */
if (team == NULL || team->nthreads == 1)
return;
/* We're no longer the owner. */
ws->ordered_owner = -1;
/* If there's only one thread in the queue, that must be us. */
if (ws->ordered_num_used == 1)
{
/* We have a similar situation as in gomp_ordered_first
where we need to post to our own release semaphore. */
gomp_sem_post (team->ordered_release[thr->ts.team_id]);
return;
}
/* If the queue is entirely full, then we move ourself to the end of
the queue merely by incrementing ordered_cur. Only if it's not
full do we have to write our id. */
if (ws->ordered_num_used < team->nthreads)
{
index = ws->ordered_cur + ws->ordered_num_used;
if (index >= team->nthreads)
index -= team->nthreads;
ws->ordered_team_ids[index] = thr->ts.team_id;
}
index = ws->ordered_cur + 1;
if (index == team->nthreads)
index = 0;
ws->ordered_cur = index;
next_id = ws->ordered_team_ids[index];
gomp_sem_post (team->ordered_release[next_id]);
}
void
GOMP_parallel_end (void)
{
if (__builtin_expect (gomp_thread_limit_var != ULONG_MAX, 0))
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
if (team && team->nthreads > 1)
{
#ifdef HAVE_SYNC_BUILTINS
__sync_fetch_and_add (&gomp_remaining_threads_count,
1UL - team->nthreads);
#else
gomp_mutex_lock (&gomp_remaining_threads_lock);
gomp_remaining_threads_count -= team->nthreads - 1;
#endif
}
}
gomp_team_end ();
}
void
gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
unsigned int generation, gen;
if (__builtin_expect ((state & 1) != 0, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
bar->awaited = bar->total;
if (__builtin_expect (team->task_count, 0))
{
gomp_barrier_handle_tasks (state);
state &= ~1;
}
else
{
__atomic_store_n (&bar->generation, state + 3, MEMMODEL_RELEASE);
futex_wake ((int *) &bar->generation, INT_MAX);
return;
}
}
generation = state;
do
{
do_wait ((int *) &bar->generation, generation);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
if (__builtin_expect (gen & 1, 0))
{
gomp_barrier_handle_tasks (state);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
}
if ((gen & 2) != 0)
generation |= 2;
}
while (gen != state + 4);
}
bool
GOMP_single_start (void)
{
#ifdef HAVE_SYNC_BUILTINS
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
unsigned long single_count;
if (__builtin_expect (team == NULL, 0))
return true;
single_count = thr->ts.single_count++;
return __sync_bool_compare_and_swap (&team->single_count, single_count,
single_count + 1L);
#else
bool ret = gomp_work_share_start (false);
if (ret)
gomp_work_share_init_done ();
gomp_work_share_end_nowait ();
return ret;
#endif
}
void
gomp_ordered_first (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_work_share *ws = thr->ts.work_share;
unsigned index;
/* Work share constructs can be orphaned. */
if (team == NULL || team->nthreads == 1)
return;
index = ws->ordered_cur + ws->ordered_num_used;
if (index >= team->nthreads)
index -= team->nthreads;
ws->ordered_team_ids[index] = thr->ts.team_id;
/* If this is the first and only thread in the queue, then there is
no one to release us when we get to our ordered section. Post to
our own release queue now so that we won't block later. */
if (ws->ordered_num_used++ == 0)
gomp_sem_post (team->ordered_release[thr->ts.team_id]);
}
void
GOMP_taskgroup_start (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
struct gomp_taskgroup *taskgroup;
/* If team is NULL, all tasks are executed as
GOMP_TASK_UNDEFERRED tasks and thus all children tasks of
taskgroup and their descendant tasks will be finished
by the time GOMP_taskgroup_end is called. */
if (team == NULL)
return;
taskgroup = gomp_malloc (sizeof (struct gomp_taskgroup));
taskgroup->prev = task->taskgroup;
taskgroup->children = NULL;
taskgroup->in_taskgroup_wait = false;
taskgroup->cancelled = false;
taskgroup->num_children = 0;
gomp_sem_init (&taskgroup->taskgroup_sem, 0);
task->taskgroup = taskgroup;
}
unsigned
GOMP_sections_next (void)
{
long s, e, ret;
#ifdef HAVE_SYNC_BUILTINS
if (gomp_iter_dynamic_next (&s, &e))
ret = s;
else
ret = 0;
#else
struct gomp_thread *thr = gomp_thread ();
gomp_mutex_lock (&thr->ts.work_share->lock);
if (gomp_iter_dynamic_next_locked (&s, &e))
ret = s;
else
ret = 0;
gomp_mutex_unlock (&thr->ts.work_share->lock);
#endif
return ret;
}
void *
GOMP_single_copy_start (void)
{
struct gomp_thread *thr = gomp_thread ();
bool first;
void *ret;
first = gomp_work_share_start (false);
gomp_mutex_unlock (&thr->ts.work_share->lock);
if (first)
ret = NULL;
else
{
gomp_barrier_wait (&thr->ts.team->barrier);
ret = thr->ts.work_share->copyprivate;
gomp_work_share_end_nowait ();
}
return ret;
}
void
gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
unsigned int generation;
if (__builtin_expect ((state & 1) != 0, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
bar->awaited = bar->total;
atomic_write_barrier ();
if (__builtin_expect (team->task_count, 0))
{
gomp_barrier_handle_tasks (state);
state &= ~1;
}
else
{
bar->generation = state + 3;
futex_wake ((int *) &bar->generation, INT_MAX);
return;
}
}
generation = state;
do
{
do_wait ((int *) &bar->generation, generation);
if (__builtin_expect (bar->generation & 1, 0))
gomp_barrier_handle_tasks (state);
if ((bar->generation & 2))
generation |= 2;
}
while (bar->generation != state + 4);
}
int
omp_in_parallel (void)
{
return gomp_thread ()->ts.active_level > 0;
}
int
omp_in_final (void)
{
struct gomp_thread *thr = gomp_thread ();
return thr->task && thr->task->final_task;
}
thr->task = NULL;
#ifdef LIBGOMP_USE_PTHREADS
pthread_exit (NULL);
#elif defined(__nvptx__)
asm ("exit;");
#else
#error gomp_free_pool_helper must terminate the thread
#endif
}
/* Free a thread pool and release its threads. */
void
gomp_free_thread (void *arg __attribute__((unused)))
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_thread_pool *pool = thr->thread_pool;
if (pool)
{
if (pool->threads_used > 0)
{
int i;
for (i = 1; i < pool->threads_used; i++)
{
struct gomp_thread *nthr = pool->threads[i];
nthr->fn = gomp_free_pool_helper;
nthr->data = pool;
}
/* This barrier undocks threads docked on pool->threads_dock. */
gomp_simple_barrier_wait (&pool->threads_dock);
/* And this waits till all threads have called gomp_barrier_wait_last
void
gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
struct gomp_work_share *work_share)
{
struct gomp_thread_start_data *start_data;
struct gomp_thread *thr, *nthr;
struct gomp_team *team;
bool nested;
unsigned i, n, old_threads_used = 0;
pthread_attr_t thread_attr, *attr;
thr = gomp_thread ();
nested = thr->ts.team != NULL;
team = new_team (nthreads, work_share);
/* Always save the previous state, even if this isn't a nested team.
In particular, we should save any work share state from an outer
orphaned work share construct. */
team->prev_ts = thr->ts;
thr->ts.team = team;
thr->ts.work_share = work_share;
thr->ts.team_id = 0;
thr->ts.work_share_generation = 0;
thr->ts.static_trip = 0;
if (nthreads == 1)
return;
i = 1;
/* We only allow the reuse of idle threads for non-nested PARALLEL
regions. This appears to be implied by the semantics of
threadprivate variables, but perhaps that's reading too much into
things. Certainly it does prevent any locking problems, since
only the initial program thread will modify gomp_threads. */
if (!nested)
{
old_threads_used = gomp_threads_used;
if (nthreads <= old_threads_used)
n = nthreads;
else if (old_threads_used == 0)
{
n = 0;
gomp_barrier_init (&gomp_threads_dock, nthreads);
}
else
{
n = old_threads_used;
/* Increase the barrier threshold to make sure all new
threads arrive before the team is released. */
gomp_barrier_reinit (&gomp_threads_dock, nthreads);
}
/* Not true yet, but soon will be. We're going to release all
threads from the dock, and those that aren't part of the
team will exit. */
gomp_threads_used = nthreads;
/* Release existing idle threads. */
for (; i < n; ++i)
{
nthr = gomp_threads[i];
nthr->ts.team = team;
nthr->ts.work_share = work_share;
nthr->ts.team_id = i;
nthr->ts.work_share_generation = 0;
nthr->ts.static_trip = 0;
nthr->fn = fn;
nthr->data = data;
team->ordered_release[i] = &nthr->release;
}
if (i == nthreads)
goto do_release;
/* If necessary, expand the size of the gomp_threads array. It is
expected that changes in the number of threads is rare, thus we
make no effort to expand gomp_threads_size geometrically. */
if (nthreads >= gomp_threads_size)
{
gomp_threads_size = nthreads + 1;
gomp_threads
= gomp_realloc (gomp_threads,
gomp_threads_size
* sizeof (struct gomp_thread_data *));
}
}
attr = &gomp_thread_attr;
if (gomp_cpu_affinity != NULL)
{
size_t stacksize;
pthread_attr_init (&thread_attr);
pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED);
if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
pthread_attr_setstacksize (&thread_attr, stacksize);
attr = &thread_attr;
}
start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
* (nthreads-i));
/* Launch new threads. */
for (; i < nthreads; ++i, ++start_data)
{
pthread_t pt;
int err;
start_data->ts.team = team;
start_data->ts.work_share = work_share;
start_data->ts.team_id = i;
start_data->ts.work_share_generation = 0;
start_data->ts.static_trip = 0;
start_data->fn = fn;
start_data->fn_data = data;
start_data->nested = nested;
if (gomp_cpu_affinity != NULL)
gomp_init_thread_affinity (attr);
err = pthread_create (&pt, attr, gomp_thread_start, start_data);
if (err != 0)
gomp_fatal ("Thread creation failed: %s", strerror (err));
}
if (gomp_cpu_affinity != NULL)
pthread_attr_destroy (&thread_attr);
do_release:
gomp_barrier_wait (nested ? &team->barrier : &gomp_threads_dock);
/* Decrease the barrier threshold to match the number of threads
that should arrive back at the end of this team. The extra
threads should be exiting. Note that we arrange for this test
to never be true for nested teams. */
if (nthreads < old_threads_used)
gomp_barrier_reinit (&gomp_threads_dock, nthreads);
}
bool
gomp_iter_dynamic_next (long *pstart, long *pend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_work_share *ws = thr->ts.work_share;
long start, end, nend, chunk, incr;
end = ws->end;
incr = ws->incr;
chunk = ws->chunk_size;
if (__builtin_expect (ws->mode, 1))
{
long tmp = __sync_fetch_and_add (&ws->next, chunk);
if (incr > 0)
{
if (tmp >= end)
return false;
nend = tmp + chunk;
if (nend > end)
nend = end;
*pstart = tmp;
*pend = nend;
return true;
}
else
{
if (tmp <= end)
return false;
nend = tmp + chunk;
if (nend < end)
nend = end;
*pstart = tmp;
*pend = nend;
return true;
}
}
start = ws->next;
while (1)
{
long left = end - start;
long tmp;
if (start == end)
return false;
if (incr < 0)
{
if (chunk < left)
chunk = left;
}
else
{
if (chunk > left)
chunk = left;
}
nend = start + chunk;
tmp = __sync_val_compare_and_swap (&ws->next, start, nend);
if (__builtin_expect (tmp == start, 1))
break;
start = tmp;
}
*pstart = start;
*pend = nend;
return true;
}
void
GOMP_taskgroup_end (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
struct gomp_taskgroup *taskgroup;
struct gomp_task *child_task = NULL;
struct gomp_task *to_free = NULL;
int do_wake = 0;
if (team == NULL)
return;
taskgroup = task->taskgroup;
/* The acquire barrier on load of taskgroup->num_children here
synchronizes with the write of 0 in gomp_task_run_post_remove_taskgroup.
It is not necessary that we synchronize with other non-0 writes at
this point, but we must ensure that all writes to memory by a
child thread task work function are seen before we exit from
GOMP_taskgroup_end. */
if (__atomic_load_n (&taskgroup->num_children, MEMMODEL_ACQUIRE) == 0)
goto finish;
gomp_mutex_lock (&team->task_lock);
while (1)
{
bool cancelled = false;
if (taskgroup->children == NULL)
{
if (taskgroup->num_children)
{
if (task->children == NULL)
goto do_wait;
child_task = task->children;
}
else
{
gomp_mutex_unlock (&team->task_lock);
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
}
goto finish;
}
}
else
child_task = taskgroup->children;
if (child_task->kind == GOMP_TASK_WAITING)
{
cancelled
= gomp_task_run_pre (child_task, child_task->parent, team);
if (__builtin_expect (cancelled, 0))
{
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
goto finish_cancelled;
}
}
else
{
child_task = NULL;
do_wait:
/* All tasks we are waiting for are already running
in other threads. Wait for them. */
taskgroup->in_taskgroup_wait = true;
}
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
{
gomp_team_barrier_wake (&team->barrier, do_wake);
do_wake = 0;
}
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
if (child_task)
{
thr->task = child_task;
child_task->fn (child_task->fn_data);
thr->task = task;
}
else
gomp_sem_wait (&taskgroup->taskgroup_sem);
gomp_mutex_lock (&team->task_lock);
if (child_task)
{
finish_cancelled:;
size_t new_tasks
= gomp_task_run_post_handle_depend (child_task, team);
gomp_task_run_post_remove_parent (child_task);
gomp_clear_parent (child_task->children);
gomp_task_run_post_remove_taskgroup (child_task);
to_free = child_task;
child_task = NULL;
team->task_count--;
if (new_tasks > 1)
{
do_wake = team->nthreads - team->task_running_count
- !task->in_tied_task;
if (do_wake > new_tasks)
do_wake = new_tasks;
}
}
}
finish:
task->taskgroup = taskgroup->prev;
gomp_sem_destroy (&taskgroup->taskgroup_sem);
free (taskgroup);
}
int
omp_get_num_threads (void)
{
struct gomp_team *team = gomp_thread ()->ts.team;
return team ? team->nthreads : 1;
}
bool
gomp_iter_ull_dynamic_next (gomp_ull *pstart, gomp_ull *pend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_work_share *ws = thr->ts.work_share;
gomp_ull start, end, nend, chunk;
end = ws->end_ull;
chunk = ws->chunk_size_ull;
if (__builtin_expect (ws->mode & 1, 1))
{
gomp_ull tmp = __sync_fetch_and_add (&ws->next_ull, chunk);
if (__builtin_expect (ws->mode & 2, 0) == 0)
{
if (tmp >= end)
return false;
nend = tmp + chunk;
if (nend > end)
nend = end;
*pstart = tmp;
*pend = nend;
return true;
}
else
{
if (tmp <= end)
return false;
nend = tmp + chunk;
if (nend < end)
nend = end;
*pstart = tmp;
*pend = nend;
return true;
}
}
start = ws->next_ull;
while (1)
{
gomp_ull left = end - start;
gomp_ull tmp;
if (start == end)
return false;
if (__builtin_expect (ws->mode & 2, 0))
{
if (chunk < left)
chunk = left;
}
else
{
if (chunk > left)
chunk = left;
}
nend = start + chunk;
tmp = __sync_val_compare_and_swap (&ws->next_ull, start, nend);
if (__builtin_expect (tmp == start, 1))
break;
start = tmp;
}
*pstart = start;
*pend = nend;
return true;
}
unsigned
gomp_resolve_num_threads (unsigned specified, unsigned count)
{
struct gomp_thread *thread = gomp_thread();
struct gomp_task_icv *icv;
unsigned threads_requested, max_num_threads, num_threads;
unsigned long remaining;
icv = gomp_icv (false);
if (specified == 1)
return 1;
else if (thread->ts.active_level >= 1 && !icv->nest_var)
return 1;
else if (thread->ts.active_level >= gomp_max_active_levels_var)
return 1;
/* If NUM_THREADS not specified, use nthreads_var. */
if (specified == 0)
threads_requested = icv->nthreads_var;
else
threads_requested = specified;
max_num_threads = threads_requested;
/* If dynamic threads are enabled, bound the number of threads
that we launch. */
if (icv->dyn_var)
{
unsigned dyn = gomp_dynamic_max_threads ();
if (dyn < max_num_threads)
max_num_threads = dyn;
/* Optimization for parallel sections. */
if (count && count < max_num_threads)
max_num_threads = count;
}
/* ULONG_MAX stands for infinity. */
if (__builtin_expect (gomp_thread_limit_var == ULONG_MAX, 1)
|| max_num_threads == 1)
return max_num_threads;
#ifdef HAVE_SYNC_BUILTINS
do
{
remaining = gomp_remaining_threads_count;
num_threads = max_num_threads;
if (num_threads > remaining)
num_threads = remaining + 1;
}
while (__sync_val_compare_and_swap (&gomp_remaining_threads_count,
remaining, remaining - num_threads + 1)
!= remaining);
#else
gomp_mutex_lock (&gomp_remaining_threads_lock);
num_threads = max_num_threads;
remaining = gomp_remaining_threads_count;
if (num_threads > remaining)
num_threads = remaining + 1;
gomp_remaining_threads_count -= num_threads - 1;
gomp_mutex_unlock (&gomp_remaining_threads_lock);
#endif
return num_threads;
}
void
gomp_task_maybe_wait_for_dependencies (void **depend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_task *task = thr->task;
struct gomp_team *team = thr->ts.team;
struct gomp_task_depend_entry elem, *ent = NULL;
struct gomp_taskwait taskwait;
struct gomp_task *last_parent_depends_on = NULL;
size_t ndepend = (uintptr_t) depend[0];
size_t nout = (uintptr_t) depend[1];
size_t i;
size_t num_awaited = 0;
struct gomp_task *child_task = NULL;
struct gomp_task *to_free = NULL;
int do_wake = 0;
gomp_mutex_lock (&team->task_lock);
for (i = 0; i < ndepend; i++)
{
elem.addr = depend[i + 2];
ent = htab_find (task->depend_hash, &elem);
for (; ent; ent = ent->next)
if (i >= nout && ent->is_in)
continue;
else
{
struct gomp_task *tsk = ent->task;
if (!tsk->parent_depends_on)
{
tsk->parent_depends_on = true;
++num_awaited;
/* If a task we need to wait for is not already
running and is ready to be scheduled, move it to
front, so that we run it as soon as possible.
We rearrange the children queue such that all
parent_depends_on tasks are first, and
last_parent_depends_on points to the last such task
we rearranged. For example, given the following
children where PD[123] are the parent_depends_on
tasks:
task->children
|
V
C1 -> C2 -> C3 -> PD1 -> PD2 -> PD3 -> C4
We rearrange such that:
task->children
| +--- last_parent_depends_on
| |
V V
PD1 -> PD2 -> PD3 -> C1 -> C2 -> C3 -> C4
*/
if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING)
{
if (last_parent_depends_on)
{
tsk->prev_child->next_child = tsk->next_child;
tsk->next_child->prev_child = tsk->prev_child;
tsk->prev_child = last_parent_depends_on;
tsk->next_child = last_parent_depends_on->next_child;
tsk->prev_child->next_child = tsk;
tsk->next_child->prev_child = tsk;
}
else if (tsk != task->children)
{
tsk->prev_child->next_child = tsk->next_child;
tsk->next_child->prev_child = tsk->prev_child;
tsk->prev_child = task->children->prev_child;
tsk->next_child = task->children;
task->children = tsk;
tsk->prev_child->next_child = tsk;
tsk->next_child->prev_child = tsk;
}
last_parent_depends_on = tsk;
}
}
}
}
if (num_awaited == 0)
{
gomp_mutex_unlock (&team->task_lock);
return;
}
memset (&taskwait, 0, sizeof (taskwait));
taskwait.n_depend = num_awaited;
taskwait.last_parent_depends_on = last_parent_depends_on;
gomp_sem_init (&taskwait.taskwait_sem, 0);
task->taskwait = &taskwait;
while (1)
{
bool cancelled = false;
if (taskwait.n_depend == 0)
{
task->taskwait = NULL;
gomp_mutex_unlock (&team->task_lock);
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
}
gomp_sem_destroy (&taskwait.taskwait_sem);
return;
}
if (task->children->kind == GOMP_TASK_WAITING)
{
child_task = task->children;
cancelled
= gomp_task_run_pre (child_task, task, team);
if (__builtin_expect (cancelled, 0))
{
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
goto finish_cancelled;
}
}
else
/* All tasks we are waiting for are already running
in other threads. Wait for them. */
taskwait.in_depend_wait = true;
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
{
gomp_team_barrier_wake (&team->barrier, do_wake);
do_wake = 0;
}
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
if (child_task)
{
thr->task = child_task;
child_task->fn (child_task->fn_data);
thr->task = task;
}
else
gomp_sem_wait (&taskwait.taskwait_sem);
gomp_mutex_lock (&team->task_lock);
if (child_task)
{
finish_cancelled:;
size_t new_tasks
= gomp_task_run_post_handle_depend (child_task, team);
if (child_task->parent_depends_on)
--taskwait.n_depend;
/* Remove child_task from sibling list. */
child_task->prev_child->next_child = child_task->next_child;
child_task->next_child->prev_child = child_task->prev_child;
if (task->children == child_task)
{
if (child_task->next_child != child_task)
task->children = child_task->next_child;
else
task->children = NULL;
}
gomp_clear_parent (child_task->children);
gomp_task_run_post_remove_taskgroup (child_task);
to_free = child_task;
child_task = NULL;
team->task_count--;
if (new_tasks > 1)
{
do_wake = team->nthreads - team->task_running_count
- !task->in_tied_task;
if (do_wake > new_tasks)
do_wake = new_tasks;
}
}
}
}
void
gomp_create_target_task (struct gomp_device_descr *devicep,
void (*fn) (void *), size_t mapnum, void **hostaddrs,
size_t *sizes, unsigned short *kinds,
unsigned int flags, void **depend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
/* If parallel or taskgroup has been cancelled, don't start new tasks. */
if (team
&& (gomp_team_barrier_cancelled (&team->barrier)
|| (thr->task->taskgroup && thr->task->taskgroup->cancelled)))
return;
struct gomp_target_task *ttask;
struct gomp_task *task;
struct gomp_task *parent = thr->task;
struct gomp_taskgroup *taskgroup = parent->taskgroup;
bool do_wake;
size_t depend_size = 0;
if (depend != NULL)
depend_size = ((uintptr_t) depend[0]
* sizeof (struct gomp_task_depend_entry));
task = gomp_malloc (sizeof (*task) + depend_size
+ sizeof (*ttask)
+ mapnum * (sizeof (void *) + sizeof (size_t)
+ sizeof (unsigned short)));
gomp_init_task (task, parent, gomp_icv (false));
task->kind = GOMP_TASK_WAITING;
task->in_tied_task = parent->in_tied_task;
task->taskgroup = taskgroup;
ttask = (struct gomp_target_task *) &task->depend[(uintptr_t) depend[0]];
ttask->devicep = devicep;
ttask->fn = fn;
ttask->mapnum = mapnum;
memcpy (ttask->hostaddrs, hostaddrs, mapnum * sizeof (void *));
ttask->sizes = (size_t *) &ttask->hostaddrs[mapnum];
memcpy (ttask->sizes, sizes, mapnum * sizeof (size_t));
ttask->kinds = (unsigned short *) &ttask->sizes[mapnum];
memcpy (ttask->kinds, kinds, mapnum * sizeof (unsigned short));
ttask->flags = flags;
task->fn = gomp_target_task_fn;
task->fn_data = ttask;
task->final_task = 0;
gomp_mutex_lock (&team->task_lock);
/* If parallel or taskgroup has been cancelled, don't start new tasks. */
if (__builtin_expect (gomp_team_barrier_cancelled (&team->barrier)
|| (taskgroup && taskgroup->cancelled), 0))
{
gomp_mutex_unlock (&team->task_lock);
gomp_finish_task (task);
free (task);
return;
}
if (taskgroup)
taskgroup->num_children++;
if (depend_size)
{
gomp_task_handle_depend (task, parent, depend);
if (task->num_dependees)
{
gomp_mutex_unlock (&team->task_lock);
return;
}
}
if (parent->children)
{
task->next_child = parent->children;
task->prev_child = parent->children->prev_child;
task->next_child->prev_child = task;
task->prev_child->next_child = task;
}
else
{
task->next_child = task;
task->prev_child = task;
}
parent->children = task;
if (taskgroup)
{
/* If applicable, place task into its taskgroup. */
if (taskgroup->children)
{
task->next_taskgroup = taskgroup->children;
task->prev_taskgroup = taskgroup->children->prev_taskgroup;
task->next_taskgroup->prev_taskgroup = task;
task->prev_taskgroup->next_taskgroup = task;
}
else
{
task->next_taskgroup = task;
task->prev_taskgroup = task;
}
taskgroup->children = task;
}
if (team->task_queue)
{
task->next_queue = team->task_queue;
task->prev_queue = team->task_queue->prev_queue;
task->next_queue->prev_queue = task;
task->prev_queue->next_queue = task;
}
else
{
task->next_queue = task;
task->prev_queue = task;
team->task_queue = task;
}
++team->task_count;
++team->task_queued_count;
gomp_team_barrier_set_task_pending (&team->barrier);
do_wake = team->task_running_count + !parent->in_tied_task
< team->nthreads;
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
gomp_team_barrier_wake (&team->barrier, 1);
}
void
GOMP_task (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *),
long arg_size, long arg_align, bool if_clause, unsigned flags,
void **depend, int priority)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
#ifdef HAVE_BROKEN_POSIX_SEMAPHORES
/* If pthread_mutex_* is used for omp_*lock*, then each task must be
tied to one thread all the time. This means UNTIED tasks must be
tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN
might be running on different thread than FN. */
if (cpyfn)
if_clause = false;
flags &= ~GOMP_TASK_FLAG_UNTIED;
#endif
/* If parallel or taskgroup has been cancelled, don't start new tasks. */
if (team
&& (gomp_team_barrier_cancelled (&team->barrier)
|| (thr->task->taskgroup && thr->task->taskgroup->cancelled)))
return;
if ((flags & GOMP_TASK_FLAG_PRIORITY) == 0)
priority = 0;
/* FIXME, use priority. */
(void) priority;
if (!if_clause || team == NULL
|| (thr->task && thr->task->final_task)
|| team->task_count > 64 * team->nthreads)
{
struct gomp_task task;
/* If there are depend clauses and earlier deferred sibling tasks
with depend clauses, check if there isn't a dependency. If there
is, we need to wait for them. There is no need to handle
depend clauses for non-deferred tasks other than this, because
the parent task is suspended until the child task finishes and thus
it can't start further child tasks. */
if ((flags & GOMP_TASK_FLAG_DEPEND)
&& thr->task && thr->task->depend_hash)
gomp_task_maybe_wait_for_dependencies (depend);
gomp_init_task (&task, thr->task, gomp_icv (false));
task.kind = GOMP_TASK_UNDEFERRED;
task.final_task = (thr->task && thr->task->final_task)
|| (flags & GOMP_TASK_FLAG_FINAL);
if (thr->task)
{
task.in_tied_task = thr->task->in_tied_task;
task.taskgroup = thr->task->taskgroup;
}
thr->task = &task;
if (__builtin_expect (cpyfn != NULL, 0))
{
char buf[arg_size + arg_align - 1];
char *arg = (char *) (((uintptr_t) buf + arg_align - 1)
& ~(uintptr_t) (arg_align - 1));
cpyfn (arg, data);
fn (arg);
}
else
fn (data);
/* Access to "children" is normally done inside a task_lock
mutex region, but the only way this particular task.children
can be set is if this thread's task work function (fn)
creates children. So since the setter is *this* thread, we
need no barriers here when testing for non-NULL. We can have
task.children set by the current thread then changed by a
child thread, but seeing a stale non-NULL value is not a
problem. Once past the task_lock acquisition, this thread
will see the real value of task.children. */
if (task.children != NULL)
{
gomp_mutex_lock (&team->task_lock);
gomp_clear_parent (task.children);
gomp_mutex_unlock (&team->task_lock);
}
gomp_end_task ();
}
else
{
struct gomp_task *task;
struct gomp_task *parent = thr->task;
struct gomp_taskgroup *taskgroup = parent->taskgroup;
char *arg;
bool do_wake;
size_t depend_size = 0;
if (flags & GOMP_TASK_FLAG_DEPEND)
depend_size = ((uintptr_t) depend[0]
* sizeof (struct gomp_task_depend_entry));
task = gomp_malloc (sizeof (*task) + depend_size
+ arg_size + arg_align - 1);
arg = (char *) (((uintptr_t) (task + 1) + depend_size + arg_align - 1)
& ~(uintptr_t) (arg_align - 1));
gomp_init_task (task, parent, gomp_icv (false));
task->kind = GOMP_TASK_UNDEFERRED;
task->in_tied_task = parent->in_tied_task;
task->taskgroup = taskgroup;
thr->task = task;
if (cpyfn)
{
cpyfn (arg, data);
task->copy_ctors_done = true;
}
else
memcpy (arg, data, arg_size);
thr->task = parent;
task->kind = GOMP_TASK_WAITING;
task->fn = fn;
task->fn_data = arg;
task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1;
gomp_mutex_lock (&team->task_lock);
/* If parallel or taskgroup has been cancelled, don't start new
tasks. */
if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier)
|| (taskgroup && taskgroup->cancelled))
&& !task->copy_ctors_done, 0))
{
gomp_mutex_unlock (&team->task_lock);
gomp_finish_task (task);
free (task);
return;
}
if (taskgroup)
taskgroup->num_children++;
if (depend_size)
{
gomp_task_handle_depend (task, parent, depend);
if (task->num_dependees)
{
gomp_mutex_unlock (&team->task_lock);
return;
}
}
if (parent->children)
{
task->next_child = parent->children;
task->prev_child = parent->children->prev_child;
task->next_child->prev_child = task;
task->prev_child->next_child = task;
}
else
{
task->next_child = task;
task->prev_child = task;
}
parent->children = task;
if (taskgroup)
{
/* If applicable, place task into its taskgroup. */
if (taskgroup->children)
{
task->next_taskgroup = taskgroup->children;
task->prev_taskgroup = taskgroup->children->prev_taskgroup;
task->next_taskgroup->prev_taskgroup = task;
task->prev_taskgroup->next_taskgroup = task;
}
else
{
task->next_taskgroup = task;
task->prev_taskgroup = task;
}
taskgroup->children = task;
}
if (team->task_queue)
{
task->next_queue = team->task_queue;
task->prev_queue = team->task_queue->prev_queue;
task->next_queue->prev_queue = task;
task->prev_queue->next_queue = task;
}
else
{
task->next_queue = task;
task->prev_queue = task;
team->task_queue = task;
}
++team->task_count;
++team->task_queued_count;
gomp_team_barrier_set_task_pending (&team->barrier);
do_wake = team->task_running_count + !parent->in_tied_task
< team->nthreads;
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
gomp_team_barrier_wake (&team->barrier, 1);
}
}
int
gomp_iter_ull_static_next (gomp_ull *pstart, gomp_ull *pend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_work_share *ws = thr->ts.work_share;
unsigned long nthreads = team ? team->nthreads : 1;
if (thr->ts.static_trip == -1)
return -1;
/* Quick test for degenerate teams and orphaned constructs. */
if (nthreads == 1)
{
*pstart = ws->next_ull;
*pend = ws->end_ull;
thr->ts.static_trip = -1;
return ws->next_ull == ws->end_ull;
}
/* We interpret chunk_size zero as "unspecified", which means that we
should break up the iterations such that each thread makes only one
trip through the outer loop. */
if (ws->chunk_size_ull == 0)
{
gomp_ull n, q, i, t, s0, e0, s, e;
if (thr->ts.static_trip > 0)
return 1;
/* Compute the total number of iterations. */
if (__builtin_expect (ws->mode, 0) == 0)
n = (ws->end_ull - ws->next_ull + ws->incr_ull - 1) / ws->incr_ull;
else
n = (ws->next_ull - ws->end_ull - ws->incr_ull - 1) / -ws->incr_ull;
i = thr->ts.team_id;
/* Compute the "zero-based" start and end points. That is, as
if the loop began at zero and incremented by one. */
q = n / nthreads;
t = n % nthreads;
if (i < t)
{
t = 0;
q++;
}
s0 = q * i + t;
e0 = s0 + q;
/* Notice when no iterations allocated for this thread. */
if (s0 >= e0)
{
thr->ts.static_trip = 1;
return 1;
}
/* Transform these to the actual start and end numbers. */
s = s0 * ws->incr_ull + ws->next_ull;
e = e0 * ws->incr_ull + ws->next_ull;
*pstart = s;
*pend = e;
thr->ts.static_trip = (e0 == n ? -1 : 1);
return 0;
}
else
{
gomp_ull n, s0, e0, i, c, s, e;
/* Otherwise, each thread gets exactly chunk_size iterations
(if available) each time through the loop. */
if (__builtin_expect (ws->mode, 0) == 0)
n = (ws->end_ull - ws->next_ull + ws->incr_ull - 1) / ws->incr_ull;
else
n = (ws->next_ull - ws->end_ull - ws->incr_ull - 1) / -ws->incr_ull;
i = thr->ts.team_id;
c = ws->chunk_size_ull;
/* Initial guess is a C sized chunk positioned nthreads iterations
in, offset by our thread number. */
s0 = (thr->ts.static_trip * (gomp_ull) nthreads + i) * c;
e0 = s0 + c;
/* Detect overflow. */
if (s0 >= n)
return 1;
if (e0 > n)
e0 = n;
/* Transform these to the actual start and end numbers. */
s = s0 * ws->incr_ull + ws->next_ull;
e = e0 * ws->incr_ull + ws->next_ull;
*pstart = s;
*pend = e;
if (e0 == n)
thr->ts.static_trip = -1;
else
thr->ts.static_trip++;
return 0;
}
}
int
omp_get_active_level (void)
{
return gomp_thread ()->ts.active_level;
}
int
omp_get_thread_num (void)
{
return gomp_thread ()->ts.team_id;
}
void
GOMP_taskwait (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
struct gomp_task *child_task = NULL;
struct gomp_task *to_free = NULL;
struct gomp_taskwait taskwait;
int do_wake = 0;
/* The acquire barrier on load of task->children here synchronizes
with the write of a NULL in gomp_task_run_post_remove_parent. It is
not necessary that we synchronize with other non-NULL writes at
this point, but we must ensure that all writes to memory by a
child thread task work function are seen before we exit from
GOMP_taskwait. */
if (task == NULL
|| __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL)
return;
memset (&taskwait, 0, sizeof (taskwait));
gomp_mutex_lock (&team->task_lock);
while (1)
{
bool cancelled = false;
if (task->children == NULL)
{
bool destroy_taskwait = task->taskwait != NULL;
task->taskwait = NULL;
gomp_mutex_unlock (&team->task_lock);
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
}
if (destroy_taskwait)
gomp_sem_destroy (&taskwait.taskwait_sem);
return;
}
if (task->children->kind == GOMP_TASK_WAITING)
{
child_task = task->children;
cancelled
= gomp_task_run_pre (child_task, task, team);
if (__builtin_expect (cancelled, 0))
{
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
goto finish_cancelled;
}
}
else
{
/* All tasks we are waiting for are already running
in other threads. Wait for them. */
if (task->taskwait == NULL)
{
taskwait.in_depend_wait = false;
gomp_sem_init (&taskwait.taskwait_sem, 0);
task->taskwait = &taskwait;
}
taskwait.in_taskwait = true;
}
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
{
gomp_team_barrier_wake (&team->barrier, do_wake);
do_wake = 0;
}
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
if (child_task)
{
thr->task = child_task;
child_task->fn (child_task->fn_data);
thr->task = task;
}
else
gomp_sem_wait (&taskwait.taskwait_sem);
gomp_mutex_lock (&team->task_lock);
if (child_task)
{
finish_cancelled:;
size_t new_tasks
= gomp_task_run_post_handle_depend (child_task, team);
/* Remove child_task from children list, and set up the next
sibling to be run. */
child_task->prev_child->next_child = child_task->next_child;
child_task->next_child->prev_child = child_task->prev_child;
if (task->children == child_task)
{
if (child_task->next_child != child_task)
task->children = child_task->next_child;
else
task->children = NULL;
}
/* Orphan all the children of CHILD_TASK. */
gomp_clear_parent (child_task->children);
/* Remove CHILD_TASK from its taskgroup. */
gomp_task_run_post_remove_taskgroup (child_task);
to_free = child_task;
child_task = NULL;
team->task_count--;
if (new_tasks > 1)
{
do_wake = team->nthreads - team->task_running_count
- !task->in_tied_task;
if (do_wake > new_tasks)
do_wake = new_tasks;
}
}
}
}
int
omp_get_level (void)
{
return gomp_thread ()->ts.level;
}
static inline int gomp_tid (void)
{
return (int) gomp_thread ();
}
unsigned
gomp_resolve_num_threads (unsigned specified, unsigned count)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_task_icv *icv;
unsigned threads_requested, max_num_threads, num_threads;
unsigned long busy;
struct gomp_thread_pool *pool;
icv = gomp_icv (false);
if (specified == 1)
return 1;
else if (thr->ts.active_level >= 1 && !icv->nest_var)
return 1;
else if (thr->ts.active_level >= gomp_max_active_levels_var)
return 1;
/* If NUM_THREADS not specified, use nthreads_var. */
if (specified == 0)
threads_requested = icv->nthreads_var;
else
threads_requested = specified;
max_num_threads = threads_requested;
/* If dynamic threads are enabled, bound the number of threads
that we launch. */
if (icv->dyn_var)
{
unsigned dyn = gomp_dynamic_max_threads ();
if (dyn < max_num_threads)
max_num_threads = dyn;
/* Optimization for parallel sections. */
if (count && count < max_num_threads)
max_num_threads = count;
}
/* UINT_MAX stands for infinity. */
if (__builtin_expect (icv->thread_limit_var == UINT_MAX, 1)
|| max_num_threads == 1)
return max_num_threads;
/* The threads_busy counter lives in thread_pool, if there
isn't a thread_pool yet, there must be just one thread
in the contention group. If thr->team is NULL, this isn't
nested parallel, so there is just one thread in the
contention group as well, no need to handle it atomically. */
pool = thr->thread_pool;
if (thr->ts.team == NULL || pool == NULL)
{
num_threads = max_num_threads;
if (num_threads > icv->thread_limit_var)
num_threads = icv->thread_limit_var;
if (pool)
pool->threads_busy = num_threads;
return num_threads;
}
#ifdef HAVE_SYNC_BUILTINS
do
{
busy = pool->threads_busy;
num_threads = max_num_threads;
if (icv->thread_limit_var - busy + 1 < num_threads)
num_threads = icv->thread_limit_var - busy + 1;
}
while (__sync_val_compare_and_swap (&pool->threads_busy,
busy, busy + num_threads - 1)
!= busy);
#else
gomp_mutex_lock (&gomp_managed_threads_lock);
num_threads = max_num_threads;
busy = pool->threads_busy;
if (icv->thread_limit_var - busy + 1 < num_threads)
num_threads = icv->thread_limit_var - busy + 1;
pool->threads_busy += num_threads - 1;
gomp_mutex_unlock (&gomp_managed_threads_lock);
#endif
return num_threads;
}
void
gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_work_share *ws = thr->ts.work_share;
unsigned int i, bits[MAX_COLLAPSED_BITS], num_bits = 0;
unsigned long ent, num_ents, elt_sz, shift_sz;
struct gomp_doacross_work_share *doacross;
if (team == NULL || team->nthreads == 1)
return;
for (i = 0; i < ncounts; i++)
{
/* If any count is 0, GOMP_doacross_{post,wait} can't be called. */
if (counts[i] == 0)
return;
if (num_bits <= MAX_COLLAPSED_BITS)
{
unsigned int this_bits;
if (counts[i] == 1)
this_bits = 1;
else
this_bits = __SIZEOF_LONG__ * __CHAR_BIT__
- __builtin_clzl (counts[i] - 1);
if (num_bits + this_bits <= MAX_COLLAPSED_BITS)
{
bits[i] = this_bits;
num_bits += this_bits;
}
else
num_bits = MAX_COLLAPSED_BITS + 1;
}
}
if (ws->sched == GFS_STATIC)
num_ents = team->nthreads;
else if (ws->sched == GFS_GUIDED)
num_ents = counts[0];
else
num_ents = (counts[0] - 1) / chunk_size + 1;
if (num_bits <= MAX_COLLAPSED_BITS)
{
elt_sz = sizeof (unsigned long);
shift_sz = ncounts * sizeof (unsigned int);
}
else
{
elt_sz = sizeof (unsigned long) * ncounts;
shift_sz = 0;
}
elt_sz = (elt_sz + 63) & ~63UL;
doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz
+ shift_sz);
doacross->chunk_size = chunk_size;
doacross->elt_sz = elt_sz;
doacross->ncounts = ncounts;
doacross->flattened = false;
doacross->array = (unsigned char *)
((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
& ~(uintptr_t) 63);
if (num_bits <= MAX_COLLAPSED_BITS)
{
unsigned int shift_count = 0;
doacross->flattened = true;
for (i = ncounts; i > 0; i--)
{
doacross->shift_counts[i - 1] = shift_count;
shift_count += bits[i - 1];
}
for (ent = 0; ent < num_ents; ent++)
*(unsigned long *) (doacross->array + ent * elt_sz) = 0;
}
else
for (ent = 0; ent < num_ents; ent++)
memset (doacross->array + ent * elt_sz, '\0',
sizeof (unsigned long) * ncounts);
if (ws->sched == GFS_STATIC && chunk_size == 0)
{
unsigned long q = counts[0] / num_ents;
unsigned long t = counts[0] % num_ents;
doacross->boundary = t * (q + 1);
doacross->q = q;
doacross->t = t;
}
ws->doacross = doacross;
}
