struct gomp_team *
gomp_new_team (unsigned nthreads)
{
struct gomp_team *team;
int i;
team = get_last_team (nthreads);
if (team == NULL)
{
size_t extra = sizeof (team->ordered_release[0])
+ sizeof (team->implicit_task[0]);
team = gomp_malloc (sizeof (*team) + nthreads * extra);
#ifndef HAVE_SYNC_BUILTINS
gomp_mutex_init (&team->work_share_list_free_lock);
#endif
gomp_barrier_init (&team->barrier, nthreads);
gomp_mutex_init (&team->task_lock);
team->nthreads = nthreads;
}
team->work_share_chunk = 8;
#ifdef HAVE_SYNC_BUILTINS
team->single_count = 0;
#endif
team->work_shares_to_free = &team->work_shares[0];
gomp_init_work_share (&team->work_shares[0], false, nthreads);
team->work_shares[0].next_alloc = NULL;
team->work_share_list_free = NULL;
team->work_share_list_alloc = &team->work_shares[1];
for (i = 1; i < 7; i++)
team->work_shares[i].next_free = &team->work_shares[i + 1];
team->work_shares[i].next_free = NULL;
gomp_sem_init (&team->master_release, 0);
team->ordered_release = (void *) &team->implicit_task[nthreads];
team->ordered_release[0] = &team->master_release;
priority_queue_init (&team->task_queue);
team->task_count = 0;
team->task_queued_count = 0;
team->task_running_count = 0;
team->work_share_cancelled = 0;
team->team_cancelled = 0;
return team;
}
initialize_team (void)
{
struct gomp_thread *thr;
#ifndef HAVE_TLS
static struct gomp_thread initial_thread_tls_data;
pthread_key_create (&gomp_tls_key, NULL);
pthread_setspecific (gomp_tls_key, &initial_thread_tls_data);
#endif
#ifdef HAVE_TLS
thr = &gomp_tls_data;
#else
thr = &initial_thread_tls_data;
#endif
gomp_sem_init (&thr->release, 0);
}
struct gomp_team *
gomp_new_team (unsigned nthreads)
{
struct gomp_team *team;
size_t size;
int i;
size = sizeof (*team) + nthreads * (sizeof (team->ordered_release[0])
+ sizeof (team->implicit_task[0]));
team = gomp_malloc (size);
team->work_share_chunk = 8;
#ifdef HAVE_SYNC_BUILTINS
team->single_count = 0;
#else
gomp_mutex_init (&team->work_share_list_free_lock);
#endif
gomp_init_work_share (&team->work_shares[0], false, nthreads);
team->work_shares[0].next_alloc = NULL;
team->work_share_list_free = NULL;
team->work_share_list_alloc = &team->work_shares[1];
for (i = 1; i < 7; i++)
team->work_shares[i].next_free = &team->work_shares[i + 1];
team->work_shares[i].next_free = NULL;
team->nthreads = nthreads;
gomp_barrier_init (&team->barrier, nthreads);
gomp_sem_init (&team->master_release, 0);
team->ordered_release = (void *) &team->implicit_task[nthreads];
team->ordered_release[0] = &team->master_release;
gomp_mutex_init (&team->task_lock);
team->task_queue = NULL;
team->task_count = 0;
team->task_running_count = 0;
return team;
}
void
GOMP_taskgroup_start (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
struct gomp_taskgroup *taskgroup;
/* If team is NULL, all tasks are executed as
GOMP_TASK_UNDEFERRED tasks and thus all children tasks of
taskgroup and their descendant tasks will be finished
by the time GOMP_taskgroup_end is called. */
if (team == NULL)
return;
taskgroup = gomp_malloc (sizeof (struct gomp_taskgroup));
taskgroup->prev = task->taskgroup;
taskgroup->children = NULL;
taskgroup->in_taskgroup_wait = false;
taskgroup->cancelled = false;
taskgroup->num_children = 0;
gomp_sem_init (&taskgroup->taskgroup_sem, 0);
task->taskgroup = taskgroup;
}
static void
allocate_thread_pool_reservoir (unsigned long count, unsigned long priority,
unsigned long scheduler)
{
struct gomp_thread_pool_reservoir *res;
struct gomp_thread_pool *pools;
unsigned long i;
size_t size;
res = gomp_thread_pool_reservoirs[scheduler];
if (res != NULL)
gomp_fatal ("Multiple thread pool reservoir initialization");
size = sizeof (*res) + count * (sizeof(pools) + sizeof(*pools));
pools = gomp_malloc (size);
memset (pools, 0, size);
res = (struct gomp_thread_pool_reservoir *) (pools + count);
res->index = count;
res->priority = priority;
gomp_sem_init (&res->available, count);
gomp_mutex_init (&res->lock);
for (i = 0; i < count; ++i)
res->pools[i] = &pools[i];
gomp_thread_pool_reservoirs[scheduler] = res;
}
static struct gomp_team *
new_team (unsigned nthreads, struct gomp_work_share *work_share)
{
struct gomp_team *team;
size_t size;
size = sizeof (*team) + nthreads * sizeof (team->ordered_release[0]);
team = gomp_malloc (size);
gomp_mutex_init (&team->work_share_lock);
team->work_shares = gomp_malloc (4 * sizeof (struct gomp_work_share *));
team->generation_mask = 3;
team->oldest_live_gen = work_share == NULL;
team->num_live_gen = work_share != NULL;
team->work_shares[0] = work_share;
team->nthreads = nthreads;
gomp_barrier_init (&team->barrier, nthreads);
gomp_sem_init (&team->master_release, 0);
team->ordered_release[0] = &team->master_release;
return team;
}
static void *
gomp_thread_start (void *xdata)
{
struct gomp_thread_start_data *data = xdata;
struct gomp_thread *thr;
struct gomp_thread_pool *pool;
void (*local_fn) (void *);
void *local_data;
#if defined HAVE_TLS || defined USE_EMUTLS
thr = &gomp_tls_data;
#else
struct gomp_thread local_thr;
thr = &local_thr;
pthread_setspecific (gomp_tls_key, thr);
#endif
gomp_sem_init (&thr->release, 0);
/* Extract what we need from data. */
local_fn = data->fn;
local_data = data->fn_data;
thr->thread_pool = data->thread_pool;
thr->ts = data->ts;
thr->task = data->task;
thr->place = data->place;
thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release;
/* Make thread pool local. */
pool = thr->thread_pool;
if (data->nested)
{
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
gomp_barrier_wait (&team->barrier);
local_fn (local_data);
gomp_team_barrier_wait_final (&team->barrier);
gomp_finish_task (task);
gomp_barrier_wait_last (&team->barrier);
}
else
{
pool->threads[thr->ts.team_id] = thr;
gomp_simple_barrier_wait (&pool->threads_dock);
do
{
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
local_fn (local_data);
gomp_team_barrier_wait_final (&team->barrier);
gomp_finish_task (task);
gomp_simple_barrier_wait (&pool->threads_dock);
local_fn = thr->fn;
local_data = thr->data;
thr->fn = NULL;
}
while (local_fn);
}
gomp_sem_destroy (&thr->release);
thr->thread_pool = NULL;
thr->task = NULL;
return NULL;
}
static void *
gomp_thread_start (void *xdata)
{
struct gomp_thread_start_data *data = xdata;
struct gomp_thread *thr;
void (*local_fn) (void *);
void *local_data;
#ifdef HAVE_TLS
thr = &gomp_tls_data;
#else
struct gomp_thread local_thr;
thr = &local_thr;
pthread_setspecific (gomp_tls_key, thr);
#endif
gomp_sem_init (&thr->release, 0);
/* Extract what we need from data. */
local_fn = data->fn;
local_data = data->fn_data;
thr->ts = data->ts;
thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release;
if (data->nested)
{
gomp_barrier_wait (&thr->ts.team->barrier);
local_fn (local_data);
gomp_barrier_wait (&thr->ts.team->barrier);
}
else
{
gomp_threads[thr->ts.team_id] = thr;
gomp_barrier_wait (&gomp_threads_dock);
do
{
struct gomp_team *team;
local_fn (local_data);
/* Clear out the team and function data. This is a debugging
signal that we're in fact back in the dock. */
team = thr->ts.team;
thr->fn = NULL;
thr->data = NULL;
thr->ts.team = NULL;
thr->ts.work_share = NULL;
thr->ts.team_id = 0;
thr->ts.work_share_generation = 0;
thr->ts.static_trip = 0;
gomp_barrier_wait (&team->barrier);
gomp_barrier_wait (&gomp_threads_dock);
local_fn = thr->fn;
local_data = thr->data;
}
while (local_fn);
}
return NULL;
}
void
gomp_task_maybe_wait_for_dependencies (void **depend)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_task *task = thr->task;
struct gomp_team *team = thr->ts.team;
struct gomp_task_depend_entry elem, *ent = NULL;
struct gomp_taskwait taskwait;
struct gomp_task *last_parent_depends_on = NULL;
size_t ndepend = (uintptr_t) depend[0];
size_t nout = (uintptr_t) depend[1];
size_t i;
size_t num_awaited = 0;
struct gomp_task *child_task = NULL;
struct gomp_task *to_free = NULL;
int do_wake = 0;
gomp_mutex_lock (&team->task_lock);
for (i = 0; i < ndepend; i++)
{
elem.addr = depend[i + 2];
ent = htab_find (task->depend_hash, &elem);
for (; ent; ent = ent->next)
if (i >= nout && ent->is_in)
continue;
else
{
struct gomp_task *tsk = ent->task;
if (!tsk->parent_depends_on)
{
tsk->parent_depends_on = true;
++num_awaited;
/* If a task we need to wait for is not already
running and is ready to be scheduled, move it to
front, so that we run it as soon as possible.
We rearrange the children queue such that all
parent_depends_on tasks are first, and
last_parent_depends_on points to the last such task
we rearranged. For example, given the following
children where PD[123] are the parent_depends_on
tasks:
task->children
|
V
C1 -> C2 -> C3 -> PD1 -> PD2 -> PD3 -> C4
We rearrange such that:
task->children
| +--- last_parent_depends_on
| |
V V
PD1 -> PD2 -> PD3 -> C1 -> C2 -> C3 -> C4
*/
if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING)
{
if (last_parent_depends_on)
{
tsk->prev_child->next_child = tsk->next_child;
tsk->next_child->prev_child = tsk->prev_child;
tsk->prev_child = last_parent_depends_on;
tsk->next_child = last_parent_depends_on->next_child;
tsk->prev_child->next_child = tsk;
tsk->next_child->prev_child = tsk;
}
else if (tsk != task->children)
{
tsk->prev_child->next_child = tsk->next_child;
tsk->next_child->prev_child = tsk->prev_child;
tsk->prev_child = task->children->prev_child;
tsk->next_child = task->children;
task->children = tsk;
tsk->prev_child->next_child = tsk;
tsk->next_child->prev_child = tsk;
}
last_parent_depends_on = tsk;
}
}
}
}
if (num_awaited == 0)
{
gomp_mutex_unlock (&team->task_lock);
return;
}
memset (&taskwait, 0, sizeof (taskwait));
taskwait.n_depend = num_awaited;
taskwait.last_parent_depends_on = last_parent_depends_on;
gomp_sem_init (&taskwait.taskwait_sem, 0);
task->taskwait = &taskwait;
while (1)
{
bool cancelled = false;
if (taskwait.n_depend == 0)
{
task->taskwait = NULL;
gomp_mutex_unlock (&team->task_lock);
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
}
gomp_sem_destroy (&taskwait.taskwait_sem);
return;
}
if (task->children->kind == GOMP_TASK_WAITING)
{
child_task = task->children;
cancelled
= gomp_task_run_pre (child_task, task, team);
if (__builtin_expect (cancelled, 0))
{
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
goto finish_cancelled;
}
}
else
/* All tasks we are waiting for are already running
in other threads. Wait for them. */
taskwait.in_depend_wait = true;
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
{
gomp_team_barrier_wake (&team->barrier, do_wake);
do_wake = 0;
}
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
if (child_task)
{
thr->task = child_task;
child_task->fn (child_task->fn_data);
thr->task = task;
}
else
gomp_sem_wait (&taskwait.taskwait_sem);
gomp_mutex_lock (&team->task_lock);
if (child_task)
{
finish_cancelled:;
size_t new_tasks
= gomp_task_run_post_handle_depend (child_task, team);
if (child_task->parent_depends_on)
--taskwait.n_depend;
/* Remove child_task from sibling list. */
child_task->prev_child->next_child = child_task->next_child;
child_task->next_child->prev_child = child_task->prev_child;
if (task->children == child_task)
{
if (child_task->next_child != child_task)
task->children = child_task->next_child;
else
task->children = NULL;
}
gomp_clear_parent (child_task->children);
gomp_task_run_post_remove_taskgroup (child_task);
to_free = child_task;
child_task = NULL;
team->task_count--;
if (new_tasks > 1)
{
do_wake = team->nthreads - team->task_running_count
- !task->in_tied_task;
if (do_wake > new_tasks)
do_wake = new_tasks;
}
}
}
}
void
GOMP_taskwait (void)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
struct gomp_task *child_task = NULL;
struct gomp_task *to_free = NULL;
struct gomp_taskwait taskwait;
int do_wake = 0;
/* The acquire barrier on load of task->children here synchronizes
with the write of a NULL in gomp_task_run_post_remove_parent. It is
not necessary that we synchronize with other non-NULL writes at
this point, but we must ensure that all writes to memory by a
child thread task work function are seen before we exit from
GOMP_taskwait. */
if (task == NULL
|| __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL)
return;
memset (&taskwait, 0, sizeof (taskwait));
gomp_mutex_lock (&team->task_lock);
while (1)
{
bool cancelled = false;
if (task->children == NULL)
{
bool destroy_taskwait = task->taskwait != NULL;
task->taskwait = NULL;
gomp_mutex_unlock (&team->task_lock);
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
}
if (destroy_taskwait)
gomp_sem_destroy (&taskwait.taskwait_sem);
return;
}
if (task->children->kind == GOMP_TASK_WAITING)
{
child_task = task->children;
cancelled
= gomp_task_run_pre (child_task, task, team);
if (__builtin_expect (cancelled, 0))
{
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
goto finish_cancelled;
}
}
else
{
/* All tasks we are waiting for are already running
in other threads. Wait for them. */
if (task->taskwait == NULL)
{
taskwait.in_depend_wait = false;
gomp_sem_init (&taskwait.taskwait_sem, 0);
task->taskwait = &taskwait;
}
taskwait.in_taskwait = true;
}
gomp_mutex_unlock (&team->task_lock);
if (do_wake)
{
gomp_team_barrier_wake (&team->barrier, do_wake);
do_wake = 0;
}
if (to_free)
{
gomp_finish_task (to_free);
free (to_free);
to_free = NULL;
}
if (child_task)
{
thr->task = child_task;
child_task->fn (child_task->fn_data);
thr->task = task;
}
else
gomp_sem_wait (&taskwait.taskwait_sem);
gomp_mutex_lock (&team->task_lock);
if (child_task)
{
finish_cancelled:;
size_t new_tasks
= gomp_task_run_post_handle_depend (child_task, team);
/* Remove child_task from children list, and set up the next
sibling to be run. */
child_task->prev_child->next_child = child_task->next_child;
child_task->next_child->prev_child = child_task->prev_child;
if (task->children == child_task)
{
if (child_task->next_child != child_task)
task->children = child_task->next_child;
else
task->children = NULL;
}
/* Orphan all the children of CHILD_TASK. */
gomp_clear_parent (child_task->children);
/* Remove CHILD_TASK from its taskgroup. */
gomp_task_run_post_remove_taskgroup (child_task);
to_free = child_task;
child_task = NULL;
team->task_count--;
if (new_tasks > 1)
{
do_wake = team->nthreads - team->task_running_count
- !task->in_tied_task;
if (do_wake > new_tasks)
do_wake = new_tasks;
}
}
}
}
