struct gomp_team * gomp_new_team (unsigned nthreads) { struct gomp_team *team; int i; team = get_last_team (nthreads); if (team == NULL) { size_t extra = sizeof (team->ordered_release[0]) + sizeof (team->implicit_task[0]); team = gomp_malloc (sizeof (*team) + nthreads * extra); #ifndef HAVE_SYNC_BUILTINS gomp_mutex_init (&team->work_share_list_free_lock); #endif gomp_barrier_init (&team->barrier, nthreads); gomp_mutex_init (&team->task_lock); team->nthreads = nthreads; } team->work_share_chunk = 8; #ifdef HAVE_SYNC_BUILTINS team->single_count = 0; #endif team->work_shares_to_free = &team->work_shares[0]; gomp_init_work_share (&team->work_shares[0], false, nthreads); team->work_shares[0].next_alloc = NULL; team->work_share_list_free = NULL; team->work_share_list_alloc = &team->work_shares[1]; for (i = 1; i < 7; i++) team->work_shares[i].next_free = &team->work_shares[i + 1]; team->work_shares[i].next_free = NULL; gomp_sem_init (&team->master_release, 0); team->ordered_release = (void *) &team->implicit_task[nthreads]; team->ordered_release[0] = &team->master_release; priority_queue_init (&team->task_queue); team->task_count = 0; team->task_queued_count = 0; team->task_running_count = 0; team->work_share_cancelled = 0; team->team_cancelled = 0; return team; }
initialize_team (void) { struct gomp_thread *thr; #ifndef HAVE_TLS static struct gomp_thread initial_thread_tls_data; pthread_key_create (&gomp_tls_key, NULL); pthread_setspecific (gomp_tls_key, &initial_thread_tls_data); #endif #ifdef HAVE_TLS thr = &gomp_tls_data; #else thr = &initial_thread_tls_data; #endif gomp_sem_init (&thr->release, 0); }
struct gomp_team * gomp_new_team (unsigned nthreads) { struct gomp_team *team; size_t size; int i; size = sizeof (*team) + nthreads * (sizeof (team->ordered_release[0]) + sizeof (team->implicit_task[0])); team = gomp_malloc (size); team->work_share_chunk = 8; #ifdef HAVE_SYNC_BUILTINS team->single_count = 0; #else gomp_mutex_init (&team->work_share_list_free_lock); #endif gomp_init_work_share (&team->work_shares[0], false, nthreads); team->work_shares[0].next_alloc = NULL; team->work_share_list_free = NULL; team->work_share_list_alloc = &team->work_shares[1]; for (i = 1; i < 7; i++) team->work_shares[i].next_free = &team->work_shares[i + 1]; team->work_shares[i].next_free = NULL; team->nthreads = nthreads; gomp_barrier_init (&team->barrier, nthreads); gomp_sem_init (&team->master_release, 0); team->ordered_release = (void *) &team->implicit_task[nthreads]; team->ordered_release[0] = &team->master_release; gomp_mutex_init (&team->task_lock); team->task_queue = NULL; team->task_count = 0; team->task_running_count = 0; return team; }
void GOMP_taskgroup_start (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_taskgroup *taskgroup; /* If team is NULL, all tasks are executed as GOMP_TASK_UNDEFERRED tasks and thus all children tasks of taskgroup and their descendant tasks will be finished by the time GOMP_taskgroup_end is called. */ if (team == NULL) return; taskgroup = gomp_malloc (sizeof (struct gomp_taskgroup)); taskgroup->prev = task->taskgroup; taskgroup->children = NULL; taskgroup->in_taskgroup_wait = false; taskgroup->cancelled = false; taskgroup->num_children = 0; gomp_sem_init (&taskgroup->taskgroup_sem, 0); task->taskgroup = taskgroup; }
static void allocate_thread_pool_reservoir (unsigned long count, unsigned long priority, unsigned long scheduler) { struct gomp_thread_pool_reservoir *res; struct gomp_thread_pool *pools; unsigned long i; size_t size; res = gomp_thread_pool_reservoirs[scheduler]; if (res != NULL) gomp_fatal ("Multiple thread pool reservoir initialization"); size = sizeof (*res) + count * (sizeof(pools) + sizeof(*pools)); pools = gomp_malloc (size); memset (pools, 0, size); res = (struct gomp_thread_pool_reservoir *) (pools + count); res->index = count; res->priority = priority; gomp_sem_init (&res->available, count); gomp_mutex_init (&res->lock); for (i = 0; i < count; ++i) res->pools[i] = &pools[i]; gomp_thread_pool_reservoirs[scheduler] = res; }
static struct gomp_team * new_team (unsigned nthreads, struct gomp_work_share *work_share) { struct gomp_team *team; size_t size; size = sizeof (*team) + nthreads * sizeof (team->ordered_release[0]); team = gomp_malloc (size); gomp_mutex_init (&team->work_share_lock); team->work_shares = gomp_malloc (4 * sizeof (struct gomp_work_share *)); team->generation_mask = 3; team->oldest_live_gen = work_share == NULL; team->num_live_gen = work_share != NULL; team->work_shares[0] = work_share; team->nthreads = nthreads; gomp_barrier_init (&team->barrier, nthreads); gomp_sem_init (&team->master_release, 0); team->ordered_release[0] = &team->master_release; return team; }
static void * gomp_thread_start (void *xdata) { struct gomp_thread_start_data *data = xdata; struct gomp_thread *thr; struct gomp_thread_pool *pool; void (*local_fn) (void *); void *local_data; #if defined HAVE_TLS || defined USE_EMUTLS thr = &gomp_tls_data; #else struct gomp_thread local_thr; thr = &local_thr; pthread_setspecific (gomp_tls_key, thr); #endif gomp_sem_init (&thr->release, 0); /* Extract what we need from data. */ local_fn = data->fn; local_data = data->fn_data; thr->thread_pool = data->thread_pool; thr->ts = data->ts; thr->task = data->task; thr->place = data->place; thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release; /* Make thread pool local. */ pool = thr->thread_pool; if (data->nested) { struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; gomp_barrier_wait (&team->barrier); local_fn (local_data); gomp_team_barrier_wait_final (&team->barrier); gomp_finish_task (task); gomp_barrier_wait_last (&team->barrier); } else { pool->threads[thr->ts.team_id] = thr; gomp_simple_barrier_wait (&pool->threads_dock); do { struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; local_fn (local_data); gomp_team_barrier_wait_final (&team->barrier); gomp_finish_task (task); gomp_simple_barrier_wait (&pool->threads_dock); local_fn = thr->fn; local_data = thr->data; thr->fn = NULL; } while (local_fn); } gomp_sem_destroy (&thr->release); thr->thread_pool = NULL; thr->task = NULL; return NULL; }
static void * gomp_thread_start (void *xdata) { struct gomp_thread_start_data *data = xdata; struct gomp_thread *thr; void (*local_fn) (void *); void *local_data; #ifdef HAVE_TLS thr = &gomp_tls_data; #else struct gomp_thread local_thr; thr = &local_thr; pthread_setspecific (gomp_tls_key, thr); #endif gomp_sem_init (&thr->release, 0); /* Extract what we need from data. */ local_fn = data->fn; local_data = data->fn_data; thr->ts = data->ts; thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release; if (data->nested) { gomp_barrier_wait (&thr->ts.team->barrier); local_fn (local_data); gomp_barrier_wait (&thr->ts.team->barrier); } else { gomp_threads[thr->ts.team_id] = thr; gomp_barrier_wait (&gomp_threads_dock); do { struct gomp_team *team; local_fn (local_data); /* Clear out the team and function data. This is a debugging signal that we're in fact back in the dock. */ team = thr->ts.team; thr->fn = NULL; thr->data = NULL; thr->ts.team = NULL; thr->ts.work_share = NULL; thr->ts.team_id = 0; thr->ts.work_share_generation = 0; thr->ts.static_trip = 0; gomp_barrier_wait (&team->barrier); gomp_barrier_wait (&gomp_threads_dock); local_fn = thr->fn; local_data = thr->data; } while (local_fn); } return NULL; }
void gomp_task_maybe_wait_for_dependencies (void **depend) { struct gomp_thread *thr = gomp_thread (); struct gomp_task *task = thr->task; struct gomp_team *team = thr->ts.team; struct gomp_task_depend_entry elem, *ent = NULL; struct gomp_taskwait taskwait; struct gomp_task *last_parent_depends_on = NULL; size_t ndepend = (uintptr_t) depend[0]; size_t nout = (uintptr_t) depend[1]; size_t i; size_t num_awaited = 0; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; int do_wake = 0; gomp_mutex_lock (&team->task_lock); for (i = 0; i < ndepend; i++) { elem.addr = depend[i + 2]; ent = htab_find (task->depend_hash, &elem); for (; ent; ent = ent->next) if (i >= nout && ent->is_in) continue; else { struct gomp_task *tsk = ent->task; if (!tsk->parent_depends_on) { tsk->parent_depends_on = true; ++num_awaited; /* If a task we need to wait for is not already running and is ready to be scheduled, move it to front, so that we run it as soon as possible. We rearrange the children queue such that all parent_depends_on tasks are first, and last_parent_depends_on points to the last such task we rearranged. For example, given the following children where PD[123] are the parent_depends_on tasks: task->children | V C1 -> C2 -> C3 -> PD1 -> PD2 -> PD3 -> C4 We rearrange such that: task->children | +--- last_parent_depends_on | | V V PD1 -> PD2 -> PD3 -> C1 -> C2 -> C3 -> C4 */ if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING) { if (last_parent_depends_on) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = last_parent_depends_on; tsk->next_child = last_parent_depends_on->next_child; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } else if (tsk != task->children) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = task->children->prev_child; tsk->next_child = task->children; task->children = tsk; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } last_parent_depends_on = tsk; } } } } if (num_awaited == 0) { gomp_mutex_unlock (&team->task_lock); return; } memset (&taskwait, 0, sizeof (taskwait)); taskwait.n_depend = num_awaited; taskwait.last_parent_depends_on = last_parent_depends_on; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; while (1) { bool cancelled = false; if (taskwait.n_depend == 0) { task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else /* All tasks we are waiting for are already running in other threads. Wait for them. */ taskwait.in_depend_wait = true; gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); if (child_task->parent_depends_on) --taskwait.n_depend; /* Remove child_task from sibling list. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } gomp_clear_parent (child_task->children); gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }
void GOMP_taskwait (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; struct gomp_taskwait taskwait; int do_wake = 0; /* The acquire barrier on load of task->children here synchronizes with the write of a NULL in gomp_task_run_post_remove_parent. It is not necessary that we synchronize with other non-NULL writes at this point, but we must ensure that all writes to memory by a child thread task work function are seen before we exit from GOMP_taskwait. */ if (task == NULL || __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL) return; memset (&taskwait, 0, sizeof (taskwait)); gomp_mutex_lock (&team->task_lock); while (1) { bool cancelled = false; if (task->children == NULL) { bool destroy_taskwait = task->taskwait != NULL; task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } if (destroy_taskwait) gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else { /* All tasks we are waiting for are already running in other threads. Wait for them. */ if (task->taskwait == NULL) { taskwait.in_depend_wait = false; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; } taskwait.in_taskwait = true; } gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); /* Remove child_task from children list, and set up the next sibling to be run. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } /* Orphan all the children of CHILD_TASK. */ gomp_clear_parent (child_task->children); /* Remove CHILD_TASK from its taskgroup. */ gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }