static void free_team (struct gomp_team *team) { free (team->work_shares); gomp_mutex_destroy (&team->work_share_lock); gomp_barrier_destroy (&team->barrier); gomp_sem_destroy (&team->master_release); free (team); }
static void gomp_free_pool_helper (void *thread_pool) { struct gomp_thread_pool *pool = (struct gomp_thread_pool *) thread_pool; gomp_barrier_wait_last (&pool->threads_dock); gomp_sem_destroy (&gomp_thread ()->release); pthread_exit (NULL); }
static void gomp_free_pool_helper (void *thread_pool) { struct gomp_thread *thr = gomp_thread (); struct gomp_thread_pool *pool = (struct gomp_thread_pool *) thread_pool; gomp_simple_barrier_wait_last (&pool->threads_dock); gomp_sem_destroy (&thr->release); thr->thread_pool = NULL; thr->task = NULL; #ifdef LIBGOMP_USE_PTHREADS pthread_exit (NULL); #elif defined(__nvptx__) asm ("exit;"); #else #error gomp_free_pool_helper must terminate the thread #endif }
static void * gomp_thread_start (void *xdata) { struct gomp_thread_start_data *data = xdata; struct gomp_thread *thr; struct gomp_thread_pool *pool; void (*local_fn) (void *); void *local_data; #if defined HAVE_TLS || defined USE_EMUTLS thr = &gomp_tls_data; #else struct gomp_thread local_thr; thr = &local_thr; pthread_setspecific (gomp_tls_key, thr); #endif gomp_sem_init (&thr->release, 0); /* Extract what we need from data. */ local_fn = data->fn; local_data = data->fn_data; thr->thread_pool = data->thread_pool; thr->ts = data->ts; thr->task = data->task; thr->place = data->place; thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release; /* Make thread pool local. */ pool = thr->thread_pool; if (data->nested) { struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; gomp_barrier_wait (&team->barrier); local_fn (local_data); gomp_team_barrier_wait_final (&team->barrier); gomp_finish_task (task); gomp_barrier_wait_last (&team->barrier); } else { pool->threads[thr->ts.team_id] = thr; gomp_simple_barrier_wait (&pool->threads_dock); do { struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; local_fn (local_data); gomp_team_barrier_wait_final (&team->barrier); gomp_finish_task (task); gomp_simple_barrier_wait (&pool->threads_dock); local_fn = thr->fn; local_data = thr->data; thr->fn = NULL; } while (local_fn); } gomp_sem_destroy (&thr->release); thr->thread_pool = NULL; thr->task = NULL; return NULL; }
void GOMP_taskgroup_end (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_taskgroup *taskgroup; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; int do_wake = 0; if (team == NULL) return; taskgroup = task->taskgroup; /* The acquire barrier on load of taskgroup->num_children here synchronizes with the write of 0 in gomp_task_run_post_remove_taskgroup. It is not necessary that we synchronize with other non-0 writes at this point, but we must ensure that all writes to memory by a child thread task work function are seen before we exit from GOMP_taskgroup_end. */ if (__atomic_load_n (&taskgroup->num_children, MEMMODEL_ACQUIRE) == 0) goto finish; gomp_mutex_lock (&team->task_lock); while (1) { bool cancelled = false; if (taskgroup->children == NULL) { if (taskgroup->num_children) { if (task->children == NULL) goto do_wait; child_task = task->children; } else { gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } goto finish; } } else child_task = taskgroup->children; if (child_task->kind == GOMP_TASK_WAITING) { cancelled = gomp_task_run_pre (child_task, child_task->parent, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else { child_task = NULL; do_wait: /* All tasks we are waiting for are already running in other threads. Wait for them. */ taskgroup->in_taskgroup_wait = true; } gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskgroup->taskgroup_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); gomp_task_run_post_remove_parent (child_task); gomp_clear_parent (child_task->children); gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } finish: task->taskgroup = taskgroup->prev; gomp_sem_destroy (&taskgroup->taskgroup_sem); free (taskgroup); }
void gomp_task_maybe_wait_for_dependencies (void **depend) { struct gomp_thread *thr = gomp_thread (); struct gomp_task *task = thr->task; struct gomp_team *team = thr->ts.team; struct gomp_task_depend_entry elem, *ent = NULL; struct gomp_taskwait taskwait; struct gomp_task *last_parent_depends_on = NULL; size_t ndepend = (uintptr_t) depend[0]; size_t nout = (uintptr_t) depend[1]; size_t i; size_t num_awaited = 0; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; int do_wake = 0; gomp_mutex_lock (&team->task_lock); for (i = 0; i < ndepend; i++) { elem.addr = depend[i + 2]; ent = htab_find (task->depend_hash, &elem); for (; ent; ent = ent->next) if (i >= nout && ent->is_in) continue; else { struct gomp_task *tsk = ent->task; if (!tsk->parent_depends_on) { tsk->parent_depends_on = true; ++num_awaited; /* If a task we need to wait for is not already running and is ready to be scheduled, move it to front, so that we run it as soon as possible. We rearrange the children queue such that all parent_depends_on tasks are first, and last_parent_depends_on points to the last such task we rearranged. For example, given the following children where PD[123] are the parent_depends_on tasks: task->children | V C1 -> C2 -> C3 -> PD1 -> PD2 -> PD3 -> C4 We rearrange such that: task->children | +--- last_parent_depends_on | | V V PD1 -> PD2 -> PD3 -> C1 -> C2 -> C3 -> C4 */ if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING) { if (last_parent_depends_on) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = last_parent_depends_on; tsk->next_child = last_parent_depends_on->next_child; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } else if (tsk != task->children) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = task->children->prev_child; tsk->next_child = task->children; task->children = tsk; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } last_parent_depends_on = tsk; } } } } if (num_awaited == 0) { gomp_mutex_unlock (&team->task_lock); return; } memset (&taskwait, 0, sizeof (taskwait)); taskwait.n_depend = num_awaited; taskwait.last_parent_depends_on = last_parent_depends_on; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; while (1) { bool cancelled = false; if (taskwait.n_depend == 0) { task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else /* All tasks we are waiting for are already running in other threads. Wait for them. */ taskwait.in_depend_wait = true; gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); if (child_task->parent_depends_on) --taskwait.n_depend; /* Remove child_task from sibling list. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } gomp_clear_parent (child_task->children); gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }
void GOMP_taskwait (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; struct gomp_taskwait taskwait; int do_wake = 0; /* The acquire barrier on load of task->children here synchronizes with the write of a NULL in gomp_task_run_post_remove_parent. It is not necessary that we synchronize with other non-NULL writes at this point, but we must ensure that all writes to memory by a child thread task work function are seen before we exit from GOMP_taskwait. */ if (task == NULL || __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL) return; memset (&taskwait, 0, sizeof (taskwait)); gomp_mutex_lock (&team->task_lock); while (1) { bool cancelled = false; if (task->children == NULL) { bool destroy_taskwait = task->taskwait != NULL; task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } if (destroy_taskwait) gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else { /* All tasks we are waiting for are already running in other threads. Wait for them. */ if (task->taskwait == NULL) { taskwait.in_depend_wait = false; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; } taskwait.in_taskwait = true; } gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); /* Remove child_task from children list, and set up the next sibling to be run. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } /* Orphan all the children of CHILD_TASK. */ gomp_clear_parent (child_task->children); /* Remove CHILD_TASK from its taskgroup. */ gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }