void gomp_ordered_static_next (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_work_share *ws = thr->ts.work_share; unsigned id = thr->ts.team_id; if (team == NULL || team->nthreads == 1) return; ws->ordered_owner = -1; /* This thread currently owns the lock. Increment the owner. */ if (++id == team->nthreads) id = 0; ws->ordered_team_ids[0] = id; gomp_sem_post (team->ordered_release[id]); }
void gomp_ordered_next (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_work_share *ws = thr->ts.work_share; unsigned index, next_id; /* Work share constructs can be orphaned. */ if (team == NULL || team->nthreads == 1) return; /* We're no longer the owner. */ ws->ordered_owner = -1; /* If there's only one thread in the queue, that must be us. */ if (ws->ordered_num_used == 1) { /* We have a similar situation as in gomp_ordered_first where we need to post to our own release semaphore. */ gomp_sem_post (team->ordered_release[thr->ts.team_id]); return; } /* If the queue is entirely full, then we move ourself to the end of the queue merely by incrementing ordered_cur. Only if it's not full do we have to write our id. */ if (ws->ordered_num_used < team->nthreads) { index = ws->ordered_cur + ws->ordered_num_used; if (index >= team->nthreads) index -= team->nthreads; ws->ordered_team_ids[index] = thr->ts.team_id; } index = ws->ordered_cur + 1; if (index == team->nthreads) index = 0; ws->ordered_cur = index; next_id = ws->ordered_team_ids[index]; gomp_sem_post (team->ordered_release[next_id]); }
void GOMP_parallel_end (void) { if (__builtin_expect (gomp_thread_limit_var != ULONG_MAX, 0)) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; if (team && team->nthreads > 1) { #ifdef HAVE_SYNC_BUILTINS __sync_fetch_and_add (&gomp_remaining_threads_count, 1UL - team->nthreads); #else gomp_mutex_lock (&gomp_remaining_threads_lock); gomp_remaining_threads_count -= team->nthreads - 1; #endif } } gomp_team_end (); }
void gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) { unsigned int generation, gen; if (__builtin_expect ((state & 1) != 0, 0)) { /* Next time we'll be awaiting TOTAL threads again. */ struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; bar->awaited = bar->total; if (__builtin_expect (team->task_count, 0)) { gomp_barrier_handle_tasks (state); state &= ~1; } else { __atomic_store_n (&bar->generation, state + 3, MEMMODEL_RELEASE); futex_wake ((int *) &bar->generation, INT_MAX); return; } } generation = state; do { do_wait ((int *) &bar->generation, generation); gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); if (__builtin_expect (gen & 1, 0)) { gomp_barrier_handle_tasks (state); gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE); } if ((gen & 2) != 0) generation |= 2; } while (gen != state + 4); }
bool GOMP_single_start (void) { #ifdef HAVE_SYNC_BUILTINS struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; unsigned long single_count; if (__builtin_expect (team == NULL, 0)) return true; single_count = thr->ts.single_count++; return __sync_bool_compare_and_swap (&team->single_count, single_count, single_count + 1L); #else bool ret = gomp_work_share_start (false); if (ret) gomp_work_share_init_done (); gomp_work_share_end_nowait (); return ret; #endif }
void gomp_ordered_first (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_work_share *ws = thr->ts.work_share; unsigned index; /* Work share constructs can be orphaned. */ if (team == NULL || team->nthreads == 1) return; index = ws->ordered_cur + ws->ordered_num_used; if (index >= team->nthreads) index -= team->nthreads; ws->ordered_team_ids[index] = thr->ts.team_id; /* If this is the first and only thread in the queue, then there is no one to release us when we get to our ordered section. Post to our own release queue now so that we won't block later. */ if (ws->ordered_num_used++ == 0) gomp_sem_post (team->ordered_release[thr->ts.team_id]); }
void GOMP_taskgroup_start (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_taskgroup *taskgroup; /* If team is NULL, all tasks are executed as GOMP_TASK_UNDEFERRED tasks and thus all children tasks of taskgroup and their descendant tasks will be finished by the time GOMP_taskgroup_end is called. */ if (team == NULL) return; taskgroup = gomp_malloc (sizeof (struct gomp_taskgroup)); taskgroup->prev = task->taskgroup; taskgroup->children = NULL; taskgroup->in_taskgroup_wait = false; taskgroup->cancelled = false; taskgroup->num_children = 0; gomp_sem_init (&taskgroup->taskgroup_sem, 0); task->taskgroup = taskgroup; }
unsigned GOMP_sections_next (void) { long s, e, ret; #ifdef HAVE_SYNC_BUILTINS if (gomp_iter_dynamic_next (&s, &e)) ret = s; else ret = 0; #else struct gomp_thread *thr = gomp_thread (); gomp_mutex_lock (&thr->ts.work_share->lock); if (gomp_iter_dynamic_next_locked (&s, &e)) ret = s; else ret = 0; gomp_mutex_unlock (&thr->ts.work_share->lock); #endif return ret; }
void * GOMP_single_copy_start (void) { struct gomp_thread *thr = gomp_thread (); bool first; void *ret; first = gomp_work_share_start (false); gomp_mutex_unlock (&thr->ts.work_share->lock); if (first) ret = NULL; else { gomp_barrier_wait (&thr->ts.team->barrier); ret = thr->ts.work_share->copyprivate; gomp_work_share_end_nowait (); } return ret; }
void gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state) { unsigned int generation; if (__builtin_expect ((state & 1) != 0, 0)) { /* Next time we'll be awaiting TOTAL threads again. */ struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; bar->awaited = bar->total; atomic_write_barrier (); if (__builtin_expect (team->task_count, 0)) { gomp_barrier_handle_tasks (state); state &= ~1; } else { bar->generation = state + 3; futex_wake ((int *) &bar->generation, INT_MAX); return; } } generation = state; do { do_wait ((int *) &bar->generation, generation); if (__builtin_expect (bar->generation & 1, 0)) gomp_barrier_handle_tasks (state); if ((bar->generation & 2)) generation |= 2; } while (bar->generation != state + 4); }
int omp_in_parallel (void) { return gomp_thread ()->ts.active_level > 0; }
int omp_in_final (void) { struct gomp_thread *thr = gomp_thread (); return thr->task && thr->task->final_task; }
thr->task = NULL; #ifdef LIBGOMP_USE_PTHREADS pthread_exit (NULL); #elif defined(__nvptx__) asm ("exit;"); #else #error gomp_free_pool_helper must terminate the thread #endif } /* Free a thread pool and release its threads. */ void gomp_free_thread (void *arg __attribute__((unused))) { struct gomp_thread *thr = gomp_thread (); struct gomp_thread_pool *pool = thr->thread_pool; if (pool) { if (pool->threads_used > 0) { int i; for (i = 1; i < pool->threads_used; i++) { struct gomp_thread *nthr = pool->threads[i]; nthr->fn = gomp_free_pool_helper; nthr->data = pool; } /* This barrier undocks threads docked on pool->threads_dock. */ gomp_simple_barrier_wait (&pool->threads_dock); /* And this waits till all threads have called gomp_barrier_wait_last
void gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads, struct gomp_work_share *work_share) { struct gomp_thread_start_data *start_data; struct gomp_thread *thr, *nthr; struct gomp_team *team; bool nested; unsigned i, n, old_threads_used = 0; pthread_attr_t thread_attr, *attr; thr = gomp_thread (); nested = thr->ts.team != NULL; team = new_team (nthreads, work_share); /* Always save the previous state, even if this isn't a nested team. In particular, we should save any work share state from an outer orphaned work share construct. */ team->prev_ts = thr->ts; thr->ts.team = team; thr->ts.work_share = work_share; thr->ts.team_id = 0; thr->ts.work_share_generation = 0; thr->ts.static_trip = 0; if (nthreads == 1) return; i = 1; /* We only allow the reuse of idle threads for non-nested PARALLEL regions. This appears to be implied by the semantics of threadprivate variables, but perhaps that's reading too much into things. Certainly it does prevent any locking problems, since only the initial program thread will modify gomp_threads. */ if (!nested) { old_threads_used = gomp_threads_used; if (nthreads <= old_threads_used) n = nthreads; else if (old_threads_used == 0) { n = 0; gomp_barrier_init (&gomp_threads_dock, nthreads); } else { n = old_threads_used; /* Increase the barrier threshold to make sure all new threads arrive before the team is released. */ gomp_barrier_reinit (&gomp_threads_dock, nthreads); } /* Not true yet, but soon will be. We're going to release all threads from the dock, and those that aren't part of the team will exit. */ gomp_threads_used = nthreads; /* Release existing idle threads. */ for (; i < n; ++i) { nthr = gomp_threads[i]; nthr->ts.team = team; nthr->ts.work_share = work_share; nthr->ts.team_id = i; nthr->ts.work_share_generation = 0; nthr->ts.static_trip = 0; nthr->fn = fn; nthr->data = data; team->ordered_release[i] = &nthr->release; } if (i == nthreads) goto do_release; /* If necessary, expand the size of the gomp_threads array. It is expected that changes in the number of threads is rare, thus we make no effort to expand gomp_threads_size geometrically. */ if (nthreads >= gomp_threads_size) { gomp_threads_size = nthreads + 1; gomp_threads = gomp_realloc (gomp_threads, gomp_threads_size * sizeof (struct gomp_thread_data *)); } } attr = &gomp_thread_attr; if (gomp_cpu_affinity != NULL) { size_t stacksize; pthread_attr_init (&thread_attr); pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED); if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize)) pthread_attr_setstacksize (&thread_attr, stacksize); attr = &thread_attr; } start_data = gomp_alloca (sizeof (struct gomp_thread_start_data) * (nthreads-i)); /* Launch new threads. */ for (; i < nthreads; ++i, ++start_data) { pthread_t pt; int err; start_data->ts.team = team; start_data->ts.work_share = work_share; start_data->ts.team_id = i; start_data->ts.work_share_generation = 0; start_data->ts.static_trip = 0; start_data->fn = fn; start_data->fn_data = data; start_data->nested = nested; if (gomp_cpu_affinity != NULL) gomp_init_thread_affinity (attr); err = pthread_create (&pt, attr, gomp_thread_start, start_data); if (err != 0) gomp_fatal ("Thread creation failed: %s", strerror (err)); } if (gomp_cpu_affinity != NULL) pthread_attr_destroy (&thread_attr); do_release: gomp_barrier_wait (nested ? &team->barrier : &gomp_threads_dock); /* Decrease the barrier threshold to match the number of threads that should arrive back at the end of this team. The extra threads should be exiting. Note that we arrange for this test to never be true for nested teams. */ if (nthreads < old_threads_used) gomp_barrier_reinit (&gomp_threads_dock, nthreads); }
bool gomp_iter_dynamic_next (long *pstart, long *pend) { struct gomp_thread *thr = gomp_thread (); struct gomp_work_share *ws = thr->ts.work_share; long start, end, nend, chunk, incr; end = ws->end; incr = ws->incr; chunk = ws->chunk_size; if (__builtin_expect (ws->mode, 1)) { long tmp = __sync_fetch_and_add (&ws->next, chunk); if (incr > 0) { if (tmp >= end) return false; nend = tmp + chunk; if (nend > end) nend = end; *pstart = tmp; *pend = nend; return true; } else { if (tmp <= end) return false; nend = tmp + chunk; if (nend < end) nend = end; *pstart = tmp; *pend = nend; return true; } } start = ws->next; while (1) { long left = end - start; long tmp; if (start == end) return false; if (incr < 0) { if (chunk < left) chunk = left; } else { if (chunk > left) chunk = left; } nend = start + chunk; tmp = __sync_val_compare_and_swap (&ws->next, start, nend); if (__builtin_expect (tmp == start, 1)) break; start = tmp; } *pstart = start; *pend = nend; return true; }
void GOMP_taskgroup_end (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_taskgroup *taskgroup; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; int do_wake = 0; if (team == NULL) return; taskgroup = task->taskgroup; /* The acquire barrier on load of taskgroup->num_children here synchronizes with the write of 0 in gomp_task_run_post_remove_taskgroup. It is not necessary that we synchronize with other non-0 writes at this point, but we must ensure that all writes to memory by a child thread task work function are seen before we exit from GOMP_taskgroup_end. */ if (__atomic_load_n (&taskgroup->num_children, MEMMODEL_ACQUIRE) == 0) goto finish; gomp_mutex_lock (&team->task_lock); while (1) { bool cancelled = false; if (taskgroup->children == NULL) { if (taskgroup->num_children) { if (task->children == NULL) goto do_wait; child_task = task->children; } else { gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } goto finish; } } else child_task = taskgroup->children; if (child_task->kind == GOMP_TASK_WAITING) { cancelled = gomp_task_run_pre (child_task, child_task->parent, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else { child_task = NULL; do_wait: /* All tasks we are waiting for are already running in other threads. Wait for them. */ taskgroup->in_taskgroup_wait = true; } gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskgroup->taskgroup_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); gomp_task_run_post_remove_parent (child_task); gomp_clear_parent (child_task->children); gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } finish: task->taskgroup = taskgroup->prev; gomp_sem_destroy (&taskgroup->taskgroup_sem); free (taskgroup); }
int omp_get_num_threads (void) { struct gomp_team *team = gomp_thread ()->ts.team; return team ? team->nthreads : 1; }
bool gomp_iter_ull_dynamic_next (gomp_ull *pstart, gomp_ull *pend) { struct gomp_thread *thr = gomp_thread (); struct gomp_work_share *ws = thr->ts.work_share; gomp_ull start, end, nend, chunk; end = ws->end_ull; chunk = ws->chunk_size_ull; if (__builtin_expect (ws->mode & 1, 1)) { gomp_ull tmp = __sync_fetch_and_add (&ws->next_ull, chunk); if (__builtin_expect (ws->mode & 2, 0) == 0) { if (tmp >= end) return false; nend = tmp + chunk; if (nend > end) nend = end; *pstart = tmp; *pend = nend; return true; } else { if (tmp <= end) return false; nend = tmp + chunk; if (nend < end) nend = end; *pstart = tmp; *pend = nend; return true; } } start = ws->next_ull; while (1) { gomp_ull left = end - start; gomp_ull tmp; if (start == end) return false; if (__builtin_expect (ws->mode & 2, 0)) { if (chunk < left) chunk = left; } else { if (chunk > left) chunk = left; } nend = start + chunk; tmp = __sync_val_compare_and_swap (&ws->next_ull, start, nend); if (__builtin_expect (tmp == start, 1)) break; start = tmp; } *pstart = start; *pend = nend; return true; }
unsigned gomp_resolve_num_threads (unsigned specified, unsigned count) { struct gomp_thread *thread = gomp_thread(); struct gomp_task_icv *icv; unsigned threads_requested, max_num_threads, num_threads; unsigned long remaining; icv = gomp_icv (false); if (specified == 1) return 1; else if (thread->ts.active_level >= 1 && !icv->nest_var) return 1; else if (thread->ts.active_level >= gomp_max_active_levels_var) return 1; /* If NUM_THREADS not specified, use nthreads_var. */ if (specified == 0) threads_requested = icv->nthreads_var; else threads_requested = specified; max_num_threads = threads_requested; /* If dynamic threads are enabled, bound the number of threads that we launch. */ if (icv->dyn_var) { unsigned dyn = gomp_dynamic_max_threads (); if (dyn < max_num_threads) max_num_threads = dyn; /* Optimization for parallel sections. */ if (count && count < max_num_threads) max_num_threads = count; } /* ULONG_MAX stands for infinity. */ if (__builtin_expect (gomp_thread_limit_var == ULONG_MAX, 1) || max_num_threads == 1) return max_num_threads; #ifdef HAVE_SYNC_BUILTINS do { remaining = gomp_remaining_threads_count; num_threads = max_num_threads; if (num_threads > remaining) num_threads = remaining + 1; } while (__sync_val_compare_and_swap (&gomp_remaining_threads_count, remaining, remaining - num_threads + 1) != remaining); #else gomp_mutex_lock (&gomp_remaining_threads_lock); num_threads = max_num_threads; remaining = gomp_remaining_threads_count; if (num_threads > remaining) num_threads = remaining + 1; gomp_remaining_threads_count -= num_threads - 1; gomp_mutex_unlock (&gomp_remaining_threads_lock); #endif return num_threads; }
void gomp_task_maybe_wait_for_dependencies (void **depend) { struct gomp_thread *thr = gomp_thread (); struct gomp_task *task = thr->task; struct gomp_team *team = thr->ts.team; struct gomp_task_depend_entry elem, *ent = NULL; struct gomp_taskwait taskwait; struct gomp_task *last_parent_depends_on = NULL; size_t ndepend = (uintptr_t) depend[0]; size_t nout = (uintptr_t) depend[1]; size_t i; size_t num_awaited = 0; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; int do_wake = 0; gomp_mutex_lock (&team->task_lock); for (i = 0; i < ndepend; i++) { elem.addr = depend[i + 2]; ent = htab_find (task->depend_hash, &elem); for (; ent; ent = ent->next) if (i >= nout && ent->is_in) continue; else { struct gomp_task *tsk = ent->task; if (!tsk->parent_depends_on) { tsk->parent_depends_on = true; ++num_awaited; /* If a task we need to wait for is not already running and is ready to be scheduled, move it to front, so that we run it as soon as possible. We rearrange the children queue such that all parent_depends_on tasks are first, and last_parent_depends_on points to the last such task we rearranged. For example, given the following children where PD[123] are the parent_depends_on tasks: task->children | V C1 -> C2 -> C3 -> PD1 -> PD2 -> PD3 -> C4 We rearrange such that: task->children | +--- last_parent_depends_on | | V V PD1 -> PD2 -> PD3 -> C1 -> C2 -> C3 -> C4 */ if (tsk->num_dependees == 0 && tsk->kind == GOMP_TASK_WAITING) { if (last_parent_depends_on) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = last_parent_depends_on; tsk->next_child = last_parent_depends_on->next_child; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } else if (tsk != task->children) { tsk->prev_child->next_child = tsk->next_child; tsk->next_child->prev_child = tsk->prev_child; tsk->prev_child = task->children->prev_child; tsk->next_child = task->children; task->children = tsk; tsk->prev_child->next_child = tsk; tsk->next_child->prev_child = tsk; } last_parent_depends_on = tsk; } } } } if (num_awaited == 0) { gomp_mutex_unlock (&team->task_lock); return; } memset (&taskwait, 0, sizeof (taskwait)); taskwait.n_depend = num_awaited; taskwait.last_parent_depends_on = last_parent_depends_on; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; while (1) { bool cancelled = false; if (taskwait.n_depend == 0) { task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else /* All tasks we are waiting for are already running in other threads. Wait for them. */ taskwait.in_depend_wait = true; gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); if (child_task->parent_depends_on) --taskwait.n_depend; /* Remove child_task from sibling list. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } gomp_clear_parent (child_task->children); gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }
void gomp_create_target_task (struct gomp_device_descr *devicep, void (*fn) (void *), size_t mapnum, void **hostaddrs, size_t *sizes, unsigned short *kinds, unsigned int flags, void **depend) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (team && (gomp_team_barrier_cancelled (&team->barrier) || (thr->task->taskgroup && thr->task->taskgroup->cancelled))) return; struct gomp_target_task *ttask; struct gomp_task *task; struct gomp_task *parent = thr->task; struct gomp_taskgroup *taskgroup = parent->taskgroup; bool do_wake; size_t depend_size = 0; if (depend != NULL) depend_size = ((uintptr_t) depend[0] * sizeof (struct gomp_task_depend_entry)); task = gomp_malloc (sizeof (*task) + depend_size + sizeof (*ttask) + mapnum * (sizeof (void *) + sizeof (size_t) + sizeof (unsigned short))); gomp_init_task (task, parent, gomp_icv (false)); task->kind = GOMP_TASK_WAITING; task->in_tied_task = parent->in_tied_task; task->taskgroup = taskgroup; ttask = (struct gomp_target_task *) &task->depend[(uintptr_t) depend[0]]; ttask->devicep = devicep; ttask->fn = fn; ttask->mapnum = mapnum; memcpy (ttask->hostaddrs, hostaddrs, mapnum * sizeof (void *)); ttask->sizes = (size_t *) &ttask->hostaddrs[mapnum]; memcpy (ttask->sizes, sizes, mapnum * sizeof (size_t)); ttask->kinds = (unsigned short *) &ttask->sizes[mapnum]; memcpy (ttask->kinds, kinds, mapnum * sizeof (unsigned short)); ttask->flags = flags; task->fn = gomp_target_task_fn; task->fn_data = ttask; task->final_task = 0; gomp_mutex_lock (&team->task_lock); /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (__builtin_expect (gomp_team_barrier_cancelled (&team->barrier) || (taskgroup && taskgroup->cancelled), 0)) { gomp_mutex_unlock (&team->task_lock); gomp_finish_task (task); free (task); return; } if (taskgroup) taskgroup->num_children++; if (depend_size) { gomp_task_handle_depend (task, parent, depend); if (task->num_dependees) { gomp_mutex_unlock (&team->task_lock); return; } } if (parent->children) { task->next_child = parent->children; task->prev_child = parent->children->prev_child; task->next_child->prev_child = task; task->prev_child->next_child = task; } else { task->next_child = task; task->prev_child = task; } parent->children = task; if (taskgroup) { /* If applicable, place task into its taskgroup. */ if (taskgroup->children) { task->next_taskgroup = taskgroup->children; task->prev_taskgroup = taskgroup->children->prev_taskgroup; task->next_taskgroup->prev_taskgroup = task; task->prev_taskgroup->next_taskgroup = task; } else { task->next_taskgroup = task; task->prev_taskgroup = task; } taskgroup->children = task; } if (team->task_queue) { task->next_queue = team->task_queue; task->prev_queue = team->task_queue->prev_queue; task->next_queue->prev_queue = task; task->prev_queue->next_queue = task; } else { task->next_queue = task; task->prev_queue = task; team->task_queue = task; } ++team->task_count; ++team->task_queued_count; gomp_team_barrier_set_task_pending (&team->barrier); do_wake = team->task_running_count + !parent->in_tied_task < team->nthreads; gomp_mutex_unlock (&team->task_lock); if (do_wake) gomp_team_barrier_wake (&team->barrier, 1); }
void GOMP_task (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *), long arg_size, long arg_align, bool if_clause, unsigned flags, void **depend, int priority) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; #ifdef HAVE_BROKEN_POSIX_SEMAPHORES /* If pthread_mutex_* is used for omp_*lock*, then each task must be tied to one thread all the time. This means UNTIED tasks must be tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN might be running on different thread than FN. */ if (cpyfn) if_clause = false; flags &= ~GOMP_TASK_FLAG_UNTIED; #endif /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (team && (gomp_team_barrier_cancelled (&team->barrier) || (thr->task->taskgroup && thr->task->taskgroup->cancelled))) return; if ((flags & GOMP_TASK_FLAG_PRIORITY) == 0) priority = 0; /* FIXME, use priority. */ (void) priority; if (!if_clause || team == NULL || (thr->task && thr->task->final_task) || team->task_count > 64 * team->nthreads) { struct gomp_task task; /* If there are depend clauses and earlier deferred sibling tasks with depend clauses, check if there isn't a dependency. If there is, we need to wait for them. There is no need to handle depend clauses for non-deferred tasks other than this, because the parent task is suspended until the child task finishes and thus it can't start further child tasks. */ if ((flags & GOMP_TASK_FLAG_DEPEND) && thr->task && thr->task->depend_hash) gomp_task_maybe_wait_for_dependencies (depend); gomp_init_task (&task, thr->task, gomp_icv (false)); task.kind = GOMP_TASK_UNDEFERRED; task.final_task = (thr->task && thr->task->final_task) || (flags & GOMP_TASK_FLAG_FINAL); if (thr->task) { task.in_tied_task = thr->task->in_tied_task; task.taskgroup = thr->task->taskgroup; } thr->task = &task; if (__builtin_expect (cpyfn != NULL, 0)) { char buf[arg_size + arg_align - 1]; char *arg = (char *) (((uintptr_t) buf + arg_align - 1) & ~(uintptr_t) (arg_align - 1)); cpyfn (arg, data); fn (arg); } else fn (data); /* Access to "children" is normally done inside a task_lock mutex region, but the only way this particular task.children can be set is if this thread's task work function (fn) creates children. So since the setter is *this* thread, we need no barriers here when testing for non-NULL. We can have task.children set by the current thread then changed by a child thread, but seeing a stale non-NULL value is not a problem. Once past the task_lock acquisition, this thread will see the real value of task.children. */ if (task.children != NULL) { gomp_mutex_lock (&team->task_lock); gomp_clear_parent (task.children); gomp_mutex_unlock (&team->task_lock); } gomp_end_task (); } else { struct gomp_task *task; struct gomp_task *parent = thr->task; struct gomp_taskgroup *taskgroup = parent->taskgroup; char *arg; bool do_wake; size_t depend_size = 0; if (flags & GOMP_TASK_FLAG_DEPEND) depend_size = ((uintptr_t) depend[0] * sizeof (struct gomp_task_depend_entry)); task = gomp_malloc (sizeof (*task) + depend_size + arg_size + arg_align - 1); arg = (char *) (((uintptr_t) (task + 1) + depend_size + arg_align - 1) & ~(uintptr_t) (arg_align - 1)); gomp_init_task (task, parent, gomp_icv (false)); task->kind = GOMP_TASK_UNDEFERRED; task->in_tied_task = parent->in_tied_task; task->taskgroup = taskgroup; thr->task = task; if (cpyfn) { cpyfn (arg, data); task->copy_ctors_done = true; } else memcpy (arg, data, arg_size); thr->task = parent; task->kind = GOMP_TASK_WAITING; task->fn = fn; task->fn_data = arg; task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1; gomp_mutex_lock (&team->task_lock); /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier) || (taskgroup && taskgroup->cancelled)) && !task->copy_ctors_done, 0)) { gomp_mutex_unlock (&team->task_lock); gomp_finish_task (task); free (task); return; } if (taskgroup) taskgroup->num_children++; if (depend_size) { gomp_task_handle_depend (task, parent, depend); if (task->num_dependees) { gomp_mutex_unlock (&team->task_lock); return; } } if (parent->children) { task->next_child = parent->children; task->prev_child = parent->children->prev_child; task->next_child->prev_child = task; task->prev_child->next_child = task; } else { task->next_child = task; task->prev_child = task; } parent->children = task; if (taskgroup) { /* If applicable, place task into its taskgroup. */ if (taskgroup->children) { task->next_taskgroup = taskgroup->children; task->prev_taskgroup = taskgroup->children->prev_taskgroup; task->next_taskgroup->prev_taskgroup = task; task->prev_taskgroup->next_taskgroup = task; } else { task->next_taskgroup = task; task->prev_taskgroup = task; } taskgroup->children = task; } if (team->task_queue) { task->next_queue = team->task_queue; task->prev_queue = team->task_queue->prev_queue; task->next_queue->prev_queue = task; task->prev_queue->next_queue = task; } else { task->next_queue = task; task->prev_queue = task; team->task_queue = task; } ++team->task_count; ++team->task_queued_count; gomp_team_barrier_set_task_pending (&team->barrier); do_wake = team->task_running_count + !parent->in_tied_task < team->nthreads; gomp_mutex_unlock (&team->task_lock); if (do_wake) gomp_team_barrier_wake (&team->barrier, 1); } }
int gomp_iter_ull_static_next (gomp_ull *pstart, gomp_ull *pend) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_work_share *ws = thr->ts.work_share; unsigned long nthreads = team ? team->nthreads : 1; if (thr->ts.static_trip == -1) return -1; /* Quick test for degenerate teams and orphaned constructs. */ if (nthreads == 1) { *pstart = ws->next_ull; *pend = ws->end_ull; thr->ts.static_trip = -1; return ws->next_ull == ws->end_ull; } /* We interpret chunk_size zero as "unspecified", which means that we should break up the iterations such that each thread makes only one trip through the outer loop. */ if (ws->chunk_size_ull == 0) { gomp_ull n, q, i, t, s0, e0, s, e; if (thr->ts.static_trip > 0) return 1; /* Compute the total number of iterations. */ if (__builtin_expect (ws->mode, 0) == 0) n = (ws->end_ull - ws->next_ull + ws->incr_ull - 1) / ws->incr_ull; else n = (ws->next_ull - ws->end_ull - ws->incr_ull - 1) / -ws->incr_ull; i = thr->ts.team_id; /* Compute the "zero-based" start and end points. That is, as if the loop began at zero and incremented by one. */ q = n / nthreads; t = n % nthreads; if (i < t) { t = 0; q++; } s0 = q * i + t; e0 = s0 + q; /* Notice when no iterations allocated for this thread. */ if (s0 >= e0) { thr->ts.static_trip = 1; return 1; } /* Transform these to the actual start and end numbers. */ s = s0 * ws->incr_ull + ws->next_ull; e = e0 * ws->incr_ull + ws->next_ull; *pstart = s; *pend = e; thr->ts.static_trip = (e0 == n ? -1 : 1); return 0; } else { gomp_ull n, s0, e0, i, c, s, e; /* Otherwise, each thread gets exactly chunk_size iterations (if available) each time through the loop. */ if (__builtin_expect (ws->mode, 0) == 0) n = (ws->end_ull - ws->next_ull + ws->incr_ull - 1) / ws->incr_ull; else n = (ws->next_ull - ws->end_ull - ws->incr_ull - 1) / -ws->incr_ull; i = thr->ts.team_id; c = ws->chunk_size_ull; /* Initial guess is a C sized chunk positioned nthreads iterations in, offset by our thread number. */ s0 = (thr->ts.static_trip * (gomp_ull) nthreads + i) * c; e0 = s0 + c; /* Detect overflow. */ if (s0 >= n) return 1; if (e0 > n) e0 = n; /* Transform these to the actual start and end numbers. */ s = s0 * ws->incr_ull + ws->next_ull; e = e0 * ws->incr_ull + ws->next_ull; *pstart = s; *pend = e; if (e0 == n) thr->ts.static_trip = -1; else thr->ts.static_trip++; return 0; } }
int omp_get_active_level (void) { return gomp_thread ()->ts.active_level; }
int omp_get_thread_num (void) { return gomp_thread ()->ts.team_id; }
void GOMP_taskwait (void) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_task *task = thr->task; struct gomp_task *child_task = NULL; struct gomp_task *to_free = NULL; struct gomp_taskwait taskwait; int do_wake = 0; /* The acquire barrier on load of task->children here synchronizes with the write of a NULL in gomp_task_run_post_remove_parent. It is not necessary that we synchronize with other non-NULL writes at this point, but we must ensure that all writes to memory by a child thread task work function are seen before we exit from GOMP_taskwait. */ if (task == NULL || __atomic_load_n (&task->children, MEMMODEL_ACQUIRE) == NULL) return; memset (&taskwait, 0, sizeof (taskwait)); gomp_mutex_lock (&team->task_lock); while (1) { bool cancelled = false; if (task->children == NULL) { bool destroy_taskwait = task->taskwait != NULL; task->taskwait = NULL; gomp_mutex_unlock (&team->task_lock); if (to_free) { gomp_finish_task (to_free); free (to_free); } if (destroy_taskwait) gomp_sem_destroy (&taskwait.taskwait_sem); return; } if (task->children->kind == GOMP_TASK_WAITING) { child_task = task->children; cancelled = gomp_task_run_pre (child_task, task, team); if (__builtin_expect (cancelled, 0)) { if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } goto finish_cancelled; } } else { /* All tasks we are waiting for are already running in other threads. Wait for them. */ if (task->taskwait == NULL) { taskwait.in_depend_wait = false; gomp_sem_init (&taskwait.taskwait_sem, 0); task->taskwait = &taskwait; } taskwait.in_taskwait = true; } gomp_mutex_unlock (&team->task_lock); if (do_wake) { gomp_team_barrier_wake (&team->barrier, do_wake); do_wake = 0; } if (to_free) { gomp_finish_task (to_free); free (to_free); to_free = NULL; } if (child_task) { thr->task = child_task; child_task->fn (child_task->fn_data); thr->task = task; } else gomp_sem_wait (&taskwait.taskwait_sem); gomp_mutex_lock (&team->task_lock); if (child_task) { finish_cancelled:; size_t new_tasks = gomp_task_run_post_handle_depend (child_task, team); /* Remove child_task from children list, and set up the next sibling to be run. */ child_task->prev_child->next_child = child_task->next_child; child_task->next_child->prev_child = child_task->prev_child; if (task->children == child_task) { if (child_task->next_child != child_task) task->children = child_task->next_child; else task->children = NULL; } /* Orphan all the children of CHILD_TASK. */ gomp_clear_parent (child_task->children); /* Remove CHILD_TASK from its taskgroup. */ gomp_task_run_post_remove_taskgroup (child_task); to_free = child_task; child_task = NULL; team->task_count--; if (new_tasks > 1) { do_wake = team->nthreads - team->task_running_count - !task->in_tied_task; if (do_wake > new_tasks) do_wake = new_tasks; } } } }
int omp_get_level (void) { return gomp_thread ()->ts.level; }
static inline int gomp_tid (void) { return (int) gomp_thread (); }
unsigned gomp_resolve_num_threads (unsigned specified, unsigned count) { struct gomp_thread *thr = gomp_thread (); struct gomp_task_icv *icv; unsigned threads_requested, max_num_threads, num_threads; unsigned long busy; struct gomp_thread_pool *pool; icv = gomp_icv (false); if (specified == 1) return 1; else if (thr->ts.active_level >= 1 && !icv->nest_var) return 1; else if (thr->ts.active_level >= gomp_max_active_levels_var) return 1; /* If NUM_THREADS not specified, use nthreads_var. */ if (specified == 0) threads_requested = icv->nthreads_var; else threads_requested = specified; max_num_threads = threads_requested; /* If dynamic threads are enabled, bound the number of threads that we launch. */ if (icv->dyn_var) { unsigned dyn = gomp_dynamic_max_threads (); if (dyn < max_num_threads) max_num_threads = dyn; /* Optimization for parallel sections. */ if (count && count < max_num_threads) max_num_threads = count; } /* UINT_MAX stands for infinity. */ if (__builtin_expect (icv->thread_limit_var == UINT_MAX, 1) || max_num_threads == 1) return max_num_threads; /* The threads_busy counter lives in thread_pool, if there isn't a thread_pool yet, there must be just one thread in the contention group. If thr->team is NULL, this isn't nested parallel, so there is just one thread in the contention group as well, no need to handle it atomically. */ pool = thr->thread_pool; if (thr->ts.team == NULL || pool == NULL) { num_threads = max_num_threads; if (num_threads > icv->thread_limit_var) num_threads = icv->thread_limit_var; if (pool) pool->threads_busy = num_threads; return num_threads; } #ifdef HAVE_SYNC_BUILTINS do { busy = pool->threads_busy; num_threads = max_num_threads; if (icv->thread_limit_var - busy + 1 < num_threads) num_threads = icv->thread_limit_var - busy + 1; } while (__sync_val_compare_and_swap (&pool->threads_busy, busy, busy + num_threads - 1) != busy); #else gomp_mutex_lock (&gomp_managed_threads_lock); num_threads = max_num_threads; busy = pool->threads_busy; if (icv->thread_limit_var - busy + 1 < num_threads) num_threads = icv->thread_limit_var - busy + 1; pool->threads_busy += num_threads - 1; gomp_mutex_unlock (&gomp_managed_threads_lock); #endif return num_threads; }
void gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; struct gomp_work_share *ws = thr->ts.work_share; unsigned int i, bits[MAX_COLLAPSED_BITS], num_bits = 0; unsigned long ent, num_ents, elt_sz, shift_sz; struct gomp_doacross_work_share *doacross; if (team == NULL || team->nthreads == 1) return; for (i = 0; i < ncounts; i++) { /* If any count is 0, GOMP_doacross_{post,wait} can't be called. */ if (counts[i] == 0) return; if (num_bits <= MAX_COLLAPSED_BITS) { unsigned int this_bits; if (counts[i] == 1) this_bits = 1; else this_bits = __SIZEOF_LONG__ * __CHAR_BIT__ - __builtin_clzl (counts[i] - 1); if (num_bits + this_bits <= MAX_COLLAPSED_BITS) { bits[i] = this_bits; num_bits += this_bits; } else num_bits = MAX_COLLAPSED_BITS + 1; } } if (ws->sched == GFS_STATIC) num_ents = team->nthreads; else if (ws->sched == GFS_GUIDED) num_ents = counts[0]; else num_ents = (counts[0] - 1) / chunk_size + 1; if (num_bits <= MAX_COLLAPSED_BITS) { elt_sz = sizeof (unsigned long); shift_sz = ncounts * sizeof (unsigned int); } else { elt_sz = sizeof (unsigned long) * ncounts; shift_sz = 0; } elt_sz = (elt_sz + 63) & ~63UL; doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz + shift_sz); doacross->chunk_size = chunk_size; doacross->elt_sz = elt_sz; doacross->ncounts = ncounts; doacross->flattened = false; doacross->array = (unsigned char *) ((((uintptr_t) (doacross + 1)) + 63 + shift_sz) & ~(uintptr_t) 63); if (num_bits <= MAX_COLLAPSED_BITS) { unsigned int shift_count = 0; doacross->flattened = true; for (i = ncounts; i > 0; i--) { doacross->shift_counts[i - 1] = shift_count; shift_count += bits[i - 1]; } for (ent = 0; ent < num_ents; ent++) *(unsigned long *) (doacross->array + ent * elt_sz) = 0; } else for (ent = 0; ent < num_ents; ent++) memset (doacross->array + ent * elt_sz, '\0', sizeof (unsigned long) * ncounts); if (ws->sched == GFS_STATIC && chunk_size == 0) { unsigned long q = counts[0] / num_ents; unsigned long t = counts[0] % num_ents; doacross->boundary = t * (q + 1); doacross->q = q; doacross->t = t; } ws->doacross = doacross; }