void * GOMP_PLUGIN_realloc (void *ptr, size_t size) { return gomp_realloc (ptr, size); }
void gomp_init_num_threads (void) { #ifdef HAVE_PTHREAD_AFFINITY_NP #if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE) gomp_cpuset_size = sysconf (_SC_NPROCESSORS_CONF); gomp_cpuset_size = CPU_ALLOC_SIZE (gomp_cpuset_size); #else gomp_cpuset_size = sizeof (cpu_set_t); #endif gomp_cpusetp = (cpu_set_t *) gomp_malloc (gomp_cpuset_size); do { int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size, gomp_cpusetp); if (ret == 0) { /* Count only the CPUs this process can use. */ gomp_global_icv.nthreads_var = gomp_cpuset_popcount (gomp_cpuset_size, gomp_cpusetp); if (gomp_global_icv.nthreads_var == 0) break; gomp_get_cpuset_size = gomp_cpuset_size; #ifdef CPU_ALLOC_SIZE unsigned long i; for (i = gomp_cpuset_size * 8; i; i--) if (CPU_ISSET_S (i - 1, gomp_cpuset_size, gomp_cpusetp)) break; gomp_cpuset_size = CPU_ALLOC_SIZE (i); #endif return; } if (ret != EINVAL) break; #ifdef CPU_ALLOC_SIZE if (gomp_cpuset_size < sizeof (cpu_set_t)) gomp_cpuset_size = sizeof (cpu_set_t); else gomp_cpuset_size = gomp_cpuset_size * 2; if (gomp_cpuset_size < 8 * sizeof (cpu_set_t)) gomp_cpusetp = (cpu_set_t *) gomp_realloc (gomp_cpusetp, gomp_cpuset_size); else { /* Avoid gomp_fatal if too large memory allocation would be requested, e.g. kernel returning EINVAL all the time. */ void *p = realloc (gomp_cpusetp, gomp_cpuset_size); if (p == NULL) break; gomp_cpusetp = (cpu_set_t *) p; } #else break; #endif } while (1); gomp_cpuset_size = 0; gomp_global_icv.nthreads_var = 1; free (gomp_cpusetp); gomp_cpusetp = NULL; #endif #if defined(__ANDROID__) gomp_global_icv.nthreads_var = sc_nprocessors_actu (); #elif defined(_SC_NPROCESSORS_ONLN) gomp_global_icv.nthreads_var = sysconf (_SC_NPROCESSORS_ONLN); #endif }
static void gomp_task_handle_depend (struct gomp_task *task, struct gomp_task *parent, void **depend) { size_t ndepend = (uintptr_t) depend[0]; size_t nout = (uintptr_t) depend[1]; size_t i; hash_entry_type ent; task->depend_count = ndepend; task->num_dependees = 0; if (parent->depend_hash == NULL) parent->depend_hash = htab_create (2 * ndepend > 12 ? 2 * ndepend : 12); for (i = 0; i < ndepend; i++) { task->depend[i].addr = depend[2 + i]; task->depend[i].next = NULL; task->depend[i].prev = NULL; task->depend[i].task = task; task->depend[i].is_in = i >= nout; task->depend[i].redundant = false; task->depend[i].redundant_out = false; hash_entry_type *slot = htab_find_slot (&parent->depend_hash, &task->depend[i], INSERT); hash_entry_type out = NULL, last = NULL; if (*slot) { /* If multiple depends on the same task are the same, all but the first one are redundant. As inout/out come first, if any of them is inout/out, it will win, which is the right semantics. */ if ((*slot)->task == task) { task->depend[i].redundant = true; continue; } for (ent = *slot; ent; ent = ent->next) { if (ent->redundant_out) break; last = ent; /* depend(in:...) doesn't depend on earlier depend(in:...). */ if (i >= nout && ent->is_in) continue; if (!ent->is_in) out = ent; struct gomp_task *tsk = ent->task; if (tsk->dependers == NULL) { tsk->dependers = gomp_malloc (sizeof (struct gomp_dependers_vec) + 6 * sizeof (struct gomp_task *)); tsk->dependers->n_elem = 1; tsk->dependers->allocated = 6; tsk->dependers->elem[0] = task; task->num_dependees++; continue; } /* We already have some other dependency on tsk from earlier depend clause. */ else if (tsk->dependers->n_elem && (tsk->dependers->elem[tsk->dependers->n_elem - 1] == task)) continue; else if (tsk->dependers->n_elem == tsk->dependers->allocated) { tsk->dependers->allocated = tsk->dependers->allocated * 2 + 2; tsk->dependers = gomp_realloc (tsk->dependers, sizeof (struct gomp_dependers_vec) + (tsk->dependers->allocated * sizeof (struct gomp_task *))); } tsk->dependers->elem[tsk->dependers->n_elem++] = task; task->num_dependees++; } task->depend[i].next = *slot; (*slot)->prev = &task->depend[i]; } *slot = &task->depend[i]; /* There is no need to store more than one depend({,in}out:) task per address in the hash table chain for the purpose of creation of deferred tasks, because each out depends on all earlier outs, thus it is enough to record just the last depend({,in}out:). For depend(in:), we need to keep all of the previous ones not terminated yet, because a later depend({,in}out:) might need to depend on all of them. So, if the new task's clause is depend({,in}out:), we know there is at most one other depend({,in}out:) clause in the list (out). For non-deferred tasks we want to see all outs, so they are moved to the end of the chain, after first redundant_out entry all following entries should be redundant_out. */ if (!task->depend[i].is_in && out) { if (out != last) { out->next->prev = out->prev; out->prev->next = out->next; out->next = last->next; out->prev = last; last->next = out; if (out->next) out->next->prev = out; } out->redundant_out = true; } } }
void gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads, struct gomp_work_share *work_share) { struct gomp_thread_start_data *start_data; struct gomp_thread *thr, *nthr; struct gomp_team *team; bool nested; unsigned i, n, old_threads_used = 0; pthread_attr_t thread_attr, *attr; thr = gomp_thread (); nested = thr->ts.team != NULL; team = new_team (nthreads, work_share); /* Always save the previous state, even if this isn't a nested team. In particular, we should save any work share state from an outer orphaned work share construct. */ team->prev_ts = thr->ts; thr->ts.team = team; thr->ts.work_share = work_share; thr->ts.team_id = 0; thr->ts.work_share_generation = 0; thr->ts.static_trip = 0; if (nthreads == 1) return; i = 1; /* We only allow the reuse of idle threads for non-nested PARALLEL regions. This appears to be implied by the semantics of threadprivate variables, but perhaps that's reading too much into things. Certainly it does prevent any locking problems, since only the initial program thread will modify gomp_threads. */ if (!nested) { old_threads_used = gomp_threads_used; if (nthreads <= old_threads_used) n = nthreads; else if (old_threads_used == 0) { n = 0; gomp_barrier_init (&gomp_threads_dock, nthreads); } else { n = old_threads_used; /* Increase the barrier threshold to make sure all new threads arrive before the team is released. */ gomp_barrier_reinit (&gomp_threads_dock, nthreads); } /* Not true yet, but soon will be. We're going to release all threads from the dock, and those that aren't part of the team will exit. */ gomp_threads_used = nthreads; /* Release existing idle threads. */ for (; i < n; ++i) { nthr = gomp_threads[i]; nthr->ts.team = team; nthr->ts.work_share = work_share; nthr->ts.team_id = i; nthr->ts.work_share_generation = 0; nthr->ts.static_trip = 0; nthr->fn = fn; nthr->data = data; team->ordered_release[i] = &nthr->release; } if (i == nthreads) goto do_release; /* If necessary, expand the size of the gomp_threads array. It is expected that changes in the number of threads is rare, thus we make no effort to expand gomp_threads_size geometrically. */ if (nthreads >= gomp_threads_size) { gomp_threads_size = nthreads + 1; gomp_threads = gomp_realloc (gomp_threads, gomp_threads_size * sizeof (struct gomp_thread_data *)); } } attr = &gomp_thread_attr; if (gomp_cpu_affinity != NULL) { size_t stacksize; pthread_attr_init (&thread_attr); pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED); if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize)) pthread_attr_setstacksize (&thread_attr, stacksize); attr = &thread_attr; } start_data = gomp_alloca (sizeof (struct gomp_thread_start_data) * (nthreads-i)); /* Launch new threads. */ for (; i < nthreads; ++i, ++start_data) { pthread_t pt; int err; start_data->ts.team = team; start_data->ts.work_share = work_share; start_data->ts.team_id = i; start_data->ts.work_share_generation = 0; start_data->ts.static_trip = 0; start_data->fn = fn; start_data->fn_data = data; start_data->nested = nested; if (gomp_cpu_affinity != NULL) gomp_init_thread_affinity (attr); err = pthread_create (&pt, attr, gomp_thread_start, start_data); if (err != 0) gomp_fatal ("Thread creation failed: %s", strerror (err)); } if (gomp_cpu_affinity != NULL) pthread_attr_destroy (&thread_attr); do_release: gomp_barrier_wait (nested ? &team->barrier : &gomp_threads_dock); /* Decrease the barrier threshold to match the number of threads that should arrive back at the end of this team. The extra threads should be exiting. Note that we arrange for this test to never be true for nested teams. */ if (nthreads < old_threads_used) gomp_barrier_reinit (&gomp_threads_dock, nthreads); }
void GOMP_task (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *), long arg_size, long arg_align, bool if_clause, unsigned flags, void **depend) { struct gomp_thread *thr = gomp_thread (); struct gomp_team *team = thr->ts.team; #ifdef HAVE_BROKEN_POSIX_SEMAPHORES /* If pthread_mutex_* is used for omp_*lock*, then each task must be tied to one thread all the time. This means UNTIED tasks must be tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN might be running on different thread than FN. */ if (cpyfn) if_clause = false; if (flags & 1) flags &= ~1; #endif /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (team && (gomp_team_barrier_cancelled (&team->barrier) || (thr->task->taskgroup && thr->task->taskgroup->cancelled))) return; if (!if_clause || team == NULL || (thr->task && thr->task->final_task) || team->task_count > 64 * team->nthreads) { struct gomp_task task; /* If there are depend clauses and earlier deferred sibling tasks with depend clauses, check if there isn't a dependency. If there is, we need to wait for them. There is no need to handle depend clauses for non-deferred tasks other than this, because the parent task is suspended until the child task finishes and thus it can't start further child tasks. */ if ((flags & 8) && thr->task && thr->task->depend_hash) gomp_task_maybe_wait_for_dependencies (depend); gomp_init_task (&task, thr->task, gomp_icv (false)); task.kind = GOMP_TASK_IFFALSE; task.final_task = (thr->task && thr->task->final_task) || (flags & 2); if (thr->task) { task.in_tied_task = thr->task->in_tied_task; task.taskgroup = thr->task->taskgroup; } thr->task = &task; if (__builtin_expect (cpyfn != NULL, 0)) { char buf[arg_size + arg_align - 1]; char *arg = (char *) (((uintptr_t) buf + arg_align - 1) & ~(uintptr_t) (arg_align - 1)); cpyfn (arg, data); fn (arg); } else fn (data); /* Access to "children" is normally done inside a task_lock mutex region, but the only way this particular task.children can be set is if this thread's task work function (fn) creates children. So since the setter is *this* thread, we need no barriers here when testing for non-NULL. We can have task.children set by the current thread then changed by a child thread, but seeing a stale non-NULL value is not a problem. Once past the task_lock acquisition, this thread will see the real value of task.children. */ if (task.children != NULL) { gomp_mutex_lock (&team->task_lock); gomp_clear_parent (task.children); gomp_mutex_unlock (&team->task_lock); } gomp_end_task (); } else { struct gomp_task *task; struct gomp_task *parent = thr->task; struct gomp_taskgroup *taskgroup = parent->taskgroup; char *arg; bool do_wake; size_t depend_size = 0; if (flags & 8) depend_size = ((uintptr_t) depend[0] * sizeof (struct gomp_task_depend_entry)); task = gomp_malloc (sizeof (*task) + depend_size + arg_size + arg_align - 1); arg = (char *) (((uintptr_t) (task + 1) + depend_size + arg_align - 1) & ~(uintptr_t) (arg_align - 1)); gomp_init_task (task, parent, gomp_icv (false)); task->kind = GOMP_TASK_IFFALSE; task->in_tied_task = parent->in_tied_task; task->taskgroup = taskgroup; thr->task = task; if (cpyfn) { cpyfn (arg, data); task->copy_ctors_done = true; } else memcpy (arg, data, arg_size); thr->task = parent; task->kind = GOMP_TASK_WAITING; task->fn = fn; task->fn_data = arg; task->final_task = (flags & 2) >> 1; gomp_mutex_lock (&team->task_lock); /* If parallel or taskgroup has been cancelled, don't start new tasks. */ if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier) || (taskgroup && taskgroup->cancelled)) && !task->copy_ctors_done, 0)) { gomp_mutex_unlock (&team->task_lock); gomp_finish_task (task); free (task); return; } if (taskgroup) taskgroup->num_children++; if (depend_size) { size_t ndepend = (uintptr_t) depend[0]; size_t nout = (uintptr_t) depend[1]; size_t i; hash_entry_type ent; task->depend_count = ndepend; task->num_dependees = 0; if (parent->depend_hash == NULL) parent->depend_hash = htab_create (2 * ndepend > 12 ? 2 * ndepend : 12); for (i = 0; i < ndepend; i++) { task->depend[i].addr = depend[2 + i]; task->depend[i].next = NULL; task->depend[i].prev = NULL; task->depend[i].task = task; task->depend[i].is_in = i >= nout; task->depend[i].redundant = false; task->depend[i].redundant_out = false; hash_entry_type *slot = htab_find_slot (&parent->depend_hash, &task->depend[i], INSERT); hash_entry_type out = NULL, last = NULL; if (*slot) { /* If multiple depends on the same task are the same, all but the first one are redundant. As inout/out come first, if any of them is inout/out, it will win, which is the right semantics. */ if ((*slot)->task == task) { task->depend[i].redundant = true; continue; } for (ent = *slot; ent; ent = ent->next) { if (ent->redundant_out) break; last = ent; /* depend(in:...) doesn't depend on earlier depend(in:...). */ if (i >= nout && ent->is_in) continue; if (!ent->is_in) out = ent; struct gomp_task *tsk = ent->task; if (tsk->dependers == NULL) { tsk->dependers = gomp_malloc (sizeof (struct gomp_dependers_vec) + 6 * sizeof (struct gomp_task *)); tsk->dependers->n_elem = 1; tsk->dependers->allocated = 6; tsk->dependers->elem[0] = task; task->num_dependees++; continue; } /* We already have some other dependency on tsk from earlier depend clause. */ else if (tsk->dependers->n_elem && (tsk->dependers->elem[tsk->dependers->n_elem - 1] == task)) continue; else if (tsk->dependers->n_elem == tsk->dependers->allocated) { tsk->dependers->allocated = tsk->dependers->allocated * 2 + 2; tsk->dependers = gomp_realloc (tsk->dependers, sizeof (struct gomp_dependers_vec) + (tsk->dependers->allocated * sizeof (struct gomp_task *))); } tsk->dependers->elem[tsk->dependers->n_elem++] = task; task->num_dependees++; } task->depend[i].next = *slot; (*slot)->prev = &task->depend[i]; } *slot = &task->depend[i]; /* There is no need to store more than one depend({,in}out:) task per address in the hash table chain for the purpose of creation of deferred tasks, because each out depends on all earlier outs, thus it is enough to record just the last depend({,in}out:). For depend(in:), we need to keep all of the previous ones not terminated yet, because a later depend({,in}out:) might need to depend on all of them. So, if the new task's clause is depend({,in}out:), we know there is at most one other depend({,in}out:) clause in the list (out). For non-deferred tasks we want to see all outs, so they are moved to the end of the chain, after first redundant_out entry all following entries should be redundant_out. */ if (!task->depend[i].is_in && out) { if (out != last) { out->next->prev = out->prev; out->prev->next = out->next; out->next = last->next; out->prev = last; last->next = out; if (out->next) out->next->prev = out; } out->redundant_out = true; } } if (task->num_dependees) { gomp_mutex_unlock (&team->task_lock); return; } } if (parent->children) { task->next_child = parent->children; task->prev_child = parent->children->prev_child; task->next_child->prev_child = task; task->prev_child->next_child = task; } else { task->next_child = task; task->prev_child = task; } parent->children = task; if (taskgroup) { if (taskgroup->children) { task->next_taskgroup = taskgroup->children; task->prev_taskgroup = taskgroup->children->prev_taskgroup; task->next_taskgroup->prev_taskgroup = task; task->prev_taskgroup->next_taskgroup = task; } else { task->next_taskgroup = task; task->prev_taskgroup = task; } taskgroup->children = task; } if (team->task_queue) { task->next_queue = team->task_queue; task->prev_queue = team->task_queue->prev_queue; task->next_queue->prev_queue = task; task->prev_queue->next_queue = task; } else { task->next_queue = task; task->prev_queue = task; team->task_queue = task; } ++team->task_count; ++team->task_queued_count; gomp_team_barrier_set_task_pending (&team->barrier); do_wake = team->task_running_count + !parent->in_tied_task < team->nthreads; gomp_mutex_unlock (&team->task_lock); if (do_wake) gomp_team_barrier_wake (&team->barrier, 1); } }