void
gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
unsigned int generation, gen;
if (__builtin_expect (state & BAR_WAS_LAST, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
bar->awaited = bar->total;
team->work_share_cancelled = 0;
if (__builtin_expect (team->task_count, 0))
{
gomp_barrier_handle_tasks (state);
state &= ~BAR_WAS_LAST;
}
else
{
state &= ~BAR_CANCELLED;
state += BAR_INCR - BAR_WAS_LAST;
__atomic_store_n (&bar->generation, state, MEMMODEL_RELEASE);
futex_wake ((int *) &bar->generation, INT_MAX);
return;
}
}
generation = state;
state &= ~BAR_CANCELLED;
do
{
do_wait ((int *) &bar->generation, generation);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
if (__builtin_expect (gen & BAR_TASK_PENDING, 0))
{
gomp_barrier_handle_tasks (state);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
}
generation |= gen & BAR_WAITING_FOR_TASK;
}
while (gen != state + BAR_INCR);
}
void
gomp_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
if (__builtin_expect ((state & 1) != 0, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
bar->awaited = bar->total;
atomic_write_barrier ();
bar->generation += 4;
futex_wake ((int *) &bar->generation, INT_MAX);
}
else
{
unsigned int generation = state;
do
do_wait ((int *) &bar->generation, generation);
while (bar->generation == generation);
}
}
int pthread_barrier_wait(pthread_barrier_t *barrier) {
// Lock barrier
pthread_mutex_lock(&barrier->lock);
// Barrier break?
if (0 < --barrier->count_left) {
futex_t event = barrier->event;
pthread_mutex_unlock(&barrier->lock);
do {
futex_wait(&barrier->event, event);
} while (barrier->event == event);
} else {
++barrier->event;
barrier->count_left = barrier->count_init;
futex_wake(&barrier->event, INT32_MAX);
pthread_mutex_unlock(&barrier->lock);
}
return 0;
}
static int SBCELT_FutexHelper() {
while (1) {
// Wait for the lib to signal us.
while (workpage->ready == 1) {
int err = futex_wait(&workpage->ready, 1, NULL);
if (err == 0) {
break;
}
}
SBCELT_DecodeSingleFrame();
workpage->ready = 1;
if (!workpage->busywait) {
futex_wake(&workpage->ready);
}
}
return -2;
}
void
gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
unsigned int generation, gen;
if (__builtin_expect ((state & 1) != 0, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
bar->awaited = bar->total;
if (__builtin_expect (team->task_count, 0))
{
gomp_barrier_handle_tasks (state);
state &= ~1;
}
else
{
__atomic_store_n (&bar->generation, state + 3, MEMMODEL_RELEASE);
futex_wake ((int *) &bar->generation, INT_MAX);
return;
}
}
generation = state;
do
{
do_wait ((int *) &bar->generation, generation);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
if (__builtin_expect (gen & 1, 0))
{
gomp_barrier_handle_tasks (state);
gen = __atomic_load_n (&bar->generation, MEMMODEL_ACQUIRE);
}
if ((gen & 2) != 0)
generation |= 2;
}
while (gen != state + 4);
}
void
gomp_barrier_wait_end (gomp_barrier_t *bar, bool last)
{
if (last)
{
bar->generation++;
futex_wake (&bar->generation, INT_MAX);
}
else
{
unsigned int generation = bar->generation;
gomp_mutex_unlock (&bar->mutex);
do
futex_wait (&bar->generation, generation);
while (bar->generation == generation);
}
if (__sync_add_and_fetch (&bar->arrived, -1) == 0)
gomp_mutex_unlock (&bar->mutex);
}
void
gomp_team_barrier_wait_end (gomp_barrier_t *bar, gomp_barrier_state_t state)
{
unsigned int generation;
if (__builtin_expect ((state & 1) != 0, 0))
{
/* Next time we'll be awaiting TOTAL threads again. */
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
bar->awaited = bar->total;
atomic_write_barrier ();
if (__builtin_expect (team->task_count, 0))
{
gomp_barrier_handle_tasks (state);
state &= ~1;
}
else
{
bar->generation = state + 3;
futex_wake ((int *) &bar->generation, INT_MAX);
return;
}
}
generation = state;
do
{
do_wait ((int *) &bar->generation, generation);
if (__builtin_expect (bar->generation & 1, 0))
gomp_barrier_handle_tasks (state);
if ((bar->generation & 2))
generation |= 2;
}
while (bar->generation != state + 4);
}
int futex_wake_some(struct sem_wakeup_info *wi, int num_semids, int num)
{
int i;
int ret;
struct lockinfo *l;
int found = 0;
for (i = 0; i < num; i++) {
l = worklist_rm();
if (!l)
break;
if (l->data != 1)
fprintf(stderr, "warning, lockinfo data was %d\n",
l->data);
l->data = 0;
ret = futex_wake(&l->data, 1, 0);
if (ret < 0) {
perror("futex wake");
exit(1);
}
found++;
}
return found;
}
int futex(int *uaddr, int op, int val,
const struct timespec *timeout,
int *uaddr2, int val3)
{
uint64_t ms_timeout = (uint64_t)-1;
assert(uaddr2 == NULL);
assert(val3 == 0);
if(timeout != NULL) {
ms_timeout = timeout->tv_sec*1000 + timeout->tv_nsec/1000000L;
assert(ms_timeout > 0);
}
run_once(futex_init());
switch(op) {
case FUTEX_WAIT:
return futex_wait(uaddr, val, ms_timeout);
case FUTEX_WAKE:
return futex_wake(uaddr, val);
default:
errno = ENOSYS;
return -1;
}
return -1;
}
static int
sync_impl_signal(struct thread_sync_impl *impl)
{
futex_wake(&impl->n, 1);
return 0;
}
void
gomp_mutex_unlock_slow (gomp_mutex_t *mutex)
{
futex_wake (mutex, 1);
}
void
gomp_sem_post_slow (gomp_sem_t *sem)
{
futex_wake (sem, 1);
}
int main(int argc, char *argv[])
{
int ret = RET_PASS;
pthread_t child;
int c;
while ((c = getopt(argc, argv, "chv:")) != -1) {
switch (c) {
case 'c':
log_color(1);
break;
case 'h':
usage(basename(argv[0]));
exit(0);
case 'v':
log_verbosity(atoi(optarg));
break;
default:
usage(basename(argv[0]));
exit(1);
}
}
printf("%s: Detect mismatched requeue_pi operations\n",
basename(argv[0]));
if (pthread_create(&child, NULL, blocking_child, NULL)) {
error("pthread_create\n", errno);
ret = RET_ERROR;
goto out;
}
/* Allow the child to block in the kernel. */
sleep(1);
/*
* The kernel should detect the waiter did not setup the
* q->requeue_pi_key and return -EINVAL. If it does not,
* it likely gave the lock to the child, which is now hung
* in the kernel.
*/
ret = futex_cmp_requeue_pi(&f1, f1, &f2, 1, 0, FUTEX_PRIVATE_FLAG);
if (ret < 0) {
if (errno == EINVAL) {
/*
* The kernel correctly detected the mismatched
* requeue_pi target and aborted. Wake the child with
* FUTEX_WAKE.
*/
ret = futex_wake(&f1, 1, FUTEX_PRIVATE_FLAG);
if (ret == 1) {
ret = RET_PASS;
} else if (ret < 0) {
error("futex_wake\n", errno);
ret = RET_ERROR;
} else {
error("futex_wake did not wake the child\n", 0);
ret = RET_ERROR;
}
} else {
error("futex_cmp_requeue_pi\n", errno);
ret = RET_ERROR;
}
} else if (ret > 0) {
fail("futex_cmp_requeue_pi failed to detect the mismatch\n");
ret = RET_FAIL;
} else {
error("futex_cmp_requeue_pi found no waiters\n", 0);
ret = RET_ERROR;
}
pthread_join(child, NULL);
if (!ret)
ret = child_ret;
out:
/* If the kernel crashes, we shouldn't return at all. */
print_result(TEST_NAME, ret);
return ret;
}
void AO_signal_value(AO_lock_ptr_t lock, AO_lock_val_t val) {
futex_wake(lock,val); // wake up exactly one thread
}
void AO_barrier_add(AO_lock_ptr_t lock, unsigned int v) {
ao_t v1 = AO_fetch_and_add_release((ao_t*)lock,(ao_t)v);
if( v1 == 1 )
futex_wake(lock,1); // wake up exactly one thread
}
int AO_unlock_g( AO_lock_ptr_t lock, AO_lock_val_t self ) {
if( AO_load((ao_t*) lock) != (ao_t) self ) return 0;
AO_store_release(lock,0);
futex_wake(lock,1); // wake up exactly one thread
return 1;
}
void
gomp_team_barrier_wake (gomp_barrier_t *bar, int count)
{
futex_wake ((int *) &bar->generation, count == 0 ? INT_MAX : count);
}
void unlock() noexcept {
if ( 1 != value_.fetch_sub( 1, std::memory_order_acquire) ) {
value_.store( 0, std::memory_order_release);
futex_wake( & value_);
}
}
/* Try to acquire write lock for RWLOCK or return after specfied time. */
int
pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
const struct timespec *abstime)
{
int result = 0;
bool wake_readers = false;
int futex_shared =
rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED;
/* Make sure we are alone. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
while (1)
{
int err;
/* Get the rwlock if there is no writer and no reader. */
if (rwlock->__data.__writer == 0 && rwlock->__data.__nr_readers == 0)
{
/* Mark self as writer. */
rwlock->__data.__writer = THREAD_GETMEM (THREAD_SELF, tid);
break;
}
/* Make sure we are not holding the rwlock as a writer. This is
a deadlock situation we recognize and report. */
if (__builtin_expect (rwlock->__data.__writer
== THREAD_GETMEM (THREAD_SELF, tid), 0))
{
result = EDEADLK;
break;
}
/* Make sure the passed in timeout value is valid. Ideally this
test would be executed once. But since it must not be
performed if we would not block at all simply moving the test
to the front is no option. Replicating all the code is
costly while this test is not. */
if (__builtin_expect (abstime->tv_nsec >= 1000000000
|| abstime->tv_nsec < 0, 0))
{
result = EINVAL;
break;
}
/* Remember that we are a writer. */
if (++rwlock->__data.__nr_writers_queued == 0)
{
/* Overflow on number of queued writers. */
--rwlock->__data.__nr_writers_queued;
result = EAGAIN;
break;
}
int waitval = rwlock->__data.__writer_wakeup;
/* Free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
/* Wait for the writer or reader(s) to finish. We handle ETIMEDOUT
below; on other return values, we decide how to continue based on
the state of the rwlock. */
err = futex_abstimed_wait (&rwlock->__data.__writer_wakeup, waitval,
abstime, futex_shared);
/* Get the lock. */
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
/* To start over again, remove the thread from the writer list. */
--rwlock->__data.__nr_writers_queued;
/* Did the futex call time out? */
if (err == ETIMEDOUT)
{
result = ETIMEDOUT;
/* If we prefer writers, it can have happened that readers blocked
for us to acquire the lock first. If we have timed out, we need
to wake such readers if there are any, and if there is no writer
currently (otherwise, the writer will take care of wake-up).
Likewise, even if we prefer readers, we can be responsible for
wake-up (see pthread_rwlock_unlock) if no reader or writer has
acquired the lock. We have timed out and thus not consumed a
futex wake-up; therefore, if there is no other blocked writer
that would consume the wake-up and thus take over responsibility,
we need to wake blocked readers. */
if ((!PTHREAD_RWLOCK_PREFER_READER_P (rwlock)
|| ((rwlock->__data.__nr_readers == 0)
&& (rwlock->__data.__nr_writers_queued == 0)))
&& (rwlock->__data.__nr_readers_queued > 0)
&& (rwlock->__data.__writer == 0))
{
++rwlock->__data.__readers_wakeup;
wake_readers = true;
}
break;
}
}
/* We are done, free the lock. */
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
/* Might be required after timeouts. */
if (wake_readers)
futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared);
return result;
}
void *signal_wakerfn(void *arg)
{
struct thread_arg *args = (struct thread_arg *)arg;
unsigned int old_val;
int nr_requeue = 0;
int task_count = 0;
int nr_wake = 1;
int i = 0;
info("Waker: waiting for waiters to block\n");
while (waiters_blocked.val < THREAD_MAX)
usleep(1000);
usleep(1000);
while (task_count < THREAD_MAX && waiters_woken.val < THREAD_MAX) {
info("task_count: %d, waiters_woken: %d\n",
task_count, waiters_woken.val);
if (args->lock) {
info("Calling FUTEX_LOCK_PI on mutex=%x @ %p\n",
f2, &f2);
futex_lock_pi(&f2, NULL, 0, FUTEX_PRIVATE_FLAG);
}
info("Waker: Calling signal\n");
/* cond_signal */
old_val = f1;
args->ret = futex_cmp_requeue_pi(&f1, old_val, &f2,
nr_wake, nr_requeue,
FUTEX_PRIVATE_FLAG);
if (args->ret < 0)
args->ret = -errno;
info("futex: %x\n", f2);
if (args->lock) {
info("Calling FUTEX_UNLOCK_PI on mutex=%x @ %p\n",
f2, &f2);
futex_unlock_pi(&f2, FUTEX_PRIVATE_FLAG);
}
info("futex: %x\n", f2);
if (args->ret < 0) {
error("FUTEX_CMP_REQUEUE_PI failed\n", errno);
args->ret = RET_ERROR;
break;
}
task_count += args->ret;
usleep(SIGNAL_PERIOD_US);
i++;
/* we have to loop at least THREAD_MAX times */
if (i > MAX_WAKE_ITERS + THREAD_MAX) {
error("max signaling iterations (%d) reached, giving up on pending waiters.\n",
0, MAX_WAKE_ITERS + THREAD_MAX);
args->ret = RET_ERROR;
break;
}
}
futex_wake(&wake_complete, 1, FUTEX_PRIVATE_FLAG);
if (args->ret >= 0)
args->ret = task_count;
info("Waker: exiting with %d\n", args->ret);
info("Waker: waiters_woken: %d\n", waiters_woken.val);
pthread_exit((void *)&args->ret);
}
__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
{
while (1)
{
int val, newval;
/* We need acquire memory order for this load because if the value
signals that initialization has finished, we need to see any
data modifications done during initialization. */
val = atomic_load_acquire (once_control);
do
{
/* Check if the initialization has already been done. */
if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
return 0;
/* We try to set the state to in-progress and having the current
fork generation. We don't need atomic accesses for the fork
generation because it's immutable in a particular process, and
forked child processes start with a single thread that modified
the generation. */
newval = __fork_generation | __PTHREAD_ONCE_INPROGRESS;
/* We need acquire memory order here for the same reason as for the
load from once_control above. */
}
while (__glibc_unlikely (!atomic_compare_exchange_weak_acquire (
once_control, &val, newval)));
/* Check if another thread already runs the initializer. */
if ((val & __PTHREAD_ONCE_INPROGRESS) != 0)
{
/* Check whether the initializer execution was interrupted by a
fork. We know that for both values, __PTHREAD_ONCE_INPROGRESS
is set and __PTHREAD_ONCE_DONE is not. */
if (val == newval)
{
/* Same generation, some other thread was faster. Wait and
retry. */
futex_wait_simple ((unsigned int *) once_control,
(unsigned int) newval, FUTEX_PRIVATE);
continue;
}
}
/* This thread is the first here. Do the initialization.
Register a cleanup handler so that in case the thread gets
interrupted the initialization can be restarted. */
pthread_cleanup_push (clear_once_control, once_control);
init_routine ();
pthread_cleanup_pop (0);
/* Mark *once_control as having finished the initialization. We need
release memory order here because we need to synchronize with other
threads that want to use the initialized data. */
atomic_store_release (once_control, __PTHREAD_ONCE_DONE);
/* Wake up all other threads. */
futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
break;
}
return 0;
}
/*
* stress_fuxex()
* stress system by futex calls. The intention is not to
* efficiently use futex, but to stress the futex system call
* by rapidly calling it on wait and wakes
*/
int stress_futex(
uint64_t *const counter,
const uint32_t instance,
const uint64_t max_ops,
const char *name)
{
uint64_t *timeout = &shared->futex.timeout[instance];
uint32_t *futex = &shared->futex.futex[instance];
pid_t pid;
again:
pid = fork();
if (pid < 0) {
if (opt_do_run && (errno == EAGAIN))
goto again;
pr_err(stderr, "%s: fork failed: errno=%d: (%s)\n",
name, errno, strerror(errno));
}
if (pid > 0) {
int status;
setpgid(pid, pgrp);
do {
int ret;
/*
* Break early in case wake gets stuck
* (which it shouldn't)
*/
if (!opt_do_run)
break;
ret = futex_wake(futex, 1);
if (opt_flags & OPT_FLAGS_VERIFY) {
if (ret < 0)
pr_fail(stderr, "%s: futex wake failed: errno=%d (%s)\n",
name, errno, strerror(errno));
}
} while (opt_do_run && (!max_ops || *counter < max_ops));
/* Kill waiter process */
(void)kill(pid, SIGKILL);
(void)waitpid(pid, &status, 0);
pr_dbg(stderr, "%s: futex timeouts: %" PRIu64 "\n", name, *timeout);
} else {
setpgid(0, pgrp);
do {
/* Small timeout to force rapid timer wakeups */
const struct timespec t = { .tv_sec = 0, .tv_nsec = 5000 };
int ret;
/* Break early before potential long wait */
if (!opt_do_run)
break;
ret = futex_wait(futex, 0, &t);
/* timeout, re-do, stress on stupid fast polling */
if ((ret < 0) && (errno == ETIMEDOUT)) {
(*timeout)++;
if (*timeout > 100000) {
pr_fail(stderr, "%s: futex wait, too "
"many timeouts: errno=%d (%s)\n",
name, errno, strerror(errno));
break;
}
} else {
if ((ret < 0) && (opt_flags & OPT_FLAGS_VERIFY)) {
pr_fail(stderr, "%s: futex wait "
"failed: errno=%d (%s)\n",
name, errno, strerror(errno));
}
(*counter)++;
}
} while (opt_do_run && (!max_ops || *counter < max_ops));
}
return EXIT_SUCCESS;
}
