int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
#ifdef __EMSCRIPTEN__
// XXX Emscripten: TODO: This is suboptimal but works naively correctly for now. The Emscripten-specific code path below
// has a bug and does not work for some reason. Figure it out and remove this code block.
__wake(&c->_c_seq, -1, 0);
return 0;
#endif
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
#ifdef __EMSCRIPTEN__
int futexResult;
do {
// XXX Emscripten: Bug, this does not work correctly.
futexResult = emscripten_futex_wake_or_requeue(&c->_c_seq, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
c->_c_seq, &m->_m_lock);
} while(futexResult == -EAGAIN);
#else
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
#endif
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
int pthread_barrier_wait(pthread_barrier_t *b)
{
int limit = b->_b_limit;
struct instance *inst;
/* Trivial case: count was set at 1 */
if (!limit) return PTHREAD_BARRIER_SERIAL_THREAD;
/* Process-shared barriers require a separate, inefficient wait */
if (limit < 0) return pshared_barrier_wait(b);
/* Otherwise we need a lock on the barrier object */
while (a_swap(&b->_b_lock, 1))
__wait(&b->_b_lock, &b->_b_waiters, 1, 1);
inst = b->_b_inst;
/* First thread to enter the barrier becomes the "instance owner" */
if (!inst) {
struct instance new_inst = { 0 };
int spins = 10000;
b->_b_inst = inst = &new_inst;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
while (spins-- && !inst->finished)
a_spin();
a_inc(&inst->finished);
while (inst->finished == 1)
__syscall(SYS_futex, &inst->finished, FUTEX_WAIT,1,0);
return PTHREAD_BARRIER_SERIAL_THREAD;
}
/* Last thread to enter the barrier wakes all non-instance-owners */
if (++inst->count == limit) {
b->_b_inst = 0;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
a_store(&inst->last, 1);
if (inst->waiters)
__wake(&inst->last, -1, 1);
} else {
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
__wait(&inst->last, &inst->waiters, 0, 1);
}
/* Last thread to exit the barrier wakes the instance owner */
if (a_fetch_add(&inst->count,-1)==1 && a_fetch_add(&inst->finished,1))
__wake(&inst->finished, 1, 1);
return 0;
}
static int pshared_barrier_wait(pthread_barrier_t *b)
{
int limit = (b->_b_limit & INT_MAX) + 1;
int ret = 0;
int v, w;
if (limit==1) return PTHREAD_BARRIER_SERIAL_THREAD;
while ((v=a_cas(&b->_b_lock, 0, limit)))
__wait(&b->_b_lock, &b->_b_waiters, v, 0);
/* Wait for <limit> threads to get to the barrier */
if (++b->_b_count == limit) {
a_store(&b->_b_count, 0);
ret = PTHREAD_BARRIER_SERIAL_THREAD;
if (b->_b_waiters2) __wake(&b->_b_count, -1, 0);
} else {
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 0);
while ((v=b->_b_count)>0)
__wait(&b->_b_count, &b->_b_waiters2, v, 0);
}
__vm_lock_impl(+1);
/* Ensure all threads have a vm lock before proceeding */
if (a_fetch_add(&b->_b_count, -1)==1-limit) {
a_store(&b->_b_count, 0);
if (b->_b_waiters2) __wake(&b->_b_count, -1, 0);
} else {
while ((v=b->_b_count))
__wait(&b->_b_count, &b->_b_waiters2, v, 0);
}
/* Perform a recursive unlock suitable for self-sync'd destruction */
do {
v = b->_b_lock;
w = b->_b_waiters;
} while (a_cas(&b->_b_lock, v, v==INT_MIN+1 ? 0 : v-1) != v);
/* Wake a thread waiting to reuse or destroy the barrier */
if (v==INT_MIN+1 || (v==1 && w))
__wake(&b->_b_lock, 1, 0);
__vm_unlock_impl();
return ret;
}
int __pthread_once(pthread_once_t *control, void (*init)(void))
{
/* Return immediately if init finished before, but ensure that
* effects of the init routine are visible to the caller. */
if (*control == 2) {
a_barrier();
return 0;
}
/* Try to enter initializing state. Four possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return
* 3 - another thread is running init, waiters present; wait */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
if (a_swap(control, 2) == 3)
__wake(control, -1, 1);
return 0;
case 1:
/* If this fails, so will __wait. */
a_cas(control, 1, 3);
case 3:
__wait(control, 0, 3, 1);
continue;
case 2:
return 0;
}
}
int __pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int type = m->_m_type & 15;
int priv = (m->_m_type & 128) ^ 128;
if (type != PTHREAD_MUTEX_NORMAL) {
self = __pthread_self();
if ((m->_m_lock&0x7fffffff) != self->tid)
return EPERM;
if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (!priv) {
self->robust_list.pending = &m->_m_next;
__vm_lock_impl(+1);
}
volatile void *prev = m->_m_prev;
volatile void *next = m->_m_next;
*(volatile void *volatile *)prev = next;
if (next != &self->robust_list.head) *(volatile void *volatile *)
((char *)next - sizeof(void *)) = prev;
}
cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0);
if (type != PTHREAD_MUTEX_NORMAL && !priv) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, priv);
return 0;
}
int pthread_once(pthread_once_t *control, void (*init)(void))
{
static int waiters;
/* Return immediately if init finished before */
if (*control == 2) return 0;
/* Try to enter initializing state. Three possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return */
for (;;) switch (a_swap(control, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
a_store(control, 2);
if (waiters) __wake(control, -1, 0);
return 0;
case 1:
__wait(control, &waiters, 1, 0);
continue;
case 2:
a_store(control, 2);
return 0;
}
}
int pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int robust = 0;
if (m->_m_type != PTHREAD_MUTEX_NORMAL) {
if (!m->_m_lock)
return EPERM;
self = __pthread_self();
if ((m->_m_lock&0x1fffffff) != self->tid)
return EPERM;
if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (m->_m_type >= 4) {
robust = 1;
self->robust_list.pending = &m->_m_next;
*(void **)m->_m_prev = m->_m_next;
if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev;
__vm_lock_impl(+1);
}
}
cont = a_swap(&m->_m_lock, 0);
if (robust) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, 0);
return 0;
}
int __pthread_once(pthread_once_t *control, void (*init)(void))
{
static int waiters;
/* Return immediately if init finished before, but ensure that
* effects of the init routine are visible to the caller. */
if (*control == 2) {
a_barrier();
return 0;
}
/* Try to enter initializing state. Three possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
a_store(control, 2);
if (waiters) __wake(control, -1, 1);
return 0;
case 1:
__wait(control, &waiters, 1, 1);
continue;
case 2:
return 0;
}
}
static inline void unlock(volatile int *lk)
{
if (lk[0]) {
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
}
static void undo(void *control)
{
/* Wake all waiters, since the waiter status is lost when
* resetting control to the initial state. */
if (a_swap(control, 0) == 3)
__wake(control, -1, 1);
}
int __pthread_once_full(pthread_once_t *control, void (*init)(void))
{
/* Try to enter initializing state. Four possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return
* 3 - another thread is running init, waiters present; wait */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
if (a_swap(control, 2) == 3)
__wake(control, -1, 1);
return 0;
case 1:
/* If this fails, so will __wait. */
a_cas(control, 1, 3);
case 3:
__wait(control, 0, 3, 1);
continue;
case 2:
return 0;
}
}
int timer_delete(timer_t t) {
if ((intptr_t)t < 0) {
pthread_t td = (void*)((uintptr_t)t << 1);
a_store(&td->timer_id, td->timer_id | INT_MIN);
__wake(&td->timer_id, 1, 1);
return 0;
}
return __syscall(SYS_timer_delete, t);
}
static void unwait(pthread_cond_t *c, pthread_mutex_t *m)
{
/* Removing a waiter is non-trivial if we could be using requeue
* based broadcast signals, due to mutex access issues, etc. */
if (c->_c_mutex == (void *)-1) {
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 0);
return;
}
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (c->_c_waiters2) c->_c_waiters2--;
else a_dec(&m->_m_waiters);
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 1);
}
int sem_post(sem_t *sem)
{
int val, waiters;
do {
val = sem->__val[0];
waiters = sem->__val[1];
if (val == SEM_VALUE_MAX) {
errno = EOVERFLOW;
return -1;
}
} while (a_cas(sem->__val, val, val+1+(val<0)) != val);
if (val<0 || waiters) __wake(sem->__val, 1, 0);
return 0;
}
int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
static inline void unlock_requeue(volatile int *l, volatile int *r, int w)
{
a_store(l, 0);
#ifdef __EMSCRIPTEN__
// Here the intent is to wake one waiter, and requeue all other waiters from waiting on address 'l'
// to wait on address 'r' instead. This is not possible at the moment with SharedArrayBuffer Atomics,
// as it does not have a "wake X waiters and requeue the rest" primitive. However this kind of
// primitive is strictly not needed, since it is more like an optimization to avoid spuriously waking
// all waiters, just to make them wait on another location immediately afterwards. Here we do exactly
// that: wake every waiter.
emscripten_futex_wake(l, 0x7FFFFFFF);
#else
if (w) __wake(l, 1, 1);
else __syscall(SYS_futex, l, FUTEX_REQUEUE|128, 0, 1, r) != -ENOSYS
|| __syscall(SYS_futex, l, FUTEX_REQUEUE, 0, 1, r);
#endif
}
int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct timespec *ts)
{
struct cm cm = { .c=c, .m=m };
int r, e=0, seq;
if (m->_m_type && (m->_m_lock&INT_MAX) != pthread_self()->tid)
return EPERM;
if (ts && ts->tv_nsec >= 1000000000UL)
return EINVAL;
pthread_testcancel();
a_inc(&c->_c_waiters);
if (c->_c_mutex != (void *)-1) {
c->_c_mutex = m;
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
c->_c_waiters2++;
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
}
seq = c->_c_seq;
pthread_mutex_unlock(m);
do e = __timedwait(&c->_c_seq, seq, c->_c_clock, ts, cleanup, &cm, 0);
while (c->_c_seq == seq && (!e || e==EINTR));
if (e == EINTR) e = 0;
unwait(c, m);
if ((r=pthread_mutex_lock(m))) return r;
return e;
}
void __aio_wake(void)
{
a_inc(&seq);
__wake(&seq, -1, 1);
}
static inline void unlock(volatile int *l)
{
if (a_swap(l, 0)==2)
__wake(l, 1, 1);
}
static void unlock(volatile int *lk)
{
if (!libc.threads_minus_1) return;
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
void __unlockfile(FILE *f)
{
a_store(&f->lock, 0);
if (f->waiters) __wake(&f->lock, 1, 1);
}
void __vm_unlock() {
if (a_fetch_add(vmlock, -1) == 1 && vmlock[1])
__wake(vmlock, -1, 1);
}
static void undo(void *control)
{
a_store(control, 0);
__wake(control, 1, 1);
}
void __vm_unlock(void)
{
int inc = vmlock[0]>0 ? -1 : 1;
if (a_fetch_add(vmlock, inc)==-inc && vmlock[1])
__wake(vmlock, -1, 1);
}
void __synccall(void (*func)(void *), void *ctx)
{
sigset_t oldmask;
int cs, i, r, pid, self;;
DIR dir = {0};
struct dirent *de;
struct sigaction sa = { .sa_flags = 0, .sa_handler = handler };
struct chain *cp, *next;
struct timespec ts;
/* Blocking signals in two steps, first only app-level signals
* before taking the lock, then all signals after taking the lock,
* is necessary to achieve AS-safety. Blocking them all first would
* deadlock if multiple threads called __synccall. Waiting to block
* any until after the lock would allow re-entry in the same thread
* with the lock already held. */
__block_app_sigs(&oldmask);
LOCK(synccall_lock);
__block_all_sigs(0);
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
head = 0;
if (!libc.threaded) goto single_threaded;
callback = func;
context = ctx;
/* This atomic store ensures that any signaled threads will see the
* above stores, and prevents more than a bounded number of threads,
* those already in pthread_create, from creating new threads until
* the value is cleared to zero again. */
a_store(&__block_new_threads, 1);
/* Block even implementation-internal signals, so that nothing
* interrupts the SIGSYNCCALL handlers. The main possible source
* of trouble is asynchronous cancellation. */
memset(&sa.sa_mask, -1, sizeof sa.sa_mask);
__libc_sigaction(SIGSYNCCALL, &sa, 0);
pid = __syscall(SYS_getpid);
self = __syscall(SYS_gettid);
/* Since opendir is not AS-safe, the DIR needs to be setup manually
* in automatic storage. Thankfully this is easy. */
dir.fd = open("/proc/self/task", O_RDONLY|O_DIRECTORY|O_CLOEXEC);
if (dir.fd < 0) goto out;
/* Initially send one signal per counted thread. But since we can't
* synchronize with thread creation/exit here, there could be too
* few signals. This initial signaling is just an optimization, not
* part of the logic. */
for (i=libc.threads_minus_1; i; i--)
__syscall(SYS_kill, pid, SIGSYNCCALL);
/* Loop scanning the kernel-provided thread list until it shows no
* threads that have not already replied to the signal. */
for (;;) {
int miss_cnt = 0;
while ((de = readdir(&dir))) {
if (!isdigit(de->d_name[0])) continue;
int tid = atoi(de->d_name);
if (tid == self || !tid) continue;
/* Set the target thread as the PI futex owner before
* checking if it's in the list of caught threads. If it
* adds itself to the list after we check for it, then
* it will see its own tid in the PI futex and perform
* the unlock operation. */
a_store(&target_tid, tid);
/* Thread-already-caught is a success condition. */
for (cp = head; cp && cp->tid != tid; cp=cp->next);
if (cp) continue;
r = -__syscall(SYS_tgkill, pid, tid, SIGSYNCCALL);
/* Target thread exit is a success condition. */
if (r == ESRCH) continue;
/* The FUTEX_LOCK_PI operation is used to loan priority
* to the target thread, which otherwise may be unable
* to run. Timeout is necessary because there is a race
* condition where the tid may be reused by a different
* process. */
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_nsec += 10000000;
if (ts.tv_nsec >= 1000000000) {
ts.tv_sec++;
ts.tv_nsec -= 1000000000;
}
r = -__syscall(SYS_futex, &target_tid,
FUTEX_LOCK_PI|FUTEX_PRIVATE, 0, &ts);
/* Obtaining the lock means the thread responded. ESRCH
* means the target thread exited, which is okay too. */
if (!r || r == ESRCH) continue;
miss_cnt++;
}
if (!miss_cnt) break;
rewinddir(&dir);
}
close(dir.fd);
/* Serialize execution of callback in caught threads. */
for (cp=head; cp; cp=cp->next) {
sem_post(&cp->target_sem);
sem_wait(&cp->caller_sem);
}
sa.sa_handler = SIG_IGN;
__libc_sigaction(SIGSYNCCALL, &sa, 0);
single_threaded:
func(ctx);
/* Only release the caught threads once all threads, including the
* caller, have returned from the callback function. */
for (cp=head; cp; cp=next) {
next = cp->next;
sem_post(&cp->target_sem);
}
out:
a_store(&__block_new_threads, 0);
__wake(&__block_new_threads, -1, 1);
pthread_setcancelstate(cs, 0);
UNLOCK(synccall_lock);
__restore_sigs(&oldmask);
}