_Noreturn void quick_exit(int code)
{
static int lock;
while (a_swap(&lock, 1)) __syscall(SYS_pause);
__funcs_on_quick_exit();
_Exit(code);
}
int __pthread_once(pthread_once_t *control, void (*init)(void))
{
/* Return immediately if init finished before, but ensure that
* effects of the init routine are visible to the caller. */
if (*control == 2) {
a_barrier();
return 0;
}
/* Try to enter initializing state. Four possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return
* 3 - another thread is running init, waiters present; wait */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
if (a_swap(control, 2) == 3)
__wake(control, -1, 1);
return 0;
case 1:
/* If this fails, so will __wait. */
a_cas(control, 1, 3);
case 3:
__wait(control, 0, 3, 1);
continue;
case 2:
return 0;
}
}
int pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int robust = 0;
if (m->_m_type != PTHREAD_MUTEX_NORMAL) {
if (!m->_m_lock)
return EPERM;
self = __pthread_self();
if ((m->_m_lock&0x1fffffff) != self->tid)
return EPERM;
if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (m->_m_type >= 4) {
robust = 1;
self->robust_list.pending = &m->_m_next;
*(void **)m->_m_prev = m->_m_next;
if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev;
__vm_lock_impl(+1);
}
}
cont = a_swap(&m->_m_lock, 0);
if (robust) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, 0);
return 0;
}
static void undo(void *control)
{
/* Wake all waiters, since the waiter status is lost when
* resetting control to the initial state. */
if (a_swap(control, 0) == 3)
__wake(control, -1, 1);
}
int __pthread_once_full(pthread_once_t *control, void (*init)(void))
{
/* Try to enter initializing state. Four possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return
* 3 - another thread is running init, waiters present; wait */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
if (a_swap(control, 2) == 3)
__wake(control, -1, 1);
return 0;
case 1:
/* If this fails, so will __wait. */
a_cas(control, 1, 3);
case 3:
__wait(control, 0, 3, 1);
continue;
case 2:
return 0;
}
}
int pthread_once(pthread_once_t *control, void (*init)(void))
{
static int waiters;
/* Return immediately if init finished before */
if (*control == 2) return 0;
/* Try to enter initializing state. Three possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return */
for (;;) switch (a_swap(control, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
a_store(control, 2);
if (waiters) __wake(control, -1, 0);
return 0;
case 1:
__wait(control, &waiters, 1, 0);
continue;
case 2:
a_store(control, 2);
return 0;
}
}
int __pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int type = m->_m_type & 15;
int priv = (m->_m_type & 128) ^ 128;
if (type != PTHREAD_MUTEX_NORMAL) {
self = __pthread_self();
if ((m->_m_lock&0x7fffffff) != self->tid)
return EPERM;
if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (!priv) {
self->robust_list.pending = &m->_m_next;
__vm_lock_impl(+1);
}
volatile void *prev = m->_m_prev;
volatile void *next = m->_m_next;
*(volatile void *volatile *)prev = next;
if (next != &self->robust_list.head) *(volatile void *volatile *)
((char *)next - sizeof(void *)) = prev;
}
cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0);
if (type != PTHREAD_MUTEX_NORMAL && !priv) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, priv);
return 0;
}
int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
#ifdef __EMSCRIPTEN__
// XXX Emscripten: TODO: This is suboptimal but works naively correctly for now. The Emscripten-specific code path below
// has a bug and does not work for some reason. Figure it out and remove this code block.
__wake(&c->_c_seq, -1, 0);
return 0;
#endif
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
#ifdef __EMSCRIPTEN__
int futexResult;
do {
// XXX Emscripten: Bug, this does not work correctly.
futexResult = emscripten_futex_wake_or_requeue(&c->_c_seq, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
c->_c_seq, &m->_m_lock);
} while(futexResult == -EAGAIN);
#else
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
#endif
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
int pthread_mutex_trylock(pthread_mutex_t *m)
{
int tid;
int own;
pthread_t self;
if (m->_m_type == PTHREAD_MUTEX_NORMAL)
return a_swap(&m->_m_lock, EBUSY);
self = pthread_self();
tid = self->tid | 0x80000000;
if (m->_m_type >= 4) {
if (!self->robust_list.off)
__syscall(SYS_set_robust_list,
&self->robust_list, 3*sizeof(long));
self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next;
self->robust_list.pending = &m->_m_next;
}
if (m->_m_lock == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) {
if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN;
m->_m_count++;
return 0;
}
own = m->_m_lock;
if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, own, tid)!=own)
return EBUSY;
m->_m_count = 1;
if (m->_m_type < 4) return 0;
if (m->_m_type >= 8) {
m->_m_lock = 0;
return ENOTRECOVERABLE;
}
m->_m_next = self->robust_list.head;
m->_m_prev = &self->robust_list.head;
if (self->robust_list.head)
self->robust_list.head[-1] = &m->_m_next;
self->robust_list.head = &m->_m_next;
self->robust_list.pending = 0;
if (own) {
m->_m_type += 8;
return EOWNERDEAD;
}
return 0;
}
int pthread_barrier_wait(pthread_barrier_t *b)
{
int limit = b->_b_limit;
struct instance *inst;
/* Trivial case: count was set at 1 */
if (!limit) return PTHREAD_BARRIER_SERIAL_THREAD;
/* Process-shared barriers require a separate, inefficient wait */
if (limit < 0) return pshared_barrier_wait(b);
/* Otherwise we need a lock on the barrier object */
while (a_swap(&b->_b_lock, 1))
__wait(&b->_b_lock, &b->_b_waiters, 1, 1);
inst = b->_b_inst;
/* First thread to enter the barrier becomes the "instance owner" */
if (!inst) {
struct instance new_inst = { 0 };
int spins = 10000;
b->_b_inst = inst = &new_inst;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
while (spins-- && !inst->finished)
a_spin();
a_inc(&inst->finished);
while (inst->finished == 1)
__syscall(SYS_futex, &inst->finished, FUTEX_WAIT,1,0);
return PTHREAD_BARRIER_SERIAL_THREAD;
}
/* Last thread to enter the barrier wakes all non-instance-owners */
if (++inst->count == limit) {
b->_b_inst = 0;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
a_store(&inst->last, 1);
if (inst->waiters)
__wake(&inst->last, -1, 1);
} else {
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
__wait(&inst->last, &inst->waiters, 0, 1);
}
/* Last thread to exit the barrier wakes the instance owner */
if (a_fetch_add(&inst->count,-1)==1 && a_fetch_add(&inst->finished,1))
__wake(&inst->finished, 1, 1);
return 0;
}
static void signalWaiters(tlTask* task) {
assert(tlTaskIsDone(task));
while (true) {
tlHandle waiting = A_PTR(a_swap(A_VAR(task->waiting), 0));
if (!waiting) return;
if (tlTaskIs(waiting)) {
tlTaskCopyValue(tlTaskAs(waiting), task);
tlTaskReady(tlTaskAs(waiting));
return;
}
tlWaitQueue* queue = tlWaitQueueAs(waiting);
while (true) {
tlTask* waiter = tlWaitQueueGet(queue);
if (!waiter) return;
tlTaskCopyValue(waiter, task);
tlTaskReady(waiter);
}
}
}
_Noreturn void exit(int code)
{
static int lock;
/* If more than one thread calls exit, hang until _Exit ends it all */
while (a_swap(&lock, 1)) __syscall(SYS_pause);
__funcs_on_exit();
#ifndef SHARED
uintptr_t a = (uintptr_t)&__fini_array_end;
for (; a>(uintptr_t)&__fini_array_start; a-=sizeof(void(*)()))
(*(void (**)())(a-sizeof(void(*)())))();
_fini();
#endif
__flush_on_exit();
__seek_on_exit();
_Exit(code);
for(;;);
}
int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct timespec *ts)
{
struct cm cm = { .c=c, .m=m };
int r, e=0, seq;
if (m->_m_type && (m->_m_lock&INT_MAX) != pthread_self()->tid)
return EPERM;
if (ts && ts->tv_nsec >= 1000000000UL)
return EINVAL;
pthread_testcancel();
a_inc(&c->_c_waiters);
if (c->_c_mutex != (void *)-1) {
c->_c_mutex = m;
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
c->_c_waiters2++;
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
}
seq = c->_c_seq;
pthread_mutex_unlock(m);
do e = __timedwait(&c->_c_seq, seq, c->_c_clock, ts, cleanup, &cm, 0);
while (c->_c_seq == seq && (!e || e==EINTR));
if (e == EINTR) e = 0;
unwait(c, m);
if ((r=pthread_mutex_lock(m))) return r;
return e;
}
static void unwait(pthread_cond_t *c, pthread_mutex_t *m)
{
/* Removing a waiter is non-trivial if we could be using requeue
* based broadcast signals, due to mutex access issues, etc. */
if (c->_c_mutex == (void *)-1) {
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 0);
return;
}
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (c->_c_waiters2) c->_c_waiters2--;
else a_dec(&m->_m_waiters);
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 1);
}
int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
int pthread_spin_lock(pthread_spinlock_t *s)
{
while (a_swap(s, 1)) a_spin();
return 0;
}
int setlogmask(int maskpri)
{
if (maskpri) return a_swap(&log_mask, maskpri);
else return log_mask;
}
static inline void lock(volatile int *lk)
{
if (libc.threads_minus_1)
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
}
static void lock(volatile int *lk)
{
if (!libc.threads_minus_1) return;
while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1);
}
static inline void unlock(volatile int *l)
{
if (a_swap(l, 0)==2)
__wake(l, 1, 1);
}
