_Noreturn void quick_exit(int code) { static int lock; while (a_swap(&lock, 1)) __syscall(SYS_pause); __funcs_on_quick_exit(); _Exit(code); }
int __pthread_once(pthread_once_t *control, void (*init)(void)) { /* Return immediately if init finished before, but ensure that * effects of the init routine are visible to the caller. */ if (*control == 2) { a_barrier(); return 0; } /* Try to enter initializing state. Four possibilities: * 0 - we're the first or the other cancelled; run init * 1 - another thread is running init; wait * 2 - another thread finished running init; just return * 3 - another thread is running init, waiters present; wait */ for (;;) switch (a_cas(control, 0, 1)) { case 0: pthread_cleanup_push(undo, control); init(); pthread_cleanup_pop(0); if (a_swap(control, 2) == 3) __wake(control, -1, 1); return 0; case 1: /* If this fails, so will __wait. */ a_cas(control, 1, 3); case 3: __wait(control, 0, 3, 1); continue; case 2: return 0; } }
int pthread_mutex_unlock(pthread_mutex_t *m) { pthread_t self; int waiters = m->_m_waiters; int cont; int robust = 0; if (m->_m_type != PTHREAD_MUTEX_NORMAL) { if (!m->_m_lock) return EPERM; self = __pthread_self(); if ((m->_m_lock&0x1fffffff) != self->tid) return EPERM; if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count) return m->_m_count--, 0; if (m->_m_type >= 4) { robust = 1; self->robust_list.pending = &m->_m_next; *(void **)m->_m_prev = m->_m_next; if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev; __vm_lock_impl(+1); } } cont = a_swap(&m->_m_lock, 0); if (robust) { self->robust_list.pending = 0; __vm_unlock_impl(); } if (waiters || cont<0) __wake(&m->_m_lock, 1, 0); return 0; }
static void undo(void *control) { /* Wake all waiters, since the waiter status is lost when * resetting control to the initial state. */ if (a_swap(control, 0) == 3) __wake(control, -1, 1); }
int __pthread_once_full(pthread_once_t *control, void (*init)(void)) { /* Try to enter initializing state. Four possibilities: * 0 - we're the first or the other cancelled; run init * 1 - another thread is running init; wait * 2 - another thread finished running init; just return * 3 - another thread is running init, waiters present; wait */ for (;;) switch (a_cas(control, 0, 1)) { case 0: pthread_cleanup_push(undo, control); init(); pthread_cleanup_pop(0); if (a_swap(control, 2) == 3) __wake(control, -1, 1); return 0; case 1: /* If this fails, so will __wait. */ a_cas(control, 1, 3); case 3: __wait(control, 0, 3, 1); continue; case 2: return 0; } }
int pthread_once(pthread_once_t *control, void (*init)(void)) { static int waiters; /* Return immediately if init finished before */ if (*control == 2) return 0; /* Try to enter initializing state. Three possibilities: * 0 - we're the first or the other cancelled; run init * 1 - another thread is running init; wait * 2 - another thread finished running init; just return */ for (;;) switch (a_swap(control, 1)) { case 0: pthread_cleanup_push(undo, control); init(); pthread_cleanup_pop(0); a_store(control, 2); if (waiters) __wake(control, -1, 0); return 0; case 1: __wait(control, &waiters, 1, 0); continue; case 2: a_store(control, 2); return 0; } }
int __pthread_mutex_unlock(pthread_mutex_t *m) { pthread_t self; int waiters = m->_m_waiters; int cont; int type = m->_m_type & 15; int priv = (m->_m_type & 128) ^ 128; if (type != PTHREAD_MUTEX_NORMAL) { self = __pthread_self(); if ((m->_m_lock&0x7fffffff) != self->tid) return EPERM; if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count) return m->_m_count--, 0; if (!priv) { self->robust_list.pending = &m->_m_next; __vm_lock_impl(+1); } volatile void *prev = m->_m_prev; volatile void *next = m->_m_next; *(volatile void *volatile *)prev = next; if (next != &self->robust_list.head) *(volatile void *volatile *) ((char *)next - sizeof(void *)) = prev; } cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0); if (type != PTHREAD_MUTEX_NORMAL && !priv) { self->robust_list.pending = 0; __vm_unlock_impl(); } if (waiters || cont<0) __wake(&m->_m_lock, 1, priv); return 0; }
int pthread_cond_broadcast(pthread_cond_t *c) { pthread_mutex_t *m; if (!c->_c_waiters) return 0; a_inc(&c->_c_seq); #ifdef __EMSCRIPTEN__ // XXX Emscripten: TODO: This is suboptimal but works naively correctly for now. The Emscripten-specific code path below // has a bug and does not work for some reason. Figure it out and remove this code block. __wake(&c->_c_seq, -1, 0); return 0; #endif /* If cond var is process-shared, simply wake all waiters. */ if (c->_c_mutex == (void *)-1) { __wake(&c->_c_seq, -1, 0); return 0; } /* Block waiters from returning so we can use the mutex. */ while (a_swap(&c->_c_lock, 1)) __wait(&c->_c_lock, &c->_c_lockwait, 1, 1); if (!c->_c_waiters) goto out; m = c->_c_mutex; /* Move waiter count to the mutex */ a_fetch_add(&m->_m_waiters, c->_c_waiters2); c->_c_waiters2 = 0; #ifdef __EMSCRIPTEN__ int futexResult; do { // XXX Emscripten: Bug, this does not work correctly. futexResult = emscripten_futex_wake_or_requeue(&c->_c_seq, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid, c->_c_seq, &m->_m_lock); } while(futexResult == -EAGAIN); #else /* Perform the futex requeue, waking one waiter unless we know * that the calling thread holds the mutex. */ __syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid, INT_MAX, &m->_m_lock); #endif out: a_store(&c->_c_lock, 0); if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0); return 0; }
int pthread_mutex_trylock(pthread_mutex_t *m) { int tid; int own; pthread_t self; if (m->_m_type == PTHREAD_MUTEX_NORMAL) return a_swap(&m->_m_lock, EBUSY); self = pthread_self(); tid = self->tid | 0x80000000; if (m->_m_type >= 4) { if (!self->robust_list.off) __syscall(SYS_set_robust_list, &self->robust_list, 3*sizeof(long)); self->robust_list.off = (char*)&m->_m_lock-(char *)&m->_m_next; self->robust_list.pending = &m->_m_next; } if (m->_m_lock == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) { if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN; m->_m_count++; return 0; } own = m->_m_lock; if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, own, tid)!=own) return EBUSY; m->_m_count = 1; if (m->_m_type < 4) return 0; if (m->_m_type >= 8) { m->_m_lock = 0; return ENOTRECOVERABLE; } m->_m_next = self->robust_list.head; m->_m_prev = &self->robust_list.head; if (self->robust_list.head) self->robust_list.head[-1] = &m->_m_next; self->robust_list.head = &m->_m_next; self->robust_list.pending = 0; if (own) { m->_m_type += 8; return EOWNERDEAD; } return 0; }
int pthread_barrier_wait(pthread_barrier_t *b) { int limit = b->_b_limit; struct instance *inst; /* Trivial case: count was set at 1 */ if (!limit) return PTHREAD_BARRIER_SERIAL_THREAD; /* Process-shared barriers require a separate, inefficient wait */ if (limit < 0) return pshared_barrier_wait(b); /* Otherwise we need a lock on the barrier object */ while (a_swap(&b->_b_lock, 1)) __wait(&b->_b_lock, &b->_b_waiters, 1, 1); inst = b->_b_inst; /* First thread to enter the barrier becomes the "instance owner" */ if (!inst) { struct instance new_inst = { 0 }; int spins = 10000; b->_b_inst = inst = &new_inst; a_store(&b->_b_lock, 0); if (b->_b_waiters) __wake(&b->_b_lock, 1, 1); while (spins-- && !inst->finished) a_spin(); a_inc(&inst->finished); while (inst->finished == 1) __syscall(SYS_futex, &inst->finished, FUTEX_WAIT,1,0); return PTHREAD_BARRIER_SERIAL_THREAD; } /* Last thread to enter the barrier wakes all non-instance-owners */ if (++inst->count == limit) { b->_b_inst = 0; a_store(&b->_b_lock, 0); if (b->_b_waiters) __wake(&b->_b_lock, 1, 1); a_store(&inst->last, 1); if (inst->waiters) __wake(&inst->last, -1, 1); } else { a_store(&b->_b_lock, 0); if (b->_b_waiters) __wake(&b->_b_lock, 1, 1); __wait(&inst->last, &inst->waiters, 0, 1); } /* Last thread to exit the barrier wakes the instance owner */ if (a_fetch_add(&inst->count,-1)==1 && a_fetch_add(&inst->finished,1)) __wake(&inst->finished, 1, 1); return 0; }
static void signalWaiters(tlTask* task) { assert(tlTaskIsDone(task)); while (true) { tlHandle waiting = A_PTR(a_swap(A_VAR(task->waiting), 0)); if (!waiting) return; if (tlTaskIs(waiting)) { tlTaskCopyValue(tlTaskAs(waiting), task); tlTaskReady(tlTaskAs(waiting)); return; } tlWaitQueue* queue = tlWaitQueueAs(waiting); while (true) { tlTask* waiter = tlWaitQueueGet(queue); if (!waiter) return; tlTaskCopyValue(waiter, task); tlTaskReady(waiter); } } }
_Noreturn void exit(int code) { static int lock; /* If more than one thread calls exit, hang until _Exit ends it all */ while (a_swap(&lock, 1)) __syscall(SYS_pause); __funcs_on_exit(); #ifndef SHARED uintptr_t a = (uintptr_t)&__fini_array_end; for (; a>(uintptr_t)&__fini_array_start; a-=sizeof(void(*)())) (*(void (**)())(a-sizeof(void(*)())))(); _fini(); #endif __flush_on_exit(); __seek_on_exit(); _Exit(code); for(;;); }
int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct timespec *ts) { struct cm cm = { .c=c, .m=m }; int r, e=0, seq; if (m->_m_type && (m->_m_lock&INT_MAX) != pthread_self()->tid) return EPERM; if (ts && ts->tv_nsec >= 1000000000UL) return EINVAL; pthread_testcancel(); a_inc(&c->_c_waiters); if (c->_c_mutex != (void *)-1) { c->_c_mutex = m; while (a_swap(&c->_c_lock, 1)) __wait(&c->_c_lock, &c->_c_lockwait, 1, 1); c->_c_waiters2++; a_store(&c->_c_lock, 0); if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1); } seq = c->_c_seq; pthread_mutex_unlock(m); do e = __timedwait(&c->_c_seq, seq, c->_c_clock, ts, cleanup, &cm, 0); while (c->_c_seq == seq && (!e || e==EINTR)); if (e == EINTR) e = 0; unwait(c, m); if ((r=pthread_mutex_lock(m))) return r; return e; }
static void unwait(pthread_cond_t *c, pthread_mutex_t *m) { /* Removing a waiter is non-trivial if we could be using requeue * based broadcast signals, due to mutex access issues, etc. */ if (c->_c_mutex == (void *)-1) { a_dec(&c->_c_waiters); if (c->_c_destroy) __wake(&c->_c_waiters, 1, 0); return; } while (a_swap(&c->_c_lock, 1)) __wait(&c->_c_lock, &c->_c_lockwait, 1, 1); if (c->_c_waiters2) c->_c_waiters2--; else a_dec(&m->_m_waiters); a_store(&c->_c_lock, 0); if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1); a_dec(&c->_c_waiters); if (c->_c_destroy) __wake(&c->_c_waiters, 1, 1); }
int pthread_cond_broadcast(pthread_cond_t *c) { pthread_mutex_t *m; if (!c->_c_waiters) return 0; a_inc(&c->_c_seq); /* If cond var is process-shared, simply wake all waiters. */ if (c->_c_mutex == (void *)-1) { __wake(&c->_c_seq, -1, 0); return 0; } /* Block waiters from returning so we can use the mutex. */ while (a_swap(&c->_c_lock, 1)) __wait(&c->_c_lock, &c->_c_lockwait, 1, 1); if (!c->_c_waiters) goto out; m = c->_c_mutex; /* Move waiter count to the mutex */ a_fetch_add(&m->_m_waiters, c->_c_waiters2); c->_c_waiters2 = 0; /* Perform the futex requeue, waking one waiter unless we know * that the calling thread holds the mutex. */ __syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid, INT_MAX, &m->_m_lock); out: a_store(&c->_c_lock, 0); if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0); return 0; }
int pthread_spin_lock(pthread_spinlock_t *s) { while (a_swap(s, 1)) a_spin(); return 0; }
int setlogmask(int maskpri) { if (maskpri) return a_swap(&log_mask, maskpri); else return log_mask; }
static inline void lock(volatile int *lk) { if (libc.threads_minus_1) while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); }
static void lock(volatile int *lk) { if (!libc.threads_minus_1) return; while(a_swap(lk, 1)) __wait(lk, lk+1, 1, 1); }
static inline void unlock(volatile int *l) { if (a_swap(l, 0)==2) __wake(l, 1, 1); }