int pthread_once(pthread_once_t *control, void (*init)(void))
{
static int waiters;
/* Return immediately if init finished before */
if (*control == 2) return 0;
/* Try to enter initializing state. Three possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return */
for (;;) switch (a_swap(control, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
a_store(control, 2);
if (waiters) __wake(control, -1, 0);
return 0;
case 1:
__wait(control, &waiters, 1, 0);
continue;
case 2:
a_store(control, 2);
return 0;
}
}
int pthread_barrier_wait(pthread_barrier_t *b)
{
int limit = b->_b_limit;
struct instance *inst;
/* Trivial case: count was set at 1 */
if (!limit) return PTHREAD_BARRIER_SERIAL_THREAD;
/* Process-shared barriers require a separate, inefficient wait */
if (limit < 0) return pshared_barrier_wait(b);
/* Otherwise we need a lock on the barrier object */
while (a_swap(&b->_b_lock, 1))
__wait(&b->_b_lock, &b->_b_waiters, 1, 1);
inst = b->_b_inst;
/* First thread to enter the barrier becomes the "instance owner" */
if (!inst) {
struct instance new_inst = { 0 };
int spins = 10000;
b->_b_inst = inst = &new_inst;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
while (spins-- && !inst->finished)
a_spin();
a_inc(&inst->finished);
while (inst->finished == 1)
__syscall(SYS_futex, &inst->finished, FUTEX_WAIT,1,0);
return PTHREAD_BARRIER_SERIAL_THREAD;
}
/* Last thread to enter the barrier wakes all non-instance-owners */
if (++inst->count == limit) {
b->_b_inst = 0;
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
a_store(&inst->last, 1);
if (inst->waiters)
__wake(&inst->last, -1, 1);
} else {
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 1);
__wait(&inst->last, &inst->waiters, 0, 1);
}
/* Last thread to exit the barrier wakes the instance owner */
if (a_fetch_add(&inst->count,-1)==1 && a_fetch_add(&inst->finished,1))
__wake(&inst->finished, 1, 1);
return 0;
}
char *bindtextdomain(const char *domainname, const char *dirname)
{
static int lock[2];
struct binding *p, *q;
if (!domainname) return 0;
if (!dirname) return gettextdir(domainname, &(size_t){0});
size_t domlen = strlen(domainname);
size_t dirlen = strlen(dirname);
if (domlen > NAME_MAX || dirlen >= PATH_MAX) {
errno = EINVAL;
return 0;
}
LOCK(lock);
for (p=bindings; p; p=p->next) {
if (!strcmp(p->domainname, domainname) &&
!strcmp(p->dirname, dirname)) {
break;
}
}
if (!p) {
p = malloc(sizeof *p + domlen + dirlen + 2);
if (!p) {
UNLOCK(lock);
return 0;
}
p->next = bindings;
p->dirlen = dirlen;
p->domainname = p->buf;
p->dirname = p->buf + domlen + 1;
memcpy(p->domainname, domainname, domlen+1);
memcpy(p->dirname, dirname, dirlen+1);
a_cas_p(&bindings, bindings, p);
}
a_store(&p->active, 1);
for (q=bindings; q; q=q->next) {
if (!strcmp(p->domainname, domainname) && q != p)
a_store(&q->active, 0);
}
UNLOCK(lock);
return (char *)p->dirname;
}
static int pshared_barrier_wait(pthread_barrier_t *b)
{
int limit = (b->_b_limit & INT_MAX) + 1;
int ret = 0;
int v, w;
if (limit==1) return PTHREAD_BARRIER_SERIAL_THREAD;
while ((v=a_cas(&b->_b_lock, 0, limit)))
__wait(&b->_b_lock, &b->_b_waiters, v, 0);
/* Wait for <limit> threads to get to the barrier */
if (++b->_b_count == limit) {
a_store(&b->_b_count, 0);
ret = PTHREAD_BARRIER_SERIAL_THREAD;
if (b->_b_waiters2) __wake(&b->_b_count, -1, 0);
} else {
a_store(&b->_b_lock, 0);
if (b->_b_waiters) __wake(&b->_b_lock, 1, 0);
while ((v=b->_b_count)>0)
__wait(&b->_b_count, &b->_b_waiters2, v, 0);
}
__vm_lock_impl(+1);
/* Ensure all threads have a vm lock before proceeding */
if (a_fetch_add(&b->_b_count, -1)==1-limit) {
a_store(&b->_b_count, 0);
if (b->_b_waiters2) __wake(&b->_b_count, -1, 0);
} else {
while ((v=b->_b_count))
__wait(&b->_b_count, &b->_b_waiters2, v, 0);
}
/* Perform a recursive unlock suitable for self-sync'd destruction */
do {
v = b->_b_lock;
w = b->_b_waiters;
} while (a_cas(&b->_b_lock, v, v==INT_MIN+1 ? 0 : v-1) != v);
/* Wake a thread waiting to reuse or destroy the barrier */
if (v==INT_MIN+1 || (v==1 && w))
__wake(&b->_b_lock, 1, 0);
__vm_unlock_impl();
return ret;
}
int __pthread_once(pthread_once_t *control, void (*init)(void))
{
static int waiters;
/* Return immediately if init finished before, but ensure that
* effects of the init routine are visible to the caller. */
if (*control == 2) {
a_barrier();
return 0;
}
/* Try to enter initializing state. Three possibilities:
* 0 - we're the first or the other cancelled; run init
* 1 - another thread is running init; wait
* 2 - another thread finished running init; just return */
for (;;) switch (a_cas(control, 0, 1)) {
case 0:
pthread_cleanup_push(undo, control);
init();
pthread_cleanup_pop(0);
a_store(control, 2);
if (waiters) __wake(control, -1, 1);
return 0;
case 1:
__wait(control, &waiters, 1, 1);
continue;
case 2:
return 0;
}
}
static inline void unlock(volatile int *lk)
{
if (lk[0]) {
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
}
void jflib::pool<T, CV>::side::signal(bool close) {
if(a_load(state_) != NONE || close) {
cond_.lock();
a_store(state_, close ? CLOSED : NONE);
cond_.broadcast();
cond_.unlock();
}
}
int timer_delete(timer_t t) {
if ((intptr_t)t < 0) {
pthread_t td = (void*)((uintptr_t)t << 1);
a_store(&td->timer_id, td->timer_id | INT_MIN);
__wake(&td->timer_id, 1, 1);
return 0;
}
return __syscall(SYS_timer_delete, t);
}
int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
#ifdef __EMSCRIPTEN__
// XXX Emscripten: TODO: This is suboptimal but works naively correctly for now. The Emscripten-specific code path below
// has a bug and does not work for some reason. Figure it out and remove this code block.
__wake(&c->_c_seq, -1, 0);
return 0;
#endif
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
#ifdef __EMSCRIPTEN__
int futexResult;
do {
// XXX Emscripten: Bug, this does not work correctly.
futexResult = emscripten_futex_wake_or_requeue(&c->_c_seq, !m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
c->_c_seq, &m->_m_lock);
} while(futexResult == -EAGAIN);
#else
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
#endif
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
static inline void unlock_requeue(volatile int *l, volatile int *r, int w)
{
a_store(l, 0);
#ifdef __EMSCRIPTEN__
// Here the intent is to wake one waiter, and requeue all other waiters from waiting on address 'l'
// to wait on address 'r' instead. This is not possible at the moment with SharedArrayBuffer Atomics,
// as it does not have a "wake X waiters and requeue the rest" primitive. However this kind of
// primitive is strictly not needed, since it is more like an optimization to avoid spuriously waking
// all waiters, just to make them wait on another location immediately afterwards. Here we do exactly
// that: wake every waiter.
emscripten_futex_wake(l, 0x7FFFFFFF);
#else
if (w) __wake(l, 1, 1);
else __syscall(SYS_futex, l, FUTEX_REQUEUE|128, 0, 1, r) != -ENOSYS
|| __syscall(SYS_futex, l, FUTEX_REQUEUE, 0, 1, r);
#endif
}
static void init_cancellation()
{
struct sigaction sa = {
.sa_flags = SA_SIGINFO | SA_RESTART,
.sa_sigaction = cancel_handler
};
sigfillset(&sa.sa_mask);
__libc_sigaction(SIGCANCEL, &sa, 0);
}
int pthread_cancel(pthread_t t)
{
static int init;
if (!init) {
init_cancellation();
init = 1;
}
a_store(&t->cancel, 1);
return pthread_kill(t, SIGCANCEL);
}
uint32_t jflib::pool<T, CV>::side::get() {
bool last_attempt = false;
uint32_t res = fifo_.dequeue();
while(res == cbT::guard) {
cond_.lock();
switch(a_load(state_)) {
case CLOSED:
if(last_attempt) {
cond_.unlock();
return cbT::guard;
} else {
last_attempt = true;
break;
}
case NONE:
a_store(state_, WAITING);
break;
case WAITING:
break;
}
res = fifo_.dequeue();
if(res == cbT::guard) {
if(last_attempt) {
cond_.unlock();
break;
}
} else {
cond_.unlock();
break;
}
do {
cond_.timedwait(5);
} while(a_load(state_) == WAITING);
cond_.unlock();
}
return res;
}
int pthread_cond_timedwait(pthread_cond_t *c, pthread_mutex_t *m, const struct timespec *ts)
{
struct cm cm = { .c=c, .m=m };
int r, e=0, seq;
if (m->_m_type && (m->_m_lock&INT_MAX) != pthread_self()->tid)
return EPERM;
if (ts && ts->tv_nsec >= 1000000000UL)
return EINVAL;
pthread_testcancel();
a_inc(&c->_c_waiters);
if (c->_c_mutex != (void *)-1) {
c->_c_mutex = m;
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
c->_c_waiters2++;
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
}
seq = c->_c_seq;
pthread_mutex_unlock(m);
do e = __timedwait(&c->_c_seq, seq, c->_c_clock, ts, cleanup, &cm, 0);
while (c->_c_seq == seq && (!e || e==EINTR));
if (e == EINTR) e = 0;
unwait(c, m);
if ((r=pthread_mutex_lock(m))) return r;
return e;
}
static void unwait(pthread_cond_t *c, pthread_mutex_t *m)
{
/* Removing a waiter is non-trivial if we could be using requeue
* based broadcast signals, due to mutex access issues, etc. */
if (c->_c_mutex == (void *)-1) {
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 0);
return;
}
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (c->_c_waiters2) c->_c_waiters2--;
else a_dec(&m->_m_waiters);
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 1);
a_dec(&c->_c_waiters);
if (c->_c_destroy) __wake(&c->_c_waiters, 1, 1);
}
int pthread_cond_broadcast(pthread_cond_t *c)
{
pthread_mutex_t *m;
if (!c->_c_waiters) return 0;
a_inc(&c->_c_seq);
/* If cond var is process-shared, simply wake all waiters. */
if (c->_c_mutex == (void *)-1) {
__wake(&c->_c_seq, -1, 0);
return 0;
}
/* Block waiters from returning so we can use the mutex. */
while (a_swap(&c->_c_lock, 1))
__wait(&c->_c_lock, &c->_c_lockwait, 1, 1);
if (!c->_c_waiters)
goto out;
m = c->_c_mutex;
/* Move waiter count to the mutex */
a_fetch_add(&m->_m_waiters, c->_c_waiters2);
c->_c_waiters2 = 0;
/* Perform the futex requeue, waking one waiter unless we know
* that the calling thread holds the mutex. */
__syscall(SYS_futex, &c->_c_seq, FUTEX_REQUEUE,
!m->_m_type || (m->_m_lock&INT_MAX)!=pthread_self()->tid,
INT_MAX, &m->_m_lock);
out:
a_store(&c->_c_lock, 0);
if (c->_c_lockwait) __wake(&c->_c_lock, 1, 0);
return 0;
}
void __do_orphaned_stdio_locks()
{
FILE *f;
for (f=__pthread_self()->stdio_locks; f; f=f->next_locked)
a_store(&f->lock, 0x40000000);
}
static void unlock(volatile int *lk)
{
if (!libc.threads_minus_1) return;
a_store(lk, 0);
if (lk[1]) __wake(lk, 1, 1);
}
void __unlockfile(FILE *f)
{
a_store(&f->lock, 0);
if (f->waiters) __wake(&f->lock, 1, 1);
}
int pthread_spin_unlock(pthread_spinlock_t *s)
{
a_store(s, 0);
return 0;
}
static void undo(void *control)
{
a_store(control, 0);
__wake(control, 1, 1);
}
void __synccall(void (*func)(void *), void *ctx)
{
sigset_t oldmask;
int cs, i, r, pid, self;;
DIR dir = {0};
struct dirent *de;
struct sigaction sa = { .sa_flags = 0, .sa_handler = handler };
struct chain *cp, *next;
struct timespec ts;
/* Blocking signals in two steps, first only app-level signals
* before taking the lock, then all signals after taking the lock,
* is necessary to achieve AS-safety. Blocking them all first would
* deadlock if multiple threads called __synccall. Waiting to block
* any until after the lock would allow re-entry in the same thread
* with the lock already held. */
__block_app_sigs(&oldmask);
LOCK(synccall_lock);
__block_all_sigs(0);
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs);
head = 0;
if (!libc.threaded) goto single_threaded;
callback = func;
context = ctx;
/* This atomic store ensures that any signaled threads will see the
* above stores, and prevents more than a bounded number of threads,
* those already in pthread_create, from creating new threads until
* the value is cleared to zero again. */
a_store(&__block_new_threads, 1);
/* Block even implementation-internal signals, so that nothing
* interrupts the SIGSYNCCALL handlers. The main possible source
* of trouble is asynchronous cancellation. */
memset(&sa.sa_mask, -1, sizeof sa.sa_mask);
__libc_sigaction(SIGSYNCCALL, &sa, 0);
pid = __syscall(SYS_getpid);
self = __syscall(SYS_gettid);
/* Since opendir is not AS-safe, the DIR needs to be setup manually
* in automatic storage. Thankfully this is easy. */
dir.fd = open("/proc/self/task", O_RDONLY|O_DIRECTORY|O_CLOEXEC);
if (dir.fd < 0) goto out;
/* Initially send one signal per counted thread. But since we can't
* synchronize with thread creation/exit here, there could be too
* few signals. This initial signaling is just an optimization, not
* part of the logic. */
for (i=libc.threads_minus_1; i; i--)
__syscall(SYS_kill, pid, SIGSYNCCALL);
/* Loop scanning the kernel-provided thread list until it shows no
* threads that have not already replied to the signal. */
for (;;) {
int miss_cnt = 0;
while ((de = readdir(&dir))) {
if (!isdigit(de->d_name[0])) continue;
int tid = atoi(de->d_name);
if (tid == self || !tid) continue;
/* Set the target thread as the PI futex owner before
* checking if it's in the list of caught threads. If it
* adds itself to the list after we check for it, then
* it will see its own tid in the PI futex and perform
* the unlock operation. */
a_store(&target_tid, tid);
/* Thread-already-caught is a success condition. */
for (cp = head; cp && cp->tid != tid; cp=cp->next);
if (cp) continue;
r = -__syscall(SYS_tgkill, pid, tid, SIGSYNCCALL);
/* Target thread exit is a success condition. */
if (r == ESRCH) continue;
/* The FUTEX_LOCK_PI operation is used to loan priority
* to the target thread, which otherwise may be unable
* to run. Timeout is necessary because there is a race
* condition where the tid may be reused by a different
* process. */
clock_gettime(CLOCK_REALTIME, &ts);
ts.tv_nsec += 10000000;
if (ts.tv_nsec >= 1000000000) {
ts.tv_sec++;
ts.tv_nsec -= 1000000000;
}
r = -__syscall(SYS_futex, &target_tid,
FUTEX_LOCK_PI|FUTEX_PRIVATE, 0, &ts);
/* Obtaining the lock means the thread responded. ESRCH
* means the target thread exited, which is okay too. */
if (!r || r == ESRCH) continue;
miss_cnt++;
}
if (!miss_cnt) break;
rewinddir(&dir);
}
close(dir.fd);
/* Serialize execution of callback in caught threads. */
for (cp=head; cp; cp=cp->next) {
sem_post(&cp->target_sem);
sem_wait(&cp->caller_sem);
}
sa.sa_handler = SIG_IGN;
__libc_sigaction(SIGSYNCCALL, &sa, 0);
single_threaded:
func(ctx);
/* Only release the caught threads once all threads, including the
* caller, have returned from the callback function. */
for (cp=head; cp; cp=next) {
next = cp->next;
sem_post(&cp->target_sem);
}
out:
a_store(&__block_new_threads, 0);
__wake(&__block_new_threads, -1, 1);
pthread_setcancelstate(cs, 0);
UNLOCK(synccall_lock);
__restore_sigs(&oldmask);
}
char *dcngettext(const char *domainname, const char *msgid1, const char *msgid2, unsigned long int n, int category)
{
static struct msgcat *volatile cats;
struct msgcat *p;
struct __locale_struct *loc = CURRENT_LOCALE;
struct __locale_map *lm;
const char *dirname, *locname, *catname;
size_t dirlen, loclen, catlen, domlen;
if (!domainname) domainname = __gettextdomain();
domlen = strlen(domainname);
if (domlen > NAME_MAX) goto notrans;
dirname = gettextdir(domainname, &dirlen);
if (!dirname) goto notrans;
switch (category) {
case LC_MESSAGES:
locname = loc->messages_name;
if (!*locname) goto notrans;
break;
case LC_TIME:
case LC_MONETARY:
case LC_COLLATE:
lm = loc->cat[category-2];
if (!lm) goto notrans;
locname = lm->name;
break;
default:
notrans:
return (char *) ((n == 1) ? msgid1 : msgid2);
}
catname = catnames[category-2];
catlen = catlens[category-2];
loclen = strlen(locname);
size_t namelen = dirlen+1 + loclen+1 + catlen+1 + domlen+3;
char name[namelen+1], *s = name;
memcpy(s, dirname, dirlen);
s[dirlen] = '/';
s += dirlen + 1;
memcpy(s, locname, loclen);
s[loclen] = '/';
s += loclen + 1;
memcpy(s, catname, catlen);
s[catlen] = '/';
s += catlen + 1;
memcpy(s, domainname, domlen);
s[domlen] = '.';
s[domlen+1] = 'm';
s[domlen+2] = 'o';
s[domlen+3] = 0;
for (p=cats; p; p=p->next)
if (!strcmp(p->name, name))
break;
if (!p) {
void *old_cats;
size_t map_size;
const void *map = __map_file(name, &map_size);
if (!map) goto notrans;
p = malloc(sizeof *p + namelen + 1);
if (!p) {
__munmap((void *)map, map_size);
goto notrans;
}
p->map = map;
p->map_size = map_size;
memcpy(p->name, name, namelen+1);
do {
old_cats = cats;
p->next = old_cats;
} while (a_cas_p(&cats, old_cats, p) != old_cats);
}
const char *trans = __mo_lookup(p->map, p->map_size, msgid1);
if (!trans) goto notrans;
/* Non-plural-processing gettext forms pass a null pointer as
* msgid2 to request that dcngettext suppress plural processing. */
if (!msgid2) return (char *)trans;
if (!p->plural_rule) {
const char *rule = "n!=1;";
unsigned long np = 2;
const char *r = __mo_lookup(p->map, p->map_size, "");
char *z;
while (r && strncmp(r, "Plural-Forms:", 13)) {
z = strchr(r, '\n');
r = z ? z+1 : 0;
}
if (r) {
r += 13;
while (isspace(*r)) r++;
if (!strncmp(r, "nplurals=", 9)) {
np = strtoul(r+9, &z, 10);
r = z;
}
while (*r && *r != ';') r++;
if (*r) {
r++;
while (isspace(*r)) r++;
if (!strncmp(r, "plural=", 7))
rule = r+7;
}
}
a_store(&p->nplurals, np);
a_cas_p(&p->plural_rule, 0, (void *)rule);
}
if (p->nplurals) {
unsigned long plural = __pleval(p->plural_rule, n);
if (plural > p->nplurals) goto notrans;
while (plural--) {
size_t rem = p->map_size - (trans - (char *)p->map);
size_t l = strnlen(trans, rem);
if (l+1 >= rem)
goto notrans;
trans += l+1;
}
}
return (char *)trans;
}
