int __pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int type = m->_m_type & 15;
int priv = (m->_m_type & 128) ^ 128;
if (type != PTHREAD_MUTEX_NORMAL) {
self = __pthread_self();
if ((m->_m_lock&0x7fffffff) != self->tid)
return EPERM;
if ((type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (!priv) {
self->robust_list.pending = &m->_m_next;
__vm_lock_impl(+1);
}
volatile void *prev = m->_m_prev;
volatile void *next = m->_m_next;
*(volatile void *volatile *)prev = next;
if (next != &self->robust_list.head) *(volatile void *volatile *)
((char *)next - sizeof(void *)) = prev;
}
cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0);
if (type != PTHREAD_MUTEX_NORMAL && !priv) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, priv);
return 0;
}
int pthread_mutex_unlock(pthread_mutex_t *m)
{
pthread_t self;
int waiters = m->_m_waiters;
int cont;
int robust = 0;
if (m->_m_type != PTHREAD_MUTEX_NORMAL) {
if (!m->_m_lock)
return EPERM;
self = __pthread_self();
if ((m->_m_lock&0x1fffffff) != self->tid)
return EPERM;
if ((m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE && m->_m_count)
return m->_m_count--, 0;
if (m->_m_type >= 4) {
robust = 1;
self->robust_list.pending = &m->_m_next;
*(void **)m->_m_prev = m->_m_next;
if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev;
__vm_lock_impl(+1);
}
}
cont = a_swap(&m->_m_lock, 0);
if (robust) {
self->robust_list.pending = 0;
__vm_unlock_impl();
}
if (waiters || cont<0)
__wake(&m->_m_lock, 1, 0);
return 0;
}
void __testcancel()
{
if (!libc.has_thread_pointer) return;
pthread_t self = __pthread_self();
if (self->cancel && !self->canceldisable)
__cancel();
}
void __cancel()
{
pthread_t self = __pthread_self();
self->canceldisable = 1;
self->cancelasync = 0;
pthread_exit(PTHREAD_CANCELED);
}
void __unlist_locked_file(FILE *f)
{
if (f->lockcount) {
if (f->next_locked) f->next_locked->prev_locked = f->prev_locked;
if (f->prev_locked) f->prev_locked->next_locked = f->next_locked;
else __pthread_self()->stdio_locks = f->next_locked;
}
}
void __init_ssp(void *entropy)
{
if (entropy) memcpy(&__stack_chk_guard, entropy, sizeof(uintptr_t));
else __stack_chk_guard = (uintptr_t)&__stack_chk_guard * 1103515245;
if (libc.has_thread_pointer)
__pthread_self()->canary = __stack_chk_guard;
}
int __lockfile(FILE *f)
{
int owner, tid = __pthread_self()->tid;
if (f->lock == tid)
return 0;
while ((owner = a_cas(&f->lock, 0, tid)))
__wait(&f->lock, &f->waiters, owner, 1);
return 1;
}
int pthread_setspecific(pthread_key_t k, const void* x) {
struct pthread* self = __pthread_self();
/* Avoid unnecessary COW */
if (self->tsd[k] != x) {
self->tsd[k] = (void*)x;
self->tsd_used = 1;
}
return 0;
}
void *__tls_get_addr(size_t *v)
{
pthread_t self = __pthread_self();
#ifdef SHARED
if (v[0]<=(size_t)self->dtv[0])
return (char *)self->dtv[v[0]]+v[1];
return __tls_get_new(v);
#else
return (char *)self->dtv[1]+v[1];
#endif
}
pthread_t __pthread_self_def()
{
static int init=0, failed=0;
if (!init) {
if (failed) return 0;
if (init_main_thread() < 0) failed = 1;
if (failed) return 0;
init = 1;
}
return __pthread_self();
}
void __reset_tls()
{
pthread_t self = __pthread_self();
struct tls_module *p;
size_t i, n = (size_t)self->dtv[0];
if (n) for (p=libc.tls_head, i=1; i<=n; i++, p=p->next) {
if (!self->dtv[i]) continue;
memcpy(self->dtv[i], p->image, p->len);
memset((char *)self->dtv[i]+p->len, 0,
p->size - p->len);
}
}
pid_t fork(void)
{
pid_t ret;
if (libc.fork_handler) libc.fork_handler(-1);
ret = syscall(SYS_fork);
if (libc.lock && !ret) {
pthread_t self = __pthread_self();
self->tid = self->pid = syscall(SYS_getpid);
libc.threads_minus_1 = 0;
}
if (libc.fork_handler) libc.fork_handler(!ret);
return ret;
}
int __pthread_mutex_trylock_owner(pthread_mutex_t* m) {
int old, own;
int type = m->_m_type & 15;
pthread_t self = __pthread_self();
int tid = self->tid;
old = m->_m_lock;
own = old & 0x7fffffff;
if (own == tid && (type & 3) == PTHREAD_MUTEX_RECURSIVE) {
if ((unsigned)m->_m_count >= INT_MAX)
return EAGAIN;
m->_m_count++;
return 0;
}
if (own == 0x40000000)
return ENOTRECOVERABLE;
if (m->_m_type & 128) {
if (!self->robust_list.off) {
self->robust_list.off = (char*)&m->_m_lock - (char*)&m->_m_next;
__syscall(SYS_set_robust_list, &self->robust_list, 3 * sizeof(long));
}
if (m->_m_waiters)
tid |= 0x80000000;
self->robust_list.pending = &m->_m_next;
}
if ((own && (!(own & 0x40000000) || !(type & 4))) ||
a_cas(&m->_m_lock, old, tid) != old) {
self->robust_list.pending = 0;
return EBUSY;
}
volatile void* next = self->robust_list.head;
m->_m_next = next;
m->_m_prev = &self->robust_list.head;
if (next != &self->robust_list.head)
*(volatile void* volatile*)((char*)next - sizeof(void*)) = &m->_m_next;
self->robust_list.head = &m->_m_next;
self->robust_list.pending = 0;
if (own) {
m->_m_count = 0;
m->_m_type |= 8;
return EOWNERDEAD;
}
return 0;
}
long (__syscall_cp)(syscall_arg_t nr,
syscall_arg_t u, syscall_arg_t v, syscall_arg_t w,
syscall_arg_t x, syscall_arg_t y, syscall_arg_t z)
{
pthread_t self;
long r;
if (!libc.has_thread_pointer || (self = __pthread_self())->canceldisable)
return __syscall(nr, u, v, w, x, y, z);
r = __syscall_cp_asm(&self->cancel, nr, u, v, w, x, y, z);
if (r==-EINTR && nr!=SYS_close && self->cancel && !self->canceldisable)
__cancel();
return r;
}
void fiber_io_check(void)
{
if (__thread_fiber != NULL) {
if (__thread_fiber->ev_fiber == NULL) {
__thread_fiber->ev_fiber = acl_fiber_create(
fiber_io_loop, __thread_fiber->event,
STACK_SIZE);
__thread_fiber->io_count = 0;
__thread_fiber->nsleeping = 0;
__thread_fiber->io_stop = 0;
ring_init(&__thread_fiber->ev_timer);
}
return;
}
if (pthread_once(&__once_control, thread_init) != 0) {
msg_fatal("%s(%d), %s: pthread_once error %s",
__FILE__, __LINE__, __FUNCTION__, last_serror());
}
var_maxfd = open_limit(0);
if (var_maxfd <= 0) {
var_maxfd = MAXFD;
}
__thread_fiber = (FIBER_TLS *) malloc(sizeof(FIBER_TLS));
__thread_fiber->event = event_create(var_maxfd);
__thread_fiber->ev_fiber = acl_fiber_create(fiber_io_loop,
__thread_fiber->event, STACK_SIZE);
__thread_fiber->io_count = 0;
__thread_fiber->nsleeping = 0;
__thread_fiber->io_stop = 0;
ring_init(&__thread_fiber->ev_timer);
#ifdef SYS_WIN
__thread_fiber->events = htable_create(var_maxfd);
#else
__thread_fiber->events = (FILE_EVENT **)
calloc(var_maxfd, sizeof(FILE_EVENT*));
#endif
if (__pthread_self() == main_thread_self()) {
__main_fiber = __thread_fiber;
atexit(fiber_io_main_free);
} else if (pthread_setspecific(__fiber_key, __thread_fiber) != 0) {
msg_fatal("pthread_setspecific error!");
}
}
void __pthread_tsd_run_dtors()
{
pthread_t self = __pthread_self();
int i, j, not_finished = self->tsd_used;
for (j=0; not_finished && j<PTHREAD_DESTRUCTOR_ITERATIONS; j++) {
not_finished = 0;
for (i=0; i<PTHREAD_KEYS_MAX; i++) {
if (self->tsd[i] && keys[i]) {
void *tmp = self->tsd[i];
self->tsd[i] = 0;
keys[i](tmp);
not_finished = 1;
}
}
}
}
int ftrylockfile(FILE *f)
{
pthread_t self = __pthread_self();
int tid = self->tid;
int owner = f->lock;
if ((owner & ~MAYBE_WAITERS) == tid) {
if (f->lockcount == LONG_MAX)
return -1;
f->lockcount++;
return 0;
}
if (owner < 0) f->lock = owner = 0;
if (owner || a_cas(&f->lock, 0, tid))
return -1;
__register_locked_file(f, self);
return 0;
}
int __pthread_key_create(pthread_key_t *k, void (*dtor)(void *))
{
unsigned i = (uintptr_t)&k / 16 % PTHREAD_KEYS_MAX;
unsigned j = i;
pthread_t self = __pthread_self();
/* This can only happen in the main thread before
* pthread_create has been called. */
if (!self->tsd) self->tsd = __pthread_tsd_main;
if (!dtor) dtor = nodtor;
do {
if (!a_cas_p(keys+j, 0, (void *)dtor)) {
*k = j;
return 0;
}
} while ((j=(j+1)%PTHREAD_KEYS_MAX) != i);
return EAGAIN;
}
static void cancel_handler(int sig, siginfo_t *si, void *ctx)
{
pthread_t self = __pthread_self();
ucontext_t *uc = ctx;
const char *ip = ((char **)&uc->uc_mcontext)[CANCEL_REG_IP];
extern const char __cp_begin[1], __cp_end[1];
if (!self->cancel || self->canceldisable) return;
_sigaddset(&uc->uc_sigmask, SIGCANCEL);
if (self->cancelasync || ip >= __cp_begin && ip < __cp_end) {
self->canceldisable = 1;
pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0);
__cancel();
}
__syscall(SYS_tgkill, self->pid, self->tid, SIGCANCEL);
}
static void cancel_handler(int sig, siginfo_t *si, void *ctx)
{
pthread_t self = __pthread_self();
ucontext_t *uc = ctx;
uintptr_t sp = ((uintptr_t *)&uc->uc_mcontext)[CANCEL_REG_SP];
uintptr_t ip = ((uintptr_t *)&uc->uc_mcontext)[CANCEL_REG_IP];
if (!self->cancel || self->canceldisable) return;
_sigaddset(&uc->uc_sigmask, SIGCANCEL);
if (self->cancelasync || sp == self->cp_sp && ip <= self->cp_ip) {
self->canceldisable = 1;
pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0);
__cancel();
}
if (self->cp_sp)
__syscall(SYS_tgkill, self->pid, self->tid, SIGCANCEL);
}
long (__syscall_cp)(long nr, long u, long v, long w, long x, long y, long z)
{
pthread_t self;
uintptr_t old_sp, old_ip;
long r;
if (!libc.main_thread || (self = __pthread_self())->canceldisable)
return __syscall(nr, u, v, w, x, y, z);
old_sp = self->cp_sp;
old_ip = self->cp_ip;
self->cp_sp = 0;
self->cp_ip = 0;
r = __syscall_cp_asm(&self->cp_sp, nr, u, v, w, x, y, z);
self->cp_ip = old_ip;
self->cp_sp = old_sp;
if (r==-EINTR && nr!=SYS_close && self->cancel && !self->canceldisable)
__cancel();
return r;
}
const char *last_serror(void)
{
char *buf;
int error = acl_fiber_last_error();
static size_t __buf_size = 4096;
if (pthread_once(&once_control, thread_buf_init) != 0) {
abort();
}
buf = (char*) pthread_getspecific(__errbuf_key);
if (buf == NULL) {
buf = (char*) malloc(__buf_size);
assert(pthread_setspecific(__errbuf_key, buf) == 0);
if (__pthread_self() == main_thread_self()) {
__main_buf = buf;
atexit(main_free_buf);
}
}
return acl_fiber_strerror(error, buf, __buf_size);
}
pid_t fork(void)
{
pid_t ret;
sigset_t set;
__fork_handler(-1);
__block_all_sigs(&set);
#ifdef SYS_fork
ret = syscall(SYS_fork);
#else
ret = syscall(SYS_clone, SIGCHLD, 0);
#endif
if (libc.has_thread_pointer && !ret) {
pthread_t self = __pthread_self();
self->tid = self->pid = __syscall(SYS_getpid);
memset(&self->robust_list, 0, sizeof self->robust_list);
libc.threads_minus_1 = 0;
}
__restore_sigs(&set);
__fork_handler(!ret);
return ret;
}
pid_t fork(void)
{
pid_t ret;
sigset_t set;
__fork_handler(-1);
__block_all_sigs(&set);
#ifdef SYS_fork
ret = syscall(SYS_fork);
#else
ret = syscall(SYS_clone, SIGCHLD, 0);
#endif
if (!ret) {
pthread_t self = __pthread_self();
self->tid = __syscall(SYS_gettid);
self->robust_list.off = 0;
self->robust_list.pending = 0;
libc.threads_minus_1 = 0;
}
__restore_sigs(&set);
__fork_handler(!ret);
return ret;
}
#include "pthread_impl.h"
int pthread_setcanceltype(int new, int* old) {
struct pthread* self = __pthread_self();
if (new > 1U)
return EINVAL;
if (old)
*old = self->cancelasync;
self->cancelasync = new;
if (new)
pthread_testcancel();
return 0;
}
#include <limits.h>
#include "pthread_impl.h"
static void dummy(void)
{
}
weak_alias(dummy, __aio_wake);
static void notify_signal(struct sigevent *sev)
{
siginfo_t si = {
.si_signo = sev->sigev_signo,
.si_value = sev->sigev_value,
.si_code = SI_ASYNCIO,
.si_pid = __pthread_self()->pid,
.si_uid = getuid()
};
__syscall(SYS_rt_sigqueueinfo, si.si_pid, si.si_signo, &si);
}
static void *io_thread(void *p)
{
struct aiocb *cb = p;
int fd = cb->aio_fildes;
void *buf = (void *)cb->aio_buf;
size_t len = cb->aio_nbytes;
off_t off = cb->aio_offset;
int op = cb->aio_lio_opcode;
struct sigevent sev = cb->aio_sigevent;
ssize_t ret;
int *__errno_location(void)
{
static int e;
return &e;
return &__pthread_self()->errno_val;
}
int *__errno_location(void)
{
static int e;
if (libc.has_thread_pointer) return &__pthread_self()->errno_val;
return &e;
}
void *pthread_getspecific(pthread_key_t k)
{
struct pthread *self = __pthread_self();
return self->tsd[k];
}
void __do_orphaned_stdio_locks()
{
FILE *f;
for (f=__pthread_self()->stdio_locks; f; f=f->next_locked)
a_store(&f->lock, 0x40000000);
}
