/* For asynchronous cancellation we use a signal. This is the handler. */
static void
sighandler_setxid (int sig, siginfo_t *si, void *ctx)
{
/* Safety check. It would be possible to call this function for
other signals and send a signal from another process. This is not
correct and might even be a security problem. Try to catch as
many incorrect invocations as possible. */
if (sig != SIGSETXID
#ifdef __ASSUME_CORRECT_SI_PID
/* Kernels before 2.5.75 stored the thread ID and not the process
ID in si_pid so we skip this test. */
|| si->si_pid != THREAD_GETMEM (THREAD_SELF, pid)
#endif
|| si->si_code != SI_TKILL)
return;
INTERNAL_SYSCALL_DECL (err);
INTERNAL_SYSCALL_NCS (__xidcmd->syscall_no, err, 3, __xidcmd->id[0],
__xidcmd->id[1], __xidcmd->id[2]);
if (atomic_decrement_val (&__xidcmd->cntr) == 0)
lll_futex_wake (&__xidcmd->cntr, 1);
/* Reset the SETXID flag. */
struct pthread *self = THREAD_SELF;
int flags = THREAD_GETMEM (self, cancelhandling);
THREAD_SETMEM (self, cancelhandling, flags & ~SETXID_BITMASK);
/* And release the futex. */
self->setxid_futex = 1;
lll_futex_wake (&self->setxid_futex, 1);
}
/* Unlock RWLOCK. */
int
attribute_protected
__pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
{
lll_lock (rwlock->__data.__lock, rwlock->__data.__shared);
if (rwlock->__data.__writer)
rwlock->__data.__writer = 0;
else
--rwlock->__data.__nr_readers;
if (rwlock->__data.__nr_readers == 0)
{
if (rwlock->__data.__nr_writers_queued)
{
++rwlock->__data.__writer_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
lll_futex_wake (&rwlock->__data.__writer_wakeup, 1,
rwlock->__data.__shared);
return 0;
}
else if (rwlock->__data.__nr_readers_queued)
{
++rwlock->__data.__readers_wakeup;
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
lll_futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX,
rwlock->__data.__shared);
return 0;
}
}
lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared);
return 0;
}
int
attribute_protected
__pthread_cond_signal (
pthread_cond_t *cond)
{
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
/* Make sure we are alone. */
lll_lock (cond->__data.__lock, pshared);
/* Are there any waiters to be woken? */
if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
{
/* Yes. Mark one of them as woken. */
++cond->__data.__wakeup_seq;
++cond->__data.__futex;
/* Wake one. */
if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
1, &cond->__data.__lock,
pshared), 0))
return 0;
lll_futex_wake (&cond->__data.__futex, 1, pshared);
}
/* We are done. */
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
void lll_unlock(OSLowLock* futex) {
//OSLog("lll_unlock: %p", futex);
int old = OSAtomicExchange(futex, 0);
if (__builtin_expect(old > 1, 0)) {
lll_futex_wake(futex, 1);
}
}
static void
clear_once_control (void *arg)
{
pthread_once_t *once_control = (pthread_once_t *) arg;
*once_control = 0;
lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
}
/* For asynchronous cancellation we use a signal. This is the handler. */
static void
sighandler_setxid (int sig, siginfo_t *si, void *ctx)
{
#ifdef __ASSUME_CORRECT_SI_PID
/* Determine the process ID. It might be negative if the thread is
in the middle of a fork() call. */
pid_t pid = THREAD_GETMEM (THREAD_SELF, pid);
if (__builtin_expect (pid < 0, 0))
pid = -pid;
#endif
/* Safety check. It would be possible to call this function for
other signals and send a signal from another process. This is not
correct and might even be a security problem. Try to catch as
many incorrect invocations as possible. */
if (sig != SIGSETXID
#ifdef __ASSUME_CORRECT_SI_PID
/* Kernels before 2.5.75 stored the thread ID and not the process
ID in si_pid so we skip this test. */
|| si->si_pid != pid
#endif
|| si->si_code != SI_TKILL)
return;
INTERNAL_SYSCALL_DECL (err);
INTERNAL_SYSCALL_NCS (__xidcmd->syscall_no, err, 3, __xidcmd->id[0],
__xidcmd->id[1], __xidcmd->id[2]);
/* Reset the SETXID flag. */
struct pthread *self = THREAD_SELF;
int flags, newval;
do
{
flags = THREAD_GETMEM (self, cancelhandling);
newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
flags & ~SETXID_BITMASK, flags);
}
while (flags != newval);
/* And release the futex. */
self->setxid_futex = 1;
lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE);
if (atomic_decrement_val (&__xidcmd->cntr) == 0)
lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE);
}
/* We use the SIGSETXID signal in the setuid, setgid, etc. implementations to
tell each thread to call the respective setxid syscall on itself. This is
the handler. */
static void
sighandler_setxid (int sig, siginfo_t *si, void *ctx)
{
/* Determine the process ID. It might be negative if the thread is
in the middle of a fork() call. */
pid_t pid = THREAD_GETMEM (THREAD_SELF, pid);
int result;
if (__glibc_unlikely (pid < 0))
pid = -pid;
/* Safety check. It would be possible to call this function for
other signals and send a signal from another process. This is not
correct and might even be a security problem. Try to catch as
many incorrect invocations as possible. */
if (sig != SIGSETXID
|| si->si_pid != pid
|| si->si_code != SI_TKILL)
return;
INTERNAL_SYSCALL_DECL (err);
result = INTERNAL_SYSCALL_NCS (__xidcmd->syscall_no, err, 3, __xidcmd->id[0],
__xidcmd->id[1], __xidcmd->id[2]);
int error = 0;
if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, err)))
error = INTERNAL_SYSCALL_ERRNO (result, err);
__nptl_setxid_error (__xidcmd, error);
/* Reset the SETXID flag. */
struct pthread *self = THREAD_SELF;
int flags, newval;
do
{
flags = THREAD_GETMEM (self, cancelhandling);
newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
flags & ~SETXID_BITMASK, flags);
}
while (flags != newval);
/* And release the futex. */
self->setxid_futex = 1;
lll_futex_wake (&self->setxid_futex, 1, LLL_PRIVATE);
if (atomic_decrement_val (&__xidcmd->cntr) == 0)
lll_futex_wake (&__xidcmd->cntr, 1, LLL_PRIVATE);
}
int
attribute_protected
__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
{
for (;;)
{
int oldval;
int newval;
/* Pseudo code:
newval = __fork_generation | 1;
oldval = *once_control;
if ((oldval & 2) == 0)
*once_control = newval;
Do this atomically.
*/
do
{
newval = __fork_generation | 1;
oldval = *once_control;
if (oldval & 2)
break;
} while (atomic_compare_and_exchange_val_acq (once_control, newval, oldval) != oldval);
/* Check if the initializer has already been done. */
if ((oldval & 2) != 0)
return 0;
/* Check if another thread already runs the initializer. */
if ((oldval & 1) == 0)
break;
/* Check whether the initializer execution was interrupted by a fork. */
if (oldval != newval)
break;
/* Same generation, some other thread was faster. Wait. */
lll_futex_wait (once_control, oldval, LLL_PRIVATE);
}
/* This thread is the first here. Do the initialization.
Register a cleanup handler so that in case the thread gets
interrupted the initialization can be restarted. */
pthread_cleanup_push (clear_once_control, once_control);
init_routine ();
pthread_cleanup_pop (0);
/* Say that the initialisation is done. */
*once_control = __fork_generation | 2;
/* Wake up all other threads. */
lll_futex_wake (once_control, INT_MAX, LLL_PRIVATE);
return 0;
}
int
lll_unlock_wake_cb (int *futex)
{
int val = atomic_exchange_rel (futex, 0);
if (__builtin_expect (val > 1, 0))
lll_futex_wake (futex, 1);
return 0;
}
int
__pthread_cond_signal (pthread_cond_t *cond)
{
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
LIBC_PROBE (cond_signal, 1, cond);
/* Make sure we are alone. */
lll_lock (cond->__data.__lock, pshared);
/* Are there any waiters to be woken? */
if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
{
/* Yes. Mark one of them as woken. */
++cond->__data.__wakeup_seq;
++cond->__data.__futex;
#if (defined lll_futex_cmp_requeue_pi \
&& defined __ASSUME_REQUEUE_PI)
pthread_mutex_t *mut = cond->__data.__mutex;
if (USE_REQUEUE_PI (mut)
/* This can only really fail with a ENOSYS, since nobody can modify
futex while we have the cond_lock. */
&& lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
&mut->__data.__lock,
cond->__data.__futex, pshared) == 0)
{
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
else
#endif
/* Wake one. */
if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
1, 1,
&cond->__data.__lock,
pshared), 0))
return 0;
/* Fallback if neither of them work. */
lll_futex_wake (&cond->__data.__futex, 1, pshared);
}
/* We are done. */
lll_unlock (cond->__data.__lock, pshared);
return 0;
}
pid_t
__libc_fork (void)
{
pid_t pid;
struct used_handler
{
struct fork_handler *handler;
struct used_handler *next;
} *allp = NULL;
/* Run all the registered preparation handlers. In reverse order.
While doing this we build up a list of all the entries. */
struct fork_handler *runp;
while ((runp = __fork_handlers) != NULL)
{
/* Make sure we read from the current RUNP pointer. */
atomic_full_barrier ();
unsigned int oldval = runp->refcntr;
if (oldval == 0)
/* This means some other thread removed the list just after
the pointer has been loaded. Try again. Either the list
is empty or we can retry it. */
continue;
/* Bump the reference counter. */
if (atomic_compare_and_exchange_bool_acq (&__fork_handlers->refcntr,
oldval + 1, oldval))
/* The value changed, try again. */
continue;
/* We bumped the reference counter for the first entry in the
list. That means that none of the following entries will
just go away. The unloading code works in the order of the
list.
While executing the registered handlers we are building a
list of all the entries so that we can go backward later on. */
while (1)
{
/* Execute the handler if there is one. */
if (runp->prepare_handler != NULL)
runp->prepare_handler ();
/* Create a new element for the list. */
struct used_handler *newp
= (struct used_handler *) alloca (sizeof (*newp));
newp->handler = runp;
newp->next = allp;
allp = newp;
/* Advance to the next handler. */
runp = runp->next;
if (runp == NULL)
break;
/* Bump the reference counter for the next entry. */
atomic_increment (&runp->refcntr);
}
/* We are done. */
break;
}
__UCLIBC_IO_MUTEX_LOCK_CANCEL_UNSAFE(_stdio_openlist_add_lock);
#ifndef NDEBUG
pid_t ppid = THREAD_GETMEM (THREAD_SELF, tid);
#endif
/* We need to prevent the getpid() code to update the PID field so
that, if a signal arrives in the child very early and the signal
handler uses getpid(), the value returned is correct. */
pid_t parentpid = THREAD_GETMEM (THREAD_SELF, pid);
THREAD_SETMEM (THREAD_SELF, pid, -parentpid);
#ifdef ARCH_FORK
pid = ARCH_FORK ();
#else
# error "ARCH_FORK must be defined so that the CLONE_SETTID flag is used"
pid = INLINE_SYSCALL (fork, 0);
#endif
if (pid == 0)
{
struct pthread *self = THREAD_SELF;
assert (THREAD_GETMEM (self, tid) != ppid);
if (__fork_generation_pointer != NULL)
*__fork_generation_pointer += 4;
/* Adjust the PID field for the new process. */
THREAD_SETMEM (self, pid, THREAD_GETMEM (self, tid));
#if HP_TIMING_AVAIL
/* The CPU clock of the thread and process have to be set to zero. */
hp_timing_t now;
HP_TIMING_NOW (now);
THREAD_SETMEM (self, cpuclock_offset, now);
GL(dl_cpuclock_offset) = now;
#endif
/* Reset the file list. These are recursive mutexes. */
fresetlockfiles ();
/* Reset locks in the I/O code. */
STDIO_INIT_MUTEX(_stdio_openlist_add_lock);
/* XXX reset any locks in dynamic loader */
/* Run the handlers registered for the child. */
while (allp != NULL)
{
if (allp->handler->child_handler != NULL)
allp->handler->child_handler ();
/* Note that we do not have to wake any possible waiter.
This is the only thread in the new process. The count
may have been bumped up by other threads doing a fork.
We reset it to 1, to avoid waiting for non-existing
thread(s) to release the count. */
allp->handler->refcntr = 1;
/* XXX We could at this point look through the object pool
and mark all objects not on the __fork_handlers list as
unused. This is necessary in case the fork() happened
while another thread called dlclose() and that call had
to create a new list. */
allp = allp->next;
}
/* Initialize the fork lock. */
__fork_lock = LLL_LOCK_INITIALIZER;
}
else
{
assert (THREAD_GETMEM (THREAD_SELF, tid) == ppid);
/* Restore the PID value. */
THREAD_SETMEM (THREAD_SELF, pid, parentpid);
/* We execute this even if the 'fork' call failed. */
__UCLIBC_IO_MUTEX_UNLOCK_CANCEL_UNSAFE(_stdio_openlist_add_lock);
/* Run the handlers registered for the parent. */
while (allp != NULL)
{
if (allp->handler->parent_handler != NULL)
allp->handler->parent_handler ();
if (atomic_decrement_and_test (&allp->handler->refcntr)
&& allp->handler->need_signal)
lll_futex_wake (allp->handler->refcntr, 1, LLL_PRIVATE);
allp = allp->next;
}
}
return pid;
}
static int
do_clone (struct pthread *pd, const struct pthread_attr *attr,
int clone_flags, int (*fct) (void *), STACK_VARIABLES_PARMS,
int stopped)
{
#ifdef PREPARE_CREATE
PREPARE_CREATE;
#endif
if (__builtin_expect (stopped != 0, 0))
/* We make sure the thread does not run far by forcing it to get a
lock. We lock it here too so that the new thread cannot continue
until we tell it to. */
lll_lock (pd->lock, LLL_PRIVATE);
/* One more thread. We cannot have the thread do this itself, since it
might exist but not have been scheduled yet by the time we've returned
and need to check the value to behave correctly. We must do it before
creating the thread, in case it does get scheduled first and then
might mistakenly think it was the only thread. In the failure case,
we momentarily store a false value; this doesn't matter because there
is no kosher thing a signal handler interrupting us right here can do
that cares whether the thread count is correct. */
atomic_increment (&__nptl_nthreads);
int rc = ARCH_CLONE (fct, STACK_VARIABLES_ARGS, clone_flags,
pd, &pd->tid, TLS_VALUE, &pd->tid);
if (__builtin_expect (rc == -1, 0))
{
atomic_decrement (&__nptl_nthreads); /* Oops, we lied for a second. */
/* Perhaps a thread wants to change the IDs and if waiting
for this stillborn thread. */
if (__builtin_expect (atomic_exchange_acq (&pd->setxid_futex, 0)
== -2, 0))
lll_futex_wake (&pd->setxid_futex, 1, LLL_PRIVATE);
/* Free the resources. */
__deallocate_stack (pd);
/* We have to translate error codes. */
return errno == ENOMEM ? EAGAIN : errno;
}
/* Now we have the possibility to set scheduling parameters etc. */
if (__builtin_expect (stopped != 0, 0))
{
INTERNAL_SYSCALL_DECL (err);
int res = 0;
/* Set the affinity mask if necessary. */
if (attr->cpuset != NULL)
{
res = INTERNAL_SYSCALL (sched_setaffinity, err, 3, pd->tid,
attr->cpusetsize, attr->cpuset);
if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (res, err), 0))
{
/* The operation failed. We have to kill the thread. First
send it the cancellation signal. */
INTERNAL_SYSCALL_DECL (err2);
err_out:
(void) INTERNAL_SYSCALL (tgkill, err2, 3,
THREAD_GETMEM (THREAD_SELF, pid),
pd->tid, SIGCANCEL);
/* We do not free the stack here because the canceled thread
itself will do this. */
return (INTERNAL_SYSCALL_ERROR_P (res, err)
? INTERNAL_SYSCALL_ERRNO (res, err)
: 0);
}
}
/* Set the scheduling parameters. */
if ((attr->flags & ATTR_FLAG_NOTINHERITSCHED) != 0)
{
res = INTERNAL_SYSCALL (sched_setscheduler, err, 3, pd->tid,
pd->schedpolicy, &pd->schedparam);
if (__builtin_expect (INTERNAL_SYSCALL_ERROR_P (res, err), 0))
goto err_out;
}
}
/* We now have for sure more than one thread. The main thread might
not yet have the flag set. No need to set the global variable
again if this is what we use. */
THREAD_SETMEM (THREAD_SELF, header.multiple_threads, 1);
return 0;
}
