__LIBC_HIDDEN__
int pthread_mutex_unlock_impl(pthread_mutex_t *mutex)
{
int mvalue, mtype, tid, shared;
if (__unlikely(mutex == NULL))
return EINVAL;
mvalue = mutex->value;
mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */
if (__likely(mtype == MUTEX_TYPE_BITS_NORMAL)) {
_normal_unlock(mutex, shared);
return 0;
}
/* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->tid;
if ( tid != MUTEX_OWNER_FROM_BITS(mvalue) )
return EPERM;
/* If the counter is > 0, we can simply decrement it atomically.
* Since other threads can mutate the lower state bits (and only the
* lower state bits), use a cmpxchg to do it.
*/
if (!MUTEX_COUNTER_BITS_IS_ZERO(mvalue)) {
for (;;) {
int newval = mvalue - MUTEX_COUNTER_BITS_ONE;
if (__likely(__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0)) {
/* success: we still own the mutex, so no memory barrier */
return 0;
}
/* the value changed, so reload and loop */
mvalue = mutex->value;
}
}
/* the counter is 0, so we're going to unlock the mutex by resetting
* its value to 'unlocked'. We need to perform a swap in order
* to read the current state, which will be 2 if there are waiters
* to awake.
*
* TODO: Change this to __bionic_swap_release when we implement it
* to get rid of the explicit memory barrier below.
*/
ANDROID_MEMBAR_FULL(); /* RELEASE BARRIER */
mvalue = __bionic_swap(mtype | shared | MUTEX_STATE_BITS_UNLOCKED, &mutex->value);
/* Wake one waiting thread, if any */
if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(mvalue)) {
__futex_wake_ex(&mutex->value, shared, 1);
}
return 0;
}
__LIBC_HIDDEN__
int pthread_mutex_trylock_impl(pthread_mutex_t *mutex)
{
int mvalue, mtype, tid, shared;
if (__unlikely(mutex == NULL))
return EINVAL;
mvalue = mutex->value;
mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */
if ( __likely(mtype == MUTEX_TYPE_BITS_NORMAL) )
{
if (__bionic_cmpxchg(shared|MUTEX_STATE_BITS_UNLOCKED,
shared|MUTEX_STATE_BITS_LOCKED_UNCONTENDED,
&mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
return EBUSY;
}
/* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->tid;
if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) )
return _recursive_increment(mutex, mvalue, mtype);
/* Same as pthread_mutex_lock, except that we don't want to wait, and
* the only operation that can succeed is a single cmpxchg to acquire the
* lock if it is released / not owned by anyone. No need for a complex loop.
*/
mtype |= shared | MUTEX_STATE_BITS_UNLOCKED;
mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
if (__likely(__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0)) {
ANDROID_MEMBAR_FULL();
return 0;
}
return EBUSY;
}
static int __pthread_mutex_timedlock(pthread_mutex_t* mutex, const timespec* abs_timeout, clockid_t clock) {
timespec ts;
int mvalue = mutex->value;
int mtype = (mvalue & MUTEX_TYPE_MASK);
int shared = (mvalue & MUTEX_SHARED_MASK);
// Handle common case first.
if (__predict_true(mtype == MUTEX_TYPE_BITS_NORMAL)) {
const int unlocked = shared | MUTEX_STATE_BITS_UNLOCKED;
const int locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
const int locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
// Fast path for uncontended lock. Note: MUTEX_TYPE_BITS_NORMAL is 0.
if (__bionic_cmpxchg(unlocked, locked_uncontended, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
// Loop while needed.
while (__bionic_swap(locked_contended, &mutex->value) != unlocked) {
if (__timespec_from_absolute(&ts, abs_timeout, clock) < 0) {
return ETIMEDOUT;
}
__futex_wait_ex(&mutex->value, shared, locked_contended, &ts);
}
ANDROID_MEMBAR_FULL();
return 0;
}
// Do we already own this recursive or error-check mutex?
pid_t tid = __get_thread()->tid;
if (tid == MUTEX_OWNER_FROM_BITS(mvalue)) {
return _recursive_increment(mutex, mvalue, mtype);
}
// The following implements the same loop as pthread_mutex_lock_impl
// but adds checks to ensure that the operation never exceeds the
// absolute expiration time.
mtype |= shared;
// First try a quick lock.
if (mvalue == mtype) {
mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
if (__predict_true(__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0)) {
ANDROID_MEMBAR_FULL();
return 0;
}
mvalue = mutex->value;
}
while (true) {
// If the value is 'unlocked', try to acquire it directly.
// NOTE: put state to 2 since we know there is contention.
if (mvalue == mtype) { // Unlocked.
mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_CONTENDED;
if (__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
// The value changed before we could lock it. We need to check
// the time to avoid livelocks, reload the value, then loop again.
if (__timespec_from_absolute(&ts, abs_timeout, clock) < 0) {
return ETIMEDOUT;
}
mvalue = mutex->value;
continue;
}
// The value is locked. If 'uncontended', try to switch its state
// to 'contented' to ensure we get woken up later.
if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(mvalue)) {
int newval = MUTEX_STATE_BITS_FLIP_CONTENTION(mvalue);
if (__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0) {
// This failed because the value changed, reload it.
mvalue = mutex->value;
} else {
// This succeeded, update mvalue.
mvalue = newval;
}
}
// Check time and update 'ts'.
if (__timespec_from_absolute(&ts, abs_timeout, clock) < 0) {
return ETIMEDOUT;
}
// Only wait to be woken up if the state is '2', otherwise we'll
// simply loop right now. This can happen when the second cmpxchg
// in our loop failed because the mutex was unlocked by another thread.
if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(mvalue)) {
if (__futex_wait_ex(&mutex->value, shared, mvalue, &ts) == -ETIMEDOUT) {
return ETIMEDOUT;
}
mvalue = mutex->value;
}
}
/* NOTREACHED */
}
__LIBC_HIDDEN__
int pthread_mutex_lock_impl(pthread_mutex_t *mutex)
{
int mvalue, mtype, tid, shared;
mvalue = mutex->value;
mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle normal case first */
if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) {
_normal_lock(mutex, shared);
return 0;
}
/* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->tid;
if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) )
return _recursive_increment(mutex, mvalue, mtype);
/* Add in shared state to avoid extra 'or' operations below */
mtype |= shared;
/* First, if the mutex is unlocked, try to quickly acquire it.
* In the optimistic case where this works, set the state to 1 to
* indicate locked with no contention */
if (mvalue == mtype) {
int newval = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
if (__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
/* argh, the value changed, reload before entering the loop */
mvalue = mutex->value;
}
for (;;) {
int newval;
/* if the mutex is unlocked, its value should be 'mtype' and
* we try to acquire it by setting its owner and state atomically.
* NOTE: We put the state to 2 since we _know_ there is contention
* when we are in this loop. This ensures all waiters will be
* unlocked.
*/
if (mvalue == mtype) {
newval = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_CONTENDED;
/* TODO: Change this to __bionic_cmpxchg_acquire when we
* implement it to get rid of the explicit memory
* barrier below.
*/
if (__predict_false(__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0)) {
mvalue = mutex->value;
continue;
}
ANDROID_MEMBAR_FULL();
return 0;
}
/* the mutex is already locked by another thread, if its state is 1
* we will change it to 2 to indicate contention. */
if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(mvalue)) {
newval = MUTEX_STATE_BITS_FLIP_CONTENTION(mvalue); /* locked state 1 => state 2 */
if (__predict_false(__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0)) {
mvalue = mutex->value;
continue;
}
mvalue = newval;
}
/* wait until the mutex is unlocked */
__futex_wait_ex(&mutex->value, shared, mvalue, NULL);
mvalue = mutex->value;
}
/* NOTREACHED */
}
__LIBC_HIDDEN__
int pthread_mutex_lock_timeout_np_impl(pthread_mutex_t *mutex, unsigned msecs)
{
clockid_t clock = CLOCK_MONOTONIC;
struct timespec abstime;
struct timespec ts;
int mvalue, mtype, tid, shared;
/* compute absolute expiration time */
__timespec_to_relative_msec(&abstime, msecs, clock);
if (__unlikely(mutex == NULL))
return EINVAL;
mvalue = mutex->value;
mtype = (mvalue & MUTEX_TYPE_MASK);
shared = (mvalue & MUTEX_SHARED_MASK);
/* Handle common case first */
if ( __likely(mtype == MUTEX_TYPE_BITS_NORMAL) )
{
const int unlocked = shared | MUTEX_STATE_BITS_UNLOCKED;
const int locked_uncontended = shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
const int locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
/* fast path for uncontended lock. Note: MUTEX_TYPE_BITS_NORMAL is 0 */
if (__bionic_cmpxchg(unlocked, locked_uncontended, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
/* loop while needed */
while (__bionic_swap(locked_contended, &mutex->value) != unlocked) {
if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
return EBUSY;
__futex_wait_ex(&mutex->value, shared, locked_contended, &ts);
}
ANDROID_MEMBAR_FULL();
return 0;
}
/* Do we already own this recursive or error-check mutex ? */
tid = __get_thread()->tid;
if ( tid == MUTEX_OWNER_FROM_BITS(mvalue) )
return _recursive_increment(mutex, mvalue, mtype);
/* the following implements the same loop than pthread_mutex_lock_impl
* but adds checks to ensure that the operation never exceeds the
* absolute expiration time.
*/
mtype |= shared;
/* first try a quick lock */
if (mvalue == mtype) {
mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
if (__likely(__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0)) {
ANDROID_MEMBAR_FULL();
return 0;
}
mvalue = mutex->value;
}
for (;;) {
struct timespec ts;
/* if the value is 'unlocked', try to acquire it directly */
/* NOTE: put state to 2 since we know there is contention */
if (mvalue == mtype) { /* unlocked */
mvalue = MUTEX_OWNER_TO_BITS(tid) | mtype | MUTEX_STATE_BITS_LOCKED_CONTENDED;
if (__bionic_cmpxchg(mtype, mvalue, &mutex->value) == 0) {
ANDROID_MEMBAR_FULL();
return 0;
}
/* the value changed before we could lock it. We need to check
* the time to avoid livelocks, reload the value, then loop again. */
if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
return EBUSY;
mvalue = mutex->value;
continue;
}
/* The value is locked. If 'uncontended', try to switch its state
* to 'contented' to ensure we get woken up later. */
if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(mvalue)) {
int newval = MUTEX_STATE_BITS_FLIP_CONTENTION(mvalue);
if (__bionic_cmpxchg(mvalue, newval, &mutex->value) != 0) {
/* this failed because the value changed, reload it */
mvalue = mutex->value;
} else {
/* this succeeded, update mvalue */
mvalue = newval;
}
}
/* check time and update 'ts' */
if (__timespec_to_absolute(&ts, &abstime, clock) < 0)
return EBUSY;
/* Only wait to be woken up if the state is '2', otherwise we'll
* simply loop right now. This can happen when the second cmpxchg
* in our loop failed because the mutex was unlocked by another
* thread.
*/
if (MUTEX_STATE_BITS_IS_LOCKED_CONTENDED(mvalue)) {
if (__futex_wait_ex(&mutex->value, shared, mvalue, &ts) == ETIMEDOUT) {
return EBUSY;
}
mvalue = mutex->value;
}
}
/* NOTREACHED */
}