/* This common inlined function is used to increment the counter of an
* errorcheck or recursive mutex.
*
* For errorcheck mutexes, it will return EDEADLK
* If the counter overflows, it will return EAGAIN
* Otherwise, it atomically increments the counter and returns 0
* after providing an acquire barrier.
*
* mtype is the current mutex type
* mvalue is the current mutex value (already loaded)
* mutex pointers to the mutex.
*/
static __inline__ __attribute__((always_inline)) int
_recursive_increment(pthread_mutex_t* mutex, int mvalue, int mtype)
{
if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) {
/* trying to re-lock a mutex we already acquired */
return EDEADLK;
}
/* Detect recursive lock overflow and return EAGAIN.
* This is safe because only the owner thread can modify the
* counter bits in the mutex value.
*/
if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(mvalue)) {
return EAGAIN;
}
/* We own the mutex, but other threads are able to change
* the lower bits (e.g. promoting it to "contended"), so we
* need to use an atomic cmpxchg loop to update the counter.
*/
for (;;) {
/* increment counter, overflow was already checked */
int newval = mvalue + MUTEX_COUNTER_BITS_ONE;
if (__predict_true(__bionic_cmpxchg(mvalue, newval, &mutex->value) == 0)) {
/* mutex is still locked, not need for a memory barrier */
return 0;
}
/* the value was changed, this happens when another thread changes
* the lower state bits from 1 to 2 to indicate contention. This
* cannot change the counter, so simply reload and try again.
*/
mvalue = mutex->value;
}
}
/* This common inlined function is used to increment the counter of a recursive mutex.
*
* If the counter overflows, it will return EAGAIN.
* Otherwise, it atomically increments the counter and returns 0.
*
*/
static inline __always_inline int __recursive_increment(pthread_mutex_internal_t* mutex,
uint16_t old_state) {
// Detect recursive lock overflow and return EAGAIN.
// This is safe because only the owner thread can modify the
// counter bits in the mutex value.
if (MUTEX_COUNTER_BITS_WILL_OVERFLOW(old_state)) {
return EAGAIN;
}
// Other threads are able to change the lower bits (e.g. promoting it to "contended"),
// but the mutex counter will not overflow. So we use atomic_fetch_add operation here.
// The mutex is still locked by current thread, so we don't need a release fence.
atomic_fetch_add_explicit(&mutex->state, MUTEX_COUNTER_BITS_ONE, memory_order_relaxed);
return 0;
}
