int sem_timedwait(sem_t* sem, const timespec* abs_timeout) {
atomic_uint* sem_count_ptr = SEM_TO_ATOMIC_POINTER(sem);
// POSIX says we need to try to decrement the semaphore
// before checking the timeout value. Note that if the
// value is currently 0, __sem_trydec() does nothing.
if (__sem_trydec(sem_count_ptr) > 0) {
return 0;
}
// Check it as per POSIX.
int result = check_timespec(abs_timeout, false);
if (result != 0) {
errno = result;
return -1;
}
unsigned int shared = SEM_GET_SHARED(sem_count_ptr);
while (true) {
// Try to grab the semaphore. If the value was 0, this will also change it to -1.
if (__sem_dec(sem_count_ptr) > 0) {
return 0;
}
// Contention detected. Wait for a wakeup event.
int result = __futex_wait_ex(sem_count_ptr, shared, shared | SEMCOUNT_MINUS_ONE, true, abs_timeout);
// Return in case of timeout or interrupt.
if (result == -ETIMEDOUT || result == -EINTR) {
errno = -result;
return -1;
}
}
}
/*
* Lock a mutex of type NORMAL.
*
* As noted above, there are three states:
* 0 (unlocked, no contention)
* 1 (locked, no contention)
* 2 (locked, contention)
*
* Non-recursive mutexes don't use the thread-id or counter fields, and the
* "type" value is zero, so the only bits that will be set are the ones in
* the lock state field.
*/
static inline __always_inline int __pthread_normal_mutex_lock(pthread_mutex_internal_t* mutex,
uint16_t shared,
bool use_realtime_clock,
const timespec* abs_timeout_or_null) {
if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) {
return 0;
}
int result = check_timespec(abs_timeout_or_null, true);
if (result != 0) {
return result;
}
ScopedTrace trace("Contending for pthread mutex");
const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED;
const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
// We want to go to sleep until the mutex is available, which requires
// promoting it to locked_contended. We need to swap in the new state
// and then wait until somebody wakes us up.
// An atomic_exchange is used to compete with other threads for the lock.
// If it returns unlocked, we have acquired the lock, otherwise another
// thread still holds the lock and we should wait again.
// If lock is acquired, an acquire fence is needed to make all memory accesses
// made by other threads visible to the current CPU.
while (atomic_exchange_explicit(&mutex->state, locked_contended,
memory_order_acquire) != unlocked) {
if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock,
abs_timeout_or_null) == -ETIMEDOUT) {
return ETIMEDOUT;
}
}
return 0;
}
static int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock,
const timespec* abs_timeout_or_null) {
if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) {
return EDEADLK;
}
while (true) {
int result = __pthread_rwlock_tryrdlock(rwlock);
if (result == 0 || result == EAGAIN) {
return result;
}
result = check_timespec(abs_timeout_or_null);
if (result != 0) {
return result;
}
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
if (__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) {
continue;
}
rwlock->pending_lock.lock();
rwlock->pending_reader_count++;
// We rely on the fact that all atomic exchange operations on the same object (here it is
// rwlock->state) always appear to occur in a single total order. If the pending flag is added
// before unlocking, the unlocking thread will wakeup the waiter. Otherwise, we will see the
// state is unlocked and will not wait anymore.
old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_READERS_FLAG,
memory_order_relaxed);
int old_serial = rwlock->pending_reader_wakeup_serial;
rwlock->pending_lock.unlock();
int futex_result = 0;
if (!__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) {
futex_result = __futex_wait_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared,
old_serial, true, abs_timeout_or_null);
}
rwlock->pending_lock.lock();
rwlock->pending_reader_count--;
if (rwlock->pending_reader_count == 0) {
atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_READERS_FLAG,
memory_order_relaxed);
}
rwlock->pending_lock.unlock();
if (futex_result == -ETIMEDOUT) {
return ETIMEDOUT;
}
}
}
static int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock,
const timespec* abs_timeout_or_null) {
if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) {
return EDEADLK;
}
while (true) {
int result = __pthread_rwlock_trywrlock(rwlock);
if (result == 0) {
return result;
}
result = check_timespec(abs_timeout_or_null);
if (result != 0) {
return result;
}
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
if (__can_acquire_write_lock(old_state)) {
continue;
}
rwlock->pending_lock.lock();
rwlock->pending_writer_count++;
old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_WRITERS_FLAG,
memory_order_relaxed);
int old_serial = rwlock->pending_writer_wakeup_serial;
rwlock->pending_lock.unlock();
int futex_result = 0;
if (!__can_acquire_write_lock(old_state)) {
futex_result = __futex_wait_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared,
old_serial, true, abs_timeout_or_null);
}
rwlock->pending_lock.lock();
rwlock->pending_writer_count--;
if (rwlock->pending_writer_count == 0) {
atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_WRITERS_FLAG,
memory_order_relaxed);
}
rwlock->pending_lock.unlock();
if (futex_result == -ETIMEDOUT) {
return ETIMEDOUT;
}
}
}
static int __pthread_cond_timedwait(pthread_cond_internal_t* cond, pthread_mutex_t* mutex,
bool use_realtime_clock, const timespec* abs_timeout_or_null) {
int result = check_timespec(abs_timeout_or_null, true);
if (result != 0) {
return result;
}
unsigned int old_state = atomic_load_explicit(&cond->state, memory_order_relaxed);
pthread_mutex_unlock(mutex);
int status = __futex_wait_ex(&cond->state, cond->process_shared(), old_state,
use_realtime_clock, abs_timeout_or_null);
pthread_mutex_lock(mutex);
if (status == -ETIMEDOUT) {
return ETIMEDOUT;
}
return 0;
}
static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex,
bool use_realtime_clock,
const timespec* abs_timeout_or_null) {
uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
uint16_t mtype = (old_state & MUTEX_TYPE_MASK);
uint16_t shared = (old_state & MUTEX_SHARED_MASK);
// Handle common case first.
if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) {
return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null);
}
// Do we already own this recursive or error-check mutex?
pid_t tid = __get_thread()->tid;
if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) {
if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) {
return EDEADLK;
}
return __recursive_increment(mutex, old_state);
}
const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED;
const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
// First, if the mutex is unlocked, try to quickly acquire it.
// In the optimistic case where this works, set the state to locked_uncontended.
if (old_state == unlocked) {
// If exchanged successfully, an acquire fence is required to make
// all memory accesses made by other threads visible to the current CPU.
if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state,
locked_uncontended, memory_order_acquire, memory_order_relaxed))) {
atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
return 0;
}
}
ScopedTrace trace("Contending for pthread mutex");
while (true) {
if (old_state == unlocked) {
// NOTE: We put the state to locked_contended since we _know_ there
// is contention when we are in this loop. This ensures all waiters
// will be unlocked.
// If exchanged successfully, an acquire fence is required to make
// all memory accesses made by other threads visible to the current CPU.
if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state,
&old_state, locked_contended,
memory_order_acquire,
memory_order_relaxed))) {
atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
return 0;
}
continue;
} else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) {
// We should set it to locked_contended beforing going to sleep. This can make
// sure waiters will be woken up eventually.
int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state);
if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state,
&old_state, new_state,
memory_order_relaxed,
memory_order_relaxed))) {
continue;
}
old_state = new_state;
}
int result = check_timespec(abs_timeout_or_null, true);
if (result != 0) {
return result;
}
// We are in locked_contended state, sleep until someone wakes us up.
if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock,
abs_timeout_or_null) == -ETIMEDOUT) {
return ETIMEDOUT;
}
old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
}
}
