int sem_timedwait(sem_t* sem, const timespec* abs_timeout) { atomic_uint* sem_count_ptr = SEM_TO_ATOMIC_POINTER(sem); // POSIX says we need to try to decrement the semaphore // before checking the timeout value. Note that if the // value is currently 0, __sem_trydec() does nothing. if (__sem_trydec(sem_count_ptr) > 0) { return 0; } // Check it as per POSIX. int result = check_timespec(abs_timeout, false); if (result != 0) { errno = result; return -1; } unsigned int shared = SEM_GET_SHARED(sem_count_ptr); while (true) { // Try to grab the semaphore. If the value was 0, this will also change it to -1. if (__sem_dec(sem_count_ptr) > 0) { return 0; } // Contention detected. Wait for a wakeup event. int result = __futex_wait_ex(sem_count_ptr, shared, shared | SEMCOUNT_MINUS_ONE, true, abs_timeout); // Return in case of timeout or interrupt. if (result == -ETIMEDOUT || result == -EINTR) { errno = -result; return -1; } } }
/* * Lock a mutex of type NORMAL. * * As noted above, there are three states: * 0 (unlocked, no contention) * 1 (locked, no contention) * 2 (locked, contention) * * Non-recursive mutexes don't use the thread-id or counter fields, and the * "type" value is zero, so the only bits that will be set are the ones in * the lock state field. */ static inline __always_inline int __pthread_normal_mutex_lock(pthread_mutex_internal_t* mutex, uint16_t shared, bool use_realtime_clock, const timespec* abs_timeout_or_null) { if (__predict_true(__pthread_normal_mutex_trylock(mutex, shared) == 0)) { return 0; } int result = check_timespec(abs_timeout_or_null, true); if (result != 0) { return result; } ScopedTrace trace("Contending for pthread mutex"); const uint16_t unlocked = shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_contended = shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; // We want to go to sleep until the mutex is available, which requires // promoting it to locked_contended. We need to swap in the new state // and then wait until somebody wakes us up. // An atomic_exchange is used to compete with other threads for the lock. // If it returns unlocked, we have acquired the lock, otherwise another // thread still holds the lock and we should wait again. // If lock is acquired, an acquire fence is needed to make all memory accesses // made by other threads visible to the current CPU. while (atomic_exchange_explicit(&mutex->state, locked_contended, memory_order_acquire) != unlocked) { if (__futex_wait_ex(&mutex->state, shared, locked_contended, use_realtime_clock, abs_timeout_or_null) == -ETIMEDOUT) { return ETIMEDOUT; } } return 0; }
static int __pthread_rwlock_timedrdlock(pthread_rwlock_internal_t* rwlock, const timespec* abs_timeout_or_null) { if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) { return EDEADLK; } while (true) { int result = __pthread_rwlock_tryrdlock(rwlock); if (result == 0 || result == EAGAIN) { return result; } result = check_timespec(abs_timeout_or_null); if (result != 0) { return result; } int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); if (__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) { continue; } rwlock->pending_lock.lock(); rwlock->pending_reader_count++; // We rely on the fact that all atomic exchange operations on the same object (here it is // rwlock->state) always appear to occur in a single total order. If the pending flag is added // before unlocking, the unlocking thread will wakeup the waiter. Otherwise, we will see the // state is unlocked and will not wait anymore. old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_READERS_FLAG, memory_order_relaxed); int old_serial = rwlock->pending_reader_wakeup_serial; rwlock->pending_lock.unlock(); int futex_result = 0; if (!__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred)) { futex_result = __futex_wait_ex(&rwlock->pending_reader_wakeup_serial, rwlock->pshared, old_serial, true, abs_timeout_or_null); } rwlock->pending_lock.lock(); rwlock->pending_reader_count--; if (rwlock->pending_reader_count == 0) { atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_READERS_FLAG, memory_order_relaxed); } rwlock->pending_lock.unlock(); if (futex_result == -ETIMEDOUT) { return ETIMEDOUT; } } }
static int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock, const timespec* abs_timeout_or_null) { if (atomic_load_explicit(&rwlock->writer_tid, memory_order_relaxed) == __get_thread()->tid) { return EDEADLK; } while (true) { int result = __pthread_rwlock_trywrlock(rwlock); if (result == 0) { return result; } result = check_timespec(abs_timeout_or_null); if (result != 0) { return result; } int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); if (__can_acquire_write_lock(old_state)) { continue; } rwlock->pending_lock.lock(); rwlock->pending_writer_count++; old_state = atomic_fetch_or_explicit(&rwlock->state, STATE_HAVE_PENDING_WRITERS_FLAG, memory_order_relaxed); int old_serial = rwlock->pending_writer_wakeup_serial; rwlock->pending_lock.unlock(); int futex_result = 0; if (!__can_acquire_write_lock(old_state)) { futex_result = __futex_wait_ex(&rwlock->pending_writer_wakeup_serial, rwlock->pshared, old_serial, true, abs_timeout_or_null); } rwlock->pending_lock.lock(); rwlock->pending_writer_count--; if (rwlock->pending_writer_count == 0) { atomic_fetch_and_explicit(&rwlock->state, ~STATE_HAVE_PENDING_WRITERS_FLAG, memory_order_relaxed); } rwlock->pending_lock.unlock(); if (futex_result == -ETIMEDOUT) { return ETIMEDOUT; } } }
static int __pthread_cond_timedwait(pthread_cond_internal_t* cond, pthread_mutex_t* mutex, bool use_realtime_clock, const timespec* abs_timeout_or_null) { int result = check_timespec(abs_timeout_or_null, true); if (result != 0) { return result; } unsigned int old_state = atomic_load_explicit(&cond->state, memory_order_relaxed); pthread_mutex_unlock(mutex); int status = __futex_wait_ex(&cond->state, cond->process_shared(), old_state, use_realtime_clock, abs_timeout_or_null); pthread_mutex_lock(mutex); if (status == -ETIMEDOUT) { return ETIMEDOUT; } return 0; }
static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex, bool use_realtime_clock, const timespec* abs_timeout_or_null) { uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); uint16_t mtype = (old_state & MUTEX_TYPE_MASK); uint16_t shared = (old_state & MUTEX_SHARED_MASK); // Handle common case first. if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) { return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null); } // Do we already own this recursive or error-check mutex? pid_t tid = __get_thread()->tid; if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { return EDEADLK; } return __recursive_increment(mutex, old_state); } const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; // First, if the mutex is unlocked, try to quickly acquire it. // In the optimistic case where this works, set the state to locked_uncontended. if (old_state == unlocked) { // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, locked_uncontended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } } ScopedTrace trace("Contending for pthread mutex"); while (true) { if (old_state == unlocked) { // NOTE: We put the state to locked_contended since we _know_ there // is contention when we are in this loop. This ensures all waiters // will be unlocked. // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, locked_contended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } continue; } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) { // We should set it to locked_contended beforing going to sleep. This can make // sure waiters will be woken up eventually. int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state); if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, new_state, memory_order_relaxed, memory_order_relaxed))) { continue; } old_state = new_state; } int result = check_timespec(abs_timeout_or_null, true); if (result != 0) { return result; } // We are in locked_contended state, sleep until someone wakes us up. if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock, abs_timeout_or_null) == -ETIMEDOUT) { return ETIMEDOUT; } old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); } }