void exchange (atomic_int *i) { int r; atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */ atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */ }
/* Test for consistency on sizes 1, 2, 4, 8, 16 and 32. */ int main () { test_struct c; atomic_store_explicit (&a, zero, memory_order_relaxed); if (memcmp (&a, &zero, size)) abort (); c = atomic_exchange_explicit (&a, ones, memory_order_seq_cst); if (memcmp (&c, &zero, size)) abort (); if (memcmp (&a, &ones, size)) abort (); b = atomic_load_explicit (&a, memory_order_relaxed); if (memcmp (&b, &ones, size)) abort (); if (!atomic_compare_exchange_strong_explicit (&a, &b, zero, memory_order_seq_cst, memory_order_acquire)) abort (); if (memcmp (&a, &zero, size)) abort (); if (atomic_compare_exchange_weak_explicit (&a, &b, ones, memory_order_seq_cst, memory_order_acquire)) abort (); if (memcmp (&b, &zero, size)) abort (); return 0; }
template<typename T> int Queue_lf_spmc<T>::pop(T& dest) { unsigned int lcl_tail, lcl_head; do{ lcl_tail = tail.load(memory_order_relaxed); lcl_head = head.load(memory_order_relaxed); if (lcl_tail == lcl_head){ // head - cap == tail //printf("empty\n"); return 0; } // try and claim the tail for our selves } while (!atomic_compare_exchange_weak_explicit(&tail, &lcl_tail, lcl_tail + 1, memory_order_release, memory_order_relaxed)); // the tail is ours. Guarded_data_tc<T>* gd = &list[lcl_tail % cap]; gd->get(dest); //printf("poping at %d, %d ,%d \n", (int)lcl_tail,(int)lcl_head, dest); gd->rescind(); return 1; }
/* add a compare-and-swap */ static inline int atomic_cas_weak(atomic_t *atomic, int *expected, int desired) { ATOMIC_IS_INITIALIZED(atomic); return atomic_compare_exchange_weak_explicit(&atomic->val, expected, desired, memory_order_seq_cst, memory_order_seq_cst); }
int pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) { pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); while (old_state >= 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, old_state + 1, memory_order_acquire, memory_order_relaxed)) { } return (old_state >= 0) ? 0 : EBUSY; }
TEST(stdatomic, atomic_compare_exchange) { atomic_int i; int expected; atomic_store(&i, 123); expected = 123; ASSERT_TRUE(atomic_compare_exchange_strong(&i, &expected, 456)); ASSERT_FALSE(atomic_compare_exchange_strong(&i, &expected, 456)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; ASSERT_TRUE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_FALSE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; int iter_count = 0; do { ++iter_count; ASSERT_LT(iter_count, 100); // Arbitrary limit on spurious compare_exchange failures. ASSERT_EQ(expected, 123); } while(!atomic_compare_exchange_weak(&i, &expected, 456)); ASSERT_FALSE(atomic_compare_exchange_weak(&i, &expected, 456)); ASSERT_EQ(456, expected); atomic_store(&i, 123); expected = 123; iter_count = 0; do { ++iter_count; ASSERT_LT(iter_count, 100); ASSERT_EQ(expected, 123); } while(!atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_FALSE(atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed, memory_order_relaxed)); ASSERT_EQ(456, expected); }
static inline __always_inline int __pthread_rwlock_trywrlock(pthread_rwlock_internal_t* rwlock) { int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); while (__predict_true(__can_acquire_write_lock(old_state))) { if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, __state_add_writer_flag(old_state), memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&rwlock->writer_tid, __get_thread()->tid, memory_order_relaxed); return 0; } } return EBUSY; }
int pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) { pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); while (old_state == 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, memory_order_acquire, memory_order_relaxed)) { } if (old_state == 0) { atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); return 0; } return EBUSY; }
static int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock, const timespec* abs_timeout_or_null) { if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id, memory_order_relaxed))) { return EDEADLK; } while (true) { int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); if (__predict_true(old_state == 0)) { if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1, memory_order_acquire, memory_order_relaxed)) { // writer_thread_id is protected by rwlock and can only be modified in rwlock write // owner thread. Other threads may read it for EDEADLK error checking, atomic operation // is safe enough for it. atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed); return 0; } } else { timespec ts; timespec* rel_timeout = NULL; if (abs_timeout_or_null != NULL) { rel_timeout = &ts; if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) { return ETIMEDOUT; } } // To avoid losing wake ups, the pending_writers increment should be observed before // futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used // here. Because only a seq_cst fence can ensure sequential consistency for non-atomic // operations in futex_wait. atomic_fetch_add_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); atomic_thread_fence(memory_order_seq_cst); int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state, rel_timeout); atomic_fetch_sub_explicit(&rwlock->pending_writers, 1, memory_order_relaxed); if (ret == -ETIMEDOUT) { return ETIMEDOUT; } } } }
static inline __always_inline int __pthread_rwlock_tryrdlock(pthread_rwlock_internal_t* rwlock) { int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); while (__predict_true(__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred))) { int new_state = old_state + STATE_READER_COUNT_CHANGE_STEP; if (__predict_false(!__state_owned_by_readers(new_state))) { // Happens when reader count overflows. return EAGAIN; } if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, new_state, memory_order_acquire, memory_order_relaxed))) { return 0; } } return EBUSY; }
int pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) { pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface); int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed); if (__predict_false(old_state == 0)) { return EPERM; } else if (old_state == -1) { if (atomic_load_explicit(&rwlock->writer_thread_id, memory_order_relaxed) != __get_thread()->tid) { return EPERM; } // We're no longer the owner. atomic_store_explicit(&rwlock->writer_thread_id, 0, memory_order_relaxed); // Change state from -1 to 0. atomic_store_explicit(&rwlock->state, 0, memory_order_release); } else { // old_state > 0 // Reduce state by 1. while (old_state > 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, old_state - 1, memory_order_release, memory_order_relaxed)) { } if (old_state <= 0) { return EPERM; } else if (old_state > 1) { return 0; } // old_state = 1, which means the last reader calling unlock. It has to wake up waiters. } // If having waiters, wake up them. // To avoid losing wake ups, the update of state should be observed before reading // pending_readers/pending_writers by all threads. Use read locking as an example: // read locking thread unlocking thread // pending_readers++; state = 0; // seq_cst fence seq_cst fence // read state for futex_wait read pending_readers for futex_wake // // So when locking and unlocking threads are running in parallel, we will not get // in a situation that the locking thread reads state as negative and needs to wait, // while the unlocking thread reads pending_readers as zero and doesn't need to wake up waiters. atomic_thread_fence(memory_order_seq_cst); if (__predict_false(atomic_load_explicit(&rwlock->pending_readers, memory_order_relaxed) > 0 || atomic_load_explicit(&rwlock->pending_writers, memory_order_relaxed) > 0)) { __futex_wake_ex(&rwlock->state, rwlock->process_shared(), INT_MAX); } return 0; }
/* NOTE: this implementation doesn't support a init function that throws a C++ exception * or calls fork() */ int pthread_once(pthread_once_t* once_control, void (*init_routine)(void)) { static_assert(sizeof(atomic_int) == sizeof(pthread_once_t), "pthread_once_t should actually be atomic_int in implementation."); // We prefer casting to atomic_int instead of declaring pthread_once_t to be atomic_int directly. // Because using the second method pollutes pthread.h, and causes an error when compiling libcxx. atomic_int* once_control_ptr = reinterpret_cast<atomic_int*>(once_control); // First check if the once is already initialized. This will be the common // case and we want to make this as fast as possible. Note that this still // requires a load_acquire operation here to ensure that all the // stores performed by the initialization function are observable on // this CPU after we exit. int old_value = atomic_load_explicit(once_control_ptr, memory_order_acquire); while (true) { if (__predict_true(old_value == ONCE_INITIALIZATION_COMPLETE)) { return 0; } // Try to atomically set the initialization underway flag. This requires a compare exchange // in a loop, and we may need to exit prematurely if the initialization is complete. if (!atomic_compare_exchange_weak_explicit(once_control_ptr, &old_value, ONCE_INITIALIZATION_UNDERWAY, memory_order_acquire, memory_order_acquire)) { continue; } if (old_value == ONCE_INITIALIZATION_NOT_YET_STARTED) { // We got here first, we can handle the initialization. (*init_routine)(); // Do a store_release indicating that initialization is complete. atomic_store_explicit(once_control_ptr, ONCE_INITIALIZATION_COMPLETE, memory_order_release); // Wake up any waiters, if any. __futex_wake_ex(once_control_ptr, 0, INT_MAX); return 0; } // The initialization is underway, wait for its finish. __futex_wait_ex(once_control_ptr, 0, old_value, NULL); old_value = atomic_load_explicit(once_control_ptr, memory_order_acquire); } }
static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex, bool use_realtime_clock, const timespec* abs_timeout_or_null) { uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); uint16_t mtype = (old_state & MUTEX_TYPE_MASK); uint16_t shared = (old_state & MUTEX_SHARED_MASK); // Handle common case first. if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) { return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null); } // Do we already own this recursive or error-check mutex? pid_t tid = __get_thread()->tid; if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) { if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) { return EDEADLK; } return __recursive_increment(mutex, old_state); } const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED; const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED; const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED; // First, if the mutex is unlocked, try to quickly acquire it. // In the optimistic case where this works, set the state to locked_uncontended. if (old_state == unlocked) { // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state, locked_uncontended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } } ScopedTrace trace("Contending for pthread mutex"); while (true) { if (old_state == unlocked) { // NOTE: We put the state to locked_contended since we _know_ there // is contention when we are in this loop. This ensures all waiters // will be unlocked. // If exchanged successfully, an acquire fence is required to make // all memory accesses made by other threads visible to the current CPU. if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, locked_contended, memory_order_acquire, memory_order_relaxed))) { atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed); return 0; } continue; } else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) { // We should set it to locked_contended beforing going to sleep. This can make // sure waiters will be woken up eventually. int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state); if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state, &old_state, new_state, memory_order_relaxed, memory_order_relaxed))) { continue; } old_state = new_state; } int result = check_timespec(abs_timeout_or_null, true); if (result != 0) { return result; } // We are in locked_contended state, sleep until someone wakes us up. if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock, abs_timeout_or_null) == -ETIMEDOUT) { return ETIMEDOUT; } old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed); } }