void
exchange (atomic_int *i)
{
int r;
atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */
atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */
atomic_compare_exchange_strong_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */
atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_release); /* { dg-warning "invalid failure memory" } */
atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_seq_cst, memory_order_acq_rel); /* { dg-warning "invalid failure memory" } */
atomic_compare_exchange_weak_explicit (i, &r, 0, memory_order_relaxed, memory_order_consume); /* { dg-warning "failure memory model cannot be stronger" } */
}
/* Test for consistency on sizes 1, 2, 4, 8, 16 and 32. */
int
main ()
{
test_struct c;
atomic_store_explicit (&a, zero, memory_order_relaxed);
if (memcmp (&a, &zero, size))
abort ();
c = atomic_exchange_explicit (&a, ones, memory_order_seq_cst);
if (memcmp (&c, &zero, size))
abort ();
if (memcmp (&a, &ones, size))
abort ();
b = atomic_load_explicit (&a, memory_order_relaxed);
if (memcmp (&b, &ones, size))
abort ();
if (!atomic_compare_exchange_strong_explicit (&a, &b, zero, memory_order_seq_cst, memory_order_acquire))
abort ();
if (memcmp (&a, &zero, size))
abort ();
if (atomic_compare_exchange_weak_explicit (&a, &b, ones, memory_order_seq_cst, memory_order_acquire))
abort ();
if (memcmp (&b, &zero, size))
abort ();
return 0;
}
template<typename T> int Queue_lf_spmc<T>::pop(T& dest)
{
unsigned int lcl_tail, lcl_head;
do{
lcl_tail = tail.load(memory_order_relaxed);
lcl_head = head.load(memory_order_relaxed);
if (lcl_tail == lcl_head){ // head - cap == tail
//printf("empty\n");
return 0;
}
// try and claim the tail for our selves
} while (!atomic_compare_exchange_weak_explicit(&tail,
&lcl_tail,
lcl_tail + 1,
memory_order_release,
memory_order_relaxed));
// the tail is ours.
Guarded_data_tc<T>* gd = &list[lcl_tail % cap];
gd->get(dest);
//printf("poping at %d, %d ,%d \n", (int)lcl_tail,(int)lcl_head, dest);
gd->rescind();
return 1;
}
/* add a compare-and-swap */
static inline int atomic_cas_weak(atomic_t *atomic, int *expected, int desired)
{
ATOMIC_IS_INITIALIZED(atomic);
return atomic_compare_exchange_weak_explicit(&atomic->val,
expected, desired,
memory_order_seq_cst,
memory_order_seq_cst);
}
int pthread_rwlock_tryrdlock(pthread_rwlock_t* rwlock_interface) {
pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
while (old_state >= 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state,
old_state + 1, memory_order_acquire, memory_order_relaxed)) {
}
return (old_state >= 0) ? 0 : EBUSY;
}
TEST(stdatomic, atomic_compare_exchange) {
atomic_int i;
int expected;
atomic_store(&i, 123);
expected = 123;
ASSERT_TRUE(atomic_compare_exchange_strong(&i, &expected, 456));
ASSERT_FALSE(atomic_compare_exchange_strong(&i, &expected, 456));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
ASSERT_TRUE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_FALSE(atomic_compare_exchange_strong_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
int iter_count = 0;
do {
++iter_count;
ASSERT_LT(iter_count, 100); // Arbitrary limit on spurious compare_exchange failures.
ASSERT_EQ(expected, 123);
} while(!atomic_compare_exchange_weak(&i, &expected, 456));
ASSERT_FALSE(atomic_compare_exchange_weak(&i, &expected, 456));
ASSERT_EQ(456, expected);
atomic_store(&i, 123);
expected = 123;
iter_count = 0;
do {
++iter_count;
ASSERT_LT(iter_count, 100);
ASSERT_EQ(expected, 123);
} while(!atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_FALSE(atomic_compare_exchange_weak_explicit(&i, &expected, 456, memory_order_relaxed,
memory_order_relaxed));
ASSERT_EQ(456, expected);
}
static inline __always_inline int __pthread_rwlock_trywrlock(pthread_rwlock_internal_t* rwlock) {
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
while (__predict_true(__can_acquire_write_lock(old_state))) {
if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state,
__state_add_writer_flag(old_state), memory_order_acquire, memory_order_relaxed))) {
atomic_store_explicit(&rwlock->writer_tid, __get_thread()->tid, memory_order_relaxed);
return 0;
}
}
return EBUSY;
}
int pthread_rwlock_trywrlock(pthread_rwlock_t* rwlock_interface) {
pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
while (old_state == 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1,
memory_order_acquire, memory_order_relaxed)) {
}
if (old_state == 0) {
atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed);
return 0;
}
return EBUSY;
}
static int __pthread_rwlock_timedwrlock(pthread_rwlock_internal_t* rwlock,
const timespec* abs_timeout_or_null) {
if (__predict_false(__get_thread()->tid == atomic_load_explicit(&rwlock->writer_thread_id,
memory_order_relaxed))) {
return EDEADLK;
}
while (true) {
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
if (__predict_true(old_state == 0)) {
if (atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, -1,
memory_order_acquire, memory_order_relaxed)) {
// writer_thread_id is protected by rwlock and can only be modified in rwlock write
// owner thread. Other threads may read it for EDEADLK error checking, atomic operation
// is safe enough for it.
atomic_store_explicit(&rwlock->writer_thread_id, __get_thread()->tid, memory_order_relaxed);
return 0;
}
} else {
timespec ts;
timespec* rel_timeout = NULL;
if (abs_timeout_or_null != NULL) {
rel_timeout = &ts;
if (!timespec_from_absolute_timespec(*rel_timeout, *abs_timeout_or_null, CLOCK_REALTIME)) {
return ETIMEDOUT;
}
}
// To avoid losing wake ups, the pending_writers increment should be observed before
// futex_wait by all threads. A seq_cst fence instead of a seq_cst operation is used
// here. Because only a seq_cst fence can ensure sequential consistency for non-atomic
// operations in futex_wait.
atomic_fetch_add_explicit(&rwlock->pending_writers, 1, memory_order_relaxed);
atomic_thread_fence(memory_order_seq_cst);
int ret = __futex_wait_ex(&rwlock->state, rwlock->process_shared(), old_state,
rel_timeout);
atomic_fetch_sub_explicit(&rwlock->pending_writers, 1, memory_order_relaxed);
if (ret == -ETIMEDOUT) {
return ETIMEDOUT;
}
}
}
}
static inline __always_inline int __pthread_rwlock_tryrdlock(pthread_rwlock_internal_t* rwlock) {
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
while (__predict_true(__can_acquire_read_lock(old_state, rwlock->writer_nonrecursive_preferred))) {
int new_state = old_state + STATE_READER_COUNT_CHANGE_STEP;
if (__predict_false(!__state_owned_by_readers(new_state))) { // Happens when reader count overflows.
return EAGAIN;
}
if (__predict_true(atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state, new_state,
memory_order_acquire, memory_order_relaxed))) {
return 0;
}
}
return EBUSY;
}
int pthread_rwlock_unlock(pthread_rwlock_t* rwlock_interface) {
pthread_rwlock_internal_t* rwlock = __get_internal_rwlock(rwlock_interface);
int old_state = atomic_load_explicit(&rwlock->state, memory_order_relaxed);
if (__predict_false(old_state == 0)) {
return EPERM;
} else if (old_state == -1) {
if (atomic_load_explicit(&rwlock->writer_thread_id, memory_order_relaxed) != __get_thread()->tid) {
return EPERM;
}
// We're no longer the owner.
atomic_store_explicit(&rwlock->writer_thread_id, 0, memory_order_relaxed);
// Change state from -1 to 0.
atomic_store_explicit(&rwlock->state, 0, memory_order_release);
} else { // old_state > 0
// Reduce state by 1.
while (old_state > 0 && !atomic_compare_exchange_weak_explicit(&rwlock->state, &old_state,
old_state - 1, memory_order_release, memory_order_relaxed)) {
}
if (old_state <= 0) {
return EPERM;
} else if (old_state > 1) {
return 0;
}
// old_state = 1, which means the last reader calling unlock. It has to wake up waiters.
}
// If having waiters, wake up them.
// To avoid losing wake ups, the update of state should be observed before reading
// pending_readers/pending_writers by all threads. Use read locking as an example:
// read locking thread unlocking thread
// pending_readers++; state = 0;
// seq_cst fence seq_cst fence
// read state for futex_wait read pending_readers for futex_wake
//
// So when locking and unlocking threads are running in parallel, we will not get
// in a situation that the locking thread reads state as negative and needs to wait,
// while the unlocking thread reads pending_readers as zero and doesn't need to wake up waiters.
atomic_thread_fence(memory_order_seq_cst);
if (__predict_false(atomic_load_explicit(&rwlock->pending_readers, memory_order_relaxed) > 0 ||
atomic_load_explicit(&rwlock->pending_writers, memory_order_relaxed) > 0)) {
__futex_wake_ex(&rwlock->state, rwlock->process_shared(), INT_MAX);
}
return 0;
}
/* NOTE: this implementation doesn't support a init function that throws a C++ exception
* or calls fork()
*/
int pthread_once(pthread_once_t* once_control, void (*init_routine)(void)) {
static_assert(sizeof(atomic_int) == sizeof(pthread_once_t),
"pthread_once_t should actually be atomic_int in implementation.");
// We prefer casting to atomic_int instead of declaring pthread_once_t to be atomic_int directly.
// Because using the second method pollutes pthread.h, and causes an error when compiling libcxx.
atomic_int* once_control_ptr = reinterpret_cast<atomic_int*>(once_control);
// First check if the once is already initialized. This will be the common
// case and we want to make this as fast as possible. Note that this still
// requires a load_acquire operation here to ensure that all the
// stores performed by the initialization function are observable on
// this CPU after we exit.
int old_value = atomic_load_explicit(once_control_ptr, memory_order_acquire);
while (true) {
if (__predict_true(old_value == ONCE_INITIALIZATION_COMPLETE)) {
return 0;
}
// Try to atomically set the initialization underway flag. This requires a compare exchange
// in a loop, and we may need to exit prematurely if the initialization is complete.
if (!atomic_compare_exchange_weak_explicit(once_control_ptr, &old_value,
ONCE_INITIALIZATION_UNDERWAY,
memory_order_acquire, memory_order_acquire)) {
continue;
}
if (old_value == ONCE_INITIALIZATION_NOT_YET_STARTED) {
// We got here first, we can handle the initialization.
(*init_routine)();
// Do a store_release indicating that initialization is complete.
atomic_store_explicit(once_control_ptr, ONCE_INITIALIZATION_COMPLETE, memory_order_release);
// Wake up any waiters, if any.
__futex_wake_ex(once_control_ptr, 0, INT_MAX);
return 0;
}
// The initialization is underway, wait for its finish.
__futex_wait_ex(once_control_ptr, 0, old_value, NULL);
old_value = atomic_load_explicit(once_control_ptr, memory_order_acquire);
}
}
static int __pthread_mutex_lock_with_timeout(pthread_mutex_internal_t* mutex,
bool use_realtime_clock,
const timespec* abs_timeout_or_null) {
uint16_t old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
uint16_t mtype = (old_state & MUTEX_TYPE_MASK);
uint16_t shared = (old_state & MUTEX_SHARED_MASK);
// Handle common case first.
if ( __predict_true(mtype == MUTEX_TYPE_BITS_NORMAL) ) {
return __pthread_normal_mutex_lock(mutex, shared, use_realtime_clock, abs_timeout_or_null);
}
// Do we already own this recursive or error-check mutex?
pid_t tid = __get_thread()->tid;
if (tid == atomic_load_explicit(&mutex->owner_tid, memory_order_relaxed)) {
if (mtype == MUTEX_TYPE_BITS_ERRORCHECK) {
return EDEADLK;
}
return __recursive_increment(mutex, old_state);
}
const uint16_t unlocked = mtype | shared | MUTEX_STATE_BITS_UNLOCKED;
const uint16_t locked_uncontended = mtype | shared | MUTEX_STATE_BITS_LOCKED_UNCONTENDED;
const uint16_t locked_contended = mtype | shared | MUTEX_STATE_BITS_LOCKED_CONTENDED;
// First, if the mutex is unlocked, try to quickly acquire it.
// In the optimistic case where this works, set the state to locked_uncontended.
if (old_state == unlocked) {
// If exchanged successfully, an acquire fence is required to make
// all memory accesses made by other threads visible to the current CPU.
if (__predict_true(atomic_compare_exchange_strong_explicit(&mutex->state, &old_state,
locked_uncontended, memory_order_acquire, memory_order_relaxed))) {
atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
return 0;
}
}
ScopedTrace trace("Contending for pthread mutex");
while (true) {
if (old_state == unlocked) {
// NOTE: We put the state to locked_contended since we _know_ there
// is contention when we are in this loop. This ensures all waiters
// will be unlocked.
// If exchanged successfully, an acquire fence is required to make
// all memory accesses made by other threads visible to the current CPU.
if (__predict_true(atomic_compare_exchange_weak_explicit(&mutex->state,
&old_state, locked_contended,
memory_order_acquire,
memory_order_relaxed))) {
atomic_store_explicit(&mutex->owner_tid, tid, memory_order_relaxed);
return 0;
}
continue;
} else if (MUTEX_STATE_BITS_IS_LOCKED_UNCONTENDED(old_state)) {
// We should set it to locked_contended beforing going to sleep. This can make
// sure waiters will be woken up eventually.
int new_state = MUTEX_STATE_BITS_FLIP_CONTENTION(old_state);
if (__predict_false(!atomic_compare_exchange_weak_explicit(&mutex->state,
&old_state, new_state,
memory_order_relaxed,
memory_order_relaxed))) {
continue;
}
old_state = new_state;
}
int result = check_timespec(abs_timeout_or_null, true);
if (result != 0) {
return result;
}
// We are in locked_contended state, sleep until someone wakes us up.
if (__recursive_or_errorcheck_mutex_wait(mutex, shared, old_state, use_realtime_clock,
abs_timeout_or_null) == -ETIMEDOUT) {
return ETIMEDOUT;
}
old_state = atomic_load_explicit(&mutex->state, memory_order_relaxed);
}
}