int
pthread_spin_lock (pthread_spinlock_t *lock)
{
int val = 0;
/* We assume that the first try mostly will be successful, thus we use
atomic_exchange if it is not implemented by a CAS loop (we also assume
that atomic_exchange can be faster if it succeeds, see
ATOMIC_EXCHANGE_USES_CAS). Otherwise, we use a weak CAS and not an
exchange so we bail out after the first failed attempt to change the
state. For the subsequent attempts we use atomic_compare_and_exchange
after we observe that the lock is not acquired.
See also comment in pthread_spin_trylock.
We use acquire MO to synchronize-with the release MO store in
pthread_spin_unlock, and thus ensure that prior critical sections
happen-before this critical section. */
#if ! ATOMIC_EXCHANGE_USES_CAS
/* Try to acquire the lock with an exchange instruction as this architecture
has such an instruction and we assume it is faster than a CAS.
The acquisition succeeds if the lock is not in an acquired state. */
if (__glibc_likely (atomic_exchange_acquire (lock, 1) == 0))
return 0;
#else
/* Try to acquire the lock with a CAS instruction as this architecture
has no exchange instruction. The acquisition succeeds if the lock is not
acquired. */
if (__glibc_likely (atomic_compare_exchange_weak_acquire (lock, &val, 1)))
return 0;
#endif
do
{
/* The lock is contended and we need to wait. Going straight back
to cmpxchg is not a good idea on many targets as that will force
expensive memory synchronizations among processors and penalize other
running threads.
There is no technical reason for throwing in a CAS every now and then,
and so far we have no evidence that it can improve performance.
If that would be the case, we have to adjust other spin-waiting loops
elsewhere, too!
Thus we use relaxed MO reads until we observe the lock to not be
acquired anymore. */
do
{
/* TODO Back-off. */
atomic_spin_nop ();
val = atomic_load_relaxed (lock);
}
while (val != 0);
/* We need acquire memory order here for the same reason as mentioned
for the first try to lock the spinlock. */
}
while (!atomic_compare_exchange_weak_acquire (lock, &val, 1));
return 0;
}
/* Test various atomic.h macros. */
static int
do_test (void)
{
atomic_t mem, expected;
int ret = 0;
#ifdef atomic_compare_and_exchange_val_acq
mem = 24;
if (atomic_compare_and_exchange_val_acq (&mem, 35, 24) != 24
|| mem != 35)
{
puts ("atomic_compare_and_exchange_val_acq test 1 failed");
ret = 1;
}
mem = 12;
if (atomic_compare_and_exchange_val_acq (&mem, 10, 15) != 12
|| mem != 12)
{
puts ("atomic_compare_and_exchange_val_acq test 2 failed");
ret = 1;
}
mem = -15;
if (atomic_compare_and_exchange_val_acq (&mem, -56, -15) != -15
|| mem != -56)
{
puts ("atomic_compare_and_exchange_val_acq test 3 failed");
ret = 1;
}
mem = -1;
if (atomic_compare_and_exchange_val_acq (&mem, 17, 0) != -1
|| mem != -1)
{
puts ("atomic_compare_and_exchange_val_acq test 4 failed");
ret = 1;
}
#endif
mem = 24;
if (atomic_compare_and_exchange_bool_acq (&mem, 35, 24)
|| mem != 35)
{
puts ("atomic_compare_and_exchange_bool_acq test 1 failed");
ret = 1;
}
mem = 12;
if (! atomic_compare_and_exchange_bool_acq (&mem, 10, 15)
|| mem != 12)
{
puts ("atomic_compare_and_exchange_bool_acq test 2 failed");
ret = 1;
}
mem = -15;
if (atomic_compare_and_exchange_bool_acq (&mem, -56, -15)
|| mem != -56)
{
puts ("atomic_compare_and_exchange_bool_acq test 3 failed");
ret = 1;
}
mem = -1;
if (! atomic_compare_and_exchange_bool_acq (&mem, 17, 0)
|| mem != -1)
{
puts ("atomic_compare_and_exchange_bool_acq test 4 failed");
ret = 1;
}
mem = 64;
if (atomic_exchange_acq (&mem, 31) != 64
|| mem != 31)
{
puts ("atomic_exchange_acq test failed");
ret = 1;
}
mem = 2;
if (atomic_exchange_and_add (&mem, 11) != 2
|| mem != 13)
{
puts ("atomic_exchange_and_add test failed");
ret = 1;
}
mem = 2;
if (atomic_exchange_and_add_acq (&mem, 11) != 2
|| mem != 13)
{
puts ("atomic_exchange_and_add test failed");
ret = 1;
}
mem = 2;
if (atomic_exchange_and_add_rel (&mem, 11) != 2
|| mem != 13)
{
puts ("atomic_exchange_and_add test failed");
ret = 1;
}
mem = -21;
atomic_add (&mem, 22);
if (mem != 1)
{
puts ("atomic_add test failed");
ret = 1;
}
mem = -1;
atomic_increment (&mem);
if (mem != 0)
{
puts ("atomic_increment test failed");
ret = 1;
}
mem = 2;
if (atomic_increment_val (&mem) != 3)
{
puts ("atomic_increment_val test failed");
ret = 1;
}
mem = 0;
if (atomic_increment_and_test (&mem)
|| mem != 1)
{
puts ("atomic_increment_and_test test 1 failed");
ret = 1;
}
mem = 35;
if (atomic_increment_and_test (&mem)
|| mem != 36)
{
puts ("atomic_increment_and_test test 2 failed");
ret = 1;
}
mem = -1;
if (! atomic_increment_and_test (&mem)
|| mem != 0)
{
puts ("atomic_increment_and_test test 3 failed");
ret = 1;
}
mem = 17;
atomic_decrement (&mem);
if (mem != 16)
{
puts ("atomic_decrement test failed");
ret = 1;
}
if (atomic_decrement_val (&mem) != 15)
{
puts ("atomic_decrement_val test failed");
ret = 1;
}
mem = 0;
if (atomic_decrement_and_test (&mem)
|| mem != -1)
{
puts ("atomic_decrement_and_test test 1 failed");
ret = 1;
}
mem = 15;
if (atomic_decrement_and_test (&mem)
|| mem != 14)
{
puts ("atomic_decrement_and_test test 2 failed");
ret = 1;
}
mem = 1;
if (! atomic_decrement_and_test (&mem)
|| mem != 0)
{
puts ("atomic_decrement_and_test test 3 failed");
ret = 1;
}
mem = 1;
if (atomic_decrement_if_positive (&mem) != 1
|| mem != 0)
{
puts ("atomic_decrement_if_positive test 1 failed");
ret = 1;
}
mem = 0;
if (atomic_decrement_if_positive (&mem) != 0
|| mem != 0)
{
puts ("atomic_decrement_if_positive test 2 failed");
ret = 1;
}
mem = -1;
if (atomic_decrement_if_positive (&mem) != -1
|| mem != -1)
{
puts ("atomic_decrement_if_positive test 3 failed");
ret = 1;
}
mem = -12;
if (! atomic_add_negative (&mem, 10)
|| mem != -2)
{
puts ("atomic_add_negative test 1 failed");
ret = 1;
}
mem = 0;
if (atomic_add_negative (&mem, 100)
|| mem != 100)
{
puts ("atomic_add_negative test 2 failed");
ret = 1;
}
mem = 15;
if (atomic_add_negative (&mem, -10)
|| mem != 5)
{
puts ("atomic_add_negative test 3 failed");
ret = 1;
}
mem = -12;
if (atomic_add_negative (&mem, 14)
|| mem != 2)
{
puts ("atomic_add_negative test 4 failed");
ret = 1;
}
mem = 0;
if (! atomic_add_negative (&mem, -1)
|| mem != -1)
{
puts ("atomic_add_negative test 5 failed");
ret = 1;
}
mem = -31;
if (atomic_add_negative (&mem, 31)
|| mem != 0)
{
puts ("atomic_add_negative test 6 failed");
ret = 1;
}
mem = -34;
if (atomic_add_zero (&mem, 31)
|| mem != -3)
{
puts ("atomic_add_zero test 1 failed");
ret = 1;
}
mem = -36;
if (! atomic_add_zero (&mem, 36)
|| mem != 0)
{
puts ("atomic_add_zero test 2 failed");
ret = 1;
}
mem = 113;
if (atomic_add_zero (&mem, -13)
|| mem != 100)
{
puts ("atomic_add_zero test 3 failed");
ret = 1;
}
mem = -18;
if (atomic_add_zero (&mem, 20)
|| mem != 2)
{
puts ("atomic_add_zero test 4 failed");
ret = 1;
}
mem = 10;
if (atomic_add_zero (&mem, -20)
|| mem != -10)
{
puts ("atomic_add_zero test 5 failed");
ret = 1;
}
mem = 10;
if (! atomic_add_zero (&mem, -10)
|| mem != 0)
{
puts ("atomic_add_zero test 6 failed");
ret = 1;
}
mem = 0;
atomic_bit_set (&mem, 1);
if (mem != 2)
{
puts ("atomic_bit_set test 1 failed");
ret = 1;
}
mem = 8;
atomic_bit_set (&mem, 3);
if (mem != 8)
{
puts ("atomic_bit_set test 2 failed");
ret = 1;
}
#ifdef TEST_ATOMIC64
mem = 16;
atomic_bit_set (&mem, 35);
if (mem != 0x800000010LL)
{
puts ("atomic_bit_set test 3 failed");
ret = 1;
}
#endif
mem = 0;
if (atomic_bit_test_set (&mem, 1)
|| mem != 2)
{
puts ("atomic_bit_test_set test 1 failed");
ret = 1;
}
mem = 8;
if (! atomic_bit_test_set (&mem, 3)
|| mem != 8)
{
puts ("atomic_bit_test_set test 2 failed");
ret = 1;
}
#ifdef TEST_ATOMIC64
mem = 16;
if (atomic_bit_test_set (&mem, 35)
|| mem != 0x800000010LL)
{
puts ("atomic_bit_test_set test 3 failed");
ret = 1;
}
mem = 0x100000000LL;
if (! atomic_bit_test_set (&mem, 32)
|| mem != 0x100000000LL)
{
puts ("atomic_bit_test_set test 4 failed");
ret = 1;
}
#endif
#ifdef catomic_compare_and_exchange_val_acq
mem = 24;
if (catomic_compare_and_exchange_val_acq (&mem, 35, 24) != 24
|| mem != 35)
{
puts ("catomic_compare_and_exchange_val_acq test 1 failed");
ret = 1;
}
mem = 12;
if (catomic_compare_and_exchange_val_acq (&mem, 10, 15) != 12
|| mem != 12)
{
puts ("catomic_compare_and_exchange_val_acq test 2 failed");
ret = 1;
}
mem = -15;
if (catomic_compare_and_exchange_val_acq (&mem, -56, -15) != -15
|| mem != -56)
{
puts ("catomic_compare_and_exchange_val_acq test 3 failed");
ret = 1;
}
mem = -1;
if (catomic_compare_and_exchange_val_acq (&mem, 17, 0) != -1
|| mem != -1)
{
puts ("catomic_compare_and_exchange_val_acq test 4 failed");
ret = 1;
}
#endif
mem = 24;
if (catomic_compare_and_exchange_bool_acq (&mem, 35, 24)
|| mem != 35)
{
puts ("catomic_compare_and_exchange_bool_acq test 1 failed");
ret = 1;
}
mem = 12;
if (! catomic_compare_and_exchange_bool_acq (&mem, 10, 15)
|| mem != 12)
{
puts ("catomic_compare_and_exchange_bool_acq test 2 failed");
ret = 1;
}
mem = -15;
if (catomic_compare_and_exchange_bool_acq (&mem, -56, -15)
|| mem != -56)
{
puts ("catomic_compare_and_exchange_bool_acq test 3 failed");
ret = 1;
}
mem = -1;
if (! catomic_compare_and_exchange_bool_acq (&mem, 17, 0)
|| mem != -1)
{
puts ("catomic_compare_and_exchange_bool_acq test 4 failed");
ret = 1;
}
mem = 2;
if (catomic_exchange_and_add (&mem, 11) != 2
|| mem != 13)
{
puts ("catomic_exchange_and_add test failed");
ret = 1;
}
mem = -21;
catomic_add (&mem, 22);
if (mem != 1)
{
puts ("catomic_add test failed");
ret = 1;
}
mem = -1;
catomic_increment (&mem);
if (mem != 0)
{
puts ("catomic_increment test failed");
ret = 1;
}
mem = 2;
if (catomic_increment_val (&mem) != 3)
{
puts ("catomic_increment_val test failed");
ret = 1;
}
mem = 17;
catomic_decrement (&mem);
if (mem != 16)
{
puts ("catomic_decrement test failed");
ret = 1;
}
if (catomic_decrement_val (&mem) != 15)
{
puts ("catomic_decrement_val test failed");
ret = 1;
}
/* Tests for C11-like atomics. */
mem = 11;
if (atomic_load_relaxed (&mem) != 11 || atomic_load_acquire (&mem) != 11)
{
puts ("atomic_load_{relaxed,acquire} test failed");
ret = 1;
}
atomic_store_relaxed (&mem, 12);
if (mem != 12)
{
puts ("atomic_store_relaxed test failed");
ret = 1;
}
atomic_store_release (&mem, 13);
if (mem != 13)
{
puts ("atomic_store_release test failed");
ret = 1;
}
mem = 14;
expected = 14;
if (!atomic_compare_exchange_weak_relaxed (&mem, &expected, 25)
|| mem != 25 || expected != 14)
{
puts ("atomic_compare_exchange_weak_relaxed test 1 failed");
ret = 1;
}
if (atomic_compare_exchange_weak_relaxed (&mem, &expected, 14)
|| mem != 25 || expected != 25)
{
puts ("atomic_compare_exchange_weak_relaxed test 2 failed");
ret = 1;
}
mem = 14;
expected = 14;
if (!atomic_compare_exchange_weak_acquire (&mem, &expected, 25)
|| mem != 25 || expected != 14)
{
puts ("atomic_compare_exchange_weak_acquire test 1 failed");
ret = 1;
}
if (atomic_compare_exchange_weak_acquire (&mem, &expected, 14)
|| mem != 25 || expected != 25)
{
puts ("atomic_compare_exchange_weak_acquire test 2 failed");
ret = 1;
}
mem = 14;
expected = 14;
if (!atomic_compare_exchange_weak_release (&mem, &expected, 25)
|| mem != 25 || expected != 14)
{
puts ("atomic_compare_exchange_weak_release test 1 failed");
ret = 1;
}
if (atomic_compare_exchange_weak_release (&mem, &expected, 14)
|| mem != 25 || expected != 25)
{
puts ("atomic_compare_exchange_weak_release test 2 failed");
ret = 1;
}
mem = 23;
if (atomic_exchange_acquire (&mem, 42) != 23 || mem != 42)
{
puts ("atomic_exchange_acquire test failed");
ret = 1;
}
mem = 23;
if (atomic_exchange_release (&mem, 42) != 23 || mem != 42)
{
puts ("atomic_exchange_release test failed");
ret = 1;
}
mem = 23;
if (atomic_fetch_add_relaxed (&mem, 1) != 23 || mem != 24)
{
puts ("atomic_fetch_add_relaxed test failed");
ret = 1;
}
mem = 23;
if (atomic_fetch_add_acquire (&mem, 1) != 23 || mem != 24)
{
puts ("atomic_fetch_add_acquire test failed");
ret = 1;
}
mem = 23;
if (atomic_fetch_add_release (&mem, 1) != 23 || mem != 24)
{
puts ("atomic_fetch_add_release test failed");
ret = 1;
}
mem = 23;
if (atomic_fetch_add_acq_rel (&mem, 1) != 23 || mem != 24)
{
puts ("atomic_fetch_add_acq_rel test failed");
ret = 1;
}
mem = 3;
if (atomic_fetch_and_acquire (&mem, 2) != 3 || mem != 2)
{
puts ("atomic_fetch_and_acquire test failed");
ret = 1;
}
mem = 4;
if (atomic_fetch_or_relaxed (&mem, 2) != 4 || mem != 6)
{
puts ("atomic_fetch_or_relaxed test failed");
ret = 1;
}
mem = 4;
if (atomic_fetch_or_acquire (&mem, 2) != 4 || mem != 6)
{
puts ("atomic_fetch_or_acquire test failed");
ret = 1;
}
/* This is a single-threaded test, so we can't test the effects of the
fences. */
atomic_thread_fence_acquire ();
atomic_thread_fence_release ();
atomic_thread_fence_seq_cst ();
return ret;
}
__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
{
while (1)
{
int val, newval;
/* We need acquire memory order for this load because if the value
signals that initialization has finished, we need to see any
data modifications done during initialization. */
val = atomic_load_acquire (once_control);
do
{
/* Check if the initialization has already been done. */
if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
return 0;
/* We try to set the state to in-progress and having the current
fork generation. We don't need atomic accesses for the fork
generation because it's immutable in a particular process, and
forked child processes start with a single thread that modified
the generation. */
newval = __fork_generation | __PTHREAD_ONCE_INPROGRESS;
/* We need acquire memory order here for the same reason as for the
load from once_control above. */
}
while (__glibc_unlikely (!atomic_compare_exchange_weak_acquire (
once_control, &val, newval)));
/* Check if another thread already runs the initializer. */
if ((val & __PTHREAD_ONCE_INPROGRESS) != 0)
{
/* Check whether the initializer execution was interrupted by a
fork. We know that for both values, __PTHREAD_ONCE_INPROGRESS
is set and __PTHREAD_ONCE_DONE is not. */
if (val == newval)
{
/* Same generation, some other thread was faster. Wait and
retry. */
futex_wait_simple ((unsigned int *) once_control,
(unsigned int) newval, FUTEX_PRIVATE);
continue;
}
}
/* This thread is the first here. Do the initialization.
Register a cleanup handler so that in case the thread gets
interrupted the initialization can be restarted. */
pthread_cleanup_push (clear_once_control, once_control);
init_routine ();
pthread_cleanup_pop (0);
/* Mark *once_control as having finished the initialization. We need
release memory order here because we need to synchronize with other
threads that want to use the initialized data. */
atomic_store_release (once_control, __PTHREAD_ONCE_DONE);
/* Wake up all other threads. */
futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE);
break;
}
return 0;
}
