/* We only want to initialize __sched_fifo_min_prio and __sched_fifo_max_prio once. The standard solution would be similar to pthread_once, but then readers would need to use an acquire fence. In this specific case, initialization is comprised of just idempotent writes to two variables that have an initial value of -1. Therefore, we can treat each variable as a separate, at-least-once initialized value. This enables using just relaxed MO loads and stores, but requires that consumers check for initialization of each value that is to be used; see __pthread_tpp_change_priority for an example. */ void __init_sched_fifo_prio (void) { atomic_store_relaxed (&__sched_fifo_max_prio, sched_get_priority_max (SCHED_FIFO)); atomic_store_relaxed (&__sched_fifo_min_prio, sched_get_priority_min (SCHED_FIFO)); }
void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms); switch (flags()->malloc_align_right) { case 0: break; case 1: right_align_mode = kRightAlignSometimes; right_align_8 = false; break; case 2: right_align_mode = kRightAlignAlways; right_align_8 = false; break; case 8: right_align_mode = kRightAlignSometimes; right_align_8 = true; break; case 9: right_align_mode = kRightAlignAlways; right_align_8 = true; break; default: Report("ERROR: unsupported value of malloc_align_right flag: %d\n", flags()->malloc_align_right); Die(); } for (uptr i = 0; i < kShadowAlignment; i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); }
int pthread_spin_init (pthread_spinlock_t *lock, int pshared) { /* Relaxed MO is fine because this is an initializing store. */ atomic_store_relaxed (lock, 0); return 0; }
do_end (void) { atomic_store_relaxed (&running, 0); int ret = pthread_join (th, NULL); if (ret != 0) { printf ("pthread_join: %s(%d)\n", strerror (ret), ret); exit (1); } printf ("Thread joined\n"); }
static void clear_once_control (void *arg) { pthread_once_t *once_control = (pthread_once_t *) arg; /* Reset to the uninitialized state here. We don't need a stronger memory order because we do not need to make any other of our writes visible to other threads that see this value: This function will be called if we get interrupted (see __pthread_once), so all we need to relay to other threads is the state being reset again. */ atomic_store_relaxed (once_control, 0); futex_wake ((unsigned int *) once_control, INT_MAX, FUTEX_PRIVATE); }
int __lll_unlock_elision (int *lock, short *adapt_count, int pshared) { /* When the lock was free we're in a transaction. */ if (*lock == 0) __libc_tend (0); else { /* Update adapt_count in the critical section to prevent a write-after-destroy error as mentioned in BZ 20822. The following update of adapt_count has to be contained within the critical region of the fall-back lock in order to not violate the mutex destruction requirements. */ short __tmp = atomic_load_relaxed (adapt_count); if (__tmp > 0) atomic_store_relaxed (adapt_count, __tmp - 1); lll_unlock ((*lock), pshared); } return 0; }
/* Test various atomic.h macros. */ static int do_test (void) { atomic_t mem, expected; int ret = 0; #ifdef atomic_compare_and_exchange_val_acq mem = 24; if (atomic_compare_and_exchange_val_acq (&mem, 35, 24) != 24 || mem != 35) { puts ("atomic_compare_and_exchange_val_acq test 1 failed"); ret = 1; } mem = 12; if (atomic_compare_and_exchange_val_acq (&mem, 10, 15) != 12 || mem != 12) { puts ("atomic_compare_and_exchange_val_acq test 2 failed"); ret = 1; } mem = -15; if (atomic_compare_and_exchange_val_acq (&mem, -56, -15) != -15 || mem != -56) { puts ("atomic_compare_and_exchange_val_acq test 3 failed"); ret = 1; } mem = -1; if (atomic_compare_and_exchange_val_acq (&mem, 17, 0) != -1 || mem != -1) { puts ("atomic_compare_and_exchange_val_acq test 4 failed"); ret = 1; } #endif mem = 24; if (atomic_compare_and_exchange_bool_acq (&mem, 35, 24) || mem != 35) { puts ("atomic_compare_and_exchange_bool_acq test 1 failed"); ret = 1; } mem = 12; if (! atomic_compare_and_exchange_bool_acq (&mem, 10, 15) || mem != 12) { puts ("atomic_compare_and_exchange_bool_acq test 2 failed"); ret = 1; } mem = -15; if (atomic_compare_and_exchange_bool_acq (&mem, -56, -15) || mem != -56) { puts ("atomic_compare_and_exchange_bool_acq test 3 failed"); ret = 1; } mem = -1; if (! atomic_compare_and_exchange_bool_acq (&mem, 17, 0) || mem != -1) { puts ("atomic_compare_and_exchange_bool_acq test 4 failed"); ret = 1; } mem = 64; if (atomic_exchange_acq (&mem, 31) != 64 || mem != 31) { puts ("atomic_exchange_acq test failed"); ret = 1; } mem = 2; if (atomic_exchange_and_add (&mem, 11) != 2 || mem != 13) { puts ("atomic_exchange_and_add test failed"); ret = 1; } mem = 2; if (atomic_exchange_and_add_acq (&mem, 11) != 2 || mem != 13) { puts ("atomic_exchange_and_add test failed"); ret = 1; } mem = 2; if (atomic_exchange_and_add_rel (&mem, 11) != 2 || mem != 13) { puts ("atomic_exchange_and_add test failed"); ret = 1; } mem = -21; atomic_add (&mem, 22); if (mem != 1) { puts ("atomic_add test failed"); ret = 1; } mem = -1; atomic_increment (&mem); if (mem != 0) { puts ("atomic_increment test failed"); ret = 1; } mem = 2; if (atomic_increment_val (&mem) != 3) { puts ("atomic_increment_val test failed"); ret = 1; } mem = 0; if (atomic_increment_and_test (&mem) || mem != 1) { puts ("atomic_increment_and_test test 1 failed"); ret = 1; } mem = 35; if (atomic_increment_and_test (&mem) || mem != 36) { puts ("atomic_increment_and_test test 2 failed"); ret = 1; } mem = -1; if (! atomic_increment_and_test (&mem) || mem != 0) { puts ("atomic_increment_and_test test 3 failed"); ret = 1; } mem = 17; atomic_decrement (&mem); if (mem != 16) { puts ("atomic_decrement test failed"); ret = 1; } if (atomic_decrement_val (&mem) != 15) { puts ("atomic_decrement_val test failed"); ret = 1; } mem = 0; if (atomic_decrement_and_test (&mem) || mem != -1) { puts ("atomic_decrement_and_test test 1 failed"); ret = 1; } mem = 15; if (atomic_decrement_and_test (&mem) || mem != 14) { puts ("atomic_decrement_and_test test 2 failed"); ret = 1; } mem = 1; if (! atomic_decrement_and_test (&mem) || mem != 0) { puts ("atomic_decrement_and_test test 3 failed"); ret = 1; } mem = 1; if (atomic_decrement_if_positive (&mem) != 1 || mem != 0) { puts ("atomic_decrement_if_positive test 1 failed"); ret = 1; } mem = 0; if (atomic_decrement_if_positive (&mem) != 0 || mem != 0) { puts ("atomic_decrement_if_positive test 2 failed"); ret = 1; } mem = -1; if (atomic_decrement_if_positive (&mem) != -1 || mem != -1) { puts ("atomic_decrement_if_positive test 3 failed"); ret = 1; } mem = -12; if (! atomic_add_negative (&mem, 10) || mem != -2) { puts ("atomic_add_negative test 1 failed"); ret = 1; } mem = 0; if (atomic_add_negative (&mem, 100) || mem != 100) { puts ("atomic_add_negative test 2 failed"); ret = 1; } mem = 15; if (atomic_add_negative (&mem, -10) || mem != 5) { puts ("atomic_add_negative test 3 failed"); ret = 1; } mem = -12; if (atomic_add_negative (&mem, 14) || mem != 2) { puts ("atomic_add_negative test 4 failed"); ret = 1; } mem = 0; if (! atomic_add_negative (&mem, -1) || mem != -1) { puts ("atomic_add_negative test 5 failed"); ret = 1; } mem = -31; if (atomic_add_negative (&mem, 31) || mem != 0) { puts ("atomic_add_negative test 6 failed"); ret = 1; } mem = -34; if (atomic_add_zero (&mem, 31) || mem != -3) { puts ("atomic_add_zero test 1 failed"); ret = 1; } mem = -36; if (! atomic_add_zero (&mem, 36) || mem != 0) { puts ("atomic_add_zero test 2 failed"); ret = 1; } mem = 113; if (atomic_add_zero (&mem, -13) || mem != 100) { puts ("atomic_add_zero test 3 failed"); ret = 1; } mem = -18; if (atomic_add_zero (&mem, 20) || mem != 2) { puts ("atomic_add_zero test 4 failed"); ret = 1; } mem = 10; if (atomic_add_zero (&mem, -20) || mem != -10) { puts ("atomic_add_zero test 5 failed"); ret = 1; } mem = 10; if (! atomic_add_zero (&mem, -10) || mem != 0) { puts ("atomic_add_zero test 6 failed"); ret = 1; } mem = 0; atomic_bit_set (&mem, 1); if (mem != 2) { puts ("atomic_bit_set test 1 failed"); ret = 1; } mem = 8; atomic_bit_set (&mem, 3); if (mem != 8) { puts ("atomic_bit_set test 2 failed"); ret = 1; } #ifdef TEST_ATOMIC64 mem = 16; atomic_bit_set (&mem, 35); if (mem != 0x800000010LL) { puts ("atomic_bit_set test 3 failed"); ret = 1; } #endif mem = 0; if (atomic_bit_test_set (&mem, 1) || mem != 2) { puts ("atomic_bit_test_set test 1 failed"); ret = 1; } mem = 8; if (! atomic_bit_test_set (&mem, 3) || mem != 8) { puts ("atomic_bit_test_set test 2 failed"); ret = 1; } #ifdef TEST_ATOMIC64 mem = 16; if (atomic_bit_test_set (&mem, 35) || mem != 0x800000010LL) { puts ("atomic_bit_test_set test 3 failed"); ret = 1; } mem = 0x100000000LL; if (! atomic_bit_test_set (&mem, 32) || mem != 0x100000000LL) { puts ("atomic_bit_test_set test 4 failed"); ret = 1; } #endif #ifdef catomic_compare_and_exchange_val_acq mem = 24; if (catomic_compare_and_exchange_val_acq (&mem, 35, 24) != 24 || mem != 35) { puts ("catomic_compare_and_exchange_val_acq test 1 failed"); ret = 1; } mem = 12; if (catomic_compare_and_exchange_val_acq (&mem, 10, 15) != 12 || mem != 12) { puts ("catomic_compare_and_exchange_val_acq test 2 failed"); ret = 1; } mem = -15; if (catomic_compare_and_exchange_val_acq (&mem, -56, -15) != -15 || mem != -56) { puts ("catomic_compare_and_exchange_val_acq test 3 failed"); ret = 1; } mem = -1; if (catomic_compare_and_exchange_val_acq (&mem, 17, 0) != -1 || mem != -1) { puts ("catomic_compare_and_exchange_val_acq test 4 failed"); ret = 1; } #endif mem = 24; if (catomic_compare_and_exchange_bool_acq (&mem, 35, 24) || mem != 35) { puts ("catomic_compare_and_exchange_bool_acq test 1 failed"); ret = 1; } mem = 12; if (! catomic_compare_and_exchange_bool_acq (&mem, 10, 15) || mem != 12) { puts ("catomic_compare_and_exchange_bool_acq test 2 failed"); ret = 1; } mem = -15; if (catomic_compare_and_exchange_bool_acq (&mem, -56, -15) || mem != -56) { puts ("catomic_compare_and_exchange_bool_acq test 3 failed"); ret = 1; } mem = -1; if (! catomic_compare_and_exchange_bool_acq (&mem, 17, 0) || mem != -1) { puts ("catomic_compare_and_exchange_bool_acq test 4 failed"); ret = 1; } mem = 2; if (catomic_exchange_and_add (&mem, 11) != 2 || mem != 13) { puts ("catomic_exchange_and_add test failed"); ret = 1; } mem = -21; catomic_add (&mem, 22); if (mem != 1) { puts ("catomic_add test failed"); ret = 1; } mem = -1; catomic_increment (&mem); if (mem != 0) { puts ("catomic_increment test failed"); ret = 1; } mem = 2; if (catomic_increment_val (&mem) != 3) { puts ("catomic_increment_val test failed"); ret = 1; } mem = 17; catomic_decrement (&mem); if (mem != 16) { puts ("catomic_decrement test failed"); ret = 1; } if (catomic_decrement_val (&mem) != 15) { puts ("catomic_decrement_val test failed"); ret = 1; } /* Tests for C11-like atomics. */ mem = 11; if (atomic_load_relaxed (&mem) != 11 || atomic_load_acquire (&mem) != 11) { puts ("atomic_load_{relaxed,acquire} test failed"); ret = 1; } atomic_store_relaxed (&mem, 12); if (mem != 12) { puts ("atomic_store_relaxed test failed"); ret = 1; } atomic_store_release (&mem, 13); if (mem != 13) { puts ("atomic_store_release test failed"); ret = 1; } mem = 14; expected = 14; if (!atomic_compare_exchange_weak_relaxed (&mem, &expected, 25) || mem != 25 || expected != 14) { puts ("atomic_compare_exchange_weak_relaxed test 1 failed"); ret = 1; } if (atomic_compare_exchange_weak_relaxed (&mem, &expected, 14) || mem != 25 || expected != 25) { puts ("atomic_compare_exchange_weak_relaxed test 2 failed"); ret = 1; } mem = 14; expected = 14; if (!atomic_compare_exchange_weak_acquire (&mem, &expected, 25) || mem != 25 || expected != 14) { puts ("atomic_compare_exchange_weak_acquire test 1 failed"); ret = 1; } if (atomic_compare_exchange_weak_acquire (&mem, &expected, 14) || mem != 25 || expected != 25) { puts ("atomic_compare_exchange_weak_acquire test 2 failed"); ret = 1; } mem = 14; expected = 14; if (!atomic_compare_exchange_weak_release (&mem, &expected, 25) || mem != 25 || expected != 14) { puts ("atomic_compare_exchange_weak_release test 1 failed"); ret = 1; } if (atomic_compare_exchange_weak_release (&mem, &expected, 14) || mem != 25 || expected != 25) { puts ("atomic_compare_exchange_weak_release test 2 failed"); ret = 1; } mem = 23; if (atomic_exchange_acquire (&mem, 42) != 23 || mem != 42) { puts ("atomic_exchange_acquire test failed"); ret = 1; } mem = 23; if (atomic_exchange_release (&mem, 42) != 23 || mem != 42) { puts ("atomic_exchange_release test failed"); ret = 1; } mem = 23; if (atomic_fetch_add_relaxed (&mem, 1) != 23 || mem != 24) { puts ("atomic_fetch_add_relaxed test failed"); ret = 1; } mem = 23; if (atomic_fetch_add_acquire (&mem, 1) != 23 || mem != 24) { puts ("atomic_fetch_add_acquire test failed"); ret = 1; } mem = 23; if (atomic_fetch_add_release (&mem, 1) != 23 || mem != 24) { puts ("atomic_fetch_add_release test failed"); ret = 1; } mem = 23; if (atomic_fetch_add_acq_rel (&mem, 1) != 23 || mem != 24) { puts ("atomic_fetch_add_acq_rel test failed"); ret = 1; } mem = 3; if (atomic_fetch_and_acquire (&mem, 2) != 3 || mem != 2) { puts ("atomic_fetch_and_acquire test failed"); ret = 1; } mem = 4; if (atomic_fetch_or_relaxed (&mem, 2) != 4 || mem != 6) { puts ("atomic_fetch_or_relaxed test failed"); ret = 1; } mem = 4; if (atomic_fetch_or_acquire (&mem, 2) != 4 || mem != 6) { puts ("atomic_fetch_or_acquire test failed"); ret = 1; } /* This is a single-threaded test, so we can't test the effects of the fences. */ atomic_thread_fence_acquire (); atomic_thread_fence_release (); atomic_thread_fence_seq_cst (); return ret; }
// Packs and stores the header, computing the checksum in the process. static INLINE void storeHeader(void *Ptr, UnpackedHeader *NewUnpackedHeader) { NewUnpackedHeader->Checksum = computeChecksum(Ptr, NewUnpackedHeader); PackedHeader NewPackedHeader = bit_cast<PackedHeader>(*NewUnpackedHeader); atomic_store_relaxed(getAtomicHeader(Ptr), NewPackedHeader); }