bool SIZE(libat_test_and_set) (UTYPE *mptr, int smodel) { UWORD wval, woldval, shift, *wptr, t; pre_barrier (smodel); if (N < WORDSIZE) { wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); shift = SIZE(INVERT_MASK); } else { wptr = (UWORD *)mptr; shift = 0; } wval = (UWORD)__GCC_ATOMIC_TEST_AND_SET_TRUEVAL << shift; woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); do { t = woldval | wval; } while (!atomic_compare_exchange_w (wptr, &woldval, t, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); post_barrier (smodel); return woldval != 0; }
UTYPE SIZE(libat_exchange) (UTYPE *mptr, UTYPE newval, int smodel) { UWORD mask, shift, woldval, wnewval, t, *wptr; pre_barrier (smodel); if (N < WORDSIZE) { wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); mask = SIZE(MASK) << shift; } else { wptr = (UWORD *)mptr; shift = 0; mask = -1; } wnewval = (UWORD)newval << shift; woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); do { t = (woldval & ~mask) | wnewval; } while (!atomic_compare_exchange_w (wptr, &woldval, t, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); post_barrier (smodel); return woldval >> shift; }
UTYPE SIZE(libat_load) (UTYPE *mptr, int smodel) { UWORD shift, t, *wptr; pre_barrier (smodel); wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); /* Exchange 0 with 0, placing the old value of *WPTR in T. */ t = 0; atomic_compare_exchange_w (wptr, &t, 0); post_barrier (smodel); return t >> shift; }
void SIZE(libat_store) (UTYPE *mptr, UTYPE newval, int smodel) { UWORD mask, shift, woldval, wnewval, t, *wptr; pre_barrier (smodel); wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); mask = SIZE(MASK) << shift; wnewval = (UWORD)newval << shift; woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); do { t = (woldval & ~mask) | wnewval; } while (!atomic_compare_exchange_w (wptr, &woldval, t)); post_barrier (smodel); }
bool SIZE(libat_compare_exchange) (UTYPE *mptr, UTYPE *eptr, UTYPE newval, int smodel, int fmodel UNUSED) { UWORD mask, shift, weval, woldval, wnewval, t, *wptr; bool ret = false; pre_barrier (smodel); if (N < WORDSIZE) { wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE); shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK); mask = SIZE(MASK) << shift; } else { wptr = (UWORD *)mptr; shift = 0; mask = -1; } weval = *eptr << shift; wnewval = (UWORD)newval << shift; woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); do { if ((woldval & mask) != weval) goto failure; t = (woldval & ~mask) | wnewval; } while (!atomic_compare_exchange_w (wptr, &woldval, t, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); ret = true; failure: *eptr = woldval >> shift; post_barrier (smodel); return ret; }
UTYPE SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel) { UWORD mask, shift, woldval, wopval, t, *wptr; pre_barrier (smodel); wptr = (UWORD *)mptr; shift = 0; mask = -1; wopval = (UWORD)opval << shift; woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED); do { t = (woldval & ~mask) | (OP(woldval, wopval) & mask); } while (!atomic_compare_exchange_w (wptr, &woldval, t, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); post_barrier (smodel); return t >> shift; }