inline uint64_t inc_and_fetch(volatile uint64_t *ptr)
{
#if ELEVELDB_IS_SOLARIS
return atomic_inc_64_nv(ptr);
#else
return __sync_add_and_fetch(ptr, 1);
#endif
}
/* Logger GID's can be used by watchers to put logs back into strict order
*/
static uint64_t logger_get_gid(void) {
static uint64_t logger_gid = 0;
#ifdef HAVE_GCC_64ATOMICS
return __sync_add_and_fetch(&logger_gid, 1);
#elif defined(__sun)
return atomic_inc_64_nv(&logger_gid);
#else
mutex_lock(&logger_atomics_mutex);
uint64_t res = ++logger_gid;
mutex_unlock(&logger_atomics_mutex);
return res;
#endif
}
/* Closed funcitons from other facilities */
void
zio_best_effort_dedup(zio_t *zio)
{
spa_t *spa = zio->io_spa;
zio_prop_t *zp = &zio->io_prop;
uint64_t val;
if (spa->spa_dedup_best_effort == 0)
return;
val = atomic_inc_64_nv(&spa->spa_dedup_rotor);
if ((val % 100) >= spa->spa_dedup_percentage)
zp->zp_dedup = 0;
}
/**
* Set new data on a thread-safe global variable
*
* @param [in] var Pointer to thread-safe global variable
* @param [in] cfdata New value for the thread-safe global variable
* @param [out] new_version New version number
*
* @return 0 on success, or a system error such as ENOMEM.
*/
int
pthread_var_set_np(pthread_var_np_t vp, void *cfdata,
uint64_t *new_version)
{
int err;
size_t i;
struct var *v;
struct vwrapper *old_wrapper = NULL;
struct vwrapper *wrapper;
struct vwrapper *tmp;
uint64_t vers;
uint64_t tmp_version;
uint64_t nref;
if (cfdata == NULL)
return EINVAL;
if (new_version == NULL)
new_version = &vers;
*new_version = 0;
/* Build a wrapper for the new value */
if ((wrapper = calloc(1, sizeof(*wrapper))) == NULL)
return errno;
/*
* The var itself holds a reference to the current value, thus its
* nref starts at 1, but that is made so further below.
*/
wrapper->dtor = vp->dtor;
wrapper->nref = 0;
wrapper->ptr = cfdata;
if ((err = pthread_mutex_lock(&vp->write_lock)) != 0) {
free(wrapper);
return err;
}
/* vp->next_version is stable because we hold the write_lock */
*new_version = wrapper->version = atomic_read_64(&vp->next_version);
/* Grab the next slot */
v = vp->vars[(*new_version + 1) & 0x1].other;
old_wrapper = atomic_read_ptr((volatile void **)&v->wrapper);
if (*new_version == 0) {
/* This is the first write; set wrapper on both slots */
for (i = 0; i < sizeof(vp->vars)/sizeof(vp->vars[0]); i++) {
v = &vp->vars[i];
nref = atomic_inc_32_nv(&wrapper->nref);
v->version = 0;
tmp = atomic_cas_ptr((volatile void **)&v->wrapper,
old_wrapper, wrapper);
assert(tmp == old_wrapper && tmp == NULL);
}
assert(nref > 1);
tmp_version = atomic_inc_64_nv(&vp->next_version);
assert(tmp_version == 1);
/* Signal waiters */
(void) pthread_mutex_lock(&vp->waiter_lock);
(void) pthread_cond_signal(&vp->waiter_cv); /* no thundering herd */
(void) pthread_mutex_unlock(&vp->waiter_lock);
return pthread_mutex_unlock(&vp->write_lock);
}
nref = atomic_inc_32_nv(&wrapper->nref);
assert(nref == 1);
assert(old_wrapper != NULL && old_wrapper->nref > 0);
/* Wait until that slot is quiescent before mutating it */
if ((err = pthread_mutex_lock(&vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
while (atomic_read_32(&v->nreaders) > 0) {
/*
* We have a separate lock for writing vs. waiting so that no
* other writer can steal our march. All writers will enter,
* all writers will finish. We got here by winning the race for
* the writer lock, so we'll hold onto it, and thus avoid having
* to restart here.
*/
if ((err = pthread_cond_wait(&vp->cv, &vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->cv_lock);
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
}
if ((err = pthread_mutex_unlock(&vp->cv_lock)) != 0) {
(void) pthread_mutex_unlock(&vp->write_lock);
free(wrapper);
return err;
}
/* Update that now quiescent slot; these are the release operations */
tmp = atomic_cas_ptr((volatile void **)&v->wrapper, old_wrapper, wrapper);
assert(tmp == old_wrapper);
v->version = *new_version;
tmp_version = atomic_inc_64_nv(&vp->next_version);
assert(tmp_version == *new_version + 1);
assert(v->version > v->other->version);
/* Release the old cf */
assert(old_wrapper != NULL && old_wrapper->nref > 0);
wrapper_free(old_wrapper);
/* Done */
return pthread_mutex_unlock(&vp->write_lock);
}
/*
* Distribute writes across special and normal vdevs in
* spa_special_to_normal-1:1 proportion
*/
static boolean_t
spa_refine_data_placement(spa_t *spa)
{
uint64_t val = atomic_inc_64_nv(&spa->spa_special_stat_rotor);
return (val % spa->spa_special_to_normal_ratio);
}
template<typename T> static T increase_nv(T *ptr) { return atomic_inc_64_nv(ptr); }