inline uint64_t inc_and_fetch(volatile uint64_t *ptr) { #if ELEVELDB_IS_SOLARIS return atomic_inc_64_nv(ptr); #else return __sync_add_and_fetch(ptr, 1); #endif }
/* Logger GID's can be used by watchers to put logs back into strict order */ static uint64_t logger_get_gid(void) { static uint64_t logger_gid = 0; #ifdef HAVE_GCC_64ATOMICS return __sync_add_and_fetch(&logger_gid, 1); #elif defined(__sun) return atomic_inc_64_nv(&logger_gid); #else mutex_lock(&logger_atomics_mutex); uint64_t res = ++logger_gid; mutex_unlock(&logger_atomics_mutex); return res; #endif }
/* Closed funcitons from other facilities */ void zio_best_effort_dedup(zio_t *zio) { spa_t *spa = zio->io_spa; zio_prop_t *zp = &zio->io_prop; uint64_t val; if (spa->spa_dedup_best_effort == 0) return; val = atomic_inc_64_nv(&spa->spa_dedup_rotor); if ((val % 100) >= spa->spa_dedup_percentage) zp->zp_dedup = 0; }
/** * Set new data on a thread-safe global variable * * @param [in] var Pointer to thread-safe global variable * @param [in] cfdata New value for the thread-safe global variable * @param [out] new_version New version number * * @return 0 on success, or a system error such as ENOMEM. */ int pthread_var_set_np(pthread_var_np_t vp, void *cfdata, uint64_t *new_version) { int err; size_t i; struct var *v; struct vwrapper *old_wrapper = NULL; struct vwrapper *wrapper; struct vwrapper *tmp; uint64_t vers; uint64_t tmp_version; uint64_t nref; if (cfdata == NULL) return EINVAL; if (new_version == NULL) new_version = &vers; *new_version = 0; /* Build a wrapper for the new value */ if ((wrapper = calloc(1, sizeof(*wrapper))) == NULL) return errno; /* * The var itself holds a reference to the current value, thus its * nref starts at 1, but that is made so further below. */ wrapper->dtor = vp->dtor; wrapper->nref = 0; wrapper->ptr = cfdata; if ((err = pthread_mutex_lock(&vp->write_lock)) != 0) { free(wrapper); return err; } /* vp->next_version is stable because we hold the write_lock */ *new_version = wrapper->version = atomic_read_64(&vp->next_version); /* Grab the next slot */ v = vp->vars[(*new_version + 1) & 0x1].other; old_wrapper = atomic_read_ptr((volatile void **)&v->wrapper); if (*new_version == 0) { /* This is the first write; set wrapper on both slots */ for (i = 0; i < sizeof(vp->vars)/sizeof(vp->vars[0]); i++) { v = &vp->vars[i]; nref = atomic_inc_32_nv(&wrapper->nref); v->version = 0; tmp = atomic_cas_ptr((volatile void **)&v->wrapper, old_wrapper, wrapper); assert(tmp == old_wrapper && tmp == NULL); } assert(nref > 1); tmp_version = atomic_inc_64_nv(&vp->next_version); assert(tmp_version == 1); /* Signal waiters */ (void) pthread_mutex_lock(&vp->waiter_lock); (void) pthread_cond_signal(&vp->waiter_cv); /* no thundering herd */ (void) pthread_mutex_unlock(&vp->waiter_lock); return pthread_mutex_unlock(&vp->write_lock); } nref = atomic_inc_32_nv(&wrapper->nref); assert(nref == 1); assert(old_wrapper != NULL && old_wrapper->nref > 0); /* Wait until that slot is quiescent before mutating it */ if ((err = pthread_mutex_lock(&vp->cv_lock)) != 0) { (void) pthread_mutex_unlock(&vp->write_lock); free(wrapper); return err; } while (atomic_read_32(&v->nreaders) > 0) { /* * We have a separate lock for writing vs. waiting so that no * other writer can steal our march. All writers will enter, * all writers will finish. We got here by winning the race for * the writer lock, so we'll hold onto it, and thus avoid having * to restart here. */ if ((err = pthread_cond_wait(&vp->cv, &vp->cv_lock)) != 0) { (void) pthread_mutex_unlock(&vp->cv_lock); (void) pthread_mutex_unlock(&vp->write_lock); free(wrapper); return err; } } if ((err = pthread_mutex_unlock(&vp->cv_lock)) != 0) { (void) pthread_mutex_unlock(&vp->write_lock); free(wrapper); return err; } /* Update that now quiescent slot; these are the release operations */ tmp = atomic_cas_ptr((volatile void **)&v->wrapper, old_wrapper, wrapper); assert(tmp == old_wrapper); v->version = *new_version; tmp_version = atomic_inc_64_nv(&vp->next_version); assert(tmp_version == *new_version + 1); assert(v->version > v->other->version); /* Release the old cf */ assert(old_wrapper != NULL && old_wrapper->nref > 0); wrapper_free(old_wrapper); /* Done */ return pthread_mutex_unlock(&vp->write_lock); }
/* * Distribute writes across special and normal vdevs in * spa_special_to_normal-1:1 proportion */ static boolean_t spa_refine_data_placement(spa_t *spa) { uint64_t val = atomic_inc_64_nv(&spa->spa_special_stat_rotor); return (val % spa->spa_special_to_normal_ratio); }
template<typename T> static T increase_nv(T *ptr) { return atomic_inc_64_nv(ptr); }