/* * igb_free_rcb_lists - Free the receive control blocks of one ring. */ static void igb_free_rcb_lists(igb_rx_data_t *rx_data) { igb_t *igb; rx_control_block_t *rcb; uint32_t rcb_count; uint32_t ref_cnt; int i; igb = rx_data->rx_ring->igb; mutex_enter(&igb->rx_pending_lock); rcb = rx_data->rcb_area; rcb_count = rx_data->ring_size + rx_data->free_list_size; for (i = 0; i < rcb_count; i++, rcb++) { ASSERT(rcb != NULL); ref_cnt = atomic_dec_32_nv(&rcb->ref_cnt); if (ref_cnt == 0) { if (rcb->mp != NULL) { freemsg(rcb->mp); rcb->mp = NULL; } igb_free_dma_buffer(&rcb->rx_buf); } else { atomic_inc_32(&rx_data->rcb_pending); atomic_inc_32(&igb->rcb_pending); } } mutex_exit(&igb->rx_pending_lock); }
void dpfree(devplcy_t *dp) { ASSERT(dp->dp_ref != 0xdeadbeef && dp->dp_ref != 0); if (atomic_dec_32_nv(&dp->dp_ref) == 0) kmem_free(dp, sizeof (*dp)); }
void weak_release() // nothrow { if( atomic_dec_32_nv( &weak_count_ ) == 0 ) { destroy(); } }
void release() // nothrow { if( atomic_dec_32_nv( &use_count_ ) == 0 ) { dispose(); weak_release(); } }
inline uint32_t dec_and_fetch(volatile uint32_t *ptr) { #if ELEVELDB_IS_SOLARIS return atomic_dec_32_nv(ptr); #else return __sync_sub_and_fetch(ptr, 1); #endif }
/** * Destroy a thread-safe global variable * * It is the caller's responsibility to ensure that no thread is using * this var and that none will use it again. * * @param [in] var The thread-safe global variable to destroy */ void pthread_var_destroy_np(pthread_var_np_t vp) { if (vp == 0) return; if (atomic_dec_32_nv(&vp->slots_in_use) > 0) return; /* defer to last reader slot release via thread key dtor */ destroy_var(vp);/* we're the last, destroy now */ }
/* * Our version of vfs_rele() that stops at 1 instead of 0, and calls * freelfsnode() instead of kmem_free(). */ static void lfs_rele(struct lfsnode *lfs, struct loinfo *li) { vfs_t *vfsp = &lfs->lfs_vfs; ASSERT(MUTEX_HELD(&li->li_lfslock)); ASSERT(vfsp->vfs_count > 1); if (atomic_dec_32_nv(&vfsp->vfs_count) == 1) freelfsnode(lfs, li); }
void crklpd_rele(credklpd_t *crkpd) { if (atomic_dec_32_nv(&crkpd->crkl_ref) == 0) { if (crkpd->crkl_reg != NULL) klpd_rele(crkpd->crkl_reg); mutex_destroy(&crkpd->crkl_lock); kmem_free(crkpd, sizeof (*crkpd)); } }
static void wrapper_free(struct vwrapper *wrapper) { if (wrapper == NULL) return; if (atomic_dec_32_nv(&wrapper->nref) > 0) return; if (wrapper->dtor != NULL) wrapper->dtor(wrapper->ptr); free(wrapper); }
void klpd_rele(klpd_reg_t *p) { if (atomic_dec_32_nv(&p->klpd_ref) == 0) { if (p->klpd_refp != NULL) klpd_unlink(p); if (p->klpd_cred != NULL) crfree(p->klpd_cred); door_ki_rele(p->klpd_door); kmem_free(p, sizeof (*p)); } }
void zrl_remove(zrlock_t *zrl) { uint32_t n; n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount); ASSERT((int32_t)n >= 0); #ifdef ZFS_DEBUG if (zrl->zr_owner == curthread) { zrl->zr_owner = NULL; zrl->zr_caller = NULL; } #endif }
/* * prop_object_release_emergency * A direct free with prop_object_release failed. * Walk down the tree until a leaf is found and * free that. Do not recurse to avoid stack overflows. * * This is a slow edge condition, but necessary to * guarantee that an object can always be freed. */ static void prop_object_release_emergency(prop_object_t obj) { struct _prop_object *po; void (*unlock)(void); prop_object_t parent = NULL; uint32_t ocnt; for (;;) { po = obj; _PROP_ASSERT(obj); if (po->po_type->pot_lock != NULL) po->po_type->pot_lock(); /* Save pointerto unlock function */ unlock = po->po_type->pot_unlock; /* Dance a bit to make sure we always get the non-racy ocnt */ ocnt = atomic_dec_32_nv(&po->po_refcnt); ocnt++; _PROP_ASSERT(ocnt != 0); if (ocnt != 1) { if (unlock != NULL) unlock(); break; } _PROP_ASSERT(po->po_type); if ((po->po_type->pot_free)(NULL, &obj) == _PROP_OBJECT_FREE_DONE) { if (unlock != NULL) unlock(); break; } if (unlock != NULL) unlock(); parent = po; atomic_inc_32(&po->po_refcnt); } _PROP_ASSERT(parent); /* One object was just freed. */ po = parent; (*po->po_type->pot_emergency_free)(parent); }
/* * Free rxbuf. */ static void vmxnet3s_free_rxbuf(vmxnet3s_softc_t *dp, vmxnet3s_rxbuf_t *rxbuf) { vmxnet3s_free(&rxbuf->dma); kmem_free(rxbuf, sizeof (vmxnet3s_rxbuf_t)); #ifndef DEBUG atomic_dec_32(&dp->rxnumbufs); #else { uint32_t nv = atomic_dec_32_nv(&dp->rxnumbufs); ASSERT(nv != (uint32_t)-1); } #endif }
/* * prop_object_release -- * Decrement the reference count on an object. * * Free the object if we are releasing the final * reference. */ void prop_object_release(prop_object_t obj) { struct _prop_object *po; struct _prop_stack stack; void (*unlock)(void); int ret; uint32_t ocnt; _prop_stack_init(&stack); do { do { po = obj; _PROP_ASSERT(obj); if (po->po_type->pot_lock != NULL) po->po_type->pot_lock(); /* Save pointer to object unlock function */ unlock = po->po_type->pot_unlock; ocnt = atomic_dec_32_nv(&po->po_refcnt); ocnt++; _PROP_ASSERT(ocnt != 0); if (ocnt != 1) { ret = 0; if (unlock != NULL) unlock(); break; } ret = (po->po_type->pot_free)(&stack, &obj); if (unlock != NULL) unlock(); if (ret == _PROP_OBJECT_FREE_DONE) break; atomic_inc_32(&po->po_refcnt); } while (ret == _PROP_OBJECT_FREE_RECURSE); if (ret == _PROP_OBJECT_FREE_FAILED) prop_object_release_emergency(obj); } while (_prop_stack_pop(&stack, &obj, NULL, NULL, NULL)); }
int do_umount(vfs_t *vfs, boolean_t force) { VFS_SYNC(vfs, 0, kcred); int ret = VFS_UNMOUNT(vfs, force ? MS_FORCE : 0, kcred); if(ret != 0) return ret; ASSERT(force || vfs->vfs_count == 1); VFS_RELE(vfs); #ifdef DEBUG fprintf(stderr, "mounted filesystems: %i\n", atomic_dec_32_nv(&mounted)); #endif return 0; }
/* Thread specific key destructor for handling thread exit */ void release_slot(void *data) { struct slot *slot = data; if (slot == NULL) return; /* Release value */ atomic_write_ptr((volatile void **)&slot->value, NULL); /* Release slot */ atomic_write_32(&slot->in_use, 0); /* * If the thread-safe global was destroyed while we held the last * slot then it falls to us to complete the destruction. */ if (atomic_dec_32_nv(&slot->vp->slots_in_use) == 0) destroy_var(slot->vp); }
/* * Drop the prom lock if it is held by the current CPU. If the lock is held * recursively, return without clearing prom_cpu. If the hold count is now * zero, clear prom_cpu and cv_signal any waiting CPU. */ void kern_postprom(void) { processorid_t cpuid = getprocessorid(); cpu_t *cp = cpu[cpuid]; if (panicstr) return; /* do not modify lock further if we have panicked */ if (prom_cpu != cp) panic("kern_postprom: not owner, cp=%p owner=%p", (void *)cp, (void *)prom_cpu); if (prom_holdcnt == 0) panic("kern_postprom: prom_holdcnt == 0, owner=%p", (void *)prom_cpu); if (atomic_dec_32_nv(&prom_holdcnt) != 0) return; /* prom lock is held recursively by this CPU */ if ((boothowto & RB_DEBUG) && prom_exit_enter_debugger) kmdb_enter(); prom_thread = NULL; membar_producer(); prom_cpu = NULL; membar_producer(); if (CPU_IN_SET(cpu_ready_set, cpuid) && cp->cpu_m.mutex_ready) { mutex_enter(&prom_mutex); cv_signal(&prom_cv); mutex_exit(&prom_mutex); kpreempt_enable(); } }
long operator--() { return atomic_dec_32_nv( &value_ ); }
/** * Get the most up to date value of the given cf var. * * @param [in] var Pointer to a cf var * @param [out] res Pointer to location where the variable's value will be output * @param [out] version Pointer (may be NULL) to 64-bit integer where the current version will be output * * @return Zero on success, a system error code otherwise */ int pthread_var_get_np(pthread_var_np_t vp, void **res, uint64_t *version) { int err = 0; int err2 = 0; uint32_t nref; struct var *v; uint64_t vers, vers2; struct vwrapper *wrapper; int got_both_slots = 0; /* Whether we incremented both slots' nreaders */ int do_signal_writer = 0; if (version == NULL) version = &vers; *version = 0; *res = NULL; if ((wrapper = pthread_getspecific(vp->tkey)) != NULL && wrapper->version == atomic_read_64(&vp->next_version) - 1) { /* Fast path */ *version = wrapper->version; *res = wrapper->ptr; return 0; } /* Get the current next version */ *version = atomic_read_64(&vp->next_version); if (*version == 0) { /* Not set yet */ assert(*version == 0 || *res != NULL); return 0; } (*version)--; /* make it the current version */ /* Get what we hope is still the current slot */ v = &vp->vars[(*version) & 0x1]; /* * We picked a slot, but we could just have lost against one or more * writers. So far nothing we've done would block any number of * them. * * We increment nreaders for the slot we picked to keep out * subsequent writers; we can then lose one more race at most. * * But we still need to learn whether we lost the race. */ (void) atomic_inc_32_nv(&v->nreaders); /* See if we won any race */ if ((vers2 = atomic_read_64(&vp->next_version)) == *version) { /* * We won, or didn't race at all. We can now safely * increment nref for the wrapped value in the current slot. * * We can still have lost one race, but this slot is now ours. * * The key here is that we updated nreaders for one slot, * which might not keep the one writer we might have been * racing with from making the then current slot the now * previous slot, but because writers are serialized it will * keep the next writer from touching the slot we thought * was the current slot. Thus here we either have the * current slot or the previous slot, and either way it's OK * for us to grab a reference to the wrapped value in the * slot we took. */ goto got_a_slot; } /* * We may have incremented nreaders for the wrong slot. Any number * of writers could have written between our getting * vp->next_version the first time, and our incrementing nreaders * for the corresponding slot. We can't incref the nref of the * value wrapper found at the slot we picked. We first have to find * the correct current slot, or ensure that no writer will release * the other slot. * * We increment the reader count on the other slot, but we do it * *before* decrementing the reader count on this one. This should * guarantee that we find the other one present by keeping * subsequent writers (subsequent to the second writer we might be * racing with) out of both slots for the time between the update of * one slot's nreaders and the other's. * * We then have to repeat the race loss detection. We need only do * this at most once. */ atomic_inc_32_nv(&v->other->nreaders); /* We hold both slots */ got_both_slots = 1; /* * vp->next_version can now increment by at most one, and we're * guaranteed to have one usable slot (whichever one we _now_ see as * the current slot, and which can still become the previous slot). */ vers2 = atomic_read_64(&vp->next_version); assert(vers2 > *version); *version = vers2 - 1; /* Select a slot that looks current in this thread */ v = &vp->vars[(*version) & 0x1]; got_a_slot: if (v->wrapper == NULL) { /* Whoa, nothing there; shouldn't happen; assert? */ assert(*version == 0); assert(*version == 0 || *res != NULL); if (got_both_slots && atomic_dec_32_nv(&v->other->nreaders) == 0) { /* Last reader of a slot -> signal writer. */ do_signal_writer = 1; } /* * Optimization TODO: * * If vp->next_version hasn't changed since earlier then we * should be able to avoid having to signal a writer when we * decrement what we know is the current slot's nreaders to * zero. This should read: * * if ((atomic_dec_32_nv(&v->nreaders) == 0 && * atomic_read_64(&vp->next_version) == vers2) || * do_signal_writer) * err2 = signal_writer(vp); */ if (atomic_dec_32_nv(&v->nreaders) == 0 || do_signal_writer) err2 = signal_writer(vp); return (err2 == 0) ? err : err2; } assert(vers2 == atomic_read_64(&vp->next_version) || (vers2 + 1) == atomic_read_64(&vp->next_version)); /* Take the wrapped value for the slot we chose */ nref = atomic_inc_32_nv(&v->wrapper->nref); assert(nref > 1); *version = v->wrapper->version; *res = atomic_read_ptr((volatile void **)&v->wrapper->ptr); assert(*res != NULL); /* * We'll release the previous wrapper and save the new one in * vp->tkey below, after releasing the slot it came from. */ wrapper = v->wrapper; /* * Release the slot(s) and signal any possible waiting writer if * either slot's nreaders drops to zero (that's what the writer will * be waiting for). * * The one blocking operation done by readers happens in * signal_writer(), but that one blocking operation is for a lock * that the writer will have or will soon have released, so it's * a practically uncontended blocking operation. */ if (got_both_slots && atomic_dec_32_nv(&v->other->nreaders) == 0) do_signal_writer = 1; if (atomic_dec_32_nv(&v->nreaders) == 0 || do_signal_writer) err2 = signal_writer(vp); /* * Release the value previously read in this thread, if any. * * Note that we call free() here, which means that we might take a * lock in free(). The application's value destructor also can do * the same. * * TODO We could use a lock-less queue/stack to queue up wrappers * for destruction by writers, then readers could be even more * light-weight. */ if (*res != pthread_getspecific(vp->tkey)) pthread_var_release_np(vp); /* Recall this value we just read */ err = pthread_setspecific(vp->tkey, wrapper); return (err2 == 0) ? err : err2; }
template<typename T> static T decrease_nv(T *ptr) { return atomic_dec_32_nv(ptr); }
/* * e1000g_rxfree_func - the call-back function to reclaim rx buffer * * This function is called when an mp is freed by the user thru * freeb call (Only for mp constructed through desballoc call) * It returns back the freed buffer to the freelist */ void e1000g_rxfree_func(p_rx_sw_packet_t packet) { e1000g_rx_data_t *rx_data; private_devi_list_t *devi_node; struct e1000g *Adapter; uint32_t ring_cnt; uint32_t ref_cnt; unsigned char *address; if (packet->ref_cnt == 0) { /* * This case only happens when rx buffers are being freed * in e1000g_stop() and freemsg() is called. */ return; } rx_data = (e1000g_rx_data_t *)(uintptr_t)packet->rx_data; if (packet->mp == NULL) { /* * Allocate a mblk that binds to the data buffer */ address = (unsigned char *)packet->rx_buf->address; if (address != NULL) { packet->mp = desballoc((unsigned char *) address, packet->rx_buf->size, BPRI_MED, &packet->free_rtn); } } /* * Enqueue the recycled packets in a recycle queue. When freelist * dries up, move the entire chain of packets from recycle queue * to freelist. This helps in avoiding per packet mutex contention * around freelist. */ mutex_enter(&rx_data->recycle_lock); QUEUE_PUSH_TAIL(&rx_data->recycle_list, &packet->Link); rx_data->recycle_freepkt++; mutex_exit(&rx_data->recycle_lock); ref_cnt = atomic_dec_32_nv(&packet->ref_cnt); if (ref_cnt == 0) { mutex_enter(&e1000g_rx_detach_lock); e1000g_free_rx_sw_packet(packet, B_FALSE); atomic_dec_32(&rx_data->pending_count); atomic_dec_32(&e1000g_mblks_pending); if ((rx_data->pending_count == 0) && (rx_data->flag & E1000G_RX_STOPPED)) { devi_node = rx_data->priv_devi_node; if (devi_node != NULL) { ring_cnt = atomic_dec_32_nv( &devi_node->pending_rx_count); if ((ring_cnt == 0) && (devi_node->flag & E1000G_PRIV_DEVI_DETACH)) { e1000g_free_priv_devi_node( devi_node); } } else { Adapter = rx_data->rx_ring->adapter; atomic_dec_32( &Adapter->pending_rx_count); } e1000g_free_rx_pending_buffers(rx_data); e1000g_free_rx_data(rx_data); } mutex_exit(&e1000g_rx_detach_lock); } }