/* No lock necessary. This function is not multiple caller safe! */ static void memblock_make_local(pa_memblock *b) { pa_assert(b); pa_atomic_dec(&b->pool->stat.n_allocated_by_type[b->type]); if (b->length <= b->pool->block_size) { struct mempool_slot *slot; if ((slot = mempool_allocate_slot(b->pool))) { void *new_data; /* We can move it into a local pool, perfect! */ new_data = mempool_slot_data(slot); memcpy(new_data, pa_atomic_ptr_load(&b->data), b->length); pa_atomic_ptr_store(&b->data, new_data); b->type = PA_MEMBLOCK_POOL_EXTERNAL; b->read_only = false; goto finish; } } /* Humm, not enough space in the pool, so lets allocate the memory with malloc() */ b->per_type.user.free_cb = pa_xfree; pa_atomic_ptr_store(&b->data, pa_xmemdup(pa_atomic_ptr_load(&b->data), b->length)); b->type = PA_MEMBLOCK_USER; b->read_only = false; finish: pa_atomic_inc(&b->pool->stat.n_allocated_by_type[b->type]); pa_atomic_inc(&b->pool->stat.n_accumulated_by_type[b->type]); memblock_wait(b); }
/* No lock necessary */ void* pa_memblock_acquire(pa_memblock *b) { pa_assert(b); pa_assert(PA_REFCNT_VALUE(b) > 0); pa_atomic_inc(&b->n_acquired); return pa_atomic_ptr_load(&b->data); }
/* No lock necessary */ static pa_memblock *memblock_shared_copy(pa_mempool *p, pa_memblock *b) { pa_memblock *n; pa_assert(p); pa_assert(b); if (b->type == PA_MEMBLOCK_IMPORTED || b->type == PA_MEMBLOCK_POOL || b->type == PA_MEMBLOCK_POOL_EXTERNAL) { pa_assert(b->pool == p); return pa_memblock_ref(b); } if (!(n = pa_memblock_new_pool(p, b->length))) return NULL; memcpy(pa_atomic_ptr_load(&n->data), pa_atomic_ptr_load(&b->data), b->length); return n; }
/* This is a copy of the function in mutex-posix.c */ pa_mutex* pa_static_mutex_get(pa_static_mutex *s, bool recursive, bool inherit_priority) { pa_mutex *m; pa_assert(s); /* First, check if already initialized and short cut */ if ((m = pa_atomic_ptr_load(&s->ptr))) return m; /* OK, not initialized, so let's allocate, and fill in */ m = pa_mutex_new(recursive, inherit_priority); if ((pa_atomic_ptr_cmpxchg(&s->ptr, NULL, m))) return m; pa_mutex_free(m); /* Him, filling in failed, so someone else must have filled in * already */ pa_assert_se(m = pa_atomic_ptr_load(&s->ptr)); return m; }
void* pa_flist_pop(pa_flist*l) { unsigned idx, n; pa_atomic_ptr_t *cells; #ifdef PROFILE unsigned len; #endif pa_assert(l); cells = PA_FLIST_CELLS(l); n = (unsigned) pa_atomic_load(&l->length) + N_EXTRA_SCAN; #ifdef PROFILE len = n; #endif _Y; idx = reduce(l, (unsigned) pa_atomic_load(&l->read_idx)); for (; n > 0 ; n--) { void *p; _Y; p = pa_atomic_ptr_load(&cells[idx]); if (p) { _Y; if (!pa_atomic_ptr_cmpxchg(&cells[idx], p, NULL)) continue; _Y; pa_atomic_inc(&l->read_idx); _Y; pa_atomic_dec(&l->length); return p; } _Y; idx = reduce(l, idx+1); } #ifdef PROFILE if (len > N_EXTRA_SCAN) pa_log_warn("Didn't find used cell after %u iterations.", len); #endif return NULL; }
void pa_flist_free(pa_flist *l, pa_free_cb_t free_cb) { pa_assert(l); pa_assert(l->name); if (free_cb) { pa_flist_elem *elem; while((elem = stack_pop(l, &l->stored))) free_cb(pa_atomic_ptr_load(&elem->ptr)); } pa_xfree(l->name); pa_xfree(l); }
void* pa_flist_pop(pa_flist *l) { pa_flist_elem *elem; void *ptr; pa_assert(l); elem = stack_pop(l, &l->stored); if (elem == NULL) return NULL; ptr = pa_atomic_ptr_load(&elem->ptr); stack_push(l, &l->empty, elem); return ptr; }
void* pa_shmasyncq_pop(pa_shmasyncq*l, int wait) { int idx; void *ret; pa_atomic_ptr_t *cells; pa_assert(l); cells = PA_SHMASYNCQ_CELLS(l); _Y; idx = reduce(l, l->read_idx); if (!(ret = pa_atomic_ptr_load(&cells[idx]))) { if (!wait) return NULL; /* pa_log("sleeping on pop"); */ do { pa_fdsem_wait(l->write_fdsem); } while (!(ret = pa_atomic_ptr_load(&cells[idx]))); } pa_assert(ret); /* Guaranteed to succeed if we only have a single reader */ pa_assert_se(pa_atomic_ptr_cmpxchg(&cells[idx], ret, NULL)); _Y; l->read_idx++; pa_fdsem_post(l->read_fdsem); return ret; }
int pa_asyncq_read_before_poll(pa_asyncq *l) { unsigned idx; pa_atomic_ptr_t *cells; pa_assert(l); cells = PA_ASYNCQ_CELLS(l); _Y; idx = reduce(l, l->read_idx); for (;;) { if (pa_atomic_ptr_load(&cells[idx])) return -1; if (pa_fdsem_before_poll(l->write_fdsem) >= 0) return 0; } }
void pa_flist_free(pa_flist *l, pa_free_cb_t free_cb) { pa_assert(l); if (free_cb) { pa_atomic_ptr_t*cells; unsigned idx; cells = PA_FLIST_CELLS(l); for (idx = 0; idx < l->size; idx ++) { void *p; if ((p = pa_atomic_ptr_load(&cells[idx]))) free_cb(p); } } pa_xfree(l); }
static void memblock_free(pa_memblock *b) { pa_assert(b); pa_assert(pa_atomic_load(&b->n_acquired) == 0); stat_remove(b); switch (b->type) { case PA_MEMBLOCK_USER : pa_assert(b->per_type.user.free_cb); b->per_type.user.free_cb(pa_atomic_ptr_load(&b->data)); /* Fall through */ case PA_MEMBLOCK_FIXED: if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; case PA_MEMBLOCK_APPENDED: /* We could attach it to unused_memblocks, but that would * probably waste some considerable amount of memory */ pa_xfree(b); break; case PA_MEMBLOCK_IMPORTED: { pa_memimport_segment *segment; pa_memimport *import; /* FIXME! This should be implemented lock-free */ pa_assert_se(segment = b->per_type.imported.segment); pa_assert_se(import = segment->import); pa_mutex_lock(import->mutex); pa_assert_se(pa_hashmap_remove(import->blocks, PA_UINT32_TO_PTR(b->per_type.imported.id))); pa_assert(segment->n_blocks >= 1); if (-- segment->n_blocks <= 0) segment_detach(segment); pa_mutex_unlock(import->mutex); import->release_cb(import, b->per_type.imported.id, import->userdata); if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; } case PA_MEMBLOCK_POOL_EXTERNAL: case PA_MEMBLOCK_POOL: { struct mempool_slot *slot; bool call_free; pa_assert_se(slot = mempool_slot_by_ptr(b->pool, pa_atomic_ptr_load(&b->data))); call_free = b->type == PA_MEMBLOCK_POOL_EXTERNAL; /* #ifdef HAVE_VALGRIND_MEMCHECK_H */ /* if (PA_UNLIKELY(pa_in_valgrind())) { */ /* VALGRIND_FREELIKE_BLOCK(slot, b->pool->block_size); */ /* } */ /* #endif */ /* The free list dimensions should easily allow all slots * to fit in, hence try harder if pushing this slot into * the free list fails */ while (pa_flist_push(b->pool->free_slots, slot) < 0) ; if (call_free) if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; } case PA_MEMBLOCK_TYPE_MAX: default: pa_assert_not_reached(); } }
static void memblock_free(pa_memblock *b) { pa_assert(b); pa_assert(pa_atomic_load(&b->n_acquired) == 0); stat_remove(b); switch (b->type) { case PA_MEMBLOCK_USER : pa_assert(b->per_type.user.free_cb); b->per_type.user.free_cb(pa_atomic_ptr_load(&b->data)); /* Fall through */ case PA_MEMBLOCK_FIXED: case PA_MEMBLOCK_APPENDED : if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; case PA_MEMBLOCK_IMPORTED : { pa_memimport_segment *segment; pa_memimport *import; /* FIXME! This should be implemented lock-free */ segment = b->per_type.imported.segment; pa_assert(segment); import = segment->import; pa_assert(import); pa_mutex_lock(import->mutex); pa_hashmap_remove(import->blocks, PA_UINT32_TO_PTR(b->per_type.imported.id)); if (-- segment->n_blocks <= 0) segment_detach(segment); pa_mutex_unlock(import->mutex); import->release_cb(import, b->per_type.imported.id, import->userdata); if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; } case PA_MEMBLOCK_POOL_EXTERNAL: case PA_MEMBLOCK_POOL: { struct mempool_slot *slot; int call_free; slot = mempool_slot_by_ptr(b->pool, pa_atomic_ptr_load(&b->data)); pa_assert(slot); call_free = b->type == PA_MEMBLOCK_POOL_EXTERNAL; /* The free list dimensions should easily allow all slots * to fit in, hence try harder if pushing this slot into * the free list fails */ while (pa_flist_push(b->pool->free_slots, slot) < 0) ; if (call_free) if (pa_flist_push(PA_STATIC_FLIST_GET(unused_memblocks), b) < 0) pa_xfree(b); break; } case PA_MEMBLOCK_TYPE_MAX: default: pa_assert_not_reached(); } }