/* No lock necessary */
static void stat_add(pa_memblock*b) {
pa_assert(b);
pa_assert(b->pool);
pa_atomic_inc(&b->pool->stat.n_allocated);
pa_atomic_add(&b->pool->stat.allocated_size, (int) b->length);
pa_atomic_inc(&b->pool->stat.n_accumulated);
pa_atomic_add(&b->pool->stat.accumulated_size, (int) b->length);
if (b->type == PA_MEMBLOCK_IMPORTED) {
pa_atomic_inc(&b->pool->stat.n_imported);
pa_atomic_add(&b->pool->stat.imported_size, (int) b->length);
}
pa_atomic_inc(&b->pool->stat.n_allocated_by_type[b->type]);
pa_atomic_inc(&b->pool->stat.n_accumulated_by_type[b->type]);
}
/* No lock necessary */
pa_memblock *pa_memblock_new_pool(pa_mempool *p, size_t length) {
pa_memblock *b = NULL;
struct mempool_slot *slot;
static int mempool_disable = 0;
pa_assert(p);
pa_assert(length);
if (mempool_disable == 0)
mempool_disable = getenv("PULSE_MEMPOOL_DISABLE") ? 1 : -1;
if (mempool_disable > 0)
return NULL;
/* If -1 is passed as length we choose the size for the caller: we
* take the largest size that fits in one of our slots. */
if (length == (size_t) -1)
length = pa_mempool_block_size_max(p);
if (p->block_size >= PA_ALIGN(sizeof(pa_memblock)) + length) {
if (!(slot = mempool_allocate_slot(p)))
return NULL;
b = mempool_slot_data(slot);
b->type = PA_MEMBLOCK_POOL;
pa_atomic_ptr_store(&b->data, (uint8_t*) b + PA_ALIGN(sizeof(pa_memblock)));
} else if (p->block_size >= length) {
if (!(slot = mempool_allocate_slot(p)))
return NULL;
if (!(b = pa_flist_pop(PA_STATIC_FLIST_GET(unused_memblocks))))
b = pa_xnew(pa_memblock, 1);
b->type = PA_MEMBLOCK_POOL_EXTERNAL;
pa_atomic_ptr_store(&b->data, mempool_slot_data(slot));
} else {
pa_log_debug("Memory block too large for pool: %lu > %lu", (unsigned long) length, (unsigned long) p->block_size);
pa_atomic_inc(&p->stat.n_too_large_for_pool);
return NULL;
}
PA_REFCNT_INIT(b);
b->pool = p;
b->read_only = b->is_silence = false;
b->length = length;
pa_atomic_store(&b->n_acquired, 0);
pa_atomic_store(&b->please_signal, 0);
stat_add(b);
return b;
}
void* pa_flist_pop(pa_flist*l) {
unsigned idx, n;
pa_atomic_ptr_t *cells;
#ifdef PROFILE
unsigned len;
#endif
pa_assert(l);
cells = PA_FLIST_CELLS(l);
n = (unsigned) pa_atomic_load(&l->length) + N_EXTRA_SCAN;
#ifdef PROFILE
len = n;
#endif
_Y;
idx = reduce(l, (unsigned) pa_atomic_load(&l->read_idx));
for (; n > 0 ; n--) {
void *p;
_Y;
p = pa_atomic_ptr_load(&cells[idx]);
if (p) {
_Y;
if (!pa_atomic_ptr_cmpxchg(&cells[idx], p, NULL))
continue;
_Y;
pa_atomic_inc(&l->read_idx);
_Y;
pa_atomic_dec(&l->length);
return p;
}
_Y;
idx = reduce(l, idx+1);
}
#ifdef PROFILE
if (len > N_EXTRA_SCAN)
pa_log_warn("Didn't find used cell after %u iterations.", len);
#endif
return NULL;
}
/* Lock free push to linked list stack */
static void stack_push(pa_flist *flist, pa_atomic_t *list, pa_flist_elem *new_elem) {
int tag, newindex, next;
pa_assert(list);
tag = pa_atomic_inc(&flist->current_tag);
newindex = new_elem - flist->table;
pa_assert(newindex >= 0 && newindex < (int) flist->size);
newindex |= (tag << flist->tag_shift) & flist->tag_mask;
do {
next = pa_atomic_load(list);
pa_atomic_store(&new_elem->next, next);
} while (!pa_atomic_cmpxchg(list, next, newindex));
}
/* No lock necessary */
static struct mempool_slot* mempool_allocate_slot(pa_mempool *p) {
struct mempool_slot *slot;
pa_assert(p);
if (!(slot = pa_flist_pop(p->free_slots))) {
int idx;
/* The free list was empty, we have to allocate a new entry */
if ((unsigned) (idx = pa_atomic_inc(&p->n_init)) >= p->n_blocks)
pa_atomic_dec(&p->n_init);
else
slot = (struct mempool_slot*) ((uint8_t*) p->memory.ptr + (p->block_size * idx));
if (!slot) {
pa_log_debug("Pool full");
pa_atomic_inc(&p->stat.n_pool_full);
return NULL;
}
}
return slot;
}
unsigned pa_aupdate_read_begin(pa_aupdate *a) {
unsigned n;
pa_assert(a);
/* Increase the lock counter */
n = (unsigned) pa_atomic_inc(&a->read_lock);
/* When n is 0 we have about 2^31 threads running that all try to
* access the data at the same time, oh my! */
pa_assert(COUNTER(n)+1 > 0);
/* The uppermost bit tells us which data to look at */
return WHICH(n);
}
/* Self locked */
static void memblock_wait(pa_memblock *b) {
pa_assert(b);
if (pa_atomic_load(&b->n_acquired) > 0) {
/* We need to wait until all threads gave up access to the
* memory block before we can go on. Unfortunately this means
* that we have to lock and wait here. Sniff! */
pa_atomic_inc(&b->please_signal);
while (pa_atomic_load(&b->n_acquired) > 0)
pa_semaphore_wait(b->pool->semaphore);
pa_atomic_dec(&b->please_signal);
}
}
int pa_fdsem_before_poll(pa_fdsem *f) {
pa_assert(f);
flush(f);
if (pa_atomic_cmpxchg(&f->data->signalled, 1, 0))
return -1;
pa_atomic_inc(&f->data->waiting);
if (pa_atomic_cmpxchg(&f->data->signalled, 1, 0)) {
pa_assert_se(pa_atomic_dec(&f->data->waiting) >= 1);
return -1;
}
return 0;
}
void pa_fdsem_wait(pa_fdsem *f) {
pa_assert(f);
flush(f);
if (pa_atomic_cmpxchg(&f->data->signalled, 1, 0))
return;
pa_atomic_inc(&f->data->waiting);
while (!pa_atomic_cmpxchg(&f->data->signalled, 1, 0)) {
char x[10];
ssize_t r;
#ifdef HAVE_SYS_EVENTFD_H
if (f->efd >= 0) {
uint64_t u;
if ((r = pa_read(f->efd, &u, sizeof(u), NULL)) != sizeof(u)) {
if (r >= 0 || errno != EINTR) {
pa_log_error("Invalid read from eventfd: %s", r < 0 ? pa_cstrerror(errno) : "EOF");
pa_assert_not_reached();
}
continue;
}
r = (ssize_t) u;
} else
#endif
if ((r = pa_read(f->fds[0], &x, sizeof(x), NULL)) <= 0) {
if (r >= 0 || errno != EINTR) {
pa_log_error("Invalid read from pipe: %s", r < 0 ? pa_cstrerror(errno) : "EOF");
pa_assert_not_reached();
}
continue;
}
pa_atomic_sub(&f->data->in_pipe, (int) r);
}
pa_assert_se(pa_atomic_dec(&f->data->waiting) >= 1);
}
void pa_fdsem_post(pa_fdsem *f) {
pa_assert(f);
if (pa_atomic_cmpxchg(&f->data->signalled, 0, 1)) {
if (pa_atomic_load(&f->data->waiting)) {
ssize_t r;
char x = 'x';
pa_atomic_inc(&f->data->in_pipe);
for (;;) {
#ifdef HAVE_SYS_EVENTFD_H
if (f->efd >= 0) {
uint64_t u = 1;
if ((r = pa_write(f->efd, &u, sizeof(u), NULL)) != sizeof(u)) {
if (r >= 0 || errno != EINTR) {
pa_log_error("Invalid write to eventfd: %s", r < 0 ? pa_cstrerror(errno) : "EOF");
pa_assert_not_reached();
}
continue;
}
} else
#endif
if ((r = pa_write(f->fds[1], &x, 1, NULL)) != 1) {
if (r >= 0 || errno != EINTR) {
pa_log_error("Invalid write to pipe: %s", r < 0 ? pa_cstrerror(errno) : "EOF");
pa_assert_not_reached();
}
continue;
}
break;
}
}
}
}
