static pool_item_t* pool_pull(pool_local_t* thread, pool_global_t* global)
{
pool_cmp_t cmp, xchg;
pool_central_t* next;
cmp.dw = global->central;
do
{
next = cmp.node;
if(next == NULL)
return NULL;
xchg.node = next->central;
xchg.aba = cmp.aba + 1;
} while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw));
pool_item_t* p = (pool_item_t*)next;
assert(next->length == global->count);
TRACK_PULL(p, next->length, global->size);
thread->pool = p->next;
thread->length = next->length - 1;
return p;
}
static void pool_push(pool_local_t* thread, pool_global_t* global)
{
pool_cmp_t cmp, xchg;
pool_central_t* p = (pool_central_t*)thread->pool;
p->length = thread->length;
cmp.dw = global->central;
do
{
p->central = cmp.node;
xchg.node = p;
xchg.aba = cmp.aba + 1;
} while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw,
__ATOMIC_RELAXED, __ATOMIC_RELAXED));
thread->pool = NULL;
thread->length = 0;
}
static void pool_push(pool_local_t* thread, pool_global_t* global)
{
pool_cmp_t cmp, xchg;
pool_central_t* p = (pool_central_t*)thread->pool;
p->length = thread->length;
thread->pool = NULL;
thread->length = 0;
assert(p->length == global->count);
TRACK_PUSH((pool_item_t*)p, p->length, global->size);
cmp.dw = global->central;
xchg.node = p;
do
{
p->central = cmp.node;
xchg.aba = cmp.aba + 1;
} while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw));
}
void* ponyint_mpmcq_pop(mpmcq_t* q)
{
mpmcq_dwcas_t cmp, xchg;
mpmcq_node_t* next;
cmp.aba = q->tail.aba;
cmp.node = q->tail.node;
do
{
// Get the next node rather than the tail. The tail is either a stub or has
// already been consumed.
next = _atomic_load(&cmp.node->next);
// Bailout if we have no next node.
if(next == NULL)
return NULL;
// Make the next node the tail, incrementing the aba counter. If this
// fails, cmp becomes the new tail and we retry the loop.
xchg.aba = cmp.aba + 1;
xchg.node = next;
} while(!_atomic_dwcas(&q->tail.dw, &cmp.dw, xchg.dw));
// We'll return the data pointer from the next node.
void* data = _atomic_load(&next->data);
// Since we will be freeing the old tail, we need to be sure no other
// consumer is still reading the old tail. To do this, we set the data
// pointer of our new tail to NULL, and we wait until the data pointer of
// the old tail is NULL.
_atomic_store(&next->data, NULL);
while(_atomic_load(&cmp.node->data) != NULL)
ponyint_cpu_relax();
// Free the old tail. The new tail is the next node.
POOL_FREE(mpmcq_node_t, cmp.node);
return data;
}
static pool_item_t* pool_pull(pool_local_t* thread, pool_global_t* global)
{
pool_cmp_t cmp, xchg;
pool_central_t* next;
cmp.dw = global->central;
do
{
next = cmp.node;
if(next == NULL)
return NULL;
xchg.node = next->central;
xchg.aba = cmp.aba + 1;
} while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw,
__ATOMIC_RELAXED, __ATOMIC_RELAXED));
pool_item_t* p = (pool_item_t*)next;
thread->pool = p->next;
thread->length = next->length - 1;
return p;
}
