static pool_item_t* pool_pull(pool_local_t* thread, pool_global_t* global) { pool_cmp_t cmp, xchg; pool_central_t* next; cmp.dw = global->central; do { next = cmp.node; if(next == NULL) return NULL; xchg.node = next->central; xchg.aba = cmp.aba + 1; } while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw)); pool_item_t* p = (pool_item_t*)next; assert(next->length == global->count); TRACK_PULL(p, next->length, global->size); thread->pool = p->next; thread->length = next->length - 1; return p; }
static void pool_push(pool_local_t* thread, pool_global_t* global) { pool_cmp_t cmp, xchg; pool_central_t* p = (pool_central_t*)thread->pool; p->length = thread->length; cmp.dw = global->central; do { p->central = cmp.node; xchg.node = p; xchg.aba = cmp.aba + 1; } while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); thread->pool = NULL; thread->length = 0; }
static void pool_push(pool_local_t* thread, pool_global_t* global) { pool_cmp_t cmp, xchg; pool_central_t* p = (pool_central_t*)thread->pool; p->length = thread->length; thread->pool = NULL; thread->length = 0; assert(p->length == global->count); TRACK_PUSH((pool_item_t*)p, p->length, global->size); cmp.dw = global->central; xchg.node = p; do { p->central = cmp.node; xchg.aba = cmp.aba + 1; } while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw)); }
void* ponyint_mpmcq_pop(mpmcq_t* q) { mpmcq_dwcas_t cmp, xchg; mpmcq_node_t* next; cmp.aba = q->tail.aba; cmp.node = q->tail.node; do { // Get the next node rather than the tail. The tail is either a stub or has // already been consumed. next = _atomic_load(&cmp.node->next); // Bailout if we have no next node. if(next == NULL) return NULL; // Make the next node the tail, incrementing the aba counter. If this // fails, cmp becomes the new tail and we retry the loop. xchg.aba = cmp.aba + 1; xchg.node = next; } while(!_atomic_dwcas(&q->tail.dw, &cmp.dw, xchg.dw)); // We'll return the data pointer from the next node. void* data = _atomic_load(&next->data); // Since we will be freeing the old tail, we need to be sure no other // consumer is still reading the old tail. To do this, we set the data // pointer of our new tail to NULL, and we wait until the data pointer of // the old tail is NULL. _atomic_store(&next->data, NULL); while(_atomic_load(&cmp.node->data) != NULL) ponyint_cpu_relax(); // Free the old tail. The new tail is the next node. POOL_FREE(mpmcq_node_t, cmp.node); return data; }
static pool_item_t* pool_pull(pool_local_t* thread, pool_global_t* global) { pool_cmp_t cmp, xchg; pool_central_t* next; cmp.dw = global->central; do { next = cmp.node; if(next == NULL) return NULL; xchg.node = next->central; xchg.aba = cmp.aba + 1; } while(!_atomic_dwcas(&global->central, &cmp.dw, xchg.dw, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); pool_item_t* p = (pool_item_t*)next; thread->pool = p->next; thread->length = next->length - 1; return p; }