void *qlfqueue_dequeue(qlfqueue_t *q)
{ /*{{{ */
void *p = NULL;
qlfqueue_node_t *head;
qlfqueue_node_t *tail;
qlfqueue_node_t *next_ptr;
qassert_ret((q != NULL), NULL);
while (1) {
head = q->head;
hazardous_ptr(0, head);
if (head != q->head) { continue; }
tail = q->tail;
next_ptr = head->next;
hazardous_ptr(1, next_ptr);
if (next_ptr == NULL) { return NULL; } /* queue is empty */
if (head == tail) { /* tail is falling behind! */
/* advance tail ptr... */
(void)qthread_cas_ptr((void **)&(q->tail), (void *)tail, next_ptr);
continue;
}
/* read value before CAS, otherwise another dequeue might free the next node */
p = next_ptr->value;
if (qthread_cas_ptr((void **)&(q->head), (void *)head, next_ptr) == head) {
break; /* success! */
}
}
hazardous_release_node(qlfqueue_pool_free_wrapper, head);
return p;
} /*}}} */
static inline qt_threadqueue_node_t *qt_internal_NEMESIS_dequeue(NEMESIS_queue *q)
{ /*{{{ */
if (!q->shadow_head) {
if (!q->head) {
return NULL;
}
q->shadow_head = q->head;
q->head = NULL;
}
qt_threadqueue_node_t *const retval = (void *volatile)(q->shadow_head);
if ((retval != NULL) && (retval != (void *)1)) {
if (retval->next != NULL) {
q->shadow_head = retval->next;
retval->next = NULL;
} else {
qt_threadqueue_node_t *old;
q->shadow_head = NULL;
old = qthread_cas_ptr(&(q->tail), retval, NULL);
if (old != retval) {
while (retval->next == NULL) SPINLOCK_BODY();
q->shadow_head = retval->next;
retval->next = NULL;
}
}
}
return retval;
} /*}}} */
qlfqueue_t *qlfqueue_create(void)
{ /*{{{ */
qlfqueue_t *q;
if (qlfqueue_node_pool == NULL) {
switch ((uintptr_t)qthread_cas_ptr(&qlfqueue_node_pool, NULL, (void *)1)) {
case 0: /* I won, I will allocate */
qlfqueue_node_pool = qpool_create_aligned(sizeof(qlfqueue_node_t), 0);
break;
case 1:
while (qlfqueue_node_pool == (void *)1) {
SPINLOCK_BODY();
}
break;
}
}
qassert_ret((qlfqueue_node_pool != NULL), NULL);
q = MALLOC(sizeof(struct qlfqueue_s));
if (q != NULL) {
q->head = (qlfqueue_node_t *)qpool_alloc(qlfqueue_node_pool);
assert(q->head != NULL);
if (q->head == NULL) { /* if we're not using asserts, fail nicely */
FREE(q, sizeof(struct qlfqueue_s));
return NULL;
}
q->tail = q->head;
q->tail->next = NULL;
}
return q;
} /*}}} */
static int qt_lf_list_delete(marked_ptr_t *head,
so_key_t hashed_key,
qt_key_t key,
qt_dict_key_equals_f op_equals,
qt_dict_cleanup_f cleanup)
{
while (1) {
marked_ptr_t *lprev;
marked_ptr_t lcur;
marked_ptr_t lnext;
if (qt_lf_list_find(head, hashed_key, key, &lprev, &lcur, &lnext, op_equals) == NULL) {
qthread_debug(ALWAYS_OUTPUT, "### inside delete - return 0\n");
return 0;
}
if (qthread_cas_ptr(&PTR_OF(lcur)->next, (void*)CONSTRUCT(0, lnext), (void*)CONSTRUCT(1, lnext)) != (void *)CONSTRUCT(0, lnext)) {
qthread_debug(ALWAYS_OUTPUT, "### inside delete - cas failed continue\n");
continue;
}
if (qthread_cas(lprev, CONSTRUCT(0, lcur), CONSTRUCT(0, lnext)) == CONSTRUCT(0, lcur)) {
if (cleanup != NULL) {
cleanup(PTR_OF(lcur)->key, NULL);
}
qpool_free(hash_entry_pool, PTR_OF(lcur));
} else {
qt_lf_list_find(head, hashed_key, key, NULL, NULL, NULL, op_equals); // needs to set cur/prev/next
}
return 1;
}
}
int qlfqueue_enqueue(qlfqueue_t *q,
void *elem)
{ /*{{{ */
qlfqueue_node_t *tail;
qlfqueue_node_t *next;
qlfqueue_node_t *node;
qassert_ret((elem != NULL), QTHREAD_BADARGS);
qassert_ret((q != NULL), QTHREAD_BADARGS);
node = (qlfqueue_node_t *)qpool_alloc(qlfqueue_node_pool);
/* these asserts should be redundant */
qassert_ret((node != NULL), QTHREAD_MALLOC_ERROR);
memset((void *)node, 0, sizeof(qlfqueue_node_t));
node->value = elem;
while (1) {
tail = q->tail;
hazardous_ptr(0, tail);
if (tail != q->tail) { continue; }
next = tail->next;
if (next != NULL) { /* tail not pointing to last node */
(void)qthread_cas_ptr((void **)&(q->tail), (void *)tail, next);
continue;
}
/* tail must be pointing to the last node */
if (qthread_cas_ptr((void **)&(tail->next), (void *)next, node) == next) {
break; /* success! */
}
}
(void)qthread_cas_ptr((void **)&(q->tail), (void *)tail, node);
hazardous_ptr(0, NULL); // release the ptr (avoid hazardptr resource exhaustion)
return QTHREAD_SUCCESS;
} /*}}} */
// old public method
static inline qt_hash qt_hash_create(qt_dict_key_equals_f eq,
qt_dict_hash_f hash,
qt_dict_cleanup_f cleanup)
{
qt_hash tmp;
if (hash_entry_pool == NULL) {
if (qthread_cas_ptr(&hash_entry_pool, NULL, (void *)1) == NULL) {
hash_entry_pool = qpool_create(sizeof(hash_entry));
} else {
while (hash_entry_pool == (void *)1) SPINLOCK_BODY();
}
}
tmp = MALLOC(sizeof(qt_dictionary));
assert(tmp);
tmp->op_equals = eq;
tmp->op_hash = hash;
tmp->op_cleanup = cleanup;
assert(tmp);
if (hard_max_buckets == 0) {
hard_max_buckets = pagesize / sizeof(marked_ptr_t);
}
tmp->B = calloc(hard_max_buckets, sizeof(marked_ptr_t));
assert(tmp->B);
tmp->size = 2;
tmp->count = 0;
{
hash_entry *dummy = qpool_alloc(hash_entry_pool);
assert(dummy);
memset(dummy, 0, sizeof(hash_entry));
tmp->B[0] = CONSTRUCT(0, dummy);
}
return tmp;
}
void *qt_hash_put_helper(qt_dictionary *h,
qt_key_t key,
void *value,
int put_choice)
{
uint64_t lkey = (uint64_t)(uintptr_t)(h->op_hash(key));
HASH_KEY(lkey);
assert(h);
unsigned bucket = BASE_SPINE_BUCKET(lkey);
hash_entry *e = qt_malloc(sizeof(hash_entry)); // XXX: should be from a memory pool
spine_element_t *child_id = &(h->base[bucket]);
spine_element_t child_val = h->base[bucket];
spine_element_t *cur_id = NULL;
spine_t *cur_spine = NULL;
unsigned depth = 0;
assert(e != NULL);
e->hashed_key = lkey;
e->key = key;
e->value = value;
e->next = NULL;
hash_entry *crt;
do {
if (child_val.e == NULL) {
// place the entry in the hash
if ((child_val.e = CAS(&(child_id->e), NULL, e)) == NULL) {
// put success: no potential colliding element was present
return value;
}
} else if (SPINE_PTR_TEST(child_val)) {
INCREMENT_COUNT(child_id, child_val);
if (cur_id) {
DECREMENT_COUNT(cur_id);
}
cur_id = child_id;
cur_spine = SPINE_PTR(h, child_val);
depth++;
assert(depth <= 11); // otherwise, something has gone horribly wrong
bucket = SPINE_BUCKET(lkey, depth);
child_id = &cur_spine->elements[bucket];
child_val = cur_spine->elements[bucket];
} else if (child_val.e->hashed_key != lkey) {
// upgrade to a spine
spine_element_t newspine, cur;
spine_t *realspine;
unsigned bucket1 = SPINE_BUCKET(child_val.e->hashed_key, depth + 1);
unsigned bucket2 = SPINE_BUCKET(lkey, depth + 1);
newspine.s.id = allocate_spine(h, &realspine);
realspine->parent = cur_id;
realspine->elements[bucket1] = child_val;
if (bucket1 != bucket2) { // both elements will be in the new spine
newspine.s.ctr = SPINE_COUNT(2); // contains 2 elements
realspine->elements[bucket2].e = e;
if ((cur.e = CAS(&(child_id->e), child_val.e, newspine.e)) == child_val.e) {
// success!
if (cur_id) {
DECREMENT_COUNT(cur_id);
}
return value;
} else {
child_val = cur;
deallocate_spine(h, newspine.s.id);
}
} else { // collision in the new spine (unusual; will use unnecessary CAS)
newspine.s.ctr = SPINE_COUNT(1); // contains 1 element (oldval)
if ((cur.e = CAS(&(child_id->e), child_val.e, newspine.e)) == child_val.e) {
// success
continue;
} else {
child_val = cur;
deallocate_spine(h, newspine.s.id);
}
}
} else {
// use the real user-equals operation to differentiate subcases
// it is possible that the element is there or it may not be there
hash_entry *head;
do {
head = child_id->e;
e->next = head;
crt = head;
// find the entry, if it is in the list
while (crt) {
if (h->op_equals(crt->key, key)) {
// already exists
if (put_choice != PUT_IF_ABSENT) {
void **crt_val_adr = &(crt->value);
void *crt_val = crt->value;
while((qthread_cas_ptr(crt_val_adr, \
crt_val, value)) != crt_val ) {
crt_val = crt->value;
}
}
if (cur_id) { DECREMENT_COUNT(cur_id); }
return crt->value;
}
crt = crt->next;
}
// and try to insert it at the head of the list;
// if the list changed, redu the work
} while (qthread_cas_ptr(&(child_id->e), head, e) != head);
// printf("IN put: (%s-%s)\n", child_id->e->key, child_id->e->value);
// if (e->next !=NULL)
// printf("next key is %s and value %s\n", e->next->key, e->next->value);
return e->value;
}
} while (1);
}