static int qt_lf_list_delete(marked_ptr_t *head,
so_key_t hashed_key,
qt_key_t key,
qt_dict_key_equals_f op_equals,
qt_dict_cleanup_f cleanup)
{
while (1) {
marked_ptr_t *lprev;
marked_ptr_t lcur;
marked_ptr_t lnext;
if (qt_lf_list_find(head, hashed_key, key, &lprev, &lcur, &lnext, op_equals) == NULL) {
qthread_debug(ALWAYS_OUTPUT, "### inside delete - return 0\n");
return 0;
}
if (qthread_cas_ptr(&PTR_OF(lcur)->next, (void*)CONSTRUCT(0, lnext), (void*)CONSTRUCT(1, lnext)) != (void *)CONSTRUCT(0, lnext)) {
qthread_debug(ALWAYS_OUTPUT, "### inside delete - cas failed continue\n");
continue;
}
if (qthread_cas(lprev, CONSTRUCT(0, lcur), CONSTRUCT(0, lnext)) == CONSTRUCT(0, lcur)) {
if (cleanup != NULL) {
cleanup(PTR_OF(lcur)->key, NULL);
}
qpool_free(hash_entry_pool, PTR_OF(lcur));
} else {
qt_lf_list_find(head, hashed_key, key, NULL, NULL, NULL, op_equals); // needs to set cur/prev/next
}
return 1;
}
}
/**
* Insert node "node" in the list starting at position "head".
* Set "ocur" to be the item with the same key if it exists
* or the item which follows if the node was added successfully.
* Set crt_node to point to the current node with key = "node->key"
* Return 0 if item existed and 1 if it was inserted
*/
static int qt_lf_list_insert(marked_ptr_t *head,
hash_entry *node,
marked_ptr_t *ocur,
hash_entry **crt_node,
qt_dict_key_equals_f op_equals)
{
so_key_t hashed_key = node->hashed_key;
qt_key_t key = node->key;
while (1) {
marked_ptr_t *lprev;
marked_ptr_t cur;
if (qt_lf_list_find(head, hashed_key, key, &lprev, &cur, NULL, op_equals) != NULL) { // needs to set cur/prev
if (ocur) { *ocur = cur; }
if (crt_node) { *crt_node = PTR_OF(cur); }
return 0;
}
node->next = (hash_entry *)CONSTRUCT(0, cur);
if (qthread_cas(lprev, (marked_ptr_t)(uintptr_t)node->next, CONSTRUCT(0, node)) == CONSTRUCT(0, cur)) {
if (ocur) { *ocur = cur; }
if (crt_node) { *crt_node = node; }
return 1;
}
}
}
void qtimer_start(qtimer_t q)
{
if (inited == 0) {
if (qthread_cas(&inited, 0, 1) == 0) {
mach_timebase_info_data_t info;
kern_return_t err = mach_timebase_info(&info);
// Convert the timebase into seconds
if (err == 0) {
conversion = 1e-9 * (double)info.numer / (double)info.denom;
}
COMPILER_FENCE;
inited = 2;
} else {
while (inited == 1) SPINLOCK_BODY();
}
}
q->start = mach_absolute_time();
}
// old public method; added last param to distinguish between put and put if absent
static inline hash_entry *qt_hash_put(qt_hash h,
qt_key_t key,
void *value,
int put_choice)
{
hash_entry *node = qpool_alloc(hash_entry_pool);
hash_entry *ret = node;
size_t bucket;
uint64_t lkey = (uint64_t)(uintptr_t)(h->op_hash(key));
HASH_KEY(lkey);
bucket = lkey % h->size;
assert(node);
assert((lkey & MSB) == 0);
node->hashed_key = so_regularkey(lkey);
node->key = key; // Also store original key!
node->value = value;
node->next = (hash_entry*)UNINITIALIZED;
if (h->B[bucket] == UNINITIALIZED) {
initialize_bucket(h, bucket);
}
if(put_choice == PUT_IF_ABSENT) {
if (!qt_lf_list_insert(&(h->B[bucket]), node, NULL, &ret, h->op_equals)) {
qpool_free(hash_entry_pool, node);
return ret->value;
}
} else {
qt_lf_force_list_insert(&(h->B[bucket]), node, h->op_equals);
}
size_t csize = h->size;
if (qthread_incr(&h->count, 1) / csize > MAX_LOAD) {
if (2 * csize <= hard_max_buckets) { // this caps the size of the hash
qthread_cas(&h->size, csize, 2 * csize);
}
}
return ret->value;
}
/**
* Insert node "node" in the list starting at position "head".
* If item exists replace it (force insert)
*/
static void qt_lf_force_list_insert(marked_ptr_t *head,
hash_entry *node,
qt_dict_key_equals_f op_equals)
{
so_key_t hashed_key = node->hashed_key;
qt_key_t key = node->key;
while (1) {
marked_ptr_t *lprev;
marked_ptr_t cur;
marked_ptr_t lnext;
if (qt_lf_list_find(head, hashed_key, key, &lprev, &cur, &lnext, op_equals) != NULL) { // needs to set cur/prev/next
node->next = (hash_entry *)CONSTRUCT(0, lnext);
} else {
node->next = (hash_entry *)CONSTRUCT(0, cur);
}
if (qthread_cas(lprev, CONSTRUCT(0, cur), CONSTRUCT(0, node)) == CONSTRUCT(0, cur)) {
return;
}
}
}
static void *qt_lf_list_find(marked_ptr_t *head,
so_key_t hashed_key,
qt_key_t key,
marked_ptr_t **oprev,
marked_ptr_t *ocur,
marked_ptr_t *onext,
qt_dict_key_equals_f op_equals)
{
so_key_t ckey;
qt_key_t okey;
void *cval;
marked_ptr_t *prev = NULL;
marked_ptr_t cur = UNINITIALIZED;
marked_ptr_t next = UNINITIALIZED;
// int foundInsertPos = 0;
while (1) {
prev = head;
cur = *prev;
while (1) {
if (PTR_OF(cur) == NULL) {
if (oprev) { *oprev = prev; }
if (ocur) { *ocur = cur; }
if (onext) { *onext = next; }
return 0;
}
next = (marked_ptr_t)(PTR_OF(cur)->next);
ckey = PTR_OF(cur)->hashed_key;
cval = PTR_OF(cur)->value;
okey = PTR_OF(cur)->key;
if (*prev != CONSTRUCT(0, cur)) {
break; // this means someone mucked with the list; start over
}
if (!MARK_OF(next)) { // if next pointer is not marked
if (ckey >= hashed_key) {
// if current key > hashed_key, the key isn't in the list; if current key == hashed_key, the key IS in the list
// if (foundInsertPos == 0) {
if (oprev) { *oprev = prev; }
if (ocur) { *ocur = cur; }
if (onext) { *onext = next; }
// foundInsertPos = 1;
// }
if(ckey == hashed_key) {
if((okey != NULL) && op_equals(okey, key)) {
/*
* if (oprev) { *oprev = prev; }
* if (ocur) { *ocur = cur; }
* if (onext) { *onext = next; }
*/
return cval;
} // else, keep looking
} else { return NULL; }
}
// but if current key < hashed_key, the we don't know yet, keep looking
prev = (marked_ptr_t *)&(PTR_OF(cur)->next);
} else {
if (qthread_cas(prev, CONSTRUCT(0, cur), CONSTRUCT(0, next)) == CONSTRUCT(0, cur)) {
qpool_free(hash_entry_pool, PTR_OF(cur));
} else {
break;
}
}
cur = next;
}
}
}
static inline aligned_t qutil_qsort_partition(struct qutil_qsort_args *args)
{ /*{{{*/
double *a = args->array;
const double pivot = args->pivot;
const size_t length = args->length;
const size_t jump = args->jump;
size_t leftwall, rightwall;
leftwall = 0;
rightwall = length - 1;
/* adjust the edges; this is critical for this algorithm */
while (a[leftwall] <= pivot) {
if ((leftwall + 1) % MT_CHUNKSIZE != 0) {
leftwall++;
} else {
leftwall += jump;
}
if (rightwall < leftwall) {
goto quickexit;
}
}
while (a[rightwall] > pivot) {
if (rightwall % MT_CHUNKSIZE != 0) {
if (rightwall == 0) {
goto quickexit;
}
rightwall--;
} else {
if (rightwall < jump) {
goto quickexit;
}
rightwall -= jump;
}
if (rightwall < leftwall) {
goto quickexit;
}
}
SWAP(double, a, leftwall, rightwall);
while (1) {
do {
leftwall += ((leftwall + 1) % MT_CHUNKSIZE != 0) ? 1 : jump;
if (rightwall < leftwall) {
goto quickexit;
}
} while (a[leftwall] <= pivot);
if (rightwall <= leftwall) {
break;
}
do {
if (rightwall % MT_CHUNKSIZE != 0) {
if (rightwall == 0) {
goto quickexit;
}
rightwall--;
} else {
if (rightwall < jump) {
goto quickexit;
}
rightwall -= jump;
}
} while (a[rightwall] > pivot);
if (rightwall <= leftwall) {
break;
}
SWAP(double, a, leftwall, rightwall);
}
quickexit:
{
aligned_t mine = leftwall + args->offset;
aligned_t cur = *args->furthest_leftwall;
aligned_t tmp;
if (mine < cur) {
tmp = cur;
do {
cur = tmp;
tmp = qthread_cas(args->furthest_leftwall, cur, mine);
} while (tmp != cur && mine < tmp);
}
}
{
aligned_t mine = rightwall + args->offset;
aligned_t cur = *args->furthest_rightwall;
aligned_t tmp;
if (mine > cur) {
tmp = cur;
do {
cur = tmp;
tmp = qthread_cas(args->furthest_rightwall, cur, mine);
} while (tmp != cur && mine > tmp);
}
}
return 0;
} /*}}}*/