bool ck_array_put(struct ck_array *array, void *value) { struct _ck_array *target; unsigned int size; /* * If no transaction copy has been necessary, attempt to do in-place * modification of the array. */ if (array->transaction == NULL) { target = array->active; if (array->n_entries == target->length) { size = target->length << 1; target = array->allocator->realloc(target, sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, sizeof(struct _ck_array) + sizeof(void *) * size, true); if (target == NULL) return false; ck_pr_store_uint(&target->length, size); /* Serialize with respect to contents. */ ck_pr_fence_store(); ck_pr_store_ptr(&array->active, target); } target->values[array->n_entries++] = value; return true; } target = array->transaction; if (array->n_entries == target->length) { size = target->length << 1; target = array->allocator->realloc(target, sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, sizeof(struct _ck_array) + sizeof(void *) * size, true); if (target == NULL) return false; target->length = size; array->transaction = target; } target->values[array->n_entries++] = value; return false; }
bool ck_array_remove(struct ck_array *array, void *value) { struct _ck_array *target; unsigned int i; if (array->transaction != NULL) { target = array->transaction; for (i = 0; i < array->n_entries; i++) { if (target->values[i] == value) { target->values[i] = target->values[--array->n_entries]; return true; } } return false; } target = array->active; for (i = 0; i < array->n_entries; i++) { if (target->values[i] == value) break; } if (i == array->n_entries) return false; /* If there are pending additions, immediately eliminate the operation. */ if (target->n_committed != array->n_entries) { ck_pr_store_ptr(&target->values[i], target->values[--array->n_entries]); return true; } /* * The assumption is that these allocations are small to begin with. * If there is no immediate opportunity for transaction, allocate a * transactional array which will be applied upon commit time. */ target = ck_array_create(array->allocator, array->n_entries); if (target == NULL) return false; memcpy(target->values, array->active->values, sizeof(void *) * array->n_entries); target->length = array->n_entries; target->n_committed = array->n_entries; target->values[i] = target->values[--array->n_entries]; array->transaction = target; return true; }
bool ck_hs_reset_size(struct ck_hs *hs, unsigned long capacity) { struct ck_hs_map *map, *previous; previous = hs->map; map = ck_hs_map_create(hs, capacity); if (map == NULL) return false; ck_pr_store_ptr(&hs->map, map); ck_hs_map_destroy(hs->m, previous, true); return true; }
void ck_barrier_tournament_init(struct ck_barrier_tournament *barrier, struct ck_barrier_tournament_round **rounds, unsigned int nthr) { unsigned int i, k, size, twok, twokm1, imod2k; ck_pr_store_uint(&barrier->tid, 0); size = ck_barrier_tournament_size(nthr); for (i = 0; i < nthr; ++i) { /* The first role is always CK_BARRIER_TOURNAMENT_DROPOUT. */ rounds[i][0].flag = 0; rounds[i][0].role = CK_BARRIER_TOURNAMENT_DROPOUT; for (k = 1, twok = 2, twokm1 = 1; k < size; ++k, twokm1 = twok, twok <<= 1) { rounds[i][k].flag = 0; imod2k = i & (twok - 1); if (imod2k == 0) { if ((i + twokm1 < nthr) && (twok < nthr)) rounds[i][k].role = CK_BARRIER_TOURNAMENT_WINNER; else if (i + twokm1 >= nthr) rounds[i][k].role = CK_BARRIER_TOURNAMENT_BYE; } if (imod2k == twokm1) rounds[i][k].role = CK_BARRIER_TOURNAMENT_LOSER; else if ((i == 0) && (twok >= nthr)) rounds[i][k].role = CK_BARRIER_TOURNAMENT_CHAMPION; if (rounds[i][k].role == CK_BARRIER_TOURNAMENT_LOSER) rounds[i][k].opponent = &rounds[i - twokm1][k].flag; else if (rounds[i][k].role == CK_BARRIER_TOURNAMENT_WINNER || rounds[i][k].role == CK_BARRIER_TOURNAMENT_CHAMPION) rounds[i][k].opponent = &rounds[i + twokm1][k].flag; } } ck_pr_store_ptr(&barrier->rounds, rounds); return; }
static void *ph_thread_boot(void *arg) { struct ph_thread_boot_data data; ph_thread_t *me; void *retval; /* copy in the boot data from the stack of our creator */ memcpy(&data, arg, sizeof(data)); me = ph_thread_init_myself(true); /* this publishes that we're ready to run to * the thread that spawned us */ ck_pr_store_ptr(data.thr, ck_pr_load_ptr(&me)); ck_pr_fence_store(); retval = data.func(data.arg); ck_epoch_barrier(&me->epoch_record); return retval; }
bool ck_array_commit(ck_array_t *array) { struct _ck_array *m = array->transaction; if (m != NULL) { struct _ck_array *p; m->n_committed = array->n_entries; ck_pr_fence_store(); p = array->active; ck_pr_store_ptr(&array->active, m); array->allocator->free(p, sizeof(struct _ck_array) + p->length * sizeof(void *), true); array->transaction = NULL; return true; } ck_pr_fence_store(); ck_pr_store_uint(&array->active->n_committed, array->n_entries); return true; }
/* * Replace prev_entry with new entry if exists, otherwise insert into bag. */ bool ck_bag_set_spmc(struct ck_bag *bag, void *compare, void *update) { struct ck_bag_block *cursor; uint16_t block_index; uint16_t n_entries_block = 0; cursor = bag->head; while (cursor != NULL) { n_entries_block = ck_bag_block_count(cursor); for (block_index = 0; block_index < n_entries_block; block_index++) { if (cursor->array[block_index] != compare) continue; ck_pr_store_ptr(&cursor->array[block_index], update); return true; } cursor = ck_bag_block_next(cursor->next.ptr); } return ck_bag_put_spmc(bag, update); }
bool ck_hs_grow(struct ck_hs *hs, unsigned long capacity) { struct ck_hs_map *map, *update; void **bucket, *previous; unsigned long k, i, j, offset, probes; restart: map = hs->map; if (map->capacity > capacity) return false; update = ck_hs_map_create(hs, capacity); if (update == NULL) return false; for (k = 0; k < map->capacity; k++) { unsigned long h; previous = map->entries[k]; if (previous == CK_HS_EMPTY || previous == CK_HS_TOMBSTONE) continue; #ifdef CK_HS_PP if (hs->mode & CK_HS_MODE_OBJECT) previous = (void *)((uintptr_t)previous & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); #endif h = hs->hf(previous, hs->seed); offset = h & update->mask; i = probes = 0; for (;;) { bucket = (void *)((uintptr_t)&update->entries[offset] & ~(CK_MD_CACHELINE - 1)); for (j = 0; j < CK_HS_PROBE_L1; j++) { void **cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1)); if (probes++ == update->probe_limit) break; if (CK_CC_LIKELY(*cursor == CK_HS_EMPTY)) { *cursor = map->entries[k]; update->n_entries++; if (probes > update->probe_maximum) update->probe_maximum = probes; break; } } if (j < CK_HS_PROBE_L1) break; offset = ck_hs_map_probe_next(update, offset, h, i++, probes); } if (probes > update->probe_limit) { /* * We have hit the probe limit, map needs to be even larger. */ ck_hs_map_destroy(hs->m, update, false); capacity <<= 1; goto restart; } } ck_pr_fence_store(); ck_pr_store_ptr(&hs->map, update); ck_hs_map_destroy(hs->m, map, true); return true; }
static inline void set_nodes(as_cluster* cluster, as_nodes* nodes) { ck_pr_fence_store(); ck_pr_store_ptr(&cluster->nodes, nodes); }
bool ck_rhs_grow(struct ck_rhs *hs, unsigned long capacity) { struct ck_rhs_map *map, *update; void *previous, *prev_saved; unsigned long k, offset, probes; restart: map = hs->map; if (map->capacity > capacity) return false; update = ck_rhs_map_create(hs, capacity); if (update == NULL) return false; for (k = 0; k < map->capacity; k++) { unsigned long h; prev_saved = previous = ck_rhs_entry(map, k); if (previous == CK_RHS_EMPTY) continue; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) previous = CK_RHS_VMA(previous); #endif h = hs->hf(previous, hs->seed); offset = h & update->mask; probes = 0; for (;;) { void **cursor = ck_rhs_entry_addr(update, offset); if (probes++ == update->probe_limit) { /* * We have hit the probe limit, map needs to be even larger. */ ck_rhs_map_destroy(hs->m, update, false); capacity <<= 1; goto restart; } if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { *cursor = prev_saved; update->n_entries++; ck_rhs_set_probes(update, offset, probes); ck_rhs_map_bound_set(update, h, probes); break; } else if (ck_rhs_probes(update, offset) < probes) { void *tmp = prev_saved; unsigned int old_probes; prev_saved = previous = *cursor; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) previous = CK_RHS_VMA(previous); #endif *cursor = tmp; ck_rhs_map_bound_set(update, h, probes); h = hs->hf(previous, hs->seed); old_probes = ck_rhs_probes(update, offset); ck_rhs_set_probes(update, offset, probes); probes = old_probes - 1; continue; } ck_rhs_wanted_inc(update, offset); offset = ck_rhs_map_probe_next(update, offset, probes); } } ck_pr_fence_store(); ck_pr_store_ptr(&hs->map, update); ck_rhs_map_destroy(hs->m, map, true); return true; }
bool ck_bag_remove_spmc(struct ck_bag *bag, void *entry) { struct ck_bag_block *cursor, *copy, *prev; uint16_t block_index, n_entries; cursor = bag->head; prev = NULL; while (cursor != NULL) { n_entries = ck_bag_block_count(cursor); for (block_index = 0; block_index < n_entries; block_index++) { if (cursor->array[block_index] == entry) goto found; } prev = cursor; cursor = ck_bag_block_next(cursor->next.ptr); } return true; found: /* Cursor points to containing block, block_index is index of deletion */ if (n_entries == 1) { /* If a block's single entry is being removed, remove the block. */ if (prev == NULL) { struct ck_bag_block *new_head = ck_bag_block_next(cursor->next.ptr); ck_pr_store_ptr(&bag->head, new_head); } else { uintptr_t next; #ifdef __x86_64__ next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) | (uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr); #else next = (uintptr_t)(void *)cursor->next.ptr; #endif ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next); } /* Remove block from available list */ if (cursor->avail_prev != NULL) cursor->avail_prev->avail_next = cursor->avail_next; if (cursor->avail_next != NULL) cursor->avail_next->avail_prev = cursor->avail_prev; bag->n_blocks--; copy = cursor->avail_next; } else { uintptr_t next_ptr; copy = allocator.malloc(bag->info.bytes); if (copy == NULL) return false; memcpy(copy, cursor, bag->info.bytes); copy->array[block_index] = copy->array[--n_entries]; next_ptr = (uintptr_t)(void *)ck_bag_block_next(copy->next.ptr); #ifdef __x86_64__ copy->next.ptr = (void *)(((uintptr_t)n_entries << 48) | next_ptr); #else copy->next.n_entries = n_entries; copy->next.ptr = (struct ck_bag_block *)next_ptr; #endif ck_pr_fence_store(); if (prev == NULL) { ck_pr_store_ptr(&bag->head, copy); } else { #ifdef __x86_64__ uintptr_t next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) | (uintptr_t)(void *)ck_bag_block_next(copy); ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next); #else ck_pr_store_ptr(&prev->next.ptr, copy); #endif } if (n_entries == bag->info.max - 1) { /* Block was previously fully, add to head of avail. list */ copy->avail_next = bag->avail_head; copy->avail_prev = NULL; bag->avail_head = copy; } } /* Update available list. */ if (bag->avail_head == cursor) bag->avail_head = copy; if (bag->avail_tail == cursor) bag->avail_tail = copy; allocator.free(cursor, sizeof(struct ck_bag_block), true); ck_pr_store_uint(&bag->n_entries, bag->n_entries - 1); return true; }
bool ck_bag_put_spmc(struct ck_bag *bag, void *entry) { struct ck_bag_block *cursor, *new_block, *new_block_prev, *new_tail; uint16_t n_entries_block; size_t blocks_alloc, i; uintptr_t next = 0; new_block = new_block_prev = new_tail = NULL; /* * Blocks with available entries are stored in * the bag's available list. */ cursor = bag->avail_head; if (cursor != NULL) { n_entries_block = ck_bag_block_count(cursor); } else { /* The bag is full, allocate a new set of blocks */ if (bag->alloc_strat == CK_BAG_ALLOCATE_GEOMETRIC) blocks_alloc = (bag->n_blocks + 1) << 1; else blocks_alloc = 1; for (i = 0; i < blocks_alloc-1; i++) { new_block = allocator.malloc(bag->info.bytes); if (new_block == NULL) return false; /* * First node is the tail of the Bag. * Second node is the new tail of the Available list. */ if (i == 0) new_tail = new_block; #ifndef __x86_64__ new_block->next.n_entries = 0; #endif new_block->next.ptr = new_block_prev; new_block->avail_next = new_block_prev; if (new_block_prev != NULL) new_block_prev->avail_prev = new_block; new_block_prev = new_block; } /* * Insert entry into last allocated block. * cursor is new head of available list. */ cursor = allocator.malloc(bag->info.bytes); cursor->avail_next = new_block; cursor->avail_prev = NULL; new_block->avail_prev = cursor; n_entries_block = 0; bag->n_blocks += blocks_alloc; /* n_blocks and n_avail_blocks? */ } /* Update the available list */ if (new_block != NULL) { if (bag->avail_tail != NULL) { cursor->avail_prev = bag->avail_tail; bag->avail_tail->avail_next = cursor; } else { /* Available list was previously empty */ bag->avail_head = cursor; } bag->avail_tail = new_tail; } else if (n_entries_block == bag->info.max-1) { /* New entry will fill up block, remove from avail list */ if (cursor->avail_prev != NULL) cursor->avail_prev->avail_next = cursor->avail_next; if (cursor->avail_next != NULL) cursor->avail_next->avail_prev = cursor->avail_prev; if (bag->avail_head == cursor) bag->avail_head = cursor->avail_next; if (bag->avail_tail == cursor) bag->avail_tail = cursor->avail_prev; /* For debugging purposes */ cursor->avail_next = NULL; cursor->avail_prev = NULL; } /* Update array and block->n_entries */ cursor->array[n_entries_block++] = entry; ck_pr_fence_store(); #ifdef __x86_64__ next = ((uintptr_t)n_entries_block << 48); #endif /* Update bag's list */ if (n_entries_block == 1) { if (bag->head != NULL) { #ifdef __x86_64__ next += ((uintptr_t)(void *)ck_bag_block_next(bag->head)); #else next = (uintptr_t)(void *)ck_bag_block_next(bag->head); #endif } #ifndef __x86_64__ ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); #endif ck_pr_store_ptr(&cursor->next.ptr, (void *)next); ck_pr_store_ptr(&bag->head, cursor); } else { #ifdef __x86_64__ next += ((uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr)); ck_pr_store_ptr(&cursor->next, (void *)next); #else ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); #endif } ck_pr_store_uint(&bag->n_entries, bag->n_entries + 1); return true; }