static inline as_seeds* swap_seeds(as_cluster* cluster, as_seeds* seeds) { ck_pr_fence_store(); as_seeds* old = ck_pr_fas_ptr(&cluster->seeds, seeds); ck_pr_fence_store(); ck_pr_inc_32(&cluster->version); return old; }
static inline as_addr_maps* swap_ip_map(as_cluster* cluster, as_addr_maps* ip_map) { ck_pr_fence_store(); as_addr_maps* old = ck_pr_fas_ptr(&cluster->ip_map, ip_map); ck_pr_fence_store(); ck_pr_inc_32(&cluster->version); return old; }
void as_cluster_set_async_max_conns_per_node(as_cluster* cluster, uint32_t async_size, uint32_t pipe_size) { // Note: This setting only affects pools in new nodes. Existing node pools are not changed. cluster->async_max_conns_per_node = async_size; cluster->pipe_max_conns_per_node = pipe_size; ck_pr_fence_store(); ck_pr_inc_32(&cluster->version); }
bool ck_array_put(struct ck_array *array, void *value) { struct _ck_array *target; unsigned int size; /* * If no transaction copy has been necessary, attempt to do in-place * modification of the array. */ if (array->transaction == NULL) { target = array->active; if (array->n_entries == target->length) { size = target->length << 1; target = array->allocator->realloc(target, sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, sizeof(struct _ck_array) + sizeof(void *) * size, true); if (target == NULL) return false; ck_pr_store_uint(&target->length, size); /* Serialize with respect to contents. */ ck_pr_fence_store(); ck_pr_store_ptr(&array->active, target); } target->values[array->n_entries++] = value; return true; } target = array->transaction; if (array->n_entries == target->length) { size = target->length << 1; target = array->allocator->realloc(target, sizeof(struct _ck_array) + sizeof(void *) * array->n_entries, sizeof(struct _ck_array) + sizeof(void *) * size, true); if (target == NULL) return false; target->length = size; array->transaction = target; } target->values[array->n_entries++] = value; return false; }
int main(void) { int r = 0; /* Below serves as a marker. */ ck_pr_sub_int(&r, 31337); /* * This is a simple test to help ensure all fences compile or crash * on target. Below are generated according to the underlying memory * model's ordering. */ ck_pr_fence_atomic(); ck_pr_fence_atomic_store(); ck_pr_fence_atomic_load(); ck_pr_fence_store_atomic(); ck_pr_fence_load_atomic(); ck_pr_fence_load(); ck_pr_fence_load_store(); ck_pr_fence_store(); ck_pr_fence_store_load(); ck_pr_fence_memory(); ck_pr_fence_release(); ck_pr_fence_acquire(); ck_pr_fence_acqrel(); ck_pr_fence_lock(); ck_pr_fence_unlock(); /* Below serves as a marker. */ ck_pr_sub_int(&r, 31337); /* The following are generating assuming RMO. */ ck_pr_fence_strict_atomic(); ck_pr_fence_strict_atomic_store(); ck_pr_fence_strict_atomic_load(); ck_pr_fence_strict_store_atomic(); ck_pr_fence_strict_load_atomic(); ck_pr_fence_strict_load(); ck_pr_fence_strict_load_store(); ck_pr_fence_strict_store(); ck_pr_fence_strict_store_load(); ck_pr_fence_strict_memory(); ck_pr_fence_strict_release(); ck_pr_fence_strict_acquire(); ck_pr_fence_strict_acqrel(); ck_pr_fence_strict_lock(); ck_pr_fence_strict_unlock(); return 0; }
bool ck_array_commit(ck_array_t *array) { struct _ck_array *m = array->transaction; if (m != NULL) { struct _ck_array *p; m->n_committed = array->n_entries; ck_pr_fence_store(); p = array->active; ck_pr_store_ptr(&array->active, m); array->allocator->free(p, sizeof(struct _ck_array) + p->length * sizeof(void *), true); array->transaction = NULL; return true; } ck_pr_fence_store(); ck_pr_store_uint(&array->active->n_committed, array->n_entries); return true; }
static void *ph_thread_boot(void *arg) { struct ph_thread_boot_data data; ph_thread_t *me; void *retval; /* copy in the boot data from the stack of our creator */ memcpy(&data, arg, sizeof(data)); me = ph_thread_init_myself(true); /* this publishes that we're ready to run to * the thread that spawned us */ ck_pr_store_ptr(data.thr, ck_pr_load_ptr(&me)); ck_pr_fence_store(); retval = data.func(data.arg); ck_epoch_barrier(&me->epoch_record); return retval; }
static void ck_barrier_combining_aux(struct ck_barrier_combining *barrier, struct ck_barrier_combining_group *tnode, unsigned int sense) { /* * If this is the last thread in the group, it moves on to the parent group. * Otherwise, it spins on this group's sense. */ if (ck_pr_faa_uint(&tnode->count, 1) == tnode->k - 1) { /* * If we are and will be the last thread entering the barrier for the * current group then signal the parent group if one exists. */ if (tnode->parent != NULL) ck_barrier_combining_aux(barrier, tnode->parent, sense); /* * Once the thread returns from its parent(s), it reinitializes the group's * arrival count and signals other threads to continue by flipping the group * sense. Order of these operations is not important since we assume a static * number of threads are members of a barrier for the lifetime of the barrier. * Since count is explicitly reinitialized, it is guaranteed that at any point * tnode->count is equivalent to tnode->k if and only if that many threads * are at the barrier. */ ck_pr_store_uint(&tnode->count, 0); ck_pr_fence_store(); ck_pr_store_uint(&tnode->sense, ~tnode->sense); } else { ck_pr_fence_memory(); while (sense != ck_pr_load_uint(&tnode->sense)) ck_pr_stall(); } return; }
static struct ck_hs_map * ck_hs_map_create(struct ck_hs *hs, unsigned long entries) { struct ck_hs_map *map; unsigned long size, n_entries, limit; n_entries = ck_internal_power_2(entries); size = sizeof(struct ck_hs_map) + (sizeof(void *) * n_entries + CK_MD_CACHELINE - 1); map = hs->m->malloc(size); if (map == NULL) return NULL; map->size = size; /* We should probably use a more intelligent heuristic for default probe length. */ limit = ck_internal_max(n_entries >> (CK_HS_PROBE_L1_SHIFT + 2), CK_HS_PROBE_L1_DEFAULT); if (limit > UINT_MAX) limit = UINT_MAX; map->probe_limit = (unsigned int)limit; map->probe_maximum = 0; map->capacity = n_entries; map->step = ck_internal_bsf(n_entries); map->mask = n_entries - 1; map->n_entries = 0; /* Align map allocation to cache line. */ map->entries = (void *)(((uintptr_t)(map + 1) + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); memset(map->entries, 0, sizeof(void *) * n_entries); memset(map->generation, 0, sizeof map->generation); /* Commit entries purge with respect to map publication. */ ck_pr_fence_store(); return map; }
static inline void ck_rhs_map_bound_set(struct ck_rhs_map *m, unsigned long h, unsigned long n_probes) { unsigned long offset = h & m->mask; struct ck_rhs_entry_desc *desc; if (n_probes > m->probe_maximum) ck_pr_store_uint(&m->probe_maximum, n_probes); if (!(m->read_mostly)) { desc = &m->entries.descs[offset]; if (desc->probe_bound < n_probes) { if (n_probes > CK_RHS_WORD_MAX) n_probes = CK_RHS_WORD_MAX; CK_RHS_STORE(&desc->probe_bound, n_probes); ck_pr_fence_store(); } } return; }
bool ck_hs_grow(struct ck_hs *hs, unsigned long capacity) { struct ck_hs_map *map, *update; void **bucket, *previous; unsigned long k, i, j, offset, probes; restart: map = hs->map; if (map->capacity > capacity) return false; update = ck_hs_map_create(hs, capacity); if (update == NULL) return false; for (k = 0; k < map->capacity; k++) { unsigned long h; previous = map->entries[k]; if (previous == CK_HS_EMPTY || previous == CK_HS_TOMBSTONE) continue; #ifdef CK_HS_PP if (hs->mode & CK_HS_MODE_OBJECT) previous = (void *)((uintptr_t)previous & (((uintptr_t)1 << CK_MD_VMA_BITS) - 1)); #endif h = hs->hf(previous, hs->seed); offset = h & update->mask; i = probes = 0; for (;;) { bucket = (void *)((uintptr_t)&update->entries[offset] & ~(CK_MD_CACHELINE - 1)); for (j = 0; j < CK_HS_PROBE_L1; j++) { void **cursor = bucket + ((j + offset) & (CK_HS_PROBE_L1 - 1)); if (probes++ == update->probe_limit) break; if (CK_CC_LIKELY(*cursor == CK_HS_EMPTY)) { *cursor = map->entries[k]; update->n_entries++; if (probes > update->probe_maximum) update->probe_maximum = probes; break; } } if (j < CK_HS_PROBE_L1) break; offset = ck_hs_map_probe_next(update, offset, h, i++, probes); } if (probes > update->probe_limit) { /* * We have hit the probe limit, map needs to be even larger. */ ck_hs_map_destroy(hs->m, update, false); capacity <<= 1; goto restart; } } ck_pr_fence_store(); ck_pr_store_ptr(&hs->map, update); ck_hs_map_destroy(hs->m, map, true); return true; }
static inline void set_nodes(as_cluster* cluster, as_nodes* nodes) { ck_pr_fence_store(); ck_pr_store_ptr(&cluster->nodes, nodes); }
bool ck_rhs_grow(struct ck_rhs *hs, unsigned long capacity) { struct ck_rhs_map *map, *update; void *previous, *prev_saved; unsigned long k, offset, probes; restart: map = hs->map; if (map->capacity > capacity) return false; update = ck_rhs_map_create(hs, capacity); if (update == NULL) return false; for (k = 0; k < map->capacity; k++) { unsigned long h; prev_saved = previous = ck_rhs_entry(map, k); if (previous == CK_RHS_EMPTY) continue; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) previous = CK_RHS_VMA(previous); #endif h = hs->hf(previous, hs->seed); offset = h & update->mask; probes = 0; for (;;) { void **cursor = ck_rhs_entry_addr(update, offset); if (probes++ == update->probe_limit) { /* * We have hit the probe limit, map needs to be even larger. */ ck_rhs_map_destroy(hs->m, update, false); capacity <<= 1; goto restart; } if (CK_CC_LIKELY(*cursor == CK_RHS_EMPTY)) { *cursor = prev_saved; update->n_entries++; ck_rhs_set_probes(update, offset, probes); ck_rhs_map_bound_set(update, h, probes); break; } else if (ck_rhs_probes(update, offset) < probes) { void *tmp = prev_saved; unsigned int old_probes; prev_saved = previous = *cursor; #ifdef CK_RHS_PP if (hs->mode & CK_RHS_MODE_OBJECT) previous = CK_RHS_VMA(previous); #endif *cursor = tmp; ck_rhs_map_bound_set(update, h, probes); h = hs->hf(previous, hs->seed); old_probes = ck_rhs_probes(update, offset); ck_rhs_set_probes(update, offset, probes); probes = old_probes - 1; continue; } ck_rhs_wanted_inc(update, offset); offset = ck_rhs_map_probe_next(update, offset, probes); } } ck_pr_fence_store(); ck_pr_store_ptr(&hs->map, update); ck_rhs_map_destroy(hs->m, map, true); return true; }
static struct ck_rhs_map * ck_rhs_map_create(struct ck_rhs *hs, unsigned long entries) { struct ck_rhs_map *map; unsigned long size, n_entries, limit; n_entries = ck_internal_power_2(entries); if (n_entries < CK_RHS_PROBE_L1) return NULL; if (hs->mode & CK_RHS_MODE_READ_MOSTLY) size = sizeof(struct ck_rhs_map) + (sizeof(void *) * n_entries + sizeof(struct ck_rhs_no_entry_desc) * n_entries + 2 * CK_MD_CACHELINE - 1); else size = sizeof(struct ck_rhs_map) + (sizeof(struct ck_rhs_entry_desc) * n_entries + CK_MD_CACHELINE - 1); map = hs->m->malloc(size); if (map == NULL) return NULL; map->read_mostly = !!(hs->mode & CK_RHS_MODE_READ_MOSTLY); map->size = size; /* We should probably use a more intelligent heuristic for default probe length. */ limit = ck_internal_max(n_entries >> (CK_RHS_PROBE_L1_SHIFT + 2), CK_RHS_PROBE_L1_DEFAULT); if (limit > UINT_MAX) limit = UINT_MAX; map->probe_limit = (unsigned int)limit; map->probe_maximum = 0; map->capacity = n_entries; map->step = ck_internal_bsf(n_entries); map->mask = n_entries - 1; map->n_entries = 0; /* Align map allocation to cache line. */ if (map->read_mostly) { map->entries.no_entries.entries = (void *)(((uintptr_t)&map[1] + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); map->entries.no_entries.descs = (void *)(((uintptr_t)map->entries.no_entries.entries + (sizeof(void *) * n_entries) + CK_MD_CACHELINE - 1) &~ (CK_MD_CACHELINE - 1)); memset(map->entries.no_entries.entries, 0, sizeof(void *) * n_entries); memset(map->entries.no_entries.descs, 0, sizeof(struct ck_rhs_no_entry_desc)); map->offset_mask = (CK_MD_CACHELINE / sizeof(void *)) - 1; map->probe_func = ck_rhs_map_probe_rm; } else { map->entries.descs = (void *)(((uintptr_t)&map[1] + CK_MD_CACHELINE - 1) & ~(CK_MD_CACHELINE - 1)); memset(map->entries.descs, 0, sizeof(struct ck_rhs_entry_desc) * n_entries); map->offset_mask = (CK_MD_CACHELINE / sizeof(struct ck_rhs_entry_desc)) - 1; map->probe_func = ck_rhs_map_probe; } memset(map->generation, 0, sizeof map->generation); /* Commit entries purge with respect to map publication. */ ck_pr_fence_store(); return map; }
bool ck_bag_remove_spmc(struct ck_bag *bag, void *entry) { struct ck_bag_block *cursor, *copy, *prev; uint16_t block_index, n_entries; cursor = bag->head; prev = NULL; while (cursor != NULL) { n_entries = ck_bag_block_count(cursor); for (block_index = 0; block_index < n_entries; block_index++) { if (cursor->array[block_index] == entry) goto found; } prev = cursor; cursor = ck_bag_block_next(cursor->next.ptr); } return true; found: /* Cursor points to containing block, block_index is index of deletion */ if (n_entries == 1) { /* If a block's single entry is being removed, remove the block. */ if (prev == NULL) { struct ck_bag_block *new_head = ck_bag_block_next(cursor->next.ptr); ck_pr_store_ptr(&bag->head, new_head); } else { uintptr_t next; #ifdef __x86_64__ next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) | (uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr); #else next = (uintptr_t)(void *)cursor->next.ptr; #endif ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next); } /* Remove block from available list */ if (cursor->avail_prev != NULL) cursor->avail_prev->avail_next = cursor->avail_next; if (cursor->avail_next != NULL) cursor->avail_next->avail_prev = cursor->avail_prev; bag->n_blocks--; copy = cursor->avail_next; } else { uintptr_t next_ptr; copy = allocator.malloc(bag->info.bytes); if (copy == NULL) return false; memcpy(copy, cursor, bag->info.bytes); copy->array[block_index] = copy->array[--n_entries]; next_ptr = (uintptr_t)(void *)ck_bag_block_next(copy->next.ptr); #ifdef __x86_64__ copy->next.ptr = (void *)(((uintptr_t)n_entries << 48) | next_ptr); #else copy->next.n_entries = n_entries; copy->next.ptr = (struct ck_bag_block *)next_ptr; #endif ck_pr_fence_store(); if (prev == NULL) { ck_pr_store_ptr(&bag->head, copy); } else { #ifdef __x86_64__ uintptr_t next = ((uintptr_t)prev->next.ptr & (CK_BAG_BLOCK_ENTRIES_MASK)) | (uintptr_t)(void *)ck_bag_block_next(copy); ck_pr_store_ptr(&prev->next.ptr, (struct ck_bag_block *)next); #else ck_pr_store_ptr(&prev->next.ptr, copy); #endif } if (n_entries == bag->info.max - 1) { /* Block was previously fully, add to head of avail. list */ copy->avail_next = bag->avail_head; copy->avail_prev = NULL; bag->avail_head = copy; } } /* Update available list. */ if (bag->avail_head == cursor) bag->avail_head = copy; if (bag->avail_tail == cursor) bag->avail_tail = copy; allocator.free(cursor, sizeof(struct ck_bag_block), true); ck_pr_store_uint(&bag->n_entries, bag->n_entries - 1); return true; }
bool ck_bag_put_spmc(struct ck_bag *bag, void *entry) { struct ck_bag_block *cursor, *new_block, *new_block_prev, *new_tail; uint16_t n_entries_block; size_t blocks_alloc, i; uintptr_t next = 0; new_block = new_block_prev = new_tail = NULL; /* * Blocks with available entries are stored in * the bag's available list. */ cursor = bag->avail_head; if (cursor != NULL) { n_entries_block = ck_bag_block_count(cursor); } else { /* The bag is full, allocate a new set of blocks */ if (bag->alloc_strat == CK_BAG_ALLOCATE_GEOMETRIC) blocks_alloc = (bag->n_blocks + 1) << 1; else blocks_alloc = 1; for (i = 0; i < blocks_alloc-1; i++) { new_block = allocator.malloc(bag->info.bytes); if (new_block == NULL) return false; /* * First node is the tail of the Bag. * Second node is the new tail of the Available list. */ if (i == 0) new_tail = new_block; #ifndef __x86_64__ new_block->next.n_entries = 0; #endif new_block->next.ptr = new_block_prev; new_block->avail_next = new_block_prev; if (new_block_prev != NULL) new_block_prev->avail_prev = new_block; new_block_prev = new_block; } /* * Insert entry into last allocated block. * cursor is new head of available list. */ cursor = allocator.malloc(bag->info.bytes); cursor->avail_next = new_block; cursor->avail_prev = NULL; new_block->avail_prev = cursor; n_entries_block = 0; bag->n_blocks += blocks_alloc; /* n_blocks and n_avail_blocks? */ } /* Update the available list */ if (new_block != NULL) { if (bag->avail_tail != NULL) { cursor->avail_prev = bag->avail_tail; bag->avail_tail->avail_next = cursor; } else { /* Available list was previously empty */ bag->avail_head = cursor; } bag->avail_tail = new_tail; } else if (n_entries_block == bag->info.max-1) { /* New entry will fill up block, remove from avail list */ if (cursor->avail_prev != NULL) cursor->avail_prev->avail_next = cursor->avail_next; if (cursor->avail_next != NULL) cursor->avail_next->avail_prev = cursor->avail_prev; if (bag->avail_head == cursor) bag->avail_head = cursor->avail_next; if (bag->avail_tail == cursor) bag->avail_tail = cursor->avail_prev; /* For debugging purposes */ cursor->avail_next = NULL; cursor->avail_prev = NULL; } /* Update array and block->n_entries */ cursor->array[n_entries_block++] = entry; ck_pr_fence_store(); #ifdef __x86_64__ next = ((uintptr_t)n_entries_block << 48); #endif /* Update bag's list */ if (n_entries_block == 1) { if (bag->head != NULL) { #ifdef __x86_64__ next += ((uintptr_t)(void *)ck_bag_block_next(bag->head)); #else next = (uintptr_t)(void *)ck_bag_block_next(bag->head); #endif } #ifndef __x86_64__ ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); #endif ck_pr_store_ptr(&cursor->next.ptr, (void *)next); ck_pr_store_ptr(&bag->head, cursor); } else { #ifdef __x86_64__ next += ((uintptr_t)(void *)ck_bag_block_next(cursor->next.ptr)); ck_pr_store_ptr(&cursor->next, (void *)next); #else ck_pr_store_ptr(&cursor->next.n_entries, (void *)(uintptr_t)n_entries_block); #endif } ck_pr_store_uint(&bag->n_entries, bag->n_entries + 1); return true; }