/*
* heap_cleanup -- cleanups the volatile heap state
*/
void
heap_cleanup(struct palloc_heap *heap)
{
struct heap_rt *rt = heap->rt;
alloc_class_collection_delete(rt->alloc_classes);
bucket_delete(rt->default_bucket);
for (unsigned i = 0; i < rt->ncaches; ++i)
bucket_group_destroy(rt->caches[i].buckets);
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_destroy(&rt->run_locks[i]);
Free(rt->caches);
for (int i = 0; i < MAX_ALLOCATION_CLASSES; ++i) {
recycler_delete(rt->recyclers[i]);
}
VALGRIND_DO_DESTROY_MEMPOOL(heap->layout);
Free(rt);
heap->rt = NULL;
}
/*
* alloc_class_collection_new -- creates a new collection of allocation classes
*/
struct alloc_class_collection *
alloc_class_collection_new()
{
LOG(10, NULL);
struct alloc_class_collection *ac = Zalloc(sizeof(*ac));
if (ac == NULL)
return NULL;
memset(ac->aclasses, 0, sizeof(ac->aclasses));
ac->granularity = ALLOC_BLOCK_SIZE;
ac->last_run_max_size = MAX_RUN_SIZE;
ac->fail_on_missing_class = 0;
ac->autogenerate_on_missing_class = 1;
size_t maps_size = (MAX_RUN_SIZE / ac->granularity) + 1;
if ((ac->class_map_by_alloc_size = Malloc(maps_size)) == NULL)
goto error;
if ((ac->class_map_by_unit_size = cuckoo_new()) == NULL)
goto error;
memset(ac->class_map_by_alloc_size, 0xFF, maps_size);
if (alloc_class_new(-1, ac, CLASS_HUGE, HEADER_COMPACT,
CHUNKSIZE, 0, 1) == NULL)
goto error;
struct alloc_class *predefined_class =
alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT,
MIN_RUN_SIZE, 0, 1);
if (predefined_class == NULL)
goto error;
for (size_t i = 0; i < FIRST_GENERATED_CLASS_SIZE / ac->granularity;
++i) {
ac->class_map_by_alloc_size[i] = predefined_class->id;
}
/*
* Based on the defined categories, a set of allocation classes is
* created. The unit size of those classes is depended on the category
* initial size and step.
*/
size_t granularity_mask = ALLOC_BLOCK_SIZE_GEN - 1;
for (int c = 1; c < MAX_ALLOC_CATEGORIES; ++c) {
size_t n = categories[c - 1].size + ALLOC_BLOCK_SIZE_GEN;
do {
if (alloc_class_find_or_create(ac, n) == NULL)
goto error;
float stepf = (float)n * categories[c].step;
size_t stepi = (size_t)stepf;
stepi = (stepf - (float)stepi < FLT_EPSILON) ?
stepi : stepi + 1;
n += (stepi + (granularity_mask)) & ~granularity_mask;
} while (n <= categories[c].size);
}
/*
* Find the largest alloc class and use it's unit size as run allocation
* threshold.
*/
uint8_t largest_aclass_slot;
for (largest_aclass_slot = MAX_ALLOCATION_CLASSES - 1;
largest_aclass_slot > 0 &&
ac->aclasses[largest_aclass_slot] == NULL;
--largest_aclass_slot) {
/* intentional NOP */
}
struct alloc_class *c = ac->aclasses[largest_aclass_slot];
/*
* The actual run might contain less unit blocks than the theoretical
* unit max variable. This may be the case for very large unit sizes.
*/
size_t real_unit_max = c->run.bitmap_nallocs < RUN_UNIT_MAX_ALLOC ?
c->run.bitmap_nallocs : RUN_UNIT_MAX_ALLOC;
size_t theoretical_run_max_size = c->unit_size * real_unit_max;
ac->last_run_max_size = MAX_RUN_SIZE > theoretical_run_max_size ?
theoretical_run_max_size : MAX_RUN_SIZE;
#ifdef DEBUG
/*
* Verify that each bucket's unit size points back to the bucket by the
* bucket map. This must be true for the default allocation classes,
* otherwise duplicate buckets will be created.
*/
for (size_t i = 0; i < MAX_ALLOCATION_CLASSES; ++i) {
struct alloc_class *c = ac->aclasses[i];
if (c != NULL && c->type == CLASS_RUN) {
ASSERTeq(i, c->id);
ASSERTeq(alloc_class_by_run(ac, c->unit_size,
c->flags, c->run.size_idx), c);
}
}
#endif
return ac;
error:
alloc_class_collection_delete(ac);
return NULL;
}
/*
* heap_boot -- opens the heap region of the pmemobj pool
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_boot(struct palloc_heap *heap, void *heap_start, uint64_t heap_size,
uint64_t run_id, void *base, struct pmem_ops *p_ops)
{
struct heap_rt *h = Malloc(sizeof(*h));
int err;
if (h == NULL) {
err = ENOMEM;
goto error_heap_malloc;
}
h->alloc_classes = alloc_class_collection_new();
if (h->alloc_classes == NULL) {
err = ENOMEM;
goto error_alloc_classes_new;
}
h->ncaches = heap_get_ncaches();
h->caches = Malloc(sizeof(struct bucket_cache) * h->ncaches);
if (h->caches == NULL) {
err = ENOMEM;
goto error_heap_cache_malloc;
}
h->max_zone = heap_max_zone(heap_size);
h->zones_exhausted = 0;
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_init(&h->run_locks[i], NULL);
heap->run_id = run_id;
heap->p_ops = *p_ops;
heap->layout = heap_start;
heap->rt = h;
heap->size = heap_size;
heap->base = base;
VALGRIND_DO_CREATE_MEMPOOL(heap->layout, 0, 0);
for (unsigned i = 0; i < h->ncaches; ++i)
bucket_group_init(h->caches[i].buckets);
size_t rec_i;
for (rec_i = 0; rec_i < MAX_ALLOCATION_CLASSES; ++rec_i) {
if ((h->recyclers[rec_i] = recycler_new(heap)) == NULL) {
err = ENOMEM;
goto error_recycler_new;
}
}
return 0;
error_recycler_new:
Free(h->caches);
for (size_t i = 0; i < rec_i; ++i)
recycler_delete(h->recyclers[i]);
error_heap_cache_malloc:
alloc_class_collection_delete(h->alloc_classes);
error_alloc_classes_new:
Free(h);
heap->rt = NULL;
error_heap_malloc:
return err;
}