/*
* heap_cleanup -- cleanups the volatile heap state
*/
void
heap_cleanup(struct palloc_heap *heap)
{
struct heap_rt *rt = heap->rt;
alloc_class_collection_delete(rt->alloc_classes);
bucket_delete(rt->default_bucket);
for (unsigned i = 0; i < rt->ncaches; ++i)
bucket_group_destroy(rt->caches[i].buckets);
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_destroy(&rt->run_locks[i]);
Free(rt->caches);
for (int i = 0; i < MAX_ALLOCATION_CLASSES; ++i) {
recycler_delete(rt->recyclers[i]);
}
VALGRIND_DO_DESTROY_MEMPOOL(heap->layout);
Free(rt);
heap->rt = NULL;
}
static void
test_recycler(void)
{
struct mock_pop *mpop = MMAP_ANON_ALIGNED(MOCK_POOL_SIZE,
Ut_mmap_align);
PMEMobjpool *pop = &mpop->p;
memset(pop, 0, MOCK_POOL_SIZE);
pop->heap_offset = (uint64_t)((uint64_t)&mpop->heap - (uint64_t)mpop);
pop->p_ops.persist = obj_heap_persist;
pop->p_ops.memset_persist = obj_heap_memset_persist;
pop->p_ops.base = pop;
pop->set = MALLOC(sizeof(*(pop->set)));
pop->set->options = 0;
pop->set->directory_based = 0;
void *heap_start = (char *)pop + pop->heap_offset;
uint64_t heap_size = MOCK_POOL_SIZE - sizeof(PMEMobjpool);
struct palloc_heap *heap = &pop->heap;
struct pmem_ops *p_ops = &pop->p_ops;
struct stats *s = stats_new(pop);
UT_ASSERTne(s, NULL);
UT_ASSERT(heap_check(heap_start, heap_size) != 0);
UT_ASSERT(heap_init(heap_start, heap_size,
&pop->heap_size, p_ops) == 0);
UT_ASSERT(heap_boot(heap, heap_start, heap_size,
&pop->heap_size,
pop, p_ops, s, pop->set) == 0);
UT_ASSERT(heap_buckets_init(heap) == 0);
UT_ASSERT(pop->heap.rt != NULL);
/* trigger heap bucket populate */
struct memory_block m = MEMORY_BLOCK_NONE;
m.size_idx = 1;
struct bucket *b = heap_bucket_acquire_by_id(heap,
DEFAULT_ALLOC_CLASS_ID);
UT_ASSERT(heap_get_bestfit_block(heap, b, &m) == 0);
heap_bucket_release(heap, b);
int ret;
struct recycler *r = recycler_new(&pop->heap, 10000 /* never recalc */);
UT_ASSERTne(r, NULL);
init_run_with_score(pop->heap.layout, 0, 64);
init_run_with_score(pop->heap.layout, 1, 128);
init_run_with_score(pop->heap.layout, 15, 0);
struct memory_block mrun = {0, 0, 1, 0};
struct memory_block mrun2 = {1, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun);
memblock_rebuild_state(&pop->heap, &mrun2);
ret = recycler_put(r, &mrun,
recycler_calc_score(&pop->heap, &mrun, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun2,
recycler_calc_score(&pop->heap, &mrun2, NULL));
UT_ASSERTeq(ret, 0);
struct memory_block mrun_ret = MEMORY_BLOCK_NONE;
mrun_ret.size_idx = 1;
struct memory_block mrun2_ret = MEMORY_BLOCK_NONE;
mrun2_ret.size_idx = 1;
ret = recycler_get(r, &mrun_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun2_ret);
UT_ASSERTeq(ret, 0);
UT_ASSERTeq(mrun2.chunk_id, mrun2_ret.chunk_id);
UT_ASSERTeq(mrun.chunk_id, mrun_ret.chunk_id);
init_run_with_score(pop->heap.layout, 7, 256);
init_run_with_score(pop->heap.layout, 2, 64);
init_run_with_score(pop->heap.layout, 5, 512);
init_run_with_score(pop->heap.layout, 10, 128);
mrun.chunk_id = 7;
mrun2.chunk_id = 2;
struct memory_block mrun3 = {5, 0, 1, 0};
struct memory_block mrun4 = {10, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun3);
memblock_rebuild_state(&pop->heap, &mrun4);
mrun_ret.size_idx = 1;
mrun2_ret.size_idx = 1;
struct memory_block mrun3_ret = MEMORY_BLOCK_NONE;
mrun3_ret.size_idx = 1;
struct memory_block mrun4_ret = MEMORY_BLOCK_NONE;
mrun4_ret.size_idx = 1;
ret = recycler_put(r, &mrun,
recycler_calc_score(&pop->heap, &mrun, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun2,
recycler_calc_score(&pop->heap, &mrun2, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun3,
recycler_calc_score(&pop->heap, &mrun3, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_put(r, &mrun4,
recycler_calc_score(&pop->heap, &mrun4, NULL));
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun2_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun4_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun_ret);
UT_ASSERTeq(ret, 0);
ret = recycler_get(r, &mrun3_ret);
UT_ASSERTeq(ret, 0);
UT_ASSERTeq(mrun.chunk_id, mrun_ret.chunk_id);
UT_ASSERTeq(mrun2.chunk_id, mrun2_ret.chunk_id);
UT_ASSERTeq(mrun3.chunk_id, mrun3_ret.chunk_id);
UT_ASSERTeq(mrun4.chunk_id, mrun4_ret.chunk_id);
init_run_with_max_block(pop->heap.layout, 1);
struct memory_block mrun5 = {1, 0, 1, 0};
memblock_rebuild_state(&pop->heap, &mrun5);
ret = recycler_put(r, &mrun5,
recycler_calc_score(&pop->heap, &mrun5, NULL));
UT_ASSERTeq(ret, 0);
struct memory_block mrun5_ret = MEMORY_BLOCK_NONE;
mrun5_ret.size_idx = 11;
ret = recycler_get(r, &mrun5_ret);
UT_ASSERTeq(ret, ENOMEM);
mrun5_ret = MEMORY_BLOCK_NONE;
mrun5_ret.size_idx = 10;
ret = recycler_get(r, &mrun5_ret);
UT_ASSERTeq(ret, 0);
recycler_delete(r);
stats_delete(pop, s);
heap_cleanup(heap);
UT_ASSERT(heap->rt == NULL);
FREE(pop->set);
MUNMAP_ANON_ALIGNED(mpop, MOCK_POOL_SIZE);
}
/*
* heap_boot -- opens the heap region of the pmemobj pool
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_boot(struct palloc_heap *heap, void *heap_start, uint64_t heap_size,
uint64_t run_id, void *base, struct pmem_ops *p_ops)
{
struct heap_rt *h = Malloc(sizeof(*h));
int err;
if (h == NULL) {
err = ENOMEM;
goto error_heap_malloc;
}
h->alloc_classes = alloc_class_collection_new();
if (h->alloc_classes == NULL) {
err = ENOMEM;
goto error_alloc_classes_new;
}
h->ncaches = heap_get_ncaches();
h->caches = Malloc(sizeof(struct bucket_cache) * h->ncaches);
if (h->caches == NULL) {
err = ENOMEM;
goto error_heap_cache_malloc;
}
h->max_zone = heap_max_zone(heap_size);
h->zones_exhausted = 0;
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_init(&h->run_locks[i], NULL);
heap->run_id = run_id;
heap->p_ops = *p_ops;
heap->layout = heap_start;
heap->rt = h;
heap->size = heap_size;
heap->base = base;
VALGRIND_DO_CREATE_MEMPOOL(heap->layout, 0, 0);
for (unsigned i = 0; i < h->ncaches; ++i)
bucket_group_init(h->caches[i].buckets);
size_t rec_i;
for (rec_i = 0; rec_i < MAX_ALLOCATION_CLASSES; ++rec_i) {
if ((h->recyclers[rec_i] = recycler_new(heap)) == NULL) {
err = ENOMEM;
goto error_recycler_new;
}
}
return 0;
error_recycler_new:
Free(h->caches);
for (size_t i = 0; i < rec_i; ++i)
recycler_delete(h->recyclers[i]);
error_heap_cache_malloc:
alloc_class_collection_delete(h->alloc_classes);
error_alloc_classes_new:
Free(h);
heap->rt = NULL;
error_heap_malloc:
return err;
}
