/*
* heap_check -- verifies if the heap is consistent and can be opened properly
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_check(void *heap_start, uint64_t heap_size)
{
if (heap_size < HEAP_MIN_SIZE) {
ERR("heap: invalid heap size");
return -1;
}
struct heap_layout *layout = heap_start;
if (heap_size != layout->header.size) {
ERR("heap: heap size missmatch");
return -1;
}
if (heap_verify_header(&layout->header))
return -1;
for (unsigned i = 0; i < heap_max_zone(layout->header.size); ++i) {
if (heap_verify_zone(ZID_TO_ZONE(layout, i)))
return -1;
}
return 0;
}
/*
* heap_init -- initializes the heap
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_init(void *heap_start, uint64_t heap_size, struct pmem_ops *p_ops)
{
if (heap_size < HEAP_MIN_SIZE)
return EINVAL;
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap_start, heap_size);
struct heap_layout *layout = heap_start;
heap_write_header(&layout->header, heap_size);
pmemops_persist(p_ops, &layout->header, sizeof(struct heap_header));
unsigned zones = heap_max_zone(heap_size);
for (unsigned i = 0; i < zones; ++i) {
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->header,
0, sizeof(struct zone_header));
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->chunk_headers,
0, sizeof(struct chunk_header));
/* only explicitly allocated chunks should be accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(
&ZID_TO_ZONE(layout, i)->chunk_headers,
sizeof(struct chunk_header));
}
return 0;
}
/*
* heap_init -- initializes the heap
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_init(PMEMobjpool *pop)
{
if (pop->heap_size < HEAP_MIN_SIZE)
return EINVAL;
struct heap_layout *layout = heap_get_layout(pop);
heap_write_header(&layout->header, pop->heap_size);
pmem_msync(&layout->header, sizeof (struct heap_header));
int zones = heap_max_zone(pop->heap_size);
for (int i = 0; i < zones; ++i) {
memset(&layout->zones[i].header, 0,
sizeof (layout->zones[i].header));
memset(&layout->zones[i].chunk_headers, 0,
sizeof (layout->zones[i].chunk_headers));
pmem_msync(&layout->zones[i].header,
sizeof (layout->zones[i].header));
pmem_msync(&layout->zones[i].chunk_headers,
sizeof (layout->zones[i].chunk_headers));
}
return 0;
}
/*
* heap_boot -- opens the heap region of the pmemobj pool
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_boot(PMEMobjpool *pop)
{
struct pmalloc_heap *h = Malloc(sizeof (*h));
int err;
if (h == NULL) {
err = ENOMEM;
goto error_heap_malloc;
}
h->max_zone = heap_max_zone(pop->heap_size);
h->zones_exhausted = 0;
h->layout = heap_get_layout(pop);
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
if ((err = pthread_mutex_init(&h->run_locks[i], NULL)) != 0)
goto error_run_lock_init;
pop->heap = h;
if ((err = heap_buckets_init(pop)) != 0)
goto error_buckets_init;
return 0;
error_buckets_init:
/* there's really no point in destroying the locks */
error_run_lock_init:
Free(h);
pop->heap = NULL;
error_heap_malloc:
return err;
}
/*
* heap_check -- verifies if the heap is consistent and can be opened properly
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_check(PMEMobjpool *pop)
{
if (pop->heap_size < HEAP_MIN_SIZE) {
ERR("heap: invalid heap size");
return -1;
}
struct heap_layout *layout = heap_get_layout(pop);
if (pop->heap_size != layout->header.size) {
ERR("heap: heap size missmatch");
return -1;
}
if (heap_verify_header(&layout->header))
return -1;
for (int i = 0; i < heap_max_zone(layout->header.size); ++i) {
if (heap_verify_zone(&layout->zones[i]))
return -1;
}
return 0;
}
/*
* heap_foreach_object -- (internal) iterates through objects in the heap
*/
void
heap_foreach_object(struct palloc_heap *heap, object_callback cb, void *arg,
struct memory_block m)
{
struct heap_layout *layout = heap->layout;
for (; m.zone_id < heap_max_zone(layout->header.size); ++m.zone_id) {
if (heap_zone_foreach_object(heap, cb, arg, &m) != 0)
break;
m.chunk_id = 0;
}
}
/*
* heap_check_remote -- verifies if the heap of a remote pool is consistent
* and can be opened properly
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_check_remote(void *heap_start, uint64_t heap_size, struct remote_ops *ops)
{
if (heap_size < HEAP_MIN_SIZE) {
ERR("heap: invalid heap size");
return -1;
}
struct heap_layout *layout = heap_start;
struct heap_header header;
if (ops->read(ops->ctx, ops->base, &header, &layout->header,
sizeof(struct heap_header))) {
ERR("heap: obj_read_remote error");
return -1;
}
if (heap_size != header.size) {
ERR("heap: heap size mismatch");
return -1;
}
if (heap_verify_header(&header))
return -1;
struct zone *zone_buff = (struct zone *)Malloc(sizeof(struct zone));
if (zone_buff == NULL) {
ERR("heap: zone_buff malloc error");
return -1;
}
for (unsigned i = 0; i < heap_max_zone(header.size); ++i) {
if (ops->read(ops->ctx, ops->base, zone_buff,
ZID_TO_ZONE(layout, i), sizeof(struct zone))) {
ERR("heap: obj_read_remote error");
goto out;
}
if (heap_verify_zone(zone_buff)) {
goto out;
}
}
Free(zone_buff);
return 0;
out:
Free(zone_buff);
return -1;
}
/*
* heap_vg_open -- notifies Valgrind about heap layout
*/
void
heap_vg_open(struct palloc_heap *heap, object_callback cb,
void *arg, int objects)
{
ASSERTne(cb, NULL);
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap->layout, heap->size);
struct heap_layout *layout = heap->layout;
VALGRIND_DO_MAKE_MEM_DEFINED(&layout->header, sizeof(layout->header));
unsigned zones = heap_max_zone(heap->size);
struct memory_block m = MEMORY_BLOCK_NONE;
for (unsigned i = 0; i < zones; ++i) {
struct zone *z = ZID_TO_ZONE(layout, i);
uint32_t chunks;
m.zone_id = i;
m.chunk_id = 0;
VALGRIND_DO_MAKE_MEM_DEFINED(&z->header, sizeof(z->header));
if (z->header.magic != ZONE_HEADER_MAGIC)
continue;
chunks = z->header.size_idx;
for (uint32_t c = 0; c < chunks; ) {
struct chunk_header *hdr = &z->chunk_headers[c];
m.chunk_id = c;
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
m.size_idx = hdr->size_idx;
heap_vg_open_chunk(heap, cb, arg, objects, &m);
m.block_off = 0;
ASSERT(hdr->size_idx > 0);
if (hdr->type == CHUNK_TYPE_RUN) {
/*
* Mark run data headers as defined.
*/
for (unsigned j = 1; j < hdr->size_idx; ++j) {
struct chunk_header *data_hdr =
&z->chunk_headers[c + j];
VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr,
sizeof(struct chunk_header));
ASSERTeq(data_hdr->type,
CHUNK_TYPE_RUN_DATA);
}
} else {
/*
* Mark unused chunk headers as not accessible.
*/
VALGRIND_DO_MAKE_MEM_NOACCESS(
&z->chunk_headers[c + 1],
(hdr->size_idx - 1) *
sizeof(struct chunk_header));
}
c += hdr->size_idx;
}
/* mark all unused chunk headers after last as not accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(&z->chunk_headers[chunks],
(MAX_CHUNK - chunks) * sizeof(struct chunk_header));
}
}
/*
* heap_boot -- opens the heap region of the pmemobj pool
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_boot(struct palloc_heap *heap, void *heap_start, uint64_t heap_size,
uint64_t run_id, void *base, struct pmem_ops *p_ops)
{
struct heap_rt *h = Malloc(sizeof(*h));
int err;
if (h == NULL) {
err = ENOMEM;
goto error_heap_malloc;
}
h->alloc_classes = alloc_class_collection_new();
if (h->alloc_classes == NULL) {
err = ENOMEM;
goto error_alloc_classes_new;
}
h->ncaches = heap_get_ncaches();
h->caches = Malloc(sizeof(struct bucket_cache) * h->ncaches);
if (h->caches == NULL) {
err = ENOMEM;
goto error_heap_cache_malloc;
}
h->max_zone = heap_max_zone(heap_size);
h->zones_exhausted = 0;
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_init(&h->run_locks[i], NULL);
heap->run_id = run_id;
heap->p_ops = *p_ops;
heap->layout = heap_start;
heap->rt = h;
heap->size = heap_size;
heap->base = base;
VALGRIND_DO_CREATE_MEMPOOL(heap->layout, 0, 0);
for (unsigned i = 0; i < h->ncaches; ++i)
bucket_group_init(h->caches[i].buckets);
size_t rec_i;
for (rec_i = 0; rec_i < MAX_ALLOCATION_CLASSES; ++rec_i) {
if ((h->recyclers[rec_i] = recycler_new(heap)) == NULL) {
err = ENOMEM;
goto error_recycler_new;
}
}
return 0;
error_recycler_new:
Free(h->caches);
for (size_t i = 0; i < rec_i; ++i)
recycler_delete(h->recyclers[i]);
error_heap_cache_malloc:
alloc_class_collection_delete(h->alloc_classes);
error_alloc_classes_new:
Free(h);
heap->rt = NULL;
error_heap_malloc:
return err;
}