/* * heap_check -- verifies if the heap is consistent and can be opened properly * * If successful function returns zero. Otherwise an error number is returned. */ int heap_check(void *heap_start, uint64_t heap_size) { if (heap_size < HEAP_MIN_SIZE) { ERR("heap: invalid heap size"); return -1; } struct heap_layout *layout = heap_start; if (heap_size != layout->header.size) { ERR("heap: heap size missmatch"); return -1; } if (heap_verify_header(&layout->header)) return -1; for (unsigned i = 0; i < heap_max_zone(layout->header.size); ++i) { if (heap_verify_zone(ZID_TO_ZONE(layout, i))) return -1; } return 0; }
/* * heap_init -- initializes the heap * * If successful function returns zero. Otherwise an error number is returned. */ int heap_init(void *heap_start, uint64_t heap_size, struct pmem_ops *p_ops) { if (heap_size < HEAP_MIN_SIZE) return EINVAL; VALGRIND_DO_MAKE_MEM_UNDEFINED(heap_start, heap_size); struct heap_layout *layout = heap_start; heap_write_header(&layout->header, heap_size); pmemops_persist(p_ops, &layout->header, sizeof(struct heap_header)); unsigned zones = heap_max_zone(heap_size); for (unsigned i = 0; i < zones; ++i) { pmemops_memset_persist(p_ops, &ZID_TO_ZONE(layout, i)->header, 0, sizeof(struct zone_header)); pmemops_memset_persist(p_ops, &ZID_TO_ZONE(layout, i)->chunk_headers, 0, sizeof(struct chunk_header)); /* only explicitly allocated chunks should be accessible */ VALGRIND_DO_MAKE_MEM_NOACCESS( &ZID_TO_ZONE(layout, i)->chunk_headers, sizeof(struct chunk_header)); } return 0; }
/* * heap_init -- initializes the heap * * If successful function returns zero. Otherwise an error number is returned. */ int heap_init(PMEMobjpool *pop) { if (pop->heap_size < HEAP_MIN_SIZE) return EINVAL; struct heap_layout *layout = heap_get_layout(pop); heap_write_header(&layout->header, pop->heap_size); pmem_msync(&layout->header, sizeof (struct heap_header)); int zones = heap_max_zone(pop->heap_size); for (int i = 0; i < zones; ++i) { memset(&layout->zones[i].header, 0, sizeof (layout->zones[i].header)); memset(&layout->zones[i].chunk_headers, 0, sizeof (layout->zones[i].chunk_headers)); pmem_msync(&layout->zones[i].header, sizeof (layout->zones[i].header)); pmem_msync(&layout->zones[i].chunk_headers, sizeof (layout->zones[i].chunk_headers)); } return 0; }
/* * heap_boot -- opens the heap region of the pmemobj pool * * If successful function returns zero. Otherwise an error number is returned. */ int heap_boot(PMEMobjpool *pop) { struct pmalloc_heap *h = Malloc(sizeof (*h)); int err; if (h == NULL) { err = ENOMEM; goto error_heap_malloc; } h->max_zone = heap_max_zone(pop->heap_size); h->zones_exhausted = 0; h->layout = heap_get_layout(pop); for (int i = 0; i < MAX_RUN_LOCKS; ++i) if ((err = pthread_mutex_init(&h->run_locks[i], NULL)) != 0) goto error_run_lock_init; pop->heap = h; if ((err = heap_buckets_init(pop)) != 0) goto error_buckets_init; return 0; error_buckets_init: /* there's really no point in destroying the locks */ error_run_lock_init: Free(h); pop->heap = NULL; error_heap_malloc: return err; }
/* * heap_check -- verifies if the heap is consistent and can be opened properly * * If successful function returns zero. Otherwise an error number is returned. */ int heap_check(PMEMobjpool *pop) { if (pop->heap_size < HEAP_MIN_SIZE) { ERR("heap: invalid heap size"); return -1; } struct heap_layout *layout = heap_get_layout(pop); if (pop->heap_size != layout->header.size) { ERR("heap: heap size missmatch"); return -1; } if (heap_verify_header(&layout->header)) return -1; for (int i = 0; i < heap_max_zone(layout->header.size); ++i) { if (heap_verify_zone(&layout->zones[i])) return -1; } return 0; }
/* * heap_foreach_object -- (internal) iterates through objects in the heap */ void heap_foreach_object(struct palloc_heap *heap, object_callback cb, void *arg, struct memory_block m) { struct heap_layout *layout = heap->layout; for (; m.zone_id < heap_max_zone(layout->header.size); ++m.zone_id) { if (heap_zone_foreach_object(heap, cb, arg, &m) != 0) break; m.chunk_id = 0; } }
/* * heap_check_remote -- verifies if the heap of a remote pool is consistent * and can be opened properly * * If successful function returns zero. Otherwise an error number is returned. */ int heap_check_remote(void *heap_start, uint64_t heap_size, struct remote_ops *ops) { if (heap_size < HEAP_MIN_SIZE) { ERR("heap: invalid heap size"); return -1; } struct heap_layout *layout = heap_start; struct heap_header header; if (ops->read(ops->ctx, ops->base, &header, &layout->header, sizeof(struct heap_header))) { ERR("heap: obj_read_remote error"); return -1; } if (heap_size != header.size) { ERR("heap: heap size mismatch"); return -1; } if (heap_verify_header(&header)) return -1; struct zone *zone_buff = (struct zone *)Malloc(sizeof(struct zone)); if (zone_buff == NULL) { ERR("heap: zone_buff malloc error"); return -1; } for (unsigned i = 0; i < heap_max_zone(header.size); ++i) { if (ops->read(ops->ctx, ops->base, zone_buff, ZID_TO_ZONE(layout, i), sizeof(struct zone))) { ERR("heap: obj_read_remote error"); goto out; } if (heap_verify_zone(zone_buff)) { goto out; } } Free(zone_buff); return 0; out: Free(zone_buff); return -1; }
/* * heap_vg_open -- notifies Valgrind about heap layout */ void heap_vg_open(struct palloc_heap *heap, object_callback cb, void *arg, int objects) { ASSERTne(cb, NULL); VALGRIND_DO_MAKE_MEM_UNDEFINED(heap->layout, heap->size); struct heap_layout *layout = heap->layout; VALGRIND_DO_MAKE_MEM_DEFINED(&layout->header, sizeof(layout->header)); unsigned zones = heap_max_zone(heap->size); struct memory_block m = MEMORY_BLOCK_NONE; for (unsigned i = 0; i < zones; ++i) { struct zone *z = ZID_TO_ZONE(layout, i); uint32_t chunks; m.zone_id = i; m.chunk_id = 0; VALGRIND_DO_MAKE_MEM_DEFINED(&z->header, sizeof(z->header)); if (z->header.magic != ZONE_HEADER_MAGIC) continue; chunks = z->header.size_idx; for (uint32_t c = 0; c < chunks; ) { struct chunk_header *hdr = &z->chunk_headers[c]; m.chunk_id = c; VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr)); m.size_idx = hdr->size_idx; heap_vg_open_chunk(heap, cb, arg, objects, &m); m.block_off = 0; ASSERT(hdr->size_idx > 0); if (hdr->type == CHUNK_TYPE_RUN) { /* * Mark run data headers as defined. */ for (unsigned j = 1; j < hdr->size_idx; ++j) { struct chunk_header *data_hdr = &z->chunk_headers[c + j]; VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr, sizeof(struct chunk_header)); ASSERTeq(data_hdr->type, CHUNK_TYPE_RUN_DATA); } } else { /* * Mark unused chunk headers as not accessible. */ VALGRIND_DO_MAKE_MEM_NOACCESS( &z->chunk_headers[c + 1], (hdr->size_idx - 1) * sizeof(struct chunk_header)); } c += hdr->size_idx; } /* mark all unused chunk headers after last as not accessible */ VALGRIND_DO_MAKE_MEM_NOACCESS(&z->chunk_headers[chunks], (MAX_CHUNK - chunks) * sizeof(struct chunk_header)); } }
/* * heap_boot -- opens the heap region of the pmemobj pool * * If successful function returns zero. Otherwise an error number is returned. */ int heap_boot(struct palloc_heap *heap, void *heap_start, uint64_t heap_size, uint64_t run_id, void *base, struct pmem_ops *p_ops) { struct heap_rt *h = Malloc(sizeof(*h)); int err; if (h == NULL) { err = ENOMEM; goto error_heap_malloc; } h->alloc_classes = alloc_class_collection_new(); if (h->alloc_classes == NULL) { err = ENOMEM; goto error_alloc_classes_new; } h->ncaches = heap_get_ncaches(); h->caches = Malloc(sizeof(struct bucket_cache) * h->ncaches); if (h->caches == NULL) { err = ENOMEM; goto error_heap_cache_malloc; } h->max_zone = heap_max_zone(heap_size); h->zones_exhausted = 0; for (int i = 0; i < MAX_RUN_LOCKS; ++i) util_mutex_init(&h->run_locks[i], NULL); heap->run_id = run_id; heap->p_ops = *p_ops; heap->layout = heap_start; heap->rt = h; heap->size = heap_size; heap->base = base; VALGRIND_DO_CREATE_MEMPOOL(heap->layout, 0, 0); for (unsigned i = 0; i < h->ncaches; ++i) bucket_group_init(h->caches[i].buckets); size_t rec_i; for (rec_i = 0; rec_i < MAX_ALLOCATION_CLASSES; ++rec_i) { if ((h->recyclers[rec_i] = recycler_new(heap)) == NULL) { err = ENOMEM; goto error_recycler_new; } } return 0; error_recycler_new: Free(h->caches); for (size_t i = 0; i < rec_i; ++i) recycler_delete(h->recyclers[i]); error_heap_cache_malloc: alloc_class_collection_delete(h->alloc_classes); error_alloc_classes_new: Free(h); heap->rt = NULL; error_heap_malloc: return err; }