/*
* lane_init -- (internal) initializes a single lane runtime variables
*/
static int
lane_init(PMEMobjpool *pop, struct lane *lane, struct lane_layout *layout,
pthread_mutex_t *mtx, pthread_mutexattr_t *attr)
{
ASSERTne(lane, NULL);
ASSERTne(mtx, NULL);
ASSERTne(attr, NULL);
int err;
util_mutex_init(mtx, attr);
lane->lock = mtx;
int i;
for (i = 0; i < MAX_LANE_SECTION; ++i) {
lane->sections[i].runtime = NULL;
lane->sections[i].layout = &layout->sections[i];
err = Section_ops[i]->construct(pop, &lane->sections[i]);
if (err != 0) {
ERR("!lane_construct_ops %d", i);
goto error_section_construct;
}
}
return 0;
error_section_construct:
for (i = i - 1; i >= 0; --i)
Section_ops[i]->destruct(pop, &lane->sections[i]);
util_mutex_destroy(lane->lock);
return err;
}
/*
* pmemblk_runtime_init -- (internal) initialize block memory pool runtime data
*/
static int
pmemblk_runtime_init(PMEMblkpool *pbp, size_t bsize, int rdonly, int is_pmem)
{
LOG(3, "pbp %p bsize %zu rdonly %d is_pmem %d",
pbp, bsize, rdonly, is_pmem);
/* remove volatile part of header */
VALGRIND_REMOVE_PMEM_MAPPING(&pbp->addr,
sizeof (struct pmemblk) -
sizeof (struct pool_hdr) -
sizeof (pbp->bsize) -
sizeof (pbp->is_zeroed));
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
pbp->rdonly = rdonly;
pbp->is_pmem = is_pmem;
pbp->data = (char *)pbp->addr +
roundup(sizeof (*pbp), BLK_FORMAT_DATA_ALIGN);
ASSERT(((char *)pbp->addr + pbp->size) >= (char *)pbp->data);
pbp->datasize = (size_t)
(((char *)pbp->addr + pbp->size) - (char *)pbp->data);
LOG(4, "data area %p data size %zu bsize %zu",
pbp->data, pbp->datasize, bsize);
long ncpus = sysconf(_SC_NPROCESSORS_ONLN);
if (ncpus < 1)
ncpus = 1;
ns_cb.ns_is_zeroed = pbp->is_zeroed;
/* things free by "goto err" if not NULL */
struct btt *bttp = NULL;
pthread_mutex_t *locks = NULL;
bttp = btt_init(pbp->datasize, (uint32_t)bsize, pbp->hdr.poolset_uuid,
(unsigned)ncpus * 2, pbp, &ns_cb);
if (bttp == NULL)
goto err; /* btt_init set errno, called LOG */
pbp->bttp = bttp;
pbp->nlane = btt_nlane(pbp->bttp);
pbp->next_lane = 0;
if ((locks = Malloc(pbp->nlane * sizeof (*locks))) == NULL) {
ERR("!Malloc for lane locks");
goto err;
}
for (unsigned i = 0; i < pbp->nlane; i++)
util_mutex_init(&locks[i], NULL);
pbp->locks = locks;
#ifdef DEBUG
/* initialize debug lock */
util_mutex_init(&pbp->write_lock, NULL);
#endif
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(pbp->addr, sizeof (struct pool_hdr));
/* the data area should be kept read-only for debug version */
RANGE_RO(pbp->data, pbp->datasize);
return 0;
err:
LOG(4, "error clean up");
int oerrno = errno;
if (locks)
Free((void *)locks);
if (bttp)
btt_fini(bttp);
errno = oerrno;
return -1;
}
/*
* heap_boot -- opens the heap region of the pmemobj pool
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_boot(struct palloc_heap *heap, void *heap_start, uint64_t heap_size,
uint64_t run_id, void *base, struct pmem_ops *p_ops)
{
struct heap_rt *h = Malloc(sizeof(*h));
int err;
if (h == NULL) {
err = ENOMEM;
goto error_heap_malloc;
}
h->alloc_classes = alloc_class_collection_new();
if (h->alloc_classes == NULL) {
err = ENOMEM;
goto error_alloc_classes_new;
}
h->ncaches = heap_get_ncaches();
h->caches = Malloc(sizeof(struct bucket_cache) * h->ncaches);
if (h->caches == NULL) {
err = ENOMEM;
goto error_heap_cache_malloc;
}
h->max_zone = heap_max_zone(heap_size);
h->zones_exhausted = 0;
for (int i = 0; i < MAX_RUN_LOCKS; ++i)
util_mutex_init(&h->run_locks[i], NULL);
heap->run_id = run_id;
heap->p_ops = *p_ops;
heap->layout = heap_start;
heap->rt = h;
heap->size = heap_size;
heap->base = base;
VALGRIND_DO_CREATE_MEMPOOL(heap->layout, 0, 0);
for (unsigned i = 0; i < h->ncaches; ++i)
bucket_group_init(h->caches[i].buckets);
size_t rec_i;
for (rec_i = 0; rec_i < MAX_ALLOCATION_CLASSES; ++rec_i) {
if ((h->recyclers[rec_i] = recycler_new(heap)) == NULL) {
err = ENOMEM;
goto error_recycler_new;
}
}
return 0;
error_recycler_new:
Free(h->caches);
for (size_t i = 0; i < rec_i; ++i)
recycler_delete(h->recyclers[i]);
error_heap_cache_malloc:
alloc_class_collection_delete(h->alloc_classes);
error_alloc_classes_new:
Free(h);
heap->rt = NULL;
error_heap_malloc:
return err;
}
