/*
* libvmmalloc_clone - (internal) clone the entire pool
*/
static void *
libvmmalloc_clone(void)
{
LOG(3, NULL);
Fd_clone = util_tmpfile(Dir, Vmp->size);
if (Fd_clone == -1)
return NULL;
void *addr = mmap(NULL, Vmp->size, PROT_READ|PROT_WRITE,
MAP_SHARED, Fd_clone, 0);
if (addr == MAP_FAILED) {
LOG(1, "!mmap");
(void) close(Fd_clone);
return NULL;
}
LOG(3, "copy the entire pool file: dst %p src %p size %zu",
addr, Vmp->addr, Vmp->size);
util_range_rw(Vmp->addr, sizeof (struct pool_hdr));
/*
* Part of vmem pool was probably freed at some point, so Valgrind
* marked it as undefined/inaccessible. We need to duplicate the whole
* pool, so as a workaround temporarily disable error reporting.
*/
VALGRIND_DO_DISABLE_ERROR_REPORTING;
memcpy(addr, Vmp->addr, Vmp->size);
VALGRIND_DO_ENABLE_ERROR_REPORTING;
util_range_none(Vmp->addr, sizeof (struct pool_hdr));
return addr;
}
/*
* pmemobj_runtime_init -- (internal) initialize runtime part of the pool header
*/
static int
pmemobj_runtime_init(PMEMobjpool *pop, int rdonly, int boot)
{
LOG(3, "pop %p rdonly %d boot %d", pop, rdonly, boot);
if (pop->replica != NULL) {
/* switch to functions that replicate data */
pop->persist = obj_rep_persist;
pop->flush = obj_rep_flush;
pop->drain = obj_rep_drain;
pop->memcpy_persist = obj_rep_memcpy_persist;
pop->memset_persist = obj_rep_memset_persist;
}
/* run_id is made unique by incrementing the previous value */
pop->run_id += 2;
if (pop->run_id == 0)
pop->run_id += 2;
pop->persist(pop, &pop->run_id, sizeof (pop->run_id));
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
pop->rdonly = rdonly;
pop->lanes = NULL;
pop->uuid_lo = pmemobj_get_uuid_lo(pop);
pop->store = (struct object_store *)
((uintptr_t)pop + pop->obj_store_offset);
if (boot) {
if ((errno = pmemobj_boot(pop)) != 0)
return -1;
if ((errno = cuckoo_insert(pools_ht, pop->uuid_lo, pop)) != 0) {
ERR("!cuckoo_insert");
return -1;
}
if ((errno = ctree_insert(pools_tree, (uint64_t)pop, pop->size))
!= 0) {
ERR("!ctree_insert");
return -1;
}
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(pop->addr, sizeof (struct pool_hdr));
return 0;
}
/*
* libvmmalloc_create -- (internal) create a memory pool in a temp file
*/
static VMEM *
libvmmalloc_create(const char *dir, size_t size)
{
LOG(3, "dir \"%s\" size %zu", dir, size);
if (size < VMMALLOC_MIN_POOL) {
LOG(1, "size %zu smaller than %zu", size, VMMALLOC_MIN_POOL);
errno = EINVAL;
return NULL;
}
/* silently enforce multiple of page size */
size = roundup(size, Pagesize);
Fd = util_tmpfile(dir, size);
if (Fd == -1)
return NULL;
void *addr;
if ((addr = util_map(Fd, size, 0, 4 << 20)) == NULL)
return NULL;
/* store opaque info at beginning of mapped area */
struct vmem *vmp = addr;
memset(&vmp->hdr, '\0', sizeof (vmp->hdr));
memcpy(vmp->hdr.signature, VMEM_HDR_SIG, POOL_HDR_SIG_LEN);
vmp->addr = addr;
vmp->size = size;
vmp->caller_mapped = 0;
/* Prepare pool for jemalloc */
if (je_vmem_pool_create((void *)((uintptr_t)addr + Header_size),
size - Header_size, 1) == NULL) {
LOG(1, "vmem pool creation failed");
util_unmap(vmp->addr, vmp->size);
return NULL;
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
LOG(3, "vmp %p", vmp);
return vmp;
}
/*
* vmem_create_in_region -- create a memory pool in a given range
*/
VMEM *
vmem_create_in_region(void *addr, size_t size)
{
vmem_init();
LOG(3, "addr %p size %zu", addr, size);
if (((uintptr_t)addr & (Pagesize - 1)) != 0) {
ERR("addr %p not aligned to pagesize %lu", addr, Pagesize);
errno = EINVAL;
return NULL;
}
if (size < VMEM_MIN_POOL) {
ERR("size %zu smaller than %zu", size, VMEM_MIN_POOL);
errno = EINVAL;
return NULL;
}
/* store opaque info at beginning of mapped area */
struct vmem *vmp = addr;
memset(&vmp->hdr, '\0', sizeof (vmp->hdr));
memcpy(vmp->hdr.signature, VMEM_HDR_SIG, POOL_HDR_SIG_LEN);
vmp->addr = addr;
vmp->size = size;
vmp->caller_mapped = 1;
/* Prepare pool for jemalloc */
if (je_vmem_pool_create((void *)((uintptr_t)addr + Header_size),
size - Header_size, 0) == NULL) {
ERR("pool creation failed");
return NULL;
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
LOG(3, "vmp %p", vmp);
return vmp;
}
/*
* vmem_pool_create_in_region -- create a memory pool in a given range
*/
VMEM *
vmem_pool_create_in_region(void *addr, size_t size)
{
LOG(3, "addr %p size %zu", addr, size);
if (size < VMEM_MIN_POOL) {
LOG(1, "size %zu smaller than %zu", size, VMEM_MIN_POOL);
errno = EINVAL;
return NULL;
}
/* store opaque info at beginning of mapped area */
struct vmem *vmp = addr;
memset(&vmp->hdr, '\0', sizeof (vmp->hdr));
strncpy(vmp->hdr.signature, VMEM_HDR_SIG, POOL_HDR_SIG_LEN);
vmp->addr = addr;
vmp->size = size;
vmp->caller_mapped = 1;
/* Prepare pool for jemalloc */
if (je_vmem_pool_create((void *)((uintptr_t)addr + Header_size),
size - Header_size, 0) == NULL) {
LOG(1, "return NULL");
return NULL;
}
#ifdef notdef
/*
* XXX not ready to call aligned util_range_none() yet,
* addr should be aligned to do that.
*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use not it is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
#endif /* notdef */
LOG(3, "vmp %p", vmp);
return vmp;
}
/*
* pmemblk_runtime_init -- (internal) initialize block memory pool runtime data
*/
static int
pmemblk_runtime_init(PMEMblkpool *pbp, size_t bsize, int rdonly, int is_pmem)
{
LOG(3, "pbp %p bsize %zu rdonly %d is_pmem %d",
pbp, bsize, rdonly, is_pmem);
/* remove volatile part of header */
VALGRIND_REMOVE_PMEM_MAPPING(&pbp->addr,
sizeof (struct pmemblk) -
sizeof (struct pool_hdr) -
sizeof (pbp->bsize) -
sizeof (pbp->is_zeroed));
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
pbp->rdonly = rdonly;
pbp->is_pmem = is_pmem;
pbp->data = (char *)pbp->addr +
roundup(sizeof (*pbp), BLK_FORMAT_DATA_ALIGN);
ASSERT(((char *)pbp->addr + pbp->size) >= (char *)pbp->data);
pbp->datasize = (size_t)
(((char *)pbp->addr + pbp->size) - (char *)pbp->data);
LOG(4, "data area %p data size %zu bsize %zu",
pbp->data, pbp->datasize, bsize);
long ncpus = sysconf(_SC_NPROCESSORS_ONLN);
if (ncpus < 1)
ncpus = 1;
ns_cb.ns_is_zeroed = pbp->is_zeroed;
/* things free by "goto err" if not NULL */
struct btt *bttp = NULL;
pthread_mutex_t *locks = NULL;
bttp = btt_init(pbp->datasize, (uint32_t)bsize, pbp->hdr.poolset_uuid,
(unsigned)ncpus * 2, pbp, &ns_cb);
if (bttp == NULL)
goto err; /* btt_init set errno, called LOG */
pbp->bttp = bttp;
pbp->nlane = btt_nlane(pbp->bttp);
pbp->next_lane = 0;
if ((locks = Malloc(pbp->nlane * sizeof (*locks))) == NULL) {
ERR("!Malloc for lane locks");
goto err;
}
for (unsigned i = 0; i < pbp->nlane; i++)
util_mutex_init(&locks[i], NULL);
pbp->locks = locks;
#ifdef DEBUG
/* initialize debug lock */
util_mutex_init(&pbp->write_lock, NULL);
#endif
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(pbp->addr, sizeof (struct pool_hdr));
/* the data area should be kept read-only for debug version */
RANGE_RO(pbp->data, pbp->datasize);
return 0;
err:
LOG(4, "error clean up");
int oerrno = errno;
if (locks)
Free((void *)locks);
if (bttp)
btt_fini(bttp);
errno = oerrno;
return -1;
}
/*
* pmemlog_map_common -- (internal) map a log memory pool
*
* This routine does all the work, but takes a rdonly flag so internal
* calls can map a read-only pool if required.
*
* If empty flag is set, the file is assumed to be a new memory pool, and
* a new pool header is created. Otherwise, a valid header must exist.
*/
static PMEMlogpool *
pmemlog_map_common(int fd, size_t poolsize, int rdonly, int empty)
{
LOG(3, "fd %d poolsize %zu rdonly %d empty %d",
fd, poolsize, rdonly, empty);
void *addr;
if ((addr = util_map(fd, poolsize, rdonly)) == NULL) {
(void) close(fd);
return NULL; /* util_map() set errno, called LOG */
}
VALGRIND_REGISTER_PMEM_MAPPING(addr, poolsize);
VALGRIND_REGISTER_PMEM_FILE(fd, addr, poolsize, 0);
(void) close(fd);
/* check if the mapped region is located in persistent memory */
int is_pmem = pmem_is_pmem(addr, poolsize);
/* opaque info lives at the beginning of mapped memory pool */
struct pmemlog *plp = addr;
if (!empty) {
struct pool_hdr hdr;
memcpy(&hdr, &plp->hdr, sizeof (hdr));
if (!util_convert_hdr(&hdr)) {
errno = EINVAL;
goto err;
}
/*
* valid header found
*/
if (strncmp(hdr.signature, LOG_HDR_SIG, POOL_HDR_SIG_LEN)) {
ERR("wrong pool type: \"%s\"", hdr.signature);
errno = EINVAL;
goto err;
}
if (hdr.major != LOG_FORMAT_MAJOR) {
ERR("log pool version %d (library expects %d)",
hdr.major, LOG_FORMAT_MAJOR);
errno = EINVAL;
goto err;
}
/* XXX - pools sets / replicas */
if (memcmp(hdr.uuid, hdr.prev_part_uuid, POOL_HDR_UUID_LEN) ||
memcmp(hdr.uuid, hdr.next_part_uuid, POOL_HDR_UUID_LEN) ||
memcmp(hdr.uuid, hdr.prev_repl_uuid, POOL_HDR_UUID_LEN) ||
memcmp(hdr.uuid, hdr.next_repl_uuid, POOL_HDR_UUID_LEN)) {
ERR("wrong UUID");
errno = EINVAL;
goto err;
}
uint64_t hdr_start = le64toh(plp->start_offset);
uint64_t hdr_end = le64toh(plp->end_offset);
uint64_t hdr_write = le64toh(plp->write_offset);
if ((hdr_start != roundup(sizeof (*plp),
LOG_FORMAT_DATA_ALIGN)) ||
(hdr_end != poolsize) || (hdr_start > hdr_end)) {
ERR("wrong start/end offsets (start: %ju end: %ju), "
"pool size %zu",
hdr_start, hdr_end, poolsize);
errno = EINVAL;
goto err;
}
if ((hdr_write > hdr_end) || (hdr_write < hdr_start)) {
ERR("wrong write offset "
"(start: %ju end: %ju write: %ju)",
hdr_start, hdr_end, hdr_write);
errno = EINVAL;
goto err;
}
LOG(3, "start: %ju, end: %ju, write: %ju",
hdr_start, hdr_end, hdr_write);
int retval = util_feature_check(&hdr, LOG_FORMAT_INCOMPAT,
LOG_FORMAT_RO_COMPAT,
LOG_FORMAT_COMPAT);
if (retval < 0)
goto err;
else if (retval == 0)
rdonly = 1;
} else {
LOG(3, "creating new log memory pool");
ASSERTeq(rdonly, 0);
struct pool_hdr *hdrp = &plp->hdr;
/* check if the pool header is all zero */
if (!util_is_zeroed(hdrp, sizeof (*hdrp))) {
ERR("Non-empty file detected");
errno = EINVAL;
goto err;
}
/* create required metadata first */
plp->start_offset = htole64(roundup(sizeof (*plp),
LOG_FORMAT_DATA_ALIGN));
plp->end_offset = htole64(poolsize);
plp->write_offset = plp->start_offset;
/* store non-volatile part of pool's descriptor */
pmem_msync(&plp->start_offset, 3 * sizeof (uint64_t));
/* create pool header */
strncpy(hdrp->signature, LOG_HDR_SIG, POOL_HDR_SIG_LEN);
hdrp->major = htole32(LOG_FORMAT_MAJOR);
hdrp->compat_features = htole32(LOG_FORMAT_COMPAT);
hdrp->incompat_features = htole32(LOG_FORMAT_INCOMPAT);
hdrp->ro_compat_features = htole32(LOG_FORMAT_RO_COMPAT);
uuid_generate(hdrp->uuid);
/* XXX - pools sets / replicas */
uuid_generate(hdrp->poolset_uuid);
memcpy(hdrp->prev_part_uuid, hdrp->uuid, POOL_HDR_UUID_LEN);
memcpy(hdrp->next_part_uuid, hdrp->uuid, POOL_HDR_UUID_LEN);
memcpy(hdrp->prev_repl_uuid, hdrp->uuid, POOL_HDR_UUID_LEN);
memcpy(hdrp->next_repl_uuid, hdrp->uuid, POOL_HDR_UUID_LEN);
hdrp->crtime = htole64((uint64_t)time(NULL));
if (util_get_arch_flags(&hdrp->arch_flags)) {
ERR("Reading architecture flags failed\n");
errno = EINVAL;
goto err;
}
hdrp->arch_flags.alignment_desc =
htole64(hdrp->arch_flags.alignment_desc);
hdrp->arch_flags.e_machine =
htole16(hdrp->arch_flags.e_machine);
util_checksum(hdrp, sizeof (*hdrp), &hdrp->checksum, 1);
/* store pool's header */
pmem_msync(hdrp, sizeof (*hdrp));
}
/* remove volatile part of header */
VALGRIND_REMOVE_PMEM_MAPPING(&plp->addr,
sizeof (struct pmemlog) -
sizeof (struct pool_hdr) -
3 * sizeof (uint64_t));
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
plp->addr = addr;
plp->size = poolsize;
plp->rdonly = rdonly;
plp->is_pmem = is_pmem;
if ((plp->rwlockp = Malloc(sizeof (*plp->rwlockp))) == NULL) {
ERR("!Malloc for a RW lock");
goto err;
}
if ((errno = pthread_rwlock_init(plp->rwlockp, NULL))) {
ERR("!pthread_rwlock_init");
goto err_free;
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
/* the rest should be kept read-only (debug version only) */
RANGE_RO(addr + sizeof (struct pool_hdr),
poolsize - sizeof (struct pool_hdr));
LOG(3, "plp %p", plp);
return plp;
err_free:
Free((void *)plp->rwlockp);
err:
LOG(4, "error clean up");
int oerrno = errno;
VALGRIND_REMOVE_PMEM_MAPPING(addr, poolsize);
util_unmap(addr, poolsize);
errno = oerrno;
return NULL;
}
/*
* pmemlog_map_common -- (internal) map a log memory pool
*
* This routine does all the work, but takes a rdonly flag so internal
* calls can map a read-only pool if required.
*/
static PMEMlog *
pmemlog_map_common(int fd, int rdonly)
{
LOG(3, "fd %d rdonly %d", fd, rdonly);
struct stat stbuf;
if (fstat(fd, &stbuf) < 0) {
LOG(1, "!fstat");
return NULL;
}
if (stbuf.st_size < PMEMLOG_MIN_POOL) {
LOG(1, "size %lld smaller than %zu",
(long long)stbuf.st_size, PMEMLOG_MIN_POOL);
errno = EINVAL;
return NULL;
}
void *addr;
if ((addr = util_map(fd, stbuf.st_size, rdonly)) == NULL)
return NULL; /* util_map() set errno, called LOG */
/* check if the mapped region is located in persistent memory */
int is_pmem = pmem_is_pmem(addr, stbuf.st_size);
/* opaque info lives at the beginning of mapped memory pool */
struct pmemlog *plp = addr;
struct pool_hdr hdr;
memcpy(&hdr, &plp->hdr, sizeof (hdr));
if (util_convert_hdr(&hdr)) {
/*
* valid header found
*/
if (strncmp(hdr.signature, LOG_HDR_SIG, POOL_HDR_SIG_LEN)) {
LOG(1, "wrong pool type: \"%s\"", hdr.signature);
errno = EINVAL;
goto err;
}
if (hdr.major != LOG_FORMAT_MAJOR) {
LOG(1, "log pool version %d (library expects %d)",
hdr.major, LOG_FORMAT_MAJOR);
errno = EINVAL;
goto err;
}
uint64_t hdr_start = le64toh(plp->start_offset);
uint64_t hdr_end = le64toh(plp->end_offset);
uint64_t hdr_write = le64toh(plp->write_offset);
if ((hdr_start != roundup(sizeof (*plp),
LOG_FORMAT_DATA_ALIGN)) ||
(hdr_end != stbuf.st_size) || (hdr_start > hdr_end)) {
LOG(1, "wrong start/end offsets (start: %ju end: %ju), "
"pool size %lld",
hdr_start, hdr_end, (long long)stbuf.st_size);
errno = EINVAL;
goto err;
}
if ((hdr_write > hdr_end) || (hdr_write < hdr_start)) {
LOG(1, "wrong write offset "
"(start: %ju end: %ju write: %ju)",
hdr_start, hdr_end, hdr_write);
errno = EINVAL;
goto err;
}
LOG(3, "start: %ju, end: %ju, write: %ju",
hdr_start, hdr_end, hdr_write);
int retval = util_feature_check(&hdr, LOG_FORMAT_INCOMPAT,
LOG_FORMAT_RO_COMPAT,
LOG_FORMAT_COMPAT);
if (retval < 0)
goto err;
else if (retval == 0)
rdonly = 1;
} else {
/*
* no valid header was found
*/
if (rdonly) {
LOG(1, "read-only and no header found");
errno = EROFS;
goto err;
}
LOG(3, "creating new log memory pool");
struct pool_hdr *hdrp = &plp->hdr;
memset(hdrp, '\0', sizeof (*hdrp));
strncpy(hdrp->signature, LOG_HDR_SIG, POOL_HDR_SIG_LEN);
hdrp->major = htole32(LOG_FORMAT_MAJOR);
hdrp->compat_features = htole32(LOG_FORMAT_COMPAT);
hdrp->incompat_features = htole32(LOG_FORMAT_INCOMPAT);
hdrp->ro_compat_features = htole32(LOG_FORMAT_RO_COMPAT);
uuid_generate(hdrp->uuid);
hdrp->crtime = htole64((uint64_t)time(NULL));
util_checksum(hdrp, sizeof (*hdrp), &hdrp->checksum, 1);
hdrp->checksum = htole64(hdrp->checksum);
/* store pool's header */
libpmem_persist(is_pmem, hdrp, sizeof (*hdrp));
/* create rest of required metadata */
plp->start_offset = htole64(roundup(sizeof (*plp),
LOG_FORMAT_DATA_ALIGN));
plp->end_offset = htole64(stbuf.st_size);
plp->write_offset = plp->start_offset;
/* store non-volatile part of pool's descriptor */
libpmem_persist(is_pmem, &plp->start_offset,
3 * sizeof (uint64_t));
}
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
plp->addr = addr;
plp->size = stbuf.st_size;
plp->rdonly = rdonly;
plp->is_pmem = is_pmem;
if ((plp->rwlockp = Malloc(sizeof (*plp->rwlockp))) == NULL) {
LOG(1, "!Malloc for a RW lock");
goto err;
}
if (pthread_rwlock_init(plp->rwlockp, NULL)) {
LOG(1, "!pthread_rwlock_init");
goto err_free;
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
/* the rest should be kept read-only (debug version only) */
RANGE_RO(addr + sizeof (struct pool_hdr),
stbuf.st_size - sizeof (struct pool_hdr));
LOG(3, "plp %p", plp);
return plp;
err_free:
Free((void *)plp->rwlockp);
err:
LOG(4, "error clean up");
int oerrno = errno;
util_unmap(addr, stbuf.st_size);
errno = oerrno;
return NULL;
}
/*
* pmemblk_map_common -- (internal) map a block memory pool
*
* This routine does all the work, but takes a rdonly flag so internal
* calls can map a read-only pool if required.
*
* Passing in bsize == 0 means a valid pool header must exist (which
* will supply the block size).
*/
static PMEMblk *
pmemblk_map_common(int fd, size_t bsize, int rdonly)
{
LOG(3, "fd %d bsize %zu rdonly %d", fd, bsize, rdonly);
/* things free by "goto err" if not NULL */
void *addr = NULL;
struct btt *bttp = NULL;
pthread_mutex_t *locks = NULL;
struct stat stbuf;
if (fstat(fd, &stbuf) < 0) {
LOG(1, "!fstat");
return NULL;
}
if (stbuf.st_size < PMEMBLK_MIN_POOL) {
LOG(1, "size %zu smaller than %zu",
stbuf.st_size, PMEMBLK_MIN_POOL);
errno = EINVAL;
return NULL;
}
if ((addr = util_map(fd, stbuf.st_size, rdonly)) == NULL)
return NULL; /* util_map() set errno, called LOG */
/* check if the mapped region is located in persistent memory */
int is_pmem = pmem_is_pmem(addr, stbuf.st_size);
/* opaque info lives at the beginning of mapped memory pool */
struct pmemblk *pbp = addr;
struct pool_hdr hdr;
memcpy(&hdr, &pbp->hdr, sizeof (hdr));
if (util_convert_hdr(&hdr)) {
/*
* valid header found
*/
if (strncmp(hdr.signature, BLK_HDR_SIG, POOL_HDR_SIG_LEN)) {
LOG(1, "wrong pool type: \"%s\"", hdr.signature);
errno = EINVAL;
goto err;
}
if (hdr.major != BLK_FORMAT_MAJOR) {
LOG(1, "blk pool version %d (library expects %d)",
hdr.major, BLK_FORMAT_MAJOR);
errno = EINVAL;
goto err;
}
size_t hdr_bsize = le32toh(pbp->bsize);
if (bsize && bsize != hdr_bsize) {
LOG(1, "wrong bsize (%zu), pool created with bsize %zu",
bsize, hdr_bsize);
errno = EINVAL;
goto err;
}
bsize = hdr_bsize;
LOG(3, "using block size from header: %zu", bsize);
int retval = util_feature_check(&hdr, BLK_FORMAT_INCOMPAT,
BLK_FORMAT_RO_COMPAT,
BLK_FORMAT_COMPAT);
if (retval < 0)
goto err;
else if (retval == 0)
rdonly = 1;
} else {
/*
* no valid header was found
*/
if (rdonly) {
LOG(1, "read-only and no header found");
errno = EROFS;
goto err;
}
LOG(3, "creating new blk memory pool");
struct pool_hdr *hdrp = &pbp->hdr;
memset(hdrp, '\0', sizeof (*hdrp));
strncpy(hdrp->signature, BLK_HDR_SIG, POOL_HDR_SIG_LEN);
hdrp->major = htole32(BLK_FORMAT_MAJOR);
hdrp->compat_features = htole32(BLK_FORMAT_COMPAT);
hdrp->incompat_features = htole32(BLK_FORMAT_INCOMPAT);
hdrp->ro_compat_features = htole32(BLK_FORMAT_RO_COMPAT);
uuid_generate(hdrp->uuid);
hdrp->crtime = htole64((uint64_t)time(NULL));
util_checksum(hdrp, sizeof (*hdrp), &hdrp->checksum, 1);
hdrp->checksum = htole64(hdrp->checksum);
/* store pool's header */
libpmem_persist(is_pmem, hdrp, sizeof (*hdrp));
/* create rest of required metadata */
pbp->bsize = htole32(bsize);
libpmem_persist(is_pmem, &pbp->bsize, sizeof (bsize));
}
/*
* Use some of the memory pool area for run-time info. This
* run-time state is never loaded from the file, it is always
* created here, so no need to worry about byte-order.
*/
pbp->addr = addr;
pbp->size = stbuf.st_size;
pbp->rdonly = rdonly;
pbp->is_pmem = is_pmem;
pbp->data = addr + roundup(sizeof (*pbp), BLK_FORMAT_DATA_ALIGN);
pbp->datasize = (pbp->addr + pbp->size) - pbp->data;
LOG(4, "data area %p data size %zu bsize %zu",
pbp->data, pbp->datasize, bsize);
int ncpus = sysconf(_SC_NPROCESSORS_ONLN);
if (ncpus < 1)
ncpus = 1;
bttp = btt_init(pbp->datasize, (uint32_t)bsize, pbp->hdr.uuid,
ncpus, pbp, &ns_cb);
if (bttp == NULL)
goto err; /* btt_init set errno, called LOG */
pbp->bttp = bttp;
pbp->nlane = btt_nlane(pbp->bttp);
pbp->next_lane = 0;
if ((locks = Malloc(pbp->nlane * sizeof (*locks))) == NULL) {
LOG(1, "!Malloc for lane locks");
goto err;
}
for (int i = 0; i < pbp->nlane; i++)
if (pthread_mutex_init(&locks[i], NULL) < 0) {
LOG(1, "!pthread_mutex_init");
goto err;
}
pbp->locks = locks;
#ifdef DEBUG
/* initialize debug lock */
if (pthread_mutex_init(&pbp->write_lock, NULL) < 0) {
LOG(1, "!pthread_mutex_init");
goto err;
}
#endif
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use not it is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
/* the data area should be kept read-only for debug version */
RANGE_RO(pbp->data, pbp->datasize);
LOG(3, "pbp %p", pbp);
return pbp;
err:
LOG(4, "error clean up");
int oerrno = errno;
if (locks)
Free((void *)locks);
if (bttp)
btt_fini(bttp);
util_unmap(addr, stbuf.st_size);
errno = oerrno;
return NULL;
}
