/*
* libvmmalloc_clone - (internal) clone the entire pool
*/
static void *
libvmmalloc_clone(void)
{
LOG(3, NULL);
Fd_clone = util_tmpfile(Dir, Vmp->size);
if (Fd_clone == -1)
return NULL;
void *addr = mmap(NULL, Vmp->size, PROT_READ|PROT_WRITE,
MAP_SHARED, Fd_clone, 0);
if (addr == MAP_FAILED) {
LOG(1, "!mmap");
(void) close(Fd_clone);
return NULL;
}
LOG(3, "copy the entire pool file: dst %p src %p size %zu",
addr, Vmp->addr, Vmp->size);
util_range_rw(Vmp->addr, sizeof (struct pool_hdr));
/*
* Part of vmem pool was probably freed at some point, so Valgrind
* marked it as undefined/inaccessible. We need to duplicate the whole
* pool, so as a workaround temporarily disable error reporting.
*/
VALGRIND_DO_DISABLE_ERROR_REPORTING;
memcpy(addr, Vmp->addr, Vmp->size);
VALGRIND_DO_ENABLE_ERROR_REPORTING;
util_range_none(Vmp->addr, sizeof (struct pool_hdr));
return addr;
}
/*
* libvmmalloc_create -- (internal) create a memory pool in a temp file
*/
static VMEM *
libvmmalloc_create(const char *dir, size_t size)
{
LOG(3, "dir \"%s\" size %zu", dir, size);
if (size < VMMALLOC_MIN_POOL) {
LOG(1, "size %zu smaller than %zu", size, VMMALLOC_MIN_POOL);
errno = EINVAL;
return NULL;
}
/* silently enforce multiple of page size */
size = roundup(size, Pagesize);
Fd = util_tmpfile(dir, size);
if (Fd == -1)
return NULL;
void *addr;
if ((addr = util_map(Fd, size, 0, 4 << 20)) == NULL)
return NULL;
/* store opaque info at beginning of mapped area */
struct vmem *vmp = addr;
memset(&vmp->hdr, '\0', sizeof (vmp->hdr));
memcpy(vmp->hdr.signature, VMEM_HDR_SIG, POOL_HDR_SIG_LEN);
vmp->addr = addr;
vmp->size = size;
vmp->caller_mapped = 0;
/* Prepare pool for jemalloc */
if (je_vmem_pool_create((void *)((uintptr_t)addr + Header_size),
size - Header_size, 1) == NULL) {
LOG(1, "vmem pool creation failed");
util_unmap(vmp->addr, vmp->size);
return NULL;
}
/*
* If possible, turn off all permissions on the pool header page.
*
* The prototype PMFS doesn't allow this when large pages are in
* use. It is not considered an error if this fails.
*/
util_range_none(addr, sizeof (struct pool_hdr));
LOG(3, "vmp %p", vmp);
return vmp;
}
/*
* util_map_tmpfile -- reserve space in an unlinked file and memory-map it
*
* size must be multiple of page size.
*/
void *
util_map_tmpfile(const char *dir, size_t size, size_t req_align)
{
int oerrno;
if (((off_t)size) < 0) {
ERR("invalid size (%zu) for off_t", size);
errno = EFBIG;
return NULL;
}
int fd = util_tmpfile(dir, "/vmem.XXXXXX");
if (fd == -1) {
LOG(2, "cannot create temporary file in dir %s", dir);
goto err;
}
if ((errno = posix_fallocate(fd, 0, (off_t)size)) != 0) {
ERR("!posix_fallocate");
goto err;
}
void *base;
if ((base = util_map(fd, size, 0, req_align)) == NULL) {
LOG(2, "cannot mmap temporary file");
goto err;
}
(void) close(fd);
return base;
err:
oerrno = errno;
if (fd != -1)
(void) close(fd);
errno = oerrno;
return NULL;
}
static inline
#endif
void *
pmem_map_fileU(const char *path, size_t len, int flags,
mode_t mode, size_t *mapped_lenp, int *is_pmemp)
{
LOG(3, "path \"%s\" size %zu flags %x mode %o mapped_lenp %p "
"is_pmemp %p", path, len, flags, mode, mapped_lenp, is_pmemp);
int oerrno;
int fd;
int open_flags = O_RDWR;
int delete_on_err = 0;
int file_type = util_file_get_type(path);
#ifdef _WIN32
open_flags |= O_BINARY;
#endif
if (file_type == OTHER_ERROR)
return NULL;
if (flags & ~(PMEM_FILE_ALL_FLAGS)) {
ERR("invalid flag specified %x", flags);
errno = EINVAL;
return NULL;
}
if (file_type == TYPE_DEVDAX) {
if (flags & ~(PMEM_DAX_VALID_FLAGS)) {
ERR("flag unsupported for Device DAX %x", flags);
errno = EINVAL;
return NULL;
} else {
/* we are ignoring all of the flags */
flags = 0;
ssize_t actual_len = util_file_get_size(path);
if (actual_len < 0) {
ERR("unable to read Device DAX size");
errno = EINVAL;
return NULL;
}
if (len != 0 && len != (size_t)actual_len) {
ERR("Device DAX length must be either 0 or "
"the exact size of the device: %zu",
actual_len);
errno = EINVAL;
return NULL;
}
len = 0;
}
}
if (flags & PMEM_FILE_CREATE) {
if ((os_off_t)len < 0) {
ERR("invalid file length %zu", len);
errno = EINVAL;
return NULL;
}
open_flags |= O_CREAT;
}
if (flags & PMEM_FILE_EXCL)
open_flags |= O_EXCL;
if ((len != 0) && !(flags & PMEM_FILE_CREATE)) {
ERR("non-zero 'len' not allowed without PMEM_FILE_CREATE");
errno = EINVAL;
return NULL;
}
if ((len == 0) && (flags & PMEM_FILE_CREATE)) {
ERR("zero 'len' not allowed with PMEM_FILE_CREATE");
errno = EINVAL;
return NULL;
}
if ((flags & PMEM_FILE_TMPFILE) && !(flags & PMEM_FILE_CREATE)) {
ERR("PMEM_FILE_TMPFILE not allowed without PMEM_FILE_CREATE");
errno = EINVAL;
return NULL;
}
if (flags & PMEM_FILE_TMPFILE) {
if ((fd = util_tmpfile(path,
OS_DIR_SEP_STR"pmem.XXXXXX",
open_flags & O_EXCL)) < 0) {
LOG(2, "failed to create temporary file at \"%s\"",
path);
return NULL;
}
} else {
if ((fd = os_open(path, open_flags, mode)) < 0) {
ERR("!open %s", path);
return NULL;
}
if ((flags & PMEM_FILE_CREATE) && (flags & PMEM_FILE_EXCL))
delete_on_err = 1;
}
if (flags & PMEM_FILE_CREATE) {
/*
* Always set length of file to 'len'.
* (May either extend or truncate existing file.)
*/
if (os_ftruncate(fd, (os_off_t)len) != 0) {
ERR("!ftruncate");
goto err;
}
if ((flags & PMEM_FILE_SPARSE) == 0) {
if ((errno = os_posix_fallocate(fd, 0,
(os_off_t)len)) != 0) {
ERR("!posix_fallocate");
goto err;
}
}
} else {
ssize_t actual_size = util_file_get_size(path);
if (actual_size < 0) {
ERR("stat %s: negative size", path);
errno = EINVAL;
goto err;
}
len = (size_t)actual_size;
}
void *addr = pmem_map_register(fd, len, path, file_type == TYPE_DEVDAX);
if (addr == NULL)
goto err;
if (mapped_lenp != NULL)
*mapped_lenp = len;
if (is_pmemp != NULL)
*is_pmemp = pmem_is_pmem(addr, len);
LOG(3, "returning %p", addr);
VALGRIND_REGISTER_PMEM_MAPPING(addr, len);
VALGRIND_REGISTER_PMEM_FILE(fd, addr, len, 0);
(void) os_close(fd);
return addr;
err:
oerrno = errno;
(void) os_close(fd);
if (delete_on_err)
(void) os_unlink(path);
errno = oerrno;
return NULL;
}