/*
* pmemlog_persist -- (internal) persist data, then metadata
*
* On entry, the write lock should be held.
*/
static void
pmemlog_persist(PMEMlogpool *plp, uint64_t new_write_offset)
{
uint64_t old_write_offset = le64toh(plp->write_offset);
size_t length = new_write_offset - old_write_offset;
/* unprotect the log space range (debug version only) */
RANGE_RW(plp->addr + old_write_offset, length);
/* persist the data */
if (plp->is_pmem)
pmem_drain(); /* data already flushed */
else
pmem_msync(plp->addr + old_write_offset, length);
/* protect the log space range (debug version only) */
RANGE_RO(plp->addr + old_write_offset, length);
/* unprotect the pool descriptor (debug version only) */
RANGE_RW(plp->addr + sizeof (struct pool_hdr), LOG_FORMAT_DATA_ALIGN);
/* write the metadata */
plp->write_offset = htole64(new_write_offset);
/* persist the metadata */
if (plp->is_pmem)
pmem_persist(&plp->write_offset, sizeof (plp->write_offset));
else
pmem_msync(&plp->write_offset, sizeof (plp->write_offset));
/* set the write-protection again (debug version only) */
RANGE_RO(plp->addr + sizeof (struct pool_hdr), LOG_FORMAT_DATA_ALIGN);
}
/*
* pmem_persist -- make any cached changes to a range of pmem persistent
*/
void
pmem_persist(void *addr, size_t len)
{
LOG(15, "addr %p len %zu", addr, len);
pmem_flush(addr, len);
pmem_drain();
}
/*
* pmem_memset_persist -- memset to pmem
*/
void *
pmem_memset_persist(void *pmemdest, int c, size_t len)
{
LOG(15, "pmemdest %p c %d len %zu", pmemdest, c, len);
PMEM_API_START();
Funcs.memset_nodrain(pmemdest, c, len, 0);
pmem_drain();
PMEM_API_END();
return pmemdest;
}
/*
* pmem_memcpy_persist -- memcpy to pmem
*/
void *
pmem_memcpy_persist(void *pmemdest, const void *src, size_t len)
{
LOG(15, "pmemdest %p src %p len %zu", pmemdest, src, len);
PMEM_API_START();
Funcs.memmove_nodrain(pmemdest, src, len, 0);
pmem_drain();
PMEM_API_END();
return pmemdest;
}
/*
* nswrite -- (internal) write data to the namespace encapsulating the BTT
*
* This routine is provided to btt_init() to allow the btt module to
* do I/O on the memory pool containing the BTT layout.
*/
static int
nswrite(void *ns, int lane, const void *buf, size_t count, off_t off)
{
struct pmemblk *pbp = (struct pmemblk *)ns;
LOG(13, "pbp %p lane %d count %zu off %lld",
pbp, lane, count, (long long)off);
if (off + count > pbp->datasize) {
ERR("offset + count (%lld) past end of data area (%zu)",
(long long)off + count, pbp->datasize);
errno = EINVAL;
return -1;
}
void *dest = pbp->data + off;
#ifdef DEBUG
/* grab debug write lock */
if ((errno = pthread_mutex_lock(&pbp->write_lock))) {
ERR("!pthread_mutex_lock");
return -1;
}
#endif
/* unprotect the memory (debug version only) */
RANGE_RW(dest, count);
if (pbp->is_pmem)
pmem_memcpy_nodrain(dest, buf, count);
else
memcpy(dest, buf, count);
/* protect the memory again (debug version only) */
RANGE_RO(dest, count);
#ifdef DEBUG
/* release debug write lock */
if ((errno = pthread_mutex_unlock(&pbp->write_lock)))
ERR("!pthread_mutex_unlock");
#endif
if (pbp->is_pmem)
pmem_drain();
else
pmem_msync(dest, count);
return 0;
}
/*
* nswrite -- (internal) write data to the namespace encapsulating the BTT
*
* This routine is provided to btt_init() to allow the btt module to
* do I/O on the memory pool containing the BTT layout.
*/
static int
nswrite(void *ns, unsigned lane, const void *buf, size_t count,
uint64_t off)
{
struct pmemblk *pbp = (struct pmemblk *)ns;
LOG(13, "pbp %p lane %u count %zu off %ju", pbp, lane, count, off);
if (off + count > pbp->datasize) {
ERR("offset + count (%zu) past end of data area (%zu)",
off + count, pbp->datasize);
errno = EINVAL;
return -1;
}
void *dest = (char *)pbp->data + off;
#ifdef DEBUG
/* grab debug write lock */
util_mutex_lock(&pbp->write_lock);
#endif
/* unprotect the memory (debug version only) */
RANGE_RW(dest, count);
if (pbp->is_pmem)
pmem_memcpy_nodrain(dest, buf, count);
else
memcpy(dest, buf, count);
/* protect the memory again (debug version only) */
RANGE_RO(dest, count);
#ifdef DEBUG
/* release debug write lock */
util_mutex_unlock(&pbp->write_lock);
#endif
if (pbp->is_pmem)
pmem_drain();
else
pmem_msync(dest, count);
return 0;
}
/*
* os_part_deep_common -- common function to handle both
* deep_persist and deep_drain part flush cases.
*/
int
os_part_deep_common(struct pool_replica *rep, unsigned partidx, void *addr,
size_t len, int flush)
{
LOG(3, "part %p part %d addr %p len %lu flush %d",
rep, partidx, addr, len, flush);
if (!rep->is_pmem) {
/*
* In case of part on non-pmem call msync on the range
* to deep flush the data. Deep drain is empty as all
* data is msynced to persistence.
*/
if (!flush)
return 0;
if (pmem_msync(addr, len)) {
LOG(1, "pmem_msync(%p, %lu)", addr, len);
return -1;
}
return 0;
}
/* Call deep flush if it was requested */
if (flush) {
LOG(15, "pmem_deep_flush addr %p, len %lu", addr, len);
pmem_deep_flush(addr, len);
}
/*
* Before deep drain call normal drain to ensure that data
* is at least in WPQ.
*/
pmem_drain();
/*
* For deep_drain on normal pmem it is enough to
* call msync on one page.
*/
if (pmem_msync(addr, MIN(Pagesize, len))) {
LOG(1, "pmem_msync(%p, %lu)", addr, len);
return -1;
}
return 0;
}
static int fio_libpmem_close_file(struct thread_data *td, struct fio_file *f)
{
struct fio_libpmem_data *fdd = FILE_ENG_DATA(f);
dprint(FD_IO,"DEBUG fio_libpmem_close_file\n");
dprint(FD_IO,"td->o.odirect %d \n",td->o.odirect);
if (!td->o.odirect) {
dprint(FD_IO,"pmem_drain\n");
pmem_drain();
}
FILE_SET_ENG_DATA(f, NULL);
free(fdd);
fio_file_clear_partial_mmap(f);
return generic_close_file(td, f);
}
/*
* pmem_memcpy -- memcpy to pmem
*/
void *
pmem_memcpy(void *pmemdest, const void *src, size_t len, unsigned flags)
{
LOG(15, "pmemdest %p src %p len %zu flags 0x%x",
pmemdest, src, len, flags);
#ifdef DEBUG
if (flags & ~PMEM_F_MEM_VALID_FLAGS)
ERR("invalid flags 0x%x", flags);
#endif
PMEM_API_START();
Funcs.memmove_nodrain(pmemdest, src, len, flags & ~PMEM_F_MEM_NODRAIN);
if ((flags & (PMEM_F_MEM_NODRAIN | PMEM_F_MEM_NOFLUSH)) == 0)
pmem_drain();
PMEM_API_END();
return pmemdest;
}
/*
* do_copy_to_pmem -- copy to pmem, postponing drain step until the end
*/
void
do_copy_to_pmem(char *pmemaddr, int srcfd, off_t len)
{
char buf[BUF_LEN];
int cc;
/* copy the file, saving the last flush step to the end */
while ((cc = read(srcfd, buf, BUF_LEN)) > 0) {
pmem_memcpy_nodrain(pmemaddr, buf, cc);
pmemaddr += cc;
}
if (cc < 0) {
perror("read");
exit(1);
}
/* perform final flush step */
pmem_drain();
}