int verify_load_state(struct thread_data *td, const char *prefix)
{
struct verify_state_hdr hdr;
void *s = NULL;
uint64_t crc;
ssize_t ret;
int fd;
if (!td->o.verify_state)
return 0;
fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0);
if (fd == -1)
return 1;
ret = read(fd, &hdr, sizeof(hdr));
if (ret != sizeof(hdr)) {
if (ret < 0)
td_verror(td, errno, "read verify state hdr");
log_err("fio: failed reading verify state header\n");
goto err;
}
hdr.version = le64_to_cpu(hdr.version);
hdr.size = le64_to_cpu(hdr.size);
hdr.crc = le64_to_cpu(hdr.crc);
if (hdr.version != VSTATE_HDR_VERSION &&
hdr.version != VSTATE_HDR_VERSION_V1) {
log_err("fio: bad version in verify state header\n");
goto err;
}
s = malloc(hdr.size);
ret = read(fd, s, hdr.size);
if (ret != hdr.size) {
if (ret < 0)
td_verror(td, errno, "read verify state");
log_err("fio: failed reading verity state\n");
goto err;
}
crc = fio_crc32c(s, hdr.size);
if (crc != hdr.crc) {
log_err("fio: verify state is corrupt\n");
goto err;
}
close(fd);
verify_convert_assign_state(td, s, hdr.version);
return 0;
err:
if (s)
free(s);
close(fd);
return 1;
}
static int pre_read_file(struct thread_data *td, struct fio_file *f)
{
int ret = 0, r, did_open = 0, old_runstate;
unsigned long long left;
unsigned int bs;
char *b;
if (td->io_ops->flags & FIO_PIPEIO)
return 0;
if (!fio_file_open(f)) {
if (td->io_ops->open_file(td, f)) {
log_err("fio: cannot pre-read, failed to open file\n");
return 1;
}
did_open = 1;
}
old_runstate = td_bump_runstate(td, TD_PRE_READING);
bs = td->o.max_bs[DDIR_READ];
b = malloc(bs);
memset(b, 0, bs);
if (lseek(f->fd, f->file_offset, SEEK_SET) < 0) {
td_verror(td, errno, "lseek");
log_err("fio: failed to lseek pre-read file\n");
ret = 1;
goto error;
}
left = f->io_size;
while (left && !td->terminate) {
if (bs > left)
bs = left;
r = read(f->fd, b, bs);
if (r == (int) bs) {
left -= bs;
continue;
} else {
td_verror(td, EIO, "pre_read");
break;
}
}
error:
td_restore_runstate(td, old_runstate);
if (did_open)
td->io_ops->close_file(td, f);
free(b);
return ret;
}
static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f,
unsigned long long off,
unsigned long long len)
{
int ret = 0;
if (len == -1ULL)
len = f->io_size;
if (off == -1ULL)
off = f->file_offset;
if (len == -1ULL || off == -1ULL)
return 0;
dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off,
len);
/*
* FIXME: add blockdev flushing too
*/
if (f->mmap_ptr) {
ret = posix_madvise(f->mmap_ptr, f->mmap_sz, POSIX_MADV_DONTNEED);
#ifdef FIO_MADV_FREE
if (f->filetype == FIO_TYPE_BD)
(void) posix_madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE);
#endif
} else if (f->filetype == FIO_TYPE_FILE) {
ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED);
} else if (f->filetype == FIO_TYPE_BD) {
ret = blockdev_invalidate_cache(f);
if (ret < 0 && errno == EACCES && geteuid()) {
if (!root_warn) {
log_err("fio: only root may flush block "
"devices. Cache flush bypassed!\n");
root_warn = 1;
}
ret = 0;
}
} else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE)
ret = 0;
if (ret < 0) {
td_verror(td, errno, "invalidate_cache");
return 1;
} else if (ret > 0) {
td_verror(td, ret, "invalidate_cache");
return 1;
}
return ret;
}
static int fio_mmapio_open(struct thread_data *td, struct fio_file *f)
{
int ret, flags;
ret = generic_open_file(td, f);
if (ret)
return ret;
/*
* for size checkup, don't mmap anything.
*/
if (!f->io_size)
return 0;
if (td_rw(td))
flags = PROT_READ | PROT_WRITE;
else if (td_write(td)) {
flags = PROT_WRITE;
if (td->o.verify != VERIFY_NONE)
flags |= PROT_READ;
} else
flags = PROT_READ;
f->mmap = mmap(NULL, f->io_size, flags, MAP_SHARED, f->fd, f->file_offset);
if (f->mmap == MAP_FAILED) {
f->mmap = NULL;
td_verror(td, errno, "mmap");
goto err;
}
if (file_invalidate_cache(td, f))
goto err;
if (!td_random(td)) {
if (madvise(f->mmap, f->io_size, MADV_SEQUENTIAL) < 0) {
td_verror(td, errno, "madvise");
goto err;
}
} else {
if (madvise(f->mmap, f->io_size, MADV_RANDOM) < 0) {
td_verror(td, errno, "madvise");
goto err;
}
}
return 0;
err:
td->io_ops->close_file(td, f);
return 1;
}
static int fio_vsyncio_commit(struct thread_data *td)
{
struct syncio_data *sd = td->io_ops->data;
struct fio_file *f;
ssize_t ret;
if (!sd->queued)
return 0;
io_u_mark_submit(td, sd->queued);
f = sd->last_file;
if (lseek(f->fd, sd->io_us[0]->offset, SEEK_SET) == -1) {
int err = -errno;
td_verror(td, errno, "lseek");
return err;
}
if (sd->last_ddir == DDIR_READ)
ret = readv(f->fd, sd->iovecs, sd->queued);
else
ret = writev(f->fd, sd->iovecs, sd->queued);
dprint(FD_IO, "vsyncio_commit: %d\n", (int) ret);
sd->events = sd->queued;
sd->queued = 0;
return fio_vsyncio_end(td, ret);
}
/*
* open iolog, check version, and call appropriate parser
*/
static int init_iolog_read(struct thread_data *td)
{
char buffer[256], *p;
FILE *f;
int ret;
f = fopen(td->o.read_iolog_file, "r");
if (!f) {
perror("fopen read iolog");
return 1;
}
p = fgets(buffer, sizeof(buffer), f);
if (!p) {
td_verror(td, errno, "iolog read");
log_err("fio: unable to read iolog\n");
fclose(f);
return 1;
}
/*
* version 2 of the iolog stores a specific string as the
* first line, check for that
*/
if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2)))
ret = read_iolog2(td, f);
else {
log_err("fio: iolog version 1 is no longer supported\n");
ret = 1;
}
fclose(f);
return ret;
}
static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u)
{
struct syslet_data *sd = td->io_ops->data;
union indirect_params params;
struct indirect_registers regs;
int ret;
fio_ro_check(td, io_u);
memset(¶ms, 0, sizeof(params));
fill_syslet_args(¶ms.syslet, sd->ring, (long)io_u, ret_func, sd->stack);
fio_syslet_prep(io_u, ®s);
ret = syscall(__NR_indirect, ®s, ¶ms, sizeof(params), 0);
if (ret == (int) io_u->xfer_buflen) {
/*
* completed sync, account. this also catches fsync().
*/
return FIO_Q_COMPLETED;
} else if (ret < 0) {
/*
* queued for async execution
*/
if (errno == ESYSLETPENDING)
return FIO_Q_QUEUED;
}
io_u->error = errno;
td_verror(td, io_u->error, "xfer");
return FIO_Q_COMPLETED;
}
void put_file_log(struct thread_data *td, struct fio_file *f)
{
int ret = put_file(td, f);
if (ret)
td_verror(td, ret, "file close");
}
static int char_size(struct thread_data *td, struct fio_file *f)
{
#ifdef FIO_HAVE_CHARDEV_SIZE
unsigned long long bytes = 0;
int r;
if (td->io_ops->open_file(td, f)) {
log_err("fio: failed opening blockdev %s for size check\n",
f->file_name);
return 1;
}
r = chardev_size(f, &bytes);
if (r) {
td_verror(td, r, "chardev_size");
goto err;
}
if (!bytes) {
log_err("%s: zero sized char device?\n", f->file_name);
goto err;
}
f->real_file_size = bytes;
td->io_ops->close_file(td, f);
return 0;
err:
td->io_ops->close_file(td, f);
return 1;
#else
f->real_file_size = -1ULL;
return 0;
#endif
}
static int fio_guasi_init(struct thread_data *td)
{
int maxthr;
struct guasi_data *ld = malloc(sizeof(*ld));
GDBG_PRINT(("fio_guasi_init(): depth=%d\n", td->o.iodepth));
memset(ld, 0, sizeof(*ld));
maxthr = td->o.iodepth > GFIO_MIN_THREADS ? td->o.iodepth: GFIO_MIN_THREADS;
if (maxthr > GFIO_MAX_THREADS)
maxthr = GFIO_MAX_THREADS;
if ((ld->hctx = guasi_create(GFIO_MIN_THREADS, maxthr, 1)) == NULL) {
td_verror(td, errno, "guasi_create");
free(ld);
return 1;
}
ld->max_reqs = td->o.iodepth;
ld->reqs = malloc(ld->max_reqs * sizeof(guasi_req_t));
ld->io_us = malloc(ld->max_reqs * sizeof(struct io_u *));
memset(ld->io_us, 0, ld->max_reqs * sizeof(struct io_u *));
ld->queued_nr = 0;
ld->reqs_nr = 0;
td->io_ops->data = ld;
GDBG_PRINT(("fio_guasi_init(): depth=%d -> %p\n", td->o.iodepth, ld));
return 0;
}
static int fio_ioring_getevents(struct thread_data *td, unsigned int min,
unsigned int max, const struct timespec *t)
{
struct ioring_data *ld = td->io_ops_data;
unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
struct ioring_options *o = td->eo;
struct io_cq_ring *ring = &ld->cq_ring;
unsigned events = 0;
int r;
ld->cq_ring_off = *ring->head;
do {
r = fio_ioring_cqring_reap(td, events, max);
if (r) {
events += r;
continue;
}
if (!o->sqpoll_thread) {
r = io_uring_enter(ld, 0, actual_min,
IORING_ENTER_GETEVENTS);
if (r < 0) {
if (errno == EAGAIN)
continue;
td_verror(td, errno, "io_uring_enter");
break;
}
}
} while (events < min);
return r < 0 ? r : events;
}
static int fio_mtd_open_file(struct thread_data *td, struct fio_file *f)
{
struct fio_mtd_data *fmd;
int ret;
ret = generic_open_file(td, f);
if (ret)
return ret;
fmd = calloc(1, sizeof(*fmd));
if (!fmd)
goto err_close;
ret = mtd_get_dev_info(desc, f->file_name, &fmd->info);
if (ret != 0) {
td_verror(td, errno, "mtd_get_dev_info");
goto err_free;
}
FILE_SET_ENG_DATA(f, fmd);
return 0;
err_free:
free(fmd);
err_close:
{
int fio_unused __ret;
__ret = generic_close_file(td, f);
return 1;
}
}
/*
* Complete a single io_u for the sync engines.
*/
int io_u_sync_complete(struct thread_data *td, struct io_u *io_u,
uint64_t *bytes)
{
struct io_completion_data icd;
init_icd(td, &icd, 1);
io_completed(td, io_u, &icd);
if (!(io_u->flags & IO_U_F_FREE_DEF))
put_io_u(td, io_u);
if (icd.error) {
td_verror(td, icd.error, "io_u_sync_complete");
return -1;
}
if (bytes) {
int ddir;
for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++)
bytes[ddir] += icd.bytes_done[ddir];
}
return 0;
}
static int bdev_size(struct thread_data *td, struct fio_file *f)
{
unsigned long long bytes = 0;
int r;
if (td->io_ops->open_file(td, f)) {
log_err("fio: failed opening blockdev %s for size check\n",
f->file_name);
return 1;
}
r = blockdev_size(f, &bytes);
if (r) {
td_verror(td, r, "blockdev_size");
goto err;
}
if (!bytes) {
log_err("%s: zero sized block device?\n", f->file_name);
goto err;
}
f->real_file_size = bytes;
td->io_ops->close_file(td, f);
return 0;
err:
td->io_ops->close_file(td, f);
return 1;
}
static int ipo_special(struct thread_data *td, struct io_piece *ipo)
{
struct fio_file *f;
int ret;
/*
* Not a special ipo
*/
if (ipo->ddir != DDIR_INVAL)
return 0;
f = td->files[ipo->fileno];
switch (ipo->file_action) {
case FIO_LOG_OPEN_FILE:
ret = td_io_open_file(td, f);
if (!ret)
break;
td_verror(td, ret, "iolog open file");
return -1;
case FIO_LOG_CLOSE_FILE:
td_io_close_file(td, f);
break;
case FIO_LOG_UNLINK_FILE:
td_io_unlink_file(td, f);
break;
default:
log_err("fio: bad file action %d\n", ipo->file_action);
break;
}
return 1;
}
static void lat_fatal(struct thread_data *td, struct io_completion_data *icd,
unsigned long tusec, unsigned long max_usec)
{
if (!td->error)
log_err("fio: latency of %lu usec exceeds specified max (%lu usec)\n", tusec, max_usec);
td_verror(td, ETIMEDOUT, "max latency exceeded");
icd->error = ETIMEDOUT;
}
static int fio_ioring_commit(struct thread_data *td)
{
struct ioring_data *ld = td->io_ops_data;
struct ioring_options *o = td->eo;
int ret;
if (!ld->queued)
return 0;
/*
* Kernel side does submission. just need to check if the ring is
* flagged as needing a kick, if so, call io_uring_enter(). This
* only happens if we've been idle too long.
*/
if (o->sqpoll_thread) {
struct io_sq_ring *ring = &ld->sq_ring;
read_barrier();
if (*ring->flags & IORING_SQ_NEED_WAKEUP)
io_uring_enter(ld, ld->queued, 0,
IORING_ENTER_SQ_WAKEUP);
ld->queued = 0;
return 0;
}
do {
unsigned start = *ld->sq_ring.head;
long nr = ld->queued;
ret = io_uring_enter(ld, nr, 0, IORING_ENTER_GETEVENTS);
if (ret > 0) {
fio_ioring_queued(td, start, ret);
io_u_mark_submit(td, ret);
ld->queued -= ret;
ret = 0;
} else if (!ret) {
io_u_mark_submit(td, ret);
continue;
} else {
if (errno == EAGAIN) {
ret = fio_ioring_cqring_reap(td, 0, ld->queued);
if (ret)
continue;
/* Shouldn't happen */
usleep(1);
continue;
}
td_verror(td, errno, "io_uring_enter submit");
break;
}
} while (ld->queued);
return ret;
}
static int fio_mmap_file(struct thread_data *td, struct fio_file *f,
size_t length, off_t off)
{
struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
int flags = 0;
if (td_rw(td) && !td->o.verify_only)
flags = PROT_READ | PROT_WRITE;
else if (td_write(td) && !td->o.verify_only) {
flags = PROT_WRITE;
if (td->o.verify != VERIFY_NONE)
flags |= PROT_READ;
} else
flags = PROT_READ;
fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off);
if (fmd->mmap_ptr == MAP_FAILED) {
fmd->mmap_ptr = NULL;
td_verror(td, errno, "mmap");
goto err;
}
if (!fio_madvise_file(td, f, length))
goto err;
if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) {
td_verror(td, errno, "madvise");
goto err;
}
#ifdef FIO_MADV_FREE
if (f->filetype == FIO_TYPE_BLOCK)
(void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE);
#endif
err:
if (td->error && fmd->mmap_ptr)
munmap(fmd->mmap_ptr, length);
return td->error;
}
static enum fio_q_status fio_mmapio_queue(struct thread_data *td,
struct io_u *io_u)
{
struct fio_file *f = io_u->file;
struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
fio_ro_check(td, io_u);
if (io_u->ddir == DDIR_READ)
memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen);
else if (io_u->ddir == DDIR_WRITE)
memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen);
else if (ddir_sync(io_u->ddir)) {
if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) {
io_u->error = errno;
td_verror(td, io_u->error, "msync");
}
} else if (io_u->ddir == DDIR_TRIM) {
int ret = do_io_u_trim(td, io_u);
if (!ret)
td_verror(td, io_u->error, "trim");
}
/*
* not really direct, but should drop the pages from the cache
*/
if (td->o.odirect && ddir_rw(io_u->ddir)) {
if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) {
io_u->error = errno;
td_verror(td, io_u->error, "msync");
}
if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) {
io_u->error = errno;
td_verror(td, io_u->error, "madvise");
}
}
return FIO_Q_COMPLETED;
}
static int fio_mtd_is_bad(struct thread_data *td,
struct fio_mtd_data *fmd,
struct io_u *io_u, int eb)
{
int ret = mtd_is_bad(&fmd->info, io_u->file->fd, eb);
if (ret == -1) {
io_u->error = errno;
td_verror(td, errno, "mtd_is_bad");
} else if (ret == 1)
io_u->error = EIO; /* Silent failure--don't flood stderr */
return ret;
}
static int file_size(struct thread_data *td, struct fio_file *f)
{
struct stat st;
if (stat(f->file_name, &st) == -1) {
td_verror(td, errno, "fstat");
return 1;
}
f->real_file_size = st.st_size;
return 0;
}
static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u)
{
struct fio_file *f = io_u->file;
unsigned long long real_off = io_u->offset - f->file_offset;
fio_ro_check(td, io_u);
if (io_u->ddir == DDIR_READ)
memcpy(io_u->xfer_buf, f->mmap + real_off, io_u->xfer_buflen);
else if (io_u->ddir == DDIR_WRITE)
memcpy(f->mmap + real_off, io_u->xfer_buf, io_u->xfer_buflen);
else if (io_u->ddir == DDIR_SYNC) {
size_t len = (f->io_size + page_size - 1) & ~page_mask;
if (msync(f->mmap, len, MS_SYNC)) {
io_u->error = errno;
td_verror(td, io_u->error, "msync");
}
}
/*
* not really direct, but should drop the pages from the cache
*/
if (td->o.odirect && io_u->ddir != DDIR_SYNC) {
size_t len = (io_u->xfer_buflen + page_size - 1) & ~page_mask;
unsigned long long off = real_off & ~page_mask;
if (msync(f->mmap + off, len, MS_SYNC) < 0) {
io_u->error = errno;
td_verror(td, io_u->error, "msync");
}
if (madvise(f->mmap + off, len, MADV_DONTNEED) < 0) {
io_u->error = errno;
td_verror(td, io_u->error, "madvise");
}
}
return FIO_Q_COMPLETED;
}
int fio_mtd_get_file_size(struct thread_data *td, struct fio_file *f)
{
struct mtd_dev_info info;
int ret = mtd_get_dev_info(desc, f->file_name, &info);
if (ret != 0) {
td_verror(td, errno, "mtd_get_dev_info");
return errno;
}
f->real_file_size = info.size;
return 0;
}
static bool fio_madvise_file(struct thread_data *td, struct fio_file *f,
size_t length)
{
struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
if (!td->o.fadvise_hint)
return true;
if (!td_random(td)) {
if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
td_verror(td, errno, "madvise");
return false;
}
} else {
if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
td_verror(td, errno, "madvise");
return false;
}
}
return true;
}
static int fio_syncio_prep(struct thread_data *td, struct io_u *io_u)
{
struct fio_file *f = io_u->file;
if (ddir_sync(io_u->ddir))
return 0;
if (lseek(f->fd, io_u->offset, SEEK_SET) == -1) {
td_verror(td, errno, "lseek");
return 1;
}
return 0;
}
static int fio_mtd_maybe_mark_bad(struct thread_data *td,
struct fio_mtd_data *fmd,
struct io_u *io_u, int eb)
{
int ret;
if (errno == EIO) {
ret = mtd_mark_bad(&fmd->info, io_u->file->fd, eb);
if (ret != 0) {
io_u->error = errno;
td_verror(td, errno, "mtd_mark_bad");
return -1;
}
}
return 0;
}
static int fio_io_end(struct thread_data *td, struct io_u *io_u, int ret)
{
if (ret != (int) io_u->xfer_buflen) {
if (ret >= 0) {
io_u->resid = io_u->xfer_buflen - ret;
io_u->error = 0;
return FIO_Q_COMPLETED;
} else
io_u->error = errno;
}
if (io_u->error)
td_verror(td, io_u->error, "xfer");
return FIO_Q_COMPLETED;
}
static int fio_solarisaio_queue(struct thread_data fio_unused *td,
struct io_u *io_u)
{
struct solarisaio_data *sd = td->io_ops->data;
struct fio_file *f = io_u->file;
off_t off;
int ret;
fio_ro_check(td, io_u);
if (io_u->ddir == DDIR_SYNC) {
if (sd->nr)
return FIO_Q_BUSY;
if (fsync(f->fd) < 0)
io_u->error = errno;
return FIO_Q_COMPLETED;
}
if (io_u->ddir == DDIR_DATASYNC) {
if (sd->nr)
return FIO_Q_BUSY;
if (fdatasync(f->fd) < 0)
io_u->error = errno;
return FIO_Q_COMPLETED;
}
if (sd->nr == sd->max_depth)
return FIO_Q_BUSY;
off = io_u->offset;
if (io_u->ddir == DDIR_READ)
ret = aioread(f->fd, io_u->xfer_buf, io_u->xfer_buflen, off,
SEEK_SET, &io_u->resultp);
else
ret = aiowrite(f->fd, io_u->xfer_buf, io_u->xfer_buflen, off,
SEEK_SET, &io_u->resultp);
if (ret) {
io_u->error = errno;
td_verror(td, io_u->error, "xfer");
return FIO_Q_COMPLETED;
}
sd->nr++;
return FIO_Q_QUEUED;
}
static int fio_gf_prep(struct thread_data *td, struct io_u *io_u)
{
struct fio_file *f = io_u->file;
struct gf_data *g = td->io_ops_data;
dprint(FD_FILE, "fio prep\n");
if (!ddir_rw(io_u->ddir))
return 0;
if (LAST_POS(f) != -1ULL && LAST_POS(f) == io_u->offset)
return 0;
if (glfs_lseek(g->fd, io_u->offset, SEEK_SET) < 0) {
td_verror(td, errno, "lseek");
return 1;
}
return 0;
}
static int fio_ime_get_file_size(struct thread_data *td, struct fio_file *f)
{
struct stat buf;
int ret;
char *ime_filename;
dprint(FD_FILE, "get file size %s\n", f->file_name);
ime_filename = fio_set_ime_filename(f->file_name);
if (ime_filename == NULL)
return 1;
ret = ime_native_stat(ime_filename, &buf);
if (ret == -1) {
td_verror(td, errno, "fstat");
return 1;
}
f->real_file_size = buf.st_size;
return 0;
}
