int verify_load_state(struct thread_data *td, const char *prefix) { struct verify_state_hdr hdr; void *s = NULL; uint64_t crc; ssize_t ret; int fd; if (!td->o.verify_state) return 0; fd = open_state_file(td->o.name, prefix, td->thread_number - 1, 0); if (fd == -1) return 1; ret = read(fd, &hdr, sizeof(hdr)); if (ret != sizeof(hdr)) { if (ret < 0) td_verror(td, errno, "read verify state hdr"); log_err("fio: failed reading verify state header\n"); goto err; } hdr.version = le64_to_cpu(hdr.version); hdr.size = le64_to_cpu(hdr.size); hdr.crc = le64_to_cpu(hdr.crc); if (hdr.version != VSTATE_HDR_VERSION && hdr.version != VSTATE_HDR_VERSION_V1) { log_err("fio: bad version in verify state header\n"); goto err; } s = malloc(hdr.size); ret = read(fd, s, hdr.size); if (ret != hdr.size) { if (ret < 0) td_verror(td, errno, "read verify state"); log_err("fio: failed reading verity state\n"); goto err; } crc = fio_crc32c(s, hdr.size); if (crc != hdr.crc) { log_err("fio: verify state is corrupt\n"); goto err; } close(fd); verify_convert_assign_state(td, s, hdr.version); return 0; err: if (s) free(s); close(fd); return 1; }
static int pre_read_file(struct thread_data *td, struct fio_file *f) { int ret = 0, r, did_open = 0, old_runstate; unsigned long long left; unsigned int bs; char *b; if (td->io_ops->flags & FIO_PIPEIO) return 0; if (!fio_file_open(f)) { if (td->io_ops->open_file(td, f)) { log_err("fio: cannot pre-read, failed to open file\n"); return 1; } did_open = 1; } old_runstate = td_bump_runstate(td, TD_PRE_READING); bs = td->o.max_bs[DDIR_READ]; b = malloc(bs); memset(b, 0, bs); if (lseek(f->fd, f->file_offset, SEEK_SET) < 0) { td_verror(td, errno, "lseek"); log_err("fio: failed to lseek pre-read file\n"); ret = 1; goto error; } left = f->io_size; while (left && !td->terminate) { if (bs > left) bs = left; r = read(f->fd, b, bs); if (r == (int) bs) { left -= bs; continue; } else { td_verror(td, EIO, "pre_read"); break; } } error: td_restore_runstate(td, old_runstate); if (did_open) td->io_ops->close_file(td, f); free(b); return ret; }
static int __file_invalidate_cache(struct thread_data *td, struct fio_file *f, unsigned long long off, unsigned long long len) { int ret = 0; if (len == -1ULL) len = f->io_size; if (off == -1ULL) off = f->file_offset; if (len == -1ULL || off == -1ULL) return 0; dprint(FD_IO, "invalidate cache %s: %llu/%llu\n", f->file_name, off, len); /* * FIXME: add blockdev flushing too */ if (f->mmap_ptr) { ret = posix_madvise(f->mmap_ptr, f->mmap_sz, POSIX_MADV_DONTNEED); #ifdef FIO_MADV_FREE if (f->filetype == FIO_TYPE_BD) (void) posix_madvise(f->mmap_ptr, f->mmap_sz, FIO_MADV_FREE); #endif } else if (f->filetype == FIO_TYPE_FILE) { ret = posix_fadvise(f->fd, off, len, POSIX_FADV_DONTNEED); } else if (f->filetype == FIO_TYPE_BD) { ret = blockdev_invalidate_cache(f); if (ret < 0 && errno == EACCES && geteuid()) { if (!root_warn) { log_err("fio: only root may flush block " "devices. Cache flush bypassed!\n"); root_warn = 1; } ret = 0; } } else if (f->filetype == FIO_TYPE_CHAR || f->filetype == FIO_TYPE_PIPE) ret = 0; if (ret < 0) { td_verror(td, errno, "invalidate_cache"); return 1; } else if (ret > 0) { td_verror(td, ret, "invalidate_cache"); return 1; } return ret; }
static int fio_mmapio_open(struct thread_data *td, struct fio_file *f) { int ret, flags; ret = generic_open_file(td, f); if (ret) return ret; /* * for size checkup, don't mmap anything. */ if (!f->io_size) return 0; if (td_rw(td)) flags = PROT_READ | PROT_WRITE; else if (td_write(td)) { flags = PROT_WRITE; if (td->o.verify != VERIFY_NONE) flags |= PROT_READ; } else flags = PROT_READ; f->mmap = mmap(NULL, f->io_size, flags, MAP_SHARED, f->fd, f->file_offset); if (f->mmap == MAP_FAILED) { f->mmap = NULL; td_verror(td, errno, "mmap"); goto err; } if (file_invalidate_cache(td, f)) goto err; if (!td_random(td)) { if (madvise(f->mmap, f->io_size, MADV_SEQUENTIAL) < 0) { td_verror(td, errno, "madvise"); goto err; } } else { if (madvise(f->mmap, f->io_size, MADV_RANDOM) < 0) { td_verror(td, errno, "madvise"); goto err; } } return 0; err: td->io_ops->close_file(td, f); return 1; }
static int fio_vsyncio_commit(struct thread_data *td) { struct syncio_data *sd = td->io_ops->data; struct fio_file *f; ssize_t ret; if (!sd->queued) return 0; io_u_mark_submit(td, sd->queued); f = sd->last_file; if (lseek(f->fd, sd->io_us[0]->offset, SEEK_SET) == -1) { int err = -errno; td_verror(td, errno, "lseek"); return err; } if (sd->last_ddir == DDIR_READ) ret = readv(f->fd, sd->iovecs, sd->queued); else ret = writev(f->fd, sd->iovecs, sd->queued); dprint(FD_IO, "vsyncio_commit: %d\n", (int) ret); sd->events = sd->queued; sd->queued = 0; return fio_vsyncio_end(td, ret); }
/* * open iolog, check version, and call appropriate parser */ static int init_iolog_read(struct thread_data *td) { char buffer[256], *p; FILE *f; int ret; f = fopen(td->o.read_iolog_file, "r"); if (!f) { perror("fopen read iolog"); return 1; } p = fgets(buffer, sizeof(buffer), f); if (!p) { td_verror(td, errno, "iolog read"); log_err("fio: unable to read iolog\n"); fclose(f); return 1; } /* * version 2 of the iolog stores a specific string as the * first line, check for that */ if (!strncmp(iolog_ver2, buffer, strlen(iolog_ver2))) ret = read_iolog2(td, f); else { log_err("fio: iolog version 1 is no longer supported\n"); ret = 1; } fclose(f); return ret; }
static int fio_syslet_queue(struct thread_data *td, struct io_u *io_u) { struct syslet_data *sd = td->io_ops->data; union indirect_params params; struct indirect_registers regs; int ret; fio_ro_check(td, io_u); memset(¶ms, 0, sizeof(params)); fill_syslet_args(¶ms.syslet, sd->ring, (long)io_u, ret_func, sd->stack); fio_syslet_prep(io_u, ®s); ret = syscall(__NR_indirect, ®s, ¶ms, sizeof(params), 0); if (ret == (int) io_u->xfer_buflen) { /* * completed sync, account. this also catches fsync(). */ return FIO_Q_COMPLETED; } else if (ret < 0) { /* * queued for async execution */ if (errno == ESYSLETPENDING) return FIO_Q_QUEUED; } io_u->error = errno; td_verror(td, io_u->error, "xfer"); return FIO_Q_COMPLETED; }
void put_file_log(struct thread_data *td, struct fio_file *f) { int ret = put_file(td, f); if (ret) td_verror(td, ret, "file close"); }
static int char_size(struct thread_data *td, struct fio_file *f) { #ifdef FIO_HAVE_CHARDEV_SIZE unsigned long long bytes = 0; int r; if (td->io_ops->open_file(td, f)) { log_err("fio: failed opening blockdev %s for size check\n", f->file_name); return 1; } r = chardev_size(f, &bytes); if (r) { td_verror(td, r, "chardev_size"); goto err; } if (!bytes) { log_err("%s: zero sized char device?\n", f->file_name); goto err; } f->real_file_size = bytes; td->io_ops->close_file(td, f); return 0; err: td->io_ops->close_file(td, f); return 1; #else f->real_file_size = -1ULL; return 0; #endif }
static int fio_guasi_init(struct thread_data *td) { int maxthr; struct guasi_data *ld = malloc(sizeof(*ld)); GDBG_PRINT(("fio_guasi_init(): depth=%d\n", td->o.iodepth)); memset(ld, 0, sizeof(*ld)); maxthr = td->o.iodepth > GFIO_MIN_THREADS ? td->o.iodepth: GFIO_MIN_THREADS; if (maxthr > GFIO_MAX_THREADS) maxthr = GFIO_MAX_THREADS; if ((ld->hctx = guasi_create(GFIO_MIN_THREADS, maxthr, 1)) == NULL) { td_verror(td, errno, "guasi_create"); free(ld); return 1; } ld->max_reqs = td->o.iodepth; ld->reqs = malloc(ld->max_reqs * sizeof(guasi_req_t)); ld->io_us = malloc(ld->max_reqs * sizeof(struct io_u *)); memset(ld->io_us, 0, ld->max_reqs * sizeof(struct io_u *)); ld->queued_nr = 0; ld->reqs_nr = 0; td->io_ops->data = ld; GDBG_PRINT(("fio_guasi_init(): depth=%d -> %p\n", td->o.iodepth, ld)); return 0; }
static int fio_ioring_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t) { struct ioring_data *ld = td->io_ops_data; unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min; struct ioring_options *o = td->eo; struct io_cq_ring *ring = &ld->cq_ring; unsigned events = 0; int r; ld->cq_ring_off = *ring->head; do { r = fio_ioring_cqring_reap(td, events, max); if (r) { events += r; continue; } if (!o->sqpoll_thread) { r = io_uring_enter(ld, 0, actual_min, IORING_ENTER_GETEVENTS); if (r < 0) { if (errno == EAGAIN) continue; td_verror(td, errno, "io_uring_enter"); break; } } } while (events < min); return r < 0 ? r : events; }
static int fio_mtd_open_file(struct thread_data *td, struct fio_file *f) { struct fio_mtd_data *fmd; int ret; ret = generic_open_file(td, f); if (ret) return ret; fmd = calloc(1, sizeof(*fmd)); if (!fmd) goto err_close; ret = mtd_get_dev_info(desc, f->file_name, &fmd->info); if (ret != 0) { td_verror(td, errno, "mtd_get_dev_info"); goto err_free; } FILE_SET_ENG_DATA(f, fmd); return 0; err_free: free(fmd); err_close: { int fio_unused __ret; __ret = generic_close_file(td, f); return 1; } }
/* * Complete a single io_u for the sync engines. */ int io_u_sync_complete(struct thread_data *td, struct io_u *io_u, uint64_t *bytes) { struct io_completion_data icd; init_icd(td, &icd, 1); io_completed(td, io_u, &icd); if (!(io_u->flags & IO_U_F_FREE_DEF)) put_io_u(td, io_u); if (icd.error) { td_verror(td, icd.error, "io_u_sync_complete"); return -1; } if (bytes) { int ddir; for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) bytes[ddir] += icd.bytes_done[ddir]; } return 0; }
static int bdev_size(struct thread_data *td, struct fio_file *f) { unsigned long long bytes = 0; int r; if (td->io_ops->open_file(td, f)) { log_err("fio: failed opening blockdev %s for size check\n", f->file_name); return 1; } r = blockdev_size(f, &bytes); if (r) { td_verror(td, r, "blockdev_size"); goto err; } if (!bytes) { log_err("%s: zero sized block device?\n", f->file_name); goto err; } f->real_file_size = bytes; td->io_ops->close_file(td, f); return 0; err: td->io_ops->close_file(td, f); return 1; }
static int ipo_special(struct thread_data *td, struct io_piece *ipo) { struct fio_file *f; int ret; /* * Not a special ipo */ if (ipo->ddir != DDIR_INVAL) return 0; f = td->files[ipo->fileno]; switch (ipo->file_action) { case FIO_LOG_OPEN_FILE: ret = td_io_open_file(td, f); if (!ret) break; td_verror(td, ret, "iolog open file"); return -1; case FIO_LOG_CLOSE_FILE: td_io_close_file(td, f); break; case FIO_LOG_UNLINK_FILE: td_io_unlink_file(td, f); break; default: log_err("fio: bad file action %d\n", ipo->file_action); break; } return 1; }
static void lat_fatal(struct thread_data *td, struct io_completion_data *icd, unsigned long tusec, unsigned long max_usec) { if (!td->error) log_err("fio: latency of %lu usec exceeds specified max (%lu usec)\n", tusec, max_usec); td_verror(td, ETIMEDOUT, "max latency exceeded"); icd->error = ETIMEDOUT; }
static int fio_ioring_commit(struct thread_data *td) { struct ioring_data *ld = td->io_ops_data; struct ioring_options *o = td->eo; int ret; if (!ld->queued) return 0; /* * Kernel side does submission. just need to check if the ring is * flagged as needing a kick, if so, call io_uring_enter(). This * only happens if we've been idle too long. */ if (o->sqpoll_thread) { struct io_sq_ring *ring = &ld->sq_ring; read_barrier(); if (*ring->flags & IORING_SQ_NEED_WAKEUP) io_uring_enter(ld, ld->queued, 0, IORING_ENTER_SQ_WAKEUP); ld->queued = 0; return 0; } do { unsigned start = *ld->sq_ring.head; long nr = ld->queued; ret = io_uring_enter(ld, nr, 0, IORING_ENTER_GETEVENTS); if (ret > 0) { fio_ioring_queued(td, start, ret); io_u_mark_submit(td, ret); ld->queued -= ret; ret = 0; } else if (!ret) { io_u_mark_submit(td, ret); continue; } else { if (errno == EAGAIN) { ret = fio_ioring_cqring_reap(td, 0, ld->queued); if (ret) continue; /* Shouldn't happen */ usleep(1); continue; } td_verror(td, errno, "io_uring_enter submit"); break; } } while (ld->queued); return ret; }
static int fio_mmap_file(struct thread_data *td, struct fio_file *f, size_t length, off_t off) { struct fio_mmap_data *fmd = FILE_ENG_DATA(f); int flags = 0; if (td_rw(td) && !td->o.verify_only) flags = PROT_READ | PROT_WRITE; else if (td_write(td) && !td->o.verify_only) { flags = PROT_WRITE; if (td->o.verify != VERIFY_NONE) flags |= PROT_READ; } else flags = PROT_READ; fmd->mmap_ptr = mmap(NULL, length, flags, MAP_SHARED, f->fd, off); if (fmd->mmap_ptr == MAP_FAILED) { fmd->mmap_ptr = NULL; td_verror(td, errno, "mmap"); goto err; } if (!fio_madvise_file(td, f, length)) goto err; if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) { td_verror(td, errno, "madvise"); goto err; } #ifdef FIO_MADV_FREE if (f->filetype == FIO_TYPE_BLOCK) (void) posix_madvise(fmd->mmap_ptr, fmd->mmap_sz, FIO_MADV_FREE); #endif err: if (td->error && fmd->mmap_ptr) munmap(fmd->mmap_ptr, length); return td->error; }
static enum fio_q_status fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; struct fio_mmap_data *fmd = FILE_ENG_DATA(f); fio_ro_check(td, io_u); if (io_u->ddir == DDIR_READ) memcpy(io_u->xfer_buf, io_u->mmap_data, io_u->xfer_buflen); else if (io_u->ddir == DDIR_WRITE) memcpy(io_u->mmap_data, io_u->xfer_buf, io_u->xfer_buflen); else if (ddir_sync(io_u->ddir)) { if (msync(fmd->mmap_ptr, fmd->mmap_sz, MS_SYNC)) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } } else if (io_u->ddir == DDIR_TRIM) { int ret = do_io_u_trim(td, io_u); if (!ret) td_verror(td, io_u->error, "trim"); } /* * not really direct, but should drop the pages from the cache */ if (td->o.odirect && ddir_rw(io_u->ddir)) { if (msync(io_u->mmap_data, io_u->xfer_buflen, MS_SYNC) < 0) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } if (posix_madvise(io_u->mmap_data, io_u->xfer_buflen, POSIX_MADV_DONTNEED) < 0) { io_u->error = errno; td_verror(td, io_u->error, "madvise"); } } return FIO_Q_COMPLETED; }
static int fio_mtd_is_bad(struct thread_data *td, struct fio_mtd_data *fmd, struct io_u *io_u, int eb) { int ret = mtd_is_bad(&fmd->info, io_u->file->fd, eb); if (ret == -1) { io_u->error = errno; td_verror(td, errno, "mtd_is_bad"); } else if (ret == 1) io_u->error = EIO; /* Silent failure--don't flood stderr */ return ret; }
static int file_size(struct thread_data *td, struct fio_file *f) { struct stat st; if (stat(f->file_name, &st) == -1) { td_verror(td, errno, "fstat"); return 1; } f->real_file_size = st.st_size; return 0; }
static int fio_mmapio_queue(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; unsigned long long real_off = io_u->offset - f->file_offset; fio_ro_check(td, io_u); if (io_u->ddir == DDIR_READ) memcpy(io_u->xfer_buf, f->mmap + real_off, io_u->xfer_buflen); else if (io_u->ddir == DDIR_WRITE) memcpy(f->mmap + real_off, io_u->xfer_buf, io_u->xfer_buflen); else if (io_u->ddir == DDIR_SYNC) { size_t len = (f->io_size + page_size - 1) & ~page_mask; if (msync(f->mmap, len, MS_SYNC)) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } } /* * not really direct, but should drop the pages from the cache */ if (td->o.odirect && io_u->ddir != DDIR_SYNC) { size_t len = (io_u->xfer_buflen + page_size - 1) & ~page_mask; unsigned long long off = real_off & ~page_mask; if (msync(f->mmap + off, len, MS_SYNC) < 0) { io_u->error = errno; td_verror(td, io_u->error, "msync"); } if (madvise(f->mmap + off, len, MADV_DONTNEED) < 0) { io_u->error = errno; td_verror(td, io_u->error, "madvise"); } } return FIO_Q_COMPLETED; }
int fio_mtd_get_file_size(struct thread_data *td, struct fio_file *f) { struct mtd_dev_info info; int ret = mtd_get_dev_info(desc, f->file_name, &info); if (ret != 0) { td_verror(td, errno, "mtd_get_dev_info"); return errno; } f->real_file_size = info.size; return 0; }
static bool fio_madvise_file(struct thread_data *td, struct fio_file *f, size_t length) { struct fio_mmap_data *fmd = FILE_ENG_DATA(f); if (!td->o.fadvise_hint) return true; if (!td_random(td)) { if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) { td_verror(td, errno, "madvise"); return false; } } else { if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) { td_verror(td, errno, "madvise"); return false; } } return true; }
static int fio_syncio_prep(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; if (ddir_sync(io_u->ddir)) return 0; if (lseek(f->fd, io_u->offset, SEEK_SET) == -1) { td_verror(td, errno, "lseek"); return 1; } return 0; }
static int fio_mtd_maybe_mark_bad(struct thread_data *td, struct fio_mtd_data *fmd, struct io_u *io_u, int eb) { int ret; if (errno == EIO) { ret = mtd_mark_bad(&fmd->info, io_u->file->fd, eb); if (ret != 0) { io_u->error = errno; td_verror(td, errno, "mtd_mark_bad"); return -1; } } return 0; }
static int fio_io_end(struct thread_data *td, struct io_u *io_u, int ret) { if (ret != (int) io_u->xfer_buflen) { if (ret >= 0) { io_u->resid = io_u->xfer_buflen - ret; io_u->error = 0; return FIO_Q_COMPLETED; } else io_u->error = errno; } if (io_u->error) td_verror(td, io_u->error, "xfer"); return FIO_Q_COMPLETED; }
static int fio_solarisaio_queue(struct thread_data fio_unused *td, struct io_u *io_u) { struct solarisaio_data *sd = td->io_ops->data; struct fio_file *f = io_u->file; off_t off; int ret; fio_ro_check(td, io_u); if (io_u->ddir == DDIR_SYNC) { if (sd->nr) return FIO_Q_BUSY; if (fsync(f->fd) < 0) io_u->error = errno; return FIO_Q_COMPLETED; } if (io_u->ddir == DDIR_DATASYNC) { if (sd->nr) return FIO_Q_BUSY; if (fdatasync(f->fd) < 0) io_u->error = errno; return FIO_Q_COMPLETED; } if (sd->nr == sd->max_depth) return FIO_Q_BUSY; off = io_u->offset; if (io_u->ddir == DDIR_READ) ret = aioread(f->fd, io_u->xfer_buf, io_u->xfer_buflen, off, SEEK_SET, &io_u->resultp); else ret = aiowrite(f->fd, io_u->xfer_buf, io_u->xfer_buflen, off, SEEK_SET, &io_u->resultp); if (ret) { io_u->error = errno; td_verror(td, io_u->error, "xfer"); return FIO_Q_COMPLETED; } sd->nr++; return FIO_Q_QUEUED; }
static int fio_gf_prep(struct thread_data *td, struct io_u *io_u) { struct fio_file *f = io_u->file; struct gf_data *g = td->io_ops_data; dprint(FD_FILE, "fio prep\n"); if (!ddir_rw(io_u->ddir)) return 0; if (LAST_POS(f) != -1ULL && LAST_POS(f) == io_u->offset) return 0; if (glfs_lseek(g->fd, io_u->offset, SEEK_SET) < 0) { td_verror(td, errno, "lseek"); return 1; } return 0; }
static int fio_ime_get_file_size(struct thread_data *td, struct fio_file *f) { struct stat buf; int ret; char *ime_filename; dprint(FD_FILE, "get file size %s\n", f->file_name); ime_filename = fio_set_ime_filename(f->file_name); if (ime_filename == NULL) return 1; ret = ime_native_stat(ime_filename, &buf); if (ret == -1) { td_verror(td, errno, "fstat"); return 1; } f->real_file_size = buf.st_size; return 0; }