/*
* Sets the status of the 'td' in the printed status map.
*/
static void check_str_update(struct thread_data *td)
{
char c = run_str[td->thread_number - 1];
switch (td->runstate) {
case TD_REAPED:
c = '_';
break;
case TD_EXITED:
c = 'E';
break;
case TD_RAMP:
c = '/';
break;
case TD_RUNNING:
if (td_rw(td)) {
if (td_random(td))
c = 'm';
else
c = 'M';
} else if (td_read(td)) {
if (td_random(td))
c = 'r';
else
c = 'R';
} else {
if (td_random(td))
c = 'w';
else
c = 'W';
}
break;
case TD_PRE_READING:
c = 'p';
break;
case TD_VERIFYING:
c = 'V';
break;
case TD_FSYNCING:
c = 'F';
break;
case TD_CREATED:
c = 'C';
break;
case TD_INITIALIZED:
c = 'I';
break;
case TD_NOT_CREATED:
c = 'P';
break;
default:
log_err("state %d\n", td->runstate);
}
run_str[td->thread_number - 1] = c;
}
static void td_fill_rand_seeds_os(struct thread_data *td)
{
os_random_seed(td->rand_seeds[FIO_RAND_BS_OFF], &td->bsrange_state);
os_random_seed(td->rand_seeds[FIO_RAND_VER_OFF], &td->verify_state);
os_random_seed(td->rand_seeds[FIO_RAND_MIX_OFF], &td->rwmix_state);
if (td->o.file_service_type == FIO_FSERVICE_RANDOM)
os_random_seed(td->rand_seeds[FIO_RAND_FILE_OFF], &td->next_file_state);
os_random_seed(td->rand_seeds[FIO_RAND_FILE_SIZE_OFF], &td->file_size_state);
os_random_seed(td->rand_seeds[FIO_RAND_TRIM_OFF], &td->trim_state);
os_random_seed(td->rand_seeds[FIO_RAND_START_DELAY], &td->delay_state);
if (!td_random(td))
return;
if (td->o.rand_repeatable)
td->rand_seeds[FIO_RAND_BLOCK_OFF] = FIO_RANDSEED * td->thread_number;
os_random_seed(td->rand_seeds[FIO_RAND_BLOCK_OFF], &td->random_state);
os_random_seed(td->rand_seeds[FIO_RAND_SEQ_RAND_READ_OFF], &td->seq_rand_state[DDIR_READ]);
os_random_seed(td->rand_seeds[FIO_RAND_SEQ_RAND_WRITE_OFF], &td->seq_rand_state[DDIR_WRITE]);
os_random_seed(td->rand_seeds[FIO_RAND_SEQ_RAND_TRIM_OFF], &td->seq_rand_state[DDIR_TRIM]);
}
static int fill_io_u(struct thread_data *td, struct io_u *io_u)
{
if (td->io_ops->flags & FIO_NOIO)
goto out;
set_rw_ddir(td, io_u);
/*
* fsync() or fdatasync() or trim etc, we are done
*/
if (!ddir_rw(io_u->ddir))
goto out;
/*
* See if it's time to switch to a new zone
*/
if (td->zone_bytes >= td->o.zone_size && td->o.zone_skip) {
td->zone_bytes = 0;
io_u->file->file_offset += td->o.zone_range + td->o.zone_skip;
io_u->file->last_pos = io_u->file->file_offset;
td->io_skip_bytes += td->o.zone_skip;
}
/*
* No log, let the seq/rand engine retrieve the next buflen and
* position.
*/
if (get_next_offset(td, io_u)) {
dprint(FD_IO, "io_u %p, failed getting offset\n", io_u);
return 1;
}
io_u->buflen = get_next_buflen(td, io_u);
if (!io_u->buflen) {
dprint(FD_IO, "io_u %p, failed getting buflen\n", io_u);
return 1;
}
if (io_u->offset + io_u->buflen > io_u->file->real_file_size) {
dprint(FD_IO, "io_u %p, offset too large\n", io_u);
dprint(FD_IO, " off=%llu/%lu > %llu\n", io_u->offset,
io_u->buflen, io_u->file->real_file_size);
return 1;
}
/*
* mark entry before potentially trimming io_u
*/
if (td_random(td) && file_randommap(td, io_u->file))
mark_random_map(td, io_u);
/*
* If using a write iolog, store this entry.
*/
out:
dprint_io_u(io_u, "fill_io_u");
td->zone_bytes += io_u->buflen;
log_io_u(td, io_u);
return 0;
}
static int get_next_block(struct thread_data *td, struct io_u *io_u,
enum fio_ddir ddir, int rw_seq, unsigned long long *b)
{
struct fio_file *f = io_u->file;
int ret;
assert(ddir_rw(ddir));
if (rw_seq) {
if (td_random(td))
ret = get_next_rand_block(td, f, ddir, b);
else
ret = get_next_seq_block(td, f, ddir, b);
} else {
io_u->flags |= IO_U_F_BUSY_OK;
if (td->o.rw_seq == RW_SEQ_SEQ) {
ret = get_next_seq_block(td, f, ddir, b);
if (ret)
ret = get_next_rand_block(td, f, ddir, b);
} else if (td->o.rw_seq == RW_SEQ_IDENT) {
if (f->last_start != -1ULL)
*b = (f->last_start - f->file_offset)
/ td->o.min_bs[ddir];
else
*b = 0;
ret = 0;
} else {
log_err("fio: unknown rw_seq=%d\n", td->o.rw_seq);
ret = 1;
}
}
return ret;
}
static int fio_mmapio_open(struct thread_data *td, struct fio_file *f)
{
int ret, flags;
ret = generic_open_file(td, f);
if (ret)
return ret;
/*
* for size checkup, don't mmap anything.
*/
if (!f->io_size)
return 0;
if (td_rw(td))
flags = PROT_READ | PROT_WRITE;
else if (td_write(td)) {
flags = PROT_WRITE;
if (td->o.verify != VERIFY_NONE)
flags |= PROT_READ;
} else
flags = PROT_READ;
f->mmap = mmap(NULL, f->io_size, flags, MAP_SHARED, f->fd, f->file_offset);
if (f->mmap == MAP_FAILED) {
f->mmap = NULL;
td_verror(td, errno, "mmap");
goto err;
}
if (file_invalidate_cache(td, f))
goto err;
if (!td_random(td)) {
if (madvise(f->mmap, f->io_size, MADV_SEQUENTIAL) < 0) {
td_verror(td, errno, "madvise");
goto err;
}
} else {
if (madvise(f->mmap, f->io_size, MADV_RANDOM) < 0) {
td_verror(td, errno, "madvise");
goto err;
}
}
return 0;
err:
td->io_ops->close_file(td, f);
return 1;
}
/*
* Sort the reads for a verify phase in batches of verifysort_nr, if
* specified.
*/
static inline int should_sort_io(struct thread_data *td)
{
if (!td->o.verifysort_nr || !td->o.do_verify)
return 0;
if (!td_random(td))
return 0;
if (td->runstate != TD_VERIFYING)
return 0;
if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE)
return 0;
return 1;
}
/*
* Sort the reads for a verify phase in batches of verifysort_nr, if
* specified.
*/
static inline bool should_sort_io(struct thread_data *td)
{
if (!td->o.verifysort_nr || !td->o.do_verify)
return false;
if (!td_random(td))
return false;
if (td->runstate != TD_VERIFYING)
return false;
if (td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE ||
td->o.random_generator == FIO_RAND_GEN_TAUSWORTHE64)
return false;
return true;
}
static int get_next_block(struct thread_data *td, struct io_u *io_u,
enum fio_ddir ddir, int rw_seq)
{
struct fio_file *f = io_u->file;
unsigned long long b, offset;
int ret;
assert(ddir_rw(ddir));
b = offset = -1ULL;
if (rw_seq) {
if (td_random(td))
ret = get_next_rand_block(td, f, ddir, &b);
else
ret = get_next_seq_offset(td, f, ddir, &offset);
} else {
io_u->flags |= IO_U_F_BUSY_OK;
if (td->o.rw_seq == RW_SEQ_SEQ) {
ret = get_next_seq_offset(td, f, ddir, &offset);
if (ret)
ret = get_next_rand_block(td, f, ddir, &b);
} else if (td->o.rw_seq == RW_SEQ_IDENT) {
if (f->last_start != -1ULL)
offset = f->last_start - f->file_offset;
else
offset = 0;
ret = 0;
} else {
log_err("fio: unknown rw_seq=%d\n", td->o.rw_seq);
ret = 1;
}
}
if (!ret) {
if (offset != -1ULL)
io_u->offset = offset;
else if (b != -1ULL)
io_u->offset = b * td->o.ba[ddir];
else {
log_err("fio: bug in offset generation\n");
ret = 1;
}
}
return ret;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*/
if (!td_random(td) || !td->o.overwrite) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
p = &td->io_hist_tree.rb_node;
parent = NULL;
/*
* Sort the entry into the verification list
*/
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->offset <= __ipo->offset)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
}
static void td_fill_rand_seeds_internal(struct thread_data *td)
{
init_rand_seed(&td->__bsrange_state, td->rand_seeds[0]);
init_rand_seed(&td->__verify_state, td->rand_seeds[1]);
init_rand_seed(&td->__rwmix_state, td->rand_seeds[2]);
if (td->o.file_service_type == FIO_FSERVICE_RANDOM)
init_rand_seed(&td->__next_file_state, td->rand_seeds[3]);
init_rand_seed(&td->__file_size_state, td->rand_seeds[5]);
init_rand_seed(&td->__trim_state, td->rand_seeds[6]);
if (!td_random(td))
return;
if (td->o.rand_repeatable)
td->rand_seeds[4] = FIO_RANDSEED * td->thread_number;
init_rand_seed(&td->__random_state, td->rand_seeds[4]);
}
static void td_fill_rand_seeds_os(struct thread_data *td)
{
os_random_seed(td->rand_seeds[0], &td->bsrange_state);
os_random_seed(td->rand_seeds[1], &td->verify_state);
os_random_seed(td->rand_seeds[2], &td->rwmix_state);
if (td->o.file_service_type == FIO_FSERVICE_RANDOM)
os_random_seed(td->rand_seeds[3], &td->next_file_state);
os_random_seed(td->rand_seeds[5], &td->file_size_state);
os_random_seed(td->rand_seeds[6], &td->trim_state);
if (!td_random(td))
return;
if (td->o.rand_repeatable)
td->rand_seeds[4] = FIO_RANDSEED * td->thread_number;
os_random_seed(td->rand_seeds[4], &td->random_state);
}
static void td_fill_rand_seeds_internal(struct thread_data *td)
{
init_rand_seed(&td->__bsrange_state, td->rand_seeds[FIO_RAND_BS_OFF]);
init_rand_seed(&td->__verify_state, td->rand_seeds[FIO_RAND_VER_OFF]);
init_rand_seed(&td->__rwmix_state, td->rand_seeds[FIO_RAND_MIX_OFF]);
if (td->o.file_service_type == FIO_FSERVICE_RANDOM)
init_rand_seed(&td->__next_file_state, td->rand_seeds[FIO_RAND_FILE_OFF]);
init_rand_seed(&td->__file_size_state, td->rand_seeds[FIO_RAND_FILE_SIZE_OFF]);
init_rand_seed(&td->__trim_state, td->rand_seeds[FIO_RAND_TRIM_OFF]);
if (!td_random(td))
return;
if (td->o.rand_repeatable)
td->rand_seeds[FIO_RAND_BLOCK_OFF] = FIO_RANDSEED * td->thread_number;
init_rand_seed(&td->__random_state, td->rand_seeds[FIO_RAND_BLOCK_OFF]);
init_rand_seed(&td->__seq_rand_state[DDIR_READ], td->rand_seeds[FIO_RAND_SEQ_RAND_READ_OFF]);
init_rand_seed(&td->__seq_rand_state[DDIR_WRITE], td->rand_seeds[FIO_RAND_SEQ_RAND_WRITE_OFF]);
init_rand_seed(&td->__seq_rand_state[DDIR_TRIM], td->rand_seeds[FIO_RAND_SEQ_RAND_TRIM_OFF]);
}
static bool fio_madvise_file(struct thread_data *td, struct fio_file *f,
size_t length)
{
struct fio_mmap_data *fmd = FILE_ENG_DATA(f);
if (!td->o.fadvise_hint)
return true;
if (!td_random(td)) {
if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) {
td_verror(td, errno, "madvise");
return false;
}
} else {
if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) {
td_verror(td, errno, "madvise");
return false;
}
}
return true;
}
/*
* Lazy way of fixing up options that depend on each other. We could also
* define option callback handlers, but this is easier.
*/
static int fixup_options(struct thread_data *td)
{
struct thread_options *o = &td->o;
int ret = 0;
#ifndef FIO_HAVE_PSHARED_MUTEX
if (!o->use_thread) {
log_info("fio: this platform does not support process shared"
" mutexes, forcing use of threads. Use the 'thread'"
" option to get rid of this warning.\n");
o->use_thread = 1;
ret = warnings_fatal;
}
#endif
if (o->write_iolog_file && o->read_iolog_file) {
log_err("fio: read iolog overrides write_iolog\n");
free(o->write_iolog_file);
o->write_iolog_file = NULL;
ret = warnings_fatal;
}
/*
* only really works with 1 file
*/
if (o->zone_size && o->open_files == 1)
o->zone_size = 0;
/*
* If zone_range isn't specified, backward compatibility dictates it
* should be made equal to zone_size.
*/
if (o->zone_size && !o->zone_range)
o->zone_range = o->zone_size;
/*
* Reads can do overwrites, we always need to pre-create the file
*/
if (td_read(td) || td_rw(td))
o->overwrite = 1;
if (!o->min_bs[DDIR_READ])
o->min_bs[DDIR_READ] = o->bs[DDIR_READ];
if (!o->max_bs[DDIR_READ])
o->max_bs[DDIR_READ] = o->bs[DDIR_READ];
if (!o->min_bs[DDIR_WRITE])
o->min_bs[DDIR_WRITE] = o->bs[DDIR_WRITE];
if (!o->max_bs[DDIR_WRITE])
o->max_bs[DDIR_WRITE] = o->bs[DDIR_WRITE];
o->rw_min_bs = min(o->min_bs[DDIR_READ], o->min_bs[DDIR_WRITE]);
/*
* For random IO, allow blockalign offset other than min_bs.
*/
if (!o->ba[DDIR_READ] || !td_random(td))
o->ba[DDIR_READ] = o->min_bs[DDIR_READ];
if (!o->ba[DDIR_WRITE] || !td_random(td))
o->ba[DDIR_WRITE] = o->min_bs[DDIR_WRITE];
if ((o->ba[DDIR_READ] != o->min_bs[DDIR_READ] ||
o->ba[DDIR_WRITE] != o->min_bs[DDIR_WRITE]) &&
!o->norandommap) {
log_err("fio: Any use of blockalign= turns off randommap\n");
o->norandommap = 1;
ret = warnings_fatal;
}
if (!o->file_size_high)
o->file_size_high = o->file_size_low;
if (o->norandommap && o->verify != VERIFY_NONE
&& !fixed_block_size(o)) {
log_err("fio: norandommap given for variable block sizes, "
"verify disabled\n");
o->verify = VERIFY_NONE;
ret = warnings_fatal;
}
if (o->bs_unaligned && (o->odirect || td->io_ops->flags & FIO_RAWIO))
log_err("fio: bs_unaligned may not work with raw io\n");
/*
* thinktime_spin must be less than thinktime
*/
if (o->thinktime_spin > o->thinktime)
o->thinktime_spin = o->thinktime;
/*
* The low water mark cannot be bigger than the iodepth
*/
if (o->iodepth_low > o->iodepth || !o->iodepth_low) {
/*
* syslet work around - if the workload is sequential,
* we want to let the queue drain all the way down to
* avoid seeking between async threads
*/
if (!strcmp(td->io_ops->name, "syslet-rw") && !td_random(td))
o->iodepth_low = 1;
else
o->iodepth_low = o->iodepth;
}
/*
* If batch number isn't set, default to the same as iodepth
*/
if (o->iodepth_batch > o->iodepth || !o->iodepth_batch)
o->iodepth_batch = o->iodepth;
if (o->nr_files > td->files_index)
o->nr_files = td->files_index;
if (o->open_files > o->nr_files || !o->open_files)
o->open_files = o->nr_files;
if (((o->rate[0] + o->rate[1]) && (o->rate_iops[0] + o->rate_iops[1]))||
((o->ratemin[0] + o->ratemin[1]) && (o->rate_iops_min[0] +
o->rate_iops_min[1]))) {
log_err("fio: rate and rate_iops are mutually exclusive\n");
ret = 1;
}
if ((o->rate[0] < o->ratemin[0]) || (o->rate[1] < o->ratemin[1]) ||
(o->rate_iops[0] < o->rate_iops_min[0]) ||
(o->rate_iops[1] < o->rate_iops_min[1])) {
log_err("fio: minimum rate exceeds rate\n");
ret = 1;
}
if (!o->timeout && o->time_based) {
log_err("fio: time_based requires a runtime/timeout setting\n");
o->time_based = 0;
ret = warnings_fatal;
}
if (o->fill_device && !o->size)
o->size = -1ULL;
if (o->verify != VERIFY_NONE) {
if (td_write(td) && o->do_verify && o->numjobs > 1) {
log_info("Multiple writers may overwrite blocks that "
"belong to other jobs. This can cause "
"verification failures.\n");
ret = warnings_fatal;
}
o->refill_buffers = 1;
if (o->max_bs[DDIR_WRITE] != o->min_bs[DDIR_WRITE] &&
!o->verify_interval)
o->verify_interval = o->min_bs[DDIR_WRITE];
}
if (o->pre_read) {
o->invalidate_cache = 0;
if (td->io_ops->flags & FIO_PIPEIO) {
log_info("fio: cannot pre-read files with an IO engine"
" that isn't seekable. Pre-read disabled.\n");
ret = warnings_fatal;
}
}
#ifndef FIO_HAVE_FDATASYNC
if (o->fdatasync_blocks) {
log_info("fio: this platform does not support fdatasync()"
" falling back to using fsync(). Use the 'fsync'"
" option instead of 'fdatasync' to get rid of"
" this warning\n");
o->fsync_blocks = o->fdatasync_blocks;
o->fdatasync_blocks = 0;
ret = warnings_fatal;
}
#endif
#ifdef WIN32
/*
* Windows doesn't support O_DIRECT or O_SYNC with the _open interface,
* so fail if we're passed those flags
*/
if ((td->io_ops->flags & FIO_SYNCIO) && (td->o.odirect || td->o.sync_io)) {
log_err("fio: Windows does not support direct or non-buffered io with"
" the synchronous ioengines. Use the 'windowsaio' ioengine"
" with 'direct=1' and 'iodepth=1' instead.\n");
ret = 1;
}
#endif
/*
* For fully compressible data, just zero them at init time.
* It's faster than repeatedly filling it.
*/
if (td->o.compress_percentage == 100) {
td->o.zero_buffers = 1;
td->o.compress_percentage = 0;
}
return ret;
}
/*
* Sets the status of the 'td' in the printed status map.
*/
static void check_str_update(struct thread_data *td)
{
char c = __run_str[td->thread_number - 1];
switch (td->runstate) {
case TD_REAPED:
if (td->error)
c = 'X';
else if (td->sig)
c = 'K';
else
c = '_';
break;
case TD_EXITED:
c = 'E';
break;
case TD_RAMP:
c = '/';
break;
case TD_RUNNING:
if (td_rw(td)) {
if (td_random(td)) {
if (td->o.rwmix[DDIR_READ] == 100)
c = 'r';
else if (td->o.rwmix[DDIR_WRITE] == 100)
c = 'w';
else
c = 'm';
} else {
if (td->o.rwmix[DDIR_READ] == 100)
c = 'R';
else if (td->o.rwmix[DDIR_WRITE] == 100)
c = 'W';
else
c = 'M';
}
} else if (td_read(td)) {
if (td_random(td))
c = 'r';
else
c = 'R';
} else if (td_write(td)) {
if (td_random(td))
c = 'w';
else
c = 'W';
} else {
if (td_random(td))
c = 'd';
else
c = 'D';
}
break;
case TD_PRE_READING:
c = 'p';
break;
case TD_VERIFYING:
c = 'V';
break;
case TD_FSYNCING:
c = 'F';
break;
case TD_FINISHING:
c = 'f';
break;
case TD_CREATED:
c = 'C';
break;
case TD_INITIALIZED:
case TD_SETTING_UP:
c = 'I';
break;
case TD_NOT_CREATED:
c = 'P';
break;
default:
log_err("state %d\n", td->runstate);
}
__run_str[td->thread_number - 1] = c;
update_condensed_str(__run_str, run_str);
}
static int fio_rdmaio_init(struct thread_data *td)
{
struct rdmaio_data *rd = td->io_ops->data;
struct rdmaio_options *o = td->eo;
unsigned int max_bs;
int ret, i;
if (td_rw(td)) {
log_err("fio: rdma connections must be read OR write\n");
return 1;
}
if (td_random(td)) {
log_err("fio: RDMA network IO can't be random\n");
return 1;
}
if (compat_options(td))
return 1;
if (!o->port) {
log_err("fio: no port has been specified which is required "
"for the rdma engine\n");
return 1;
}
if (check_set_rlimits(td))
return 1;
rd->rdma_protocol = o->verb;
rd->cq_event_num = 0;
rd->cm_channel = rdma_create_event_channel();
if (!rd->cm_channel) {
log_err("fio: rdma_create_event_channel fail\n");
return 1;
}
ret = rdma_create_id(rd->cm_channel, &rd->cm_id, rd, RDMA_PS_TCP);
if (ret) {
log_err("fio: rdma_create_id fail\n");
return 1;
}
if ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE) ||
(rd->rdma_protocol == FIO_RDMA_MEM_READ)) {
rd->rmt_us =
malloc(FIO_RDMA_MAX_IO_DEPTH * sizeof(struct remote_u));
memset(rd->rmt_us, 0,
FIO_RDMA_MAX_IO_DEPTH * sizeof(struct remote_u));
rd->rmt_nr = 0;
}
rd->io_us_queued = malloc(td->o.iodepth * sizeof(struct io_u *));
memset(rd->io_us_queued, 0, td->o.iodepth * sizeof(struct io_u *));
rd->io_u_queued_nr = 0;
rd->io_us_flight = malloc(td->o.iodepth * sizeof(struct io_u *));
memset(rd->io_us_flight, 0, td->o.iodepth * sizeof(struct io_u *));
rd->io_u_flight_nr = 0;
rd->io_us_completed = malloc(td->o.iodepth * sizeof(struct io_u *));
memset(rd->io_us_completed, 0, td->o.iodepth * sizeof(struct io_u *));
rd->io_u_completed_nr = 0;
if (td_read(td)) { /* READ as the server */
rd->is_client = 0;
td->flags |= TD_F_NO_PROGRESS;
/* server rd->rdma_buf_len will be setup after got request */
ret = fio_rdmaio_setup_listen(td, o->port);
} else { /* WRITE as the client */
rd->is_client = 1;
ret = fio_rdmaio_setup_connect(td, td->o.filename, o->port);
}
max_bs = max(td->o.max_bs[DDIR_READ], td->o.max_bs[DDIR_WRITE]);
rd->send_buf.max_bs = htonl(max_bs);
/* register each io_u in the free list */
for (i = 0; i < td->io_u_freelist.nr; i++) {
struct io_u *io_u = td->io_u_freelist.io_us[i];
io_u->engine_data = malloc(sizeof(struct rdma_io_u_data));
memset(io_u->engine_data, 0, sizeof(struct rdma_io_u_data));
((struct rdma_io_u_data *)io_u->engine_data)->wr_id = i;
io_u->mr = ibv_reg_mr(rd->pd, io_u->buf, max_bs,
IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_READ |
IBV_ACCESS_REMOTE_WRITE);
if (io_u->mr == NULL) {
log_err("fio: ibv_reg_mr io_u failed\n");
return 1;
}
rd->send_buf.rmt_us[i].buf =
htonll((uint64_t) (unsigned long)io_u->buf);
rd->send_buf.rmt_us[i].rkey = htonl(io_u->mr->rkey);
rd->send_buf.rmt_us[i].size = htonl(max_bs);
#if 0
log_info("fio: Send rkey %x addr %" PRIx64 " len %d to client\n", io_u->mr->rkey, io_u->buf, max_bs); */
#endif
}
rd->send_buf.nr = htonl(i);
return ret;
}
/*
* log a successful write, so we can unwind the log for verify
*/
void log_io_piece(struct thread_data *td, struct io_u *io_u)
{
struct rb_node **p, *parent;
struct io_piece *ipo, *__ipo;
ipo = malloc(sizeof(struct io_piece));
init_ipo(ipo);
ipo->file = io_u->file;
ipo->offset = io_u->offset;
ipo->len = io_u->buflen;
ipo->numberio = io_u->numberio;
ipo->flags = IP_F_IN_FLIGHT;
io_u->ipo = ipo;
if (io_u_should_trim(td, io_u)) {
flist_add_tail(&ipo->trim_list, &td->trim_list);
td->trim_entries++;
}
/*
* We don't need to sort the entries, if:
*
* Sequential writes, or
* Random writes that lay out the file as it goes along
*
* For both these cases, just reading back data in the order we
* wrote it out is the fastest.
*
* One exception is if we don't have a random map AND we are doing
* verifies, in that case we need to check for duplicate blocks and
* drop the old one, which we rely on the rb insert/lookup for
* handling.
*/
if (((!td->o.verifysort) || !td_random(td) || !td->o.overwrite) &&
(file_randommap(td, ipo->file) || td->o.verify == VERIFY_NONE)) {
INIT_FLIST_HEAD(&ipo->list);
flist_add_tail(&ipo->list, &td->io_hist_list);
ipo->flags |= IP_F_ONLIST;
td->io_hist_len++;
return;
}
RB_CLEAR_NODE(&ipo->rb_node);
/*
* Sort the entry into the verification list
*/
restart:
p = &td->io_hist_tree.rb_node;
parent = NULL;
while (*p) {
parent = *p;
__ipo = rb_entry(parent, struct io_piece, rb_node);
if (ipo->file < __ipo->file)
p = &(*p)->rb_left;
else if (ipo->file > __ipo->file)
p = &(*p)->rb_right;
else if (ipo->offset < __ipo->offset)
p = &(*p)->rb_left;
else if (ipo->offset > __ipo->offset)
p = &(*p)->rb_right;
else {
dprint(FD_IO, "iolog: overlap %llu/%lu, %llu/%lu",
__ipo->offset, __ipo->len,
ipo->offset, ipo->len);
td->io_hist_len--;
rb_erase(parent, &td->io_hist_tree);
remove_trim_entry(td, __ipo);
free(__ipo);
goto restart;
}
}
rb_link_node(&ipo->rb_node, parent, p);
rb_insert_color(&ipo->rb_node, &td->io_hist_tree);
ipo->flags |= IP_F_ONRB;
td->io_hist_len++;
}
/*
* Lazy way of fixing up options that depend on each other. We could also
* define option callback handlers, but this is easier.
*/
static int fixup_options(struct thread_data *td)
{
struct thread_options *o = &td->o;
int ret = 0;
#ifndef FIO_HAVE_PSHARED_MUTEX
if (!o->use_thread) {
log_info("fio: this platform does not support process shared"
" mutexes, forcing use of threads. Use the 'thread'"
" option to get rid of this warning.\n");
o->use_thread = 1;
ret = warnings_fatal;
}
#endif
if (o->write_iolog_file && o->read_iolog_file) {
log_err("fio: read iolog overrides write_iolog\n");
free(o->write_iolog_file);
o->write_iolog_file = NULL;
ret = warnings_fatal;
}
/*
* only really works with 1 file
*/
if (o->zone_size && o->open_files > 1)
o->zone_size = 0;
/*
* If zone_range isn't specified, backward compatibility dictates it
* should be made equal to zone_size.
*/
if (o->zone_size && !o->zone_range)
o->zone_range = o->zone_size;
/*
* Reads can do overwrites, we always need to pre-create the file
*/
if (td_read(td) || td_rw(td))
o->overwrite = 1;
if (!o->min_bs[DDIR_READ])
o->min_bs[DDIR_READ] = o->bs[DDIR_READ];
if (!o->max_bs[DDIR_READ])
o->max_bs[DDIR_READ] = o->bs[DDIR_READ];
if (!o->min_bs[DDIR_WRITE])
o->min_bs[DDIR_WRITE] = o->bs[DDIR_WRITE];
if (!o->max_bs[DDIR_WRITE])
o->max_bs[DDIR_WRITE] = o->bs[DDIR_WRITE];
if (!o->min_bs[DDIR_TRIM])
o->min_bs[DDIR_TRIM] = o->bs[DDIR_TRIM];
if (!o->max_bs[DDIR_TRIM])
o->max_bs[DDIR_TRIM] = o->bs[DDIR_TRIM];
o->rw_min_bs = min(o->min_bs[DDIR_READ], o->min_bs[DDIR_WRITE]);
o->rw_min_bs = min(o->min_bs[DDIR_TRIM], o->rw_min_bs);
/*
* For random IO, allow blockalign offset other than min_bs.
*/
if (!o->ba[DDIR_READ] || !td_random(td))
o->ba[DDIR_READ] = o->min_bs[DDIR_READ];
if (!o->ba[DDIR_WRITE] || !td_random(td))
o->ba[DDIR_WRITE] = o->min_bs[DDIR_WRITE];
if (!o->ba[DDIR_TRIM] || !td_random(td))
o->ba[DDIR_TRIM] = o->min_bs[DDIR_TRIM];
if ((o->ba[DDIR_READ] != o->min_bs[DDIR_READ] ||
o->ba[DDIR_WRITE] != o->min_bs[DDIR_WRITE] ||
o->ba[DDIR_TRIM] != o->min_bs[DDIR_TRIM]) &&
!o->norandommap) {
log_err("fio: Any use of blockalign= turns off randommap\n");
o->norandommap = 1;
ret = warnings_fatal;
}
if (!o->file_size_high)
o->file_size_high = o->file_size_low;
if (o->start_delay_high)
o->start_delay = get_rand_start_delay(td);
if (o->norandommap && o->verify != VERIFY_NONE
&& !fixed_block_size(o)) {
log_err("fio: norandommap given for variable block sizes, "
"verify disabled\n");
o->verify = VERIFY_NONE;
ret = warnings_fatal;
}
if (o->bs_unaligned && (o->odirect || td->io_ops->flags & FIO_RAWIO))
log_err("fio: bs_unaligned may not work with raw io\n");
/*
* thinktime_spin must be less than thinktime
*/
if (o->thinktime_spin > o->thinktime)
o->thinktime_spin = o->thinktime;
/*
* The low water mark cannot be bigger than the iodepth
*/
if (o->iodepth_low > o->iodepth || !o->iodepth_low)
o->iodepth_low = o->iodepth;
/*
* If batch number isn't set, default to the same as iodepth
*/
if (o->iodepth_batch > o->iodepth || !o->iodepth_batch)
o->iodepth_batch = o->iodepth;
if (o->nr_files > td->files_index)
o->nr_files = td->files_index;
if (o->open_files > o->nr_files || !o->open_files)
o->open_files = o->nr_files;
if (((o->rate[DDIR_READ] + o->rate[DDIR_WRITE] + o->rate[DDIR_TRIM]) &&
(o->rate_iops[DDIR_READ] + o->rate_iops[DDIR_WRITE] + o->rate_iops[DDIR_TRIM])) ||
((o->ratemin[DDIR_READ] + o->ratemin[DDIR_WRITE] + o->ratemin[DDIR_TRIM]) &&
(o->rate_iops_min[DDIR_READ] + o->rate_iops_min[DDIR_WRITE] + o->rate_iops_min[DDIR_TRIM]))) {
log_err("fio: rate and rate_iops are mutually exclusive\n");
ret = 1;
}
if ((o->rate[DDIR_READ] < o->ratemin[DDIR_READ]) ||
(o->rate[DDIR_WRITE] < o->ratemin[DDIR_WRITE]) ||
(o->rate[DDIR_TRIM] < o->ratemin[DDIR_TRIM]) ||
(o->rate_iops[DDIR_READ] < o->rate_iops_min[DDIR_READ]) ||
(o->rate_iops[DDIR_WRITE] < o->rate_iops_min[DDIR_WRITE]) ||
(o->rate_iops[DDIR_TRIM] < o->rate_iops_min[DDIR_TRIM])) {
log_err("fio: minimum rate exceeds rate\n");
ret = 1;
}
if (!o->timeout && o->time_based) {
log_err("fio: time_based requires a runtime/timeout setting\n");
o->time_based = 0;
ret = warnings_fatal;
}
if (o->fill_device && !o->size)
o->size = -1ULL;
if (o->verify != VERIFY_NONE) {
if (td_write(td) && o->do_verify && o->numjobs > 1) {
log_info("Multiple writers may overwrite blocks that "
"belong to other jobs. This can cause "
"verification failures.\n");
ret = warnings_fatal;
}
o->refill_buffers = 1;
if (o->max_bs[DDIR_WRITE] != o->min_bs[DDIR_WRITE] &&
!o->verify_interval)
o->verify_interval = o->min_bs[DDIR_WRITE];
/*
* Verify interval must be smaller or equal to the
* write size.
*/
if (o->verify_interval > o->min_bs[DDIR_WRITE])
o->verify_interval = o->min_bs[DDIR_WRITE];
else if (td_read(td) && o->verify_interval > o->min_bs[DDIR_READ])
o->verify_interval = o->min_bs[DDIR_READ];
}
if (o->pre_read) {
o->invalidate_cache = 0;
if (td->io_ops->flags & FIO_PIPEIO) {
log_info("fio: cannot pre-read files with an IO engine"
" that isn't seekable. Pre-read disabled.\n");
ret = warnings_fatal;
}
}
if (!o->unit_base) {
if (td->io_ops->flags & FIO_BIT_BASED)
o->unit_base = 1;
else
o->unit_base = 8;
}
#ifndef CONFIG_FDATASYNC
if (o->fdatasync_blocks) {
log_info("fio: this platform does not support fdatasync()"
" falling back to using fsync(). Use the 'fsync'"
" option instead of 'fdatasync' to get rid of"
" this warning\n");
o->fsync_blocks = o->fdatasync_blocks;
o->fdatasync_blocks = 0;
ret = warnings_fatal;
}
#endif
#ifdef WIN32
/*
* Windows doesn't support O_DIRECT or O_SYNC with the _open interface,
* so fail if we're passed those flags
*/
if ((td->io_ops->flags & FIO_SYNCIO) && (td->o.odirect || td->o.sync_io)) {
log_err("fio: Windows does not support direct or non-buffered io with"
" the synchronous ioengines. Use the 'windowsaio' ioengine"
" with 'direct=1' and 'iodepth=1' instead.\n");
ret = 1;
}
#endif
/*
* For fully compressible data, just zero them at init time.
* It's faster than repeatedly filling it.
*/
if (td->o.compress_percentage == 100) {
td->o.zero_buffers = 1;
td->o.compress_percentage = 0;
}
/*
* Using a non-uniform random distribution excludes usage of
* a random map
*/
if (td->o.random_distribution != FIO_RAND_DIST_RANDOM)
td->o.norandommap = 1;
/*
* If size is set but less than the min block size, complain
*/
if (o->size && o->size < td_min_bs(td)) {
log_err("fio: size too small, must be larger than the IO size: %llu\n", (unsigned long long) o->size);
ret = 1;
}
/*
* O_ATOMIC implies O_DIRECT
*/
if (td->o.oatomic)
td->o.odirect = 1;
/*
* If randseed is set, that overrides randrepeat
*/
if (td->o.rand_seed)
td->o.rand_repeatable = 0;
if ((td->io_ops->flags & FIO_NOEXTEND) && td->o.file_append) {
log_err("fio: can't append/extent with IO engine %s\n", td->io_ops->name);
ret = 1;
}
return ret;
}