static int aio_submit(struct aio_output *output, struct aio_mref_aspect *mref_a, bool use_fdsync)
{
struct mref_object *mref = mref_a->object;
mm_segment_t oldfs;
int res;
struct iocb iocb = {
.aio_data = (__u64)mref_a,
.aio_lio_opcode = use_fdsync ? IOCB_CMD_FDSYNC : (mref->ref_rw != 0 ? IOCB_CMD_PWRITE : IOCB_CMD_PREAD),
.aio_fildes = output->fd,
.aio_buf = (unsigned long)mref->ref_data,
.aio_nbytes = mref->ref_len,
.aio_offset = mref->ref_pos,
// .aio_reqprio = something(mref->ref_prio) field exists, but not yet implemented in kernelspace :(
};
struct iocb *iocbp = &iocb;
unsigned long long latency;
mars_trace(mref, "aio_submit");
if (unlikely(output->fd < 0)) {
MARS_ERR("bad fd = %d\n", output->fd);
res = -EBADF;
goto done;
}
oldfs = get_fs();
set_fs(get_ds());
latency = TIME_STATS(&timings[mref->ref_rw & 1], res = sys_io_submit(output->ctxp, 1, &iocbp));
set_fs(oldfs);
threshold_check(&aio_submit_threshold, latency);
atomic_inc(&output->total_submit_count);
if (likely(res >= 0)) {
atomic_inc(&output->submit_count);
} else if (likely(res == -EAGAIN)) {
atomic_inc(&output->total_again_count);
} else {
MARS_ERR("error = %d\n", res);
}
done:
return res;
}
static int aio_submit_dummy(struct aio_output *output)
{
mm_segment_t oldfs;
int res;
int dummy;
struct iocb iocb = {
.aio_buf = (__u64)&dummy,
};
struct iocb *iocbp = &iocb;
oldfs = get_fs();
set_fs(get_ds());
res = sys_io_submit(output->ctxp, 1, &iocbp);
set_fs(oldfs);
if (likely(res >= 0)) {
atomic_inc(&output->submit_count);
}
return res;
}
static
int aio_start_thread(
struct aio_output *output,
struct aio_threadinfo *tinfo,
int(*fn)(void*),
char class)
{
int j;
for (j = 0; j < MARS_PRIO_NR; j++) {
INIT_LIST_HEAD(&tinfo->mref_list[j]);
}
tinfo->output = output;
spin_lock_init(&tinfo->lock);
init_waitqueue_head(&tinfo->event);
init_waitqueue_head(&tinfo->terminate_event);
tinfo->terminated = false;
tinfo->thread = brick_thread_create(fn, tinfo, "mars_aio_%c%d", class, output->index);
if (unlikely(!tinfo->thread)) {
MARS_ERR("cannot create thread\n");
return -ENOENT;
}
return 0;
}
static
void aio_stop_thread(struct aio_output *output, int i, bool do_submit_dummy)
{
struct aio_threadinfo *tinfo = &output->tinfo[i];
if (tinfo->thread) {
MARS_DBG("stopping thread %d ...\n", i);
brick_thread_stop_nowait(tinfo->thread);
// workaround for waking up the receiver thread. TODO: check whether signal handlong could do better.
if (do_submit_dummy) {
MARS_DBG("submitting dummy for wakeup %d...\n", i);
use_fake_mm();
aio_submit_dummy(output);
if (likely(current->mm)) {
unuse_fake_mm();
}
}
// wait for termination
MARS_DBG("waiting for thread %d ...\n", i);
wait_event_interruptible_timeout(
tinfo->terminate_event,
tinfo->terminated,
(60 - i * 2) * HZ);
if (likely(tinfo->terminated)) {
brick_thread_stop(tinfo->thread);
} else {
MARS_ERR("thread %d did not terminate - leaving a zombie\n", i);
}
}
}
static
int aio_sync(struct file *file)
{
int err;
switch (aio_sync_mode) {
case 1:
#if defined(S_BIAS) || (defined(RHEL_MAJOR) && (RHEL_MAJOR < 7))
err = vfs_fsync_range(file, file->f_path.dentry, 0, LLONG_MAX, 1);
#else
err = vfs_fsync_range(file, 0, LLONG_MAX, 1);
#endif
break;
case 2:
#if defined(S_BIAS) || (defined(RHEL_MAJOR) && (RHEL_MAJOR < 7))
err = vfs_fsync_range(file, file->f_path.dentry, 0, LLONG_MAX, 0);
#else
err = vfs_fsync_range(file, 0, LLONG_MAX, 0);
#endif
break;
default:
err = filemap_write_and_wait_range(file->f_mapping, 0, LLONG_MAX);
}
return err;
}
/*
* stress_aio_linux
* stress asynchronous I/O using the linux specific aio ABI
*/
int stress_aio_linux(
uint64_t *const counter,
const uint32_t instance,
const uint64_t max_ops,
const char *name)
{
int fd, rc = EXIT_FAILURE;
char filename[PATH_MAX];
const pid_t pid = getpid();
aio_context_t ctx = 0;
if (!set_aio_linux_requests) {
if (opt_flags & OPT_FLAGS_MAXIMIZE)
opt_aio_linux_requests = MAX_AIO_REQUESTS;
if (opt_flags & OPT_FLAGS_MINIMIZE)
opt_aio_linux_requests = MIN_AIO_REQUESTS;
}
if (sys_io_setup(opt_aio_linux_requests, &ctx) < 0) {
pr_failed_err(name, "io_setup");
return EXIT_FAILURE;
}
if (stress_temp_dir_mk(name, pid, instance) < 0) {
return EXIT_FAILURE;
}
(void)stress_temp_filename(filename, sizeof(filename),
name, pid, instance, mwc32());
(void)umask(0077);
if ((fd = open(filename, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR)) < 0) {
pr_failed_err(name, "open");
goto finish;
}
(void)unlink(filename);
do {
struct iocb cb[opt_aio_linux_requests];
struct iocb *cbs[opt_aio_linux_requests];
struct io_event events[opt_aio_linux_requests];
uint8_t buffers[opt_aio_linux_requests][BUFFER_SZ];
int ret, i;
long n;
for (i = 0; i < opt_aio_linux_requests; i++)
aio_linux_fill_buffer(i, buffers[i], BUFFER_SZ);
memset(cb, 0, sizeof(cb));
for (i = 0; i < opt_aio_linux_requests; i++) {
cb[i].aio_fildes = fd;
cb[i].aio_lio_opcode = IOCB_CMD_PWRITE;
cb[i].aio_buf = (long)buffers[i];
cb[i].aio_offset = mwc16() * BUFFER_SZ;
cb[i].aio_nbytes = BUFFER_SZ;
cbs[i] = &cb[i];
}
ret = sys_io_submit(ctx, opt_aio_linux_requests, cbs);
if (ret < 0) {
if (errno == EAGAIN)
continue;
pr_failed_err(name, "io_submit");
break;
}
n = opt_aio_linux_requests;
do {
struct timespec timeout, *timeout_ptr;
if (clock_gettime(CLOCK_REALTIME, &timeout) < 0) {
timeout_ptr = NULL;
} else {
timeout.tv_nsec += 1000000;
if (timeout.tv_nsec > 1000000000) {
timeout.tv_nsec -= 1000000000;
timeout.tv_sec++;
}
timeout_ptr = &timeout;
}
ret = sys_io_getevents(ctx, 1, n, events, timeout_ptr);
if (ret < 0) {
if ((errno == EINTR) && (opt_do_run))
continue;
pr_failed_err(name, "io_getevents");
break;
} else {
n -= ret;
}
} while ((n > 0) && opt_do_run);
(*counter)++;
} while (opt_do_run && (!max_ops || *counter < max_ops));
rc = EXIT_SUCCESS;
(void)close(fd);
finish:
(void)sys_io_destroy(ctx);
(void)stress_temp_dir_rm(name, pid, instance);
return rc;
}
