static void
nfsio_write_page(struct ploop_io * io, struct ploop_request * preq,
struct page * page, sector_t sec, int fua)
{
struct inode *inode = io->files.inode;
struct nfs_write_data *nreq;
int err;
nfsio_prepare_io_request(preq);
ploop_prepare_tracker(preq, sec);
nreq = wbio_init((loff_t)sec << 9, page, 0, PAGE_SIZE, preq, inode);
if (nreq == NULL) {
PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
goto out;
}
atomic_inc(&preq->io_count);
err = wbio_submit(io, nreq, &nfsio_write_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
nfsio_complete_io_request(preq);
}
out:
nfsio_complete_io_request(preq);
}
/*
* WRITE case:
*
* sbl is the list of bio; the first bio in the list and iblk specify
* destination file offset; the content of bios in sbl is scattered source
* buffer.
*
* The goal is to write source buffer to the file with given offset. We're
* doing it by stuffing as many bvecs from source to kreqs as possible and
* submitting kreqs to in-kernel aio.
*
* READ case:
*
* The same as WRITE, but here the file plays the role of source and the
* content of bios in sbl plays the role of destination.
*/
static void kaio_sbl_submit(struct file *file, struct ploop_request *preq,
unsigned long rw, struct bio_list *sbl,
iblock_t iblk, size_t size)
{
struct bio *bio = sbl->head;
int idx = 0;
loff_t off = bio->bi_sector;
off = ((loff_t)iblk << preq->plo->cluster_log) |
(off & ((1<<preq->plo->cluster_log) - 1));
if (rw & REQ_WRITE)
ploop_prepare_tracker(preq, off);
off <<= 9;
/* since now 'off' always points to a position in the file to X-mit */
WARN_ONCE(!(file->f_flags & O_DIRECT), "File opened w/o O_DIRECT");
ploop_prepare_io_request(preq);
size <<= 9;
while (size > 0) {
struct kaio_req *kreq;
int nr_segs;
size_t copy;
int err;
kreq = kaio_kreq_alloc(preq, &nr_segs);
if (!kreq) {
PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
break;
}
copy = kaio_kreq_pack(kreq, &nr_segs, &bio, &idx, size);
atomic_inc(&preq->io_count);
err = kaio_kernel_submit(file, kreq, nr_segs, copy, off, rw);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
ploop_complete_io_request(preq);
kfree(kreq);
break;
}
off += copy;
size -= copy;
}
kaio_complete_io_request(preq);
}
static void
kaio_submit(struct ploop_io *io, struct ploop_request * preq,
unsigned long rw,
struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
struct bio * b;
if (rw & BIO_FLUSH) {
spin_lock_irq(&io->plo->lock);
kaio_queue_fsync_req(preq);
io->plo->st.bio_syncwait++;
spin_unlock_irq(&io->plo->lock);
return;
}
ploop_prepare_io_request(preq);
if (iblk == PLOOP_ZERO_INDEX)
iblk = 0;
if (rw & (1<<BIO_RW)) {
loff_t off = sbl->head->bi_sector;
off = ((loff_t)iblk << preq->plo->cluster_log) |
(off & ((1<<preq->plo->cluster_log) - 1));
ploop_prepare_tracker(preq, off);
}
for (b = sbl->head; b != NULL; b = b->bi_next) {
int err;
atomic_inc(&preq->io_count);
err = kaio_kernel_submit(io->files.file, b, preq, iblk, rw);
if (err) {
ploop_set_error(preq, err);
ploop_complete_io_request(preq);
break;
}
}
kaio_complete_io_request(preq);
}
static int
rbio_submit(struct ploop_io * io, struct nfs_read_data * nreq,
const struct rpc_call_ops * cb)
{
struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
struct inode *inode = io->files.inode;
struct rpc_task *task;
struct rpc_message msg = {
.rpc_cred = ctx->cred,
};
struct rpc_task_setup task_setup_data = {
.rpc_client = NFS_CLIENT(inode),
.rpc_message = &msg,
.callback_ops = cb,
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
.workqueue = nfsio_workqueue,
#endif
.flags = RPC_TASK_ASYNC,
};
nreq->res.count = nreq->args.count;
nreq->header->cred = msg.rpc_cred;
nreq->args.context = ctx;
task_setup_data.task = &nreq->task;
task_setup_data.callback_data = nreq;
msg.rpc_argp = &nreq->args;
msg.rpc_resp = &nreq->res;
NFS_PROTO(inode)->read_setup(nreq, &msg);
task = rpc_run_task(&task_setup_data);
if (unlikely(IS_ERR(task)))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
#else
static int
rbio_submit(struct ploop_io * io, struct nfs_read_data * nreq,
const struct rpc_call_ops * cb)
{
struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
struct inode *inode = io->files.inode;
nreq->res.count = nreq->args.count;
nreq->cred = ctx->cred;
nreq->args.context = ctx;
rpc_init_task(&nreq->task, NFS_CLIENT(inode), RPC_TASK_ASYNC, cb, nreq);
NFS_PROTO(inode)->read_setup(nreq);
nreq->task.tk_cookie = (unsigned long) inode;
lock_kernel();
rpc_execute(&nreq->task);
unlock_kernel();
return 0;
}
#endif
static void
nfsio_submit_read(struct ploop_io *io, struct ploop_request * preq,
struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
struct inode *inode = io->files.inode;
size_t rsize = NFS_SERVER(inode)->rsize;
struct nfs_read_data *nreq = NULL;
loff_t pos;
unsigned int prev_end;
struct bio * b;
ploop_prepare_io_request(preq);
pos = sbl->head->bi_sector;
pos = ((loff_t)iblk << preq->plo->cluster_log) | (pos & ((1<<preq->plo->cluster_log) - 1));
pos <<= 9;
prev_end = PAGE_SIZE;
for (b = sbl->head; b != NULL; b = b->bi_next) {
int bv_idx;
for (bv_idx = 0; bv_idx < b->bi_vcnt; bv_idx++) {
struct bio_vec * bv = &b->bi_io_vec[bv_idx];
if (nreq && nreq->args.count + bv->bv_len <= rsize) {
if (nreq->pages.pagevec[nreq->pages.npages-1] == bv->bv_page &&
prev_end == bv->bv_offset) {
nreq->args.count += bv->bv_len;
pos += bv->bv_len;
prev_end += bv->bv_len;
continue;
}
if (nreq->pages.npages < MAX_NBIO_PAGES &&
bv->bv_offset == 0 && prev_end == PAGE_SIZE) {
nreq->args.count += bv->bv_len;
nreq->pages.pagevec[nreq->pages.npages] = bv->bv_page;
nreq->pages.npages++;
pos += bv->bv_len;
prev_end = bv->bv_offset + bv->bv_len;
continue;
}
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = rbio_submit(io, nreq, &nfsio_read_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
ploop_complete_io_request(preq);
goto out;
}
}
nreq = rbio_init(pos, bv->bv_page, bv->bv_offset,
bv->bv_len, preq, inode);
if (nreq == NULL) {
PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
goto out;
}
pos += bv->bv_len;
prev_end = bv->bv_offset + bv->bv_len;
}
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = rbio_submit(io, nreq, &nfsio_read_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
ploop_complete_io_request(preq);
goto out;
}
}
out:
ploop_complete_io_request(preq);
}
static void nfsio_write_result(struct rpc_task *task, void *calldata)
{
struct nfs_write_data *data = calldata;
struct nfs_writeargs *argp = &data->args;
struct nfs_writeres *resp = &data->res;
int status;
status = NFS_PROTO(data->header->inode)->write_done(task, data);
if (status != 0)
return;
if (task->tk_status >= 0 && resp->count < argp->count)
task->tk_status = -EIO;
}
static void nfsio_write_release(void *calldata)
{
struct nfs_write_data *nreq = calldata;
struct ploop_request *preq = (struct ploop_request *) nreq->header->req;
int status = nreq->task.tk_status;
if (unlikely(status < 0))
PLOOP_REQ_SET_ERROR(preq, status);
if (!preq->error &&
nreq->res.verf->committed != NFS_FILE_SYNC) {
if (!test_and_set_bit(PLOOP_REQ_UNSTABLE, &preq->state))
memcpy(&preq->verf, &nreq->res.verf->verifier, 8);
}
nfsio_complete_io_request(preq);
nfsio_wbio_release(calldata);
}
static const struct rpc_call_ops nfsio_write_ops = {
.rpc_call_done = nfsio_write_result,
.rpc_release = nfsio_write_release,
};
static struct nfs_write_data *
wbio_init(loff_t pos, struct page * page, unsigned int off, unsigned int len,
void * priv, struct inode * inode)
{
struct nfs_write_data * nreq;
nreq = nfsio_wbio_alloc(MAX_NBIO_PAGES);
if (unlikely(nreq == NULL))
return NULL;
nreq->args.offset = pos;
nreq->args.pgbase = off;
nreq->args.count = len;
nreq->pages.pagevec[0] = page;
nreq->pages.npages = 1;
nreq->header->req = priv;
nreq->header->inode = inode;
nreq->args.fh = NFS_FH(inode);
nreq->args.pages = nreq->pages.pagevec;
nreq->args.stable = NFS_UNSTABLE;
nreq->res.fattr = &nreq->fattr;
nreq->res.verf = &nreq->verf;
return nreq;
}
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)
static int wbio_submit(struct ploop_io * io, struct nfs_write_data *nreq,
const struct rpc_call_ops * cb)
{
struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
struct inode *inode = io->files.inode;
struct rpc_task *task;
struct rpc_message msg = {
.rpc_cred = ctx->cred,
};
struct rpc_task_setup task_setup_data = {
.rpc_client = NFS_CLIENT(inode),
.rpc_message = &msg,
.callback_ops = cb,
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
.workqueue = nfsio_workqueue,
#endif
.flags = RPC_TASK_ASYNC,
};
if (verify_bounce(nreq))
return -ENOMEM;
nreq->res.count = nreq->args.count;
nreq->args.context = ctx;
nreq->header->cred = msg.rpc_cred;
task_setup_data.task = &nreq->task;
task_setup_data.callback_data = nreq;
msg.rpc_argp = &nreq->args;
msg.rpc_resp = &nreq->res;
NFS_PROTO(inode)->write_setup(nreq, &msg);
task = rpc_run_task(&task_setup_data);
if (unlikely(IS_ERR(task)))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
#else
static int wbio_submit(struct ploop_io * io, struct nfs_write_data *nreq,
const struct rpc_call_ops * cb)
{
struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
struct inode *inode = io->files.inode;
if (verify_bounce(nreq))
return -ENOMEM;
nreq->res.count = nreq->args.count;
nreq->args.context = ctx;
nreq->cred = ctx->cred;
rpc_init_task(&nreq->task, NFS_CLIENT(inode), RPC_TASK_ASYNC, cb, nreq);
NFS_PROTO(inode)->write_setup(nreq, NFS_UNSTABLE);
nreq->task.tk_priority = RPC_PRIORITY_NORMAL;
nreq->task.tk_cookie = (unsigned long) inode;
lock_kernel();
rpc_execute(&nreq->task);
unlock_kernel();
return 0;
}
#endif
static void
nfsio_submit_write(struct ploop_io *io, struct ploop_request * preq,
struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
struct inode *inode = io->files.inode;
size_t wsize = NFS_SERVER(inode)->wsize;
struct nfs_write_data *nreq = NULL;
loff_t pos;
struct bio * b;
unsigned int prev_end;
nfsio_prepare_io_request(preq);
pos = sbl->head->bi_sector;
pos = ((loff_t)iblk << preq->plo->cluster_log) | (pos & ((1<<preq->plo->cluster_log) - 1));
ploop_prepare_tracker(preq, pos);
pos <<= 9;
prev_end = PAGE_SIZE;
for (b = sbl->head; b != NULL; b = b->bi_next) {
int bv_idx;
for (bv_idx = 0; bv_idx < b->bi_vcnt; bv_idx++) {
struct bio_vec * bv = &b->bi_io_vec[bv_idx];
if (nreq && nreq->args.count + bv->bv_len <= wsize) {
if (nreq->pages.pagevec[nreq->pages.npages-1] == bv->bv_page &&
prev_end == bv->bv_offset) {
nreq->args.count += bv->bv_len;
pos += bv->bv_len;
prev_end += bv->bv_len;
continue;
}
if (nreq->pages.npages < MAX_NBIO_PAGES &&
bv->bv_offset == 0 && prev_end == PAGE_SIZE) {
nreq->args.count += bv->bv_len;
nreq->pages.pagevec[nreq->pages.npages] = bv->bv_page;
nreq->pages.npages++;
pos += bv->bv_len;
prev_end = bv->bv_offset + bv->bv_len;
continue;
}
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = wbio_submit(io, nreq, &nfsio_write_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
nfsio_complete_io_request(preq);
goto out;
}
}
nreq = wbio_init(pos, bv->bv_page, bv->bv_offset,
bv->bv_len, preq, inode);
if (nreq == NULL) {
PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
goto out;
}
prev_end = bv->bv_offset + bv->bv_len;
pos += bv->bv_len;
}
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = wbio_submit(io, nreq, &nfsio_write_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
nfsio_complete_io_request(preq);
}
}
out:
nfsio_complete_io_request(preq);
}
static void
nfsio_submit(struct ploop_io *io, struct ploop_request * preq,
unsigned long rw,
struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
if (iblk == PLOOP_ZERO_INDEX)
iblk = 0;
if (rw & (1<<BIO_RW))
nfsio_submit_write(io, preq, sbl, iblk, size);
else
nfsio_submit_read(io, preq, sbl, iblk, size);
}
struct bio_list_walk
{
struct bio * cur;
int idx;
int bv_off;
};
static void
nfsio_submit_write_pad(struct ploop_io *io, struct ploop_request * preq,
struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
struct inode *inode = io->files.inode;
size_t wsize = NFS_SERVER(inode)->wsize;
struct nfs_write_data *nreq = NULL;
struct bio_list_walk bw;
unsigned prev_end;
loff_t pos, end_pos, start, end;
/* pos..end_pos is the range which we are going to write */
pos = (loff_t)iblk << (preq->plo->cluster_log + 9);
end_pos = pos + (1 << (preq->plo->cluster_log + 9));
/* start..end is data that we have. The rest must be zero padded. */
start = pos + ((sbl->head->bi_sector & ((1<<preq->plo->cluster_log) - 1)) << 9);
end = start + (size << 9);
nfsio_prepare_io_request(preq);
ploop_prepare_tracker(preq, start >> 9);
prev_end = PAGE_SIZE;
#if 1
/* GCC, shut up! */
bw.cur = sbl->head;
bw.idx = 0;
bw.bv_off = 0;
BUG_ON(bw.cur->bi_io_vec[0].bv_len & 511);
#endif
while (pos < end_pos) {
struct page * page;
unsigned int poff, plen;
if (pos < start) {
page = ZERO_PAGE(0);
poff = 0;
plen = start - pos;
if (plen > PAGE_SIZE)
plen = PAGE_SIZE;
} else if (pos >= end) {
page = ZERO_PAGE(0);
poff = 0;
plen = end_pos - pos;
if (plen > PAGE_SIZE)
plen = PAGE_SIZE;
} else {
/* pos >= start && pos < end */
struct bio_vec * bv;
if (pos == start) {
bw.cur = sbl->head;
bw.idx = 0;
bw.bv_off = 0;
BUG_ON(bw.cur->bi_io_vec[0].bv_len & 511);
}
bv = bw.cur->bi_io_vec + bw.idx;
if (bw.bv_off >= bv->bv_len) {
bw.idx++;
bv++;
bw.bv_off = 0;
if (bw.idx >= bw.cur->bi_vcnt) {
bw.cur = bw.cur->bi_next;
bw.idx = 0;
bw.bv_off = 0;
bv = bw.cur->bi_io_vec;
}
BUG_ON(bv->bv_len & 511);
}
page = bv->bv_page;
poff = bv->bv_offset + bw.bv_off;
plen = bv->bv_len - bw.bv_off;
}
if (nreq && nreq->args.count + plen <= wsize) {
if (nreq->pages.pagevec[nreq->pages.npages-1] == page &&
prev_end == poff) {
nreq->args.count += plen;
pos += plen;
bw.bv_off += plen;
prev_end += plen;
continue;
}
if (nreq->pages.npages < MAX_NBIO_PAGES &&
poff == 0 && prev_end == PAGE_SIZE) {
nreq->args.count += plen;
nreq->pages.pagevec[nreq->pages.npages] = page;
nreq->pages.npages++;
pos += plen;
bw.bv_off += plen;
prev_end = poff + plen;
continue;
}
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = wbio_submit(io, nreq, &nfsio_write_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
nfsio_complete_io_request(preq);
goto out;
}
}
nreq = wbio_init(pos, page, poff, plen, preq, inode);
if (nreq == NULL) {
PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
goto out;
}
prev_end = poff + plen;
pos += plen;
bw.bv_off += plen;
}
if (nreq) {
int err;
atomic_inc(&preq->io_count);
err = wbio_submit(io, nreq, &nfsio_write_ops);
if (err) {
PLOOP_REQ_SET_ERROR(preq, err);
nfsio_complete_io_request(preq);
}
}
out:
nfsio_complete_io_request(preq);
}
static void
nfsio_submit_alloc(struct ploop_io *io, struct ploop_request * preq,
struct bio_list * sbl, unsigned int size)
{
iblock_t iblk = io->alloc_head++;
if (!(io->files.file->f_mode & FMODE_WRITE)) {
PLOOP_FAIL_REQUEST(preq, -EBADF);
return;
}
preq->iblock = iblk;
preq->eng_state = PLOOP_E_DATA_WBI;
nfsio_submit_write_pad(io, preq, sbl, iblk, size);
}
static void nfsio_destroy(struct ploop_io * io)
{
if (io->fsync_thread) {
kthread_stop(io->fsync_thread);
io->fsync_thread = NULL;
}
if (io->files.file) {
struct file * file = io->files.file;
mutex_lock(&io->plo->sysfs_mutex);
io->files.file = NULL;
if (io->files.mapping)
(void)invalidate_inode_pages2(io->files.mapping);
mutex_unlock(&io->plo->sysfs_mutex);
fput(file);
}
}
static int nfsio_sync(struct ploop_io * io)
{
return 0;
}
static int nfsio_stop(struct ploop_io * io)
{
return 0;
}
static int
nfsio_init(struct ploop_io * io)
{
INIT_LIST_HEAD(&io->fsync_queue);
init_waitqueue_head(&io->fsync_waitq);
return 0;
}