예제 #1
0
static void
nfsio_write_page(struct ploop_io * io, struct ploop_request * preq,
		 struct page * page, sector_t sec, int fua)
{
	struct inode *inode = io->files.inode;
	struct nfs_write_data *nreq;
	int err;

	nfsio_prepare_io_request(preq);
	ploop_prepare_tracker(preq, sec);

	nreq = wbio_init((loff_t)sec << 9, page, 0, PAGE_SIZE, preq, inode);

	if (nreq == NULL) {
		PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
		goto out;
	}

	atomic_inc(&preq->io_count);
	err = wbio_submit(io, nreq, &nfsio_write_ops);
	if (err) {
		PLOOP_REQ_SET_ERROR(preq, err);
		nfsio_complete_io_request(preq);
	}

out:
	nfsio_complete_io_request(preq);
}
예제 #2
0
/*
 * WRITE case:
 *
 * sbl is the list of bio; the first bio in the list and iblk specify
 * destination file offset; the content of bios in sbl is scattered source
 * buffer.
 *
 * The goal is to write source buffer to the file with given offset. We're
 * doing it by stuffing as many bvecs from source to kreqs as possible and
 * submitting kreqs to in-kernel aio.
 *
 * READ case:
 *
 * The same as WRITE, but here the file plays the role of source and the
 * content of bios in sbl plays the role of destination.
 */
static void kaio_sbl_submit(struct file *file, struct ploop_request *preq,
                            unsigned long rw, struct bio_list *sbl,
                            iblock_t iblk, size_t size)
{
    struct bio *bio = sbl->head;
    int idx = 0;

    loff_t off = bio->bi_sector;
    off = ((loff_t)iblk << preq->plo->cluster_log) |
          (off & ((1<<preq->plo->cluster_log) - 1));

    if (rw & REQ_WRITE)
        ploop_prepare_tracker(preq, off);

    off <<= 9;
    /* since now 'off' always points to a position in the file to X-mit */

    WARN_ONCE(!(file->f_flags & O_DIRECT), "File opened w/o O_DIRECT");

    ploop_prepare_io_request(preq);

    size <<= 9;
    while (size > 0) {
        struct kaio_req *kreq;
        int nr_segs;
        size_t copy;
        int err;

        kreq = kaio_kreq_alloc(preq, &nr_segs);
        if (!kreq) {
            PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
            break;
        }

        copy = kaio_kreq_pack(kreq, &nr_segs, &bio, &idx, size);

        atomic_inc(&preq->io_count);
        err = kaio_kernel_submit(file, kreq, nr_segs, copy, off, rw);
        if (err) {
            PLOOP_REQ_SET_ERROR(preq, err);
            ploop_complete_io_request(preq);
            kfree(kreq);
            break;
        }

        off += copy;
        size -= copy;
    }

    kaio_complete_io_request(preq);
}
예제 #3
0
static void
kaio_submit(struct ploop_io *io, struct ploop_request * preq,
	     unsigned long rw,
	     struct bio_list *sbl, iblock_t iblk, unsigned int size)
{

	struct bio * b;

	if (rw & BIO_FLUSH) {
		spin_lock_irq(&io->plo->lock);
		kaio_queue_fsync_req(preq);
		io->plo->st.bio_syncwait++;
		spin_unlock_irq(&io->plo->lock);
		return;
	}

	ploop_prepare_io_request(preq);

	if (iblk == PLOOP_ZERO_INDEX)
		iblk = 0;

	if (rw & (1<<BIO_RW)) {
		loff_t off = sbl->head->bi_sector;
		off = ((loff_t)iblk << preq->plo->cluster_log) |
			(off & ((1<<preq->plo->cluster_log) - 1));
		ploop_prepare_tracker(preq, off);
	}

	for (b = sbl->head; b != NULL; b = b->bi_next) {
		int err;

		atomic_inc(&preq->io_count);
		err = kaio_kernel_submit(io->files.file, b, preq, iblk, rw);
		if (err) {
			ploop_set_error(preq, err);
			ploop_complete_io_request(preq);
			break;
		}
	}

	kaio_complete_io_request(preq);
}
예제 #4
0
static int
rbio_submit(struct ploop_io * io, struct nfs_read_data * nreq,
	    const struct rpc_call_ops * cb)
{
	struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
	struct inode *inode = io->files.inode;
	struct rpc_task *task;

	struct rpc_message msg = {
		.rpc_cred = ctx->cred,
	};

	struct rpc_task_setup task_setup_data = {
		.rpc_client = NFS_CLIENT(inode),
		.rpc_message = &msg,
		.callback_ops = cb,
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
		.workqueue = nfsio_workqueue,
#endif
		.flags = RPC_TASK_ASYNC,
	};

	nreq->res.count = nreq->args.count;
	nreq->header->cred = msg.rpc_cred;
	nreq->args.context = ctx;

	task_setup_data.task = &nreq->task;
	task_setup_data.callback_data = nreq;
	msg.rpc_argp = &nreq->args;
	msg.rpc_resp = &nreq->res;
	NFS_PROTO(inode)->read_setup(nreq, &msg);

	task = rpc_run_task(&task_setup_data);
	if (unlikely(IS_ERR(task)))
		return PTR_ERR(task);

	rpc_put_task(task);
	return 0;
}

#else

static int
rbio_submit(struct ploop_io * io, struct nfs_read_data * nreq,
	    const struct rpc_call_ops * cb)
{
	struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
	struct inode *inode = io->files.inode;

	nreq->res.count = nreq->args.count;
	nreq->cred = ctx->cred;
	nreq->args.context = ctx;

	rpc_init_task(&nreq->task, NFS_CLIENT(inode), RPC_TASK_ASYNC, cb, nreq);
	NFS_PROTO(inode)->read_setup(nreq);

	nreq->task.tk_cookie = (unsigned long) inode;

	lock_kernel();
	rpc_execute(&nreq->task);
	unlock_kernel();
	return 0;
}
#endif

static void
nfsio_submit_read(struct ploop_io *io, struct ploop_request * preq,
		  struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
	struct inode *inode = io->files.inode;
	size_t rsize = NFS_SERVER(inode)->rsize;
	struct nfs_read_data *nreq = NULL;
	loff_t pos;
	unsigned int prev_end;
	struct bio * b;

	ploop_prepare_io_request(preq);

	pos = sbl->head->bi_sector;
	pos = ((loff_t)iblk << preq->plo->cluster_log) | (pos & ((1<<preq->plo->cluster_log) - 1));
	pos <<= 9;

	prev_end = PAGE_SIZE;

	for (b = sbl->head; b != NULL; b = b->bi_next) {
		int bv_idx;

		for (bv_idx = 0; bv_idx < b->bi_vcnt; bv_idx++) {
			struct bio_vec * bv = &b->bi_io_vec[bv_idx];

			if (nreq && nreq->args.count + bv->bv_len <= rsize) {
				if (nreq->pages.pagevec[nreq->pages.npages-1] == bv->bv_page &&
				    prev_end == bv->bv_offset) {
					nreq->args.count += bv->bv_len;
					pos += bv->bv_len;
					prev_end += bv->bv_len;
					continue;
				}
				if (nreq->pages.npages < MAX_NBIO_PAGES &&
				    bv->bv_offset == 0 && prev_end == PAGE_SIZE) {
					nreq->args.count += bv->bv_len;
					nreq->pages.pagevec[nreq->pages.npages] = bv->bv_page;
					nreq->pages.npages++;
					pos += bv->bv_len;
					prev_end = bv->bv_offset + bv->bv_len;
					continue;
				}
			}

			if (nreq) {
				int err;

				atomic_inc(&preq->io_count);

				err = rbio_submit(io, nreq, &nfsio_read_ops);
				if (err) {
					PLOOP_REQ_SET_ERROR(preq, err);
					ploop_complete_io_request(preq);
					goto out;
				}
			}

			nreq = rbio_init(pos, bv->bv_page, bv->bv_offset,
					 bv->bv_len, preq, inode);

			if (nreq == NULL) {
				PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
				goto out;
			}

			pos += bv->bv_len;
			prev_end = bv->bv_offset + bv->bv_len;
		}
	}

	if (nreq) {
		int err;

		atomic_inc(&preq->io_count);

		err = rbio_submit(io, nreq, &nfsio_read_ops);
		if (err) {
			PLOOP_REQ_SET_ERROR(preq, err);
			ploop_complete_io_request(preq);
			goto out;
		}
	}

out:
	ploop_complete_io_request(preq);
}

static void nfsio_write_result(struct rpc_task *task, void *calldata)
{
	struct nfs_write_data *data = calldata;
	struct nfs_writeargs	*argp = &data->args;
	struct nfs_writeres	*resp = &data->res;
	int status;

	status = NFS_PROTO(data->header->inode)->write_done(task, data);
	if (status != 0)
		return;

	if (task->tk_status >= 0 && resp->count < argp->count)
		task->tk_status = -EIO;
}

static void nfsio_write_release(void *calldata)
{
	struct nfs_write_data *nreq = calldata;
	struct ploop_request *preq = (struct ploop_request *) nreq->header->req;
	int status = nreq->task.tk_status;

	if (unlikely(status < 0))
		PLOOP_REQ_SET_ERROR(preq, status);

	if (!preq->error &&
	    nreq->res.verf->committed != NFS_FILE_SYNC) {
		if (!test_and_set_bit(PLOOP_REQ_UNSTABLE, &preq->state))
			memcpy(&preq->verf, &nreq->res.verf->verifier, 8);
	}
	nfsio_complete_io_request(preq);

	nfsio_wbio_release(calldata);
}

static const struct rpc_call_ops nfsio_write_ops = {
	.rpc_call_done = nfsio_write_result,
	.rpc_release = nfsio_write_release,
};

static struct nfs_write_data *
wbio_init(loff_t pos, struct page * page, unsigned int off, unsigned int len,
	  void * priv, struct inode * inode)
{
	struct nfs_write_data * nreq;

	nreq = nfsio_wbio_alloc(MAX_NBIO_PAGES);
	if (unlikely(nreq == NULL))
		return NULL;

	nreq->args.offset = pos;
	nreq->args.pgbase = off;
	nreq->args.count = len;
	nreq->pages.pagevec[0] = page;
	nreq->pages.npages = 1;
	nreq->header->req = priv;
	nreq->header->inode = inode;
	nreq->args.fh = NFS_FH(inode);
	nreq->args.pages = nreq->pages.pagevec;
	nreq->args.stable = NFS_UNSTABLE;
	nreq->res.fattr = &nreq->fattr;
	nreq->res.verf = &nreq->verf;
	return nreq;
}

#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,18)

static int wbio_submit(struct ploop_io * io, struct nfs_write_data *nreq,
		       const struct rpc_call_ops * cb)
{
	struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
	struct inode *inode = io->files.inode;

	struct rpc_task *task;
	struct rpc_message msg = {
		.rpc_cred = ctx->cred,
	};

	struct rpc_task_setup task_setup_data = {
		.rpc_client = NFS_CLIENT(inode),
		.rpc_message = &msg,
		.callback_ops = cb,
#if LINUX_VERSION_CODE > KERNEL_VERSION(2,6,25)
		.workqueue = nfsio_workqueue,
#endif
		.flags = RPC_TASK_ASYNC,
	};

	if (verify_bounce(nreq))
		return -ENOMEM;

	nreq->res.count = nreq->args.count;
	nreq->args.context = ctx;
	nreq->header->cred = msg.rpc_cred;

	task_setup_data.task = &nreq->task;
	task_setup_data.callback_data = nreq;
	msg.rpc_argp = &nreq->args;
	msg.rpc_resp = &nreq->res;
	NFS_PROTO(inode)->write_setup(nreq, &msg);

	task = rpc_run_task(&task_setup_data);
	if (unlikely(IS_ERR(task)))
		return PTR_ERR(task);
	rpc_put_task(task);
	return 0;
}

#else

static int wbio_submit(struct ploop_io * io, struct nfs_write_data *nreq,
		       const struct rpc_call_ops * cb)
{
	struct nfs_open_context *ctx = nfs_file_open_context(io->files.file);
	struct inode *inode = io->files.inode;

	if (verify_bounce(nreq))
		return -ENOMEM;

	nreq->res.count = nreq->args.count;
	nreq->args.context = ctx;
	nreq->cred = ctx->cred;

	rpc_init_task(&nreq->task, NFS_CLIENT(inode), RPC_TASK_ASYNC, cb, nreq);
	NFS_PROTO(inode)->write_setup(nreq, NFS_UNSTABLE);

	nreq->task.tk_priority = RPC_PRIORITY_NORMAL;
	nreq->task.tk_cookie = (unsigned long) inode;

	lock_kernel();
	rpc_execute(&nreq->task);
	unlock_kernel();
	return 0;
}

#endif


static void
nfsio_submit_write(struct ploop_io *io, struct ploop_request * preq,
		   struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
	struct inode *inode = io->files.inode;
	size_t wsize = NFS_SERVER(inode)->wsize;
	struct nfs_write_data *nreq = NULL;
	loff_t pos;
	struct bio * b;
	unsigned int prev_end;

	nfsio_prepare_io_request(preq);

	pos = sbl->head->bi_sector;
	pos = ((loff_t)iblk << preq->plo->cluster_log) | (pos & ((1<<preq->plo->cluster_log) - 1));
	ploop_prepare_tracker(preq, pos);
	pos <<= 9;

	prev_end = PAGE_SIZE;

	for (b = sbl->head; b != NULL; b = b->bi_next) {
		int bv_idx;

		for (bv_idx = 0; bv_idx < b->bi_vcnt; bv_idx++) {
			struct bio_vec * bv = &b->bi_io_vec[bv_idx];

			if (nreq && nreq->args.count + bv->bv_len <= wsize) {
				if (nreq->pages.pagevec[nreq->pages.npages-1] == bv->bv_page &&
				    prev_end == bv->bv_offset) {
					nreq->args.count += bv->bv_len;
					pos += bv->bv_len;
					prev_end += bv->bv_len;
					continue;
				}
				if (nreq->pages.npages < MAX_NBIO_PAGES &&
				    bv->bv_offset == 0 && prev_end == PAGE_SIZE) {
					nreq->args.count += bv->bv_len;
					nreq->pages.pagevec[nreq->pages.npages] = bv->bv_page;
					nreq->pages.npages++;
					pos += bv->bv_len;
					prev_end = bv->bv_offset + bv->bv_len;
					continue;
				}
			}

			if (nreq) {
				int err;
				atomic_inc(&preq->io_count);
				err = wbio_submit(io, nreq, &nfsio_write_ops);
				if (err) {
					PLOOP_REQ_SET_ERROR(preq, err);
					nfsio_complete_io_request(preq);
					goto out;
				}
			}

			nreq = wbio_init(pos, bv->bv_page, bv->bv_offset,
					 bv->bv_len, preq, inode);

			if (nreq == NULL) {
				PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
				goto out;
			}

			prev_end = bv->bv_offset + bv->bv_len;
			pos += bv->bv_len;
		}
	}

	if (nreq) {
		int err;
		atomic_inc(&preq->io_count);
		err = wbio_submit(io, nreq, &nfsio_write_ops);
		if (err) {
			PLOOP_REQ_SET_ERROR(preq, err);
			nfsio_complete_io_request(preq);
		}
	}

out:
	nfsio_complete_io_request(preq);
}

static void
nfsio_submit(struct ploop_io *io, struct ploop_request * preq,
	     unsigned long rw,
	     struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
	if (iblk == PLOOP_ZERO_INDEX)
		iblk = 0;

	if (rw & (1<<BIO_RW))
		nfsio_submit_write(io, preq, sbl, iblk, size);
	else
		nfsio_submit_read(io, preq, sbl, iblk, size);
}

struct bio_list_walk
{
	struct bio * cur;
	int idx;
	int bv_off;
};

static void
nfsio_submit_write_pad(struct ploop_io *io, struct ploop_request * preq,
		       struct bio_list *sbl, iblock_t iblk, unsigned int size)
{
	struct inode *inode = io->files.inode;
	size_t wsize = NFS_SERVER(inode)->wsize;
	struct nfs_write_data *nreq = NULL;
	struct bio_list_walk bw;
	unsigned prev_end;

	loff_t pos, end_pos, start, end;

	/* pos..end_pos is the range which we are going to write */
	pos = (loff_t)iblk << (preq->plo->cluster_log + 9);
	end_pos = pos + (1 << (preq->plo->cluster_log + 9));

	/* start..end is data that we have. The rest must be zero padded. */
	start = pos + ((sbl->head->bi_sector & ((1<<preq->plo->cluster_log) - 1)) << 9);
	end = start + (size << 9);

	nfsio_prepare_io_request(preq);
	ploop_prepare_tracker(preq, start >> 9);

	prev_end = PAGE_SIZE;

#if 1
	/* GCC, shut up! */
	bw.cur = sbl->head;
	bw.idx = 0;
	bw.bv_off = 0;
	BUG_ON(bw.cur->bi_io_vec[0].bv_len & 511);
#endif

	while (pos < end_pos) {
		struct page * page;
		unsigned int poff, plen;

		if (pos < start) {
			page = ZERO_PAGE(0);
			poff = 0;
			plen = start - pos;
			if (plen > PAGE_SIZE)
				plen = PAGE_SIZE;
		} else if (pos >= end) {
			page = ZERO_PAGE(0);
			poff = 0;
			plen = end_pos - pos;
			if (plen > PAGE_SIZE)
				plen = PAGE_SIZE;
		} else {
			/* pos >= start && pos < end */
			struct bio_vec * bv;

			if (pos == start) {
				bw.cur = sbl->head;
				bw.idx = 0;
				bw.bv_off = 0;
				BUG_ON(bw.cur->bi_io_vec[0].bv_len & 511);
			}
			bv = bw.cur->bi_io_vec + bw.idx;

			if (bw.bv_off >= bv->bv_len) {
				bw.idx++;
				bv++;
				bw.bv_off = 0;
				if (bw.idx >= bw.cur->bi_vcnt) {
					bw.cur = bw.cur->bi_next;
					bw.idx = 0;
					bw.bv_off = 0;
					bv = bw.cur->bi_io_vec;
				}
				BUG_ON(bv->bv_len & 511);
			}

			page = bv->bv_page;
			poff = bv->bv_offset + bw.bv_off;
			plen = bv->bv_len - bw.bv_off;
		}

		if (nreq && nreq->args.count + plen <= wsize) {
			if (nreq->pages.pagevec[nreq->pages.npages-1] == page &&
			    prev_end == poff) {
				nreq->args.count += plen;
				pos += plen;
				bw.bv_off += plen;
				prev_end += plen;
				continue;
			}
			if (nreq->pages.npages < MAX_NBIO_PAGES &&
			    poff == 0 && prev_end == PAGE_SIZE) {
				nreq->args.count += plen;
				nreq->pages.pagevec[nreq->pages.npages] = page;
				nreq->pages.npages++;
				pos += plen;
				bw.bv_off += plen;
				prev_end = poff + plen;
				continue;
			}
		}

		if (nreq) {
			int err;
			atomic_inc(&preq->io_count);
			err = wbio_submit(io, nreq, &nfsio_write_ops);
			if (err) {
				PLOOP_REQ_SET_ERROR(preq, err);
				nfsio_complete_io_request(preq);
				goto out;
			}
		}

		nreq = wbio_init(pos, page, poff, plen, preq, inode);

		if (nreq == NULL) {
			PLOOP_REQ_SET_ERROR(preq, -ENOMEM);
			goto out;
		}

		prev_end = poff + plen;
		pos += plen;
		bw.bv_off += plen;
	}

	if (nreq) {
		int err;
		atomic_inc(&preq->io_count);
		err = wbio_submit(io, nreq, &nfsio_write_ops);
		if (err) {
			PLOOP_REQ_SET_ERROR(preq, err);
			nfsio_complete_io_request(preq);
		}
	}

out:
	nfsio_complete_io_request(preq);
}


static void
nfsio_submit_alloc(struct ploop_io *io, struct ploop_request * preq,
		 struct bio_list * sbl, unsigned int size)
{
	iblock_t iblk = io->alloc_head++;

	if (!(io->files.file->f_mode & FMODE_WRITE)) {
		PLOOP_FAIL_REQUEST(preq, -EBADF);
		return;
	}
	preq->iblock = iblk;
	preq->eng_state = PLOOP_E_DATA_WBI;

	nfsio_submit_write_pad(io, preq, sbl, iblk, size);
}

static void nfsio_destroy(struct ploop_io * io)
{
	if (io->fsync_thread) {
		kthread_stop(io->fsync_thread);
		io->fsync_thread = NULL;
	}

	if (io->files.file) {
		struct file * file = io->files.file;
		mutex_lock(&io->plo->sysfs_mutex);
		io->files.file = NULL;
		if (io->files.mapping)
			(void)invalidate_inode_pages2(io->files.mapping);
		mutex_unlock(&io->plo->sysfs_mutex);
		fput(file);
	}
}

static int nfsio_sync(struct ploop_io * io)
{
	return 0;
}

static int nfsio_stop(struct ploop_io * io)
{
	return 0;
}


static int
nfsio_init(struct ploop_io * io)
{
	INIT_LIST_HEAD(&io->fsync_queue);
	init_waitqueue_head(&io->fsync_waitq);
	return 0;
}