/**
* nfs_coalesce_requests - Split coalesced requests out from a list.
* @head: source list
* @dst: destination list
* @nmax: maximum number of requests to coalesce
*
* Moves a maximum of 'nmax' elements from one list to another.
* The elements are checked to ensure that they form a contiguous set
* of pages, and that the RPC credentials are the same.
*/
int
nfs_coalesce_requests(struct list_head *head, struct list_head *dst,
unsigned int nmax)
{
struct nfs_page *req = NULL;
unsigned int npages = 0;
while (!list_empty(head)) {
struct nfs_page *prev = req;
req = nfs_list_entry(head->next);
if (prev) {
if (req->wb_context->cred != prev->wb_context->cred)
break;
if (req->wb_context->lockowner != prev->wb_context->lockowner)
break;
if (req->wb_context->state != prev->wb_context->state)
break;
if (req->wb_index != (prev->wb_index + 1))
break;
if (req->wb_pgbase != 0)
break;
}
nfs_list_remove_request(req);
nfs_list_add_request(req, dst);
npages++;
if (req->wb_pgbase + req->wb_bytes != PAGE_CACHE_SIZE)
break;
if (npages >= nmax)
break;
}
return npages;
}
/**
* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
* @desc: destination io descriptor
* @req: request
*
* Returns true if the request 'req' was successfully coalesced into the
* existing list of pages 'desc'.
*/
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
struct nfs_page *req)
{
size_t newlen = req->wb_bytes;
if (desc->pg_count != 0) {
struct nfs_page *prev;
/*
* FIXME: ideally we should be able to coalesce all requests
* that are not block boundary aligned, but currently this
* is problematic for the case of bsize < PAGE_CACHE_SIZE,
* since nfs_flush_multi and nfs_pagein_multi assume you
* can have only one struct nfs_page.
*/
if (desc->pg_bsize < PAGE_SIZE)
return 0;
newlen += desc->pg_count;
if (newlen > desc->pg_bsize)
return 0;
prev = nfs_list_entry(desc->pg_list.prev);
if (!nfs_can_coalesce_requests(prev, req))
return 0;
} else
desc->pg_base = req->wb_pgbase;
nfs_list_remove_request(req);
nfs_list_add_request(req, &desc->pg_list);
desc->pg_count = newlen;
return 1;
}
/**
* nfs_scan_list - Scan a list for matching requests
* @head: One of the NFS inode request lists
* @dst: Destination list
* @idx_start: lower bound of page->index to scan
* @npages: idx_start + npages sets the upper bound to scan.
*
* Moves elements from one of the inode request lists.
* If the number of requests is set to 0, the entire address_space
* starting at index idx_start, is scanned.
* The requests are *not* checked to ensure that they form a contiguous set.
* You must be holding the inode's req_lock when calling this function
*/
int
nfs_scan_list(struct list_head *head, struct list_head *dst,
unsigned long idx_start, unsigned int npages)
{
struct list_head *pos, *tmp;
struct nfs_page *req;
unsigned long idx_end;
int res;
res = 0;
if (npages == 0)
idx_end = ~0;
else
idx_end = idx_start + npages - 1;
list_for_each_safe(pos, tmp, head) {
req = nfs_list_entry(pos);
if (req->wb_index < idx_start)
continue;
if (req->wb_index > idx_end)
break;
if (!nfs_lock_request(req))
continue;
nfs_list_remove_request(req);
nfs_list_add_request(req, dst);
res++;
}
/*
* Add a request to the inode's commit list.
*/
static inline void
nfs_mark_request_commit(struct nfs_page *req)
{
struct inode *inode = req->wb_inode;
spin_lock(&nfs_wreq_lock);
nfs_list_add_request(req, &inode->u.nfs_i.commit);
inode->u.nfs_i.ncommit++;
__nfs_del_lru(req);
__nfs_add_lru(&NFS_SERVER(inode)->lru_commit, req);
spin_unlock(&nfs_wreq_lock);
mark_inode_dirty(inode);
}
/**
* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
* @desc: destination io descriptor
* @req: request
*
* Returns true if the request 'req' was successfully coalesced into the
* existing list of pages 'desc'.
*/
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
struct nfs_page *req)
{
if (desc->pg_count != 0) {
struct nfs_page *prev;
prev = nfs_list_entry(desc->pg_list.prev);
if (!nfs_can_coalesce_requests(prev, req, desc))
return 0;
} else {
desc->pg_base = req->wb_pgbase;
}
nfs_list_remove_request(req);
nfs_list_add_request(req, &desc->pg_list);
desc->pg_count += req->wb_bytes;
return 1;
}
/**
* nfs_scan_list - Scan a list for matching requests
* @nfsi: NFS inode
* @dst: Destination list
* @idx_start: lower bound of page->index to scan
* @npages: idx_start + npages sets the upper bound to scan.
* @tag: tag to scan for
*
* Moves elements from one of the inode request lists.
* If the number of requests is set to 0, the entire address_space
* starting at index idx_start, is scanned.
* The requests are *not* checked to ensure that they form a contiguous set.
* You must be holding the inode's i_lock when calling this function
*/
int nfs_scan_list(struct nfs_inode *nfsi,
struct list_head *dst, pgoff_t idx_start,
unsigned int npages, int tag)
{
struct nfs_page *pgvec[NFS_SCAN_MAXENTRIES];
struct nfs_page *req;
pgoff_t idx_end;
int found, i;
int res;
struct list_head *list;
res = 0;
if (npages == 0)
idx_end = ~0;
else
idx_end = idx_start + npages - 1;
for (;;) {
found = radix_tree_gang_lookup_tag(&nfsi->nfs_page_tree,
(void **)&pgvec[0], idx_start,
NFS_SCAN_MAXENTRIES, tag);
if (found <= 0)
break;
for (i = 0; i < found; i++) {
req = pgvec[i];
if (req->wb_index > idx_end)
goto out;
idx_start = req->wb_index + 1;
if (nfs_set_page_tag_locked(req)) {
kref_get(&req->wb_kref);
radix_tree_tag_clear(&nfsi->nfs_page_tree,
req->wb_index, tag);
list = pnfs_choose_commit_list(req, dst);
nfs_list_add_request(req, list);
res++;
if (res == INT_MAX)
goto out;
}
}
/* for latency reduction */
cond_resched_lock(&nfsi->vfs_inode.i_lock);
}
out:
return res;
}
/*
* Create an RPC task for the given read or write request and kick it.
* The page must have been locked by the caller.
*
* It may happen that the page we're passed is not marked dirty.
* This is the case if nfs_updatepage detects a conflicting request
* that has been written but not committed.
*/
int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
struct nfs_pgio_header *hdr)
{
struct nfs_page *req;
struct page **pages,
*last_page;
struct list_head *head = &desc->pg_list;
struct nfs_commit_info cinfo;
unsigned int pagecount, pageused;
pagecount = nfs_page_array_len(desc->pg_base, desc->pg_count);
if (!nfs_pgarray_set(&hdr->page_array, pagecount))
return nfs_pgio_error(desc, hdr);
nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
pages = hdr->page_array.pagevec;
last_page = NULL;
pageused = 0;
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_list_add_request(req, &hdr->pages);
if (WARN_ON_ONCE(pageused >= pagecount))
return nfs_pgio_error(desc, hdr);
if (!last_page || last_page != req->wb_page) {
*pages++ = last_page = req->wb_page;
pageused++;
}
}
if (WARN_ON_ONCE(pageused != pagecount))
return nfs_pgio_error(desc, hdr);
if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
(desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
desc->pg_ioflags &= ~FLUSH_COND_STABLE;
/* Set up the argument struct */
nfs_pgio_rpcsetup(hdr, desc->pg_count, 0, desc->pg_ioflags, &cinfo);
desc->pg_rpc_callops = &nfs_pgio_common_ops;
return 0;
}
/*
* nfs_pageio_resend - Transfer requests to new descriptor and resend
* @hdr - the pgio header to move request from
* @desc - the pageio descriptor to add requests to
*
* Try to move each request (nfs_page) from @hdr to @desc then attempt
* to send them.
*
* Returns 0 on success and < 0 on error.
*/
int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
struct nfs_pgio_header *hdr)
{
LIST_HEAD(failed);
desc->pg_dreq = hdr->dreq;
while (!list_empty(&hdr->pages)) {
struct nfs_page *req = nfs_list_entry(hdr->pages.next);
nfs_list_remove_request(req);
if (!nfs_pageio_add_request(desc, req))
nfs_list_add_request(req, &failed);
}
nfs_pageio_complete(desc);
if (!list_empty(&failed)) {
list_move(&failed, &hdr->pages);
return -EIO;
}
return 0;
}
/*
* Set up the argument/result storage required for the RPC call.
*/
static int nfs_write_rpcsetup(struct nfs_page *req,
struct nfs_write_data *data,
const struct rpc_call_ops *call_ops,
unsigned int count, unsigned int offset,
int how)
{
struct inode *inode = req->wb_context->path.dentry->d_inode;
int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
int priority = flush_task_priority(how);
struct rpc_task *task;
struct rpc_message msg = {
.rpc_argp = &data->args,
.rpc_resp = &data->res,
.rpc_cred = req->wb_context->cred,
};
struct rpc_task_setup task_setup_data = {
.rpc_client = NFS_CLIENT(inode),
.task = &data->task,
.rpc_message = &msg,
.callback_ops = call_ops,
.callback_data = data,
.workqueue = nfsiod_workqueue,
.flags = flags,
.priority = priority,
};
/* Set up the RPC argument and reply structs
* NB: take care not to mess about with data->commit et al. */
data->req = req;
data->inode = inode = req->wb_context->path.dentry->d_inode;
data->cred = msg.rpc_cred;
data->args.fh = NFS_FH(inode);
data->args.offset = req_offset(req) + offset;
data->args.pgbase = req->wb_pgbase + offset;
data->args.pages = data->pagevec;
data->args.count = count;
data->args.context = get_nfs_open_context(req->wb_context);
data->args.stable = NFS_UNSTABLE;
if (how & FLUSH_STABLE) {
data->args.stable = NFS_DATA_SYNC;
if (!nfs_need_commit(NFS_I(inode)))
data->args.stable = NFS_FILE_SYNC;
}
data->res.fattr = &data->fattr;
data->res.count = count;
data->res.verf = &data->verf;
nfs_fattr_init(&data->fattr);
/* Set up the initial task struct. */
NFS_PROTO(inode)->write_setup(data, &msg);
dprintk("NFS: %5u initiated write call "
"(req %s/%lld, %u bytes @ offset %llu)\n",
data->task.tk_pid,
inode->i_sb->s_id,
(long long)NFS_FILEID(inode),
count,
(unsigned long long)data->args.offset);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
/* If a nfs_flush_* function fails, it should remove reqs from @head and
* call this on each, which will prepare them to be retried on next
* writeback using standard nfs.
*/
static void nfs_redirty_request(struct nfs_page *req)
{
nfs_mark_request_dirty(req);
nfs_end_page_writeback(req->wb_page);
nfs_clear_page_tag_locked(req);
}
/*
* Generate multiple small requests to write out a single
* contiguous dirty area on one page.
*/
static int nfs_flush_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
{
struct nfs_page *req = nfs_list_entry(head->next);
struct page *page = req->wb_page;
struct nfs_write_data *data;
size_t wsize = NFS_SERVER(inode)->wsize, nbytes;
unsigned int offset;
int requests = 0;
int ret = 0;
LIST_HEAD(list);
nfs_list_remove_request(req);
nbytes = count;
do {
size_t len = min(nbytes, wsize);
data = nfs_writedata_alloc(1);
if (!data)
goto out_bad;
list_add(&data->pages, &list);
requests++;
nbytes -= len;
} while (nbytes != 0);
atomic_set(&req->wb_complete, requests);
ClearPageError(page);
offset = 0;
nbytes = count;
do {
int ret2;
data = list_entry(list.next, struct nfs_write_data, pages);
list_del_init(&data->pages);
data->pagevec[0] = page;
if (nbytes < wsize)
wsize = nbytes;
ret2 = nfs_write_rpcsetup(req, data, &nfs_write_partial_ops,
wsize, offset, how);
if (ret == 0)
ret = ret2;
offset += wsize;
nbytes -= wsize;
} while (nbytes != 0);
return ret;
out_bad:
while (!list_empty(&list)) {
data = list_entry(list.next, struct nfs_write_data, pages);
list_del(&data->pages);
nfs_writedata_release(data);
}
nfs_redirty_request(req);
return -ENOMEM;
}
/*
* Create an RPC task for the given write request and kick it.
* The page must have been locked by the caller.
*
* It may happen that the page we're passed is not marked dirty.
* This is the case if nfs_updatepage detects a conflicting request
* that has been written but not committed.
*/
static int nfs_flush_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
{
struct nfs_page *req;
struct page **pages;
struct nfs_write_data *data;
data = nfs_writedata_alloc(npages);
if (!data)
goto out_bad;
pages = data->pagevec;
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_list_add_request(req, &data->pages);
ClearPageError(req->wb_page);
*pages++ = req->wb_page;
}
req = nfs_list_entry(data->pages.next);
/* Set up the argument struct */
return nfs_write_rpcsetup(req, data, &nfs_write_full_ops, count, 0, how);
out_bad:
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_redirty_request(req);
}
return -ENOMEM;
}
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
struct inode *inode, int ioflags)
{
size_t wsize = NFS_SERVER(inode)->wsize;
if (wsize < PAGE_CACHE_SIZE)
nfs_pageio_init(pgio, inode, nfs_flush_multi, wsize, ioflags);
else
nfs_pageio_init(pgio, inode, nfs_flush_one, wsize, ioflags);
}
