/**
* nfs_scan_list - Scan a list for matching requests
* @head: One of the NFS inode request lists
* @dst: Destination list
* @idx_start: lower bound of page->index to scan
* @npages: idx_start + npages sets the upper bound to scan.
*
* Moves elements from one of the inode request lists.
* If the number of requests is set to 0, the entire address_space
* starting at index idx_start, is scanned.
* The requests are *not* checked to ensure that they form a contiguous set.
* You must be holding the inode's req_lock when calling this function
*/
int
nfs_scan_list(struct list_head *head, struct list_head *dst,
unsigned long idx_start, unsigned int npages)
{
struct list_head *pos, *tmp;
struct nfs_page *req;
unsigned long idx_end;
int res;
res = 0;
if (npages == 0)
idx_end = ~0;
else
idx_end = idx_start + npages - 1;
list_for_each_safe(pos, tmp, head) {
req = nfs_list_entry(pos);
if (req->wb_index < idx_start)
continue;
if (req->wb_index > idx_end)
break;
if (!nfs_lock_request(req))
continue;
nfs_list_remove_request(req);
nfs_list_add_request(req, dst);
res++;
}
/**
* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
* @desc: destination io descriptor
* @req: request
*
* Returns true if the request 'req' was successfully coalesced into the
* existing list of pages 'desc'.
*/
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
struct nfs_page *req)
{
size_t newlen = req->wb_bytes;
if (desc->pg_count != 0) {
struct nfs_page *prev;
/*
* FIXME: ideally we should be able to coalesce all requests
* that are not block boundary aligned, but currently this
* is problematic for the case of bsize < PAGE_CACHE_SIZE,
* since nfs_flush_multi and nfs_pagein_multi assume you
* can have only one struct nfs_page.
*/
if (desc->pg_bsize < PAGE_SIZE)
return 0;
newlen += desc->pg_count;
if (newlen > desc->pg_bsize)
return 0;
prev = nfs_list_entry(desc->pg_list.prev);
if (!nfs_can_coalesce_requests(prev, req))
return 0;
} else
desc->pg_base = req->wb_pgbase;
nfs_list_remove_request(req);
nfs_list_add_request(req, &desc->pg_list);
desc->pg_count = newlen;
return 1;
}
/**
* nfs_coalesce_requests - Split coalesced requests out from a list.
* @head: source list
* @dst: destination list
* @nmax: maximum number of requests to coalesce
*
* Moves a maximum of 'nmax' elements from one list to another.
* The elements are checked to ensure that they form a contiguous set
* of pages, and that the RPC credentials are the same.
*/
int
nfs_coalesce_requests(struct list_head *head, struct list_head *dst,
unsigned int nmax)
{
struct nfs_page *req = NULL;
unsigned int npages = 0;
while (!list_empty(head)) {
struct nfs_page *prev = req;
req = nfs_list_entry(head->next);
if (prev) {
if (req->wb_context->cred != prev->wb_context->cred)
break;
if (req->wb_context->lockowner != prev->wb_context->lockowner)
break;
if (req->wb_context->state != prev->wb_context->state)
break;
if (req->wb_index != (prev->wb_index + 1))
break;
if (req->wb_pgbase != 0)
break;
}
nfs_list_remove_request(req);
nfs_list_add_request(req, dst);
npages++;
if (req->wb_pgbase + req->wb_bytes != PAGE_CACHE_SIZE)
break;
if (npages >= nmax)
break;
}
return npages;
}
/**
* nfs_pageio_cond_complete - Conditional I/O completion
* @desc: pointer to io descriptor
* @index: page index
*
* It is important to ensure that processes don't try to take locks
* on non-contiguous ranges of pages as that might deadlock. This
* function should be called before attempting to wait on a locked
* nfs_page. It will complete the I/O if the page index 'index'
* is not contiguous with the existing list of pages in 'desc'.
*/
void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
{
if (!list_empty(&desc->pg_list)) {
struct nfs_page *prev = nfs_list_entry(desc->pg_list.prev);
if (index != prev->wb_index + 1)
nfs_pageio_complete(desc);
}
}
static void nfs_cancel_commit_list(struct list_head *head)
{
struct nfs_page *req;
while(!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_clear_request_commit(req);
nfs_inode_remove_request(req);
nfs_unlock_request(req);
}
}
void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
struct nfs_pgio_header *hdr,
void (*release)(struct nfs_pgio_header *hdr))
{
hdr->req = nfs_list_entry(desc->pg_list.next);
hdr->inode = desc->pg_inode;
hdr->cred = hdr->req->wb_context->cred;
hdr->io_start = req_offset(hdr->req);
hdr->good_bytes = desc->pg_count;
hdr->dreq = desc->pg_dreq;
hdr->release = release;
hdr->completion_ops = desc->pg_completion_ops;
if (hdr->completion_ops->init_hdr)
hdr->completion_ops->init_hdr(hdr);
}
/**
* nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
* @desc: destination io descriptor
* @req: request
*
* Returns true if the request 'req' was successfully coalesced into the
* existing list of pages 'desc'.
*/
static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
struct nfs_page *req)
{
if (desc->pg_count != 0) {
struct nfs_page *prev;
prev = nfs_list_entry(desc->pg_list.prev);
if (!nfs_can_coalesce_requests(prev, req, desc))
return 0;
} else {
desc->pg_base = req->wb_pgbase;
}
nfs_list_remove_request(req);
nfs_list_add_request(req, &desc->pg_list);
desc->pg_count += req->wb_bytes;
return 1;
}
/*
* Create an RPC task for the given read or write request and kick it.
* The page must have been locked by the caller.
*
* It may happen that the page we're passed is not marked dirty.
* This is the case if nfs_updatepage detects a conflicting request
* that has been written but not committed.
*/
int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
struct nfs_pgio_header *hdr)
{
struct nfs_page *req;
struct page **pages,
*last_page;
struct list_head *head = &desc->pg_list;
struct nfs_commit_info cinfo;
unsigned int pagecount, pageused;
pagecount = nfs_page_array_len(desc->pg_base, desc->pg_count);
if (!nfs_pgarray_set(&hdr->page_array, pagecount))
return nfs_pgio_error(desc, hdr);
nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
pages = hdr->page_array.pagevec;
last_page = NULL;
pageused = 0;
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_list_add_request(req, &hdr->pages);
if (WARN_ON_ONCE(pageused >= pagecount))
return nfs_pgio_error(desc, hdr);
if (!last_page || last_page != req->wb_page) {
*pages++ = last_page = req->wb_page;
pageused++;
}
}
if (WARN_ON_ONCE(pageused != pagecount))
return nfs_pgio_error(desc, hdr);
if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
(desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
desc->pg_ioflags &= ~FLUSH_COND_STABLE;
/* Set up the argument struct */
nfs_pgio_rpcsetup(hdr, desc->pg_count, 0, desc->pg_ioflags, &cinfo);
desc->pg_rpc_callops = &nfs_pgio_common_ops;
return 0;
}
/**
* nfs_list_add_request - Insert a request into a sorted list
* @req: request
* @head: head of list into which to insert the request.
*
* Note that the wb_list is sorted by page index in order to facilitate
* coalescing of requests.
* We use an insertion sort that is optimized for the case of appended
* writes.
*/
void
nfs_list_add_request(struct nfs_page *req, struct list_head *head)
{
struct list_head *pos;
#ifdef NFS_PARANOIA
if (!list_empty(&req->wb_list)) {
printk(KERN_ERR "NFS: Add to list failed!\n");
BUG();
}
#endif
list_for_each_prev(pos, head) {
struct nfs_page *p = nfs_list_entry(pos);
if (p->wb_index < req->wb_index)
break;
}
list_add(&req->wb_list, pos);
req->wb_list_head = head;
}
static void nfs_writeback_release_full(void *calldata)
{
struct nfs_write_data *data = calldata;
int status = data->task.tk_status;
/* Update attributes as result of writeback. */
while (!list_empty(&data->pages)) {
struct nfs_page *req = nfs_list_entry(data->pages.next);
struct page *page = req->wb_page;
nfs_list_remove_request(req);
dprintk("NFS: %5u write (%s/%lld %d@%lld)",
data->task.tk_pid,
req->wb_context->path.dentry->d_inode->i_sb->s_id,
(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
req->wb_bytes,
(long long)req_offset(req));
if (status < 0) {
nfs_set_pageerror(page);
nfs_context_set_write_error(req->wb_context, status);
dprintk(", error = %d\n", status);
goto remove_request;
}
if (nfs_write_need_commit(data)) {
memcpy(&req->wb_verf, &data->verf, sizeof(req->wb_verf));
nfs_mark_request_commit(req);
nfs_end_page_writeback(page);
dprintk(" marked for commit\n");
goto next;
}
dprintk(" OK\n");
remove_request:
nfs_end_page_writeback(page);
nfs_inode_remove_request(req);
next:
nfs_clear_page_tag_locked(req);
}
nfs_writedata_release(calldata);
}
/*
* nfs_pageio_resend - Transfer requests to new descriptor and resend
* @hdr - the pgio header to move request from
* @desc - the pageio descriptor to add requests to
*
* Try to move each request (nfs_page) from @hdr to @desc then attempt
* to send them.
*
* Returns 0 on success and < 0 on error.
*/
int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
struct nfs_pgio_header *hdr)
{
LIST_HEAD(failed);
desc->pg_dreq = hdr->dreq;
while (!list_empty(&hdr->pages)) {
struct nfs_page *req = nfs_list_entry(hdr->pages.next);
nfs_list_remove_request(req);
if (!nfs_pageio_add_request(desc, req))
nfs_list_add_request(req, &failed);
}
nfs_pageio_complete(desc);
if (!list_empty(&failed)) {
list_move(&failed, &hdr->pages);
return -EIO;
}
return 0;
}
/*
* Set up the argument/result storage required for the RPC call.
*/
static int nfs_write_rpcsetup(struct nfs_page *req,
struct nfs_write_data *data,
const struct rpc_call_ops *call_ops,
unsigned int count, unsigned int offset,
int how)
{
struct inode *inode = req->wb_context->path.dentry->d_inode;
int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
int priority = flush_task_priority(how);
struct rpc_task *task;
struct rpc_message msg = {
.rpc_argp = &data->args,
.rpc_resp = &data->res,
.rpc_cred = req->wb_context->cred,
};
struct rpc_task_setup task_setup_data = {
.rpc_client = NFS_CLIENT(inode),
.task = &data->task,
.rpc_message = &msg,
.callback_ops = call_ops,
.callback_data = data,
.workqueue = nfsiod_workqueue,
.flags = flags,
.priority = priority,
};
/* Set up the RPC argument and reply structs
* NB: take care not to mess about with data->commit et al. */
data->req = req;
data->inode = inode = req->wb_context->path.dentry->d_inode;
data->cred = msg.rpc_cred;
data->args.fh = NFS_FH(inode);
data->args.offset = req_offset(req) + offset;
data->args.pgbase = req->wb_pgbase + offset;
data->args.pages = data->pagevec;
data->args.count = count;
data->args.context = get_nfs_open_context(req->wb_context);
data->args.stable = NFS_UNSTABLE;
if (how & FLUSH_STABLE) {
data->args.stable = NFS_DATA_SYNC;
if (!nfs_need_commit(NFS_I(inode)))
data->args.stable = NFS_FILE_SYNC;
}
data->res.fattr = &data->fattr;
data->res.count = count;
data->res.verf = &data->verf;
nfs_fattr_init(&data->fattr);
/* Set up the initial task struct. */
NFS_PROTO(inode)->write_setup(data, &msg);
dprintk("NFS: %5u initiated write call "
"(req %s/%lld, %u bytes @ offset %llu)\n",
data->task.tk_pid,
inode->i_sb->s_id,
(long long)NFS_FILEID(inode),
count,
(unsigned long long)data->args.offset);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
/* If a nfs_flush_* function fails, it should remove reqs from @head and
* call this on each, which will prepare them to be retried on next
* writeback using standard nfs.
*/
static void nfs_redirty_request(struct nfs_page *req)
{
nfs_mark_request_dirty(req);
nfs_end_page_writeback(req->wb_page);
nfs_clear_page_tag_locked(req);
}
/*
* Generate multiple small requests to write out a single
* contiguous dirty area on one page.
*/
static int nfs_flush_multi(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
{
struct nfs_page *req = nfs_list_entry(head->next);
struct page *page = req->wb_page;
struct nfs_write_data *data;
size_t wsize = NFS_SERVER(inode)->wsize, nbytes;
unsigned int offset;
int requests = 0;
int ret = 0;
LIST_HEAD(list);
nfs_list_remove_request(req);
nbytes = count;
do {
size_t len = min(nbytes, wsize);
data = nfs_writedata_alloc(1);
if (!data)
goto out_bad;
list_add(&data->pages, &list);
requests++;
nbytes -= len;
} while (nbytes != 0);
atomic_set(&req->wb_complete, requests);
ClearPageError(page);
offset = 0;
nbytes = count;
do {
int ret2;
data = list_entry(list.next, struct nfs_write_data, pages);
list_del_init(&data->pages);
data->pagevec[0] = page;
if (nbytes < wsize)
wsize = nbytes;
ret2 = nfs_write_rpcsetup(req, data, &nfs_write_partial_ops,
wsize, offset, how);
if (ret == 0)
ret = ret2;
offset += wsize;
nbytes -= wsize;
} while (nbytes != 0);
return ret;
out_bad:
while (!list_empty(&list)) {
data = list_entry(list.next, struct nfs_write_data, pages);
list_del(&data->pages);
nfs_writedata_release(data);
}
nfs_redirty_request(req);
return -ENOMEM;
}
/*
* Create an RPC task for the given write request and kick it.
* The page must have been locked by the caller.
*
* It may happen that the page we're passed is not marked dirty.
* This is the case if nfs_updatepage detects a conflicting request
* that has been written but not committed.
*/
static int nfs_flush_one(struct inode *inode, struct list_head *head, unsigned int npages, size_t count, int how)
{
struct nfs_page *req;
struct page **pages;
struct nfs_write_data *data;
data = nfs_writedata_alloc(npages);
if (!data)
goto out_bad;
pages = data->pagevec;
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_list_add_request(req, &data->pages);
ClearPageError(req->wb_page);
*pages++ = req->wb_page;
}
req = nfs_list_entry(data->pages.next);
/* Set up the argument struct */
return nfs_write_rpcsetup(req, data, &nfs_write_full_ops, count, 0, how);
out_bad:
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_redirty_request(req);
}
return -ENOMEM;
}
static void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
struct inode *inode, int ioflags)
{
size_t wsize = NFS_SERVER(inode)->wsize;
if (wsize < PAGE_CACHE_SIZE)
nfs_pageio_init(pgio, inode, nfs_flush_multi, wsize, ioflags);
else
nfs_pageio_init(pgio, inode, nfs_flush_one, wsize, ioflags);
}
/*
* Set up the argument/result storage required for the RPC call.
*/
static int nfs_commit_rpcsetup(struct list_head *head,
struct nfs_write_data *data,
int how)
{
struct nfs_page *first = nfs_list_entry(head->next);
struct inode *inode = first->wb_context->path.dentry->d_inode;
int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
int priority = flush_task_priority(how);
struct rpc_task *task;
struct rpc_message msg = {
.rpc_argp = &data->args,
.rpc_resp = &data->res,
.rpc_cred = first->wb_context->cred,
};
struct rpc_task_setup task_setup_data = {
.task = &data->task,
.rpc_client = NFS_CLIENT(inode),
.rpc_message = &msg,
.callback_ops = &nfs_commit_ops,
.callback_data = data,
.workqueue = nfsiod_workqueue,
.flags = flags,
.priority = priority,
};
/* Set up the RPC argument and reply structs
* NB: take care not to mess about with data->commit et al. */
list_splice_init(head, &data->pages);
data->inode = inode;
data->cred = msg.rpc_cred;
data->args.fh = NFS_FH(data->inode);
/* Note: we always request a commit of the entire inode */
data->args.offset = 0;
data->args.count = 0;
data->args.context = get_nfs_open_context(first->wb_context);
data->res.count = 0;
data->res.fattr = &data->fattr;
data->res.verf = &data->verf;
nfs_fattr_init(&data->fattr);
/* Set up the initial task struct. */
NFS_PROTO(inode)->commit_setup(data, &msg);
dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
/*
* Commit dirty pages
*/
static int
nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
struct nfs_write_data *data;
struct nfs_page *req;
data = nfs_commitdata_alloc();
if (!data)
goto out_bad;
/* Set up the argument struct */
return nfs_commit_rpcsetup(head, data, how);
out_bad:
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_mark_request_commit(req);
dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
BDI_RECLAIMABLE);
nfs_clear_page_tag_locked(req);
}
return -ENOMEM;
}
/*
* COMMIT call returned
*/
static void nfs_commit_done(struct rpc_task *task, void *calldata)
{
struct nfs_write_data *data = calldata;
dprintk("NFS: %5u nfs_commit_done (status %d)\n",
task->tk_pid, task->tk_status);
/* Call the NFS version-specific code */
if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
return;
}
static void nfs_commit_release(void *calldata)
{
struct nfs_write_data *data = calldata;
struct nfs_page *req;
int status = data->task.tk_status;
while (!list_empty(&data->pages)) {
req = nfs_list_entry(data->pages.next);
nfs_list_remove_request(req);
nfs_clear_request_commit(req);
dprintk("NFS: commit (%s/%lld %d@%lld)",
req->wb_context->path.dentry->d_inode->i_sb->s_id,
(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
req->wb_bytes,
(long long)req_offset(req));
if (status < 0) {
nfs_context_set_write_error(req->wb_context, status);
nfs_inode_remove_request(req);
dprintk(", error = %d\n", status);
goto next;
}
/* Okay, COMMIT succeeded, apparently. Check the verifier
* returned by the server against all stored verfs. */
if (!memcmp(req->wb_verf.verifier, data->verf.verifier, sizeof(data->verf.verifier))) {
/* We have a match */
nfs_inode_remove_request(req);
dprintk(" OK\n");
goto next;
}
/* We have a mismatch. Write the page again */
dprintk(" mismatch\n");
nfs_mark_request_dirty(req);
next:
nfs_clear_page_tag_locked(req);
}
nfs_commitdata_release(calldata);
}
static const struct rpc_call_ops nfs_commit_ops = {
#if defined(CONFIG_NFS_V4_1)
.rpc_call_prepare = nfs_write_prepare,
#endif /* CONFIG_NFS_V4_1 */
.rpc_call_done = nfs_commit_done,
.rpc_release = nfs_commit_release,
};
int nfs_commit_inode(struct inode *inode, int how)
{
LIST_HEAD(head);
int res;
spin_lock(&inode->i_lock);
res = nfs_scan_commit(inode, &head, 0, 0);
spin_unlock(&inode->i_lock);
if (res) {
int error = nfs_commit_list(inode, &head, how);
if (error < 0)
return error;
}
return res;
}
#else
static inline int nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
return 0;
}
#endif
long nfs_sync_mapping_wait(struct address_space *mapping, struct writeback_control *wbc, int how)
{
struct inode *inode = mapping->host;
pgoff_t idx_start, idx_end;
unsigned int npages = 0;
LIST_HEAD(head);
int nocommit = how & FLUSH_NOCOMMIT;
long pages, ret;
/* FIXME */
if (wbc->range_cyclic)
idx_start = 0;
else {
idx_start = wbc->range_start >> PAGE_CACHE_SHIFT;
idx_end = wbc->range_end >> PAGE_CACHE_SHIFT;
if (idx_end > idx_start) {
pgoff_t l_npages = 1 + idx_end - idx_start;
npages = l_npages;
if (sizeof(npages) != sizeof(l_npages) &&
(pgoff_t)npages != l_npages)
npages = 0;
}
}
how &= ~FLUSH_NOCOMMIT;
spin_lock(&inode->i_lock);
do {
ret = nfs_wait_on_requests_locked(inode, idx_start, npages);
if (ret != 0)
continue;
if (nocommit)
break;
pages = nfs_scan_commit(inode, &head, idx_start, npages);
if (pages == 0)
break;
if (how & FLUSH_INVALIDATE) {
spin_unlock(&inode->i_lock);
nfs_cancel_commit_list(&head);
ret = pages;
spin_lock(&inode->i_lock);
continue;
}
pages += nfs_scan_commit(inode, &head, 0, 0);
spin_unlock(&inode->i_lock);
ret = nfs_commit_list(inode, &head, how);
spin_lock(&inode->i_lock);
} while (ret >= 0);
spin_unlock(&inode->i_lock);
return ret;
}
static int __nfs_write_mapping(struct address_space *mapping, struct writeback_control *wbc, int how)
{
int ret;
ret = nfs_writepages(mapping, wbc);
if (ret < 0)
goto out;
ret = nfs_sync_mapping_wait(mapping, wbc, how);
if (ret < 0)
goto out;
return 0;
out:
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
return ret;
}
