static void __sweep(struct dm_deferred_set *ds, struct list_head *head)
{
while ((ds->sweeper != ds->current_entry) &&
!ds->entries[ds->sweeper].count) {
list_splice_init(&ds->entries[ds->sweeper].work_items, head);
ds->sweeper = ds_next(ds->sweeper);
}
if ((ds->sweeper == ds->current_entry) && !ds->entries[ds->sweeper].count)
list_splice_init(&ds->entries[ds->sweeper].work_items, head);
}
/*
* Shifts all regions down one level. This has no effect on the order of
* the queue.
*/
static void queue_shift_down(struct queue *q)
{
unsigned level;
for (level = 1; level < NR_QUEUE_LEVELS; level++)
list_splice_init(q->qs + level, q->qs + level - 1);
}
static void __linkwatch_run_queue(int urgent_only)
{
struct net_device *dev;
LIST_HEAD(wrk);
if (!urgent_only)
linkwatch_nextevent = jiffies + HZ;
else if (time_after(linkwatch_nextevent, jiffies + HZ))
linkwatch_nextevent = jiffies;
clear_bit(LW_URGENT, &linkwatch_flags);
spin_lock_irq(&lweventlist_lock);
list_splice_init(&lweventlist, &wrk);
while (!list_empty(&wrk)) {
dev = list_first_entry(&wrk, struct net_device, link_watch_list);
list_del_init(&dev->link_watch_list);
if (urgent_only && !linkwatch_urgent_event(dev)) {
list_add_tail(&dev->link_watch_list, &lweventlist);
continue;
}
spin_unlock_irq(&lweventlist_lock);
linkwatch_do_dev(dev);
spin_lock_irq(&lweventlist_lock);
}
if (!list_empty(&lweventlist))
linkwatch_schedule_work(0);
spin_unlock_irq(&lweventlist_lock);
}
static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
/*
* If a CPU goes away, splice its entries to the current CPU
* and trigger a run of the softirq
*/
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
int cpu = (unsigned long) hcpu;
int i;
local_irq_disable();
for (i = 0; i < NR_SOFTIRQS; i++) {
struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
struct list_head *local_head;
if (list_empty(head))
continue;
local_head = &__get_cpu_var(softirq_work_list[i]);
list_splice_init(head, local_head);
raise_softirq_irqoff(i);
}
local_irq_enable();
}
return NOTIFY_OK;
}
static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
{
LIST_HEAD(head);
do {
list_splice_init(&desc->pg_list, &head);
desc->pg_bytes_written -= desc->pg_count;
desc->pg_count = 0;
desc->pg_base = 0;
desc->pg_recoalesce = 0;
while (!list_empty(&head)) {
struct nfs_page *req;
req = list_first_entry(&head, struct nfs_page, wb_list);
nfs_list_remove_request(req);
if (__nfs_pageio_add_request(desc, req))
continue;
if (desc->pg_error < 0)
return 0;
break;
}
} while (desc->pg_recoalesce);
return 1;
}
/* Auditor comments: This merge does not know if merged sets contain
blocks pairs (As for wandered sets) or extents, so it cannot really merge
overlapping ranges if there is some. So I believe it may lead to
some blocks being presented several times in one blocknr_set. To help
debugging such problems it might help to check for duplicate entries on
actual processing of this set. Testing this kind of stuff right here is
also complicated by the fact that these sets are not sorted and going
through whole set on each element addition is going to be CPU-heavy task */
void blocknr_set_merge(struct list_head *from, struct list_head *into)
{
blocknr_set_entry *bse_into = NULL;
/* If @from is empty, no work to perform. */
if (list_empty(from))
return;
/* If @into is not empty, try merging partial-entries. */
if (!list_empty(into)) {
/* Neither set is empty, pop the front to members and try to
combine them. */
blocknr_set_entry *bse_from;
unsigned into_avail;
bse_into = list_entry(into->next, blocknr_set_entry, link);
list_del_init(&bse_into->link);
bse_from = list_entry(from->next, blocknr_set_entry, link);
list_del_init(&bse_from->link);
/* Combine singles. */
for (into_avail = bse_avail(bse_into);
into_avail != 0 && bse_from->nr_singles != 0;
into_avail -= 1) {
bse_put_single(bse_into,
&bse_from->entries[--bse_from->
nr_singles]);
}
/* Combine pairs. */
for (; into_avail > 1 && bse_from->nr_pairs != 0;
into_avail -= 2) {
struct blocknr_pair *pair =
bse_get_pair(bse_from, --bse_from->nr_pairs);
bse_put_pair(bse_into, &pair->a, &pair->b);
}
/* If bse_from is empty, delete it now. */
if (bse_avail(bse_from) == BLOCKNR_SET_ENTRIES_NUMBER) {
bse_free(bse_from);
} else {
/* Otherwise, bse_into is full or nearly full (e.g.,
it could have one slot avail and bse_from has one
pair left). Push it back onto the list. bse_from
becomes bse_into, which will be the new partial. */
list_add(&bse_into->link, into);
bse_into = bse_from;
}
}
/* Splice lists together. */
list_splice_init(from, into->prev);
/* Add the partial entry back to the head of the list. */
if (bse_into != NULL)
list_add(&bse_into->link, into);
}
/*
* When inserting, destroying or reaping, it's first necessary to update the
* lists relative to a particular time. In the case of destroying, that time
* will be well in the future to ensure that all items are moved to the reap
* list. In all other cases though, the time will be the current time.
*
* This function enters a loop, moving the contents of the LRU list to the reap
* list again and again until either a) the lists are all empty, or b) time zero
* has been advanced sufficiently to be within the immediate element lifetime.
*
* Case a) above is detected by counting how many groups are migrated and
* stopping when they've all been moved. Case b) is detected by monitoring the
* time_zero field, which is updated as each group is migrated.
*
* The return value is the earliest time that more migration could be needed, or
* zero if there's no need to schedule more work because the lists are empty.
*/
STATIC unsigned long
_xfs_mru_cache_migrate(
xfs_mru_cache_t *mru,
unsigned long now)
{
unsigned int grp;
unsigned int migrated = 0;
struct list_head *lru_list;
/* Nothing to do if the data store is empty. */
if (!mru->time_zero)
return 0;
/* While time zero is older than the time spanned by all the lists. */
while (mru->time_zero <= now - mru->grp_count * mru->grp_time) {
/*
* If the LRU list isn't empty, migrate its elements to the tail
* of the reap list.
*/
lru_list = mru->lists + mru->lru_grp;
if (!list_empty(lru_list))
list_splice_init(lru_list, mru->reap_list.prev);
/*
* Advance the LRU group number, freeing the old LRU list to
* become the new MRU list; advance time zero accordingly.
*/
mru->lru_grp = (mru->lru_grp + 1) % mru->grp_count;
mru->time_zero += mru->grp_time;
/*
* If reaping is so far behind that all the elements on all the
* lists have been migrated to the reap list, it's now empty.
*/
if (++migrated == mru->grp_count) {
mru->lru_grp = 0;
mru->time_zero = 0;
return 0;
}
}
/* Find the first non-empty list from the LRU end. */
for (grp = 0; grp < mru->grp_count; grp++) {
/* Check the grp'th list from the LRU end. */
lru_list = mru->lists + ((mru->lru_grp + grp) % mru->grp_count);
if (!list_empty(lru_list))
return mru->time_zero +
(mru->grp_count + grp) * mru->grp_time;
}
/* All the lists must be empty. */
mru->lru_grp = 0;
mru->time_zero = 0;
return 0;
}
/* Do the collect_pages job on a single CPU: assumes that all other
* CPUs have been stopped during a panic. If this isn't true for
* some arch, this will have to be implemented separately in each arch.
*/
static void
collect_pages_from_single_cpu(struct page_collection *pc)
{
struct trace_cpu_data *tcd;
int i, j;
tcd_for_each(tcd, i, j) {
list_splice_init(&tcd->tcd_pages, &pc->pc_pages);
tcd->tcd_cur_pages = 0;
}
enum gcerror gcmmu_destroy_context(struct gccorecontext *gccorecontext,
struct gcmmucontext *gcmmucontext)
{
enum gcerror gcerror;
struct gcmmu *gcmmu = &gccorecontext->gcmmu;
struct list_head *head;
struct gcmmuarena *arena;
struct gcmmustlbblock *nextblock;
GCENTER(GCZONE_CONTEXT);
if (gcmmucontext == NULL) {
gcerror = GCERR_MMU_CTXT_BAD;
goto exit;
}
/* Unmap the command queue. */
gcerror = gcqueue_unmap(gccorecontext, gcmmucontext);
if (gcerror != GCERR_NONE)
goto exit;
/* Free allocated arenas. */
while (!list_empty(&gcmmucontext->allocated)) {
head = gcmmucontext->allocated.next;
arena = list_entry(head, struct gcmmuarena, link);
release_physical_pages(arena);
list_move(head, &gcmmucontext->vacant);
}
/* Free slave tables. */
while (gcmmucontext->slavealloc != NULL) {
gc_free_cached(&gcmmucontext->slavealloc->pages);
nextblock = gcmmucontext->slavealloc->next;
kfree(gcmmucontext->slavealloc);
gcmmucontext->slavealloc = nextblock;
}
/* Free the master table. */
gc_free_cached(&gcmmucontext->master);
/* Free arenas. */
GCLOCK(&gcmmu->lock);
list_splice_init(&gcmmucontext->vacant, &gcmmu->vacarena);
GCUNLOCK(&gcmmu->lock);
/* Dereference. */
gcmmu->refcount -= 1;
GCEXIT(GCZONE_CONTEXT);
return GCERR_NONE;
exit:
GCEXITARG(GCZONE_CONTEXT, "gcerror = 0x%08X\n", gcerror);
return gcerror;
}
static void delayed_fput(struct work_struct *unused)
{
LIST_HEAD(head);
spin_lock_irq(&delayed_fput_lock);
list_splice_init(&delayed_fput_list, &head);
spin_unlock_irq(&delayed_fput_lock);
while (!list_empty(&head)) {
struct file *f = list_first_entry(&head, struct file, f_u.fu_list);
list_del_init(&f->f_u.fu_list);
__fput(f);
}
}
/* This worker cleans all PID related data on this node. */
static void put_pid_worker(struct work_struct *work)
{
LIST_HEAD(work_list);
struct pid_kddm_object *obj, *n;
/* Remove the current job-list */
spin_lock(&put_pid_wq_lock);
list_splice_init(&put_pid_wq_head, &work_list);
spin_unlock(&put_pid_wq_lock);
list_for_each_entry_safe(obj, n, &work_list, wq)
__put_pid(obj);
}
static int blk_softirq_cpu_dead(unsigned int cpu)
{
/*
* If a CPU goes away, splice its entries to the current CPU
* and trigger a run of the softirq
*/
local_irq_disable();
list_splice_init(&per_cpu(blk_cpu_done, cpu),
this_cpu_ptr(&blk_cpu_done));
raise_softirq_irqoff(BLOCK_SOFTIRQ);
local_irq_enable();
return 0;
}
void global_flush_tlb(void)
{
LIST_HEAD(l);
struct page *pg, *next;
BUG_ON(irqs_disabled());
spin_lock_irq(&cpa_lock);
list_splice_init(&df_list, &l);
spin_unlock_irq(&cpa_lock);
flush_map();
list_for_each_entry_safe(pg, next, &l, lru)
__free_page(pg);
}
void isci_terminate_pending_requests(struct isci_host *ihost,
struct isci_remote_device *idev)
{
struct completion request_completion;
enum isci_request_status old_state;
unsigned long flags;
LIST_HEAD(list);
spin_lock_irqsave(&ihost->scic_lock, flags);
list_splice_init(&idev->reqs_in_process, &list);
while (!list_empty(&list)) {
struct isci_request *ireq = list_entry(list.next, typeof(*ireq), dev_node);
old_state = isci_request_change_started_to_newstate(ireq,
&request_completion,
terminating);
switch (old_state) {
case started:
case completed:
case aborting:
break;
default:
list_move(&ireq->dev_node, &idev->reqs_in_process);
ireq = NULL;
break;
}
if (!ireq)
continue;
spin_unlock_irqrestore(&ihost->scic_lock, flags);
init_completion(&request_completion);
dev_dbg(&ihost->pdev->dev,
"%s: idev=%p request=%p; task=%p old_state=%d\n",
__func__, idev, ireq,
(!test_bit(IREQ_TMF, &ireq->flags)
? isci_request_access_task(ireq)
: NULL),
old_state);
isci_terminate_request_core(ihost, idev, ireq);
spin_lock_irqsave(&ihost->scic_lock, flags);
}
spin_unlock_irqrestore(&ihost->scic_lock, flags);
}
/*
* Replay pending frontend request like orphan, etc. I.e. this starts
* to modify FS.
*/
int replay_stage3(struct replay *rp, int apply)
{
if(DEBUG_MODE_K==1)
{
printk(KERN_INFO"%25s %25s %4d #in\n",__FILE__,__func__,__LINE__);
}
struct sb *sb = rp->sb;
LIST_HEAD(orphan_in_otree);
list_splice_init(&rp->orphan_in_otree, &orphan_in_otree);
replay_done(rp);
/* Start logging after replay_done() */
replay_iput_orphan_inodes(sb, &orphan_in_otree, apply);
return 0;
}
static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self,
unsigned long action, void *hcpu)
{
/*
* If a CPU goes away, splice its entries to the current CPU
* and trigger a run of the softirq
*/
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
int cpu = (unsigned long) hcpu;
local_irq_disable();
list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
&__get_cpu_var(blk_cpu_iopoll));
__raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
local_irq_enable();
}
return NOTIFY_OK;
}
static int blk_cpu_notify(struct notifier_block *self, unsigned long action,
void *hcpu)
{
/*
* If a CPU goes away, splice its entries to the current CPU
* and trigger a run of the softirq
*/
if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
int cpu = (unsigned long) hcpu;
local_irq_disable();
list_splice_init(&per_cpu(blk_cpu_done, cpu),
this_cpu_ptr(&blk_cpu_done));
raise_softirq_irqoff(BLOCK_SOFTIRQ);
local_irq_enable();
}
return NOTIFY_OK;
}
static void __linkwatch_run_queue(int urgent_only)
{
struct net_device *dev;
LIST_HEAD(wrk);
/*
* Limit the number of linkwatch events to one
* per second so that a runaway driver does not
* cause a storm of messages on the netlink
* socket. This limit does not apply to up events
* while the device qdisc is down.
*/
if (!urgent_only)
linkwatch_nextevent = jiffies + HZ;
/* Limit wrap-around effect on delay. */
else if (time_after(linkwatch_nextevent, jiffies + HZ))
linkwatch_nextevent = jiffies;
clear_bit(LW_URGENT, &linkwatch_flags);
spin_lock_irq(&lweventlist_lock);
list_splice_init(&lweventlist, &wrk);
while (!list_empty(&wrk)) {
dev = list_first_entry(&wrk, struct net_device, link_watch_list);
list_del_init(&dev->link_watch_list);
if (urgent_only && !linkwatch_urgent_event(dev)) {
list_add_tail(&dev->link_watch_list, &lweventlist);
continue;
}
spin_unlock_irq(&lweventlist_lock);
linkwatch_do_dev(dev);
spin_lock_irq(&lweventlist_lock);
}
if (!list_empty(&lweventlist))
linkwatch_schedule_work(0);
spin_unlock_irq(&lweventlist_lock);
}
static void
rpc_timeout_upcall_queue(struct work_struct *work)
{
LIST_HEAD(free_list);
struct rpc_pipe *pipe =
container_of(work, struct rpc_pipe, queue_timeout.work);
void (*destroy_msg)(struct rpc_pipe_msg *);
struct dentry *dentry;
spin_lock(&pipe->lock);
destroy_msg = pipe->ops->destroy_msg;
if (pipe->nreaders == 0) {
list_splice_init(&pipe->pipe, &free_list);
pipe->pipelen = 0;
}
dentry = dget(pipe->dentry);
spin_unlock(&pipe->lock);
rpc_purge_list(dentry ? &RPC_I(dentry->d_inode)->waitq : NULL,
&free_list, destroy_msg, -ETIMEDOUT);
dput(dentry);
}
static void ipoib_cm_start_rx_drain(struct ipoib_dev_priv *priv)
{
struct ib_send_wr *bad_wr;
struct ipoib_cm_rx *p;
/* We only reserved 1 extra slot in CQ for drain WRs, so
* make sure we have at most 1 outstanding WR. */
if (list_empty(&priv->cm.rx_flush_list) ||
!list_empty(&priv->cm.rx_drain_list))
return;
/*
* QPs on flush list are error state. This way, a "flush
* error" WC will be immediately generated for each WR we post.
*/
p = list_entry(priv->cm.rx_flush_list.next, typeof(*p), list);
if (ib_post_send(p->qp, &ipoib_cm_rx_drain_wr, &bad_wr))
ipoib_warn(priv, "failed to post drain wr\n");
list_splice_init(&priv->cm.rx_flush_list, &priv->cm.rx_drain_list);
}
static void take_over_work(struct ehca_comp_pool *pool, int cpu)
{
struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
LIST_HEAD(list);
struct ehca_cq *cq;
unsigned long flags_cct;
spin_lock_irqsave(&cct->task_lock, flags_cct);
list_splice_init(&cct->cq_list, &list);
while (!list_empty(&list)) {
cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
list_del(&cq->entry);
__queue_comp_task(cq, this_cpu_ptr(pool->cpu_comp_tasks));
}
spin_unlock_irqrestore(&cct->task_lock, flags_cct);
}
static void
rpc_timeout_upcall_queue(struct work_struct *work)
{
LIST_HEAD(free_list);
struct rpc_inode *rpci =
container_of(work, struct rpc_inode, queue_timeout.work);
struct inode *inode = &rpci->vfs_inode;
void (*destroy_msg)(struct rpc_pipe_msg *);
spin_lock(&inode->i_lock);
if (rpci->ops == NULL) {
spin_unlock(&inode->i_lock);
return;
}
destroy_msg = rpci->ops->destroy_msg;
if (rpci->nreaders == 0) {
list_splice_init(&rpci->pipe, &free_list);
rpci->pipelen = 0;
}
spin_unlock(&inode->i_lock);
rpc_purge_list(rpci, &free_list, destroy_msg, -ETIMEDOUT);
}
/*
* Set up the argument/result storage required for the RPC call.
*/
static int nfs_commit_rpcsetup(struct list_head *head,
struct nfs_write_data *data,
int how)
{
struct nfs_page *first = nfs_list_entry(head->next);
struct inode *inode = first->wb_context->path.dentry->d_inode;
int flags = (how & FLUSH_SYNC) ? 0 : RPC_TASK_ASYNC;
int priority = flush_task_priority(how);
struct rpc_task *task;
struct rpc_message msg = {
.rpc_argp = &data->args,
.rpc_resp = &data->res,
.rpc_cred = first->wb_context->cred,
};
struct rpc_task_setup task_setup_data = {
.task = &data->task,
.rpc_client = NFS_CLIENT(inode),
.rpc_message = &msg,
.callback_ops = &nfs_commit_ops,
.callback_data = data,
.workqueue = nfsiod_workqueue,
.flags = flags,
.priority = priority,
};
/* Set up the RPC argument and reply structs
* NB: take care not to mess about with data->commit et al. */
list_splice_init(head, &data->pages);
data->inode = inode;
data->cred = msg.rpc_cred;
data->args.fh = NFS_FH(data->inode);
/* Note: we always request a commit of the entire inode */
data->args.offset = 0;
data->args.count = 0;
data->args.context = get_nfs_open_context(first->wb_context);
data->res.count = 0;
data->res.fattr = &data->fattr;
data->res.verf = &data->verf;
nfs_fattr_init(&data->fattr);
/* Set up the initial task struct. */
NFS_PROTO(inode)->commit_setup(data, &msg);
dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
task = rpc_run_task(&task_setup_data);
if (IS_ERR(task))
return PTR_ERR(task);
rpc_put_task(task);
return 0;
}
/*
* Commit dirty pages
*/
static int
nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
struct nfs_write_data *data;
struct nfs_page *req;
data = nfs_commitdata_alloc();
if (!data)
goto out_bad;
/* Set up the argument struct */
return nfs_commit_rpcsetup(head, data, how);
out_bad:
while (!list_empty(head)) {
req = nfs_list_entry(head->next);
nfs_list_remove_request(req);
nfs_mark_request_commit(req);
dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
dec_bdi_stat(req->wb_page->mapping->backing_dev_info,
BDI_RECLAIMABLE);
nfs_clear_page_tag_locked(req);
}
return -ENOMEM;
}
/*
* COMMIT call returned
*/
static void nfs_commit_done(struct rpc_task *task, void *calldata)
{
struct nfs_write_data *data = calldata;
dprintk("NFS: %5u nfs_commit_done (status %d)\n",
task->tk_pid, task->tk_status);
/* Call the NFS version-specific code */
if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
return;
}
static void nfs_commit_release(void *calldata)
{
struct nfs_write_data *data = calldata;
struct nfs_page *req;
int status = data->task.tk_status;
while (!list_empty(&data->pages)) {
req = nfs_list_entry(data->pages.next);
nfs_list_remove_request(req);
nfs_clear_request_commit(req);
dprintk("NFS: commit (%s/%lld %d@%lld)",
req->wb_context->path.dentry->d_inode->i_sb->s_id,
(long long)NFS_FILEID(req->wb_context->path.dentry->d_inode),
req->wb_bytes,
(long long)req_offset(req));
if (status < 0) {
nfs_context_set_write_error(req->wb_context, status);
nfs_inode_remove_request(req);
dprintk(", error = %d\n", status);
goto next;
}
/* Okay, COMMIT succeeded, apparently. Check the verifier
* returned by the server against all stored verfs. */
if (!memcmp(req->wb_verf.verifier, data->verf.verifier, sizeof(data->verf.verifier))) {
/* We have a match */
nfs_inode_remove_request(req);
dprintk(" OK\n");
goto next;
}
/* We have a mismatch. Write the page again */
dprintk(" mismatch\n");
nfs_mark_request_dirty(req);
next:
nfs_clear_page_tag_locked(req);
}
nfs_commitdata_release(calldata);
}
static const struct rpc_call_ops nfs_commit_ops = {
#if defined(CONFIG_NFS_V4_1)
.rpc_call_prepare = nfs_write_prepare,
#endif /* CONFIG_NFS_V4_1 */
.rpc_call_done = nfs_commit_done,
.rpc_release = nfs_commit_release,
};
int nfs_commit_inode(struct inode *inode, int how)
{
LIST_HEAD(head);
int res;
spin_lock(&inode->i_lock);
res = nfs_scan_commit(inode, &head, 0, 0);
spin_unlock(&inode->i_lock);
if (res) {
int error = nfs_commit_list(inode, &head, how);
if (error < 0)
return error;
}
return res;
}
#else
static inline int nfs_commit_list(struct inode *inode, struct list_head *head, int how)
{
return 0;
}
#endif
long nfs_sync_mapping_wait(struct address_space *mapping, struct writeback_control *wbc, int how)
{
struct inode *inode = mapping->host;
pgoff_t idx_start, idx_end;
unsigned int npages = 0;
LIST_HEAD(head);
int nocommit = how & FLUSH_NOCOMMIT;
long pages, ret;
/* FIXME */
if (wbc->range_cyclic)
idx_start = 0;
else {
idx_start = wbc->range_start >> PAGE_CACHE_SHIFT;
idx_end = wbc->range_end >> PAGE_CACHE_SHIFT;
if (idx_end > idx_start) {
pgoff_t l_npages = 1 + idx_end - idx_start;
npages = l_npages;
if (sizeof(npages) != sizeof(l_npages) &&
(pgoff_t)npages != l_npages)
npages = 0;
}
}
how &= ~FLUSH_NOCOMMIT;
spin_lock(&inode->i_lock);
do {
ret = nfs_wait_on_requests_locked(inode, idx_start, npages);
if (ret != 0)
continue;
if (nocommit)
break;
pages = nfs_scan_commit(inode, &head, idx_start, npages);
if (pages == 0)
break;
if (how & FLUSH_INVALIDATE) {
spin_unlock(&inode->i_lock);
nfs_cancel_commit_list(&head);
ret = pages;
spin_lock(&inode->i_lock);
continue;
}
pages += nfs_scan_commit(inode, &head, 0, 0);
spin_unlock(&inode->i_lock);
ret = nfs_commit_list(inode, &head, how);
spin_lock(&inode->i_lock);
} while (ret >= 0);
spin_unlock(&inode->i_lock);
return ret;
}
static int __nfs_write_mapping(struct address_space *mapping, struct writeback_control *wbc, int how)
{
int ret;
ret = nfs_writepages(mapping, wbc);
if (ret < 0)
goto out;
ret = nfs_sync_mapping_wait(mapping, wbc, how);
if (ret < 0)
goto out;
return 0;
out:
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
return ret;
}
/**
* isci_terminate_pending_requests() - This function will change the all of the
* requests on the given device's state to "aborting", will terminate the
* requests, and wait for them to complete. This function must only be
* called from a thread that can wait. Note that the requests are all
* terminated and completed (back to the host, if started there).
* @isci_host: This parameter specifies SCU.
* @idev: This parameter specifies the target.
*
*/
void isci_terminate_pending_requests(struct isci_host *ihost,
struct isci_remote_device *idev)
{
struct completion request_completion;
enum isci_request_status old_state;
unsigned long flags;
LIST_HEAD(list);
spin_lock_irqsave(&ihost->scic_lock, flags);
list_splice_init(&idev->reqs_in_process, &list);
/* assumes that isci_terminate_request_core deletes from the list */
while (!list_empty(&list)) {
struct isci_request *ireq = list_entry(list.next, typeof(*ireq), dev_node);
/* Change state to "terminating" if it is currently
* "started".
*/
old_state = isci_request_change_started_to_newstate(ireq,
&request_completion,
terminating);
switch (old_state) {
case started:
case completed:
case aborting:
break;
default:
/* termination in progress, or otherwise dispositioned.
* We know the request was on 'list' so should be safe
* to move it back to reqs_in_process
*/
list_move(&ireq->dev_node, &idev->reqs_in_process);
ireq = NULL;
break;
}
if (!ireq)
continue;
spin_unlock_irqrestore(&ihost->scic_lock, flags);
init_completion(&request_completion);
dev_dbg(&ihost->pdev->dev,
"%s: idev=%p request=%p; task=%p old_state=%d\n",
__func__, idev, ireq,
(!test_bit(IREQ_TMF, &ireq->flags)
? isci_request_access_task(ireq)
: NULL),
old_state);
/* If the old_state is started:
* This request was not already being aborted. If it had been,
* then the aborting I/O (ie. the TMF request) would not be in
* the aborting state, and thus would be terminated here. Note
* that since the TMF completion's call to the kernel function
* "complete()" does not happen until the pending I/O request
* terminate fully completes, we do not have to implement a
* special wait here for already aborting requests - the
* termination of the TMF request will force the request
* to finish it's already started terminate.
*
* If old_state == completed:
* This request completed from the SCU hardware perspective
* and now just needs cleaning up in terms of freeing the
* request and potentially calling up to libsas.
*
* If old_state == aborting:
* This request has already gone through a TMF timeout, but may
* not have been terminated; needs cleaning up at least.
*/
isci_terminate_request_core(ihost, idev, ireq);
spin_lock_irqsave(&ihost->scic_lock, flags);
}
spin_unlock_irqrestore(&ihost->scic_lock, flags);
}
static void vsp1_dl_list_free(struct vsp1_dl_list *dl)
{
vsp1_dl_body_cleanup(&dl->body0);
list_splice_init(&dl->fragments, &dl->dlm->gc_fragments);
kfree(dl);
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
size_t buf_len, size_t period_len,
enum dma_transfer_direction direction,
unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *first = NULL, *prev = NULL;
unsigned int periods = buf_len / period_len;
int i;
unsigned long irqflags;
dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
__func__, &buf_addr, buf_len, period_len,
direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
if (!is_slave_direction(direction)) {
dev_err(chan2dev(chan), "invalid DMA direction\n");
return NULL;
}
if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
dev_err(chan2dev(chan), "channel currently used\n");
return NULL;
}
if (at_xdmac_compute_chan_conf(chan, direction))
return NULL;
for (i = 0; i < periods; i++) {
struct at_xdmac_desc *desc = NULL;
spin_lock_irqsave(&atchan->lock, irqflags);
desc = at_xdmac_get_desc(atchan);
if (!desc) {
dev_err(chan2dev(chan), "can't get descriptor\n");
if (first)
list_splice_init(&first->descs_list, &atchan->free_descs_list);
spin_unlock_irqrestore(&atchan->lock, irqflags);
return NULL;
}
spin_unlock_irqrestore(&atchan->lock, irqflags);
dev_dbg(chan2dev(chan),
"%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
__func__, desc, &desc->tx_dma_desc.phys);
if (direction == DMA_DEV_TO_MEM) {
desc->lld.mbr_sa = atchan->sconfig.src_addr;
desc->lld.mbr_da = buf_addr + i * period_len;
} else {
desc->lld.mbr_sa = buf_addr + i * period_len;
desc->lld.mbr_da = atchan->sconfig.dst_addr;
}
desc->lld.mbr_cfg = atchan->cfg;
desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1
| AT_XDMAC_MBR_UBC_NDEN
| AT_XDMAC_MBR_UBC_NSEN
| period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
dev_dbg(chan2dev(chan),
"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
/* Chain lld. */
if (prev)
at_xdmac_queue_desc(chan, prev, desc);
prev = desc;
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
}
prev->lld.mbr_nda = first->tx_dma_desc.phys;
dev_dbg(chan2dev(chan),
"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
__func__, prev, &prev->lld.mbr_nda);
first->tx_dma_desc.flags = flags;
first->xfer_size = buf_len;
first->direction = direction;
return &first->tx_dma_desc;
}
static struct dma_async_tx_descriptor *
at_xdmac_prep_interleaved(struct dma_chan *chan,
struct dma_interleaved_template *xt,
unsigned long flags)
{
struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
struct at_xdmac_desc *prev = NULL, *first = NULL;
struct data_chunk *chunk, *prev_chunk = NULL;
dma_addr_t dst_addr, src_addr;
size_t dst_skip, src_skip, len = 0;
size_t prev_dst_icg = 0, prev_src_icg = 0;
int i;
if (!xt || (xt->numf != 1) || (xt->dir != DMA_MEM_TO_MEM))
return NULL;
dev_dbg(chan2dev(chan), "%s: src=0x%08x, dest=0x%08x, numf=%d, frame_size=%d, flags=0x%lx\n",
__func__, xt->src_start, xt->dst_start, xt->numf,
xt->frame_size, flags);
src_addr = xt->src_start;
dst_addr = xt->dst_start;
for (i = 0; i < xt->frame_size; i++) {
struct at_xdmac_desc *desc;
size_t src_icg, dst_icg;
chunk = xt->sgl + i;
dst_icg = dmaengine_get_dst_icg(xt, chunk);
src_icg = dmaengine_get_src_icg(xt, chunk);
src_skip = chunk->size + src_icg;
dst_skip = chunk->size + dst_icg;
dev_dbg(chan2dev(chan),
"%s: chunk size=%d, src icg=%d, dst icg=%d\n",
__func__, chunk->size, src_icg, dst_icg);
/*
* Handle the case where we just have the same
* transfer to setup, we can just increase the
* block number and reuse the same descriptor.
*/
if (prev_chunk && prev &&
(prev_chunk->size == chunk->size) &&
(prev_src_icg == src_icg) &&
(prev_dst_icg == dst_icg)) {
dev_dbg(chan2dev(chan),
"%s: same configuration that the previous chunk, merging the descriptors...\n",
__func__);
at_xdmac_increment_block_count(chan, prev);
continue;
}
desc = at_xdmac_interleaved_queue_desc(chan, atchan,
prev,
src_addr, dst_addr,
xt, chunk);
if (!desc) {
list_splice_init(&first->descs_list,
&atchan->free_descs_list);
return NULL;
}
if (!first)
first = desc;
dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
__func__, desc, first);
list_add_tail(&desc->desc_node, &first->descs_list);
if (xt->src_sgl)
src_addr += src_skip;
if (xt->dst_sgl)
dst_addr += dst_skip;
len += chunk->size;
prev_chunk = chunk;
prev_dst_icg = dst_icg;
prev_src_icg = src_icg;
prev = desc;
}
first->tx_dma_desc.cookie = -EBUSY;
first->tx_dma_desc.flags = flags;
first->xfer_size = len;
return &first->tx_dma_desc;
}
static void free_fixup_list(struct list_head *fixuplist)
{
GCLOCK(&g_fixuplock);
list_splice_init(fixuplist, &g_fixupvac);
GCUNLOCK(&g_fixuplock);
}
/*
* this adds all existing backrefs (inline backrefs, backrefs and delayed
* refs) for the given bytenr to the refs list, merges duplicates and resolves
* indirect refs to their parent bytenr.
* When roots are found, they're added to the roots list
*
* FIXME some caching might speed things up
*/
static int find_parent_nodes(struct btrfs_trans_handle *trans,
struct btrfs_fs_info *fs_info, u64 bytenr,
u64 time_seq, struct ulist *refs,
struct ulist *roots, const u64 *extent_item_pos)
{
struct btrfs_key key;
struct btrfs_path *path;
struct btrfs_delayed_ref_root *delayed_refs = NULL;
struct btrfs_delayed_ref_head *head;
int info_level = 0;
int ret;
struct list_head prefs_delayed;
struct list_head prefs;
struct __prelim_ref *ref;
INIT_LIST_HEAD(&prefs);
INIT_LIST_HEAD(&prefs_delayed);
key.objectid = bytenr;
key.offset = (u64)-1;
if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
key.type = BTRFS_METADATA_ITEM_KEY;
else
key.type = BTRFS_EXTENT_ITEM_KEY;
path = btrfs_alloc_path();
if (!path)
return -ENOMEM;
if (!trans)
path->search_commit_root = 1;
/*
* grab both a lock on the path and a lock on the delayed ref head.
* We need both to get a consistent picture of how the refs look
* at a specified point in time
*/
again:
head = NULL;
ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 0);
if (ret < 0)
goto out;
BUG_ON(ret == 0);
if (trans) {
/*
* look if there are updates for this ref queued and lock the
* head
*/
delayed_refs = &trans->transaction->delayed_refs;
spin_lock(&delayed_refs->lock);
head = btrfs_find_delayed_ref_head(trans, bytenr);
if (head) {
if (!mutex_trylock(&head->mutex)) {
atomic_inc(&head->node.refs);
spin_unlock(&delayed_refs->lock);
btrfs_release_path(path);
/*
* Mutex was contended, block until it's
* released and try again
*/
mutex_lock(&head->mutex);
mutex_unlock(&head->mutex);
btrfs_put_delayed_ref(&head->node);
goto again;
}
ret = __add_delayed_refs(head, time_seq,
&prefs_delayed);
mutex_unlock(&head->mutex);
if (ret) {
spin_unlock(&delayed_refs->lock);
goto out;
}
}
spin_unlock(&delayed_refs->lock);
}
if (path->slots[0]) {
struct extent_buffer *leaf;
int slot;
path->slots[0]--;
leaf = path->nodes[0];
slot = path->slots[0];
btrfs_item_key_to_cpu(leaf, &key, slot);
if (key.objectid == bytenr &&
(key.type == BTRFS_EXTENT_ITEM_KEY ||
key.type == BTRFS_METADATA_ITEM_KEY)) {
ret = __add_inline_refs(fs_info, path, bytenr,
&info_level, &prefs);
if (ret)
goto out;
ret = __add_keyed_refs(fs_info, path, bytenr,
info_level, &prefs);
if (ret)
goto out;
}
}
btrfs_release_path(path);
list_splice_init(&prefs_delayed, &prefs);
ret = __add_missing_keys(fs_info, &prefs);
if (ret)
goto out;
__merge_refs(&prefs, 1);
ret = __resolve_indirect_refs(fs_info, path, time_seq, &prefs,
extent_item_pos);
if (ret)
goto out;
__merge_refs(&prefs, 2);
while (!list_empty(&prefs)) {
ref = list_first_entry(&prefs, struct __prelim_ref, list);
list_del(&ref->list);
WARN_ON(ref->count < 0);
if (ref->count && ref->root_id && ref->parent == 0) {
/* no parent == root of tree */
ret = ulist_add(roots, ref->root_id, 0, GFP_NOFS);
if (ret < 0)
goto out;
}
if (ref->count && ref->parent) {
struct extent_inode_elem *eie = NULL;
if (extent_item_pos && !ref->inode_list) {
u32 bsz;
struct extent_buffer *eb;
bsz = btrfs_level_size(fs_info->extent_root,
info_level);
eb = read_tree_block(fs_info->extent_root,
ref->parent, bsz, 0);
if (!eb || !extent_buffer_uptodate(eb)) {
free_extent_buffer(eb);
ret = -EIO;
goto out;
}
ret = find_extent_in_eb(eb, bytenr,
*extent_item_pos, &eie);
ref->inode_list = eie;
free_extent_buffer(eb);
}
ret = ulist_add_merge(refs, ref->parent,
(uintptr_t)ref->inode_list,
(u64 *)&eie, GFP_NOFS);
if (ret < 0)
goto out;
if (!ret && extent_item_pos) {
/*
* we've recorded that parent, so we must extend
* its inode list here
*/
BUG_ON(!eie);
while (eie->next)
eie = eie->next;
eie->next = ref->inode_list;
}
}
kfree(ref);
}
out:
btrfs_free_path(path);
while (!list_empty(&prefs)) {
ref = list_first_entry(&prefs, struct __prelim_ref, list);
list_del(&ref->list);
kfree(ref);
}
while (!list_empty(&prefs_delayed)) {
ref = list_first_entry(&prefs_delayed, struct __prelim_ref,
list);
list_del(&ref->list);
kfree(ref);
}
return ret;
}
static void free_schedunmap_list(struct list_head *schedunmaplist)
{
GCLOCK(&g_unmaplock);
list_splice_init(schedunmaplist, &g_unmapvac);
GCUNLOCK(&g_unmaplock);
}
