/* * check a range of space and convert unwritten extents to written. */ int ext4_end_io_nolock(ext4_io_end_t *io) { struct inode *inode = io->inode; loff_t offset = io->offset; ssize_t size = io->size; int ret = 0; ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," "list->prev 0x%p\n", io, inode->i_ino, io->list.next, io->list.prev); if (list_empty(&io->list)) return ret; if (!(io->flag & EXT4_IO_END_UNWRITTEN)) return ret; ret = ext4_convert_unwritten_extents(inode, offset, size); if (ret < 0) { printk(KERN_EMERG "%s: failed to convert unwritten " "extents to written extents, error is %d " "io is still on inode %lu aio dio list\n", __func__, ret, inode->i_ino); return ret; } if (io->iocb) aio_complete(io->iocb, io->result, 0); /* clear the DIO AIO unwritten flag */ io->flag &= ~EXT4_IO_END_UNWRITTEN; return ret; }
/* * check a range of space and convert unwritten extents to written. * * Called with inode->i_mutex; we depend on this when we manipulate * io->flag, since we could otherwise race with ext4_flush_completed_IO() */ int ext4_end_io_nolock(ext4_io_end_t *io) { struct inode *inode = io->inode; loff_t offset = io->offset; ssize_t size = io->size; int ret = 0; ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," "list->prev 0x%p\n", io, inode->i_ino, io->list.next, io->list.prev); ret = ext4_convert_unwritten_extents(inode, offset, size); if (ret < 0) { ext4_msg(inode->i_sb, KERN_EMERG, "failed to convert unwritten extents to written " "extents -- potential data loss! " "(inode %lu, offset %llu, size %zd, error %d)", inode->i_ino, offset, size, ret); } /* Wake up anyone waiting on unwritten extent conversion */ if (atomic_dec_and_test(&EXT4_I(inode)->i_aiodio_unwritten)) wake_up_all(ext4_ioend_wq(io->inode)); if (io->flag & EXT4_IO_END_DIRECT) inode_dio_done(inode); if (io->iocb) aio_complete(io->iocb, io->result, 0); return ret; }
static void soo_aio_cancel(struct kaiocb *job) { struct socket *so; struct sockbuf *sb; long done; int opcode; so = job->fd_file->f_data; opcode = job->uaiocb.aio_lio_opcode; if (opcode == LIO_READ) sb = &so->so_rcv; else { MPASS(opcode == LIO_WRITE); sb = &so->so_snd; } SOCKBUF_LOCK(sb); if (!aio_cancel_cleared(job)) TAILQ_REMOVE(&sb->sb_aiojobq, job, list); if (TAILQ_EMPTY(&sb->sb_aiojobq)) sb->sb_flags &= ~SB_AIO; SOCKBUF_UNLOCK(sb); done = job->aio_done; if (done != 0) aio_complete(job, done, 0); else aio_cancel(job); }
int async_poll(struct kiocb *iocb, int events) { unsigned int mask; async_poll_table *pasync; poll_table *p; /* Fast path */ if (iocb->filp->f_op && iocb->filp->f_op->poll) { mask = iocb->filp->f_op->poll(iocb->filp, NULL); mask &= events | POLLERR | POLLHUP; if (mask & events) return mask; } pasync = kmem_cache_alloc(async_poll_table_cache, SLAB_KERNEL); if (!pasync) return -ENOMEM; p = (poll_table *)pasync; poll_initwait(p); wtd_set_action(&pasync->wtd, async_poll_complete, pasync); p->iocb = iocb; pasync->wake = 0; pasync->sync = 0; pasync->events = events; pasync->pt_page.entry = pasync->pt_page.entries; pasync->pt_page.size = sizeof(pasync->pt_page); p->table = &pasync->pt_page; iocb->data = p; iocb->users ++; wmb(); mask = DEFAULT_POLLMASK; if (iocb->filp->f_op && iocb->filp->f_op->poll) mask = iocb->filp->f_op->poll(iocb->filp, p); mask &= events | POLLERR | POLLHUP; if (mask && xchg(&iocb->data, NULL)) { poll_freewait(p); aio_complete(iocb, mask, 0); } iocb->cancel = async_poll_cancel; aio_put_req(iocb); return 0; }
static void async_poll_waiter(wait_queue_t *wait) { struct poll_table_entry *entry = (struct poll_table_entry *)wait; async_poll_table *pasync = (async_poll_table *)(entry->p); struct kiocb *iocb = pasync->pt.iocb; unsigned int mask; mask = iocb->filp->f_op->poll(iocb->filp, NULL); mask &= pasync->events | POLLERR | POLLHUP; if (mask) { poll_table *p2 = xchg(&iocb->data, NULL); if (p2) { __poll_freewait(p2, wait); aio_complete(iocb, mask, 0); } return; } }
/* * IO write completion. */ STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; /* * For unwritten extents we need to issue transactions to convert a * range to normal written extens after the data I/O has finished. */ if (ioend->io_type == IO_UNWRITTEN && likely(!ioend->io_error && !XFS_FORCED_SHUTDOWN(ip->i_mount))) { error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); if (error) ioend->io_error = error; } /* * We might have to update the on-disk file size after extending * writes. */ error = xfs_setfilesize(ioend); ASSERT(!error || error == EAGAIN); /* * If we didn't complete processing of the ioend, requeue it to the * tail of the workqueue for another attempt later. Otherwise destroy * it. */ if (error == EAGAIN) { atomic_inc(&ioend->io_remaining); xfs_finish_ioend(ioend); /* ensure we don't spin on blocked ioends */ delay(1); } else { if (ioend->io_iocb) aio_complete(ioend->io_iocb, ioend->io_result, 0); xfs_destroy_ioend(ioend); } }
/* * check a range of space and convert unwritten extents to written. */ int ext4_end_io_nolock(ext4_io_end_t *io) { struct inode *inode = io->inode; loff_t offset = io->offset; ssize_t size = io->size; wait_queue_head_t *wq; int ret = 0; ext4_debug("ext4_end_io_nolock: io 0x%p from inode %lu,list->next 0x%p," "list->prev 0x%p\n", io, inode->i_ino, io->list.next, io->list.prev); if (list_empty(&io->list)) return ret; if (!(io->flag & EXT4_IO_END_UNWRITTEN)) return ret; ret = ext4_convert_unwritten_extents(inode, offset, size); if (ret < 0) { printk(KERN_EMERG "%s: failed to convert unwritten " "extents to written extents, error is %d " "io is still on inode %lu aio dio list\n", __func__, ret, inode->i_ino); return ret; } if (io->iocb) aio_complete(io->iocb, io->result, 0); /* clear the DIO AIO unwritten flag */ if (io->flag & EXT4_IO_END_UNWRITTEN) { io->flag &= ~EXT4_IO_END_UNWRITTEN; /* Wake up anyone waiting on unwritten extent conversion */ wq = ext4_ioend_wq(io->inode); if (atomic_dec_and_test(&EXT4_I(inode)->i_aiodio_unwritten) && waitqueue_active(wq)) { wake_up_all(wq); } } return ret; }
STATIC void xfs_destroy_ioend( xfs_ioend_t *ioend) { struct buffer_head *bh, *next; for (bh = ioend->io_buffer_head; bh; bh = next) { next = bh->b_private; bh->b_end_io(bh, !ioend->io_error); } if (ioend->io_iocb) { if (ioend->io_isasync) { aio_complete(ioend->io_iocb, ioend->io_error ? ioend->io_error : ioend->io_result, 0); } inode_dio_done(ioend->io_inode); } mempool_free(ioend, xfs_ioend_pool); }
static void ext4_release_io_end(ext4_io_end_t *io_end) { struct bio *bio, *next_bio; BUG_ON(!list_empty(&io_end->list)); BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); WARN_ON(io_end->handle); if (atomic_dec_and_test(&EXT4_I(io_end->inode)->i_ioend_count)) wake_up_all(ext4_ioend_wq(io_end->inode)); for (bio = io_end->bio; bio; bio = next_bio) { next_bio = bio->bi_private; ext4_finish_bio(bio); bio_put(bio); } if (io_end->flag & EXT4_IO_END_DIRECT) inode_dio_done(io_end->inode); if (io_end->iocb) aio_complete(io_end->iocb, io_end->result, 0); kmem_cache_free(io_end_cachep, io_end); }
void async_poll_complete(void *data) { async_poll_table *pasync = data; poll_table *p = data; struct kiocb *iocb = p->iocb; unsigned int mask; pasync->wake = 0; wmb(); do { mask = iocb->filp->f_op->poll(iocb->filp, p); mask &= pasync->events | POLLERR | POLLHUP; if (mask) { poll_table *p2 = xchg(&iocb->data, NULL); if (p2) { poll_freewait(p2); aio_complete(iocb, mask, 0); } return; } pasync->sync = 0; wmb(); } while (pasync->wake); }
/* * "Complete" an asynchronous operation. */ static void char_device_do_deferred_op(void *p) { struct async_work *stuff = (struct async_work *) p; aio_complete(stuff->iocb, stuff->result, 0); kfree(stuff); }
static void soaio_process_job(struct socket *so, struct sockbuf *sb, struct kaiocb *job) { struct ucred *td_savedcred; struct thread *td; struct file *fp; struct uio uio; struct iovec iov; size_t cnt; int error, flags; SOCKBUF_UNLOCK(sb); aio_switch_vmspace(job); td = curthread; fp = job->fd_file; retry: td_savedcred = td->td_ucred; td->td_ucred = job->cred; cnt = job->uaiocb.aio_nbytes; iov.iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf; iov.iov_len = cnt; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = cnt; uio.uio_segflg = UIO_USERSPACE; uio.uio_td = td; flags = MSG_NBIO; /* TODO: Charge ru_msg* to job. */ if (sb == &so->so_rcv) { uio.uio_rw = UIO_READ; #ifdef MAC error = mac_socket_check_receive(fp->f_cred, so); if (error == 0) #endif error = soreceive(so, NULL, &uio, NULL, NULL, &flags); } else { uio.uio_rw = UIO_WRITE; #ifdef MAC error = mac_socket_check_send(fp->f_cred, so); if (error == 0) #endif error = sosend(so, NULL, &uio, NULL, NULL, flags, td); if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0) { PROC_LOCK(job->userproc); kern_psignal(job->userproc, SIGPIPE); PROC_UNLOCK(job->userproc); } } cnt -= uio.uio_resid; td->td_ucred = td_savedcred; /* XXX: Not sure if this is needed? */ if (cnt != 0 && (error == ERESTART || error == EINTR || error == EWOULDBLOCK)) error = 0; if (error == EWOULDBLOCK) { /* * A read() or write() on the socket raced with this * request. If the socket is now ready, try again. * If it is not, place this request at the head of the * queue to try again when the socket is ready. */ SOCKBUF_LOCK(sb); empty_results++; if (soaio_ready(so, sb)) { empty_retries++; SOCKBUF_UNLOCK(sb); goto retry; } if (!aio_set_cancel_function(job, soo_aio_cancel)) { MPASS(cnt == 0); SOCKBUF_UNLOCK(sb); aio_cancel(job); SOCKBUF_LOCK(sb); } else { TAILQ_INSERT_HEAD(&sb->sb_aiojobq, job, list); } } else { aio_complete(job, cnt, error); SOCKBUF_LOCK(sb); } }
/* * "Complete" an asynchronous operation. */ static void scullc_do_deferred_op(struct work_struct *work) { struct async_work *stuff = (struct async_work *) work; aio_complete(stuff->iocb, stuff->result, 0); kfree(stuff); }
static void scullc_do_deferred_op(struct work_struct *work) { struct async_work * stuff = container_of(work, struct async_work, dwork.work); /*Long*/ aio_complete(stuff->iocb, stuff->result, 0); kfree(stuff); }
static void soaio_process_job(struct socket *so, struct sockbuf *sb, struct kaiocb *job) { struct ucred *td_savedcred; struct thread *td; struct file *fp; struct uio uio; struct iovec iov; size_t cnt, done; long ru_before; int error, flags; SOCKBUF_UNLOCK(sb); aio_switch_vmspace(job); td = curthread; fp = job->fd_file; retry: td_savedcred = td->td_ucred; td->td_ucred = job->cred; done = job->aio_done; cnt = job->uaiocb.aio_nbytes - done; iov.iov_base = (void *)((uintptr_t)job->uaiocb.aio_buf + done); iov.iov_len = cnt; uio.uio_iov = &iov; uio.uio_iovcnt = 1; uio.uio_offset = 0; uio.uio_resid = cnt; uio.uio_segflg = UIO_USERSPACE; uio.uio_td = td; flags = MSG_NBIO; /* * For resource usage accounting, only count a completed request * as a single message to avoid counting multiple calls to * sosend/soreceive on a blocking socket. */ if (sb == &so->so_rcv) { uio.uio_rw = UIO_READ; ru_before = td->td_ru.ru_msgrcv; #ifdef MAC error = mac_socket_check_receive(fp->f_cred, so); if (error == 0) #endif error = soreceive(so, NULL, &uio, NULL, NULL, &flags); if (td->td_ru.ru_msgrcv != ru_before) job->msgrcv = 1; } else { uio.uio_rw = UIO_WRITE; ru_before = td->td_ru.ru_msgsnd; #ifdef MAC error = mac_socket_check_send(fp->f_cred, so); if (error == 0) #endif error = sosend(so, NULL, &uio, NULL, NULL, flags, td); if (td->td_ru.ru_msgsnd != ru_before) job->msgsnd = 1; if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0) { PROC_LOCK(job->userproc); kern_psignal(job->userproc, SIGPIPE); PROC_UNLOCK(job->userproc); } } done += cnt - uio.uio_resid; job->aio_done = done; td->td_ucred = td_savedcred; if (error == EWOULDBLOCK) { /* * The request was either partially completed or not * completed at all due to racing with a read() or * write() on the socket. If the socket is * non-blocking, return with any partial completion. * If the socket is blocking or if no progress has * been made, requeue this request at the head of the * queue to try again when the socket is ready. */ MPASS(done != job->uaiocb.aio_nbytes); SOCKBUF_LOCK(sb); if (done == 0 || !(so->so_state & SS_NBIO)) { empty_results++; if (soaio_ready(so, sb)) { empty_retries++; SOCKBUF_UNLOCK(sb); goto retry; } if (!aio_set_cancel_function(job, soo_aio_cancel)) { SOCKBUF_UNLOCK(sb); if (done != 0) aio_complete(job, done, 0); else aio_cancel(job); SOCKBUF_LOCK(sb); } else { TAILQ_INSERT_HEAD(&sb->sb_aiojobq, job, list); } return; } SOCKBUF_UNLOCK(sb); } if (done != 0 && (error == ERESTART || error == EINTR || error == EWOULDBLOCK)) error = 0; if (error) aio_complete(job, -1, error); else aio_complete(job, done, 0); SOCKBUF_LOCK(sb); }