static noinline ssize_t xfs_file_dax_write( struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = iocb->ki_filp->f_mapping->host; struct xfs_inode *ip = XFS_I(inode); int iolock = XFS_IOLOCK_EXCL; ssize_t ret, error = 0; size_t count; loff_t pos; xfs_ilock(ip, iolock); ret = xfs_file_aio_write_checks(iocb, from, &iolock); if (ret) goto out; pos = iocb->ki_pos; count = iov_iter_count(from); trace_xfs_file_dax_write(ip, count, pos); ret = dax_iomap_rw(iocb, from, &xfs_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); error = xfs_setfilesize(ip, pos, ret); } out: xfs_iunlock(ip, iolock); return error ? error : ret; }
/* * IO write completion. */ STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { ioend->io_error = -EIO; goto done; } if (ioend->io_error) goto done; /* * For unwritten extents we need to issue transactions to convert a * range to normal written extens after the data I/O has finished. */ if (ioend->io_type == XFS_IO_UNWRITTEN) { error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); } else if (ioend->io_isdirect && xfs_ioend_is_append(ioend)) { /* * For direct I/O we do not know if we need to allocate blocks * or not so we can't preallocate an append transaction as that * results in nested reservations and log space deadlocks. Hence * allocate the transaction here. While this is sub-optimal and * can block IO completion for some time, we're stuck with doing * it this way until we can pass the ioend to the direct IO * allocation callbacks and avoid nesting that way. */ error = xfs_setfilesize_trans_alloc(ioend); if (error) goto done; error = xfs_setfilesize(ioend); } else if (ioend->io_append_trans) { error = xfs_setfilesize(ioend); } else { ASSERT(!xfs_ioend_is_append(ioend)); } done: if (error) ioend->io_error = -error; xfs_destroy_ioend(ioend); }
static int xfs_dio_write_end_io( struct kiocb *iocb, ssize_t size, unsigned flags) { struct inode *inode = file_inode(iocb->ki_filp); struct xfs_inode *ip = XFS_I(inode); loff_t offset = iocb->ki_pos; int error = 0; trace_xfs_end_io_direct_write(ip, offset, size); if (XFS_FORCED_SHUTDOWN(ip->i_mount)) return -EIO; if (size <= 0) return size; if (flags & IOMAP_DIO_COW) { error = xfs_reflink_end_cow(ip, offset, size); if (error) return error; } /* * Unwritten conversion updates the in-core isize after extent * conversion but before updating the on-disk size. Updating isize any * earlier allows a racing dio read to find unwritten extents before * they are converted. */ if (flags & IOMAP_DIO_UNWRITTEN) return xfs_iomap_write_unwritten(ip, offset, size, true); /* * We need to update the in-core inode size here so that we don't end up * with the on-disk inode size being outside the in-core inode size. We * have no other method of updating EOF for AIO, so always do it here * if necessary. * * We need to lock the test/set EOF update as we can be racing with * other IO completions here to update the EOF. Failing to serialise * here can result in EOF moving backwards and Bad Things Happen when * that occurs. */ spin_lock(&ip->i_flags_lock); if (offset + size > i_size_read(inode)) { i_size_write(inode, offset + size); spin_unlock(&ip->i_flags_lock); error = xfs_setfilesize(ip, offset, size); } else { spin_unlock(&ip->i_flags_lock); } return error; }
STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { ioend->io_error = -EIO; goto done; } if (ioend->io_error) goto done; /* * For unwritten extents we need to issue transactions to convert a * range to normal written extens after the data I/O has finished. */ if (ioend->io_type == IO_UNWRITTEN) { /* * For buffered I/O we never preallocate a transaction when * doing the unwritten extent conversion, but for direct I/O * we do not know if we are converting an unwritten extent * or not at the point where we preallocate the transaction. */ if (ioend->io_append_trans) { ASSERT(ioend->io_isdirect); current_set_flags_nested( &ioend->io_append_trans->t_pflags, PF_FSTRANS); xfs_trans_cancel(ioend->io_append_trans, 0); } error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); if (error) { ioend->io_error = -error; goto done; } } else if (ioend->io_append_trans) { error = xfs_setfilesize(ioend); if (error) ioend->io_error = -error; } else { ASSERT(!xfs_ioend_is_append(ioend)); } done: xfs_destroy_ioend(ioend); }
/* * IO write completion. */ STATIC void xfs_end_io( struct work_struct *work) { xfs_ioend_t *ioend = container_of(work, xfs_ioend_t, io_work); struct xfs_inode *ip = XFS_I(ioend->io_inode); int error = 0; if (XFS_FORCED_SHUTDOWN(ip->i_mount)) { error = -EIO; goto done; } if (ioend->io_error) goto done; /* * For unwritten extents we need to issue transactions to convert a * range to normal written extens after the data I/O has finished. */ if (ioend->io_type == IO_UNWRITTEN) { error = xfs_iomap_write_unwritten(ip, ioend->io_offset, ioend->io_size); if (error) { ioend->io_error = -error; goto done; } } /* * We might have to update the on-disk file size after extending * writes. */ error = xfs_setfilesize(ioend); ASSERT(!error || error == EAGAIN); done: /* * If we didn't complete processing of the ioend, requeue it to the * tail of the workqueue for another attempt later. Otherwise destroy * it. */ if (error == EAGAIN) { atomic_inc(&ioend->io_remaining); xfs_finish_ioend(ioend); /* ensure we don't spin on blocked ioends */ delay(1); } else { xfs_destroy_ioend(ioend); } }
static noinline ssize_t xfs_file_dax_write( struct kiocb *iocb, struct iov_iter *from) { struct inode *inode = iocb->ki_filp->f_mapping->host; struct xfs_inode *ip = XFS_I(inode); int iolock = XFS_IOLOCK_EXCL; ssize_t ret, error = 0; size_t count; loff_t pos; if (iocb->ki_flags & IOCB_NOWAIT) { if (!xfs_ilock_nowait(ip, iolock)) return -EAGAIN; } else { xfs_ilock(ip, iolock); } ret = xfs_file_aio_write_checks(iocb, from, &iolock); if (ret) goto out; pos = iocb->ki_pos; count = iov_iter_count(from); trace_xfs_file_dax_write(ip, count, pos); ret = dax_iomap_rw(iocb, from, &xfs_iomap_ops); if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { i_size_write(inode, iocb->ki_pos); error = xfs_setfilesize(ip, pos, ret); } out: xfs_iunlock(ip, iolock); if (error) return error; if (ret > 0) { XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret); /* Handle various SYNC-type writes */ ret = generic_write_sync(iocb, ret); } return ret; }
STATIC int xfs_setfilesize_ioend( struct xfs_ioend *ioend) { struct xfs_inode *ip = XFS_I(ioend->io_inode); struct xfs_trans *tp = ioend->io_append_trans; /* * The transaction may have been allocated in the I/O submission thread, * thus we need to mark ourselves as being in a transaction manually. * Similarly for freeze protection. */ current_set_flags_nested(&tp->t_pflags, PF_FSTRANS); rwsem_acquire_read(&VFS_I(ip)->i_sb->s_writers.lock_map[SB_FREEZE_FS-1], 0, 1, _THIS_IP_); return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size); }
STATIC int xfs_setfilesize_ioend( struct xfs_ioend *ioend) { struct xfs_inode *ip = XFS_I(ioend->io_inode); struct xfs_trans *tp = ioend->io_append_trans; /* * The transaction may have been allocated in the I/O submission thread, * thus we need to mark ourselves as being in a transaction manually. * Similarly for freeze protection. */ current_set_flags_nested(&tp->t_pflags, PF_FSTRANS); __sb_writers_acquired(VFS_I(ip)->i_sb, SB_FREEZE_FS); /* we abort the update if there was an IO error */ if (ioend->io_error) { xfs_trans_cancel(tp); return ioend->io_error; } return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size); }