/*
* Find the CoW reservation for a given byte offset of a file.
*/
bool
xfs_reflink_find_cow_mapping(
struct xfs_inode *ip,
xfs_off_t offset,
struct xfs_bmbt_irec *imap)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
xfs_fileoff_t offset_fsb;
struct xfs_bmbt_irec got;
xfs_extnum_t idx;
ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED));
ASSERT(xfs_is_reflink_inode(ip));
offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
return false;
if (got.br_startoff > offset_fsb)
return false;
trace_xfs_reflink_find_cow_mapping(ip, offset, 1, XFS_IO_OVERWRITE,
&got);
*imap = got;
return true;
}
/*
* Trim an extent to end at the next CoW reservation past offset_fsb.
*/
void
xfs_reflink_trim_irec_to_next_cow(
struct xfs_inode *ip,
xfs_fileoff_t offset_fsb,
struct xfs_bmbt_irec *imap)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
struct xfs_bmbt_irec got;
xfs_extnum_t idx;
if (!xfs_is_reflink_inode(ip))
return;
/* Find the extent in the CoW fork. */
if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
return;
/* This is the extent before; try sliding up one. */
if (got.br_startoff < offset_fsb) {
if (!xfs_iext_get_extent(ifp, idx + 1, &got))
return;
}
if (got.br_startoff >= imap->br_startoff + imap->br_blockcount)
return;
imap->br_blockcount = got.br_startoff - imap->br_startoff;
trace_xfs_reflink_trim_irec(ip, imap);
}
/* Convert all of the unwritten CoW extents in a file's range to real ones. */
int
xfs_reflink_convert_cow(
struct xfs_inode *ip,
xfs_off_t offset,
xfs_off_t count)
{
struct xfs_bmbt_irec got;
struct xfs_defer_ops dfops;
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
xfs_extnum_t idx;
bool found;
int error = 0;
xfs_ilock(ip, XFS_ILOCK_EXCL);
/* Convert all the extents to real from unwritten. */
for (found = xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got);
found && got.br_startoff < end_fsb;
found = xfs_iext_get_extent(ifp, ++idx, &got)) {
error = xfs_reflink_convert_cow_extent(ip, &got, offset_fsb,
end_fsb - offset_fsb, &dfops);
if (error)
break;
}
/* Finish up. */
xfs_iunlock(ip, XFS_ILOCK_EXCL);
return error;
}
/*
* Trim the passed in imap to the next shared/unshared extent boundary, and
* if imap->br_startoff points to a shared extent reserve space for it in the
* COW fork. In this case *shared is set to true, else to false.
*
* Note that imap will always contain the block numbers for the existing blocks
* in the data fork, as the upper layers need them for read-modify-write
* operations.
*/
int
xfs_reflink_reserve_cow(
struct xfs_inode *ip,
struct xfs_bmbt_irec *imap,
bool *shared)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
struct xfs_bmbt_irec got;
int error = 0;
bool eof = false, trimmed;
xfs_extnum_t idx;
/*
* Search the COW fork extent list first. This serves two purposes:
* first this implement the speculative preallocation using cowextisze,
* so that we also unshared block adjacent to shared blocks instead
* of just the shared blocks themselves. Second the lookup in the
* extent list is generally faster than going out to the shared extent
* tree.
*/
if (!xfs_iext_lookup_extent(ip, ifp, imap->br_startoff, &idx, &got))
eof = true;
if (!eof && got.br_startoff <= imap->br_startoff) {
trace_xfs_reflink_cow_found(ip, imap);
xfs_trim_extent(imap, got.br_startoff, got.br_blockcount);
*shared = true;
return 0;
}
/* Trim the mapping to the nearest shared extent boundary. */
error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed);
if (error)
return error;
/* Not shared? Just report the (potentially capped) extent. */
if (!*shared)
return 0;
/*
* Fork all the shared blocks from our write offset until the end of
* the extent.
*/
error = xfs_qm_dqattach_locked(ip, 0);
if (error)
return error;
error = xfs_bmapi_reserve_delalloc(ip, XFS_COW_FORK, imap->br_startoff,
imap->br_blockcount, 0, &got, &idx, eof);
if (error == -ENOSPC || error == -EDQUOT)
trace_xfs_reflink_cow_enospc(ip, imap);
if (error)
return error;
trace_xfs_reflink_cow_alloc(ip, &got);
return 0;
}
/* Does this inode need the reflink flag? */
int
xfs_reflink_inode_has_shared_extents(
struct xfs_trans *tp,
struct xfs_inode *ip,
bool *has_shared)
{
struct xfs_bmbt_irec got;
struct xfs_mount *mp = ip->i_mount;
struct xfs_ifork *ifp;
xfs_agnumber_t agno;
xfs_agblock_t agbno;
xfs_extlen_t aglen;
xfs_agblock_t rbno;
xfs_extlen_t rlen;
xfs_extnum_t idx;
bool found;
int error;
ifp = XFS_IFORK_PTR(ip, XFS_DATA_FORK);
if (!(ifp->if_flags & XFS_IFEXTENTS)) {
error = xfs_iread_extents(tp, ip, XFS_DATA_FORK);
if (error)
return error;
}
*has_shared = false;
found = xfs_iext_lookup_extent(ip, ifp, 0, &idx, &got);
while (found) {
if (isnullstartblock(got.br_startblock) ||
got.br_state != XFS_EXT_NORM)
goto next;
agno = XFS_FSB_TO_AGNO(mp, got.br_startblock);
agbno = XFS_FSB_TO_AGBNO(mp, got.br_startblock);
aglen = got.br_blockcount;
error = xfs_reflink_find_shared(mp, tp, agno, agbno, aglen,
&rbno, &rlen, false);
if (error)
return error;
/* Is there still a shared block here? */
if (rbno != NULLAGBLOCK) {
*has_shared = true;
return 0;
}
next:
found = xfs_iext_get_extent(ifp, ++idx, &got);
}
return 0;
}
/*
* Advance to the next id in the current chunk, or if at the
* end of the chunk, skip ahead to first id in next allocated chunk
* using the SEEK_DATA interface.
*/
static int
xfs_dq_get_next_id(
struct xfs_mount *mp,
uint type,
xfs_dqid_t *id)
{
struct xfs_inode *quotip = xfs_quota_inode(mp, type);
xfs_dqid_t next_id = *id + 1; /* simple advance */
uint lock_flags;
struct xfs_bmbt_irec got;
struct xfs_iext_cursor cur;
xfs_fsblock_t start;
int error = 0;
/* If we'd wrap past the max ID, stop */
if (next_id < *id)
return -ENOENT;
/* If new ID is within the current chunk, advancing it sufficed */
if (next_id % mp->m_quotainfo->qi_dqperchunk) {
*id = next_id;
return 0;
}
/* Nope, next_id is now past the current chunk, so find the next one */
start = (xfs_fsblock_t)next_id / mp->m_quotainfo->qi_dqperchunk;
lock_flags = xfs_ilock_data_map_shared(quotip);
if (!(quotip->i_df.if_flags & XFS_IFEXTENTS)) {
error = xfs_iread_extents(NULL, quotip, XFS_DATA_FORK);
if (error)
return error;
}
if (xfs_iext_lookup_extent(quotip, "ip->i_df, start, &cur, &got)) {
/* contiguous chunk, bump startoff for the id calculation */
if (got.br_startoff < start)
got.br_startoff = start;
*id = got.br_startoff * mp->m_quotainfo->qi_dqperchunk;
} else {
error = -ENOENT;
}
xfs_iunlock(quotip, lock_flags);
return error;
}
/*
* Find the extent that maps the given range in the COW fork. Even if the extent
* is not shared we might have a preallocation for it in the COW fork. If so we
* use it that rather than trigger a new allocation.
*/
static int
xfs_find_trim_cow_extent(
struct xfs_inode *ip,
struct xfs_bmbt_irec *imap,
bool *shared,
bool *found)
{
xfs_fileoff_t offset_fsb = imap->br_startoff;
xfs_filblks_t count_fsb = imap->br_blockcount;
struct xfs_iext_cursor icur;
struct xfs_bmbt_irec got;
*found = false;
/*
* If we don't find an overlapping extent, trim the range we need to
* allocate to fit the hole we found.
*/
if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
got.br_startoff = offset_fsb + count_fsb;
if (got.br_startoff > offset_fsb) {
xfs_trim_extent(imap, imap->br_startoff,
got.br_startoff - imap->br_startoff);
return xfs_inode_need_cow(ip, imap, shared);
}
*shared = true;
if (isnullstartblock(got.br_startblock)) {
xfs_trim_extent(imap, got.br_startoff, got.br_blockcount);
return 0;
}
/* real extent found - no need to allocate */
xfs_trim_extent(&got, offset_fsb, count_fsb);
*imap = got;
*found = true;
return 0;
}
static int
xfs_reflink_convert_cow_locked(
struct xfs_inode *ip,
xfs_fileoff_t offset_fsb,
xfs_filblks_t count_fsb)
{
struct xfs_iext_cursor icur;
struct xfs_bmbt_irec got;
struct xfs_btree_cur *dummy_cur = NULL;
int dummy_logflags;
int error = 0;
if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got))
return 0;
do {
if (got.br_startoff >= offset_fsb + count_fsb)
break;
if (got.br_state == XFS_EXT_NORM)
continue;
if (WARN_ON_ONCE(isnullstartblock(got.br_startblock)))
return -EIO;
xfs_trim_extent(&got, offset_fsb, count_fsb);
if (!got.br_blockcount)
continue;
got.br_state = XFS_EXT_NORM;
error = xfs_bmap_add_extent_unwritten_real(NULL, ip,
XFS_COW_FORK, &icur, &dummy_cur, &got,
&dummy_logflags);
if (error)
return error;
} while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got));
return error;
}
/*
* Remap parts of a file's data fork after a successful CoW.
*/
int
xfs_reflink_end_cow(
struct xfs_inode *ip,
xfs_off_t offset,
xfs_off_t count)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
struct xfs_bmbt_irec got, del;
struct xfs_trans *tp;
xfs_fileoff_t offset_fsb;
xfs_fileoff_t end_fsb;
xfs_fsblock_t firstfsb;
struct xfs_defer_ops dfops;
int error;
unsigned int resblks;
xfs_filblks_t rlen;
xfs_extnum_t idx;
trace_xfs_reflink_end_cow(ip, offset, count);
/* No COW extents? That's easy! */
if (ifp->if_bytes == 0)
return 0;
offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset);
end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count);
/*
* Start a rolling transaction to switch the mappings. We're
* unlikely ever to have to remap 16T worth of single-block
* extents, so just cap the worst case extent count to 2^32-1.
* Stick a warning in just in case, and avoid 64-bit division.
*/
BUILD_BUG_ON(MAX_RW_COUNT > UINT_MAX);
if (end_fsb - offset_fsb > UINT_MAX) {
error = -EFSCORRUPTED;
xfs_force_shutdown(ip->i_mount, SHUTDOWN_CORRUPT_INCORE);
ASSERT(0);
goto out;
}
resblks = XFS_NEXTENTADD_SPACE_RES(ip->i_mount,
(unsigned int)(end_fsb - offset_fsb),
XFS_DATA_FORK);
error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write,
resblks, 0, 0, &tp);
if (error)
goto out;
xfs_ilock(ip, XFS_ILOCK_EXCL);
xfs_trans_ijoin(tp, ip, 0);
/* If there is a hole at end_fsb - 1 go to the previous extent */
if (!xfs_iext_lookup_extent(ip, ifp, end_fsb - 1, &idx, &got) ||
got.br_startoff > end_fsb) {
/*
* In case of racing, overlapping AIO writes no COW extents
* might be left by the time I/O completes for the loser of
* the race. In that case we are done.
*/
if (idx <= 0)
goto out_cancel;
xfs_iext_get_extent(ifp, --idx, &got);
}
/* Walk backwards until we're out of the I/O range... */
while (got.br_startoff + got.br_blockcount > offset_fsb) {
del = got;
xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb);
/* Extent delete may have bumped idx forward */
if (!del.br_blockcount) {
idx--;
goto next_extent;
}
ASSERT(!isnullstartblock(got.br_startblock));
/*
* Don't remap unwritten extents; these are
* speculatively preallocated CoW extents that have been
* allocated but have not yet been involved in a write.
*/
if (got.br_state == XFS_EXT_UNWRITTEN) {
idx--;
goto next_extent;
}
/* Unmap the old blocks in the data fork. */
xfs_defer_init(&dfops, &firstfsb);
rlen = del.br_blockcount;
error = __xfs_bunmapi(tp, ip, del.br_startoff, &rlen, 0, 1,
&firstfsb, &dfops);
if (error)
goto out_defer;
/* Trim the extent to whatever got unmapped. */
if (rlen) {
xfs_trim_extent(&del, del.br_startoff + rlen,
del.br_blockcount - rlen);
}
trace_xfs_reflink_cow_remap(ip, &del);
/* Free the CoW orphan record. */
error = xfs_refcount_free_cow_extent(tp->t_mountp, &dfops,
del.br_startblock, del.br_blockcount);
if (error)
goto out_defer;
/* Map the new blocks into the data fork. */
error = xfs_bmap_map_extent(tp->t_mountp, &dfops, ip, &del);
if (error)
goto out_defer;
/* Remove the mapping from the CoW fork. */
xfs_bmap_del_extent_cow(ip, &idx, &got, &del);
xfs_defer_ijoin(&dfops, ip);
error = xfs_defer_finish(&tp, &dfops);
if (error)
goto out_defer;
next_extent:
if (!xfs_iext_get_extent(ifp, idx, &got))
break;
}
error = xfs_trans_commit(tp);
xfs_iunlock(ip, XFS_ILOCK_EXCL);
if (error)
goto out;
return 0;
out_defer:
xfs_defer_cancel(&dfops);
out_cancel:
xfs_trans_cancel(tp);
xfs_iunlock(ip, XFS_ILOCK_EXCL);
out:
trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_);
return error;
}
/*
* Cancel CoW reservations for some block range of an inode.
*
* If cancel_real is true this function cancels all COW fork extents for the
* inode; if cancel_real is false, real extents are not cleared.
*/
int
xfs_reflink_cancel_cow_blocks(
struct xfs_inode *ip,
struct xfs_trans **tpp,
xfs_fileoff_t offset_fsb,
xfs_fileoff_t end_fsb,
bool cancel_real)
{
struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, XFS_COW_FORK);
struct xfs_bmbt_irec got, del;
xfs_extnum_t idx;
xfs_fsblock_t firstfsb;
struct xfs_defer_ops dfops;
int error = 0;
if (!xfs_is_reflink_inode(ip))
return 0;
if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &idx, &got))
return 0;
while (got.br_startoff < end_fsb) {
del = got;
xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb);
trace_xfs_reflink_cancel_cow(ip, &del);
if (isnullstartblock(del.br_startblock)) {
error = xfs_bmap_del_extent_delay(ip, XFS_COW_FORK,
&idx, &got, &del);
if (error)
break;
} else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) {
xfs_trans_ijoin(*tpp, ip, 0);
xfs_defer_init(&dfops, &firstfsb);
/* Free the CoW orphan record. */
error = xfs_refcount_free_cow_extent(ip->i_mount,
&dfops, del.br_startblock,
del.br_blockcount);
if (error)
break;
xfs_bmap_add_free(ip->i_mount, &dfops,
del.br_startblock, del.br_blockcount,
NULL);
/* Update quota accounting */
xfs_trans_mod_dquot_byino(*tpp, ip, XFS_TRANS_DQ_BCOUNT,
-(long)del.br_blockcount);
/* Roll the transaction */
xfs_defer_ijoin(&dfops, ip);
error = xfs_defer_finish(tpp, &dfops);
if (error) {
xfs_defer_cancel(&dfops);
break;
}
/* Remove the mapping from the CoW fork. */
xfs_bmap_del_extent_cow(ip, &idx, &got, &del);
}
if (!xfs_iext_get_extent(ifp, ++idx, &got))
break;
}
/* clear tag if cow fork is emptied */
if (!ifp->if_bytes)
xfs_inode_clear_cowblocks_tag(ip);
return error;
}
/* Allocate all CoW reservations covering a range of blocks in a file. */
int
xfs_reflink_allocate_cow(
struct xfs_inode *ip,
struct xfs_bmbt_irec *imap,
bool *shared,
uint *lockmode)
{
struct xfs_mount *mp = ip->i_mount;
xfs_fileoff_t offset_fsb = imap->br_startoff;
xfs_filblks_t count_fsb = imap->br_blockcount;
struct xfs_bmbt_irec got;
struct xfs_defer_ops dfops;
struct xfs_trans *tp = NULL;
xfs_fsblock_t first_block;
int nimaps, error = 0;
bool trimmed;
xfs_filblks_t resaligned;
xfs_extlen_t resblks = 0;
xfs_extnum_t idx;
retry:
ASSERT(xfs_is_reflink_inode(ip));
ASSERT(xfs_isilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED));
/*
* Even if the extent is not shared we might have a preallocation for
* it in the COW fork. If so use it.
*/
if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &idx, &got) &&
got.br_startoff <= offset_fsb) {
*shared = true;
/* If we have a real allocation in the COW fork we're done. */
if (!isnullstartblock(got.br_startblock)) {
xfs_trim_extent(&got, offset_fsb, count_fsb);
*imap = got;
goto convert;
}
xfs_trim_extent(imap, got.br_startoff, got.br_blockcount);
} else {
error = xfs_reflink_trim_around_shared(ip, imap, shared, &trimmed);
if (error || !*shared)
goto out;
}
if (!tp) {
resaligned = xfs_aligned_fsb_count(imap->br_startoff,
imap->br_blockcount, xfs_get_cowextsz_hint(ip));
resblks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned);
xfs_iunlock(ip, *lockmode);
error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, 0, &tp);
*lockmode = XFS_ILOCK_EXCL;
xfs_ilock(ip, *lockmode);
if (error)
return error;
error = xfs_qm_dqattach_locked(ip, 0);
if (error)
goto out;
goto retry;
}
error = xfs_trans_reserve_quota_nblks(tp, ip, resblks, 0,
XFS_QMOPT_RES_REGBLKS);
if (error)
goto out;
xfs_trans_ijoin(tp, ip, 0);
xfs_defer_init(&dfops, &first_block);
nimaps = 1;
/* Allocate the entire reservation as unwritten blocks. */
error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount,
XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, &first_block,
resblks, imap, &nimaps, &dfops);
if (error)
goto out_bmap_cancel;
/* Finish up. */
error = xfs_defer_finish(&tp, &dfops);
if (error)
goto out_bmap_cancel;
error = xfs_trans_commit(tp);
if (error)
return error;
convert:
return xfs_reflink_convert_cow_extent(ip, imap, offset_fsb, count_fsb,
&dfops);
out_bmap_cancel:
xfs_defer_cancel(&dfops);
xfs_trans_unreserve_quota_nblks(tp, ip, (long)resblks, 0,
XFS_QMOPT_RES_REGBLKS);
out:
if (tp)
xfs_trans_cancel(tp);
return error;
}