/*ARGSUSED*/
static kmem_cbrc_t
zfs_znode_move(void *buf, void *newbuf, size_t size, void *arg)
{
znode_t *ozp = buf, *nzp = newbuf;
zfsvfs_t *zfsvfs;
vnode_t *vp;
/*
* The znode is on the file system's list of known znodes if the vfs
* pointer is valid. We set the low bit of the vfs pointer when freeing
* the znode to invalidate it, and the memory patterns written by kmem
* (baddcafe and deadbeef) set at least one of the two low bits. A newly
* created znode sets the vfs pointer last of all to indicate that the
* znode is known and in a valid state to be moved by this function.
*/
zfsvfs = ozp->z_zfsvfs;
if (!POINTER_IS_VALID(zfsvfs)) {
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_invalid);
return (KMEM_CBRC_DONT_KNOW);
}
/*
* Ensure that the filesystem is not unmounted during the move.
*/
if (zfs_enter(zfsvfs) != 0) { /* ZFS_ENTER */
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_unmounted);
return (KMEM_CBRC_DONT_KNOW);
}
mutex_enter(&zfsvfs->z_znodes_lock);
/*
* Recheck the vfs pointer in case the znode was removed just before
* acquiring the lock.
*/
if (zfsvfs != ozp->z_zfsvfs) {
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_recheck_invalid);
return (KMEM_CBRC_DONT_KNOW);
}
/*
* At this point we know that as long as we hold z_znodes_lock, the
* znode cannot be freed and fields within the znode can be safely
* accessed. Now, prevent a race with zfs_zget().
*/
if (ZFS_OBJ_HOLD_TRYENTER(zfsvfs, ozp->z_id) == 0) {
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_obj_held);
return (KMEM_CBRC_LATER);
}
vp = ZTOV(ozp);
if (mutex_tryenter(&vp->v_lock) == 0) {
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_vnode_locked);
return (KMEM_CBRC_LATER);
}
/* Only move znodes that are referenced _only_ by the DNLC. */
if (vp->v_count != 1 || !vn_in_dnlc(vp)) {
mutex_exit(&vp->v_lock);
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_not_only_dnlc);
return (KMEM_CBRC_LATER);
}
/*
* The znode is known and in a valid state to move. We're holding the
* locks needed to execute the critical section.
*/
zfs_znode_move_impl(ozp, nzp);
mutex_exit(&vp->v_lock);
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
list_link_replace(&ozp->z_link_node, &nzp->z_link_node);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
return (KMEM_CBRC_YES);
}
/*ARGSUSED*/
static kmem_cbrc_t
zfs_znode_move(void *buf, void *newbuf, size_t size, void *arg)
{
znode_t *ozp = buf, *nzp = newbuf;
zfsvfs_t *zfsvfs;
vnode_t *vp;
/*
* The znode is on the file system's list of known znodes if the vfs
* pointer is valid. We set the low bit of the vfs pointer when freeing
* the znode to invalidate it, and the memory patterns written by kmem
* (baddcafe and deadbeef) set at least one of the two low bits. A newly
* created znode sets the vfs pointer last of all to indicate that the
* znode is known and in a valid state to be moved by this function.
*/
zfsvfs = ozp->z_zfsvfs;
if (!POINTER_IS_VALID(zfsvfs)) {
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_invalid);
return (KMEM_CBRC_DONT_KNOW);
}
/*
* Close a small window in which it's possible that the filesystem could
* be unmounted and freed, and zfsvfs, though valid in the previous
* statement, could point to unrelated memory by the time we try to
* prevent the filesystem from being unmounted.
*/
rw_enter(&zfsvfs_lock, RW_WRITER);
if (zfsvfs != ozp->z_zfsvfs) {
rw_exit(&zfsvfs_lock);
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_recheck1);
return (KMEM_CBRC_DONT_KNOW);
}
/*
* If the znode is still valid, then so is the file system. We know that
* no valid file system can be freed while we hold zfsvfs_lock, so we
* can safely ensure that the filesystem is not and will not be
* unmounted. The next statement is equivalent to ZFS_ENTER().
*/
rrw_enter(&zfsvfs->z_teardown_lock, RW_READER, FTAG);
if (zfsvfs->z_unmounted) {
ZFS_EXIT(zfsvfs);
rw_exit(&zfsvfs_lock);
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_unmounted);
return (KMEM_CBRC_DONT_KNOW);
}
rw_exit(&zfsvfs_lock);
mutex_enter(&zfsvfs->z_znodes_lock);
/*
* Recheck the vfs pointer in case the znode was removed just before
* acquiring the lock.
*/
if (zfsvfs != ozp->z_zfsvfs) {
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_zfsvfs_recheck2);
return (KMEM_CBRC_DONT_KNOW);
}
/*
* At this point we know that as long as we hold z_znodes_lock, the
* znode cannot be freed and fields within the znode can be safely
* accessed. Now, prevent a race with zfs_zget().
*/
if (ZFS_OBJ_HOLD_TRYENTER(zfsvfs, ozp->z_id) == 0) {
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_obj_held);
return (KMEM_CBRC_LATER);
}
vp = ZTOV(ozp);
if (mutex_tryenter(&vp->v_lock) == 0) {
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_vnode_locked);
return (KMEM_CBRC_LATER);
}
/* Only move znodes that are referenced _only_ by the DNLC. */
if (vp->v_count != 1 || !vn_in_dnlc(vp)) {
mutex_exit(&vp->v_lock);
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
ZNODE_STAT_ADD(znode_move_stats.zms_not_only_dnlc);
return (KMEM_CBRC_LATER);
}
/*
* The znode is known and in a valid state to move. We're holding the
* locks needed to execute the critical section.
*/
zfs_znode_move_impl(ozp, nzp);
mutex_exit(&vp->v_lock);
ZFS_OBJ_HOLD_EXIT(zfsvfs, ozp->z_id);
list_link_replace(&ozp->z_link_node, &nzp->z_link_node);
mutex_exit(&zfsvfs->z_znodes_lock);
ZFS_EXIT(zfsvfs);
return (KMEM_CBRC_YES);
}