void
kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
off_t write_offset, caddr_t addr, vm_size_t write_length,
off_t discard_offset, off_t discard_end)
{
int error;
kprintf("kern_close_file_for_direct_io\n");
if (!ref) return;
if (ref->vp)
{
int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
void * p1;
void * p2;
if (ref->vp->v_type == VREG)
{
p1 = &ref->device;
p2 = kernproc;
do_ioctl = &file_ioctl;
}
else
{
/* Partition. */
p1 = ref->vp;
p2 = ref->ctx;
do_ioctl = &device_ioctl;
}
(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
if (addr && write_length)
{
(void) kern_write_file(ref, write_offset, addr, write_length);
}
if (discard_offset && discard_end && !ref->pinned)
{
(void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, discard_end);
}
error = vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
kprintf("vnode_close(%d)\n", error);
}
vfs_context_rele(ref->ctx);
ref->ctx = NULL;
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
}
void
kern_close_file_for_direct_io(struct kern_direct_file_io_ref_t * ref,
off_t write_offset, void * addr, size_t write_length,
off_t discard_offset, off_t discard_end)
{
int error;
printf("kern_close_file_for_direct_io(%p)\n", ref);
if (!ref) return;
if (ref->vp)
{
int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
void * p1;
void * p2;
discard_offset = ((discard_offset + ref->blksize - 1) & ~(((off_t) ref->blksize) - 1));
discard_end = ((discard_end) & ~(((off_t) ref->blksize) - 1));
if (ref->vp->v_type == VREG)
{
p1 = &ref->device;
p2 = kernproc;
do_ioctl = &file_ioctl;
}
else
{
/* Partition. */
p1 = ref->vp;
p2 = ref->ctx;
do_ioctl = &device_ioctl;
}
(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
//XXX If unmapping extents then don't also need to unpin; except ...
//XXX if file unaligned (HFS 4k / Fusion 128k) then pin is superset and
//XXX unmap is subset, so save extra walk over file extents (and the risk
//XXX that CF drain starts) vs leaving partial units pinned to SSD
//XXX (until whatever was sharing also unmaps). Err on cleaning up fully.
boolean_t will_unmap = (!ref->pinned || ref->cf) && (discard_end > discard_offset);
boolean_t will_unpin = (ref->pinned && ref->cf /* && !will_unmap */);
(void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (will_unpin) ? ref->filelength : 0);
if (will_unmap)
{
(void) kern_ioctl_file_extents(ref, DKIOCUNMAP, discard_offset, (ref->cf) ? ref->filelength : discard_end);
}
if (ref->frozen)
{
(void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
}
if (ref->wbcranged)
{
(void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
}
if (addr && write_length)
{
(void) kern_write_file(ref, write_offset, addr, write_length, IO_SKIP_ENCRYPTION);
}
error = vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
kprintf("vnode_close(%d)\n", error);
}
ref->ctx = NULL;
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
}
struct kern_direct_file_io_ref_t *
kern_open_file_for_direct_io(const char * name,
kern_get_file_extents_callback_t callback,
void * callback_ref,
dev_t * partition_device_result,
dev_t * image_device_result,
uint64_t * partitionbase_result,
uint64_t * maxiocount_result,
uint32_t * oflags,
off_t offset,
caddr_t addr,
vm_size_t len)
{
struct kern_direct_file_io_ref_t * ref;
proc_t p;
struct vnode_attr va;
int error;
off_t f_offset;
uint64_t fileblk;
size_t filechunk;
uint64_t physoffset;
dev_t device;
dev_t target = 0;
int isssd = 0;
uint32_t flags = 0;
uint32_t blksize;
off_t maxiocount, count;
boolean_t locked = FALSE;
int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
void * p1 = NULL;
void * p2 = NULL;
error = EFAULT;
ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
if (!ref)
{
error = EFAULT;
goto out;
}
bzero(ref, sizeof(*ref));
p = kernproc;
ref->ctx = vfs_context_create(vfs_context_current());
if ((error = vnode_open(name, (O_CREAT | FWRITE), (0), 0, &ref->vp, ref->ctx)))
goto out;
if (addr && len)
{
if ((error = kern_write_file(ref, offset, addr, len)))
goto out;
}
VATTR_INIT(&va);
VATTR_WANTED(&va, va_rdev);
VATTR_WANTED(&va, va_fsid);
VATTR_WANTED(&va, va_data_size);
VATTR_WANTED(&va, va_nlink);
error = EFAULT;
if (vnode_getattr(ref->vp, &va, ref->ctx))
goto out;
kprintf("vp va_rdev major %d minor %d\n", major(va.va_rdev), minor(va.va_rdev));
kprintf("vp va_fsid major %d minor %d\n", major(va.va_fsid), minor(va.va_fsid));
kprintf("vp size %qd\n", va.va_data_size);
if (ref->vp->v_type == VREG)
{
/* Don't dump files with links. */
if (va.va_nlink != 1)
goto out;
device = va.va_fsid;
p1 = &device;
p2 = p;
do_ioctl = &file_ioctl;
}
else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
{
/* Partition. */
device = va.va_rdev;
p1 = ref->vp;
p2 = ref->ctx;
do_ioctl = &device_ioctl;
}
else
{
/* Don't dump to non-regular files. */
error = EFAULT;
goto out;
}
ref->device = device;
// get block size
error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
if (error)
goto out;
if (ref->vp->v_type == VREG)
ref->filelength = va.va_data_size;
else
{
error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
if (error)
goto out;
ref->filelength = fileblk * ref->blksize;
}
// pin logical extents
error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
if (error && (ENOTTY != error)) goto out;
ref->pinned = (error == 0);
// generate the block list
error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
if (error)
goto out;
locked = TRUE;
f_offset = 0;
while (f_offset < ref->filelength)
{
if (ref->vp->v_type == VREG)
{
filechunk = 1*1024*1024*1024;
daddr64_t blkno;
error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno, &filechunk, NULL, 0, NULL);
if (error)
goto out;
fileblk = blkno * ref->blksize;
}
else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
{
fileblk = f_offset;
filechunk = f_offset ? 0 : ref->filelength;
}
physoffset = 0;
while (physoffset < filechunk)
{
dk_physical_extent_t getphysreq;
bzero(&getphysreq, sizeof(getphysreq));
getphysreq.offset = fileblk + physoffset;
getphysreq.length = (filechunk - physoffset);
error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
if (error)
goto out;
if (!target)
{
target = getphysreq.dev;
}
else if (target != getphysreq.dev)
{
error = ENOTSUP;
goto out;
}
callback(callback_ref, getphysreq.offset, getphysreq.length);
physoffset += getphysreq.length;
}
f_offset += filechunk;
}
callback(callback_ref, 0ULL, 0ULL);
if (ref->vp->v_type == VREG)
p1 = ⌖
// get partition base
error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
if (error)
goto out;
// get block size & constraints
error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
if (error)
goto out;
maxiocount = 1*1024*1024*1024;
error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
if (error)
count = 0;
count *= blksize;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
if (error)
count = 0;
count *= blksize;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
kprintf("max io 0x%qx bytes\n", maxiocount);
if (maxiocount_result)
*maxiocount_result = maxiocount;
error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
if (!error && isssd)
flags |= kIOHibernateOptionSSD;
if (partition_device_result)
*partition_device_result = device;
if (image_device_result)
*image_device_result = target;
if (flags)
*oflags = flags;
out:
kprintf("kern_open_file_for_direct_io(%d)\n", error);
if (error && locked)
{
p1 = &device;
(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
}
if (error && ref)
{
if (ref->vp)
{
vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
}
vfs_context_rele(ref->ctx);
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
ref = NULL;
}
return(ref);
}
struct kern_direct_file_io_ref_t *
kern_open_file_for_direct_io(const char * name,
uint32_t iflags,
kern_get_file_extents_callback_t callback,
void * callback_ref,
off_t set_file_size,
off_t fs_free_size,
off_t write_file_offset,
void * write_file_addr,
size_t write_file_len,
dev_t * partition_device_result,
dev_t * image_device_result,
uint64_t * partitionbase_result,
uint64_t * maxiocount_result,
uint32_t * oflags)
{
struct kern_direct_file_io_ref_t * ref;
proc_t p;
struct vnode_attr va;
dk_apfs_wbc_range_t wbc_range;
int error;
off_t f_offset;
uint64_t fileblk;
size_t filechunk;
uint64_t physoffset, minoffset;
dev_t device;
dev_t target = 0;
int isssd = 0;
uint32_t flags = 0;
uint32_t blksize;
off_t maxiocount, count, segcount, wbctotal;
boolean_t locked = FALSE;
int fmode, cmode;
struct nameidata nd;
u_int32_t ndflags;
off_t mpFree;
int (*do_ioctl)(void * p1, void * p2, u_long theIoctl, caddr_t result);
void * p1 = NULL;
void * p2 = NULL;
error = EFAULT;
ref = (struct kern_direct_file_io_ref_t *) kalloc(sizeof(struct kern_direct_file_io_ref_t));
if (!ref)
{
error = EFAULT;
goto out;
}
bzero(ref, sizeof(*ref));
p = kernproc;
ref->ctx = vfs_context_kernel();
fmode = (kIOPolledFileCreate & iflags) ? (O_CREAT | FWRITE) : FWRITE;
cmode = S_IRUSR | S_IWUSR;
ndflags = NOFOLLOW;
NDINIT(&nd, LOOKUP, OP_OPEN, ndflags, UIO_SYSSPACE, CAST_USER_ADDR_T(name), ref->ctx);
VATTR_INIT(&va);
VATTR_SET(&va, va_mode, cmode);
VATTR_SET(&va, va_dataprotect_flags, VA_DP_RAWENCRYPTED);
VATTR_SET(&va, va_dataprotect_class, PROTECTION_CLASS_D);
if ((error = vn_open_auth(&nd, &fmode, &va))) {
kprintf("vn_open_auth(fmode: %d, cmode: %d) failed with error: %d\n", fmode, cmode, error);
goto out;
}
ref->vp = nd.ni_vp;
if (ref->vp->v_type == VREG)
{
vnode_lock_spin(ref->vp);
SET(ref->vp->v_flag, VSWAP);
vnode_unlock(ref->vp);
}
if (write_file_addr && write_file_len)
{
if ((error = kern_write_file(ref, write_file_offset, write_file_addr, write_file_len, IO_SKIP_ENCRYPTION))) {
kprintf("kern_write_file() failed with error: %d\n", error);
goto out;
}
}
VATTR_INIT(&va);
VATTR_WANTED(&va, va_rdev);
VATTR_WANTED(&va, va_fsid);
VATTR_WANTED(&va, va_devid);
VATTR_WANTED(&va, va_data_size);
VATTR_WANTED(&va, va_data_alloc);
VATTR_WANTED(&va, va_nlink);
error = EFAULT;
if (vnode_getattr(ref->vp, &va, ref->ctx)) goto out;
wbctotal = 0;
mpFree = freespace_mb(ref->vp);
mpFree <<= 20;
kprintf("kern_direct_file(%s): vp size %qd, alloc %qd, mp free %qd, keep free %qd\n",
name, va.va_data_size, va.va_data_alloc, mpFree, fs_free_size);
if (ref->vp->v_type == VREG)
{
/* Don't dump files with links. */
if (va.va_nlink != 1) goto out;
device = (VATTR_IS_SUPPORTED(&va, va_devid)) ? va.va_devid : va.va_fsid;
ref->filelength = va.va_data_size;
p1 = &device;
p2 = p;
do_ioctl = &file_ioctl;
if (kIOPolledFileHibernate & iflags)
{
error = do_ioctl(p1, p2, DKIOCAPFSGETWBCRANGE, (caddr_t) &wbc_range);
ref->wbcranged = (error == 0);
}
if (ref->wbcranged)
{
uint32_t idx;
assert(wbc_range.count <= (sizeof(wbc_range.extents) / sizeof(wbc_range.extents[0])));
for (idx = 0; idx < wbc_range.count; idx++) wbctotal += wbc_range.extents[idx].length;
kprintf("kern_direct_file(%s): wbc %qd\n", name, wbctotal);
if (wbctotal) target = wbc_range.dev;
}
if (set_file_size)
{
if (wbctotal)
{
if (wbctotal >= set_file_size) set_file_size = HIBERNATE_MIN_FILE_SIZE;
else
{
set_file_size -= wbctotal;
if (set_file_size < HIBERNATE_MIN_FILE_SIZE) set_file_size = HIBERNATE_MIN_FILE_SIZE;
}
}
if (fs_free_size)
{
mpFree += va.va_data_alloc;
if ((mpFree < set_file_size) || ((mpFree - set_file_size) < fs_free_size))
{
error = ENOSPC;
goto out;
}
}
error = vnode_setsize(ref->vp, set_file_size, IO_NOZEROFILL | IO_NOAUTH, ref->ctx);
if (error) goto out;
ref->filelength = set_file_size;
}
}
else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
{
/* Partition. */
device = va.va_rdev;
p1 = ref->vp;
p2 = ref->ctx;
do_ioctl = &device_ioctl;
}
else
{
/* Don't dump to non-regular files. */
error = EFAULT;
goto out;
}
ref->device = device;
// probe for CF
dk_corestorage_info_t cs_info;
memset(&cs_info, 0, sizeof(dk_corestorage_info_t));
error = do_ioctl(p1, p2, DKIOCCORESTORAGE, (caddr_t)&cs_info);
ref->cf = (error == 0) && (cs_info.flags & DK_CORESTORAGE_ENABLE_HOTFILES);
// get block size
error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &ref->blksize);
if (error)
goto out;
minoffset = HIBERNATE_MIN_PHYSICAL_LBA * ref->blksize;
if (ref->vp->v_type != VREG)
{
error = do_ioctl(p1, p2, DKIOCGETBLOCKCOUNT, (caddr_t) &fileblk);
if (error) goto out;
ref->filelength = fileblk * ref->blksize;
}
// pin logical extents, CS version
error = kern_ioctl_file_extents(ref, _DKIOCCSPINEXTENT, 0, ref->filelength);
if (error && (ENOTTY != error)) goto out;
ref->pinned = (error == 0);
// pin logical extents, apfs version
error = VNOP_IOCTL(ref->vp, FSCTL_FREEZE_EXTENTS, NULL, 0, ref->ctx);
if (error && (ENOTTY != error)) goto out;
ref->frozen = (error == 0);
// generate the block list
error = do_ioctl(p1, p2, DKIOCLOCKPHYSICALEXTENTS, NULL);
if (error) goto out;
locked = TRUE;
f_offset = 0;
for (; f_offset < ref->filelength; f_offset += filechunk)
{
if (ref->vp->v_type == VREG)
{
filechunk = 1*1024*1024*1024;
daddr64_t blkno;
error = VNOP_BLOCKMAP(ref->vp, f_offset, filechunk, &blkno,
&filechunk, NULL, VNODE_WRITE | VNODE_BLOCKMAP_NO_TRACK, NULL);
if (error) goto out;
if (-1LL == blkno) continue;
fileblk = blkno * ref->blksize;
}
else if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
{
fileblk = f_offset;
filechunk = f_offset ? 0 : ref->filelength;
}
physoffset = 0;
while (physoffset < filechunk)
{
dk_physical_extent_t getphysreq;
bzero(&getphysreq, sizeof(getphysreq));
getphysreq.offset = fileblk + physoffset;
getphysreq.length = (filechunk - physoffset);
error = do_ioctl(p1, p2, DKIOCGETPHYSICALEXTENT, (caddr_t) &getphysreq);
if (error) goto out;
if (!target)
{
target = getphysreq.dev;
}
else if (target != getphysreq.dev)
{
error = ENOTSUP;
goto out;
}
assert(getphysreq.offset >= minoffset);
#if HIBFRAGMENT
uint64_t rev;
for (rev = 4096; rev <= getphysreq.length; rev += 4096)
{
callback(callback_ref, getphysreq.offset + getphysreq.length - rev, 4096);
}
#else
callback(callback_ref, getphysreq.offset, getphysreq.length);
#endif
physoffset += getphysreq.length;
}
}
if (ref->wbcranged)
{
uint32_t idx;
for (idx = 0; idx < wbc_range.count; idx++)
{
assert(wbc_range.extents[idx].offset >= minoffset);
callback(callback_ref, wbc_range.extents[idx].offset, wbc_range.extents[idx].length);
}
}
callback(callback_ref, 0ULL, 0ULL);
if (ref->vp->v_type == VREG) p1 = ⌖
else
{
p1 = ⌖
p2 = p;
do_ioctl = &file_ioctl;
}
// get partition base
if (partitionbase_result)
{
error = do_ioctl(p1, p2, DKIOCGETBASE, (caddr_t) partitionbase_result);
if (error)
goto out;
}
// get block size & constraints
error = do_ioctl(p1, p2, DKIOCGETBLOCKSIZE, (caddr_t) &blksize);
if (error)
goto out;
maxiocount = 1*1024*1024*1024;
error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTREAD, (caddr_t) &count);
if (error)
count = 0;
count *= blksize;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBLOCKCOUNTWRITE, (caddr_t) &count);
if (error)
count = 0;
count *= blksize;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTREAD, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXBYTECOUNTWRITE, (caddr_t) &count);
if (error)
count = 0;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTREAD, (caddr_t) &count);
if (!error)
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTREAD, (caddr_t) &segcount);
if (error)
count = segcount = 0;
count *= segcount;
if (count && (count < maxiocount))
maxiocount = count;
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTBYTECOUNTWRITE, (caddr_t) &count);
if (!error)
error = do_ioctl(p1, p2, DKIOCGETMAXSEGMENTCOUNTWRITE, (caddr_t) &segcount);
if (error)
count = segcount = 0;
count *= segcount;
if (count && (count < maxiocount))
maxiocount = count;
kprintf("max io 0x%qx bytes\n", maxiocount);
if (maxiocount_result)
*maxiocount_result = maxiocount;
error = do_ioctl(p1, p2, DKIOCISSOLIDSTATE, (caddr_t)&isssd);
if (!error && isssd)
flags |= kIOPolledFileSSD;
if (partition_device_result)
*partition_device_result = device;
if (image_device_result)
*image_device_result = target;
if (oflags)
*oflags = flags;
if ((ref->vp->v_type == VBLK) || (ref->vp->v_type == VCHR))
{
vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
ref->ctx = NULL;
}
out:
printf("kern_open_file_for_direct_io(%p, %d)\n", ref, error);
if (error && locked)
{
p1 = &device;
(void) do_ioctl(p1, p2, DKIOCUNLOCKPHYSICALEXTENTS, NULL);
}
if (error && ref)
{
if (ref->vp)
{
(void) kern_ioctl_file_extents(ref, _DKIOCCSUNPINEXTENT, 0, (ref->pinned && ref->cf) ? ref->filelength : 0);
if (ref->frozen)
{
(void) VNOP_IOCTL(ref->vp, FSCTL_THAW_EXTENTS, NULL, 0, ref->ctx);
}
if (ref->wbcranged)
{
(void) do_ioctl(p1, p2, DKIOCAPFSRELEASEWBCRANGE, (caddr_t) NULL);
}
vnode_close(ref->vp, FWRITE, ref->ctx);
ref->vp = NULLVP;
}
ref->ctx = NULL;
kfree(ref, sizeof(struct kern_direct_file_io_ref_t));
ref = NULL;
}
return(ref);
}
