/*
* Remove the specified directory entry from the hash. The entry to remove
* is defined by the name in `dirp', which must exist at the specified
* `offset' within the directory.
*/
void
ufsdirhash_remove(struct inode *ip, struct direct *dirp, doff_t offset)
{
struct dirhash *dh;
int slot;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
if ((dh = ip->i_dirhash) == NULL)
return;
DIRHASH_LOCK(dh);
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return;
}
KASSERT(offset < dh->dh_dirblks * dirblksiz);
/* Find the entry */
slot = ufsdirhash_findslot(dh, dirp->d_name, dirp->d_namlen, offset);
/* Remove the hash entry. */
ufsdirhash_delslot(dh, slot);
/* Update the per-block summary info. */
ufsdirhash_adjfree(dh, offset, UFS_DIRSIZ(0, dirp, needswap), dirblksiz);
DIRHASH_UNLOCK(dh);
}
/*
* Initialize the vnode associated with a new inode, handle aliased
* vnodes.
*/
void
ufs_vinit(struct mount *mntp, int (**specops)(void *), int (**fifoops)(void *),
struct vnode **vpp)
{
struct timeval tv;
struct inode *ip;
struct vnode *vp;
dev_t rdev;
struct ufsmount *ump;
vp = *vpp;
ip = VTOI(vp);
switch(vp->v_type = IFTOVT(ip->i_mode)) {
case VCHR:
case VBLK:
vp->v_op = specops;
ump = ip->i_ump;
if (ump->um_fstype == UFS1)
rdev = (dev_t)ufs_rw32(ip->i_ffs1_rdev,
UFS_MPNEEDSWAP(ump));
else
rdev = (dev_t)ufs_rw64(ip->i_ffs2_rdev,
UFS_MPNEEDSWAP(ump));
spec_node_init(vp, rdev);
break;
case VFIFO:
vp->v_op = fifoops;
break;
case VNON:
case VBAD:
case VSOCK:
case VLNK:
case VDIR:
case VREG:
break;
}
if (ip->i_number == UFS_ROOTINO)
vp->v_vflag |= VV_ROOT;
/*
* Initialize modrev times
*/
getmicrouptime(&tv);
ip->i_modrev = (uint64_t)(uint)tv.tv_sec << 32
| tv.tv_usec * 4294u;
*vpp = vp;
}
/*
* Insert information into the hash about a new directory entry. dirp
* points to a struct direct containing the entry, and offset specifies
* the offset of this entry.
*/
void
ufsdirhash_add(struct inode *ip, struct direct *dirp, doff_t offset)
{
struct dirhash *dh;
int slot;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
if ((dh = ip->i_dirhash) == NULL)
return;
DIRHASH_LOCK(dh);
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return;
}
KASSERT(offset < dh->dh_dirblks * dirblksiz);
/*
* Normal hash usage is < 66%. If the usage gets too high then
* remove the hash entirely and let it be rebuilt later.
*/
if (dh->dh_hused >= (dh->dh_hlen * 3) / 4) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return;
}
/* Find a free hash slot (empty or deleted), and add the entry. */
slot = ufsdirhash_hash(dh, dirp->d_name, dirp->d_namlen);
while (DH_ENTRY(dh, slot) >= 0)
slot = WRAPINCR(slot, dh->dh_hlen);
if (DH_ENTRY(dh, slot) == DIRHASH_EMPTY)
dh->dh_hused++;
DH_ENTRY(dh, slot) = offset;
/* Update the per-block summary info. */
ufsdirhash_adjfree(dh, offset, -UFS_DIRSIZ(0, dirp, needswap), dirblksiz);
DIRHASH_UNLOCK(dh);
}
int
ufs_mkdir(void *v)
{
struct vop_mkdir_v3_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
} */ *ap = v;
struct vnode *dvp = ap->a_dvp, *tvp;
struct vattr *vap = ap->a_vap;
struct componentname *cnp = ap->a_cnp;
struct inode *ip, *dp = VTOI(dvp);
struct buf *bp;
struct dirtemplate dirtemplate;
struct direct *newdir;
int error, dmode;
struct ufsmount *ump = dp->i_ump;
int dirblksiz = ump->um_dirblksiz;
struct ufs_lookup_results *ulr;
fstrans_start(dvp->v_mount, FSTRANS_SHARED);
/* XXX should handle this material another way */
ulr = &dp->i_crap;
UFS_CHECK_CRAPCOUNTER(dp);
if ((nlink_t)dp->i_nlink >= LINK_MAX) {
error = EMLINK;
goto out;
}
dmode = vap->va_mode & ACCESSPERMS;
dmode |= IFDIR;
/*
* Must simulate part of ufs_makeinode here to acquire the inode,
* but not have it entered in the parent directory. The entry is
* made later after writing "." and ".." entries.
*/
if ((error = UFS_VALLOC(dvp, dmode, cnp->cn_cred, ap->a_vpp)) != 0)
goto out;
tvp = *ap->a_vpp;
ip = VTOI(tvp);
error = UFS_WAPBL_BEGIN(ap->a_dvp->v_mount);
if (error) {
UFS_VFREE(tvp, ip->i_number, dmode);
vput(tvp);
goto out;
}
ip->i_uid = kauth_cred_geteuid(cnp->cn_cred);
DIP_ASSIGN(ip, uid, ip->i_uid);
ip->i_gid = dp->i_gid;
DIP_ASSIGN(ip, gid, ip->i_gid);
#if defined(QUOTA) || defined(QUOTA2)
if ((error = chkiq(ip, 1, cnp->cn_cred, 0))) {
UFS_VFREE(tvp, ip->i_number, dmode);
UFS_WAPBL_END(dvp->v_mount);
fstrans_done(dvp->v_mount);
vput(tvp);
return (error);
}
#endif
ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE;
ip->i_mode = dmode;
DIP_ASSIGN(ip, mode, dmode);
tvp->v_type = VDIR; /* Rest init'd in getnewvnode(). */
ip->i_nlink = 2;
DIP_ASSIGN(ip, nlink, 2);
if (cnp->cn_flags & ISWHITEOUT) {
ip->i_flags |= UF_OPAQUE;
DIP_ASSIGN(ip, flags, ip->i_flags);
}
/*
* Bump link count in parent directory to reflect work done below.
* Should be done before reference is created so cleanup is
* possible if we crash.
*/
dp->i_nlink++;
DIP_ASSIGN(dp, nlink, dp->i_nlink);
dp->i_flag |= IN_CHANGE;
if ((error = UFS_UPDATE(dvp, NULL, NULL, UPDATE_DIROP)) != 0)
goto bad;
/*
* Initialize directory with "." and ".." from static template.
*/
dirtemplate = mastertemplate;
dirtemplate.dotdot_reclen = dirblksiz - dirtemplate.dot_reclen;
dirtemplate.dot_ino = ufs_rw32(ip->i_number, UFS_MPNEEDSWAP(ump));
dirtemplate.dotdot_ino = ufs_rw32(dp->i_number, UFS_MPNEEDSWAP(ump));
dirtemplate.dot_reclen = ufs_rw16(dirtemplate.dot_reclen,
UFS_MPNEEDSWAP(ump));
dirtemplate.dotdot_reclen = ufs_rw16(dirtemplate.dotdot_reclen,
UFS_MPNEEDSWAP(ump));
if (ump->um_maxsymlinklen <= 0) {
#if BYTE_ORDER == LITTLE_ENDIAN
if (UFS_MPNEEDSWAP(ump) == 0)
#else
if (UFS_MPNEEDSWAP(ump) != 0)
#endif
{
dirtemplate.dot_type = dirtemplate.dot_namlen;
dirtemplate.dotdot_type = dirtemplate.dotdot_namlen;
dirtemplate.dot_namlen = dirtemplate.dotdot_namlen = 0;
} else
dirtemplate.dot_type = dirtemplate.dotdot_type = 0;
}
if ((error = UFS_BALLOC(tvp, (off_t)0, dirblksiz, cnp->cn_cred,
B_CLRBUF, &bp)) != 0)
goto bad;
ip->i_size = dirblksiz;
DIP_ASSIGN(ip, size, dirblksiz);
ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE;
uvm_vnp_setsize(tvp, ip->i_size);
memcpy((void *)bp->b_data, (void *)&dirtemplate, sizeof dirtemplate);
/*
* Directory set up, now install its entry in the parent directory.
* We must write out the buffer containing the new directory body
* before entering the new name in the parent.
*/
if ((error = VOP_BWRITE(bp->b_vp, bp)) != 0)
goto bad;
if ((error = UFS_UPDATE(tvp, NULL, NULL, UPDATE_DIROP)) != 0) {
goto bad;
}
newdir = pool_cache_get(ufs_direct_cache, PR_WAITOK);
ufs_makedirentry(ip, cnp, newdir);
error = ufs_direnter(dvp, ulr, tvp, newdir, cnp, bp);
pool_cache_put(ufs_direct_cache, newdir);
bad:
if (error == 0) {
VN_KNOTE(dvp, NOTE_WRITE | NOTE_LINK);
VOP_UNLOCK(tvp);
UFS_WAPBL_END(dvp->v_mount);
} else {
dp->i_nlink--;
DIP_ASSIGN(dp, nlink, dp->i_nlink);
dp->i_flag |= IN_CHANGE;
UFS_WAPBL_UPDATE(dvp, NULL, NULL, UPDATE_DIROP);
/*
* No need to do an explicit UFS_TRUNCATE here, vrele will
* do this for us because we set the link count to 0.
*/
ip->i_nlink = 0;
DIP_ASSIGN(ip, nlink, 0);
ip->i_flag |= IN_CHANGE;
UFS_WAPBL_UPDATE(tvp, NULL, NULL, UPDATE_DIROP);
UFS_WAPBL_END(dvp->v_mount);
vput(tvp);
}
out:
fstrans_done(dvp->v_mount);
return (error);
}
/* ARGSUSED */
int
ufs_getattr(void *v)
{
struct vop_getattr_args /* {
struct vnode *a_vp;
struct vattr *a_vap;
kauth_cred_t a_cred;
} */ *ap = v;
struct vnode *vp;
struct inode *ip;
struct vattr *vap;
vp = ap->a_vp;
ip = VTOI(vp);
vap = ap->a_vap;
fstrans_start(vp->v_mount, FSTRANS_SHARED);
UFS_ITIMES(vp, NULL, NULL, NULL);
/*
* Copy from inode table
*/
vap->va_fsid = ip->i_dev;
vap->va_fileid = ip->i_number;
vap->va_mode = ip->i_mode & ALLPERMS;
vap->va_nlink = ip->i_nlink;
vap->va_uid = ip->i_uid;
vap->va_gid = ip->i_gid;
vap->va_size = vp->v_size;
if (ip->i_ump->um_fstype == UFS1) {
vap->va_rdev = (dev_t)ufs_rw32(ip->i_ffs1_rdev,
UFS_MPNEEDSWAP(ip->i_ump));
vap->va_atime.tv_sec = ip->i_ffs1_atime;
vap->va_atime.tv_nsec = ip->i_ffs1_atimensec;
vap->va_mtime.tv_sec = ip->i_ffs1_mtime;
vap->va_mtime.tv_nsec = ip->i_ffs1_mtimensec;
vap->va_ctime.tv_sec = ip->i_ffs1_ctime;
vap->va_ctime.tv_nsec = ip->i_ffs1_ctimensec;
vap->va_birthtime.tv_sec = 0;
vap->va_birthtime.tv_nsec = 0;
vap->va_bytes = dbtob((u_quad_t)ip->i_ffs1_blocks);
} else {
vap->va_rdev = (dev_t)ufs_rw64(ip->i_ffs2_rdev,
UFS_MPNEEDSWAP(ip->i_ump));
vap->va_atime.tv_sec = ip->i_ffs2_atime;
vap->va_atime.tv_nsec = ip->i_ffs2_atimensec;
vap->va_mtime.tv_sec = ip->i_ffs2_mtime;
vap->va_mtime.tv_nsec = ip->i_ffs2_mtimensec;
vap->va_ctime.tv_sec = ip->i_ffs2_ctime;
vap->va_ctime.tv_nsec = ip->i_ffs2_ctimensec;
vap->va_birthtime.tv_sec = ip->i_ffs2_birthtime;
vap->va_birthtime.tv_nsec = ip->i_ffs2_birthnsec;
vap->va_bytes = dbtob(ip->i_ffs2_blocks);
}
vap->va_gen = ip->i_gen;
vap->va_flags = ip->i_flags;
/* this doesn't belong here */
if (vp->v_type == VBLK)
vap->va_blocksize = BLKDEV_IOSIZE;
else if (vp->v_type == VCHR)
vap->va_blocksize = MAXBSIZE;
else
vap->va_blocksize = vp->v_mount->mnt_stat.f_iosize;
vap->va_type = vp->v_type;
vap->va_filerev = ip->i_modrev;
fstrans_done(vp->v_mount);
return (0);
}
/* ARGSUSED */
int
ufs_mknod(void *v)
{
struct vop_mknod_v3_args /* {
struct vnode *a_dvp;
struct vnode **a_vpp;
struct componentname *a_cnp;
struct vattr *a_vap;
} */ *ap = v;
struct vattr *vap;
struct vnode **vpp;
struct inode *ip;
int error;
struct mount *mp;
ino_t ino;
struct ufs_lookup_results *ulr;
vap = ap->a_vap;
vpp = ap->a_vpp;
/* XXX should handle this material another way */
ulr = &VTOI(ap->a_dvp)->i_crap;
UFS_CHECK_CRAPCOUNTER(VTOI(ap->a_dvp));
/*
* UFS_WAPBL_BEGIN1(dvp->v_mount, dvp) performed by successful
* ufs_makeinode
*/
fstrans_start(ap->a_dvp->v_mount, FSTRANS_SHARED);
if ((error =
ufs_makeinode(MAKEIMODE(vap->va_type, vap->va_mode),
ap->a_dvp, ulr, vpp, ap->a_cnp)) != 0)
goto out;
VN_KNOTE(ap->a_dvp, NOTE_WRITE);
ip = VTOI(*vpp);
mp = (*vpp)->v_mount;
ino = ip->i_number;
ip->i_flag |= IN_ACCESS | IN_CHANGE | IN_UPDATE;
if (vap->va_rdev != VNOVAL) {
struct ufsmount *ump = ip->i_ump;
/*
* Want to be able to use this to make badblock
* inodes, so don't truncate the dev number.
*/
if (ump->um_fstype == UFS1)
ip->i_ffs1_rdev = ufs_rw32(vap->va_rdev,
UFS_MPNEEDSWAP(ump));
else
ip->i_ffs2_rdev = ufs_rw64(vap->va_rdev,
UFS_MPNEEDSWAP(ump));
}
UFS_WAPBL_UPDATE(*vpp, NULL, NULL, 0);
UFS_WAPBL_END1(ap->a_dvp->v_mount, ap->a_dvp);
/*
* Remove inode so that it will be reloaded by vcache_get and
* checked to see if it is an alias of an existing entry in
* the inode cache.
*/
(*vpp)->v_type = VNON;
VOP_UNLOCK(*vpp);
vgone(*vpp);
error = vcache_get(mp, &ino, sizeof(ino), vpp);
out:
fstrans_done(ap->a_dvp->v_mount);
if (error != 0) {
*vpp = NULL;
return (error);
}
return (0);
}
/*
* Vnode op for reading directories.
*
* This routine handles converting from the on-disk directory format
* "struct direct" to the in-memory format "struct dirent" as well as
* byte swapping the entries if necessary.
*/
int
ufs_readdir(void *v)
{
struct vop_readdir_args /* {
struct vnode *a_vp;
struct uio *a_uio;
kauth_cred_t a_cred;
int *a_eofflag;
off_t **a_cookies;
int *ncookies;
} */ *ap = v;
struct vnode *vp = ap->a_vp;
struct direct *cdp, *ecdp;
struct dirent *ndp;
char *cdbuf, *ndbuf, *endp;
struct uio auio, *uio;
struct iovec aiov;
int error;
size_t count, ccount, rcount, cdbufsz, ndbufsz;
off_t off, *ccp;
off_t startoff;
size_t skipbytes;
struct ufsmount *ump = VFSTOUFS(vp->v_mount);
int nswap = UFS_MPNEEDSWAP(ump);
#if BYTE_ORDER == LITTLE_ENDIAN
int needswap = ump->um_maxsymlinklen <= 0 && nswap == 0;
#else
int needswap = ump->um_maxsymlinklen <= 0 && nswap != 0;
#endif
uio = ap->a_uio;
count = uio->uio_resid;
rcount = count - ((uio->uio_offset + count) & (ump->um_dirblksiz - 1));
if (rcount < _DIRENT_MINSIZE(cdp) || count < _DIRENT_MINSIZE(ndp))
return EINVAL;
startoff = uio->uio_offset & ~(ump->um_dirblksiz - 1);
skipbytes = uio->uio_offset - startoff;
rcount += skipbytes;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = startoff;
auio.uio_resid = rcount;
UIO_SETUP_SYSSPACE(&auio);
auio.uio_rw = UIO_READ;
cdbufsz = rcount;
cdbuf = kmem_alloc(cdbufsz, KM_SLEEP);
aiov.iov_base = cdbuf;
aiov.iov_len = rcount;
error = VOP_READ(vp, &auio, 0, ap->a_cred);
if (error != 0) {
kmem_free(cdbuf, cdbufsz);
return error;
}
rcount -= auio.uio_resid;
cdp = (struct direct *)(void *)cdbuf;
ecdp = (struct direct *)(void *)&cdbuf[rcount];
ndbufsz = count;
ndbuf = kmem_alloc(ndbufsz, KM_SLEEP);
ndp = (struct dirent *)(void *)ndbuf;
endp = &ndbuf[count];
off = uio->uio_offset;
if (ap->a_cookies) {
ccount = rcount / _DIRENT_RECLEN(cdp, 1);
ccp = *(ap->a_cookies) = malloc(ccount * sizeof(*ccp),
M_TEMP, M_WAITOK);
} else {
/* XXX: GCC */
ccount = 0;
ccp = NULL;
}
while (cdp < ecdp) {
cdp->d_reclen = ufs_rw16(cdp->d_reclen, nswap);
if (skipbytes > 0) {
if (cdp->d_reclen <= skipbytes) {
skipbytes -= cdp->d_reclen;
cdp = _DIRENT_NEXT(cdp);
continue;
}
/*
* invalid cookie.
*/
error = EINVAL;
goto out;
}
if (cdp->d_reclen == 0) {
struct dirent *ondp = ndp;
ndp->d_reclen = _DIRENT_MINSIZE(ndp);
ndp = _DIRENT_NEXT(ndp);
ondp->d_reclen = 0;
cdp = ecdp;
break;
}
if (needswap) {
ndp->d_type = cdp->d_namlen;
ndp->d_namlen = cdp->d_type;
} else {
ndp->d_type = cdp->d_type;
ndp->d_namlen = cdp->d_namlen;
}
ndp->d_reclen = _DIRENT_RECLEN(ndp, ndp->d_namlen);
if ((char *)(void *)ndp + ndp->d_reclen +
_DIRENT_MINSIZE(ndp) > endp)
break;
ndp->d_fileno = ufs_rw32(cdp->d_ino, nswap);
(void)memcpy(ndp->d_name, cdp->d_name, ndp->d_namlen);
memset(&ndp->d_name[ndp->d_namlen], 0,
ndp->d_reclen - _DIRENT_NAMEOFF(ndp) - ndp->d_namlen);
off += cdp->d_reclen;
if (ap->a_cookies) {
KASSERT(ccp - *(ap->a_cookies) < ccount);
*(ccp++) = off;
}
ndp = _DIRENT_NEXT(ndp);
cdp = _DIRENT_NEXT(cdp);
}
count = ((char *)(void *)ndp - ndbuf);
error = uiomove(ndbuf, count, uio);
out:
if (ap->a_cookies) {
if (error) {
free(*(ap->a_cookies), M_TEMP);
*(ap->a_cookies) = NULL;
*(ap->a_ncookies) = 0;
} else {
*ap->a_ncookies = ccp - *(ap->a_cookies);
}
}
uio->uio_offset = off;
kmem_free(ndbuf, ndbufsz);
kmem_free(cdbuf, cdbufsz);
*ap->a_eofflag = VTOI(vp)->i_size <= uio->uio_offset;
return error;
}
/*
* Debugging function to check that the dirhash information about
* a directory block matches its actual contents. Panics if a mismatch
* is detected.
*
* On entry, `sbuf' should point to the start of an in-core
* DIRBLKSIZ-sized directory block, and `offset' should contain the
* offset from the start of the directory of that block.
*/
void
ufsdirhash_checkblock(struct inode *ip, char *sbuf, doff_t offset)
{
struct dirhash *dh;
struct direct *dp;
int block, ffslot, i, nfree;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
if (!ufs_dirhashcheck)
return;
if ((dh = ip->i_dirhash) == NULL)
return;
DIRHASH_LOCK(dh);
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return;
}
block = offset / dirblksiz;
if ((offset & (dirblksiz - 1)) != 0 || block >= dh->dh_dirblks)
panic("ufsdirhash_checkblock: bad offset");
nfree = 0;
for (i = 0; i < dirblksiz; i += dp->d_reclen) {
dp = (struct direct *)(sbuf + i);
if (dp->d_reclen == 0 || i + dp->d_reclen > dirblksiz)
panic("ufsdirhash_checkblock: bad dir");
if (dp->d_ino == 0) {
#if 0
/*
* XXX entries with d_ino == 0 should only occur
* at the start of a DIRBLKSIZ block. However the
* ufs code is tolerant of such entries at other
* offsets, and fsck does not fix them.
*/
if (i != 0)
panic("ufsdirhash_checkblock: bad dir inode");
#endif
nfree += dp->d_reclen;
continue;
}
/* Check that the entry exists (will panic if it doesn't). */
ufsdirhash_findslot(dh, dp->d_name, dp->d_namlen, offset + i);
nfree += dp->d_reclen - UFS_DIRSIZ(0, dp, needswap);
}
if (i != dirblksiz)
panic("ufsdirhash_checkblock: bad dir end");
if (dh->dh_blkfree[block] * DIRALIGN != nfree)
panic("ufsdirhash_checkblock: bad free count");
ffslot = BLKFREE2IDX(nfree / DIRALIGN);
for (i = 0; i <= DH_NFSTATS; i++)
if (dh->dh_firstfree[i] == block && i != ffslot)
panic("ufsdirhash_checkblock: bad first-free");
if (dh->dh_firstfree[ffslot] == -1)
panic("ufsdirhash_checkblock: missing first-free entry");
DIRHASH_UNLOCK(dh);
}
/*
* Find a directory block with room for 'slotneeded' bytes. Returns
* the offset of the directory entry that begins the free space.
* This will either be the offset of an existing entry that has free
* space at the end, or the offset of an entry with d_ino == 0 at
* the start of a UFS_DIRBLKSIZ block.
*
* To use the space, the caller may need to compact existing entries in
* the directory. The total number of bytes in all of the entries involved
* in the compaction is stored in *slotsize. In other words, all of
* the entries that must be compacted are exactly contained in the
* region beginning at the returned offset and spanning *slotsize bytes.
*
* Returns -1 if no space was found, indicating that the directory
* must be extended.
*/
doff_t
ufsdirhash_findfree(struct inode *ip, int slotneeded, int *slotsize)
{
struct direct *dp;
struct dirhash *dh;
struct buf *bp;
doff_t pos, slotstart;
int dirblock, error, freebytes, i;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
if ((dh = ip->i_dirhash) == NULL)
return (-1);
DIRHASH_LOCK(dh);
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return (-1);
}
/* Find a directory block with the desired free space. */
dirblock = -1;
for (i = howmany(slotneeded, DIRALIGN); i <= DH_NFSTATS; i++)
if ((dirblock = dh->dh_firstfree[i]) != -1)
break;
if (dirblock == -1) {
DIRHASH_UNLOCK(dh);
return (-1);
}
KASSERT(dirblock < dh->dh_nblk &&
dh->dh_blkfree[dirblock] >= howmany(slotneeded, DIRALIGN));
pos = dirblock * dirblksiz;
error = ufs_blkatoff(ip->i_vnode, (off_t)pos, (void *)&dp, &bp, false);
if (error) {
DIRHASH_UNLOCK(dh);
return (-1);
}
/* Find the first entry with free space. */
for (i = 0; i < dirblksiz; ) {
if (dp->d_reclen == 0) {
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
return (-1);
}
if (dp->d_ino == 0 || dp->d_reclen > UFS_DIRSIZ(0, dp, needswap))
break;
i += dp->d_reclen;
dp = (struct direct *)((char *)dp + dp->d_reclen);
}
if (i > dirblksiz) {
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
return (-1);
}
slotstart = pos + i;
/* Find the range of entries needed to get enough space */
freebytes = 0;
while (i < dirblksiz && freebytes < slotneeded) {
freebytes += dp->d_reclen;
if (dp->d_ino != 0)
freebytes -= UFS_DIRSIZ(0, dp, needswap);
if (dp->d_reclen == 0) {
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
return (-1);
}
i += dp->d_reclen;
dp = (struct direct *)((char *)dp + dp->d_reclen);
}
if (i > dirblksiz) {
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
return (-1);
}
if (freebytes < slotneeded)
panic("ufsdirhash_findfree: free mismatch");
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
*slotsize = pos + i - slotstart;
return (slotstart);
}
/*
* Find the offset of the specified name within the given inode.
* Returns 0 on success, ENOENT if the entry does not exist, or
* EJUSTRETURN if the caller should revert to a linear search.
*
* If successful, the directory offset is stored in *offp, and a
* pointer to a struct buf containing the entry is stored in *bpp. If
* prevoffp is non-NULL, the offset of the previous entry within
* the UFS_DIRBLKSIZ-sized block is stored in *prevoffp (if the entry
* is the first in a block, the start of the block is used).
*/
int
ufsdirhash_lookup(struct inode *ip, const char *name, int namelen, doff_t *offp,
struct buf **bpp, doff_t *prevoffp)
{
struct dirhash *dh, *dh_next;
struct direct *dp;
struct vnode *vp;
struct buf *bp;
doff_t blkoff, bmask, offset, prevoff;
int i, slot;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
if ((dh = ip->i_dirhash) == NULL)
return (EJUSTRETURN);
/*
* Move this dirhash towards the end of the list if it has a
* score higher than the next entry, and acquire the dh_lock.
* Optimise the case where it's already the last by performing
* an unlocked read of the TAILQ_NEXT pointer.
*
* In both cases, end up holding just dh_lock.
*/
if (TAILQ_NEXT(dh, dh_list) != NULL) {
DIRHASHLIST_LOCK();
DIRHASH_LOCK(dh);
/*
* If the new score will be greater than that of the next
* entry, then move this entry past it. With both mutexes
* held, dh_next won't go away, but its dh_score could
* change; that's not important since it is just a hint.
*/
if (dh->dh_hash != NULL &&
(dh_next = TAILQ_NEXT(dh, dh_list)) != NULL &&
dh->dh_score >= dh_next->dh_score) {
KASSERT(dh->dh_onlist);
TAILQ_REMOVE(&ufsdirhash_list, dh, dh_list);
TAILQ_INSERT_AFTER(&ufsdirhash_list, dh_next, dh,
dh_list);
}
DIRHASHLIST_UNLOCK();
} else {
/* Already the last, though that could change as we wait. */
DIRHASH_LOCK(dh);
}
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
ufsdirhash_free(ip);
return (EJUSTRETURN);
}
/* Update the score. */
if (dh->dh_score < DH_SCOREMAX)
dh->dh_score++;
vp = ip->i_vnode;
bmask = VFSTOUFS(vp->v_mount)->um_mountp->mnt_stat.f_iosize - 1;
blkoff = -1;
bp = NULL;
restart:
slot = ufsdirhash_hash(dh, name, namelen);
if (dh->dh_seqopt) {
/*
* Sequential access optimisation. dh_seqoff contains the
* offset of the directory entry immediately following
* the last entry that was looked up. Check if this offset
* appears in the hash chain for the name we are looking for.
*/
for (i = slot; (offset = DH_ENTRY(dh, i)) != DIRHASH_EMPTY;
i = WRAPINCR(i, dh->dh_hlen))
if (offset == dh->dh_seqoff)
break;
if (offset == dh->dh_seqoff) {
/*
* We found an entry with the expected offset. This
* is probably the entry we want, but if not, the
* code below will turn off seqoff and retry.
*/
slot = i;
} else
dh->dh_seqopt = 0;
}
for (; (offset = DH_ENTRY(dh, slot)) != DIRHASH_EMPTY;
slot = WRAPINCR(slot, dh->dh_hlen)) {
if (offset == DIRHASH_DEL)
continue;
if (offset < 0 || offset >= ip->i_size)
panic("ufsdirhash_lookup: bad offset in hash array");
if ((offset & ~bmask) != blkoff) {
if (bp != NULL)
brelse(bp, 0);
blkoff = offset & ~bmask;
if (ufs_blkatoff(vp, (off_t)blkoff,
NULL, &bp, false) != 0) {
DIRHASH_UNLOCK(dh);
return (EJUSTRETURN);
}
}
dp = (struct direct *)((char *)bp->b_data + (offset & bmask));
if (dp->d_reclen == 0 || dp->d_reclen >
dirblksiz - (offset & (dirblksiz - 1))) {
/* Corrupted directory. */
DIRHASH_UNLOCK(dh);
brelse(bp, 0);
return (EJUSTRETURN);
}
if (dp->d_namlen == namelen &&
memcmp(dp->d_name, name, namelen) == 0) {
/* Found. Get the prev offset if needed. */
if (prevoffp != NULL) {
if (offset & (dirblksiz - 1)) {
prevoff = ufsdirhash_getprev(dp,
offset, dirblksiz);
if (prevoff == -1) {
brelse(bp, 0);
return (EJUSTRETURN);
}
} else
prevoff = offset;
*prevoffp = prevoff;
}
/* Check for sequential access, and update offset. */
if (dh->dh_seqopt == 0 && dh->dh_seqoff == offset)
dh->dh_seqopt = 1;
dh->dh_seqoff = offset + UFS_DIRSIZ(0, dp, needswap);
DIRHASH_UNLOCK(dh);
*bpp = bp;
*offp = offset;
return (0);
}
if (dh->dh_hash == NULL) {
DIRHASH_UNLOCK(dh);
if (bp != NULL)
brelse(bp, 0);
ufsdirhash_free(ip);
return (EJUSTRETURN);
}
/*
* When the name doesn't match in the seqopt case, go back
* and search normally.
*/
if (dh->dh_seqopt) {
dh->dh_seqopt = 0;
goto restart;
}
}
DIRHASH_UNLOCK(dh);
if (bp != NULL)
brelse(bp, 0);
return (ENOENT);
}
/*
* Attempt to build up a hash table for the directory contents in
* inode 'ip'. Returns 0 on success, or -1 of the operation failed.
*/
int
ufsdirhash_build(struct inode *ip)
{
struct dirhash *dh;
struct buf *bp = NULL;
struct direct *ep;
struct vnode *vp;
doff_t bmask, pos;
int dirblocks, i, j, memreqd, nblocks, narrays, nslots, slot;
const int needswap = UFS_MPNEEDSWAP(ip->i_ump);
int dirblksiz = ip->i_ump->um_dirblksiz;
/* Check if we can/should use dirhash. */
if (ip->i_dirhash == NULL) {
if (ip->i_size < (ufs_dirhashminblks * dirblksiz) || OFSFMT(ip))
return (-1);
} else {
/* Hash exists, but sysctls could have changed. */
if (ip->i_size < (ufs_dirhashminblks * dirblksiz) ||
ufs_dirhashmem > ufs_dirhashmaxmem) {
ufsdirhash_free(ip);
return (-1);
}
/* Check if hash exists and is intact (note: unlocked read). */
if (ip->i_dirhash->dh_hash != NULL)
return (0);
/* Free the old, recycled hash and build a new one. */
ufsdirhash_free(ip);
}
/* Don't hash removed directories. */
if (ip->i_nlink == 0)
return (-1);
vp = ip->i_vnode;
/* Allocate 50% more entries than this dir size could ever need. */
KASSERT(ip->i_size >= dirblksiz);
nslots = ip->i_size / UFS_DIRECTSIZ(1);
nslots = (nslots * 3 + 1) / 2;
narrays = howmany(nslots, DH_NBLKOFF);
nslots = narrays * DH_NBLKOFF;
dirblocks = howmany(ip->i_size, dirblksiz);
nblocks = (dirblocks * 3 + 1) / 2;
memreqd = sizeof(*dh) + narrays * sizeof(*dh->dh_hash) +
narrays * DH_NBLKOFF * sizeof(**dh->dh_hash) +
nblocks * sizeof(*dh->dh_blkfree);
while (atomic_add_int_nv(&ufs_dirhashmem, memreqd) >
ufs_dirhashmaxmem) {
atomic_add_int(&ufs_dirhashmem, -memreqd);
if (memreqd > ufs_dirhashmaxmem / 2)
return (-1);
/* Try to free some space. */
if (ufsdirhash_recycle(memreqd) != 0)
return (-1);
else
DIRHASHLIST_UNLOCK();
}
/*
* Use non-blocking mallocs so that we will revert to a linear
* lookup on failure rather than potentially blocking forever.
*/
dh = pool_cache_get(ufsdirhash_cache, PR_NOWAIT);
if (dh == NULL) {
atomic_add_int(&ufs_dirhashmem, -memreqd);
return (-1);
}
memset(dh, 0, sizeof(*dh));
mutex_init(&dh->dh_lock, MUTEX_DEFAULT, IPL_NONE);
DIRHASH_LOCK(dh);
dh->dh_hashsz = narrays * sizeof(dh->dh_hash[0]);
dh->dh_hash = kmem_zalloc(dh->dh_hashsz, KM_NOSLEEP);
dh->dh_blkfreesz = nblocks * sizeof(dh->dh_blkfree[0]);
dh->dh_blkfree = kmem_zalloc(dh->dh_blkfreesz, KM_NOSLEEP);
if (dh->dh_hash == NULL || dh->dh_blkfree == NULL)
goto fail;
for (i = 0; i < narrays; i++) {
if ((dh->dh_hash[i] = DIRHASH_BLKALLOC()) == NULL)
goto fail;
for (j = 0; j < DH_NBLKOFF; j++)
dh->dh_hash[i][j] = DIRHASH_EMPTY;
}
/* Initialise the hash table and block statistics. */
dh->dh_narrays = narrays;
dh->dh_hlen = nslots;
dh->dh_nblk = nblocks;
dh->dh_dirblks = dirblocks;
for (i = 0; i < dirblocks; i++)
dh->dh_blkfree[i] = dirblksiz / DIRALIGN;
for (i = 0; i < DH_NFSTATS; i++)
dh->dh_firstfree[i] = -1;
dh->dh_firstfree[DH_NFSTATS] = 0;
dh->dh_seqopt = 0;
dh->dh_seqoff = 0;
dh->dh_score = DH_SCOREINIT;
ip->i_dirhash = dh;
bmask = VFSTOUFS(vp->v_mount)->um_mountp->mnt_stat.f_iosize - 1;
pos = 0;
while (pos < ip->i_size) {
if ((curcpu()->ci_schedstate.spc_flags & SPCF_SHOULDYIELD)
!= 0) {
preempt();
}
/* If necessary, get the next directory block. */
if ((pos & bmask) == 0) {
if (bp != NULL)
brelse(bp, 0);
if (ufs_blkatoff(vp, (off_t)pos, NULL, &bp, false) != 0)
goto fail;
}
/* Add this entry to the hash. */
ep = (struct direct *)((char *)bp->b_data + (pos & bmask));
if (ep->d_reclen == 0 || ep->d_reclen >
dirblksiz - (pos & (dirblksiz - 1))) {
/* Corrupted directory. */
brelse(bp, 0);
goto fail;
}
if (ep->d_ino != 0) {
/* Add the entry (simplified ufsdirhash_add). */
slot = ufsdirhash_hash(dh, ep->d_name, ep->d_namlen);
while (DH_ENTRY(dh, slot) != DIRHASH_EMPTY)
slot = WRAPINCR(slot, dh->dh_hlen);
dh->dh_hused++;
DH_ENTRY(dh, slot) = pos;
ufsdirhash_adjfree(dh, pos, -UFS_DIRSIZ(0, ep, needswap),
dirblksiz);
}
pos += ep->d_reclen;
}
if (bp != NULL)
brelse(bp, 0);
DIRHASHLIST_LOCK();
TAILQ_INSERT_TAIL(&ufsdirhash_list, dh, dh_list);
dh->dh_onlist = 1;
DIRHASH_UNLOCK(dh);
DIRHASHLIST_UNLOCK();
return (0);
fail:
DIRHASH_UNLOCK(dh);
if (dh->dh_hash != NULL) {
for (i = 0; i < narrays; i++)
if (dh->dh_hash[i] != NULL)
DIRHASH_BLKFREE(dh->dh_hash[i]);
kmem_free(dh->dh_hash, dh->dh_hashsz);
}
if (dh->dh_blkfree != NULL)
kmem_free(dh->dh_blkfree, dh->dh_blkfreesz);
mutex_destroy(&dh->dh_lock);
pool_cache_put(ufsdirhash_cache, dh);
ip->i_dirhash = NULL;
atomic_add_int(&ufs_dirhashmem, -memreqd);
return (-1);
}
