/*
* Lock an inode, with SYNCQ semantics.
*
* HAMMER2 offers shared and exclusive locks on inodes. Pass a mask of
* flags for options:
*
* - pass HAMMER2_RESOLVE_SHARED if a shared lock is desired. The
* inode locking function will automatically set the RDONLY flag.
* shared locks are not subject to SYNCQ semantics, exclusive locks
* are.
*
* - pass HAMMER2_RESOLVE_ALWAYS if you need the inode's meta-data.
* Most front-end inode locks do.
*
* - pass HAMMER2_RESOLVE_NEVER if you do not want to require that
* the inode data be resolved. This is used by the syncthr because
* it can run on an unresolved/out-of-sync cluster, and also by the
* vnode reclamation code to avoid unnecessary I/O (particularly when
* disposing of hundreds of thousands of cached vnodes).
*
* This function, along with lock4, has SYNCQ semantics. If the inode being
* locked is on the SYNCQ, that is it has been staged by the syncer, we must
* block until the operation is complete (even if we can lock the inode). In
* order to reduce the stall time, we re-order the inode to the front of the
* pmp->syncq prior to blocking. This reordering VERY significantly improves
* performance.
*
* The inode locking function locks the inode itself, resolves any stale
* chains in the inode's cluster, and allocates a fresh copy of the
* cluster with 1 ref and all the underlying chains locked.
*
* ip->cluster will be stable while the inode is locked.
*
* NOTE: We don't combine the inode/chain lock because putting away an
* inode would otherwise confuse multiple lock holders of the inode.
*/
void
hammer2_inode_lock(hammer2_inode_t *ip, int how)
{
hammer2_pfs_t *pmp;
hammer2_inode_ref(ip);
pmp = ip->pmp;
/*
* Inode structure mutex - Shared lock
*/
if (how & HAMMER2_RESOLVE_SHARED) {
hammer2_mtx_sh(&ip->lock);
return;
}
/*
* Inode structure mutex - Exclusive lock
*
* An exclusive lock (if not recursive) must wait for inodes on
* SYNCQ to flush first, to ensure that meta-data dependencies such
* as the nlink count and related directory entries are not split
* across flushes.
*
* If the vnode is locked by the current thread it must be unlocked
* across the tsleep() to avoid a deadlock.
*/
hammer2_mtx_ex(&ip->lock);
if (hammer2_mtx_refs(&ip->lock) > 1)
return;
while ((ip->flags & HAMMER2_INODE_SYNCQ) && pmp) {
hammer2_spin_ex(&pmp->list_spin);
if (ip->flags & HAMMER2_INODE_SYNCQ) {
tsleep_interlock(&ip->flags, 0);
atomic_set_int(&ip->flags, HAMMER2_INODE_SYNCQ_WAKEUP);
TAILQ_REMOVE(&pmp->syncq, ip, entry);
TAILQ_INSERT_HEAD(&pmp->syncq, ip, entry);
hammer2_spin_unex(&pmp->list_spin);
hammer2_mtx_unlock(&ip->lock);
tsleep(&ip->flags, PINTERLOCKED, "h2sync", 0);
hammer2_mtx_ex(&ip->lock);
continue;
}
hammer2_spin_unex(&pmp->list_spin);
break;
}
}
/*
* Upgrade a shared inode lock to exclusive and return. If the inode lock
* is already held exclusively this is a NOP.
*
* The caller MUST hold the inode lock either shared or exclusive on call
* and will own the lock exclusively on return.
*
* Returns non-zero if the lock was already exclusive prior to the upgrade.
*/
int
hammer2_inode_lock_upgrade(hammer2_inode_t *ip)
{
int wasexclusive;
if (mtx_islocked_ex(&ip->lock)) {
wasexclusive = 1;
} else {
hammer2_mtx_unlock(&ip->lock);
hammer2_mtx_ex(&ip->lock);
wasexclusive = 0;
}
return wasexclusive;
}
/*
* XXX this API needs a rewrite. It needs to be split into a
* hammer2_inode_alloc() and hammer2_inode_build() to allow us to get
* rid of the inode/chain lock reversal fudge.
*
* Returns the inode associated with the passed-in cluster, allocating a new
* hammer2_inode structure if necessary, then synchronizing it to the passed
* xop cluster. When synchronizing, if idx >= 0, only cluster index (idx)
* is synchronized. Otherwise the whole cluster is synchronized. inum will
* be extracted from the passed-in xop and the inum argument will be ignored.
*
* If xop is passed as NULL then a new hammer2_inode is allocated with the
* specified inum, and returned. For normal inodes, the inode will be
* indexed in memory and if it already exists the existing ip will be
* returned instead of allocating a new one. The superroot and PFS inodes
* are not indexed in memory.
*
* The passed-in cluster must be locked and will remain locked on return.
* The returned inode will be locked and the caller may dispose of both
* via hammer2_inode_unlock() + hammer2_inode_drop(). However, if the caller
* needs to resolve a hardlink it must ref/unlock/relock/drop the inode.
*
* The hammer2_inode structure regulates the interface between the high level
* kernel VNOPS API and the filesystem backend (the chains).
*
* On return the inode is locked with the supplied cluster.
*/
hammer2_inode_t *
hammer2_inode_get(hammer2_pfs_t *pmp, hammer2_xop_head_t *xop,
hammer2_tid_t inum, int idx)
{
hammer2_inode_t *nip;
const hammer2_inode_data_t *iptmp;
const hammer2_inode_data_t *nipdata;
KKASSERT(xop == NULL ||
hammer2_cluster_type(&xop->cluster) ==
HAMMER2_BREF_TYPE_INODE);
KKASSERT(pmp);
/*
* Interlocked lookup/ref of the inode. This code is only needed
* when looking up inodes with nlinks != 0 (TODO: optimize out
* otherwise and test for duplicates).
*
* Cluster can be NULL during the initial pfs allocation.
*/
if (xop) {
iptmp = &hammer2_xop_gdata(xop)->ipdata;
inum = iptmp->meta.inum;
hammer2_xop_pdata(xop);
}
again:
nip = hammer2_inode_lookup(pmp, inum);
if (nip) {
/*
* We may have to unhold the cluster to avoid a deadlock
* against vnlru (and possibly other XOPs).
*/
if (xop) {
if (hammer2_mtx_ex_try(&nip->lock) != 0) {
hammer2_cluster_unhold(&xop->cluster);
hammer2_mtx_ex(&nip->lock);
hammer2_cluster_rehold(&xop->cluster);
}
} else {
hammer2_mtx_ex(&nip->lock);
}
/*
* Handle SMP race (not applicable to the super-root spmp
* which can't index inodes due to duplicative inode numbers).
*/
if (pmp->spmp_hmp == NULL &&
(nip->flags & HAMMER2_INODE_ONRBTREE) == 0) {
hammer2_mtx_unlock(&nip->lock);
hammer2_inode_drop(nip);
goto again;
}
if (xop) {
if (idx >= 0)
hammer2_inode_repoint_one(nip, &xop->cluster,
idx);
else
hammer2_inode_repoint(nip, NULL, &xop->cluster);
}
return nip;
}
/*
* We couldn't find the inode number, create a new inode and try to
* insert it, handle insertion races.
*/
nip = kmalloc(sizeof(*nip), pmp->minode, M_WAITOK | M_ZERO);
spin_init(&nip->cluster_spin, "h2clspin");
atomic_add_long(&pmp->inmem_inodes, 1);
if (pmp->spmp_hmp)
nip->flags = HAMMER2_INODE_SROOT;
/*
* Initialize nip's cluster. A cluster is provided for normal
* inodes but typically not for the super-root or PFS inodes.
*/
nip->cluster.refs = 1;
nip->cluster.pmp = pmp;
nip->cluster.flags |= HAMMER2_CLUSTER_INODE;
if (xop) {
nipdata = &hammer2_xop_gdata(xop)->ipdata;
nip->meta = nipdata->meta;
hammer2_xop_pdata(xop);
atomic_set_int(&nip->flags, HAMMER2_INODE_METAGOOD);
hammer2_inode_repoint(nip, NULL, &xop->cluster);
} else {
nip->meta.inum = inum; /* PFS inum is always 1 XXX */
/* mtime will be updated when a cluster is available */
atomic_set_int(&nip->flags, HAMMER2_INODE_METAGOOD); /*XXX*/
}
nip->pmp = pmp;
/*
* ref and lock on nip gives it state compatible to after a
* hammer2_inode_lock() call.
*/
nip->refs = 1;
hammer2_mtx_init(&nip->lock, "h2inode");
hammer2_mtx_init(&nip->truncate_lock, "h2trunc");
hammer2_mtx_ex(&nip->lock);
TAILQ_INIT(&nip->depend_static.sideq);
/* combination of thread lock and chain lock == inode lock */
/*
* Attempt to add the inode. If it fails we raced another inode
* get. Undo all the work and try again.
*/
if (pmp->spmp_hmp == NULL) {
hammer2_spin_ex(&pmp->inum_spin);
if (RB_INSERT(hammer2_inode_tree, &pmp->inum_tree, nip)) {
hammer2_spin_unex(&pmp->inum_spin);
hammer2_mtx_unlock(&nip->lock);
hammer2_inode_drop(nip);
goto again;
}
atomic_set_int(&nip->flags, HAMMER2_INODE_ONRBTREE);
++pmp->inum_count;
hammer2_spin_unex(&pmp->inum_spin);
}
return (nip);
}
/*
* Exclusively lock up to four inodes, in order, with SYNCQ semantics.
* ip1 and ip2 must not be NULL. ip3 and ip4 may be NULL, but if ip3 is
* NULL then ip4 must also be NULL.
*
* This creates a dependency between up to four inodes.
*/
void
hammer2_inode_lock4(hammer2_inode_t *ip1, hammer2_inode_t *ip2,
hammer2_inode_t *ip3, hammer2_inode_t *ip4)
{
hammer2_inode_t *ips[4];
hammer2_inode_t *iptmp;
hammer2_inode_t *ipslp;
hammer2_depend_t *depend;
hammer2_pfs_t *pmp;
size_t count;
size_t i;
pmp = ip1->pmp; /* may be NULL */
KKASSERT(pmp == ip2->pmp);
ips[0] = ip1;
ips[1] = ip2;
if (ip3 == NULL) {
count = 2;
} else if (ip4 == NULL) {
count = 3;
ips[2] = ip3;
KKASSERT(pmp == ip3->pmp);
} else {
count = 4;
ips[2] = ip3;
ips[3] = ip4;
KKASSERT(pmp == ip3->pmp);
KKASSERT(pmp == ip4->pmp);
}
for (i = 0; i < count; ++i)
hammer2_inode_ref(ips[i]);
restart:
/*
* Lock the inodes in order
*/
for (i = 0; i < count; ++i) {
hammer2_mtx_ex(&ips[i]->lock);
}
/*
* Associate dependencies, record the first inode found on SYNCQ
* (operation is allowed to proceed for inodes on PASS2) for our
* sleep operation, this inode is theoretically the last one sync'd
* in the sequence.
*
* All inodes found on SYNCQ are moved to the head of the syncq
* to reduce stalls.
*/
hammer2_spin_ex(&pmp->list_spin);
depend = NULL;
ipslp = NULL;
for (i = 0; i < count; ++i) {
iptmp = ips[i];
depend = hammer2_inode_setdepend_locked(iptmp, depend);
if (iptmp->flags & HAMMER2_INODE_SYNCQ) {
TAILQ_REMOVE(&pmp->syncq, iptmp, entry);
TAILQ_INSERT_HEAD(&pmp->syncq, iptmp, entry);
if (ipslp == NULL)
ipslp = iptmp;
}
}
hammer2_spin_unex(&pmp->list_spin);
/*
* Block and retry if any of the inodes are on SYNCQ. It is
* important that we allow the operation to proceed in the
* PASS2 case, to avoid deadlocking against the vnode.
*/
if (ipslp) {
for (i = 0; i < count; ++i)
hammer2_mtx_unlock(&ips[i]->lock);
tsleep(&ipslp->flags, 0, "h2sync", 2);
goto restart;
}
}
/*
* Mount-wide locks
*/
void
hammer2_dev_exlock(hammer2_dev_t *hmp)
{
hammer2_mtx_ex(&hmp->vchain.lock);
}
