void di_write_unlock(struct dentry *d)
{
au_dbg_verify_dinode(d);
if (d->d_inode)
ii_write_unlock(d->d_inode);
au_rw_write_unlock(&au_di(d)->di_rwsem);
}
void di_write_unlock(struct dentry *d)
{
LKTRTrace("%.*s\n", AuDLNPair(d));
SiMustAnyLock(d->d_sb);
if (d->d_inode)
ii_write_unlock(d->d_inode);
au_rw_write_unlock(&au_di(d)->di_rwsem);
au_dbg_locked_di_unreg(d, AuLock_IW);
}
void di_read_unlock(struct dentry *d, int flags)
{
if (d->d_inode) {
if (au_ftest_lock(flags, IW))
ii_write_unlock(d->d_inode);
else if (au_ftest_lock(flags, IR))
ii_read_unlock(d->d_inode);
}
au_rw_read_unlock(&au_di(d)->di_rwsem);
}
static int aufs_commit_metadata(struct inode *inode)
{
int err;
aufs_bindex_t bindex;
struct super_block *sb;
struct inode *h_inode;
int (*f)(struct inode *inode);
sb = inode->i_sb;
si_read_lock(sb, AuLock_FLUSH);
ii_write_lock_child(inode);
bindex = au_ibstart(inode);
AuDebugOn(bindex < 0);
h_inode = au_h_iptr(inode, bindex);
f = h_inode->i_sb->s_export_op->commit_metadata;
if (f)
err = f(h_inode);
else {
struct writeback_control wbc = {
.sync_mode = WB_SYNC_ALL,
.nr_to_write = 0 /* metadata only */
};
err = sync_inode(h_inode, &wbc);
}
au_cpup_attr_timesizes(inode);
ii_write_unlock(inode);
si_read_unlock(sb);
return err;
}
/* ---------------------------------------------------------------------- */
static struct export_operations aufs_export_op = {
.fh_to_dentry = aufs_fh_to_dentry,
/* .fh_to_parent = aufs_fh_to_parent, */
.encode_fh = aufs_encode_fh,
.commit_metadata = aufs_commit_metadata
};
void au_export_init(struct super_block *sb)
{
struct au_sbinfo *sbinfo;
__u32 u;
sb->s_export_op = &aufs_export_op;
sbinfo = au_sbi(sb);
sbinfo->si_xigen = NULL;
get_random_bytes(&u, sizeof(u));
BUILD_BUG_ON(sizeof(u) != sizeof(int));
atomic_set(&sbinfo->si_xigen_next, u);
}
static int test_inode_busy(struct super_block *sb, aufs_bindex_t bindex,
unsigned int sigen, const unsigned int verbose)
{
int err;
unsigned long long max, ull;
struct inode *i, **array;
aufs_bindex_t bstart, bend;
array = au_iarray_alloc(sb, &max);
err = PTR_ERR(array);
if (IS_ERR(array))
goto out;
err = 0;
AuDbg("b%d\n", bindex);
for (ull = 0; !err && ull < max; ull++) {
i = array[ull];
if (i->i_ino == AUFS_ROOT_INO)
continue;
/* AuDbgInode(i); */
if (au_iigen(i) == sigen)
ii_read_lock_child(i);
else {
ii_write_lock_child(i);
err = au_refresh_hinode_self(i);
au_iigen_dec(i);
if (!err)
ii_downgrade_lock(i);
else {
ii_write_unlock(i);
break;
}
}
bstart = au_ibstart(i);
bend = au_ibend(i);
if (bstart <= bindex
&& bindex <= bend
&& au_h_iptr(i, bindex)
&& au_test_ibusy(i, bstart, bend)) {
err = -EBUSY;
AuVerbose(verbose, "busy i%lu\n", i->i_ino);
AuDbgInode(i);
}
ii_read_unlock(i);
}
au_iarray_free(array, max);
out:
return err;
}
void di_read_unlock(struct dentry *d, int flags)
{
if (d->d_inode) {
if (au_ftest_lock(flags, IW)) {
au_dbg_verify_dinode(d);
ii_write_unlock(d->d_inode);
} else if (au_ftest_lock(flags, IR)) {
au_dbg_verify_dinode(d);
ii_read_unlock(d->d_inode);
}
}
au_rw_read_unlock(&au_di(d)->di_rwsem);
}
void di_read_unlock(struct dentry *d, int flags)
{
LKTRTrace("%.*s\n", AuDLNPair(d));
SiMustAnyLock(d->d_sb);
if (d->d_inode) {
if (au_ftest_lock(flags, IW))
ii_write_unlock(d->d_inode);
else if (au_ftest_lock(flags, IR))
ii_read_unlock(d->d_inode);
}
au_rw_read_unlock(&au_di(d)->di_rwsem);
au_dbg_locked_di_unreg(d, flags);
}
static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
struct nameidata *nd)
{
struct dentry *ret, *parent;
struct inode *inode;
struct super_block *sb;
int err, npositive;
IMustLock(dir);
sb = dir->i_sb;
si_read_lock(sb, AuLock_FLUSH);
ret = ERR_PTR(-ENAMETOOLONG);
if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
goto out;
err = au_di_init(dentry);
ret = ERR_PTR(err);
if (unlikely(err))
goto out;
parent = dentry->d_parent; /* dir inode is locked */
di_read_lock_parent(parent, AuLock_IR);
npositive = au_lkup_dentry(dentry, au_dbstart(parent), /*type*/0, nd);
di_read_unlock(parent, AuLock_IR);
err = npositive;
ret = ERR_PTR(err);
if (unlikely(err < 0))
goto out_unlock;
inode = NULL;
if (npositive) {
inode = au_new_inode(dentry, /*must_new*/0);
ret = (void *)inode;
}
if (IS_ERR(inode))
goto out_unlock;
ret = d_splice_alias(inode, dentry);
if (unlikely(IS_ERR(ret) && inode))
ii_write_unlock(inode);
out_unlock:
di_write_unlock(dentry);
out:
si_read_unlock(sb);
return ret;
}
/*
* successful returns with iinfo write_locked
* minus: errno
* zero: success, matched
* plus: no error, but unmatched
*/
static int reval_inode(struct inode *inode, struct dentry *dentry)
{
int err;
unsigned int gen;
struct au_iigen iigen;
aufs_bindex_t bindex, bend;
struct inode *h_inode, *h_dinode;
/*
* before this function, if aufs got any iinfo lock, it must be only
* one, the parent dir.
* it can happen by UDBA and the obsoleted inode number.
*/
err = -EIO;
if (unlikely(inode->i_ino == parent_ino(dentry)))
goto out;
err = 1;
ii_write_lock_new_child(inode);
h_dinode = au_h_dptr(dentry, au_dbstart(dentry))->d_inode;
bend = au_ibend(inode);
for (bindex = au_ibstart(inode); bindex <= bend; bindex++) {
h_inode = au_h_iptr(inode, bindex);
if (!h_inode || h_inode != h_dinode)
continue;
err = 0;
gen = au_iigen(inode, &iigen);
if (gen == au_digen(dentry)
&& !au_ig_ftest(iigen.ig_flags, HALF_REFRESHED))
break;
/* fully refresh inode using dentry */
err = au_refresh_hinode(inode, dentry);
if (!err)
au_update_iigen(inode, /*half*/0);
break;
}
if (unlikely(err))
ii_write_unlock(inode);
out:
return err;
}
/* successful returns with iinfo write_locked */
static int reval_inode(struct inode *inode, struct dentry *dentry, int *matched)
{
int err;
aufs_bindex_t bindex, bend;
struct inode *h_inode, *h_dinode;
*matched = 0;
/*
* before this function, if aufs got any iinfo lock, it must be only
* one, the parent dir.
* it can happen by UDBA and the obsoleted inode number.
*/
err = -EIO;
if (unlikely(inode->i_ino == parent_ino(dentry)))
goto out;
err = 0;
ii_write_lock_new_child(inode);
h_dinode = au_h_dptr(dentry, au_dbstart(dentry))->d_inode;
bend = au_ibend(inode);
for (bindex = au_ibstart(inode); bindex <= bend; bindex++) {
h_inode = au_h_iptr(inode, bindex);
if (h_inode && h_inode == h_dinode) {
*matched = 1;
err = 0;
if (au_iigen(inode) != au_digen(dentry))
err = au_refresh_hinode(inode, dentry);
break;
}
}
if (unlikely(err))
ii_write_unlock(inode);
out:
return err;
}
static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
struct nameidata *nd)
{
struct dentry *ret, *parent;
struct inode *inode;
struct super_block *sb;
int err, npositive, lc_idx;
IMustLock(dir);
sb = dir->i_sb;
err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
ret = ERR_PTR(err);
if (unlikely(err))
goto out;
ret = ERR_PTR(-ENAMETOOLONG);
if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
goto out_si;
err = au_di_init(dentry);
ret = ERR_PTR(err);
if (unlikely(err))
goto out_si;
inode = NULL;
npositive = 0; /* suppress a warning */
parent = dentry->d_parent; /* dir inode is locked */
di_read_lock_parent(parent, AuLock_IR);
err = au_alive_dir(parent);
if (!err)
err = au_digen_test(parent, au_sigen(sb));
if (!err) {
npositive = au_lkup_dentry(dentry, au_dbstart(parent),
/*type*/0, nd);
err = npositive;
}
di_read_unlock(parent, AuLock_IR);
ret = ERR_PTR(err);
if (unlikely(err < 0))
goto out_unlock;
if (npositive) {
inode = au_new_inode(dentry, /*must_new*/0);
ret = (void *)inode;
}
if (IS_ERR(inode)) {
inode = NULL;
goto out_unlock;
}
ret = d_splice_alias(inode, dentry);
if (unlikely(IS_ERR(ret) && inode)) {
ii_write_unlock(inode);
lc_idx = AuLcNonDir_IIINFO;
if (S_ISLNK(inode->i_mode))
lc_idx = AuLcSymlink_IIINFO;
else if (S_ISDIR(inode->i_mode))
lc_idx = AuLcDir_IIINFO;
au_rw_class(&au_ii(inode)->ii_rwsem, au_lc_key + lc_idx);
iput(inode);
}
out_unlock:
di_write_unlock(dentry);
if (unlikely(IS_ERR(ret) && inode)) {
lc_idx = AuLcNonDir_DIINFO;
if (S_ISLNK(inode->i_mode))
lc_idx = AuLcSymlink_DIINFO;
else if (S_ISDIR(inode->i_mode))
lc_idx = AuLcDir_DIINFO;
au_rw_class(&au_di(dentry)->di_rwsem, au_lc_key + lc_idx);
}
out_si:
si_read_unlock(sb);
out:
return ret;
}
void di_write_unlock(struct dentry *d)
{
if (d->d_inode)
ii_write_unlock(d->d_inode);
au_rw_write_unlock(&au_di(d)->di_rwsem);
}
static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
unsigned int flags)
{
struct dentry *ret, *parent;
struct inode *inode;
struct super_block *sb;
int err, npositive;
IMustLock(dir);
/* todo: support rcu-walk? */
ret = ERR_PTR(-ECHILD);
if (flags & LOOKUP_RCU)
goto out;
ret = ERR_PTR(-ENAMETOOLONG);
if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
goto out;
sb = dir->i_sb;
err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
ret = ERR_PTR(err);
if (unlikely(err))
goto out;
err = au_di_init(dentry);
ret = ERR_PTR(err);
if (unlikely(err))
goto out_si;
inode = NULL;
npositive = 0; /* suppress a warning */
parent = dentry->d_parent; /* dir inode is locked */
di_read_lock_parent(parent, AuLock_IR);
err = au_alive_dir(parent);
if (!err)
err = au_digen_test(parent, au_sigen(sb));
if (!err) {
/* regardless LOOKUP_CREATE, always ALLOW_NEG */
npositive = au_lkup_dentry(dentry, au_dbtop(parent),
AuLkup_ALLOW_NEG);
err = npositive;
}
di_read_unlock(parent, AuLock_IR);
ret = ERR_PTR(err);
if (unlikely(err < 0))
goto out_unlock;
if (npositive) {
inode = au_new_inode(dentry, /*must_new*/0);
if (IS_ERR(inode)) {
ret = (void *)inode;
inode = NULL;
goto out_unlock;
}
}
if (inode)
atomic_inc(&inode->i_count);
ret = d_splice_alias(inode, dentry);
#if 0
if (unlikely(d_need_lookup(dentry))) {
spin_lock(&dentry->d_lock);
dentry->d_flags &= ~DCACHE_NEED_LOOKUP;
spin_unlock(&dentry->d_lock);
} else
#endif
if (inode) {
if (!IS_ERR(ret)) {
iput(inode);
if (ret && ret != dentry)
ii_write_unlock(inode);
} else {
ii_write_unlock(inode);
iput(inode);
inode = NULL;
}
}
out_unlock:
di_write_unlock(dentry);
out_si:
si_read_unlock(sb);
out:
return ret;
}
/* todo: return with unlocked? */
struct inode *au_new_inode(struct dentry *dentry, int must_new)
{
struct inode *inode, *h_inode;
struct dentry *h_dentry;
struct super_block *sb;
struct mutex *mtx;
ino_t h_ino, ino;
int err;
aufs_bindex_t bstart;
sb = dentry->d_sb;
bstart = au_dbstart(dentry);
h_dentry = au_h_dptr(dentry, bstart);
h_inode = h_dentry->d_inode;
h_ino = h_inode->i_ino;
/*
* stop 'race'-ing between hardlinks under different
* parents.
*/
mtx = NULL;
if (!S_ISDIR(h_inode->i_mode))
mtx = &au_sbr(sb, bstart)->br_xino.xi_nondir_mtx;
new_ino:
if (mtx)
mutex_lock(mtx);
err = au_xino_read(sb, bstart, h_ino, &ino);
inode = ERR_PTR(err);
if (unlikely(err))
goto out;
if (!ino) {
ino = au_xino_new_ino(sb);
if (unlikely(!ino)) {
inode = ERR_PTR(-EIO);
goto out;
}
}
AuDbg("i%lu\n", (unsigned long)ino);
inode = au_iget_locked(sb, ino);
err = PTR_ERR(inode);
if (IS_ERR(inode))
goto out;
AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW));
if (inode->i_state & I_NEW) {
/* verbose coding for lock class name */
if (unlikely(S_ISLNK(h_inode->i_mode)))
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcSymlink_IIINFO);
else if (unlikely(S_ISDIR(h_inode->i_mode)))
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcDir_IIINFO);
else /* likely */
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcNonDir_IIINFO);
ii_write_lock_new_child(inode);
err = set_inode(inode, dentry);
if (!err) {
unlock_new_inode(inode);
goto out; /* success */
}
/*
* iget_failed() calls iput(), but we need to call
* ii_write_unlock() after iget_failed(). so dirty hack for
* i_count.
*/
atomic_inc(&inode->i_count);
iget_failed(inode);
ii_write_unlock(inode);
au_xino_write(sb, bstart, h_ino, /*ino*/0);
/* ignore this error */
goto out_iput;
} else if (!must_new && !IS_DEADDIR(inode) && inode->i_nlink) {
/*
* horrible race condition between lookup, readdir and copyup
* (or something).
*/
if (mtx)
mutex_unlock(mtx);
err = reval_inode(inode, dentry);
if (unlikely(err < 0)) {
mtx = NULL;
goto out_iput;
}
if (!err) {
mtx = NULL;
goto out; /* success */
} else if (mtx)
mutex_lock(mtx);
}
if (unlikely(au_test_fs_unique_ino(h_dentry->d_inode)))
AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir,"
" b%d, %s, %pd, hi%lu, i%lu.\n",
bstart, au_sbtype(h_dentry->d_sb), dentry,
(unsigned long)h_ino, (unsigned long)ino);
ino = 0;
err = au_xino_write(sb, bstart, h_ino, /*ino*/0);
if (!err) {
iput(inode);
if (mtx)
mutex_unlock(mtx);
goto new_ino;
}
out_iput:
iput(inode);
inode = ERR_PTR(err);
out:
if (mtx)
mutex_unlock(mtx);
return inode;
}
/* todo: return with unlocked? */
struct inode *au_new_inode(struct dentry *dentry, int must_new)
{
struct inode *inode, *h_inode;
struct dentry *h_dentry;
struct super_block *sb;
struct mutex *mtx;
ino_t h_ino, ino;
int err, match;
aufs_bindex_t bstart;
sb = dentry->d_sb;
bstart = au_dbstart(dentry);
h_dentry = au_h_dptr(dentry, bstart);
h_inode = h_dentry->d_inode;
h_ino = h_inode->i_ino;
/*
* stop 'race'-ing between hardlinks under different
* parents.
*/
mtx = NULL;
if (!S_ISDIR(h_inode->i_mode))
mtx = &au_sbr(sb, bstart)->br_xino.xi_nondir_mtx;
new_ino:
if (mtx)
mutex_lock(mtx);
err = au_xino_read(sb, bstart, h_ino, &ino);
inode = ERR_PTR(err);
if (unlikely(err))
goto out;
if (!ino) {
ino = au_xino_new_ino(sb);
if (unlikely(!ino)) {
inode = ERR_PTR(-EIO);
goto out;
}
}
AuDbg("i%lu\n", (unsigned long)ino);
inode = au_iget_locked(sb, ino);
err = PTR_ERR(inode);
if (IS_ERR(inode))
goto out;
AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW));
if (inode->i_state & I_NEW) {
ii_write_lock_new_child(inode);
err = set_inode(inode, dentry);
if (!err) {
unlock_new_inode(inode);
goto out; /* success */
}
ii_write_unlock(inode);
iget_failed(inode);
goto out_err;
} else if (!must_new) {
/*
* horrible race condition between lookup, readdir and copyup
* (or something).
*/
if (mtx)
mutex_unlock(mtx);
err = reval_inode(inode, dentry, &match);
if (!err) {
mtx = NULL;
goto out; /* success */
} else if (match) {
mtx = NULL;
goto out_iput;
} else if (mtx)
mutex_lock(mtx);
}
if (unlikely(au_test_fs_unique_ino(h_dentry->d_inode)))
AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir,"
" b%d, %s, %.*s, hi%lu, i%lu.\n",
bstart, au_sbtype(h_dentry->d_sb), AuDLNPair(dentry),
(unsigned long)h_ino, (unsigned long)ino);
ino = 0;
err = au_xino_write(sb, bstart, h_ino, /*ino*/0);
if (!err) {
iput(inode);
if (mtx)
mutex_unlock(mtx);
goto new_ino;
}
out_iput:
iput(inode);
out_err:
inode = ERR_PTR(err);
out:
if (mtx)
mutex_unlock(mtx);
return inode;
}
/* todo: return with unlocked? */
struct inode *au_new_inode(struct dentry *dentry, int must_new)
{
struct inode *inode;
struct dentry *h_dentry;
struct super_block *sb;
ino_t h_ino, ino;
int err, match;
aufs_bindex_t bstart;
sb = dentry->d_sb;
bstart = au_dbstart(dentry);
h_dentry = au_h_dptr(dentry, bstart);
h_ino = h_dentry->d_inode->i_ino;
err = au_xino_read(sb, bstart, h_ino, &ino);
inode = ERR_PTR(err);
if (unlikely(err))
goto out;
new_ino:
if (!ino) {
ino = au_xino_new_ino(sb);
if (unlikely(!ino)) {
inode = ERR_PTR(-EIO);
goto out;
}
}
AuDbg("i%lu\n", (unsigned long)ino);
inode = au_iget_locked(sb, ino);
err = PTR_ERR(inode);
if (IS_ERR(inode))
goto out;
AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW));
if (inode->i_state & I_NEW) {
ii_write_lock_new_child(inode);
err = set_inode(inode, dentry);
if (!err) {
unlock_new_inode(inode);
goto out; /* success */
}
ii_write_unlock(inode);
iget_failed(inode);
goto out_err;
} else if (!must_new) {
err = reval_inode(inode, dentry, &match);
if (!err)
goto out; /* success */
else if (match)
goto out_iput;
}
if (unlikely(au_test_fs_unique_ino(h_dentry->d_inode)))
AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir,"
" b%d, %s, %.*s, hi%lu, i%lu.\n",
bstart, au_sbtype(h_dentry->d_sb), AuDLNPair(dentry),
(unsigned long)h_ino, (unsigned long)ino);
ino = 0;
err = au_xino_write(sb, bstart, h_ino, /*ino*/0);
if (!err) {
iput(inode);
goto new_ino;
}
out_iput:
iput(inode);
out_err:
inode = ERR_PTR(err);
out:
return inode;
}
static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
unsigned int flags)
{
struct dentry *ret, *parent;
struct inode *inode;
struct super_block *sb;
int err, npositive;
IMustLock(dir);
/* todo: support rcu-walk? */
ret = ERR_PTR(-ECHILD);
if (flags & LOOKUP_RCU)
goto out;
ret = ERR_PTR(-ENAMETOOLONG);
if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
goto out;
sb = dir->i_sb;
err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
ret = ERR_PTR(err);
if (unlikely(err))
goto out;
err = au_di_init(dentry);
ret = ERR_PTR(err);
if (unlikely(err))
goto out_si;
inode = NULL;
npositive = 0; /* suppress a warning */
parent = dentry->d_parent; /* dir inode is locked */
di_read_lock_parent(parent, AuLock_IR);
err = au_alive_dir(parent);
if (!err)
err = au_digen_test(parent, au_sigen(sb));
if (!err) {
npositive = au_lkup_dentry(dentry, au_dbstart(parent),
/*type*/0);
err = npositive;
}
di_read_unlock(parent, AuLock_IR);
ret = ERR_PTR(err);
if (unlikely(err < 0))
goto out_unlock;
if (npositive) {
inode = au_new_inode(dentry, /*must_new*/0);
if (IS_ERR(inode)) {
ret = (void *)inode;
inode = NULL;
goto out_unlock;
}
}
if (inode)
atomic_inc(&inode->i_count);
ret = d_splice_alias(inode, dentry);
#if 0
if (unlikely(d_need_lookup(dentry))) {
spin_lock(&dentry->d_lock);
dentry->d_flags &= ~DCACHE_NEED_LOOKUP;
spin_unlock(&dentry->d_lock);
} else
#endif
if (inode) {
if (!IS_ERR(ret)) {
iput(inode);
if (ret && ret != dentry)
ii_write_unlock(inode);
} else {
ii_write_unlock(inode);
iput(inode);
inode = NULL;
}
}
out_unlock:
di_write_unlock(dentry);
if (inode) {
/* verbose coding for lock class name */
if (unlikely(S_ISLNK(inode->i_mode)))
au_rw_class(&au_di(dentry)->di_rwsem,
au_lc_key + AuLcSymlink_DIINFO);
else if (unlikely(S_ISDIR(inode->i_mode)))
au_rw_class(&au_di(dentry)->di_rwsem,
au_lc_key + AuLcDir_DIINFO);
else /* likely */
au_rw_class(&au_di(dentry)->di_rwsem,
au_lc_key + AuLcNonDir_DIINFO);
}
out_si:
si_read_unlock(sb);
out:
return ret;
}
