static void au_br_do_del(struct super_block *sb, aufs_bindex_t bindex,
struct au_branch *br)
{
aufs_bindex_t bend;
struct au_sbinfo *sbinfo;
struct dentry *root, *h_root;
struct inode *inode, *h_inode;
struct au_hinode *hinode;
SiMustWriteLock(sb);
root = sb->s_root;
inode = root->d_inode;
sbinfo = au_sbi(sb);
bend = sbinfo->si_bend;
h_root = au_h_dptr(root, bindex);
hinode = au_hi(inode, bindex);
h_inode = au_igrab(hinode->hi_inode);
au_hiput(hinode);
au_sbilist_lock();
au_br_do_del_brp(sbinfo, bindex, bend);
au_br_do_del_hdp(au_di(root), bindex, bend);
au_br_do_del_hip(au_ii(inode), bindex, bend);
au_sbilist_unlock();
dput(h_root);
iput(h_inode);
au_br_do_free(br);
}
static int do_pri_dentry(aufs_bindex_t bindex, struct dentry *dentry)
{
struct dentry *wh = NULL;
int hn;
if (!dentry || IS_ERR(dentry)) {
dpri("d%d: err %ld\n", bindex, PTR_ERR(dentry));
return -1;
}
/* do not call dget_parent() here */
/* note: access d_xxx without d_lock */
dpri("d%d: %.*s?/%.*s, %s, cnt %d, flags 0x%x\n",
bindex,
AuDLNPair(dentry->d_parent), AuDLNPair(dentry),
dentry->d_sb ? au_sbtype(dentry->d_sb) : "??",
d_count(dentry), dentry->d_flags);
hn = -1;
if (bindex >= 0 && dentry->d_inode && au_test_aufs(dentry->d_sb)) {
struct au_iinfo *iinfo = au_ii(dentry->d_inode);
if (iinfo) {
hn = !!au_hn(iinfo->ii_hinode + bindex);
wh = iinfo->ii_hinode[0 + bindex].hi_whdentry;
}
}
do_pri_inode(bindex, dentry->d_inode, hn, wh);
return 0;
}
int au_refresh_hinode_self(struct inode *inode, int do_attr)
{
int err, e;
aufs_bindex_t bindex, new_bindex;
unsigned char update;
struct au_hinode *p, *q, tmp;
struct super_block *sb;
struct au_iinfo *iinfo;
IiMustWriteLock(inode);
update = 0;
sb = inode->i_sb;
iinfo = au_ii(inode);
err = au_ii_realloc(iinfo, au_sbend(sb) + 1);
if (unlikely(err))
goto out;
p = iinfo->ii_hinode + iinfo->ii_bstart;
err = 0;
for (bindex = iinfo->ii_bstart; bindex <= iinfo->ii_bend;
bindex++, p++) {
if (!p->hi_inode)
continue;
new_bindex = au_br_index(sb, p->hi_id);
if (new_bindex == bindex)
continue;
if (new_bindex < 0) {
update = 1;
au_hiput(p);
p->hi_inode = NULL;
continue;
}
if (new_bindex < iinfo->ii_bstart)
iinfo->ii_bstart = new_bindex;
if (iinfo->ii_bend < new_bindex)
iinfo->ii_bend = new_bindex;
/* swap two lower inode, and loop again */
q = iinfo->ii_hinode + new_bindex;
tmp = *q;
*q = *p;
*p = tmp;
if (tmp.hi_inode) {
bindex--;
p--;
}
}
au_update_ibrange(inode, /*do_put_zero*/0);
e = au_dy_irefresh(inode);
if (unlikely(e && !err))
err = e;
if (do_attr)
au_refresh_hinode_attr(inode, update && S_ISDIR(inode->i_mode));
out:
return err;
}
void au_cpup_igen(struct inode *inode, struct inode *h_inode)
{
struct au_iinfo *iinfo = au_ii(inode);
IiMustWriteLock(inode);
iinfo->ii_higen = h_inode->i_generation;
iinfo->ii_hsb1 = h_inode->i_sb;
}
int au_refresh_hinode(struct inode *inode, struct dentry *dentry)
{
int err, e, update;
unsigned int flags;
umode_t mode;
aufs_bindex_t bindex, bend;
unsigned char isdir;
struct au_hinode *p;
struct au_iinfo *iinfo;
err = au_ii_refresh(inode, &update);
if (unlikely(err))
goto out;
update = 0;
iinfo = au_ii(inode);
p = iinfo->ii_hinode + iinfo->ii_bstart;
mode = (inode->i_mode & S_IFMT);
isdir = S_ISDIR(mode);
flags = au_hi_flags(inode, isdir);
bend = au_dbend(dentry);
for (bindex = au_dbstart(dentry); bindex <= bend; bindex++) {
struct inode *h_i;
struct dentry *h_d;
h_d = au_h_dptr(dentry, bindex);
if (!h_d || !h_d->d_inode)
continue;
AuDebugOn(mode != (h_d->d_inode->i_mode & S_IFMT));
if (iinfo->ii_bstart <= bindex && bindex <= iinfo->ii_bend) {
h_i = au_h_iptr(inode, bindex);
if (h_i) {
if (h_i == h_d->d_inode)
continue;
err = -EIO;
break;
}
}
if (bindex < iinfo->ii_bstart)
iinfo->ii_bstart = bindex;
if (iinfo->ii_bend < bindex)
iinfo->ii_bend = bindex;
au_set_h_iptr(inode, bindex, au_igrab(h_d->d_inode), flags);
update = 1;
}
au_update_ibrange(inode, /*do_put_zero*/0);
e = au_dy_irefresh(inode);
if (unlikely(e && !err))
err = e;
if (!err)
au_refresh_hinode_attr(inode, update && isdir);
out:
AuTraceErr(err);
return err;
}
int au_refresh_hinode(struct inode *inode, struct dentry *dentry)
{
int err, update;
unsigned int flags;
aufs_bindex_t bindex, bend;
unsigned char isdir;
struct inode *first;
struct au_hinode *p;
struct au_iinfo *iinfo;
err = au_refresh_hinode_self(inode, /*do_attr*/0);
if (unlikely(err))
goto out;
update = 0;
iinfo = au_ii(inode);
p = iinfo->ii_hinode + iinfo->ii_bstart;
first = p->hi_inode;
isdir = S_ISDIR(inode->i_mode);
flags = au_hi_flags(inode, isdir);
bend = au_dbend(dentry);
for (bindex = au_dbstart(dentry); bindex <= bend; bindex++) {
struct inode *h_i;
struct dentry *h_d;
h_d = au_h_dptr(dentry, bindex);
if (!h_d || !h_d->d_inode)
continue;
if (iinfo->ii_bstart <= bindex && bindex <= iinfo->ii_bend) {
h_i = au_h_iptr(inode, bindex);
if (h_i) {
if (h_i == h_d->d_inode)
continue;
err = -EIO;
break;
}
}
if (bindex < iinfo->ii_bstart)
iinfo->ii_bstart = bindex;
if (iinfo->ii_bend < bindex)
iinfo->ii_bend = bindex;
au_set_h_iptr(inode, bindex, au_igrab(h_d->d_inode), flags);
update = 1;
}
au_update_brange(inode, /*do_put_zero*/0);
if (unlikely(err))
goto out;
au_refresh_hinode_attr(inode, update && isdir);
out:
return err;
}
static void au_br_do_add(struct super_block *sb, struct dentry *h_dentry,
struct au_branch *br, aufs_bindex_t bindex)
{
struct dentry *root;
struct inode *root_inode;
aufs_bindex_t bend, amount;
root = sb->s_root;
root_inode = root->d_inode;
bend = au_sbend(sb);
amount = bend + 1 - bindex;
au_br_do_add_brp(au_sbi(sb), bindex, br, bend, amount);
au_br_do_add_hdp(au_di(root), bindex, bend, amount);
au_br_do_add_hip(au_ii(root_inode), bindex, bend, amount);
au_set_h_dptr(root, bindex, dget(h_dentry));
au_set_h_iptr(root_inode, bindex, au_igrab(h_dentry->d_inode),
/*flags*/0);
}
/*
* returns a newly allocated branch. @new_nbranch is a number of branches
* after adding a branch.
*/
static struct au_branch *au_br_alloc(struct super_block *sb, int new_nbranch,
int perm)
{
struct au_branch *add_branch;
struct dentry *root;
int err;
err = -ENOMEM;
root = sb->s_root;
add_branch = kmalloc(sizeof(*add_branch), GFP_NOFS);
if (unlikely(!add_branch))
goto out;
err = au_hnotify_init_br(add_branch, perm);
if (unlikely(err))
goto out_br;
add_branch->br_wbr = NULL;
if (au_br_writable(perm)) {
/* may be freed separately at changing the branch permission */
add_branch->br_wbr = kmalloc(sizeof(*add_branch->br_wbr),
GFP_NOFS);
if (unlikely(!add_branch->br_wbr))
goto out_hnotify;
}
err = au_sbr_realloc(au_sbi(sb), new_nbranch);
if (!err)
err = au_di_realloc(au_di(root), new_nbranch);
if (!err)
err = au_ii_realloc(au_ii(root->d_inode), new_nbranch);
if (!err)
return add_branch; /* success */
kfree(add_branch->br_wbr);
out_hnotify:
au_hnotify_fin_br(add_branch);
out_br:
kfree(add_branch);
out:
return ERR_PTR(err);
}
void au_dpri_inode(struct inode *inode)
{
struct au_iinfo *iinfo;
aufs_bindex_t bindex;
int err;
err = do_pri_inode(-1, inode, NULL);
if (err || !au_test_aufs(inode->i_sb))
return;
iinfo = au_ii(inode);
if (!iinfo)
return;
dpri("i-1: bstart %d, bend %d, gen %d\n",
iinfo->ii_bstart, iinfo->ii_bend, au_iigen(inode));
if (iinfo->ii_bstart < 0)
return;
for (bindex = iinfo->ii_bstart; bindex <= iinfo->ii_bend; bindex++)
do_pri_inode(bindex, iinfo->ii_hinode[0 + bindex].hi_inode,
iinfo->ii_hinode[0 + bindex].hi_whdentry);
}
static int do_pri_dentry(aufs_bindex_t bindex, struct dentry *dentry)
{
struct dentry *wh = NULL;
if (!dentry || IS_ERR(dentry)) {
dpri("d%d: err %ld\n", bindex, PTR_ERR(dentry));
return -1;
}
/* do not call dget_parent() here */
dpri("d%d: %.*s?/%.*s, %s, cnt %d, flags 0x%x\n",
bindex,
AuDLNPair(dentry->d_parent), AuDLNPair(dentry),
dentry->d_sb ? au_sbtype(dentry->d_sb) : "??",
atomic_read(&dentry->d_count), dentry->d_flags);
if (bindex >= 0 && dentry->d_inode && au_test_aufs(dentry->d_sb)) {
struct au_iinfo *iinfo = au_ii(dentry->d_inode);
if (iinfo)
wh = iinfo->ii_hinode[0 + bindex].hi_whdentry;
}
do_pri_inode(bindex, dentry->d_inode, wh);
return 0;
}
static int set_inode(struct inode *inode, struct dentry *dentry)
{
int err;
unsigned int flags;
umode_t mode;
aufs_bindex_t bindex, bstart, btail;
unsigned char isdir;
struct dentry *h_dentry;
struct inode *h_inode;
struct au_iinfo *iinfo;
IiMustWriteLock(inode);
err = 0;
isdir = 0;
bstart = au_dbstart(dentry);
h_inode = au_h_dptr(dentry, bstart)->d_inode;
mode = h_inode->i_mode;
switch (mode & S_IFMT) {
case S_IFREG:
btail = au_dbtail(dentry);
inode->i_op = &aufs_iop;
inode->i_fop = &aufs_file_fop;
inode->i_mapping->a_ops = &aufs_aop;
break;
case S_IFDIR:
isdir = 1;
btail = au_dbtaildir(dentry);
inode->i_op = &aufs_dir_iop;
inode->i_fop = &aufs_dir_fop;
break;
case S_IFLNK:
btail = au_dbtail(dentry);
inode->i_op = &aufs_symlink_iop;
break;
case S_IFBLK:
case S_IFCHR:
case S_IFIFO:
case S_IFSOCK:
btail = au_dbtail(dentry);
inode->i_op = &aufs_iop;
au_init_special_fop(inode, mode, h_inode->i_rdev);
break;
default:
AuIOErr("Unknown file type 0%o\n", mode);
err = -EIO;
goto out;
}
/* do not set inotify for whiteouted dirs (SHWH mode) */
flags = au_hi_flags(inode, isdir);
if (au_opt_test(au_mntflags(dentry->d_sb), SHWH)
&& au_ftest_hi(flags, HINOTIFY)
&& dentry->d_name.len > AUFS_WH_PFX_LEN
&& !memcmp(dentry->d_name.name, AUFS_WH_PFX, AUFS_WH_PFX_LEN))
au_fclr_hi(flags, HINOTIFY);
iinfo = au_ii(inode);
iinfo->ii_bstart = bstart;
iinfo->ii_bend = btail;
for (bindex = bstart; bindex <= btail; bindex++) {
h_dentry = au_h_dptr(dentry, bindex);
if (h_dentry)
au_set_h_iptr(inode, bindex,
au_igrab(h_dentry->d_inode), flags);
}
au_cpup_attr_all(inode, /*force*/1);
out:
return err;
}
static struct dentry *aufs_lookup(struct inode *dir, struct dentry *dentry,
struct nameidata *nd)
{
struct dentry *ret, *parent;
struct inode *inode;
struct super_block *sb;
int err, npositive, lc_idx;
IMustLock(dir);
sb = dir->i_sb;
err = si_read_lock(sb, AuLock_FLUSH | AuLock_NOPLM);
ret = ERR_PTR(err);
if (unlikely(err))
goto out;
ret = ERR_PTR(-ENAMETOOLONG);
if (unlikely(dentry->d_name.len > AUFS_MAX_NAMELEN))
goto out_si;
err = au_di_init(dentry);
ret = ERR_PTR(err);
if (unlikely(err))
goto out_si;
inode = NULL;
npositive = 0; /* suppress a warning */
parent = dentry->d_parent; /* dir inode is locked */
di_read_lock_parent(parent, AuLock_IR);
err = au_alive_dir(parent);
if (!err)
err = au_digen_test(parent, au_sigen(sb));
if (!err) {
npositive = au_lkup_dentry(dentry, au_dbstart(parent),
/*type*/0, nd);
err = npositive;
}
di_read_unlock(parent, AuLock_IR);
ret = ERR_PTR(err);
if (unlikely(err < 0))
goto out_unlock;
if (npositive) {
inode = au_new_inode(dentry, /*must_new*/0);
ret = (void *)inode;
}
if (IS_ERR(inode)) {
inode = NULL;
goto out_unlock;
}
ret = d_splice_alias(inode, dentry);
if (unlikely(IS_ERR(ret) && inode)) {
ii_write_unlock(inode);
lc_idx = AuLcNonDir_IIINFO;
if (S_ISLNK(inode->i_mode))
lc_idx = AuLcSymlink_IIINFO;
else if (S_ISDIR(inode->i_mode))
lc_idx = AuLcDir_IIINFO;
au_rw_class(&au_ii(inode)->ii_rwsem, au_lc_key + lc_idx);
iput(inode);
}
out_unlock:
di_write_unlock(dentry);
if (unlikely(IS_ERR(ret) && inode)) {
lc_idx = AuLcNonDir_DIINFO;
if (S_ISLNK(inode->i_mode))
lc_idx = AuLcSymlink_DIINFO;
else if (S_ISDIR(inode->i_mode))
lc_idx = AuLcDir_DIINFO;
au_rw_class(&au_di(dentry)->di_rwsem, au_lc_key + lc_idx);
}
out_si:
si_read_unlock(sb);
out:
return ret;
}
static int au_ii_refresh(struct inode *inode, int *update)
{
int err, e;
umode_t type;
aufs_bindex_t bindex, new_bindex;
struct super_block *sb;
struct au_iinfo *iinfo;
struct au_hinode *p, *q, tmp;
IiMustWriteLock(inode);
*update = 0;
sb = inode->i_sb;
type = inode->i_mode & S_IFMT;
iinfo = au_ii(inode);
err = au_ii_realloc(iinfo, au_sbend(sb) + 1);
if (unlikely(err))
goto out;
AuDebugOn(iinfo->ii_bstart < 0);
p = iinfo->ii_hinode + iinfo->ii_bstart;
for (bindex = iinfo->ii_bstart; bindex <= iinfo->ii_bend;
bindex++, p++) {
if (!p->hi_inode)
continue;
AuDebugOn(type != (p->hi_inode->i_mode & S_IFMT));
new_bindex = au_br_index(sb, p->hi_id);
if (new_bindex == bindex)
continue;
if (new_bindex < 0) {
*update = 1;
au_hiput(p);
p->hi_inode = NULL;
continue;
}
if (new_bindex < iinfo->ii_bstart)
iinfo->ii_bstart = new_bindex;
if (iinfo->ii_bend < new_bindex)
iinfo->ii_bend = new_bindex;
/* swap two lower inode, and loop again */
q = iinfo->ii_hinode + new_bindex;
tmp = *q;
*q = *p;
*p = tmp;
if (tmp.hi_inode) {
bindex--;
p--;
}
}
au_update_ibrange(inode, /*do_put_zero*/0);
e = au_dy_irefresh(inode);
if (unlikely(e && !err))
err = e;
out:
AuTraceErr(err);
return err;
}
/* todo: return with unlocked? */
struct inode *au_new_inode(struct dentry *dentry, int must_new)
{
struct inode *inode, *h_inode;
struct dentry *h_dentry;
struct super_block *sb;
struct mutex *mtx;
ino_t h_ino, ino;
int err;
aufs_bindex_t bstart;
sb = dentry->d_sb;
bstart = au_dbstart(dentry);
h_dentry = au_h_dptr(dentry, bstart);
h_inode = h_dentry->d_inode;
h_ino = h_inode->i_ino;
/*
* stop 'race'-ing between hardlinks under different
* parents.
*/
mtx = NULL;
if (!S_ISDIR(h_inode->i_mode))
mtx = &au_sbr(sb, bstart)->br_xino.xi_nondir_mtx;
new_ino:
if (mtx)
mutex_lock(mtx);
err = au_xino_read(sb, bstart, h_ino, &ino);
inode = ERR_PTR(err);
if (unlikely(err))
goto out;
if (!ino) {
ino = au_xino_new_ino(sb);
if (unlikely(!ino)) {
inode = ERR_PTR(-EIO);
goto out;
}
}
AuDbg("i%lu\n", (unsigned long)ino);
inode = au_iget_locked(sb, ino);
err = PTR_ERR(inode);
if (IS_ERR(inode))
goto out;
AuDbg("%lx, new %d\n", inode->i_state, !!(inode->i_state & I_NEW));
if (inode->i_state & I_NEW) {
/* verbose coding for lock class name */
if (unlikely(S_ISLNK(h_inode->i_mode)))
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcSymlink_IIINFO);
else if (unlikely(S_ISDIR(h_inode->i_mode)))
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcDir_IIINFO);
else /* likely */
au_rw_class(&au_ii(inode)->ii_rwsem,
au_lc_key + AuLcNonDir_IIINFO);
ii_write_lock_new_child(inode);
err = set_inode(inode, dentry);
if (!err) {
unlock_new_inode(inode);
goto out; /* success */
}
/*
* iget_failed() calls iput(), but we need to call
* ii_write_unlock() after iget_failed(). so dirty hack for
* i_count.
*/
atomic_inc(&inode->i_count);
iget_failed(inode);
ii_write_unlock(inode);
au_xino_write(sb, bstart, h_ino, /*ino*/0);
/* ignore this error */
goto out_iput;
} else if (!must_new && !IS_DEADDIR(inode) && inode->i_nlink) {
/*
* horrible race condition between lookup, readdir and copyup
* (or something).
*/
if (mtx)
mutex_unlock(mtx);
err = reval_inode(inode, dentry);
if (unlikely(err < 0)) {
mtx = NULL;
goto out_iput;
}
if (!err) {
mtx = NULL;
goto out; /* success */
} else if (mtx)
mutex_lock(mtx);
}
if (unlikely(au_test_fs_unique_ino(h_dentry->d_inode)))
AuWarn1("Warning: Un-notified UDBA or repeatedly renamed dir,"
" b%d, %s, %pd, hi%lu, i%lu.\n",
bstart, au_sbtype(h_dentry->d_sb), dentry,
(unsigned long)h_ino, (unsigned long)ino);
ino = 0;
err = au_xino_write(sb, bstart, h_ino, /*ino*/0);
if (!err) {
iput(inode);
if (mtx)
mutex_unlock(mtx);
goto new_ino;
}
out_iput:
iput(inode);
inode = ERR_PTR(err);
out:
if (mtx)
mutex_unlock(mtx);
return inode;
}
