static int au_dpages_append(struct au_dcsub_pages *dpages,
struct dentry *dentry, gfp_t gfp)
{
int err, sz;
struct au_dpage *dpage;
void *p;
dpage = dpages->dpages + dpages->ndpage - 1;
sz = PAGE_SIZE / sizeof(dentry);
if (unlikely(dpage->ndentry >= sz)) {
AuLabel(new dpage);
err = -ENOMEM;
sz = dpages->ndpage * sizeof(*dpages->dpages);
p = au_kzrealloc(dpages->dpages, sz,
sz + sizeof(*dpages->dpages), gfp);
if (unlikely(!p))
goto out;
dpages->dpages = p;
dpage = dpages->dpages + dpages->ndpage;
p = (void *)__get_free_page(gfp);
if (unlikely(!p))
goto out;
dpage->ndentry = 0;
dpage->dentries = p;
dpages->ndpage++;
}
/* d_count can be zero */
dpage->dentries[dpage->ndentry++] = dget_locked(dentry);
return 0; /* success */
out:
return err;
}
int aufs_rmdir(struct inode *dir, struct dentry *dentry)
{
int err, rmdir_later;
aufs_bindex_t bwh, bindex, bstart;
struct au_dtime dt;
struct au_pin pin;
struct inode *inode;
struct dentry *parent, *wh_dentry, *h_dentry;
struct au_whtmp_rmdir *args;
IMustLock(dir);
err = aufs_read_lock(dentry, AuLock_DW | AuLock_FLUSH | AuLock_GEN);
if (unlikely(err))
goto out;
err = au_alive_dir(dentry);
if (unlikely(err))
goto out_unlock;
inode = dentry->d_inode;
IMustLock(inode);
err = -ENOTDIR;
if (unlikely(!S_ISDIR(inode->i_mode)))
goto out_unlock; /* possible? */
err = -ENOMEM;
args = au_whtmp_rmdir_alloc(dir->i_sb, GFP_NOFS);
if (unlikely(!args))
goto out_unlock;
parent = dentry->d_parent; /* dir inode is locked */
di_write_lock_parent(parent);
err = au_test_empty(dentry, &args->whlist);
if (unlikely(err))
goto out_parent;
bstart = au_dbstart(dentry);
bwh = au_dbwh(dentry);
bindex = -1;
wh_dentry = lock_hdir_create_wh(dentry, /*isdir*/1, &bindex, &dt, &pin);
err = PTR_ERR(wh_dentry);
if (IS_ERR(wh_dentry))
goto out_parent;
h_dentry = au_h_dptr(dentry, bstart);
dget(h_dentry);
rmdir_later = 0;
if (bindex == bstart) {
err = renwh_and_rmdir(dentry, bstart, &args->whlist, dir);
if (err > 0) {
rmdir_later = err;
err = 0;
}
} else {
/* stop monitoring */
au_hn_free(au_hi(inode, bstart));
/* dir inode is locked */
IMustLock(wh_dentry->d_parent->d_inode);
err = 0;
}
if (!err) {
vfsub_dead_dir(inode);
au_set_dbdiropq(dentry, -1);
epilog(dir, dentry, bindex);
if (rmdir_later) {
au_whtmp_kick_rmdir(dir, bstart, h_dentry, args);
args = NULL;
}
goto out_unpin; /* success */
}
/* revert */
AuLabel(revert);
if (wh_dentry) {
int rerr;
rerr = do_revert(err, dir, bindex, bwh, wh_dentry, dentry, &dt);
if (rerr)
err = rerr;
}
out_unpin:
au_unpin(&pin);
dput(wh_dentry);
dput(h_dentry);
out_parent:
di_write_unlock(parent);
if (args)
au_whtmp_rmdir_free(args);
out_unlock:
aufs_read_unlock(dentry, AuLock_DW);
out:
AuTraceErr(err);
return err;
}
static void dump_opts(struct au_opts *opts)
{
#ifdef CONFIG_AUFS_DEBUG
/* reduce stack space */
union {
struct au_opt_add *add;
struct au_opt_del *del;
struct au_opt_mod *mod;
struct au_opt_xino *xino;
struct au_opt_xino_itrunc *xino_itrunc;
struct au_opt_wbr_create *create;
} u;
struct au_opt *opt;
opt = opts->opt;
while (opt->type != Opt_tail) {
switch (opt->type) {
case Opt_add:
u.add = &opt->add;
AuDbg("add {b%d, %s, 0x%x, %p}\n",
u.add->bindex, u.add->pathname, u.add->perm,
u.add->path.dentry);
break;
case Opt_del:
case Opt_idel:
u.del = &opt->del;
AuDbg("del {%s, %p}\n",
u.del->pathname, u.del->h_path.dentry);
break;
case Opt_mod:
case Opt_imod:
u.mod = &opt->mod;
AuDbg("mod {%s, 0x%x, %p}\n",
u.mod->path, u.mod->perm, u.mod->h_root);
break;
case Opt_append:
u.add = &opt->add;
AuDbg("append {b%d, %s, 0x%x, %p}\n",
u.add->bindex, u.add->pathname, u.add->perm,
u.add->path.dentry);
break;
case Opt_prepend:
u.add = &opt->add;
AuDbg("prepend {b%d, %s, 0x%x, %p}\n",
u.add->bindex, u.add->pathname, u.add->perm,
u.add->path.dentry);
break;
case Opt_dirwh:
AuDbg("dirwh %d\n", opt->dirwh);
break;
case Opt_rdcache:
AuDbg("rdcache %d\n", opt->rdcache);
break;
case Opt_rdblk:
AuDbg("rdblk %u\n", opt->rdblk);
break;
case Opt_rdblk_def:
AuDbg("rdblk_def\n");
break;
case Opt_rdhash:
AuDbg("rdhash %u\n", opt->rdhash);
break;
case Opt_rdhash_def:
AuDbg("rdhash_def\n");
break;
case Opt_xino:
u.xino = &opt->xino;
AuDbg("xino {%s %.*s}\n",
u.xino->path,
AuDLNPair(u.xino->file->f_dentry));
break;
case Opt_trunc_xino:
AuLabel(trunc_xino);
break;
case Opt_notrunc_xino:
AuLabel(notrunc_xino);
break;
case Opt_trunc_xino_path:
case Opt_itrunc_xino:
u.xino_itrunc = &opt->xino_itrunc;
AuDbg("trunc_xino %d\n", u.xino_itrunc->bindex);
break;
case Opt_noxino:
AuLabel(noxino);
break;
case Opt_trunc_xib:
AuLabel(trunc_xib);
break;
case Opt_notrunc_xib:
AuLabel(notrunc_xib);
break;
case Opt_shwh:
AuLabel(shwh);
break;
case Opt_noshwh:
AuLabel(noshwh);
break;
case Opt_plink:
AuLabel(plink);
break;
case Opt_noplink:
AuLabel(noplink);
break;
case Opt_list_plink:
AuLabel(list_plink);
break;
case Opt_udba:
AuDbg("udba %d, %s\n",
opt->udba, au_optstr_udba(opt->udba));
break;
case Opt_dio:
AuLabel(dio);
break;
case Opt_nodio:
AuLabel(nodio);
break;
case Opt_diropq_a:
AuLabel(diropq_a);
break;
case Opt_diropq_w:
AuLabel(diropq_w);
break;
case Opt_warn_perm:
AuLabel(warn_perm);
break;
case Opt_nowarn_perm:
AuLabel(nowarn_perm);
break;
case Opt_refrof:
AuLabel(refrof);
break;
case Opt_norefrof:
AuLabel(norefrof);
break;
case Opt_verbose:
AuLabel(verbose);
break;
case Opt_noverbose:
AuLabel(noverbose);
break;
case Opt_sum:
AuLabel(sum);
break;
case Opt_nosum:
AuLabel(nosum);
break;
case Opt_wsum:
AuLabel(wsum);
break;
case Opt_wbr_create:
u.create = &opt->wbr_create;
AuDbg("create %d, %s\n", u.create->wbr_create,
au_optstr_wbr_create(u.create->wbr_create));
switch (u.create->wbr_create) {
case AuWbrCreate_MFSV:
case AuWbrCreate_PMFSV:
AuDbg("%d sec\n", u.create->mfs_second);
break;
case AuWbrCreate_MFSRR:
AuDbg("%llu watermark\n",
u.create->mfsrr_watermark);
break;
case AuWbrCreate_MFSRRV:
AuDbg("%llu watermark, %d sec\n",
u.create->mfsrr_watermark,
u.create->mfs_second);
break;
}
break;
case Opt_wbr_copyup:
AuDbg("copyup %d, %s\n", opt->wbr_copyup,
au_optstr_wbr_copyup(opt->wbr_copyup));
break;
default:
BUG();
}
opt++;
}
#endif
}
static int au_do_copy_file(struct file *dst, struct file *src, loff_t len,
char *buf, unsigned long blksize)
{
int err;
size_t sz, rbytes, wbytes;
unsigned char all_zero;
char *p, *zp;
struct mutex *h_mtx;
/* reduce stack usage */
struct iattr *ia;
zp = page_address(ZERO_PAGE(0));
if (unlikely(!zp))
return -ENOMEM; /* possible? */
err = 0;
all_zero = 0;
while (len) {
AuDbg("len %lld\n", len);
sz = blksize;
if (len < blksize)
sz = len;
rbytes = 0;
/* todo: signal_pending? */
while (!rbytes || err == -EAGAIN || err == -EINTR) {
rbytes = vfsub_read_k(src, buf, sz, &src->f_pos);
err = rbytes;
}
if (unlikely(err < 0))
break;
all_zero = 0;
if (len >= rbytes && rbytes == blksize)
all_zero = !memcmp(buf, zp, rbytes);
if (!all_zero) {
wbytes = rbytes;
p = buf;
while (wbytes) {
size_t b;
b = vfsub_write_k(dst, p, wbytes, &dst->f_pos);
err = b;
/* todo: signal_pending? */
if (unlikely(err == -EAGAIN || err == -EINTR))
continue;
if (unlikely(err < 0))
break;
wbytes -= b;
p += b;
}
} else {
loff_t res;
AuLabel(hole);
res = vfsub_llseek(dst, rbytes, SEEK_CUR);
err = res;
if (unlikely(res < 0))
break;
}
len -= rbytes;
err = 0;
}
/* the last block may be a hole */
if (!err && all_zero) {
AuLabel(last hole);
err = 1;
if (au_test_nfs(dst->f_dentry->d_sb)) {
/* nfs requires this step to make last hole */
/* is this only nfs? */
do {
/* todo: signal_pending? */
err = vfsub_write_k(dst, "\0", 1, &dst->f_pos);
} while (err == -EAGAIN || err == -EINTR);
if (err == 1)
dst->f_pos--;
}
if (err == 1) {
ia = (void *)buf;
ia->ia_size = dst->f_pos;
ia->ia_valid = ATTR_SIZE | ATTR_FILE;
ia->ia_file = dst;
h_mtx = &dst->f_dentry->d_inode->i_mutex;
mutex_lock_nested(h_mtx, AuLsc_I_CHILD2);
err = vfsub_notify_change(&dst->f_path, ia);
mutex_unlock(h_mtx);
}
}
return err;
}
/*
* returns the number of lower positive dentries,
* otherwise an error.
* can be called at unlinking with @type is zero.
*/
int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t bstart, mode_t type,
struct nameidata *nd)
{
int npositive, err;
aufs_bindex_t bindex, btail, bdiropq;
unsigned char isdir;
struct qstr whname;
struct au_do_lookup_args args = {
.flags = 0,
.type = type,
.nd = nd
};
const struct qstr *name = &dentry->d_name;
struct dentry *parent;
struct inode *inode;
err = au_test_shwh(dentry->d_sb, name);
if (unlikely(err))
goto out;
err = au_wh_name_alloc(&whname, name);
if (unlikely(err))
goto out;
inode = dentry->d_inode;
isdir = !!(inode && S_ISDIR(inode->i_mode));
if (!type)
au_fset_lkup(args.flags, ALLOW_NEG);
npositive = 0;
parent = dget_parent(dentry);
btail = au_dbtaildir(parent);
for (bindex = bstart; bindex <= btail; bindex++) {
struct dentry *h_parent, *h_dentry;
struct inode *h_inode, *h_dir;
h_dentry = au_h_dptr(dentry, bindex);
if (h_dentry) {
if (h_dentry->d_inode)
npositive++;
if (type != S_IFDIR)
break;
continue;
}
h_parent = au_h_dptr(parent, bindex);
if (!h_parent)
continue;
h_dir = h_parent->d_inode;
if (!h_dir || !S_ISDIR(h_dir->i_mode))
continue;
mutex_lock_nested(&h_dir->i_mutex, AuLsc_I_PARENT);
h_dentry = au_do_lookup(h_parent, dentry, bindex, &whname,
&args);
mutex_unlock(&h_dir->i_mutex);
err = PTR_ERR(h_dentry);
if (IS_ERR(h_dentry))
goto out_parent;
au_fclr_lkup(args.flags, ALLOW_NEG);
if (au_dbwh(dentry) >= 0)
break;
if (!h_dentry)
continue;
h_inode = h_dentry->d_inode;
if (!h_inode)
continue;
npositive++;
if (!args.type)
args.type = h_inode->i_mode & S_IFMT;
if (args.type != S_IFDIR)
break;
else if (isdir) {
/* the type of lower may be different */
bdiropq = au_dbdiropq(dentry);
if (bdiropq >= 0 && bdiropq <= bindex)
break;
}
}
if (npositive) {
AuLabel(positive);
au_update_dbstart(dentry);
}
err = npositive;
if (unlikely(!au_opt_test(au_mntflags(dentry->d_sb), UDBA_NONE)
&& au_dbstart(dentry) < 0)) {
err = -EIO;
AuIOErr("both of real entry and whiteout found, %.*s, err %d\n",
AuDLNPair(dentry), err);
}
out_parent:
dput(parent);
kfree(whname.name);
out:
return err;
}
struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent,
struct au_branch *br)
{
struct dentry *dentry;
int wkq_err;
if (!au_test_h_perm_sio(parent->d_inode, MAY_EXEC))
dentry = au_lkup_one(name, parent, br, /*nd*/NULL);
else {
struct au_lkup_one_args args = {
.errp = &dentry,
.name = name,
.h_parent = parent,
.br = br,
.nd = NULL
};
wkq_err = au_wkq_wait(au_call_lkup_one, &args);
if (unlikely(wkq_err))
dentry = ERR_PTR(wkq_err);
}
return dentry;
}
/*
* lookup @dentry on @bindex which should be negative.
*/
int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex)
{
int err;
struct dentry *parent, *h_parent, *h_dentry;
parent = dget_parent(dentry);
h_parent = au_h_dptr(parent, bindex);
h_dentry = au_sio_lkup_one(&dentry->d_name, h_parent,
au_sbr(dentry->d_sb, bindex));
err = PTR_ERR(h_dentry);
if (IS_ERR(h_dentry))
goto out;
if (unlikely(h_dentry->d_inode)) {
err = -EIO;
AuIOErr("%.*s should be negative on b%d.\n",
AuDLNPair(h_dentry), bindex);
dput(h_dentry);
goto out;
}
err = 0;
if (bindex < au_dbstart(dentry))
au_set_dbstart(dentry, bindex);
if (au_dbend(dentry) < bindex)
au_set_dbend(dentry, bindex);
au_set_h_dptr(dentry, bindex, h_dentry);
out:
dput(parent);
return err;
}
int aufs_mkdir(struct inode *dir, struct dentry *dentry, int mode)
{
int err, rerr;
aufs_bindex_t bindex;
unsigned char diropq;
struct path h_path;
struct dentry *wh_dentry, *parent, *opq_dentry;
struct mutex *h_mtx;
struct super_block *sb;
struct {
struct au_pin pin;
struct au_dtime dt;
} *a; /* reduce the stack usage */
struct au_wr_dir_args wr_dir_args = {
.force_btgt = -1,
.flags = AuWrDir_ADD_ENTRY | AuWrDir_ISDIR
};
IMustLock(dir);
err = -ENOMEM;
a = kmalloc(sizeof(*a), GFP_NOFS);
if (unlikely(!a))
goto out;
err = aufs_read_lock(dentry, AuLock_DW | AuLock_GEN);
if (unlikely(err))
goto out_free;
err = au_d_may_add(dentry);
if (unlikely(err))
goto out_unlock;
parent = dentry->d_parent; /* dir inode is locked */
di_write_lock_parent(parent);
wh_dentry = lock_hdir_lkup_wh(dentry, &a->dt, /*src_dentry*/NULL,
&a->pin, &wr_dir_args);
err = PTR_ERR(wh_dentry);
if (IS_ERR(wh_dentry))
goto out_parent;
sb = dentry->d_sb;
bindex = au_dbstart(dentry);
h_path.dentry = au_h_dptr(dentry, bindex);
h_path.mnt = au_sbr_mnt(sb, bindex);
err = vfsub_mkdir(au_pinned_h_dir(&a->pin), &h_path, mode);
if (unlikely(err))
goto out_unpin;
/* make the dir opaque */
diropq = 0;
h_mtx = &h_path.dentry->d_inode->i_mutex;
if (wh_dentry
|| au_opt_test(au_mntflags(sb), ALWAYS_DIROPQ)) {
mutex_lock_nested(h_mtx, AuLsc_I_CHILD);
opq_dentry = au_diropq_create(dentry, bindex);
mutex_unlock(h_mtx);
err = PTR_ERR(opq_dentry);
if (IS_ERR(opq_dentry))
goto out_dir;
dput(opq_dentry);
diropq = 1;
}
err = epilog(dir, bindex, wh_dentry, dentry);
if (!err) {
inc_nlink(dir);
goto out_unpin; /* success */
}
/* revert */
if (diropq) {
AuLabel(revert opq);
mutex_lock_nested(h_mtx, AuLsc_I_CHILD);
rerr = au_diropq_remove(dentry, bindex);
mutex_unlock(h_mtx);
if (rerr) {
AuIOErr("%.*s reverting diropq failed(%d, %d)\n",
AuDLNPair(dentry), err, rerr);
err = -EIO;
}
}
out_dir:
AuLabel(revert dir);
rerr = vfsub_rmdir(au_pinned_h_dir(&a->pin), &h_path);
if (rerr) {
AuIOErr("%.*s reverting dir failed(%d, %d)\n",
AuDLNPair(dentry), err, rerr);
err = -EIO;
}
au_dtime_revert(&a->dt);
out_unpin:
au_unpin(&a->pin);
dput(wh_dentry);
out_parent:
di_write_unlock(parent);
out_unlock:
if (unlikely(err)) {
au_update_dbstart(dentry);
d_drop(dentry);
}
aufs_read_unlock(dentry, AuLock_DW);
out_free:
kfree(a);
out:
return err;
}
/*
* returns the number of lower positive dentries,
* otherwise an error.
* can be called at unlinking with @type is zero.
*/
int au_lkup_dentry(struct dentry *dentry, aufs_bindex_t bstart, mode_t type)
{
int npositive, err;
aufs_bindex_t bindex, btail, bdiropq;
unsigned char isdir, dirperm1;
struct qstr whname;
struct au_do_lookup_args args = {
.flags = 0,
.type = type
};
const struct qstr *name = &dentry->d_name;
struct dentry *parent;
struct super_block *sb;
sb = dentry->d_sb;
err = au_test_shwh(sb, name);
if (unlikely(err))
goto out;
err = au_wh_name_alloc(&whname, name);
if (unlikely(err))
goto out;
isdir = !!d_is_dir(dentry);
if (!type)
au_fset_lkup(args.flags, ALLOW_NEG);
dirperm1 = !!au_opt_test(au_mntflags(sb), DIRPERM1);
npositive = 0;
parent = dget_parent(dentry);
btail = au_dbtaildir(parent);
for (bindex = bstart; bindex <= btail; bindex++) {
struct dentry *h_parent, *h_dentry;
struct inode *h_inode, *h_dir;
h_dentry = au_h_dptr(dentry, bindex);
if (h_dentry) {
if (d_is_positive(h_dentry))
npositive++;
if (type != S_IFDIR)
break;
continue;
}
h_parent = au_h_dptr(parent, bindex);
if (!h_parent || !d_is_dir(h_parent))
continue;
h_dir = d_inode(h_parent);
mutex_lock_nested(&h_dir->i_mutex, AuLsc_I_PARENT);
h_dentry = au_do_lookup(h_parent, dentry, bindex, &whname,
&args);
mutex_unlock(&h_dir->i_mutex);
err = PTR_ERR(h_dentry);
if (IS_ERR(h_dentry))
goto out_parent;
if (h_dentry)
au_fclr_lkup(args.flags, ALLOW_NEG);
if (dirperm1)
au_fset_lkup(args.flags, IGNORE_PERM);
if (au_dbwh(dentry) >= 0)
break;
if (!h_dentry)
continue;
if (d_is_negative(h_dentry))
continue;
h_inode = d_inode(h_dentry);
npositive++;
if (!args.type)
args.type = h_inode->i_mode & S_IFMT;
if (args.type != S_IFDIR)
break;
else if (isdir) {
/* the type of lower may be different */
bdiropq = au_dbdiropq(dentry);
if (bdiropq >= 0 && bdiropq <= bindex)
break;
}
}
if (npositive) {
AuLabel(positive);
au_update_dbstart(dentry);
}
err = npositive;
if (unlikely(!au_opt_test(au_mntflags(sb), UDBA_NONE)
&& au_dbstart(dentry) < 0)) {
err = -EIO;
AuIOErr("both of real entry and whiteout found, %pd, err %d\n",
dentry, err);
}
out_parent:
dput(parent);
kfree(whname.name);
out:
return err;
}
struct dentry *au_sio_lkup_one(struct qstr *name, struct dentry *parent)
{
struct dentry *dentry;
int wkq_err;
if (!au_test_h_perm_sio(d_inode(parent), MAY_EXEC))
dentry = vfsub_lkup_one(name, parent);
else {
struct vfsub_lkup_one_args args = {
.errp = &dentry,
.name = name,
.parent = parent
};
wkq_err = au_wkq_wait(vfsub_call_lkup_one, &args);
if (unlikely(wkq_err))
dentry = ERR_PTR(wkq_err);
}
return dentry;
}
/*
* lookup @dentry on @bindex which should be negative.
*/
int au_lkup_neg(struct dentry *dentry, aufs_bindex_t bindex, int wh)
{
int err;
struct dentry *parent, *h_parent, *h_dentry;
struct au_branch *br;
parent = dget_parent(dentry);
h_parent = au_h_dptr(parent, bindex);
br = au_sbr(dentry->d_sb, bindex);
if (wh)
h_dentry = au_whtmp_lkup(h_parent, br, &dentry->d_name);
else
h_dentry = au_sio_lkup_one(&dentry->d_name, h_parent);
err = PTR_ERR(h_dentry);
if (IS_ERR(h_dentry))
goto out;
if (unlikely(d_is_positive(h_dentry))) {
err = -EIO;
AuIOErr("%pd should be negative on b%d.\n", h_dentry, bindex);
dput(h_dentry);
goto out;
}
err = 0;
if (bindex < au_dbstart(dentry))
au_set_dbstart(dentry, bindex);
if (au_dbend(dentry) < bindex)
au_set_dbend(dentry, bindex);
au_set_h_dptr(dentry, bindex, h_dentry);
out:
dput(parent);
return err;
}