/*
* Decode the result of a readdir call.
* We're not really decoding anymore, we just leave the buffer untouched
* and only check that it is syntactically correct.
* The real decoding happens in nfs_decode_entry below, called directly
* from nfs_readdir for each entry.
*/
static int
nfs_xdr_readdirres(struct rpc_rqst *req, __be32 *p, void *dummy)
{
struct xdr_buf *rcvbuf = &req->rq_rcv_buf;
struct kvec *iov = rcvbuf->head;
struct page **page;
size_t hdrlen;
unsigned int pglen, recvd;
u32 len;
int status, nr = 0;
__be32 *end, *entry, *kaddr;
if ((status = ntohl(*p++)))
return nfs_stat_to_errno(status);
hdrlen = (u8 *) p - (u8 *) iov->iov_base;
if (iov->iov_len < hdrlen) {
dprintk("NFS: READDIR reply header overflowed:"
"length %Zu > %Zu\n", hdrlen, iov->iov_len);
return -errno_NFSERR_IO;
} else if (iov->iov_len != hdrlen) {
dprintk("NFS: READDIR header is short. iovec will be shifted.\n");
xdr_shift_buf(rcvbuf, iov->iov_len - hdrlen);
}
pglen = rcvbuf->page_len;
recvd = rcvbuf->len - hdrlen;
if (pglen > recvd)
pglen = recvd;
page = rcvbuf->pages;
kaddr = p = kmap_atomic(*page, KM_USER0);
end = (__be32 *)((char *)p + pglen);
entry = p;
/* Make sure the packet actually has a value_follows and EOF entry */
if ((entry + 1) > end)
goto short_pkt;
for (; *p++; nr++) {
if (p + 2 > end)
goto short_pkt;
p++; /* fileid */
len = ntohl(*p++);
p += XDR_QUADLEN(len) + 1; /* name plus cookie */
if (len > NFS2_MAXNAMLEN) {
dprintk("NFS: giant filename in readdir (len 0x%x)!\n",
len);
goto err_unmap;
}
if (p + 2 > end)
goto short_pkt;
entry = p;
}
/*
* Apparently some server sends responses that are a valid size, but
* contain no entries, and have value_follows==0 and EOF==0. For
* those, just set the EOF marker.
*/
if (!nr && entry[1] == 0) {
dprintk("NFS: readdir reply truncated!\n");
entry[1] = 1;
}
out:
kunmap_atomic(kaddr, KM_USER0);
return nr;
short_pkt:
/*
* When we get a short packet there are 2 possibilities. We can
* return an error, or fix up the response to look like a valid
* response and return what we have so far. If there are no
* entries and the packet was short, then return -EIO. If there
* are valid entries in the response, return them and pretend that
* the call was successful, but incomplete. The caller can retry the
* readdir starting at the last cookie.
*/
entry[0] = entry[1] = 0;
if (!nr)
nr = -errno_NFSERR_IO;
goto out;
err_unmap:
nr = -errno_NFSERR_IO;
goto out;
}
static int nilfs_cpfile_clear_snapshot(struct inode *cpfile, __u64 cno)
{
struct buffer_head *header_bh, *next_bh, *prev_bh, *cp_bh;
struct nilfs_cpfile_header *header;
struct nilfs_checkpoint *cp;
struct nilfs_snapshot_list *list;
__u64 next, prev;
void *kaddr;
int ret;
if (cno == 0)
return -ENOENT; /* checkpoint number 0 is invalid */
down_write(&NILFS_MDT(cpfile)->mi_sem);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh);
if (ret < 0)
goto out_sem;
kaddr = kmap_atomic(cp_bh->b_page);
cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr);
if (nilfs_checkpoint_invalid(cp)) {
ret = -ENOENT;
kunmap_atomic(kaddr);
goto out_cp;
}
if (!nilfs_checkpoint_snapshot(cp)) {
ret = 0;
kunmap_atomic(kaddr);
goto out_cp;
}
list = &cp->cp_snapshot_list;
next = le64_to_cpu(list->ssl_next);
prev = le64_to_cpu(list->ssl_prev);
kunmap_atomic(kaddr);
ret = nilfs_cpfile_get_header_block(cpfile, &header_bh);
if (ret < 0)
goto out_cp;
if (next != 0) {
ret = nilfs_cpfile_get_checkpoint_block(cpfile, next, 0,
&next_bh);
if (ret < 0)
goto out_header;
} else {
next_bh = header_bh;
get_bh(next_bh);
}
if (prev != 0) {
ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0,
&prev_bh);
if (ret < 0)
goto out_next;
} else {
prev_bh = header_bh;
get_bh(prev_bh);
}
kaddr = kmap_atomic(next_bh->b_page);
list = nilfs_cpfile_block_get_snapshot_list(
cpfile, next, next_bh, kaddr);
list->ssl_prev = cpu_to_le64(prev);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(prev_bh->b_page);
list = nilfs_cpfile_block_get_snapshot_list(
cpfile, prev, prev_bh, kaddr);
list->ssl_next = cpu_to_le64(next);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(cp_bh->b_page);
cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr);
cp->cp_snapshot_list.ssl_next = cpu_to_le64(0);
cp->cp_snapshot_list.ssl_prev = cpu_to_le64(0);
nilfs_checkpoint_clear_snapshot(cp);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(header_bh->b_page);
header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr);
le64_add_cpu(&header->ch_nsnapshots, -1);
kunmap_atomic(kaddr);
mark_buffer_dirty(next_bh);
mark_buffer_dirty(prev_bh);
mark_buffer_dirty(cp_bh);
mark_buffer_dirty(header_bh);
nilfs_mdt_mark_dirty(cpfile);
brelse(prev_bh);
out_next:
brelse(next_bh);
out_header:
brelse(header_bh);
out_cp:
brelse(cp_bh);
out_sem:
up_write(&NILFS_MDT(cpfile)->mi_sem);
return ret;
}
/*
* Attempts to free an entry by adding a page to the swap cache,
* decompressing the entry data into the page, and issuing a
* bio write to write the page back to the swap device.
*
* This can be thought of as a "resumed writeback" of the page
* to the swap device. We are basically resuming the same swap
* writeback path that was intercepted with the frontswap_store()
* in the first place. After the page has been decompressed into
* the swap cache, the compressed version stored by zswap can be
* freed.
*/
static int zswap_writeback_entry(struct zpool *pool, unsigned long handle)
{
struct zswap_header *zhdr;
swp_entry_t swpentry;
struct zswap_tree *tree;
pgoff_t offset;
struct zswap_entry *entry;
struct page *page;
u8 *src, *dst;
unsigned int dlen;
int ret;
struct writeback_control wbc = {
.sync_mode = WB_SYNC_NONE,
};
/* extract swpentry from data */
zhdr = zpool_map_handle(pool, handle, ZPOOL_MM_RO);
swpentry = zhdr->swpentry; /* here */
zpool_unmap_handle(pool, handle);
tree = zswap_trees[swp_type(swpentry)];
offset = swp_offset(swpentry);
/* find and ref zswap entry */
spin_lock(&tree->lock);
entry = zswap_entry_find_get(&tree->rbroot, offset);
if (!entry) {
/* entry was invalidated */
spin_unlock(&tree->lock);
return 0;
}
spin_unlock(&tree->lock);
BUG_ON(offset != entry->offset);
/* try to allocate swap cache page */
switch (zswap_get_swap_cache_page(swpentry, &page)) {
case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */
ret = -ENOMEM;
goto fail;
case ZSWAP_SWAPCACHE_EXIST:
/* page is already in the swap cache, ignore for now */
page_cache_release(page);
ret = -EEXIST;
goto fail;
case ZSWAP_SWAPCACHE_NEW: /* page is locked */
/* decompress */
dlen = PAGE_SIZE;
src = (u8 *)zpool_map_handle(zswap_pool, entry->handle,
ZPOOL_MM_RO) + sizeof(struct zswap_header);
dst = kmap_atomic(page);
ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src,
entry->length, dst, &dlen);
kunmap_atomic(dst);
zpool_unmap_handle(zswap_pool, entry->handle);
BUG_ON(ret);
BUG_ON(dlen != PAGE_SIZE);
/* page is up to date */
SetPageUptodate(page);
}
/* move it to the tail of the inactive list after end_writeback */
SetPageReclaim(page);
/* start writeback */
__swap_writepage(page, &wbc, end_swap_bio_write);
page_cache_release(page);
zswap_written_back_pages++;
spin_lock(&tree->lock);
/* drop local reference */
zswap_entry_put(tree, entry);
/*
* There are two possible situations for entry here:
* (1) refcount is 1(normal case), entry is valid and on the tree
* (2) refcount is 0, entry is freed and not on the tree
* because invalidate happened during writeback
* search the tree and free the entry if find entry
*/
if (entry == zswap_rb_search(&tree->rbroot, offset))
zswap_entry_put(tree, entry);
spin_unlock(&tree->lock);
goto end;
/*
* if we get here due to ZSWAP_SWAPCACHE_EXIST
* a load may happening concurrently
* it is safe and okay to not free the entry
* if we free the entry in the following put
* it it either okay to return !0
*/
fail:
spin_lock(&tree->lock);
zswap_entry_put(tree, entry);
spin_unlock(&tree->lock);
end:
return ret;
}
*/
for (bytes = 0; bytes < length; offset = 0, bytes += copied) {
entry = squashfs_cache_get(sb, msblk->block_cache, block, 0);
if (entry->error) {
ERROR("Unable to read symlink [%llx:%x]\n",
squashfs_i(inode)->start,
squashfs_i(inode)->offset);
squashfs_cache_put(entry);
goto error_out;
}
<<<<<<< HEAD
pageaddr = kmap_atomic(page);
=======
<<<<<<< HEAD
pageaddr = kmap_atomic(page);
=======
pageaddr = kmap_atomic(page, KM_USER0);
>>>>>>> 58a75b6a81be54a8b491263ca1af243e9d8617b9
>>>>>>> ae1773bb70f3d7cf73324ce8fba787e01d8fa9f2
copied = squashfs_copy_data(pageaddr + bytes, entry, offset,
length - bytes);
if (copied == length - bytes)
memset(pageaddr + length, 0, PAGE_CACHE_SIZE - length);
else
block = entry->next_index;
<<<<<<< HEAD
kunmap_atomic(pageaddr);
=======
<<<<<<< HEAD
kunmap_atomic(pageaddr);
static ssize_t nilfs_cpfile_do_get_ssinfo(struct inode *cpfile, __u64 *cnop,
void *buf, unsigned int cisz,
size_t nci)
{
struct buffer_head *bh;
struct nilfs_cpfile_header *header;
struct nilfs_checkpoint *cp;
struct nilfs_cpinfo *ci = buf;
__u64 curr = *cnop, next;
unsigned long curr_blkoff, next_blkoff;
void *kaddr;
int n = 0, ret;
down_read(&NILFS_MDT(cpfile)->mi_sem);
if (curr == 0) {
ret = nilfs_cpfile_get_header_block(cpfile, &bh);
if (ret < 0)
goto out;
kaddr = kmap_atomic(bh->b_page);
header = nilfs_cpfile_block_get_header(cpfile, bh, kaddr);
curr = le64_to_cpu(header->ch_snapshot_list.ssl_next);
kunmap_atomic(kaddr);
brelse(bh);
if (curr == 0) {
ret = 0;
goto out;
}
} else if (unlikely(curr == ~(__u64)0)) {
ret = 0;
goto out;
}
curr_blkoff = nilfs_cpfile_get_blkoff(cpfile, curr);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr, 0, &bh);
if (unlikely(ret < 0)) {
if (ret == -ENOENT)
ret = 0; /* No snapshots (started from a hole block) */
goto out;
}
kaddr = kmap_atomic(bh->b_page);
while (n < nci) {
cp = nilfs_cpfile_block_get_checkpoint(cpfile, curr, bh, kaddr);
curr = ~(__u64)0; /* Terminator */
if (unlikely(nilfs_checkpoint_invalid(cp) ||
!nilfs_checkpoint_snapshot(cp)))
break;
nilfs_cpfile_checkpoint_to_cpinfo(cpfile, cp, ci);
ci = (void *)ci + cisz;
n++;
next = le64_to_cpu(cp->cp_snapshot_list.ssl_next);
if (next == 0)
break; /* reach end of the snapshot list */
next_blkoff = nilfs_cpfile_get_blkoff(cpfile, next);
if (curr_blkoff != next_blkoff) {
kunmap_atomic(kaddr);
brelse(bh);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, next,
0, &bh);
if (unlikely(ret < 0)) {
WARN_ON(ret == -ENOENT);
goto out;
}
kaddr = kmap_atomic(bh->b_page);
}
curr = next;
curr_blkoff = next_blkoff;
}
kunmap_atomic(kaddr);
brelse(bh);
*cnop = curr;
ret = n;
out:
up_read(&NILFS_MDT(cpfile)->mi_sem);
return ret;
}
int direct2indirect(struct reiserfs_transaction_handle *th, struct inode *inode,
struct treepath *path, struct buffer_head *unbh,
loff_t tail_offset)
{
struct super_block *sb = inode->i_sb;
struct buffer_head *up_to_date_bh;
struct item_head *p_le_ih = PATH_PITEM_HEAD(path);
unsigned long total_tail = 0;
struct cpu_key end_key;
struct item_head ind_ih;
int blk_size, retval;
unp_t unfm_ptr;
BUG_ON(!th->t_trans_id);
REISERFS_SB(sb)->s_direct2indirect++;
blk_size = sb->s_blocksize;
copy_item_head(&ind_ih, p_le_ih);
set_le_ih_k_offset(&ind_ih, tail_offset);
set_le_ih_k_type(&ind_ih, TYPE_INDIRECT);
make_cpu_key(&end_key, inode, tail_offset, TYPE_INDIRECT, 4);
if (search_for_position_by_key(sb, &end_key, path) == POSITION_FOUND) {
reiserfs_error(sb, "PAP-14030",
"pasted or inserted byte exists in "
"the tree %K. Use fsck to repair.", &end_key);
pathrelse(path);
return -EIO;
}
p_le_ih = PATH_PITEM_HEAD(path);
unfm_ptr = cpu_to_le32(unbh->b_blocknr);
if (is_statdata_le_ih(p_le_ih)) {
set_ih_free_space(&ind_ih, 0);
put_ih_item_len(&ind_ih, UNFM_P_SIZE);
PATH_LAST_POSITION(path)++;
retval =
reiserfs_insert_item(th, path, &end_key, &ind_ih, inode,
(char *)&unfm_ptr);
} else {
retval = reiserfs_paste_into_item(th, path, &end_key, inode,
(char *)&unfm_ptr,
UNFM_P_SIZE);
}
if (retval) {
return retval;
}
make_cpu_key(&end_key, inode, max_reiserfs_offset(inode), TYPE_DIRECT,
4);
while (1) {
int tail_size;
if (search_for_position_by_key(sb, &end_key, path) ==
POSITION_FOUND)
reiserfs_panic(sb, "PAP-14050",
"direct item (%K) not found", &end_key);
p_le_ih = PATH_PITEM_HEAD(path);
RFALSE(!is_direct_le_ih(p_le_ih),
"vs-14055: direct item expected(%K), found %h",
&end_key, p_le_ih);
tail_size = (le_ih_k_offset(p_le_ih) & (blk_size - 1))
+ ih_item_len(p_le_ih) - 1;
if (!unbh->b_page || buffer_uptodate(unbh)
|| PageUptodate(unbh->b_page)) {
up_to_date_bh = NULL;
} else {
up_to_date_bh = unbh;
}
retval = reiserfs_delete_item(th, path, &end_key, inode,
up_to_date_bh);
total_tail += retval;
if (tail_size == retval)
break;
}
if (up_to_date_bh) {
unsigned pgoff =
(tail_offset + total_tail - 1) & (PAGE_CACHE_SIZE - 1);
char *kaddr = kmap_atomic(up_to_date_bh->b_page);
memset(kaddr + pgoff, 0, blk_size - total_tail);
kunmap_atomic(kaddr);
}
REISERFS_I(inode)->i_first_direct_byte = U32_MAX;
return 0;
}
int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
unsigned int count)
{
struct iscsi_conn *conn = task->conn;
struct iscsi_tcp_task *tcp_task = task->dd_data;
struct cxgb3i_task_data *tdata = tcp_task->dd_data;
struct sk_buff *skb = tdata->skb;
unsigned int datalen = count;
int i, padlen = iscsi_padding(count);
struct page *pg;
cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
task, task->sc, offset, count, skb);
skb_put(skb, task->hdr_len);
tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
if (!count)
return 0;
if (task->sc) {
struct scsi_data_buffer *sdb = scsi_out(task->sc);
struct scatterlist *sg = NULL;
int err;
tdata->offset = offset;
tdata->count = count;
err = sgl_seek_offset(sdb->table.sgl, sdb->table.nents,
tdata->offset, &tdata->sgoffset, &sg);
if (err < 0) {
cxgb3i_log_warn("tpdu, sgl %u, bad offset %u/%u.\n",
sdb->table.nents, tdata->offset,
sdb->length);
return err;
}
err = sgl_read_to_frags(sg, tdata->sgoffset, tdata->count,
tdata->frags, MAX_PDU_FRAGS);
if (err < 0) {
cxgb3i_log_warn("tpdu, sgl %u, bad offset %u + %u.\n",
sdb->table.nents, tdata->offset,
tdata->count);
return err;
}
tdata->nr_frags = err;
if (tdata->nr_frags > MAX_SKB_FRAGS ||
(padlen && tdata->nr_frags == MAX_SKB_FRAGS)) {
char *dst = skb->data + task->hdr_len;
skb_frag_t *frag = tdata->frags;
/* data fits in the skb's headroom */
for (i = 0; i < tdata->nr_frags; i++, frag++) {
char *src = kmap_atomic(frag->page,
KM_SOFTIRQ0);
memcpy(dst, src+frag->page_offset, frag->size);
dst += frag->size;
kunmap_atomic(src, KM_SOFTIRQ0);
}
if (padlen) {
memset(dst, 0, padlen);
padlen = 0;
}
skb_put(skb, count + padlen);
} else {
/* data fit into frag_list */
for (i = 0; i < tdata->nr_frags; i++)
get_page(tdata->frags[i].page);
memcpy(skb_shinfo(skb)->frags, tdata->frags,
sizeof(skb_frag_t) * tdata->nr_frags);
skb_shinfo(skb)->nr_frags = tdata->nr_frags;
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
} else {
pg = virt_to_page(task->data);
get_page(pg);
skb_fill_page_desc(skb, 0, pg, offset_in_page(task->data),
count);
skb->len += count;
skb->data_len += count;
skb->truesize += count;
}
if (padlen) {
i = skb_shinfo(skb)->nr_frags;
get_page(pad_page);
skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, pad_page, 0,
padlen);
skb->data_len += padlen;
skb->truesize += padlen;
skb->len += padlen;
}
return 0;
}
static int mcopy_atomic_pte(struct mm_struct *dst_mm,
pmd_t *dst_pmd,
struct vm_area_struct *dst_vma,
unsigned long dst_addr,
unsigned long src_addr,
struct page **pagep)
{
struct mem_cgroup *memcg;
pte_t _dst_pte, *dst_pte;
spinlock_t *ptl;
void *page_kaddr;
int ret;
struct page *page;
if (!*pagep) {
ret = -ENOMEM;
page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, dst_vma, dst_addr);
if (!page)
goto out;
page_kaddr = kmap_atomic(page);
ret = copy_from_user(page_kaddr,
(const void __user *) src_addr,
PAGE_SIZE);
kunmap_atomic(page_kaddr);
/* fallback to copy_from_user outside mmap_sem */
if (unlikely(ret)) {
ret = -EFAULT;
*pagep = page;
/* don't free the page */
goto out;
}
} else {
page = *pagep;
*pagep = NULL;
}
/*
* The memory barrier inside __SetPageUptodate makes sure that
* preceeding stores to the page contents become visible before
* the set_pte_at() write.
*/
__SetPageUptodate(page);
ret = -ENOMEM;
if (mem_cgroup_try_charge(page, dst_mm, GFP_KERNEL, &memcg, false))
goto out_release;
_dst_pte = mk_pte(page, dst_vma->vm_page_prot);
if (dst_vma->vm_flags & VM_WRITE)
_dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
ret = -EEXIST;
dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
if (!pte_none(*dst_pte))
goto out_release_uncharge_unlock;
inc_mm_counter(dst_mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, dst_vma, dst_addr, false);
mem_cgroup_commit_charge(page, memcg, false, false);
lru_cache_add_active_or_unevictable(page, dst_vma);
set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
/* No need to invalidate - it was non-present before */
update_mmu_cache(dst_vma, dst_addr, dst_pte);
pte_unmap_unlock(dst_pte, ptl);
ret = 0;
out:
return ret;
out_release_uncharge_unlock:
pte_unmap_unlock(dst_pte, ptl);
mem_cgroup_cancel_charge(page, memcg, false);
out_release:
page_cache_release(page);
goto out;
}
/*
* create a vfsmount to be automounted
*/
static struct vfsmount *afs_mntpt_do_automount(struct dentry *mntpt)
{
struct afs_super_info *super;
struct vfsmount *mnt;
struct afs_vnode *vnode;
struct page *page;
char *devname, *options;
bool rwpath = false;
int ret;
_enter("{%pd}", mntpt);
BUG_ON(!d_inode(mntpt));
ret = -ENOMEM;
devname = (char *) get_zeroed_page(GFP_KERNEL);
if (!devname)
goto error_no_devname;
options = (char *) get_zeroed_page(GFP_KERNEL);
if (!options)
goto error_no_options;
vnode = AFS_FS_I(d_inode(mntpt));
if (test_bit(AFS_VNODE_PSEUDODIR, &vnode->flags)) {
/* if the directory is a pseudo directory, use the d_name */
static const char afs_root_cell[] = ":root.cell.";
unsigned size = mntpt->d_name.len;
ret = -ENOENT;
if (size < 2 || size > AFS_MAXCELLNAME)
goto error_no_page;
if (mntpt->d_name.name[0] == '.') {
devname[0] = '#';
memcpy(devname + 1, mntpt->d_name.name, size - 1);
memcpy(devname + size, afs_root_cell,
sizeof(afs_root_cell));
rwpath = true;
} else {
devname[0] = '%';
memcpy(devname + 1, mntpt->d_name.name, size);
memcpy(devname + size + 1, afs_root_cell,
sizeof(afs_root_cell));
}
} else {
/* read the contents of the AFS special symlink */
loff_t size = i_size_read(d_inode(mntpt));
char *buf;
ret = -EINVAL;
if (size > PAGE_SIZE - 1)
goto error_no_page;
page = read_mapping_page(d_inode(mntpt)->i_mapping, 0, NULL);
if (IS_ERR(page)) {
ret = PTR_ERR(page);
goto error_no_page;
}
ret = -EIO;
if (PageError(page))
goto error;
buf = kmap_atomic(page);
memcpy(devname, buf, size);
kunmap_atomic(buf);
put_page(page);
page = NULL;
}
/* work out what options we want */
super = AFS_FS_S(mntpt->d_sb);
memcpy(options, "cell=", 5);
strcpy(options + 5, super->volume->cell->name);
if (super->volume->type == AFSVL_RWVOL || rwpath)
strcat(options, ",rwpath");
/* try and do the mount */
_debug("--- attempting mount %s -o %s ---", devname, options);
mnt = vfs_submount(mntpt, &afs_fs_type, devname, options);
_debug("--- mount result %p ---", mnt);
free_page((unsigned long) devname);
free_page((unsigned long) options);
_leave(" = %p", mnt);
return mnt;
error:
put_page(page);
error_no_page:
free_page((unsigned long) options);
error_no_options:
free_page((unsigned long) devname);
error_no_devname:
_leave(" = %d", ret);
return ERR_PTR(ret);
}
static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len)
{
char *to = kmap_atomic(page);
memcpy(to + offset, from, len);
kunmap_atomic(to);
}
static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,
struct iov_iter *i)
{
size_t skip, copy, left, wanted;
const struct iovec *iov;
char __user *buf;
void *kaddr, *to;
if (unlikely(bytes > i->count))
bytes = i->count;
if (unlikely(!bytes))
return 0;
wanted = bytes;
iov = i->iov;
skip = i->iov_offset;
buf = iov->iov_base + skip;
copy = min(bytes, iov->iov_len - skip);
if (!fault_in_pages_readable(buf, copy)) {
kaddr = kmap_atomic(page);
to = kaddr + offset;
/* first chunk, usually the only one */
left = __copy_from_user_inatomic(to, buf, copy);
copy -= left;
skip += copy;
to += copy;
bytes -= copy;
while (unlikely(!left && bytes)) {
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
left = __copy_from_user_inatomic(to, buf, copy);
copy -= left;
skip = copy;
to += copy;
bytes -= copy;
}
if (likely(!bytes)) {
kunmap_atomic(kaddr);
goto done;
}
offset = to - kaddr;
buf += copy;
kunmap_atomic(kaddr);
copy = min(bytes, iov->iov_len - skip);
}
/* Too bad - revert to non-atomic kmap */
kaddr = kmap(page);
to = kaddr + offset;
left = __copy_from_user(to, buf, copy);
copy -= left;
skip += copy;
to += copy;
bytes -= copy;
while (unlikely(!left && bytes)) {
iov++;
buf = iov->iov_base;
copy = min(bytes, iov->iov_len);
left = __copy_from_user(to, buf, copy);
copy -= left;
skip = copy;
to += copy;
bytes -= copy;
}
kunmap(page);
done:
if (skip == iov->iov_len) {
iov++;
skip = 0;
}
i->count -= wanted - bytes;
i->nr_segs -= iov - i->iov;
i->iov = iov;
i->iov_offset = skip;
return wanted - bytes;
}
static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len)
{
char *from = kmap_atomic(page);
memcpy(to, from + offset, len);
kunmap_atomic(from);
}
/* path points to first direct item of the file regarless of how many of
them are there */
int direct2indirect (struct reiserfs_transaction_handle *th, struct inode * inode,
struct path * path, struct buffer_head * unbh,
loff_t tail_offset)
{
struct super_block * sb = inode->i_sb;
struct buffer_head *up_to_date_bh ;
struct item_head * p_le_ih = PATH_PITEM_HEAD (path);
unsigned long total_tail = 0 ;
struct cpu_key end_key; /* Key to search for the last byte of the
converted item. */
struct item_head ind_ih; /* new indirect item to be inserted or
key of unfm pointer to be pasted */
int n_blk_size,
n_retval; /* returned value for reiserfs_insert_item and clones */
unp_t unfm_ptr; /* Handle on an unformatted node
that will be inserted in the
tree. */
REISERFS_SB(sb)->s_direct2indirect ++;
n_blk_size = sb->s_blocksize;
/* and key to search for append or insert pointer to the new
unformatted node. */
copy_item_head (&ind_ih, p_le_ih);
set_le_ih_k_offset (&ind_ih, tail_offset);
set_le_ih_k_type (&ind_ih, TYPE_INDIRECT);
/* Set the key to search for the place for new unfm pointer */
make_cpu_key (&end_key, inode, tail_offset, TYPE_INDIRECT, 4);
// FIXME: we could avoid this
if ( search_for_position_by_key (sb, &end_key, path) == POSITION_FOUND ) {
reiserfs_warning (sb, "PAP-14030: direct2indirect: "
"pasted or inserted byte exists in the tree %K. "
"Use fsck to repair.", &end_key);
pathrelse(path);
return -EIO;
}
p_le_ih = PATH_PITEM_HEAD (path);
unfm_ptr = cpu_to_le32 (unbh->b_blocknr);
if ( is_statdata_le_ih (p_le_ih) ) {
/* Insert new indirect item. */
set_ih_free_space (&ind_ih, 0); /* delete at nearest future */
put_ih_item_len( &ind_ih, UNFM_P_SIZE );
PATH_LAST_POSITION (path)++;
n_retval = reiserfs_insert_item (th, path, &end_key, &ind_ih, inode,
(char *)&unfm_ptr);
} else {
/* Paste into last indirect item of an object. */
n_retval = reiserfs_paste_into_item(th, path, &end_key, inode,
(char *)&unfm_ptr, UNFM_P_SIZE);
}
if ( n_retval ) {
return n_retval;
}
// note: from here there are two keys which have matching first
// three key components. They only differ by the fourth one.
/* Set the key to search for the direct items of the file */
make_cpu_key (&end_key, inode, max_reiserfs_offset (inode), TYPE_DIRECT, 4);
/* Move bytes from the direct items to the new unformatted node
and delete them. */
while (1) {
int tail_size;
/* end_key.k_offset is set so, that we will always have found
last item of the file */
if ( search_for_position_by_key (sb, &end_key, path) == POSITION_FOUND )
reiserfs_panic (sb, "PAP-14050: direct2indirect: "
"direct item (%K) not found", &end_key);
p_le_ih = PATH_PITEM_HEAD (path);
RFALSE( !is_direct_le_ih (p_le_ih),
"vs-14055: direct item expected(%K), found %h",
&end_key, p_le_ih);
tail_size = (le_ih_k_offset (p_le_ih) & (n_blk_size - 1))
+ ih_item_len(p_le_ih) - 1;
/* we only send the unbh pointer if the buffer is not up to date.
** this avoids overwriting good data from writepage() with old data
** from the disk or buffer cache
** Special case: unbh->b_page will be NULL if we are coming through
** DIRECT_IO handler here.
*/
if (!unbh->b_page || buffer_uptodate(unbh) || PageUptodate(unbh->b_page)) {
up_to_date_bh = NULL ;
} else {
up_to_date_bh = unbh ;
}
n_retval = reiserfs_delete_item (th, path, &end_key, inode,
up_to_date_bh) ;
total_tail += n_retval ;
if (tail_size == n_retval)
// done: file does not have direct items anymore
break;
}
/* if we've copied bytes from disk into the page, we need to zero
** out the unused part of the block (it was not up to date before)
*/
if (up_to_date_bh) {
unsigned pgoff = (tail_offset + total_tail - 1) & (PAGE_CACHE_SIZE - 1);
char *kaddr=kmap_atomic(up_to_date_bh->b_page, KM_USER0);
memset(kaddr + pgoff, 0, n_blk_size - total_tail) ;
kunmap_atomic(kaddr, KM_USER0);
}
REISERFS_I(inode)->i_first_direct_byte = U32_MAX;
return 0;
}
int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage)
{
struct address_space *swap_space;
struct file *swap_storage;
struct page *from_page;
struct page *to_page;
void *from_virtual;
void *to_virtual;
int i;
BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated);
BUG_ON(ttm->caching_state != tt_cached);
/*
* For user buffers, just unpin the pages, as there should be
* vma references.
*/
if (ttm->page_flags & TTM_PAGE_FLAG_USER) {
ttm_tt_free_user_pages(ttm);
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
ttm->swap_storage = NULL;
return 0;
}
if (!persistant_swap_storage) {
swap_storage = shmem_file_setup("ttm swap",
ttm->num_pages << PAGE_SHIFT,
0);
if (unlikely(IS_ERR(swap_storage))) {
printk(KERN_ERR "Failed allocating swap storage.\n");
return -ENOMEM;
}
} else
swap_storage = persistant_swap_storage;
swap_space = swap_storage->f_path.dentry->d_inode->i_mapping;
for (i = 0; i < ttm->num_pages; ++i) {
from_page = ttm->pages[i];
if (unlikely(from_page == NULL))
continue;
to_page = read_mapping_page(swap_space, i, NULL);
if (unlikely(to_page == NULL))
goto out_err;
preempt_disable();
from_virtual = kmap_atomic(from_page, KM_USER0);
to_virtual = kmap_atomic(to_page, KM_USER1);
memcpy(to_virtual, from_virtual, PAGE_SIZE);
kunmap_atomic(to_virtual, KM_USER1);
kunmap_atomic(from_virtual, KM_USER0);
preempt_enable();
set_page_dirty(to_page);
mark_page_accessed(to_page);
page_cache_release(to_page);
}
ttm_tt_free_alloced_pages(ttm);
ttm->swap_storage = swap_storage;
ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED;
if (persistant_swap_storage)
ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP;
return 0;
out_err:
if (!persistant_swap_storage)
fput(swap_storage);
return -ENOMEM;
}
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
int offset)
{
int ret = 0;
size_t clen;
unsigned long handle;
struct page *page;
unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
struct zram_meta *meta = zram->meta;
page = bvec->bv_page;
src = meta->compress_buffer;
if (is_partial_io(bvec)) {
/*
* This is a partial IO. We need to read the full page
* before to write the changes.
*/
uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
if (!uncmem) {
ret = -ENOMEM;
goto out;
}
ret = zram_decompress_page(zram, uncmem, index);
if (ret)
goto out;
}
/*
* System overwrites unused sectors. Free memory associated
* with this sector now.
*/
if (meta->table[index].handle ||
zram_test_flag(meta, index, ZRAM_ZERO))
zram_free_page(zram, index);
user_mem = kmap_atomic(page);
if (is_partial_io(bvec)) {
memcpy(uncmem + offset, user_mem + bvec->bv_offset,
bvec->bv_len);
kunmap_atomic(user_mem);
user_mem = NULL;
} else {
uncmem = user_mem;
}
if (page_zero_filled(uncmem)) {
kunmap_atomic(user_mem);
if (is_partial_io(bvec))
kfree(uncmem);
zram->stats.pages_zero++;
zram_set_flag(meta, index, ZRAM_ZERO);
ret = 0;
goto out;
}
ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen,
meta->compress_workmem);
if (!is_partial_io(bvec)) {
kunmap_atomic(user_mem);
user_mem = NULL;
uncmem = NULL;
}
if (unlikely(ret != LZO_E_OK)) {
pr_err("Compression failed! err=%d\n", ret);
goto out;
}
if (unlikely(clen > max_zpage_size)) {
zram->stats.bad_compress++;
clen = PAGE_SIZE;
src = NULL;
if (is_partial_io(bvec))
src = uncmem;
}
handle = zs_malloc(meta->mem_pool, clen);
if (!handle) {
pr_info("Error allocating memory for compressed "
"page: %u, size=%zu\n", index, clen);
ret = -ENOMEM;
goto out;
}
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
src = kmap_atomic(page);
memcpy(cmem, src, clen);
if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
kunmap_atomic(src);
zs_unmap_object(meta->mem_pool, handle);
meta->table[index].handle = handle;
meta->table[index].size = clen;
/* Update stats */
zram_stat64_add(zram, &zram->stats.compr_size, clen);
zram->stats.pages_stored++;
if (clen <= PAGE_SIZE / 2)
zram->stats.good_compress++;
out:
if (is_partial_io(bvec))
kfree(uncmem);
if (ret)
zram_stat64_inc(zram, &zram->stats.failed_writes);
return ret;
}
static int uloop_xfer(struct loop_device *lo, int cmd,
struct page *raw_page, unsigned int raw_off,
struct page *loop_page, unsigned int loop_off,
int size, sector_t real_block)
{
int err, set, bit, sz;
struct ulo_dev *dev;
char *raw_buf, *loop_buf;
loff_t pos, loff;
unsigned long pindex;
struct mutex *mtx;
const unsigned long bmp_page_bytes = PAGE_SIZE * BITS_PER_BYTE;
#if 0
UloDbg("raw_off %u, loop_off %u, sz %d, real_block %lu\n",
raw_off, loop_off, size, real_block);
#endif
#if 0
err = -EACCES;
if (unlikely(cmd != READ))
goto out;
#endif
err = -ESRCH;
dev = lo->private_data;
if (unlikely(!dev || !dev->bmp))
goto out;
pos = real_block;
//pos *= KERNEL_SECTOR_SIZE;
pos *= 512;
//pos += loop_off;
#if 0
UloDbg("pos %Lu, raw_off %u, loop_off %u, sz %d, real_block %lu\n",
pos, raw_off, loop_off, size, real_block);
#endif
// todo: optimize (or make intelligent) this loop
err = 0;
sz = size;
mtx = &dev->bmpmtx;
while (sz > 0) {
/* test bitmap */
set = 1;
loff = pos / PAGE_SIZE;
pindex = loff / bmp_page_bytes;
bit = loff % bmp_page_bytes;
//Dbg("pindex %lu, bit %d\n", pindex, bit);
mutex_lock(mtx);
err = ulo_bmp_pindex(dev, pindex);
if (!err)
set = test_bit(bit, dev->bmpbuf);
mutex_unlock(mtx);
if (unlikely(err))
goto out;
/* xfer by userspace */
if (!set) {
err = ulo_xfer_begin(dev, pos, PAGE_SIZE);
if (!err)
err = ulo_xfer_end(dev, pos, PAGE_SIZE, pindex,
bit);
if (unlikely(err))
goto out;
}
sz -= PAGE_SIZE;
pos += PAGE_SIZE;
}
/* satisfy the request */
if (!err) {
raw_buf = kmap_atomic(raw_page, KM_USER0) + raw_off;
loop_buf = kmap_atomic(loop_page, KM_USER1) + loop_off;
memcpy(loop_buf, raw_buf, size);
kunmap_atomic(raw_buf, KM_USER0);
kunmap_atomic(loop_buf, KM_USER1);
cond_resched();
}
out:
UloDbgErr(err);
return err;
}
static int ext4_destroy_inline_data_nolock(handle_t *handle,
struct inode *inode)
{
struct ext4_inode_info *ei = EXT4_I(inode);
struct ext4_xattr_ibody_find is = {
.s = { .not_found = 0, },
};
struct ext4_xattr_info i = {
.name_index = EXT4_XATTR_INDEX_SYSTEM,
.name = EXT4_XATTR_SYSTEM_DATA,
.value = NULL,
.value_len = 0,
};
int error;
if (!ei->i_inline_off)
return 0;
error = ext4_get_inode_loc(inode, &is.iloc);
if (error)
return error;
error = ext4_xattr_ibody_find(inode, &i, &is);
if (error)
goto out;
error = ext4_journal_get_write_access(handle, is.iloc.bh);
if (error)
goto out;
error = ext4_xattr_ibody_inline_set(handle, inode, &i, &is);
if (error)
goto out;
memset((void *)ext4_raw_inode(&is.iloc)->i_block,
0, EXT4_MIN_INLINE_DATA_SIZE);
if (EXT4_HAS_INCOMPAT_FEATURE(inode->i_sb,
EXT4_FEATURE_INCOMPAT_EXTENTS)) {
if (S_ISDIR(inode->i_mode) ||
S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) {
ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS);
ext4_ext_tree_init(handle, inode);
}
}
ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA);
get_bh(is.iloc.bh);
error = ext4_mark_iloc_dirty(handle, inode, &is.iloc);
EXT4_I(inode)->i_inline_off = 0;
EXT4_I(inode)->i_inline_size = 0;
ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
out:
brelse(is.iloc.bh);
if (error == -ENODATA)
error = 0;
return error;
}
static int ext4_read_inline_page(struct inode *inode, struct page *page)
{
void *kaddr;
int ret = 0;
size_t len;
struct ext4_iloc iloc;
BUG_ON(!PageLocked(page));
BUG_ON(!ext4_has_inline_data(inode));
BUG_ON(page->index);
if (!EXT4_I(inode)->i_inline_off) {
ext4_warning(inode->i_sb, "inode %lu doesn't have inline data.",
inode->i_ino);
goto out;
}
ret = ext4_get_inode_loc(inode, &iloc);
if (ret)
goto out;
len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode));
kaddr = kmap_atomic(page);
ret = ext4_read_inline_data(inode, kaddr, len, &iloc);
flush_dcache_page(page);
kunmap_atomic(kaddr);
zero_user_segment(page, len, PAGE_CACHE_SIZE);
SetPageUptodate(page);
brelse(iloc.bh);
out:
return ret;
}
static inline char *wbsd_map_sg(struct wbsd_host *host)
{
return kmap_atomic(sg_page(host->cur_sg)) + host->cur_sg->offset;
}
static inline dma_addr_t map_single_or_page(struct device *dev, void *ptr,
struct page *page, unsigned long offset, size_t size,
enum dma_data_direction dir)
{
struct dmabounce_device_info *device_info = dev->archdata.dmabounce;
dma_addr_t dma_addr;
int needs_bounce = 0;
if (device_info)
DO_STATS ( device_info->map_op_count++ );
if (page)
dma_addr = page_to_dma(dev, page) + offset;
else
dma_addr = virt_to_dma(dev, ptr);
if (dev->dma_mask) {
unsigned long mask = *dev->dma_mask;
unsigned long limit;
limit = (mask - 1) | mask;
limit = (limit + 1) & ~limit;
if (limit && size > limit) {
dev_err(dev, "DMA mapping too big (requested %#x "
"mask %#Lx)\n", size, *dev->dma_mask);
return ~0;
}
/*
* Figure out if we need to bounce from the DMA mask.
*/
needs_bounce = (dma_addr & ~mask) ||
(limit && (dma_addr + size > limit));
}
if (device_info && (needs_bounce || dma_needs_bounce(dev, dma_addr, size))) {
struct safe_buffer *buf;
buf = alloc_safe_buffer(device_info, ptr, page, offset, size, dir);
if (buf == 0) {
dev_err(dev, "%s: unable to map unsafe buffer %p!\n",
__func__, ptr);
return 0;
}
if (buf->page)
dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped "
"to %p (dma=%#x)\n", __func__,
page_address(buf->page),
page_to_dma(dev, buf->page),
buf->safe, buf->safe_dma_addr);
else
dev_dbg(dev, "%s: unsafe buffer %p (dma=%#x) mapped "
"to %p (dma=%#x)\n", __func__,
buf->ptr, virt_to_dma(dev, buf->ptr),
buf->safe, buf->safe_dma_addr);
if ((dir == DMA_TO_DEVICE) ||
(dir == DMA_BIDIRECTIONAL)) {
if (page)
ptr = kmap_atomic(page, KM_BOUNCE_READ) + offset;
dev_dbg(dev, "%s: copy unsafe %p to safe %p, size %d\n",
__func__, ptr, buf->safe, size);
memcpy(buf->safe, ptr, size);
wmb();
if (page)
kunmap_atomic(ptr - offset, KM_BOUNCE_READ);
}
dma_addr = buf->safe_dma_addr;
} else {
/*
* We don't need to sync the DMA buffer since
* it was allocated via the coherent allocators.
*/
if (page)
__dma_page_cpu_to_dev(page, offset, size, dir);
else
__dma_single_cpu_to_dev(ptr, size, dir);
}
return dma_addr;
}
static int copy_user_bh(struct page *to, struct inode *inode,
struct buffer_head *bh, unsigned long vaddr)
{
struct blk_dax_ctl dax = {
.sector = to_sector(bh, inode),
.size = bh->b_size,
};
struct block_device *bdev = bh->b_bdev;
void *vto;
if (dax_map_atomic(bdev, &dax) < 0)
return PTR_ERR(dax.addr);
vto = kmap_atomic(to);
copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
kunmap_atomic(vto);
dax_unmap_atomic(bdev, &dax);
return 0;
}
#define NO_SECTOR -1
#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_CACHE_SHIFT))
static int dax_radix_entry(struct address_space *mapping, pgoff_t index,
sector_t sector, bool pmd_entry, bool dirty)
{
struct radix_tree_root *page_tree = &mapping->page_tree;
pgoff_t pmd_index = DAX_PMD_INDEX(index);
int type, error = 0;
void *entry;
WARN_ON_ONCE(pmd_entry && !dirty);
if (dirty)
__mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
spin_lock_irq(&mapping->tree_lock);
entry = radix_tree_lookup(page_tree, pmd_index);
if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) {
index = pmd_index;
goto dirty;
}
entry = radix_tree_lookup(page_tree, index);
if (entry) {
type = RADIX_DAX_TYPE(entry);
if (WARN_ON_ONCE(type != RADIX_DAX_PTE &&
type != RADIX_DAX_PMD)) {
error = -EIO;
goto unlock;
}
if (!pmd_entry || type == RADIX_DAX_PMD)
goto dirty;
/*
* We only insert dirty PMD entries into the radix tree. This
* means we don't need to worry about removing a dirty PTE
* entry and inserting a clean PMD entry, thus reducing the
* range we would flush with a follow-up fsync/msync call.
*/
radix_tree_delete(&mapping->page_tree, index);
mapping->nrexceptional--;
}
if (sector == NO_SECTOR) {
/*
* This can happen during correct operation if our pfn_mkwrite
* fault raced against a hole punch operation. If this
* happens the pte that was hole punched will have been
* unmapped and the radix tree entry will have been removed by
* the time we are called, but the call will still happen. We
* will return all the way up to wp_pfn_shared(), where the
* pte_same() check will fail, eventually causing page fault
* to be retried by the CPU.
*/
goto unlock;
}
error = radix_tree_insert(page_tree, index,
RADIX_DAX_ENTRY(sector, pmd_entry));
if (error)
goto unlock;
mapping->nrexceptional++;
dirty:
if (dirty)
radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
unlock:
spin_unlock_irq(&mapping->tree_lock);
return error;
}
static int dax_writeback_one(struct block_device *bdev,
struct address_space *mapping, pgoff_t index, void *entry)
{
struct radix_tree_root *page_tree = &mapping->page_tree;
int type = RADIX_DAX_TYPE(entry);
struct radix_tree_node *node;
struct blk_dax_ctl dax;
void **slot;
int ret = 0;
spin_lock_irq(&mapping->tree_lock);
/*
* Regular page slots are stabilized by the page lock even
* without the tree itself locked. These unlocked entries
* need verification under the tree lock.
*/
if (!__radix_tree_lookup(page_tree, index, &node, &slot))
goto unlock;
if (*slot != entry)
goto unlock;
/* another fsync thread may have already written back this entry */
if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
goto unlock;
if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) {
ret = -EIO;
goto unlock;
}
dax.sector = RADIX_DAX_SECTOR(entry);
dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE);
spin_unlock_irq(&mapping->tree_lock);
/*
* We cannot hold tree_lock while calling dax_map_atomic() because it
* eventually calls cond_resched().
*/
ret = dax_map_atomic(bdev, &dax);
if (ret < 0)
return ret;
if (WARN_ON_ONCE(ret < dax.size)) {
ret = -EIO;
goto unmap;
}
wb_cache_pmem(dax.addr, dax.size);
spin_lock_irq(&mapping->tree_lock);
radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
spin_unlock_irq(&mapping->tree_lock);
unmap:
dax_unmap_atomic(bdev, &dax);
return ret;
unlock:
spin_unlock_irq(&mapping->tree_lock);
return ret;
}
/*
* Flush the mapping to the persistent domain within the byte range of [start,
* end]. This is required by data integrity operations to ensure file data is
* on persistent storage prior to completion of the operation.
*/
int dax_writeback_mapping_range(struct address_space *mapping,
struct block_device *bdev, struct writeback_control *wbc)
{
struct inode *inode = mapping->host;
pgoff_t start_index, end_index, pmd_index;
pgoff_t indices[PAGEVEC_SIZE];
struct pagevec pvec;
bool done = false;
int i, ret = 0;
void *entry;
if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
return -EIO;
if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
return 0;
start_index = wbc->range_start >> PAGE_CACHE_SHIFT;
end_index = wbc->range_end >> PAGE_CACHE_SHIFT;
pmd_index = DAX_PMD_INDEX(start_index);
rcu_read_lock();
entry = radix_tree_lookup(&mapping->page_tree, pmd_index);
rcu_read_unlock();
/* see if the start of our range is covered by a PMD entry */
if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD)
start_index = pmd_index;
tag_pages_for_writeback(mapping, start_index, end_index);
pagevec_init(&pvec, 0);
while (!done) {
pvec.nr = find_get_entries_tag(mapping, start_index,
PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
pvec.pages, indices);
if (pvec.nr == 0)
break;
for (i = 0; i < pvec.nr; i++) {
if (indices[i] > end_index) {
done = true;
break;
}
ret = dax_writeback_one(bdev, mapping, indices[i],
pvec.pages[i]);
if (ret < 0)
return ret;
}
}
wmb_pmem();
return 0;
}
EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh,
struct vm_area_struct *vma, struct vm_fault *vmf)
{
unsigned long vaddr = (unsigned long)vmf->virtual_address;
struct address_space *mapping = inode->i_mapping;
struct block_device *bdev = bh->b_bdev;
struct blk_dax_ctl dax = {
.sector = to_sector(bh, inode),
.size = bh->b_size,
};
pgoff_t size;
int error;
i_mmap_lock_read(mapping);
/*
* Check truncate didn't happen while we were allocating a block.
* If it did, this block may or may not be still allocated to the
* file. We can't tell the filesystem to free it because we can't
* take i_mutex here. In the worst case, the file still has blocks
* allocated past the end of the file.
*/
size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (unlikely(vmf->pgoff >= size)) {
error = -EIO;
goto out;
}
if (dax_map_atomic(bdev, &dax) < 0) {
error = PTR_ERR(dax.addr);
goto out;
}
if (buffer_unwritten(bh) || buffer_new(bh)) {
clear_pmem(dax.addr, PAGE_SIZE);
wmb_pmem();
}
dax_unmap_atomic(bdev, &dax);
error = dax_radix_entry(mapping, vmf->pgoff, dax.sector, false,
vmf->flags & FAULT_FLAG_WRITE);
if (error)
goto out;
error = vm_insert_mixed(vma, vaddr, dax.pfn);
out:
i_mmap_unlock_read(mapping);
return error;
}
/**
* __dax_fault - handle a page fault on a DAX file
* @vma: The virtual memory area where the fault occurred
* @vmf: The description of the fault
* @get_block: The filesystem method used to translate file offsets to blocks
* @complete_unwritten: The filesystem method used to convert unwritten blocks
* to written so the data written to them is exposed. This is required for
* required by write faults for filesystems that will return unwritten
* extent mappings from @get_block, but it is optional for reads as
* dax_insert_mapping() will always zero unwritten blocks. If the fs does
* not support unwritten extents, the it should pass NULL.
*
* When a page fault occurs, filesystems may call this helper in their
* fault handler for DAX files. __dax_fault() assumes the caller has done all
* the necessary locking for the page fault to proceed successfully.
*/
int __dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
get_block_t get_block, dax_iodone_t complete_unwritten)
{
struct file *file = vma->vm_file;
struct address_space *mapping = file->f_mapping;
struct inode *inode = mapping->host;
struct page *page;
struct buffer_head bh;
unsigned long vaddr = (unsigned long)vmf->virtual_address;
unsigned blkbits = inode->i_blkbits;
sector_t block;
pgoff_t size;
int error;
int major = 0;
size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (vmf->pgoff >= size)
return VM_FAULT_SIGBUS;
memset(&bh, 0, sizeof(bh));
block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits);
bh.b_bdev = inode->i_sb->s_bdev;
bh.b_size = PAGE_SIZE;
repeat:
page = find_get_page(mapping, vmf->pgoff);
if (page) {
if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) {
page_cache_release(page);
return VM_FAULT_RETRY;
}
if (unlikely(page->mapping != mapping)) {
unlock_page(page);
page_cache_release(page);
goto repeat;
}
size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (unlikely(vmf->pgoff >= size)) {
/*
* We have a struct page covering a hole in the file
* from a read fault and we've raced with a truncate
*/
error = -EIO;
goto unlock_page;
}
}
error = get_block(inode, block, &bh, 0);
if (!error && (bh.b_size < PAGE_SIZE))
error = -EIO; /* fs corruption? */
if (error)
goto unlock_page;
if (!buffer_mapped(&bh) && !buffer_unwritten(&bh) && !vmf->cow_page) {
if (vmf->flags & FAULT_FLAG_WRITE) {
error = get_block(inode, block, &bh, 1);
count_vm_event(PGMAJFAULT);
mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
major = VM_FAULT_MAJOR;
if (!error && (bh.b_size < PAGE_SIZE))
error = -EIO;
if (error)
goto unlock_page;
} else {
return dax_load_hole(mapping, page, vmf);
}
}
if (vmf->cow_page) {
struct page *new_page = vmf->cow_page;
if (buffer_written(&bh))
error = copy_user_bh(new_page, inode, &bh, vaddr);
else
clear_user_highpage(new_page, vaddr);
if (error)
goto unlock_page;
vmf->page = page;
if (!page) {
i_mmap_lock_read(mapping);
/* Check we didn't race with truncate */
size = (i_size_read(inode) + PAGE_SIZE - 1) >>
PAGE_SHIFT;
if (vmf->pgoff >= size) {
i_mmap_unlock_read(mapping);
error = -EIO;
goto out;
}
}
return VM_FAULT_LOCKED;
}
/* Check we didn't race with a read fault installing a new page */
if (!page && major)
page = find_lock_page(mapping, vmf->pgoff);
if (page) {
unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
PAGE_CACHE_SIZE, 0);
delete_from_page_cache(page);
unlock_page(page);
page_cache_release(page);
page = NULL;
}
/*
* If we successfully insert the new mapping over an unwritten extent,
* we need to ensure we convert the unwritten extent. If there is an
* error inserting the mapping, the filesystem needs to leave it as
* unwritten to prevent exposure of the stale underlying data to
* userspace, but we still need to call the completion function so
* the private resources on the mapping buffer can be released. We
* indicate what the callback should do via the uptodate variable, same
* as for normal BH based IO completions.
*/
error = dax_insert_mapping(inode, &bh, vma, vmf);
if (buffer_unwritten(&bh)) {
if (complete_unwritten)
complete_unwritten(&bh, !error);
else
WARN_ON_ONCE(!(vmf->flags & FAULT_FLAG_WRITE));
}
out:
if (error == -ENOMEM)
return VM_FAULT_OOM | major;
/* -EBUSY is fine, somebody else faulted on the same PTE */
if ((error < 0) && (error != -EBUSY))
return VM_FAULT_SIGBUS | major;
return VM_FAULT_NOPAGE | major;
unlock_page:
if (page) {
unlock_page(page);
page_cache_release(page);
}
goto out;
}
static int ocfs2_symlink_get_block(struct inode *inode, sector_t iblock,
struct buffer_head *bh_result, int create)
{
int err = -EIO;
int status;
struct ocfs2_dinode *fe = NULL;
struct buffer_head *bh = NULL;
struct buffer_head *buffer_cache_bh = NULL;
struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
void *kaddr;
trace_ocfs2_symlink_get_block(
(unsigned long long)OCFS2_I(inode)->ip_blkno,
(unsigned long long)iblock, bh_result, create);
BUG_ON(ocfs2_inode_is_fast_symlink(inode));
if ((iblock << inode->i_sb->s_blocksize_bits) > PATH_MAX + 1) {
mlog(ML_ERROR, "block offset > PATH_MAX: %llu",
(unsigned long long)iblock);
goto bail;
}
status = ocfs2_read_inode_block(inode, &bh);
if (status < 0) {
mlog_errno(status);
goto bail;
}
fe = (struct ocfs2_dinode *) bh->b_data;
if ((u64)iblock >= ocfs2_clusters_to_blocks(inode->i_sb,
le32_to_cpu(fe->i_clusters))) {
mlog(ML_ERROR, "block offset is outside the allocated size: "
"%llu\n", (unsigned long long)iblock);
goto bail;
}
if (!buffer_uptodate(bh_result) && ocfs2_inode_is_new(inode)) {
u64 blkno = le64_to_cpu(fe->id2.i_list.l_recs[0].e_blkno) +
iblock;
buffer_cache_bh = sb_getblk(osb->sb, blkno);
if (!buffer_cache_bh) {
mlog(ML_ERROR, "couldn't getblock for symlink!\n");
goto bail;
}
if (buffer_jbd(buffer_cache_bh)
&& ocfs2_inode_is_new(inode)) {
kaddr = kmap_atomic(bh_result->b_page);
if (!kaddr) {
mlog(ML_ERROR, "couldn't kmap!\n");
goto bail;
}
memcpy(kaddr + (bh_result->b_size * iblock),
buffer_cache_bh->b_data,
bh_result->b_size);
kunmap_atomic(kaddr);
set_buffer_uptodate(bh_result);
}
brelse(buffer_cache_bh);
}
map_bh(bh_result, inode->i_sb,
le64_to_cpu(fe->id2.i_list.l_recs[0].e_blkno) + iblock);
err = 0;
bail:
brelse(bh);
return err;
}
static int
ecryptfs_copy_up_encrypted_with_header(struct page *page,
struct ecryptfs_crypt_stat *crypt_stat)
{
loff_t extent_num_in_page = 0;
loff_t num_extents_per_page = (PAGE_CACHE_SIZE
/ crypt_stat->extent_size);
int rc = 0;
while (extent_num_in_page < num_extents_per_page) {
loff_t view_extent_num = ((((loff_t)page->index)
* num_extents_per_page)
+ extent_num_in_page);
size_t num_header_extents_at_front =
(crypt_stat->metadata_size / crypt_stat->extent_size);
if (view_extent_num < num_header_extents_at_front) {
char *page_virt;
page_virt = kmap_atomic(page);
memset(page_virt, 0, PAGE_CACHE_SIZE);
if (view_extent_num == 0) {
size_t written;
rc = ecryptfs_read_xattr_region(
page_virt, page->mapping->host);
strip_xattr_flag(page_virt + 16, crypt_stat);
ecryptfs_write_header_metadata(page_virt + 20,
crypt_stat,
&written);
}
kunmap_atomic(page_virt);
flush_dcache_page(page);
if (rc) {
printk(KERN_ERR "%s: Error reading xattr "
"region; rc = [%d]\n", __func__, rc);
goto out;
}
} else {
loff_t lower_offset =
((view_extent_num * crypt_stat->extent_size)
- crypt_stat->metadata_size);
rc = ecryptfs_read_lower_page_segment(
page, (lower_offset >> PAGE_CACHE_SHIFT),
(lower_offset & ~PAGE_CACHE_MASK),
crypt_stat->extent_size, page->mapping->host);
if (rc) {
printk(KERN_ERR "%s: Error attempting to read "
"extent at offset [%lld] in the lower "
"file; rc = [%d]\n", __func__,
lower_offset, rc);
goto out;
}
}
extent_num_in_page++;
}
out:
return rc;
}
/**
* nilfs_cpfile_delete_checkpoints - delete checkpoints
* @cpfile: inode of checkpoint file
* @start: start checkpoint number
* @end: end checkpoint numer
*
* Description: nilfs_cpfile_delete_checkpoints() deletes the checkpoints in
* the period from @start to @end, excluding @end itself. The checkpoints
* which have been already deleted are ignored.
*
* Return Value: On success, 0 is returned. On error, one of the following
* negative error codes is returned.
*
* %-EIO - I/O error.
*
* %-ENOMEM - Insufficient amount of memory available.
*
* %-EINVAL - invalid checkpoints.
*/
int nilfs_cpfile_delete_checkpoints(struct inode *cpfile,
__u64 start,
__u64 end)
{
struct buffer_head *header_bh, *cp_bh;
struct nilfs_cpfile_header *header;
struct nilfs_checkpoint *cp;
size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size;
__u64 cno;
void *kaddr;
unsigned long tnicps;
int ret, ncps, nicps, nss, count, i;
if (unlikely(start == 0 || start > end)) {
printk(KERN_ERR "%s: invalid range of checkpoint numbers: "
"[%llu, %llu)\n", __func__,
(unsigned long long)start, (unsigned long long)end);
return -EINVAL;
}
down_write(&NILFS_MDT(cpfile)->mi_sem);
ret = nilfs_cpfile_get_header_block(cpfile, &header_bh);
if (ret < 0)
goto out_sem;
tnicps = 0;
nss = 0;
for (cno = start; cno < end; cno += ncps) {
ncps = nilfs_cpfile_checkpoints_in_block(cpfile, cno, end);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh);
if (ret < 0) {
if (ret != -ENOENT)
break;
/* skip hole */
ret = 0;
continue;
}
kaddr = kmap_atomic(cp_bh->b_page);
cp = nilfs_cpfile_block_get_checkpoint(
cpfile, cno, cp_bh, kaddr);
nicps = 0;
for (i = 0; i < ncps; i++, cp = (void *)cp + cpsz) {
if (nilfs_checkpoint_snapshot(cp)) {
nss++;
} else if (!nilfs_checkpoint_invalid(cp)) {
nilfs_checkpoint_set_invalid(cp);
nicps++;
}
}
if (nicps > 0) {
tnicps += nicps;
mark_buffer_dirty(cp_bh);
nilfs_mdt_mark_dirty(cpfile);
if (!nilfs_cpfile_is_in_first(cpfile, cno)) {
count =
nilfs_cpfile_block_sub_valid_checkpoints(
cpfile, cp_bh, kaddr, nicps);
if (count == 0) {
/* make hole */
kunmap_atomic(kaddr);
brelse(cp_bh);
ret =
nilfs_cpfile_delete_checkpoint_block(
cpfile, cno);
if (ret == 0)
continue;
printk(KERN_ERR
"%s: cannot delete block\n",
__func__);
break;
}
}
}
kunmap_atomic(kaddr);
brelse(cp_bh);
}
if (tnicps > 0) {
kaddr = kmap_atomic(header_bh->b_page);
header = nilfs_cpfile_block_get_header(cpfile, header_bh,
kaddr);
le64_add_cpu(&header->ch_ncheckpoints, -(u64)tnicps);
mark_buffer_dirty(header_bh);
nilfs_mdt_mark_dirty(cpfile);
kunmap_atomic(kaddr);
}
brelse(header_bh);
if (nss > 0)
ret = -EBUSY;
out_sem:
up_write(&NILFS_MDT(cpfile)->mi_sem);
return ret;
}
int journal_write_metadata_buffer(transaction_t *transaction,
struct journal_head *jh_in,
struct journal_head **jh_out,
unsigned int blocknr)
{
int need_copy_out = 0;
int done_copy_out = 0;
int do_escape = 0;
char *mapped_data;
struct buffer_head *new_bh;
struct journal_head *new_jh;
struct page *new_page;
unsigned int new_offset;
struct buffer_head *bh_in = jh2bh(jh_in);
journal_t *journal = transaction->t_journal;
/*
* The buffer really shouldn't be locked: only the current committing
* transaction is allowed to write it, so nobody else is allowed
* to do any IO.
*
* akpm: except if we're journalling data, and write() output is
* also part of a shared mapping, and another thread has
* decided to launch a writepage() against this buffer.
*/
J_ASSERT_BH(bh_in, buffer_jbddirty(bh_in));
new_bh = alloc_buffer_head(GFP_NOFS|__GFP_NOFAIL);
/* keep subsequent assertions sane */
new_bh->b_state = 0;
init_buffer(new_bh, NULL, NULL);
atomic_set(&new_bh->b_count, 1);
new_jh = journal_add_journal_head(new_bh); /* This sleeps */
/*
* If a new transaction has already done a buffer copy-out, then
* we use that version of the data for the commit.
*/
jbd_lock_bh_state(bh_in);
repeat:
if (jh_in->b_frozen_data) {
done_copy_out = 1;
new_page = virt_to_page(jh_in->b_frozen_data);
new_offset = offset_in_page(jh_in->b_frozen_data);
} else {
new_page = jh2bh(jh_in)->b_page;
new_offset = offset_in_page(jh2bh(jh_in)->b_data);
}
mapped_data = kmap_atomic(new_page);
/*
* Check for escaping
*/
if (*((__be32 *)(mapped_data + new_offset)) ==
cpu_to_be32(JFS_MAGIC_NUMBER)) {
need_copy_out = 1;
do_escape = 1;
}
kunmap_atomic(mapped_data);
/*
* Do we need to do a data copy?
*/
if (need_copy_out && !done_copy_out) {
char *tmp;
jbd_unlock_bh_state(bh_in);
tmp = jbd_alloc(bh_in->b_size, GFP_NOFS);
jbd_lock_bh_state(bh_in);
if (jh_in->b_frozen_data) {
jbd_free(tmp, bh_in->b_size);
goto repeat;
}
jh_in->b_frozen_data = tmp;
mapped_data = kmap_atomic(new_page);
memcpy(tmp, mapped_data + new_offset, jh2bh(jh_in)->b_size);
kunmap_atomic(mapped_data);
new_page = virt_to_page(tmp);
new_offset = offset_in_page(tmp);
done_copy_out = 1;
}
/*
* Did we need to do an escaping? Now we've done all the
* copying, we can finally do so.
*/
if (do_escape) {
mapped_data = kmap_atomic(new_page);
*((unsigned int *)(mapped_data + new_offset)) = 0;
kunmap_atomic(mapped_data);
}
set_bh_page(new_bh, new_page, new_offset);
new_jh->b_transaction = NULL;
new_bh->b_size = jh2bh(jh_in)->b_size;
new_bh->b_bdev = transaction->t_journal->j_dev;
new_bh->b_blocknr = blocknr;
set_buffer_mapped(new_bh);
set_buffer_dirty(new_bh);
*jh_out = new_jh;
/*
* The to-be-written buffer needs to get moved to the io queue,
* and the original buffer whose contents we are shadowing or
* copying is moved to the transaction's shadow queue.
*/
JBUFFER_TRACE(jh_in, "file as BJ_Shadow");
spin_lock(&journal->j_list_lock);
__journal_file_buffer(jh_in, transaction, BJ_Shadow);
spin_unlock(&journal->j_list_lock);
jbd_unlock_bh_state(bh_in);
JBUFFER_TRACE(new_jh, "file as BJ_IO");
journal_file_buffer(new_jh, transaction, BJ_IO);
return do_escape | (done_copy_out << 1);
}
static int nilfs_cpfile_set_snapshot(struct inode *cpfile, __u64 cno)
{
struct buffer_head *header_bh, *curr_bh, *prev_bh, *cp_bh;
struct nilfs_cpfile_header *header;
struct nilfs_checkpoint *cp;
struct nilfs_snapshot_list *list;
__u64 curr, prev;
unsigned long curr_blkoff, prev_blkoff;
void *kaddr;
int ret;
if (cno == 0)
return -ENOENT; /* checkpoint number 0 is invalid */
down_write(&NILFS_MDT(cpfile)->mi_sem);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh);
if (ret < 0)
goto out_sem;
kaddr = kmap_atomic(cp_bh->b_page);
cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr);
if (nilfs_checkpoint_invalid(cp)) {
ret = -ENOENT;
kunmap_atomic(kaddr);
goto out_cp;
}
if (nilfs_checkpoint_snapshot(cp)) {
ret = 0;
kunmap_atomic(kaddr);
goto out_cp;
}
kunmap_atomic(kaddr);
ret = nilfs_cpfile_get_header_block(cpfile, &header_bh);
if (ret < 0)
goto out_cp;
kaddr = kmap_atomic(header_bh->b_page);
header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr);
list = &header->ch_snapshot_list;
curr_bh = header_bh;
get_bh(curr_bh);
curr = 0;
curr_blkoff = 0;
prev = le64_to_cpu(list->ssl_prev);
while (prev > cno) {
prev_blkoff = nilfs_cpfile_get_blkoff(cpfile, prev);
curr = prev;
if (curr_blkoff != prev_blkoff) {
kunmap_atomic(kaddr);
brelse(curr_bh);
ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr,
0, &curr_bh);
if (ret < 0)
goto out_header;
kaddr = kmap_atomic(curr_bh->b_page);
}
curr_blkoff = prev_blkoff;
cp = nilfs_cpfile_block_get_checkpoint(
cpfile, curr, curr_bh, kaddr);
list = &cp->cp_snapshot_list;
prev = le64_to_cpu(list->ssl_prev);
}
kunmap_atomic(kaddr);
if (prev != 0) {
ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0,
&prev_bh);
if (ret < 0)
goto out_curr;
} else {
prev_bh = header_bh;
get_bh(prev_bh);
}
kaddr = kmap_atomic(curr_bh->b_page);
list = nilfs_cpfile_block_get_snapshot_list(
cpfile, curr, curr_bh, kaddr);
list->ssl_prev = cpu_to_le64(cno);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(cp_bh->b_page);
cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr);
cp->cp_snapshot_list.ssl_next = cpu_to_le64(curr);
cp->cp_snapshot_list.ssl_prev = cpu_to_le64(prev);
nilfs_checkpoint_set_snapshot(cp);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(prev_bh->b_page);
list = nilfs_cpfile_block_get_snapshot_list(
cpfile, prev, prev_bh, kaddr);
list->ssl_next = cpu_to_le64(cno);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(header_bh->b_page);
header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr);
le64_add_cpu(&header->ch_nsnapshots, 1);
kunmap_atomic(kaddr);
mark_buffer_dirty(prev_bh);
mark_buffer_dirty(curr_bh);
mark_buffer_dirty(cp_bh);
mark_buffer_dirty(header_bh);
nilfs_mdt_mark_dirty(cpfile);
brelse(prev_bh);
out_curr:
brelse(curr_bh);
out_header:
brelse(header_bh);
out_cp:
brelse(cp_bh);
out_sem:
up_write(&NILFS_MDT(cpfile)->mi_sem);
return ret;
}
/*
* Clear the user page. No aliasing to deal with so we can just
* attack the kernel's existing mapping of this page.
*/
static void v6_clear_user_highpage_nonaliasing(struct page *page, unsigned long vaddr)
{
void *kaddr = kmap_atomic(page, KM_USER0);
clear_page(kaddr);
kunmap_atomic(kaddr, KM_USER0);
}
/*
* netvsc_recv_callback - Callback when we receive a packet from the
* "wire" on the specified device.
*/
static int netvsc_recv_callback(struct hv_device *device_obj,
struct hv_netvsc_packet *packet)
{
struct vm_device *device_ctx = to_vm_device(device_obj);
struct net_device *net = dev_get_drvdata(&device_ctx->device);
struct sk_buff *skb;
void *data;
int i;
unsigned long flags;
DPRINT_ENTER(NETVSC_DRV);
if (!net) {
DPRINT_ERR(NETVSC_DRV, "got receive callback but net device "
"not initialized yet");
return 0;
}
/* Allocate a skb - TODO direct I/O to pages? */
skb = netdev_alloc_skb_ip_align(net, packet->TotalDataBufferLength);
if (unlikely(!skb)) {
++net->stats.rx_dropped;
return 0;
}
/* for kmap_atomic */
local_irq_save(flags);
/*
* Copy to skb. This copy is needed here since the memory pointed by
* hv_netvsc_packet cannot be deallocated
*/
for (i = 0; i < packet->PageBufferCount; i++) {
data = kmap_atomic(pfn_to_page(packet->PageBuffers[i].Pfn),
KM_IRQ1);
data = (void *)(unsigned long)data +
packet->PageBuffers[i].Offset;
memcpy(skb_put(skb, packet->PageBuffers[i].Length), data,
packet->PageBuffers[i].Length);
kunmap_atomic((void *)((unsigned long)data -
packet->PageBuffers[i].Offset), KM_IRQ1);
}
local_irq_restore(flags);
skb->protocol = eth_type_trans(skb, net);
skb->ip_summed = CHECKSUM_NONE;
net->stats.rx_packets++;
net->stats.rx_bytes += skb->len;
/*
* Pass the skb back up. Network stack will deallocate the skb when it
* is done.
* TODO - use NAPI?
*/
netif_rx(skb);
DPRINT_DBG(NETVSC_DRV, "# of recvs %lu total size %lu",
net->stats.rx_packets, net->stats.rx_bytes);
DPRINT_EXIT(NETVSC_DRV);
return 0;
}
static inline void __clear_user_highpage(struct page *page, unsigned long vaddr)
{
void *kaddr = kmap_atomic(page);
clear_page(kaddr);
kunmap_atomic(kaddr);
}
/* attempts to compress and store an single page */
static int zswap_frontswap_store(unsigned type, pgoff_t offset,
struct page *page)
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry, *dupentry;
int ret;
unsigned int dlen = PAGE_SIZE, len;
unsigned long handle;
char *buf;
u8 *src, *dst;
struct zswap_header *zhdr;
if (!tree) {
ret = -ENODEV;
goto reject;
}
/* if this page got EIO on pageout before, give up immediately */
if (PageError(page)) {
ret = -ENOMEM;
goto reject;
}
/* reclaim space if needed */
if (zswap_is_full()) {
zswap_pool_limit_hit++;
if (zpool_shrink(zswap_pool, 1, NULL)) {
zswap_reject_reclaim_fail++;
ret = -ENOMEM;
goto reject;
}
}
/* allocate entry */
entry = zswap_entry_cache_alloc(GFP_KERNEL);
if (!entry) {
zswap_reject_kmemcache_fail++;
ret = -ENOMEM;
goto reject;
}
/* compress */
dst = get_cpu_var(zswap_dstmem);
src = kmap_atomic(page);
ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
kunmap_atomic(src);
if (ret) {
ret = -EINVAL;
goto freepage;
}
/* store */
len = dlen + sizeof(struct zswap_header);
ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN,
&handle);
if (ret == -ENOSPC) {
zswap_reject_compress_poor++;
goto freepage;
}
if (ret) {
zswap_reject_alloc_fail++;
goto freepage;
}
zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW);
zhdr->swpentry = swp_entry(type, offset);
buf = (u8 *)(zhdr + 1);
memcpy(buf, dst, dlen);
zpool_unmap_handle(zswap_pool, handle);
put_cpu_var(zswap_dstmem);
/* populate entry */
entry->offset = offset;
entry->handle = handle;
entry->length = dlen;
/* map */
spin_lock(&tree->lock);
do {
ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
if (ret == -EEXIST) {
zswap_duplicate_entry++;
/* remove from rbtree */
zswap_rb_erase(&tree->rbroot, dupentry);
zswap_entry_put(tree, dupentry);
}
} while (ret == -EEXIST);
spin_unlock(&tree->lock);
/* update stats */
atomic_inc(&zswap_stored_pages);
zswap_pool_total_size = zpool_get_total_size(zswap_pool);
return 0;
freepage:
put_cpu_var(zswap_dstmem);
zswap_entry_cache_free(entry);
reject:
return ret;
}
static int vmw_gmr_build_descriptors(struct list_head *desc_pages,
struct page *pages[],
unsigned long num_pages)
{
struct page *page, *next;
struct svga_guest_mem_descriptor *page_virtual = NULL;
struct svga_guest_mem_descriptor *desc_virtual = NULL;
unsigned int desc_per_page;
unsigned long prev_pfn;
unsigned long pfn;
int ret;
desc_per_page = PAGE_SIZE /
sizeof(struct svga_guest_mem_descriptor) - 1;
while (likely(num_pages != 0)) {
page = alloc_page(__GFP_HIGHMEM);
if (unlikely(page == NULL)) {
ret = -ENOMEM;
goto out_err;
}
list_add_tail(&page->lru, desc_pages);
/*
* Point previous page terminating descriptor to this
* page before unmapping it.
*/
if (likely(page_virtual != NULL)) {
desc_virtual->ppn = page_to_pfn(page);
kunmap_atomic(page_virtual, KM_USER0);
}
page_virtual = kmap_atomic(page, KM_USER0);
desc_virtual = page_virtual - 1;
prev_pfn = ~(0UL);
while (likely(num_pages != 0)) {
pfn = page_to_pfn(*pages);
if (pfn != prev_pfn + 1) {
if (desc_virtual - page_virtual ==
desc_per_page - 1)
break;
(++desc_virtual)->ppn = cpu_to_le32(pfn);
desc_virtual->num_pages = cpu_to_le32(1);
} else {
uint32_t tmp =
le32_to_cpu(desc_virtual->num_pages);
desc_virtual->num_pages = cpu_to_le32(tmp + 1);
}
prev_pfn = pfn;
--num_pages;
++pages;
}
(++desc_virtual)->ppn = cpu_to_le32(0);
desc_virtual->num_pages = cpu_to_le32(0);
}
if (likely(page_virtual != NULL))
kunmap_atomic(page_virtual, KM_USER0);
return 0;
out_err:
list_for_each_entry_safe(page, next, desc_pages, lru) {
list_del_init(&page->lru);
__free_page(page);
}