/**
* read_cache_pages - populate an address space with some pages & start reads against them
* @mapping: the address_space
* @pages: The address of a list_head which contains the target pages. These
* pages have their ->index populated and are otherwise uninitialised.
* @filler: callback routine for filling a single page.
* @data: private data for the callback routine.
*
* Hides the details of the LRU cache etc from the filesystems.
*/
int read_cache_pages(struct address_space *mapping, struct list_head *pages,
int (*filler)(void *, struct page *), void *data)
{
struct page *page;
int ret = 0;
while (!list_empty(pages)) {
page = list_to_page(pages);
list_del(&page->lru);
if (add_to_page_cache_lru(page, mapping,
page->index, GFP_KERNEL)) {
read_cache_pages_invalidate_page(mapping, page);
continue;
}
page_cache_release(page);
ret = filler(data, page);
if (unlikely(ret)) {
read_cache_pages_invalidate_pages(mapping, pages);
break;
}
task_io_account_read(PAGE_CACHE_SIZE);
}
return ret;
}
/**
* mpage_readpages - populate an address space with some pages & start reads against them
* @mapping: the address_space
* @pages: The address of a list_head which contains the target pages. These
* pages have their ->index populated and are otherwise uninitialised.
* The page at @pages->prev has the lowest file offset, and reads should be
* issued in @pages->prev to @pages->next order.
* @nr_pages: The number of pages at *@pages
* @get_block: The filesystem's block mapper function.
*
* This function walks the pages and the blocks within each page, building and
* emitting large BIOs.
*
* If anything unusual happens, such as:
*
* - encountering a page which has buffers
* - encountering a page which has a non-hole after a hole
* - encountering a page with non-contiguous blocks
*
* then this code just gives up and calls the buffer_head-based read function.
* It does handle a page which has holes at the end - that is a common case:
* the end-of-file on blocksize < PAGE_CACHE_SIZE setups.
*
* BH_Boundary explanation:
*
* There is a problem. The mpage read code assembles several pages, gets all
* their disk mappings, and then submits them all. That's fine, but obtaining
* the disk mappings may require I/O. Reads of indirect blocks, for example.
*
* So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
* submitted in the following order:
* 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
*
* because the indirect block has to be read to get the mappings of blocks
* 13,14,15,16. Obviously, this impacts performance.
*
* So what we do it to allow the filesystem's get_block() function to set
* BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block
* after this one will require I/O against a block which is probably close to
* this one. So you should push what I/O you have currently accumulated.
*
* This all causes the disk requests to be issued in the correct order.
*/
int
mpage_readpages(struct address_space *mapping, struct list_head *pages,
unsigned nr_pages, get_block_t get_block)
{
struct bio *bio = NULL;
unsigned page_idx;
sector_t last_block_in_bio = 0;
struct buffer_head map_bh;
unsigned long first_logical_block = 0;
gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL);
map_bh.b_state = 0;
map_bh.b_size = 0;
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
struct page *page = list_entry(pages->prev, struct page, lru);
prefetchw(&page->flags);
list_del(&page->lru);
if (!add_to_page_cache_lru(page, mapping,
page->index,
gfp)) {
bio = do_mpage_readpage(bio, page,
nr_pages - page_idx,
&last_block_in_bio, &map_bh,
&first_logical_block,
get_block, gfp);
}
page_cache_release(page);
}
BUG_ON(!list_empty(pages));
if (bio)
mpage_bio_submit(READ, bio);
return 0;
}
/* part of tail2extent. replace tail items with extent one. Content of tail
items (@count bytes) being cut are copied already into
pages. extent_writepage method is called to create extents corresponding to
those pages */
static int replace(struct inode *inode, struct page **pages, unsigned nr_pages, int count)
{
int result;
unsigned i;
STORE_COUNTERS;
if (nr_pages == 0)
return 0;
assert("vs-596", pages[0]);
/* cut copied items */
result = cut_formatting_items(inode, page_offset(pages[0]), count);
if (result)
return result;
CHECK_COUNTERS;
/* put into tree replacement for just removed items: extent item, namely */
for (i = 0; i < nr_pages; i++) {
result = add_to_page_cache_lru(pages[i], inode->i_mapping,
pages[i]->index,
mapping_gfp_mask(inode->
i_mapping));
if (result)
break;
unlock_page(pages[i]);
result = find_or_create_extent(pages[i]);
if (result)
break;
SetPageUptodate(pages[i]);
}
return result;
}
/* part of tail2extent. replace tail items with extent one. Content of tail
items (@count bytes) being cut are copied already into
pages. extent_writepage method is called to create extents corresponding to
those pages */
static int replace(struct inode *inode, struct page **pages, unsigned nr_pages, int count)
{
int result;
unsigned i;
STORE_COUNTERS;
if (nr_pages == 0)
return 0;
assert("vs-596", pages[0]);
/* cut copied items */
result = cut_formatting_items(inode, page_offset(pages[0]), count);
if (result)
return result;
CHECK_COUNTERS;
/* put into tree replacement for just removed items: extent item, namely */
for (i = 0; i < nr_pages; i++) {
result = add_to_page_cache_lru(pages[i], inode->i_mapping,
pages[i]->index,
mapping_gfp_mask(inode->
i_mapping));
if (result)
break;
unlock_page(pages[i]);
result = find_or_create_extent(pages[i]);
if (result) {
/*
* Unsuccess in critical place:
* tail has been removed,
* but extent hasn't been created
*/
warning("edward-1572",
"Report the error code %i to developers. Run FSCK",
result);
break;
}
SetPageUptodate(pages[i]);
}
return result;
}
int ext4_mpage_readpages(struct address_space *mapping,
struct list_head *pages, struct page *page,
unsigned nr_pages, bool is_readahead)
{
struct bio *bio = NULL;
sector_t last_block_in_bio = 0;
struct inode *inode = mapping->host;
const unsigned blkbits = inode->i_blkbits;
const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
const unsigned blocksize = 1 << blkbits;
sector_t block_in_file;
sector_t last_block;
sector_t last_block_in_file;
sector_t blocks[MAX_BUF_PER_PAGE];
unsigned page_block;
struct block_device *bdev = inode->i_sb->s_bdev;
int length;
unsigned relative_block = 0;
struct ext4_map_blocks map;
map.m_pblk = 0;
map.m_lblk = 0;
map.m_len = 0;
map.m_flags = 0;
for (; nr_pages; nr_pages--) {
int fully_mapped = 1;
unsigned first_hole = blocks_per_page;
prefetchw(&page->flags);
if (pages) {
page = lru_to_page(pages);
list_del(&page->lru);
if (add_to_page_cache_lru(page, mapping, page->index,
readahead_gfp_mask(mapping)))
goto next_page;
}
if (page_has_buffers(page))
goto confused;
block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
last_block = block_in_file + nr_pages * blocks_per_page;
last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
if (last_block > last_block_in_file)
last_block = last_block_in_file;
page_block = 0;
/*
* Map blocks using the previous result first.
*/
if ((map.m_flags & EXT4_MAP_MAPPED) &&
block_in_file > map.m_lblk &&
block_in_file < (map.m_lblk + map.m_len)) {
unsigned map_offset = block_in_file - map.m_lblk;
unsigned last = map.m_len - map_offset;
for (relative_block = 0; ; relative_block++) {
if (relative_block == last) {
/* needed? */
map.m_flags &= ~EXT4_MAP_MAPPED;
break;
}
if (page_block == blocks_per_page)
break;
blocks[page_block] = map.m_pblk + map_offset +
relative_block;
page_block++;
block_in_file++;
}
}
/*
* Then do more ext4_map_blocks() calls until we are
* done with this page.
*/
while (page_block < blocks_per_page) {
if (block_in_file < last_block) {
map.m_lblk = block_in_file;
map.m_len = last_block - block_in_file;
if (ext4_map_blocks(NULL, inode, &map, 0) < 0) {
set_error_page:
SetPageError(page);
zero_user_segment(page, 0,
PAGE_SIZE);
unlock_page(page);
goto next_page;
}
}
if ((map.m_flags & EXT4_MAP_MAPPED) == 0) {
fully_mapped = 0;
if (first_hole == blocks_per_page)
first_hole = page_block;
page_block++;
block_in_file++;
continue;
}
if (first_hole != blocks_per_page)
goto confused; /* hole -> non-hole */
/* Contiguous blocks? */
if (page_block && blocks[page_block-1] != map.m_pblk-1)
goto confused;
for (relative_block = 0; ; relative_block++) {
if (relative_block == map.m_len) {
/* needed? */
map.m_flags &= ~EXT4_MAP_MAPPED;
break;
} else if (page_block == blocks_per_page)
break;
blocks[page_block] = map.m_pblk+relative_block;
page_block++;
block_in_file++;
}
}
if (first_hole != blocks_per_page) {
zero_user_segment(page, first_hole << blkbits,
PAGE_SIZE);
if (first_hole == 0) {
SetPageUptodate(page);
unlock_page(page);
goto next_page;
}
} else if (fully_mapped) {
SetPageMappedToDisk(page);
}
if (fully_mapped && blocks_per_page == 1 &&
!PageUptodate(page) && cleancache_get_page(page) == 0) {
SetPageUptodate(page);
goto confused;
}
/*
* This page will go to BIO. Do we need to send this
* BIO off first?
*/
if (bio && (last_block_in_bio != blocks[0] - 1)) {
submit_and_realloc:
submit_bio(bio);
bio = NULL;
}
if (bio == NULL) {
struct fscrypt_ctx *ctx = NULL;
if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) {
ctx = fscrypt_get_ctx(inode, GFP_NOFS);
if (IS_ERR(ctx))
goto set_error_page;
}
bio = bio_alloc(GFP_KERNEL,
min_t(int, nr_pages, BIO_MAX_PAGES));
if (!bio) {
if (ctx)
fscrypt_release_ctx(ctx);
goto set_error_page;
}
bio_set_dev(bio, bdev);
bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9);
bio->bi_end_io = mpage_end_io;
bio->bi_private = ctx;
bio_set_op_attrs(bio, REQ_OP_READ,
is_readahead ? REQ_RAHEAD : 0);
}
length = first_hole << blkbits;
if (bio_add_page(bio, page, length, 0) < length)
goto submit_and_realloc;
if (((map.m_flags & EXT4_MAP_BOUNDARY) &&
(relative_block == map.m_len)) ||
(first_hole != blocks_per_page)) {
submit_bio(bio);
bio = NULL;
} else
last_block_in_bio = blocks[blocks_per_page - 1];
goto next_page;
confused:
if (bio) {
submit_bio(bio);
bio = NULL;
}
if (!PageUptodate(page))
block_read_full_page(page, ext4_get_block);
else
unlock_page(page);
next_page:
if (pages)
put_page(page);
}
BUG_ON(pages && !list_empty(pages));
if (bio)
submit_bio(bio);
return 0;
}
/*
* This function was originally taken from fs/mpage.c, and customized for f2fs.
* Major change was from block_size == page_size in f2fs by default.
*/
static int f2fs_mpage_readpages(struct address_space *mapping,
struct list_head *pages, struct page *page,
unsigned nr_pages)
{
struct bio *bio = NULL;
unsigned page_idx;
sector_t last_block_in_bio = 0;
struct inode *inode = mapping->host;
const unsigned blkbits = inode->i_blkbits;
const unsigned blocksize = 1 << blkbits;
sector_t block_in_file;
sector_t last_block;
sector_t last_block_in_file;
sector_t block_nr;
struct block_device *bdev = inode->i_sb->s_bdev;
struct f2fs_map_blocks map;
map.m_pblk = 0;
map.m_lblk = 0;
map.m_len = 0;
map.m_flags = 0;
for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
prefetchw(&page->flags);
if (pages) {
page = list_entry(pages->prev, struct page, lru);
list_del(&page->lru);
if (add_to_page_cache_lru(page, mapping,
page->index, GFP_KERNEL))
goto next_page;
}
block_in_file = (sector_t)page->index;
last_block = block_in_file + nr_pages;
last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
blkbits;
if (last_block > last_block_in_file)
last_block = last_block_in_file;
/*
* Map blocks using the previous result first.
*/
if ((map.m_flags & F2FS_MAP_MAPPED) &&
block_in_file > map.m_lblk &&
block_in_file < (map.m_lblk + map.m_len))
goto got_it;
/*
* Then do more f2fs_map_blocks() calls until we are
* done with this page.
*/
map.m_flags = 0;
if (block_in_file < last_block) {
map.m_lblk = block_in_file;
map.m_len = last_block - block_in_file;
if (f2fs_map_blocks(inode, &map, 0, false))
goto set_error_page;
}
got_it:
if ((map.m_flags & F2FS_MAP_MAPPED)) {
block_nr = map.m_pblk + block_in_file - map.m_lblk;
SetPageMappedToDisk(page);
if (!PageUptodate(page) && !cleancache_get_page(page)) {
SetPageUptodate(page);
goto confused;
}
} else {
zero_user_segment(page, 0, PAGE_CACHE_SIZE);
SetPageUptodate(page);
unlock_page(page);
goto next_page;
}
/*
* This page will go to BIO. Do we need to send this
* BIO off first?
*/
if (bio && (last_block_in_bio != block_nr - 1)) {
submit_and_realloc:
submit_bio(READ, bio);
bio = NULL;
}
if (bio == NULL) {
struct f2fs_crypto_ctx *ctx = NULL;
if (f2fs_encrypted_inode(inode) &&
S_ISREG(inode->i_mode)) {
struct page *cpage;
ctx = f2fs_get_crypto_ctx(inode);
if (IS_ERR(ctx))
goto set_error_page;
/* wait the page to be moved by cleaning */
cpage = find_lock_page(
META_MAPPING(F2FS_I_SB(inode)),
block_nr);
if (cpage) {
f2fs_wait_on_page_writeback(cpage,
DATA);
f2fs_put_page(cpage, 1);
}
}
bio = bio_alloc(GFP_KERNEL,
min_t(int, nr_pages, BIO_MAX_PAGES));
if (!bio) {
if (ctx)
f2fs_release_crypto_ctx(ctx);
goto set_error_page;
}
bio->bi_bdev = bdev;
bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(block_nr);
bio->bi_end_io = f2fs_read_end_io;
bio->bi_private = ctx;
}
if (bio_add_page(bio, page, blocksize, 0) < blocksize)
goto submit_and_realloc;
last_block_in_bio = block_nr;
goto next_page;
set_error_page:
SetPageError(page);
zero_user_segment(page, 0, PAGE_CACHE_SIZE);
unlock_page(page);
goto next_page;
confused:
if (bio) {
submit_bio(READ, bio);
bio = NULL;
}
unlock_page(page);
next_page:
if (pages)
page_cache_release(page);
}
BUG_ON(pages && !list_empty(pages));
if (bio)
submit_bio(READ, bio);
return 0;
}
static int
__generic_file_splice_read(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe, size_t len,
unsigned int flags)
{
struct address_space *mapping = in->f_mapping;
unsigned int loff, nr_pages;
struct page *pages[PIPE_BUFFERS];
struct partial_page partial[PIPE_BUFFERS];
struct page *page;
pgoff_t index, end_index;
loff_t isize;
size_t total_len;
int error, page_nr;
struct splice_pipe_desc spd = {
.pages = pages,
.partial = partial,
.flags = flags,
.ops = &page_cache_pipe_buf_ops,
};
index = *ppos >> PAGE_CACHE_SHIFT;
loff = *ppos & ~PAGE_CACHE_MASK;
nr_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
if (nr_pages > PIPE_BUFFERS)
nr_pages = PIPE_BUFFERS;
/*
* Initiate read-ahead on this page range. however, don't call into
* read-ahead if this is a non-zero offset (we are likely doing small
* chunk splice and the page is already there) for a single page.
*/
if (!loff || nr_pages > 1)
page_cache_readahead(mapping, &in->f_ra, in, index, nr_pages);
/*
* Now fill in the holes:
*/
error = 0;
total_len = 0;
/*
* Lookup the (hopefully) full range of pages we need.
*/
spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages);
/*
* If find_get_pages_contig() returned fewer pages than we needed,
* allocate the rest.
*/
index += spd.nr_pages;
while (spd.nr_pages < nr_pages) {
/*
* Page could be there, find_get_pages_contig() breaks on
* the first hole.
*/
page = find_get_page(mapping, index);
if (!page) {
/*
* Make sure the read-ahead engine is notified
* about this failure.
*/
handle_ra_miss(mapping, &in->f_ra, index);
/*
* page didn't exist, allocate one.
*/
page = page_cache_alloc_cold(mapping);
if (!page)
break;
error = add_to_page_cache_lru(page, mapping, index,
GFP_KERNEL);
if (unlikely(error)) {
page_cache_release(page);
if (error == -EEXIST)
continue;
break;
}
/*
* add_to_page_cache() locks the page, unlock it
* to avoid convoluting the logic below even more.
*/
unlock_page(page);
}
pages[spd.nr_pages++] = page;
index++;
}
/*
* Now loop over the map and see if we need to start IO on any
* pages, fill in the partial map, etc.
*/
index = *ppos >> PAGE_CACHE_SHIFT;
nr_pages = spd.nr_pages;
spd.nr_pages = 0;
for (page_nr = 0; page_nr < nr_pages; page_nr++) {
unsigned int this_len;
if (!len)
break;
/*
* this_len is the max we'll use from this page
*/
this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff);
page = pages[page_nr];
/*
* If the page isn't uptodate, we may need to start io on it
*/
if (!PageUptodate(page)) {
/*
* If in nonblock mode then dont block on waiting
* for an in-flight io page
*/
if (flags & SPLICE_F_NONBLOCK)
break;
lock_page(page);
/*
* page was truncated, stop here. if this isn't the
* first page, we'll just complete what we already
* added
*/
if (!page->mapping) {
unlock_page(page);
break;
}
/*
* page was already under io and is now done, great
*/
if (PageUptodate(page)) {
unlock_page(page);
goto fill_it;
}
/*
* need to read in the page
*/
error = mapping->a_ops->readpage(in, page);
if (unlikely(error)) {
/*
* We really should re-lookup the page here,
* but it complicates things a lot. Instead
* lets just do what we already stored, and
* we'll get it the next time we are called.
*/
if (error == AOP_TRUNCATED_PAGE)
error = 0;
break;
}
/*
* i_size must be checked after ->readpage().
*/
isize = i_size_read(mapping->host);
end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
if (unlikely(!isize || index > end_index))
break;
/*
* if this is the last page, see if we need to shrink
* the length and stop
*/
if (end_index == index) {
loff = PAGE_CACHE_SIZE - (isize & ~PAGE_CACHE_MASK);
if (total_len + loff > isize)
break;
/*
* force quit after adding this page
*/
len = this_len;
this_len = min(this_len, loff);
loff = 0;
}
}
fill_it:
partial[page_nr].offset = loff;
partial[page_nr].len = this_len;
len -= this_len;
total_len += this_len;
loff = 0;
spd.nr_pages++;
index++;
}
/*
* Release any pages at the end, if we quit early. 'i' is how far
* we got, 'nr_pages' is how many pages are in the map.
*/
while (page_nr < nr_pages)
page_cache_release(pages[page_nr++]);
if (spd.nr_pages)
return splice_to_pipe(pipe, &spd);
return error;
}
/*
* This is a little more tricky than the file -> pipe splicing. There are
* basically three cases:
*
* - Destination page already exists in the address space and there
* are users of it. For that case we have no other option that
* copying the data. Tough luck.
* - Destination page already exists in the address space, but there
* are no users of it. Make sure it's uptodate, then drop it. Fall
* through to last case.
* - Destination page does not exist, we can add the pipe page to
* the page cache and avoid the copy.
*
* If asked to move pages to the output file (SPLICE_F_MOVE is set in
* sd->flags), we attempt to migrate pages from the pipe to the output
* file address space page cache. This is possible if no one else has
* the pipe page referenced outside of the pipe and page cache. If
* SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
* a new page in the output file page cache and fill/dirty that.
*/
static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
struct splice_desc *sd)
{
struct file *file = sd->file;
struct address_space *mapping = file->f_mapping;
unsigned int offset, this_len;
struct page *page;
pgoff_t index;
int ret;
/*
* make sure the data in this buffer is uptodate
*/
ret = buf->ops->pin(pipe, buf);
if (unlikely(ret))
return ret;
index = sd->pos >> PAGE_CACHE_SHIFT;
offset = sd->pos & ~PAGE_CACHE_MASK;
this_len = sd->len;
if (this_len + offset > PAGE_CACHE_SIZE)
this_len = PAGE_CACHE_SIZE - offset;
/*
* Reuse buf page, if SPLICE_F_MOVE is set and we are doing a full
* page.
*/
if ((sd->flags & SPLICE_F_MOVE) && this_len == PAGE_CACHE_SIZE) {
/*
* If steal succeeds, buf->page is now pruned from the
* pagecache and we can reuse it. The page will also be
* locked on successful return.
*/
if (buf->ops->steal(pipe, buf))
goto find_page;
page = buf->page;
if (add_to_page_cache(page, mapping, index, GFP_KERNEL)) {
unlock_page(page);
goto find_page;
}
page_cache_get(page);
if (!(buf->flags & PIPE_BUF_FLAG_LRU))
lru_cache_add(page);
} else {
find_page:
page = find_lock_page(mapping, index);
if (!page) {
ret = -ENOMEM;
page = page_cache_alloc_cold(mapping);
if (unlikely(!page))
goto out_ret;
/*
* This will also lock the page
*/
ret = add_to_page_cache_lru(page, mapping, index,
GFP_KERNEL);
if (unlikely(ret))
goto out;
}
/*
* We get here with the page locked. If the page is also
* uptodate, we don't need to do more. If it isn't, we
* may need to bring it in if we are not going to overwrite
* the full page.
*/
if (!PageUptodate(page)) {
if (this_len < PAGE_CACHE_SIZE) {
ret = mapping->a_ops->readpage(file, page);
if (unlikely(ret))
goto out;
lock_page(page);
if (!PageUptodate(page)) {
/*
* Page got invalidated, repeat.
*/
if (!page->mapping) {
unlock_page(page);
page_cache_release(page);
goto find_page;
}
ret = -EIO;
goto out;
}
} else
SetPageUptodate(page);
}
}
ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
if (unlikely(ret)) {
loff_t isize = i_size_read(mapping->host);
if (ret != AOP_TRUNCATED_PAGE)
unlock_page(page);
page_cache_release(page);
if (ret == AOP_TRUNCATED_PAGE)
goto find_page;
/*
* prepare_write() may have instantiated a few blocks
* outside i_size. Trim these off again.
*/
if (sd->pos + this_len > isize)
vmtruncate(mapping->host, isize);
goto out_ret;
}
if (buf->page != page) {
/*
* Careful, ->map() uses KM_USER0!
*/
char *src = buf->ops->map(pipe, buf, 1);
char *dst = kmap_atomic(page, KM_USER1);
memcpy(dst + offset, src + buf->offset, this_len);
flush_dcache_page(page);
kunmap_atomic(dst, KM_USER1);
buf->ops->unmap(pipe, buf, src);
}
ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
if (!ret) {
/*
* Return the number of bytes written and mark page as
* accessed, we are now done!
*/
ret = this_len;
mark_page_accessed(page);
balance_dirty_pages_ratelimited(mapping);
} else if (ret == AOP_TRUNCATED_PAGE) {
page_cache_release(page);
goto find_page;
}
out:
page_cache_release(page);
unlock_page(page);
out_ret:
return ret;
}
/*
* ecryptfs_readpages
* This is kernel code, not Java.
*
* Read in multiple pages and decrypt them if necessary.
*/
static int ecryptfs_readpages(struct file *filp, struct address_space *mapping,
struct list_head *pages, unsigned nr_pages)
{
struct ecryptfs_crypt_stat *crypt_stat =
&ecryptfs_inode_to_private(mapping->host)->crypt_stat;
struct page **pgs = NULL;
unsigned int page_idx = 0;
int rc = 0;
int nodec = 0; //no decryption needed flag
/* u32 sz = 0; */
if (!crypt_stat
|| !(crypt_stat->flags & ECRYPTFS_ENCRYPTED)
|| (crypt_stat->flags & ECRYPTFS_NEW_FILE)
|| (crypt_stat->flags & ECRYPTFS_VIEW_AS_ENCRYPTED)) {
nodec = 1;
}
if (!nodec) {
/* sz = __ilog2_u32((u32)__roundup_pow_of_two(nr_pages*sizeof(struct page*))); */
/* pgs = (struct page **)__get_free_pages(GFP_KERNEL, sz); */
pgs = (struct page **)kmalloc(nr_pages*sizeof(struct page*), GFP_KERNEL);
if (!pgs) {
return -EFAULT;
}
}
/* printk("[g-ecryptfs] Info: in read_pages read %d pages %d: \n", nr_pages, nodec); */
/* dump_stack(); */
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
struct page *page = list_entry(pages->prev, struct page, lru);
list_del(&page->lru);
if (add_to_page_cache_lru(page, mapping, page->index, GFP_KERNEL)) {
printk("[g-eCryptfs] INFO: cannot add page %lu to cache lru\n",
(unsigned long)(page->index));
} else {
if (nodec)
rc |= ecryptfs_readpage(filp, page);
}
if (nodec)
page_cache_release(page);
else
pgs[page_idx] = page;
}
if (!nodec) {
rc = ecryptfs_decrypt_pages(pgs, nr_pages);
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
if (rc)
ClearPageUptodate(pgs[page_idx]);
else
SetPageUptodate(pgs[page_idx]);
unlock_page(pgs[page_idx]);
page_cache_release(pgs[page_idx]);
}
kfree(pgs);
/* free_pages((unsigned long)pgs, sz); */
}
return 0;
}
/*
* This is a little more tricky than the file -> pipe splicing. There are
* basically three cases:
*
* - Destination page already exists in the address space and there
* are users of it. For that case we have no other option that
* copying the data. Tough luck.
* - Destination page already exists in the address space, but there
* are no users of it. Make sure it's uptodate, then drop it. Fall
* through to last case.
* - Destination page does not exist, we can add the pipe page to
* the page cache and avoid the copy.
*
* If asked to move pages to the output file (SPLICE_F_MOVE is set in
* sd->flags), we attempt to migrate pages from the pipe to the output
* file address space page cache. This is possible if no one else has
* the pipe page referenced outside of the pipe and page cache. If
* SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create
* a new page in the output file page cache and fill/dirty that.
*/
static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
struct splice_desc *sd)
{
struct file *file = sd->file;
struct address_space *mapping = file->f_mapping;
unsigned int offset, this_len;
struct page *page;
pgoff_t index;
int ret;
/*
* make sure the data in this buffer is uptodate
*/
ret = buf->ops->pin(pipe, buf);
if (unlikely(ret))
return ret;
index = sd->pos >> PAGE_CACHE_SHIFT;
offset = sd->pos & ~PAGE_CACHE_MASK;
this_len = sd->len;
if (this_len + offset > PAGE_CACHE_SIZE)
this_len = PAGE_CACHE_SIZE - offset;
find_page:
page = find_lock_page(mapping, index);
if (!page) {
ret = -ENOMEM;
page = page_cache_alloc_cold(mapping);
if (unlikely(!page))
goto out_ret;
/*
* This will also lock the page
*/
ret = add_to_page_cache_lru(page, mapping, index,
GFP_KERNEL);
if (unlikely(ret))
goto out;
}
ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len);
if (unlikely(ret)) {
loff_t isize = i_size_read(mapping->host);
if (ret != AOP_TRUNCATED_PAGE)
unlock_page(page);
page_cache_release(page);
if (ret == AOP_TRUNCATED_PAGE)
goto find_page;
/*
* prepare_write() may have instantiated a few blocks
* outside i_size. Trim these off again.
*/
if (sd->pos + this_len > isize)
vmtruncate(mapping->host, isize);
goto out_ret;
}
if (buf->page != page) {
/*
* Careful, ->map() uses KM_USER0!
*/
char *src = buf->ops->map(pipe, buf, 1);
char *dst = kmap_atomic(page, KM_USER1);
memcpy(dst + offset, src + buf->offset, this_len);
flush_dcache_page(page);
kunmap_atomic(dst, KM_USER1);
buf->ops->unmap(pipe, buf, src);
}
ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len);
if (ret) {
if (ret == AOP_TRUNCATED_PAGE) {
page_cache_release(page);
goto find_page;
}
if (ret < 0)
goto out;
/*
* Partial write has happened, so 'ret' already initialized by
* number of bytes written, Where is nothing we have to do here.
*/
} else
ret = this_len;
/*
* Return the number of bytes written and mark page as
* accessed, we are now done!
*/
mark_page_accessed(page);
balance_dirty_pages_ratelimited(mapping);
out:
page_cache_release(page);
unlock_page(page);
out_ret:
return ret;
}
int
mpage_readpages_compressed(struct address_space *mapping, struct list_head *pages,
unsigned nr_pages, get_block_t get_block)
{
struct bio *bio = NULL;
struct inode *inode = mapping->host;
unsigned page_idx, count, nr_to_read;
sector_t last_block_in_bio = 0;
struct buffer_head map_bh;
unsigned long first_logical_block = 0;
struct compressed_bio *cb;
struct page *page;
loff_t isize = i_size_read(inode);
unsigned long prev_index = 0, end_index = ((isize - 1) >> PAGE_CACHE_SHIFT);
struct list_head *list;
list = pages->prev;
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
page = list_entry(list, struct page, lru);
prev_index = page->index;
list = list->prev;
}
if (prev_index == end_index || nr_pages >= COMPRESSION_STRIDE_LEN)
goto again;
/* Start Readahead : mm/readahead.c*/
prev_index++;
nr_to_read = COMPRESSION_STRIDE_LEN - nr_pages;
printk(KERN_INFO "Start Readahead for %u pages", nr_to_read);
for (page_idx = 0; page_idx < nr_to_read; page_idx++) {
pgoff_t page_offset = prev_index + page_idx;
if (page_offset > end_index)
break;
rcu_read_lock();
page = radix_tree_lookup(&mapping->page_tree, page_offset);
rcu_read_unlock();
if (page)
continue;
page = page_cache_alloc_readahead(mapping);
if (!page) {
printk(KERN_INFO "Page Readahead Failed");
break;
}
page->index = page_offset;
list_add(&page->lru, pages);
if (page_idx == nr_to_read)
SetPageReadahead(page);
nr_pages++;
}
again:
cb = NULL;
map_bh.b_state = 0;
map_bh.b_size = 0;
printk(KERN_INFO "\n\n==> IN MPAGE_READPAGES | nr_pages : %u", nr_pages);
count = min_t(unsigned, nr_pages, COMPRESSION_STRIDE_LEN);
for (page_idx = 0; page_idx < count; page_idx++) {
if (list_empty(pages->prev))
break;
page = list_entry(pages->prev, struct page, lru);
prefetchw(&page->flags);
list_del(&page->lru);
if (!add_to_page_cache_lru(page, mapping,
page->index, GFP_KERNEL)) {
/* first_logical : first_logical_block_of_extent
* last_blk_in_bio : increments to last physical of bio
*/
printk(KERN_INFO "\n IN DO_MPAGE_READPAGE");
bio = do_mpage_readpage(bio, page,
nr_pages - page_idx,
&last_block_in_bio, &map_bh,
&first_logical_block, &cb,
get_block);
assert(cb);
printk(KERN_INFO "\n OUT DO_MPAGE_READPAGE");
}
page_cache_release(page);
}
printk(KERN_INFO "\n\n==>OUT MPAGE_READPAGES | first_logical : %lu",first_logical_block);
/* create and submit bio for compressed_read */
for (page_idx = 0; page_idx < cb->nr_pages; page_idx++) {
page = alloc_page(GFP_NOFS |__GFP_HIGHMEM);
page->mapping = NULL;
page->index = cb->start + page_idx;
cb->compressed_pages[page_idx] = page;
/* Try to add pages to exists bio */
if (!bio || !bio_add_page(bio, page, PAGE_CACHE_SIZE, 0)) {
/* Couldn't add. So submit old bio and allocate new bio */
if (bio)
bio = mpage_bio_submit(READ, bio);
bio = mpage_alloc(map_bh.b_bdev, (map_bh.b_blocknr + page_idx) << (cb->inode->i_blkbits - 9),
min_t(int, cb->nr_pages - page_idx, bio_get_nr_vecs(map_bh.b_bdev)),
GFP_NOFS);
bio->bi_private = cb;
if (!bio_add_page(bio, page, PAGE_CACHE_SIZE, 0))
assert(0); /* why? */
}
}
if (bio)
bio = mpage_bio_submit(READ, bio);
nr_pages -= count;
if(nr_pages > 0)
goto again;
BUG_ON(!list_empty(pages));
return 0;
}
/**
* scfs_readpages
*
* Parameters:
* @file: upper file
* @*mapping: address_space struct for the file
* @*pages: list of pages to read in
* @nr_pages: number of pages to read in
*
* Return:
* SCFS_SUCCESS if success, otherwise if error
*
* Description:
* - Asynchronously read pages for readahead. A scaling number of background threads
* will read & decompress them in a slightly deferred but parallelized manner.
*/
static int
scfs_readpages(struct file *file, struct address_space *mapping,
struct list_head *pages, unsigned nr_pages)
{
struct scfs_inode_info *sii = SCFS_I(file->f_mapping->host);
struct scfs_sb_info *sbi = SCFS_S(file->f_mapping->host->i_sb);
struct file *lower_file = NULL;
struct page *page;
struct scfs_cinfo cinfo;
loff_t i_size;
pgoff_t start, end;
int page_idx, page_idx_readahead = 1024, ret = 0;
int readahead_page = 0;
int prev_cbi = 0;
int prev_cluster = -1, cur_cluster = -1;
int cluster_idx = 0;
i_size = i_size_read(&sii->vfs_inode);
if (!i_size) {
SCFS_PRINT("file %s: i_size is zero, "
"flags 0x%x sii->clust_info_size %d\n",
file->f_path.dentry->d_name.name, sii->flags,
sii->cinfo_array_size);
return 0;
}
#ifdef SCFS_ASYNC_READ_PROFILE
atomic_add(nr_pages, &sbi->scfs_standby_readpage_count);
#endif
#ifdef SCFS_NOTIFY_RANDOM_READ
lower_file = scfs_lower_file(file);
if (!lower_file) {
SCFS_PRINT_ERROR("file %s: lower file is null!\n",
file->f_path.dentry->d_name.name);
return -EINVAL;
}
/* if the read request was random (enough), hint it to the lower file.
* scfs_sequential_page_number is the tunable threshold.
* filemap.c will later on refer to this FMODE_RANDOM flag.
*/
spin_lock(&lower_file->f_lock);
if (nr_pages > sbi->scfs_sequential_page_number)
lower_file->f_mode &= ~FMODE_RANDOM;
else
lower_file->f_mode |= FMODE_RANDOM;
spin_unlock(&lower_file->f_lock);
#endif
lower_file = scfs_lower_file(file);
page = list_entry(pages->prev, struct page, lru);
cluster_idx = page->index / (sii->cluster_size / PAGE_SIZE);
if (sii->compressed) {
mutex_lock(&sii->cinfo_mutex);
ret = get_cluster_info(file, cluster_idx, &cinfo);
mutex_unlock(&sii->cinfo_mutex);
if (ret) {
SCFS_PRINT_ERROR("err in get_cluster_info, ret : %d,"
"i_size %lld\n", ret, i_size);
return ret;
}
if (!cinfo.size || cinfo.size > sii->cluster_size) {
SCFS_PRINT_ERROR("file %s: cinfo is invalid, "
"clust %u cinfo.size %u\n",
file->f_path.dentry->d_name.name,
cluster_idx, cinfo.size);
return -EINVAL;
}
start = (pgoff_t)(cinfo.offset / PAGE_SIZE);
} else {
start = (pgoff_t)(cluster_idx * sii->cluster_size / PAGE_SIZE);
}
cluster_idx = (page->index + nr_pages - 1) / (sii->cluster_size / PAGE_SIZE);
if (sii->compressed) {
mutex_lock(&sii->cinfo_mutex);
ret = get_cluster_info(file, cluster_idx, &cinfo);
mutex_unlock(&sii->cinfo_mutex);
if (ret) {
SCFS_PRINT_ERROR("err in get_cluster_info, ret : %d,"
"i_size %lld\n", ret, i_size);
return ret;
}
if (!cinfo.size || cinfo.size > sii->cluster_size) {
SCFS_PRINT_ERROR("file %s: cinfo is invalid, "
"clust %u cinfo.size %u\n",
file->f_path.dentry->d_name.name,
cluster_idx, cinfo.size);
return -EINVAL;
}
end = (pgoff_t)((cinfo.offset + cinfo.size -1) / PAGE_SIZE);
} else {
end = (pgoff_t)(((cluster_idx + 1) * sii->cluster_size - 1) / PAGE_SIZE);
/* check upper inode size */
/* out of range? on compressed file, it is handled returning error,
which one is right? */
if (end > (i_size / PAGE_SIZE))
end = (i_size / PAGE_SIZE);
}
force_page_cache_readahead(lower_file->f_mapping, lower_file,
start, (unsigned long)(end - start +1));
for (page_idx = 0; page_idx < nr_pages; page_idx++) {
page = list_entry(pages->prev, struct page, lru);
list_del(&page->lru);
if (PageReadahead(page))
page_idx_readahead = page_idx;
ret = add_to_page_cache_lru(page, mapping,
page->index, GFP_KERNEL);
if (ret) {
SCFS_PRINT("adding to page cache failed, "
"page %x page->idx %d ret %d\n",
page, page->index, ret);
page_cache_release(page);
continue;
}
/* memory buffer is full or synchronous read request -
call scfs_readpage to read now */
if (sbi->page_buffer_next_filling_index_smb ==
MAX_PAGE_BUFFER_SIZE_SMB || page_idx < page_idx_readahead) {
cur_cluster = PAGE_TO_CLUSTER_INDEX(page, sii);
if (prev_cluster == cur_cluster && prev_cbi > 0)
prev_cbi = _scfs_readpage(file, page, prev_cbi - 1);
else
prev_cbi = _scfs_readpage(file, page, -1);
prev_cluster = cur_cluster;
page_cache_release(page); /* refer line 701 */
} else {
spin_lock(&sbi->spinlock_smb);
/* Queue is not full so add the page into the queue.
Also, here we increase file->f_count to protect
the file structs from multi-threaded accesses */
atomic_long_inc(&SCFS_F(file)->lower_file->f_count);
atomic_long_inc(&file->f_count);
sbi->page_buffer_smb[sbi->page_buffer_next_filling_index_smb] = page;
sbi->file_buffer_smb[sbi->page_buffer_next_filling_index_smb++] = file;
/* check whether page buffer is full and set page buffer full if needed */
if (((sbi->page_buffer_next_filling_index_smb == MAX_PAGE_BUFFER_SIZE_SMB) &&
sbi->page_buffer_next_io_index_smb == 0) ||
(sbi->page_buffer_next_filling_index_smb ==
sbi->page_buffer_next_io_index_smb))
sbi->page_buffer_next_filling_index_smb = MAX_PAGE_BUFFER_SIZE_SMB;
else if (sbi->page_buffer_next_filling_index_smb == MAX_PAGE_BUFFER_SIZE_SMB)
sbi->page_buffer_next_filling_index_smb = 0;
spin_unlock(&sbi->spinlock_smb);
++readahead_page;
}
//page_cache_release(page);
}
if (readahead_page > 0)
wakeup_smb_thread(sbi);
SCFS_PRINT("<e>\n");
#ifdef SCFS_ASYNC_READ_PROFILE
atomic_sub(nr_pages, &sbi->scfs_standby_readpage_count);
#endif
return 0;
}
