Buddy::Buddy (mword phys, mword virt, mword f_addr, size_t size)
{
// Compute maximum aligned block size
unsigned long bit = bit_scan_reverse (size);
// Compute maximum aligned physical block address (base)
base = phys_to_virt (align_up (phys, 1ul << bit));
// Convert block size to page order
order = bit + 1 - PAGE_BITS;
trace (TRACE_MEMORY, "POOL: %#010lx-%#010lx O:%lu",
phys,
phys + size,
order);
// Allocate block-list heads
size -= order * sizeof *head;
head = reinterpret_cast<Block *>(virt + size);
// Allocate block-index storage
size -= size / (PAGE_SIZE + sizeof *index) * sizeof *index;
size &= ~PAGE_MASK;
min_idx = page_to_index (virt);
max_idx = page_to_index (virt + size);
index = reinterpret_cast<Block *>(virt + size) - min_idx;
for (unsigned i = 0; i < order; i++)
head[i].next = head[i].prev = head + i;
for (mword i = f_addr; i < virt + size; i += PAGE_SIZE)
free (i);
}
/*
* Free physically contiguous memory region.
* @param virt Linear block base address
*/
void Buddy::free (mword virt)
{
signed long idx = page_to_index (virt);
// Ensure virt is within allocator range
assert (idx >= min_idx && idx < max_idx);
Block *block = index_to_block (idx);
// Ensure block is marked as used
assert (block->tag == Block::Used);
// Ensure corresponding physical block is order-aligned
assert ((virt_to_phys (virt) & ((1ul << (block->ord + PAGE_BITS)) - 1)) == 0);
Lock_guard <Spinlock> guard (lock);
unsigned short ord;
for (ord = block->ord; ord < order - 1; ord++) {
// Compute block index and corresponding buddy index
signed long block_idx = block_to_index (block);
signed long buddy_idx = block_idx ^ (1ul << ord);
// Buddy outside mempool
if (buddy_idx < min_idx || buddy_idx >= max_idx)
break;
Block *buddy = index_to_block (buddy_idx);
// Buddy in use or fragmented
if (buddy->tag == Block::Used || buddy->ord != ord)
break;
// Dequeue buddy from block list
buddy->prev->next = buddy->next;
buddy->next->prev = buddy->prev;
// Merge block with buddy
if (buddy < block)
block = buddy;
}
block->ord = ord;
block->tag = Block::Free;
// Enqueue final-size block
Block *h = head + ord;
block->prev = h;
block->next = h->next;
block->next->prev = h->next = block;
}
int bio_add_page(bio_t *bio, struct page *page, unsigned int rw)
{
struct bio_vec *bv = bio->bi_vec;
bv = &bio->bi_vec[bio->bi_idex++];
bv->bv_page = page;
bio->bi_idex++;
bio->bi_size += PAGE_SIZE;
bio->bi_rw = rw ;
bio->bi_front_nr = page_to_index(page);
bio->bi_back_nr = bio->bi_front_nr;
return 0;
}
/**
* invalidate_inode_pages2_range - remove range of pages from an address_space
* @mapping: the address_space
* @start: the page offset 'from' which to invalidate
* @end: the page offset 'to' which to invalidate (inclusive)
*
* Any pages which are found to be mapped into pagetables are unmapped prior to
* invalidation.
*
* Returns -EBUSY if any pages could not be invalidated.
*/
int invalidate_inode_pages2_range(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
pgoff_t indices[PAGEVEC_SIZE];
struct pagevec pvec;
pgoff_t index;
int i;
int ret = 0;
int ret2 = 0;
int did_range_unmap = 0;
cleancache_invalidate_inode(mapping);
pagevec_init(&pvec, 0);
index = start;
while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
index = indices[i];
if (index > end)
break;
if (radix_tree_exceptional_entry(page)) {
clear_exceptional_entry(mapping, index, page);
continue;
}
lock_page(page);
WARN_ON(page_to_index(page) != index);
if (page->mapping != mapping) {
unlock_page(page);
continue;
}
wait_on_page_writeback(page);
if (page_mapped(page)) {
if (!did_range_unmap) {
/*
* Zap the rest of the file in one hit.
*/
unmap_mapping_range(mapping,
(loff_t)index << PAGE_SHIFT,
(loff_t)(1 + end - index)
<< PAGE_SHIFT,
0);
did_range_unmap = 1;
} else {
/*
* Just zap this page
*/
unmap_mapping_range(mapping,
(loff_t)index << PAGE_SHIFT,
PAGE_SIZE, 0);
}
}
BUG_ON(page_mapped(page));
ret2 = do_launder_page(mapping, page);
if (ret2 == 0) {
if (!invalidate_complete_page2(mapping, page))
ret2 = -EBUSY;
}
if (ret2 < 0)
ret = ret2;
unlock_page(page);
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
index++;
}
cleancache_invalidate_inode(mapping);
return ret;
}
/**
* invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
* @mapping: the address_space which holds the pages to invalidate
* @start: the offset 'from' which to invalidate
* @end: the offset 'to' which to invalidate (inclusive)
*
* This function only removes the unlocked pages, if you want to
* remove all the pages of one inode, you must call truncate_inode_pages.
*
* invalidate_mapping_pages() will not block on IO activity. It will not
* invalidate pages which are dirty, locked, under writeback or mapped into
* pagetables.
*/
unsigned long invalidate_mapping_pages(struct address_space *mapping,
pgoff_t start, pgoff_t end)
{
pgoff_t indices[PAGEVEC_SIZE];
struct pagevec pvec;
pgoff_t index = start;
unsigned long ret;
unsigned long count = 0;
int i;
pagevec_init(&pvec, 0);
while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
index = indices[i];
if (index > end)
break;
if (radix_tree_exceptional_entry(page)) {
clear_exceptional_entry(mapping, index, page);
continue;
}
if (!trylock_page(page))
continue;
WARN_ON(page_to_index(page) != index);
/* Middle of THP: skip */
if (PageTransTail(page)) {
unlock_page(page);
continue;
} else if (PageTransHuge(page)) {
index += HPAGE_PMD_NR - 1;
i += HPAGE_PMD_NR - 1;
/* 'end' is in the middle of THP */
if (index == round_down(end, HPAGE_PMD_NR))
continue;
}
ret = invalidate_inode_page(page);
unlock_page(page);
/*
* Invalidation is a hint that the page is no longer
* of interest and try to speed up its reclaim.
*/
if (!ret)
deactivate_file_page(page);
count += ret;
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
index++;
}
return count;
}
/**
* truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
* @mapping: mapping to truncate
* @lstart: offset from which to truncate
* @lend: offset to which to truncate (inclusive)
*
* Truncate the page cache, removing the pages that are between
* specified offsets (and zeroing out partial pages
* if lstart or lend + 1 is not page aligned).
*
* Truncate takes two passes - the first pass is nonblocking. It will not
* block on page locks and it will not block on writeback. The second pass
* will wait. This is to prevent as much IO as possible in the affected region.
* The first pass will remove most pages, so the search cost of the second pass
* is low.
*
* We pass down the cache-hot hint to the page freeing code. Even if the
* mapping is large, it is probably the case that the final pages are the most
* recently touched, and freeing happens in ascending file offset order.
*
* Note that since ->invalidatepage() accepts range to invalidate
* truncate_inode_pages_range is able to handle cases where lend + 1 is not
* page aligned properly.
*/
void truncate_inode_pages_range(struct address_space *mapping,
loff_t lstart, loff_t lend)
{
pgoff_t start; /* inclusive */
pgoff_t end; /* exclusive */
unsigned int partial_start; /* inclusive */
unsigned int partial_end; /* exclusive */
struct pagevec pvec;
pgoff_t indices[PAGEVEC_SIZE];
pgoff_t index;
int i;
cleancache_invalidate_inode(mapping);
if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
return;
/* Offsets within partial pages */
partial_start = lstart & (PAGE_SIZE - 1);
partial_end = (lend + 1) & (PAGE_SIZE - 1);
/*
* 'start' and 'end' always covers the range of pages to be fully
* truncated. Partial pages are covered with 'partial_start' at the
* start of the range and 'partial_end' at the end of the range.
* Note that 'end' is exclusive while 'lend' is inclusive.
*/
start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (lend == -1)
/*
* lend == -1 indicates end-of-file so we have to set 'end'
* to the highest possible pgoff_t and since the type is
* unsigned we're using -1.
*/
end = -1;
else
end = (lend + 1) >> PAGE_SHIFT;
pagevec_init(&pvec, 0);
index = start;
while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
min(end - index, (pgoff_t)PAGEVEC_SIZE),
indices)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
index = indices[i];
if (index >= end)
break;
if (radix_tree_exceptional_entry(page)) {
clear_exceptional_entry(mapping, index, page);
continue;
}
if (!trylock_page(page))
continue;
WARN_ON(page_to_index(page) != index);
if (PageWriteback(page)) {
unlock_page(page);
continue;
}
truncate_inode_page(mapping, page);
unlock_page(page);
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
cond_resched();
index++;
}
if (partial_start) {
struct page *page = find_lock_page(mapping, start - 1);
if (page) {
unsigned int top = PAGE_SIZE;
if (start > end) {
/* Truncation within a single page */
top = partial_end;
partial_end = 0;
}
wait_on_page_writeback(page);
zero_user_segment(page, partial_start, top);
cleancache_invalidate_page(mapping, page);
if (page_has_private(page))
do_invalidatepage(page, partial_start,
top - partial_start);
unlock_page(page);
put_page(page);
}
}
if (partial_end) {
struct page *page = find_lock_page(mapping, end);
if (page) {
wait_on_page_writeback(page);
zero_user_segment(page, 0, partial_end);
cleancache_invalidate_page(mapping, page);
if (page_has_private(page))
do_invalidatepage(page, 0,
partial_end);
unlock_page(page);
put_page(page);
}
}
/*
* If the truncation happened within a single page no pages
* will be released, just zeroed, so we can bail out now.
*/
if (start >= end)
return;
index = start;
for ( ; ; ) {
cond_resched();
if (!pagevec_lookup_entries(&pvec, mapping, index,
min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
/* If all gone from start onwards, we're done */
if (index == start)
break;
/* Otherwise restart to make sure all gone */
index = start;
continue;
}
if (index == start && indices[0] >= end) {
/* All gone out of hole to be punched, we're done */
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
break;
}
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
/* We rely upon deletion not changing page->index */
index = indices[i];
if (index >= end) {
/* Restart punch to make sure all gone */
index = start - 1;
break;
}
if (radix_tree_exceptional_entry(page)) {
clear_exceptional_entry(mapping, index, page);
continue;
}
lock_page(page);
WARN_ON(page_to_index(page) != index);
wait_on_page_writeback(page);
truncate_inode_page(mapping, page);
unlock_page(page);
}
pagevec_remove_exceptionals(&pvec);
pagevec_release(&pvec);
index++;
}
cleancache_invalidate_inode(mapping);
}
void mem_pool::mem_pool_impl::free_page(const char *page)
{
assert(page != 0);
int index = page_to_index(page);
free_page(index);
}