/**
* toi_end_bio - bio completion function.
* @bio: bio that has completed.
* @err: Error value. Yes, like end_swap_bio_read, we ignore it.
*
* Function called by the block driver from interrupt context when I/O is
* completed. If we were writing the page, we want to free it and will have
* set bio->bi_private to the parameter we should use in telling the page
* allocation accounting code what the page was allocated for. If we're
* reading the page, it will be in the singly linked list made from
* page->private pointers.
**/
static void toi_end_bio(struct bio *bio, int err)
{
struct page *page = bio->bi_io_vec[0].bv_page;
/* hib_log("err %d flags 0x%08x\n", err, bio->bi_flags); */
if (!err)
BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
unlock_page(page);
bio_put(bio);
if (waiting_on == page)
waiting_on = NULL;
put_page(page);
if (bio->bi_private)
toi__free_page((int)((unsigned long)bio->bi_private), page);
bio_put(bio);
atomic_dec(&toi_io_in_progress);
atomic_inc(&toi_io_done);
wake_up(&num_in_progress_wait);
}
static void free_conflicting_pages(void)
{
while (first_conflicting_page) {
struct page *next =
*((struct page **) kmap(first_conflicting_page));
kunmap(first_conflicting_page);
toi__free_page(29, first_conflicting_page);
first_conflicting_page = next;
}
}
/**
* free_pbe_list - free page backup entries used by the atomic copy code.
* @list: List to free.
* @highmem: Whether the list is in highmem.
*
* Normally, this function isn't used. If, however, we need to abort before
* doing the atomic copy, we use this to free the pbes previously allocated.
**/
static void free_pbe_list(struct pbe **list, int highmem)
{
while (*list) {
int i;
struct pbe *free_pbe, *next_page = NULL;
struct page *page;
if (highmem) {
page = (struct page *) *list;
free_pbe = (struct pbe *) kmap(page);
} else {
page = virt_to_page(*list);
free_pbe = *list;
}
for (i = 0; i < PBES_PER_PAGE; i++) {
if (!free_pbe)
break;
if (highmem)
toi__free_page(29, free_pbe->address);
else
toi_free_page(29,
(unsigned long) free_pbe->address);
free_pbe = free_pbe->next;
}
if (highmem) {
if (free_pbe)
next_page = free_pbe;
kunmap(page);
} else {
if (free_pbe)
next_page = free_pbe;
}
toi__free_page(29, page);
*list = (struct pbe *) next_page;
};
}
/**
* get_pageset1_load_addresses - generate pbes for conflicting pages
*
* We check here that pagedir & pages it points to won't collide
* with pages where we're going to restore from the loaded pages
* later.
*
* Returns:
* Zero on success, one if couldn't find enough pages (shouldn't
* happen).
**/
int toi_get_pageset1_load_addresses(void)
{
int pfn, highallocd = 0, lowallocd = 0;
int low_needed = pagedir1.size - get_highmem_size(pagedir1);
int high_needed = get_highmem_size(pagedir1);
int low_pages_for_highmem = 0;
gfp_t flags = GFP_ATOMIC | __GFP_NOWARN | __GFP_HIGHMEM;
struct page *page, *high_pbe_page = NULL, *last_high_pbe_page = NULL,
*low_pbe_page, *last_low_pbe_page = NULL;
struct pbe **last_high_pbe_ptr = &restore_highmem_pblist,
*this_high_pbe = NULL;
int orig_low_pfn, orig_high_pfn;
int high_pbes_done = 0, low_pbes_done = 0;
int low_direct = 0, high_direct = 0, result = 0, i;
int high_page = 1, high_offset = 0, low_page = 1, low_offset = 0;
memory_bm_set_iterators(pageset1_map, 3);
memory_bm_position_reset(pageset1_map);
memory_bm_set_iterators(pageset1_copy_map, 2);
memory_bm_position_reset(pageset1_copy_map);
last_low_pbe_ptr = &restore_pblist;
/* First, allocate pages for the start of our pbe lists. */
if (high_needed) {
high_pbe_page = ___toi_get_nonconflicting_page(1);
if (!high_pbe_page) {
result = -ENOMEM;
goto out;
}
this_high_pbe = (struct pbe *) kmap(high_pbe_page);
memset(this_high_pbe, 0, PAGE_SIZE);
}
low_pbe_page = ___toi_get_nonconflicting_page(0);
if (!low_pbe_page) {
result = -ENOMEM;
goto out;
}
this_low_pbe = (struct pbe *) page_address(low_pbe_page);
/*
* Next, allocate the number of pages we need.
*/
i = low_needed + high_needed;
do {
int is_high;
if (i == low_needed)
flags &= ~__GFP_HIGHMEM;
page = toi_alloc_page(30, flags);
BUG_ON(!page);
SetPagePageset1Copy(page);
is_high = PageHighMem(page);
if (PagePageset1(page)) {
if (is_high)
high_direct++;
else
low_direct++;
} else {
if (is_high)
highallocd++;
else
lowallocd++;
}
} while (--i);
high_needed -= high_direct;
low_needed -= low_direct;
/*
* Do we need to use some lowmem pages for the copies of highmem
* pages?
*/
if (high_needed > highallocd) {
low_pages_for_highmem = high_needed - highallocd;
high_needed -= low_pages_for_highmem;
low_needed += low_pages_for_highmem;
}
/*
* Now generate our pbes (which will be used for the atomic restore),
* and free unneeded pages.
*/
memory_bm_position_reset(pageset1_copy_map);
for (pfn = memory_bm_next_pfn_index(pageset1_copy_map, 1); pfn != BM_END_OF_MAP;
pfn = memory_bm_next_pfn_index(pageset1_copy_map, 1)) {
int is_high;
page = pfn_to_page(pfn);
is_high = PageHighMem(page);
if (PagePageset1(page))
continue;
/* Nope. We're going to use this page. Add a pbe. */
if (is_high || low_pages_for_highmem) {
struct page *orig_page;
high_pbes_done++;
if (!is_high)
low_pages_for_highmem--;
do {
orig_high_pfn = memory_bm_next_pfn_index(pageset1_map, 1);
BUG_ON(orig_high_pfn == BM_END_OF_MAP);
orig_page = pfn_to_page(orig_high_pfn);
} while (!PageHighMem(orig_page) ||
PagePageset1Copy(orig_page));
this_high_pbe->orig_address = orig_page;
this_high_pbe->address = page;
this_high_pbe->next = NULL;
toi_message(TOI_PAGEDIR, TOI_VERBOSE, 0, "High pbe %d/%d: %p(%d)=>%p",
high_page, high_offset, page, orig_high_pfn, orig_page);
if (last_high_pbe_page != high_pbe_page) {
*last_high_pbe_ptr =
(struct pbe *) high_pbe_page;
if (last_high_pbe_page) {
kunmap(last_high_pbe_page);
high_page++;
high_offset = 0;
} else
high_offset++;
last_high_pbe_page = high_pbe_page;
} else {
*last_high_pbe_ptr = this_high_pbe;
high_offset++;
}
last_high_pbe_ptr = &this_high_pbe->next;
this_high_pbe = get_next_pbe(&high_pbe_page,
this_high_pbe, 1);
if (IS_ERR(this_high_pbe)) {
printk(KERN_INFO
"This high pbe is an error.\n");
return -ENOMEM;
}
} else {
struct page *orig_page;
low_pbes_done++;
do {
orig_low_pfn = memory_bm_next_pfn_index(pageset1_map, 2);
BUG_ON(orig_low_pfn == BM_END_OF_MAP);
orig_page = pfn_to_page(orig_low_pfn);
} while (PageHighMem(orig_page) ||
PagePageset1Copy(orig_page));
this_low_pbe->orig_address = page_address(orig_page);
this_low_pbe->address = page_address(page);
this_low_pbe->next = NULL;
toi_message(TOI_PAGEDIR, TOI_VERBOSE, 0, "Low pbe %d/%d: %p(%d)=>%p",
low_page, low_offset, this_low_pbe->orig_address,
orig_low_pfn, this_low_pbe->address);
*last_low_pbe_ptr = this_low_pbe;
last_low_pbe_ptr = &this_low_pbe->next;
this_low_pbe = get_next_pbe(&low_pbe_page,
this_low_pbe, 0);
if (low_pbe_page != last_low_pbe_page) {
if (last_low_pbe_page) {
low_page++;
low_offset = 0;
}
last_low_pbe_page = low_pbe_page;
} else
low_offset++;
if (IS_ERR(this_low_pbe)) {
printk(KERN_INFO "this_low_pbe is an error.\n");
return -ENOMEM;
}
}
}
if (high_pbe_page)
kunmap(high_pbe_page);
if (last_high_pbe_page != high_pbe_page) {
if (last_high_pbe_page)
kunmap(last_high_pbe_page);
toi__free_page(29, high_pbe_page);
}
free_conflicting_pages();
out:
memory_bm_set_iterators(pageset1_map, 1);
memory_bm_set_iterators(pageset1_copy_map, 1);
return result;
}