static void copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm) { struct zone *zone; unsigned long pfn; for_each_zone (zone) { unsigned long max_zone_pfn; if (is_highmem(zone)) continue; mark_free_pages(zone); max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) if (saveable_page(pfn)) memory_bm_set_bit(orig_bm, pfn); } memory_bm_position_reset(orig_bm); memory_bm_position_reset(copy_bm); do { pfn = memory_bm_next_pfn(orig_bm); if (likely(pfn != BM_END_OF_MAP)) { struct page *page; void *src; page = pfn_to_page(pfn); src = page_address(page); page = pfn_to_page(memory_bm_next_pfn(copy_bm)); copy_data_page(page_address(page), src); } } while (pfn != BM_END_OF_MAP); }
void check_checksums(void) { int pfn, index = 0, cpu = smp_processor_id(); char current_checksum[CHECKSUM_SIZE]; struct cpu_context *ctx = &per_cpu(contexts, cpu); if (!toi_checksum_ops.enabled) { toi_message(TOI_IO, TOI_VERBOSE, 0, "Checksumming disabled."); return; } next_page = (unsigned long) page_list; toi_num_resaved = 0; this_checksum = 0; toi_message(TOI_IO, TOI_VERBOSE, 0, "Verifying checksums."); memory_bm_position_reset(pageset2_map); for (pfn = memory_bm_next_pfn(pageset2_map); pfn != BM_END_OF_MAP; pfn = memory_bm_next_pfn(pageset2_map)) { int ret; char *pa; struct page *page = pfn_to_page(pfn); if (index % CHECKSUMS_PER_PAGE) { this_checksum += CHECKSUM_SIZE; } else { this_checksum = next_page + sizeof(void *); next_page = *((unsigned long *) next_page); } /* Done when IRQs disabled so must be atomic */ pa = kmap_atomic(page); memcpy(ctx->buf, pa, PAGE_SIZE); kunmap_atomic(pa); ret = crypto_hash_digest(&ctx->desc, ctx->sg, PAGE_SIZE, current_checksum); if (ret) { printk(KERN_INFO "Digest failed. Returned %d.\n", ret); return; } if (memcmp(current_checksum, (char *) this_checksum, CHECKSUM_SIZE)) { toi_message(TOI_IO, TOI_VERBOSE, 0, "Resaving %ld.", pfn); SetPageResave(pfn_to_page(pfn)); toi_num_resaved++; if (test_action_state(TOI_ABORT_ON_RESAVE_NEEDED)) set_abort_result(TOI_RESAVE_NEEDED); } index++; } toi_message(TOI_IO, TOI_VERBOSE, 0, "Checksum verification complete."); }
/** * toi_copy_pageset1 - do the atomic copy of pageset1 * * Make the atomic copy of pageset1. We can't use copy_page (as we once did) * because we can't be sure what side effects it has. On my old Duron, with * 3DNOW, kernel_fpu_begin increments preempt count, making our preempt * count at resume time 4 instead of 3. * * We don't want to call kmap_atomic unconditionally because it has the side * effect of incrementing the preempt count, which will leave it one too high * post resume (the page containing the preempt count will be copied after * its incremented. This is essentially the same problem. **/ void toi_copy_pageset1(void) { int i; unsigned long source_index, dest_index; memory_bm_position_reset(pageset1_map); memory_bm_position_reset(pageset1_copy_map); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); for (i = 0; i < pagedir1.size; i++) { unsigned long *origvirt, *copyvirt; struct page *origpage, *copypage; int loop = (PAGE_SIZE / sizeof(unsigned long)) - 1, was_present1, was_present2; origpage = pfn_to_page(source_index); copypage = pfn_to_page(dest_index); origvirt = PageHighMem(origpage) ? kmap_atomic(origpage) : page_address(origpage); copyvirt = PageHighMem(copypage) ? kmap_atomic(copypage) : page_address(copypage); was_present1 = kernel_page_present(origpage); if (!was_present1) kernel_map_pages(origpage, 1, 1); was_present2 = kernel_page_present(copypage); if (!was_present2) kernel_map_pages(copypage, 1, 1); while (loop >= 0) { *(copyvirt + loop) = *(origvirt + loop); loop--; } if (!was_present1) kernel_map_pages(origpage, 1, 0); if (!was_present2) kernel_map_pages(copypage, 1, 0); if (PageHighMem(origpage)) kunmap_atomic(origvirt); if (PageHighMem(copypage)) kunmap_atomic(copyvirt); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); } }
static void duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src) { unsigned long pfn; memory_bm_position_reset(src); pfn = memory_bm_next_pfn(src); while (pfn != BM_END_OF_MAP) { memory_bm_set_bit(dst, pfn); pfn = memory_bm_next_pfn(src); } }
static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca) { struct pbe *pbe; struct page *page = pfn_to_page(memory_bm_next_pfn(bm)); if (PageNosave(page) && PageNosaveFree(page)) /* We have allocated the "original" page frame and we can * use it directly to store the loaded page. */ return page_address(page); /* The "original" page frame has not been allocated and we have to * use a "safe" page frame to store the loaded page. */ pbe = chain_alloc(ca, sizeof(struct pbe)); if (!pbe) { swsusp_free(); return NULL; } pbe->orig_address = (unsigned long)page_address(page); pbe->address = (unsigned long)safe_pages_list; safe_pages_list = safe_pages_list->next; pbe->next = restore_pblist; restore_pblist = pbe; return (void *)pbe->address; }
static int mark_unsafe_pages(struct memory_bitmap *bm) { struct zone *zone; unsigned long pfn, max_zone_pfn; /* Clear page flags */ for_each_zone (zone) { max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages; for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) if (pfn_valid(pfn)) ClearPageNosaveFree(pfn_to_page(pfn)); } /* Mark pages that correspond to the "original" pfns as "unsafe" */ memory_bm_position_reset(bm); do { pfn = memory_bm_next_pfn(bm); if (likely(pfn != BM_END_OF_MAP)) { if (likely(pfn_valid(pfn))) SetPageNosaveFree(pfn_to_page(pfn)); else return -EFAULT; } } while (pfn != BM_END_OF_MAP); allocated_unsafe_pages = 0; return 0; }
static inline void pack_pfns(unsigned long *buf, struct memory_bitmap *bm) { int j; for (j = 0; j < PAGE_SIZE / sizeof(long); j++) { buf[j] = memory_bm_next_pfn(bm); if (unlikely(buf[j] == BM_END_OF_MAP)) break; } }
int snapshot_read_next(struct snapshot_handle *handle, size_t count) { if (handle->cur > nr_meta_pages + nr_copy_pages) return 0; if (!buffer) { /* This makes the buffer be freed by swsusp_free() */ buffer = alloc_image_page(GFP_ATOMIC, PG_ANY); if (!buffer) return -ENOMEM; } if (!handle->offset) { init_header((struct swsusp_info *)buffer); handle->buffer = buffer; memory_bm_position_reset(&orig_bm); memory_bm_position_reset(©_bm); } if (handle->prev < handle->cur) { if (handle->cur <= nr_meta_pages) { memset(buffer, 0, PAGE_SIZE); pack_pfns(buffer, &orig_bm); } else { unsigned long pfn = memory_bm_next_pfn(©_bm); handle->buffer = page_address(pfn_to_page(pfn)); } handle->prev = handle->cur; } handle->buf_offset = handle->cur_offset; if (handle->cur_offset + count >= PAGE_SIZE) { count = PAGE_SIZE - handle->cur_offset; handle->cur_offset = 0; handle->cur++; } else { handle->cur_offset += count; } handle->offset += count; return count; }
struct page *___toi_get_nonconflicting_page(int can_be_highmem) { struct page *page; gfp_t flags = TOI_ATOMIC_GFP; if (can_be_highmem) flags |= __GFP_HIGHMEM; if (test_toi_state(TOI_LOADING_ALT_IMAGE) && pageset2_map && (ptoi_pfn != BM_END_OF_MAP)) { do { ptoi_pfn = memory_bm_next_pfn(pageset2_map); if (ptoi_pfn != BM_END_OF_MAP) { page = pfn_to_page(ptoi_pfn); if (!PagePageset1(page) && (can_be_highmem || !PageHighMem(page))) return page; } } while (ptoi_pfn != BM_END_OF_MAP); } do { page = toi_alloc_page(29, flags); if (!page) { printk(KERN_INFO "Failed to get nonconflicting " "page.\n"); return NULL; } if (PagePageset1(page)) { struct page **next = (struct page **) kmap(page); *next = first_conflicting_page; first_conflicting_page = page; kunmap(page); } } while (PagePageset1(page)); return page; }
/** * toi_copy_pageset1 - do the atomic copy of pageset1 * * Make the atomic copy of pageset1. We can't use copy_page (as we once did) * because we can't be sure what side effects it has. On my old Duron, with * 3DNOW, kernel_fpu_begin increments preempt count, making our preempt * count at resume time 4 instead of 3. * * We don't want to call kmap_atomic unconditionally because it has the side * effect of incrementing the preempt count, which will leave it one too high * post resume (the page containing the preempt count will be copied after * its incremented. This is essentially the same problem. **/ void toi_copy_pageset1(void) { int i; unsigned long source_index, dest_index; memory_bm_position_reset(pageset1_map); memory_bm_position_reset(pageset1_copy_map); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); for (i = 0; i < pagedir1.size; i++) { unsigned long *origvirt, *copyvirt; struct page *origpage, *copypage; int loop = (PAGE_SIZE / sizeof(unsigned long)) - 1, was_present1, was_present2; #ifdef CONFIG_TOI_ENHANCE if (!pfn_valid(source_index) || !pfn_valid(dest_index)) { pr_emerg("[%s] (%d) dest_index:%lu, source_index:%lu\n", __func__, i, dest_index, source_index); set_abort_result(TOI_ARCH_PREPARE_FAILED); return; } #endif origpage = pfn_to_page(source_index); copypage = pfn_to_page(dest_index); origvirt = PageHighMem(origpage) ? kmap_atomic(origpage) : page_address(origpage); copyvirt = PageHighMem(copypage) ? kmap_atomic(copypage) : page_address(copypage); was_present1 = kernel_page_present(origpage); if (!was_present1) kernel_map_pages(origpage, 1, 1); was_present2 = kernel_page_present(copypage); if (!was_present2) kernel_map_pages(copypage, 1, 1); while (loop >= 0) { *(copyvirt + loop) = *(origvirt + loop); loop--; } if (!was_present1) kernel_map_pages(origpage, 1, 0); if (!was_present2) kernel_map_pages(copypage, 1, 0); if (PageHighMem(origpage)) kunmap_atomic(origvirt); if (PageHighMem(copypage)) kunmap_atomic(copyvirt); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); } }