void __flush_icache_page(struct vm_area_struct *vma, struct page *page) { unsigned long addr; if (PageHighMem(page)) return; addr = (unsigned long) page_address(page); flush_data_cache_page(addr); }
void show_mem(unsigned int filter) { pg_data_t *pgdat; unsigned long total = 0, reserved = 0, shared = 0, nonshared = 0, highmem = 0; printk("Mem-Info:\n"); show_free_areas(filter); if (filter & SHOW_MEM_FILTER_PAGE_COUNT) return; for_each_online_pgdat(pgdat) { unsigned long i, flags; pgdat_resize_lock(pgdat, &flags); for (i = 0; i < pgdat->node_spanned_pages; i++) { struct page *page; unsigned long pfn = pgdat->node_start_pfn + i; if (unlikely(!(i % MAX_ORDER_NR_PAGES))) touch_nmi_watchdog(); if (!pfn_valid(pfn)) continue; page = pfn_to_page(pfn); if (PageHighMem(page)) highmem++; if (PageReserved(page)) reserved++; else if (page_count(page) == 1) nonshared++; else if (page_count(page) > 1) shared += page_count(page) - 1; total++; } pgdat_resize_unlock(pgdat, &flags); } printk("%lu pages RAM\n", total); #ifdef CONFIG_HIGHMEM printk("%lu pages HighMem\n", highmem); #endif printk("%lu pages reserved\n", reserved); printk("%lu pages shared\n", shared); printk("%lu pages non-shared\n", nonshared); #ifdef CONFIG_QUICKLIST printk("%lu pages in pagetable cache\n", quicklist_total_size()); #endif }
static inline void pgd_walk_set_prot(void *pt, pgprot_t flags) { struct page *page = virt_to_page(pt); unsigned long pfn = page_to_pfn(page); if (PageHighMem(page)) return; BUG_ON(HYPERVISOR_update_va_mapping( (unsigned long)__va(pfn << PAGE_SHIFT), pfn_pte(pfn, flags), 0)); }
/* balloon_append: add the given page to the balloon. */ static void balloon_append(struct page *page) { /* Lowmem is re-populated first, so highmem pages go at list tail. */ if (PageHighMem(page)) { list_add_tail(PAGE_TO_LIST(page), &ballooned_pages); balloon_high++; } else { list_add(PAGE_TO_LIST(page), &ballooned_pages); balloon_low++; } }
static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, void (*op)(const void *, size_t, int)) { /* * A single sg entry may refer to multiple physically contiguous * pages. But we still need to process highmem pages individually. * If highmem is not configured then the bulk of this loop gets * optimized out. */ size_t left = size; do { size_t len = left; void *vaddr; if (PageHighMem(page)) { if (len + offset > PAGE_SIZE) { if (offset >= PAGE_SIZE) { page += offset / PAGE_SIZE; offset %= PAGE_SIZE; } len = PAGE_SIZE - offset; } vaddr = kmap_high_get(page); if (vaddr) { vaddr += offset; op(vaddr, len, dir); kunmap_high(page); } else if (cache_is_vipt()) { pte_t saved_pte; vaddr = kmap_high_l1_vipt(page, &saved_pte); if(vaddr) //TI { //TI op(vaddr + offset, len, dir); kunmap_high_l1_vipt(page, saved_pte); }else //TI printk(KERN_ERR "SSN: 1 vaddr is NULL\n");//TI } else //TI printk(KERN_ERR "SSN: 2 vaddr is NULL\n"); //TI } else { vaddr = page_address(page) + offset; if(vaddr) //TI op(vaddr, len, dir); else //TI printk(KERN_ERR "SSN: 3 vaddr is NULL\n"); //TI } offset = 0; page++; left -= len; } while (left); }
/* balloon_append: add the given page to the balloon. */ static void __balloon_append(struct page *page) { /* Lowmem is re-populated first, so highmem pages go at list tail. */ if (PageHighMem(page)) { list_add_tail(&page->lru, &ballooned_pages); balloon_stats.balloon_high++; } else { list_add(&page->lru, &ballooned_pages); balloon_stats.balloon_low++; } }
static inline void *coherent_kvaddr(struct page *page, unsigned long base, unsigned long vaddr, unsigned long *paddr) { if (PageHighMem(page) || !DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { *paddr = page_to_phys(page); return (void *)(base + (vaddr & DCACHE_ALIAS_MASK)); } else { *paddr = 0; return page_to_virt(page); } }
void __flush_dcache_page(struct page *page) { #ifdef CONFIG_RALINK_SOC void *addr; if (page_mapping(page) && !page_mapped(page)) { SetPageDcacheDirty(page); return; } #else struct address_space *mapping = page_mapping(page); unsigned long addr; if (PageHighMem(page)) return; if (mapping && !mapping_mapped(mapping)) { SetPageDcacheDirty(page); return; } #endif /* * We could delay the flush for the !page_mapping case too. But that * case is for exec env/arg pages and those are %99 certainly going to * get faulted into the tlb (and thus flushed) anyways. */ #ifdef CONFIG_RALINK_SOC if (PageHighMem(page)) { addr = kmap_atomic(page, KM_PTE1); flush_data_cache_page((unsigned long)addr); kunmap_atomic(addr, KM_PTE1); } else { addr = (void *) page_address(page); flush_data_cache_page((unsigned long)addr); } ClearPageDcacheDirty(page); #else addr = (unsigned long) page_address(page); flush_data_cache_page(addr); #endif }
struct scatterlist* videobuf_pages_to_sg(struct page **pages, int nr_pages, int offset) { struct scatterlist *sglist; int i = 0; if (NULL == pages[0]) return NULL; sglist = kmalloc(sizeof(*sglist) * nr_pages, GFP_KERNEL); if (NULL == sglist) return NULL; memset(sglist, 0, sizeof(*sglist) * nr_pages); if (NULL == pages[0]) goto nopage; if (PageHighMem(pages[0])) /* DMA to highmem pages might not work */ goto highmem; sglist[0].page = pages[0]; sglist[0].offset = offset; sglist[0].length = PAGE_SIZE - offset; for (i = 1; i < nr_pages; i++) { if (NULL == pages[i]) goto nopage; if (PageHighMem(pages[i])) goto highmem; sglist[i].page = pages[i]; sglist[i].length = PAGE_SIZE; } return sglist; nopage: dprintk(2,"sgl: oops - no page\n"); kfree(sglist); return NULL; highmem: dprintk(2,"sgl: oops - highmem page\n"); kfree(sglist); return NULL; }
void *__kmap(struct page *page) { void *addr; might_sleep(); if (!PageHighMem(page)) return page_address(page); addr = kmap_high(page); flush_tlb_one((unsigned long)addr); return addr; }
static void *try_ram_remap(resource_size_t offset, size_t size, unsigned long flags) { unsigned long pfn = PHYS_PFN(offset); /* In the simple case just return the existing linear address */ if (pfn_valid(pfn) && !PageHighMem(pfn_to_page(pfn)) && arch_memremap_can_ram_remap(offset, size, flags)) return __va(offset); return NULL; /* fallback to arch_memremap_wb */ }
static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, int enc) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); struct aead_request *subreq = &rctx->subreq; struct scatterlist *dst = req->dst; struct scatterlist *cipher = rctx->cipher; struct scatterlist *payload = rctx->payload; struct scatterlist *assoc = rctx->assoc; unsigned int authsize = crypto_aead_authsize(aead); unsigned int assoclen = req->assoclen; struct page *dstp; u8 *vdst; u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), crypto_aead_alignmask(ctx->child) + 1); memcpy(iv, ctx->nonce, 4); memcpy(iv + 4, req->iv, 8); /* construct cipher/plaintext */ if (enc) memset(rctx->auth_tag, 0, authsize); else scatterwalk_map_and_copy(rctx->auth_tag, dst, req->cryptlen - authsize, authsize, 0); sg_init_one(cipher, rctx->auth_tag, authsize); /* construct the aad */ dstp = sg_page(dst); vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; sg_init_table(payload, 2); sg_set_buf(payload, req->iv, 8); scatterwalk_crypto_chain(payload, dst, vdst == req->iv + 8, 2); assoclen += 8 + req->cryptlen - (enc ? 0 : authsize); sg_init_table(assoc, 2); sg_set_page(assoc, sg_page(req->assoc), req->assoc->length, req->assoc->offset); scatterwalk_crypto_chain(assoc, payload, 0, 2); aead_request_set_tfm(subreq, ctx->child); aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv); aead_request_set_assoc(subreq, assoc, assoclen); return subreq; }
static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, void (*op)(const void *, size_t, int)) { unsigned long pfn; size_t left = size; pfn = page_to_pfn(page) + offset / PAGE_SIZE; offset %= PAGE_SIZE; /* * A single sg entry may refer to multiple physically contiguous * pages. But we still need to process highmem pages individually. * If highmem is not configured then the bulk of this loop gets * optimized out. */ /* size_t left = size; */ do { size_t len = left; void *vaddr; page = pfn_to_page(pfn); if (PageHighMem(page)) { /* if (len + offset > PAGE_SIZE) { if (offset >= PAGE_SIZE) { page += offset / PAGE_SIZE; offset %= PAGE_SIZE; } */ if (len + offset > PAGE_SIZE) len = PAGE_SIZE - offset; /* } */ vaddr = kmap_high_get(page); if (vaddr) { vaddr += offset; op(vaddr, len, dir); kunmap_high(page); } else if (cache_is_vipt()) { /* unmapped pages might still be cached */ vaddr = kmap_atomic(page); op(vaddr + offset, len, dir); kunmap_atomic(vaddr); } } else { vaddr = page_address(page) + offset; op(vaddr, len, dir); } offset = 0; /* page++; */ pfn++; left -= len; } while (left); }
void *os_kmap_sgptr(PSG psg) { struct page * page = (struct page *)(HPT_UPTR)(psg->addr.bus>>32); if (page) return (PageHighMem(page)? (char *)kmap_atomic(page) : (char *)page_address(page)) + (psg->addr.bus & 0xffffffff); else return psg->addr._logical; }
static void poison_page(struct page *page) { void *addr; if (PageHighMem(page)) { poison_highpage(page); return; } set_page_poison(page); addr = page_address(page); memset(addr, PAGE_POISON, PAGE_SIZE); }
static void __dma_remap(struct page *page, size_t size, pgprot_t prot) { unsigned long start = (unsigned long) page_address(page); unsigned end = start + size; if (PageHighMem(page)) return; apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); dsb(); flush_tlb_kernel_range(start, end); }
void __flush_anon_page(struct page *page, unsigned long vmaddr) { #ifdef CONFIG_RALINK_SOC if (!PageHighMem(page)) { unsigned long addr = (unsigned long) page_address(page); if (pages_do_alias(addr, vmaddr & PAGE_MASK)) { if (page_mapped(page) && !Page_dcache_dirty(page)) { void *kaddr; kaddr = kmap_coherent(page, vmaddr); flush_data_cache_page((unsigned long)kaddr); kunmap_coherent(); } else { flush_data_cache_page(addr); ClearPageDcacheDirty(page); } } } else { void *laddr = lowmem_page_address(page); if (pages_do_alias((unsigned long)laddr, vmaddr & PAGE_MASK)) { if (page_mapped(page) && !Page_dcache_dirty(page)) { void *kaddr; kaddr = kmap_coherent(page, vmaddr); flush_data_cache_page((unsigned long)kaddr); kunmap_coherent(); } else { void *kaddr; kaddr = kmap_atomic(page, KM_PTE1); flush_data_cache_page((unsigned long)kaddr); kunmap_atomic(kaddr, KM_PTE1); ClearPageDcacheDirty(page); } } } #else unsigned long addr = (unsigned long) page_address(page); if (pages_do_alias(addr, vmaddr)) { if (page_mapped(page) && !Page_dcache_dirty(page)) { void *kaddr; kaddr = kmap_coherent(page, vmaddr); flush_data_cache_page((unsigned long)kaddr); kunmap_coherent(); } else flush_data_cache_page(addr); } #endif }
static int __change_page_attr(struct page *page, pgprot_t prot) { pte_t *kpte; unsigned long address; struct page *kpte_page; BUG_ON(PageHighMem(page)); address = (unsigned long)page_address(page); kpte = lookup_address(address); if (!kpte) return -EINVAL; kpte_page = virt_to_page(kpte); if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) { if ((pte_val(*kpte) & _PAGE_PSE) == 0) { set_pte_atomic(kpte, mk_pte(page, prot)); } else { pgprot_t ref_prot; struct page *split; ref_prot = ((address & LARGE_PAGE_MASK) < (unsigned long)&_etext) ? PAGE_KERNEL_EXEC : PAGE_KERNEL; split = split_large_page(address, prot, ref_prot); if (!split) return -ENOMEM; set_pmd_pte(kpte,address,mk_pte(split, ref_prot)); kpte_page = split; } page_private(kpte_page)++; } else if ((pte_val(*kpte) & _PAGE_PSE) == 0) { set_pte_atomic(kpte, mk_pte(page, PAGE_KERNEL)); BUG_ON(page_private(kpte_page) == 0); page_private(kpte_page)--; } else BUG(); /* * If the pte was reserved, it means it was created at boot * time (not via split_large_page) and in turn we must not * replace it with a largepage. */ if (!PageReserved(kpte_page)) { if (cpu_has_pse && (page_private(kpte_page) == 0)) { ClearPagePrivate(kpte_page); list_add(&kpte_page->lru, &df_list); revert_page(kpte_page, address); } } return 0; }
static int ion_page_pool_add(struct ion_page_pool *pool, struct page *page) { mutex_lock(&pool->mutex); if (PageHighMem(page)) { list_add_tail(&page->lru, &pool->high_items); pool->high_count++; } else { list_add_tail(&page->lru, &pool->low_items); pool->low_count++; } mutex_unlock(&pool->mutex); return 0; }
void cma_unmap_kernel(u32 phys_addr, size_t size, void *cpu_addr) { struct page *page = phys_to_page(phys_addr); BUG_ON(unlikely(!pfn_valid(__phys_to_pfn(phys_addr)))); size = PAGE_ALIGN(size); if (PageHighMem(page)) __dma_free_remap(cpu_addr, size); else __dma_remap(page, size, pgprot_kernel); }
/* * Ensure cache coherency between kernel mapping and userspace mapping * of this page. */ void flush_dcache_page(struct page *page) { struct address_space *mapping = page_mapping(page); if (!PageHighMem(page) && mapping && !mapping_mapped(mapping)) set_bit(PG_dcache_dirty, &page->flags); else { __flush_dcache_page(mapping, page); if (mapping) __flush_icache_all(); } }
static void unpoison_page(struct page *page) { if (PageHighMem(page)) { unpoison_highpage(page); return; } if (page_poison(page)) { void *addr = page_address(page); check_poison_mem(addr, PAGE_SIZE); clear_page_poison(page); } }
/* * Return a scatterlist for a an array of userpages (NULL on errors). * Memory for the scatterlist is allocated using kmalloc. The caller * must free the memory. */ static struct scatterlist *videobuf_pages_to_sg(struct page **pages, int nr_pages, int offset, size_t size) { struct scatterlist *sglist; int i; if (NULL == pages[0]) return NULL; sglist = vmalloc(nr_pages * sizeof(*sglist)); if (NULL == sglist) return NULL; sg_init_table(sglist, nr_pages); if (PageHighMem(pages[0])) /* DMA to highmem pages might not work */ goto highmem; sg_set_page(&sglist[0], pages[0], min_t(size_t, PAGE_SIZE - offset, size), offset); size -= min_t(size_t, PAGE_SIZE - offset, size); for (i = 1; i < nr_pages; i++) { if (NULL == pages[i]) goto nopage; if (PageHighMem(pages[i])) goto highmem; sg_set_page(&sglist[i], pages[i], min_t(size_t, PAGE_SIZE, size), 0); size -= min_t(size_t, PAGE_SIZE, size); } return sglist; nopage: dprintk(2, "sgl: oops - no page\n"); vfree(sglist); return NULL; highmem: dprintk(2, "sgl: oops - highmem page\n"); vfree(sglist); return NULL; }
void kernel_map_pages(struct page *page, int numpages, int enable) { if (PageHighMem(page)) return; /* the return value is ignored - the calls cannot fail, * large pages are disabled at boot time. */ change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0)); /* we should perform an IPI and flush all tlbs, * but that can deadlock->flush only current cpu. */ __flush_tlb_all(); }
struct scatterlist* videobuf_pages_to_sg(struct page **pages, int nr_pages, int offset) { struct scatterlist *sglist; int i = 0; if (NULL == pages[0]) return NULL; sglist = kcalloc(nr_pages, sizeof(*sglist), GFP_KERNEL); if (NULL == sglist) return NULL; sg_init_table(sglist, nr_pages); if (NULL == pages[0]) goto nopage; if (PageHighMem(pages[0])) /* DMA to highmem pages might not work */ goto highmem; sg_set_page(&sglist[0], pages[0], PAGE_SIZE - offset, offset); for (i = 1; i < nr_pages; i++) { if (NULL == pages[i]) goto nopage; if (PageHighMem(pages[i])) goto highmem; sg_set_page(&sglist[i], pages[i], PAGE_SIZE, 0); } return sglist; nopage: dprintk(2,"sgl: oops - no page\n"); kfree(sglist); return NULL; highmem: dprintk(2,"sgl: oops - highmem page\n"); kfree(sglist); return NULL; }
static void arc_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, struct dma_attrs *attrs) { struct page *page = virt_to_page(dma_handle); int is_non_coh = 1; is_non_coh = dma_get_attr(DMA_ATTR_NON_CONSISTENT, attrs) || (is_isa_arcv2() && ioc_exists); if (PageHighMem(page) || !is_non_coh) iounmap((void __force __iomem *)vaddr); __free_pages(page, get_order(size)); }
/* if pagedir_p != NULL it also copies the counted pages */ static int count_and_copy_data_pages(struct pbe *pagedir_p) { int chunk_size; int nr_copy_pages = 0; int pfn; struct page *page; #ifdef CONFIG_DISCONTIGMEM panic("Discontingmem not supported"); #else BUG_ON (max_pfn != num_physpages); #endif for (pfn = 0; pfn < max_pfn; pfn++) { page = pfn_to_page(pfn); if (PageHighMem(page)) panic("Swsusp not supported on highmem boxes. Send 1GB of RAM to <*****@*****.**> and try again ;-)."); if (!PageReserved(page)) { if (PageNosave(page)) continue; if ((chunk_size=is_head_of_free_region(page))!=0) { pfn += chunk_size - 1; continue; } } else if (PageReserved(page)) { BUG_ON (PageNosave(page)); /* * Just copy whole code segment. Hopefully it is not that big. */ if ((ADDRESS(pfn) >= (unsigned long) ADDRESS2(&__nosave_begin)) && (ADDRESS(pfn) < (unsigned long) ADDRESS2(&__nosave_end))) { PRINTK("[nosave %lx]", ADDRESS(pfn)); continue; } /* Hmm, perhaps copying all reserved pages is not too healthy as they may contain critical bios data? */ } else BUG(); nr_copy_pages++; if (pagedir_p) { pagedir_p->orig_address = ADDRESS(pfn); copy_page((void *) pagedir_p->address, (void *) pagedir_p->orig_address); pagedir_p++; } } return nr_copy_pages; }
void arch_dma_free(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { phys_addr_t paddr = dma_handle; struct page *page = virt_to_page(paddr); int is_non_coh = 1; is_non_coh = (attrs & DMA_ATTR_NON_CONSISTENT) || (is_isa_arcv2() && ioc_enable); if (PageHighMem(page) || !is_non_coh) iounmap((void __force __iomem *)vaddr); __free_pages(page, get_order(size)); }
static inline int ttm_tt_set_page_caching(struct page *p, enum ttm_caching_state c_state) { if (PageHighMem(p)) return 0; switch (c_state) { case tt_cached: return set_pages_wb(p, 1); case tt_wc: return set_memory_wc((unsigned long) page_address(p), 1); default: return set_pages_uc(p, 1); } }
static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) { mutex_lock(&pool->mutex); if (PageHighMem(page)) { list_add_tail(&page->lru, &pool->high_items); pool->high_count++; } else { list_add_tail(&page->lru, &pool->low_items); pool->low_count++; } mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, 1 << pool->order); mutex_unlock(&pool->mutex); }