void show_mem(unsigned int filter) { int free = 0, total = 0, reserved = 0; int shared = 0, cached = 0, slab = 0, i; struct meminfo *mi = &meminfo; printk("Mem-info:\n"); show_free_areas(filter); for_each_bank(i, mi) { struct membank *bank = &mi->bank[i]; unsigned int pfn1, pfn2; struct page *page, *end; pfn1 = bank_pfn_start(bank); pfn2 = bank_pfn_end(bank); page = pfn_to_page(pfn1); end = pfn_to_page(pfn2 - 1) + 1; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (!page_count(page)) free++; else shared += page_count(page) - 1; page++; } while (page < end); } printk(KERN_INFO "%d pages of RAM\n", total); printk(KERN_INFO "%d free pages\n", free); printk(KERN_INFO "%d reserved pages\n", reserved); printk(KERN_INFO "%d slab pages\n", slab); printk(KERN_INFO "%d pages shared\n", shared); printk(KERN_INFO "%d pages swap cached\n", cached); }
/* /proc/kpagecount - an array exposing page counts * * Each entry is a u64 representing the corresponding * physical page count. */ static ssize_t kpagecount_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) { u64 __user *out = (u64 __user *)buf; struct page *ppage; unsigned long src = *ppos; unsigned long pfn; unsigned long max_pfn_kpmsize = max_pfn * KPMSIZE; ssize_t ret = 0; u64 pcount; pfn = src / KPMSIZE; if (src != max_pfn_kpmsize) count = min_t(size_t, count, max_pfn_kpmsize - src); if (src & KPMMASK || count & KPMMASK) return -EINVAL; while (count > 0) { if (pfn_valid(pfn)) ppage = pfn_to_page(pfn); else ppage = NULL; if (!ppage || PageSlab(ppage)) pcount = 0; else pcount = page_mapcount(ppage); if (put_user(pcount, out)) { ret = -EFAULT; break; } pfn++; out++; count -= KPMSIZE; } *ppos += (char __user *)out - buf; if (!ret) ret = (char __user *)out - buf; return ret; }
/* * This keeps memory configuration data used by a couple memory * initialization functions, as well as show_mem() for the skipping * of holes in the memory map. It is populated by arm_add_memory(). */ void show_mem(unsigned int filter) { int free = 0, total = 0, reserved = 0; int shared = 0, cached = 0, slab = 0; struct memblock_region *reg; printk("Mem-info:\n"); show_free_areas(filter); if (filter & SHOW_MEM_FILTER_PAGE_COUNT) return; for_each_memblock(memory, reg) { unsigned int pfn1, pfn2; struct page *page, *end; pfn1 = memblock_region_memory_base_pfn(reg); pfn2 = memblock_region_memory_end_pfn(reg); page = pfn_to_page(pfn1); end = pfn_to_page(pfn2 - 1) + 1; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (!page_count(page)) free++; else shared += page_count(page) - 1; page++; #ifdef CONFIG_SPARSEMEM pfn1++; if (!(pfn1 % PAGES_PER_SECTION)) page = pfn_to_page(pfn1); } while (pfn1 < pfn2); #else } while (page < end);
void show_mem(unsigned int filter) { int free = 0, total = 0, reserved = 0; int shared = 0, cached = 0, slab = 0, i; struct meminfo * mi = &meminfo; printk("Mem-info:\n"); show_free_areas(filter); for_each_bank (i, mi) { struct membank *bank = &mi->bank[i]; unsigned int pfn1, pfn2; struct page *page, *end; pfn1 = bank_pfn_start(bank); pfn2 = bank_pfn_end(bank); page = pfn_to_page(pfn1); end = pfn_to_page(pfn2 - 1) + 1; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (!page_count(page)) free++; else shared += page_count(page) - 1; page++; #ifdef CONFIG_SPARSEMEM pfn1++; if (!(pfn1 % PAGES_PER_SECTION)) page = pfn_to_page(pfn1); } while (pfn1 < pfn2); #else } while (page < end); #endif }
static int _phymem_pages_proc_show(struct seq_file *m, void *v) { int free = 0, total = 0, reserved = 0; int other = 0, shared = 0, cached = 0, slab = 0, node, i; struct meminfo * mi = &meminfo; for_each_bank (i,mi) { struct membank *bank = &mi->bank[i]; unsigned int pfn1, pfn2; struct page *page, *end; pfn1 = bank_pfn_start(bank); pfn2 = bank_pfn_end(bank); page = pfn_to_page(pfn1); end = pfn_to_page(pfn2 - 1) + 1; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (page_count(page) > 1) shared++; else if (!page_count(page)) free++; else other++; page++; }while (page < end); } seq_printf(m, "pages of RAM %d\n", total); seq_printf(m, "free pages %d\n", free); seq_printf(m, "reserved pages %d\n", reserved); seq_printf(m, "slab pages %d\n", slab); seq_printf(m, "pages shared %d\n", shared); seq_printf(m, "pages swap cached %d\n", cached); seq_printf(m, "other pages %d\n", other); return 0; }
void show_mem(void) { int free = 0, total = 0, reserved = 0; int shared = 0, cached = 0, slab = 0, node, i; struct meminfo * mi = &meminfo; printk("Mem-info:\n"); show_free_areas(); printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); for_each_online_node(node) { pg_data_t *n = NODE_DATA(node); struct page *map = n->node_mem_map - n->node_start_pfn; for_each_nodebank (i,mi,node) { unsigned int pfn1, pfn2; struct page *page, *end; pfn1 = __phys_to_pfn(mi->bank[i].start); pfn2 = __phys_to_pfn(mi->bank[i].size + mi->bank[i].start); page = map + pfn1; end = map + pfn2; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (!page_count(page)) free++; else shared += page_count(page) - 1; page++; } while (page < end); } }
/* * Return the total memory allocated for this pointer, not * just what the caller asked for. * * Doesn't have to be accurate, i.e. may have races. */ unsigned int kobjsize(const void *objp) { struct page *page; /* * If the object we have should not have ksize performed on it, * return size of 0 */ if (!objp || !virt_addr_valid(objp)) return 0; page = virt_to_head_page(objp); /* * If the allocator sets PageSlab, we know the pointer came from * kmalloc(). */ if (PageSlab(page)) return ksize(objp); /* * If it's not a compound page, see if we have a matching VMA * region. This test is intentionally done in reverse order, * so if there's no VMA, we still fall through and hand back * PAGE_SIZE for 0-order pages. */ if (!PageCompound(page)) { struct vm_area_struct *vma; vma = find_vma(current->mm, (unsigned long)objp); if (vma) return vma->vm_end - vma->vm_start; } /* * The ksize() function is only guaranteed to work for pointers * returned by kmalloc(). So handle arbitrary pointers here. */ return PAGE_SIZE << compound_order(page); }
/* * Show free area list (used inside shift_scroll-lock stuff) * We also calculate the percentage fragmentation. We do this by counting the * memory on each free list with the exception of the first item on the list. */ int Jcmd_mem(char *arg1, char *arg2) { unsigned long order, flags; unsigned long total = 0; spin_lock_irqsave(&free_area_lock, flags); for (order = 0; order < NR_MEM_LISTS; order++) { struct page * tmp; unsigned long nr = 0; for (tmp = free_mem_area[order].next; tmp != memory_head(free_mem_area+order); tmp = tmp->next) { nr++; } total += nr << order; ut_printf("%d(%d): count:%d static count:%d total:%d (%dM)\n", order,1<<order, nr, free_mem_area[order].stat_count, (nr << order), ((nr << order)*PAGE_SIZE)/(1024*1024)); } spin_unlock_irqrestore(&free_area_lock, flags); ut_printf("total Free pages = %d (%dM) Actual pages: %d (%dM) pagecachesize: %dM , freepages:%d\n", total, (total * 4) / 1024,g_stat_mem_size/PAGE_SIZE,g_stat_mem_size/(1024*1024),g_pagecache_size/(1024*1024),g_nr_free_pages); int slab=0; int referenced=0; int reserved=0; int dma=0; unsigned long va_end=(unsigned long)__va(g_phy_mem_size); page_struct_t *p; p = g_mem_map + MAP_NR(va_end); do { --p; if (PageReserved(p)) reserved++; if (PageDMA(p)) dma++; if (PageReferenced(p))referenced++; if (PageSlab(p)) slab++; } while (p > g_mem_map); ut_printf(" reserved :%d(%dM) referenced:%d dma:%d slab:%d stat_allocs:%d stat_frees: %d\n\n",reserved,(reserved*PAGE_SIZE)/(1024*1024),referenced,dma,slab,stat_allocs,stat_frees); if ((arg1 != 0) && (ut_strcmp(arg1,"all")==0)) Jcmd_jslab(0,0); return 1; }
void show_mem(void) { int free = 0, total = 0, reserved = 0; int shared = 0, cached = 0, slab = 0, node; printk("Mem-info:\n"); show_free_areas(); printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); for_each_online_node(node) { struct page *page, *end; page = NODE_MEM_MAP(node); end = page + NODE_DATA(node)->node_spanned_pages; do { total++; if (PageReserved(page)) reserved++; else if (PageSwapCache(page)) cached++; else if (PageSlab(page)) slab++; else if (!page_count(page)) free++; else shared += page_count(page) - 1; page++; } while (page < end); } printk("%d pages of RAM\n", total); printk("%d free pages\n", free); printk("%d reserved pages\n", reserved); printk("%d slab pages\n", slab); printk("%d pages shared\n", shared); printk("%d pages swap cached\n", cached); }
/** * sg_miter_stop - stop mapping iteration * @miter: sg mapping iter to be stopped * * Description: * Stops mapping iterator @miter. @miter should have been started * started using sg_miter_start(). A stopped iteration can be * resumed by calling sg_miter_next() on it. This is useful when * resources (kmap) need to be released during iteration. * * Context: * IRQ disabled if the SG_MITER_ATOMIC is set. Don't care otherwise. */ void sg_miter_stop(struct sg_mapping_iter *miter) { WARN_ON(miter->consumed > miter->length); /* drop resources from the last iteration */ if (miter->addr) { miter->__offset += miter->consumed; if ((miter->__flags & SG_MITER_TO_SG) && !PageSlab(miter->page)) flush_kernel_dcache_page(miter->page); if (miter->__flags & SG_MITER_ATOMIC) { WARN_ON(!irqs_disabled()); kunmap_atomic(miter->addr); } else kunmap(miter->page); miter->page = NULL; miter->addr = NULL; miter->length = 0; miter->consumed = 0; } }
static void print_address_description(void *addr) { struct page *page = addr_to_page(addr); dump_stack(); pr_err("\n"); if (page && PageSlab(page)) { struct kmem_cache *cache = page->slab_cache; void *object = nearest_obj(cache, page, addr); describe_object(cache, object, addr); } if (kernel_or_module_addr(addr) && !init_task_stack_addr(addr)) { pr_err("The buggy address belongs to the variable:\n"); pr_err(" %pS\n", addr); } if (page) { pr_err("The buggy address belongs to the page:\n"); dump_page(page, "kasan: bad access detected"); } }
void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask) { struct page_ext *page_ext = lookup_page_ext(page); struct stack_trace trace = { .nr_entries = 0, .max_entries = ARRAY_SIZE(page_ext->trace_entries), .entries = &page_ext->trace_entries[0], .skip = 3, }; save_stack_trace(&trace); page_ext->order = order; page_ext->gfp_mask = gfp_mask; page_ext->nr_entries = trace.nr_entries; __set_bit(PAGE_EXT_OWNER, &page_ext->flags); } static ssize_t print_page_owner(char __user *buf, size_t count, unsigned long pfn, struct page *page, struct page_ext *page_ext) { int ret; int pageblock_mt, page_mt; char *kbuf; struct stack_trace trace = { .nr_entries = page_ext->nr_entries, .entries = &page_ext->trace_entries[0], }; kbuf = kmalloc(count, GFP_KERNEL); if (!kbuf) return -ENOMEM; ret = snprintf(kbuf, count, "Page allocated via order %u, mask 0x%x\n", page_ext->order, page_ext->gfp_mask); if (ret >= count) goto err; /* Print information relevant to grouping pages by mobility */ pageblock_mt = get_pfnblock_migratetype(page, pfn); page_mt = gfpflags_to_migratetype(page_ext->gfp_mask); ret += snprintf(kbuf + ret, count - ret, "PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n", pfn, pfn >> pageblock_order, pageblock_mt, pageblock_mt != page_mt ? "Fallback" : " ", PageLocked(page) ? "K" : " ", PageError(page) ? "E" : " ", PageReferenced(page) ? "R" : " ", PageUptodate(page) ? "U" : " ", PageDirty(page) ? "D" : " ", PageLRU(page) ? "L" : " ", PageActive(page) ? "A" : " ", PageSlab(page) ? "S" : " ", PageWriteback(page) ? "W" : " ", PageCompound(page) ? "C" : " ", PageSwapCache(page) ? "B" : " ", PageMappedToDisk(page) ? "M" : " "); if (ret >= count) goto err; ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0); if (ret >= count) goto err; ret += snprintf(kbuf + ret, count - ret, "\n"); if (ret >= count) goto err; if (copy_to_user(buf, kbuf, ret)) ret = -EFAULT; kfree(kbuf); return ret; err: kfree(kbuf); return -ENOMEM; } static ssize_t read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) { unsigned long pfn; struct page *page; struct page_ext *page_ext; if (!page_owner_inited) return -EINVAL; page = NULL; pfn = min_low_pfn + *ppos; /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) pfn++; drain_all_pages(NULL); /* Find an allocated page */ for (; pfn < max_pfn; pfn++) { /* * If the new page is in a new MAX_ORDER_NR_PAGES area, * validate the area as existing, skip it if not */ if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { pfn += MAX_ORDER_NR_PAGES - 1; continue; } /* Check for holes within a MAX_ORDER area */ if (!pfn_valid_within(pfn)) continue; page = pfn_to_page(pfn); if (PageBuddy(page)) { unsigned long freepage_order = page_order_unsafe(page); if (freepage_order < MAX_ORDER) pfn += (1UL << freepage_order) - 1; continue; } page_ext = lookup_page_ext(page); /* * Some pages could be missed by concurrent allocation or free, * because we don't hold the zone lock. */ if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) continue; /* Record the next PFN to read in the file offset */ *ppos = (pfn - min_low_pfn) + 1; return print_page_owner(buf, count, pfn, page, page_ext); } return 0; }
static void put_compound_page(struct page *page) { struct page *page_head; if (likely(!PageTail(page))) { if (put_page_testzero(page)) { /* * By the time all refcounts have been released * split_huge_page cannot run anymore from under us. */ if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } return; } /* __split_huge_page_refcount can run under us */ page_head = compound_trans_head(page); /* * THP can not break up slab pages so avoid taking * compound_lock() and skip the tail page refcounting (in * _mapcount) too. Slab performs non-atomic bit ops on * page->flags for better performance. In particular * slab_unlock() in slub used to be a hot path. It is still * hot on arches that do not support * this_cpu_cmpxchg_double(). * * If "page" is part of a slab or hugetlbfs page it cannot be * splitted and the head page cannot change from under us. And * if "page" is part of a THP page under splitting, if the * head page pointed by the THP tail isn't a THP head anymore, * we'll find PageTail clear after smp_rmb() and we'll treat * it as a single page. */ if (!__compound_tail_refcounted(page_head)) { /* * If "page" is a THP tail, we must read the tail page * flags after the head page flags. The * split_huge_page side enforces write memory barriers * between clearing PageTail and before the head page * can be freed and reallocated. */ smp_rmb(); if (likely(PageTail(page))) { /* * __split_huge_page_refcount cannot race * here. */ VM_BUG_ON(!PageHead(page_head)); VM_BUG_ON(page_mapcount(page) != 0); if (put_page_testzero(page_head)) { /* * If this is the tail of a slab * compound page, the tail pin must * not be the last reference held on * the page, because the PG_slab * cannot be cleared before all tail * pins (which skips the _mapcount * tail refcounting) have been * released. For hugetlbfs the tail * pin may be the last reference on * the page instead, because * PageHeadHuge will not go away until * the compound page enters the buddy * allocator. */ VM_BUG_ON(PageSlab(page_head)); __put_compound_page(page_head); } return; } else /* * __split_huge_page_refcount run before us, * "page" was a THP tail. The split page_head * has been freed and reallocated as slab or * hugetlbfs page of smaller order (only * possible if reallocated as slab on x86). */ goto out_put_single; } if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * page_head wasn't a dangling pointer but it may not * be a head page anymore by the time we obtain the * lock. That is ok as long as it can't be freed from * under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { /* * The head page may have been freed * and reallocated as a compound page * of smaller order and then freed * again. All we know is that it * cannot have become: a THP page, a * compound page of higher order, a * tail page. That is because we * still hold the refcount of the * split THP tail and page_head was * the THP head before the split. */ if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on the * compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } }
static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * THP can not break up slab pages so avoid taking * compound_lock(). Slab performs non-atomic bit ops * on page->flags for better performance. In particular * slab_unlock() in slub used to be a hot path. It is * still hot on arches that do not support * this_cpu_cmpxchg_double(). */ if (PageSlab(page_head)) { if (PageTail(page)) { if (put_page_testzero(page_head)) VM_BUG_ON(1); atomic_dec(&page->_mapcount); goto skip_lock_tail; } else goto skip_lock; } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); skip_lock: if (put_page_testzero(page_head)) __put_single_page(page_head); out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on * the compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); skip_lock_tail: if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }
static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * THP can not break up slab pages so avoid taking * compound_lock(). Slab performs non-atomic bit ops * on page->flags for better performance. In particular * slab_unlock() in slub used to be a hot path. It is * still hot on arches that do not support * this_cpu_cmpxchg_double(). */ if (PageSlab(page_head) || PageHeadHuge(page_head)) { if (likely(PageTail(page))) { /* * __split_huge_page_refcount * cannot race here. */ VM_BUG_ON(!PageHead(page_head)); atomic_dec(&page->_mapcount); if (put_page_testzero(page_head)) VM_BUG_ON(1); if (put_page_testzero(page_head)) __put_compound_page(page_head); return; } else /* * __split_huge_page_refcount * run before us, "page" was a * THP tail. The split * page_head has been freed * and reallocated as slab or * hugetlbfs page of smaller * order (only possible if * reallocated as slab on * x86). */ goto skip_lock; } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); skip_lock: if (put_page_testzero(page_head)) { /* * The head page may have been * freed and reallocated as a * compound page of smaller * order and then freed again. * All we know is that it * cannot have become: a THP * page, a compound page of * higher order, a tail page. * That is because we still * hold the refcount of the * split THP tail and * page_head was the THP head * before the split. */ if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on * the compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }
/* * is_slob_page: True for all slob pages (false for bigblock pages) */ static inline int is_slob_page(struct slob_page *sp) { return PageSlab((struct page *)sp); }
u64 stable_page_flags(struct page *page) { u64 k; u64 u; /* * pseudo flag: KPF_NOPAGE * it differentiates a memory hole from a page with no flags */ if (!page) return 1 << KPF_NOPAGE; k = page->flags; u = 0; /* * pseudo flags for the well known (anonymous) memory mapped pages * * Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the * simple test in page_mapcount() is not enough. */ if (!PageSlab(page) && page_mapcount(page)) u |= 1 << KPF_MMAP; if (PageAnon(page)) u |= 1 << KPF_ANON; if (PageKsm(page)) u |= 1 << KPF_KSM; /* * compound pages: export both head/tail info * they together define a compound page's start/end pos and order */ if (PageHead(page)) u |= 1 << KPF_COMPOUND_HEAD; if (PageTail(page)) u |= 1 << KPF_COMPOUND_TAIL; if (PageHuge(page)) u |= 1 << KPF_HUGE; /* * PageTransCompound can be true for non-huge compound pages (slab * pages or pages allocated by drivers with __GFP_COMP) because it * just checks PG_head/PG_tail, so we need to check PageLRU/PageAnon * to make sure a given page is a thp, not a non-huge compound page. */ else if (PageTransCompound(page)) { struct page *head = compound_head(page); if (PageLRU(head) || PageAnon(head)) u |= 1 << KPF_THP; else if (is_huge_zero_page(head)) { u |= 1 << KPF_ZERO_PAGE; u |= 1 << KPF_THP; } } else if (is_zero_pfn(page_to_pfn(page))) u |= 1 << KPF_ZERO_PAGE; /* * Caveats on high order pages: page->_count will only be set * -1 on the head page; SLUB/SLQB do the same for PG_slab; * SLOB won't set PG_slab at all on compound pages. */ if (PageBuddy(page)) u |= 1 << KPF_BUDDY; if (PageBalloon(page)) u |= 1 << KPF_BALLOON; if (page_is_idle(page)) u |= 1 << KPF_IDLE; u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); u |= kpf_copy_bit(k, KPF_SLAB, PG_slab); u |= kpf_copy_bit(k, KPF_ERROR, PG_error); u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback); u |= kpf_copy_bit(k, KPF_LRU, PG_lru); u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced); u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); u |= kpf_copy_bit(k, KPF_SWAPCACHE, PG_swapcache); u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); #ifdef CONFIG_MEMORY_FAILURE u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); #endif #ifdef CONFIG_ARCH_USES_PG_UNCACHED u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached); #endif u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk); u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); return u; };
u64 stable_page_flags(struct page *page) { u64 k; u64 u; /* * pseudo flag: KPF_NOPAGE * it differentiates a memory hole from a page with no flags */ if (!page) return 1 << KPF_NOPAGE; k = page->flags; u = 0; /* * pseudo flags for the well known (anonymous) memory mapped pages * * Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the * simple test in page_mapped() is not enough. */ if (!PageSlab(page) && page_mapped(page)) u |= 1 << KPF_MMAP; if (PageAnon(page)) u |= 1 << KPF_ANON; if (PageKsm(page)) u |= 1 << KPF_KSM; /* * compound pages: export both head/tail info * they together define a compound page's start/end pos and order */ if (PageHead(page)) u |= 1 << KPF_COMPOUND_HEAD; if (PageTail(page)) u |= 1 << KPF_COMPOUND_TAIL; if (PageHuge(page)) u |= 1 << KPF_HUGE; u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); /* * Caveats on high order pages: * PG_buddy will only be set on the head page; SLUB/SLQB do the same * for PG_slab; SLOB won't set PG_slab at all on compound pages. */ u |= kpf_copy_bit(k, KPF_SLAB, PG_slab); u |= kpf_copy_bit(k, KPF_BUDDY, PG_buddy); u |= kpf_copy_bit(k, KPF_ERROR, PG_error); u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback); u |= kpf_copy_bit(k, KPF_LRU, PG_lru); u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced); u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); u |= kpf_copy_bit(k, KPF_SWAPCACHE, PG_swapcache); u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); #ifdef CONFIG_MEMORY_FAILURE u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); #endif #ifdef CONFIG_IA64_UNCACHED_ALLOCATOR u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached); #endif u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk); u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); return u; };
/* * This function is exported but must not be called by anything other * than get_page(). It implements the slow path of get_page(). */ bool __get_page_tail(struct page *page) { /* * This takes care of get_page() if run on a tail page * returned by one of the get_user_pages/follow_page variants. * get_user_pages/follow_page itself doesn't need the compound * lock because it runs __get_page_tail_foll() under the * proper PT lock that already serializes against * split_huge_page(). */ unsigned long flags; bool got = false; struct page *page_head; /* * If this is a hugetlbfs page it cannot be split under us. Simply * increment refcount for the head page. */ if (PageHuge(page)) { page_head = compound_head(page); atomic_inc(&page_head->_count); got = true; goto out; } page_head = compound_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { /* Ref to put_compound_page() comment. */ if (PageSlab(page_head)) { if (likely(PageTail(page))) { /* * This is a hugetlbfs page or a slab * page. __split_huge_page_refcount * cannot race here. */ VM_BUG_ON(!PageHead(page_head)); __get_page_tail_foll(page, false); return true; } else { /* * __split_huge_page_refcount run * before us, "page" was a THP * tail. The split page_head has been * freed and reallocated as slab or * hugetlbfs page of smaller order * (only possible if reallocated as * slab on x86). */ put_page(page_head); return false; } } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); /* here __split_huge_page_refcount won't run anymore */ if (likely(PageTail(page))) { __get_page_tail_foll(page, false); got = true; } compound_unlock_irqrestore(page_head, flags); if (unlikely(!got)) put_page(page_head); } out: return got; }