void show_mem(unsigned int filter)
{
int free = 0, total = 0, reserved = 0;
int shared = 0, cached = 0, slab = 0, i;
struct meminfo *mi = &meminfo;
printk("Mem-info:\n");
show_free_areas(filter);
for_each_bank(i, mi) {
struct membank *bank = &mi->bank[i];
unsigned int pfn1, pfn2;
struct page *page, *end;
pfn1 = bank_pfn_start(bank);
pfn2 = bank_pfn_end(bank);
page = pfn_to_page(pfn1);
end = pfn_to_page(pfn2 - 1) + 1;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (!page_count(page))
free++;
else
shared += page_count(page) - 1;
page++;
} while (page < end);
}
printk(KERN_INFO "%d pages of RAM\n", total);
printk(KERN_INFO "%d free pages\n", free);
printk(KERN_INFO "%d reserved pages\n", reserved);
printk(KERN_INFO "%d slab pages\n", slab);
printk(KERN_INFO "%d pages shared\n", shared);
printk(KERN_INFO "%d pages swap cached\n", cached);
}
/* /proc/kpagecount - an array exposing page counts
*
* Each entry is a u64 representing the corresponding
* physical page count.
*/
static ssize_t kpagecount_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
u64 __user *out = (u64 __user *)buf;
struct page *ppage;
unsigned long src = *ppos;
unsigned long pfn;
unsigned long max_pfn_kpmsize = max_pfn * KPMSIZE;
ssize_t ret = 0;
u64 pcount;
pfn = src / KPMSIZE;
if (src != max_pfn_kpmsize)
count = min_t(size_t, count, max_pfn_kpmsize - src);
if (src & KPMMASK || count & KPMMASK)
return -EINVAL;
while (count > 0) {
if (pfn_valid(pfn))
ppage = pfn_to_page(pfn);
else
ppage = NULL;
if (!ppage || PageSlab(ppage))
pcount = 0;
else
pcount = page_mapcount(ppage);
if (put_user(pcount, out)) {
ret = -EFAULT;
break;
}
pfn++;
out++;
count -= KPMSIZE;
}
*ppos += (char __user *)out - buf;
if (!ret)
ret = (char __user *)out - buf;
return ret;
}
/*
* This keeps memory configuration data used by a couple memory
* initialization functions, as well as show_mem() for the skipping
* of holes in the memory map. It is populated by arm_add_memory().
*/
void show_mem(unsigned int filter)
{
int free = 0, total = 0, reserved = 0;
int shared = 0, cached = 0, slab = 0;
struct memblock_region *reg;
printk("Mem-info:\n");
show_free_areas(filter);
if (filter & SHOW_MEM_FILTER_PAGE_COUNT)
return;
for_each_memblock(memory, reg) {
unsigned int pfn1, pfn2;
struct page *page, *end;
pfn1 = memblock_region_memory_base_pfn(reg);
pfn2 = memblock_region_memory_end_pfn(reg);
page = pfn_to_page(pfn1);
end = pfn_to_page(pfn2 - 1) + 1;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (!page_count(page))
free++;
else
shared += page_count(page) - 1;
page++;
#ifdef CONFIG_SPARSEMEM
pfn1++;
if (!(pfn1 % PAGES_PER_SECTION))
page = pfn_to_page(pfn1);
} while (pfn1 < pfn2);
#else
} while (page < end);
void show_mem(unsigned int filter)
{
int free = 0, total = 0, reserved = 0;
int shared = 0, cached = 0, slab = 0, i;
struct meminfo * mi = &meminfo;
printk("Mem-info:\n");
show_free_areas(filter);
for_each_bank (i, mi) {
struct membank *bank = &mi->bank[i];
unsigned int pfn1, pfn2;
struct page *page, *end;
pfn1 = bank_pfn_start(bank);
pfn2 = bank_pfn_end(bank);
page = pfn_to_page(pfn1);
end = pfn_to_page(pfn2 - 1) + 1;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (!page_count(page))
free++;
else
shared += page_count(page) - 1;
page++;
#ifdef CONFIG_SPARSEMEM
pfn1++;
if (!(pfn1 % PAGES_PER_SECTION))
page = pfn_to_page(pfn1);
} while (pfn1 < pfn2);
#else
} while (page < end);
#endif
}
static int _phymem_pages_proc_show(struct seq_file *m, void *v)
{
int free = 0, total = 0, reserved = 0;
int other = 0, shared = 0, cached = 0, slab = 0, node, i;
struct meminfo * mi = &meminfo;
for_each_bank (i,mi) {
struct membank *bank = &mi->bank[i];
unsigned int pfn1, pfn2;
struct page *page, *end;
pfn1 = bank_pfn_start(bank);
pfn2 = bank_pfn_end(bank);
page = pfn_to_page(pfn1);
end = pfn_to_page(pfn2 - 1) + 1;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (page_count(page) > 1)
shared++;
else if (!page_count(page))
free++;
else
other++;
page++;
}while (page < end);
}
seq_printf(m, "pages of RAM %d\n", total);
seq_printf(m, "free pages %d\n", free);
seq_printf(m, "reserved pages %d\n", reserved);
seq_printf(m, "slab pages %d\n", slab);
seq_printf(m, "pages shared %d\n", shared);
seq_printf(m, "pages swap cached %d\n", cached);
seq_printf(m, "other pages %d\n", other);
return 0;
}
void show_mem(void)
{
int free = 0, total = 0, reserved = 0;
int shared = 0, cached = 0, slab = 0, node, i;
struct meminfo * mi = &meminfo;
printk("Mem-info:\n");
show_free_areas();
printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
for_each_online_node(node) {
pg_data_t *n = NODE_DATA(node);
struct page *map = n->node_mem_map - n->node_start_pfn;
for_each_nodebank (i,mi,node) {
unsigned int pfn1, pfn2;
struct page *page, *end;
pfn1 = __phys_to_pfn(mi->bank[i].start);
pfn2 = __phys_to_pfn(mi->bank[i].size + mi->bank[i].start);
page = map + pfn1;
end = map + pfn2;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (!page_count(page))
free++;
else
shared += page_count(page) - 1;
page++;
} while (page < end);
}
}
/*
* Return the total memory allocated for this pointer, not
* just what the caller asked for.
*
* Doesn't have to be accurate, i.e. may have races.
*/
unsigned int kobjsize(const void *objp)
{
struct page *page;
/*
* If the object we have should not have ksize performed on it,
* return size of 0
*/
if (!objp || !virt_addr_valid(objp))
return 0;
page = virt_to_head_page(objp);
/*
* If the allocator sets PageSlab, we know the pointer came from
* kmalloc().
*/
if (PageSlab(page))
return ksize(objp);
/*
* If it's not a compound page, see if we have a matching VMA
* region. This test is intentionally done in reverse order,
* so if there's no VMA, we still fall through and hand back
* PAGE_SIZE for 0-order pages.
*/
if (!PageCompound(page)) {
struct vm_area_struct *vma;
vma = find_vma(current->mm, (unsigned long)objp);
if (vma)
return vma->vm_end - vma->vm_start;
}
/*
* The ksize() function is only guaranteed to work for pointers
* returned by kmalloc(). So handle arbitrary pointers here.
*/
return PAGE_SIZE << compound_order(page);
}
/*
* Show free area list (used inside shift_scroll-lock stuff)
* We also calculate the percentage fragmentation. We do this by counting the
* memory on each free list with the exception of the first item on the list.
*/
int Jcmd_mem(char *arg1, char *arg2) {
unsigned long order, flags;
unsigned long total = 0;
spin_lock_irqsave(&free_area_lock, flags);
for (order = 0; order < NR_MEM_LISTS; order++) {
struct page * tmp;
unsigned long nr = 0;
for (tmp = free_mem_area[order].next;
tmp != memory_head(free_mem_area+order); tmp = tmp->next) {
nr++;
}
total += nr << order;
ut_printf("%d(%d): count:%d static count:%d total:%d (%dM)\n", order,1<<order, nr,
free_mem_area[order].stat_count, (nr << order), ((nr << order)*PAGE_SIZE)/(1024*1024));
}
spin_unlock_irqrestore(&free_area_lock, flags);
ut_printf("total Free pages = %d (%dM) Actual pages: %d (%dM) pagecachesize: %dM , freepages:%d\n", total, (total * 4) / 1024,g_stat_mem_size/PAGE_SIZE,g_stat_mem_size/(1024*1024),g_pagecache_size/(1024*1024),g_nr_free_pages);
int slab=0;
int referenced=0;
int reserved=0;
int dma=0;
unsigned long va_end=(unsigned long)__va(g_phy_mem_size);
page_struct_t *p;
p = g_mem_map + MAP_NR(va_end);
do {
--p;
if (PageReserved(p)) reserved++;
if (PageDMA(p)) dma++;
if (PageReferenced(p))referenced++;
if (PageSlab(p)) slab++;
} while (p > g_mem_map);
ut_printf(" reserved :%d(%dM) referenced:%d dma:%d slab:%d stat_allocs:%d stat_frees: %d\n\n",reserved,(reserved*PAGE_SIZE)/(1024*1024),referenced,dma,slab,stat_allocs,stat_frees);
if ((arg1 != 0) && (ut_strcmp(arg1,"all")==0))
Jcmd_jslab(0,0);
return 1;
}
void show_mem(void)
{
int free = 0, total = 0, reserved = 0;
int shared = 0, cached = 0, slab = 0, node;
printk("Mem-info:\n");
show_free_areas();
printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
for_each_online_node(node) {
struct page *page, *end;
page = NODE_MEM_MAP(node);
end = page + NODE_DATA(node)->node_spanned_pages;
do {
total++;
if (PageReserved(page))
reserved++;
else if (PageSwapCache(page))
cached++;
else if (PageSlab(page))
slab++;
else if (!page_count(page))
free++;
else
shared += page_count(page) - 1;
page++;
} while (page < end);
}
printk("%d pages of RAM\n", total);
printk("%d free pages\n", free);
printk("%d reserved pages\n", reserved);
printk("%d slab pages\n", slab);
printk("%d pages shared\n", shared);
printk("%d pages swap cached\n", cached);
}
/**
* sg_miter_stop - stop mapping iteration
* @miter: sg mapping iter to be stopped
*
* Description:
* Stops mapping iterator @miter. @miter should have been started
* started using sg_miter_start(). A stopped iteration can be
* resumed by calling sg_miter_next() on it. This is useful when
* resources (kmap) need to be released during iteration.
*
* Context:
* IRQ disabled if the SG_MITER_ATOMIC is set. Don't care otherwise.
*/
void sg_miter_stop(struct sg_mapping_iter *miter)
{
WARN_ON(miter->consumed > miter->length);
/* drop resources from the last iteration */
if (miter->addr) {
miter->__offset += miter->consumed;
if ((miter->__flags & SG_MITER_TO_SG) && !PageSlab(miter->page))
flush_kernel_dcache_page(miter->page);
if (miter->__flags & SG_MITER_ATOMIC) {
WARN_ON(!irqs_disabled());
kunmap_atomic(miter->addr);
} else
kunmap(miter->page);
miter->page = NULL;
miter->addr = NULL;
miter->length = 0;
miter->consumed = 0;
}
}
static void print_address_description(void *addr)
{
struct page *page = addr_to_page(addr);
dump_stack();
pr_err("\n");
if (page && PageSlab(page)) {
struct kmem_cache *cache = page->slab_cache;
void *object = nearest_obj(cache, page, addr);
describe_object(cache, object, addr);
}
if (kernel_or_module_addr(addr) && !init_task_stack_addr(addr)) {
pr_err("The buggy address belongs to the variable:\n");
pr_err(" %pS\n", addr);
}
if (page) {
pr_err("The buggy address belongs to the page:\n");
dump_page(page, "kasan: bad access detected");
}
}
void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask)
{
struct page_ext *page_ext = lookup_page_ext(page);
struct stack_trace trace = {
.nr_entries = 0,
.max_entries = ARRAY_SIZE(page_ext->trace_entries),
.entries = &page_ext->trace_entries[0],
.skip = 3,
};
save_stack_trace(&trace);
page_ext->order = order;
page_ext->gfp_mask = gfp_mask;
page_ext->nr_entries = trace.nr_entries;
__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
}
static ssize_t
print_page_owner(char __user *buf, size_t count, unsigned long pfn,
struct page *page, struct page_ext *page_ext)
{
int ret;
int pageblock_mt, page_mt;
char *kbuf;
struct stack_trace trace = {
.nr_entries = page_ext->nr_entries,
.entries = &page_ext->trace_entries[0],
};
kbuf = kmalloc(count, GFP_KERNEL);
if (!kbuf)
return -ENOMEM;
ret = snprintf(kbuf, count,
"Page allocated via order %u, mask 0x%x\n",
page_ext->order, page_ext->gfp_mask);
if (ret >= count)
goto err;
/* Print information relevant to grouping pages by mobility */
pageblock_mt = get_pfnblock_migratetype(page, pfn);
page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);
ret += snprintf(kbuf + ret, count - ret,
"PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n",
pfn,
pfn >> pageblock_order,
pageblock_mt,
pageblock_mt != page_mt ? "Fallback" : " ",
PageLocked(page) ? "K" : " ",
PageError(page) ? "E" : " ",
PageReferenced(page) ? "R" : " ",
PageUptodate(page) ? "U" : " ",
PageDirty(page) ? "D" : " ",
PageLRU(page) ? "L" : " ",
PageActive(page) ? "A" : " ",
PageSlab(page) ? "S" : " ",
PageWriteback(page) ? "W" : " ",
PageCompound(page) ? "C" : " ",
PageSwapCache(page) ? "B" : " ",
PageMappedToDisk(page) ? "M" : " ");
if (ret >= count)
goto err;
ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
if (ret >= count)
goto err;
ret += snprintf(kbuf + ret, count - ret, "\n");
if (ret >= count)
goto err;
if (copy_to_user(buf, kbuf, ret))
ret = -EFAULT;
kfree(kbuf);
return ret;
err:
kfree(kbuf);
return -ENOMEM;
}
static ssize_t
read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
unsigned long pfn;
struct page *page;
struct page_ext *page_ext;
if (!page_owner_inited)
return -EINVAL;
page = NULL;
pfn = min_low_pfn + *ppos;
/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
pfn++;
drain_all_pages(NULL);
/* Find an allocated page */
for (; pfn < max_pfn; pfn++) {
/*
* If the new page is in a new MAX_ORDER_NR_PAGES area,
* validate the area as existing, skip it if not
*/
if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
pfn += MAX_ORDER_NR_PAGES - 1;
continue;
}
/* Check for holes within a MAX_ORDER area */
if (!pfn_valid_within(pfn))
continue;
page = pfn_to_page(pfn);
if (PageBuddy(page)) {
unsigned long freepage_order = page_order_unsafe(page);
if (freepage_order < MAX_ORDER)
pfn += (1UL << freepage_order) - 1;
continue;
}
page_ext = lookup_page_ext(page);
/*
* Some pages could be missed by concurrent allocation or free,
* because we don't hold the zone lock.
*/
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
continue;
/* Record the next PFN to read in the file offset */
*ppos = (pfn - min_low_pfn) + 1;
return print_page_owner(buf, count, pfn, page, page_ext);
}
return 0;
}
static void put_compound_page(struct page *page)
{
struct page *page_head;
if (likely(!PageTail(page))) {
if (put_page_testzero(page)) {
/*
* By the time all refcounts have been released
* split_huge_page cannot run anymore from under us.
*/
if (PageHead(page))
__put_compound_page(page);
else
__put_single_page(page);
}
return;
}
/* __split_huge_page_refcount can run under us */
page_head = compound_trans_head(page);
/*
* THP can not break up slab pages so avoid taking
* compound_lock() and skip the tail page refcounting (in
* _mapcount) too. Slab performs non-atomic bit ops on
* page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is still
* hot on arches that do not support
* this_cpu_cmpxchg_double().
*
* If "page" is part of a slab or hugetlbfs page it cannot be
* splitted and the head page cannot change from under us. And
* if "page" is part of a THP page under splitting, if the
* head page pointed by the THP tail isn't a THP head anymore,
* we'll find PageTail clear after smp_rmb() and we'll treat
* it as a single page.
*/
if (!__compound_tail_refcounted(page_head)) {
/*
* If "page" is a THP tail, we must read the tail page
* flags after the head page flags. The
* split_huge_page side enforces write memory barriers
* between clearing PageTail and before the head page
* can be freed and reallocated.
*/
smp_rmb();
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount cannot race
* here.
*/
VM_BUG_ON(!PageHead(page_head));
VM_BUG_ON(page_mapcount(page) != 0);
if (put_page_testzero(page_head)) {
/*
* If this is the tail of a slab
* compound page, the tail pin must
* not be the last reference held on
* the page, because the PG_slab
* cannot be cleared before all tail
* pins (which skips the _mapcount
* tail refcounting) have been
* released. For hugetlbfs the tail
* pin may be the last reference on
* the page instead, because
* PageHeadHuge will not go away until
* the compound page enters the buddy
* allocator.
*/
VM_BUG_ON(PageSlab(page_head));
__put_compound_page(page_head);
}
return;
} else
/*
* __split_huge_page_refcount run before us,
* "page" was a THP tail. The split page_head
* has been freed and reallocated as slab or
* hugetlbfs page of smaller order (only
* possible if reallocated as slab on x86).
*/
goto out_put_single;
}
if (likely(page != page_head && get_page_unless_zero(page_head))) {
unsigned long flags;
/*
* page_head wasn't a dangling pointer but it may not
* be a head page anymore by the time we obtain the
* lock. That is ok as long as it can't be freed from
* under us.
*/
flags = compound_lock_irqsave(page_head);
if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags);
if (put_page_testzero(page_head)) {
/*
* The head page may have been freed
* and reallocated as a compound page
* of smaller order and then freed
* again. All we know is that it
* cannot have become: a THP page, a
* compound page of higher order, a
* tail page. That is because we
* still hold the refcount of the
* split THP tail and page_head was
* the THP head before the split.
*/
if (PageHead(page_head))
__put_compound_page(page_head);
else
__put_single_page(page_head);
}
out_put_single:
if (put_page_testzero(page))
__put_single_page(page);
return;
}
VM_BUG_ON(page_head != page->first_page);
/*
* We can release the refcount taken by
* get_page_unless_zero() now that
* __split_huge_page_refcount() is blocked on the
* compound_lock.
*/
if (put_page_testzero(page_head))
VM_BUG_ON(1);
/* __split_huge_page_refcount will wait now */
VM_BUG_ON(page_mapcount(page) <= 0);
atomic_dec(&page->_mapcount);
VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
VM_BUG_ON(atomic_read(&page->_count) != 0);
compound_unlock_irqrestore(page_head, flags);
if (put_page_testzero(page_head)) {
if (PageHead(page_head))
__put_compound_page(page_head);
else
__put_single_page(page_head);
}
} else {
/* page_head is a dangling pointer */
VM_BUG_ON(PageTail(page));
goto out_put_single;
}
}
static void put_compound_page(struct page *page)
{
if (unlikely(PageTail(page))) {
/* __split_huge_page_refcount can run under us */
struct page *page_head = compound_trans_head(page);
if (likely(page != page_head &&
get_page_unless_zero(page_head))) {
unsigned long flags;
/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is
* still hot on arches that do not support
* this_cpu_cmpxchg_double().
*/
if (PageSlab(page_head)) {
if (PageTail(page)) {
if (put_page_testzero(page_head))
VM_BUG_ON(1);
atomic_dec(&page->_mapcount);
goto skip_lock_tail;
} else
goto skip_lock;
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags);
skip_lock:
if (put_page_testzero(page_head))
__put_single_page(page_head);
out_put_single:
if (put_page_testzero(page))
__put_single_page(page);
return;
}
VM_BUG_ON(page_head != page->first_page);
/*
* We can release the refcount taken by
* get_page_unless_zero() now that
* __split_huge_page_refcount() is blocked on
* the compound_lock.
*/
if (put_page_testzero(page_head))
VM_BUG_ON(1);
/* __split_huge_page_refcount will wait now */
VM_BUG_ON(page_mapcount(page) <= 0);
atomic_dec(&page->_mapcount);
VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
VM_BUG_ON(atomic_read(&page->_count) != 0);
compound_unlock_irqrestore(page_head, flags);
skip_lock_tail:
if (put_page_testzero(page_head)) {
if (PageHead(page_head))
__put_compound_page(page_head);
else
__put_single_page(page_head);
}
} else {
/* page_head is a dangling pointer */
VM_BUG_ON(PageTail(page));
goto out_put_single;
}
} else if (put_page_testzero(page)) {
if (PageHead(page))
__put_compound_page(page);
else
__put_single_page(page);
}
}
static void put_compound_page(struct page *page)
{
if (unlikely(PageTail(page))) {
/* __split_huge_page_refcount can run under us */
struct page *page_head = compound_head(page);
if (likely(page != page_head &&
get_page_unless_zero(page_head))) {
unsigned long flags;
/*
* THP can not break up slab pages so avoid taking
* compound_lock(). Slab performs non-atomic bit ops
* on page->flags for better performance. In particular
* slab_unlock() in slub used to be a hot path. It is
* still hot on arches that do not support
* this_cpu_cmpxchg_double().
*/
if (PageSlab(page_head) || PageHeadHuge(page_head)) {
if (likely(PageTail(page))) {
/*
* __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
atomic_dec(&page->_mapcount);
if (put_page_testzero(page_head))
VM_BUG_ON(1);
if (put_page_testzero(page_head))
__put_compound_page(page_head);
return;
} else
/*
* __split_huge_page_refcount
* run before us, "page" was a
* THP tail. The split
* page_head has been freed
* and reallocated as slab or
* hugetlbfs page of smaller
* order (only possible if
* reallocated as slab on
* x86).
*/
goto skip_lock;
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
if (unlikely(!PageTail(page))) {
/* __split_huge_page_refcount run before us */
compound_unlock_irqrestore(page_head, flags);
skip_lock:
if (put_page_testzero(page_head)) {
/*
* The head page may have been
* freed and reallocated as a
* compound page of smaller
* order and then freed again.
* All we know is that it
* cannot have become: a THP
* page, a compound page of
* higher order, a tail page.
* That is because we still
* hold the refcount of the
* split THP tail and
* page_head was the THP head
* before the split.
*/
if (PageHead(page_head))
__put_compound_page(page_head);
else
__put_single_page(page_head);
}
out_put_single:
if (put_page_testzero(page))
__put_single_page(page);
return;
}
VM_BUG_ON(page_head != page->first_page);
/*
* We can release the refcount taken by
* get_page_unless_zero() now that
* __split_huge_page_refcount() is blocked on
* the compound_lock.
*/
if (put_page_testzero(page_head))
VM_BUG_ON(1);
/* __split_huge_page_refcount will wait now */
VM_BUG_ON(page_mapcount(page) <= 0);
atomic_dec(&page->_mapcount);
VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
VM_BUG_ON(atomic_read(&page->_count) != 0);
compound_unlock_irqrestore(page_head, flags);
if (put_page_testzero(page_head)) {
if (PageHead(page_head))
__put_compound_page(page_head);
else
__put_single_page(page_head);
}
} else {
/* page_head is a dangling pointer */
VM_BUG_ON(PageTail(page));
goto out_put_single;
}
} else if (put_page_testzero(page)) {
if (PageHead(page))
__put_compound_page(page);
else
__put_single_page(page);
}
}
/*
* is_slob_page: True for all slob pages (false for bigblock pages)
*/
static inline int is_slob_page(struct slob_page *sp)
{
return PageSlab((struct page *)sp);
}
u64 stable_page_flags(struct page *page)
{
u64 k;
u64 u;
/*
* pseudo flag: KPF_NOPAGE
* it differentiates a memory hole from a page with no flags
*/
if (!page)
return 1 << KPF_NOPAGE;
k = page->flags;
u = 0;
/*
* pseudo flags for the well known (anonymous) memory mapped pages
*
* Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the
* simple test in page_mapcount() is not enough.
*/
if (!PageSlab(page) && page_mapcount(page))
u |= 1 << KPF_MMAP;
if (PageAnon(page))
u |= 1 << KPF_ANON;
if (PageKsm(page))
u |= 1 << KPF_KSM;
/*
* compound pages: export both head/tail info
* they together define a compound page's start/end pos and order
*/
if (PageHead(page))
u |= 1 << KPF_COMPOUND_HEAD;
if (PageTail(page))
u |= 1 << KPF_COMPOUND_TAIL;
if (PageHuge(page))
u |= 1 << KPF_HUGE;
/*
* PageTransCompound can be true for non-huge compound pages (slab
* pages or pages allocated by drivers with __GFP_COMP) because it
* just checks PG_head/PG_tail, so we need to check PageLRU/PageAnon
* to make sure a given page is a thp, not a non-huge compound page.
*/
else if (PageTransCompound(page)) {
struct page *head = compound_head(page);
if (PageLRU(head) || PageAnon(head))
u |= 1 << KPF_THP;
else if (is_huge_zero_page(head)) {
u |= 1 << KPF_ZERO_PAGE;
u |= 1 << KPF_THP;
}
} else if (is_zero_pfn(page_to_pfn(page)))
u |= 1 << KPF_ZERO_PAGE;
/*
* Caveats on high order pages: page->_count will only be set
* -1 on the head page; SLUB/SLQB do the same for PG_slab;
* SLOB won't set PG_slab at all on compound pages.
*/
if (PageBuddy(page))
u |= 1 << KPF_BUDDY;
if (PageBalloon(page))
u |= 1 << KPF_BALLOON;
if (page_is_idle(page))
u |= 1 << KPF_IDLE;
u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
u |= kpf_copy_bit(k, KPF_ERROR, PG_error);
u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty);
u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate);
u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback);
u |= kpf_copy_bit(k, KPF_LRU, PG_lru);
u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced);
u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active);
u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim);
u |= kpf_copy_bit(k, KPF_SWAPCACHE, PG_swapcache);
u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked);
u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable);
u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked);
#ifdef CONFIG_MEMORY_FAILURE
u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison);
#endif
#ifdef CONFIG_ARCH_USES_PG_UNCACHED
u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached);
#endif
u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);
u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk);
u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private);
u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
return u;
};
u64 stable_page_flags(struct page *page)
{
u64 k;
u64 u;
/*
* pseudo flag: KPF_NOPAGE
* it differentiates a memory hole from a page with no flags
*/
if (!page)
return 1 << KPF_NOPAGE;
k = page->flags;
u = 0;
/*
* pseudo flags for the well known (anonymous) memory mapped pages
*
* Note that page->_mapcount is overloaded in SLOB/SLUB/SLQB, so the
* simple test in page_mapped() is not enough.
*/
if (!PageSlab(page) && page_mapped(page))
u |= 1 << KPF_MMAP;
if (PageAnon(page))
u |= 1 << KPF_ANON;
if (PageKsm(page))
u |= 1 << KPF_KSM;
/*
* compound pages: export both head/tail info
* they together define a compound page's start/end pos and order
*/
if (PageHead(page))
u |= 1 << KPF_COMPOUND_HEAD;
if (PageTail(page))
u |= 1 << KPF_COMPOUND_TAIL;
if (PageHuge(page))
u |= 1 << KPF_HUGE;
u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked);
/*
* Caveats on high order pages:
* PG_buddy will only be set on the head page; SLUB/SLQB do the same
* for PG_slab; SLOB won't set PG_slab at all on compound pages.
*/
u |= kpf_copy_bit(k, KPF_SLAB, PG_slab);
u |= kpf_copy_bit(k, KPF_BUDDY, PG_buddy);
u |= kpf_copy_bit(k, KPF_ERROR, PG_error);
u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty);
u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate);
u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback);
u |= kpf_copy_bit(k, KPF_LRU, PG_lru);
u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced);
u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active);
u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim);
u |= kpf_copy_bit(k, KPF_SWAPCACHE, PG_swapcache);
u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked);
u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable);
u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked);
#ifdef CONFIG_MEMORY_FAILURE
u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison);
#endif
#ifdef CONFIG_IA64_UNCACHED_ALLOCATOR
u |= kpf_copy_bit(k, KPF_UNCACHED, PG_uncached);
#endif
u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved);
u |= kpf_copy_bit(k, KPF_MAPPEDTODISK, PG_mappedtodisk);
u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private);
u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2);
u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1);
u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1);
return u;
};
/*
* This function is exported but must not be called by anything other
* than get_page(). It implements the slow path of get_page().
*/
bool __get_page_tail(struct page *page)
{
/*
* This takes care of get_page() if run on a tail page
* returned by one of the get_user_pages/follow_page variants.
* get_user_pages/follow_page itself doesn't need the compound
* lock because it runs __get_page_tail_foll() under the
* proper PT lock that already serializes against
* split_huge_page().
*/
unsigned long flags;
bool got = false;
struct page *page_head;
/*
* If this is a hugetlbfs page it cannot be split under us. Simply
* increment refcount for the head page.
*/
if (PageHuge(page)) {
page_head = compound_head(page);
atomic_inc(&page_head->_count);
got = true;
goto out;
}
page_head = compound_head(page);
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/* Ref to put_compound_page() comment. */
if (PageSlab(page_head)) {
if (likely(PageTail(page))) {
/*
* This is a hugetlbfs page or a slab
* page. __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
__get_page_tail_foll(page, false);
return true;
} else {
/*
* __split_huge_page_refcount run
* before us, "page" was a THP
* tail. The split page_head has been
* freed and reallocated as slab or
* hugetlbfs page of smaller order
* (only possible if reallocated as
* slab on x86).
*/
put_page(page_head);
return false;
}
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
/* here __split_huge_page_refcount won't run anymore */
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
got = true;
}
compound_unlock_irqrestore(page_head, flags);
if (unlikely(!got))
put_page(page_head);
}
out:
return got;
}
