static
void extract_stack(struct task_struct *p, char *stacktxt, int skip)
{
struct stack_trace trace;
unsigned long entries[32];
char tmp[48];
int i, j;
size_t frame_len;
trace.nr_entries = 0;
trace.max_entries = ARRAY_SIZE(entries);
trace.entries = entries;
trace.skip = 0;
save_stack_trace(&trace);
// print_stack_trace(&trace, 0);
j = 0;
for (i = 0; i < trace.nr_entries; i++) {
if (i < skip)
continue;
snprintf(tmp, 48, "%pS\n", (void *) trace.entries[i]);
frame_len = strlen(tmp);
snprintf(stacktxt + j, MAX_STACK_TXT - j, tmp);
j += frame_len;
if (MAX_STACK_TXT - j < 0)
return;
}
}
/* this function may be time consuming, move out from table set/get? */
ktap_str_t *kp_obj_kstack2str(ktap_state_t *ks, uint16_t depth, uint16_t skip)
{
struct stack_trace trace;
unsigned long *bt;
char *btstr, *p;
int i;
bt = kp_this_cpu_print_buffer(ks); /* use print percpu buffer */
trace.nr_entries = 0;
trace.skip = skip;
trace.max_entries = depth;
trace.entries = (unsigned long *)(bt + 1);
save_stack_trace(&trace);
/* convert backtrace to string */
p = btstr = kp_this_cpu_temp_buffer(ks);
for (i = 0; i < trace.nr_entries; i++) {
unsigned long addr = trace.entries[i];
if (addr == ULONG_MAX)
break;
p += sprint_symbol(p, addr);
*p++ = '\n';
}
return kp_str_new(ks, btstr, p - btstr);
}
static void set_track(struct kmem_cache *s, void *object,
enum track_item alloc, unsigned long addr)
{
struct track *p = get_track(s, object, alloc);
if (addr) {
#ifdef CONFIG_STACKTRACE
struct stack_trace trace;
int i;
trace.nr_entries = 0;
trace.max_entries = TRACK_ADDRS_COUNT;
trace.entries = p->addrs;
trace.skip = 3;
save_stack_trace(&trace);
/* See rant in lockdep.c */
if (trace.nr_entries != 0 &&
trace.entries[trace.nr_entries - 1] == ULONG_MAX)
trace.nr_entries--;
for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++)
p->addrs[i] = 0;
#endif
p->addr = addr;
p->cpu = smp_processor_id();
p->pid = current->pid;
p->when = jiffies;
} else
memset(p, 0, sizeof(struct track));
}
static
void htc_trace_pages_user(struct page *page, int numpages, int free)
{
struct page_user_trace* user_trace;
struct stack_trace trace;
unsigned long* entries;
int nr_entries;
if (unlikely(!page))
return;
user_trace = page_to_trace(page, free);
entries = user_trace->entries;
nr_entries = page_to_entries_size(page, free);
user_trace->pid = current->pid;
user_trace->tgid = current->tgid;
memcpy(user_trace->comm, current->comm,
sizeof(user_trace->comm));
memcpy(user_trace->tgcomm, current->group_leader->comm,
sizeof(user_trace->comm));
if (unlikely(!nr_entries))
return;
memset(entries, 0, nr_entries * sizeof(*entries));
trace.max_entries = nr_entries;
trace.entries = entries;
trace.nr_entries = 0;
trace.skip = 3;
save_stack_trace(&trace);
for (; page++, numpages > 1; numpages--)
memcpy(page_to_trace(page, free), user_trace, sizeof(*user_trace));
}
static int kplib_backtrace(ktap_state *ks)
{
struct stack_trace trace;
int skip = 10, max_entries = 10;
int n = kp_arg_nr(ks);
ktap_btrace *bt;
if (n >= 1) {
kp_arg_check(ks, 1, KTAP_TYPE_NUMBER);
skip = nvalue(kp_arg(ks, 1));
}
if (n >= 2) {
kp_arg_check(ks, 2, KTAP_TYPE_NUMBER);
max_entries = nvalue(kp_arg(ks, 2));
max_entries = min(max_entries, KTAP_MAX_STACK_ENTRIES);
}
bt = kp_percpu_data(ks, KTAP_PERCPU_DATA_BTRACE);
trace.nr_entries = 0;
trace.skip = skip;
trace.max_entries = max_entries;
trace.entries = (unsigned long *)(bt + 1);
save_stack_trace(&trace);
bt->nr_entries = trace.nr_entries;
set_btrace(ks->top, bt);
incr_top(ks);
return 1;
}
static bool fail_stacktrace(struct fault_attr *attr)
{
struct stack_trace trace;
int depth = attr->stacktrace_depth;
unsigned long entries[MAX_STACK_TRACE_DEPTH];
int n;
bool found = (attr->require_start == 0 && attr->require_end == ULONG_MAX);
if (depth == 0)
return found;
trace.nr_entries = 0;
trace.entries = entries;
trace.max_entries = depth;
trace.skip = 1;
save_stack_trace(&trace);
for (n = 0; n < trace.nr_entries; n++) {
if (attr->reject_start <= entries[n] &&
entries[n] < attr->reject_end)
return false;
if (attr->require_start <= entries[n] &&
entries[n] < attr->require_end)
found = true;
}
return found;
}
/**
* vos_mem_save_stack_trace() - Save stack trace of the caller
* @mem_struct: Pointer to the memory structure where to save the stack trace
*
* Return: None
*/
static inline void vos_mem_save_stack_trace(struct s_vos_mem_struct* mem_struct)
{
struct stack_trace *trace = &mem_struct->trace;
trace->nr_entries = 0;
trace->max_entries = VOS_MEM_MAX_STACK_TRACE;
trace->entries = mem_struct->stack_trace;
trace->skip = 2;
save_stack_trace(trace);
}
static __always_inline depot_stack_handle_t create_dummy_stack(void)
{
unsigned long entries[4];
struct stack_trace dummy;
dummy.nr_entries = 0;
dummy.max_entries = ARRAY_SIZE(entries);
dummy.entries = &entries[0];
dummy.skip = 0;
save_stack_trace(&dummy);
return depot_save_stack(&dummy, GFP_KERNEL);
}
static noinline void register_failure_stack(void)
{
unsigned long entries[4];
struct stack_trace failure;
failure.nr_entries = 0;
failure.max_entries = ARRAY_SIZE(entries);
failure.entries = &entries[0];
failure.skip = 0;
save_stack_trace(&failure);
failure_handle = depot_save_stack(&failure, GFP_KERNEL);
}
static noinline void register_dummy_stack(void)
{
unsigned long entries[4];
struct stack_trace dummy;
dummy.nr_entries = 0;
dummy.max_entries = ARRAY_SIZE(entries);
dummy.entries = &entries[0];
dummy.skip = 0;
save_stack_trace(&dummy);
dummy_handle = depot_save_stack(&dummy, GFP_KERNEL);
}
static void wcnss_prealloc_save_stack_trace(struct wcnss_prealloc *entry)
{
struct stack_trace *trace = &entry->trace;
memset(&entry->stack_trace, 0, sizeof(entry->stack_trace));
trace->nr_entries = 0;
trace->max_entries = WCNSS_MAX_STACK_TRACE;
trace->entries = entry->stack_trace;
trace->skip = 2;
save_stack_trace(trace);
return;
}
static int ktap_lib_backtrace(ktap_state *ks)
{
struct stack_trace trace;
ktap_btrace *bt;
bt = kp_percpu_data(KTAP_PERCPU_DATA_BTRACE);
trace.nr_entries = 0;
trace.skip = 10;
trace.max_entries = KTAP_STACK_MAX_ENTRIES;
trace.entries = &bt->entries[0];
save_stack_trace(&trace);
bt->nr_entries = trace.nr_entries;
setbtvalue(ks->top, bt);
incr_top(ks);
return 1;
}
/*
* Save the context of a kmemcheck bug.
*/
void kmemcheck_error_save_bug(struct pt_regs *regs)
{
struct kmemcheck_error *e;
e = error_next_wr();
if (!e)
return;
e->type = KMEMCHECK_ERROR_BUG;
memcpy(&e->regs, regs, sizeof(*regs));
e->trace.nr_entries = 0;
e->trace.entries = e->trace_entries;
e->trace.max_entries = ARRAY_SIZE(e->trace_entries);
e->trace.skip = 1;
save_stack_trace(&e->trace);
tasklet_hi_schedule_first(&kmemcheck_tasklet);
}
static inline depot_stack_handle_t save_stack(gfp_t flags)
{
unsigned long entries[KASAN_STACK_DEPTH];
struct stack_trace trace = {
.nr_entries = 0,
.entries = entries,
.max_entries = KASAN_STACK_DEPTH,
.skip = 0
};
save_stack_trace(&trace);
filter_irq_stacks(&trace);
if (trace.nr_entries != 0 &&
trace.entries[trace.nr_entries-1] == ULONG_MAX)
trace.nr_entries--;
return depot_save_stack(&trace, flags);
}
static inline void set_track(struct kasan_track *track, gfp_t flags)
{
track->pid = current->pid;
track->stack = save_stack(flags);
}
static void backtrace_test_saved(void)
{
struct stack_trace trace;
unsigned long entries[8];
#ifdef CONFIG_DEBUG_PRINTK
printk("Testing a saved backtrace.\n");
#else
;
#endif
#ifdef CONFIG_DEBUG_PRINTK
printk("The following trace is a kernel self test and not a bug!\n");
#else
;
#endif
trace.nr_entries = 0;
trace.max_entries = ARRAY_SIZE(entries);
trace.entries = entries;
trace.skip = 0;
save_stack_trace(&trace);
print_stack_trace(&trace, 0);
}
void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask)
{
struct page_ext *page_ext = lookup_page_ext(page);
struct stack_trace trace = {
.nr_entries = 0,
.max_entries = ARRAY_SIZE(page_ext->trace_entries),
.entries = &page_ext->trace_entries[0],
.skip = 3,
};
save_stack_trace(&trace);
page_ext->order = order;
page_ext->gfp_mask = gfp_mask;
page_ext->nr_entries = trace.nr_entries;
__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
}
static ssize_t
print_page_owner(char __user *buf, size_t count, unsigned long pfn,
struct page *page, struct page_ext *page_ext)
{
int ret;
int pageblock_mt, page_mt;
char *kbuf;
struct stack_trace trace = {
.nr_entries = page_ext->nr_entries,
.entries = &page_ext->trace_entries[0],
};
kbuf = kmalloc(count, GFP_KERNEL);
if (!kbuf)
return -ENOMEM;
ret = snprintf(kbuf, count,
"Page allocated via order %u, mask 0x%x\n",
page_ext->order, page_ext->gfp_mask);
if (ret >= count)
goto err;
/* Print information relevant to grouping pages by mobility */
pageblock_mt = get_pfnblock_migratetype(page, pfn);
page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);
ret += snprintf(kbuf + ret, count - ret,
"PFN %lu Block %lu type %d %s Flags %s%s%s%s%s%s%s%s%s%s%s%s\n",
pfn,
pfn >> pageblock_order,
pageblock_mt,
pageblock_mt != page_mt ? "Fallback" : " ",
PageLocked(page) ? "K" : " ",
PageError(page) ? "E" : " ",
PageReferenced(page) ? "R" : " ",
PageUptodate(page) ? "U" : " ",
PageDirty(page) ? "D" : " ",
PageLRU(page) ? "L" : " ",
PageActive(page) ? "A" : " ",
PageSlab(page) ? "S" : " ",
PageWriteback(page) ? "W" : " ",
PageCompound(page) ? "C" : " ",
PageSwapCache(page) ? "B" : " ",
PageMappedToDisk(page) ? "M" : " ");
if (ret >= count)
goto err;
ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
if (ret >= count)
goto err;
ret += snprintf(kbuf + ret, count - ret, "\n");
if (ret >= count)
goto err;
if (copy_to_user(buf, kbuf, ret))
ret = -EFAULT;
kfree(kbuf);
return ret;
err:
kfree(kbuf);
return -ENOMEM;
}
static ssize_t
read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
unsigned long pfn;
struct page *page;
struct page_ext *page_ext;
if (!page_owner_inited)
return -EINVAL;
page = NULL;
pfn = min_low_pfn + *ppos;
/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
pfn++;
drain_all_pages(NULL);
/* Find an allocated page */
for (; pfn < max_pfn; pfn++) {
/*
* If the new page is in a new MAX_ORDER_NR_PAGES area,
* validate the area as existing, skip it if not
*/
if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
pfn += MAX_ORDER_NR_PAGES - 1;
continue;
}
/* Check for holes within a MAX_ORDER area */
if (!pfn_valid_within(pfn))
continue;
page = pfn_to_page(pfn);
if (PageBuddy(page)) {
unsigned long freepage_order = page_order_unsafe(page);
if (freepage_order < MAX_ORDER)
pfn += (1UL << freepage_order) - 1;
continue;
}
page_ext = lookup_page_ext(page);
/*
* Some pages could be missed by concurrent allocation or free,
* because we don't hold the zone lock.
*/
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
continue;
/* Record the next PFN to read in the file offset */
*ppos = (pfn - min_low_pfn) + 1;
return print_page_owner(buf, count, pfn, page, page_ext);
}
return 0;
}
static noinline void save_stack(struct drm_mm_node *node)
{
unsigned long entries[STACKDEPTH];
struct stack_trace trace = {
.entries = entries,
.max_entries = STACKDEPTH,
.skip = 1
};
save_stack_trace(&trace);
if (trace.nr_entries != 0 &&
trace.entries[trace.nr_entries-1] == ULONG_MAX)
trace.nr_entries--;
/* May be called under spinlock, so avoid sleeping */
node->stack = depot_save_stack(&trace, GFP_NOWAIT);
}
static void show_leaks(struct drm_mm *mm)
{
struct drm_mm_node *node;
unsigned long entries[STACKDEPTH];
char *buf;
buf = kmalloc(BUFSZ, GFP_KERNEL);
if (!buf)
return;
list_for_each_entry(node, drm_mm_nodes(mm), node_list) {
struct stack_trace trace = {
.entries = entries,
.max_entries = STACKDEPTH
};
if (!node->stack) {
DRM_ERROR("node [%08llx + %08llx]: unknown owner\n",
node->start, node->size);
continue;
}
depot_fetch_stack(node->stack, &trace);
snprint_stack_trace(buf, BUFSZ, &trace, 0);
DRM_ERROR("node [%08llx + %08llx]: inserted at\n%s",
node->start, node->size, buf);
}
kfree(buf);
}
#undef STACKDEPTH
#undef BUFSZ
#else
static void save_stack(struct drm_mm_node *node) { }
static void show_leaks(struct drm_mm *mm) { }
#endif
#define START(node) ((node)->start)
#define LAST(node) ((node)->start + (node)->size - 1)
INTERVAL_TREE_DEFINE(struct drm_mm_node, rb,
u64, __subtree_last,
START, LAST, static inline, drm_mm_interval_tree)
struct drm_mm_node *
__drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last)
{
return drm_mm_interval_tree_iter_first((struct rb_root_cached *)&mm->interval_tree,
start, last) ?: (struct drm_mm_node *)&mm->head_node;
}
EXPORT_SYMBOL(__drm_mm_interval_first);
static void drm_mm_interval_tree_add_node(struct drm_mm_node *hole_node,
struct drm_mm_node *node)
{
struct drm_mm *mm = hole_node->mm;
struct rb_node **link, *rb;
struct drm_mm_node *parent;
bool leftmost = true;
node->__subtree_last = LAST(node);
if (hole_node->allocated) {
rb = &hole_node->rb;
while (rb) {
parent = rb_entry(rb, struct drm_mm_node, rb);
if (parent->__subtree_last >= node->__subtree_last)
break;
parent->__subtree_last = node->__subtree_last;
rb = rb_parent(rb);
}
rb = &hole_node->rb;
link = &hole_node->rb.rb_right;
leftmost = false;
} else {
/**
* stack_trace_save - Save a stack trace into a storage array
* @store: Pointer to storage array
* @size: Size of the storage array
* @skipnr: Number of entries to skip at the start of the stack trace
*
* Return: Number of trace entries stored
*/
unsigned int stack_trace_save(unsigned long *store, unsigned int size,
unsigned int skipnr)
{
struct stack_trace trace = {
.entries = store,
.max_entries = size,
.skip = skipnr + 1,
};
save_stack_trace(&trace);
return trace.nr_entries;
}
EXPORT_SYMBOL_GPL(stack_trace_save);
/**
* stack_trace_save_tsk - Save a task stack trace into a storage array
* @task: The task to examine
* @store: Pointer to storage array
* @size: Size of the storage array
* @skipnr: Number of entries to skip at the start of the stack trace
*
* Return: Number of trace entries stored
*/
unsigned int stack_trace_save_tsk(struct task_struct *task,
unsigned long *store, unsigned int size,
unsigned int skipnr)
{
struct stack_trace trace = {
.entries = store,
.max_entries = size,
.skip = skipnr + 1,
};
save_stack_trace_tsk(task, &trace);
return trace.nr_entries;
}
/**
* stack_trace_save_regs - Save a stack trace based on pt_regs into a storage array
* @regs: Pointer to pt_regs to examine
* @store: Pointer to storage array
* @size: Size of the storage array
* @skipnr: Number of entries to skip at the start of the stack trace
*
* Return: Number of trace entries stored
*/
unsigned int stack_trace_save_regs(struct pt_regs *regs, unsigned long *store,
unsigned int size, unsigned int skipnr)
{
struct stack_trace trace = {
.entries = store,
.max_entries = size,
.skip = skipnr,
};
save_stack_trace_regs(regs, &trace);
return trace.nr_entries;
}
#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
/**
* stack_trace_save_tsk_reliable - Save task stack with verification
* @tsk: Pointer to the task to examine
* @store: Pointer to storage array
* @size: Size of the storage array
*
* Return: An error if it detects any unreliable features of the
* stack. Otherwise it guarantees that the stack trace is
* reliable and returns the number of entries stored.
*
* If the task is not 'current', the caller *must* ensure the task is inactive.
*/
int stack_trace_save_tsk_reliable(struct task_struct *tsk, unsigned long *store,
unsigned int size)
{
struct stack_trace trace = {
.entries = store,
.max_entries = size,
};
int ret = save_stack_trace_tsk_reliable(tsk, &trace);
return ret ? ret : trace.nr_entries;
}
#endif
#ifdef CONFIG_USER_STACKTRACE_SUPPORT
/**
* stack_trace_save_user - Save a user space stack trace into a storage array
* @store: Pointer to storage array
* @size: Size of the storage array
*
* Return: Number of trace entries stored
*/
unsigned int stack_trace_save_user(unsigned long *store, unsigned int size)
{
struct stack_trace trace = {
.entries = store,
.max_entries = size,
};
save_stack_trace_user(&trace);
return trace.nr_entries;
}
static noinline depot_stack_handle_t save_stack(gfp_t flags)
{
unsigned long entries[PAGE_OWNER_STACK_DEPTH];
struct stack_trace trace = {
.nr_entries = 0,
.entries = entries,
.max_entries = PAGE_OWNER_STACK_DEPTH,
.skip = 2
};
depot_stack_handle_t handle;
save_stack_trace(&trace);
if (trace.nr_entries != 0 &&
trace.entries[trace.nr_entries-1] == ULONG_MAX)
trace.nr_entries--;
/*
* We need to check recursion here because our request to stackdepot
* could trigger memory allocation to save new entry. New memory
* allocation would reach here and call depot_save_stack() again
* if we don't catch it. There is still not enough memory in stackdepot
* so it would try to allocate memory again and loop forever.
*/
if (check_recursive_alloc(&trace, _RET_IP_))
return dummy_handle;
handle = depot_save_stack(&trace, flags);
if (!handle)
handle = failure_handle;
return handle;
}
static inline void __set_page_owner_handle(struct page_ext *page_ext,
depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)
{
struct page_owner *page_owner;
page_owner = get_page_owner(page_ext);
page_owner->handle = handle;
page_owner->order = order;
page_owner->gfp_mask = gfp_mask;
page_owner->last_migrate_reason = -1;
__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
}
noinline void __set_page_owner(struct page *page, unsigned int order,
gfp_t gfp_mask)
{
struct page_ext *page_ext = lookup_page_ext(page);
depot_stack_handle_t handle;
if (unlikely(!page_ext))
return;
handle = save_stack(gfp_mask);
__set_page_owner_handle(page_ext, handle, order, gfp_mask);
}
void __set_page_owner_migrate_reason(struct page *page, int reason)
{
struct page_ext *page_ext = lookup_page_ext(page);
struct page_owner *page_owner;
if (unlikely(!page_ext))
return;
page_owner = get_page_owner(page_ext);
page_owner->last_migrate_reason = reason;
}
void __split_page_owner(struct page *page, unsigned int order)
{
int i;
struct page_ext *page_ext = lookup_page_ext(page);
struct page_owner *page_owner;
if (unlikely(!page_ext))
return;
page_owner = get_page_owner(page_ext);
page_owner->order = 0;
for (i = 1; i < (1 << order); i++)
__copy_page_owner(page, page + i);
}
void __set_page_owner(struct page *page, unsigned int order, gfp_t gfp_mask)
{
struct page_ext *page_ext = lookup_page_ext(page);
struct stack_trace trace = {
.nr_entries = 0,
.max_entries = ARRAY_SIZE(page_ext->trace_entries),
.entries = &page_ext->trace_entries[0],
.skip = 3,
};
if (unlikely(!page_ext))
return;
save_stack_trace(&trace);
page_ext->order = order;
page_ext->gfp_mask = gfp_mask;
page_ext->nr_entries = trace.nr_entries;
page_ext->last_migrate_reason = -1;
__set_bit(PAGE_EXT_OWNER, &page_ext->flags);
}
void __set_page_owner_migrate_reason(struct page *page, int reason)
{
struct page_ext *page_ext = lookup_page_ext(page);
if (unlikely(!page_ext))
return;
page_ext->last_migrate_reason = reason;
}
gfp_t __get_page_owner_gfp(struct page *page)
{
struct page_ext *page_ext = lookup_page_ext(page);
if (unlikely(!page_ext))
/*
* The caller just returns 0 if no valid gfp
* So return 0 here too.
*/
return 0;
return page_ext->gfp_mask;
}
void __copy_page_owner(struct page *oldpage, struct page *newpage)
{
struct page_ext *old_ext = lookup_page_ext(oldpage);
struct page_ext *new_ext = lookup_page_ext(newpage);
int i;
if (unlikely(!old_ext || !new_ext))
return;
new_ext->order = old_ext->order;
new_ext->gfp_mask = old_ext->gfp_mask;
new_ext->nr_entries = old_ext->nr_entries;
for (i = 0; i < ARRAY_SIZE(new_ext->trace_entries); i++)
new_ext->trace_entries[i] = old_ext->trace_entries[i];
/*
* We don't clear the bit on the oldpage as it's going to be freed
* after migration. Until then, the info can be useful in case of
* a bug, and the overal stats will be off a bit only temporarily.
* Also, migrate_misplaced_transhuge_page() can still fail the
* migration and then we want the oldpage to retain the info. But
* in that case we also don't need to explicitly clear the info from
* the new page, which will be freed.
*/
__set_bit(PAGE_EXT_OWNER, &new_ext->flags);
}
static ssize_t
print_page_owner(char __user *buf, size_t count, unsigned long pfn,
struct page *page, struct page_ext *page_ext)
{
int ret;
int pageblock_mt, page_mt;
char *kbuf;
struct stack_trace trace = {
.nr_entries = page_ext->nr_entries,
.entries = &page_ext->trace_entries[0],
};
kbuf = kmalloc(count, GFP_KERNEL);
if (!kbuf)
return -ENOMEM;
ret = snprintf(kbuf, count,
"Page allocated via order %u, mask %#x(%pGg)\n",
page_ext->order, page_ext->gfp_mask,
&page_ext->gfp_mask);
if (ret >= count)
goto err;
/* Print information relevant to grouping pages by mobility */
pageblock_mt = get_pageblock_migratetype(page);
page_mt = gfpflags_to_migratetype(page_ext->gfp_mask);
ret += snprintf(kbuf + ret, count - ret,
"PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n",
pfn,
migratetype_names[page_mt],
pfn >> pageblock_order,
migratetype_names[pageblock_mt],
page->flags, &page->flags);
if (ret >= count)
goto err;
ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0);
if (ret >= count)
goto err;
if (page_ext->last_migrate_reason != -1) {
ret += snprintf(kbuf + ret, count - ret,
"Page has been migrated, last migrate reason: %s\n",
migrate_reason_names[page_ext->last_migrate_reason]);
if (ret >= count)
goto err;
}
ret += snprintf(kbuf + ret, count - ret, "\n");
if (ret >= count)
goto err;
if (copy_to_user(buf, kbuf, ret))
ret = -EFAULT;
kfree(kbuf);
return ret;
err:
kfree(kbuf);
return -ENOMEM;
}
void __dump_page_owner(struct page *page)
{
struct page_ext *page_ext = lookup_page_ext(page);
struct stack_trace trace = {
.nr_entries = page_ext->nr_entries,
.entries = &page_ext->trace_entries[0],
};
gfp_t gfp_mask = page_ext->gfp_mask;
int mt = gfpflags_to_migratetype(gfp_mask);
if (unlikely(!page_ext)) {
pr_alert("There is not page extension available.\n");
return;
}
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
pr_alert("page_owner info is not active (free page?)\n");
return;
}
pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg)\n",
page_ext->order, migratetype_names[mt], gfp_mask, &gfp_mask);
print_stack_trace(&trace, 0);
if (page_ext->last_migrate_reason != -1)
pr_alert("page has been migrated, last migrate reason: %s\n",
migrate_reason_names[page_ext->last_migrate_reason]);
}
static ssize_t
read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
unsigned long pfn;
struct page *page;
struct page_ext *page_ext;
if (!static_branch_unlikely(&page_owner_inited))
return -EINVAL;
page = NULL;
pfn = min_low_pfn + *ppos;
/* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
pfn++;
drain_all_pages(NULL);
/* Find an allocated page */
for (; pfn < max_pfn; pfn++) {
/*
* If the new page is in a new MAX_ORDER_NR_PAGES area,
* validate the area as existing, skip it if not
*/
if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
pfn += MAX_ORDER_NR_PAGES - 1;
continue;
}
/* Check for holes within a MAX_ORDER area */
if (!pfn_valid_within(pfn))
continue;
page = pfn_to_page(pfn);
if (PageBuddy(page)) {
unsigned long freepage_order = page_order_unsafe(page);
if (freepage_order < MAX_ORDER)
pfn += (1UL << freepage_order) - 1;
continue;
}
page_ext = lookup_page_ext(page);
if (unlikely(!page_ext))
continue;
/*
* Some pages could be missed by concurrent allocation or free,
* because we don't hold the zone lock.
*/
if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
continue;
/* Record the next PFN to read in the file offset */
*ppos = (pfn - min_low_pfn) + 1;
return print_page_owner(buf, count, pfn, page, page_ext);
}
return 0;
}
static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
{
struct page *page;
struct page_ext *page_ext;
unsigned long pfn = zone->zone_start_pfn, block_end_pfn;
unsigned long end_pfn = pfn + zone->spanned_pages;
unsigned long count = 0;
/* Scan block by block. First and last block may be incomplete */
pfn = zone->zone_start_pfn;
/*
* Walk the zone in pageblock_nr_pages steps. If a page block spans
* a zone boundary, it will be double counted between zones. This does
* not matter as the mixed block count will still be correct
*/
for (; pfn < end_pfn; ) {
if (!pfn_valid(pfn)) {
pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
continue;
}
block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
block_end_pfn = min(block_end_pfn, end_pfn);
page = pfn_to_page(pfn);
for (; pfn < block_end_pfn; pfn++) {
if (!pfn_valid_within(pfn))
continue;
page = pfn_to_page(pfn);
if (page_zone(page) != zone)
continue;
/*
* We are safe to check buddy flag and order, because
* this is init stage and only single thread runs.
*/
if (PageBuddy(page)) {
pfn += (1UL << page_order(page)) - 1;
continue;
}
if (PageReserved(page))
continue;
page_ext = lookup_page_ext(page);
if (unlikely(!page_ext))
continue;
/* Maybe overraping zone */
if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
continue;
/* Found early allocated page */
set_page_owner(page, 0, 0);
count++;
}
}
pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
pgdat->node_id, zone->name, count);
}
static void init_zones_in_node(pg_data_t *pgdat)
{
struct zone *zone;
struct zone *node_zones = pgdat->node_zones;
unsigned long flags;
for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
if (!populated_zone(zone))
continue;
spin_lock_irqsave(&zone->lock, flags);
init_pages_in_zone(pgdat, zone);
spin_unlock_irqrestore(&zone->lock, flags);
}
}
static void init_early_allocated_pages(void)
{
pg_data_t *pgdat;
drain_all_pages(NULL);
for_each_online_pgdat(pgdat)
init_zones_in_node(pgdat);
}
