static int __save_stack_trace_reliable(struct stack_trace *trace, struct task_struct *task) { struct unwind_state state; struct pt_regs *regs; unsigned long addr; for (unwind_start(&state, task, NULL, NULL); !unwind_done(&state); unwind_next_frame(&state)) { regs = unwind_get_entry_regs(&state); if (regs) { /* * Kernel mode registers on the stack indicate an * in-kernel interrupt or exception (e.g., preemption * or a page fault), which can make frame pointers * unreliable. */ if (!user_mode(regs)) return -EINVAL; /* * The last frame contains the user mode syscall * pt_regs. Skip it and finish the unwind. */ unwind_next_frame(&state); if (!unwind_done(&state)) { STACKTRACE_DUMP_ONCE(task); return -EINVAL; } break; } addr = unwind_get_return_address(&state); /* * A NULL or invalid return address probably means there's some * generated code which __kernel_text_address() doesn't know * about. */ if (!addr) { STACKTRACE_DUMP_ONCE(task); return -EINVAL; } if (save_stack_address(trace, addr, false)) return -EINVAL; } /* Check for stack corruption */ if (unwind_error(&state)) { STACKTRACE_DUMP_ONCE(task); return -EINVAL; } if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = ULONG_MAX; return 0; }
void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame) { unsigned long *bp; memset(state, 0, sizeof(*state)); state->task = task; state->got_irq = (regs); /* Don't even attempt to start from user mode regs: */ if (regs && user_mode(regs)) { state->stack_info.type = STACK_TYPE_UNKNOWN; return; } bp = get_frame_pointer(task, regs); /* Initialize stack info and make sure the frame data is accessible: */ get_stack_info(bp, state->task, &state->stack_info, &state->stack_mask); update_stack_state(state, bp); /* * The caller can provide the address of the first frame directly * (first_frame) or indirectly (regs->sp) to indicate which stack frame * to start unwinding at. Skip ahead until we reach it. */ while (!unwind_done(state) && (!on_stack(&state->stack_info, first_frame, sizeof(long)) || state->bp < first_frame)) unwind_next_frame(state); }
unsigned long *unwind_get_return_address_ptr(struct unwind_state *state) { if (unwind_done(state)) return NULL; return state->regs ? &state->regs->ip : state->bp + 1; }
unsigned long unwind_get_return_address(struct unwind_state *state) { if (unwind_done(state)) return 0; return __kernel_text_address(state->ip) ? state->ip : 0; }
unsigned long unwind_get_return_address(struct unwind_state *state) { if (unwind_done(state)) return 0; return ftrace_graph_ret_addr(state->task, &state->graph_idx, *state->sp, state->sp); }
unsigned long unwind_get_return_address(struct unwind_state *state) { unsigned long addr; unsigned long *addr_p = unwind_get_return_address_ptr(state); if (unwind_done(state)) return 0; if (state->regs && user_mode(state->regs)) return 0; addr = READ_ONCE_TASK_STACK(state->task, *addr_p); addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr, addr_p); return __kernel_text_address(addr) ? addr : 0; }
bool unwind_next_frame(struct unwind_state *state) { struct stack_info *info = &state->stack_info; if (unwind_done(state)) return false; do { for (state->sp++; state->sp < info->end; state->sp++) if (__kernel_text_address(*state->sp)) return true; state->sp = info->next_sp; } while (!get_stack_info(state->sp, state->task, info, &state->stack_mask)); return false; }
void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame) { unsigned long *bp, *frame; size_t len; memset(state, 0, sizeof(*state)); state->task = task; /* don't even attempt to start from user mode regs */ if (regs && user_mode(regs)) { state->stack_info.type = STACK_TYPE_UNKNOWN; return; } /* set up the starting stack frame */ bp = get_frame_pointer(task, regs); regs = decode_frame_pointer(bp); if (regs) { state->regs = regs; frame = (unsigned long *)regs; len = sizeof(*regs); } else { state->bp = bp; frame = bp; len = FRAME_HEADER_SIZE; } /* initialize stack info and make sure the frame data is accessible */ get_stack_info(frame, state->task, &state->stack_info, &state->stack_mask); update_stack_state(state, frame, len); /* * The caller can provide the address of the first frame directly * (first_frame) or indirectly (regs->sp) to indicate which stack frame * to start unwinding at. Skip ahead until we reach it. */ while (!unwind_done(state) && (!on_stack(&state->stack_info, first_frame, sizeof(long)) || state->bp < first_frame)) unwind_next_frame(state); }
static void __save_stack_trace(struct stack_trace *trace, struct task_struct *task, struct pt_regs *regs, bool nosched) { struct unwind_state state; unsigned long addr; if (regs) save_stack_address(trace, regs->ip, nosched); for (unwind_start(&state, task, regs, NULL); !unwind_done(&state); unwind_next_frame(&state)) { addr = unwind_get_return_address(&state); if (!addr || save_stack_address(trace, addr, nosched)) break; } if (trace->nr_entries < trace->max_entries) trace->entries[trace->nr_entries++] = ULONG_MAX; }
void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame) { memset(state, 0, sizeof(*state)); state->task = task; state->sp = first_frame; get_stack_info(first_frame, state->task, &state->stack_info, &state->stack_mask); /* * The caller can provide the address of the first frame directly * (first_frame) or indirectly (regs->sp) to indicate which stack frame * to start unwinding at. Skip ahead until we reach it. */ if (!unwind_done(state) && (!on_stack(&state->stack_info, first_frame, sizeof(long)) || !__kernel_text_address(*first_frame))) unwind_next_frame(state); }
unsigned long unwind_get_return_address(struct unwind_state *state) { unsigned long addr; unsigned long *addr_p = unwind_get_return_address_ptr(state); if (unwind_done(state)) return 0; if (state->regs && user_mode(state->regs)) return 0; addr = ftrace_graph_ret_addr(state->task, &state->graph_idx, *addr_p, addr_p); if (!__kernel_text_address(addr)) { printk_deferred_once(KERN_WARNING "WARNING: unrecognized kernel stack return address %p at %p in %s:%d\n", (void *)addr, addr_p, state->task->comm, state->task->pid); return 0; } return addr; }
bool unwind_next_frame(struct unwind_state *state) { struct pt_regs *regs; unsigned long *next_bp, *next_frame; size_t next_len; enum stack_type prev_type = state->stack_info.type; if (unwind_done(state)) return false; /* have we reached the end? */ if (state->regs && user_mode(state->regs)) goto the_end; if (is_last_task_frame(state)) { regs = task_pt_regs(state->task); /* * kthreads (other than the boot CPU's idle thread) have some * partial regs at the end of their stack which were placed * there by copy_thread_tls(). But the regs don't have any * useful information, so we can skip them. * * This user_mode() check is slightly broader than a PF_KTHREAD * check because it also catches the awkward situation where a * newly forked kthread transitions into a user task by calling * do_execve(), which eventually clears PF_KTHREAD. */ if (!user_mode(regs)) goto the_end; /* * We're almost at the end, but not quite: there's still the * syscall regs frame. Entry code doesn't encode the regs * pointer for syscalls, so we have to set it manually. */ state->regs = regs; state->bp = NULL; return true; } /* get the next frame pointer */ if (state->regs) next_bp = (unsigned long *)state->regs->bp; else next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task,*state->bp); /* is the next frame pointer an encoded pointer to pt_regs? */ regs = decode_frame_pointer(next_bp); if (regs) { next_frame = (unsigned long *)regs; next_len = sizeof(*regs); } else { next_frame = next_bp; next_len = FRAME_HEADER_SIZE; } /* make sure the next frame's data is accessible */ if (!update_stack_state(state, next_frame, next_len)) { /* * Don't warn on bad regs->bp. An interrupt in entry code * might cause a false positive warning. */ if (state->regs) goto the_end; goto bad_address; } /* Make sure it only unwinds up and doesn't overlap the last frame: */ if (state->stack_info.type == prev_type) { if (state->regs && (void *)next_frame < (void *)state->regs + regs_size(state->regs)) goto bad_address; if (state->bp && (void *)next_frame < (void *)state->bp + FRAME_HEADER_SIZE) goto bad_address; } /* move to the next frame */ if (regs) { state->regs = regs; state->bp = NULL; } else { state->bp = next_bp; state->regs = NULL; } return true; bad_address: /* * When unwinding a non-current task, the task might actually be * running on another CPU, in which case it could be modifying its * stack while we're reading it. This is generally not a problem and * can be ignored as long as the caller understands that unwinding * another task will not always succeed. */ if (state->task != current) goto the_end; if (state->regs) { printk_deferred_once(KERN_WARNING "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", state->regs, state->task->comm, state->task->pid, next_frame); unwind_dump(state, (unsigned long *)state->regs); } else { printk_deferred_once(KERN_WARNING "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n", state->bp, state->task->comm, state->task->pid, next_frame); unwind_dump(state, state->bp); } the_end: state->stack_info.type = STACK_TYPE_UNKNOWN; return false; }
bool unwind_next_frame(struct unwind_state *state) { struct pt_regs *regs; unsigned long *next_bp; if (unwind_done(state)) return false; /* Have we reached the end? */ if (state->regs && user_mode(state->regs)) goto the_end; if (is_last_task_frame(state)) { regs = task_pt_regs(state->task); /* * kthreads (other than the boot CPU's idle thread) have some * partial regs at the end of their stack which were placed * there by copy_thread_tls(). But the regs don't have any * useful information, so we can skip them. * * This user_mode() check is slightly broader than a PF_KTHREAD * check because it also catches the awkward situation where a * newly forked kthread transitions into a user task by calling * do_execve(), which eventually clears PF_KTHREAD. */ if (!user_mode(regs)) goto the_end; /* * We're almost at the end, but not quite: there's still the * syscall regs frame. Entry code doesn't encode the regs * pointer for syscalls, so we have to set it manually. */ state->regs = regs; state->bp = NULL; state->ip = 0; return true; } /* Get the next frame pointer: */ if (state->regs) next_bp = (unsigned long *)state->regs->bp; else next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp); /* Move to the next frame if it's safe: */ if (!update_stack_state(state, next_bp)) goto bad_address; return true; bad_address: state->error = true; /* * When unwinding a non-current task, the task might actually be * running on another CPU, in which case it could be modifying its * stack while we're reading it. This is generally not a problem and * can be ignored as long as the caller understands that unwinding * another task will not always succeed. */ if (state->task != current) goto the_end; /* * Don't warn if the unwinder got lost due to an interrupt in entry * code or in the C handler before the first frame pointer got set up: */ if (state->got_irq && in_entry_code(state->ip)) goto the_end; if (state->regs && state->regs->sp >= (unsigned long)last_aligned_frame(state) && state->regs->sp < (unsigned long)task_pt_regs(state->task)) goto the_end; if (state->regs) { printk_deferred_once(KERN_WARNING "WARNING: kernel stack regs at %p in %s:%d has bad 'bp' value %p\n", state->regs, state->task->comm, state->task->pid, next_bp); unwind_dump(state); } else { printk_deferred_once(KERN_WARNING "WARNING: kernel stack frame pointer at %p in %s:%d has bad value %p\n", state->bp, state->task->comm, state->task->pid, next_bp); unwind_dump(state); } the_end: state->stack_info.type = STACK_TYPE_UNKNOWN; return false; }