/*
* callback from kmsg_dump. (s2,l2) has the most recently
* written bytes, older bytes are in (s1,l1). Save as much
* as we can from the end of the buffer.
*/
static void pstore_dump(struct kmsg_dumper *dumper,
enum kmsg_dump_reason reason,
const char *s1, unsigned long l1,
const char *s2, unsigned long l2)
{
unsigned long s1_start, s2_start;
unsigned long l1_cpy, l2_cpy;
unsigned long size, total = 0;
char *dst;
const char *why;
u64 id;
int hsize, ret;
unsigned int part = 1;
unsigned long flags = 0;
int is_locked = 0;
why = get_reason_str(reason);
if (in_nmi()) {
is_locked = spin_trylock(&psinfo->buf_lock);
if (!is_locked)
pr_err("pstore dump routine blocked in NMI, may corrupt error record\n");
} else
spin_lock_irqsave(&psinfo->buf_lock, flags);
oopscount++;
while (total < kmsg_bytes) {
dst = psinfo->buf;
hsize = sprintf(dst, "%s#%d Part%d\n", why, oopscount, part);
size = psinfo->bufsize - hsize;
dst += hsize;
l2_cpy = min(l2, size);
l1_cpy = min(l1, size - l2_cpy);
if (l1_cpy + l2_cpy == 0)
break;
s2_start = l2 - l2_cpy;
s1_start = l1 - l1_cpy;
memcpy(dst, s1 + s1_start, l1_cpy);
memcpy(dst + l1_cpy, s2 + s2_start, l2_cpy);
ret = psinfo->write(PSTORE_TYPE_DMESG, reason, &id, part,
oopscount, hsize + l1_cpy + l2_cpy, psinfo);
if (ret == 0 && reason == KMSG_DUMP_OOPS && pstore_is_mounted())
pstore_new_entry = 1;
l1 -= l1_cpy;
l2 -= l2_cpy;
total += l1_cpy + l2_cpy;
part++;
}
if (in_nmi()) {
if (is_locked)
spin_unlock(&psinfo->buf_lock);
} else
spin_unlock_irqrestore(&psinfo->buf_lock, flags);
}
static int erst_exec_stall_while_true(struct apei_exec_context *ctx,
struct acpi_whea_header *entry)
{
int rc;
u64 val;
u64 timeout = FIRMWARE_TIMEOUT;
u64 stall_time;
if (ctx->var1 > FIRMWARE_MAX_STALL) {
if (!in_nmi())
pr_warn(FW_WARN
"Too long stall time for stall while true instruction: 0x%llx.\n",
ctx->var1);
stall_time = FIRMWARE_MAX_STALL;
} else
stall_time = ctx->var1;
for (;;) {
rc = __apei_exec_read_register(entry, &val);
if (rc)
return rc;
if (val != ctx->value)
break;
if (erst_timedout(&timeout, stall_time * NSEC_PER_USEC))
return -EIO;
}
return 0;
}
static void inline procfile_context_print(void)
{
printk_d("preemptible 0x%x\n", preemptible());
printk_d("in_atomic_preempt_off 0x%x\n", in_atomic_preempt_off());
printk_d("in_atomic 0x%x\n", in_atomic());
printk_d("in_nmi 0x%lx\n", in_nmi());
printk_d("in_serving_softirq 0x%lx\n", in_serving_softirq());
printk_d("in_interrupt 0x%lx\n", in_interrupt());
printk_d("in_softirq 0x%lx\n", in_softirq());
printk_d("in_irq 0x%lx\n", in_irq());
printk_d("preempt_count 0x%x\n", preempt_count());
printk_d("irqs_disabled 0x%x\n", irqs_disabled());
if(current) {
printk_d("task->comm %s\n", current->comm);
printk_d("task->flags 0x%x\n", current->flags);
printk_d("task->state %lu\n", current->state);
printk_d("task->usage %d\n", atomic_read(&(current->usage)));
printk_d("task->prio %d\n", current->prio);
printk_d("task->static_prio %d\n", current->static_prio);
printk_d("task->normal_prio %d\n", current->normal_prio);
printk_d("task->rt_priority %d\n", current->rt_priority);
printk_d("task->policy %d\n", current->policy);
printk_d("task->pid %d\n", current->pid);
printk_d("task->tgid %d\n", current->tgid);
}
else
printk_d("task pointer NULL\n");
}
/*
* best effort, GUP based copy_from_user() that assumes IRQ or NMI context
*/
static unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
{
unsigned long offset, addr = (unsigned long)from;
int type = in_nmi() ? KM_NMI : KM_IRQ0;
unsigned long size, len = 0;
struct page *page;
void *map;
int ret;
do {
ret = __get_user_pages_fast(addr, 1, 0, &page);
if (!ret)
break;
offset = addr & (PAGE_SIZE - 1);
size = min(PAGE_SIZE - offset, n - len);
map = kmap_atomic(page, type);
memcpy(to, map+offset, size);
kunmap_atomic(map, type);
put_page(page);
len += size;
to += size;
addr += size;
} while (len < n);
return len;
}
bool kmemcheck_fault(struct pt_regs *regs, unsigned long address,
unsigned long error_code)
{
pte_t *pte;
/*
* XXX: Is it safe to assume that memory accesses from virtual 86
* mode or non-kernel code segments will _never_ access kernel
* memory (e.g. tracked pages)? For now, we need this to avoid
* invoking kmemcheck for PnP BIOS calls.
*/
if (regs->flags & X86_VM_MASK)
return false;
if (regs->cs != __KERNEL_CS)
return false;
pte = kmemcheck_pte_lookup(address);
if (!pte)
return false;
WARN_ON_ONCE(in_nmi());
if (error_code & 2)
kmemcheck_access(regs, address, KMEMCHECK_WRITE);
else
kmemcheck_access(regs, address, KMEMCHECK_READ);
kmemcheck_show(regs);
return true;
}
/**
* trace_call_bpf - invoke BPF program
* @prog: BPF program
* @ctx: opaque context pointer
*
* kprobe handlers execute BPF programs via this helper.
* Can be used from static tracepoints in the future.
*
* Return: BPF programs always return an integer which is interpreted by
* kprobe handler as:
* 0 - return from kprobe (event is filtered out)
* 1 - store kprobe event into ring buffer
* Other values are reserved and currently alias to 1
*/
unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx)
{
unsigned int ret;
if (in_nmi()) /* not supported yet */
return 1;
preempt_disable();
if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
/*
* since some bpf program is already running on this cpu,
* don't call into another bpf program (same or different)
* and don't send kprobe event into ring-buffer,
* so return zero here
*/
ret = 0;
goto out;
}
rcu_read_lock();
ret = BPF_PROG_RUN(prog, ctx);
rcu_read_unlock();
out:
__this_cpu_dec(bpf_prog_active);
preempt_enable();
return ret;
}
u64 notrace trace_clock_global(void)
{
unsigned long flags;
int this_cpu;
u64 now;
local_irq_save(flags);
this_cpu = raw_smp_processor_id();
now = cpu_clock(this_cpu);
if (unlikely(in_nmi()))
goto out;
arch_spin_lock(&trace_clock_struct.lock);
if ((s64)(now - trace_clock_struct.prev_time) < 0)
now = trace_clock_struct.prev_time + 1;
trace_clock_struct.prev_time = now;
arch_spin_unlock(&trace_clock_struct.lock);
out:
local_irq_restore(flags);
return now;
}
static void prof_syscall_exit(struct pt_regs *regs, long ret)
{
struct syscall_metadata *sys_data;
struct syscall_trace_exit *rec;
unsigned long flags;
int syscall_nr;
char *raw_data;
int size;
int cpu;
syscall_nr = syscall_get_nr(current, regs);
if (!test_bit(syscall_nr, enabled_prof_exit_syscalls))
return;
sys_data = syscall_nr_to_meta(syscall_nr);
if (!sys_data)
return;
/* We can probably do that at build time */
size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
/*
* Impossible, but be paranoid with the future
* How to put this check outside runtime?
*/
if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
"exit event has grown above profile buffer size"))
return;
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
cpu = smp_processor_id();
if (in_nmi())
raw_data = rcu_dereference(trace_profile_buf_nmi);
else
raw_data = rcu_dereference(trace_profile_buf);
if (!raw_data)
goto end;
raw_data = per_cpu_ptr(raw_data, cpu);
/* zero the dead bytes from align to not leak stack to user */
*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
rec = (struct syscall_trace_exit *)raw_data;
tracing_generic_entry_update(&rec->ent, 0, 0);
rec->ent.type = sys_data->exit_id;
rec->nr = syscall_nr;
rec->ret = syscall_get_return_value(current, regs);
perf_tp_event(sys_data->exit_id, 0, 1, rec, size);
end:
local_irq_restore(flags);
}
u64 sched_clock_cpu(int cpu)
{
u64 now, clock, this_clock, remote_clock;
struct sched_clock_data *scd;
if (sched_clock_stable)
return sched_clock();
scd = cpu_sdc(cpu);
/*
* Normally this is not called in NMI context - but if it is,
* trying to do any locking here is totally lethal.
*/
if (unlikely(in_nmi()))
return scd->clock;
if (unlikely(!sched_clock_running))
return 0ull;
WARN_ON_ONCE(!irqs_disabled());
now = sched_clock();
if (cpu != raw_smp_processor_id()) {
struct sched_clock_data *my_scd = this_scd();
lock_double_clock(scd, my_scd);
this_clock = __update_sched_clock(my_scd, now);
remote_clock = scd->clock;
/*
* Use the opportunity that we have both locks
* taken to couple the two clocks: we take the
* larger time as the latest time for both
* runqueues. (this creates monotonic movement)
*/
if (likely((s64)(remote_clock - this_clock) < 0)) {
clock = this_clock;
scd->clock = clock;
} else {
/*
* Should be rare, but possible:
*/
clock = remote_clock;
my_scd->clock = remote_clock;
}
__raw_spin_unlock(&my_scd->lock);
} else {
__raw_spin_lock(&scd->lock);
clock = __update_sched_clock(scd, now);
}
__raw_spin_unlock(&scd->lock);
return clock;
}
static void prof_syscall_enter(struct pt_regs *regs, long id)
{
struct syscall_metadata *sys_data;
struct syscall_trace_enter *rec;
unsigned long flags;
char *raw_data;
int syscall_nr;
int size;
int cpu;
syscall_nr = syscall_get_nr(current, regs);
if (!test_bit(syscall_nr, enabled_prof_enter_syscalls))
return;
sys_data = syscall_nr_to_meta(syscall_nr);
if (!sys_data)
return;
/* get the size after alignment with the u32 buffer size field */
size = sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
size = ALIGN(size + sizeof(u32), sizeof(u64));
size -= sizeof(u32);
if (WARN_ONCE(size > FTRACE_MAX_PROFILE_SIZE,
"profile buffer not large enough"))
return;
/* Protect the per cpu buffer, begin the rcu read side */
local_irq_save(flags);
cpu = smp_processor_id();
if (in_nmi())
raw_data = rcu_dereference(trace_profile_buf_nmi);
else
raw_data = rcu_dereference(trace_profile_buf);
if (!raw_data)
goto end;
raw_data = per_cpu_ptr(raw_data, cpu);
/* zero the dead bytes from align to not leak stack to user */
*(u64 *)(&raw_data[size - sizeof(u64)]) = 0ULL;
rec = (struct syscall_trace_enter *) raw_data;
tracing_generic_entry_update(&rec->ent, 0, 0);
rec->ent.type = sys_data->enter_id;
rec->nr = syscall_nr;
syscall_get_arguments(current, regs, 0, sys_data->nb_args,
(unsigned long *)&rec->args);
perf_tp_event(sys_data->enter_id, 0, 1, rec, size);
end:
local_irq_restore(flags);
}
__visible void trace_hardirqs_off_caller(unsigned long caller_addr)
{
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, caller_addr);
if (!in_nmi())
trace_irq_disable_rcuidle(CALLER_ADDR0, caller_addr);
}
lockdep_hardirqs_off(CALLER_ADDR0);
}
void trace_hardirqs_off(void)
{
if (!this_cpu_read(tracing_irq_cpu)) {
this_cpu_write(tracing_irq_cpu, 1);
tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1);
if (!in_nmi())
trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
}
lockdep_hardirqs_off(CALLER_ADDR0);
}
void trace_hardirqs_on(void)
{
if (this_cpu_read(tracing_irq_cpu)) {
if (!in_nmi())
trace_irq_enable_rcuidle(CALLER_ADDR0, CALLER_ADDR1);
tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1);
this_cpu_write(tracing_irq_cpu, 0);
}
lockdep_hardirqs_on(CALLER_ADDR0);
}
/*
* Function to get the timestamp.
* FIXME: import the goodness from the LTTng trace clock
*/
static inline u64 trace_clock_monotonic_wrapper(void)
{
ktime_t ktime;
/*
* Refuse to trace from NMIs with this wrapper, because an NMI could
* nest over the xtime write seqlock and deadlock.
*/
if (in_nmi())
return (u64) -EIO;
ktime = ktime_get();
return ktime_to_ns(ktime);
}
/*
* callback from kmsg_dump. (s2,l2) has the most recently
* written bytes, older bytes are in (s1,l1). Save as much
* as we can from the end of the buffer.
*/
static void pstore_dump(struct kmsg_dumper *dumper,
enum kmsg_dump_reason reason)
{
unsigned long total = 0;
const char *why;
u64 id;
unsigned int part = 1;
unsigned long flags = 0;
int is_locked = 0;
int ret;
why = get_reason_str(reason);
if (pstore_cannot_block_path(reason)) {
is_locked = spin_trylock_irqsave(&psinfo->buf_lock, flags);
if (!is_locked) {
pr_err("pstore dump routine blocked in %s path, may corrupt error record\n"
, in_nmi() ? "NMI" : why);
}
} else
spin_lock_irqsave(&psinfo->buf_lock, flags);
oopscount++;
while (total < kmsg_bytes) {
char *dst;
unsigned long size;
int hsize;
size_t len;
dst = psinfo->buf;
hsize = sprintf(dst, "%s#%d Part%d\n", why, oopscount, part);
size = psinfo->bufsize - hsize;
dst += hsize;
if (!kmsg_dump_get_buffer(dumper, true, dst, size, &len))
break;
ret = psinfo->write(PSTORE_TYPE_DMESG, reason, &id, part,
oopscount, hsize + len, psinfo);
if (ret == 0 && reason == KMSG_DUMP_OOPS && pstore_is_mounted())
pstore_new_entry = 1;
total += hsize + len;
part++;
}
if (pstore_cannot_block_path(reason)) {
if (is_locked)
spin_unlock_irqrestore(&psinfo->buf_lock, flags);
} else
spin_unlock_irqrestore(&psinfo->buf_lock, flags);
}
static int erst_exec_stall(struct apei_exec_context *ctx,
struct acpi_whea_header *entry)
{
u64 stall_time;
if (ctx->value > FIRMWARE_MAX_STALL) {
if (!in_nmi())
pr_warn(FW_WARN
"Too long stall time for stall instruction: 0x%llx.\n",
ctx->value);
stall_time = FIRMWARE_MAX_STALL;
} else
stall_time = ctx->value;
udelay(stall_time);
return 0;
}
/*
* Should pstore_dump() wait for a concurrent pstore_dump()? If
* not, the current pstore_dump() will report a failure to dump
* and return.
*/
static bool pstore_cannot_wait(enum kmsg_dump_reason reason)
{
/* In NMI path, pstore shouldn't block regardless of reason. */
if (in_nmi())
return true;
switch (reason) {
/* In panic case, other cpus are stopped by smp_send_stop(). */
case KMSG_DUMP_PANIC:
/* Emergency restart shouldn't be blocked. */
case KMSG_DUMP_EMERG:
return true;
default:
return false;
}
}
bool rcu_lockdep_current_cpu_online(void)
{
struct rcu_data *rdp;
struct rcu_node *rnp;
bool ret;
if (in_nmi())
return 1;
preempt_disable();
rdp = &__get_cpu_var(rcu_sched_data);
rnp = rdp->mynode;
ret = (rdp->grpmask & rnp->qsmaskinit) ||
!rcu_scheduler_fully_active;
preempt_enable();
return ret;
}
/*
* Enqueue the irq_work @work on @cpu unless it's already pending
* somewhere.
*
* Can be re-enqueued while the callback is still in progress.
*/
bool irq_work_queue_on(struct irq_work *work, int cpu)
{
/* All work should have been flushed before going offline */
WARN_ON_ONCE(cpu_is_offline(cpu));
/* Arch remote IPI send/receive backend aren't NMI safe */
WARN_ON_ONCE(in_nmi());
/* Only queue if not already pending */
if (!irq_work_claim(work))
return false;
if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
arch_send_call_function_single_ipi(cpu);
return true;
}
/*
* We can receive a page fault from a migrating PTE at any time.
* Handle it by just waiting until the fault resolves.
*
* It's also possible to get a migrating kernel PTE that resolves
* itself during the downcall from hypervisor to Linux. We just check
* here to see if the PTE seems valid, and if so we retry it.
*
* NOTE! We MUST NOT take any locks for this case. We may be in an
* interrupt or a critical region, and must do as little as possible.
* Similarly, we can't use atomic ops here, since we may be handling a
* fault caused by an atomic op access.
*
* If we find a migrating PTE while we're in an NMI context, and we're
* at a PC that has a registered exception handler, we don't wait,
* since this thread may (e.g.) have been interrupted while migrating
* its own stack, which would then cause us to self-deadlock.
*/
static int handle_migrating_pte(pgd_t *pgd, int fault_num,
unsigned long address, unsigned long pc,
int is_kernel_mode, int write)
{
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
pte_t pteval;
if (pgd_addr_invalid(address))
return 0;
pgd += pgd_index(address);
pud = pud_offset(pgd, address);
if (!pud || !pud_present(*pud))
return 0;
pmd = pmd_offset(pud, address);
if (!pmd || !pmd_present(*pmd))
return 0;
pte = pmd_huge_page(*pmd) ? ((pte_t *)pmd) :
pte_offset_kernel(pmd, address);
pteval = *pte;
if (pte_migrating(pteval)) {
if (in_nmi() && search_exception_tables(pc))
return 0;
wait_for_migration(pte);
return 1;
}
if (!is_kernel_mode || !pte_present(pteval))
return 0;
if (fault_num == INT_ITLB_MISS) {
if (pte_exec(pteval))
return 1;
} else if (write) {
if (pte_write(pteval))
return 1;
} else {
if (pte_read(pteval))
return 1;
}
return 0;
}
/**
* trace_call_bpf - invoke BPF program
* @call: tracepoint event
* @ctx: opaque context pointer
*
* kprobe handlers execute BPF programs via this helper.
* Can be used from static tracepoints in the future.
*
* Return: BPF programs always return an integer which is interpreted by
* kprobe handler as:
* 0 - return from kprobe (event is filtered out)
* 1 - store kprobe event into ring buffer
* Other values are reserved and currently alias to 1
*/
unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
{
unsigned int ret;
if (in_nmi()) /* not supported yet */
return 1;
preempt_disable();
if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
/*
* since some bpf program is already running on this cpu,
* don't call into another bpf program (same or different)
* and don't send kprobe event into ring-buffer,
* so return zero here
*/
ret = 0;
goto out;
}
/*
* Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
* to all call sites, we did a bpf_prog_array_valid() there to check
* whether call->prog_array is empty or not, which is
* a heurisitc to speed up execution.
*
* If bpf_prog_array_valid() fetched prog_array was
* non-NULL, we go into trace_call_bpf() and do the actual
* proper rcu_dereference() under RCU lock.
* If it turns out that prog_array is NULL then, we bail out.
* For the opposite, if the bpf_prog_array_valid() fetched pointer
* was NULL, you'll skip the prog_array with the risk of missing
* out of events when it was updated in between this and the
* rcu_dereference() which is accepted risk.
*/
ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN);
out:
__this_cpu_dec(bpf_prog_active);
preempt_enable();
return ret;
}
/*
* dump a block of kernel memory from around the given address
*/
static void show_data(unsigned long addr, int nbytes, const char *name)
{
int i, j;
int nlines;
u32 *p;
/*
* don't attempt to dump non-kernel addresses or
* values that are probably just small negative numbers
*/
if (addr < PAGE_OFFSET || addr > -256UL)
return;
printk("\n%s: %#lx:\n", name, addr);
/*
* round address down to a 32 bit boundary
* and always dump a multiple of 32 bytes
*/
p = (u32 *)(addr & ~(sizeof(u32) - 1));
nbytes += (addr & (sizeof(u32) - 1));
nlines = (nbytes + 31) / 32;
for (i = 0; i < nlines; i++) {
/*
* just display low 16 bits of address to keep
* each line of the dump < 80 characters
*/
printk("%04lx ", (unsigned long)p & 0xffff);
for (j = 0; j < 8; j++) {
u32 data;
if (in_nmi() || probe_kernel_address(p, data)) {
printk(" ********");
} else {
printk(" %08x", data);
}
++p;
}
printk("\n");
}
}
/*
* Hook the return address and push it in the stack of return addresses
* in current thread info.
*/
unsigned long prepare_ftrace_return(unsigned long ip, unsigned long parent)
{
struct ftrace_graph_ent trace;
/* Nmi's are currently unsupported. */
if (unlikely(in_nmi()))
goto out;
if (unlikely(atomic_read(¤t->tracing_graph_pause)))
goto out;
if (ftrace_push_return_trace(parent, ip, &trace.depth, 0) == -EBUSY)
goto out;
trace.func = ftrace_mcount_call_adjust(ip) & PSW_ADDR_INSN;
/* Only trace if the calling function expects to. */
if (!ftrace_graph_entry(&trace)) {
current->curr_ret_stack--;
goto out;
}
parent = (unsigned long)return_to_handler;
out:
return parent;
}
void gr_handle_kernel_exploit(void)
{
#ifdef CONFIG_GRKERNSEC_KERN_LOCKOUT
const struct cred *cred;
struct task_struct *tsk, *tsk2;
struct user_struct *user;
uid_t uid;
if (in_irq() || in_serving_softirq() || in_nmi())
panic("grsec: halting the system due to suspicious kernel crash caused in interrupt context");
uid = current_uid();
if (uid == 0)
panic("grsec: halting the system due to suspicious kernel crash caused by root");
else {
/* kill all the processes of this user, hold a reference
to their creds struct, and prevent them from creating
another process until system reset
*/
printk(KERN_ALERT "grsec: banning user with uid %u until system restart for suspicious kernel crash\n", uid);
/* we intentionally leak this ref */
user = get_uid(current->cred->user);
if (user) {
user->banned = 1;
user->ban_expires = ~0UL;
}
read_lock(&tasklist_lock);
do_each_thread(tsk2, tsk) {
cred = __task_cred(tsk);
if (cred->uid == uid)
gr_fake_force_sig(SIGKILL, tsk);
} while_each_thread(tsk2, tsk);
read_unlock(&tasklist_lock);
}
u64 notrace trace_clock_global(void)
{
unsigned long flags;
int this_cpu;
u64 now;
local_irq_save(flags);
this_cpu = raw_smp_processor_id();
now = cpu_clock(this_cpu);
/*
* If in an NMI context then dont risk lockups and return the
* cpu_clock() time:
*/
if (unlikely(in_nmi()))
goto out;
arch_spin_lock(&trace_clock_struct.lock);
/*
* TODO: if this happens often then maybe we should reset
* my_scd->clock to prev_time+1, to make sure
* we start ticking with the local clock from now on?
*/
if ((s64)(now - trace_clock_struct.prev_time) < 0)
now = trace_clock_struct.prev_time + 1;
trace_clock_struct.prev_time = now;
arch_spin_unlock(&trace_clock_struct.lock);
out:
local_irq_restore(flags);
return now;
}
int kdb_stub(struct kgdb_state *ks)
{
int error = 0;
kdb_bp_t *bp;
unsigned long addr = kgdb_arch_pc(ks->ex_vector, ks->linux_regs);
kdb_reason_t reason = KDB_REASON_OOPS;
kdb_dbtrap_t db_result = KDB_DB_NOBPT;
int i;
kdb_ks = ks;
if (KDB_STATE(REENTRY)) {
reason = KDB_REASON_SWITCH;
KDB_STATE_CLEAR(REENTRY);
addr = instruction_pointer(ks->linux_regs);
}
ks->pass_exception = 0;
if (atomic_read(&kgdb_setting_breakpoint))
reason = KDB_REASON_KEYBOARD;
if (in_nmi())
reason = KDB_REASON_NMI;
for (i = 0, bp = kdb_breakpoints; i < KDB_MAXBPT; i++, bp++) {
if ((bp->bp_enabled) && (bp->bp_addr == addr)) {
reason = KDB_REASON_BREAK;
db_result = KDB_DB_BPT;
if (addr != instruction_pointer(ks->linux_regs))
kgdb_arch_set_pc(ks->linux_regs, addr);
break;
}
}
if (reason == KDB_REASON_BREAK || reason == KDB_REASON_SWITCH) {
for (i = 0, bp = kdb_breakpoints; i < KDB_MAXBPT; i++, bp++) {
if (bp->bp_free)
continue;
if (bp->bp_addr == addr) {
bp->bp_delay = 1;
bp->bp_delayed = 1;
/*
* SSBPT is set when the kernel debugger must single step a
* task in order to re-establish an instruction breakpoint
* which uses the instruction replacement mechanism. It is
* cleared by any action that removes the need to single-step
* the breakpoint.
*/
reason = KDB_REASON_BREAK;
db_result = KDB_DB_BPT;
KDB_STATE_SET(SSBPT);
break;
}
}
}
if (reason != KDB_REASON_BREAK && ks->ex_vector == 0 &&
ks->signo == SIGTRAP) {
reason = KDB_REASON_SSTEP;
db_result = KDB_DB_BPT;
}
/* Set initial kdb state variables */
KDB_STATE_CLEAR(KGDB_TRANS);
kdb_common_init_state(ks);
/* Remove any breakpoints as needed by kdb and clear single step */
kdb_bp_remove();
KDB_STATE_CLEAR(DOING_SS);
KDB_STATE_SET(PAGER);
/* zero out any offline cpu data */
for_each_present_cpu(i) {
if (!cpu_online(i)) {
kgdb_info[i].debuggerinfo = NULL;
kgdb_info[i].task = NULL;
}
}
if (ks->err_code == DIE_OOPS || reason == KDB_REASON_OOPS) {
ks->pass_exception = 1;
KDB_FLAG_SET(CATASTROPHIC);
}
if (KDB_STATE(SSBPT) && reason == KDB_REASON_SSTEP) {
KDB_STATE_CLEAR(SSBPT);
KDB_STATE_CLEAR(DOING_SS);
} else {
/* Start kdb main loop */
error = kdb_main_loop(KDB_REASON_ENTER, reason,
ks->err_code, db_result, ks->linux_regs);
}
/*
* Upon exit from the kdb main loop setup break points and restart
* the system based on the requested continue state
*/
kdb_common_deinit_state();
KDB_STATE_CLEAR(PAGER);
kdbnearsym_cleanup();
if (error == KDB_CMD_KGDB) {
if (KDB_STATE(DOING_KGDB))
KDB_STATE_CLEAR(DOING_KGDB);
return DBG_PASS_EVENT;
}
kdb_bp_install(ks->linux_regs);
dbg_activate_sw_breakpoints();
/* Set the exit state to a single step or a continue */
if (KDB_STATE(DOING_SS))
gdbstub_state(ks, "s");
else
gdbstub_state(ks, "c");
KDB_FLAG_CLEAR(CATASTROPHIC);
/* Invoke arch specific exception handling prior to system resume */
kgdb_info[ks->cpu].ret_state = gdbstub_state(ks, "e");
if (ks->pass_exception)
kgdb_info[ks->cpu].ret_state = 1;
if (error == KDB_CMD_CPU) {
KDB_STATE_SET(REENTRY);
/*
* Force clear the single step bit because kdb emulates this
* differently vs the gdbstub
*/
kgdb_single_step = 0;
dbg_deactivate_sw_breakpoints();
return DBG_SWITCH_CPU_EVENT;
}
return kgdb_info[ks->cpu].ret_state;
}
/*
* callback from kmsg_dump. Save as much as we can (up to kmsg_bytes) from the
* end of the buffer.
*/
static void pstore_dump(struct kmsg_dumper *dumper,
enum kmsg_dump_reason reason)
{
unsigned long total = 0;
const char *why;
unsigned int part = 1;
int ret;
why = get_reason_str(reason);
if (down_trylock(&psinfo->buf_lock)) {
/* Failed to acquire lock: give up if we cannot wait. */
if (pstore_cannot_wait(reason)) {
pr_err("dump skipped in %s path: may corrupt error record\n",
in_nmi() ? "NMI" : why);
return;
}
if (down_interruptible(&psinfo->buf_lock)) {
pr_err("could not grab semaphore?!\n");
return;
}
}
oopscount++;
while (total < kmsg_bytes) {
char *dst;
size_t dst_size;
int header_size;
int zipped_len = -1;
size_t dump_size;
struct pstore_record record;
pstore_record_init(&record, psinfo);
record.type = PSTORE_TYPE_DMESG;
record.count = oopscount;
record.reason = reason;
record.part = part;
record.buf = psinfo->buf;
if (big_oops_buf) {
dst = big_oops_buf;
dst_size = big_oops_buf_sz;
} else {
dst = psinfo->buf;
dst_size = psinfo->bufsize;
}
/* Write dump header. */
header_size = snprintf(dst, dst_size, "%s#%d Part%u\n", why,
oopscount, part);
dst_size -= header_size;
/* Write dump contents. */
if (!kmsg_dump_get_buffer(dumper, true, dst + header_size,
dst_size, &dump_size))
break;
if (big_oops_buf) {
zipped_len = pstore_compress(dst, psinfo->buf,
header_size + dump_size,
psinfo->bufsize);
if (zipped_len > 0) {
record.compressed = true;
record.size = zipped_len;
} else {
record.size = copy_kmsg_to_buffer(header_size,
dump_size);
}
} else {
record.size = header_size + dump_size;
}
ret = psinfo->write(&record);
if (ret == 0 && reason == KMSG_DUMP_OOPS && pstore_is_mounted())
pstore_new_entry = 1;
total += record.size;
part++;
}
up(&psinfo->buf_lock);
}
void trace_preempt_off(unsigned long a0, unsigned long a1)
{
if (!in_nmi())
trace_preempt_disable_rcuidle(a0, a1);
tracer_preempt_off(a0, a1);
}
/*
* callback from kmsg_dump. (s2,l2) has the most recently
* written bytes, older bytes are in (s1,l1). Save as much
* as we can from the end of the buffer.
*/
static void pstore_dump(struct kmsg_dumper *dumper,
enum kmsg_dump_reason reason)
{
unsigned long total = 0;
const char *why;
u64 id;
unsigned int part = 1;
unsigned long flags = 0;
int is_locked;
int ret;
why = get_reason_str(reason);
if (pstore_cannot_block_path(reason)) {
is_locked = spin_trylock_irqsave(&psinfo->buf_lock, flags);
if (!is_locked) {
pr_err("pstore dump routine blocked in %s path, may corrupt error record\n"
, in_nmi() ? "NMI" : why);
}
} else {
spin_lock_irqsave(&psinfo->buf_lock, flags);
is_locked = 1;
}
oopscount++;
while (total < kmsg_bytes) {
char *dst;
unsigned long size;
int hsize;
int zipped_len = -1;
size_t len;
bool compressed = false;
size_t total_len;
if (big_oops_buf && is_locked) {
dst = big_oops_buf;
size = big_oops_buf_sz;
} else {
dst = psinfo->buf;
size = psinfo->bufsize;
}
hsize = sprintf(dst, "%s#%d Part%u\n", why, oopscount, part);
size -= hsize;
if (!kmsg_dump_get_buffer(dumper, true, dst + hsize,
size, &len))
break;
if (big_oops_buf && is_locked) {
zipped_len = pstore_compress(dst, psinfo->buf,
hsize + len, psinfo->bufsize);
if (zipped_len > 0) {
compressed = true;
total_len = zipped_len;
} else {
total_len = copy_kmsg_to_buffer(hsize, len);
}
} else {
total_len = hsize + len;
}
ret = psinfo->write(PSTORE_TYPE_DMESG, reason, &id, part,
oopscount, compressed, total_len, psinfo);
if (ret == 0 && reason == KMSG_DUMP_OOPS && pstore_is_mounted())
pstore_new_entry = 1;
total += total_len;
part++;
}
if (is_locked)
spin_unlock_irqrestore(&psinfo->buf_lock, flags);
}
