/* * get_mce_event: * mce Pointer to machine_check_event structure to be filled. * release Flag to indicate whether to free the event slot or not. * 0 <= do not release the mce event. Caller will invoke * release_mce_event() once event has been consumed. * 1 <= release the slot. * * return 1 = success * 0 = failure * * get_mce_event() will be called by platform specific machine check * handle routine and in KVM. * When we call get_mce_event(), we are still in interrupt context and * preemption will not be scheduled until ret_from_expect() routine * is called. */ int get_mce_event(struct machine_check_event *mce, bool release) { int index = __this_cpu_read(mce_nest_count) - 1; struct machine_check_event *mc_evt; int ret = 0; /* Sanity check */ if (index < 0) return ret; /* Check if we have MCE info to process. */ if (index < MAX_MC_EVT) { mc_evt = this_cpu_ptr(&mce_event[index]); /* Copy the event structure and release the original */ if (mce) *mce = *mc_evt; if (release) mc_evt->in_use = 0; ret = 1; } /* Decrement the count to free the slot. */ if (release) __this_cpu_dec(mce_nest_count); return ret; }
/** * trace_call_bpf - invoke BPF program * @prog: BPF program * @ctx: opaque context pointer * * kprobe handlers execute BPF programs via this helper. * Can be used from static tracepoints in the future. * * Return: BPF programs always return an integer which is interpreted by * kprobe handler as: * 0 - return from kprobe (event is filtered out) * 1 - store kprobe event into ring buffer * Other values are reserved and currently alias to 1 */ unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx) { unsigned int ret; if (in_nmi()) /* not supported yet */ return 1; preempt_disable(); if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { /* * since some bpf program is already running on this cpu, * don't call into another bpf program (same or different) * and don't send kprobe event into ring-buffer, * so return zero here */ ret = 0; goto out; } rcu_read_lock(); ret = BPF_PROG_RUN(prog, ctx); rcu_read_unlock(); out: __this_cpu_dec(bpf_prog_active); preempt_enable(); return ret; }
static void queue_ue_paddr(unsigned long paddr) { int index; index = __this_cpu_inc_return(rtas_ue_count) - 1; if (index >= MAX_MC_EVT) { __this_cpu_dec(rtas_ue_count); return; } this_cpu_write(rtas_ue_paddr[index], paddr); schedule_work(&hwpoison_work); }
static void pseries_hwpoison_work_fn(struct work_struct *work) { unsigned long paddr; int index; while (__this_cpu_read(rtas_ue_count) > 0) { index = __this_cpu_read(rtas_ue_count) - 1; paddr = __this_cpu_read(rtas_ue_paddr[index]); memory_failure(paddr >> PAGE_SHIFT, 0); __this_cpu_dec(rtas_ue_count); } }
/* * Queue up the MCE event which then can be handled later. */ void machine_check_ue_event(struct machine_check_event *evt) { int index; index = __this_cpu_inc_return(mce_ue_count) - 1; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { __this_cpu_dec(mce_ue_count); return; } memcpy(this_cpu_ptr(&mce_ue_event_queue[index]), evt, sizeof(*evt)); /* Queue work to process this event later. */ schedule_work(&mce_ue_event_work); }
/* * process pending MCE event from the mce event queue. This function will be * called during syscall exit. */ static void machine_check_process_queued_event(struct irq_work *work) { int index; /* * For now just print it to console. * TODO: log this error event to FSP or nvram. */ while (__this_cpu_read(mce_queue_count) > 0) { index = __this_cpu_read(mce_queue_count) - 1; machine_check_print_event_info( this_cpu_ptr(&mce_event_queue[index])); __this_cpu_dec(mce_queue_count); } }
/* * process pending MCE event from the mce event queue. This function will be * called during syscall exit. */ static void machine_check_process_queued_event(struct irq_work *work) { int index; add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); /* * For now just print it to console. * TODO: log this error event to FSP or nvram. */ while (__this_cpu_read(mce_queue_count) > 0) { index = __this_cpu_read(mce_queue_count) - 1; machine_check_print_event_info( this_cpu_ptr(&mce_event_queue[index]), false); __this_cpu_dec(mce_queue_count); } }
/* * Queue up the MCE event which then can be handled later. */ void machine_check_queue_event(void) { int index; struct machine_check_event evt; if (!get_mce_event(&evt, MCE_EVENT_RELEASE)) return; index = __this_cpu_inc_return(mce_queue_count) - 1; /* If queue is full, just return for now. */ if (index >= MAX_MC_EVT) { __this_cpu_dec(mce_queue_count); return; } memcpy(this_cpu_ptr(&mce_event_queue[index]), &evt, sizeof(evt)); /* Queue irq work to process this event later. */ irq_work_queue(&mce_event_process_work); }
/** * trace_call_bpf - invoke BPF program * @call: tracepoint event * @ctx: opaque context pointer * * kprobe handlers execute BPF programs via this helper. * Can be used from static tracepoints in the future. * * Return: BPF programs always return an integer which is interpreted by * kprobe handler as: * 0 - return from kprobe (event is filtered out) * 1 - store kprobe event into ring buffer * Other values are reserved and currently alias to 1 */ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) { unsigned int ret; if (in_nmi()) /* not supported yet */ return 1; preempt_disable(); if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { /* * since some bpf program is already running on this cpu, * don't call into another bpf program (same or different) * and don't send kprobe event into ring-buffer, * so return zero here */ ret = 0; goto out; } /* * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock * to all call sites, we did a bpf_prog_array_valid() there to check * whether call->prog_array is empty or not, which is * a heurisitc to speed up execution. * * If bpf_prog_array_valid() fetched prog_array was * non-NULL, we go into trace_call_bpf() and do the actual * proper rcu_dereference() under RCU lock. * If it turns out that prog_array is NULL then, we bail out. * For the opposite, if the bpf_prog_array_valid() fetched pointer * was NULL, you'll skip the prog_array with the risk of missing * out of events when it was updated in between this and the * rcu_dereference() which is accepted risk. */ ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN); out: __this_cpu_dec(bpf_prog_active); preempt_enable(); return ret; }