/* Create perf event sample with the counter sets as raw data. The sample * is then pushed to the event subsystem and the function checks for * possible event overflows. If an event overflow occurs, the PMU is * stopped. * * Return non-zero if an event overflow occurred. */ static int cf_diag_push_sample(struct perf_event *event, struct cf_diag_csd *csd) { struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; int overflow; /* Setup perf sample */ perf_sample_data_init(&data, 0, event->hw.last_period); memset(®s, 0, sizeof(regs)); memset(&raw, 0, sizeof(raw)); if (event->attr.sample_type & PERF_SAMPLE_CPU) data.cpu_entry.cpu = event->cpu; if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.frag.size = csd->used; raw.frag.data = csd->data; raw.size = csd->used; data.raw = &raw; } overflow = perf_event_overflow(event, &data, ®s); debug_sprintf_event(cf_diag_dbg, 6, "%s event %p cpu %d sample_type %#llx raw %d " "ov %d\n", __func__, event, event->cpu, event->attr.sample_type, raw.size, overflow); if (overflow) event->pmu->stop(event, 0); perf_event_update_userpage(event); return overflow; }
static __always_inline u64 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, u64 flags, struct perf_raw_record *raw) { struct bpf_array *array = container_of(map, struct bpf_array, map); unsigned int cpu = smp_processor_id(); u64 index = flags & BPF_F_INDEX_MASK; struct perf_sample_data sample_data; struct bpf_event_entry *ee; struct perf_event *event; if (index == BPF_F_CURRENT_CPU) index = cpu; if (unlikely(index >= array->map.max_entries)) return -E2BIG; ee = READ_ONCE(array->ptrs[index]); if (!ee) return -ENOENT; event = ee->event; if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) return -EINVAL; if (unlikely(event->oncpu != cpu)) return -EOPNOTSUPP; perf_sample_data_init(&sample_data, 0, 0); sample_data.raw = raw; perf_event_output(event, &sample_data, regs); return 0; }
BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, u64, flags, void *, data, u64, size) { struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd); struct perf_raw_record raw = { .frag = { .size = size, .data = data, }, }; if (unlikely(flags & ~(BPF_F_INDEX_MASK))) return -EINVAL; perf_sample_data_init(sd, 0, 0); sd->raw = &raw; return __bpf_perf_event_output(regs, map, flags, sd); }
static u64 bpf_perf_event_output(u64 r1, u64 r2, u64 flags, u64 r4, u64 size) { struct pt_regs *regs = (struct pt_regs *) (long) r1; struct bpf_map *map = (struct bpf_map *) (long) r2; struct bpf_array *array = container_of(map, struct bpf_array, map); unsigned int cpu = smp_processor_id(); u64 index = flags & BPF_F_INDEX_MASK; void *data = (void *) (long) r4; struct perf_sample_data sample_data; struct bpf_event_entry *ee; struct perf_event *event; struct perf_raw_record raw = { .size = size, .data = data, }; if (unlikely(flags & ~(BPF_F_INDEX_MASK))) return -EINVAL; if (index == BPF_F_CURRENT_CPU) index = cpu; if (unlikely(index >= array->map.max_entries)) return -E2BIG; ee = READ_ONCE(array->ptrs[index]); if (!ee) return -ENOENT; event = ee->event; if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) return -EINVAL; if (unlikely(event->oncpu != cpu)) return -EOPNOTSUPP; perf_sample_data_init(&sample_data, 0, 0); sample_data.raw = &raw; perf_event_output(event, &sample_data, regs); return 0; }
static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) { struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); struct perf_event *event = pcpu->event; struct hw_perf_event *hwc = &event->hw; struct perf_sample_data data; struct perf_raw_record raw; struct pt_regs regs; struct perf_ibs_data ibs_data; int offset, size, check_rip, offset_max, throttle = 0; unsigned int msr; u64 *buf, *config, period; if (!test_bit(IBS_STARTED, pcpu->state)) { /* * Catch spurious interrupts after stopping IBS: After * disabling IBS there could be still incoming NMIs * with samples that even have the valid bit cleared. * Mark all this NMIs as handled. */ return test_and_clear_bit(IBS_STOPPING, pcpu->state) ? 1 : 0; } msr = hwc->config_base; buf = ibs_data.regs; rdmsrl(msr, *buf); if (!(*buf++ & perf_ibs->valid_mask)) return 0; config = &ibs_data.regs[0]; perf_ibs_event_update(perf_ibs, event, config); perf_sample_data_init(&data, 0, hwc->last_period); if (!perf_ibs_set_period(perf_ibs, hwc, &period)) goto out; /* no sw counter overflow */ ibs_data.caps = ibs_caps; size = 1; offset = 1; check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK)); if (event->attr.sample_type & PERF_SAMPLE_RAW) offset_max = perf_ibs->offset_max; else if (check_rip) offset_max = 2; else offset_max = 1; do { rdmsrl(msr + offset, *buf++); size++; offset = find_next_bit(perf_ibs->offset_mask, perf_ibs->offset_max, offset + 1); } while (offset < offset_max); if (event->attr.sample_type & PERF_SAMPLE_RAW) { /* * Read IbsBrTarget and IbsOpData4 separately * depending on their availability. * Can't add to offset_max as they are staggered */ if (ibs_caps & IBS_CAPS_BRNTRGT) { rdmsrl(MSR_AMD64_IBSBRTARGET, *buf++); size++; } if (ibs_caps & IBS_CAPS_OPDATA4) { rdmsrl(MSR_AMD64_IBSOPDATA4, *buf++); size++; } } ibs_data.size = sizeof(u64) * size; regs = *iregs; if (check_rip && (ibs_data.regs[2] & IBS_RIP_INVALID)) { regs.flags &= ~PERF_EFLAGS_EXACT; } else { set_linear_ip(®s, ibs_data.regs[1]); regs.flags |= PERF_EFLAGS_EXACT; } if (event->attr.sample_type & PERF_SAMPLE_RAW) { raw.size = sizeof(u32) + ibs_data.size; raw.data = ibs_data.data; data.raw = &raw; } throttle = perf_event_overflow(event, &data, ®s); out: if (throttle) perf_ibs_disable_event(perf_ibs, hwc, *config); else perf_ibs_enable_event(perf_ibs, hwc, period >> 4); perf_event_update_userpage(event); return 1; }
static irqreturn_t armv8pmu_handle_irq(int irq_num, void *dev) { u32 pmovsr; struct perf_sample_data data; struct arm_pmu *cpu_pmu = (struct arm_pmu *)dev; struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); struct pt_regs *regs; int idx; /* * Get and reset the IRQ flags */ pmovsr = armv8pmu_getreset_flags(); /* * Did an overflow occur? */ if (!armv8pmu_has_overflowed(pmovsr)) return IRQ_NONE; /* * Handle the counter(s) overflow(s) */ regs = get_irq_regs(); for (idx = 0; idx < cpu_pmu->num_events; ++idx) { struct perf_event *event = cpuc->events[idx]; struct hw_perf_event *hwc; /* Ignore if we don't have an event. */ if (!event) continue; /* * We have a single interrupt for all counters. Check that * each counter has overflowed before we process it. */ if (!armv8pmu_counter_has_overflowed(pmovsr, idx)) continue; hwc = &event->hw; armpmu_event_update(event); perf_sample_data_init(&data, 0, hwc->last_period); if (!armpmu_event_set_period(event)) continue; if (perf_event_overflow(event, &data, regs)) cpu_pmu->disable(event); } /* * Handle the pending perf events. * * Note: this call *must* be run with interrupts disabled. For * platforms that can have the PMU interrupts raised as an NMI, this * will not work. */ irq_work_run(); return IRQ_HANDLED; }