static void __bts_event_start(struct perf_event *event) { struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct bts_buffer *buf = perf_get_aux(&bts->handle); u64 config = 0; if (!buf || bts_buffer_is_full(buf, bts)) return; event->hw.state = 0; if (!buf->snapshot) config |= ARCH_PERFMON_EVENTSEL_INT; if (!event->attr.exclude_kernel) config |= ARCH_PERFMON_EVENTSEL_OS; if (!event->attr.exclude_user) config |= ARCH_PERFMON_EVENTSEL_USR; bts_config_buffer(buf); /* * local barrier to make sure that ds configuration made it * before we enable BTS */ wmb(); intel_pmu_enable_bts(config); }
static void bts_update(struct bts_ctx *bts) { int cpu = raw_smp_processor_id(); struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; struct bts_buffer *buf = perf_get_aux(&bts->handle); unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head; if (!buf) return; head = index + bts_buffer_offset(buf, buf->cur_buf); old = local_xchg(&buf->head, head); if (!buf->snapshot) { if (old == head) return; if (ds->bts_index >= ds->bts_absolute_maximum) local_inc(&buf->lost); /* * old and head are always in the same physical buffer, so we * can subtract them to get the data size. */ local_add(head - old, &buf->data_size); } else { local_set(&buf->data_size, head); } }
static void bts_event_stop(struct perf_event *event, int flags) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct bts_buffer *buf = NULL; int state = READ_ONCE(bts->state); if (state == BTS_STATE_ACTIVE) __bts_event_stop(event, BTS_STATE_STOPPED); if (state != BTS_STATE_STOPPED) buf = perf_get_aux(&bts->handle); event->hw.state |= PERF_HES_STOPPED; if (flags & PERF_EF_UPDATE) { bts_update(bts); if (buf) { if (buf->snapshot) bts->handle.head = local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0), !!local_xchg(&buf->lost, 0)); } cpuc->ds->bts_index = bts->ds_back.bts_buffer_base; cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base; cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum; cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold; } }
static void __bts_event_start(struct perf_event *event) { struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct bts_buffer *buf = perf_get_aux(&bts->handle); u64 config = 0; if (!buf->snapshot) config |= ARCH_PERFMON_EVENTSEL_INT; if (!event->attr.exclude_kernel) config |= ARCH_PERFMON_EVENTSEL_OS; if (!event->attr.exclude_user) config |= ARCH_PERFMON_EVENTSEL_USR; bts_config_buffer(buf); /* * local barrier to make sure that ds configuration made it * before we enable BTS and bts::state goes ACTIVE */ wmb(); /* INACTIVE/STOPPED -> ACTIVE */ WRITE_ONCE(bts->state, BTS_STATE_ACTIVE); intel_pmu_enable_bts(config); }
int intel_bts_interrupt(void) { struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct perf_event *event = bts->handle.event; struct bts_buffer *buf; s64 old_head; int err; if (!event || !bts->started) return 0; buf = perf_get_aux(&bts->handle); /* * Skip snapshot counters: they don't use the interrupt, but * there's no other way of telling, because the pointer will * keep moving */ if (!buf || buf->snapshot) return 0; old_head = local_read(&buf->head); bts_update(bts); /* no new data */ if (old_head == local_read(&buf->head)) return 0; perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0), !!local_xchg(&buf->lost, 0)); buf = perf_aux_output_begin(&bts->handle, event); if (!buf) return 1; err = bts_buffer_reset(buf, &bts->handle); if (err) perf_aux_output_end(&bts->handle, 0, false); return 1; }
static void bts_event_del(struct perf_event *event, int mode) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct bts_buffer *buf = perf_get_aux(&bts->handle); bts_event_stop(event, PERF_EF_UPDATE); if (buf) { if (buf->snapshot) bts->handle.head = local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0), !!local_xchg(&buf->lost, 0)); } cpuc->ds->bts_index = bts->ds_back.bts_buffer_base; cpuc->ds->bts_buffer_base = bts->ds_back.bts_buffer_base; cpuc->ds->bts_absolute_maximum = bts->ds_back.bts_absolute_maximum; cpuc->ds->bts_interrupt_threshold = bts->ds_back.bts_interrupt_threshold; }
int intel_bts_interrupt(void) { struct debug_store *ds = this_cpu_ptr(&cpu_hw_events)->ds; struct bts_ctx *bts = this_cpu_ptr(&bts_ctx); struct perf_event *event = bts->handle.event; struct bts_buffer *buf; s64 old_head; int err = -ENOSPC, handled = 0; /* * The only surefire way of knowing if this NMI is ours is by checking * the write ptr against the PMI threshold. */ if (ds->bts_index >= ds->bts_interrupt_threshold) handled = 1; /* * this is wrapped in intel_bts_enable_local/intel_bts_disable_local, * so we can only be INACTIVE or STOPPED */ if (READ_ONCE(bts->state) == BTS_STATE_STOPPED) return handled; buf = perf_get_aux(&bts->handle); if (!buf) return handled; /* * Skip snapshot counters: they don't use the interrupt, but * there's no other way of telling, because the pointer will * keep moving */ if (buf->snapshot) return 0; old_head = local_read(&buf->head); bts_update(bts); /* no new data */ if (old_head == local_read(&buf->head)) return handled; perf_aux_output_end(&bts->handle, local_xchg(&buf->data_size, 0), !!local_xchg(&buf->lost, 0)); buf = perf_aux_output_begin(&bts->handle, event); if (buf) err = bts_buffer_reset(buf, &bts->handle); if (err) { WRITE_ONCE(bts->state, BTS_STATE_STOPPED); if (buf) { /* * BTS_STATE_STOPPED should be visible before * cleared handle::event */ barrier(); perf_aux_output_end(&bts->handle, 0, false); } } return 1; }