void perf_output_end(struct perf_output_handle *handle) { struct perf_event *event = handle->event; struct ring_buffer *rb = handle->rb; int wakeup_events = event->attr.wakeup_events; if (handle->sample && wakeup_events) { int events = local_inc_return(&rb->events); if (events >= wakeup_events) { local_sub(wakeup_events, &rb->events); local_inc(&rb->wakeup); } } perf_output_put_handle(handle); rcu_read_unlock(); }
int perf_output_begin(struct perf_output_handle *handle, struct perf_event *event, unsigned int size) { struct ring_buffer *rb; unsigned long tail, offset, head; int have_lost; struct perf_sample_data sample_data; struct { struct perf_event_header header; u64 id; u64 lost; } lost_event; rcu_read_lock(); /* * For inherited events we send all the output towards the parent. */ if (event->parent) event = event->parent; rb = rcu_dereference(event->rb); if (!rb) goto out; handle->rb = rb; handle->event = event; if (!rb->nr_pages) goto out; have_lost = local_read(&rb->lost); if (have_lost) { lost_event.header.size = sizeof(lost_event); perf_event_header__init_id(&lost_event.header, &sample_data, event); size += lost_event.header.size; } perf_output_get_handle(handle); do { /* * Userspace could choose to issue a mb() before updating the * tail pointer. So that all reads will be completed before the * write is issued. */ tail = ACCESS_ONCE(rb->user_page->data_tail); smp_rmb(); offset = head = local_read(&rb->head); head += size; if (unlikely(!perf_output_space(rb, tail, offset, head))) goto fail; } while (local_cmpxchg(&rb->head, offset, head) != offset); if (head - local_read(&rb->wakeup) > rb->watermark) local_add(rb->watermark, &rb->wakeup); handle->page = offset >> (PAGE_SHIFT + page_order(rb)); handle->page &= rb->nr_pages - 1; handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1); handle->addr = rb->data_pages[handle->page]; handle->addr += handle->size; handle->size = (PAGE_SIZE << page_order(rb)) - handle->size; if (have_lost) { lost_event.header.type = PERF_RECORD_LOST; lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, &sample_data); } return 0; fail: local_inc(&rb->lost); perf_output_put_handle(handle); out: rcu_read_unlock(); return -ENOSPC; }
void perf_output_end(struct perf_output_handle *handle) { perf_output_put_handle(handle); rcu_read_unlock(); }