static void bts_update(struct bts_ctx *bts) { int cpu = raw_smp_processor_id(); struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; struct bts_buffer *buf = perf_get_aux(&bts->handle); unsigned long index = ds->bts_index - ds->bts_buffer_base, old, head; if (!buf) return; head = index + bts_buffer_offset(buf, buf->cur_buf); old = local_xchg(&buf->head, head); if (!buf->snapshot) { if (old == head) return; if (ds->bts_index >= ds->bts_absolute_maximum) local_inc(&buf->lost); /* * old and head are always in the same physical buffer, so we * can subtract them to get the data size. */ local_add(head - old, &buf->data_size); } else { local_set(&buf->data_size, head); } }
void ctype_list_init(char *module) { Module = tdrpStrDup(module); ListBuf = tdrpBufCreate(); /* * add the names of the known types */ local_add("boolean", "tdrp_bool_t"); local_add("char*", "char*"); local_add("string", "char*"); local_add("int", "int"); local_add("long", "long"); local_add("float", "float"); local_add("double", "double"); }
static void etb_update_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config) { bool lost = false; int i, cur; u8 *buf_ptr; const u32 *barrier; u32 read_ptr, write_ptr, capacity; u32 status, read_data, to_read; unsigned long offset; struct cs_buffers *buf = sink_config; struct etb_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); if (!buf) return; capacity = drvdata->buffer_depth * ETB_FRAME_SIZE_WORDS; etb_disable_hw(drvdata); CS_UNLOCK(drvdata->base); /* unit is in words, not bytes */ read_ptr = readl_relaxed(drvdata->base + ETB_RAM_READ_POINTER); write_ptr = readl_relaxed(drvdata->base + ETB_RAM_WRITE_POINTER); /* * Entries should be aligned to the frame size. If they are not * go back to the last alignment point to give decoding tools a * chance to fix things. */ if (write_ptr % ETB_FRAME_SIZE_WORDS) { dev_err(drvdata->dev, "write_ptr: %lu not aligned to formatter frame size\n", (unsigned long)write_ptr); write_ptr &= ~(ETB_FRAME_SIZE_WORDS - 1); lost = true; } /* * Get a hold of the status register and see if a wrap around * has occurred. If so adjust things accordingly. Otherwise * start at the beginning and go until the write pointer has * been reached. */ status = readl_relaxed(drvdata->base + ETB_STATUS_REG); if (status & ETB_STATUS_RAM_FULL) { lost = true; to_read = capacity; read_ptr = write_ptr; } else { to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->buffer_depth); to_read *= ETB_FRAME_SIZE_WORDS; } /* * Make sure we don't overwrite data that hasn't been consumed yet. * It is entirely possible that the HW buffer has more data than the * ring buffer can currently handle. If so adjust the start address * to take only the last traces. * * In snapshot mode we are looking to get the latest traces only and as * such, we don't care about not overwriting data that hasn't been * processed by user space. */ if (!buf->snapshot && to_read > handle->size) { u32 mask = ~(ETB_FRAME_SIZE_WORDS - 1); /* The new read pointer must be frame size aligned */ to_read = handle->size & mask; /* * Move the RAM read pointer up, keeping in mind that * everything is in frame size units. */ read_ptr = (write_ptr + drvdata->buffer_depth) - to_read / ETB_FRAME_SIZE_WORDS; /* Wrap around if need be*/ if (read_ptr > (drvdata->buffer_depth - 1)) read_ptr -= drvdata->buffer_depth; /* let the decoder know we've skipped ahead */ lost = true; } if (lost) perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED); /* finally tell HW where we want to start reading from */ writel_relaxed(read_ptr, drvdata->base + ETB_RAM_READ_POINTER); cur = buf->cur; offset = buf->offset; barrier = barrier_pkt; for (i = 0; i < to_read; i += 4) { buf_ptr = buf->data_pages[cur] + offset; read_data = readl_relaxed(drvdata->base + ETB_RAM_READ_DATA_REG); if (lost && *barrier) { read_data = *barrier; barrier++; } *(u32 *)buf_ptr = read_data; buf_ptr += 4; offset += 4; if (offset >= PAGE_SIZE) { offset = 0; cur++; /* wrap around at the end of the buffer */ cur &= buf->nr_pages - 1; } } /* reset ETB buffer for next run */ writel_relaxed(0x0, drvdata->base + ETB_RAM_READ_POINTER); writel_relaxed(0x0, drvdata->base + ETB_RAM_WRITE_POINTER); /* * In snapshot mode all we have to do is communicate to * perf_aux_output_end() the address of the current head. In full * trace mode the same function expects a size to move rb->aux_head * forward. */ if (buf->snapshot) local_set(&buf->data_size, (cur * PAGE_SIZE) + offset); else local_add(to_read, &buf->data_size); etb_enable_hw(drvdata); CS_LOCK(drvdata->base); }
int perf_output_begin(struct perf_output_handle *handle, struct perf_event *event, unsigned int size) { struct ring_buffer *rb; unsigned long tail, offset, head; int have_lost; struct perf_sample_data sample_data; struct { struct perf_event_header header; u64 id; u64 lost; } lost_event; rcu_read_lock(); /* * For inherited events we send all the output towards the parent. */ if (event->parent) event = event->parent; rb = rcu_dereference(event->rb); if (!rb) goto out; handle->rb = rb; handle->event = event; if (!rb->nr_pages) goto out; have_lost = local_read(&rb->lost); if (have_lost) { lost_event.header.size = sizeof(lost_event); perf_event_header__init_id(&lost_event.header, &sample_data, event); size += lost_event.header.size; } perf_output_get_handle(handle); do { /* * Userspace could choose to issue a mb() before updating the * tail pointer. So that all reads will be completed before the * write is issued. */ tail = ACCESS_ONCE(rb->user_page->data_tail); smp_rmb(); offset = head = local_read(&rb->head); head += size; if (unlikely(!perf_output_space(rb, tail, offset, head))) goto fail; } while (local_cmpxchg(&rb->head, offset, head) != offset); if (head - local_read(&rb->wakeup) > rb->watermark) local_add(rb->watermark, &rb->wakeup); handle->page = offset >> (PAGE_SHIFT + page_order(rb)); handle->page &= rb->nr_pages - 1; handle->size = offset & ((PAGE_SIZE << page_order(rb)) - 1); handle->addr = rb->data_pages[handle->page]; handle->addr += handle->size; handle->size = (PAGE_SIZE << page_order(rb)) - handle->size; if (have_lost) { lost_event.header.type = PERF_RECORD_LOST; lost_event.header.misc = 0; lost_event.id = event->id; lost_event.lost = local_xchg(&rb->lost, 0); perf_output_put(handle, lost_event); perf_event__output_id_sample(event, handle, &sample_data); } return 0; fail: local_inc(&rb->lost); perf_output_put_handle(handle); out: rcu_read_unlock(); return -ENOSPC; }
static void tmc_update_etf_buffer(struct coresight_device *csdev, struct perf_output_handle *handle, void *sink_config) { int i, cur; u32 *buf_ptr; u32 read_ptr, write_ptr; u32 status, to_read; unsigned long offset; struct cs_buffers *buf = sink_config; struct tmc_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); if (!buf) return; /* This shouldn't happen */ if (WARN_ON_ONCE(local_read(&drvdata->mode) != CS_MODE_PERF)) return; CS_UNLOCK(drvdata->base); tmc_flush_and_stop(drvdata); read_ptr = readl_relaxed(drvdata->base + TMC_RRP); write_ptr = readl_relaxed(drvdata->base + TMC_RWP); /* * Get a hold of the status register and see if a wrap around * has occurred. If so adjust things accordingly. */ status = readl_relaxed(drvdata->base + TMC_STS); if (status & TMC_STS_FULL) { local_inc(&buf->lost); to_read = drvdata->size; } else { to_read = CIRC_CNT(write_ptr, read_ptr, drvdata->size); } /* * The TMC RAM buffer may be bigger than the space available in the * perf ring buffer (handle->size). If so advance the RRP so that we * get the latest trace data. */ if (to_read > handle->size) { u32 mask = 0; /* * The value written to RRP must be byte-address aligned to * the width of the trace memory databus _and_ to a frame * boundary (16 byte), whichever is the biggest. For example, * for 32-bit, 64-bit and 128-bit wide trace memory, the four * LSBs must be 0s. For 256-bit wide trace memory, the five * LSBs must be 0s. */ switch (drvdata->memwidth) { case TMC_MEM_INTF_WIDTH_32BITS: case TMC_MEM_INTF_WIDTH_64BITS: case TMC_MEM_INTF_WIDTH_128BITS: mask = GENMASK(31, 5); break; case TMC_MEM_INTF_WIDTH_256BITS: mask = GENMASK(31, 6); break; } /* * Make sure the new size is aligned in accordance with the * requirement explained above. */ to_read = handle->size & mask; /* Move the RAM read pointer up */ read_ptr = (write_ptr + drvdata->size) - to_read; /* Make sure we are still within our limits */ if (read_ptr > (drvdata->size - 1)) read_ptr -= drvdata->size; /* Tell the HW */ writel_relaxed(read_ptr, drvdata->base + TMC_RRP); local_inc(&buf->lost); } cur = buf->cur; offset = buf->offset; /* for every byte to read */ for (i = 0; i < to_read; i += 4) { buf_ptr = buf->data_pages[cur] + offset; *buf_ptr = readl_relaxed(drvdata->base + TMC_RRD); offset += 4; if (offset >= PAGE_SIZE) { offset = 0; cur++; /* wrap around at the end of the buffer */ cur &= buf->nr_pages - 1; } } /* * In snapshot mode all we have to do is communicate to * perf_aux_output_end() the address of the current head. In full * trace mode the same function expects a size to move rb->aux_head * forward. */ if (buf->snapshot) local_set(&buf->data_size, (cur * PAGE_SIZE) + offset); else local_add(to_read, &buf->data_size); CS_LOCK(drvdata->base); }