int
pmc_uncore_initialize(struct pmc_mdep *md, int maxcpu)
{
uncore_cputype = md->pmd_cputype;
uncore_pmcmask = 0;
/*
* Initialize programmable counters.
*/
uncore_ucp_npmc = 8;
uncore_ucp_width = 48;
uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1);
ucp_initialize(md, maxcpu, uncore_ucp_npmc, uncore_ucp_width);
/*
* Initialize fixed function counters, if present.
*/
uncore_ucf_ri = uncore_ucp_npmc;
uncore_ucf_npmc = 1;
uncore_ucf_width = 48;
ucf_initialize(md, maxcpu, uncore_ucf_npmc, uncore_ucf_width);
uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype);
PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask,
uncore_ucf_ri);
uncore_pcpu = malloc(sizeof(*uncore_pcpu) * maxcpu, M_PMC,
M_ZERO | M_WAITOK);
return (0);
}
static int
ucf_stop_pmc(int cpu, int ri)
{
uint32_t fc;
struct uncore_cpu *ucfc;
PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
fc = (UCF_MASK << (ri * 4));
ucfc->pc_ucfctrl &= ~fc;
PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucf_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
uint32_t caps, flags;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
if (ri < 0 || ri > uncore_ucf_npmc)
return (EINVAL);
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_UCF ||
(caps & UCF_PMC_CAPS) != caps)
return (EINVAL);
ev = pm->pm_event;
if (ev < PMC_EV_UCF_FIRST || ev > PMC_EV_UCF_LAST)
return (EINVAL);
flags = UCF_EN;
pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4));
PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx",
(uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl);
return (0);
}
static int
mpc7xxx_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
KASSERT(pm,
("[core,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = mpc7xxx_pmcn_read(ri);
PMCDBG2(MDP,REA,2,"ppc-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = POWERPC_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
return 0;
}
static int
arm64_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
uint32_t caps, config;
struct arm64_cpu *pac;
enum pmc_event pe;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < arm64_npmcs,
("[arm64,%d] illegal row index %d", __LINE__, ri));
pac = arm64_pcpu[cpu];
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_ARMV8) {
return (EINVAL);
}
pe = a->pm_ev;
config = (pe & EVENT_ID_MASK);
pm->pm_md.pm_arm64.pm_arm64_evsel = config;
PMCDBG2(MDP, ALL, 2, "arm64-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
pmclog_get_buffer(struct pmc_owner *po)
{
struct pmclog_buffer *plb;
mtx_assert(&po->po_mtx, MA_OWNED);
KASSERT(po->po_curbuf == NULL,
("[pmclog,%d] po=%p current buffer still valid", __LINE__, po));
mtx_lock_spin(&pmc_bufferlist_mtx);
if ((plb = TAILQ_FIRST(&pmc_bufferlist)) != NULL)
TAILQ_REMOVE(&pmc_bufferlist, plb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
PMCDBG2(LOG,GTB,1, "po=%p plb=%p", po, plb);
#ifdef HWPMC_DEBUG
if (plb)
KASSERT(plb->plb_ptr == plb->plb_base &&
plb->plb_base < plb->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p "
"base=%p fence=%p", __LINE__, po, plb->plb_ptr,
plb->plb_base, plb->plb_fence));
#endif
po->po_curbuf = plb;
/* update stats */
atomic_add_int(&pmc_stats.pm_buffer_requests, 1);
if (plb == NULL)
atomic_add_int(&pmc_stats.pm_buffer_requests_failed, 1);
return (plb ? 0 : ENOMEM);
}
static int
ucp_stop_pmc(int cpu, int ri)
{
struct pmc *pm;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri);
/* stop hw. */
wrmsr(SELECTSEL(uncore_cputype) + ri, 0);
do {
cc->pc_resync = 0;
cc->pc_globalctrl &= ~(1ULL << ri);
wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
ucf_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct uncore_cpu *ucfc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl;
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucp_start_pmc(int cpu, int ri)
{
struct pmc *pm;
uint32_t evsel;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] starting cpu%d,ri%d with no pmc configured",
__LINE__, cpu, ri));
PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri);
evsel = pm->pm_md.pm_ucp.pm_ucp_evsel;
PMCDBG4(MDP,STA,2,
"ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel);
/* Event specific configuration. */
switch (pm->pm_event) {
case PMC_EV_UCP_EVENT_0CH_04H_E:
case PMC_EV_UCP_EVENT_0CH_08H_E:
wrmsr(MSR_GQ_SNOOP_MESF,0x2);
break;
case PMC_EV_UCP_EVENT_0CH_04H_F:
case PMC_EV_UCP_EVENT_0CH_08H_F:
wrmsr(MSR_GQ_SNOOP_MESF,0x8);
break;
case PMC_EV_UCP_EVENT_0CH_04H_M:
case PMC_EV_UCP_EVENT_0CH_08H_M:
wrmsr(MSR_GQ_SNOOP_MESF,0x1);
break;
case PMC_EV_UCP_EVENT_0CH_04H_S:
case PMC_EV_UCP_EVENT_0CH_08H_S:
wrmsr(MSR_GQ_SNOOP_MESF,0x4);
break;
default:
break;
}
wrmsr(SELECTSEL(uncore_cputype) + ri, evsel);
do {
cc->pc_resync = 0;
cc->pc_globalctrl |= (1ULL << ri);
wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
int
pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
{
struct proc *p;
cap_rights_t rights;
int error;
sx_assert(&pmc_sx, SA_XLOCKED);
PMCDBG2(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
p = po->po_owner;
/* return EBUSY if a log file was already present */
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
return (EBUSY);
KASSERT(po->po_file == NULL,
("[pmclog,%d] po=%p file (%p) already present", __LINE__, po,
po->po_file));
/* get a reference to the file state */
error = fget_write(curthread, logfd,
cap_rights_init(&rights, CAP_WRITE), &po->po_file);
if (error)
goto error;
/* mark process as owning a log file */
po->po_flags |= PMC_PO_OWNS_LOGFILE;
/* mark process as using HWPMCs */
PROC_LOCK(p);
p->p_flag |= P_HWPMC;
PROC_UNLOCK(p);
/* create a log initialization entry */
PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
sizeof(struct pmclog_initialize));
PMCLOG_EMIT32(PMC_VERSION);
PMCLOG_EMIT32(md->pmd_cputype);
PMCLOG_DESPATCH(po);
return (0);
error:
KASSERT(po->po_kthread == NULL, ("[pmclog,%d] po=%p kthread not "
"stopped", __LINE__, po));
if (po->po_file)
(void) fdrop(po->po_file, curthread);
po->po_file = NULL; /* clear file and error state */
po->po_error = 0;
po->po_flags &= ~PMC_PO_OWNS_LOGFILE;
return (error);
}
static int
p4_get_msr(int ri, uint32_t *msr)
{
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] ri %d out of range", __LINE__, ri));
*msr = p4_pmcdesc[ri].pm_pmc_msr - P4_PERFCTR_MSR_FIRST;
PMCDBG2(MDP,OPS, 1, "ri=%d getmsr=0x%x", ri, *msr);
return 0;
}
void
pmclog_process_pmcdetach(struct pmc *pm, pid_t pid)
{
struct pmc_owner *po;
PMCDBG2(LOG,ATT,1,"!pm=%p pid=%d", pm, pid);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PMCDETACH, sizeof(struct pmclog_pmcdetach));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pid);
PMCLOG_DESPATCH(po);
}
int
pmclog_process_userlog(struct pmc_owner *po, struct pmc_op_writelog *wl)
{
int error;
PMCDBG2(LOG,WRI,1, "writelog po=%p ud=0x%x", po, wl->pm_userdata);
error = 0;
PMCLOG_RESERVE_WITH_ERROR(po, USERDATA,
sizeof(struct pmclog_userdata));
PMCLOG_EMIT32(wl->pm_userdata);
PMCLOG_DESPATCH(po);
error:
return (error);
}
void
pmclog_process_pmcattach(struct pmc *pm, pid_t pid, char *path)
{
int pathlen, recordlen;
struct pmc_owner *po;
PMCDBG2(LOG,ATT,1,"pm=%p pid=%d", pm, pid);
po = pm->pm_owner;
pathlen = strlen(path) + 1; /* #bytes for the string */
recordlen = offsetof(struct pmclog_pmcattach, pl_pathname) + pathlen;
PMCLOG_RESERVE(po, PMCATTACH, recordlen);
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pid);
PMCLOG_EMITSTRING(path, pathlen);
PMCLOG_DESPATCH(po);
}
static int
mpc7xxx_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event pe;
uint32_t caps, config, counter;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row index %d", __LINE__, ri));
caps = a->pm_caps;
pe = a->pm_ev;
for (i = 0; i < nitems(mpc7xxx_event_codes); i++) {
if (mpc7xxx_event_codes[i].pe_ev == pe) {
config = mpc7xxx_event_codes[i].pe_code;
counter = mpc7xxx_event_codes[i].pe_counter_mask;
break;
}
}
if (i == nitems(mpc7xxx_event_codes))
return (EINVAL);
if ((counter & (1 << ri)) == 0)
return (EINVAL);
if (caps & PMC_CAP_SYSTEM)
config |= POWERPC_PMC_KERNEL_ENABLE;
if (caps & PMC_CAP_USER)
config |= POWERPC_PMC_USER_ENABLE;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
config |= POWERPC_PMC_ENABLE;
pm->pm_md.pm_powerpc.pm_powerpc_evsel = config;
PMCDBG2(MDP,ALL,2,"powerpc-allocate ri=%d -> config=0x%x", ri, config);
return 0;
}
static int
arm64_read_pmc(int cpu, int ri, pmc_value_t *v)
{
pmc_value_t tmp;
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < arm64_npmcs,
("[arm64,%d] illegal row index %d", __LINE__, ri));
pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
tmp = arm64_pmcn_read(ri);
PMCDBG2(MDP, REA, 2, "arm64-read id=%d -> %jd", ri, tmp);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = ARMV8_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
return 0;
}
uint32_t
mips_get_perfctl(int cpu, int ri, uint32_t event, uint32_t caps)
{
uint32_t config;
config = event;
config <<= MIPS24K_PMC_SELECT;
if (caps & PMC_CAP_SYSTEM)
config |= (MIPS24K_PMC_SUPER_ENABLE |
MIPS24K_PMC_KERNEL_ENABLE);
if (caps & PMC_CAP_USER)
config |= MIPS24K_PMC_USER_ENABLE;
if ((caps & (PMC_CAP_USER | PMC_CAP_SYSTEM)) == 0)
config |= MIPS24K_PMC_ENABLE;
if (caps & PMC_CAP_INTERRUPT)
config |= MIPS24K_PMC_INTERRUPT_ENABLE;
PMCDBG2(MDP,ALL,2,"mips24k-get_perfctl ri=%d -> config=0x%x", ri, config);
return (config);
}
static int
p4_describe(int cpu, int ri, struct pmc_info *pi,
struct pmc **ppmc)
{
int error;
size_t copied;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] row-index %d out of range", __LINE__, ri));
PMCDBG2(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri);
if (P4_CPU_IS_HTT_SECONDARY(cpu))
return (EINVAL);
pd = &p4_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static void
pmclog_loop(void *arg)
{
int error;
struct pmc_owner *po;
struct pmclog_buffer *lb;
struct proc *p;
struct ucred *ownercred;
struct ucred *mycred;
struct thread *td;
struct uio auio;
struct iovec aiov;
size_t nbytes;
po = (struct pmc_owner *) arg;
p = po->po_owner;
td = curthread;
mycred = td->td_ucred;
PROC_LOCK(p);
ownercred = crhold(p->p_ucred);
PROC_UNLOCK(p);
PMCDBG2(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
KASSERT(po->po_kthread == curthread->td_proc,
("[pmclog,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__,
po, po->po_kthread, curthread->td_proc));
lb = NULL;
/*
* Loop waiting for I/O requests to be added to the owner
* struct's queue. The loop is exited when the log file
* is deconfigured.
*/
mtx_lock(&pmc_kthread_mtx);
for (;;) {
/* check if we've been asked to exit */
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
break;
if (lb == NULL) { /* look for a fresh buffer to write */
mtx_lock_spin(&po->po_mtx);
if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
mtx_unlock_spin(&po->po_mtx);
/* No more buffers and shutdown required. */
if (po->po_flags & PMC_PO_SHUTDOWN) {
mtx_unlock(&pmc_kthread_mtx);
/*
* Close the file to get PMCLOG_EOF
* error in pmclog(3).
*/
fo_close(po->po_file, curthread);
mtx_lock(&pmc_kthread_mtx);
break;
}
(void) msleep(po, &pmc_kthread_mtx, PWAIT,
"pmcloop", 0);
continue;
}
TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
mtx_unlock_spin(&po->po_mtx);
}
mtx_unlock(&pmc_kthread_mtx);
/* process the request */
PMCDBG3(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
lb->plb_base, lb->plb_ptr);
/* change our thread's credentials before issuing the I/O */
aiov.iov_base = lb->plb_base;
aiov.iov_len = nbytes = lb->plb_ptr - lb->plb_base;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = -1;
auio.uio_resid = nbytes;
auio.uio_rw = UIO_WRITE;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_td = td;
/* switch thread credentials -- see kern_ktrace.c */
td->td_ucred = ownercred;
error = fo_write(po->po_file, &auio, ownercred, 0, td);
td->td_ucred = mycred;
if (error) {
/* XXX some errors are recoverable */
/* send a SIGIO to the owner and exit */
PROC_LOCK(p);
kern_psignal(p, SIGIO);
PROC_UNLOCK(p);
mtx_lock(&pmc_kthread_mtx);
po->po_error = error; /* save for flush log */
PMCDBG2(LOG,WRI,2, "po=%p error=%d", po, error);
break;
}
mtx_lock(&pmc_kthread_mtx);
/* put the used buffer back into the global pool */
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
lb = NULL;
}
wakeup_one(po->po_kthread);
po->po_kthread = NULL;
mtx_unlock(&pmc_kthread_mtx);
/* return the current I/O buffer to the global pool */
if (lb) {
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/*
* Exit this thread, signalling the waiter
*/
crfree(ownercred);
kproc_exit(0);
}
static uint32_t *
pmclog_reserve(struct pmc_owner *po, int length)
{
uintptr_t newptr, oldptr;
uint32_t *lh;
struct timespec ts;
PMCDBG2(LOG,ALL,1, "po=%p len=%d", po, length);
KASSERT(length % sizeof(uint32_t) == 0,
("[pmclog,%d] length not a multiple of word size", __LINE__));
mtx_lock_spin(&po->po_mtx);
/* No more data when shutdown in progress. */
if (po->po_flags & PMC_PO_SHUTDOWN) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
if (po->po_curbuf == NULL)
if (pmclog_get_buffer(po) != 0) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] po=%p no current buffer", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base &&
po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
po->po_curbuf->plb_fence));
oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
newptr = oldptr + length;
KASSERT(oldptr != (uintptr_t) NULL,
("[pmclog,%d] po=%p Null log buffer pointer", __LINE__, po));
/*
* If we have space in the current buffer, return a pointer to
* available space with the PO structure locked.
*/
if (newptr <= (uintptr_t) po->po_curbuf->plb_fence) {
po->po_curbuf->plb_ptr = (char *) newptr;
goto done;
}
/*
* Otherwise, schedule the current buffer for output and get a
* fresh buffer.
*/
pmclog_schedule_io(po);
if (pmclog_get_buffer(po) != 0) {
mtx_unlock_spin(&po->po_mtx);
return (NULL);
}
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] po=%p no current buffer", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr != NULL,
("[pmclog,%d] null return from pmc_get_log_buffer", __LINE__));
KASSERT(po->po_curbuf->plb_ptr == po->po_curbuf->plb_base &&
po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] po=%p buffer invariants: ptr=%p base=%p fence=%p",
__LINE__, po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base,
po->po_curbuf->plb_fence));
oldptr = (uintptr_t) po->po_curbuf->plb_ptr;
done:
lh = (uint32_t *) oldptr;
lh++; /* skip header */
getnanotime(&ts); /* fill in the timestamp */
*lh++ = ts.tv_sec & 0xFFFFFFFF;
*lh++ = ts.tv_nsec & 0xFFFFFFF;
return ((uint32_t *) oldptr);
}
static int
p4_pcpu_init(struct pmc_mdep *md, int cpu)
{
char *pescr;
int n, first_ri, phycpu;
struct pmc_hw *phw;
struct p4_cpu *p4c;
struct pmc_cpu *pc, *plc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG2(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,
pmc_cpu_is_primary(cpu) != 0);
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
/*
* The two CPUs in an HT pair share their per-cpu state.
*
* For HT capable CPUs, we assume that the two logical
* processors in the HT pair get two consecutive CPU ids
* starting with an even id #.
*
* The primary CPU (the even numbered CPU of the pair) would
* have been initialized prior to the initialization for the
* secondary.
*/
if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) {
p4_system_has_htt = 1;
phycpu = P4_TO_HTT_PRIMARY(cpu);
pc = pmc_pcpu[phycpu];
plc = pmc_pcpu[cpu];
KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
PMCDBG3(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
phycpu, pc);
KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
__LINE__, cpu, phycpu));
/* PMCs are shared with the physical CPU. */
for (n = 0; n < P4_NPMCS; n++)
plc->pc_hwpmcs[n + first_ri] =
pc->pc_hwpmcs[n + first_ri];
return (0);
}
p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
p4_pcpu[cpu] = p4c;
phw = p4c->pc_p4pmcs;
for (n = 0; n < P4_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
pescr = p4c->pc_escrs;
for (n = 0; n < P4_NESCR; n++)
*pescr++ = P4_INVALID_PMC_INDEX;
mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
return (0);
}
static int
p4_intr(int cpu, struct trapframe *tf)
{
uint32_t cccrval, ovf_mask, ovf_partner;
int did_interrupt, error, ri;
struct p4_cpu *pc;
struct pmc *pm;
pmc_value_t v;
PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
ovf_mask |= P4_CCCR_OVF;
if (p4_system_has_htt)
ovf_partner = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T0 : P4_CCCR_OVF_PMI_T1;
else
ovf_partner = 0;
did_interrupt = 0;
if (p4_system_has_htt)
P4_PCPU_ACQ_INTR_SPINLOCK(pc);
/*
* Loop through all CCCRs, looking for ones that have
* interrupted this CPU.
*/
for (ri = 0; ri < P4_NPMCS; ri++) {
/*
* Check if our partner logical CPU has already marked
* this PMC has having interrupted it. If so, reset
* the flag and process the interrupt, but leave the
* hardware alone.
*/
if (p4_system_has_htt && P4_PCPU_GET_INTRFLAG(pc,ri)) {
P4_PCPU_SET_INTRFLAG(pc,ri,0);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs.
* Ignore PMCs not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
continue;
}
/*
* Fresh interrupt. Look for the CCCR_OVF bit
* and the OVF_Tx bit for this logical
* processor being set.
*/
cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
if ((cccrval & ovf_mask) != ovf_mask)
continue;
/*
* If the other logical CPU would also have been
* interrupted due to the PMC being shared, record
* this fact in the per-cpu saved interrupt flag
* bitmask.
*/
if (p4_system_has_htt && (cccrval & ovf_partner))
P4_PCPU_SET_INTRFLAG(pc, ri, 1);
v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
PMCDBG2(MDP,INT, 2, "ri=%d v=%jx", ri, v);
/* Stop the counter, and reset the overflow bit */
cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs. Ignore PMCs
* not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
/*
* Process the interrupt. Re-enable the PMC if
* processing was successful.
*/
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
/*
* Only the first processor executing the NMI handler
* in a HTT pair will restart a PMC, and that too
* only if there were no errors.
*/
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
pm->pm_sc.pm_reloadcount);
wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
if (error == 0)
wrmsr(P4_CCCR_MSR_FIRST + ri,
cccrval | P4_CCCR_ENABLE);
}
/* allow the other CPU to proceed */
if (p4_system_has_htt)
P4_PCPU_REL_INTR_SPINLOCK(pc);
/*
* On Intel P4 CPUs, the PMC 'pcint' entry in the LAPIC gets
* masked when a PMC interrupts the CPU. We need to unmask
* the interrupt source explicitly.
*/
if (did_interrupt)
lapic_reenable_pmc();
atomic_add_int(did_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (did_interrupt);
}
static int
p4_stop_pmc(int cpu, int ri)
{
int rc;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row index %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
KASSERT(pm != NULL,
("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));
PMCDBG2(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(pd->pm_cccr_msr,
pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
return (0);
}
/*
* Thread mode PMCs.
*
* On HTT machines, this PMC may be in use by two threads
* running on two logical CPUS. Thus we look at the
* 'runcount' field and only turn off the appropriate TO/T1
* bits (and keep the PMC running) if two logical CPUs were
* using the PMC.
*
*/
/* bits to mask */
cccrtbits = P4_CCCR_OVF_PMI_T0;
escrtbits = P4_ESCR_T0_OS | P4_ESCR_T0_USR;
if (P4_CPU_IS_HTT_SECONDARY(cpu)) {
cccrtbits <<= 1;
escrtbits >>= 2;
}
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 2 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
--rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
/* Stop this PMC */
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
escrvalue = rdmsr(escrmsr);
/* The current CPU should be running on this PMC */
KASSERT(escrvalue & escrtbits,
("[p4,%d] ESCR T0/T1 mismatch cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,
escrvalue, escrtbits));
KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)) ||
(cccrvalue & cccrtbits),
("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
"tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
/* get the current hardware reading */
tmp = rdmsr(pd->pm_pmc_msr);
if (rc == 1) { /* need to keep the PMC running */
escrvalue &= ~escrtbits;
cccrvalue &= ~cccrtbits;
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
}
mtx_unlock_spin(&pc->pc_mtx);
PMCDBG5(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrval=0x%x", cpu, rc, ri, escrmsr, escrvalue);
PMCDBG2(MDP,STO,2, "cccrval=0x%x v=%jx", cccrvalue, tmp);
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */
tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
return 0;
}
static int
p4_start_pmc(int cpu, int ri)
{
int rc;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));
PMCDBG2(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] wrong PMC class %d", __LINE__,
pd->pm_descr.pd_class));
/* retrieve the desired CCCR/ESCR values from the PMC */
cccrvalue = pm->pm_md.pm_p4.pm_p4_cccrvalue;
escrvalue = pm->pm_md.pm_p4.pm_p4_escrvalue;
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
/* extract and zero the logical processor selection bits */
cccrtbits = cccrvalue & P4_CCCR_OVF_PMI_T0;
escrtbits = escrvalue & (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
cccrvalue &= ~P4_CCCR_OVF_PMI_T0;
escrvalue &= ~(P4_ESCR_T0_OS|P4_ESCR_T0_USR);
if (P4_CPU_IS_HTT_SECONDARY(cpu)) { /* shift T0 bits to T1 position */
cccrtbits <<= 1;
escrtbits >>= 2;
}
/* start system mode PMCs directly */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(escrmsr, escrvalue | escrtbits);
wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
return 0;
}
/*
* Thread mode PMCs
*
* On HTT machines, the same PMC could be scheduled on the
* same physical CPU twice (once for each logical CPU), for
* example, if two threads of a multi-threaded process get
* scheduled on the same CPU.
*
*/
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 0 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
if (rc == 0) { /* 1st CPU and the non-HTT case */
KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
cpu, ri, pd->pm_cccr_msr));
/* write out the low 40 bits of the saved value to hardware */
wrmsr(pd->pm_pmc_msr,
P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
} else if (rc == 1) { /* 2nd CPU */
/*
* Stop the PMC and retrieve the CCCR and ESCR values
* from their MSRs, and turn on the additional T[0/1]
* bits for the 2nd CPU.
*/
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
/* check that the configuration bits read back match the PMC */
KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
("[p4,%d] Extra CCCR bits cpu=%d rc=%d ri=%d "
"cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,
cccrvalue & P4_CCCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
KASSERT(cccrvalue & P4_CCCR_ENABLE,
("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
__LINE__, rc, cpu, ri));
KASSERT((cccrvalue & cccrtbits) == 0,
("[p4,%d] CCCR T0/T1 mismatch rc=%d cpu=%d ri=%d"
"cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
cccrvalue, cccrtbits));
escrvalue = rdmsr(escrmsr);
KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK),
("[p4,%d] Extra ESCR bits cpu=%d rc=%d ri=%d "
"escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
escrvalue & P4_ESCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
KASSERT((escrvalue & escrtbits) == 0,
("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
"escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
rc, cpu, ri, escrmsr, escrvalue, escrtbits));
}
/* Enable the correct bits for this CPU. */
escrvalue |= escrtbits;
cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
/* Save HW value at the time of starting hardware */
P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);
/* Program the ESCR and CCCR and start the PMC */
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
++rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
mtx_unlock_spin(&pc->pc_mtx);
PMCDBG6(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
"escrmsr=0x%x escrvalue=0x%x", cpu, rc,
ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue);
PMCDBG2(MDP,STA,2,"cccr_config=0x%x v=%jx",
cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));
return (0);
}
int
pmclog_configure_log(struct pmc_mdep *md, struct pmc_owner *po, int logfd)
{
int error;
struct proc *p;
cap_rights_t rights;
/*
* As long as it is possible to get a LOR between pmc_sx lock and
* proctree/allproc sx locks used for adding a new process, assure
* the former is not held here.
*/
sx_assert(&pmc_sx, SA_UNLOCKED);
PMCDBG2(LOG,CFG,1, "config po=%p logfd=%d", po, logfd);
p = po->po_owner;
/* return EBUSY if a log file was already present */
if (po->po_flags & PMC_PO_OWNS_LOGFILE)
return (EBUSY);
KASSERT(po->po_kthread == NULL,
("[pmclog,%d] po=%p kthread (%p) already present", __LINE__, po,
po->po_kthread));
KASSERT(po->po_file == NULL,
("[pmclog,%d] po=%p file (%p) already present", __LINE__, po,
po->po_file));
/* get a reference to the file state */
error = fget_write(curthread, logfd,
cap_rights_init(&rights, CAP_WRITE), &po->po_file);
if (error)
goto error;
/* mark process as owning a log file */
po->po_flags |= PMC_PO_OWNS_LOGFILE;
error = kproc_create(pmclog_loop, po, &po->po_kthread,
RFHIGHPID, 0, "hwpmc: proc(%d)", p->p_pid);
if (error)
goto error;
/* mark process as using HWPMCs */
PROC_LOCK(p);
p->p_flag |= P_HWPMC;
PROC_UNLOCK(p);
/* create a log initialization entry */
PMCLOG_RESERVE_WITH_ERROR(po, INITIALIZE,
sizeof(struct pmclog_initialize));
PMCLOG_EMIT32(PMC_VERSION);
PMCLOG_EMIT32(md->pmd_cputype);
PMCLOG_DESPATCH(po);
return (0);
error:
/* shutdown the thread */
if (po->po_kthread)
pmclog_stop_kthread(po);
KASSERT(po->po_kthread == NULL, ("[pmclog,%d] po=%p kthread not "
"stopped", __LINE__, po));
if (po->po_file)
(void) fdrop(po->po_file, curthread);
po->po_file = NULL; /* clear file and error state */
po->po_error = 0;
return (error);
}
