static int
ucf_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
enum pmc_event ev;
uint32_t caps, flags;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
if (ri < 0 || ri > uncore_ucf_npmc)
return (EINVAL);
caps = a->pm_caps;
if (a->pm_class != PMC_CLASS_UCF ||
(caps & UCF_PMC_CAPS) != caps)
return (EINVAL);
ev = pm->pm_event;
if (ev < PMC_EV_UCF_FIRST || ev > PMC_EV_UCF_LAST)
return (EINVAL);
flags = UCF_EN;
pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4));
PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx",
(uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl);
return (0);
}
static int
mpc7xxx_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct pmc_cpu *pc;
struct powerpc_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] wrong cpu number %d", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"powerpc-init cpu=%d", cpu);
powerpc_pcpu[cpu] = pac = malloc(sizeof(struct powerpc_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_ppcpmcs = malloc(sizeof(struct pmc_hw) * MPC7XXX_MAX_PMCS,
M_PMC, M_WAITOK|M_ZERO);
pac->pc_class = PMC_CLASS_PPC7450;
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_POWERPC].pcd_ri;
KASSERT(pc != NULL, ("[powerpc,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_ppcpmcs; i < MPC7XXX_MAX_PMCS; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/* Clear the MMCRs, and set FC, to disable all PMCs. */
mtspr(SPR_MMCR0, SPR_MMCR0_FC | SPR_MMCR0_PMXE |
SPR_MMCR0_FCECE | SPR_MMCR0_PMC1CE | SPR_MMCR0_PMCNCE);
mtspr(SPR_MMCR1, 0);
return 0;
}
static int
p4_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct p4_cpu *p4c;
struct pmc_cpu *pc;
PMCDBG1(MDP,INI,0, "p4-cleanup cpu=%d", cpu);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
for (i = 0; i < P4_NPMCS; i++)
pc->pc_hwpmcs[i + first_ri] = NULL;
if (!pmc_cpu_is_primary(cpu) && (cpu & 1))
return (0);
p4c = p4_pcpu[cpu];
KASSERT(p4c != NULL, ("[p4,%d] NULL pcpu", __LINE__));
/* Turn off all PMCs on this CPU */
for (i = 0; i < P4_NPMCS - 1; i++)
wrmsr(P4_CCCR_MSR_FIRST + i,
rdmsr(P4_CCCR_MSR_FIRST + i) & ~P4_CCCR_ENABLE);
mtx_destroy(&p4c->pc_mtx);
free(p4c, M_PMC);
p4_pcpu[cpu] = NULL;
return (0);
}
static int
ucf_stop_pmc(int cpu, int ri)
{
uint32_t fc;
struct uncore_cpu *ucfc;
PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
fc = (UCF_MASK << (ri * 4));
ucfc->pc_ucfctrl &= ~fc;
PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
void
pmclog_process_pmcallocate(struct pmc *pm)
{
struct pmc_owner *po;
struct pmc_soft *ps;
po = pm->pm_owner;
PMCDBG1(LOG,ALL,1, "pm=%p", pm);
if (PMC_TO_CLASS(pm) == PMC_CLASS_SOFT) {
PMCLOG_RESERVE(po, PMCALLOCATEDYN,
sizeof(struct pmclog_pmcallocatedyn));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
ps = pmc_soft_ev_acquire(pm->pm_event);
if (ps != NULL)
PMCLOG_EMITSTRING(ps->ps_ev.pm_ev_name,PMC_NAME_MAX);
else
PMCLOG_EMITNULLSTRING(PMC_NAME_MAX);
pmc_soft_ev_release(ps);
PMCLOG_DESPATCH(po);
} else {
PMCLOG_RESERVE(po, PMCALLOCATE,
sizeof(struct pmclog_pmcallocate));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(pm->pm_event);
PMCLOG_EMIT32(pm->pm_flags);
PMCLOG_DESPATCH(po);
}
}
int
pmclog_close(struct pmc_owner *po)
{
PMCDBG1(LOG,CLO,1, "po=%p", po);
mtx_lock(&pmc_kthread_mtx);
/*
* Schedule the current buffer.
*/
mtx_lock_spin(&po->po_mtx);
if (po->po_curbuf)
pmclog_schedule_io(po);
else
wakeup_one(po);
mtx_unlock_spin(&po->po_mtx);
/*
* Initiate shutdown: no new data queued,
* thread will close file on last block.
*/
po->po_flags |= PMC_PO_SHUTDOWN;
mtx_unlock(&pmc_kthread_mtx);
return (0);
}
static void
pmclog_schedule_io(struct pmc_owner *po)
{
KASSERT(po->po_curbuf != NULL,
("[pmclog,%d] schedule_io with null buffer po=%p", __LINE__, po));
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
PMCDBG1(LOG,SIO, 1, "po=%p", po);
mtx_assert(&po->po_mtx, MA_OWNED);
/*
* Add the current buffer to the tail of the buffer list and
* wakeup the helper.
*/
TAILQ_INSERT_TAIL(&po->po_logbuffers, po->po_curbuf, plb_next);
po->po_curbuf = NULL;
wakeup_one(po);
}
static int
p4_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
struct p4_cpu *pc;
enum pmc_mode mode;
struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));
KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm),
("[p4,%d] class mismatch pd %d != id class %d", __LINE__,
pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
mode = PMC_TO_MODE(pm);
PMCDBG3(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
if (PMC_IS_VIRTUAL_MODE(mode)) {
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */
tmp += (P4_PERFCTR_MASK + 1) -
P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);
}
if (PMC_IS_SAMPLING_MODE(mode)) /* undo transformation */
*v = P4_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
PMCDBG1(MDP,REA,2, "p4-read -> %jx", *v);
return (0);
}
int
pmclog_deconfigure_log(struct pmc_owner *po)
{
int error;
struct pmclog_buffer *lb;
PMCDBG1(LOG,CFG,1, "de-config po=%p", po);
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
return (EINVAL);
KASSERT(po->po_sscount == 0,
("[pmclog,%d] po=%p still owning SS PMCs", __LINE__, po));
KASSERT(po->po_file != NULL,
("[pmclog,%d] po=%p no log file", __LINE__, po));
/* stop the kthread, this will reset the 'OWNS_LOGFILE' flag */
pmclog_stop_kthread(po);
KASSERT(po->po_kthread == NULL,
("[pmclog,%d] po=%p kthread not stopped", __LINE__, po));
/* return all queued log buffers to the global pool */
while ((lb = TAILQ_FIRST(&po->po_logbuffers)) != NULL) {
TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/* return the 'current' buffer to the global pool */
if ((lb = po->po_curbuf) != NULL) {
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/* drop a reference to the fd */
if (po->po_file != NULL) {
error = fdrop(po->po_file, curthread);
po->po_file = NULL;
} else
error = 0;
po->po_error = 0;
return (error);
}
static void
pmclog_release(struct pmc_owner *po)
{
KASSERT(po->po_curbuf->plb_ptr >= po->po_curbuf->plb_base,
("[pmclog,%d] buffer invariants po=%p ptr=%p base=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_base));
KASSERT(po->po_curbuf->plb_ptr <= po->po_curbuf->plb_fence,
("[pmclog,%d] buffer invariants po=%p ptr=%p fenc=%p", __LINE__,
po, po->po_curbuf->plb_ptr, po->po_curbuf->plb_fence));
/* schedule an I/O if we've filled a buffer */
if (po->po_curbuf->plb_ptr >= po->po_curbuf->plb_fence)
pmclog_schedule_io(po);
mtx_unlock_spin(&po->po_mtx);
PMCDBG1(LOG,REL,1, "po=%p", po);
}
int
pmclog_flush(struct pmc_owner *po)
{
int error;
struct pmclog_buffer *lb;
PMCDBG1(LOG,FLS,1, "po=%p", po);
/*
* If there is a pending error recorded by the logger thread,
* return that.
*/
if (po->po_error)
return (po->po_error);
error = 0;
/*
* Check that we do have an active log file.
*/
mtx_lock(&pmc_kthread_mtx);
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0) {
error = EINVAL;
goto error;
}
/*
* Schedule the current buffer if any and not empty.
*/
mtx_lock_spin(&po->po_mtx);
lb = po->po_curbuf;
if (lb && lb->plb_ptr != lb->plb_base) {
pmclog_schedule_io(po);
} else
error = ENOBUFS;
mtx_unlock_spin(&po->po_mtx);
error:
mtx_unlock(&pmc_kthread_mtx);
return (error);
}
static int
uncore_pcpu_fini(struct pmc_mdep *md, int cpu)
{
int uncore_ri, n, npmc;
struct pmc_cpu *pc;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] insane cpu number (%d)", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu);
if ((cc = uncore_pcpu[cpu]) == NULL)
return (0);
uncore_pcpu[cpu] = NULL;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__,
cpu));
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;
uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;
for (n = 0; n < npmc; n++)
wrmsr(SELECTSEL(uncore_cputype) + n, 0);
wrmsr(UCF_CTRL, 0);
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;
for (n = 0; n < npmc; n++)
pc->pc_hwpmcs[n + uncore_ri] = NULL;
free(cc, M_PMC);
return (0);
}
static int
arm64_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct arm64_cpu *pac;
struct pmc_hw *phw;
struct pmc_cpu *pc;
uint64_t pmcr;
int first_ri;
int i;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] wrong cpu number %d", __LINE__, cpu));
PMCDBG1(MDP, INI, 1, "arm64-init cpu=%d", cpu);
arm64_pcpu[cpu] = pac = malloc(sizeof(struct arm64_cpu), M_PMC,
M_WAITOK | M_ZERO);
pac->pc_arm64pmcs = malloc(sizeof(struct pmc_hw) * arm64_npmcs,
M_PMC, M_WAITOK | M_ZERO);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8].pcd_ri;
KASSERT(pc != NULL, ("[arm64,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_arm64pmcs; i < arm64_npmcs; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/* Enable unit */
pmcr = arm64_pmcr_read();
pmcr |= PMCR_E;
arm64_pmcr_write(pmcr);
return (0);
}
static int
uncore_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct uncore_cpu *cc;
struct pmc_hw *phw;
int uncore_ri, n, npmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[ucf,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu);
uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;
cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw),
M_PMC, M_WAITOK | M_ZERO);
uncore_pcpu[cpu] = cc;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL && cc != NULL,
("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) |
PMC_PHW_INDEX_TO_STATE(n + uncore_ri);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + uncore_ri] = phw;
}
return (0);
}
struct pmc_mdep *
pmc_arm64_initialize()
{
struct pmc_mdep *pmc_mdep;
struct pmc_classdep *pcd;
int idcode;
int reg;
reg = arm64_pmcr_read();
arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT;
idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT;
PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs);
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
*/
arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(),
M_PMC, M_WAITOK | M_ZERO);
/* Just one class */
pmc_mdep = pmc_mdep_alloc(1);
switch (idcode) {
case PMCR_IDCODE_CORTEX_A57:
case PMCR_IDCODE_CORTEX_A72:
pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57;
break;
default:
case PMCR_IDCODE_CORTEX_A53:
pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
break;
}
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8];
pcd->pcd_caps = ARMV8_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_ARMV8;
pcd->pcd_num = arm64_npmcs;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 32;
pcd->pcd_allocate_pmc = arm64_allocate_pmc;
pcd->pcd_config_pmc = arm64_config_pmc;
pcd->pcd_pcpu_fini = arm64_pcpu_fini;
pcd->pcd_pcpu_init = arm64_pcpu_init;
pcd->pcd_describe = arm64_describe;
pcd->pcd_get_config = arm64_get_config;
pcd->pcd_read_pmc = arm64_read_pmc;
pcd->pcd_release_pmc = arm64_release_pmc;
pcd->pcd_start_pmc = arm64_start_pmc;
pcd->pcd_stop_pmc = arm64_stop_pmc;
pcd->pcd_write_pmc = arm64_write_pmc;
pmc_mdep->pmd_intr = arm64_intr;
pmc_mdep->pmd_switch_in = arm64_switch_in;
pmc_mdep->pmd_switch_out = arm64_switch_out;
pmc_mdep->pmd_npmc += arm64_npmcs;
return (pmc_mdep);
}
