static int
p4_describe(int cpu, int ri, struct pmc_info *pi,
struct pmc **ppmc)
{
int error;
size_t copied;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] row-index %d out of range", __LINE__, ri));
PMCDBG(MDP,OPS,1,"p4-describe cpu=%d ri=%d", cpu, ri);
if (P4_CPU_IS_HTT_SECONDARY(cpu))
return (EINVAL);
pd = &p4_pmcdesc[ri];
if ((error = copystr(pd->pm_descr.pd_name, pi->pm_name,
PMC_NAME_MAX, &copied)) != 0)
return (error);
pi->pm_class = pd->pm_descr.pd_class;
if (p4_pcpu[cpu]->pc_p4pmcs[ri].phw_state & PMC_PHW_FLAG_IS_ENABLED) {
pi->pm_enabled = TRUE;
*ppmc = p4_pcpu[cpu]->pc_p4pmcs[ri].phw_pmc;
} else {
pi->pm_enabled = FALSE;
*ppmc = NULL;
}
return (0);
}
static int
mips24k_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, i;
struct pmc_cpu *pc;
struct mips24k_cpu *pac;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] wrong cpu number %d", __LINE__, cpu));
PMCDBG(MDP,INI,1,"mips-init cpu=%d", cpu);
mips24k_pcpu[cpu] = pac = malloc(sizeof(struct mips24k_cpu), M_PMC,
M_WAITOK|M_ZERO);
pac->pc_mipspmcs = malloc(sizeof(struct pmc_hw) * mips24k_npmcs,
M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_MIPS24K].pcd_ri;
KASSERT(pc != NULL, ("[mips,%d] NULL per-cpu pointer", __LINE__));
for (i = 0, phw = pac->pc_mipspmcs; i < mips24k_npmcs; i++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(i);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[i + first_ri] = phw;
}
/*
* Clear the counter control register which has the effect
* of disabling counting.
*/
for (i = 0; i < mips24k_npmcs; i++)
mips24k_pmcn_write(i, 0);
return 0;
}
static int
soft_pcpu_init(struct pmc_mdep *md, int cpu)
{
int first_ri, n;
struct pmc_cpu *pc;
struct soft_cpu *soft_pc;
struct pmc_hw *phw;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[soft,%d] illegal cpu %d", __LINE__, cpu));
KASSERT(soft_pcpu, ("[soft,%d] null pcpu", __LINE__));
KASSERT(soft_pcpu[cpu] == NULL, ("[soft,%d] non-null per-cpu",
__LINE__));
soft_pc = malloc(sizeof(struct soft_cpu), M_PMC, M_WAITOK|M_ZERO);
if (soft_pc == NULL)
return (ENOMEM);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[soft,%d] cpu %d null per-cpu", __LINE__, cpu));
soft_pcpu[cpu] = soft_pc;
phw = soft_pc->soft_hw;
first_ri = md->pmd_classdep[PMC_CLASS_INDEX_SOFT].pcd_ri;
for (n = 0; n < SOFT_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
return (0);
}
static int
uncore_pcpu_init(struct pmc_mdep *md, int cpu)
{
struct pmc_cpu *pc;
struct uncore_cpu *cc;
struct pmc_hw *phw;
int uncore_ri, n, npmc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[ucf,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu);
uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;
npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;
npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;
cc = malloc(sizeof(struct uncore_cpu) + npmc * sizeof(struct pmc_hw),
M_PMC, M_WAITOK | M_ZERO);
uncore_pcpu[cpu] = cc;
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL && cc != NULL,
("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) |
PMC_PHW_INDEX_TO_STATE(n + uncore_ri);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + uncore_ri] = phw;
}
return (0);
}
static int
ucp_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int n;
enum pmc_event ev;
struct ucp_event_descr *ie;
uint32_t caps, config, cpuflag, evsel;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index value %d", __LINE__, ri));
/* check requested capabilities */
caps = a->pm_caps;
if ((UCP_PMC_CAPS & caps) != caps)
return (EPERM);
ev = pm->pm_event;
switch (uncore_cputype) {
case PMC_CPU_INTEL_HASWELL:
case PMC_CPU_INTEL_SANDYBRIDGE:
if (ucp_event_sb_hw_ok_on_counter(ev, ri) == 0)
return (EINVAL);
break;
default:
break;
}
/*
* Look for an event descriptor with matching CPU and event id
* fields.
*/
switch (uncore_cputype) {
case PMC_CPU_INTEL_COREI7:
cpuflag = UCP_F_I7;
break;
case PMC_CPU_INTEL_HASWELL:
cpuflag = UCP_F_HW;
break;
case PMC_CPU_INTEL_SANDYBRIDGE:
cpuflag = UCP_F_SB;
break;
case PMC_CPU_INTEL_WESTMERE:
cpuflag = UCP_F_WM;
break;
default:
return (EINVAL);
}
for (n = 0, ie = ucp_events; n < nucp_events; n++, ie++)
if (ie->ucp_ev == ev && ie->ucp_flags & cpuflag)
break;
if (n == nucp_events)
return (EINVAL);
/*
* A matching event descriptor has been found, so start
* assembling the contents of the event select register.
*/
evsel = ie->ucp_evcode | UCP_EN;
config = a->pm_md.pm_ucp.pm_ucp_config & ~UCP_F_CMASK;
/*
* If the event uses a fixed umask value, reject any umask
* bits set by the user.
*/
if (ie->ucp_flags & UCP_F_FM) {
if (UCP_UMASK(config) != 0)
return (EINVAL);
evsel |= (ie->ucp_umask << 8);
} else
return (EINVAL);
if (caps & PMC_CAP_THRESHOLD)
evsel |= (a->pm_md.pm_ucp.pm_ucp_config & UCP_F_CMASK);
if (caps & PMC_CAP_EDGE)
evsel |= UCP_EDGE;
if (caps & PMC_CAP_INVERT)
evsel |= UCP_INV;
pm->pm_md.pm_ucp.pm_ucp_evsel = evsel;
return (0);
}
struct pmc_mdep *
pmc_arm64_initialize()
{
struct pmc_mdep *pmc_mdep;
struct pmc_classdep *pcd;
int idcode;
int reg;
reg = arm64_pmcr_read();
arm64_npmcs = (reg & PMCR_N_MASK) >> PMCR_N_SHIFT;
idcode = (reg & PMCR_IDCODE_MASK) >> PMCR_IDCODE_SHIFT;
PMCDBG1(MDP, INI, 1, "arm64-init npmcs=%d", arm64_npmcs);
/*
* Allocate space for pointers to PMC HW descriptors and for
* the MDEP structure used by MI code.
*/
arm64_pcpu = malloc(sizeof(struct arm64_cpu *) * pmc_cpu_max(),
M_PMC, M_WAITOK | M_ZERO);
/* Just one class */
pmc_mdep = pmc_mdep_alloc(1);
switch (idcode) {
case PMCR_IDCODE_CORTEX_A57:
case PMCR_IDCODE_CORTEX_A72:
pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A57;
break;
default:
case PMCR_IDCODE_CORTEX_A53:
pmc_mdep->pmd_cputype = PMC_CPU_ARMV8_CORTEX_A53;
break;
}
pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_ARMV8];
pcd->pcd_caps = ARMV8_PMC_CAPS;
pcd->pcd_class = PMC_CLASS_ARMV8;
pcd->pcd_num = arm64_npmcs;
pcd->pcd_ri = pmc_mdep->pmd_npmc;
pcd->pcd_width = 32;
pcd->pcd_allocate_pmc = arm64_allocate_pmc;
pcd->pcd_config_pmc = arm64_config_pmc;
pcd->pcd_pcpu_fini = arm64_pcpu_fini;
pcd->pcd_pcpu_init = arm64_pcpu_init;
pcd->pcd_describe = arm64_describe;
pcd->pcd_get_config = arm64_get_config;
pcd->pcd_read_pmc = arm64_read_pmc;
pcd->pcd_release_pmc = arm64_release_pmc;
pcd->pcd_start_pmc = arm64_start_pmc;
pcd->pcd_stop_pmc = arm64_stop_pmc;
pcd->pcd_write_pmc = arm64_write_pmc;
pmc_mdep->pmd_intr = arm64_intr;
pmc_mdep->pmd_switch_in = arm64_switch_in;
pmc_mdep->pmd_switch_out = arm64_switch_out;
pmc_mdep->pmd_npmc += arm64_npmcs;
return (pmc_mdep);
}
static int
mpc7xxx_intr(int cpu, struct trapframe *tf)
{
int i, error, retval;
uint32_t config;
struct pmc *pm;
struct powerpc_cpu *pac;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC;
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
for (i = 0; i < MPC7XXX_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!MPC7XXX_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the counter if logging fails. */
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error != 0)
mpc7xxx_stop_pmc(cpu, i);
/* reload count. */
mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
static int
mips_pmc_intr(int cpu, struct trapframe *tf)
{
int error;
int retval, ri;
struct pmc *pm;
struct mips_cpu *pc;
uint32_t r0, r2;
pmc_value_t r;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] CPU %d out of range", __LINE__, cpu));
retval = 0;
pc = mips_pcpu[cpu];
/* Stop PMCs without clearing the counter */
r0 = mips_rd_perfcnt0();
mips_wr_perfcnt0(r0 & ~(0x1f));
r2 = mips_rd_perfcnt2();
mips_wr_perfcnt2(r2 & ~(0x1f));
for (ri = 0; ri < mips_npmcs; ri++) {
pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
if (pm == NULL)
continue;
if (! PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
r = mips_pmcn_read(ri);
/* If bit 31 is set, the counter has overflowed */
if ((r & (1UL << (mips_pmc_spec.ps_counter_width - 1))) == 0)
continue;
retval = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error) {
/* Clear/disable the relevant counter */
if (ri == 0)
r0 = 0;
else if (ri == 1)
r2 = 0;
mips_stop_pmc(cpu, ri);
}
/* Reload sampling count */
mips_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount);
}
/*
* Re-enable the PMC counters where they left off.
*
* Any counter which overflowed will have its sample count
* reloaded in the loop above.
*/
mips_wr_perfcnt0(r0);
mips_wr_perfcnt2(r2);
return retval;
}
static int
p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int found, n, m;
uint32_t caps, cccrvalue, escrvalue, tflags;
enum pmc_p4escr escr;
struct p4_cpu *pc;
struct p4_event_descr *pevent;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index value %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
PMCDBG(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
"reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
pm->pm_caps);
/* check class */
if (pd->pm_descr.pd_class != a->pm_class)
return (EINVAL);
/* check requested capabilities */
caps = a->pm_caps;
if ((pd->pm_descr.pd_caps & caps) != caps)
return (EPERM);
/*
* If the system has HTT enabled, and the desired allocation
* mode is process-private, and the PMC row disposition is not
* FREE (0), decline the allocation.
*/
if (p4_system_has_htt &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
pmc_getrowdisp(ri) != 0)
return (EBUSY);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__,
pd->pm_descr.pd_class));
if (pm->pm_event < PMC_EV_P4_FIRST ||
pm->pm_event > PMC_EV_P4_LAST)
return (EINVAL);
if ((pevent = p4_find_event(pm->pm_event)) == NULL)
return (ESRCH);
PMCDBG(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}",
pevent->pm_event, pevent->pm_escr_eventselect,
pevent->pm_cccr_select, pevent->pm_is_ti_event);
/*
* Some PMC events are 'thread independent'and therefore
* cannot be used for process-private modes if HTT is being
* used.
*/
if (P4_EVENT_IS_TI(pevent) &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
p4_system_has_htt)
return (EINVAL);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
found = 0;
/* look for a suitable ESCR for this event */
for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) {
if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE)
break; /* out of ESCRs */
/*
* Check ESCR row disposition.
*
* If the request is for a system-mode PMC, then the
* ESCR row should not be in process-virtual mode, and
* should also be free on the current CPU.
*/
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
continue;
}
/*
* If the request is for a process-virtual PMC, and if
* HTT is not enabled, we can use an ESCR row that is
* either FREE or already in process mode.
*
* If HTT is enabled, then we need to ensure that a
* given ESCR is never allocated to two PMCS that
* could run simultaneously on the two logical CPUs of
* a CPU package. We ensure this be only allocating
* ESCRs from rows marked as 'FREE'.
*/
if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
if (p4_system_has_htt) {
if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
continue;
} else
if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr))
continue;
}
/*
* We found a suitable ESCR for this event. Now check if
* this escr can work with the PMC at row-index 'ri'.
*/
for (m = 0; m < P4_MAX_PMC_PER_ESCR; m++)
if (p4_escrs[escr].pm_pmcs[m] == pd->pm_pmcnum) {
found = 1;
break;
}
}
if (found == 0)
return (ESRCH);
KASSERT((int) escr >= 0 && escr < P4_NESCR,
("[p4,%d] illegal ESCR value %d", __LINE__, escr));
/* mark ESCR row mode */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
P4_ESCR_MARK_ROW_STANDALONE(escr);
} else {
KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX,
("[p4,%d] escr[%d] already in use", __LINE__, escr));
P4_ESCR_MARK_ROW_THREAD(escr);
}
pm->pm_md.pm_p4.pm_p4_escrmsr = p4_escrs[escr].pm_escr_msr;
pm->pm_md.pm_p4.pm_p4_escr = escr;
cccrvalue = P4_CCCR_TO_ESCR_SELECT(pevent->pm_cccr_select);
escrvalue = P4_ESCR_TO_EVENT_SELECT(pevent->pm_escr_eventselect);
/* CCCR fields */
if (caps & PMC_CAP_THRESHOLD)
cccrvalue |= (a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_THRESHOLD_MASK) | P4_CCCR_COMPARE;
if (caps & PMC_CAP_EDGE)
cccrvalue |= P4_CCCR_EDGE;
if (caps & PMC_CAP_INVERT)
cccrvalue |= P4_CCCR_COMPLEMENT;
if (p4_system_has_htt)
cccrvalue |= a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_ACTIVE_THREAD_MASK;
else /* no HTT; thread field should be '11b' */
cccrvalue |= P4_CCCR_TO_ACTIVE_THREAD(0x3);
if (caps & PMC_CAP_CASCADE)
cccrvalue |= P4_CCCR_CASCADE;
/* On HTT systems the PMI T0 field may get moved to T1 at pmc start */
if (caps & PMC_CAP_INTERRUPT)
cccrvalue |= P4_CCCR_OVF_PMI_T0;
/* ESCR fields */
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK;
if (caps & PMC_CAP_TAGGING)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_TAG_VALUE_MASK) | P4_ESCR_TAG_ENABLE;
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK);
/* HTT: T0_{OS,USR} bits may get moved to T1 at pmc start */
tflags = 0;
if (caps & PMC_CAP_SYSTEM)
tflags |= P4_ESCR_T0_OS;
if (caps & PMC_CAP_USER)
tflags |= P4_ESCR_T0_USR;
if (tflags == 0)
tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
escrvalue |= tflags;
pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue;
pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
PMCDBG(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
"escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select,
cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
return (0);
}
static int
p4_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
struct p4_cpu *pc;
int cfgflags, cpuflag;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
PMCDBG(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
phw = &pc->pc_p4pmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL ||
(p4_system_has_htt && phw->phw_pmc == pm),
("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
mtx_lock_spin(&pc->pc_mtx);
cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal cfgflags cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
KASSERT(cfgflags == 0 || phw->phw_pmc,
("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
__LINE__, cpu, ri));
cpuflag = P4_CPU_TO_FLAG(cpu);
if (pm) { /* config */
if (cfgflags == 0)
phw->phw_pmc = pm;
KASSERT(phw->phw_pmc == pm,
("[p4,%d] cpu=%d ri=%d config %p != hw %p",
__LINE__, cpu, ri, pm, phw->phw_pmc));
cfgflags |= cpuflag;
} else { /* unconfig */
cfgflags &= ~cpuflag;
if (cfgflags == 0)
phw->phw_pmc = NULL;
}
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
mtx_unlock_spin(&pc->pc_mtx);
return (0);
}
static int
p4_pcpu_init(struct pmc_mdep *md, int cpu)
{
char *pescr;
int n, first_ri, phycpu;
struct pmc_hw *phw;
struct p4_cpu *p4c;
struct pmc_cpu *pc, *plc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,
pmc_cpu_is_primary(cpu) != 0);
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
/*
* The two CPUs in an HT pair share their per-cpu state.
*
* For HT capable CPUs, we assume that the two logical
* processors in the HT pair get two consecutive CPU ids
* starting with an even id #.
*
* The primary CPU (the even numbered CPU of the pair) would
* have been initialized prior to the initialization for the
* secondary.
*/
if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) {
p4_system_has_htt = 1;
phycpu = P4_TO_HTT_PRIMARY(cpu);
pc = pmc_pcpu[phycpu];
plc = pmc_pcpu[cpu];
KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
PMCDBG(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
phycpu, pc);
KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
__LINE__, cpu, phycpu));
/* PMCs are shared with the physical CPU. */
for (n = 0; n < P4_NPMCS; n++)
plc->pc_hwpmcs[n + first_ri] =
pc->pc_hwpmcs[n + first_ri];
return (0);
}
p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
if (p4c == NULL)
return (ENOMEM);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
p4_pcpu[cpu] = p4c;
phw = p4c->pc_p4pmcs;
for (n = 0; n < P4_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
pescr = p4c->pc_escrs;
for (n = 0; n < P4_NESCR; n++)
*pescr++ = P4_INVALID_PMC_INDEX;
mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
return (0);
}
static int
p4_stop_pmc(int cpu, int ri)
{
int rc;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row index %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
KASSERT(pm != NULL,
("[p4,%d] null pmc for cpu%d, ri%d", __LINE__, cpu, ri));
PMCDBG(MDP,STO,1, "p4-stop cpu=%d ri=%d", cpu, ri);
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(pd->pm_cccr_msr,
pm->pm_md.pm_p4.pm_p4_cccrvalue & ~P4_CCCR_ENABLE);
return (0);
}
/*
* Thread mode PMCs.
*
* On HTT machines, this PMC may be in use by two threads
* running on two logical CPUS. Thus we look at the
* 'runcount' field and only turn off the appropriate TO/T1
* bits (and keep the PMC running) if two logical CPUs were
* using the PMC.
*
*/
/* bits to mask */
cccrtbits = P4_CCCR_OVF_PMI_T0;
escrtbits = P4_ESCR_T0_OS | P4_ESCR_T0_USR;
if (P4_CPU_IS_HTT_SECONDARY(cpu)) {
cccrtbits <<= 1;
escrtbits >>= 2;
}
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 2 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
--rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
/* Stop this PMC */
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
escrvalue = rdmsr(escrmsr);
/* The current CPU should be running on this PMC */
KASSERT(escrvalue & escrtbits,
("[p4,%d] ESCR T0/T1 mismatch cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrvalue=0x%x tbits=0x%x", __LINE__, cpu, rc, ri, escrmsr,
escrvalue, escrtbits));
KASSERT(PMC_IS_COUNTING_MODE(PMC_TO_MODE(pm)) ||
(cccrvalue & cccrtbits),
("[p4,%d] CCCR T0/T1 mismatch cpu=%d ri=%d cccrvalue=0x%x "
"tbits=0x%x", __LINE__, cpu, ri, cccrvalue, cccrtbits));
/* get the current hardware reading */
tmp = rdmsr(pd->pm_pmc_msr);
if (rc == 1) { /* need to keep the PMC running */
escrvalue &= ~escrtbits;
cccrvalue &= ~cccrtbits;
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
}
mtx_unlock_spin(&pc->pc_mtx);
PMCDBG(MDP,STO,2, "p4-stop cpu=%d rc=%d ri=%d escrmsr=0x%x "
"escrval=0x%x cccrval=0x%x v=%jx", cpu, rc, ri, escrmsr,
escrvalue, cccrvalue, tmp);
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit counter overflow */
tmp += (P4_PERFCTR_MASK + 1) - P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
P4_PCPU_PMC_VALUE(pc,ri,cpu) += tmp;
return 0;
}
static int
p4_start_pmc(int cpu, int ri)
{
int rc;
struct pmc *pm;
struct p4_cpu *pc;
struct p4pmc_descr *pd;
uint32_t cccrvalue, cccrtbits, escrvalue, escrmsr, escrtbits;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] starting cpu%d,pmc%d with null pmc", __LINE__, cpu, ri));
PMCDBG(MDP,STA,1, "p4-start cpu=%d ri=%d", cpu, ri);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] wrong PMC class %d", __LINE__,
pd->pm_descr.pd_class));
/* retrieve the desired CCCR/ESCR values from the PMC */
cccrvalue = pm->pm_md.pm_p4.pm_p4_cccrvalue;
escrvalue = pm->pm_md.pm_p4.pm_p4_escrvalue;
escrmsr = pm->pm_md.pm_p4.pm_p4_escrmsr;
/* extract and zero the logical processor selection bits */
cccrtbits = cccrvalue & P4_CCCR_OVF_PMI_T0;
escrtbits = escrvalue & (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
cccrvalue &= ~P4_CCCR_OVF_PMI_T0;
escrvalue &= ~(P4_ESCR_T0_OS|P4_ESCR_T0_USR);
if (P4_CPU_IS_HTT_SECONDARY(cpu)) { /* shift T0 bits to T1 position */
cccrtbits <<= 1;
escrtbits >>= 2;
}
/* start system mode PMCs directly */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
wrmsr(escrmsr, escrvalue | escrtbits);
wrmsr(pd->pm_cccr_msr, cccrvalue | cccrtbits | P4_CCCR_ENABLE);
return 0;
}
/*
* Thread mode PMCs
*
* On HTT machines, the same PMC could be scheduled on the
* same physical CPU twice (once for each logical CPU), for
* example, if two threads of a multi-threaded process get
* scheduled on the same CPU.
*
*/
mtx_lock_spin(&pc->pc_mtx);
rc = P4_PCPU_GET_RUNCOUNT(pc,ri);
KASSERT(rc == 0 || rc == 1,
("[p4,%d] illegal runcount cpu=%d ri=%d rc=%d", __LINE__, cpu, ri,
rc));
if (rc == 0) { /* 1st CPU and the non-HTT case */
KASSERT(P4_PMC_IS_STOPPED(pd->pm_cccr_msr),
("[p4,%d] cpu=%d ri=%d cccr=0x%x not stopped", __LINE__,
cpu, ri, pd->pm_cccr_msr));
/* write out the low 40 bits of the saved value to hardware */
wrmsr(pd->pm_pmc_msr,
P4_PCPU_PMC_VALUE(pc,ri,cpu) & P4_PERFCTR_MASK);
} else if (rc == 1) { /* 2nd CPU */
/*
* Stop the PMC and retrieve the CCCR and ESCR values
* from their MSRs, and turn on the additional T[0/1]
* bits for the 2nd CPU.
*/
cccrvalue = rdmsr(pd->pm_cccr_msr);
wrmsr(pd->pm_cccr_msr, cccrvalue & ~P4_CCCR_ENABLE);
/* check that the configuration bits read back match the PMC */
KASSERT((cccrvalue & P4_CCCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK),
("[p4,%d] Extra CCCR bits cpu=%d rc=%d ri=%d "
"cccr=0x%x PMC=0x%x", __LINE__, cpu, rc, ri,
cccrvalue & P4_CCCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_cccrvalue & P4_CCCR_Tx_MASK));
KASSERT(cccrvalue & P4_CCCR_ENABLE,
("[p4,%d] 2nd cpu rc=%d cpu=%d ri=%d not running",
__LINE__, rc, cpu, ri));
KASSERT((cccrvalue & cccrtbits) == 0,
("[p4,%d] CCCR T0/T1 mismatch rc=%d cpu=%d ri=%d"
"cccrvalue=0x%x tbits=0x%x", __LINE__, rc, cpu, ri,
cccrvalue, cccrtbits));
escrvalue = rdmsr(escrmsr);
KASSERT((escrvalue & P4_ESCR_Tx_MASK) ==
(pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK),
("[p4,%d] Extra ESCR bits cpu=%d rc=%d ri=%d "
"escr=0x%x pm=0x%x", __LINE__, cpu, rc, ri,
escrvalue & P4_ESCR_Tx_MASK,
pm->pm_md.pm_p4.pm_p4_escrvalue & P4_ESCR_Tx_MASK));
KASSERT((escrvalue & escrtbits) == 0,
("[p4,%d] ESCR T0/T1 mismatch rc=%d cpu=%d ri=%d "
"escrmsr=0x%x escrvalue=0x%x tbits=0x%x", __LINE__,
rc, cpu, ri, escrmsr, escrvalue, escrtbits));
}
/* Enable the correct bits for this CPU. */
escrvalue |= escrtbits;
cccrvalue |= cccrtbits | P4_CCCR_ENABLE;
/* Save HW value at the time of starting hardware */
P4_PCPU_HW_VALUE(pc,ri,cpu) = rdmsr(pd->pm_pmc_msr);
/* Program the ESCR and CCCR and start the PMC */
wrmsr(escrmsr, escrvalue);
wrmsr(pd->pm_cccr_msr, cccrvalue);
++rc;
P4_PCPU_SET_RUNCOUNT(pc,ri,rc);
mtx_unlock_spin(&pc->pc_mtx);
PMCDBG(MDP,STA,2,"p4-start cpu=%d rc=%d ri=%d escr=%d "
"escrmsr=0x%x escrvalue=0x%x cccr_config=0x%x v=%jx", cpu, rc,
ri, pm->pm_md.pm_p4.pm_p4_escr, escrmsr, escrvalue,
cccrvalue, P4_PCPU_HW_VALUE(pc,ri,cpu));
return (0);
}
