static int
ucp_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu,
ri));
tmp = rdmsr(UCP_PMC0 + ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = ucp_perfctr_value_to_reload_count(tmp);
else
*v = tmp;
PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
ri, *v);
return (0);
}
static int
ucf_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct uncore_cpu *cc;
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri));
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = ucf_reload_count_to_perfctr_value(v);
wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */
wrmsr(UCF_CTR0 + ri, v);
wrmsr(UCF_CTRL, cc->pc_ucfctrl);
PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ",
cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL));
return (0);
}
static int
ucf_stop_pmc(int cpu, int ri)
{
uint32_t fc;
struct uncore_cpu *ucfc;
PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
fc = (UCF_MASK << (ri * 4));
ucfc->pc_ucfctrl &= ~fc;
PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucf_start_pmc(int cpu, int ri)
{
struct pmc *pm;
struct uncore_cpu *ucfc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri);
ucfc = uncore_pcpu[cpu];
pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl;
wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);
do {
ucfc->pc_resync = 0;
ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype)));
wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);
} while (ucfc->pc_resync != 0);
PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)",
ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),
ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));
return (0);
}
static int
ucp_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__,
cpu, ri));
PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
UCP_PMC0 + ri, v);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = ucp_reload_count_to_perfctr_value(v);
/*
* Write the new value to the counter. The counter will be in
* a stopped state when the pcd_write() entry point is called.
*/
wrmsr(UCP_PMC0 + ri, v);
return (0);
}
static int
ucp_start_pmc(int cpu, int ri)
{
struct pmc *pm;
uint32_t evsel;
struct uncore_cpu *cc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
cc = uncore_pcpu[cpu];
pm = cc->pc_uncorepmcs[ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] starting cpu%d,ri%d with no pmc configured",
__LINE__, cpu, ri));
PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri);
evsel = pm->pm_md.pm_ucp.pm_ucp_evsel;
PMCDBG4(MDP,STA,2,
"ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel);
/* Event specific configuration. */
switch (pm->pm_event) {
case PMC_EV_UCP_EVENT_0CH_04H_E:
case PMC_EV_UCP_EVENT_0CH_08H_E:
wrmsr(MSR_GQ_SNOOP_MESF,0x2);
break;
case PMC_EV_UCP_EVENT_0CH_04H_F:
case PMC_EV_UCP_EVENT_0CH_08H_F:
wrmsr(MSR_GQ_SNOOP_MESF,0x8);
break;
case PMC_EV_UCP_EVENT_0CH_04H_M:
case PMC_EV_UCP_EVENT_0CH_08H_M:
wrmsr(MSR_GQ_SNOOP_MESF,0x1);
break;
case PMC_EV_UCP_EVENT_0CH_04H_S:
case PMC_EV_UCP_EVENT_0CH_08H_S:
wrmsr(MSR_GQ_SNOOP_MESF,0x4);
break;
default:
break;
}
wrmsr(SELECTSEL(uncore_cputype) + ri, evsel);
do {
cc->pc_resync = 0;
cc->pc_globalctrl |= (1ULL << ri);
wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);
} while (cc->pc_resync != 0);
return (0);
}
static int
p4_write_pmc(int cpu, int ri, pmc_value_t v)
{
enum pmc_mode mode;
struct pmc *pm;
struct p4_cpu *pc;
const struct pmc_hw *phw;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[amd,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[amd,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
phw = &pc->pc_p4pmcs[ri];
pm = phw->phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__,
cpu, ri));
mode = PMC_TO_MODE(pm);
PMCDBG4(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri,
mode, v);
/*
* write the PMC value to the register/saved value: for
* sampling mode PMCs, the value to be programmed into the PMC
* counter is -(C+1) where 'C' is the requested sample rate.
*/
if (PMC_IS_SAMPLING_MODE(mode))
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
if (PMC_IS_SYSTEM_MODE(mode))
wrmsr(pd->pm_pmc_msr, v);
else
P4_PCPU_PMC_VALUE(pc,ri,cpu) = v;
return (0);
}
static int
p4_allocate_pmc(int cpu, int ri, struct pmc *pm,
const struct pmc_op_pmcallocate *a)
{
int found, n, m;
uint32_t caps, cccrvalue, escrvalue, tflags;
enum pmc_p4escr escr;
struct p4_cpu *pc;
struct p4_event_descr *pevent;
const struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index value %d", __LINE__, ri));
pd = &p4_pmcdesc[ri];
PMCDBG4(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x "
"reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps,
pm->pm_caps);
/* check class */
if (pd->pm_descr.pd_class != a->pm_class)
return (EINVAL);
/* check requested capabilities */
caps = a->pm_caps;
if ((pd->pm_descr.pd_caps & caps) != caps)
return (EPERM);
/*
* If the system has HTT enabled, and the desired allocation
* mode is process-private, and the PMC row disposition is not
* FREE (0), decline the allocation.
*/
if (p4_system_has_htt &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
pmc_getrowdisp(ri) != 0)
return (EBUSY);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__,
pd->pm_descr.pd_class));
if (pm->pm_event < PMC_EV_P4_FIRST ||
pm->pm_event > PMC_EV_P4_LAST)
return (EINVAL);
if ((pevent = p4_find_event(pm->pm_event)) == NULL)
return (ESRCH);
PMCDBG4(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}",
pevent->pm_event, pevent->pm_escr_eventselect,
pevent->pm_cccr_select, pevent->pm_is_ti_event);
/*
* Some PMC events are 'thread independent'and therefore
* cannot be used for process-private modes if HTT is being
* used.
*/
if (P4_EVENT_IS_TI(pevent) &&
PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) &&
p4_system_has_htt)
return (EINVAL);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
found = 0;
/* look for a suitable ESCR for this event */
for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) {
if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE)
break; /* out of ESCRs */
/*
* Check ESCR row disposition.
*
* If the request is for a system-mode PMC, then the
* ESCR row should not be in process-virtual mode, and
* should also be free on the current CPU.
*/
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
if (P4_ESCR_ROW_DISP_IS_THREAD(escr) ||
pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX)
continue;
}
/*
* If the request is for a process-virtual PMC, and if
* HTT is not enabled, we can use an ESCR row that is
* either FREE or already in process mode.
*
* If HTT is enabled, then we need to ensure that a
* given ESCR is never allocated to two PMCS that
* could run simultaneously on the two logical CPUs of
* a CPU package. We ensure this be only allocating
* ESCRs from rows marked as 'FREE'.
*/
if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) {
if (p4_system_has_htt) {
if (!P4_ESCR_ROW_DISP_IS_FREE(escr))
continue;
} else
if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr))
continue;
}
/*
* We found a suitable ESCR for this event. Now check if
* this escr can work with the PMC at row-index 'ri'.
*/
for (m = 0; m < P4_MAX_PMC_PER_ESCR; m++)
if (p4_escrs[escr].pm_pmcs[m] == pd->pm_pmcnum) {
found = 1;
break;
}
}
if (found == 0)
return (ESRCH);
KASSERT((int) escr >= 0 && escr < P4_NESCR,
("[p4,%d] illegal ESCR value %d", __LINE__, escr));
/* mark ESCR row mode */
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */
P4_ESCR_MARK_ROW_STANDALONE(escr);
} else {
KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX,
("[p4,%d] escr[%d] already in use", __LINE__, escr));
P4_ESCR_MARK_ROW_THREAD(escr);
}
pm->pm_md.pm_p4.pm_p4_escrmsr = p4_escrs[escr].pm_escr_msr;
pm->pm_md.pm_p4.pm_p4_escr = escr;
cccrvalue = P4_CCCR_TO_ESCR_SELECT(pevent->pm_cccr_select);
escrvalue = P4_ESCR_TO_EVENT_SELECT(pevent->pm_escr_eventselect);
/* CCCR fields */
if (caps & PMC_CAP_THRESHOLD)
cccrvalue |= (a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_THRESHOLD_MASK) | P4_CCCR_COMPARE;
if (caps & PMC_CAP_EDGE)
cccrvalue |= P4_CCCR_EDGE;
if (caps & PMC_CAP_INVERT)
cccrvalue |= P4_CCCR_COMPLEMENT;
if (p4_system_has_htt)
cccrvalue |= a->pm_md.pm_p4.pm_p4_cccrconfig &
P4_CCCR_ACTIVE_THREAD_MASK;
else /* no HTT; thread field should be '11b' */
cccrvalue |= P4_CCCR_TO_ACTIVE_THREAD(0x3);
if (caps & PMC_CAP_CASCADE)
cccrvalue |= P4_CCCR_CASCADE;
/* On HTT systems the PMI T0 field may get moved to T1 at pmc start */
if (caps & PMC_CAP_INTERRUPT)
cccrvalue |= P4_CCCR_OVF_PMI_T0;
/* ESCR fields */
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK;
if (caps & PMC_CAP_TAGGING)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_TAG_VALUE_MASK) | P4_ESCR_TAG_ENABLE;
if (caps & PMC_CAP_QUALIFIER)
escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig &
P4_ESCR_EVENT_MASK_MASK);
/* HTT: T0_{OS,USR} bits may get moved to T1 at pmc start */
tflags = 0;
if (caps & PMC_CAP_SYSTEM)
tflags |= P4_ESCR_T0_OS;
if (caps & PMC_CAP_USER)
tflags |= P4_ESCR_T0_USR;
if (tflags == 0)
tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR);
escrvalue |= tflags;
pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue;
pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue;
PMCDBG5(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x "
"escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select,
cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue);
return (0);
}
