static int ucp_read_pmc(int cpu, int ri, pmc_value_t *v) { struct pmc *pm; pmc_value_t tmp; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); tmp = rdmsr(UCP_PMC0 + ri); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) *v = ucp_perfctr_value_to_reload_count(tmp); else *v = tmp; PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri, ri, *v); return (0); }
static int ucf_write_pmc(int cpu, int ri, pmc_value_t v) { struct uncore_cpu *cc; struct pmc *pm; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu %d ri %d pmc not configured", __LINE__, cpu, ri)); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucf_reload_count_to_perfctr_value(v); wrmsr(UCF_CTRL, 0); /* Turn off fixed counters */ wrmsr(UCF_CTR0 + ri, v); wrmsr(UCF_CTRL, cc->pc_ucfctrl); PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ", cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL)); return (0); }
static int ucf_stop_pmc(int cpu, int ri) { uint32_t fc; struct uncore_cpu *ucfc; PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); fc = (UCF_MASK << (ri * 4)); ucfc->pc_ucfctrl &= ~fc; PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl); wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl &= ~(1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STO,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); }
static int ucf_start_pmc(int cpu, int ri) { struct pmc *pm; struct uncore_cpu *ucfc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucf_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri); ucfc = uncore_pcpu[cpu]; pm = ucfc->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc; ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl; wrmsr(UCF_CTRL, ucfc->pc_ucfctrl); do { ucfc->pc_resync = 0; ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype))); wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl); } while (ucfc->pc_resync != 0); PMCDBG4(MDP,STA,1,"ucfctrl=%x(%x) globalctrl=%jx(%jx)", ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL), ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL)); return (0); }
static int ucp_write_pmc(int cpu, int ri, pmc_value_t v) { struct pmc *pm; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal cpu value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] cpu%d ri%d no configured PMC to stop", __LINE__, cpu, ri)); PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri, UCP_PMC0 + ri, v); if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) v = ucp_reload_count_to_perfctr_value(v); /* * Write the new value to the counter. The counter will be in * a stopped state when the pcd_write() entry point is called. */ wrmsr(UCP_PMC0 + ri, v); return (0); }
static int ucp_start_pmc(int cpu, int ri) { struct pmc *pm; uint32_t evsel; struct uncore_cpu *cc; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[uncore,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < uncore_ucp_npmc, ("[uncore,%d] illegal row-index %d", __LINE__, ri)); cc = uncore_pcpu[cpu]; pm = cc->pc_uncorepmcs[ri].phw_pmc; KASSERT(pm, ("[uncore,%d] starting cpu%d,ri%d with no pmc configured", __LINE__, cpu, ri)); PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri); evsel = pm->pm_md.pm_ucp.pm_ucp_evsel; PMCDBG4(MDP,STA,2, "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x", cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel); /* Event specific configuration. */ switch (pm->pm_event) { case PMC_EV_UCP_EVENT_0CH_04H_E: case PMC_EV_UCP_EVENT_0CH_08H_E: wrmsr(MSR_GQ_SNOOP_MESF,0x2); break; case PMC_EV_UCP_EVENT_0CH_04H_F: case PMC_EV_UCP_EVENT_0CH_08H_F: wrmsr(MSR_GQ_SNOOP_MESF,0x8); break; case PMC_EV_UCP_EVENT_0CH_04H_M: case PMC_EV_UCP_EVENT_0CH_08H_M: wrmsr(MSR_GQ_SNOOP_MESF,0x1); break; case PMC_EV_UCP_EVENT_0CH_04H_S: case PMC_EV_UCP_EVENT_0CH_08H_S: wrmsr(MSR_GQ_SNOOP_MESF,0x4); break; default: break; } wrmsr(SELECTSEL(uncore_cputype) + ri, evsel); do { cc->pc_resync = 0; cc->pc_globalctrl |= (1ULL << ri); wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl); } while (cc->pc_resync != 0); return (0); }
static int p4_write_pmc(int cpu, int ri, pmc_value_t v) { enum pmc_mode mode; struct pmc *pm; struct p4_cpu *pc; const struct pmc_hw *phw; const struct p4pmc_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[amd,%d] illegal CPU value %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < P4_NPMCS, ("[amd,%d] illegal row-index %d", __LINE__, ri)); pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; phw = &pc->pc_p4pmcs[ri]; pm = phw->phw_pmc; pd = &p4_pmcdesc[ri]; KASSERT(pm != NULL, ("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri)); mode = PMC_TO_MODE(pm); PMCDBG4(MDP,WRI,1, "p4-write cpu=%d ri=%d mode=%d v=%jx", cpu, ri, mode, v); /* * write the PMC value to the register/saved value: for * sampling mode PMCs, the value to be programmed into the PMC * counter is -(C+1) where 'C' is the requested sample rate. */ if (PMC_IS_SAMPLING_MODE(mode)) v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(v); if (PMC_IS_SYSTEM_MODE(mode)) wrmsr(pd->pm_pmc_msr, v); else P4_PCPU_PMC_VALUE(pc,ri,cpu) = v; return (0); }
static int p4_allocate_pmc(int cpu, int ri, struct pmc *pm, const struct pmc_op_pmcallocate *a) { int found, n, m; uint32_t caps, cccrvalue, escrvalue, tflags; enum pmc_p4escr escr; struct p4_cpu *pc; struct p4_event_descr *pevent; const struct p4pmc_descr *pd; KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), ("[p4,%d] illegal CPU %d", __LINE__, cpu)); KASSERT(ri >= 0 && ri < P4_NPMCS, ("[p4,%d] illegal row-index value %d", __LINE__, ri)); pd = &p4_pmcdesc[ri]; PMCDBG4(MDP,ALL,1, "p4-allocate ri=%d class=%d pmccaps=0x%x " "reqcaps=0x%x", ri, pd->pm_descr.pd_class, pd->pm_descr.pd_caps, pm->pm_caps); /* check class */ if (pd->pm_descr.pd_class != a->pm_class) return (EINVAL); /* check requested capabilities */ caps = a->pm_caps; if ((pd->pm_descr.pd_caps & caps) != caps) return (EPERM); /* * If the system has HTT enabled, and the desired allocation * mode is process-private, and the PMC row disposition is not * FREE (0), decline the allocation. */ if (p4_system_has_htt && PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) && pmc_getrowdisp(ri) != 0) return (EBUSY); KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4, ("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class)); if (pm->pm_event < PMC_EV_P4_FIRST || pm->pm_event > PMC_EV_P4_LAST) return (EINVAL); if ((pevent = p4_find_event(pm->pm_event)) == NULL) return (ESRCH); PMCDBG4(MDP,ALL,2, "pevent={ev=%d,escrsel=0x%x,cccrsel=0x%x,isti=%d}", pevent->pm_event, pevent->pm_escr_eventselect, pevent->pm_cccr_select, pevent->pm_is_ti_event); /* * Some PMC events are 'thread independent'and therefore * cannot be used for process-private modes if HTT is being * used. */ if (P4_EVENT_IS_TI(pevent) && PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm)) && p4_system_has_htt) return (EINVAL); pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)]; found = 0; /* look for a suitable ESCR for this event */ for (n = 0; n < P4_MAX_ESCR_PER_EVENT && !found; n++) { if ((escr = pevent->pm_escrs[n]) == P4_ESCR_NONE) break; /* out of ESCRs */ /* * Check ESCR row disposition. * * If the request is for a system-mode PMC, then the * ESCR row should not be in process-virtual mode, and * should also be free on the current CPU. */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { if (P4_ESCR_ROW_DISP_IS_THREAD(escr) || pc->pc_escrs[escr] != P4_INVALID_PMC_INDEX) continue; } /* * If the request is for a process-virtual PMC, and if * HTT is not enabled, we can use an ESCR row that is * either FREE or already in process mode. * * If HTT is enabled, then we need to ensure that a * given ESCR is never allocated to two PMCS that * could run simultaneously on the two logical CPUs of * a CPU package. We ensure this be only allocating * ESCRs from rows marked as 'FREE'. */ if (PMC_IS_VIRTUAL_MODE(PMC_TO_MODE(pm))) { if (p4_system_has_htt) { if (!P4_ESCR_ROW_DISP_IS_FREE(escr)) continue; } else if (P4_ESCR_ROW_DISP_IS_STANDALONE(escr)) continue; } /* * We found a suitable ESCR for this event. Now check if * this escr can work with the PMC at row-index 'ri'. */ for (m = 0; m < P4_MAX_PMC_PER_ESCR; m++) if (p4_escrs[escr].pm_pmcs[m] == pd->pm_pmcnum) { found = 1; break; } } if (found == 0) return (ESRCH); KASSERT((int) escr >= 0 && escr < P4_NESCR, ("[p4,%d] illegal ESCR value %d", __LINE__, escr)); /* mark ESCR row mode */ if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) { pc->pc_escrs[escr] = ri; /* mark ESCR as in use on this cpu */ P4_ESCR_MARK_ROW_STANDALONE(escr); } else { KASSERT(pc->pc_escrs[escr] == P4_INVALID_PMC_INDEX, ("[p4,%d] escr[%d] already in use", __LINE__, escr)); P4_ESCR_MARK_ROW_THREAD(escr); } pm->pm_md.pm_p4.pm_p4_escrmsr = p4_escrs[escr].pm_escr_msr; pm->pm_md.pm_p4.pm_p4_escr = escr; cccrvalue = P4_CCCR_TO_ESCR_SELECT(pevent->pm_cccr_select); escrvalue = P4_ESCR_TO_EVENT_SELECT(pevent->pm_escr_eventselect); /* CCCR fields */ if (caps & PMC_CAP_THRESHOLD) cccrvalue |= (a->pm_md.pm_p4.pm_p4_cccrconfig & P4_CCCR_THRESHOLD_MASK) | P4_CCCR_COMPARE; if (caps & PMC_CAP_EDGE) cccrvalue |= P4_CCCR_EDGE; if (caps & PMC_CAP_INVERT) cccrvalue |= P4_CCCR_COMPLEMENT; if (p4_system_has_htt) cccrvalue |= a->pm_md.pm_p4.pm_p4_cccrconfig & P4_CCCR_ACTIVE_THREAD_MASK; else /* no HTT; thread field should be '11b' */ cccrvalue |= P4_CCCR_TO_ACTIVE_THREAD(0x3); if (caps & PMC_CAP_CASCADE) cccrvalue |= P4_CCCR_CASCADE; /* On HTT systems the PMI T0 field may get moved to T1 at pmc start */ if (caps & PMC_CAP_INTERRUPT) cccrvalue |= P4_CCCR_OVF_PMI_T0; /* ESCR fields */ if (caps & PMC_CAP_QUALIFIER) escrvalue |= a->pm_md.pm_p4.pm_p4_escrconfig & P4_ESCR_EVENT_MASK_MASK; if (caps & PMC_CAP_TAGGING) escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig & P4_ESCR_TAG_VALUE_MASK) | P4_ESCR_TAG_ENABLE; if (caps & PMC_CAP_QUALIFIER) escrvalue |= (a->pm_md.pm_p4.pm_p4_escrconfig & P4_ESCR_EVENT_MASK_MASK); /* HTT: T0_{OS,USR} bits may get moved to T1 at pmc start */ tflags = 0; if (caps & PMC_CAP_SYSTEM) tflags |= P4_ESCR_T0_OS; if (caps & PMC_CAP_USER) tflags |= P4_ESCR_T0_USR; if (tflags == 0) tflags = (P4_ESCR_T0_OS|P4_ESCR_T0_USR); escrvalue |= tflags; pm->pm_md.pm_p4.pm_p4_cccrvalue = cccrvalue; pm->pm_md.pm_p4.pm_p4_escrvalue = escrvalue; PMCDBG5(MDP,ALL,2, "p4-allocate cccrsel=0x%x cccrval=0x%x " "escr=%d escrmsr=0x%x escrval=0x%x", pevent->pm_cccr_select, cccrvalue, escr, pm->pm_md.pm_p4.pm_p4_escrmsr, escrvalue); return (0); }