int
pmc_soft_intr(struct pmckern_soft *ks)
{
struct pmc *pm;
struct soft_cpu *pc;
int ri, processed, error, user_mode;
KASSERT(ks->pm_cpu >= 0 && ks->pm_cpu < pmc_cpu_max(),
("[soft,%d] CPU %d out of range", __LINE__, ks->pm_cpu));
processed = 0;
pc = soft_pcpu[ks->pm_cpu];
for (ri = 0; ri < SOFT_NPMCS; ri++) {
pm = pc->soft_hw[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
pm->pm_event != ks->pm_ev) {
continue;
}
processed = 1;
pc->soft_values[ri]++;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
user_mode = TRAPF_USERMODE(ks->pm_tf);
error = pmc_process_interrupt(ks->pm_cpu, PMC_SR, pm,
ks->pm_tf, user_mode);
if (error) {
soft_stop_pmc(ks->pm_cpu, ri);
continue;
}
if (user_mode) {
/* If in user mode setup AST to process
* callchain out of interrupt context.
*/
curthread->td_flags |= TDF_ASTPENDING;
}
}
}
atomic_add_int(processed ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (processed);
}
static int
arm64_intr(struct trapframe *tf)
{
struct arm64_cpu *pc;
int retval, ri;
struct pmc *pm;
int error;
int reg, cpu;
cpu = curcpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] CPU %d out of range", __LINE__, cpu));
retval = 0;
pc = arm64_pcpu[cpu];
for (ri = 0; ri < arm64_npmcs; ri++) {
pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
if (pm == NULL)
continue;
if (!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
/* Check if counter is overflowed */
reg = (1 << ri);
if ((READ_SPECIALREG(PMOVSCLR_EL0) & reg) == 0)
continue;
/* Clear Overflow Flag */
WRITE_SPECIALREG(PMOVSCLR_EL0, reg);
isb();
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (error)
arm64_stop_pmc(cpu, ri);
/* Reload sampling count */
arm64_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount);
}
return (retval);
}
static int
mpc7xxx_intr(int cpu, struct trapframe *tf)
{
int i, error, retval;
uint32_t config;
struct pmc *pm;
struct powerpc_cpu *pac;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
PMCDBG(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC;
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
for (i = 0; i < MPC7XXX_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!MPC7XXX_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the counter if logging fails. */
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error != 0)
mpc7xxx_stop_pmc(cpu, i);
/* reload count. */
mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
atomic_add_int(retval ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
static int
mips_pmc_intr(int cpu, struct trapframe *tf)
{
int error;
int retval, ri;
struct pmc *pm;
struct mips_cpu *pc;
uint32_t r0, r2;
pmc_value_t r;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[mips,%d] CPU %d out of range", __LINE__, cpu));
retval = 0;
pc = mips_pcpu[cpu];
/* Stop PMCs without clearing the counter */
r0 = mips_rd_perfcnt0();
mips_wr_perfcnt0(r0 & ~(0x1f));
r2 = mips_rd_perfcnt2();
mips_wr_perfcnt2(r2 & ~(0x1f));
for (ri = 0; ri < mips_npmcs; ri++) {
pm = mips_pcpu[cpu]->pc_mipspmcs[ri].phw_pmc;
if (pm == NULL)
continue;
if (! PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
continue;
r = mips_pmcn_read(ri);
/* If bit 31 is set, the counter has overflowed */
if ((r & (1UL << (mips_pmc_spec.ps_counter_width - 1))) == 0)
continue;
retval = 1;
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
if (error) {
/* Clear/disable the relevant counter */
if (ri == 0)
r0 = 0;
else if (ri == 1)
r2 = 0;
mips_stop_pmc(cpu, ri);
}
/* Reload sampling count */
mips_write_pmc(cpu, ri, pm->pm_sc.pm_reloadcount);
}
/*
* Re-enable the PMC counters where they left off.
*
* Any counter which overflowed will have its sample count
* reloaded in the loop above.
*/
mips_wr_perfcnt0(r0);
mips_wr_perfcnt2(r2);
return retval;
}
static int
p4_intr(int cpu, struct trapframe *tf)
{
uint32_t cccrval, ovf_mask, ovf_partner;
int did_interrupt, error, ri;
struct p4_cpu *pc;
struct pmc *pm;
pmc_value_t v;
PMCDBG(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
ovf_mask |= P4_CCCR_OVF;
if (p4_system_has_htt)
ovf_partner = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T0 : P4_CCCR_OVF_PMI_T1;
else
ovf_partner = 0;
did_interrupt = 0;
if (p4_system_has_htt)
P4_PCPU_ACQ_INTR_SPINLOCK(pc);
/*
* Loop through all CCCRs, looking for ones that have
* interrupted this CPU.
*/
for (ri = 0; ri < P4_NPMCS; ri++) {
/*
* Check if our partner logical CPU has already marked
* this PMC has having interrupted it. If so, reset
* the flag and process the interrupt, but leave the
* hardware alone.
*/
if (p4_system_has_htt && P4_PCPU_GET_INTRFLAG(pc,ri)) {
P4_PCPU_SET_INTRFLAG(pc,ri,0);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs.
* Ignore PMCs not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
continue;
}
/*
* Fresh interrupt. Look for the CCCR_OVF bit
* and the OVF_Tx bit for this logical
* processor being set.
*/
cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
if ((cccrval & ovf_mask) != ovf_mask)
continue;
/*
* If the other logical CPU would also have been
* interrupted due to the PMC being shared, record
* this fact in the per-cpu saved interrupt flag
* bitmask.
*/
if (p4_system_has_htt && (cccrval & ovf_partner))
P4_PCPU_SET_INTRFLAG(pc, ri, 1);
v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
PMCDBG(MDP,INT, 2, "ri=%d v=%jx", ri, v);
/* Stop the counter, and reset the overflow bit */
cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs. Ignore PMCs
* not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
/*
* Process the interrupt. Re-enable the PMC if
* processing was successful.
*/
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
/*
* Only the first processor executing the NMI handler
* in a HTT pair will restart a PMC, and that too
* only if there were no errors.
*/
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
pm->pm_sc.pm_reloadcount);
wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
if (error == 0)
wrmsr(P4_CCCR_MSR_FIRST + ri,
cccrval | P4_CCCR_ENABLE);
}
/* allow the other CPU to proceed */
if (p4_system_has_htt)
P4_PCPU_REL_INTR_SPINLOCK(pc);
/*
* On Intel P4 CPUs, the PMC 'pcint' entry in the LAPIC gets
* masked when a PMC interrupts the CPU. We need to unmask
* the interrupt source explicitly.
*/
if (did_interrupt)
lapic_reenable_pmc();
atomic_add_int(did_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (did_interrupt);
}
