static int
ucf_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal cpu value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
pm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;
KASSERT(pm,
("[uncore,%d] cpu %d ri %d(%d) pmc not configured", __LINE__, cpu,
ri, ri + uncore_ucf_ri));
tmp = rdmsr(UCF_CTR0 + ri);
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
*v = ucf_perfctr_value_to_reload_count(tmp);
else
*v = tmp;
PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v);
return (0);
}
static int
p4_release_pmc(int cpu, int ri, struct pmc *pm)
{
enum pmc_p4escr escr;
struct p4_cpu *pc;
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
escr = pm->pm_md.pm_p4.pm_p4_escr;
PMCDBG3(MDP,REL,1, "p4-release cpu=%d ri=%d escr=%d", cpu, ri, escr);
if (PMC_IS_SYSTEM_MODE(PMC_TO_MODE(pm))) {
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
KASSERT(pc->pc_p4pmcs[ri].phw_pmc == NULL,
("[p4,%d] releasing configured PMC ri=%d", __LINE__, ri));
P4_ESCR_UNMARK_ROW_STANDALONE(escr);
KASSERT(pc->pc_escrs[escr] == ri,
("[p4,%d] escr[%d] not allocated to ri %d", __LINE__,
escr, ri));
pc->pc_escrs[escr] = P4_INVALID_PMC_INDEX; /* mark as free */
} else
P4_ESCR_UNMARK_ROW_THREAD(escr);
return (0);
}
static int
p4_read_pmc(int cpu, int ri, pmc_value_t *v)
{
struct pmc *pm;
pmc_value_t tmp;
struct p4_cpu *pc;
enum pmc_mode mode;
struct p4pmc_descr *pd;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
pm = pc->pc_p4pmcs[ri].phw_pmc;
pd = &p4_pmcdesc[ri];
KASSERT(pm != NULL,
("[p4,%d] No owner for HWPMC [cpu%d,pmc%d]", __LINE__, cpu, ri));
KASSERT(pd->pm_descr.pd_class == PMC_TO_CLASS(pm),
("[p4,%d] class mismatch pd %d != id class %d", __LINE__,
pd->pm_descr.pd_class, PMC_TO_CLASS(pm)));
mode = PMC_TO_MODE(pm);
PMCDBG3(MDP,REA,1, "p4-read cpu=%d ri=%d mode=%d", cpu, ri, mode);
KASSERT(pd->pm_descr.pd_class == PMC_CLASS_P4,
("[p4,%d] unknown PMC class %d", __LINE__, pd->pm_descr.pd_class));
tmp = rdmsr(p4_pmcdesc[ri].pm_pmc_msr);
if (PMC_IS_VIRTUAL_MODE(mode)) {
if (tmp < P4_PCPU_HW_VALUE(pc,ri,cpu)) /* 40 bit overflow */
tmp += (P4_PERFCTR_MASK + 1) -
P4_PCPU_HW_VALUE(pc,ri,cpu);
else
tmp -= P4_PCPU_HW_VALUE(pc,ri,cpu);
tmp += P4_PCPU_PMC_VALUE(pc,ri,cpu);
}
if (PMC_IS_SAMPLING_MODE(mode)) /* undo transformation */
*v = P4_PERFCTR_VALUE_TO_RELOAD_COUNT(tmp);
else
*v = tmp;
PMCDBG1(MDP,REA,2, "p4-read -> %jx", *v);
return (0);
}
static int
ucf_release_pmc(int cpu, int ri, struct pmc *pmc)
{
PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucf_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL,
("[uncore,%d] PHW pmc non-NULL", __LINE__));
return (0);
}
static int
ucp_config_pmc(int cpu, int ri, struct pmc *pm)
{
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[uncore,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < uncore_ucp_npmc,
("[uncore,%d] illegal row-index %d", __LINE__, ri));
PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__,
cpu));
uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm;
return (0);
}
void
pmclog_process_procexit(struct pmc *pm, struct pmc_process *pp)
{
int ri;
struct pmc_owner *po;
ri = PMC_TO_ROWINDEX(pm);
PMCDBG3(LOG,EXT,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
pp->pp_pmcs[ri].pp_pmcval);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PROCEXIT, sizeof(struct pmclog_procexit));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT64(pp->pp_pmcs[ri].pp_pmcval);
PMCLOG_EMIT32(pp->pp_proc->p_pid);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_procexec(struct pmc_owner *po, pmc_id_t pmid, pid_t pid,
uintfptr_t startaddr, char *path)
{
int pathlen, recordlen;
PMCDBG3(LOG,EXC,1,"po=%p pid=%d path=\"%s\"", po, pid, path);
pathlen = strlen(path) + 1; /* #bytes for the path */
recordlen = offsetof(struct pmclog_procexec, pl_pathname) + pathlen;
PMCLOG_RESERVE(po, PROCEXEC, recordlen);
PMCLOG_EMIT32(pid);
PMCLOG_EMITADDR(startaddr);
PMCLOG_EMIT32(pmid);
PMCLOG_EMITSTRING(path,pathlen);
PMCLOG_DESPATCH(po);
}
void
pmclog_process_proccsw(struct pmc *pm, struct pmc_process *pp, pmc_value_t v)
{
struct pmc_owner *po;
KASSERT(pm->pm_flags & PMC_F_LOG_PROCCSW,
("[pmclog,%d] log-process-csw called gratuitously", __LINE__));
PMCDBG3(LOG,SWO,1,"pm=%p pid=%d v=%jx", pm, pp->pp_proc->p_pid,
v);
po = pm->pm_owner;
PMCLOG_RESERVE(po, PROCCSW, sizeof(struct pmclog_proccsw));
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT64(v);
PMCLOG_EMIT32(pp->pp_proc->p_pid);
PMCLOG_DESPATCH(po);
}
static int
mpc7xxx_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
pm = powerpc_pcpu[cpu]->pc_ppcpmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = POWERPC_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
PMCDBG3(MDP,WRI,1,"powerpc-write cpu=%d ri=%d v=%jx", cpu, ri, v);
mpc7xxx_pmcn_write(ri, v);
return 0;
}
static int
arm64_write_pmc(int cpu, int ri, pmc_value_t v)
{
struct pmc *pm;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < arm64_npmcs,
("[arm64,%d] illegal row-index %d", __LINE__, ri));
pm = arm64_pcpu[cpu]->pc_arm64pmcs[ri].phw_pmc;
if (PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm)))
v = ARMV8_RELOAD_COUNT_TO_PERFCTR_VALUE(v);
PMCDBG3(MDP, WRI, 1, "arm64-write cpu=%d ri=%d v=%jx", cpu, ri, v);
arm64_pmcn_write(ri, v);
return 0;
}
static int
mpc7xxx_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < MPC7XXX_MAX_PMCS,
("[powerpc,%d] illegal row-index %d", __LINE__, ri));
phw = &powerpc_pcpu[cpu]->pc_ppcpmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[powerpc,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
void
pmclog_process_callchain(struct pmc *pm, struct pmc_sample *ps)
{
int n, recordlen;
uint32_t flags;
struct pmc_owner *po;
PMCDBG3(LOG,SAM,1,"pm=%p pid=%d n=%d", pm, ps->ps_pid,
ps->ps_nsamples);
recordlen = offsetof(struct pmclog_callchain, pl_pc) +
ps->ps_nsamples * sizeof(uintfptr_t);
po = pm->pm_owner;
flags = PMC_CALLCHAIN_TO_CPUFLAGS(ps->ps_cpu,ps->ps_flags);
PMCLOG_RESERVE(po, CALLCHAIN, recordlen);
PMCLOG_EMIT32(ps->ps_pid);
PMCLOG_EMIT32(pm->pm_id);
PMCLOG_EMIT32(flags);
for (n = 0; n < ps->ps_nsamples; n++)
PMCLOG_EMITADDR(ps->ps_pc[n]);
PMCLOG_DESPATCH(po);
}
static int
arm64_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[arm64,%d] illegal CPU value %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < arm64_npmcs,
("[arm64,%d] illegal row-index %d", __LINE__, ri));
phw = &arm64_pcpu[cpu]->pc_arm64pmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL,
("[arm64,%d] pm=%p phw->pm=%p hwpmc not unconfigured",
__LINE__, pm, phw->phw_pmc));
phw->phw_pmc = pm;
return 0;
}
static int
mpc7xxx_intr(struct trapframe *tf)
{
int i, error, retval, cpu;
uint32_t config;
struct pmc *pm;
struct powerpc_cpu *pac;
cpu = curcpu;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[powerpc,%d] out of range CPU %d", __LINE__, cpu));
PMCDBG3(MDP,INT,1, "cpu=%d tf=%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
retval = 0;
pac = powerpc_pcpu[cpu];
config = mfspr(SPR_MMCR0) & ~SPR_MMCR0_FC;
/*
* look for all PMCs that have interrupted:
* - look for a running, sampling PMC which has overflowed
* and which has a valid 'struct pmc' association
*
* If found, we call a helper to process the interrupt.
*/
for (i = 0; i < MPC7XXX_MAX_PMCS; i++) {
if ((pm = pac->pc_ppcpmcs[i].phw_pmc) == NULL ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
if (!MPC7XXX_PMC_HAS_OVERFLOWED(i))
continue;
retval = 1; /* Found an interrupting PMC. */
if (pm->pm_state != PMC_STATE_RUNNING)
continue;
/* Stop the counter if logging fails. */
error = pmc_process_interrupt(PMC_HR, pm, tf);
if (error != 0)
mpc7xxx_stop_pmc(cpu, i);
/* reload count. */
mpc7xxx_write_pmc(cpu, i, pm->pm_sc.pm_reloadcount);
}
if (retval)
counter_u64_add(pmc_stats.pm_intr_processed, 1);
else
counter_u64_add(pmc_stats.pm_intr_ignored, 1);
/* Re-enable PERF exceptions. */
if (retval)
mtspr(SPR_MMCR0, config | SPR_MMCR0_PMXE);
return (retval);
}
static int
p4_config_pmc(int cpu, int ri, struct pmc *pm)
{
struct pmc_hw *phw;
struct p4_cpu *pc;
int cfgflags, cpuflag;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] illegal CPU %d", __LINE__, cpu));
KASSERT(ri >= 0 && ri < P4_NPMCS,
("[p4,%d] illegal row-index %d", __LINE__, ri));
PMCDBG3(MDP,CFG,1, "cpu=%d ri=%d pm=%p", cpu, ri, pm);
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
phw = &pc->pc_p4pmcs[ri];
KASSERT(pm == NULL || phw->phw_pmc == NULL ||
(p4_system_has_htt && phw->phw_pmc == pm),
("[p4,%d] hwpmc not unconfigured before re-config", __LINE__));
mtx_lock_spin(&pc->pc_mtx);
cfgflags = P4_PCPU_GET_CFGFLAGS(pc,ri);
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal cfgflags cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
KASSERT(cfgflags == 0 || phw->phw_pmc,
("[p4,%d] cpu=%d ri=%d pmc configured with zero cfg count",
__LINE__, cpu, ri));
cpuflag = P4_CPU_TO_FLAG(cpu);
if (pm) { /* config */
if (cfgflags == 0)
phw->phw_pmc = pm;
KASSERT(phw->phw_pmc == pm,
("[p4,%d] cpu=%d ri=%d config %p != hw %p",
__LINE__, cpu, ri, pm, phw->phw_pmc));
cfgflags |= cpuflag;
} else { /* unconfig */
cfgflags &= ~cpuflag;
if (cfgflags == 0)
phw->phw_pmc = NULL;
}
KASSERT(cfgflags >= 0 || cfgflags <= 3,
("[p4,%d] illegal runcount cfg=%d on cpu=%d ri=%d", __LINE__,
cfgflags, cpu, ri));
P4_PCPU_SET_CFGFLAGS(pc,ri,cfgflags);
mtx_unlock_spin(&pc->pc_mtx);
return (0);
}
static int
p4_pcpu_init(struct pmc_mdep *md, int cpu)
{
char *pescr;
int n, first_ri, phycpu;
struct pmc_hw *phw;
struct p4_cpu *p4c;
struct pmc_cpu *pc, *plc;
KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
("[p4,%d] insane cpu number %d", __LINE__, cpu));
PMCDBG2(MDP,INI,0, "p4-init cpu=%d is-primary=%d", cpu,
pmc_cpu_is_primary(cpu) != 0);
first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_P4].pcd_ri;
/*
* The two CPUs in an HT pair share their per-cpu state.
*
* For HT capable CPUs, we assume that the two logical
* processors in the HT pair get two consecutive CPU ids
* starting with an even id #.
*
* The primary CPU (the even numbered CPU of the pair) would
* have been initialized prior to the initialization for the
* secondary.
*/
if (!pmc_cpu_is_primary(cpu) && (cpu & 1)) {
p4_system_has_htt = 1;
phycpu = P4_TO_HTT_PRIMARY(cpu);
pc = pmc_pcpu[phycpu];
plc = pmc_pcpu[cpu];
KASSERT(plc != pc, ("[p4,%d] per-cpu config error", __LINE__));
PMCDBG3(MDP,INI,1, "p4-init cpu=%d phycpu=%d pc=%p", cpu,
phycpu, pc);
KASSERT(pc, ("[p4,%d] Null Per-Cpu state cpu=%d phycpu=%d",
__LINE__, cpu, phycpu));
/* PMCs are shared with the physical CPU. */
for (n = 0; n < P4_NPMCS; n++)
plc->pc_hwpmcs[n + first_ri] =
pc->pc_hwpmcs[n + first_ri];
return (0);
}
p4c = malloc(sizeof(struct p4_cpu), M_PMC, M_WAITOK|M_ZERO);
pc = pmc_pcpu[cpu];
KASSERT(pc != NULL, ("[p4,%d] cpu %d null per-cpu", __LINE__, cpu));
p4_pcpu[cpu] = p4c;
phw = p4c->pc_p4pmcs;
for (n = 0; n < P4_NPMCS; n++, phw++) {
phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n);
phw->phw_pmc = NULL;
pc->pc_hwpmcs[n + first_ri] = phw;
}
pescr = p4c->pc_escrs;
for (n = 0; n < P4_NESCR; n++)
*pescr++ = P4_INVALID_PMC_INDEX;
mtx_init(&p4c->pc_mtx, "p4-pcpu", "pmc-leaf", MTX_SPIN);
return (0);
}
static int
p4_intr(int cpu, struct trapframe *tf)
{
uint32_t cccrval, ovf_mask, ovf_partner;
int did_interrupt, error, ri;
struct p4_cpu *pc;
struct pmc *pm;
pmc_value_t v;
PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
TRAPF_USERMODE(tf));
pc = p4_pcpu[P4_TO_HTT_PRIMARY(cpu)];
ovf_mask = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T1 : P4_CCCR_OVF_PMI_T0;
ovf_mask |= P4_CCCR_OVF;
if (p4_system_has_htt)
ovf_partner = P4_CPU_IS_HTT_SECONDARY(cpu) ?
P4_CCCR_OVF_PMI_T0 : P4_CCCR_OVF_PMI_T1;
else
ovf_partner = 0;
did_interrupt = 0;
if (p4_system_has_htt)
P4_PCPU_ACQ_INTR_SPINLOCK(pc);
/*
* Loop through all CCCRs, looking for ones that have
* interrupted this CPU.
*/
for (ri = 0; ri < P4_NPMCS; ri++) {
/*
* Check if our partner logical CPU has already marked
* this PMC has having interrupted it. If so, reset
* the flag and process the interrupt, but leave the
* hardware alone.
*/
if (p4_system_has_htt && P4_PCPU_GET_INTRFLAG(pc,ri)) {
P4_PCPU_SET_INTRFLAG(pc,ri,0);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs.
* Ignore PMCs not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
(void) pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
continue;
}
/*
* Fresh interrupt. Look for the CCCR_OVF bit
* and the OVF_Tx bit for this logical
* processor being set.
*/
cccrval = rdmsr(P4_CCCR_MSR_FIRST + ri);
if ((cccrval & ovf_mask) != ovf_mask)
continue;
/*
* If the other logical CPU would also have been
* interrupted due to the PMC being shared, record
* this fact in the per-cpu saved interrupt flag
* bitmask.
*/
if (p4_system_has_htt && (cccrval & ovf_partner))
P4_PCPU_SET_INTRFLAG(pc, ri, 1);
v = rdmsr(P4_PERFCTR_MSR_FIRST + ri);
PMCDBG2(MDP,INT, 2, "ri=%d v=%jx", ri, v);
/* Stop the counter, and reset the overflow bit */
cccrval &= ~(P4_CCCR_OVF | P4_CCCR_ENABLE);
wrmsr(P4_CCCR_MSR_FIRST + ri, cccrval);
did_interrupt = 1;
/*
* Ignore de-configured or stopped PMCs. Ignore PMCs
* not in sampling mode.
*/
pm = pc->pc_p4pmcs[ri].phw_pmc;
if (pm == NULL ||
pm->pm_state != PMC_STATE_RUNNING ||
!PMC_IS_SAMPLING_MODE(PMC_TO_MODE(pm))) {
continue;
}
/*
* Process the interrupt. Re-enable the PMC if
* processing was successful.
*/
error = pmc_process_interrupt(cpu, PMC_HR, pm, tf,
TRAPF_USERMODE(tf));
/*
* Only the first processor executing the NMI handler
* in a HTT pair will restart a PMC, and that too
* only if there were no errors.
*/
v = P4_RELOAD_COUNT_TO_PERFCTR_VALUE(
pm->pm_sc.pm_reloadcount);
wrmsr(P4_PERFCTR_MSR_FIRST + ri, v);
if (error == 0)
wrmsr(P4_CCCR_MSR_FIRST + ri,
cccrval | P4_CCCR_ENABLE);
}
/* allow the other CPU to proceed */
if (p4_system_has_htt)
P4_PCPU_REL_INTR_SPINLOCK(pc);
/*
* On Intel P4 CPUs, the PMC 'pcint' entry in the LAPIC gets
* masked when a PMC interrupts the CPU. We need to unmask
* the interrupt source explicitly.
*/
if (did_interrupt)
lapic_reenable_pmc();
atomic_add_int(did_interrupt ? &pmc_stats.pm_intr_processed :
&pmc_stats.pm_intr_ignored, 1);
return (did_interrupt);
}
static void
pmclog_loop(void *arg)
{
int error;
struct pmc_owner *po;
struct pmclog_buffer *lb;
struct proc *p;
struct ucred *ownercred;
struct ucred *mycred;
struct thread *td;
struct uio auio;
struct iovec aiov;
size_t nbytes;
po = (struct pmc_owner *) arg;
p = po->po_owner;
td = curthread;
mycred = td->td_ucred;
PROC_LOCK(p);
ownercred = crhold(p->p_ucred);
PROC_UNLOCK(p);
PMCDBG2(LOG,INI,1, "po=%p kt=%p", po, po->po_kthread);
KASSERT(po->po_kthread == curthread->td_proc,
("[pmclog,%d] proc mismatch po=%p po/kt=%p curproc=%p", __LINE__,
po, po->po_kthread, curthread->td_proc));
lb = NULL;
/*
* Loop waiting for I/O requests to be added to the owner
* struct's queue. The loop is exited when the log file
* is deconfigured.
*/
mtx_lock(&pmc_kthread_mtx);
for (;;) {
/* check if we've been asked to exit */
if ((po->po_flags & PMC_PO_OWNS_LOGFILE) == 0)
break;
if (lb == NULL) { /* look for a fresh buffer to write */
mtx_lock_spin(&po->po_mtx);
if ((lb = TAILQ_FIRST(&po->po_logbuffers)) == NULL) {
mtx_unlock_spin(&po->po_mtx);
/* No more buffers and shutdown required. */
if (po->po_flags & PMC_PO_SHUTDOWN) {
mtx_unlock(&pmc_kthread_mtx);
/*
* Close the file to get PMCLOG_EOF
* error in pmclog(3).
*/
fo_close(po->po_file, curthread);
mtx_lock(&pmc_kthread_mtx);
break;
}
(void) msleep(po, &pmc_kthread_mtx, PWAIT,
"pmcloop", 0);
continue;
}
TAILQ_REMOVE(&po->po_logbuffers, lb, plb_next);
mtx_unlock_spin(&po->po_mtx);
}
mtx_unlock(&pmc_kthread_mtx);
/* process the request */
PMCDBG3(LOG,WRI,2, "po=%p base=%p ptr=%p", po,
lb->plb_base, lb->plb_ptr);
/* change our thread's credentials before issuing the I/O */
aiov.iov_base = lb->plb_base;
aiov.iov_len = nbytes = lb->plb_ptr - lb->plb_base;
auio.uio_iov = &aiov;
auio.uio_iovcnt = 1;
auio.uio_offset = -1;
auio.uio_resid = nbytes;
auio.uio_rw = UIO_WRITE;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_td = td;
/* switch thread credentials -- see kern_ktrace.c */
td->td_ucred = ownercred;
error = fo_write(po->po_file, &auio, ownercred, 0, td);
td->td_ucred = mycred;
if (error) {
/* XXX some errors are recoverable */
/* send a SIGIO to the owner and exit */
PROC_LOCK(p);
kern_psignal(p, SIGIO);
PROC_UNLOCK(p);
mtx_lock(&pmc_kthread_mtx);
po->po_error = error; /* save for flush log */
PMCDBG2(LOG,WRI,2, "po=%p error=%d", po, error);
break;
}
mtx_lock(&pmc_kthread_mtx);
/* put the used buffer back into the global pool */
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
lb = NULL;
}
wakeup_one(po->po_kthread);
po->po_kthread = NULL;
mtx_unlock(&pmc_kthread_mtx);
/* return the current I/O buffer to the global pool */
if (lb) {
PMCLOG_INIT_BUFFER_DESCRIPTOR(lb);
mtx_lock_spin(&pmc_bufferlist_mtx);
TAILQ_INSERT_HEAD(&pmc_bufferlist, lb, plb_next);
mtx_unlock_spin(&pmc_bufferlist_mtx);
}
/*
* Exit this thread, signalling the waiter
*/
crfree(ownercred);
kproc_exit(0);
}
