inline void perfctr_cpu_suspend(struct perfctr_cpu_state *state)
{
unsigned int i, cstatus, nractrs;
struct perfctr_low_ctrs now;
int cpu_id;
cpu_id = hard_smp_processor_id() / 4;
spin_lock (&pmc_resource[cpu_id].lock);
if ( pmc_resource[cpu_id].current_thread != netlogic_thr_id() ) {
// printk (KERN_INFO "PMCounters do not belong to this process[%d]\n", current->pid);
spin_unlock (&pmc_resource[cpu_id].lock);
return;
}
pmc_resource[cpu_id].current_thread = -1;
spin_unlock (&pmc_resource[cpu_id].lock);
// To prevent polluting the numbers, can we freeze the counters
// here, as early as possible ?
if (perfctr_cstatus_has_ictrs(state->cstatus)) {
perfctr_cpu_isuspend(state);
}
perfctr_cpu_read_counters(state, &now);
cstatus = state->cstatus;
if (perfctr_cstatus_has_tsc(cstatus)) {
state->tsc_sum += now.tsc - state->tsc_start;
}
nractrs = perfctr_cstatus_nractrs(cstatus);
for(i = 0; i < nractrs; ++i) {
state->pmc[i].sum += now.pmc[i] - state->pmc[i].start;
}
}
static void do_vperfctr_release(struct vperfctr *child_perfctr, struct task_struct *parent_tsk)
{
struct vperfctr *parent_perfctr;
unsigned int cstatus, nrctrs, i;
parent_perfctr = parent_tsk->thread.perfctr;
if (parent_perfctr && child_perfctr) {
// since more than one child can try to add to parent's
// counters, we need a lock
spin_lock(&parent_perfctr->children_lock);
if (parent_perfctr->inheritance_id == child_perfctr->inheritance_id) {
cstatus = parent_perfctr->cpu_state.cstatus;
if (perfctr_cstatus_has_tsc(cstatus))
parent_perfctr->children.tsc +=
child_perfctr->cpu_state.tsc_sum +
child_perfctr->children.tsc;
nrctrs = perfctr_cstatus_nrctrs(cstatus);
for(i = 0; i < nrctrs; ++i)
parent_perfctr->children.pmc[i] +=
child_perfctr->cpu_state.pmc[i].sum +
child_perfctr->children.pmc[i];
}
spin_unlock(&parent_perfctr->children_lock);
}
// now that we reaped the data from child's task structure
// the child's task structure can be freed. Only the child's
// vperfctr structure seems to be released. Is the 'task_struct'
// released in __vperfctr_release() itself? Doesn't seem so.
schedule_put_vperfctr(child_perfctr);
}
// Sampling only a-mode registers
void perfctr_cpu_sample(struct perfctr_cpu_state *state)
{
unsigned int i, cstatus, nractrs;
struct perfctr_low_ctrs now;
int cpu_id;
cpu_id = hard_smp_processor_id() / 4;
spin_lock (&pmc_resource[cpu_id].lock);
if ( pmc_resource[cpu_id].current_thread != netlogic_thr_id() ) {
// printk (KERN_INFO "PMCounters do not belong to this process[%d]\n", current->pid);
spin_unlock (&pmc_resource[cpu_id].lock);
return;
}
spin_unlock (&pmc_resource[cpu_id].lock);
perfctr_cpu_read_counters(state, &now); // reads only a-mode registers
cstatus = state->cstatus;
if (perfctr_cstatus_has_tsc(cstatus)) {
state->tsc_sum += now.tsc - state->tsc_start;
// one needs to update the start status as we continue to gather
// statistics without interruption
state->tsc_start = now.tsc;
}
nractrs = perfctr_cstatus_nractrs(cstatus);
for(i = 0; i < nractrs; ++i) {
state->pmc[i].sum += now.pmc[i] - state->pmc[i].start;
state->pmc[i].start = now.pmc[i];
}
}
static inline void vperfctr_handle_overflow(struct task_struct *tsk,
struct vperfctr *perfctr)
{
unsigned int pmc_mask;
siginfo_t si;
sigset_t old_blocked;
pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state);
if (!pmc_mask) {
printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n", __FUNCTION__, tsk->pid);
return;
}
/* suspend a-mode and i-mode PMCs, leaving only TSC on */
/* XXX: some people also want to suspend the TSC */
// we are storing 'cpu_state.cstatus' in 'iresume_cstatus' because
// in vperfctr_resume, we only want to read the status of those
// In the following TSC is resumed and continues to collect the
// stats
// if 'perfctr->iresume_cstatus' is not updated below, vperfctr_iresume() fails
// as it thinks it was spuriously called inspite of absence of i-mode counters.
// vperfctr_iresume() -> ... -> do_vperfctr_iresume() is a different thread of
// execution from vperfctr_resume() -> ... -> vperfctr_iresume() -> __write_control() ->
// ...
perfctr->iresume_cstatus = perfctr->cpu_state.cstatus;
if (perfctr_cstatus_has_tsc(perfctr->iresume_cstatus)) {
perfctr->cpu_state.cstatus = perfctr_mk_cstatus(1, 0, 0);
vperfctr_resume(perfctr);
}
else {
perfctr->cpu_state.cstatus = 0;
}
// the following siginfo_t structure helps the kernel in invoking
// the correct signal handler. Is that right ?
// what's the deal with si_errno? what does it say ? si_code ?
si.si_signo = perfctr->si_signo;
si.si_errno = 0;
si.si_code = SI_PMC_OVF;
si.si_pmc_ovf_mask = pmc_mask;
/* deliver signal without waking up the receiver */
spin_lock_irq(&tsk->sighand->siglock);
old_blocked = tsk->blocked;
sigaddset(&tsk->blocked, si.si_signo);
spin_unlock_irq(&tsk->sighand->siglock);
if (!send_sig_info(si.si_signo, &si, tsk)) {
send_sig(si.si_signo, tsk, 1);
}
spin_lock_irq(&tsk->sighand->siglock);
tsk->blocked = old_blocked;
recalc_sigpending();
spin_unlock_irq(&tsk->sighand->siglock);
}
static void vperfctr_handle_overflow(struct task_struct *tsk,
struct vperfctr *perfctr)
{
unsigned int pmc_mask;
siginfo_t si;
sigset_t old_blocked;
pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state);
if (!pmc_mask) {
#ifdef CONFIG_PPC64
/* On some hardware (ppc64, in particular) it's
* impossible to control interrupts finely enough to
* eliminate overflows on counters we don't care
* about. So in this case just restart the counters
* and keep going. */
vperfctr_resume(perfctr);
#else
printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n",
__FUNCTION__, tsk->id);
#endif
return;
}
perfctr->ireload_needed = 1;
/* suspend a-mode and i-mode PMCs, leaving only TSC on */
/* XXX: some people also want to suspend the TSC */
perfctr->resume_cstatus = perfctr->cpu_state.user.cstatus;
if (perfctr_cstatus_has_tsc(perfctr->resume_cstatus)) {
perfctr->cpu_state.user.cstatus = perfctr_mk_cstatus(1, 0, 0);
vperfctr_resume(perfctr);
} else
perfctr->cpu_state.user.cstatus = 0;
si.si_signo = perfctr->si_signo;
si.si_errno = 0;
si.si_code = SI_PMC_OVF;
si.si_pmc_ovf_mask = pmc_mask;
/* deliver signal without waking up the receiver */
spin_lock_irq(&tsk->sighand->siglock);
old_blocked = tsk->blocked;
sigaddset(&tsk->blocked, si.si_signo);
spin_unlock_irq(&tsk->sighand->siglock);
if (!send_sig_info(si.si_signo, &si, tsk))
send_sig(si.si_signo, tsk, 1);
spin_lock_irq(&tsk->sighand->siglock);
tsk->blocked = old_blocked;
recalc_sigpending();
spin_unlock_irq(&tsk->sighand->siglock);
}
static void mips_read_counters(struct perfctr_cpu_state *state,
struct perfctr_low_ctrs *ctrs)
{
unsigned int cstatus, nrctrs, i;
cstatus = state->cstatus;
if (perfctr_cstatus_has_tsc(cstatus)) {
ctrs->tsc = read_c0_count();
}
nrctrs = perfctr_cstatus_nractrs(cstatus);
for(i = 0; i < nrctrs; ++i) {
unsigned int pmc = state->pmc[i].map;
ctrs->pmc[i] = read_pmc(pmc);
}
}
static int vperfctr_enable_control(struct vperfctr *perfctr, struct task_struct *tsk)
{
int err;
unsigned int next_cstatus;
unsigned int nrctrs, i;
if (perfctr->cpu_state.control.header.nractrs ||
perfctr->cpu_state.control.header.nrictrs) {
cpumask_t old_mask, new_mask;
//old_mask = tsk->cpus_allowed;
old_mask = tsk->cpu_mask;
cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask);
if (cpus_empty(new_mask))
return -EINVAL;
if (!cpus_equal(new_mask, old_mask))
set_cpus_allowed(tsk, new_mask);
}
perfctr->cpu_state.user.cstatus = 0;
perfctr->resume_cstatus = 0;
/* remote access note: perfctr_cpu_update_control() is ok */
err = perfctr_cpu_update_control(&perfctr->cpu_state, 0);
if (err < 0)
return err;
next_cstatus = perfctr->cpu_state.user.cstatus;
if (!perfctr_cstatus_enabled(next_cstatus))
return 0;
if (!perfctr_cstatus_has_tsc(next_cstatus))
perfctr->cpu_state.user.tsc_sum = 0;
nrctrs = perfctr_cstatus_nrctrs(next_cstatus);
for(i = 0; i < nrctrs; ++i)
if (!(perfctr->preserve & (1<<i)))
perfctr->cpu_state.user.pmc[i].sum = 0;
spin_lock(&perfctr->children_lock);
perfctr->inheritance_id = new_inheritance_id();
memset(&perfctr->children, 0, sizeof perfctr->children);
spin_unlock(&perfctr->children_lock);
return 0;
}
inline void perfctr_cpu_resume(struct perfctr_cpu_state *state)
{
int cpu_id;
cpu_id = hard_smp_processor_id() / 4;
spin_lock (&pmc_resource[cpu_id].lock);
if ( pmc_resource[cpu_id].current_thread != -1 ) {
// printk (KERN_INFO "PMCounters unavailable for process %d\n", current->pid);
spin_unlock (&pmc_resource[cpu_id].lock);
return;
}
pmc_resource[cpu_id].current_thread = netlogic_thr_id();
spin_unlock (&pmc_resource[cpu_id].lock);
if (perfctr_cstatus_has_ictrs(state->cstatus)) {
perfctr_cpu_iresume(state);
}
// the counters are triggered, having been frozen in _iresume()
// that preceded this point. So, the model is to trigger the
// registere to collect the numbers and record the start state
// that completes the 'resume' process.
perfctr_cpu_write_control(state);
{
struct perfctr_low_ctrs now;
unsigned int i, cstatus, nrctrs;
perfctr_cpu_read_counters(state, &now);
cstatus = state->cstatus;
// the start state of the registers has to be recorded only
// in resume() and that is what is being done.
if (perfctr_cstatus_has_tsc(cstatus)) {
state->tsc_start = now.tsc;
}
nrctrs = perfctr_cstatus_nractrs(cstatus);
for (i = 0; i < nrctrs; ++i) {
state->pmc[i].start = now.pmc[i];
}
}
/* XXX: if (SMP && start.tsc == now.tsc) ++now.tsc; */
}
static void vperfctr_handle_overflow(struct task_struct *tsk,
struct vperfctr *perfctr)
{
unsigned int pmc_mask;
siginfo_t si;
sigset_t old_blocked;
pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state);
if (!pmc_mask) {
printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n",
__FUNCTION__, tsk->pid);
return;
}
/* suspend a-mode and i-mode PMCs, leaving only TSC on */
/* XXX: some people also want to suspend the TSC */
perfctr->iresume_cstatus = perfctr->cpu_state.cstatus;
if (perfctr_cstatus_has_tsc(perfctr->iresume_cstatus)) {
perfctr->cpu_state.cstatus = perfctr_mk_cstatus(1, 0, 0);
vperfctr_resume(perfctr);
} else
perfctr->cpu_state.cstatus = 0;
si.si_signo = perfctr->si_signo;
si.si_errno = 0;
si.si_code = SI_PMC_OVF;
si.si_pmc_ovf_mask = pmc_mask;
/* deliver signal without waking up the receiver */
spin_lock_irq(&task_siglock(tsk));
old_blocked = tsk->blocked;
sigaddset(&tsk->blocked, si.si_signo);
spin_unlock_irq(&task_siglock(tsk));
if (!send_sig_info(si.si_signo, &si, tsk))
send_sig(si.si_signo, tsk, 1);
spin_lock_irq(&task_siglock(tsk));
tsk->blocked = old_blocked;
recalc_sigpending();
spin_unlock_irq(&task_siglock(tsk));
}
/* release_task() -> perfctr_release_task() -> __vperfctr_release().
* A task is being released. If it inherited its perfctr settings
* from its parent, then merge its final counts back into the parent.
* Then unlink the child's perfctr.
* PRE: caller has write_lock_irq(&tasklist_lock).
* PREEMPT note: preemption is disabled due to tasklist_lock.
*
* When current == parent_tsk, the child's counts can be merged
* into the parent's immediately. This is the common case.
*
* When current != parent_tsk, the parent must be task_lock()ed
* before its perfctr state can be accessed. task_lock() is illegal
* here due to the write_lock_irq(&tasklist_lock) in release_task(),
* so the operation is done via schedule_work().
*/
static void do_vperfctr_release(struct vperfctr *child_perfctr, struct task_struct *parent_tsk)
{
struct vperfctr *parent_perfctr;
unsigned int cstatus, nrctrs, i;
parent_perfctr = parent_tsk->arch.thread.perfctr;
if (parent_perfctr && child_perfctr) {
spin_lock(&parent_perfctr->children_lock);
if (parent_perfctr->inheritance_id == child_perfctr->inheritance_id) {
cstatus = parent_perfctr->cpu_state.user.cstatus;
if (perfctr_cstatus_has_tsc(cstatus))
parent_perfctr->children.tsc +=
child_perfctr->cpu_state.user.tsc_sum +
child_perfctr->children.tsc;
nrctrs = perfctr_cstatus_nrctrs(cstatus);
for(i = 0; i < nrctrs; ++i)
parent_perfctr->children.pmc[i] +=
child_perfctr->cpu_state.user.pmc[i].sum +
child_perfctr->children.pmc[i];
}
spin_unlock(&parent_perfctr->children_lock);
}
schedule_put_vperfctr(child_perfctr);
}
static int do_vperfctr_control(struct vperfctr *perfctr,
const struct vperfctr_control __user *argp,
unsigned int argbytes,
struct task_struct *tsk)
{
struct vperfctr_control *control;
int err;
unsigned int next_cstatus;
unsigned int nrctrs, i;
if (!tsk) {
return -ESRCH; /* attempt to update unlinked perfctr */
}
/* The control object can be large (over 300 bytes on i386),
so kmalloc() it instead of storing it on the stack.
We must use task-private storage to prevent racing with a
monitor process attaching to us before the non-preemptible
perfctr update step. Therefore we cannot store the copy
in the perfctr object itself. */
control = kmalloc(sizeof(*control), GFP_USER);
if (!control) {
return -ENOMEM;
}
err = -EINVAL;
if (argbytes > sizeof *control) {
goto out_kfree;
}
err = -EFAULT;
if (copy_from_user(control, argp, argbytes)) {
goto out_kfree;
}
if (argbytes < sizeof *control)
memset((char*)control + argbytes, 0, sizeof *control - argbytes);
// figure out what is happening in the following 'if' loop
if (control->cpu_control.nractrs || control->cpu_control.nrictrs) {
cpumask_t old_mask, new_mask;
old_mask = tsk->cpus_allowed;
cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask);
err = -EINVAL;
if (cpus_empty(new_mask)) {
goto out_kfree;
}
if (!cpus_equal(new_mask, old_mask))
set_cpus_allowed(tsk, new_mask);
}
/* PREEMPT note: preemption is disabled over the entire
region since we're updating an active perfctr. */
preempt_disable();
// the task whose control register I am changing might actually be
// in suspended state. That can happen when the other is executing
// under the control of another task as in the case of debugging
// or ptrace. However, if the write_control is done for the current
// executing process, first suspend them and then do the update
// why are we resetting 'perfctr->cpu_state.cstatus' ?
if (IS_RUNNING(perfctr)) {
if (tsk == current)
vperfctr_suspend(perfctr);
// not sure why we are zeroing out the following explicitly
perfctr->cpu_state.cstatus = 0;
vperfctr_clear_iresume_cstatus(perfctr);
}
// coying the user-specified control values to 'state'
perfctr->cpu_state.control = control->cpu_control;
/* remote access note: perfctr_cpu_update_control() is ok */
err = perfctr_cpu_update_control(&perfctr->cpu_state, 0);
if (err < 0) {
goto out;
}
next_cstatus = perfctr->cpu_state.cstatus;
if (!perfctr_cstatus_enabled(next_cstatus))
goto out;
/* XXX: validate si_signo? */
perfctr->si_signo = control->si_signo;
if (!perfctr_cstatus_has_tsc(next_cstatus))
perfctr->cpu_state.tsc_sum = 0;
nrctrs = perfctr_cstatus_nrctrs(next_cstatus);
for(i = 0; i < nrctrs; ++i)
if (!(control->preserve & (1<<i)))
perfctr->cpu_state.pmc[i].sum = 0;
// I am not sure why we are removing the inheritance just because
// we updated the control information. True, because the children might
// be performing something else. So, the control will have to be set
// before spawning any children
spin_lock(&perfctr->children_lock);
perfctr->inheritance_id = new_inheritance_id();
memset(&perfctr->children, 0, sizeof perfctr->children);
spin_unlock(&perfctr->children_lock);
if (tsk == current) {
vperfctr_resume(perfctr);
}
out:
preempt_enable();
out_kfree:
kfree(control);
return err;
}
static int sys_vperfctr_control(struct vperfctr *perfctr,
struct perfctr_struct_buf *argp,
struct task_struct *tsk)
{
struct vperfctr_control control;
int err;
unsigned int next_cstatus;
unsigned int nrctrs, i;
cpumask_t cpumask;
if (!tsk)
return -ESRCH; /* attempt to update unlinked perfctr */
err = perfctr_copy_from_user(&control, argp, &vperfctr_control_sdesc);
if (err)
return err;
/* Step 1: Update the control but keep the counters disabled.
PREEMPT note: Preemption is disabled since we're updating
an active perfctr. */
preempt_disable();
if (IS_RUNNING(perfctr)) {
if (tsk == current)
vperfctr_suspend(perfctr);
perfctr->cpu_state.cstatus = 0;
vperfctr_clear_iresume_cstatus(perfctr);
}
perfctr->cpu_state.control = control.cpu_control;
/* remote access note: perfctr_cpu_update_control() is ok */
cpus_setall(cpumask);
#ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK
/* make a stopped vperfctr have an unconstrained cpumask */
perfctr->cpumask = cpumask;
#endif
err = perfctr_cpu_update_control(&perfctr->cpu_state, &cpumask);
if (err < 0) {
next_cstatus = 0;
} else {
next_cstatus = perfctr->cpu_state.cstatus;
perfctr->cpu_state.cstatus = 0;
perfctr->updater_tgid = current->tgid;
#ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK
perfctr->cpumask = cpumask;
#endif
}
preempt_enable_no_resched();
if (!perfctr_cstatus_enabled(next_cstatus))
return err;
#ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK
/* Step 2: Update the task's CPU affinity mask.
PREEMPT note: Preemption must be enabled for set_cpus_allowed(). */
if (control.cpu_control.nractrs || control.cpu_control.nrictrs) {
cpumask_t old_mask, new_mask;
old_mask = tsk->cpus_allowed;
cpus_and(new_mask, old_mask, cpumask);
if (cpus_empty(new_mask))
return -EINVAL;
if (!cpus_equal(new_mask, old_mask))
set_cpus_allowed(tsk, new_mask);
}
#endif
/* Step 3: Enable the counters with the new control and affinity.
PREEMPT note: Preemption is disabled since we're updating
an active perfctr. */
preempt_disable();
/* We had to enable preemption above for set_cpus_allowed() so we may
have lost a race with a concurrent update via the remote control
interface. If so then we must abort our update of this perfctr. */
if (perfctr->updater_tgid != current->tgid) {
printk(KERN_WARNING "perfctr: control update by task %d"
" was lost due to race with update by task %d\n",
current->tgid, perfctr->updater_tgid);
err = -EBUSY;
} else {
/* XXX: validate si_signo? */
perfctr->si_signo = control.si_signo;
perfctr->cpu_state.cstatus = next_cstatus;
if (!perfctr_cstatus_has_tsc(next_cstatus))
perfctr->cpu_state.tsc_sum = 0;
nrctrs = perfctr_cstatus_nrctrs(next_cstatus);
for(i = 0; i < nrctrs; ++i)
if (!(control.preserve & (1<<i)))
perfctr->cpu_state.pmc[i].sum = 0;
perfctr->flags = control.flags;
if (tsk == current)
vperfctr_resume(perfctr);
}
preempt_enable();
return err;
}
