inline void perfctr_cpu_suspend(struct perfctr_cpu_state *state) { unsigned int i, cstatus, nractrs; struct perfctr_low_ctrs now; int cpu_id; cpu_id = hard_smp_processor_id() / 4; spin_lock (&pmc_resource[cpu_id].lock); if ( pmc_resource[cpu_id].current_thread != netlogic_thr_id() ) { // printk (KERN_INFO "PMCounters do not belong to this process[%d]\n", current->pid); spin_unlock (&pmc_resource[cpu_id].lock); return; } pmc_resource[cpu_id].current_thread = -1; spin_unlock (&pmc_resource[cpu_id].lock); // To prevent polluting the numbers, can we freeze the counters // here, as early as possible ? if (perfctr_cstatus_has_ictrs(state->cstatus)) { perfctr_cpu_isuspend(state); } perfctr_cpu_read_counters(state, &now); cstatus = state->cstatus; if (perfctr_cstatus_has_tsc(cstatus)) { state->tsc_sum += now.tsc - state->tsc_start; } nractrs = perfctr_cstatus_nractrs(cstatus); for(i = 0; i < nractrs; ++i) { state->pmc[i].sum += now.pmc[i] - state->pmc[i].start; } }
static void do_vperfctr_release(struct vperfctr *child_perfctr, struct task_struct *parent_tsk) { struct vperfctr *parent_perfctr; unsigned int cstatus, nrctrs, i; parent_perfctr = parent_tsk->thread.perfctr; if (parent_perfctr && child_perfctr) { // since more than one child can try to add to parent's // counters, we need a lock spin_lock(&parent_perfctr->children_lock); if (parent_perfctr->inheritance_id == child_perfctr->inheritance_id) { cstatus = parent_perfctr->cpu_state.cstatus; if (perfctr_cstatus_has_tsc(cstatus)) parent_perfctr->children.tsc += child_perfctr->cpu_state.tsc_sum + child_perfctr->children.tsc; nrctrs = perfctr_cstatus_nrctrs(cstatus); for(i = 0; i < nrctrs; ++i) parent_perfctr->children.pmc[i] += child_perfctr->cpu_state.pmc[i].sum + child_perfctr->children.pmc[i]; } spin_unlock(&parent_perfctr->children_lock); } // now that we reaped the data from child's task structure // the child's task structure can be freed. Only the child's // vperfctr structure seems to be released. Is the 'task_struct' // released in __vperfctr_release() itself? Doesn't seem so. schedule_put_vperfctr(child_perfctr); }
// Sampling only a-mode registers void perfctr_cpu_sample(struct perfctr_cpu_state *state) { unsigned int i, cstatus, nractrs; struct perfctr_low_ctrs now; int cpu_id; cpu_id = hard_smp_processor_id() / 4; spin_lock (&pmc_resource[cpu_id].lock); if ( pmc_resource[cpu_id].current_thread != netlogic_thr_id() ) { // printk (KERN_INFO "PMCounters do not belong to this process[%d]\n", current->pid); spin_unlock (&pmc_resource[cpu_id].lock); return; } spin_unlock (&pmc_resource[cpu_id].lock); perfctr_cpu_read_counters(state, &now); // reads only a-mode registers cstatus = state->cstatus; if (perfctr_cstatus_has_tsc(cstatus)) { state->tsc_sum += now.tsc - state->tsc_start; // one needs to update the start status as we continue to gather // statistics without interruption state->tsc_start = now.tsc; } nractrs = perfctr_cstatus_nractrs(cstatus); for(i = 0; i < nractrs; ++i) { state->pmc[i].sum += now.pmc[i] - state->pmc[i].start; state->pmc[i].start = now.pmc[i]; } }
static inline void vperfctr_handle_overflow(struct task_struct *tsk, struct vperfctr *perfctr) { unsigned int pmc_mask; siginfo_t si; sigset_t old_blocked; pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state); if (!pmc_mask) { printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n", __FUNCTION__, tsk->pid); return; } /* suspend a-mode and i-mode PMCs, leaving only TSC on */ /* XXX: some people also want to suspend the TSC */ // we are storing 'cpu_state.cstatus' in 'iresume_cstatus' because // in vperfctr_resume, we only want to read the status of those // In the following TSC is resumed and continues to collect the // stats // if 'perfctr->iresume_cstatus' is not updated below, vperfctr_iresume() fails // as it thinks it was spuriously called inspite of absence of i-mode counters. // vperfctr_iresume() -> ... -> do_vperfctr_iresume() is a different thread of // execution from vperfctr_resume() -> ... -> vperfctr_iresume() -> __write_control() -> // ... perfctr->iresume_cstatus = perfctr->cpu_state.cstatus; if (perfctr_cstatus_has_tsc(perfctr->iresume_cstatus)) { perfctr->cpu_state.cstatus = perfctr_mk_cstatus(1, 0, 0); vperfctr_resume(perfctr); } else { perfctr->cpu_state.cstatus = 0; } // the following siginfo_t structure helps the kernel in invoking // the correct signal handler. Is that right ? // what's the deal with si_errno? what does it say ? si_code ? si.si_signo = perfctr->si_signo; si.si_errno = 0; si.si_code = SI_PMC_OVF; si.si_pmc_ovf_mask = pmc_mask; /* deliver signal without waking up the receiver */ spin_lock_irq(&tsk->sighand->siglock); old_blocked = tsk->blocked; sigaddset(&tsk->blocked, si.si_signo); spin_unlock_irq(&tsk->sighand->siglock); if (!send_sig_info(si.si_signo, &si, tsk)) { send_sig(si.si_signo, tsk, 1); } spin_lock_irq(&tsk->sighand->siglock); tsk->blocked = old_blocked; recalc_sigpending(); spin_unlock_irq(&tsk->sighand->siglock); }
static void vperfctr_handle_overflow(struct task_struct *tsk, struct vperfctr *perfctr) { unsigned int pmc_mask; siginfo_t si; sigset_t old_blocked; pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state); if (!pmc_mask) { #ifdef CONFIG_PPC64 /* On some hardware (ppc64, in particular) it's * impossible to control interrupts finely enough to * eliminate overflows on counters we don't care * about. So in this case just restart the counters * and keep going. */ vperfctr_resume(perfctr); #else printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n", __FUNCTION__, tsk->id); #endif return; } perfctr->ireload_needed = 1; /* suspend a-mode and i-mode PMCs, leaving only TSC on */ /* XXX: some people also want to suspend the TSC */ perfctr->resume_cstatus = perfctr->cpu_state.user.cstatus; if (perfctr_cstatus_has_tsc(perfctr->resume_cstatus)) { perfctr->cpu_state.user.cstatus = perfctr_mk_cstatus(1, 0, 0); vperfctr_resume(perfctr); } else perfctr->cpu_state.user.cstatus = 0; si.si_signo = perfctr->si_signo; si.si_errno = 0; si.si_code = SI_PMC_OVF; si.si_pmc_ovf_mask = pmc_mask; /* deliver signal without waking up the receiver */ spin_lock_irq(&tsk->sighand->siglock); old_blocked = tsk->blocked; sigaddset(&tsk->blocked, si.si_signo); spin_unlock_irq(&tsk->sighand->siglock); if (!send_sig_info(si.si_signo, &si, tsk)) send_sig(si.si_signo, tsk, 1); spin_lock_irq(&tsk->sighand->siglock); tsk->blocked = old_blocked; recalc_sigpending(); spin_unlock_irq(&tsk->sighand->siglock); }
static void mips_read_counters(struct perfctr_cpu_state *state, struct perfctr_low_ctrs *ctrs) { unsigned int cstatus, nrctrs, i; cstatus = state->cstatus; if (perfctr_cstatus_has_tsc(cstatus)) { ctrs->tsc = read_c0_count(); } nrctrs = perfctr_cstatus_nractrs(cstatus); for(i = 0; i < nrctrs; ++i) { unsigned int pmc = state->pmc[i].map; ctrs->pmc[i] = read_pmc(pmc); } }
static int vperfctr_enable_control(struct vperfctr *perfctr, struct task_struct *tsk) { int err; unsigned int next_cstatus; unsigned int nrctrs, i; if (perfctr->cpu_state.control.header.nractrs || perfctr->cpu_state.control.header.nrictrs) { cpumask_t old_mask, new_mask; //old_mask = tsk->cpus_allowed; old_mask = tsk->cpu_mask; cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask); if (cpus_empty(new_mask)) return -EINVAL; if (!cpus_equal(new_mask, old_mask)) set_cpus_allowed(tsk, new_mask); } perfctr->cpu_state.user.cstatus = 0; perfctr->resume_cstatus = 0; /* remote access note: perfctr_cpu_update_control() is ok */ err = perfctr_cpu_update_control(&perfctr->cpu_state, 0); if (err < 0) return err; next_cstatus = perfctr->cpu_state.user.cstatus; if (!perfctr_cstatus_enabled(next_cstatus)) return 0; if (!perfctr_cstatus_has_tsc(next_cstatus)) perfctr->cpu_state.user.tsc_sum = 0; nrctrs = perfctr_cstatus_nrctrs(next_cstatus); for(i = 0; i < nrctrs; ++i) if (!(perfctr->preserve & (1<<i))) perfctr->cpu_state.user.pmc[i].sum = 0; spin_lock(&perfctr->children_lock); perfctr->inheritance_id = new_inheritance_id(); memset(&perfctr->children, 0, sizeof perfctr->children); spin_unlock(&perfctr->children_lock); return 0; }
inline void perfctr_cpu_resume(struct perfctr_cpu_state *state) { int cpu_id; cpu_id = hard_smp_processor_id() / 4; spin_lock (&pmc_resource[cpu_id].lock); if ( pmc_resource[cpu_id].current_thread != -1 ) { // printk (KERN_INFO "PMCounters unavailable for process %d\n", current->pid); spin_unlock (&pmc_resource[cpu_id].lock); return; } pmc_resource[cpu_id].current_thread = netlogic_thr_id(); spin_unlock (&pmc_resource[cpu_id].lock); if (perfctr_cstatus_has_ictrs(state->cstatus)) { perfctr_cpu_iresume(state); } // the counters are triggered, having been frozen in _iresume() // that preceded this point. So, the model is to trigger the // registere to collect the numbers and record the start state // that completes the 'resume' process. perfctr_cpu_write_control(state); { struct perfctr_low_ctrs now; unsigned int i, cstatus, nrctrs; perfctr_cpu_read_counters(state, &now); cstatus = state->cstatus; // the start state of the registers has to be recorded only // in resume() and that is what is being done. if (perfctr_cstatus_has_tsc(cstatus)) { state->tsc_start = now.tsc; } nrctrs = perfctr_cstatus_nractrs(cstatus); for (i = 0; i < nrctrs; ++i) { state->pmc[i].start = now.pmc[i]; } } /* XXX: if (SMP && start.tsc == now.tsc) ++now.tsc; */ }
static void vperfctr_handle_overflow(struct task_struct *tsk, struct vperfctr *perfctr) { unsigned int pmc_mask; siginfo_t si; sigset_t old_blocked; pmc_mask = perfctr_cpu_identify_overflow(&perfctr->cpu_state); if (!pmc_mask) { printk(KERN_ERR "%s: BUG! pid %d has unidentifiable overflow source\n", __FUNCTION__, tsk->pid); return; } /* suspend a-mode and i-mode PMCs, leaving only TSC on */ /* XXX: some people also want to suspend the TSC */ perfctr->iresume_cstatus = perfctr->cpu_state.cstatus; if (perfctr_cstatus_has_tsc(perfctr->iresume_cstatus)) { perfctr->cpu_state.cstatus = perfctr_mk_cstatus(1, 0, 0); vperfctr_resume(perfctr); } else perfctr->cpu_state.cstatus = 0; si.si_signo = perfctr->si_signo; si.si_errno = 0; si.si_code = SI_PMC_OVF; si.si_pmc_ovf_mask = pmc_mask; /* deliver signal without waking up the receiver */ spin_lock_irq(&task_siglock(tsk)); old_blocked = tsk->blocked; sigaddset(&tsk->blocked, si.si_signo); spin_unlock_irq(&task_siglock(tsk)); if (!send_sig_info(si.si_signo, &si, tsk)) send_sig(si.si_signo, tsk, 1); spin_lock_irq(&task_siglock(tsk)); tsk->blocked = old_blocked; recalc_sigpending(); spin_unlock_irq(&task_siglock(tsk)); }
/* release_task() -> perfctr_release_task() -> __vperfctr_release(). * A task is being released. If it inherited its perfctr settings * from its parent, then merge its final counts back into the parent. * Then unlink the child's perfctr. * PRE: caller has write_lock_irq(&tasklist_lock). * PREEMPT note: preemption is disabled due to tasklist_lock. * * When current == parent_tsk, the child's counts can be merged * into the parent's immediately. This is the common case. * * When current != parent_tsk, the parent must be task_lock()ed * before its perfctr state can be accessed. task_lock() is illegal * here due to the write_lock_irq(&tasklist_lock) in release_task(), * so the operation is done via schedule_work(). */ static void do_vperfctr_release(struct vperfctr *child_perfctr, struct task_struct *parent_tsk) { struct vperfctr *parent_perfctr; unsigned int cstatus, nrctrs, i; parent_perfctr = parent_tsk->arch.thread.perfctr; if (parent_perfctr && child_perfctr) { spin_lock(&parent_perfctr->children_lock); if (parent_perfctr->inheritance_id == child_perfctr->inheritance_id) { cstatus = parent_perfctr->cpu_state.user.cstatus; if (perfctr_cstatus_has_tsc(cstatus)) parent_perfctr->children.tsc += child_perfctr->cpu_state.user.tsc_sum + child_perfctr->children.tsc; nrctrs = perfctr_cstatus_nrctrs(cstatus); for(i = 0; i < nrctrs; ++i) parent_perfctr->children.pmc[i] += child_perfctr->cpu_state.user.pmc[i].sum + child_perfctr->children.pmc[i]; } spin_unlock(&parent_perfctr->children_lock); } schedule_put_vperfctr(child_perfctr); }
static int do_vperfctr_control(struct vperfctr *perfctr, const struct vperfctr_control __user *argp, unsigned int argbytes, struct task_struct *tsk) { struct vperfctr_control *control; int err; unsigned int next_cstatus; unsigned int nrctrs, i; if (!tsk) { return -ESRCH; /* attempt to update unlinked perfctr */ } /* The control object can be large (over 300 bytes on i386), so kmalloc() it instead of storing it on the stack. We must use task-private storage to prevent racing with a monitor process attaching to us before the non-preemptible perfctr update step. Therefore we cannot store the copy in the perfctr object itself. */ control = kmalloc(sizeof(*control), GFP_USER); if (!control) { return -ENOMEM; } err = -EINVAL; if (argbytes > sizeof *control) { goto out_kfree; } err = -EFAULT; if (copy_from_user(control, argp, argbytes)) { goto out_kfree; } if (argbytes < sizeof *control) memset((char*)control + argbytes, 0, sizeof *control - argbytes); // figure out what is happening in the following 'if' loop if (control->cpu_control.nractrs || control->cpu_control.nrictrs) { cpumask_t old_mask, new_mask; old_mask = tsk->cpus_allowed; cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask); err = -EINVAL; if (cpus_empty(new_mask)) { goto out_kfree; } if (!cpus_equal(new_mask, old_mask)) set_cpus_allowed(tsk, new_mask); } /* PREEMPT note: preemption is disabled over the entire region since we're updating an active perfctr. */ preempt_disable(); // the task whose control register I am changing might actually be // in suspended state. That can happen when the other is executing // under the control of another task as in the case of debugging // or ptrace. However, if the write_control is done for the current // executing process, first suspend them and then do the update // why are we resetting 'perfctr->cpu_state.cstatus' ? if (IS_RUNNING(perfctr)) { if (tsk == current) vperfctr_suspend(perfctr); // not sure why we are zeroing out the following explicitly perfctr->cpu_state.cstatus = 0; vperfctr_clear_iresume_cstatus(perfctr); } // coying the user-specified control values to 'state' perfctr->cpu_state.control = control->cpu_control; /* remote access note: perfctr_cpu_update_control() is ok */ err = perfctr_cpu_update_control(&perfctr->cpu_state, 0); if (err < 0) { goto out; } next_cstatus = perfctr->cpu_state.cstatus; if (!perfctr_cstatus_enabled(next_cstatus)) goto out; /* XXX: validate si_signo? */ perfctr->si_signo = control->si_signo; if (!perfctr_cstatus_has_tsc(next_cstatus)) perfctr->cpu_state.tsc_sum = 0; nrctrs = perfctr_cstatus_nrctrs(next_cstatus); for(i = 0; i < nrctrs; ++i) if (!(control->preserve & (1<<i))) perfctr->cpu_state.pmc[i].sum = 0; // I am not sure why we are removing the inheritance just because // we updated the control information. True, because the children might // be performing something else. So, the control will have to be set // before spawning any children spin_lock(&perfctr->children_lock); perfctr->inheritance_id = new_inheritance_id(); memset(&perfctr->children, 0, sizeof perfctr->children); spin_unlock(&perfctr->children_lock); if (tsk == current) { vperfctr_resume(perfctr); } out: preempt_enable(); out_kfree: kfree(control); return err; }
static int sys_vperfctr_control(struct vperfctr *perfctr, struct perfctr_struct_buf *argp, struct task_struct *tsk) { struct vperfctr_control control; int err; unsigned int next_cstatus; unsigned int nrctrs, i; cpumask_t cpumask; if (!tsk) return -ESRCH; /* attempt to update unlinked perfctr */ err = perfctr_copy_from_user(&control, argp, &vperfctr_control_sdesc); if (err) return err; /* Step 1: Update the control but keep the counters disabled. PREEMPT note: Preemption is disabled since we're updating an active perfctr. */ preempt_disable(); if (IS_RUNNING(perfctr)) { if (tsk == current) vperfctr_suspend(perfctr); perfctr->cpu_state.cstatus = 0; vperfctr_clear_iresume_cstatus(perfctr); } perfctr->cpu_state.control = control.cpu_control; /* remote access note: perfctr_cpu_update_control() is ok */ cpus_setall(cpumask); #ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK /* make a stopped vperfctr have an unconstrained cpumask */ perfctr->cpumask = cpumask; #endif err = perfctr_cpu_update_control(&perfctr->cpu_state, &cpumask); if (err < 0) { next_cstatus = 0; } else { next_cstatus = perfctr->cpu_state.cstatus; perfctr->cpu_state.cstatus = 0; perfctr->updater_tgid = current->tgid; #ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK perfctr->cpumask = cpumask; #endif } preempt_enable_no_resched(); if (!perfctr_cstatus_enabled(next_cstatus)) return err; #ifdef CONFIG_PERFCTR_CPUS_FORBIDDEN_MASK /* Step 2: Update the task's CPU affinity mask. PREEMPT note: Preemption must be enabled for set_cpus_allowed(). */ if (control.cpu_control.nractrs || control.cpu_control.nrictrs) { cpumask_t old_mask, new_mask; old_mask = tsk->cpus_allowed; cpus_and(new_mask, old_mask, cpumask); if (cpus_empty(new_mask)) return -EINVAL; if (!cpus_equal(new_mask, old_mask)) set_cpus_allowed(tsk, new_mask); } #endif /* Step 3: Enable the counters with the new control and affinity. PREEMPT note: Preemption is disabled since we're updating an active perfctr. */ preempt_disable(); /* We had to enable preemption above for set_cpus_allowed() so we may have lost a race with a concurrent update via the remote control interface. If so then we must abort our update of this perfctr. */ if (perfctr->updater_tgid != current->tgid) { printk(KERN_WARNING "perfctr: control update by task %d" " was lost due to race with update by task %d\n", current->tgid, perfctr->updater_tgid); err = -EBUSY; } else { /* XXX: validate si_signo? */ perfctr->si_signo = control.si_signo; perfctr->cpu_state.cstatus = next_cstatus; if (!perfctr_cstatus_has_tsc(next_cstatus)) perfctr->cpu_state.tsc_sum = 0; nrctrs = perfctr_cstatus_nrctrs(next_cstatus); for(i = 0; i < nrctrs; ++i) if (!(control.preserve & (1<<i))) perfctr->cpu_state.pmc[i].sum = 0; perfctr->flags = control.flags; if (tsk == current) vperfctr_resume(perfctr); } preempt_enable(); return err; }