void cpuidle_wakeup_mwait(cpumask_t *mask)
{
cpumask_t target;
unsigned int cpu;
cpus_and(target, *mask, cpuidle_mwait_flags);
/* CPU is MWAITing on the cpuidle_mwait_wakeup flag. */
for_each_cpu_mask(cpu, target)
mwait_wakeup(cpu) = 0;
cpus_andnot(*mask, *mask, target);
}
static int vperfctr_enable_control(struct vperfctr *perfctr, struct task_struct *tsk)
{
int err;
unsigned int next_cstatus;
unsigned int nrctrs, i;
if (perfctr->cpu_state.control.header.nractrs ||
perfctr->cpu_state.control.header.nrictrs) {
cpumask_t old_mask, new_mask;
//old_mask = tsk->cpus_allowed;
old_mask = tsk->cpu_mask;
cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask);
if (cpus_empty(new_mask))
return -EINVAL;
if (!cpus_equal(new_mask, old_mask))
set_cpus_allowed(tsk, new_mask);
}
perfctr->cpu_state.user.cstatus = 0;
perfctr->resume_cstatus = 0;
/* remote access note: perfctr_cpu_update_control() is ok */
err = perfctr_cpu_update_control(&perfctr->cpu_state, 0);
if (err < 0)
return err;
next_cstatus = perfctr->cpu_state.user.cstatus;
if (!perfctr_cstatus_enabled(next_cstatus))
return 0;
if (!perfctr_cstatus_has_tsc(next_cstatus))
perfctr->cpu_state.user.tsc_sum = 0;
nrctrs = perfctr_cstatus_nrctrs(next_cstatus);
for(i = 0; i < nrctrs; ++i)
if (!(perfctr->preserve & (1<<i)))
perfctr->cpu_state.user.pmc[i].sum = 0;
spin_lock(&perfctr->children_lock);
perfctr->inheritance_id = new_inheritance_id();
memset(&perfctr->children, 0, sizeof perfctr->children);
spin_unlock(&perfctr->children_lock);
return 0;
}
static void __clear_irq_vector(int irq)
{
int vector, cpu;
cpumask_t mask;
cpumask_t domain;
struct irq_cfg *cfg = &irq_cfg[irq];
BUG_ON((unsigned)irq >= NR_IRQS);
BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED);
vector = cfg->vector;
domain = cfg->domain;
cpus_and(mask, cfg->domain, cpu_online_map);
for_each_cpu_mask(cpu, mask)
per_cpu(vector_irq, cpu)[vector] = -1;
cfg->vector = IRQ_VECTOR_UNASSIGNED;
cfg->domain = CPU_MASK_NONE;
irq_status[irq] = IRQ_UNUSED;
cpus_andnot(vector_table[vector], vector_table[vector], domain);
}
/*
* Register a new batch of callbacks, and start it up if there is currently no
* active batch and the batch to be registered has not already occurred.
* Caller must hold rcu_ctrlblk.lock.
*/
static void rcu_start_batch(struct rcu_ctrlblk *rcp)
{
if (rcp->next_pending &&
rcp->completed == rcp->cur) {
rcp->next_pending = 0;
/*
* next_pending == 0 must be visible in
* __rcu_process_callbacks() before it can see new value of cur.
*/
smp_wmb();
rcp->cur++;
/*
* Accessing nohz_cpu_mask before incrementing rcp->cur needs a
* Barrier Otherwise it can cause tickless idle CPUs to be
* included in rcp->cpumask, which will extend graceperiods
* unnecessarily.
*/
smp_mb();
cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask);
rcp->signaled = 0;
}
}
static int do_vperfctr_control(struct vperfctr *perfctr,
const struct vperfctr_control __user *argp,
unsigned int argbytes,
struct task_struct *tsk)
{
struct vperfctr_control *control;
int err;
unsigned int next_cstatus;
unsigned int nrctrs, i;
if (!tsk) {
return -ESRCH; /* attempt to update unlinked perfctr */
}
/* The control object can be large (over 300 bytes on i386),
so kmalloc() it instead of storing it on the stack.
We must use task-private storage to prevent racing with a
monitor process attaching to us before the non-preemptible
perfctr update step. Therefore we cannot store the copy
in the perfctr object itself. */
control = kmalloc(sizeof(*control), GFP_USER);
if (!control) {
return -ENOMEM;
}
err = -EINVAL;
if (argbytes > sizeof *control) {
goto out_kfree;
}
err = -EFAULT;
if (copy_from_user(control, argp, argbytes)) {
goto out_kfree;
}
if (argbytes < sizeof *control)
memset((char*)control + argbytes, 0, sizeof *control - argbytes);
// figure out what is happening in the following 'if' loop
if (control->cpu_control.nractrs || control->cpu_control.nrictrs) {
cpumask_t old_mask, new_mask;
old_mask = tsk->cpus_allowed;
cpus_andnot(new_mask, old_mask, perfctr_cpus_forbidden_mask);
err = -EINVAL;
if (cpus_empty(new_mask)) {
goto out_kfree;
}
if (!cpus_equal(new_mask, old_mask))
set_cpus_allowed(tsk, new_mask);
}
/* PREEMPT note: preemption is disabled over the entire
region since we're updating an active perfctr. */
preempt_disable();
// the task whose control register I am changing might actually be
// in suspended state. That can happen when the other is executing
// under the control of another task as in the case of debugging
// or ptrace. However, if the write_control is done for the current
// executing process, first suspend them and then do the update
// why are we resetting 'perfctr->cpu_state.cstatus' ?
if (IS_RUNNING(perfctr)) {
if (tsk == current)
vperfctr_suspend(perfctr);
// not sure why we are zeroing out the following explicitly
perfctr->cpu_state.cstatus = 0;
vperfctr_clear_iresume_cstatus(perfctr);
}
// coying the user-specified control values to 'state'
perfctr->cpu_state.control = control->cpu_control;
/* remote access note: perfctr_cpu_update_control() is ok */
err = perfctr_cpu_update_control(&perfctr->cpu_state, 0);
if (err < 0) {
goto out;
}
next_cstatus = perfctr->cpu_state.cstatus;
if (!perfctr_cstatus_enabled(next_cstatus))
goto out;
/* XXX: validate si_signo? */
perfctr->si_signo = control->si_signo;
if (!perfctr_cstatus_has_tsc(next_cstatus))
perfctr->cpu_state.tsc_sum = 0;
nrctrs = perfctr_cstatus_nrctrs(next_cstatus);
for(i = 0; i < nrctrs; ++i)
if (!(control->preserve & (1<<i)))
perfctr->cpu_state.pmc[i].sum = 0;
// I am not sure why we are removing the inheritance just because
// we updated the control information. True, because the children might
// be performing something else. So, the control will have to be set
// before spawning any children
spin_lock(&perfctr->children_lock);
perfctr->inheritance_id = new_inheritance_id();
memset(&perfctr->children, 0, sizeof perfctr->children);
spin_unlock(&perfctr->children_lock);
if (tsk == current) {
vperfctr_resume(perfctr);
}
out:
preempt_enable();
out_kfree:
kfree(control);
return err;
}
unsigned long ipipe_critical_enter(void (*syncfn)(void))
{
int cpu __maybe_unused, n __maybe_unused;
unsigned long flags, loops __maybe_unused;
cpumask_t allbutself __maybe_unused;
flags = hard_local_irq_save();
if (num_online_cpus() == 1)
return flags;
#ifdef CONFIG_SMP
cpu = ipipe_processor_id();
if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
while (test_and_set_bit(0, &__ipipe_critical_lock)) {
n = 0;
hard_local_irq_enable();
do
cpu_relax();
while (++n < cpu);
hard_local_irq_disable();
}
restart:
spin_lock(&__ipipe_cpu_barrier);
__ipipe_cpu_sync = syncfn;
cpus_clear(__ipipe_cpu_pass_map);
cpu_set(cpu, __ipipe_cpu_pass_map);
/*
* Send the sync IPI to all processors but the current
* one.
*/
cpus_andnot(allbutself, cpu_online_map, __ipipe_cpu_pass_map);
ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself);
loops = IPIPE_CRITICAL_TIMEOUT;
while (!cpus_equal(__ipipe_cpu_sync_map, allbutself)) {
if (--loops > 0) {
cpu_relax();
continue;
}
/*
* We ran into a deadlock due to a contended
* rwlock. Cancel this round and retry.
*/
__ipipe_cpu_sync = NULL;
spin_unlock(&__ipipe_cpu_barrier);
/*
* Ensure all CPUs consumed the IPI to avoid
* running __ipipe_cpu_sync prematurely. This
* usually resolves the deadlock reason too.
*/
while (!cpus_equal(cpu_online_map, __ipipe_cpu_pass_map))
cpu_relax();
goto restart;
}
}
atomic_inc(&__ipipe_critical_count);
#endif /* CONFIG_SMP */
return flags;
}
