static __init int jump_label_test(void) { int i; for (i = 0; i < 2; i++) { WARN_ON(static_key_enabled(&sk_true.key) != true); WARN_ON(static_key_enabled(&sk_false.key) != false); WARN_ON(!static_branch_likely(&sk_true)); WARN_ON(!static_branch_unlikely(&sk_true)); WARN_ON(static_branch_likely(&sk_false)); WARN_ON(static_branch_unlikely(&sk_false)); static_branch_disable(&sk_true); static_branch_enable(&sk_false); WARN_ON(static_key_enabled(&sk_true.key) == true); WARN_ON(static_key_enabled(&sk_false.key) == false); WARN_ON(static_branch_likely(&sk_true)); WARN_ON(static_branch_unlikely(&sk_true)); WARN_ON(!static_branch_likely(&sk_false)); WARN_ON(!static_branch_unlikely(&sk_false)); static_branch_enable(&sk_true); static_branch_disable(&sk_false); } return 0; }
int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) { int full; if (static_branch_likely(&psi_disabled)) return -EOPNOTSUPP; update_stats(group); for (full = 0; full < 2 - (res == PSI_CPU); full++) { unsigned long avg[3]; u64 total; int w; for (w = 0; w < 3; w++) avg[w] = group->avg[res * 2 + full][w]; total = div_u64(group->total[res * 2 + full], NSEC_PER_USEC); seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n", full ? "full" : "some", LOAD_INT(avg[0]), LOAD_FRAC(avg[0]), LOAD_INT(avg[1]), LOAD_FRAC(avg[1]), LOAD_INT(avg[2]), LOAD_FRAC(avg[2]), total); } return 0; }
void psi_cgroup_free(struct cgroup *cgroup) { if (static_branch_likely(&psi_disabled)) return; cancel_delayed_work_sync(&cgroup->psi.clock_work); free_percpu(cgroup->psi.pcpu); }
int psi_cgroup_alloc(struct cgroup *cgroup) { if (static_branch_likely(&psi_disabled)) return 0; cgroup->psi.pcpu = alloc_percpu(struct psi_group_cpu); if (!cgroup->psi.pcpu) return -ENOMEM; group_init(&cgroup->psi); return 0; }
/** * psi_memstall_leave - mark the end of an memory stall section * @flags: flags to handle nested memdelay sections * * Marks the calling task as no longer stalled due to lack of memory. */ void psi_memstall_leave(unsigned long *flags) { struct rq_flags rf; struct rq *rq; if (static_branch_likely(&psi_disabled)) return; if (*flags) return; /* * PF_MEMSTALL clearing & accounting needs to be atomic wrt * changes to the task's scheduling state, otherwise we could * race with CPU migration. */ rq = this_rq_lock_irq(&rf); current->flags &= ~PF_MEMSTALL; psi_task_change(current, TSK_MEMSTALL, 0); rq_unlock_irq(rq, &rf); }
/** * cgroup_move_task - move task to a different cgroup * @task: the task * @to: the target css_set * * Move task to a new cgroup and safely migrate its associated stall * state between the different groups. * * This function acquires the task's rq lock to lock out concurrent * changes to the task's scheduling state and - in case the task is * running - concurrent changes to its stall state. */ void cgroup_move_task(struct task_struct *task, struct css_set *to) { unsigned int task_flags = 0; struct rq_flags rf; struct rq *rq; if (static_branch_likely(&psi_disabled)) { /* * Lame to do this here, but the scheduler cannot be locked * from the outside, so we move cgroups from inside sched/. */ rcu_assign_pointer(task->cgroups, to); return; } rq = task_rq_lock(task, &rf); if (task_on_rq_queued(task)) task_flags = TSK_RUNNING; else if (task->in_iowait) task_flags = TSK_IOWAIT; if (task->flags & PF_MEMSTALL) task_flags |= TSK_MEMSTALL; if (task_flags) psi_task_change(task, task_flags, 0); /* See comment above */ rcu_assign_pointer(task->cgroups, to); if (task_flags) psi_task_change(task, 0, task_flags); task_rq_unlock(rq, task, &rf); }
static void cond_ibpb(struct task_struct *next) { if (!next || !next->mm) return; /* * Both, the conditional and the always IBPB mode use the mm * pointer to avoid the IBPB when switching between tasks of the * same process. Using the mm pointer instead of mm->context.ctx_id * opens a hypothetical hole vs. mm_struct reuse, which is more or * less impossible to control by an attacker. Aside of that it * would only affect the first schedule so the theoretically * exposed data is not really interesting. */ if (static_branch_likely(&switch_mm_cond_ibpb)) { unsigned long prev_mm, next_mm; /* * This is a bit more complex than the always mode because * it has to handle two cases: * * 1) Switch from a user space task (potential attacker) * which has TIF_SPEC_IB set to a user space task * (potential victim) which has TIF_SPEC_IB not set. * * 2) Switch from a user space task (potential attacker) * which has TIF_SPEC_IB not set to a user space task * (potential victim) which has TIF_SPEC_IB set. * * This could be done by unconditionally issuing IBPB when * a task which has TIF_SPEC_IB set is either scheduled in * or out. Though that results in two flushes when: * * - the same user space task is scheduled out and later * scheduled in again and only a kernel thread ran in * between. * * - a user space task belonging to the same process is * scheduled in after a kernel thread ran in between * * - a user space task belonging to the same process is * scheduled in immediately. * * Optimize this with reasonably small overhead for the * above cases. Mangle the TIF_SPEC_IB bit into the mm * pointer of the incoming task which is stored in * cpu_tlbstate.last_user_mm_ibpb for comparison. */ next_mm = mm_mangle_tif_spec_ib(next); prev_mm = this_cpu_read(cpu_tlbstate.last_user_mm_ibpb); /* * Issue IBPB only if the mm's are different and one or * both have the IBPB bit set. */ if (next_mm != prev_mm && (next_mm | prev_mm) & LAST_USER_MM_IBPB) indirect_branch_prediction_barrier(); this_cpu_write(cpu_tlbstate.last_user_mm_ibpb, next_mm); } if (static_branch_unlikely(&switch_mm_always_ibpb)) { /* * Only flush when switching to a user space task with a * different context than the user space task which ran * last on this CPU. */ if (this_cpu_read(cpu_tlbstate.last_user_mm) != next->mm) { indirect_branch_prediction_barrier(); this_cpu_write(cpu_tlbstate.last_user_mm, next->mm); } } }
int sched_clock_stable(void) { return static_branch_likely(&__sched_clock_stable); }