/* * Check if exiting cpu is the designated reader. If so migrate the * events when there is a valid target available */ static int cstate_cpu_exit(unsigned int cpu) { unsigned int target; if (has_cstate_core && cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask)) { target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); /* Migrate events if there is a valid target */ if (target < nr_cpu_ids) { cpumask_set_cpu(target, &cstate_core_cpu_mask); perf_pmu_migrate_context(&cstate_core_pmu, cpu, target); } } if (has_cstate_pkg && cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask)) { target = cpumask_any_but(topology_core_cpumask(cpu), cpu); /* Migrate events if there is a valid target */ if (target < nr_cpu_ids) { cpumask_set_cpu(target, &cstate_pkg_cpu_mask); perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target); } } return 0; }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start) { struct mm_struct *mm = vma->vm_mm; preempt_disable(); if (current->active_mm == mm) { if (current->mm) { /* * Implicit full barrier (INVLPG) that synchronizes * with switch_mm. */ __flush_tlb_one(start); } else { leave_mm(smp_processor_id()); /* Synchronize with switch_mm. */ smp_mb(); } } if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) flush_tlb_others(mm_cpumask(mm), mm, start, 0UL); preempt_enable(); }
static void tegra_cpufreq_hotplug(NvRmPmRequest req) { int rc = 0; #ifdef CONFIG_HOTPLUG_CPU unsigned int cpu; int policy = atomic_read(&hotplug_policy); smp_rmb(); if (disable_hotplug) return; if (req & NvRmPmRequest_CpuOnFlag && (policy > 1 || !policy)) { struct cpumask m; cpumask_andnot(&m, cpu_present_mask, cpu_online_mask); cpu = cpumask_any(&m); if (cpu_present(cpu) && !cpu_online(cpu)) rc = cpu_up(cpu); } else if (req & NvRmPmRequest_CpuOffFlag && (policy < NR_CPUS || !policy)) { cpu = cpumask_any_but(cpu_online_mask, 0); if (cpu_present(cpu) && cpu_online(cpu)) rc = cpu_down(cpu); } #endif if (rc) pr_err("%s: error %d servicing hot plug request\n", __func__, rc); }
/* * Store and check the TSC ADJUST MSR if available */ bool tsc_store_and_check_tsc_adjust(bool bootcpu) { struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); unsigned int refcpu, cpu = smp_processor_id(); struct cpumask *mask; s64 bootval; if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) return false; rdmsrl(MSR_IA32_TSC_ADJUST, bootval); cur->bootval = bootval; cur->nextcheck = jiffies + HZ; cur->warned = false; /* * Check whether this CPU is the first in a package to come up. In * this case do not check the boot value against another package * because the new package might have been physically hotplugged, * where TSC_ADJUST is expected to be different. When called on the * boot CPU topology_core_cpumask() might not be available yet. */ mask = topology_core_cpumask(cpu); refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids; if (refcpu >= nr_cpu_ids) { tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu); return false; } ref = per_cpu_ptr(&tsc_adjust, refcpu); /* * Compare the boot value and complain if it differs in the * package. */ if (bootval != ref->bootval) { pr_warn(FW_BUG "TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n", refcpu, ref->bootval, cpu, bootval); } /* * The TSC_ADJUST values in a package must be the same. If the boot * value on this newly upcoming CPU differs from the adjustment * value of the already online CPU in this package, set it to that * adjusted value. */ if (bootval != ref->adjusted) { pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n", refcpu, ref->adjusted, cpu, bootval); cur->adjusted = ref->adjusted; wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); } /* * We have the TSCs forced to be in sync on this package. Skip sync * test: */ return true; }
void flush_tlb_current_task(void) { struct mm_struct *mm = current->mm; preempt_disable(); local_flush_tlb(); if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids) flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL); preempt_enable(); }
void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long vmflag) { unsigned long addr; /* do a global flush by default */ unsigned long base_pages_to_flush = TLB_FLUSH_ALL; preempt_disable(); if (current->active_mm != mm) { /* Synchronize with switch_mm. */ smp_mb(); goto out; } if (!current->mm) { leave_mm(smp_processor_id()); /* Synchronize with switch_mm. */ smp_mb(); goto out; } if ((end != TLB_FLUSH_ALL) && !(vmflag & VM_HUGETLB)) base_pages_to_flush = (end - start) >> PAGE_SHIFT; /* * Both branches below are implicit full barriers (MOV to CR or * INVLPG) that synchronize with switch_mm. */ if (base_pages_to_flush > tlb_single_page_flush_ceiling) { base_pages_to_flush = TLB_FLUSH_ALL; count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); local_flush_tlb(); } else { /* flush range by one by one 'invlpg' */ for (addr = start; addr < end; addr += PAGE_SIZE) { count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); __flush_tlb_single(addr); } } trace_tlb_flush(TLB_LOCAL_MM_SHOOTDOWN, base_pages_to_flush); out: if (base_pages_to_flush == TLB_FLUSH_ALL) { start = 0UL; end = TLB_FLUSH_ALL; } if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) flush_tlb_others(mm_cpumask(mm), mm, start, end); preempt_enable(); }
static void bpmp_irq_clr_affinity(int cpu) { int nr_cpus = num_present_cpus(); int new_cpu; int r; int i; for (i = cpu; i < ARRAY_SIZE(cpu_irqs); i += nr_cpus) { new_cpu = cpumask_any_but(cpu_online_mask, cpu); r = irq_set_affinity(cpu_irqs[i], cpumask_of(new_cpu)); WARN_ON(r); } }
void flush_tlb_current_task(void) { struct mm_struct *mm = current->mm; preempt_disable(); count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); local_flush_tlb(); trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL); if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); preempt_enable(); }
void flush_tlb_mm(struct mm_struct *mm) { preempt_disable(); if (current->active_mm == mm) { if (current->mm) local_flush_tlb(); else leave_mm(smp_processor_id()); } if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids) flush_tlb_others(&mm->cpu_vm_mask, mm, TLB_FLUSH_ALL); preempt_enable(); }
void flush_tlb_current_task(void) { struct mm_struct *mm = current->mm; preempt_disable(); count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); /* This is an implicit full barrier that synchronizes with switch_mm. */ local_flush_tlb(); trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL); if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); preempt_enable(); }
static int power_cpu_init(unsigned int cpu) { int target; /* * 1) If any CPU is set at cpu_mask in the same compute unit, do * nothing. * 2) If no CPU is set at cpu_mask in the same compute unit, * set current ONLINE CPU. * * Note: if there is a CPU aside of the new one already in the * sibling mask, then it is also in cpu_mask. */ target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); if (target >= nr_cpumask_bits) cpumask_set_cpu(cpu, &cpu_mask); return 0; }
void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) { struct mm_struct *mm = vma->vm_mm; preempt_disable(); if (current->active_mm == mm) { if (current->mm) __flush_tlb_one(va); else leave_mm(smp_processor_id()); } if (cpumask_any_but(&mm->cpu_vm_mask, smp_processor_id()) < nr_cpu_ids) flush_tlb_others(&mm->cpu_vm_mask, mm, va); preempt_enable(); }
static int power_cpu_exit(unsigned int cpu) { int target; if (!cpumask_test_and_clear_cpu(cpu, &cpu_mask)) return 0; /* * Find a new CPU on the same compute unit, if was set in cpumask * and still some CPUs on compute unit. Then migrate event and * context to new CPU. */ target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); if (target < nr_cpumask_bits) { cpumask_set_cpu(target, &cpu_mask); perf_pmu_migrate_context(&pmu_class, cpu, target); } return 0; }