void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) { union jump_code_union code; if (type == JUMP_LABEL_ENABLE) { code.jump = 0xe9; code.offset = entry->target - (entry->code + JUMP_LABEL_NOP_SIZE); } else memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE); get_online_cpus(); mutex_lock(&text_mutex); text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE); mutex_unlock(&text_mutex); put_online_cpus(); }
int config_L2(int size) { int cur_size = get_l2c_size(); if (size != SZ_256K && size != SZ_512K) { printk("inlvalid input size %x\n", size); return -1; } if (in_interrupt()) { printk(KERN_ERR "Cannot use %s in interrupt/softirq context\n", __func__); return -1; } if (size == cur_size) { printk("Config L2 size %x is equal to current L2 size %x\n", size, cur_size); return 0; } atomic_set(&L1_flush_done, 0); get_online_cpus(); //printk("[Config L2] Config L2 start, on line cpu = %d\n",num_online_cpus()); /* disable cache and flush L1 */ on_each_cpu((smp_call_func_t)atomic_flush, NULL, true); //while(atomic_read(&L1_flush_done) != num_online_cpus()); //printk("[Config L2] L1 flush done\n"); /* flush L2 */ inner_dcache_flush_L2(); //printk("[Config L2] L2 flush done\n"); /* change L2 size */ config_L2_size(size); //printk("[Config L2] Change L2 flush size done(size = %d)\n",size); /* enable cache */ atomic_set(&L1_flush_done, 0); on_each_cpu((smp_call_func_t)__enable_cache, NULL, true); //update cr_alignment for other kernel function usage cr_alignment = cr_alignment | (0x4); //C1_CBIT put_online_cpus(); printk("Config L2 size %x done\n", size); return 0; }
static void nmi_shutdown(void) { struct op_msrs *msrs; get_online_cpus(); unregister_cpu_notifier(&oprofile_cpu_nb); on_each_cpu(nmi_cpu_shutdown, NULL, 1); nmi_enabled = 0; ctr_running = 0; put_online_cpus(); /* make variables visible to the nmi handler: */ smp_mb(); unregister_nmi_handler(NMI_LOCAL, "oprofile"); msrs = &get_cpu_var(cpu_msrs); model->shutdown(msrs); free_msrs(); put_cpu_var(cpu_msrs); }
/* Input boost main boost function */ static void __cpuinit ib_boost_main(struct work_struct *work) { unsigned int cpu, nr_cpus_to_boost, nr_boosted = 0; get_online_cpus(); /* Max. of 3 CPUs can be boosted at any given time */ nr_cpus_to_boost = num_online_cpus() > 2 ? num_online_cpus() - 1 : 1; for_each_online_cpu(cpu) { /* Calculate boost duration for each CPU (CPU0 is boosted the longest) */ /* TODO: Make this more standard and configurable from sysfs */ boost_ms[cpu] = 1500 - (cpu * 200) - (nr_cpus_to_boost * 250); cpu_boost_cpu(cpu); nr_boosted++; if (nr_boosted == nr_cpus_to_boost) break; } put_online_cpus(); }
/* * STP work. Check for the STP state and take over the clock * synchronization if the STP clock source is usable. */ static void stp_work_fn(struct work_struct *work) { struct clock_sync_data stp_sync; int rc; /* prevent multiple execution. */ mutex_lock(&stp_work_mutex); if (!stp_online) { chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000, NULL); del_timer_sync(&stp_timer); goto out_unlock; } rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0xb0e0, NULL); if (rc) goto out_unlock; rc = chsc_sstpi(stp_page, &stp_info, sizeof(struct stp_sstpi)); if (rc || stp_info.c == 0) goto out_unlock; /* Skip synchronization if the clock is already in sync. */ if (check_sync_clock()) goto out_unlock; memset(&stp_sync, 0, sizeof(stp_sync)); get_online_cpus(); atomic_set(&stp_sync.cpus, num_online_cpus() - 1); stop_machine(stp_sync_clock, &stp_sync, cpu_online_mask); put_online_cpus(); if (!check_sync_clock()) /* * There is a usable clock but the synchonization failed. * Retry after a second. */ mod_timer(&stp_timer, jiffies + HZ); out_unlock: mutex_unlock(&stp_work_mutex); }
static int bts_trace_init(struct trace_array *tr) { int cpu; hw_branch_trace = tr; trace_hw_branches_enabled = 0; get_online_cpus(); for_each_online_cpu(cpu) { bts_trace_init_cpu(cpu); if (likely(per_cpu(hwb_tracer, cpu))) trace_hw_branches_enabled = 1; } trace_hw_branches_suspended = 0; put_online_cpus(); /* If we could not enable tracing on a single cpu, we fail. */ return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP; }
static void trace_bts_prepare(struct trace_iterator *iter) { int cpu; get_online_cpus(); for_each_online_cpu(cpu) if (likely(per_cpu(hwb_tracer, cpu))) ds_suspend_bts(per_cpu(hwb_tracer, cpu)); /* * We need to collect the trace on the respective cpu since ftrace * implicitly adds the record for the current cpu. * Once that is more flexible, we could collect the data from any cpu. */ on_each_cpu(trace_bts_cpu, iter->tr, 1); for_each_online_cpu(cpu) if (likely(per_cpu(hwb_tracer, cpu))) ds_suspend_bts(per_cpu(hwb_tracer, cpu)); put_online_cpus(); }
static int bts_trace_init(struct trace_array *tr) { int cpu; hw_branch_trace = tr; trace_hw_branches_enabled = 0; get_online_cpus(); for_each_online_cpu(cpu) { bts_trace_init_cpu(cpu); if (likely(per_cpu(tracer, cpu))) trace_hw_branches_enabled = 1; } trace_hw_branches_suspended = 0; put_online_cpus(); return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP; }
static void fb_bpf_cleanup_filter_cpus(struct fblock *fb) { unsigned int cpu; struct fb_bpf_priv __percpu *fb_priv; if (!fb) return; rcu_read_lock(); fb_priv = (struct fb_bpf_priv __percpu *) rcu_dereference_raw(fb->private_data); rcu_read_unlock(); get_online_cpus(); for_each_online_cpu(cpu) { struct fb_bpf_priv *fb_priv_cpu; fb_priv_cpu = per_cpu_ptr(fb_priv, cpu); fb_bpf_cleanup_filter(fb_priv_cpu); } put_online_cpus(); }
static int update_cpu_max_freq(int cpu, uint32_t max_freq) { int ret = 0; if (max_freq != UINT_MAX) pr_info("%s: Limiting cpu%d max frequency to %d\n", KBUILD_MODNAME, cpu, max_freq); else pr_info("%s: Max frequency reset for cpu%d\n", KBUILD_MODNAME, cpu); get_online_cpus(); for_each_online_cpu(cpu) { if (cpufreq_update_policy(cpu)) pr_info("%s: Unable to update policy for cpu:%d\n", KBUILD_MODNAME, cpu); } put_online_cpus(); return ret; }
/* * We will reset the cpu frequencies limits here. The core online/offline * status will be carried over to the process stopping the msm_thermal, as * we dont want to online a core and bring in the thermal issues. */ static void __ref disable_msm_thermal(void) { int cpu = 0; /* make sure check_temp is no longer running */ cancel_delayed_work(&check_temp_work); flush_scheduled_work(); if (limited_max_freq == UINT_MAX) return; limited_max_freq = UINT_MAX; get_online_cpus(); for_each_online_cpu(cpu) { if (cpufreq_update_policy(cpu)) pr_info("%s: Unable to update policy for cpu:%d\n", KBUILD_MODNAME, cpu); } put_online_cpus(); }
static void __init acpi_cpufreq_boost_init(void) { if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) { msrs = msrs_alloc(); if (!msrs) return; acpi_cpufreq_driver.boost_supported = true; acpi_cpufreq_driver.boost_enabled = boost_state(0); get_online_cpus(); /* Force all MSRs to the same value */ boost_set_msrs(acpi_cpufreq_driver.boost_enabled, cpu_online_mask); register_cpu_notifier(&boost_nb); put_online_cpus(); } }
static int fb_bpf_proc_show_filter(struct seq_file *m, void *v) { unsigned long flags; struct fblock *fb = (struct fblock *) m->private; struct fb_bpf_priv *fb_priv_cpu; struct sk_filter *sf; get_online_cpus(); rcu_read_lock(); fb_priv_cpu = this_cpu_ptr(rcu_dereference_raw(fb->private_data)); rcu_read_unlock(); spin_lock_irqsave(&fb_priv_cpu->flock, flags); sf = fb_priv_cpu->filter; if (sf) { unsigned int i; if (sf->bpf_func == sk_run_filter) seq_puts(m, "bpf jit: 0\n"); else seq_puts(m, "bpf jit: 1\n"); seq_puts(m, "code:\n"); for (i = 0; i < sf->len; ++i) { char sline[32]; memset(sline, 0, sizeof(sline)); snprintf(sline, sizeof(sline), "{ 0x%x, %u, %u, 0x%x }\n", sf->insns[i].code, sf->insns[i].jt, sf->insns[i].jf, sf->insns[i].k); sline[sizeof(sline) - 1] = 0; seq_puts(m, sline); } } spin_unlock_irqrestore(&fb_priv_cpu->flock, flags); put_online_cpus(); return 0; }
static void raise_mce(struct mce *m) { int context = MCJ_CTX(m->inject_flags); inject_mce(m); if (context == MCJ_CTX_RANDOM) return; #ifdef CONFIG_X86_LOCAL_APIC if (m->inject_flags & MCJ_NMI_BROADCAST) { unsigned long start; int cpu; get_online_cpus(); mce_inject_cpumask = cpu_online_map; cpu_clear(get_cpu(), mce_inject_cpumask); for_each_online_cpu(cpu) { struct mce *mcpu = &per_cpu(injectm, cpu); if (!mcpu->finished || MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM) cpu_clear(cpu, mce_inject_cpumask); } if (!cpus_empty(mce_inject_cpumask)) apic->send_IPI_mask(&mce_inject_cpumask, NMI_VECTOR); start = jiffies; while (!cpus_empty(mce_inject_cpumask)) { if (!time_before(jiffies, start + 2*HZ)) { printk(KERN_ERR "Timeout waiting for mce inject NMI %lx\n", *cpus_addr(mce_inject_cpumask)); break; } cpu_relax(); } raise_local(); put_cpu(); put_online_cpus(); } else
static void acpi_processor_remove(struct acpi_device *device) { struct acpi_processor *pr; if (!device || !acpi_driver_data(device)) return; pr = acpi_driver_data(device); if (pr->id >= nr_cpu_ids) goto out; /* * The only reason why we ever get here is CPU hot-removal. The CPU is * already offline and the ACPI device removal locking prevents it from * being put back online at this point. * * Unbind the driver from the processor device and detach it from the * ACPI companion object. */ device_release_driver(pr->dev); acpi_unbind_one(pr->dev); /* Clean up. */ per_cpu(processor_device_array, pr->id) = NULL; per_cpu(processors, pr->id) = NULL; try_offline_node(cpu_to_node(pr->id)); /* Remove the CPU. */ get_online_cpus(); arch_unregister_cpu(pr->id); acpi_unmap_lsapic(pr->id); put_online_cpus(); out: free_cpumask_var(pr->throttling.shared_cpu_map); kfree(pr); }
/* * Take a map of online CPUs and the number of available interrupt vectors * and generate an output cpumask suitable for spreading MSI/MSI-X vectors * so that they are distributed as good as possible around the CPUs. If * more vectors than CPUs are available we'll map one to each CPU, * otherwise we map one to the first sibling of each socket. * * If there are more vectors than CPUs we will still only have one bit * set per CPU, but interrupt code will keep on assigning the vectors from * the start of the bitmap until we run out of vectors. */ struct cpumask *irq_create_affinity_mask(unsigned int *nr_vecs) { struct cpumask *affinity_mask; unsigned int max_vecs = *nr_vecs; if (max_vecs == 1) return NULL; affinity_mask = kzalloc(cpumask_size(), GFP_KERNEL); if (!affinity_mask) { *nr_vecs = 1; return NULL; } get_online_cpus(); if (max_vecs >= num_online_cpus()) { cpumask_copy(affinity_mask, cpu_online_mask); *nr_vecs = num_online_cpus(); } else { unsigned int vecs = 0, cpu; for_each_online_cpu(cpu) { if (cpu == get_first_sibling(cpu)) { cpumask_set_cpu(cpu, affinity_mask); vecs++; } if (--max_vecs == 0) break; } *nr_vecs = vecs; } put_online_cpus(); return affinity_mask; }
static void etm_disable_sysfs(struct coresight_device *csdev) { struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent); /* * Taking hotplug lock here protects from clocks getting disabled * with tracing being left on (crash scenario) if user disable occurs * after cpu online mask indicates the cpu is offline but before the * DYING hotplug callback is serviced by the ETM driver. */ get_online_cpus(); spin_lock(&drvdata->spinlock); /* * Executing etm_disable_hw on the cpu whose ETM is being disabled * ensures that register writes occur when cpu is powered. */ smp_call_function_single(drvdata->cpu, etm_disable_hw, drvdata, 1); spin_unlock(&drvdata->spinlock); put_online_cpus(); dev_info(drvdata->dev, "ETM tracing disabled\n"); }
/*H:020 * Now the Switcher is mapped and every thing else is ready, we need to do * some more i386-specific initialization. */ void __init lguest_arch_host_init(void) { int i; /* * Most of the x86/switcher_32.S doesn't care that it's been moved; on * Intel, jumps are relative, and it doesn't access any references to * external code or data. * * The only exception is the interrupt handlers in switcher.S: their * addresses are placed in a table (default_idt_entries), so we need to * update the table with the new addresses. switcher_offset() is a * convenience function which returns the distance between the * compiled-in switcher code and the high-mapped copy we just made. */ for (i = 0; i < IDT_ENTRIES; i++) default_idt_entries[i] += switcher_offset(); /* * Set up the Switcher's per-cpu areas. * * Each CPU gets two pages of its own within the high-mapped region * (aka. "struct lguest_pages"). Much of this can be initialized now, * but some depends on what Guest we are running (which is set up in * copy_in_guest_info()). */ for_each_possible_cpu(i) { /* lguest_pages() returns this CPU's two pages. */ struct lguest_pages *pages = lguest_pages(i); /* This is a convenience pointer to make the code neater. */ struct lguest_ro_state *state = &pages->state; /* * The Global Descriptor Table: the Host has a different one * for each CPU. We keep a descriptor for the GDT which says * where it is and how big it is (the size is actually the last * byte, not the size, hence the "-1"). */ state->host_gdt_desc.size = GDT_SIZE-1; state->host_gdt_desc.address = (long)get_cpu_gdt_table(i); /* * All CPUs on the Host use the same Interrupt Descriptor * Table, so we just use store_idt(), which gets this CPU's IDT * descriptor. */ store_idt(&state->host_idt_desc); /* * The descriptors for the Guest's GDT and IDT can be filled * out now, too. We copy the GDT & IDT into ->guest_gdt and * ->guest_idt before actually running the Guest. */ state->guest_idt_desc.size = sizeof(state->guest_idt)-1; state->guest_idt_desc.address = (long)&state->guest_idt; state->guest_gdt_desc.size = sizeof(state->guest_gdt)-1; state->guest_gdt_desc.address = (long)&state->guest_gdt; /* * We know where we want the stack to be when the Guest enters * the Switcher: in pages->regs. The stack grows upwards, so * we start it at the end of that structure. */ state->guest_tss.sp0 = (long)(&pages->regs + 1); /* * And this is the GDT entry to use for the stack: we keep a * couple of special LGUEST entries. */ state->guest_tss.ss0 = LGUEST_DS; /* * x86 can have a finegrained bitmap which indicates what I/O * ports the process can use. We set it to the end of our * structure, meaning "none". */ state->guest_tss.io_bitmap_base = sizeof(state->guest_tss); /* * Some GDT entries are the same across all Guests, so we can * set them up now. */ setup_default_gdt_entries(state); /* Most IDT entries are the same for all Guests, too.*/ setup_default_idt_entries(state, default_idt_entries); /* * The Host needs to be able to use the LGUEST segments on this * CPU, too, so put them in the Host GDT. */ get_cpu_gdt_table(i)[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT; get_cpu_gdt_table(i)[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT; } /* * In the Switcher, we want the %cs segment register to use the * LGUEST_CS GDT entry: we've put that in the Host and Guest GDTs, so * it will be undisturbed when we switch. To change %cs and jump we * need this structure to feed to Intel's "lcall" instruction. */ lguest_entry.offset = (long)switch_to_guest + switcher_offset(); lguest_entry.segment = LGUEST_CS; /* * Finally, we need to turn off "Page Global Enable". PGE is an * optimization where page table entries are specially marked to show * they never change. The Host kernel marks all the kernel pages this * way because it's always present, even when userspace is running. * * Lguest breaks this: unbeknownst to the rest of the Host kernel, we * switch to the Guest kernel. If you don't disable this on all CPUs, * you'll get really weird bugs that you'll chase for two days. * * I used to turn PGE off every time we switched to the Guest and back * on when we return, but that slowed the Switcher down noticibly. */ /* * We don't need the complexity of CPUs coming and going while we're * doing this. */ get_online_cpus(); if (boot_cpu_has(X86_FEATURE_PGE)) { /* We have a broader idea of "global". */ /* Remember that this was originally set (for cleanup). */ cpu_had_pge = 1; /* * adjust_pge is a helper function which sets or unsets the PGE * bit on its CPU, depending on the argument (0 == unset). */ on_each_cpu(adjust_pge, (void *)0, 1); /* Turn off the feature in the global feature set. */ clear_cpu_cap(&boot_cpu_data, X86_FEATURE_PGE); } put_online_cpus(); }
/* * Allow secondary CPU threads to come online again */ void uninhibit_secondary_onlining(void) { get_online_cpus(); atomic_dec(&secondary_inhibit_count); put_online_cpus(); }
/* * mipsmt_sys_sched_setaffinity - set the cpu affinity of a process */ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr) { cpumask_var_t cpus_allowed, new_mask, effective_mask; struct thread_info *ti; struct task_struct *p; int retval; if (len < sizeof(new_mask)) return -EINVAL; if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) return -EFAULT; get_online_cpus(); rcu_read_lock(); p = find_process_by_pid(pid); if (!p) { rcu_read_unlock(); put_online_cpus(); return -ESRCH; } /* Prevent p going away */ get_task_struct(p); rcu_read_unlock(); if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { retval = -ENOMEM; goto out_put_task; } if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out_free_cpus_allowed; } if (!alloc_cpumask_var(&effective_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out_free_new_mask; } retval = -EPERM; if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) goto out_unlock; retval = security_task_setscheduler(p); if (retval) goto out_unlock; /* Record new user-specified CPU set for future reference */ cpumask_copy(&p->thread.user_cpus_allowed, new_mask); again: /* Compute new global allowed CPU set if necessary */ ti = task_thread_info(p); if (test_ti_thread_flag(ti, TIF_FPUBOUND) && cpus_intersects(*new_mask, mt_fpu_cpumask)) { cpus_and(*effective_mask, *new_mask, mt_fpu_cpumask); retval = set_cpus_allowed_ptr(p, effective_mask); } else { cpumask_copy(effective_mask, new_mask); clear_ti_thread_flag(ti, TIF_FPUBOUND); retval = set_cpus_allowed_ptr(p, new_mask); } if (!retval) { cpuset_cpus_allowed(p, cpus_allowed); if (!cpumask_subset(effective_mask, cpus_allowed)) { /* * We must have raced with a concurrent cpuset * update. Just reset the cpus_allowed to the * cpuset's cpus_allowed */ cpumask_copy(new_mask, cpus_allowed); goto again; } } out_unlock: free_cpumask_var(effective_mask); out_free_new_mask: free_cpumask_var(new_mask); out_free_cpus_allowed: free_cpumask_var(cpus_allowed); out_put_task: put_task_struct(p); put_online_cpus(); return retval; }
static int clamp_thread(void *arg) { int cpunr = (unsigned long)arg; DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0); static const struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO/2, }; unsigned int count = 0; unsigned int target_ratio; set_bit(cpunr, cpu_clamping_mask); set_freezable(); init_timer_on_stack(&wakeup_timer); sched_setscheduler(current, SCHED_FIFO, ¶m); while (true == clamping && !kthread_should_stop() && cpu_online(cpunr)) { int sleeptime; unsigned long target_jiffies; unsigned int guard; unsigned int compensated_ratio; int interval; /* jiffies to sleep for each attempt */ unsigned int duration_jiffies = msecs_to_jiffies(duration); unsigned int window_size_now; try_to_freeze(); /* * make sure user selected ratio does not take effect until * the next round. adjust target_ratio if user has changed * target such that we can converge quickly. */ target_ratio = set_target_ratio; guard = 1 + target_ratio/20; window_size_now = window_size; count++; /* * systems may have different ability to enter package level * c-states, thus we need to compensate the injected idle ratio * to achieve the actual target reported by the HW. */ compensated_ratio = target_ratio + get_compensation(target_ratio); if (compensated_ratio <= 0) compensated_ratio = 1; interval = duration_jiffies * 100 / compensated_ratio; /* align idle time */ target_jiffies = roundup(jiffies, interval); sleeptime = target_jiffies - jiffies; if (sleeptime <= 0) sleeptime = 1; schedule_timeout_interruptible(sleeptime); /* * only elected controlling cpu can collect stats and update * control parameters. */ if (cpunr == control_cpu && !(count%window_size_now)) { should_skip = powerclamp_adjust_controls(target_ratio, guard, window_size_now); smp_mb(); } if (should_skip) continue; target_jiffies = jiffies + duration_jiffies; mod_timer(&wakeup_timer, target_jiffies); if (unlikely(local_softirq_pending())) continue; /* * stop tick sched during idle time, interrupts are still * allowed. thus jiffies are updated properly. */ preempt_disable(); /* mwait until target jiffies is reached */ while (time_before(jiffies, target_jiffies)) { unsigned long ecx = 1; unsigned long eax = target_mwait; /* * REVISIT: may call enter_idle() to notify drivers who * can save power during cpu idle. same for exit_idle() */ local_touch_nmi(); stop_critical_timings(); mwait_idle_with_hints(eax, ecx); start_critical_timings(); atomic_inc(&idle_wakeup_counter); } preempt_enable(); } del_timer_sync(&wakeup_timer); clear_bit(cpunr, cpu_clamping_mask); return 0; } /* * 1 HZ polling while clamping is active, useful for userspace * to monitor actual idle ratio. */ static void poll_pkg_cstate(struct work_struct *dummy); static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate); static void poll_pkg_cstate(struct work_struct *dummy) { static u64 msr_last; static u64 tsc_last; static unsigned long jiffies_last; u64 msr_now; unsigned long jiffies_now; u64 tsc_now; u64 val64; msr_now = pkg_state_counter(); tsc_now = rdtsc(); jiffies_now = jiffies; /* calculate pkg cstate vs tsc ratio */ if (!msr_last || !tsc_last) pkg_cstate_ratio_cur = 1; else { if (tsc_now - tsc_last) { val64 = 100 * (msr_now - msr_last); do_div(val64, (tsc_now - tsc_last)); pkg_cstate_ratio_cur = val64; } } /* update record */ msr_last = msr_now; jiffies_last = jiffies_now; tsc_last = tsc_now; if (true == clamping) schedule_delayed_work(&poll_pkg_cstate_work, HZ); } static int start_power_clamp(void) { unsigned long cpu; struct task_struct *thread; set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1); /* prevent cpu hotplug */ get_online_cpus(); /* prefer BSP */ control_cpu = 0; if (!cpu_online(control_cpu)) control_cpu = smp_processor_id(); clamping = true; schedule_delayed_work(&poll_pkg_cstate_work, 0); /* start one thread per online cpu */ for_each_online_cpu(cpu) { struct task_struct **p = per_cpu_ptr(powerclamp_thread, cpu); thread = kthread_create_on_node(clamp_thread, (void *) cpu, cpu_to_node(cpu), "kidle_inject/%ld", cpu); /* bind to cpu here */ if (likely(!IS_ERR(thread))) { kthread_bind(thread, cpu); wake_up_process(thread); *p = thread; } } put_online_cpus(); return 0; }
int ds_selftest_bts(void) { struct ds_selftest_bts_conf conf; unsigned char buffer[BUFFER_SIZE], *small_buffer; unsigned long irq; int cpu; printk(KERN_INFO "[ds] bts selftest..."); conf.error = 0; small_buffer = (unsigned char *)ALIGN((unsigned long)buffer, 8) + 8; get_online_cpus(); for_each_online_cpu(cpu) { conf.suspend = ds_suspend_bts_wrap; conf.resume = ds_resume_bts_wrap; conf.tracer = ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE, NULL, (size_t)-1, BTS_KERNEL); ds_selftest_bts_cpu(&conf); if (conf.error >= 0) conf.error = ds_selftest_bts_bad_request_task(buffer); ds_release_bts(conf.tracer); if (conf.error < 0) goto out; conf.suspend = ds_suspend_bts_noirq; conf.resume = ds_resume_bts_noirq; conf.tracer = ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE, NULL, (size_t)-1, BTS_KERNEL); smp_call_function_single(cpu, ds_selftest_bts_cpu, &conf, 1); if (conf.error >= 0) { conf.error = ds_selftest_bts_bad_release_noirq(cpu, conf.tracer); /* We must not release the tracer twice. */ if (conf.error < 0) conf.tracer = NULL; } if (conf.error >= 0) conf.error = ds_selftest_bts_bad_request_task(buffer); smp_call_function_single(cpu, ds_release_bts_noirq_wrap, conf.tracer, 1); if (conf.error < 0) goto out; } conf.suspend = ds_suspend_bts_wrap; conf.resume = ds_resume_bts_wrap; conf.tracer = ds_request_bts_task(current, buffer, BUFFER_SIZE, NULL, (size_t)-1, BTS_KERNEL); ds_selftest_bts_cpu(&conf); if (conf.error >= 0) conf.error = ds_selftest_bts_bad_request_cpu(0, buffer); ds_release_bts(conf.tracer); if (conf.error < 0) goto out; conf.suspend = ds_suspend_bts_noirq; conf.resume = ds_resume_bts_noirq; conf.tracer = ds_request_bts_task(current, small_buffer, SMALL_BUFFER_SIZE, NULL, (size_t)-1, BTS_KERNEL); local_irq_save(irq); ds_selftest_bts_cpu(&conf); if (conf.error >= 0) conf.error = ds_selftest_bts_bad_request_cpu(0, buffer); ds_release_bts_noirq(conf.tracer); local_irq_restore(irq); if (conf.error < 0) goto out; conf.error = 0; out: put_online_cpus(); printk(KERN_CONT "%s.\n", (conf.error ? "failed" : "passed")); return conf.error; }
static int etm_probe(struct amba_device *adev, const struct amba_id *id) { int ret; void __iomem *base; struct device *dev = &adev->dev; struct coresight_platform_data *pdata = NULL; struct etm_drvdata *drvdata; struct resource *res = &adev->res; struct coresight_desc *desc; struct device_node *np = adev->dev.of_node; desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL); if (!desc) return -ENOMEM; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; if (np) { pdata = of_get_coresight_platform_data(dev, np); if (IS_ERR(pdata)) return PTR_ERR(pdata); adev->dev.platform_data = pdata; drvdata->use_cp14 = of_property_read_bool(np, "arm,cp14"); } drvdata->dev = &adev->dev; dev_set_drvdata(dev, drvdata); /* Validity for the resource is already checked by the AMBA core */ base = devm_ioremap_resource(dev, res); if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; spin_lock_init(&drvdata->spinlock); drvdata->atclk = devm_clk_get(&adev->dev, "atclk"); /* optional */ if (!IS_ERR(drvdata->atclk)) { ret = clk_prepare_enable(drvdata->atclk); if (ret) return ret; } drvdata->cpu = pdata ? pdata->cpu : 0; get_online_cpus(); etmdrvdata[drvdata->cpu] = drvdata; if (!smp_call_function_single(drvdata->cpu, etm_os_unlock, drvdata, 1)) drvdata->os_unlock = true; if (smp_call_function_single(drvdata->cpu, etm_init_arch_data, drvdata, 1)) dev_err(dev, "ETM arch init failed\n"); if (!etm_count++) register_hotcpu_notifier(&etm_cpu_notifier); put_online_cpus(); if (etm_arch_supported(drvdata->arch) == false) { ret = -EINVAL; goto err_arch_supported; } etm_init_default_data(drvdata); desc->type = CORESIGHT_DEV_TYPE_SOURCE; desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; desc->ops = &etm_cs_ops; desc->pdata = pdata; desc->dev = dev; desc->groups = coresight_etm_groups; drvdata->csdev = coresight_register(desc); if (IS_ERR(drvdata->csdev)) { ret = PTR_ERR(drvdata->csdev); goto err_arch_supported; } pm_runtime_put(&adev->dev); dev_info(dev, "%s initialized\n", (char *)id->data); if (boot_enable) { coresight_enable(drvdata->csdev); drvdata->boot_enable = true; } return 0; err_arch_supported: if (--etm_count == 0) unregister_hotcpu_notifier(&etm_cpu_notifier); return ret; }
/* * mipsmt_sys_sched_setaffinity - set the cpu affinity of a process */ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr) { cpumask_t new_mask; cpumask_t effective_mask; int retval; struct task_struct *p; struct thread_info *ti; uid_t euid; if (len < sizeof(new_mask)) return -EINVAL; if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) return -EFAULT; get_online_cpus(); read_lock(&tasklist_lock); p = find_process_by_pid(pid); if (!p) { read_unlock(&tasklist_lock); put_online_cpus(); return -ESRCH; } /* * It is not safe to call set_cpus_allowed with the * tasklist_lock held. We will bump the task_struct's * usage count and drop tasklist_lock before invoking * set_cpus_allowed. */ get_task_struct(p); euid = current_euid(); retval = -EPERM; if (euid != p->cred->euid && euid != p->cred->uid && !capable(CAP_SYS_NICE)) { read_unlock(&tasklist_lock); goto out_unlock; } retval = security_task_setscheduler(p, 0, NULL); if (retval) goto out_unlock; /* Record new user-specified CPU set for future reference */ p->thread.user_cpus_allowed = new_mask; /* Unlock the task list */ read_unlock(&tasklist_lock); /* Compute new global allowed CPU set if necessary */ ti = task_thread_info(p); if (test_ti_thread_flag(ti, TIF_FPUBOUND) && cpus_intersects(new_mask, mt_fpu_cpumask)) { cpus_and(effective_mask, new_mask, mt_fpu_cpumask); retval = set_cpus_allowed_ptr(p, &effective_mask); } else { clear_ti_thread_flag(ti, TIF_FPUBOUND); retval = set_cpus_allowed_ptr(p, &new_mask); } out_unlock: put_task_struct(p); put_online_cpus(); return retval; }
/** * mtrr_add_page - Add a memory type region * @base: Physical base address of region in pages (in units of 4 kB!) * @size: Physical size of region in pages (4 kB) * @type: Type of MTRR desired * @increment: If this is true do usage counting on the region * * Memory type region registers control the caching on newer Intel and * non Intel processors. This function allows drivers to request an * MTRR is added. The details and hardware specifics of each processor's * implementation are hidden from the caller, but nevertheless the * caller should expect to need to provide a power of two size on an * equivalent power of two boundary. * * If the region cannot be added either because all regions are in use * or the CPU cannot support it a negative value is returned. On success * the register number for this entry is returned, but should be treated * as a cookie only. * * On a multiprocessor machine the changes are made to all processors. * This is required on x86 by the Intel processors. * * The available types are * * %MTRR_TYPE_UNCACHABLE - No caching * * %MTRR_TYPE_WRBACK - Write data back in bursts whenever * * %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts * * %MTRR_TYPE_WRTHROUGH - Cache reads but not writes * * BUGS: Needs a quiet flag for the cases where drivers do not mind * failures and do not wish system log messages to be sent. */ int mtrr_add_page(unsigned long base, unsigned long size, unsigned int type, bool increment) { unsigned long lbase, lsize; int i, replace, error; mtrr_type ltype; if (!mtrr_if) return -ENXIO; error = mtrr_if->validate_add_page(base, size, type); if (error) return error; if (type >= MTRR_NUM_TYPES) { pr_warning("mtrr: type: %u invalid\n", type); return -EINVAL; } /* If the type is WC, check that this processor supports it */ if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) { pr_warning("mtrr: your processor doesn't support write-combining\n"); return -ENOSYS; } if (!size) { pr_warning("mtrr: zero sized request\n"); return -EINVAL; } if (base & size_or_mask || size & size_or_mask) { pr_warning("mtrr: base or size exceeds the MTRR width\n"); return -EINVAL; } error = -EINVAL; replace = -1; /* No CPU hotplug when we change MTRR entries */ get_online_cpus(); /* Search for existing MTRR */ mutex_lock(&mtrr_mutex); for (i = 0; i < num_var_ranges; ++i) { mtrr_if->get(i, &lbase, &lsize, <ype); if (!lsize || base > lbase + lsize - 1 || base + size - 1 < lbase) continue; /* * At this point we know there is some kind of * overlap/enclosure */ if (base < lbase || base + size - 1 > lbase + lsize - 1) { if (base <= lbase && base + size - 1 >= lbase + lsize - 1) { /* New region encloses an existing region */ if (type == ltype) { replace = replace == -1 ? i : -2; continue; } else if (types_compatible(type, ltype)) continue; } pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing" " 0x%lx000,0x%lx000\n", base, size, lbase, lsize); goto out; } /* New region is enclosed by an existing region */ if (ltype != type) { if (types_compatible(type, ltype)) continue; pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n", base, size, mtrr_attrib_to_str(ltype), mtrr_attrib_to_str(type)); goto out; } if (increment) ++mtrr_usage_table[i]; error = i; goto out; } /* Search for an empty MTRR */ i = mtrr_if->get_free_region(base, size, replace); if (i >= 0) { set_mtrr(i, base, size, type); if (likely(replace < 0)) { mtrr_usage_table[i] = 1; } else { mtrr_usage_table[i] = mtrr_usage_table[replace]; if (increment) mtrr_usage_table[i]++; if (unlikely(replace != i)) { set_mtrr(replace, 0, 0, 0); mtrr_usage_table[replace] = 0; } } } else { pr_info("mtrr: no more MTRRs available\n"); } error = i; out: mutex_unlock(&mtrr_mutex); put_online_cpus(); return error; }
static int etm4_probe(struct amba_device *adev, const struct amba_id *id) { int ret; void __iomem *base; struct device *dev = &adev->dev; struct coresight_platform_data *pdata = NULL; struct etmv4_drvdata *drvdata; struct resource *res = &adev->res; struct coresight_desc desc = { 0 }; struct device_node *np = adev->dev.of_node; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; if (np) { pdata = of_get_coresight_platform_data(dev, np); if (IS_ERR(pdata)) return PTR_ERR(pdata); adev->dev.platform_data = pdata; } drvdata->dev = &adev->dev; dev_set_drvdata(dev, drvdata); /* Validity for the resource is already checked by the AMBA core */ base = devm_ioremap_resource(dev, res); if (IS_ERR(base)) return PTR_ERR(base); drvdata->base = base; spin_lock_init(&drvdata->spinlock); drvdata->cpu = pdata ? pdata->cpu : 0; get_online_cpus(); etmdrvdata[drvdata->cpu] = drvdata; if (smp_call_function_single(drvdata->cpu, etm4_init_arch_data, drvdata, 1)) dev_err(dev, "ETM arch init failed\n"); if (!etm4_count++) { cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING, "arm/coresight4:starting", etm4_starting_cpu, etm4_dying_cpu); ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/coresight4:online", etm4_online_cpu, NULL); if (ret < 0) goto err_arch_supported; hp_online = ret; } put_online_cpus(); if (etm4_arch_supported(drvdata->arch) == false) { ret = -EINVAL; goto err_arch_supported; } etm4_init_trace_id(drvdata); etm4_set_default(&drvdata->config); desc.type = CORESIGHT_DEV_TYPE_SOURCE; desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC; desc.ops = &etm4_cs_ops; desc.pdata = pdata; desc.dev = dev; desc.groups = coresight_etmv4_groups; drvdata->csdev = coresight_register(&desc); if (IS_ERR(drvdata->csdev)) { ret = PTR_ERR(drvdata->csdev); goto err_arch_supported; } ret = etm_perf_symlink(drvdata->csdev, true); if (ret) { coresight_unregister(drvdata->csdev); goto err_arch_supported; } pm_runtime_put(&adev->dev); dev_info(dev, "%s initialized\n", (char *)id->data); if (boot_enable) { coresight_enable(drvdata->csdev); drvdata->boot_enable = true; } return 0; err_arch_supported: if (--etm4_count == 0) { cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING); if (hp_online) cpuhp_remove_state_nocalls(hp_online); } return ret; }
/* * Accumulate the vm event counters across all CPUs. * The result is unavoidably approximate - it can change * during and after execution of this function. */ void all_vm_events(unsigned long *ret) { get_online_cpus(); sum_vm_events(ret); put_online_cpus(); }
void kvm_hv_vm_deactivated(void) { get_online_cpus(); atomic_dec(&hv_vm_count); put_online_cpus(); }
static int boost_mig_sync_thread(void *data) { int dest_cpu = (int) data; int src_cpu, ret; struct cpu_sync *s = &per_cpu(sync_info, dest_cpu); struct cpufreq_policy dest_policy; struct cpufreq_policy src_policy; unsigned long flags; while(1) { ret = wait_event_interruptible(s->sync_wq, s->pending || kthread_should_stop()); if (kthread_should_stop()) break; if (ret == -ERESTARTSYS) continue; spin_lock_irqsave(&s->lock, flags); s->pending = false; src_cpu = s->src_cpu; spin_unlock_irqrestore(&s->lock, flags); ret = cpufreq_get_policy(&src_policy, src_cpu); if (ret) continue; ret = cpufreq_get_policy(&dest_policy, dest_cpu); if (ret) continue; if (dest_policy.cur >= src_policy.cur ) { pr_debug("No sync. CPU%d@%dKHz >= CPU%d@%dKHz\n", dest_cpu, dest_policy.cur, src_cpu, src_policy.cur); continue; } if (sync_threshold && (dest_policy.cur >= sync_threshold)) continue; cancel_delayed_work_sync(&s->boost_rem); if (sync_threshold) { if (src_policy.cur >= sync_threshold) s->boost_min = sync_threshold; else s->boost_min = src_policy.cur; } else { s->boost_min = src_policy.cur; } /* Force policy re-evaluation to trigger adjust notifier. */ get_online_cpus(); if (cpu_online(dest_cpu)) { cpufreq_update_policy(dest_cpu); queue_delayed_work_on(dest_cpu, cpu_boost_wq, &s->boost_rem, msecs_to_jiffies(boost_ms)); } else { s->boost_min = 0; } put_online_cpus(); } return 0; }
static int boost_mig_sync_thread(void *data) { int dest_cpu = (int) data; int src_cpu, ret; struct cpu_sync *s = &per_cpu(sync_info, dest_cpu); struct cpufreq_policy dest_policy; struct cpufreq_policy src_policy; unsigned long flags; unsigned int req_freq; if (!cpuboost_enable) return 0; while (1) { wait_event(s->sync_wq, s->pending || kthread_should_stop()); #ifdef CONFIG_IRLED_GPIO if (unlikely(gir_boost_disable)) { pr_debug("[GPIO_IR][%s] continue~!(cpu:%d)\n", __func__, raw_smp_processor_id()); continue; } #endif if (kthread_should_stop()) break; spin_lock_irqsave(&s->lock, flags); s->pending = false; src_cpu = s->src_cpu; spin_unlock_irqrestore(&s->lock, flags); ret = cpufreq_get_policy(&src_policy, src_cpu); if (ret) continue; ret = cpufreq_get_policy(&dest_policy, dest_cpu); if (ret) continue; if (s->task_load < migration_load_threshold) continue; if (s->task_load < migration_load_threshold) continue; req_freq = load_based_syncs ? (dest_policy.max * s->task_load) / 100 : src_policy.cur; if (req_freq <= dest_policy.cpuinfo.min_freq) { pr_debug("No sync. Sync Freq:%u\n", req_freq); continue; } if (sync_threshold) req_freq = min(sync_threshold, req_freq); cancel_delayed_work_sync(&s->boost_rem); #ifdef CONFIG_CPUFREQ_HARDLIMIT s->boost_min = check_cpufreq_hardlimit(req_freq); #else s->boost_min = req_freq; #endif /* Force policy re-evaluation to trigger adjust notifier. */ get_online_cpus(); if (cpu_online(src_cpu)) /* * Send an unchanged policy update to the source * CPU. Even though the policy isn't changed from * its existing boosted or non-boosted state * notifying the source CPU will let the governor * know a boost happened on another CPU and that it * should re-evaluate the frequency at the next timer * event without interference from a min sample time. */ cpufreq_update_policy(src_cpu); if (cpu_online(dest_cpu)) { cpufreq_update_policy(dest_cpu); queue_delayed_work_on(dest_cpu, cpu_boost_wq, &s->boost_rem, msecs_to_jiffies(boost_ms)); } else { s->boost_min = 0; } put_online_cpus(); } return 0; }