void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
union jump_code_union code;
if (type == JUMP_LABEL_ENABLE) {
code.jump = 0xe9;
code.offset = entry->target -
(entry->code + JUMP_LABEL_NOP_SIZE);
} else
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
get_online_cpus();
mutex_lock(&text_mutex);
text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
mutex_unlock(&text_mutex);
put_online_cpus();
}
int config_L2(int size)
{
int cur_size = get_l2c_size();
if (size != SZ_256K && size != SZ_512K) {
printk("inlvalid input size %x\n", size);
return -1;
}
if (in_interrupt()) {
printk(KERN_ERR "Cannot use %s in interrupt/softirq context\n",
__func__);
return -1;
}
if (size == cur_size) {
printk("Config L2 size %x is equal to current L2 size %x\n",
size, cur_size);
return 0;
}
atomic_set(&L1_flush_done, 0);
get_online_cpus();
//printk("[Config L2] Config L2 start, on line cpu = %d\n",num_online_cpus());
/* disable cache and flush L1 */
on_each_cpu((smp_call_func_t)atomic_flush, NULL, true);
//while(atomic_read(&L1_flush_done) != num_online_cpus());
//printk("[Config L2] L1 flush done\n");
/* flush L2 */
inner_dcache_flush_L2();
//printk("[Config L2] L2 flush done\n");
/* change L2 size */
config_L2_size(size);
//printk("[Config L2] Change L2 flush size done(size = %d)\n",size);
/* enable cache */
atomic_set(&L1_flush_done, 0);
on_each_cpu((smp_call_func_t)__enable_cache, NULL, true);
//update cr_alignment for other kernel function usage
cr_alignment = cr_alignment | (0x4); //C1_CBIT
put_online_cpus();
printk("Config L2 size %x done\n", size);
return 0;
}
static void nmi_shutdown(void)
{
struct op_msrs *msrs;
get_online_cpus();
unregister_cpu_notifier(&oprofile_cpu_nb);
on_each_cpu(nmi_cpu_shutdown, NULL, 1);
nmi_enabled = 0;
ctr_running = 0;
put_online_cpus();
/* make variables visible to the nmi handler: */
smp_mb();
unregister_nmi_handler(NMI_LOCAL, "oprofile");
msrs = &get_cpu_var(cpu_msrs);
model->shutdown(msrs);
free_msrs();
put_cpu_var(cpu_msrs);
}
/* Input boost main boost function */
static void __cpuinit ib_boost_main(struct work_struct *work)
{
unsigned int cpu, nr_cpus_to_boost, nr_boosted = 0;
get_online_cpus();
/* Max. of 3 CPUs can be boosted at any given time */
nr_cpus_to_boost = num_online_cpus() > 2 ? num_online_cpus() - 1 : 1;
for_each_online_cpu(cpu) {
/* Calculate boost duration for each CPU (CPU0 is boosted the longest) */
/* TODO: Make this more standard and configurable from sysfs */
boost_ms[cpu] = 1500 - (cpu * 200) - (nr_cpus_to_boost * 250);
cpu_boost_cpu(cpu);
nr_boosted++;
if (nr_boosted == nr_cpus_to_boost)
break;
}
put_online_cpus();
}
/*
* STP work. Check for the STP state and take over the clock
* synchronization if the STP clock source is usable.
*/
static void stp_work_fn(struct work_struct *work)
{
struct clock_sync_data stp_sync;
int rc;
/* prevent multiple execution. */
mutex_lock(&stp_work_mutex);
if (!stp_online) {
chsc_sstpc(stp_page, STP_OP_CTRL, 0x0000, NULL);
del_timer_sync(&stp_timer);
goto out_unlock;
}
rc = chsc_sstpc(stp_page, STP_OP_CTRL, 0xb0e0, NULL);
if (rc)
goto out_unlock;
rc = chsc_sstpi(stp_page, &stp_info, sizeof(struct stp_sstpi));
if (rc || stp_info.c == 0)
goto out_unlock;
/* Skip synchronization if the clock is already in sync. */
if (check_sync_clock())
goto out_unlock;
memset(&stp_sync, 0, sizeof(stp_sync));
get_online_cpus();
atomic_set(&stp_sync.cpus, num_online_cpus() - 1);
stop_machine(stp_sync_clock, &stp_sync, cpu_online_mask);
put_online_cpus();
if (!check_sync_clock())
/*
* There is a usable clock but the synchonization failed.
* Retry after a second.
*/
mod_timer(&stp_timer, jiffies + HZ);
out_unlock:
mutex_unlock(&stp_work_mutex);
}
static int bts_trace_init(struct trace_array *tr)
{
int cpu;
hw_branch_trace = tr;
trace_hw_branches_enabled = 0;
get_online_cpus();
for_each_online_cpu(cpu) {
bts_trace_init_cpu(cpu);
if (likely(per_cpu(hwb_tracer, cpu)))
trace_hw_branches_enabled = 1;
}
trace_hw_branches_suspended = 0;
put_online_cpus();
/* If we could not enable tracing on a single cpu, we fail. */
return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
}
static void trace_bts_prepare(struct trace_iterator *iter)
{
int cpu;
get_online_cpus();
for_each_online_cpu(cpu)
if (likely(per_cpu(hwb_tracer, cpu)))
ds_suspend_bts(per_cpu(hwb_tracer, cpu));
/*
* We need to collect the trace on the respective cpu since ftrace
* implicitly adds the record for the current cpu.
* Once that is more flexible, we could collect the data from any cpu.
*/
on_each_cpu(trace_bts_cpu, iter->tr, 1);
for_each_online_cpu(cpu)
if (likely(per_cpu(hwb_tracer, cpu)))
ds_suspend_bts(per_cpu(hwb_tracer, cpu));
put_online_cpus();
}
static int bts_trace_init(struct trace_array *tr)
{
int cpu;
hw_branch_trace = tr;
trace_hw_branches_enabled = 0;
get_online_cpus();
for_each_online_cpu(cpu) {
bts_trace_init_cpu(cpu);
if (likely(per_cpu(tracer, cpu)))
trace_hw_branches_enabled = 1;
}
trace_hw_branches_suspended = 0;
put_online_cpus();
return trace_hw_branches_enabled ? 0 : -EOPNOTSUPP;
}
static void fb_bpf_cleanup_filter_cpus(struct fblock *fb)
{
unsigned int cpu;
struct fb_bpf_priv __percpu *fb_priv;
if (!fb)
return;
rcu_read_lock();
fb_priv = (struct fb_bpf_priv __percpu *) rcu_dereference_raw(fb->private_data);
rcu_read_unlock();
get_online_cpus();
for_each_online_cpu(cpu) {
struct fb_bpf_priv *fb_priv_cpu;
fb_priv_cpu = per_cpu_ptr(fb_priv, cpu);
fb_bpf_cleanup_filter(fb_priv_cpu);
}
put_online_cpus();
}
static int update_cpu_max_freq(int cpu, uint32_t max_freq)
{
int ret = 0;
if (max_freq != UINT_MAX)
pr_info("%s: Limiting cpu%d max frequency to %d\n",
KBUILD_MODNAME, cpu, max_freq);
else
pr_info("%s: Max frequency reset for cpu%d\n",
KBUILD_MODNAME, cpu);
get_online_cpus();
for_each_online_cpu(cpu) {
if (cpufreq_update_policy(cpu))
pr_info("%s: Unable to update policy for cpu:%d\n",
KBUILD_MODNAME, cpu);
}
put_online_cpus();
return ret;
}
/*
* We will reset the cpu frequencies limits here. The core online/offline
* status will be carried over to the process stopping the msm_thermal, as
* we dont want to online a core and bring in the thermal issues.
*/
static void __ref disable_msm_thermal(void)
{
int cpu = 0;
/* make sure check_temp is no longer running */
cancel_delayed_work(&check_temp_work);
flush_scheduled_work();
if (limited_max_freq == UINT_MAX)
return;
limited_max_freq = UINT_MAX;
get_online_cpus();
for_each_online_cpu(cpu) {
if (cpufreq_update_policy(cpu))
pr_info("%s: Unable to update policy for cpu:%d\n",
KBUILD_MODNAME, cpu);
}
put_online_cpus();
}
static void __init acpi_cpufreq_boost_init(void)
{
if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) {
msrs = msrs_alloc();
if (!msrs)
return;
acpi_cpufreq_driver.boost_supported = true;
acpi_cpufreq_driver.boost_enabled = boost_state(0);
get_online_cpus();
/* Force all MSRs to the same value */
boost_set_msrs(acpi_cpufreq_driver.boost_enabled,
cpu_online_mask);
register_cpu_notifier(&boost_nb);
put_online_cpus();
}
}
static int fb_bpf_proc_show_filter(struct seq_file *m, void *v)
{
unsigned long flags;
struct fblock *fb = (struct fblock *) m->private;
struct fb_bpf_priv *fb_priv_cpu;
struct sk_filter *sf;
get_online_cpus();
rcu_read_lock();
fb_priv_cpu = this_cpu_ptr(rcu_dereference_raw(fb->private_data));
rcu_read_unlock();
spin_lock_irqsave(&fb_priv_cpu->flock, flags);
sf = fb_priv_cpu->filter;
if (sf) {
unsigned int i;
if (sf->bpf_func == sk_run_filter)
seq_puts(m, "bpf jit: 0\n");
else
seq_puts(m, "bpf jit: 1\n");
seq_puts(m, "code:\n");
for (i = 0; i < sf->len; ++i) {
char sline[32];
memset(sline, 0, sizeof(sline));
snprintf(sline, sizeof(sline),
"{ 0x%x, %u, %u, 0x%x }\n",
sf->insns[i].code,
sf->insns[i].jt,
sf->insns[i].jf,
sf->insns[i].k);
sline[sizeof(sline) - 1] = 0;
seq_puts(m, sline);
}
}
spin_unlock_irqrestore(&fb_priv_cpu->flock, flags);
put_online_cpus();
return 0;
}
static void raise_mce(struct mce *m)
{
int context = MCJ_CTX(m->inject_flags);
inject_mce(m);
if (context == MCJ_CTX_RANDOM)
return;
#ifdef CONFIG_X86_LOCAL_APIC
if (m->inject_flags & MCJ_NMI_BROADCAST) {
unsigned long start;
int cpu;
get_online_cpus();
mce_inject_cpumask = cpu_online_map;
cpu_clear(get_cpu(), mce_inject_cpumask);
for_each_online_cpu(cpu) {
struct mce *mcpu = &per_cpu(injectm, cpu);
if (!mcpu->finished ||
MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM)
cpu_clear(cpu, mce_inject_cpumask);
}
if (!cpus_empty(mce_inject_cpumask))
apic->send_IPI_mask(&mce_inject_cpumask, NMI_VECTOR);
start = jiffies;
while (!cpus_empty(mce_inject_cpumask)) {
if (!time_before(jiffies, start + 2*HZ)) {
printk(KERN_ERR
"Timeout waiting for mce inject NMI %lx\n",
*cpus_addr(mce_inject_cpumask));
break;
}
cpu_relax();
}
raise_local();
put_cpu();
put_online_cpus();
} else
static void acpi_processor_remove(struct acpi_device *device)
{
struct acpi_processor *pr;
if (!device || !acpi_driver_data(device))
return;
pr = acpi_driver_data(device);
if (pr->id >= nr_cpu_ids)
goto out;
/*
* The only reason why we ever get here is CPU hot-removal. The CPU is
* already offline and the ACPI device removal locking prevents it from
* being put back online at this point.
*
* Unbind the driver from the processor device and detach it from the
* ACPI companion object.
*/
device_release_driver(pr->dev);
acpi_unbind_one(pr->dev);
/* Clean up. */
per_cpu(processor_device_array, pr->id) = NULL;
per_cpu(processors, pr->id) = NULL;
try_offline_node(cpu_to_node(pr->id));
/* Remove the CPU. */
get_online_cpus();
arch_unregister_cpu(pr->id);
acpi_unmap_lsapic(pr->id);
put_online_cpus();
out:
free_cpumask_var(pr->throttling.shared_cpu_map);
kfree(pr);
}
/*
* Take a map of online CPUs and the number of available interrupt vectors
* and generate an output cpumask suitable for spreading MSI/MSI-X vectors
* so that they are distributed as good as possible around the CPUs. If
* more vectors than CPUs are available we'll map one to each CPU,
* otherwise we map one to the first sibling of each socket.
*
* If there are more vectors than CPUs we will still only have one bit
* set per CPU, but interrupt code will keep on assigning the vectors from
* the start of the bitmap until we run out of vectors.
*/
struct cpumask *irq_create_affinity_mask(unsigned int *nr_vecs)
{
struct cpumask *affinity_mask;
unsigned int max_vecs = *nr_vecs;
if (max_vecs == 1)
return NULL;
affinity_mask = kzalloc(cpumask_size(), GFP_KERNEL);
if (!affinity_mask) {
*nr_vecs = 1;
return NULL;
}
get_online_cpus();
if (max_vecs >= num_online_cpus()) {
cpumask_copy(affinity_mask, cpu_online_mask);
*nr_vecs = num_online_cpus();
} else {
unsigned int vecs = 0, cpu;
for_each_online_cpu(cpu) {
if (cpu == get_first_sibling(cpu)) {
cpumask_set_cpu(cpu, affinity_mask);
vecs++;
}
if (--max_vecs == 0)
break;
}
*nr_vecs = vecs;
}
put_online_cpus();
return affinity_mask;
}
static void etm_disable_sysfs(struct coresight_device *csdev)
{
struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
/*
* Taking hotplug lock here protects from clocks getting disabled
* with tracing being left on (crash scenario) if user disable occurs
* after cpu online mask indicates the cpu is offline but before the
* DYING hotplug callback is serviced by the ETM driver.
*/
get_online_cpus();
spin_lock(&drvdata->spinlock);
/*
* Executing etm_disable_hw on the cpu whose ETM is being disabled
* ensures that register writes occur when cpu is powered.
*/
smp_call_function_single(drvdata->cpu, etm_disable_hw, drvdata, 1);
spin_unlock(&drvdata->spinlock);
put_online_cpus();
dev_info(drvdata->dev, "ETM tracing disabled\n");
}
/*H:020
* Now the Switcher is mapped and every thing else is ready, we need to do
* some more i386-specific initialization.
*/
void __init lguest_arch_host_init(void)
{
int i;
/*
* Most of the x86/switcher_32.S doesn't care that it's been moved; on
* Intel, jumps are relative, and it doesn't access any references to
* external code or data.
*
* The only exception is the interrupt handlers in switcher.S: their
* addresses are placed in a table (default_idt_entries), so we need to
* update the table with the new addresses. switcher_offset() is a
* convenience function which returns the distance between the
* compiled-in switcher code and the high-mapped copy we just made.
*/
for (i = 0; i < IDT_ENTRIES; i++)
default_idt_entries[i] += switcher_offset();
/*
* Set up the Switcher's per-cpu areas.
*
* Each CPU gets two pages of its own within the high-mapped region
* (aka. "struct lguest_pages"). Much of this can be initialized now,
* but some depends on what Guest we are running (which is set up in
* copy_in_guest_info()).
*/
for_each_possible_cpu(i) {
/* lguest_pages() returns this CPU's two pages. */
struct lguest_pages *pages = lguest_pages(i);
/* This is a convenience pointer to make the code neater. */
struct lguest_ro_state *state = &pages->state;
/*
* The Global Descriptor Table: the Host has a different one
* for each CPU. We keep a descriptor for the GDT which says
* where it is and how big it is (the size is actually the last
* byte, not the size, hence the "-1").
*/
state->host_gdt_desc.size = GDT_SIZE-1;
state->host_gdt_desc.address = (long)get_cpu_gdt_table(i);
/*
* All CPUs on the Host use the same Interrupt Descriptor
* Table, so we just use store_idt(), which gets this CPU's IDT
* descriptor.
*/
store_idt(&state->host_idt_desc);
/*
* The descriptors for the Guest's GDT and IDT can be filled
* out now, too. We copy the GDT & IDT into ->guest_gdt and
* ->guest_idt before actually running the Guest.
*/
state->guest_idt_desc.size = sizeof(state->guest_idt)-1;
state->guest_idt_desc.address = (long)&state->guest_idt;
state->guest_gdt_desc.size = sizeof(state->guest_gdt)-1;
state->guest_gdt_desc.address = (long)&state->guest_gdt;
/*
* We know where we want the stack to be when the Guest enters
* the Switcher: in pages->regs. The stack grows upwards, so
* we start it at the end of that structure.
*/
state->guest_tss.sp0 = (long)(&pages->regs + 1);
/*
* And this is the GDT entry to use for the stack: we keep a
* couple of special LGUEST entries.
*/
state->guest_tss.ss0 = LGUEST_DS;
/*
* x86 can have a finegrained bitmap which indicates what I/O
* ports the process can use. We set it to the end of our
* structure, meaning "none".
*/
state->guest_tss.io_bitmap_base = sizeof(state->guest_tss);
/*
* Some GDT entries are the same across all Guests, so we can
* set them up now.
*/
setup_default_gdt_entries(state);
/* Most IDT entries are the same for all Guests, too.*/
setup_default_idt_entries(state, default_idt_entries);
/*
* The Host needs to be able to use the LGUEST segments on this
* CPU, too, so put them in the Host GDT.
*/
get_cpu_gdt_table(i)[GDT_ENTRY_LGUEST_CS] = FULL_EXEC_SEGMENT;
get_cpu_gdt_table(i)[GDT_ENTRY_LGUEST_DS] = FULL_SEGMENT;
}
/*
* In the Switcher, we want the %cs segment register to use the
* LGUEST_CS GDT entry: we've put that in the Host and Guest GDTs, so
* it will be undisturbed when we switch. To change %cs and jump we
* need this structure to feed to Intel's "lcall" instruction.
*/
lguest_entry.offset = (long)switch_to_guest + switcher_offset();
lguest_entry.segment = LGUEST_CS;
/*
* Finally, we need to turn off "Page Global Enable". PGE is an
* optimization where page table entries are specially marked to show
* they never change. The Host kernel marks all the kernel pages this
* way because it's always present, even when userspace is running.
*
* Lguest breaks this: unbeknownst to the rest of the Host kernel, we
* switch to the Guest kernel. If you don't disable this on all CPUs,
* you'll get really weird bugs that you'll chase for two days.
*
* I used to turn PGE off every time we switched to the Guest and back
* on when we return, but that slowed the Switcher down noticibly.
*/
/*
* We don't need the complexity of CPUs coming and going while we're
* doing this.
*/
get_online_cpus();
if (boot_cpu_has(X86_FEATURE_PGE)) { /* We have a broader idea of "global". */
/* Remember that this was originally set (for cleanup). */
cpu_had_pge = 1;
/*
* adjust_pge is a helper function which sets or unsets the PGE
* bit on its CPU, depending on the argument (0 == unset).
*/
on_each_cpu(adjust_pge, (void *)0, 1);
/* Turn off the feature in the global feature set. */
clear_cpu_cap(&boot_cpu_data, X86_FEATURE_PGE);
}
put_online_cpus();
}
/*
* Allow secondary CPU threads to come online again
*/
void uninhibit_secondary_onlining(void)
{
get_online_cpus();
atomic_dec(&secondary_inhibit_count);
put_online_cpus();
}
/*
* mipsmt_sys_sched_setaffinity - set the cpu affinity of a process
*/
asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len,
unsigned long __user *user_mask_ptr)
{
cpumask_var_t cpus_allowed, new_mask, effective_mask;
struct thread_info *ti;
struct task_struct *p;
int retval;
if (len < sizeof(new_mask))
return -EINVAL;
if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask)))
return -EFAULT;
get_online_cpus();
rcu_read_lock();
p = find_process_by_pid(pid);
if (!p) {
rcu_read_unlock();
put_online_cpus();
return -ESRCH;
}
/* Prevent p going away */
get_task_struct(p);
rcu_read_unlock();
if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
retval = -ENOMEM;
goto out_put_task;
}
if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
retval = -ENOMEM;
goto out_free_cpus_allowed;
}
if (!alloc_cpumask_var(&effective_mask, GFP_KERNEL)) {
retval = -ENOMEM;
goto out_free_new_mask;
}
retval = -EPERM;
if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
goto out_unlock;
retval = security_task_setscheduler(p);
if (retval)
goto out_unlock;
/* Record new user-specified CPU set for future reference */
cpumask_copy(&p->thread.user_cpus_allowed, new_mask);
again:
/* Compute new global allowed CPU set if necessary */
ti = task_thread_info(p);
if (test_ti_thread_flag(ti, TIF_FPUBOUND) &&
cpus_intersects(*new_mask, mt_fpu_cpumask)) {
cpus_and(*effective_mask, *new_mask, mt_fpu_cpumask);
retval = set_cpus_allowed_ptr(p, effective_mask);
} else {
cpumask_copy(effective_mask, new_mask);
clear_ti_thread_flag(ti, TIF_FPUBOUND);
retval = set_cpus_allowed_ptr(p, new_mask);
}
if (!retval) {
cpuset_cpus_allowed(p, cpus_allowed);
if (!cpumask_subset(effective_mask, cpus_allowed)) {
/*
* We must have raced with a concurrent cpuset
* update. Just reset the cpus_allowed to the
* cpuset's cpus_allowed
*/
cpumask_copy(new_mask, cpus_allowed);
goto again;
}
}
out_unlock:
free_cpumask_var(effective_mask);
out_free_new_mask:
free_cpumask_var(new_mask);
out_free_cpus_allowed:
free_cpumask_var(cpus_allowed);
out_put_task:
put_task_struct(p);
put_online_cpus();
return retval;
}
static int clamp_thread(void *arg)
{
int cpunr = (unsigned long)arg;
DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0);
static const struct sched_param param = {
.sched_priority = MAX_USER_RT_PRIO/2,
};
unsigned int count = 0;
unsigned int target_ratio;
set_bit(cpunr, cpu_clamping_mask);
set_freezable();
init_timer_on_stack(&wakeup_timer);
sched_setscheduler(current, SCHED_FIFO, ¶m);
while (true == clamping && !kthread_should_stop() &&
cpu_online(cpunr)) {
int sleeptime;
unsigned long target_jiffies;
unsigned int guard;
unsigned int compensated_ratio;
int interval; /* jiffies to sleep for each attempt */
unsigned int duration_jiffies = msecs_to_jiffies(duration);
unsigned int window_size_now;
try_to_freeze();
/*
* make sure user selected ratio does not take effect until
* the next round. adjust target_ratio if user has changed
* target such that we can converge quickly.
*/
target_ratio = set_target_ratio;
guard = 1 + target_ratio/20;
window_size_now = window_size;
count++;
/*
* systems may have different ability to enter package level
* c-states, thus we need to compensate the injected idle ratio
* to achieve the actual target reported by the HW.
*/
compensated_ratio = target_ratio +
get_compensation(target_ratio);
if (compensated_ratio <= 0)
compensated_ratio = 1;
interval = duration_jiffies * 100 / compensated_ratio;
/* align idle time */
target_jiffies = roundup(jiffies, interval);
sleeptime = target_jiffies - jiffies;
if (sleeptime <= 0)
sleeptime = 1;
schedule_timeout_interruptible(sleeptime);
/*
* only elected controlling cpu can collect stats and update
* control parameters.
*/
if (cpunr == control_cpu && !(count%window_size_now)) {
should_skip =
powerclamp_adjust_controls(target_ratio,
guard, window_size_now);
smp_mb();
}
if (should_skip)
continue;
target_jiffies = jiffies + duration_jiffies;
mod_timer(&wakeup_timer, target_jiffies);
if (unlikely(local_softirq_pending()))
continue;
/*
* stop tick sched during idle time, interrupts are still
* allowed. thus jiffies are updated properly.
*/
preempt_disable();
/* mwait until target jiffies is reached */
while (time_before(jiffies, target_jiffies)) {
unsigned long ecx = 1;
unsigned long eax = target_mwait;
/*
* REVISIT: may call enter_idle() to notify drivers who
* can save power during cpu idle. same for exit_idle()
*/
local_touch_nmi();
stop_critical_timings();
mwait_idle_with_hints(eax, ecx);
start_critical_timings();
atomic_inc(&idle_wakeup_counter);
}
preempt_enable();
}
del_timer_sync(&wakeup_timer);
clear_bit(cpunr, cpu_clamping_mask);
return 0;
}
/*
* 1 HZ polling while clamping is active, useful for userspace
* to monitor actual idle ratio.
*/
static void poll_pkg_cstate(struct work_struct *dummy);
static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
static void poll_pkg_cstate(struct work_struct *dummy)
{
static u64 msr_last;
static u64 tsc_last;
static unsigned long jiffies_last;
u64 msr_now;
unsigned long jiffies_now;
u64 tsc_now;
u64 val64;
msr_now = pkg_state_counter();
tsc_now = rdtsc();
jiffies_now = jiffies;
/* calculate pkg cstate vs tsc ratio */
if (!msr_last || !tsc_last)
pkg_cstate_ratio_cur = 1;
else {
if (tsc_now - tsc_last) {
val64 = 100 * (msr_now - msr_last);
do_div(val64, (tsc_now - tsc_last));
pkg_cstate_ratio_cur = val64;
}
}
/* update record */
msr_last = msr_now;
jiffies_last = jiffies_now;
tsc_last = tsc_now;
if (true == clamping)
schedule_delayed_work(&poll_pkg_cstate_work, HZ);
}
static int start_power_clamp(void)
{
unsigned long cpu;
struct task_struct *thread;
set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
/* prevent cpu hotplug */
get_online_cpus();
/* prefer BSP */
control_cpu = 0;
if (!cpu_online(control_cpu))
control_cpu = smp_processor_id();
clamping = true;
schedule_delayed_work(&poll_pkg_cstate_work, 0);
/* start one thread per online cpu */
for_each_online_cpu(cpu) {
struct task_struct **p =
per_cpu_ptr(powerclamp_thread, cpu);
thread = kthread_create_on_node(clamp_thread,
(void *) cpu,
cpu_to_node(cpu),
"kidle_inject/%ld", cpu);
/* bind to cpu here */
if (likely(!IS_ERR(thread))) {
kthread_bind(thread, cpu);
wake_up_process(thread);
*p = thread;
}
}
put_online_cpus();
return 0;
}
int ds_selftest_bts(void)
{
struct ds_selftest_bts_conf conf;
unsigned char buffer[BUFFER_SIZE], *small_buffer;
unsigned long irq;
int cpu;
printk(KERN_INFO "[ds] bts selftest...");
conf.error = 0;
small_buffer = (unsigned char *)ALIGN((unsigned long)buffer, 8) + 8;
get_online_cpus();
for_each_online_cpu(cpu) {
conf.suspend = ds_suspend_bts_wrap;
conf.resume = ds_resume_bts_wrap;
conf.tracer =
ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE,
NULL, (size_t)-1, BTS_KERNEL);
ds_selftest_bts_cpu(&conf);
if (conf.error >= 0)
conf.error = ds_selftest_bts_bad_request_task(buffer);
ds_release_bts(conf.tracer);
if (conf.error < 0)
goto out;
conf.suspend = ds_suspend_bts_noirq;
conf.resume = ds_resume_bts_noirq;
conf.tracer =
ds_request_bts_cpu(cpu, buffer, BUFFER_SIZE,
NULL, (size_t)-1, BTS_KERNEL);
smp_call_function_single(cpu, ds_selftest_bts_cpu, &conf, 1);
if (conf.error >= 0) {
conf.error =
ds_selftest_bts_bad_release_noirq(cpu,
conf.tracer);
/* We must not release the tracer twice. */
if (conf.error < 0)
conf.tracer = NULL;
}
if (conf.error >= 0)
conf.error = ds_selftest_bts_bad_request_task(buffer);
smp_call_function_single(cpu, ds_release_bts_noirq_wrap,
conf.tracer, 1);
if (conf.error < 0)
goto out;
}
conf.suspend = ds_suspend_bts_wrap;
conf.resume = ds_resume_bts_wrap;
conf.tracer =
ds_request_bts_task(current, buffer, BUFFER_SIZE,
NULL, (size_t)-1, BTS_KERNEL);
ds_selftest_bts_cpu(&conf);
if (conf.error >= 0)
conf.error = ds_selftest_bts_bad_request_cpu(0, buffer);
ds_release_bts(conf.tracer);
if (conf.error < 0)
goto out;
conf.suspend = ds_suspend_bts_noirq;
conf.resume = ds_resume_bts_noirq;
conf.tracer =
ds_request_bts_task(current, small_buffer, SMALL_BUFFER_SIZE,
NULL, (size_t)-1, BTS_KERNEL);
local_irq_save(irq);
ds_selftest_bts_cpu(&conf);
if (conf.error >= 0)
conf.error = ds_selftest_bts_bad_request_cpu(0, buffer);
ds_release_bts_noirq(conf.tracer);
local_irq_restore(irq);
if (conf.error < 0)
goto out;
conf.error = 0;
out:
put_online_cpus();
printk(KERN_CONT "%s.\n", (conf.error ? "failed" : "passed"));
return conf.error;
}
static int etm_probe(struct amba_device *adev, const struct amba_id *id)
{
int ret;
void __iomem *base;
struct device *dev = &adev->dev;
struct coresight_platform_data *pdata = NULL;
struct etm_drvdata *drvdata;
struct resource *res = &adev->res;
struct coresight_desc *desc;
struct device_node *np = adev->dev.of_node;
desc = devm_kzalloc(dev, sizeof(*desc), GFP_KERNEL);
if (!desc)
return -ENOMEM;
drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
if (!drvdata)
return -ENOMEM;
if (np) {
pdata = of_get_coresight_platform_data(dev, np);
if (IS_ERR(pdata))
return PTR_ERR(pdata);
adev->dev.platform_data = pdata;
drvdata->use_cp14 = of_property_read_bool(np, "arm,cp14");
}
drvdata->dev = &adev->dev;
dev_set_drvdata(dev, drvdata);
/* Validity for the resource is already checked by the AMBA core */
base = devm_ioremap_resource(dev, res);
if (IS_ERR(base))
return PTR_ERR(base);
drvdata->base = base;
spin_lock_init(&drvdata->spinlock);
drvdata->atclk = devm_clk_get(&adev->dev, "atclk"); /* optional */
if (!IS_ERR(drvdata->atclk)) {
ret = clk_prepare_enable(drvdata->atclk);
if (ret)
return ret;
}
drvdata->cpu = pdata ? pdata->cpu : 0;
get_online_cpus();
etmdrvdata[drvdata->cpu] = drvdata;
if (!smp_call_function_single(drvdata->cpu, etm_os_unlock, drvdata, 1))
drvdata->os_unlock = true;
if (smp_call_function_single(drvdata->cpu,
etm_init_arch_data, drvdata, 1))
dev_err(dev, "ETM arch init failed\n");
if (!etm_count++)
register_hotcpu_notifier(&etm_cpu_notifier);
put_online_cpus();
if (etm_arch_supported(drvdata->arch) == false) {
ret = -EINVAL;
goto err_arch_supported;
}
etm_init_default_data(drvdata);
desc->type = CORESIGHT_DEV_TYPE_SOURCE;
desc->subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
desc->ops = &etm_cs_ops;
desc->pdata = pdata;
desc->dev = dev;
desc->groups = coresight_etm_groups;
drvdata->csdev = coresight_register(desc);
if (IS_ERR(drvdata->csdev)) {
ret = PTR_ERR(drvdata->csdev);
goto err_arch_supported;
}
pm_runtime_put(&adev->dev);
dev_info(dev, "%s initialized\n", (char *)id->data);
if (boot_enable) {
coresight_enable(drvdata->csdev);
drvdata->boot_enable = true;
}
return 0;
err_arch_supported:
if (--etm_count == 0)
unregister_hotcpu_notifier(&etm_cpu_notifier);
return ret;
}
/*
* mipsmt_sys_sched_setaffinity - set the cpu affinity of a process
*/
asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len,
unsigned long __user *user_mask_ptr)
{
cpumask_t new_mask;
cpumask_t effective_mask;
int retval;
struct task_struct *p;
struct thread_info *ti;
uid_t euid;
if (len < sizeof(new_mask))
return -EINVAL;
if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask)))
return -EFAULT;
get_online_cpus();
read_lock(&tasklist_lock);
p = find_process_by_pid(pid);
if (!p) {
read_unlock(&tasklist_lock);
put_online_cpus();
return -ESRCH;
}
/*
* It is not safe to call set_cpus_allowed with the
* tasklist_lock held. We will bump the task_struct's
* usage count and drop tasklist_lock before invoking
* set_cpus_allowed.
*/
get_task_struct(p);
euid = current_euid();
retval = -EPERM;
if (euid != p->cred->euid && euid != p->cred->uid &&
!capable(CAP_SYS_NICE)) {
read_unlock(&tasklist_lock);
goto out_unlock;
}
retval = security_task_setscheduler(p, 0, NULL);
if (retval)
goto out_unlock;
/* Record new user-specified CPU set for future reference */
p->thread.user_cpus_allowed = new_mask;
/* Unlock the task list */
read_unlock(&tasklist_lock);
/* Compute new global allowed CPU set if necessary */
ti = task_thread_info(p);
if (test_ti_thread_flag(ti, TIF_FPUBOUND) &&
cpus_intersects(new_mask, mt_fpu_cpumask)) {
cpus_and(effective_mask, new_mask, mt_fpu_cpumask);
retval = set_cpus_allowed_ptr(p, &effective_mask);
} else {
clear_ti_thread_flag(ti, TIF_FPUBOUND);
retval = set_cpus_allowed_ptr(p, &new_mask);
}
out_unlock:
put_task_struct(p);
put_online_cpus();
return retval;
}
/**
* mtrr_add_page - Add a memory type region
* @base: Physical base address of region in pages (in units of 4 kB!)
* @size: Physical size of region in pages (4 kB)
* @type: Type of MTRR desired
* @increment: If this is true do usage counting on the region
*
* Memory type region registers control the caching on newer Intel and
* non Intel processors. This function allows drivers to request an
* MTRR is added. The details and hardware specifics of each processor's
* implementation are hidden from the caller, but nevertheless the
* caller should expect to need to provide a power of two size on an
* equivalent power of two boundary.
*
* If the region cannot be added either because all regions are in use
* or the CPU cannot support it a negative value is returned. On success
* the register number for this entry is returned, but should be treated
* as a cookie only.
*
* On a multiprocessor machine the changes are made to all processors.
* This is required on x86 by the Intel processors.
*
* The available types are
*
* %MTRR_TYPE_UNCACHABLE - No caching
*
* %MTRR_TYPE_WRBACK - Write data back in bursts whenever
*
* %MTRR_TYPE_WRCOMB - Write data back soon but allow bursts
*
* %MTRR_TYPE_WRTHROUGH - Cache reads but not writes
*
* BUGS: Needs a quiet flag for the cases where drivers do not mind
* failures and do not wish system log messages to be sent.
*/
int mtrr_add_page(unsigned long base, unsigned long size,
unsigned int type, bool increment)
{
unsigned long lbase, lsize;
int i, replace, error;
mtrr_type ltype;
if (!mtrr_if)
return -ENXIO;
error = mtrr_if->validate_add_page(base, size, type);
if (error)
return error;
if (type >= MTRR_NUM_TYPES) {
pr_warning("mtrr: type: %u invalid\n", type);
return -EINVAL;
}
/* If the type is WC, check that this processor supports it */
if ((type == MTRR_TYPE_WRCOMB) && !have_wrcomb()) {
pr_warning("mtrr: your processor doesn't support write-combining\n");
return -ENOSYS;
}
if (!size) {
pr_warning("mtrr: zero sized request\n");
return -EINVAL;
}
if (base & size_or_mask || size & size_or_mask) {
pr_warning("mtrr: base or size exceeds the MTRR width\n");
return -EINVAL;
}
error = -EINVAL;
replace = -1;
/* No CPU hotplug when we change MTRR entries */
get_online_cpus();
/* Search for existing MTRR */
mutex_lock(&mtrr_mutex);
for (i = 0; i < num_var_ranges; ++i) {
mtrr_if->get(i, &lbase, &lsize, <ype);
if (!lsize || base > lbase + lsize - 1 ||
base + size - 1 < lbase)
continue;
/*
* At this point we know there is some kind of
* overlap/enclosure
*/
if (base < lbase || base + size - 1 > lbase + lsize - 1) {
if (base <= lbase &&
base + size - 1 >= lbase + lsize - 1) {
/* New region encloses an existing region */
if (type == ltype) {
replace = replace == -1 ? i : -2;
continue;
} else if (types_compatible(type, ltype))
continue;
}
pr_warning("mtrr: 0x%lx000,0x%lx000 overlaps existing"
" 0x%lx000,0x%lx000\n", base, size, lbase,
lsize);
goto out;
}
/* New region is enclosed by an existing region */
if (ltype != type) {
if (types_compatible(type, ltype))
continue;
pr_warning("mtrr: type mismatch for %lx000,%lx000 old: %s new: %s\n",
base, size, mtrr_attrib_to_str(ltype),
mtrr_attrib_to_str(type));
goto out;
}
if (increment)
++mtrr_usage_table[i];
error = i;
goto out;
}
/* Search for an empty MTRR */
i = mtrr_if->get_free_region(base, size, replace);
if (i >= 0) {
set_mtrr(i, base, size, type);
if (likely(replace < 0)) {
mtrr_usage_table[i] = 1;
} else {
mtrr_usage_table[i] = mtrr_usage_table[replace];
if (increment)
mtrr_usage_table[i]++;
if (unlikely(replace != i)) {
set_mtrr(replace, 0, 0, 0);
mtrr_usage_table[replace] = 0;
}
}
} else {
pr_info("mtrr: no more MTRRs available\n");
}
error = i;
out:
mutex_unlock(&mtrr_mutex);
put_online_cpus();
return error;
}
static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
{
int ret;
void __iomem *base;
struct device *dev = &adev->dev;
struct coresight_platform_data *pdata = NULL;
struct etmv4_drvdata *drvdata;
struct resource *res = &adev->res;
struct coresight_desc desc = { 0 };
struct device_node *np = adev->dev.of_node;
drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
if (!drvdata)
return -ENOMEM;
if (np) {
pdata = of_get_coresight_platform_data(dev, np);
if (IS_ERR(pdata))
return PTR_ERR(pdata);
adev->dev.platform_data = pdata;
}
drvdata->dev = &adev->dev;
dev_set_drvdata(dev, drvdata);
/* Validity for the resource is already checked by the AMBA core */
base = devm_ioremap_resource(dev, res);
if (IS_ERR(base))
return PTR_ERR(base);
drvdata->base = base;
spin_lock_init(&drvdata->spinlock);
drvdata->cpu = pdata ? pdata->cpu : 0;
get_online_cpus();
etmdrvdata[drvdata->cpu] = drvdata;
if (smp_call_function_single(drvdata->cpu,
etm4_init_arch_data, drvdata, 1))
dev_err(dev, "ETM arch init failed\n");
if (!etm4_count++) {
cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
"arm/coresight4:starting",
etm4_starting_cpu, etm4_dying_cpu);
ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
"arm/coresight4:online",
etm4_online_cpu, NULL);
if (ret < 0)
goto err_arch_supported;
hp_online = ret;
}
put_online_cpus();
if (etm4_arch_supported(drvdata->arch) == false) {
ret = -EINVAL;
goto err_arch_supported;
}
etm4_init_trace_id(drvdata);
etm4_set_default(&drvdata->config);
desc.type = CORESIGHT_DEV_TYPE_SOURCE;
desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
desc.ops = &etm4_cs_ops;
desc.pdata = pdata;
desc.dev = dev;
desc.groups = coresight_etmv4_groups;
drvdata->csdev = coresight_register(&desc);
if (IS_ERR(drvdata->csdev)) {
ret = PTR_ERR(drvdata->csdev);
goto err_arch_supported;
}
ret = etm_perf_symlink(drvdata->csdev, true);
if (ret) {
coresight_unregister(drvdata->csdev);
goto err_arch_supported;
}
pm_runtime_put(&adev->dev);
dev_info(dev, "%s initialized\n", (char *)id->data);
if (boot_enable) {
coresight_enable(drvdata->csdev);
drvdata->boot_enable = true;
}
return 0;
err_arch_supported:
if (--etm4_count == 0) {
cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
if (hp_online)
cpuhp_remove_state_nocalls(hp_online);
}
return ret;
}
/*
* Accumulate the vm event counters across all CPUs.
* The result is unavoidably approximate - it can change
* during and after execution of this function.
*/
void all_vm_events(unsigned long *ret)
{
get_online_cpus();
sum_vm_events(ret);
put_online_cpus();
}
void kvm_hv_vm_deactivated(void)
{
get_online_cpus();
atomic_dec(&hv_vm_count);
put_online_cpus();
}
static int boost_mig_sync_thread(void *data)
{
int dest_cpu = (int) data;
int src_cpu, ret;
struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);
struct cpufreq_policy dest_policy;
struct cpufreq_policy src_policy;
unsigned long flags;
while(1) {
ret = wait_event_interruptible(s->sync_wq, s->pending ||
kthread_should_stop());
if (kthread_should_stop())
break;
if (ret == -ERESTARTSYS)
continue;
spin_lock_irqsave(&s->lock, flags);
s->pending = false;
src_cpu = s->src_cpu;
spin_unlock_irqrestore(&s->lock, flags);
ret = cpufreq_get_policy(&src_policy, src_cpu);
if (ret)
continue;
ret = cpufreq_get_policy(&dest_policy, dest_cpu);
if (ret)
continue;
if (dest_policy.cur >= src_policy.cur ) {
pr_debug("No sync. CPU%d@%dKHz >= CPU%d@%dKHz\n",
dest_cpu, dest_policy.cur, src_cpu, src_policy.cur);
continue;
}
if (sync_threshold && (dest_policy.cur >= sync_threshold))
continue;
cancel_delayed_work_sync(&s->boost_rem);
if (sync_threshold) {
if (src_policy.cur >= sync_threshold)
s->boost_min = sync_threshold;
else
s->boost_min = src_policy.cur;
} else {
s->boost_min = src_policy.cur;
}
/* Force policy re-evaluation to trigger adjust notifier. */
get_online_cpus();
if (cpu_online(dest_cpu)) {
cpufreq_update_policy(dest_cpu);
queue_delayed_work_on(dest_cpu, cpu_boost_wq,
&s->boost_rem, msecs_to_jiffies(boost_ms));
} else {
s->boost_min = 0;
}
put_online_cpus();
}
return 0;
}
static int boost_mig_sync_thread(void *data)
{
int dest_cpu = (int) data;
int src_cpu, ret;
struct cpu_sync *s = &per_cpu(sync_info, dest_cpu);
struct cpufreq_policy dest_policy;
struct cpufreq_policy src_policy;
unsigned long flags;
unsigned int req_freq;
if (!cpuboost_enable) return 0;
while (1) {
wait_event(s->sync_wq, s->pending || kthread_should_stop());
#ifdef CONFIG_IRLED_GPIO
if (unlikely(gir_boost_disable)) {
pr_debug("[GPIO_IR][%s] continue~!(cpu:%d)\n",
__func__, raw_smp_processor_id());
continue;
}
#endif
if (kthread_should_stop())
break;
spin_lock_irqsave(&s->lock, flags);
s->pending = false;
src_cpu = s->src_cpu;
spin_unlock_irqrestore(&s->lock, flags);
ret = cpufreq_get_policy(&src_policy, src_cpu);
if (ret)
continue;
ret = cpufreq_get_policy(&dest_policy, dest_cpu);
if (ret)
continue;
if (s->task_load < migration_load_threshold)
continue;
if (s->task_load < migration_load_threshold)
continue;
req_freq = load_based_syncs ?
(dest_policy.max * s->task_load) / 100 : src_policy.cur;
if (req_freq <= dest_policy.cpuinfo.min_freq) {
pr_debug("No sync. Sync Freq:%u\n", req_freq);
continue;
}
if (sync_threshold)
req_freq = min(sync_threshold, req_freq);
cancel_delayed_work_sync(&s->boost_rem);
#ifdef CONFIG_CPUFREQ_HARDLIMIT
s->boost_min = check_cpufreq_hardlimit(req_freq);
#else
s->boost_min = req_freq;
#endif
/* Force policy re-evaluation to trigger adjust notifier. */
get_online_cpus();
if (cpu_online(src_cpu))
/*
* Send an unchanged policy update to the source
* CPU. Even though the policy isn't changed from
* its existing boosted or non-boosted state
* notifying the source CPU will let the governor
* know a boost happened on another CPU and that it
* should re-evaluate the frequency at the next timer
* event without interference from a min sample time.
*/
cpufreq_update_policy(src_cpu);
if (cpu_online(dest_cpu)) {
cpufreq_update_policy(dest_cpu);
queue_delayed_work_on(dest_cpu, cpu_boost_wq,
&s->boost_rem, msecs_to_jiffies(boost_ms));
} else {
s->boost_min = 0;
}
put_online_cpus();
}
return 0;
}
