void native_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm, unsigned long start,
unsigned long end)
{
struct flush_tlb_info info;
if (end == 0)
end = start + PAGE_SIZE;
info.flush_mm = mm;
info.flush_start = start;
info.flush_end = end;
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
if (end == TLB_FLUSH_ALL)
trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
else
trace_tlb_flush(TLB_REMOTE_SEND_IPI,
(end - start) >> PAGE_SHIFT);
if (is_uv_system()) {
unsigned int cpu;
cpu = smp_processor_id();
cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
if (cpumask)
smp_call_function_many(cpumask, flush_tlb_func,
&info, 1);
return;
}
smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
}
/***********************************************************
* mv_eth_start -- *
* start a network device. connect and enable interrupts *
* set hw defaults. fill rx buffers. restart phy link *
* auto neg. set device link flags. report status. *
***********************************************************/
static int mv_eth_start(struct net_device *dev)
{
struct eth_port *priv = MV_ETH_PRIV(dev);
int group;
/* in default link is down */
netif_carrier_off(dev);
/* Stop the TX queue - it will be enabled upon PHY status change after link-up interrupt/timer */
netif_tx_stop_all_queues(dev);
/* fill rx buffers, start rx/tx activity, set coalescing */
if (mv_eth_start_internals(priv, dev->mtu) != 0) {
printk(KERN_ERR "%s: start internals failed\n", dev->name);
goto error;
}
/* enable polling on the port, must be used after netif_poll_disable */
if (priv->flags & MV_ETH_F_CONNECT_LINUX)
for (group = 0; group < CONFIG_MV_ETH_NAPI_GROUPS; group++)
napi_enable(priv->napiGroup[group]);
if ((priv->flags & MV_ETH_F_LINK_UP) && !(priv->flags & MV_ETH_F_EXT_SWITCH)) {
if (mv_eth_ctrl_is_tx_enabled(priv)) {
netif_carrier_on(dev);
netif_tx_wake_all_queues(dev);
}
printk(KERN_NOTICE "%s: link up\n", dev->name);
}
#ifdef CONFIG_MV_ETH_SWITCH_LINK
if (priv->flags & MV_ETH_F_EXT_SWITCH) {
struct eth_netdev *dev_priv = MV_DEV_PRIV(dev);
dev_priv->link_map = 0;
mv_switch_link_update_event(dev_priv->port_map, 1);
}
#endif /* CONFIG_MV_ETH_SWITCH_LINK */
if (priv->flags & MV_ETH_F_CONNECT_LINUX) {
/* connect to port interrupt line */
if (request_irq(dev->irq, mv_eth_isr, (IRQF_DISABLED|IRQF_SAMPLE_RANDOM), "mv_eth", priv)) {
printk(KERN_ERR "cannot request irq %d for %s port %d\n", dev->irq, dev->name, priv->port);
if (priv->flags & MV_ETH_F_CONNECT_LINUX)
napi_disable(priv->napiGroup[CPU_GROUP_DEF]);
goto error;
}
/* unmask interrupts */
mv_eth_interrupts_unmask(priv);
smp_call_function_many(cpu_online_mask, (smp_call_func_t)mv_eth_interrupts_unmask, (void *)priv, 1);
printk(KERN_NOTICE "%s: started\n", dev->name);
}
return 0;
error:
printk(KERN_ERR "%s: start failed\n", dev->name);
return -1;
}
/*
* This variant of smp_call_function is used for index cacheops only.
*/
static inline void r4k_indexop_on_each_cpu(void (*func) (void *info), void *info)
{
preempt_disable();
#ifdef CONFIG_SMP
if (!cpu_has_safe_index_cacheops) {
if (smp_num_siblings > 1) {
cpumask_t tmp_mask = INIT_CPUMASK;
int cpu, this_cpu, n = 0;
/* If processor hasn't safe index cachops (likely)
then run cache flush on other CPUs.
But I assume that siblings have common L1 cache, so -
- run cache flush only once per sibling group. LY22 */
this_cpu = smp_processor_id();
for_each_online_cpu(cpu) {
if (cpumask_test_cpu(cpu, (&per_cpu(cpu_sibling_map, this_cpu))))
continue;
if (cpumask_intersects(&tmp_mask, (&per_cpu(cpu_sibling_map, cpu))))
continue;
cpu_set(cpu, tmp_mask);
n++;
}
if (n)
smp_call_function_many(&tmp_mask, func, info, 1);
} else
/*
* Use IPI to get all target uvhubs to release resources held by
* a given sending cpu number.
*/
static void reset_with_ipi(struct bau_targ_hubmask *distribution, int sender)
{
int uvhub;
int maskbits;
cpumask_t mask;
struct reset_args reset_args;
pax_track_stack();
reset_args.sender = sender;
cpus_clear(mask);
/* find a single cpu for each uvhub in this distribution mask */
maskbits = sizeof(struct bau_targ_hubmask) * BITSPERBYTE;
for (uvhub = 0; uvhub < maskbits; uvhub++) {
int cpu;
if (!bau_uvhub_isset(uvhub, distribution))
continue;
/* find a cpu for this uvhub */
cpu = uvhub_to_first_cpu(uvhub);
cpu_set(cpu, mask);
}
/* IPI all cpus; preemption is already disabled */
smp_call_function_many(&mask, do_reset, (void *)&reset_args, 1);
return;
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
struct cpumask *cpu_mask;
unsigned int pid;
preempt_disable();
pid = vma ? vma->vm_mm->context.id : 0;
if (unlikely(pid == MMU_NO_CONTEXT))
goto bail;
cpu_mask = mm_cpumask(vma->vm_mm);
if (!cpumask_equal(cpu_mask, cpumask_of(smp_processor_id()))) {
/* If broadcast tlbivax is supported, use it */
if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
if (lock)
spin_lock(&tlbivax_lock);
_tlbivax_bcast(vmaddr, pid);
if (lock)
spin_unlock(&tlbivax_lock);
goto bail;
} else {
struct tlb_flush_param p = { .pid = pid, .addr = vmaddr };
/* Ignores smp_processor_id() even if set in cpu_mask */
smp_call_function_many(cpu_mask,
do_flush_tlb_page_ipi, &p, 1);
}
}
_tlbil_va(vmaddr, pid);
bail:
preempt_enable();
}
static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
{
int cpu;
cpumask_t mask = { CPU_BITS_NONE };
if (!erratum_a15_798181())
return;
preempt_disable();
dummy_flush_tlb_a15_erratum();
for_each_online_cpu(cpu) {
if (cpu == smp_processor_id())
continue;
/*
* We only need to send an IPI if the other CPUs are running
* the same mm (and ASID) as the one being invalidated. There
* is no need for locking around the current_mm check since
* the switch_mm() function has a dmb() for this erratum in
* case a context switch happens on another CPU after the
* condition below.
*/
if (mm == per_cpu(current_mm, cpu))
cpumask_set_cpu(cpu, &mask);
}
smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
preempt_enable_no_resched();
}
/*
* smp_call_function - run a function on all other CPUs.
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
* @nonatomic: currently unused.
* @wait: If true, wait (atomically) until function has completed on other
* CPUs.
*
* Returns 0 on success, else a negative status code. Does not return until
* remote CPUs are nearly ready to execute func or are or have executed.
*
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler.
* Actually there are a few legal cases, like panic.
*/
int smp_call_function(void (*func) (void *info), void *info, int nonatomic,
int wait)
{
cpumask_t thismask = CPU_MASK_NONE;
cpus_or(thismask, cpu_online_map, thismask);
smp_call_function_many(&thismask, func, info, nonatomic, wait);
return 0;
}
static void drv_write(struct drv_cmd *cmd)
{
int this_cpu;
this_cpu = get_cpu();
if (cpumask_test_cpu(this_cpu, cmd->mask))
do_drv_write(cmd);
smp_call_function_many(cmd->mask, do_drv_write, cmd, 1);
put_cpu();
}
/*
* Kick all full dynticks CPUs in order to force these to re-evaluate
* their dependency on the tick and restart it if necessary.
*/
void tick_nohz_full_kick_all(void)
{
if (!have_nohz_full_mask)
return;
preempt_disable();
smp_call_function_many(nohz_full_mask,
nohz_full_kick_ipi, NULL, false);
preempt_enable();
}
/*
* This function initiates iCache flush. Call this function to
* to flush icaches on MIC
*/
void mic_flush_icache(void)
{
if (icache_snoop)
return;
preempt_disable();
/* flush icache on the current CPU */
icache_flush_handler();
smp_call_function_many(cpu_online_mask, icache_flush, NULL, 1);
preempt_enable();
}
static void on_each_cpu_mask(void (*func)(void *), void *info, int wait,
const struct cpumask *mask)
{
preempt_disable();
smp_call_function_many(mask, func, info, wait);
if (cpumask_test_cpu(smp_processor_id(), mask))
func(info);
preempt_enable();
}
static void broadcast_tlb_a15_erratum(void)
{
if (!erratum_a15_798181())
return;
preempt_disable();
dummy_flush_tlb_a15_erratum();
smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
NULL, 1);
preempt_enable_no_resched();
}
/*
* Set the icache snoop 0 - off, 1 - on
*/
static int icache_snoop_set (void *data, u64 val)
{
if (icache_snoop != (!!val)) {
preempt_disable();
icache_snoop = !!val;
set_icache_snoop_handler();
smp_call_function_many(cpu_online_mask, set_icache_snoop, NULL, 1);
preempt_enable();
}
return 0;
}
void native_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm, unsigned long start,
unsigned long end)
{
struct flush_tlb_info info;
info.flush_mm = mm;
info.flush_start = start;
info.flush_end = end;
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
if (is_uv_system()) {
unsigned int cpu;
cpu = smp_processor_id();
cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
if (cpumask)
smp_call_function_many(cpumask, flush_tlb_func,
&info, 1);
return;
}
smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
}
/*
* This function gets called when a part of the kernel has a new latency
* requirement. This means we need to get all processors out of their C-state,
* and then recalculate a new suitable C-state. Just do a cross-cpu IPI; that
* wakes them all right up.
*/
static int cpuidle_latency_notify(struct notifier_block *b,
unsigned long l, void *v)
{
const struct cpumask *cpus;
cpus = v ?: cpu_online_mask;
preempt_disable();
smp_call_function_many(cpus, smp_callback, NULL, 1);
preempt_enable();
return NOTIFY_OK;
}
static void exit_flush_lazy_tlbs(struct mm_struct *mm)
{
/*
* Would be nice if this was async so it could be run in
* parallel with our local flush, but generic code does not
* give a good API for it. Could extend the generic code or
* make a special powerpc IPI for flushing TLBs.
* For now it's not too performance critical.
*/
smp_call_function_many(mm_cpumask(mm), do_exit_flush_lazy_tlb,
(void *)mm, 1);
mm_reset_thread_local(mm);
}
static void raise_mce(struct mce *m)
{
int context = MCJ_CTX(m->inject_flags);
inject_mce(m);
if (context == MCJ_CTX_RANDOM)
return;
#ifdef CONFIG_X86_LOCAL_APIC
if (m->inject_flags & (MCJ_IRQ_BRAODCAST | MCJ_NMI_BROADCAST)) {
unsigned long start;
int cpu;
get_online_cpus();
cpumask_copy(mce_inject_cpumask, cpu_online_mask);
cpumask_clear_cpu(get_cpu(), mce_inject_cpumask);
for_each_online_cpu(cpu) {
struct mce *mcpu = &per_cpu(injectm, cpu);
if (!mcpu->finished ||
MCJ_CTX(mcpu->inject_flags) != MCJ_CTX_RANDOM)
cpumask_clear_cpu(cpu, mce_inject_cpumask);
}
if (!cpumask_empty(mce_inject_cpumask)) {
if (m->inject_flags & MCJ_IRQ_BRAODCAST) {
/*
* don't wait because mce_irq_ipi is necessary
* to be sync with following raise_local
*/
preempt_disable();
smp_call_function_many(mce_inject_cpumask,
mce_irq_ipi, NULL, 0);
preempt_enable();
} else if (m->inject_flags & MCJ_NMI_BROADCAST)
apic->send_IPI_mask(mce_inject_cpumask,
NMI_VECTOR);
}
start = jiffies;
while (!cpumask_empty(mce_inject_cpumask)) {
if (!time_before(jiffies, start + 2*HZ)) {
printk(KERN_ERR
"Timeout waiting for mce inject %lx\n",
*cpumask_bits(mce_inject_cpumask));
break;
}
cpu_relax();
}
raise_local();
put_cpu();
put_online_cpus();
} else
static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
{
int this_cpu;
cpumask_t mask = { CPU_BITS_NONE };
if (!has_erratum_a15_798181)
return;
dummy_flush_tlb_a15_erratum();
this_cpu = get_cpu();
a15_erratum_get_cpumask(this_cpu, mm, &mask);
smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
put_cpu();
}
/* context.lock is held */
static void install_ldt(struct mm_struct *current_mm,
struct ldt_struct *ldt)
{
/* Synchronizes with smp_read_barrier_depends in load_mm_ldt. */
barrier();
ACCESS_ONCE(current_mm->context.ldt) = ldt;
/* Activate the LDT for all CPUs using current_mm. */
smp_call_function_many(mm_cpumask(current_mm), flush_ldt, current_mm,
true);
local_irq_disable();
flush_ldt(current_mm);
local_irq_enable();
}
void flush_tlb_mm(struct mm_struct *mm)
{
unsigned int pid;
preempt_disable();
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
goto no_context;
if (!cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) {
struct tlb_flush_param p = { .pid = pid };
/* Ignores smp_processor_id() even if set. */
smp_call_function_many(mm_cpumask(mm),
do_flush_tlb_mm_ipi, &p, 1);
}
_tlbil_pid(pid);
no_context:
preempt_enable();
}
static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no,
struct msr *msrs,
void (*msr_func) (void *info))
{
struct msr_info rv;
int this_cpu;
memset(&rv, 0, sizeof(rv));
rv.msrs = msrs;
rv.msr_no = msr_no;
this_cpu = get_cpu();
if (cpumask_test_cpu(this_cpu, mask))
msr_func(&rv);
smp_call_function_many(mask, msr_func, &rv, 1);
put_cpu();
}
/* This gets called just before system reboots */
void opal_flash_term_callback(void)
{
struct cpumask mask;
if (update_flash_data.status != FLASH_IMG_READY)
return;
pr_alert("FLASH: Flashing new firmware\n");
pr_alert("FLASH: Image is %u bytes\n", image_data.size);
pr_alert("FLASH: Performing flash and reboot/shutdown\n");
pr_alert("FLASH: This will take several minutes. Do not power off!\n");
/* Small delay to help getting the above message out */
msleep(500);
/* Return secondary CPUs to firmware */
cpumask_copy(&mask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &mask);
if (!cpumask_empty(&mask))
smp_call_function_many(&mask,
flash_return_cpu, NULL, false);
/* Hard disable interrupts */
hard_irq_disable();
}
RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
{
IPRT_LINUX_SAVE_EFL_AC();
int rc;
RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
/*
* Check that both CPUs are online before doing the broadcast call.
*/
RTThreadPreemptDisable(&PreemptState);
if ( RTMpIsCpuOnline(idCpu1)
&& RTMpIsCpuOnline(idCpu2))
{
/*
* Use the smp_call_function variant taking a cpu mask where available,
* falling back on broadcast with filter. Slight snag if one of the
* CPUs is the one we're running on, we must do the call and the post
* call wait ourselves.
*/
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpumask_t DstCpuMask;
#endif
RTCPUID idCpuSelf = RTMpCpuId();
bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
RTMPARGS Args;
Args.pfnWorker = pfnWorker;
Args.pvUser1 = pvUser1;
Args.pvUser2 = pvUser2;
Args.idCpu = idCpu1;
Args.idCpu2 = idCpu2;
Args.cHits = 0;
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
cpumask_clear(&DstCpuMask);
cpumask_set_cpu(idCpu1, &DstCpuMask);
cpumask_set_cpu(idCpu2, &DstCpuMask);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
cpus_clear(DstCpuMask);
cpu_set(idCpu1, DstCpuMask);
cpu_set(idCpu2, DstCpuMask);
#endif
#if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
rc = 0;
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
#else /* older kernels */
rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
#endif /* older kernels */
Assert(rc == 0);
/* Call ourselves if necessary and wait for the other party to be done. */
if (fCallSelf)
{
uint32_t cLoops = 0;
rtmpLinuxWrapper(&Args);
while (ASMAtomicReadU32(&Args.cHits) < 2)
{
if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
break;
cLoops++;
ASMNopPause();
}
}
Assert(Args.cHits <= 2);
if (Args.cHits == 2)
rc = VINF_SUCCESS;
else if (Args.cHits == 1)
rc = VERR_NOT_ALL_CPUS_SHOWED;
else if (Args.cHits == 0)
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_IPE_1;
}
/*
* A CPU must be present to be considered just offline.
*/
else if ( RTMpIsCpuPresent(idCpu1)
&& RTMpIsCpuPresent(idCpu2))
rc = VERR_CPU_OFFLINE;
else
rc = VERR_CPU_NOT_FOUND;
RTThreadPreemptRestore(&PreemptState);;
IPRT_LINUX_RESTORE_EFL_AC();
return rc;
}
/*
* Serialize against find_current_mm_pte which does lock-less
* lookup in page tables with local interrupts disabled. For huge pages
* it casts pmd_t to pte_t. Since format of pte_t is different from
* pmd_t we want to prevent transit from pmd pointing to page table
* to pmd pointing to huge page (and back) while interrupts are disabled.
* We clear pmd to possibly replace it with page table pointer in
* different code paths. So make sure we wait for the parallel
* find_current_mm_pte to finish.
*/
void serialize_against_pte_lookup(struct mm_struct *mm)
{
smp_mb();
smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
}
static u32 drv_read(struct acpi_cpufreq_data *data, const struct cpumask *mask)
{
struct acpi_processor_performance *perf = to_perf_data(data);
struct drv_cmd cmd = {
.reg = &perf->control_register,
.func.read = data->cpu_freq_read,
};
int err;
err = smp_call_function_any(mask, do_drv_read, &cmd, 1);
WARN_ON_ONCE(err); /* smp_call_function_any() was buggy? */
return cmd.val;
}
/* Called via smp_call_function_many(), on the target CPUs */
static void do_drv_write(void *_cmd)
{
struct drv_cmd *cmd = _cmd;
cmd->func.write(cmd->reg, cmd->val);
}
static void drv_write(struct acpi_cpufreq_data *data,
const struct cpumask *mask, u32 val)
{
struct acpi_processor_performance *perf = to_perf_data(data);
struct drv_cmd cmd = {
.reg = &perf->control_register,
.val = val,
.func.write = data->cpu_freq_write,
};
int this_cpu;
this_cpu = get_cpu();
if (cpumask_test_cpu(this_cpu, mask))
do_drv_write(&cmd);
smp_call_function_many(mask, do_drv_write, &cmd, 1);
put_cpu();
}
static u32 get_cur_val(const struct cpumask *mask, struct acpi_cpufreq_data *data)
{
u32 val;
if (unlikely(cpumask_empty(mask)))
return 0;
val = drv_read(data, mask);
pr_debug("get_cur_val = %u\n", val);
return val;
}
static unsigned int get_cur_freq_on_cpu(unsigned int cpu)
{
struct acpi_cpufreq_data *data;
struct cpufreq_policy *policy;
unsigned int freq;
unsigned int cached_freq;
pr_debug("get_cur_freq_on_cpu (%d)\n", cpu);
policy = cpufreq_cpu_get_raw(cpu);
if (unlikely(!policy))
return 0;
data = policy->driver_data;
if (unlikely(!data || !policy->freq_table))
return 0;
cached_freq = policy->freq_table[to_perf_data(data)->state].frequency;
freq = extract_freq(policy, get_cur_val(cpumask_of(cpu), data));
if (freq != cached_freq) {
/*
* The dreaded BIOS frequency change behind our back.
* Force set the frequency on next target call.
*/
data->resume = 1;
}
pr_debug("cur freq = %u\n", freq);
return freq;
}
static unsigned int check_freqs(struct cpufreq_policy *policy,
const struct cpumask *mask, unsigned int freq)
{
struct acpi_cpufreq_data *data = policy->driver_data;
unsigned int cur_freq;
unsigned int i;
for (i = 0; i < 100; i++) {
cur_freq = extract_freq(policy, get_cur_val(mask, data));
if (cur_freq == freq)
return 1;
udelay(10);
}
return 0;
}
static int acpi_cpufreq_target(struct cpufreq_policy *policy,
unsigned int index)
{
struct acpi_cpufreq_data *data = policy->driver_data;
struct acpi_processor_performance *perf;
const struct cpumask *mask;
unsigned int next_perf_state = 0; /* Index into perf table */
int result = 0;
if (unlikely(!data)) {
return -ENODEV;
}
perf = to_perf_data(data);
next_perf_state = policy->freq_table[index].driver_data;
if (perf->state == next_perf_state) {
if (unlikely(data->resume)) {
pr_debug("Called after resume, resetting to P%d\n",
next_perf_state);
data->resume = 0;
} else {
pr_debug("Already at target state (P%d)\n",
next_perf_state);
return 0;
}
}
/*
* The core won't allow CPUs to go away until the governor has been
* stopped, so we can rely on the stability of policy->cpus.
*/
mask = policy->shared_type == CPUFREQ_SHARED_TYPE_ANY ?
cpumask_of(policy->cpu) : policy->cpus;
drv_write(data, mask, perf->states[next_perf_state].control);
if (acpi_pstate_strict) {
if (!check_freqs(policy, mask,
policy->freq_table[index].frequency)) {
pr_debug("acpi_cpufreq_target failed (%d)\n",
policy->cpu);
result = -EAGAIN;
}
}
if (!result)
perf->state = next_perf_state;
return result;
}
unsigned int acpi_cpufreq_fast_switch(struct cpufreq_policy *policy,
unsigned int target_freq)
{
struct acpi_cpufreq_data *data = policy->driver_data;
struct acpi_processor_performance *perf;
struct cpufreq_frequency_table *entry;
unsigned int next_perf_state, next_freq, index;
/*
* Find the closest frequency above target_freq.
*/
if (policy->cached_target_freq == target_freq)
index = policy->cached_resolved_idx;
else
index = cpufreq_table_find_index_dl(policy, target_freq);
entry = &policy->freq_table[index];
next_freq = entry->frequency;
next_perf_state = entry->driver_data;
perf = to_perf_data(data);
if (perf->state == next_perf_state) {
if (unlikely(data->resume))
data->resume = 0;
else
return next_freq;
}
data->cpu_freq_write(&perf->control_register,
perf->states[next_perf_state].control);
perf->state = next_perf_state;
return next_freq;
}
static unsigned long
acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu)
{
struct acpi_processor_performance *perf;
perf = to_perf_data(data);
if (cpu_khz) {
/* search the closest match to cpu_khz */
unsigned int i;
unsigned long freq;
unsigned long freqn = perf->states[0].core_frequency * 1000;
for (i = 0; i < (perf->state_count-1); i++) {
freq = freqn;
freqn = perf->states[i+1].core_frequency * 1000;
if ((2 * cpu_khz) > (freqn + freq)) {
perf->state = i;
return freq;
}
}
perf->state = perf->state_count-1;
return freqn;
} else {
/* assume CPU is at P0... */
perf->state = 0;
return perf->states[0].core_frequency * 1000;
}
}
static void free_acpi_perf_data(void)
{
unsigned int i;
/* Freeing a NULL pointer is OK, and alloc_percpu zeroes. */
for_each_possible_cpu(i)
free_cpumask_var(per_cpu_ptr(acpi_perf_data, i)
->shared_cpu_map);
free_percpu(acpi_perf_data);
}
void force_vm_exit(const cpumask_t *mask)
{
smp_call_function_many(mask, exit_vm_noop, NULL, true);
}
static void l4x_unmap_sync_mm(struct mm_struct *mm)
{
cpumask_t mask = { CPU_BITS_NONE };
a15_erratum_get_cpumask(smp_processor_id(), mm, &mask);
smp_call_function_many(&mask, l4x_dummy_remote_func, NULL, 1);
}
void force_vm_exit(const cpumask_t *mask)
{
preempt_disable();
smp_call_function_many(mask, exit_vm_noop, NULL, true);
preempt_enable();
}
