static void flat_vector_allocation_domain(int cpu, struct cpumask *retmask)
{
/* Careful. Some cpus do not strictly honor the set of cpus
* specified in the interrupt destination when using lowest
* priority interrupt delivery mode.
*
* In particular there was a hyperthreading cpu observed to
* deliver interrupts to the wrong hyperthread when only one
* hyperthread was specified in the interrupt desitination.
*/
cpumask_clear(retmask);
cpumask_bits(retmask)[0] = APIC_ALL_CPUS;
}
int config_L2(int size)
{
int i;
struct cpumask mask;
int cur_size = get_l2c_size();
if (size != SZ_256K && size != SZ_512K) {
printk("inlvalid input size %x\n", size);
return -1;
}
if (in_interrupt()) {
printk(KERN_ERR "Cannot use %s in interrupt/softirq context\n",
__func__);
return -1;
}
if (size == cur_size) {
printk("Config L2 size %x is equal to current L2 size %x\n",
size, cur_size);
return 0;
}
cpumask_clear(&mask);
for(i = 0; i < get_cluster_core_count(); i++)
cpumask_set_cpu(i, &mask);
atomic_set(&L1_flush_done, 0);
get_online_cpus();
//printk("[Config L2] Config L2 start, on line cpu = %d\n",num_online_cpus());
/* disable cache and flush L1 on Cluster0*/
on_each_cpu_mask(&mask, (smp_call_func_t)atomic_flush, NULL, true);
//while(atomic_read(&L1_flush_done) != num_online_cpus());
//printk("[Config L2] L1 flush done\n");
/* Only need to flush Cluster0's L2 */
smp_call_function_any(&mask, (smp_call_func_t)inner_dcache_flush_L2, NULL, true);
//printk("[Config L2] L2 flush done\n");
/* change L2 size */
config_L2_size(size);
//printk("[Config L2] Change L2 flush size done(size = %d)\n",size);
/* enable Cluster0's cache */
atomic_set(&L1_flush_done, 0);
on_each_cpu_mask(&mask, (smp_call_func_t)__enable_cache, NULL, true);
//update cr_alignment for other kernel function usage
cr_alignment = cr_alignment | (0x4); //C1_CBIT
put_online_cpus();
printk("Config L2 size %x done\n", size);
return 0;
}
void move_masked_irq(int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
if (likely(!(desc->status & IRQ_MOVE_PENDING)))
return;
/*
* Paranoia: cpu-local interrupts shouldn't be calling in here anyway.
*/
if (CHECK_IRQ_PER_CPU(desc->status)) {
WARN_ON(1);
return;
}
desc->status &= ~IRQ_MOVE_PENDING;
if (unlikely(cpumask_empty(desc->pending_mask)))
return;
if (!desc->chip->set_affinity)
return;
assert_spin_locked(&desc->lock);
/*
* If there was a valid mask to work with, please
* do the disable, re-program, enable sequence.
* This is *not* particularly important for level triggered
* but in a edge trigger case, we might be setting rte
* when an active trigger is comming in. This could
* cause some ioapics to mal-function.
* Being paranoid i guess!
*
* For correct operation this depends on the caller
* masking the irqs.
*/
if (likely(cpumask_any_and(desc->pending_mask, cpu_online_mask)
< nr_cpu_ids)) {
int ret = chip->irq_set_affinity(irq, desc->pending_mask);
switch (ret) {
case IRQ_SET_MASK_OK:
cpumask_copy(desc->affinity, desc->pending_mask);
case IRQ_SET_MASK_OK_NOCOPY:
irq_set_thread_affinity(desc);
}
}
cpumask_clear(desc->pending_mask);
}
static void desc_smp_init(struct irq_desc *desc, int node,
const struct cpumask *affinity)
{
if (!affinity)
affinity = irq_default_affinity;
cpumask_copy(desc->irq_common_data.affinity, affinity);
#ifdef CONFIG_GENERIC_PENDING_IRQ
cpumask_clear(desc->pending_mask);
#endif
#ifdef CONFIG_NUMA
desc->irq_common_data.node = node;
#endif
}
/*
* In XLP cpu mask for setting affinity of an interrupt cannot span multiple
* nodes. Although this is not a h/w restriction, the effort to implement
* this feature does not justify the potential benefit; not only that handling
* non local interrupts are slightly slower, it could be expensive in terms of
* memory access and other resource utilization
*
* @node : node to which mask `mask` to be restricted
* @src : mask to restrict
* @dst : restricted mask (result)
*/
void constrict_mask_to_node(u8 node, struct cpumask *dst, const struct cpumask *src)
{
// char buf[140];
int i;
cpumask_clear(dst);
for (i = NLM_MAX_CPU_PER_NODE * node;
i < (NLM_MAX_CPU_PER_NODE *(node + 1)); i++) {
cpumask_set_cpu(i, dst);
}
cpumask_and(dst, dst, &phys_cpu_present_map);
cpumask_and(dst, dst, src);
return;
}
static int set_cpu_min_freq(const char *buf, const struct kernel_param *kp)
{
int i, j, ntokens = 0;
unsigned int val, cpu;
const char *cp = buf;
struct cpu_status *i_cpu_stats;
struct cpufreq_policy policy;
cpumask_var_t limit_mask;
int ret;
while ((cp = strpbrk(cp + 1, " :")))
ntokens++;
if (!(ntokens % 2))
return -EINVAL;
cp = buf;
cpumask_clear(limit_mask);
for (i = 0; i < ntokens; i += 2) {
if (sscanf(cp, "%u:%u", &cpu, &val) != 2)
return -EINVAL;
if (cpu > (num_present_cpus() - 1))
return -EINVAL;
i_cpu_stats = &per_cpu(cpu_stats, cpu);
i_cpu_stats->min = val;
cpumask_set_cpu(cpu, limit_mask);
cp = strchr(cp, ' ');
cp++;
}
get_online_cpus();
for_each_cpu(i, limit_mask) {
i_cpu_stats = &per_cpu(cpu_stats, i);
if (cpufreq_get_policy(&policy, i))
continue;
if (cpu_online(i) && (policy.min != i_cpu_stats->min)) {
ret = cpufreq_update_policy(i);
if (ret)
continue;
}
for_each_cpu(j, policy.related_cpus)
cpumask_clear_cpu(j, limit_mask);
}
// ARM10C 20141004
// desc: kmem_cache#28-o0, node: 0
static void desc_smp_init(struct irq_desc *desc, int node)
{
// desc->irq_data.node: (kmem_cache#28-o0)->irq_data.node, node: 0
desc->irq_data.node = node;
// desc->irq_data.node: (kmem_cache#28-o0)->irq_data.node: 0
// desc->irq_data.affinity: (kmem_cache#28-o0)->irq_data.affinity,
// irq_default_affinity->bits[0]: 0xF
cpumask_copy(desc->irq_data.affinity, irq_default_affinity);
// desc->irq_data.affinity: (kmem_cache#28-o0)->irq_data.affinity.bits[0]: 0xF
#ifdef CONFIG_GENERIC_PENDING_IRQ // CONFIG_GENERIC_PENDING_IRQ=n
cpumask_clear(desc->pending_mask);
#endif
}
static int __stp_alloc_ring_buffer(void)
{
int i;
unsigned long buffer_size = _stp_bufsize * 1024 * 1024;
if (!alloc_cpumask_var(&_stp_relay_data.trace_reader_cpumask,
(GFP_KERNEL & ~__GFP_WAIT)))
goto fail;
cpumask_clear(_stp_relay_data.trace_reader_cpumask);
if (buffer_size == 0) {
dbug_trans(1, "using default buffer size...\n");
buffer_size = _stp_nsubbufs * _stp_subbuf_size;
}
dbug_trans(1, "using buffer size %lu...\n", buffer_size);
/* The number passed to ring_buffer_alloc() is per cpu. Our
* 'buffer_size' is a total number of bytes to allocate. So,
* we need to divide buffer_size by the number of cpus. */
buffer_size /= num_online_cpus();
dbug_trans(1, "%lu\n", buffer_size);
_stp_relay_data.rb = ring_buffer_alloc(buffer_size, 0);
if (!_stp_relay_data.rb)
goto fail;
/* Increment _stp_allocated_memory and
_stp_allocated_net_memory to approximately account for
buffers allocated by ring_buffer_alloc. */
{
#ifndef DIV_ROUND_UP
#define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
#endif
u64 relay_pages;
relay_pages = DIV_ROUND_UP (buffer_size, PAGE_SIZE);
if (relay_pages < 2) relay_pages = 2;
relay_pages *= num_online_cpus();
_stp_allocated_net_memory += relay_pages * PAGE_SIZE;
_stp_allocated_memory += relay_pages * PAGE_SIZE;
}
dbug_trans(0, "size = %lu\n", ring_buffer_size(_stp_relay_data.rb));
return 0;
fail:
__stp_free_ring_buffer();
return -ENOMEM;
}
/*
* Allocate node_to_cpumask_map based on number of available nodes
* Requires node_possible_map to be valid.
*
* Note: cpumask_of_node() is not valid until after this is done.
* (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
*/
static void __init setup_node_to_cpumask_map(void)
{
int node;
/* setup nr_node_ids if not done yet */
if (nr_node_ids == MAX_NUMNODES)
setup_nr_node_ids();
/* allocate and clear the mapping */
for (node = 0; node < nr_node_ids; node++) {
alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
cpumask_clear(node_to_cpumask_map[node]);
}
/* cpumask_of_node() will now work */
pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
}
static void crash_kexec_prepare_cpus(int cpu)
{
unsigned int msecs;
unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */
crash_send_ipi(crash_ipi_callback);
smp_wmb();
/*
* FIXME: Until we will have the way to stop other CPUs reliably,
* the crash CPU will send an IPI and wait for other CPUs to
* respond.
* Delay of at least 10 seconds.
*/
printk(KERN_EMERG "Sending IPI to other cpus...\n");
msecs = 10000;
while ((cpumask_weight(&cpus_in_crash) < ncpus) && (--msecs > 0)) {
cpu_relax();
mdelay(1);
}
/* Would it be better to replace the trap vector here? */
/*
* FIXME: In case if we do not get all CPUs, one possibility: ask the
* user to do soft reset such that we get all.
* Soft-reset will be used until better mechanism is implemented.
*/
if (cpumask_weight(&cpus_in_crash) < ncpus) {
printk(KERN_EMERG "done waiting: %d cpu(s) not responding\n",
ncpus - cpumask_weight(&cpus_in_crash));
printk(KERN_EMERG "Activate soft-reset to stop other cpu(s)\n");
cpumask_clear(&cpus_in_sr);
atomic_set(&enter_on_soft_reset, 0);
while (cpumask_weight(&cpus_in_crash) < ncpus)
cpu_relax();
}
/*
* Make sure all CPUs are entered via soft-reset if the kdump is
* invoked using soft-reset.
*/
if (cpumask_test_cpu(cpu, &cpus_in_sr))
crash_soft_reset_check(cpu);
/* Leave the IPI callback set */
}
static void tzdev_migrate_threads(int cpu)
{
struct task_struct *thread;
cpumask_t next_cpumask;
cpumask_clear(&next_cpumask);
cpumask_set_cpu(cpu, &next_cpumask);
while ((thread = tzdev_get_next_thread(&next_cpumask))) {
pr_notice("Migrate thread pid = %d to cpu = %d\n", thread->pid, cpu);
/* We shouldn't fail here because of we wrap this code by
* get_online_cpus() / put_online_cpus() */
BUG_ON(set_cpus_allowed(thread, next_cpumask));
put_task_struct(thread);
}
BUG_ON(set_cpus_allowed(current, next_cpumask));
}
int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues,
const struct cpumask *online_mask)
{
unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling;
cpumask_var_t cpus;
if (!alloc_cpumask_var(&cpus, GFP_ATOMIC))
return 1;
cpumask_clear(cpus);
nr_cpus = nr_uniq_cpus = 0;
for_each_cpu(i, online_mask) {
nr_cpus++;
first_sibling = get_first_sibling(i);
if (!cpumask_test_cpu(first_sibling, cpus))
nr_uniq_cpus++;
cpumask_set_cpu(i, cpus);
}
/*
* Initializes PIC ITE entries PRM 9.5.6.26
* XLP restricts CPU affinity to 8 groups. Though configurable, they are
* programmed to have the following patterns.
* 0 => Only 0th cpu on the node
* 1 => All local threads in node; mask = (0xffffffff) on node
* 2 => cpu0-15 on node; mask = 0x0000ffff & online_cpu_mask on nodes
* 3 => cpu15-31 on node; mask = 0xffff0000 & online_cpu_mask on node
* 4 => All cpus on all nodes; i.e.,
* mask = (0xffffffff_ffffffff_ffffffff_ffffffff & physical online cpu map)
* These are programmer defined groups and can be changed as warranted.
* Added 5 => CPUs 0-11
* Added 6 => CPUs 0-7
* Added 7 => CPUs 0-3
* Actual programmed value will take into consideration cpu_online_mask.
*
* There is a major issue that needs addressing when run in multi node mode
* Number of nodes must be determined and programmed correctly, if a bit in ITE
* is programmed without physical thread being present, when interrupt is
* dispatched to that CPU under global scheme, system would hang. Thus this
* scenario should be avoided. That is why phys_cpu_present_map is used
*
* This function simply initializes the xlp_ites entries with proposed
* CPUmasks. */
static void xlp_ites_init(void)
{
u64 bm = 0x1;
u8 node;
struct cpumask m;
cpumask_clear(&m);
for_each_online_node(node) {
/* Simply set the static pattern in all */
bm = 1;
u32_to_cpumask(&xlp_ites[node][0], bm);
cpumask_shift_left(&xlp_ites[node][0], &xlp_ites[node][0], NLM_MAX_CPU_PER_NODE * node); /* directs only to cpu0 of node `node` */
bm = 0xffffffff;
u32_to_cpumask(&xlp_ites[node][1], bm);
cpumask_shift_left(&xlp_ites[node][1], &xlp_ites[node][1], NLM_MAX_CPU_PER_NODE * node); /* directs to all cpus of node `node` */
cpumask_or(&m, &m, &xlp_ites[node][1]);
bm = 0x0000ffff;
u32_to_cpumask(&xlp_ites[node][2], bm);
cpumask_shift_left(&xlp_ites[node][2], &xlp_ites[node][2], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */
bm = 0xffff0000;
u32_to_cpumask(&xlp_ites[node][3], bm);
cpumask_shift_left(&xlp_ites[node][3], &xlp_ites[node][3], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */
bm = 0x000000ff;
u32_to_cpumask(&xlp_ites[node][5], bm);
cpumask_shift_left(&xlp_ites[node][5], &xlp_ites[node][5], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */
bm = 0x000000f0;
u32_to_cpumask(&xlp_ites[node][6], bm);
cpumask_shift_left(&xlp_ites[node][6], &xlp_ites[node][6], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */
bm = 0x0000000f;
u32_to_cpumask(&xlp_ites[node][7], bm);
cpumask_shift_left(&xlp_ites[node][7], &xlp_ites[node][7], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */
}
for_each_online_node(node) {
cpumask_copy(&xlp_ites[node][4], &m);
}
// dump_all_ites();
}
int disable_nonboot_cpus(void)
{
int cpu, first_cpu, error = 0;
cpu_maps_update_begin();
first_cpu = cpumask_first(cpu_online_mask);
/*
* We take down all of the non-boot CPUs in one shot to avoid races
* with the userspace trying to use the CPU hotplug at the same time
*/
cpumask_clear(frozen_cpus);
arch_disable_nonboot_cpus_begin();
#ifdef CONFIG_DEBUG_PRINTK
printk("Disabling non-boot CPUs ...\n");
#else
;
#endif
for_each_online_cpu(cpu) {
if (cpu == first_cpu)
continue;
error = _cpu_down(cpu, 1);
if (!error)
cpumask_set_cpu(cpu, frozen_cpus);
else {
printk(KERN_ERR "Error taking CPU%d down: %d\n",
cpu, error);
break;
}
}
arch_disable_nonboot_cpus_end();
if (!error) {
BUG_ON(num_online_cpus() > 1);
/* Make sure the CPUs won't be enabled by someone else */
cpu_hotplug_disabled = 1;
} else {
printk(KERN_ERR "Non-boot CPUs are not disabled\n");
}
cpu_maps_update_done();
return error;
}
void irq_move_masked_irq(struct irq_data *idata)
{
struct irq_desc *desc = irq_data_to_desc(idata);
struct irq_chip *chip = desc->irq_data.chip;
if (likely(!irqd_is_setaffinity_pending(&desc->irq_data)))
return;
irqd_clr_move_pending(&desc->irq_data);
/*
* Paranoia: cpu-local interrupts shouldn't be calling in here anyway.
*/
if (irqd_is_per_cpu(&desc->irq_data)) {
WARN_ON(1);
return;
}
if (unlikely(cpumask_empty(desc->pending_mask)))
return;
if (!chip->irq_set_affinity)
return;
assert_raw_spin_locked(&desc->lock);
/*
* If there was a valid mask to work with, please
* do the disable, re-program, enable sequence.
* This is *not* particularly important for level triggered
* but in a edge trigger case, we might be setting rte
* when an active trigger is coming in. This could
* cause some ioapics to mal-function.
* Being paranoid i guess!
*
* For correct operation this depends on the caller
* masking the irqs.
*/
if (cpumask_any_and(desc->pending_mask, cpu_online_mask) < nr_cpu_ids)
irq_do_set_affinity(&desc->irq_data, desc->pending_mask, false);
cpumask_clear(desc->pending_mask);
}
void __init arch_init_irq(void)
{
int irq;
#ifdef CONFIG_SMP
/* Set the default affinity to the boot cpu. */
cpumask_clear(irq_default_affinity);
cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
#endif
if (NR_IRQS < OCTEON_IRQ_LAST)
pr_err("octeon_irq_init: NR_IRQS is set too low\n");
/* 0 - 15 reserved for i8259 master and slave controller. */
/* 17 - 23 Mips internal */
for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++) {
set_irq_chip_and_handler(irq, &octeon_irq_chip_core,
handle_percpu_irq);
}
/* 24 - 87 CIU_INT_SUM0 */
for (irq = OCTEON_IRQ_WORKQ0; irq <= OCTEON_IRQ_BOOTDMA; irq++) {
set_irq_chip_and_handler(irq, &octeon_irq_chip_ciu0,
handle_percpu_irq);
}
/* 88 - 151 CIU_INT_SUM1 */
for (irq = OCTEON_IRQ_WDOG0; irq <= OCTEON_IRQ_RESERVED151; irq++) {
set_irq_chip_and_handler(irq, &octeon_irq_chip_ciu1,
handle_percpu_irq);
}
#ifdef CONFIG_PCI_MSI
/* 152 - 215 PCI/PCIe MSI interrupts */
for (irq = OCTEON_IRQ_MSI_BIT0; irq <= OCTEON_IRQ_MSI_BIT63; irq++) {
set_irq_chip_and_handler(irq, &octeon_irq_chip_msi,
handle_percpu_irq);
}
#endif
set_c0_status(0x300 << 2);
}
static void run_parallel_many_CPUs_bulk(enum queue_behavior_type type,
uint32_t loops, int q_size, int prefill,
int CPUs, int bulk)
{
struct alf_queue *queue = NULL;
cpumask_t cpumask;
int i;
if (CPUs == 0)
return;
if (!(queue = alloc_and_init_queue(q_size, prefill)))
return; /* fail */
/* Restrict the CPUs to run on
*/
if (verbose)
pr_info("Limit to %d parallel CPUs (bulk:%d)\n", CPUs, bulk);
cpumask_clear(&cpumask);
for (i = 0; i < CPUs ; i++) {
cpumask_set_cpu(i, &cpumask);
}
if (type & SPSC) {
if (CPUs > 2) {
pr_err("%s() ERR SPSC does not support CPUs > 2\n",
__func__);
goto out;
}
run_parallel("alf_queue_BULK_SPSC_parallel_many_CPUs",
loops, &cpumask, bulk, queue,
time_bench_CPU_BULK_enq_or_deq_spsc);
} else if (type & MPMC) {
run_parallel("alf_queue_BULK_MPMC_parallel_many_CPUs",
loops, &cpumask, bulk, queue,
time_bench_CPU_BULK_enq_or_deq_mpmc);
} else {
pr_err("%s() WRONG TYPE!!! FIX\n", __func__);
}
out:
alf_queue_free(queue);
}
int disable_nonboot_cpus(void)
{
int cpu, first_cpu, error = 0;
cpu_maps_update_begin();
first_cpu = cpumask_first(cpu_online_mask);
/*
* We take down all of the non-boot CPUs in one shot to avoid races
* with the userspace trying to use the CPU hotplug at the same time
*/
cpumask_clear(frozen_cpus);
pr_info("Disabling non-boot CPUs ...\n");
for_each_online_cpu(cpu) {
if (cpu == first_cpu)
continue;
trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
error = _cpu_down(cpu, 1);
trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
if (!error)
cpumask_set_cpu(cpu, frozen_cpus);
else {
pr_err("Error taking CPU%d down: %d\n", cpu, error);
break;
}
}
if (!error)
BUG_ON(num_online_cpus() > 1);
else
pr_err("Non-boot CPUs are not disabled\n");
/*
* Make sure the CPUs won't be enabled by someone else. We need to do
* this even in case of failure as all disable_nonboot_cpus() users are
* supposed to do enable_nonboot_cpus() on the failure path.
*/
cpu_hotplug_disabled++;
cpu_maps_update_done();
return error;
}
/*
* Handle oneshot mode broadcasting
*/
static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
{
struct tick_device *td;
ktime_t now, next_event;
int cpu;
spin_lock(&tick_broadcast_lock);
again:
dev->next_event.tv64 = KTIME_MAX;
next_event.tv64 = KTIME_MAX;
cpumask_clear(to_cpumask(tmpmask));
now = ktime_get();
/* Find all expired events */
for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
td = &per_cpu(tick_cpu_device, cpu);
if (td->evtdev->next_event.tv64 <= now.tv64)
cpumask_set_cpu(cpu, to_cpumask(tmpmask));
else if (td->evtdev->next_event.tv64 < next_event.tv64)
next_event.tv64 = td->evtdev->next_event.tv64;
}
static ssize_t
mode_store(struct device *dev, struct device_attribute *attr,
const char *buf, size_t count)
{
if (!gbcl)
return -EPERM;
if (!strcmp(buf, "enable")) {
bcl_mode_set(BCL_DEVICE_ENABLED);
bcl_update_online_mask();
pr_info("bcl enabled\n");
} else if (!strcmp(buf, "disable")) {
bcl_mode_set(BCL_DEVICE_DISABLED);
cpumask_clear(bcl_cpu_online_mask);
pr_info("bcl disabled\n");
} else {
return -EINVAL;
}
return count;
}
static int dbg_set_cpu_affinity(const char *val, struct kernel_param *kp)
{
char *endptr;
pid_t pid;
int cpu;
struct cpumask mask;
long ret;
pid = (pid_t)memparse(val, &endptr);
if (*endptr != '@') {
pr_info("%s: invalid input strin: %s\n", __func__, val);
return -EINVAL;
}
cpu = memparse(++endptr, &endptr);
cpumask_clear(&mask);
cpumask_set_cpu(cpu, &mask);
pr_info("%s: Setting %d cpu affinity to cpu%d\n",
__func__, pid, cpu);
ret = sched_setaffinity(pid, &mask);
pr_info("%s: sched_setaffinity returned %ld\n", __func__, ret);
return 0;
}
static void round_robin_cpu(unsigned int tsk_index)
{
struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
cpumask_var_t tmp;
int cpu;
unsigned long min_weight = -1;
unsigned long uninitialized_var(preferred_cpu);
if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
return;
mutex_lock(&round_robin_lock);
cpumask_clear(tmp);
for_each_cpu(cpu, pad_busy_cpus)
cpumask_or(tmp, tmp, topology_thread_cpumask(cpu));
cpumask_andnot(tmp, cpu_online_mask, tmp);
/* avoid HT sibilings if possible */
if (cpumask_empty(tmp))
cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
if (cpumask_empty(tmp)) {
mutex_unlock(&round_robin_lock);
return;
}
for_each_cpu(cpu, tmp) {
if (cpu_weight[cpu] < min_weight) {
min_weight = cpu_weight[cpu];
preferred_cpu = cpu;
}
}
if (tsk_in_cpu[tsk_index] != -1)
cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
tsk_in_cpu[tsk_index] = preferred_cpu;
cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
cpu_weight[preferred_cpu]++;
mutex_unlock(&round_robin_lock);
set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
}
void noinline bench_outstanding_parallel_cpus(uint32_t loops, int nr_cpus,
int outstanding_pages)
{
const char *desc = "parallel_cpus";
struct time_bench_sync sync;
struct time_bench_cpu *cpu_tasks;
struct cpumask my_cpumask;
int i;
/* Allocate records for CPUs */
cpu_tasks = kzalloc(sizeof(*cpu_tasks) * nr_cpus, GFP_KERNEL);
/* Reduce number of CPUs to run on */
cpumask_clear(&my_cpumask);
for (i = 0; i < nr_cpus ; i++) {
cpumask_set_cpu(i, &my_cpumask);
}
pr_info("Limit to %d parallel CPUs\n", nr_cpus);
time_bench_run_concurrent(loops, outstanding_pages, NULL,
&my_cpumask, &sync, cpu_tasks,
time_alloc_pages_outstanding);
time_bench_print_stats_cpumask(desc, cpu_tasks, &my_cpumask);
kfree(cpu_tasks);
}
/* Return a mask of the cpus whose caches currently own these pages. */
static void homecache_mask(struct page *page, int pages,
struct cpumask *home_mask)
{
int i;
cpumask_clear(home_mask);
for (i = 0; i < pages; ++i) {
int home = page_home(&page[i]);
if (home == PAGE_HOME_IMMUTABLE ||
home == PAGE_HOME_INCOHERENT) {
cpumask_copy(home_mask, cpu_possible_mask);
return;
}
#if CHIP_HAS_CBOX_HOME_MAP()
if (home == PAGE_HOME_HASH) {
cpumask_or(home_mask, home_mask, &hash_for_home_map);
continue;
}
#endif
if (home == PAGE_HOME_UNCACHED)
continue;
BUG_ON(home < 0 || home >= NR_CPUS);
cpumask_set_cpu(home, home_mask);
}
}
static int bcl_probe(struct platform_device *pdev)
{
struct bcl_context *bcl = NULL;
int ret = 0;
enum bcl_device_mode bcl_mode = BCL_DEVICE_DISABLED;
bcl = devm_kzalloc(&pdev->dev, sizeof(struct bcl_context), GFP_KERNEL);
if (!bcl) {
pr_err("Cannot allocate bcl_context\n");
return -ENOMEM;
}
/* For BCL */
/* Init default BCL params */
if (of_property_read_bool(pdev->dev.of_node, "qcom,bcl-enable"))
bcl_mode = BCL_DEVICE_ENABLED;
else
bcl_mode = BCL_DEVICE_DISABLED;
bcl->bcl_mode = BCL_DEVICE_DISABLED;
bcl->dev = &pdev->dev;
bcl->bcl_monitor_type = BCL_IAVAIL_MONITOR_TYPE;
bcl->bcl_threshold_mode[BCL_LOW_THRESHOLD_TYPE] =
BCL_IAVAIL_THRESHOLD_DISABLED;
bcl->bcl_threshold_mode[BCL_HIGH_THRESHOLD_TYPE] =
BCL_IAVAIL_THRESHOLD_DISABLED;
bcl->bcl_threshold_value_ma[BCL_LOW_THRESHOLD_TYPE] = 0;
bcl->bcl_threshold_value_ma[BCL_HIGH_THRESHOLD_TYPE] = 0;
bcl->bcl_vbat_min = BATTERY_VOLTAGE_MIN;
snprintf(bcl->bcl_type, BCL_NAME_LENGTH, "%s",
bcl_type[BCL_IAVAIL_MONITOR_TYPE]);
bcl->bcl_poll_interval_msec = BCL_POLL_INTERVAL;
if (of_property_read_bool(pdev->dev.of_node, "qcom,bcl-no-bms"))
bcl->bcl_no_bms = true;
else
bcl->bcl_no_bms = false;
bcl_frequency_mask = get_mask_from_core_handle(pdev,
"qcom,bcl-freq-control-list");
bcl_hotplug_mask = get_mask_from_core_handle(pdev,
"qcom,bcl-hotplug-list");
bcl_soc_hotplug_mask = get_mask_from_core_handle(pdev,
"qcom,bcl-soc-hotplug-list");
if (!bcl_hotplug_mask && !bcl_soc_hotplug_mask)
bcl_hotplug_enabled = false;
else
bcl_hotplug_enabled = true;
if (of_property_read_bool(pdev->dev.of_node,
"qcom,bcl-framework-interface"))
ret = probe_bcl_periph_prop(bcl);
else
ret = probe_btm_properties(bcl);
if (ret == -EPROBE_DEFER)
return ret;
ret = create_bcl_sysfs(bcl);
if (ret < 0) {
pr_err("Cannot create bcl sysfs\n");
return ret;
}
cpumask_clear(bcl_cpu_online_mask);
bcl_psy.name = bcl_psy_name;
bcl_psy.type = POWER_SUPPLY_TYPE_BMS;
bcl_psy.get_property = bcl_battery_get_property;
bcl_psy.set_property = bcl_battery_set_property;
bcl_psy.num_properties = 0;
#ifndef CONFIG_LGE_PM
bcl_psy.external_power_changed = power_supply_callback;
#endif
gbcl = bcl;
platform_set_drvdata(pdev, bcl);
INIT_DEFERRABLE_WORK(&bcl->bcl_iavail_work, bcl_iavail_work);
INIT_WORK(&bcl_hotplug_work, bcl_handle_hotplug);
if (bcl_hotplug_enabled)
register_cpu_notifier(&bcl_cpu_notifier);
if (bcl_mode == BCL_DEVICE_ENABLED)
bcl_mode_set(bcl_mode);
return 0;
}
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
unsigned long charge;
struct mempolicy *pol;
down_write(&oldmm->mmap_sem);
flush_cache_dup_mm(oldmm);
/*
* Not linked in yet - no deadlock potential:
*/
down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
mm->locked_vm = 0;
mm->mmap = NULL;
mm->mmap_cache = NULL;
mm->free_area_cache = oldmm->mmap_base;
mm->cached_hole_size = ~0UL;
mm->map_count = 0;
cpumask_clear(mm_cpumask(mm));
mm->mm_rb = RB_ROOT;
rb_link = &mm->mm_rb.rb_node;
rb_parent = NULL;
pprev = &mm->mmap;
retval = ksm_fork(mm, oldmm);
if (retval)
goto out;
for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
struct file *file;
if (mpnt->vm_flags & VM_DONTCOPY) {
long pages = vma_pages(mpnt);
mm->total_vm -= pages;
vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
-pages);
continue;
}
charge = 0;
if (mpnt->vm_flags & VM_ACCOUNT) {
unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT;
if (security_vm_enough_memory(len))
goto fail_nomem;
charge = len;
}
tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
if (!tmp)
goto fail_nomem;
*tmp = *mpnt;
pol = mpol_dup(vma_policy(mpnt));
retval = PTR_ERR(pol);
if (IS_ERR(pol))
goto fail_nomem_policy;
vma_set_policy(tmp, pol);
tmp->vm_flags &= ~VM_LOCKED;
tmp->vm_mm = mm;
tmp->vm_next = NULL;
anon_vma_link(tmp);
file = tmp->vm_file;
if (file) {
struct inode *inode = file->f_path.dentry->d_inode;
struct address_space *mapping = file->f_mapping;
get_file(file);
if (tmp->vm_flags & VM_DENYWRITE)
atomic_dec(&inode->i_writecount);
spin_lock(&mapping->i_mmap_lock);
if (tmp->vm_flags & VM_SHARED)
mapping->i_mmap_writable++;
tmp->vm_truncate_count = mpnt->vm_truncate_count;
flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */
vma_prio_tree_add(tmp, mpnt);
flush_dcache_mmap_unlock(mapping);
spin_unlock(&mapping->i_mmap_lock);
}
/*
* Clear hugetlb-related page reserves for children. This only
* affects MAP_PRIVATE mappings. Faults generated by the child
* are not guaranteed to succeed, even if read-only
*/
if (is_vm_hugetlb_page(tmp))
reset_vma_resv_huge_pages(tmp);
/*
* Link in the new vma and copy the page table entries.
*/
*pprev = tmp;
pprev = &tmp->vm_next;
__vma_link_rb(mm, tmp, rb_link, rb_parent);
rb_link = &tmp->vm_rb.rb_right;
rb_parent = &tmp->vm_rb;
mm->map_count++;
retval = copy_page_range(mm, oldmm, mpnt);
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
if (retval)
goto out;
}
static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy)
{
unsigned int i;
unsigned int valid_states = 0;
unsigned int cpu = policy->cpu;
struct acpi_cpufreq_data *data;
unsigned int result = 0;
struct cpuinfo_x86 *c = &cpu_data(policy->cpu);
struct acpi_processor_performance *perf;
#ifdef CONFIG_SMP
static int blacklisted;
#endif
pr_debug("acpi_cpufreq_cpu_init\n");
#ifdef CONFIG_SMP
if (blacklisted)
return blacklisted;
blacklisted = acpi_cpufreq_blacklist(c);
if (blacklisted)
return blacklisted;
#endif
data = kzalloc(sizeof(*data), GFP_KERNEL);
if (!data)
return -ENOMEM;
if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) {
result = -ENOMEM;
goto err_free;
}
perf = per_cpu_ptr(acpi_perf_data, cpu);
data->acpi_perf_cpu = cpu;
policy->driver_data = data;
if (cpu_has(c, X86_FEATURE_CONSTANT_TSC))
acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS;
result = acpi_processor_register_performance(perf, cpu);
if (result)
goto err_free_mask;
policy->shared_type = perf->shared_type;
/*
* Will let policy->cpus know about dependency only when software
* coordination is required.
*/
if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL ||
policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) {
cpumask_copy(policy->cpus, perf->shared_cpu_map);
}
cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map);
#ifdef CONFIG_SMP
dmi_check_system(sw_any_bug_dmi_table);
if (bios_with_sw_any_bug && !policy_is_shared(policy)) {
policy->shared_type = CPUFREQ_SHARED_TYPE_ALL;
cpumask_copy(policy->cpus, topology_core_cpumask(cpu));
}
if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) {
cpumask_clear(policy->cpus);
cpumask_set_cpu(cpu, policy->cpus);
cpumask_copy(data->freqdomain_cpus,
topology_sibling_cpumask(cpu));
policy->shared_type = CPUFREQ_SHARED_TYPE_HW;
pr_info_once(PFX "overriding BIOS provided _PSD data\n");
}
#endif
/* capability check */
if (perf->state_count <= 1) {
pr_debug("No P-States\n");
result = -ENODEV;
goto err_unreg;
}
if (perf->control_register.space_id != perf->status_register.space_id) {
result = -ENODEV;
goto err_unreg;
}
switch (perf->control_register.space_id) {
case ACPI_ADR_SPACE_SYSTEM_IO:
if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
boot_cpu_data.x86 == 0xf) {
pr_debug("AMD K8 systems must use native drivers.\n");
result = -ENODEV;
goto err_unreg;
}
pr_debug("SYSTEM IO addr space\n");
data->cpu_feature = SYSTEM_IO_CAPABLE;
break;
case ACPI_ADR_SPACE_FIXED_HARDWARE:
pr_debug("HARDWARE addr space\n");
if (check_est_cpu(cpu)) {
data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE;
break;
}
if (check_amd_hwpstate_cpu(cpu)) {
data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE;
break;
}
result = -ENODEV;
goto err_unreg;
default:
pr_debug("Unknown addr space %d\n",
(u32) (perf->control_register.space_id));
result = -ENODEV;
goto err_unreg;
}
data->freq_table = kzalloc(sizeof(*data->freq_table) *
(perf->state_count+1), GFP_KERNEL);
if (!data->freq_table) {
result = -ENOMEM;
goto err_unreg;
}
/* detect transition latency */
policy->cpuinfo.transition_latency = 0;
for (i = 0; i < perf->state_count; i++) {
if ((perf->states[i].transition_latency * 1000) >
policy->cpuinfo.transition_latency)
policy->cpuinfo.transition_latency =
perf->states[i].transition_latency * 1000;
}
/* Check for high latency (>20uS) from buggy BIOSes, like on T42 */
if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE &&
policy->cpuinfo.transition_latency > 20 * 1000) {
policy->cpuinfo.transition_latency = 20 * 1000;
printk_once(KERN_INFO
"P-state transition latency capped at 20 uS\n");
}
/* table init */
for (i = 0; i < perf->state_count; i++) {
if (i > 0 && perf->states[i].core_frequency >=
data->freq_table[valid_states-1].frequency / 1000)
continue;
data->freq_table[valid_states].driver_data = i;
data->freq_table[valid_states].frequency =
perf->states[i].core_frequency * 1000;
valid_states++;
}
data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END;
perf->state = 0;
result = cpufreq_table_validate_and_show(policy, data->freq_table);
if (result)
goto err_freqfree;
if (perf->states[0].core_frequency * 1000 != policy->cpuinfo.max_freq)
printk(KERN_WARNING FW_WARN "P-state 0 is not max freq\n");
switch (perf->control_register.space_id) {
case ACPI_ADR_SPACE_SYSTEM_IO:
/*
* The core will not set policy->cur, because
* cpufreq_driver->get is NULL, so we need to set it here.
* However, we have to guess it, because the current speed is
* unknown and not detectable via IO ports.
*/
policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu);
break;
case ACPI_ADR_SPACE_FIXED_HARDWARE:
acpi_cpufreq_driver.get = get_cur_freq_on_cpu;
break;
default:
break;
}
/* notify BIOS that we exist */
acpi_processor_notify_smm(THIS_MODULE);
pr_debug("CPU%u - ACPI performance management activated.\n", cpu);
for (i = 0; i < perf->state_count; i++)
pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n",
(i == perf->state ? '*' : ' '), i,
(u32) perf->states[i].core_frequency,
(u32) perf->states[i].power,
(u32) perf->states[i].transition_latency);
/*
* the first call to ->target() should result in us actually
* writing something to the appropriate registers.
*/
data->resume = 1;
return result;
err_freqfree:
kfree(data->freq_table);
err_unreg:
acpi_processor_unregister_performance(cpu);
err_free_mask:
free_cpumask_var(data->freqdomain_cpus);
err_free:
kfree(data);
policy->driver_data = NULL;
return result;
}
/*
* This wrapper function around hv_flush_remote() does several things:
*
* - Provides a return value error-checking panic path, since
* there's never any good reason for hv_flush_remote() to fail.
* - Accepts a 32-bit PFN rather than a 64-bit PA, which generally
* is the type that Linux wants to pass around anyway.
* - Centralizes the mark_caches_evicted() handling.
* - Canonicalizes that lengths of zero make cpumasks NULL.
* - Handles deferring TLB flushes for dataplane tiles.
* - Tracks remote interrupts in the per-cpu irq_cpustat_t.
*
* Note that we have to wait until the cache flush completes before
* updating the per-cpu last_cache_flush word, since otherwise another
* concurrent flush can race, conclude the flush has already
* completed, and start to use the page while it's still dirty
* remotely (running concurrently with the actual evict, presumably).
*/
void flush_remote(unsigned long cache_pfn, unsigned long cache_control,
const struct cpumask *cache_cpumask_orig,
HV_VirtAddr tlb_va, unsigned long tlb_length,
unsigned long tlb_pgsize,
const struct cpumask *tlb_cpumask_orig,
HV_Remote_ASID *asids, int asidcount)
{
int rc;
int timestamp = 0; /* happy compiler */
struct cpumask cache_cpumask_copy, tlb_cpumask_copy;
struct cpumask *cache_cpumask, *tlb_cpumask;
HV_PhysAddr cache_pa;
char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5];
mb(); /* provided just to simplify "magic hypervisor" mode */
/*
* Canonicalize and copy the cpumasks.
*/
if (cache_cpumask_orig && cache_control) {
cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig);
cache_cpumask = &cache_cpumask_copy;
} else {
cpumask_clear(&cache_cpumask_copy);
cache_cpumask = NULL;
}
if (cache_cpumask == NULL)
cache_control = 0;
if (tlb_cpumask_orig && tlb_length) {
cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig);
tlb_cpumask = &tlb_cpumask_copy;
} else {
cpumask_clear(&tlb_cpumask_copy);
tlb_cpumask = NULL;
}
hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length,
asids, asidcount);
cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT;
if (cache_control & HV_FLUSH_EVICT_L2)
timestamp = mark_caches_evicted_start();
rc = hv_flush_remote(cache_pa, cache_control,
cpumask_bits(cache_cpumask),
tlb_va, tlb_length, tlb_pgsize,
cpumask_bits(tlb_cpumask),
asids, asidcount);
if (cache_control & HV_FLUSH_EVICT_L2)
mark_caches_evicted_finish(cache_cpumask, timestamp);
if (rc == 0)
return;
cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy);
cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy);
pr_err("hv_flush_remote(%#llx, %#lx, %p [%s],"
" %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n",
cache_pa, cache_control, cache_cpumask, cache_buf,
(unsigned long)tlb_va, tlb_length, tlb_pgsize,
tlb_cpumask, tlb_buf,
asids, asidcount, rc);
panic("Unsafe to continue.");
}
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
{
struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
struct rb_node **rb_link, *rb_parent;
int retval;
unsigned long charge;
uprobe_start_dup_mmap();
down_write(&oldmm->mmap_sem);
flush_cache_dup_mm(oldmm);
uprobe_dup_mmap(oldmm, mm);
/*
* Not linked in yet - no deadlock potential:
*/
down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING);
mm->locked_vm = 0;
mm->mmap = NULL;
mm->mmap_cache = NULL;
mm->map_count = 0;
cpumask_clear(mm_cpumask(mm));
mm->mm_rb = RB_ROOT;
rb_link = &mm->mm_rb.rb_node;
rb_parent = NULL;
pprev = &mm->mmap;
retval = ksm_fork(mm, oldmm);
if (retval)
goto out;
retval = khugepaged_fork(mm, oldmm);
if (retval)
goto out;
prev = NULL;
for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
struct file *file;
if (mpnt->vm_flags & VM_DONTCOPY) {
vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file,
-vma_pages(mpnt));
continue;
}
charge = 0;
if (mpnt->vm_flags & VM_ACCOUNT) {
unsigned long len = vma_pages(mpnt);
if (security_vm_enough_memory_mm(oldmm, len)) /* sic */
goto fail_nomem;
charge = len;
}
tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
if (!tmp)
goto fail_nomem;
*tmp = *mpnt;
INIT_LIST_HEAD(&tmp->anon_vma_chain);
retval = vma_dup_policy(mpnt, tmp);
if (retval)
goto fail_nomem_policy;
tmp->vm_mm = mm;
if (anon_vma_fork(tmp, mpnt))
goto fail_nomem_anon_vma_fork;
tmp->vm_flags &= ~VM_LOCKED;
tmp->vm_next = tmp->vm_prev = NULL;
file = tmp->vm_file;
if (file) {
struct inode *inode = file_inode(file);
struct address_space *mapping = file->f_mapping;
get_file(file);
if (tmp->vm_flags & VM_DENYWRITE)
atomic_dec(&inode->i_writecount);
mutex_lock(&mapping->i_mmap_mutex);
if (tmp->vm_flags & VM_SHARED)
mapping->i_mmap_writable++;
flush_dcache_mmap_lock(mapping);
/* insert tmp into the share list, just after mpnt */
if (unlikely(tmp->vm_flags & VM_NONLINEAR))
vma_nonlinear_insert(tmp,
&mapping->i_mmap_nonlinear);
else
vma_interval_tree_insert_after(tmp, mpnt,
&mapping->i_mmap);
flush_dcache_mmap_unlock(mapping);
mutex_unlock(&mapping->i_mmap_mutex);
}
/*
* Clear hugetlb-related page reserves for children. This only
* affects MAP_PRIVATE mappings. Faults generated by the child
* are not guaranteed to succeed, even if read-only
*/
if (is_vm_hugetlb_page(tmp))
reset_vma_resv_huge_pages(tmp);
/*
* Link in the new vma and copy the page table entries.
*/
*pprev = tmp;
pprev = &tmp->vm_next;
tmp->vm_prev = prev;
prev = tmp;
__vma_link_rb(mm, tmp, rb_link, rb_parent);
rb_link = &tmp->vm_rb.rb_right;
rb_parent = &tmp->vm_rb;
mm->map_count++;
retval = copy_page_range(mm, oldmm, mpnt);
if (tmp->vm_ops && tmp->vm_ops->open)
tmp->vm_ops->open(tmp);
if (retval)
goto out;
}
/* a new mm has just been created */
arch_dup_mmap(oldmm, mm);
retval = 0;
out:
up_write(&mm->mmap_sem);
flush_tlb_mm(oldmm);
up_write(&oldmm->mmap_sem);
uprobe_end_dup_mmap();
return retval;
fail_nomem_anon_vma_fork:
mpol_put(vma_policy(tmp));
fail_nomem_policy:
kmem_cache_free(vm_area_cachep, tmp);
fail_nomem:
retval = -ENOMEM;
vm_unacct_memory(charge);
goto out;
}
/*
* Handle oneshot mode broadcasting
*/
static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
{
struct tick_device *td;
ktime_t now, next_event;
int cpu, next_cpu = 0;
bool bc_local;
raw_spin_lock(&tick_broadcast_lock);
dev->next_event = KTIME_MAX;
next_event = KTIME_MAX;
cpumask_clear(tmpmask);
now = ktime_get();
/* Find all expired events */
for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
/*
* Required for !SMP because for_each_cpu() reports
* unconditionally CPU0 as set on UP kernels.
*/
if (!IS_ENABLED(CONFIG_SMP) &&
cpumask_empty(tick_broadcast_oneshot_mask))
break;
td = &per_cpu(tick_cpu_device, cpu);
if (td->evtdev->next_event <= now) {
cpumask_set_cpu(cpu, tmpmask);
/*
* Mark the remote cpu in the pending mask, so
* it can avoid reprogramming the cpu local
* timer in tick_broadcast_oneshot_control().
*/
cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
} else if (td->evtdev->next_event < next_event) {
next_event = td->evtdev->next_event;
next_cpu = cpu;
}
}
/*
* Remove the current cpu from the pending mask. The event is
* delivered immediately in tick_do_broadcast() !
*/
cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
/* Take care of enforced broadcast requests */
cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
cpumask_clear(tick_broadcast_force_mask);
/*
* Sanity check. Catch the case where we try to broadcast to
* offline cpus.
*/
if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
cpumask_and(tmpmask, tmpmask, cpu_online_mask);
/*
* Wakeup the cpus which have an expired event.
*/
bc_local = tick_do_broadcast(tmpmask);
/*
* Two reasons for reprogram:
*
* - The global event did not expire any CPU local
* events. This happens in dyntick mode, as the maximum PIT
* delta is quite small.
*
* - There are pending events on sleeping CPUs which were not
* in the event mask
*/
if (next_event != KTIME_MAX)
tick_broadcast_set_event(dev, next_cpu, next_event);
raw_spin_unlock(&tick_broadcast_lock);
if (bc_local) {
td = this_cpu_ptr(&tick_cpu_device);
td->evtdev->event_handler(td->evtdev);
}
}