static void flat_vector_allocation_domain(int cpu, struct cpumask *retmask) { /* Careful. Some cpus do not strictly honor the set of cpus * specified in the interrupt destination when using lowest * priority interrupt delivery mode. * * In particular there was a hyperthreading cpu observed to * deliver interrupts to the wrong hyperthread when only one * hyperthread was specified in the interrupt desitination. */ cpumask_clear(retmask); cpumask_bits(retmask)[0] = APIC_ALL_CPUS; }
int config_L2(int size) { int i; struct cpumask mask; int cur_size = get_l2c_size(); if (size != SZ_256K && size != SZ_512K) { printk("inlvalid input size %x\n", size); return -1; } if (in_interrupt()) { printk(KERN_ERR "Cannot use %s in interrupt/softirq context\n", __func__); return -1; } if (size == cur_size) { printk("Config L2 size %x is equal to current L2 size %x\n", size, cur_size); return 0; } cpumask_clear(&mask); for(i = 0; i < get_cluster_core_count(); i++) cpumask_set_cpu(i, &mask); atomic_set(&L1_flush_done, 0); get_online_cpus(); //printk("[Config L2] Config L2 start, on line cpu = %d\n",num_online_cpus()); /* disable cache and flush L1 on Cluster0*/ on_each_cpu_mask(&mask, (smp_call_func_t)atomic_flush, NULL, true); //while(atomic_read(&L1_flush_done) != num_online_cpus()); //printk("[Config L2] L1 flush done\n"); /* Only need to flush Cluster0's L2 */ smp_call_function_any(&mask, (smp_call_func_t)inner_dcache_flush_L2, NULL, true); //printk("[Config L2] L2 flush done\n"); /* change L2 size */ config_L2_size(size); //printk("[Config L2] Change L2 flush size done(size = %d)\n",size); /* enable Cluster0's cache */ atomic_set(&L1_flush_done, 0); on_each_cpu_mask(&mask, (smp_call_func_t)__enable_cache, NULL, true); //update cr_alignment for other kernel function usage cr_alignment = cr_alignment | (0x4); //C1_CBIT put_online_cpus(); printk("Config L2 size %x done\n", size); return 0; }
void move_masked_irq(int irq) { struct irq_desc *desc = irq_to_desc(irq); if (likely(!(desc->status & IRQ_MOVE_PENDING))) return; /* * Paranoia: cpu-local interrupts shouldn't be calling in here anyway. */ if (CHECK_IRQ_PER_CPU(desc->status)) { WARN_ON(1); return; } desc->status &= ~IRQ_MOVE_PENDING; if (unlikely(cpumask_empty(desc->pending_mask))) return; if (!desc->chip->set_affinity) return; assert_spin_locked(&desc->lock); /* * If there was a valid mask to work with, please * do the disable, re-program, enable sequence. * This is *not* particularly important for level triggered * but in a edge trigger case, we might be setting rte * when an active trigger is comming in. This could * cause some ioapics to mal-function. * Being paranoid i guess! * * For correct operation this depends on the caller * masking the irqs. */ if (likely(cpumask_any_and(desc->pending_mask, cpu_online_mask) < nr_cpu_ids)) { int ret = chip->irq_set_affinity(irq, desc->pending_mask); switch (ret) { case IRQ_SET_MASK_OK: cpumask_copy(desc->affinity, desc->pending_mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); } } cpumask_clear(desc->pending_mask); }
static void desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { if (!affinity) affinity = irq_default_affinity; cpumask_copy(desc->irq_common_data.affinity, affinity); #ifdef CONFIG_GENERIC_PENDING_IRQ cpumask_clear(desc->pending_mask); #endif #ifdef CONFIG_NUMA desc->irq_common_data.node = node; #endif }
/* * In XLP cpu mask for setting affinity of an interrupt cannot span multiple * nodes. Although this is not a h/w restriction, the effort to implement * this feature does not justify the potential benefit; not only that handling * non local interrupts are slightly slower, it could be expensive in terms of * memory access and other resource utilization * * @node : node to which mask `mask` to be restricted * @src : mask to restrict * @dst : restricted mask (result) */ void constrict_mask_to_node(u8 node, struct cpumask *dst, const struct cpumask *src) { // char buf[140]; int i; cpumask_clear(dst); for (i = NLM_MAX_CPU_PER_NODE * node; i < (NLM_MAX_CPU_PER_NODE *(node + 1)); i++) { cpumask_set_cpu(i, dst); } cpumask_and(dst, dst, &phys_cpu_present_map); cpumask_and(dst, dst, src); return; }
static int set_cpu_min_freq(const char *buf, const struct kernel_param *kp) { int i, j, ntokens = 0; unsigned int val, cpu; const char *cp = buf; struct cpu_status *i_cpu_stats; struct cpufreq_policy policy; cpumask_var_t limit_mask; int ret; while ((cp = strpbrk(cp + 1, " :"))) ntokens++; if (!(ntokens % 2)) return -EINVAL; cp = buf; cpumask_clear(limit_mask); for (i = 0; i < ntokens; i += 2) { if (sscanf(cp, "%u:%u", &cpu, &val) != 2) return -EINVAL; if (cpu > (num_present_cpus() - 1)) return -EINVAL; i_cpu_stats = &per_cpu(cpu_stats, cpu); i_cpu_stats->min = val; cpumask_set_cpu(cpu, limit_mask); cp = strchr(cp, ' '); cp++; } get_online_cpus(); for_each_cpu(i, limit_mask) { i_cpu_stats = &per_cpu(cpu_stats, i); if (cpufreq_get_policy(&policy, i)) continue; if (cpu_online(i) && (policy.min != i_cpu_stats->min)) { ret = cpufreq_update_policy(i); if (ret) continue; } for_each_cpu(j, policy.related_cpus) cpumask_clear_cpu(j, limit_mask); }
// ARM10C 20141004 // desc: kmem_cache#28-o0, node: 0 static void desc_smp_init(struct irq_desc *desc, int node) { // desc->irq_data.node: (kmem_cache#28-o0)->irq_data.node, node: 0 desc->irq_data.node = node; // desc->irq_data.node: (kmem_cache#28-o0)->irq_data.node: 0 // desc->irq_data.affinity: (kmem_cache#28-o0)->irq_data.affinity, // irq_default_affinity->bits[0]: 0xF cpumask_copy(desc->irq_data.affinity, irq_default_affinity); // desc->irq_data.affinity: (kmem_cache#28-o0)->irq_data.affinity.bits[0]: 0xF #ifdef CONFIG_GENERIC_PENDING_IRQ // CONFIG_GENERIC_PENDING_IRQ=n cpumask_clear(desc->pending_mask); #endif }
static int __stp_alloc_ring_buffer(void) { int i; unsigned long buffer_size = _stp_bufsize * 1024 * 1024; if (!alloc_cpumask_var(&_stp_relay_data.trace_reader_cpumask, (GFP_KERNEL & ~__GFP_WAIT))) goto fail; cpumask_clear(_stp_relay_data.trace_reader_cpumask); if (buffer_size == 0) { dbug_trans(1, "using default buffer size...\n"); buffer_size = _stp_nsubbufs * _stp_subbuf_size; } dbug_trans(1, "using buffer size %lu...\n", buffer_size); /* The number passed to ring_buffer_alloc() is per cpu. Our * 'buffer_size' is a total number of bytes to allocate. So, * we need to divide buffer_size by the number of cpus. */ buffer_size /= num_online_cpus(); dbug_trans(1, "%lu\n", buffer_size); _stp_relay_data.rb = ring_buffer_alloc(buffer_size, 0); if (!_stp_relay_data.rb) goto fail; /* Increment _stp_allocated_memory and _stp_allocated_net_memory to approximately account for buffers allocated by ring_buffer_alloc. */ { #ifndef DIV_ROUND_UP #define DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d)) #endif u64 relay_pages; relay_pages = DIV_ROUND_UP (buffer_size, PAGE_SIZE); if (relay_pages < 2) relay_pages = 2; relay_pages *= num_online_cpus(); _stp_allocated_net_memory += relay_pages * PAGE_SIZE; _stp_allocated_memory += relay_pages * PAGE_SIZE; } dbug_trans(0, "size = %lu\n", ring_buffer_size(_stp_relay_data.rb)); return 0; fail: __stp_free_ring_buffer(); return -ENOMEM; }
/* * Allocate node_to_cpumask_map based on number of available nodes * Requires node_possible_map to be valid. * * Note: cpumask_of_node() is not valid until after this is done. * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.) */ static void __init setup_node_to_cpumask_map(void) { int node; /* setup nr_node_ids if not done yet */ if (nr_node_ids == MAX_NUMNODES) setup_nr_node_ids(); /* allocate and clear the mapping */ for (node = 0; node < nr_node_ids; node++) { alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); cpumask_clear(node_to_cpumask_map[node]); } /* cpumask_of_node() will now work */ pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids); }
static void crash_kexec_prepare_cpus(int cpu) { unsigned int msecs; unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */ crash_send_ipi(crash_ipi_callback); smp_wmb(); /* * FIXME: Until we will have the way to stop other CPUs reliably, * the crash CPU will send an IPI and wait for other CPUs to * respond. * Delay of at least 10 seconds. */ printk(KERN_EMERG "Sending IPI to other cpus...\n"); msecs = 10000; while ((cpumask_weight(&cpus_in_crash) < ncpus) && (--msecs > 0)) { cpu_relax(); mdelay(1); } /* Would it be better to replace the trap vector here? */ /* * FIXME: In case if we do not get all CPUs, one possibility: ask the * user to do soft reset such that we get all. * Soft-reset will be used until better mechanism is implemented. */ if (cpumask_weight(&cpus_in_crash) < ncpus) { printk(KERN_EMERG "done waiting: %d cpu(s) not responding\n", ncpus - cpumask_weight(&cpus_in_crash)); printk(KERN_EMERG "Activate soft-reset to stop other cpu(s)\n"); cpumask_clear(&cpus_in_sr); atomic_set(&enter_on_soft_reset, 0); while (cpumask_weight(&cpus_in_crash) < ncpus) cpu_relax(); } /* * Make sure all CPUs are entered via soft-reset if the kdump is * invoked using soft-reset. */ if (cpumask_test_cpu(cpu, &cpus_in_sr)) crash_soft_reset_check(cpu); /* Leave the IPI callback set */ }
static void tzdev_migrate_threads(int cpu) { struct task_struct *thread; cpumask_t next_cpumask; cpumask_clear(&next_cpumask); cpumask_set_cpu(cpu, &next_cpumask); while ((thread = tzdev_get_next_thread(&next_cpumask))) { pr_notice("Migrate thread pid = %d to cpu = %d\n", thread->pid, cpu); /* We shouldn't fail here because of we wrap this code by * get_online_cpus() / put_online_cpus() */ BUG_ON(set_cpus_allowed(thread, next_cpumask)); put_task_struct(thread); } BUG_ON(set_cpus_allowed(current, next_cpumask)); }
int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues, const struct cpumask *online_mask) { unsigned int i, nr_cpus, nr_uniq_cpus, queue, first_sibling; cpumask_var_t cpus; if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) return 1; cpumask_clear(cpus); nr_cpus = nr_uniq_cpus = 0; for_each_cpu(i, online_mask) { nr_cpus++; first_sibling = get_first_sibling(i); if (!cpumask_test_cpu(first_sibling, cpus)) nr_uniq_cpus++; cpumask_set_cpu(i, cpus); }
/* * Initializes PIC ITE entries PRM 9.5.6.26 * XLP restricts CPU affinity to 8 groups. Though configurable, they are * programmed to have the following patterns. * 0 => Only 0th cpu on the node * 1 => All local threads in node; mask = (0xffffffff) on node * 2 => cpu0-15 on node; mask = 0x0000ffff & online_cpu_mask on nodes * 3 => cpu15-31 on node; mask = 0xffff0000 & online_cpu_mask on node * 4 => All cpus on all nodes; i.e., * mask = (0xffffffff_ffffffff_ffffffff_ffffffff & physical online cpu map) * These are programmer defined groups and can be changed as warranted. * Added 5 => CPUs 0-11 * Added 6 => CPUs 0-7 * Added 7 => CPUs 0-3 * Actual programmed value will take into consideration cpu_online_mask. * * There is a major issue that needs addressing when run in multi node mode * Number of nodes must be determined and programmed correctly, if a bit in ITE * is programmed without physical thread being present, when interrupt is * dispatched to that CPU under global scheme, system would hang. Thus this * scenario should be avoided. That is why phys_cpu_present_map is used * * This function simply initializes the xlp_ites entries with proposed * CPUmasks. */ static void xlp_ites_init(void) { u64 bm = 0x1; u8 node; struct cpumask m; cpumask_clear(&m); for_each_online_node(node) { /* Simply set the static pattern in all */ bm = 1; u32_to_cpumask(&xlp_ites[node][0], bm); cpumask_shift_left(&xlp_ites[node][0], &xlp_ites[node][0], NLM_MAX_CPU_PER_NODE * node); /* directs only to cpu0 of node `node` */ bm = 0xffffffff; u32_to_cpumask(&xlp_ites[node][1], bm); cpumask_shift_left(&xlp_ites[node][1], &xlp_ites[node][1], NLM_MAX_CPU_PER_NODE * node); /* directs to all cpus of node `node` */ cpumask_or(&m, &m, &xlp_ites[node][1]); bm = 0x0000ffff; u32_to_cpumask(&xlp_ites[node][2], bm); cpumask_shift_left(&xlp_ites[node][2], &xlp_ites[node][2], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */ bm = 0xffff0000; u32_to_cpumask(&xlp_ites[node][3], bm); cpumask_shift_left(&xlp_ites[node][3], &xlp_ites[node][3], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */ bm = 0x000000ff; u32_to_cpumask(&xlp_ites[node][5], bm); cpumask_shift_left(&xlp_ites[node][5], &xlp_ites[node][5], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */ bm = 0x000000f0; u32_to_cpumask(&xlp_ites[node][6], bm); cpumask_shift_left(&xlp_ites[node][6], &xlp_ites[node][6], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */ bm = 0x0000000f; u32_to_cpumask(&xlp_ites[node][7], bm); cpumask_shift_left(&xlp_ites[node][7], &xlp_ites[node][7], NLM_MAX_CPU_PER_NODE * node); /* directs to specified cpus of node `node` */ } for_each_online_node(node) { cpumask_copy(&xlp_ites[node][4], &m); } // dump_all_ites(); }
int disable_nonboot_cpus(void) { int cpu, first_cpu, error = 0; cpu_maps_update_begin(); first_cpu = cpumask_first(cpu_online_mask); /* * We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time */ cpumask_clear(frozen_cpus); arch_disable_nonboot_cpus_begin(); #ifdef CONFIG_DEBUG_PRINTK printk("Disabling non-boot CPUs ...\n"); #else ; #endif for_each_online_cpu(cpu) { if (cpu == first_cpu) continue; error = _cpu_down(cpu, 1); if (!error) cpumask_set_cpu(cpu, frozen_cpus); else { printk(KERN_ERR "Error taking CPU%d down: %d\n", cpu, error); break; } } arch_disable_nonboot_cpus_end(); if (!error) { BUG_ON(num_online_cpus() > 1); /* Make sure the CPUs won't be enabled by someone else */ cpu_hotplug_disabled = 1; } else { printk(KERN_ERR "Non-boot CPUs are not disabled\n"); } cpu_maps_update_done(); return error; }
void irq_move_masked_irq(struct irq_data *idata) { struct irq_desc *desc = irq_data_to_desc(idata); struct irq_chip *chip = desc->irq_data.chip; if (likely(!irqd_is_setaffinity_pending(&desc->irq_data))) return; irqd_clr_move_pending(&desc->irq_data); /* * Paranoia: cpu-local interrupts shouldn't be calling in here anyway. */ if (irqd_is_per_cpu(&desc->irq_data)) { WARN_ON(1); return; } if (unlikely(cpumask_empty(desc->pending_mask))) return; if (!chip->irq_set_affinity) return; assert_raw_spin_locked(&desc->lock); /* * If there was a valid mask to work with, please * do the disable, re-program, enable sequence. * This is *not* particularly important for level triggered * but in a edge trigger case, we might be setting rte * when an active trigger is coming in. This could * cause some ioapics to mal-function. * Being paranoid i guess! * * For correct operation this depends on the caller * masking the irqs. */ if (cpumask_any_and(desc->pending_mask, cpu_online_mask) < nr_cpu_ids) irq_do_set_affinity(&desc->irq_data, desc->pending_mask, false); cpumask_clear(desc->pending_mask); }
void __init arch_init_irq(void) { int irq; #ifdef CONFIG_SMP /* Set the default affinity to the boot cpu. */ cpumask_clear(irq_default_affinity); cpumask_set_cpu(smp_processor_id(), irq_default_affinity); #endif if (NR_IRQS < OCTEON_IRQ_LAST) pr_err("octeon_irq_init: NR_IRQS is set too low\n"); /* 0 - 15 reserved for i8259 master and slave controller. */ /* 17 - 23 Mips internal */ for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++) { set_irq_chip_and_handler(irq, &octeon_irq_chip_core, handle_percpu_irq); } /* 24 - 87 CIU_INT_SUM0 */ for (irq = OCTEON_IRQ_WORKQ0; irq <= OCTEON_IRQ_BOOTDMA; irq++) { set_irq_chip_and_handler(irq, &octeon_irq_chip_ciu0, handle_percpu_irq); } /* 88 - 151 CIU_INT_SUM1 */ for (irq = OCTEON_IRQ_WDOG0; irq <= OCTEON_IRQ_RESERVED151; irq++) { set_irq_chip_and_handler(irq, &octeon_irq_chip_ciu1, handle_percpu_irq); } #ifdef CONFIG_PCI_MSI /* 152 - 215 PCI/PCIe MSI interrupts */ for (irq = OCTEON_IRQ_MSI_BIT0; irq <= OCTEON_IRQ_MSI_BIT63; irq++) { set_irq_chip_and_handler(irq, &octeon_irq_chip_msi, handle_percpu_irq); } #endif set_c0_status(0x300 << 2); }
static void run_parallel_many_CPUs_bulk(enum queue_behavior_type type, uint32_t loops, int q_size, int prefill, int CPUs, int bulk) { struct alf_queue *queue = NULL; cpumask_t cpumask; int i; if (CPUs == 0) return; if (!(queue = alloc_and_init_queue(q_size, prefill))) return; /* fail */ /* Restrict the CPUs to run on */ if (verbose) pr_info("Limit to %d parallel CPUs (bulk:%d)\n", CPUs, bulk); cpumask_clear(&cpumask); for (i = 0; i < CPUs ; i++) { cpumask_set_cpu(i, &cpumask); } if (type & SPSC) { if (CPUs > 2) { pr_err("%s() ERR SPSC does not support CPUs > 2\n", __func__); goto out; } run_parallel("alf_queue_BULK_SPSC_parallel_many_CPUs", loops, &cpumask, bulk, queue, time_bench_CPU_BULK_enq_or_deq_spsc); } else if (type & MPMC) { run_parallel("alf_queue_BULK_MPMC_parallel_many_CPUs", loops, &cpumask, bulk, queue, time_bench_CPU_BULK_enq_or_deq_mpmc); } else { pr_err("%s() WRONG TYPE!!! FIX\n", __func__); } out: alf_queue_free(queue); }
int disable_nonboot_cpus(void) { int cpu, first_cpu, error = 0; cpu_maps_update_begin(); first_cpu = cpumask_first(cpu_online_mask); /* * We take down all of the non-boot CPUs in one shot to avoid races * with the userspace trying to use the CPU hotplug at the same time */ cpumask_clear(frozen_cpus); pr_info("Disabling non-boot CPUs ...\n"); for_each_online_cpu(cpu) { if (cpu == first_cpu) continue; trace_suspend_resume(TPS("CPU_OFF"), cpu, true); error = _cpu_down(cpu, 1); trace_suspend_resume(TPS("CPU_OFF"), cpu, false); if (!error) cpumask_set_cpu(cpu, frozen_cpus); else { pr_err("Error taking CPU%d down: %d\n", cpu, error); break; } } if (!error) BUG_ON(num_online_cpus() > 1); else pr_err("Non-boot CPUs are not disabled\n"); /* * Make sure the CPUs won't be enabled by someone else. We need to do * this even in case of failure as all disable_nonboot_cpus() users are * supposed to do enable_nonboot_cpus() on the failure path. */ cpu_hotplug_disabled++; cpu_maps_update_done(); return error; }
/* * Handle oneshot mode broadcasting */ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) { struct tick_device *td; ktime_t now, next_event; int cpu; spin_lock(&tick_broadcast_lock); again: dev->next_event.tv64 = KTIME_MAX; next_event.tv64 = KTIME_MAX; cpumask_clear(to_cpumask(tmpmask)); now = ktime_get(); /* Find all expired events */ for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) { td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event.tv64 <= now.tv64) cpumask_set_cpu(cpu, to_cpumask(tmpmask)); else if (td->evtdev->next_event.tv64 < next_event.tv64) next_event.tv64 = td->evtdev->next_event.tv64; }
static ssize_t mode_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { if (!gbcl) return -EPERM; if (!strcmp(buf, "enable")) { bcl_mode_set(BCL_DEVICE_ENABLED); bcl_update_online_mask(); pr_info("bcl enabled\n"); } else if (!strcmp(buf, "disable")) { bcl_mode_set(BCL_DEVICE_DISABLED); cpumask_clear(bcl_cpu_online_mask); pr_info("bcl disabled\n"); } else { return -EINVAL; } return count; }
static int dbg_set_cpu_affinity(const char *val, struct kernel_param *kp) { char *endptr; pid_t pid; int cpu; struct cpumask mask; long ret; pid = (pid_t)memparse(val, &endptr); if (*endptr != '@') { pr_info("%s: invalid input strin: %s\n", __func__, val); return -EINVAL; } cpu = memparse(++endptr, &endptr); cpumask_clear(&mask); cpumask_set_cpu(cpu, &mask); pr_info("%s: Setting %d cpu affinity to cpu%d\n", __func__, pid, cpu); ret = sched_setaffinity(pid, &mask); pr_info("%s: sched_setaffinity returned %ld\n", __func__, ret); return 0; }
static void round_robin_cpu(unsigned int tsk_index) { struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits); cpumask_var_t tmp; int cpu; unsigned long min_weight = -1; unsigned long uninitialized_var(preferred_cpu); if (!alloc_cpumask_var(&tmp, GFP_KERNEL)) return; mutex_lock(&round_robin_lock); cpumask_clear(tmp); for_each_cpu(cpu, pad_busy_cpus) cpumask_or(tmp, tmp, topology_thread_cpumask(cpu)); cpumask_andnot(tmp, cpu_online_mask, tmp); /* avoid HT sibilings if possible */ if (cpumask_empty(tmp)) cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus); if (cpumask_empty(tmp)) { mutex_unlock(&round_robin_lock); return; } for_each_cpu(cpu, tmp) { if (cpu_weight[cpu] < min_weight) { min_weight = cpu_weight[cpu]; preferred_cpu = cpu; } } if (tsk_in_cpu[tsk_index] != -1) cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus); tsk_in_cpu[tsk_index] = preferred_cpu; cpumask_set_cpu(preferred_cpu, pad_busy_cpus); cpu_weight[preferred_cpu]++; mutex_unlock(&round_robin_lock); set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu)); }
void noinline bench_outstanding_parallel_cpus(uint32_t loops, int nr_cpus, int outstanding_pages) { const char *desc = "parallel_cpus"; struct time_bench_sync sync; struct time_bench_cpu *cpu_tasks; struct cpumask my_cpumask; int i; /* Allocate records for CPUs */ cpu_tasks = kzalloc(sizeof(*cpu_tasks) * nr_cpus, GFP_KERNEL); /* Reduce number of CPUs to run on */ cpumask_clear(&my_cpumask); for (i = 0; i < nr_cpus ; i++) { cpumask_set_cpu(i, &my_cpumask); } pr_info("Limit to %d parallel CPUs\n", nr_cpus); time_bench_run_concurrent(loops, outstanding_pages, NULL, &my_cpumask, &sync, cpu_tasks, time_alloc_pages_outstanding); time_bench_print_stats_cpumask(desc, cpu_tasks, &my_cpumask); kfree(cpu_tasks); }
/* Return a mask of the cpus whose caches currently own these pages. */ static void homecache_mask(struct page *page, int pages, struct cpumask *home_mask) { int i; cpumask_clear(home_mask); for (i = 0; i < pages; ++i) { int home = page_home(&page[i]); if (home == PAGE_HOME_IMMUTABLE || home == PAGE_HOME_INCOHERENT) { cpumask_copy(home_mask, cpu_possible_mask); return; } #if CHIP_HAS_CBOX_HOME_MAP() if (home == PAGE_HOME_HASH) { cpumask_or(home_mask, home_mask, &hash_for_home_map); continue; } #endif if (home == PAGE_HOME_UNCACHED) continue; BUG_ON(home < 0 || home >= NR_CPUS); cpumask_set_cpu(home, home_mask); } }
static int bcl_probe(struct platform_device *pdev) { struct bcl_context *bcl = NULL; int ret = 0; enum bcl_device_mode bcl_mode = BCL_DEVICE_DISABLED; bcl = devm_kzalloc(&pdev->dev, sizeof(struct bcl_context), GFP_KERNEL); if (!bcl) { pr_err("Cannot allocate bcl_context\n"); return -ENOMEM; } /* For BCL */ /* Init default BCL params */ if (of_property_read_bool(pdev->dev.of_node, "qcom,bcl-enable")) bcl_mode = BCL_DEVICE_ENABLED; else bcl_mode = BCL_DEVICE_DISABLED; bcl->bcl_mode = BCL_DEVICE_DISABLED; bcl->dev = &pdev->dev; bcl->bcl_monitor_type = BCL_IAVAIL_MONITOR_TYPE; bcl->bcl_threshold_mode[BCL_LOW_THRESHOLD_TYPE] = BCL_IAVAIL_THRESHOLD_DISABLED; bcl->bcl_threshold_mode[BCL_HIGH_THRESHOLD_TYPE] = BCL_IAVAIL_THRESHOLD_DISABLED; bcl->bcl_threshold_value_ma[BCL_LOW_THRESHOLD_TYPE] = 0; bcl->bcl_threshold_value_ma[BCL_HIGH_THRESHOLD_TYPE] = 0; bcl->bcl_vbat_min = BATTERY_VOLTAGE_MIN; snprintf(bcl->bcl_type, BCL_NAME_LENGTH, "%s", bcl_type[BCL_IAVAIL_MONITOR_TYPE]); bcl->bcl_poll_interval_msec = BCL_POLL_INTERVAL; if (of_property_read_bool(pdev->dev.of_node, "qcom,bcl-no-bms")) bcl->bcl_no_bms = true; else bcl->bcl_no_bms = false; bcl_frequency_mask = get_mask_from_core_handle(pdev, "qcom,bcl-freq-control-list"); bcl_hotplug_mask = get_mask_from_core_handle(pdev, "qcom,bcl-hotplug-list"); bcl_soc_hotplug_mask = get_mask_from_core_handle(pdev, "qcom,bcl-soc-hotplug-list"); if (!bcl_hotplug_mask && !bcl_soc_hotplug_mask) bcl_hotplug_enabled = false; else bcl_hotplug_enabled = true; if (of_property_read_bool(pdev->dev.of_node, "qcom,bcl-framework-interface")) ret = probe_bcl_periph_prop(bcl); else ret = probe_btm_properties(bcl); if (ret == -EPROBE_DEFER) return ret; ret = create_bcl_sysfs(bcl); if (ret < 0) { pr_err("Cannot create bcl sysfs\n"); return ret; } cpumask_clear(bcl_cpu_online_mask); bcl_psy.name = bcl_psy_name; bcl_psy.type = POWER_SUPPLY_TYPE_BMS; bcl_psy.get_property = bcl_battery_get_property; bcl_psy.set_property = bcl_battery_set_property; bcl_psy.num_properties = 0; #ifndef CONFIG_LGE_PM bcl_psy.external_power_changed = power_supply_callback; #endif gbcl = bcl; platform_set_drvdata(pdev, bcl); INIT_DEFERRABLE_WORK(&bcl->bcl_iavail_work, bcl_iavail_work); INIT_WORK(&bcl_hotplug_work, bcl_handle_hotplug); if (bcl_hotplug_enabled) register_cpu_notifier(&bcl_cpu_notifier); if (bcl_mode == BCL_DEVICE_ENABLED) bcl_mode_set(bcl_mode); return 0; }
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) { struct vm_area_struct *mpnt, *tmp, **pprev; struct rb_node **rb_link, *rb_parent; int retval; unsigned long charge; struct mempolicy *pol; down_write(&oldmm->mmap_sem); flush_cache_dup_mm(oldmm); /* * Not linked in yet - no deadlock potential: */ down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); mm->locked_vm = 0; mm->mmap = NULL; mm->mmap_cache = NULL; mm->free_area_cache = oldmm->mmap_base; mm->cached_hole_size = ~0UL; mm->map_count = 0; cpumask_clear(mm_cpumask(mm)); mm->mm_rb = RB_ROOT; rb_link = &mm->mm_rb.rb_node; rb_parent = NULL; pprev = &mm->mmap; retval = ksm_fork(mm, oldmm); if (retval) goto out; for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { struct file *file; if (mpnt->vm_flags & VM_DONTCOPY) { long pages = vma_pages(mpnt); mm->total_vm -= pages; vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file, -pages); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { unsigned int len = (mpnt->vm_end - mpnt->vm_start) >> PAGE_SHIFT; if (security_vm_enough_memory(len)) goto fail_nomem; charge = len; } tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); if (!tmp) goto fail_nomem; *tmp = *mpnt; pol = mpol_dup(vma_policy(mpnt)); retval = PTR_ERR(pol); if (IS_ERR(pol)) goto fail_nomem_policy; vma_set_policy(tmp, pol); tmp->vm_flags &= ~VM_LOCKED; tmp->vm_mm = mm; tmp->vm_next = NULL; anon_vma_link(tmp); file = tmp->vm_file; if (file) { struct inode *inode = file->f_path.dentry->d_inode; struct address_space *mapping = file->f_mapping; get_file(file); if (tmp->vm_flags & VM_DENYWRITE) atomic_dec(&inode->i_writecount); spin_lock(&mapping->i_mmap_lock); if (tmp->vm_flags & VM_SHARED) mapping->i_mmap_writable++; tmp->vm_truncate_count = mpnt->vm_truncate_count; flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ vma_prio_tree_add(tmp, mpnt); flush_dcache_mmap_unlock(mapping); spin_unlock(&mapping->i_mmap_lock); } /* * Clear hugetlb-related page reserves for children. This only * affects MAP_PRIVATE mappings. Faults generated by the child * are not guaranteed to succeed, even if read-only */ if (is_vm_hugetlb_page(tmp)) reset_vma_resv_huge_pages(tmp); /* * Link in the new vma and copy the page table entries. */ *pprev = tmp; pprev = &tmp->vm_next; __vma_link_rb(mm, tmp, rb_link, rb_parent); rb_link = &tmp->vm_rb.rb_right; rb_parent = &tmp->vm_rb; mm->map_count++; retval = copy_page_range(mm, oldmm, mpnt); if (tmp->vm_ops && tmp->vm_ops->open) tmp->vm_ops->open(tmp); if (retval) goto out; }
static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) { unsigned int i; unsigned int valid_states = 0; unsigned int cpu = policy->cpu; struct acpi_cpufreq_data *data; unsigned int result = 0; struct cpuinfo_x86 *c = &cpu_data(policy->cpu); struct acpi_processor_performance *perf; #ifdef CONFIG_SMP static int blacklisted; #endif pr_debug("acpi_cpufreq_cpu_init\n"); #ifdef CONFIG_SMP if (blacklisted) return blacklisted; blacklisted = acpi_cpufreq_blacklist(c); if (blacklisted) return blacklisted; #endif data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; if (!zalloc_cpumask_var(&data->freqdomain_cpus, GFP_KERNEL)) { result = -ENOMEM; goto err_free; } perf = per_cpu_ptr(acpi_perf_data, cpu); data->acpi_perf_cpu = cpu; policy->driver_data = data; if (cpu_has(c, X86_FEATURE_CONSTANT_TSC)) acpi_cpufreq_driver.flags |= CPUFREQ_CONST_LOOPS; result = acpi_processor_register_performance(perf, cpu); if (result) goto err_free_mask; policy->shared_type = perf->shared_type; /* * Will let policy->cpus know about dependency only when software * coordination is required. */ if (policy->shared_type == CPUFREQ_SHARED_TYPE_ALL || policy->shared_type == CPUFREQ_SHARED_TYPE_ANY) { cpumask_copy(policy->cpus, perf->shared_cpu_map); } cpumask_copy(data->freqdomain_cpus, perf->shared_cpu_map); #ifdef CONFIG_SMP dmi_check_system(sw_any_bug_dmi_table); if (bios_with_sw_any_bug && !policy_is_shared(policy)) { policy->shared_type = CPUFREQ_SHARED_TYPE_ALL; cpumask_copy(policy->cpus, topology_core_cpumask(cpu)); } if (check_amd_hwpstate_cpu(cpu) && !acpi_pstate_strict) { cpumask_clear(policy->cpus); cpumask_set_cpu(cpu, policy->cpus); cpumask_copy(data->freqdomain_cpus, topology_sibling_cpumask(cpu)); policy->shared_type = CPUFREQ_SHARED_TYPE_HW; pr_info_once(PFX "overriding BIOS provided _PSD data\n"); } #endif /* capability check */ if (perf->state_count <= 1) { pr_debug("No P-States\n"); result = -ENODEV; goto err_unreg; } if (perf->control_register.space_id != perf->status_register.space_id) { result = -ENODEV; goto err_unreg; } switch (perf->control_register.space_id) { case ACPI_ADR_SPACE_SYSTEM_IO: if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD && boot_cpu_data.x86 == 0xf) { pr_debug("AMD K8 systems must use native drivers.\n"); result = -ENODEV; goto err_unreg; } pr_debug("SYSTEM IO addr space\n"); data->cpu_feature = SYSTEM_IO_CAPABLE; break; case ACPI_ADR_SPACE_FIXED_HARDWARE: pr_debug("HARDWARE addr space\n"); if (check_est_cpu(cpu)) { data->cpu_feature = SYSTEM_INTEL_MSR_CAPABLE; break; } if (check_amd_hwpstate_cpu(cpu)) { data->cpu_feature = SYSTEM_AMD_MSR_CAPABLE; break; } result = -ENODEV; goto err_unreg; default: pr_debug("Unknown addr space %d\n", (u32) (perf->control_register.space_id)); result = -ENODEV; goto err_unreg; } data->freq_table = kzalloc(sizeof(*data->freq_table) * (perf->state_count+1), GFP_KERNEL); if (!data->freq_table) { result = -ENOMEM; goto err_unreg; } /* detect transition latency */ policy->cpuinfo.transition_latency = 0; for (i = 0; i < perf->state_count; i++) { if ((perf->states[i].transition_latency * 1000) > policy->cpuinfo.transition_latency) policy->cpuinfo.transition_latency = perf->states[i].transition_latency * 1000; } /* Check for high latency (>20uS) from buggy BIOSes, like on T42 */ if (perf->control_register.space_id == ACPI_ADR_SPACE_FIXED_HARDWARE && policy->cpuinfo.transition_latency > 20 * 1000) { policy->cpuinfo.transition_latency = 20 * 1000; printk_once(KERN_INFO "P-state transition latency capped at 20 uS\n"); } /* table init */ for (i = 0; i < perf->state_count; i++) { if (i > 0 && perf->states[i].core_frequency >= data->freq_table[valid_states-1].frequency / 1000) continue; data->freq_table[valid_states].driver_data = i; data->freq_table[valid_states].frequency = perf->states[i].core_frequency * 1000; valid_states++; } data->freq_table[valid_states].frequency = CPUFREQ_TABLE_END; perf->state = 0; result = cpufreq_table_validate_and_show(policy, data->freq_table); if (result) goto err_freqfree; if (perf->states[0].core_frequency * 1000 != policy->cpuinfo.max_freq) printk(KERN_WARNING FW_WARN "P-state 0 is not max freq\n"); switch (perf->control_register.space_id) { case ACPI_ADR_SPACE_SYSTEM_IO: /* * The core will not set policy->cur, because * cpufreq_driver->get is NULL, so we need to set it here. * However, we have to guess it, because the current speed is * unknown and not detectable via IO ports. */ policy->cur = acpi_cpufreq_guess_freq(data, policy->cpu); break; case ACPI_ADR_SPACE_FIXED_HARDWARE: acpi_cpufreq_driver.get = get_cur_freq_on_cpu; break; default: break; } /* notify BIOS that we exist */ acpi_processor_notify_smm(THIS_MODULE); pr_debug("CPU%u - ACPI performance management activated.\n", cpu); for (i = 0; i < perf->state_count; i++) pr_debug(" %cP%d: %d MHz, %d mW, %d uS\n", (i == perf->state ? '*' : ' '), i, (u32) perf->states[i].core_frequency, (u32) perf->states[i].power, (u32) perf->states[i].transition_latency); /* * the first call to ->target() should result in us actually * writing something to the appropriate registers. */ data->resume = 1; return result; err_freqfree: kfree(data->freq_table); err_unreg: acpi_processor_unregister_performance(cpu); err_free_mask: free_cpumask_var(data->freqdomain_cpus); err_free: kfree(data); policy->driver_data = NULL; return result; }
/* * This wrapper function around hv_flush_remote() does several things: * * - Provides a return value error-checking panic path, since * there's never any good reason for hv_flush_remote() to fail. * - Accepts a 32-bit PFN rather than a 64-bit PA, which generally * is the type that Linux wants to pass around anyway. * - Centralizes the mark_caches_evicted() handling. * - Canonicalizes that lengths of zero make cpumasks NULL. * - Handles deferring TLB flushes for dataplane tiles. * - Tracks remote interrupts in the per-cpu irq_cpustat_t. * * Note that we have to wait until the cache flush completes before * updating the per-cpu last_cache_flush word, since otherwise another * concurrent flush can race, conclude the flush has already * completed, and start to use the page while it's still dirty * remotely (running concurrently with the actual evict, presumably). */ void flush_remote(unsigned long cache_pfn, unsigned long cache_control, const struct cpumask *cache_cpumask_orig, HV_VirtAddr tlb_va, unsigned long tlb_length, unsigned long tlb_pgsize, const struct cpumask *tlb_cpumask_orig, HV_Remote_ASID *asids, int asidcount) { int rc; int timestamp = 0; /* happy compiler */ struct cpumask cache_cpumask_copy, tlb_cpumask_copy; struct cpumask *cache_cpumask, *tlb_cpumask; HV_PhysAddr cache_pa; char cache_buf[NR_CPUS*5], tlb_buf[NR_CPUS*5]; mb(); /* provided just to simplify "magic hypervisor" mode */ /* * Canonicalize and copy the cpumasks. */ if (cache_cpumask_orig && cache_control) { cpumask_copy(&cache_cpumask_copy, cache_cpumask_orig); cache_cpumask = &cache_cpumask_copy; } else { cpumask_clear(&cache_cpumask_copy); cache_cpumask = NULL; } if (cache_cpumask == NULL) cache_control = 0; if (tlb_cpumask_orig && tlb_length) { cpumask_copy(&tlb_cpumask_copy, tlb_cpumask_orig); tlb_cpumask = &tlb_cpumask_copy; } else { cpumask_clear(&tlb_cpumask_copy); tlb_cpumask = NULL; } hv_flush_update(cache_cpumask, tlb_cpumask, tlb_va, tlb_length, asids, asidcount); cache_pa = (HV_PhysAddr)cache_pfn << PAGE_SHIFT; if (cache_control & HV_FLUSH_EVICT_L2) timestamp = mark_caches_evicted_start(); rc = hv_flush_remote(cache_pa, cache_control, cpumask_bits(cache_cpumask), tlb_va, tlb_length, tlb_pgsize, cpumask_bits(tlb_cpumask), asids, asidcount); if (cache_control & HV_FLUSH_EVICT_L2) mark_caches_evicted_finish(cache_cpumask, timestamp); if (rc == 0) return; cpumask_scnprintf(cache_buf, sizeof(cache_buf), &cache_cpumask_copy); cpumask_scnprintf(tlb_buf, sizeof(tlb_buf), &tlb_cpumask_copy); pr_err("hv_flush_remote(%#llx, %#lx, %p [%s]," " %#lx, %#lx, %#lx, %p [%s], %p, %d) = %d\n", cache_pa, cache_control, cache_cpumask, cache_buf, (unsigned long)tlb_va, tlb_length, tlb_pgsize, tlb_cpumask, tlb_buf, asids, asidcount, rc); panic("Unsafe to continue."); }
static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) { struct vm_area_struct *mpnt, *tmp, *prev, **pprev; struct rb_node **rb_link, *rb_parent; int retval; unsigned long charge; uprobe_start_dup_mmap(); down_write(&oldmm->mmap_sem); flush_cache_dup_mm(oldmm); uprobe_dup_mmap(oldmm, mm); /* * Not linked in yet - no deadlock potential: */ down_write_nested(&mm->mmap_sem, SINGLE_DEPTH_NESTING); mm->locked_vm = 0; mm->mmap = NULL; mm->mmap_cache = NULL; mm->map_count = 0; cpumask_clear(mm_cpumask(mm)); mm->mm_rb = RB_ROOT; rb_link = &mm->mm_rb.rb_node; rb_parent = NULL; pprev = &mm->mmap; retval = ksm_fork(mm, oldmm); if (retval) goto out; retval = khugepaged_fork(mm, oldmm); if (retval) goto out; prev = NULL; for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) { struct file *file; if (mpnt->vm_flags & VM_DONTCOPY) { vm_stat_account(mm, mpnt->vm_flags, mpnt->vm_file, -vma_pages(mpnt)); continue; } charge = 0; if (mpnt->vm_flags & VM_ACCOUNT) { unsigned long len = vma_pages(mpnt); if (security_vm_enough_memory_mm(oldmm, len)) /* sic */ goto fail_nomem; charge = len; } tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); if (!tmp) goto fail_nomem; *tmp = *mpnt; INIT_LIST_HEAD(&tmp->anon_vma_chain); retval = vma_dup_policy(mpnt, tmp); if (retval) goto fail_nomem_policy; tmp->vm_mm = mm; if (anon_vma_fork(tmp, mpnt)) goto fail_nomem_anon_vma_fork; tmp->vm_flags &= ~VM_LOCKED; tmp->vm_next = tmp->vm_prev = NULL; file = tmp->vm_file; if (file) { struct inode *inode = file_inode(file); struct address_space *mapping = file->f_mapping; get_file(file); if (tmp->vm_flags & VM_DENYWRITE) atomic_dec(&inode->i_writecount); mutex_lock(&mapping->i_mmap_mutex); if (tmp->vm_flags & VM_SHARED) mapping->i_mmap_writable++; flush_dcache_mmap_lock(mapping); /* insert tmp into the share list, just after mpnt */ if (unlikely(tmp->vm_flags & VM_NONLINEAR)) vma_nonlinear_insert(tmp, &mapping->i_mmap_nonlinear); else vma_interval_tree_insert_after(tmp, mpnt, &mapping->i_mmap); flush_dcache_mmap_unlock(mapping); mutex_unlock(&mapping->i_mmap_mutex); } /* * Clear hugetlb-related page reserves for children. This only * affects MAP_PRIVATE mappings. Faults generated by the child * are not guaranteed to succeed, even if read-only */ if (is_vm_hugetlb_page(tmp)) reset_vma_resv_huge_pages(tmp); /* * Link in the new vma and copy the page table entries. */ *pprev = tmp; pprev = &tmp->vm_next; tmp->vm_prev = prev; prev = tmp; __vma_link_rb(mm, tmp, rb_link, rb_parent); rb_link = &tmp->vm_rb.rb_right; rb_parent = &tmp->vm_rb; mm->map_count++; retval = copy_page_range(mm, oldmm, mpnt); if (tmp->vm_ops && tmp->vm_ops->open) tmp->vm_ops->open(tmp); if (retval) goto out; } /* a new mm has just been created */ arch_dup_mmap(oldmm, mm); retval = 0; out: up_write(&mm->mmap_sem); flush_tlb_mm(oldmm); up_write(&oldmm->mmap_sem); uprobe_end_dup_mmap(); return retval; fail_nomem_anon_vma_fork: mpol_put(vma_policy(tmp)); fail_nomem_policy: kmem_cache_free(vm_area_cachep, tmp); fail_nomem: retval = -ENOMEM; vm_unacct_memory(charge); goto out; }
/* * Handle oneshot mode broadcasting */ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) { struct tick_device *td; ktime_t now, next_event; int cpu, next_cpu = 0; bool bc_local; raw_spin_lock(&tick_broadcast_lock); dev->next_event = KTIME_MAX; next_event = KTIME_MAX; cpumask_clear(tmpmask); now = ktime_get(); /* Find all expired events */ for_each_cpu(cpu, tick_broadcast_oneshot_mask) { /* * Required for !SMP because for_each_cpu() reports * unconditionally CPU0 as set on UP kernels. */ if (!IS_ENABLED(CONFIG_SMP) && cpumask_empty(tick_broadcast_oneshot_mask)) break; td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event <= now) { cpumask_set_cpu(cpu, tmpmask); /* * Mark the remote cpu in the pending mask, so * it can avoid reprogramming the cpu local * timer in tick_broadcast_oneshot_control(). */ cpumask_set_cpu(cpu, tick_broadcast_pending_mask); } else if (td->evtdev->next_event < next_event) { next_event = td->evtdev->next_event; next_cpu = cpu; } } /* * Remove the current cpu from the pending mask. The event is * delivered immediately in tick_do_broadcast() ! */ cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask); /* Take care of enforced broadcast requests */ cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask); cpumask_clear(tick_broadcast_force_mask); /* * Sanity check. Catch the case where we try to broadcast to * offline cpus. */ if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask))) cpumask_and(tmpmask, tmpmask, cpu_online_mask); /* * Wakeup the cpus which have an expired event. */ bc_local = tick_do_broadcast(tmpmask); /* * Two reasons for reprogram: * * - The global event did not expire any CPU local * events. This happens in dyntick mode, as the maximum PIT * delta is quite small. * * - There are pending events on sleeping CPUs which were not * in the event mask */ if (next_event != KTIME_MAX) tick_broadcast_set_event(dev, next_cpu, next_event); raw_spin_unlock(&tick_broadcast_lock); if (bc_local) { td = this_cpu_ptr(&tick_cpu_device); td->evtdev->event_handler(td->evtdev); } }