/* * Periodic broadcast: * - invoke the broadcast handlers */ static void tick_do_periodic_broadcast(void) { raw_spin_lock(&tick_broadcast_lock); cpumask_and(to_cpumask(tmpmask), cpu_online_mask, tick_get_broadcast_mask()); tick_do_broadcast(to_cpumask(tmpmask)); raw_spin_unlock(&tick_broadcast_lock); }
static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, const char *name) { int ret = -ENOMEM; struct pcrypt_cpumask *mask; get_online_cpus(); pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1, name); if (!pcrypt->wq) goto err; pcrypt->pinst = padata_alloc_possible(pcrypt->wq); if (!pcrypt->pinst) goto err_destroy_workqueue; mask = kmalloc(sizeof(*mask), GFP_KERNEL); if (!mask) goto err_free_padata; if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { kfree(mask); goto err_free_padata; } cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask); rcu_assign_pointer(pcrypt->cb_cpumask, mask); pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); if (ret) goto err_free_cpumask; ret = pcrypt_sysfs_add(pcrypt->pinst, name); if (ret) goto err_unregister_notifier; put_online_cpus(); return ret; err_unregister_notifier: padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); err_free_cpumask: free_cpumask_var(mask->mask); kfree(mask); err_free_padata: padata_free(pcrypt->pinst); err_destroy_workqueue: destroy_workqueue(pcrypt->wq); err: put_online_cpus(); return ret; }
/* * Generic version of the affinity autoselector. */ static int setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask) { struct irq_chip *chip = irq_desc_get_chip(desc); struct cpumask *set = irq_default_affinity; int ret, node = desc->irq_data.node; /* Excludes PER_CPU and NO_BALANCE interrupts */ if (!irq_can_set_affinity(irq)) return 0; /* * Preserve an userspace affinity setup, but make sure that * one of the targets is online. */ if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { if (cpumask_intersects(desc->irq_data.affinity, cpu_online_mask)) set = desc->irq_data.affinity; else irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); } cpumask_and(mask, cpu_online_mask, set); if (node != NUMA_NO_NODE) { const struct cpumask *nodemask = cpumask_of_node(node); /* make sure at least one of the cpus in nodemask is online */ if (cpumask_intersects(mask, nodemask)) cpumask_and(mask, mask, nodemask); } ret = chip->irq_set_affinity(&desc->irq_data, mask, false); switch (ret) { case IRQ_SET_MASK_OK: cpumask_copy(desc->irq_data.affinity, mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); } return 0; }
void cpuidle_wakeup_mwait(cpumask_t *mask) { cpumask_t target; unsigned int cpu; cpumask_and(&target, mask, &cpuidle_mwait_flags); /* CPU is MWAITing on the cpuidle_mwait_wakeup flag. */ for_each_cpu(cpu, &target) mwait_wakeup(cpu) = 0; cpumask_andnot(mask, mask, &target); }
/* * cpudl_find - find the best (later-dl) CPU in the system * @cp: the cpudl max-heap context * @p: the task * @later_mask: a mask to fill in with the selected CPUs (or NULL) * * Returns: int - best CPU (heap maximum if suitable) */ int cpudl_find(struct cpudl *cp, struct task_struct *p, struct cpumask *later_mask) { int best_cpu = -1; const struct sched_dl_entity *dl_se = &p->dl; if (later_mask && cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed) && cpumask_and(later_mask, later_mask, cpu_active_mask)) { best_cpu = cpumask_any(later_mask); goto out; } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) && dl_time_before(dl_se->deadline, cp->elements[0].dl)) { best_cpu = cpudl_maximum(cp); if (later_mask) cpumask_set_cpu(best_cpu, later_mask); } out: WARN_ON(best_cpu != -1 && !cpu_present(best_cpu)); return best_cpu; }
static int setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask) { struct irq_chip *chip = irq_desc_get_chip(desc); struct cpumask *set = irq_default_affinity; int ret, node = desc->irq_data.node; if (!irq_can_set_affinity(irq)) return 0; if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { if (cpumask_intersects(desc->irq_data.affinity, cpu_online_mask)) set = desc->irq_data.affinity; else irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); } cpumask_and(mask, cpu_online_mask, set); if (node != NUMA_NO_NODE) { const struct cpumask *nodemask = cpumask_of_node(node); if (cpumask_intersects(mask, nodemask)) cpumask_and(mask, mask, nodemask); } ret = chip->irq_set_affinity(&desc->irq_data, mask, false); switch (ret) { case IRQ_SET_MASK_OK: cpumask_copy(desc->irq_data.affinity, mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); } return 0; }
void __init arch_get_hmp_domains(struct list_head *hmp_domains_list) { struct hmp_domain *domain; arch_get_fast_and_slow_cpus(&hmp_fast_cpu_mask, &hmp_slow_cpu_mask); /* * Initialize hmp_domains * Must be ordered with respect to compute capacity. * Fastest domain at head of list. */ if(!cpumask_empty(&hmp_slow_cpu_mask)) { domain = (struct hmp_domain *) kmalloc(sizeof(struct hmp_domain), GFP_KERNEL); cpumask_copy(&domain->possible_cpus, &hmp_slow_cpu_mask); cpumask_and(&domain->cpus, cpu_online_mask, &domain->possible_cpus); list_add(&domain->hmp_domains, hmp_domains_list); } domain = (struct hmp_domain *) kmalloc(sizeof(struct hmp_domain), GFP_KERNEL); cpumask_copy(&domain->possible_cpus, &hmp_fast_cpu_mask); cpumask_and(&domain->cpus, cpu_online_mask, &domain->possible_cpus); list_add(&domain->hmp_domains, hmp_domains_list); }
void move_masked_irq(int irq) { struct irq_desc *desc = irq_to_desc(irq); if (likely(!(desc->status & IRQ_MOVE_PENDING))) return; /* * Paranoia: cpu-local interrupts shouldn't be calling in here anyway. */ if (CHECK_IRQ_PER_CPU(desc->status)) { WARN_ON(1); return; } desc->status &= ~IRQ_MOVE_PENDING; if (unlikely(cpumask_empty(desc->pending_mask))) return; if (!desc->chip->set_affinity) return; assert_spin_locked(&desc->lock); /* * If there was a valid mask to work with, please * do the disable, re-program, enable sequence. * This is *not* particularly important for level triggered * but in a edge trigger case, we might be setting rte * when an active trigger is comming in. This could * cause some ioapics to mal-function. * Being paranoid i guess! * * For correct operation this depends on the caller * masking the irqs. */ if (likely(cpumask_any_and(desc->pending_mask, cpu_online_mask) < nr_cpu_ids)) { cpumask_and(desc->affinity, desc->pending_mask, cpu_online_mask); desc->chip->set_affinity(irq, desc->affinity); } cpumask_clear(desc->pending_mask); }
void irq_complete_move(unsigned irq) { struct irq_cfg *cfg = &irq_cfg[irq]; cpumask_t cleanup_mask; int i; if (likely(!cfg->move_in_progress)) return; if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain))) return; cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask); cfg->move_cleanup_count = cpus_weight(cleanup_mask); for_each_cpu_mask(i, cleanup_mask) platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0); cfg->move_in_progress = 0; }
static inline int find_unassigned_vector(cpumask_t domain) { cpumask_t mask; int pos, vector; cpumask_and(&mask, &domain, cpu_online_mask); if (cpus_empty(mask)) return -EINVAL; for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) { vector = IA64_FIRST_DEVICE_VECTOR + pos; cpus_and(mask, domain, vector_table[vector]); if (!cpus_empty(mask)) continue; return vector; } return -ENOSPC; }
/** * flush_tlb_others - Tell the specified CPUs to invalidate their TLBs * @cpumask: The list of CPUs to target. * @mm: The VM context to flush from (if va!=FLUSH_ALL). * @va: Virtual address to flush or FLUSH_ALL to flush everything. */ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, unsigned long va) { cpumask_t tmp; /* A couple of sanity checks (to be removed): * - mask must not be empty * - current CPU must not be in mask * - we do not send IPIs to as-yet unbooted CPUs. */ BUG_ON(!mm); BUG_ON(cpumask_empty(&cpumask)); BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask)); cpumask_and(&tmp, &cpumask, cpu_online_mask); BUG_ON(!cpumask_equal(&cpumask, &tmp)); /* I'm not happy about this global shared spinlock in the MM hot path, * but we'll see how contended it is. * * Temporarily this turns IRQs off, so that lockups are detected by the * NMI watchdog. */ spin_lock(&tlbstate_lock); flush_mm = mm; flush_va = va; #if NR_CPUS <= BITS_PER_LONG atomic_set_mask(cpumask.bits[0], &flush_cpumask.bits[0]); #else #error Not supported. #endif /* FIXME: if NR_CPUS>=3, change send_IPI_mask */ smp_call_function(smp_flush_tlb, NULL, 1); while (!cpumask_empty(&flush_cpumask)) /* Lockup detection does not belong here */ smp_mb(); flush_mm = NULL; flush_va = 0; spin_unlock(&tlbstate_lock); }
static void __clear_irq_vector(int irq) { int vector, cpu; cpumask_t mask; cpumask_t domain; struct irq_cfg *cfg = &irq_cfg[irq]; BUG_ON((unsigned)irq >= NR_IRQS); BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); vector = cfg->vector; domain = cfg->domain; cpumask_and(&mask, &cfg->domain, cpu_online_mask); for_each_cpu_mask(cpu, mask) per_cpu(vector_irq, cpu)[vector] = -1; cfg->vector = IRQ_VECTOR_UNASSIGNED; cfg->domain = CPU_MASK_NONE; irq_status[irq] = IRQ_UNUSED; cpus_andnot(vector_table[vector], vector_table[vector], domain); }
/* * Debug related code, dump vcpu/cpu information */ static void rt_dump_vcpu(const struct scheduler *ops, const struct rt_vcpu *svc) { cpumask_t *cpupool_mask, *mask; ASSERT(svc != NULL); /* idle vcpu */ if( svc->sdom == NULL ) { printk("\n"); return; } /* * We can't just use 'cpumask_scratch' because the dumping can * happen from a pCPU outside of this scheduler's cpupool, and * hence it's not right to use the pCPU's scratch mask (which * may even not exist!). On the other hand, it is safe to use * svc->vcpu->processor's own scratch space, since we hold the * runqueue lock. */ mask = _cpumask_scratch[svc->vcpu->processor]; cpupool_mask = cpupool_domain_cpumask(svc->vcpu->domain); cpumask_and(mask, cpupool_mask, svc->vcpu->cpu_hard_affinity); cpulist_scnprintf(keyhandler_scratch, sizeof(keyhandler_scratch), mask); printk("[%5d.%-2u] cpu %u, (%"PRI_stime", %"PRI_stime")," " cur_b=%"PRI_stime" cur_d=%"PRI_stime" last_start=%"PRI_stime"\n" " \t\t onQ=%d runnable=%d flags=%x effective hard_affinity=%s\n", svc->vcpu->domain->domain_id, svc->vcpu->vcpu_id, svc->vcpu->processor, svc->period, svc->budget, svc->cur_budget, svc->cur_deadline, svc->last_start, __vcpu_on_q(svc), vcpu_runnable(svc->vcpu), svc->flags, keyhandler_scratch); }
SYSCALL_DEFINE3(fairsched_cpumask, unsigned int, id, unsigned int, len, unsigned long __user *, user_mask_ptr) { struct cgroup *cgrp; int retval; cpumask_var_t new_mask, in_mask; if (!capable_setveid()) return -EPERM; if (id == 0) return -EINVAL; cgrp = fairsched_open(id); if (IS_ERR(cgrp)) return PTR_ERR(cgrp); if (!alloc_cpumask_var(&in_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out; } if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out_free_in_mask; } retval = get_user_cpu_mask(user_mask_ptr, len, in_mask); if (retval == 0) { cpumask_and(new_mask, in_mask, cpu_active_mask); cgroup_lock(); retval = cgroup_set_cpumask(cgrp, new_mask); cgroup_unlock(); } free_cpumask_var(new_mask); out_free_in_mask: free_cpumask_var(in_mask); out: cgroup_kernel_close(cgrp); return retval; }
void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags) { ASSERT(local_irq_is_enabled()); if ( cpumask_test_cpu(smp_processor_id(), mask) ) flush_area_local(va, flags); if ( !cpumask_subset(mask, cpumask_of(smp_processor_id())) ) { spin_lock(&flush_lock); cpumask_and(&flush_cpumask, mask, &cpu_online_map); cpumask_clear_cpu(smp_processor_id(), &flush_cpumask); flush_va = va; flush_flags = flags; send_IPI_mask(&flush_cpumask, INVALIDATE_TLB_VECTOR); while ( !cpumask_empty(&flush_cpumask) ) cpu_relax(); spin_unlock(&flush_lock); } }
static int idu_irq_set_affinity(struct irq_data *data, const struct cpumask *cpumask, bool force) { unsigned long flags; cpumask_t online; /* errout if no online cpu per @cpumask */ if (!cpumask_and(&online, cpumask, cpu_online_mask)) return -EINVAL; raw_spin_lock_irqsave(&mcip_lock, flags); idu_set_dest(data->hwirq, cpumask_bits(&online)[0]); idu_set_mode(data->hwirq, IDU_M_TRIG_LEVEL, IDU_M_DISTRI_RR); raw_spin_unlock_irqrestore(&mcip_lock, flags); return IRQ_SET_MASK_OK; }
/* * Generic version of the affinity autoselector. */ static int setup_affinity(unsigned int irq, struct irq_desc *desc) { if (!irq_can_set_affinity(irq)) return 0; /* * Preserve an userspace affinity setup, but make sure that * one of the targets is online. */ if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { if (cpumask_any_and(desc->irq_data.affinity, cpu_online_mask) < nr_cpu_ids) goto set_affinity; else desc->status &= ~IRQ_AFFINITY_SET; } cpumask_and(desc->irq_data.affinity, cpu_online_mask, irq_default_affinity); set_affinity: desc->irq_data.chip->irq_set_affinity(&desc->irq_data, desc->irq_data.affinity, false); return 0; }
/* This function would program ITE values on node given by the cpumask * @cpumask : cpumask to program on ITE * @node : node on which ITE should be programmed * @ite : ITE to program * @scope : program ITE only on the given node (0) or all nodes (1) */ void xlp_cpumask_to_node_ite(const struct cpumask *m, u8 node, u8 ite, u8 scope) { __label__ prog_all; struct cpumask t; int cpu = (node * NLM_MAX_CPU_PER_NODE), last; if (scope != 0) goto prog_all; /* When the scope is 0, program node ITEs with target as * local cpus only */ last = cpu + NLM_MAX_CPU_PER_NODE - 1; if (last >= NR_CPUS) return; cpumask_and(&t, m, &phys_cpu_present_map); for (; cpu <= last; cpu++) { cpumask_test_cpu(cpu, &t) ? xlp_ite_cpu_set(node, cpu, ite) : xlp_ite_cpu_clear(node, cpu, ite); } return; prog_all: /* Here we program the specified ITE in all nodes with the cpumask * passed. */ /* TBD TODO */ return; }
static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg) { struct irq_cfg *cfg = irq_cfg + irq; unsigned dest; cpumask_t mask; cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask); dest = cpu_physical_id(first_cpu(mask)); msg->address_hi = 0; msg->address_lo = MSI_ADDR_HEADER | MSI_ADDR_DEST_MODE_PHYS | MSI_ADDR_REDIRECTION_CPU | MSI_ADDR_DEST_ID_CPU(dest); msg->data = MSI_DATA_TRIGGER_EDGE | MSI_DATA_LEVEL_ASSERT | MSI_DATA_DELIVERY_FIXED | MSI_DATA_VECTOR(cfg->vector); return 0; }
static int __bind_irq_vector(int irq, int vector, cpumask_t domain) { cpumask_t mask; int cpu; struct irq_cfg *cfg = &irq_cfg[irq]; BUG_ON((unsigned)irq >= NR_IRQS); BUG_ON((unsigned)vector >= IA64_NUM_VECTORS); cpumask_and(&mask, &domain, cpu_online_mask); if (cpus_empty(mask)) return -EINVAL; if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain)) return 0; if (cfg->vector != IRQ_VECTOR_UNASSIGNED) return -EBUSY; for_each_cpu_mask(cpu, mask) per_cpu(vector_irq, cpu)[vector] = irq; cfg->vector = vector; cfg->domain = domain; irq_status[irq] = IRQ_USED; cpus_or(vector_table[vector], vector_table[vector], domain); return 0; }
static void gic_set_affinity(unsigned int irq, const struct cpumask *cpumask) { cpumask_t tmp = CPU_MASK_NONE; unsigned long flags; int i; pr_debug(KERN_DEBUG "%s called\n", __func__); irq -= _irqbase; cpumask_and(&tmp, cpumask, cpu_online_mask); if (cpus_empty(tmp)) return; /* Assumption : cpumask refers to a single CPU */ spin_lock_irqsave(&gic_lock, flags); for (;;) { /* Re-route this IRQ */ GIC_SH_MAP_TO_VPE_SMASK(irq, first_cpu(tmp)); /* * FIXME: assumption that _intrmap is ordered and has no holes */ /* Update the intr_map */ _intrmap[irq].cpunum = first_cpu(tmp); /* Update the pcpu_masks */ for (i = 0; i < NR_CPUS; i++) clear_bit(irq, pcpu_masks[i].pcpu_mask); set_bit(irq, pcpu_masks[first_cpu(tmp)].pcpu_mask); } irq_desc[irq].affinity = *cpumask; spin_unlock_irqrestore(&gic_lock, flags); }
int cpupri_find(struct cpupri *cp, struct task_struct *p, struct cpumask *lowest_mask) { int idx = 0; int task_pri = convert_prio(p->prio); if (task_pri >= MAX_RT_PRIO) return 0; for (idx = 0; idx < task_pri; idx++) { struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; int skip = 0; if (!atomic_read(&(vec)->count)) skip = 1; smp_rmb(); if (skip) continue; if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids) continue; if (lowest_mask) { cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask); if (cpumask_any(lowest_mask) >= nr_cpu_ids) continue; } return 1; } return 0; }
/* * Handle oneshot mode broadcasting */ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) { struct tick_device *td; ktime_t now, next_event; int cpu, next_cpu = 0; bool bc_local; raw_spin_lock(&tick_broadcast_lock); dev->next_event = KTIME_MAX; next_event = KTIME_MAX; cpumask_clear(tmpmask); now = ktime_get(); /* Find all expired events */ for_each_cpu(cpu, tick_broadcast_oneshot_mask) { /* * Required for !SMP because for_each_cpu() reports * unconditionally CPU0 as set on UP kernels. */ if (!IS_ENABLED(CONFIG_SMP) && cpumask_empty(tick_broadcast_oneshot_mask)) break; td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event <= now) { cpumask_set_cpu(cpu, tmpmask); /* * Mark the remote cpu in the pending mask, so * it can avoid reprogramming the cpu local * timer in tick_broadcast_oneshot_control(). */ cpumask_set_cpu(cpu, tick_broadcast_pending_mask); } else if (td->evtdev->next_event < next_event) { next_event = td->evtdev->next_event; next_cpu = cpu; } } /* * Remove the current cpu from the pending mask. The event is * delivered immediately in tick_do_broadcast() ! */ cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask); /* Take care of enforced broadcast requests */ cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask); cpumask_clear(tick_broadcast_force_mask); /* * Sanity check. Catch the case where we try to broadcast to * offline cpus. */ if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask))) cpumask_and(tmpmask, tmpmask, cpu_online_mask); /* * Wakeup the cpus which have an expired event. */ bc_local = tick_do_broadcast(tmpmask); /* * Two reasons for reprogram: * * - The global event did not expire any CPU local * events. This happens in dyntick mode, as the maximum PIT * delta is quite small. * * - There are pending events on sleeping CPUs which were not * in the event mask */ if (next_event != KTIME_MAX) tick_broadcast_set_event(dev, next_cpu, next_event); raw_spin_unlock(&tick_broadcast_lock); if (bc_local) { td = this_cpu_ptr(&tick_cpu_device); td->evtdev->event_handler(td->evtdev); } }
/* * Periodic broadcast: * - invoke the broadcast handlers */ static bool tick_do_periodic_broadcast(void) { cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); return tick_do_broadcast(tmpmask); }
/* * This maps the physical memory to kernel virtual address space, a total * of max_low_pfn pages, by creating page tables starting from address * PAGE_OFFSET. * * This routine transitions us from using a set of compiled-in large * pages to using some more precise caching, including removing access * to code pages mapped at PAGE_OFFSET (executed only at MEM_SV_START) * marking read-only data as locally cacheable, striping the remaining * .data and .bss across all the available tiles, and removing access * to pages above the top of RAM (thus ensuring a page fault from a bad * virtual address rather than a hypervisor shoot down for accessing * memory outside the assigned limits). */ static void __init kernel_physical_mapping_init(pgd_t *pgd_base) { unsigned long long irqmask; unsigned long address, pfn; pmd_t *pmd; pte_t *pte; int pte_ofs; const struct cpumask *my_cpu_mask = cpumask_of(smp_processor_id()); struct cpumask kstripe_mask; int rc, i; #if CHIP_HAS_CBOX_HOME_MAP() if (ktext_arg_seen && ktext_hash) { pr_warning("warning: \"ktext\" boot argument ignored" " if \"kcache_hash\" sets up text hash-for-home\n"); ktext_small = 0; } if (kdata_arg_seen && kdata_hash) { pr_warning("warning: \"kdata\" boot argument ignored" " if \"kcache_hash\" sets up data hash-for-home\n"); } if (kdata_huge && !hash_default) { pr_warning("warning: disabling \"kdata=huge\"; requires" " kcache_hash=all or =allbutstack\n"); kdata_huge = 0; } #endif /* * Set up a mask for cpus to use for kernel striping. * This is normally all cpus, but minus dataplane cpus if any. * If the dataplane covers the whole chip, we stripe over * the whole chip too. */ cpumask_copy(&kstripe_mask, cpu_possible_mask); if (!kdata_arg_seen) kdata_mask = kstripe_mask; /* Allocate and fill in L2 page tables */ for (i = 0; i < MAX_NUMNODES; ++i) { #ifdef CONFIG_HIGHMEM unsigned long end_pfn = node_lowmem_end_pfn[i]; #else unsigned long end_pfn = node_end_pfn[i]; #endif unsigned long end_huge_pfn = 0; /* Pre-shatter the last huge page to allow per-cpu pages. */ if (kdata_huge) end_huge_pfn = end_pfn - (HPAGE_SIZE >> PAGE_SHIFT); pfn = node_start_pfn[i]; /* Allocate enough memory to hold L2 page tables for node. */ init_prealloc_ptes(i, end_pfn - pfn); address = (unsigned long) pfn_to_kaddr(pfn); while (pfn < end_pfn) { BUG_ON(address & (HPAGE_SIZE-1)); pmd = get_pmd(pgtables, address); pte = get_prealloc_pte(pfn); if (pfn < end_huge_pfn) { pgprot_t prot = init_pgprot(address); *(pte_t *)pmd = pte_mkhuge(pfn_pte(pfn, prot)); for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE; pfn++, pte_ofs++, address += PAGE_SIZE) pte[pte_ofs] = pfn_pte(pfn, prot); } else { if (kdata_huge) printk(KERN_DEBUG "pre-shattered huge" " page at %#lx\n", address); for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE; pfn++, pte_ofs++, address += PAGE_SIZE) { pgprot_t prot = init_pgprot(address); pte[pte_ofs] = pfn_pte(pfn, prot); } assign_pte(pmd, pte); } } } /* * Set or check ktext_map now that we have cpu_possible_mask * and kstripe_mask to work with. */ if (ktext_all) cpumask_copy(&ktext_mask, cpu_possible_mask); else if (ktext_nondataplane) ktext_mask = kstripe_mask; else if (!cpumask_empty(&ktext_mask)) { /* Sanity-check any mask that was requested */ struct cpumask bad; cpumask_andnot(&bad, &ktext_mask, cpu_possible_mask); cpumask_and(&ktext_mask, &ktext_mask, cpu_possible_mask); if (!cpumask_empty(&bad)) { char buf[NR_CPUS * 5]; cpulist_scnprintf(buf, sizeof(buf), &bad); pr_info("ktext: not using unavailable cpus %s\n", buf); } if (cpumask_empty(&ktext_mask)) { pr_warning("ktext: no valid cpus; caching on %d.\n", smp_processor_id()); cpumask_copy(&ktext_mask, cpumask_of(smp_processor_id())); } } address = MEM_SV_INTRPT; pmd = get_pmd(pgtables, address); pfn = 0; /* code starts at PA 0 */ if (ktext_small) { /* Allocate an L2 PTE for the kernel text */ int cpu = 0; pgprot_t prot = construct_pgprot(PAGE_KERNEL_EXEC, PAGE_HOME_IMMUTABLE); if (ktext_local) { if (ktext_nocache) prot = hv_pte_set_mode(prot, HV_PTE_MODE_UNCACHED); else prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_NO_L3); } else { prot = hv_pte_set_mode(prot, HV_PTE_MODE_CACHE_TILE_L3); cpu = cpumask_first(&ktext_mask); prot = ktext_set_nocache(prot); } BUG_ON(address != (unsigned long)_stext); pte = NULL; for (; address < (unsigned long)_einittext; pfn++, address += PAGE_SIZE) { pte_ofs = pte_index(address); if (pte_ofs == 0) { if (pte) assign_pte(pmd++, pte); pte = alloc_pte(); } if (!ktext_local) { prot = set_remote_cache_cpu(prot, cpu); cpu = cpumask_next(cpu, &ktext_mask); if (cpu == NR_CPUS) cpu = cpumask_first(&ktext_mask); } pte[pte_ofs] = pfn_pte(pfn, prot); } if (pte) assign_pte(pmd, pte); } else { pte_t pteval = pfn_pte(0, PAGE_KERNEL_EXEC); pteval = pte_mkhuge(pteval); #if CHIP_HAS_CBOX_HOME_MAP() if (ktext_hash) { pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_HASH_L3); pteval = ktext_set_nocache(pteval); } else #endif /* CHIP_HAS_CBOX_HOME_MAP() */ if (cpumask_weight(&ktext_mask) == 1) { pteval = set_remote_cache_cpu(pteval, cpumask_first(&ktext_mask)); pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_TILE_L3); pteval = ktext_set_nocache(pteval); } else if (ktext_nocache) pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_UNCACHED); else pteval = hv_pte_set_mode(pteval, HV_PTE_MODE_CACHE_NO_L3); for (; address < (unsigned long)_einittext; pfn += PFN_DOWN(HPAGE_SIZE), address += HPAGE_SIZE) *(pte_t *)(pmd++) = pfn_pte(pfn, pteval); } /* Set swapper_pgprot here so it is flushed to memory right away. */ swapper_pgprot = init_pgprot((unsigned long)swapper_pg_dir); /* * Since we may be changing the caching of the stack and page * table itself, we invoke an assembly helper to do the * following steps: * * - flush the cache so we start with an empty slate * - install pgtables[] as the real page table * - flush the TLB so the new page table takes effect */ irqmask = interrupt_mask_save_mask(); interrupt_mask_set_mask(-1ULL); rc = flush_and_install_context(__pa(pgtables), init_pgprot((unsigned long)pgtables), __get_cpu_var(current_asid), cpumask_bits(my_cpu_mask)); interrupt_mask_restore_mask(irqmask); BUG_ON(rc != 0); /* Copy the page table back to the normal swapper_pg_dir. */ memcpy(pgd_base, pgtables, sizeof(pgtables)); __install_page_table(pgd_base, __get_cpu_var(current_asid), swapper_pgprot); /* * We just read swapper_pgprot and thus brought it into the cache, * with its new home & caching mode. When we start the other CPUs, * they're going to reference swapper_pgprot via their initial fake * VA-is-PA mappings, which cache everything locally. At that * time, if it's in our cache with a conflicting home, the * simulator's coherence checker will complain. So, flush it out * of our cache; we're not going to ever use it again anyway. */ __insn_finv(&swapper_pgprot); }
/* * Periodic broadcast: * - invoke the broadcast handlers */ static void tick_do_periodic_broadcast(void) { cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); tick_do_broadcast(tmpmask); }
/** * cpupri_find - find the best (lowest-pri) CPU in the system * @cp: The cpupri context * @p: The task * @lowest_mask: A mask to fill in with selected CPUs (or NULL) * * Note: This function returns the recommended CPUs as calculated during the * current invocation. By the time the call returns, the CPUs may have in * fact changed priorities any number of times. While not ideal, it is not * an issue of correctness since the normal rebalancer logic will correct * any discrepancies created by racing against the uncertainty of the current * priority configuration. * * Returns: (int)bool - CPUs were found */ int cpupri_find(struct cpupri *cp, struct task_struct *p, struct cpumask *lowest_mask) { int idx = 0; int task_pri = convert_prio(p->prio); if (task_pri >= MAX_RT_PRIO) return 0; for (idx = 0; idx < task_pri; idx++) { struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; int skip = 0; if (!atomic_read(&(vec)->count)) skip = 1; /* * When looking at the vector, we need to read the counter, * do a memory barrier, then read the mask. * * Note: This is still all racey, but we can deal with it. * Ideally, we only want to look at masks that are set. * * If a mask is not set, then the only thing wrong is that we * did a little more work than necessary. * * If we read a zero count but the mask is set, because of the * memory barriers, that can only happen when the highest prio * task for a run queue has left the run queue, in which case, * it will be followed by a pull. If the task we are processing * fails to find a proper place to go, that pull request will * pull this task if the run queue is running at a lower * priority. */ smp_rmb(); /* Need to do the rmb for every iteration */ if (skip) continue; if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids) continue; if (lowest_mask) { cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask); /* * We have to ensure that we have at least one bit * still set in the array, since the map could have * been concurrently emptied between the first and * second reads of vec->mask. If we hit this * condition, simply act as though we never hit this * priority level and continue on. */ if (cpumask_any(lowest_mask) >= nr_cpu_ids) continue; } return 1; } return 0; }
int cps_pm_enter_state(enum cps_pm_state state) { unsigned cpu = smp_processor_id(); unsigned core = current_cpu_data.core; unsigned online, left; cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled); u32 *core_ready_count, *nc_core_ready_count; void *nc_addr; cps_nc_entry_fn entry; struct core_boot_config *core_cfg; struct vpe_boot_config *vpe_cfg; /* Check that there is an entry function for this state */ entry = per_cpu(nc_asm_enter, core)[state]; if (!entry) return -EINVAL; /* Calculate which coupled CPUs (VPEs) are online */ #ifdef CONFIG_MIPS_MT if (cpu_online(cpu)) { cpumask_and(coupled_mask, cpu_online_mask, this_cpu_ptr(&cpu_sibling_map)); online = cpumask_weight(coupled_mask); cpumask_clear_cpu(cpu, coupled_mask); } else #endif { cpumask_clear(coupled_mask); online = 1; } /* Setup the VPE to run mips_cps_pm_restore when started again */ if (state == CPS_PM_POWER_GATED) { /* Power gating relies upon CPS SMP */ if (!mips_cps_smp_in_use()) return -EINVAL; core_cfg = &mips_cps_core_bootcfg[core]; vpe_cfg = &core_cfg->vpe_config[current_cpu_data.vpe_id]; vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; vpe_cfg->gp = (unsigned long)current_thread_info(); vpe_cfg->sp = 0; } /* Indicate that this CPU might not be coherent */ cpumask_clear_cpu(cpu, &cpu_coherent_mask); smp_mb__after_clear_bit(); /* Create a non-coherent mapping of the core ready_count */ core_ready_count = per_cpu(ready_count, core); nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), (unsigned long)core_ready_count); nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); nc_core_ready_count = nc_addr; /* Ensure ready_count is zero-initialised before the assembly runs */ ACCESS_ONCE(*nc_core_ready_count) = 0; coupled_barrier(&per_cpu(pm_barrier, core), online); /* Run the generated entry code */ left = entry(online, nc_core_ready_count); /* Remove the non-coherent mapping of ready_count */ kunmap_noncoherent(); /* Indicate that this CPU is definitely coherent */ cpumask_set_cpu(cpu, &cpu_coherent_mask); /* * If this VPE is the first to leave the non-coherent wait state then * it needs to wake up any coupled VPEs still running their wait * instruction so that they return to cpuidle, which can then complete * coordination between the coupled VPEs & provide the governor with * a chance to reflect on the length of time the VPEs were in the * idle state. */ if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) arch_send_call_function_ipi_mask(coupled_mask); return 0; }
/* * hps algo - hmp */ void hps_algo_hmp(void) { unsigned int cpu; unsigned int val; struct cpumask little_online_cpumask; struct cpumask big_online_cpumask; unsigned int little_num_base, little_num_limit, little_num_online; unsigned int big_num_base, big_num_limit, big_num_online; //log purpose char str1[64]; char str2[64]; int i, j; char * str1_ptr = str1; char * str2_ptr = str2; /* * run algo or not by hps_ctxt.enabled */ if (!hps_ctxt.enabled) { atomic_set(&hps_ctxt.is_ondemand, 0); return; } /* * calculate cpu loading */ hps_ctxt.cur_loads = 0; str1_ptr = str1; str2_ptr = str2; for_each_possible_cpu(cpu) { per_cpu(hps_percpu_ctxt, cpu).load = hps_cpu_get_percpu_load(cpu); hps_ctxt.cur_loads += per_cpu(hps_percpu_ctxt, cpu).load; if (hps_ctxt.cur_dump_enabled) { if (cpu_online(cpu)) i = sprintf(str1_ptr, "%4u", 1); else i = sprintf(str1_ptr, "%4u", 0); str1_ptr += i; j = sprintf(str2_ptr, "%4u", per_cpu(hps_percpu_ctxt, cpu).load); str2_ptr += j; } } hps_ctxt.cur_nr_heavy_task = hps_cpu_get_nr_heavy_task(); hps_cpu_get_tlp(&hps_ctxt.cur_tlp, &hps_ctxt.cur_iowait); /* * algo - begin */ mutex_lock(&hps_ctxt.lock); hps_ctxt.action = ACTION_NONE; atomic_set(&hps_ctxt.is_ondemand, 0); /* * algo - get boundary */ little_num_limit = min(hps_ctxt.little_num_limit_thermal, hps_ctxt.little_num_limit_low_battery); little_num_base = hps_ctxt.little_num_base_perf_serv; cpumask_and(&little_online_cpumask, &hps_ctxt.little_cpumask, cpu_online_mask); little_num_online = cpumask_weight(&little_online_cpumask); //TODO: no need if is_hmp big_num_limit = min(hps_ctxt.big_num_limit_thermal, hps_ctxt.big_num_limit_low_battery); big_num_base = max(hps_ctxt.cur_nr_heavy_task, hps_ctxt.big_num_base_perf_serv); cpumask_and(&big_online_cpumask, &hps_ctxt.big_cpumask, cpu_online_mask); big_num_online = cpumask_weight(&big_online_cpumask); if (hps_ctxt.cur_dump_enabled) { hps_debug(" CPU:%s\n", str1); hps_debug("LOAD:%s\n", str2); hps_debug("loads(%u), hvy_tsk(%u), tlp(%u), iowait(%u), limit_t(%u)(%u), limit_lb(%u)(%u), base_ps(%u)(%u)\n", hps_ctxt.cur_loads, hps_ctxt.cur_nr_heavy_task, hps_ctxt.cur_tlp, hps_ctxt.cur_iowait, hps_ctxt.little_num_limit_thermal, hps_ctxt.big_num_limit_thermal, hps_ctxt.little_num_limit_low_battery, hps_ctxt.big_num_limit_low_battery, hps_ctxt.little_num_base_perf_serv, hps_ctxt.big_num_base_perf_serv); } //ALGO_LIMIT: /* * algo - thermal, low battery */ if (big_num_online > big_num_limit) { val = big_num_online - big_num_limit; for (cpu = hps_ctxt.big_cpu_id_max; cpu >= hps_ctxt.big_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_LIMIT_BIG, (unsigned long *)&hps_ctxt.action); } if (little_num_online > little_num_limit) { val = little_num_online - little_num_limit; for (cpu = hps_ctxt.little_cpu_id_max; cpu > hps_ctxt.little_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &little_online_cpumask); --little_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_LIMIT_LITTLE, (unsigned long *)&hps_ctxt.action); } if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_BASE: /* * algo - PerfService, heavy task detect */ BUG_ON(big_num_online > big_num_limit); BUG_ON(little_num_online > little_num_limit); if ((big_num_online < big_num_base) && (big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { val = min(big_num_base, big_num_limit) - big_num_online; for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_BASE_BIG, (unsigned long *)&hps_ctxt.action); } if ((little_num_online < little_num_base) && (little_num_online < little_num_limit) && (little_num_online + big_num_online < hps_ctxt.little_num_base_perf_serv + hps_ctxt.big_num_base_perf_serv)) { val = min(little_num_base, little_num_limit) - little_num_online; if (big_num_online > hps_ctxt.big_num_base_perf_serv) val -= big_num_online - hps_ctxt.big_num_base_perf_serv; for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; if (--val == 0) break; } } BUG_ON(val); set_bit(ACTION_BASE_LITTLE, (unsigned long *)&hps_ctxt.action); } if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; /* * update history - tlp */ val = hps_ctxt.tlp_history[hps_ctxt.tlp_history_index]; hps_ctxt.tlp_history[hps_ctxt.tlp_history_index] = hps_ctxt.cur_tlp; hps_ctxt.tlp_sum += hps_ctxt.cur_tlp; hps_ctxt.tlp_history_index = (hps_ctxt.tlp_history_index + 1 == hps_ctxt.tlp_times) ? 0 : hps_ctxt.tlp_history_index + 1; ++hps_ctxt.tlp_count; if (hps_ctxt.tlp_count > hps_ctxt.tlp_times) { BUG_ON(hps_ctxt.tlp_sum < val); hps_ctxt.tlp_sum -= val; hps_ctxt.tlp_avg = hps_ctxt.tlp_sum / hps_ctxt.tlp_times; } else { hps_ctxt.tlp_avg = hps_ctxt.tlp_sum / hps_ctxt.tlp_count; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_tlp(0); //ALGO_RUSH_BOOST: /* * algo - rush boost */ if (hps_ctxt.rush_boost_enabled) { if (hps_ctxt.cur_loads > hps_ctxt.rush_boost_threshold * (little_num_online + big_num_online)) ++hps_ctxt.rush_count; else hps_ctxt.rush_count = 0; if ((hps_ctxt.rush_count >= hps_ctxt.rush_boost_times) && ((little_num_online + big_num_online) * 100 < hps_ctxt.tlp_avg)) { val = hps_ctxt.tlp_avg / 100 + (hps_ctxt.tlp_avg % 100 ? 1 : 0); BUG_ON(!(val > little_num_online + big_num_online)); if (val > num_possible_cpus()) val = num_possible_cpus(); val -= little_num_online + big_num_online; if ((val) && (little_num_online < little_num_limit)) { for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; if (--val == 0) break; } } set_bit(ACTION_RUSH_BOOST_LITTLE, (unsigned long *)&hps_ctxt.action); } else if ((val) && (big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; if (--val == 0) break; } } set_bit(ACTION_RUSH_BOOST_BIG, (unsigned long *)&hps_ctxt.action); } } } //if (hps_ctxt.rush_boost_enabled) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_UP: /* * algo - cpu up */ if ((little_num_online + big_num_online) < num_possible_cpus()) { /* * update history - up */ val = hps_ctxt.up_loads_history[hps_ctxt.up_loads_history_index]; hps_ctxt.up_loads_history[hps_ctxt.up_loads_history_index] = hps_ctxt.cur_loads; hps_ctxt.up_loads_sum += hps_ctxt.cur_loads; hps_ctxt.up_loads_history_index = (hps_ctxt.up_loads_history_index + 1 == hps_ctxt.up_times) ? 0 : hps_ctxt.up_loads_history_index + 1; ++hps_ctxt.up_loads_count; //XXX: use >= or >, which is benifit? use > if (hps_ctxt.up_loads_count > hps_ctxt.up_times) { BUG_ON(hps_ctxt.up_loads_sum < val); hps_ctxt.up_loads_sum -= val; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_up(0); if (hps_ctxt.up_loads_count >= hps_ctxt.up_times) { if (hps_ctxt.up_loads_sum > hps_ctxt.up_threshold * hps_ctxt.up_times * (little_num_online + big_num_online)) { if (little_num_online < little_num_limit) { for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; break; } } set_bit(ACTION_UP_LITTLE, (unsigned long *)&hps_ctxt.action); } else if ((big_num_online < big_num_limit) && (hps_ctxt.state == STATE_LATE_RESUME)) { for (cpu = hps_ctxt.big_cpu_id_min; cpu <= hps_ctxt.big_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &big_online_cpumask); ++big_num_online; break; } } set_bit(ACTION_UP_BIG, (unsigned long *)&hps_ctxt.action); } } } //if (hps_ctxt.up_loads_count >= hps_ctxt.up_times) } //if ((little_num_online + big_num_online) < num_possible_cpus()) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_DOWN: /* * algo - cpu down (inc. quick landing) */ if (little_num_online + big_num_online > 1) { /* * update history - down */ val = hps_ctxt.down_loads_history[hps_ctxt.down_loads_history_index]; hps_ctxt.down_loads_history[hps_ctxt.down_loads_history_index] = hps_ctxt.cur_loads; hps_ctxt.down_loads_sum += hps_ctxt.cur_loads; hps_ctxt.down_loads_history_index = (hps_ctxt.down_loads_history_index + 1 == hps_ctxt.down_times) ? 0 : hps_ctxt.down_loads_history_index + 1; ++hps_ctxt.down_loads_count; //XXX: use >= or >, which is benifit? use > if (hps_ctxt.down_loads_count > hps_ctxt.down_times) { BUG_ON(hps_ctxt.down_loads_sum < val); hps_ctxt.down_loads_sum -= val; } if (hps_ctxt.stats_dump_enabled) hps_ctxt_print_algo_stats_down(0); if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) { unsigned int down_threshold = hps_ctxt.down_threshold * hps_ctxt.down_times; val = little_num_online + big_num_online; while (hps_ctxt.down_loads_sum < down_threshold * (val - 1)) --val; val = little_num_online + big_num_online - val; if ((val) && (big_num_online > big_num_base)) { for (cpu = hps_ctxt.big_cpu_id_max; cpu >= hps_ctxt.big_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &big_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; if (--val == 0) break; } } set_bit(ACTION_DOWN_BIG, (unsigned long *)&hps_ctxt.action); } else if ((val) && (little_num_online > little_num_base)) { for (cpu = hps_ctxt.little_cpu_id_max; cpu > hps_ctxt.little_cpu_id_min; --cpu) { if (cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_down(cpu); cpumask_clear_cpu(cpu, &little_online_cpumask); --little_num_online; if (--val == 0) break; } } set_bit(ACTION_DOWN_LITTLE, (unsigned long *)&hps_ctxt.action); } } //if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) } //if (little_num_online + big_num_online > 1) if (hps_ctxt.action) goto ALGO_END_WITH_ACTION; //ALGO_BIG_TO_LITTLE: /* * algo - b2L */ if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) { if ((little_num_online < little_num_limit) && (big_num_online > big_num_base)) { //find last online big for (val = hps_ctxt.big_cpu_id_max; val >= hps_ctxt.big_cpu_id_min; --val) { if (cpumask_test_cpu(val, &big_online_cpumask)) break; } BUG_ON(val < hps_ctxt.big_cpu_id_min); //verify whether b2L will open 1 little if (per_cpu(hps_percpu_ctxt, val).load * CPU_DMIPS_BIG_LITTLE_DIFF / 100 + hps_ctxt.up_loads_sum / hps_ctxt.up_times <= hps_ctxt.up_threshold * (little_num_online + big_num_online)) { //up 1 little for (cpu = hps_ctxt.little_cpu_id_min; cpu <= hps_ctxt.little_cpu_id_max; ++cpu) { if (!cpumask_test_cpu(cpu, &little_online_cpumask)) { cpu_up(cpu); cpumask_set_cpu(cpu, &little_online_cpumask); ++little_num_online; break; } } //down 1 big cpu_down(val); cpumask_clear_cpu(cpu, &big_online_cpumask); --big_num_online; set_bit(ACTION_BIG_TO_LITTLE, (unsigned long *)&hps_ctxt.action); } } //if ((little_num_online < little_num_limit) && (big_num_online > big_num_base)) } //if (hps_ctxt.down_loads_count >= hps_ctxt.down_times) if (!hps_ctxt.action) goto ALGO_END_WO_ACTION; /* * algo - end */ ALGO_END_WITH_ACTION: hps_warn("(%04x)(%u)(%u)action end(%u)(%u)(%u)(%u) (%u)(%u)(%u)(%u)(%u)(%u) (%u)(%u)(%u) (%u)(%u)(%u) (%u)(%u)(%u)(%u)(%u)\n", hps_ctxt.action, little_num_online, big_num_online, hps_ctxt.cur_loads, hps_ctxt.cur_tlp, hps_ctxt.cur_iowait, hps_ctxt.cur_nr_heavy_task, hps_ctxt.little_num_limit_thermal, hps_ctxt.big_num_limit_thermal, hps_ctxt.little_num_limit_low_battery, hps_ctxt.big_num_limit_low_battery, hps_ctxt.little_num_base_perf_serv, hps_ctxt.big_num_base_perf_serv, hps_ctxt.up_loads_sum, hps_ctxt.up_loads_count, hps_ctxt.up_loads_history_index, hps_ctxt.down_loads_sum, hps_ctxt.down_loads_count, hps_ctxt.down_loads_history_index, hps_ctxt.rush_count, hps_ctxt.tlp_sum, hps_ctxt.tlp_count, hps_ctxt.tlp_history_index, hps_ctxt.tlp_avg); hps_ctxt_reset_stas_nolock(); ALGO_END_WO_ACTION: mutex_unlock(&hps_ctxt.lock); return; }
/* Propagate any changes to the watchdog threads */ static void proc_watchdog_update(void) { /* Remove impossible cpus to keep sysctl output clean. */ cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask); lockup_detector_reconfigure(); }