/* * Generic version of the affinity autoselector. */ static int setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask) { struct cpumask *set = irq_default_affinity; int node = desc->irq_data.node; /* Excludes PER_CPU and NO_BALANCE interrupts */ if (!irq_can_set_affinity(irq)) return 0; /* * Preserve an userspace affinity setup, but make sure that * one of the targets is online. */ if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { if (cpumask_intersects(desc->irq_data.affinity, cpu_online_mask)) set = desc->irq_data.affinity; else irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); } cpumask_and(mask, cpu_online_mask, set); if (node != NUMA_NO_NODE) { const struct cpumask *nodemask = cpumask_of_node(node); /* make sure at least one of the cpus in nodemask is online */ if (cpumask_intersects(mask, nodemask)) cpumask_and(mask, mask, nodemask); } irq_do_set_affinity(&desc->irq_data, mask, false); return 0; }
/* * Generic version of the affinity autoselector. */ static int setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask) { struct irq_chip *chip = irq_desc_get_chip(desc); struct cpumask *set = irq_default_affinity; int ret; /* Excludes PER_CPU and NO_BALANCE interrupts */ if (!irq_can_set_affinity(irq)) return 0; /* * Preserve an userspace affinity setup, but make sure that * one of the targets is online. */ if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { if (cpumask_intersects(desc->irq_data.affinity, cpu_online_mask)) set = desc->irq_data.affinity; else irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); } cpumask_and(mask, cpu_online_mask, set); ret = chip->irq_set_affinity(&desc->irq_data, mask, false); switch (ret) { case IRQ_SET_MASK_OK: cpumask_copy(desc->irq_data.affinity, mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); } return 0; }
/* * Generic version of the affinity autoselector. */ static int setup_affinity(unsigned int irq, struct irq_desc *desc) { if (!irq_can_set_affinity(irq)) return 0; /* * Preserve an userspace affinity setup, but make sure that * one of the targets is online. */ if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { if (cpumask_any_and(desc->affinity, cpu_online_mask) < nr_cpu_ids) goto set_affinity; else desc->status &= ~IRQ_AFFINITY_SET; } cpumask_and(desc->affinity, cpu_online_mask, irq_default_affinity); if (desc->node != NUMA_NO_NODE) { const struct cpumask *nodemask = cpumask_of_node(desc->node); /* make sure at least one of the cpus in nodemask is online */ if (cpumask_intersects(desc->affinity, nodemask)) cpumask_and(desc->affinity, desc->affinity, nodemask); } set_affinity: desc->chip->set_affinity(irq, desc->affinity); return 0; }
static ssize_t default_affinity_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { cpumask_var_t new_value; int err; if (!alloc_cpumask_var(&new_value, GFP_KERNEL)) return -ENOMEM; err = cpumask_parse_user(buffer, count, new_value); if (err) goto out; if (!is_affinity_mask_valid(new_value)) { err = -EINVAL; goto out; } /* * Do not allow disabling IRQs completely - it's a too easy * way to make the system unusable accidentally :-) At least * one online CPU still has to be targeted. */ if (!cpumask_intersects(new_value, cpu_online_mask)) { err = -EINVAL; goto out; } cpumask_copy(irq_default_affinity, new_value); err = count; out: free_cpumask_var(new_value); return err; }
/* * This variant of smp_call_function is used for index cacheops only. */ static inline void r4k_indexop_on_each_cpu(void (*func) (void *info), void *info) { preempt_disable(); #ifdef CONFIG_SMP if (!cpu_has_safe_index_cacheops) { if (smp_num_siblings > 1) { cpumask_t tmp_mask = INIT_CPUMASK; int cpu, this_cpu, n = 0; /* If processor hasn't safe index cachops (likely) then run cache flush on other CPUs. But I assume that siblings have common L1 cache, so - - run cache flush only once per sibling group. LY22 */ this_cpu = smp_processor_id(); for_each_online_cpu(cpu) { if (cpumask_test_cpu(cpu, (&per_cpu(cpu_sibling_map, this_cpu)))) continue; if (cpumask_intersects(&tmp_mask, (&per_cpu(cpu_sibling_map, cpu)))) continue; cpu_set(cpu, tmp_mask); n++; } if (n) smp_call_function_many(&tmp_mask, func, info, 1); } else
/* * This is held with the irq desc lock held, so we don't require any * additional locking here at the intc desc level. The affinity mask is * later tested in the enable/disable paths. */ static int intc_set_affinity(struct irq_data *data, const struct cpumask *cpumask, bool force) { if (!cpumask_intersects(cpumask, cpu_online_mask)) return -1; cpumask_copy(data->affinity, cpumask); return IRQ_SET_MASK_OK_NOCOPY; }
/* * Migrate the IO-APIC irq in the presence of intr-remapping. * * For both level and edge triggered, irq migration is a simple atomic * update(of vector and cpu destination) of IRTE and flush the hardware cache. * * For level triggered, we eliminate the io-apic RTE modification (with the * updated vector information), by using a virtual vector (io-apic pin number). * Real vector that is used for interrupting cpu will be coming from * the interrupt-remapping table entry. * * As the migration is a simple atomic update of IRTE, the same mechanism * is used to migrate MSI irq's in the presence of interrupt-remapping. */ static int intel_ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { struct irq_cfg *cfg = data->chip_data; unsigned int dest, irq = data->irq; struct irte irte; int err; if (!config_enabled(CONFIG_SMP)) return -EINVAL; if (!cpumask_intersects(mask, cpu_online_mask)) return -EINVAL; if (get_irte(irq, &irte)) return -EBUSY; err = assign_irq_vector(irq, cfg, mask); if (err) return err; err = apic->cpu_mask_to_apicid_and(cfg->domain, mask, &dest); if (err) { if (assign_irq_vector(irq, cfg, data->affinity)) pr_err("Failed to recover vector for irq %d\n", irq); return err; } irte.vector = cfg->vector; irte.dest_id = IRTE_DEST(dest); /* * Atomically updates the IRTE with the new destination, vector * and flushes the interrupt entry cache. */ modify_irte(irq, &irte); /* * After this point, all the interrupts will start arriving * at the new destination. So, time to cleanup the previous * vector allocation. */ if (cfg->move_in_progress) send_cleanup_vector(cfg); cpumask_copy(data->affinity, mask); return 0; }
static int setup_affinity(unsigned int irq, struct irq_desc *desc, struct cpumask *mask) { struct irq_chip *chip = irq_desc_get_chip(desc); struct cpumask *set = irq_default_affinity; int ret, node = desc->irq_data.node; if (!irq_can_set_affinity(irq)) return 0; if (irqd_has_set(&desc->irq_data, IRQD_AFFINITY_SET)) { if (cpumask_intersects(desc->irq_data.affinity, cpu_online_mask)) set = desc->irq_data.affinity; else irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); } cpumask_and(mask, cpu_online_mask, set); if (node != NUMA_NO_NODE) { const struct cpumask *nodemask = cpumask_of_node(node); if (cpumask_intersects(mask, nodemask)) cpumask_and(mask, mask, nodemask); } ret = chip->irq_set_affinity(&desc->irq_data, mask, false); switch (ret) { case IRQ_SET_MASK_OK: cpumask_copy(desc->irq_data.affinity, mask); case IRQ_SET_MASK_OK_NOCOPY: irq_set_thread_affinity(desc); } return 0; }
/* Cause workers to balance themselves appropriately to the available CPU cores. In case of 2 workers, balance on two CPU sets which are exact xor of each other. */ static void cpu_balance2(struct task_struct *task, unsigned int worker_id) { const unsigned long cpumasks[2] = { 0x55555555UL, 0xAAAAAAAAUL }; struct cpumask cpumask = { CPU_BITS_NONE }; int ret; /* Test if CPU mask has desired "holes". */ cpumask_set_cpu(1 - worker_id, &cpumask); if (!cpumask_intersects(&task->cpus_allowed, &cpumask)) { /* The mask is already correct, no need to balance. */ return; } cpumask.bits[0] = cpumasks[worker_id]; /* Move the worker to the designated CPU. */ ret = set_cpus_allowed_ptr(task, &cpumask); /* ret == 0 => Changed to the appropriate CPU. ret == -EINVAL => The desired CPU is not currently available. */ }
static ssize_t irq_affinity_proc_write(struct file *file, const char __user *buffer, size_t count, loff_t *pos) { unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data; cpumask_var_t new_value; int err; if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity || irq_balancing_disabled(irq)) return -EIO; if (!alloc_cpumask_var(&new_value, GFP_KERNEL)) return -ENOMEM; err = cpumask_parse_user(buffer, count, new_value); if (err) goto free_cpumask; if (!is_affinity_mask_valid(new_value)) { err = -EINVAL; goto free_cpumask; } /* * Do not allow disabling IRQs completely - it's a too easy * way to make the system unusable accidentally :-) At least * one online CPU still has to be targeted. */ if (!cpumask_intersects(new_value, cpu_online_mask)) { /* Special case for empty set - allow the architecture code to set default SMP affinity. */ err = irq_select_affinity_usr(irq) ? -EINVAL : count; } else { irq_set_affinity(irq, new_value); err = count; } free_cpumask: free_cpumask_var(new_value); return err; }
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, struct cpumask *groupmask) { struct sched_group *group = sd->groups; cpumask_clear(groupmask); printk(KERN_DEBUG "%*s domain-%d: ", level, "", level); if (!(sd->flags & SD_LOAD_BALANCE)) { printk("does not load-balance\n"); if (sd->parent) printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent"); return -1; } printk(KERN_CONT "span=%*pbl level=%s\n", cpumask_pr_args(sched_domain_span(sd)), sd->name); if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu); } if (!cpumask_test_cpu(cpu, sched_group_span(group))) { printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu); } printk(KERN_DEBUG "%*s groups:", level + 1, ""); do { if (!group) { printk("\n"); printk(KERN_ERR "ERROR: group is NULL\n"); break; } if (!cpumask_weight(sched_group_span(group))) { printk(KERN_CONT "\n"); printk(KERN_ERR "ERROR: empty group\n"); break; } if (!(sd->flags & SD_OVERLAP) && cpumask_intersects(groupmask, sched_group_span(group))) { printk(KERN_CONT "\n"); printk(KERN_ERR "ERROR: repeated CPUs\n"); break; } cpumask_or(groupmask, groupmask, sched_group_span(group)); printk(KERN_CONT " %d:{ span=%*pbl", group->sgc->id, cpumask_pr_args(sched_group_span(group))); if ((sd->flags & SD_OVERLAP) && !cpumask_equal(group_balance_mask(group), sched_group_span(group))) { printk(KERN_CONT " mask=%*pbl", cpumask_pr_args(group_balance_mask(group))); } if (group->sgc->capacity != SCHED_CAPACITY_SCALE) printk(KERN_CONT " cap=%lu", group->sgc->capacity); if (group == sd->groups && sd->child && !cpumask_equal(sched_domain_span(sd->child), sched_group_span(group))) { printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n"); } printk(KERN_CONT " }"); group = group->next; if (group != sd->groups) printk(KERN_CONT ","); } while (group != sd->groups); printk(KERN_CONT "\n"); if (!cpumask_equal(sched_domain_span(sd), groupmask)) printk(KERN_ERR "ERROR: groups don't span domain->span\n"); if (sd->parent && !cpumask_subset(groupmask, sched_domain_span(sd->parent))) printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n"); return 0; }
/* * mipsmt_sys_sched_setaffinity - set the cpu affinity of a process */ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, unsigned long __user *user_mask_ptr) { cpumask_var_t cpus_allowed, new_mask, effective_mask; struct thread_info *ti; struct task_struct *p; int retval; if (len < sizeof(new_mask)) return -EINVAL; if (copy_from_user(&new_mask, user_mask_ptr, sizeof(new_mask))) return -EFAULT; get_online_cpus(); rcu_read_lock(); p = find_process_by_pid(pid); if (!p) { rcu_read_unlock(); put_online_cpus(); return -ESRCH; } /* Prevent p going away */ get_task_struct(p); rcu_read_unlock(); if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { retval = -ENOMEM; goto out_put_task; } if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out_free_cpus_allowed; } if (!alloc_cpumask_var(&effective_mask, GFP_KERNEL)) { retval = -ENOMEM; goto out_free_new_mask; } if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) { retval = -EPERM; goto out_unlock; } retval = security_task_setscheduler(p); if (retval) goto out_unlock; /* Record new user-specified CPU set for future reference */ cpumask_copy(&p->thread.user_cpus_allowed, new_mask); again: /* Compute new global allowed CPU set if necessary */ ti = task_thread_info(p); if (test_ti_thread_flag(ti, TIF_FPUBOUND) && cpumask_intersects(new_mask, &mt_fpu_cpumask)) { cpumask_and(effective_mask, new_mask, &mt_fpu_cpumask); retval = set_cpus_allowed_ptr(p, effective_mask); } else { cpumask_copy(effective_mask, new_mask); clear_ti_thread_flag(ti, TIF_FPUBOUND); retval = set_cpus_allowed_ptr(p, new_mask); } if (!retval) { cpuset_cpus_allowed(p, cpus_allowed); if (!cpumask_subset(effective_mask, cpus_allowed)) { /* * We must have raced with a concurrent cpuset * update. Just reset the cpus_allowed to the * cpuset's cpus_allowed */ cpumask_copy(new_mask, cpus_allowed); goto again; } } out_unlock: free_cpumask_var(effective_mask); out_free_new_mask: free_cpumask_var(new_mask); out_free_cpus_allowed: free_cpumask_var(cpus_allowed); out_put_task: put_task_struct(p); put_online_cpus(); return retval; }