static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
unsigned long count, void *data)
{
unsigned int irq = (int)(long)data, full_count = count, err;
cpumask_t new_value, tmp;
if (!irq_desc[irq].chip->set_affinity || no_irq_affinity ||
irq_balancing_disabled(irq))
return -EIO;
err = cpumask_parse_user(buffer, count, new_value);
if (err)
return err;
if (!is_affinity_mask_valid(new_value))
return -EINVAL;
/*
* Do not allow disabling IRQs completely - it's a too easy
* way to make the system unusable accidentally :-) At least
* one online CPU still has to be targeted.
*/
cpus_and(tmp, new_value, cpu_online_map);
if (cpus_empty(tmp))
/* Special case for empty set - allow the architecture
code to set default SMP affinity. */
return select_smp_affinity(irq) ? -EINVAL : full_count;
irq_set_affinity(irq, new_value);
return full_count;
}
static ssize_t default_affinity_write(struct file *file,
const char __user *buffer, size_t count, loff_t *ppos)
{
cpumask_var_t new_value;
int err;
if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
return -ENOMEM;
err = cpumask_parse_user(buffer, count, new_value);
if (err)
goto out;
if (!is_affinity_mask_valid(new_value)) {
err = -EINVAL;
goto out;
}
/*
* Do not allow disabling IRQs completely - it's a too easy
* way to make the system unusable accidentally :-) At least
* one online CPU still has to be targeted.
*/
if (!cpumask_intersects(new_value, cpu_online_mask)) {
err = -EINVAL;
goto out;
}
cpumask_copy(irq_default_affinity, new_value);
err = count;
out:
free_cpumask_var(new_value);
return err;
}
static ssize_t default_affinity_write(struct file *file,
const char __user *buffer, size_t count, loff_t *ppos)
{
cpumask_t new_value;
int err;
err = cpumask_parse_user(buffer, count, new_value);
if (err)
return err;
if (!is_affinity_mask_valid(new_value))
return -EINVAL;
/*
* Do not allow disabling IRQs completely - it's a too easy
* way to make the system unusable accidentally :-) At least
* one online CPU still has to be targeted.
*/
if (!cpus_intersects(new_value, cpu_online_map))
return -EINVAL;
irq_default_affinity = new_value;
return count;
}
static ssize_t irq_affinity_proc_write(struct file *file,
const char __user *buffer, size_t count, loff_t *pos)
{
unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data;
cpumask_t new_value;
int err;
if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity ||
irq_balancing_disabled(irq))
return -EIO;
err = cpumask_parse_user(buffer, count, new_value);
if (err)
return err;
if (!is_affinity_mask_valid(new_value))
return -EINVAL;
/*
* Do not allow disabling IRQs completely - it's a too easy
* way to make the system unusable accidentally :-) At least
* one online CPU still has to be targeted.
*/
if (!cpus_intersects(new_value, cpu_online_map))
/* Special case for empty set - allow the architecture
code to set default SMP affinity. */
return irq_select_affinity(irq) ? -EINVAL : count;
irq_set_affinity(irq, new_value);
return count;
}
static int prof_cpu_mask_write_proc (struct file *file, const char __user *buffer,
unsigned long count, void *data)
{
cpumask_t *mask = (cpumask_t *)data;
unsigned long full_count = count, err;
cpumask_t new_value;
err = cpumask_parse_user(buffer, count, new_value);
if (err)
return err;
*mask = new_value;
return full_count;
}
static ssize_t prof_cpu_mask_proc_write(struct file *file,
const char __user *buffer, size_t count, loff_t *pos)
{
cpumask_var_t new_value;
int err;
if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
return -ENOMEM;
err = cpumask_parse_user(buffer, count, new_value);
if (!err) {
cpumask_copy(prof_cpu_mask, new_value);
err = count;
}
free_cpumask_var(new_value);
return err;
}
static int rps_sock_flow_sysctl(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp, loff_t *ppos)
{
unsigned int orig_size, size;
int ret, i;
struct ctl_table tmp = {
.data = &size,
.maxlen = sizeof(size),
.mode = table->mode
};
struct rps_sock_flow_table *orig_sock_table, *sock_table;
static DEFINE_MUTEX(sock_flow_mutex);
mutex_lock(&sock_flow_mutex);
orig_sock_table = rcu_dereference_protected(rps_sock_flow_table,
lockdep_is_held(&sock_flow_mutex));
size = orig_size = orig_sock_table ? orig_sock_table->mask + 1 : 0;
ret = proc_dointvec(&tmp, write, buffer, lenp, ppos);
if (write) {
if (size) {
if (size > 1<<29) {
/* Enforce limit to prevent overflow */
mutex_unlock(&sock_flow_mutex);
return -EINVAL;
}
size = roundup_pow_of_two(size);
if (size != orig_size) {
sock_table =
vmalloc(RPS_SOCK_FLOW_TABLE_SIZE(size));
if (!sock_table) {
mutex_unlock(&sock_flow_mutex);
return -ENOMEM;
}
rps_cpu_mask = roundup_pow_of_two(nr_cpu_ids) - 1;
sock_table->mask = size - 1;
} else
sock_table = orig_sock_table;
for (i = 0; i < size; i++)
sock_table->ents[i] = RPS_NO_CPU;
} else
sock_table = NULL;
if (sock_table != orig_sock_table) {
rcu_assign_pointer(rps_sock_flow_table, sock_table);
if (sock_table)
static_key_slow_inc(&rps_needed);
if (orig_sock_table) {
static_key_slow_dec(&rps_needed);
synchronize_rcu();
vfree(orig_sock_table);
}
}
}
mutex_unlock(&sock_flow_mutex);
return ret;
}
#endif /* CONFIG_RPS */
#ifdef CONFIG_NET_FLOW_LIMIT
static DEFINE_MUTEX(flow_limit_update_mutex);
static int flow_limit_cpu_sysctl(struct ctl_table *table, int write,
void __user *buffer, size_t *lenp,
loff_t *ppos)
{
struct sd_flow_limit *cur;
struct softnet_data *sd;
cpumask_var_t mask;
int i, len, ret = 0;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
return -ENOMEM;
if (write) {
ret = cpumask_parse_user(buffer, *lenp, mask);
if (ret)
goto done;
mutex_lock(&flow_limit_update_mutex);
len = sizeof(*cur) + netdev_flow_limit_table_len;
for_each_possible_cpu(i) {
sd = &per_cpu(softnet_data, i);
cur = rcu_dereference_protected(sd->flow_limit,
lockdep_is_held(&flow_limit_update_mutex));
if (cur && !cpumask_test_cpu(i, mask)) {
RCU_INIT_POINTER(sd->flow_limit, NULL);
synchronize_rcu();
kfree(cur);
} else if (!cur && cpumask_test_cpu(i, mask)) {
cur = kzalloc_node(len, GFP_KERNEL,
cpu_to_node(i));
if (!cur) {
/* not unwinding previous changes */
ret = -ENOMEM;
goto write_unlock;
}
cur->num_buckets = netdev_flow_limit_table_len;
rcu_assign_pointer(sd->flow_limit, cur);
}
}
write_unlock:
mutex_unlock(&flow_limit_update_mutex);
} else {
char kbuf[128];
if (*ppos || !*lenp) {
*lenp = 0;
goto done;
}
cpumask_clear(mask);
rcu_read_lock();
for_each_possible_cpu(i) {
sd = &per_cpu(softnet_data, i);
if (rcu_dereference(sd->flow_limit))
cpumask_set_cpu(i, mask);
}
rcu_read_unlock();
len = min(sizeof(kbuf) - 1, *lenp);
len = scnprintf(kbuf, len, "%*pb", cpumask_pr_args(mask));
if (!len) {
*lenp = 0;
goto done;
}
if (len < *lenp)
kbuf[len++] = '\n';
if (copy_to_user(buffer, kbuf, len)) {
ret = -EFAULT;
goto done;
}
*lenp = len;
*ppos += len;
}
done:
free_cpumask_var(mask);
return ret;
}
static void do_one_cpu(char *path)
{
struct topo_obj *cpu;
FILE *file;
char new_path[PATH_MAX];
cpumask_t cache_mask, package_mask;
struct topo_obj *cache;
struct topo_obj *package;
DIR *dir;
struct dirent *entry;
int nodeid;
int packageid = 0;
unsigned int max_cache_index, cache_index, cache_stat;
/* skip offline cpus */
snprintf(new_path, PATH_MAX, "%s/online", path);
file = fopen(new_path, "r");
if (file) {
char *line = NULL;
size_t size = 0;
if (getline(&line, &size, file)==0)
return;
fclose(file);
if (line && line[0]=='0') {
free(line);
return;
}
free(line);
}
cpu = calloc(sizeof(struct topo_obj), 1);
if (!cpu)
return;
cpu->obj_type = OBJ_TYPE_CPU;
cpu->number = strtoul(&path[27], NULL, 10);
cpu_set(cpu->number, cpu_possible_map);
cpu_set(cpu->number, cpu->mask);
/*
* Default the cache_domain mask to be equal to the cpu
*/
cpus_clear(cache_mask);
cpu_set(cpu->number, cache_mask);
/* if the cpu is on the banned list, just don't add it */
if (cpus_intersects(cpu->mask, banned_cpus)) {
free(cpu);
/* even though we don't use the cpu we do need to count it */
core_count++;
return;
}
/* try to read the package mask; if it doesn't exist assume solitary */
snprintf(new_path, PATH_MAX, "%s/topology/core_siblings", path);
file = fopen(new_path, "r");
cpu_set(cpu->number, package_mask);
if (file) {
char *line = NULL;
size_t size = 0;
if (getline(&line, &size, file))
cpumask_parse_user(line, strlen(line), package_mask);
fclose(file);
free(line);
}
/* try to read the package id */
snprintf(new_path, PATH_MAX, "%s/topology/physical_package_id", path);
file = fopen(new_path, "r");
if (file) {
char *line = NULL;
size_t size = 0;
if (getline(&line, &size, file))
packageid = strtoul(line, NULL, 10);
fclose(file);
free(line);
}
/* try to read the cache mask; if it doesn't exist assume solitary */
/* We want the deepest cache level available */
cpu_set(cpu->number, cache_mask);
max_cache_index = 0;
cache_index = 1;
cache_stat = 0;
do {
struct stat sb;
snprintf(new_path, PATH_MAX, "%s/cache/index%d/shared_cpu_map", path, cache_index);
cache_stat = stat(new_path, &sb);
if (!cache_stat) {
max_cache_index = cache_index;
if (max_cache_index == deepest_cache)
break;
cache_index ++;
}
} while(!cache_stat);
if (max_cache_index > 0) {
snprintf(new_path, PATH_MAX, "%s/cache/index%d/shared_cpu_map", path, max_cache_index);
file = fopen(new_path, "r");
if (file) {
char *line = NULL;
size_t size = 0;
if (getline(&line, &size, file))
cpumask_parse_user(line, strlen(line), cache_mask);
fclose(file);
free(line);
}
}
nodeid=-1;
if (numa_avail) {
dir = opendir(path);
do {
entry = readdir(dir);
if (!entry)
break;
if (strstr(entry->d_name, "node")) {
nodeid = strtoul(&entry->d_name[4], NULL, 10);
break;
}
} while (entry);
closedir(dir);
}
/*
blank out the banned cpus from the various masks so that interrupts
will never be told to go there
*/
cpus_and(cache_mask, cache_mask, unbanned_cpus);
cpus_and(package_mask, package_mask, unbanned_cpus);
cache = add_cpu_to_cache_domain(cpu, cache_mask);
package = add_cache_domain_to_package(cache, packageid, package_mask);
add_package_to_node(package, nodeid);
cpu->obj_type_list = &cpus;
cpus = g_list_append(cpus, cpu);
core_count++;
}
int main(int argc, char** argv)
{
if (argc>1 && strstr(argv[1],"debug"))
debug_mode=1;
if (argc>1 && strstr(argv[1],"oneshot"))
one_shot_mode=1;
if (getenv("IRQBALANCE_BANNED_CPUS")) {
cpumask_parse_user(getenv("IRQBALANCE_BANNED_CPUS"), strlen(getenv("IRQBALANCE_BANNED_CPUS")), banned_cpus);
}
if (getenv("IRQBALANCE_ONESHOT"))
one_shot_mode=1;
if (getenv("IRQBALANCE_DEBUG"))
debug_mode=1;
parse_cpu_tree();
/* On single core UP systems irqbalance obviously has no work to do */
if (core_count<2)
exit(EXIT_SUCCESS);
/* On dual core/hyperthreading shared cache systems just do a one shot setup */
if (cache_domain_count==1)
one_shot_mode = 1;
if (!debug_mode)
if (daemon(0,0))
exit(EXIT_FAILURE);
#ifdef HAVE_LIBCAP_NG
// Drop capabilities
capng_clear(CAPNG_SELECT_BOTH);
capng_lock();
capng_apply(CAPNG_SELECT_BOTH);
#endif
parse_proc_interrupts();
sleep(SLEEP_INTERVAL/4);
reset_counts();
parse_proc_interrupts();
pci_numa_scan();
calculate_workload();
sort_irq_list();
if (debug_mode)
dump_workloads();
while (1) {
sleep_approx(SLEEP_INTERVAL);
if (debug_mode)
printf("\n\n\n-----------------------------------------------------------------------------\n");
check_power_mode();
parse_proc_interrupts();
/* cope with cpu hotplug -- detected during /proc/interrupts parsing */
if (need_cpu_rescan) {
need_cpu_rescan = 0;
/* if there's a hotplug event we better turn off power mode for a bit until things settle */
power_mode = 0;
if (debug_mode)
printf("Rescanning cpu topology \n");
reset_counts();
clear_work_stats();
clear_cpu_tree();
parse_cpu_tree();
}
/* deal with NAPI */
account_for_nic_stats();
calculate_workload();
/* to cope with dynamic configurations we scan for new numa information
* once every 5 minutes
*/
if (counter % NUMA_REFRESH_INTERVAL == 16)
pci_numa_scan();
calculate_placement();
activate_mapping();
if (debug_mode)
dump_tree();
if (one_shot_mode)
break;
counter++;
}
return EXIT_SUCCESS;
}
