/* Translate global CPU index to local CPU index. This is needed for
* Power7 processors with multi-threading disabled. On those processors,
* the CPU mask has gaps for the unused threads (different from Intel
* processors) which need to be skipped over in the mask used in the
* set system call. */
void reset_cpuset(cpu_set_t *new_mask, cpu_set_t *cur_mask)
{
cpu_set_t full_mask, newer_mask;
int cur_offset, new_offset = 0, last_set = -1;
if (!_is_power_cpu())
return;
if (slurm_getaffinity(1, sizeof(full_mask), &full_mask)) {
/* Try to get full CPU mask from process init */
CPU_ZERO(&full_mask);
#ifdef __FreeBSD__
CPU_OR(&full_mask, cur_mask);
#else
CPU_OR(&full_mask, &full_mask, cur_mask);
#endif
}
CPU_ZERO(&newer_mask);
for (cur_offset = 0; cur_offset < CPU_SETSIZE; cur_offset++) {
if (!CPU_ISSET(cur_offset, &full_mask))
continue;
if (CPU_ISSET(new_offset, new_mask)) {
CPU_SET(cur_offset, &newer_mask);
last_set = cur_offset;
}
new_offset++;
}
CPU_ZERO(new_mask);
for (cur_offset = 0; cur_offset <= last_set; cur_offset++) {
if (CPU_ISSET(cur_offset, &newer_mask))
CPU_SET(cur_offset, new_mask);
}
}
static int
smp_topo_addleaf(struct cpu_group *parent, struct cpu_group *child, int share,
int count, int flags, int start)
{
char cpusetbuf[CPUSETBUFSIZ], cpusetbuf2[CPUSETBUFSIZ];
cpuset_t mask;
int i;
CPU_ZERO(&mask);
for (i = 0; i < count; i++, start++)
CPU_SET(start, &mask);
child->cg_parent = parent;
child->cg_child = NULL;
child->cg_children = 0;
child->cg_level = share;
child->cg_count = count;
child->cg_flags = flags;
child->cg_mask = mask;
parent->cg_children++;
for (; parent != NULL; parent = parent->cg_parent) {
if (CPU_OVERLAP(&parent->cg_mask, &child->cg_mask))
panic("Duplicate children in %p. mask (%s) child (%s)",
parent,
cpusetobj_strprint(cpusetbuf, &parent->cg_mask),
cpusetobj_strprint(cpusetbuf2, &child->cg_mask));
CPU_OR(&parent->cg_mask, &child->cg_mask);
parent->cg_count += child->cg_count;
}
return (start);
}
struct mxq_job_list *group_list_add_job(struct mxq_group_list *glist, struct mxq_job *job)
{
struct mxq_server *server;
struct mxq_job_list *jlist;
struct mxq_user_list *ulist;
struct mxq_group *group;
assert(glist);
assert(glist->user);
assert(glist->user->server);
assert(job->job_status == MXQ_JOB_STATUS_RUNNING || job->job_status == MXQ_JOB_STATUS_LOADED);
group = &glist->group;
ulist = glist->user;
server = ulist->server;
jlist = mx_calloc_forever(1, sizeof(*jlist));
memcpy(&jlist->job, job, sizeof(*job));
jlist->group = glist;
jlist->next = glist->jobs;
glist->jobs = jlist;
glist->job_cnt++;
ulist->job_cnt++;
server->job_cnt++;
glist->slots_running += glist->slots_per_job;
ulist->slots_running += glist->slots_per_job;
server->slots_running += glist->slots_per_job;
glist->threads_running += group->job_threads;
ulist->threads_running += group->job_threads;
server->threads_running += group->job_threads;
CPU_OR(&server->cpu_set_running, &server->cpu_set_running, &job->host_cpu_set);
glist->jobs_running++;
ulist->jobs_running++;
server->jobs_running++;
glist->memory_used += group->job_memory;
ulist->memory_used += group->job_memory;
server->memory_used += group->job_memory;
return jlist;
}
/* We intercept this call. */
int numa_sched_setaffinity(pid_t pid, struct bitmask *mask) {
cpu_set_t requested_mask[CPU_SETSIZE], allowed_mask[CPU_SETSIZE], lnuma_mask[CPU_SETSIZE];
static void * (*real_function)();
int n_cpus;
int allow_change;
n_cpus = (int) sysconf(_SC_NPROCESSORS_CONF);
fprintf(stderr, "There are %d CPUs.\n", n_cpus);
/* Check whether the requested mask is allowed. */
/* First gets the list of LSB-allocated CPUs. If it's empty, we */
/* check if we are running exclusively on the node. */
allow_change = 0;
if (get_allowed_CPUs(allowed_mask)>0) {
int bit;
CPU_ZERO(lnuma_mask);
for (bit = 0; bit < n_cpus; bit++)
if(((1L << bit) & *(mask->maskp)) != 0 )
CPU_SET(bit,lnuma_mask);
CPU_OR(requested_mask, lnuma_mask, allowed_mask);
allow_change = CPU_EQUAL(requested_mask, allowed_mask);
} else {
allow_change = have_full_node(n_cpus);
}
if (allow_change) {
real_function = (void *(*) ()) dlsym(RTLD_NEXT, "sched_setaffinity");
return (int) real_function(pid, sizeof(lnuma_mask),lnuma_mask);
} else {
char *env_var;
if ((env_var = getenv("AFFINITY_NO_COMPLAIN")))
return 0;
/*
* The requested mask does not match with LSF one, we give to numactl
* the mask defined by LSF
*/
else{
fprintf(stderr, "Using cores from cpuset.\n");
real_function = (void *(*) ()) dlsym(RTLD_NEXT, "sched_setaffinity");
return (int) real_function(pid, sizeof(allowed_mask),allowed_mask);
}
}
}
cpu_set_t seissol::parallel::getWorkerUnionMask() {
cpu_set_t workerUnion;
CPU_ZERO(&workerUnion);
#ifdef _OPENMP
#pragma omp parallel
{
cpu_set_t worker;
CPU_ZERO(&worker);
sched_getaffinity(0, sizeof(cpu_set_t), &worker);
#pragma omp critical
{
CPU_OR(&workerUnion, &workerUnion, &worker);
}
}
#else
sched_getaffinity(0, sizeof(cpu_set_t), &workerUnion);
#endif
return workerUnion;
}
/*
* stress_tlb_shootdown()
* stress out TLB shootdowns
*/
static int stress_tlb_shootdown(const args_t *args)
{
const size_t page_size = args->page_size;
const size_t mmap_size = page_size * MMAP_PAGES;
pid_t pids[MAX_TLB_PROCS];
cpu_set_t proc_mask_initial;
if (sched_getaffinity(0, sizeof(proc_mask_initial), &proc_mask_initial) < 0) {
pr_fail_err("could not get CPU affinity");
return EXIT_FAILURE;
}
do {
uint8_t *mem, *ptr;
int retry = 128;
cpu_set_t proc_mask;
int32_t tlb_procs, i;
const int32_t max_cpus = stress_get_processors_configured();
CPU_ZERO(&proc_mask);
CPU_OR(&proc_mask, &proc_mask_initial, &proc_mask);
tlb_procs = max_cpus;
if (tlb_procs > MAX_TLB_PROCS)
tlb_procs = MAX_TLB_PROCS;
if (tlb_procs < MIN_TLB_PROCS)
tlb_procs = MIN_TLB_PROCS;
for (;;) {
mem = mmap(NULL, mmap_size, PROT_WRITE | PROT_READ,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if ((void *)mem == MAP_FAILED) {
if ((errno == EAGAIN) ||
(errno == ENOMEM) ||
(errno == ENFILE)) {
if (--retry < 0)
return EXIT_NO_RESOURCE;
} else {
pr_fail_err("mmap");
}
} else {
break;
}
}
(void)memset(mem, 0, mmap_size);
for (i = 0; i < tlb_procs; i++)
pids[i] = -1;
for (i = 0; i < tlb_procs; i++) {
int32_t j, cpu = -1;
for (j = 0; j < max_cpus; j++) {
if (CPU_ISSET(j, &proc_mask)) {
cpu = j;
CPU_CLR(j, &proc_mask);
break;
}
}
if (cpu == -1)
break;
pids[i] = fork();
if (pids[i] < 0)
break;
if (pids[i] == 0) {
cpu_set_t mask;
char buffer[page_size];
(void)setpgid(0, g_pgrp);
stress_parent_died_alarm();
/* Make sure this is killable by OOM killer */
set_oom_adjustment(args->name, true);
CPU_ZERO(&mask);
CPU_SET(cpu % max_cpus, &mask);
(void)sched_setaffinity(args->pid, sizeof(mask), &mask);
for (ptr = mem; ptr < mem + mmap_size; ptr += page_size) {
/* Force tlb shoot down on page */
(void)mprotect(ptr, page_size, PROT_READ);
(void)memcpy(buffer, ptr, page_size);
(void)munmap(ptr, page_size);
}
_exit(0);
}
}
for (i = 0; i < tlb_procs; i++) {
if (pids[i] != -1) {
int status, ret;
ret = shim_waitpid(pids[i], &status, 0);
if ((ret < 0) && (errno == EINTR)) {
int j;
/*
* We got interrupted, so assume
* it was the alarm (timedout) or
* SIGINT so force terminate
*/
for (j = i; j < tlb_procs; j++) {
if (pids[j] != -1)
(void)kill(pids[j], SIGKILL);
}
/* re-wait on the failed wait */
(void)shim_waitpid(pids[i], &status, 0);
/* and continue waitpid on the pids */
}
}
}
(void)munmap(mem, mmap_size);
(void)sched_setaffinity(0, sizeof(proc_mask_initial), &proc_mask_initial);
inc_counter(args);
} while (keep_stressing());
return EXIT_SUCCESS;
}
/* We intercept this call. */
int sched_setaffinity(pid_t pid, size_t cpusetsize, const cpu_set_t *mask) {
cpu_set_t requested_mask[CPU_SETSIZE], allowed_mask[CPU_SETSIZE], used_mask[CPU_SETSIZE];
static void * (*real_function)();
int n_cpus,c_cpus;
int allow_change;
int *mapp_allowed_cpus,*l_allowed_mask;
n_cpus = (int) sysconf(_SC_NPROCESSORS_CONF);
fprintf(stderr, "There are %d CPUs.\n", n_cpus);
/* Check whether the requested mask is allowed. */
/* First gets the list of LSB-allocated CPUs. If it's empty, we
* check if we are running exclusively on the node. */
allow_change = 0;
c_cpus = get_allowed_CPUs(allowed_mask);
CPU_ZERO(requested_mask);
if (c_cpus > 0) {
CPU_OR(requested_mask, mask, allowed_mask);
allow_change = CPU_EQUAL(requested_mask, allowed_mask);
} else {
allow_change = have_full_node(n_cpus);
}
if (allow_change) {
fprintf(stderr, "Change allowed.\n");
real_function = (void *(*) ()) dlsym(RTLD_NEXT, "sched_setaffinity");
return (int) real_function(pid, cpusetsize, mask);
} else {
char *env_var;
if ((env_var = getenv("AFFINITY_NO_COMPLAIN")))
return 0;
/*
* The requested mask does not match with LSF one, we shuffle the
* user mask
* Algorithm to get the mapping - Urban Borstnik
* 1. Let M(:) ← -1.
* 1. For each p in A, let M(p) ← p.
* 2. Let i←x|A(x)>|A| // I.e., find first entry in A that is greater than
* the requested core count.
* 3. For each p in P where M(p)<0, do
* let M(p) = A(i)
* i = (i+1) % |A|
*/
else{
int p,greater,bit;
fprintf(stderr, "Shuffling.\n");
mapp_allowed_cpus = malloc(n_cpus*sizeof(int));
memset (mapp_allowed_cpus, -1, n_cpus*sizeof (int) );
l_allowed_mask = calloc(c_cpus,sizeof(int));
get_logical_allowed_CPUs(l_allowed_mask);
greater = c_cpus;
for(p=0; p < c_cpus; p++)
{
mapp_allowed_cpus[l_allowed_mask[p]] = l_allowed_mask[p];
if(l_allowed_mask[p] > greater)
greater = p;
}
int index = greater;
for(p=0; p < n_cpus; p++)
if(mapp_allowed_cpus[p] == -1)
{
mapp_allowed_cpus[p] = l_allowed_mask[index];
index = (index+1) % c_cpus;
}
CPU_ZERO(used_mask);
for (bit=0;bit<n_cpus;bit++)
if(CPU_ISSET(bit,mask)){
CPU_SET(mapp_allowed_cpus[bit],used_mask);
}
free(mapp_allowed_cpus);
free(l_allowed_mask);
real_function = (void *(*) ()) dlsym(RTLD_NEXT, "sched_setaffinity");
return (int) real_function(pid, sizeof(used_mask), used_mask);
}
}
}
int
sys_cpuset_getaffinity(struct thread *td, struct cpuset_getaffinity_args *uap)
{
struct thread *ttd;
struct cpuset *nset;
struct cpuset *set;
struct proc *p;
cpuset_t *mask;
int error;
size_t size;
if (uap->cpusetsize < sizeof(cpuset_t) ||
uap->cpusetsize > CPU_MAXSIZE / NBBY)
return (ERANGE);
size = uap->cpusetsize;
mask = malloc(size, M_TEMP, M_WAITOK | M_ZERO);
error = cpuset_which(uap->which, uap->id, &p, &ttd, &set);
if (error)
goto out;
switch (uap->level) {
case CPU_LEVEL_ROOT:
case CPU_LEVEL_CPUSET:
switch (uap->which) {
case CPU_WHICH_TID:
case CPU_WHICH_PID:
thread_lock(ttd);
set = cpuset_ref(ttd->td_cpuset);
thread_unlock(ttd);
break;
case CPU_WHICH_CPUSET:
case CPU_WHICH_JAIL:
break;
case CPU_WHICH_IRQ:
error = EINVAL;
goto out;
}
if (uap->level == CPU_LEVEL_ROOT)
nset = cpuset_refroot(set);
else
nset = cpuset_refbase(set);
CPU_COPY(&nset->cs_mask, mask);
cpuset_rel(nset);
break;
case CPU_LEVEL_WHICH:
switch (uap->which) {
case CPU_WHICH_TID:
thread_lock(ttd);
CPU_COPY(&ttd->td_cpuset->cs_mask, mask);
thread_unlock(ttd);
break;
case CPU_WHICH_PID:
FOREACH_THREAD_IN_PROC(p, ttd) {
thread_lock(ttd);
CPU_OR(mask, &ttd->td_cpuset->cs_mask);
thread_unlock(ttd);
}
break;
case CPU_WHICH_CPUSET:
case CPU_WHICH_JAIL:
CPU_COPY(&set->cs_mask, mask);
break;
case CPU_WHICH_IRQ:
error = intr_getaffinity(uap->id, mask);
break;
}
break;
default:
error = EINVAL;
break;
}
int
test_affinity1(void)
#endif
{
unsigned int cpu;
cpu_set_t newmask;
cpu_set_t src1mask;
cpu_set_t src2mask;
cpu_set_t src3mask;
CPU_ZERO(&newmask);
CPU_ZERO(&src1mask);
memset(&src2mask, 0, sizeof(cpu_set_t));
assert(memcmp(&src1mask, &src2mask, sizeof(cpu_set_t)) == 0);
assert(CPU_EQUAL(&src1mask, &src2mask));
assert(CPU_COUNT(&src1mask) == 0);
CPU_ZERO(&src1mask);
CPU_ZERO(&src2mask);
CPU_ZERO(&src3mask);
for (cpu = 0; cpu < sizeof(cpu_set_t)*8; cpu += 2)
{
CPU_SET(cpu, &src1mask); /* 0b01010101010101010101010101010101 */
}
for (cpu = 0; cpu < sizeof(cpu_set_t)*4; cpu++)
{
CPU_SET(cpu, &src2mask); /* 0b00000000000000001111111111111111 */
}
for (cpu = sizeof(cpu_set_t)*4; cpu < sizeof(cpu_set_t)*8; cpu += 2)
{
CPU_SET(cpu, &src2mask); /* 0b01010101010101011111111111111111 */
}
for (cpu = 0; cpu < sizeof(cpu_set_t)*8; cpu += 2)
{
CPU_SET(cpu, &src3mask); /* 0b01010101010101010101010101010101 */
}
assert(CPU_COUNT(&src1mask) == (sizeof(cpu_set_t)*4));
assert(CPU_COUNT(&src2mask) == ((sizeof(cpu_set_t)*4 + (sizeof(cpu_set_t)*2))));
assert(CPU_COUNT(&src3mask) == (sizeof(cpu_set_t)*4));
CPU_SET(0, &newmask);
CPU_SET(1, &newmask);
CPU_SET(3, &newmask);
assert(CPU_ISSET(1, &newmask));
CPU_CLR(1, &newmask);
assert(!CPU_ISSET(1, &newmask));
CPU_OR(&newmask, &src1mask, &src2mask);
assert(CPU_EQUAL(&newmask, &src2mask));
CPU_AND(&newmask, &src1mask, &src2mask);
assert(CPU_EQUAL(&newmask, &src1mask));
CPU_XOR(&newmask, &src1mask, &src3mask);
memset(&src2mask, 0, sizeof(cpu_set_t));
assert(memcmp(&newmask, &src2mask, sizeof(cpu_set_t)) == 0);
/*
* Need to confirm the bitwise logical right-shift in CpuCount().
* i.e. zeros inserted into MSB on shift because cpu_set_t is
* unsigned.
*/
CPU_ZERO(&src1mask);
for (cpu = 1; cpu < sizeof(cpu_set_t)*8; cpu += 2)
{
CPU_SET(cpu, &src1mask); /* 0b10101010101010101010101010101010 */
}
assert(CPU_ISSET(sizeof(cpu_set_t)*8-1, &src1mask));
assert(CPU_COUNT(&src1mask) == (sizeof(cpu_set_t)*4));
return 0;
}
void odp_cpumask_or(odp_cpumask_t *dest, const odp_cpumask_t *src1,
const odp_cpumask_t *src2)
{
CPU_OR(&dest->set, &src1->set, &src2->set);
}
