/* Translate global CPU index to local CPU index. This is needed for
* Power7 processors with multi-threading disabled. On those processors,
* the CPU mask has gaps for the unused threads (different from Intel
* processors) which need to be skipped over in the mask used in the
* set system call. */
void reset_cpuset(cpu_set_t *new_mask, cpu_set_t *cur_mask)
{
cpu_set_t full_mask, newer_mask;
int cur_offset, new_offset = 0, last_set = -1;
if (!_is_power_cpu())
return;
if (slurm_getaffinity(1, sizeof(full_mask), &full_mask)) {
/* Try to get full CPU mask from process init */
CPU_ZERO(&full_mask);
#ifdef __FreeBSD__
CPU_OR(&full_mask, cur_mask);
#else
CPU_OR(&full_mask, &full_mask, cur_mask);
#endif
}
CPU_ZERO(&newer_mask);
for (cur_offset = 0; cur_offset < CPU_SETSIZE; cur_offset++) {
if (!CPU_ISSET(cur_offset, &full_mask))
continue;
if (CPU_ISSET(new_offset, new_mask)) {
CPU_SET(cur_offset, &newer_mask);
last_set = cur_offset;
}
new_offset++;
}
CPU_ZERO(new_mask);
for (cur_offset = 0; cur_offset <= last_set; cur_offset++) {
if (CPU_ISSET(cur_offset, &newer_mask))
CPU_SET(cur_offset, new_mask);
}
}
/*
* init() is called when the plugin is loaded, before any other functions
* are called. Put global initialization here.
*/
extern int init (void)
{
cpu_set_t cur_mask;
char mstr[1 + CPU_SETSIZE / 4];
slurm_getaffinity(0, sizeof(cur_mask), &cur_mask);
cpuset_to_str(&cur_mask, mstr);
verbose("%s loaded with CPU mask %s", plugin_name, mstr);
return SLURM_SUCCESS;
}
/*
* task_pre_launch() is called prior to exec of application task.
* It is followed by TaskProlog program (from slurm.conf) and
* --task-prolog (from srun command line).
*/
extern int task_pre_launch (slurmd_job_t *job)
{
char base[PATH_MAX], path[PATH_MAX];
int rc = SLURM_SUCCESS;
debug("affinity task_pre_launch:%u.%u, task:%u bind:%u",
job->jobid, job->stepid, job->envtp->procid,
job->cpu_bind_type);
if (conf->task_plugin_param & CPU_BIND_CPUSETS) {
info("Using cpuset affinity for tasks");
#ifdef MULTIPLE_SLURMD
if (snprintf(base, PATH_MAX, "%s/slurm_%s_%u",
CPUSET_DIR,
(conf->node_name != NULL)?conf->node_name:"",
job->jobid) > PATH_MAX) {
error("cpuset path too long");
return SLURM_ERROR;
}
#else
if (snprintf(base, PATH_MAX, "%s/slurm%u",
CPUSET_DIR, job->jobid) > PATH_MAX) {
error("cpuset path too long");
return SLURM_ERROR;
}
#endif
if (snprintf(path, PATH_MAX, "%s/slurm%u.%u_%d",
base, job->jobid, job->stepid,
job->envtp->localid) > PATH_MAX) {
error("cpuset path too long");
return SLURM_ERROR;
}
} else
info("Using sched_affinity for tasks");
/*** CPU binding support ***/
if (job->cpu_bind_type) {
cpu_set_t new_mask, cur_mask;
pid_t mypid = job->envtp->task_pid;
slurm_getaffinity(mypid, sizeof(cur_mask), &cur_mask);
if (get_cpuset(&new_mask, job) &&
(!(job->cpu_bind_type & CPU_BIND_NONE))) {
if (conf->task_plugin_param & CPU_BIND_CPUSETS) {
rc = slurm_set_cpuset(base, path, mypid,
sizeof(new_mask),
&new_mask);
slurm_get_cpuset(path, mypid,
sizeof(cur_mask),
&cur_mask);
} else {
rc = slurm_setaffinity(mypid,
sizeof(new_mask),
&new_mask);
slurm_getaffinity(mypid,
sizeof(cur_mask),
&cur_mask);
}
}
slurm_chkaffinity(rc ? &cur_mask : &new_mask,
job, rc);
} else if (job->mem_bind_type &&
(conf->task_plugin_param & CPU_BIND_CPUSETS)) {
cpu_set_t cur_mask;
pid_t mypid = job->envtp->task_pid;
/* Establish cpuset just for the memory binding */
slurm_getaffinity(mypid, sizeof(cur_mask), &cur_mask);
rc = slurm_set_cpuset(base, path,
(pid_t) job->envtp->task_pid,
sizeof(cur_mask), &cur_mask);
}
#ifdef HAVE_NUMA
if ((conf->task_plugin_param & CPU_BIND_CPUSETS) &&
(slurm_memset_available() >= 0)) {
nodemask_t new_mask, cur_mask;
cur_mask = numa_get_membind();
if (get_memset(&new_mask, job) &&
(!(job->mem_bind_type & MEM_BIND_NONE))) {
slurm_set_memset(path, &new_mask);
if (numa_available() >= 0)
numa_set_membind(&new_mask);
cur_mask = new_mask;
}
slurm_chk_memset(&cur_mask, job);
} else if (job->mem_bind_type && (numa_available() >= 0)) {
nodemask_t new_mask, cur_mask;
cur_mask = numa_get_membind();
if (get_memset(&new_mask, job)
&& (!(job->mem_bind_type & MEM_BIND_NONE))) {
numa_set_membind(&new_mask);
cur_mask = new_mask;
}
slurm_chk_memset(&cur_mask, job);
}
#endif
return rc;
}