/*
* lllp_distribution
*
* Note: lllp stands for Lowest Level of Logical Processors.
*
* When automatic binding is enabled:
* - no binding flags set >= CPU_BIND_NONE, and
* - a auto binding level selected CPU_BIND_TO_{SOCKETS,CORES,THREADS}
* Otherwise limit job step to the allocated CPUs
*
* generate the appropriate cpu_bind type and string which results in
* the specified lllp distribution.
*
* IN/OUT- job launch request (cpu_bind_type and cpu_bind updated)
* IN- global task id array
*/
void lllp_distribution(launch_tasks_request_msg_t *req, uint32_t node_id)
{
int rc = SLURM_SUCCESS;
bitstr_t **masks = NULL;
char buf_type[100];
int maxtasks = req->tasks_to_launch[(int)node_id];
int whole_nodes, whole_sockets, whole_cores, whole_threads;
int part_sockets, part_cores;
const uint32_t *gtid = req->global_task_ids[(int)node_id];
static uint16_t bind_entity = CPU_BIND_TO_THREADS | CPU_BIND_TO_CORES |
CPU_BIND_TO_SOCKETS | CPU_BIND_TO_LDOMS;
static uint16_t bind_mode = CPU_BIND_NONE | CPU_BIND_MASK |
CPU_BIND_RANK | CPU_BIND_MAP |
CPU_BIND_LDMASK | CPU_BIND_LDRANK |
CPU_BIND_LDMAP;
static int only_one_thread_per_core = -1;
if (only_one_thread_per_core == -1) {
if (conf->cpus == (conf->sockets * conf->cores))
only_one_thread_per_core = 1;
else
only_one_thread_per_core = 0;
}
/* If we are telling the system we only want to use 1 thread
* per core with the CPUs node option this is the easiest way
* to portray that to the affinity plugin.
*/
if (only_one_thread_per_core)
req->cpu_bind_type |= CPU_BIND_ONE_THREAD_PER_CORE;
if (req->cpu_bind_type & bind_mode) {
/* Explicit step binding specified by user */
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
if ((whole_nodes == 0) && avail_mask &&
(req->job_core_spec == (uint16_t) NO_VAL)) {
info("task/affinity: entire node must be allocated, "
"disabling affinity");
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type &= (~bind_mode);
req->cpu_bind_type |= CPU_BIND_MASK;
} else {
if (req->job_core_spec == (uint16_t) NO_VAL) {
if (req->cpu_bind_type & CPU_BIND_MASK)
_validate_mask(req, avail_mask);
else if (req->cpu_bind_type & CPU_BIND_MAP)
_validate_map(req, avail_mask);
}
xfree(avail_mask);
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] manual binding: %s",
req->job_id, buf_type);
return;
}
if (!(req->cpu_bind_type & bind_entity)) {
/* No bind unit (sockets, cores) specified by user,
* pick something reasonable */
uint32_t task_plugin_param = slurm_get_task_plugin_param();
bool auto_def_set = false;
int spec_thread_cnt = 0;
int max_tasks = req->tasks_to_launch[(int)node_id] *
req->cpus_per_task;
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
debug("binding tasks:%d to "
"nodes:%d sockets:%d:%d cores:%d:%d threads:%d",
max_tasks, whole_nodes, whole_sockets ,part_sockets,
whole_cores, part_cores, whole_threads);
if ((req->job_core_spec != (uint16_t) NO_VAL) &&
(req->job_core_spec & CORE_SPEC_THREAD) &&
(req->job_core_spec != CORE_SPEC_THREAD)) {
spec_thread_cnt = req->job_core_spec &
(~CORE_SPEC_THREAD);
}
if (((max_tasks == whole_sockets) && (part_sockets == 0)) ||
(spec_thread_cnt &&
(max_tasks == (whole_sockets + part_sockets)))) {
req->cpu_bind_type |= CPU_BIND_TO_SOCKETS;
goto make_auto;
}
if (((max_tasks == whole_cores) && (part_cores == 0)) ||
(spec_thread_cnt &&
(max_tasks == (whole_cores + part_cores)))) {
req->cpu_bind_type |= CPU_BIND_TO_CORES;
goto make_auto;
}
if (max_tasks == whole_threads) {
req->cpu_bind_type |= CPU_BIND_TO_THREADS;
goto make_auto;
}
if (task_plugin_param & CPU_AUTO_BIND_TO_THREADS) {
auto_def_set = true;
req->cpu_bind_type |= CPU_BIND_TO_THREADS;
goto make_auto;
} else if (task_plugin_param & CPU_AUTO_BIND_TO_CORES) {
auto_def_set = true;
req->cpu_bind_type |= CPU_BIND_TO_CORES;
goto make_auto;
} else if (task_plugin_param & CPU_AUTO_BIND_TO_SOCKETS) {
auto_def_set = true;
req->cpu_bind_type |= CPU_BIND_TO_SOCKETS;
goto make_auto;
}
if (avail_mask) {
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type |= CPU_BIND_MASK;
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] auto binding off: %s",
req->job_id, buf_type);
return;
make_auto: xfree(avail_mask);
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] %s auto binding: "
"%s, dist %d", req->job_id,
(auto_def_set) ? "default" : "implicit",
buf_type, req->task_dist);
} else {
/* Explicit bind unit (sockets, cores) specified by user */
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] binding: %s, dist %d",
req->job_id, buf_type, req->task_dist);
}
switch (req->task_dist & SLURM_DIST_STATE_BASE) {
case SLURM_DIST_BLOCK_BLOCK:
case SLURM_DIST_CYCLIC_BLOCK:
case SLURM_DIST_PLANE:
/* tasks are distributed in blocks within a plane */
rc = _task_layout_lllp_block(req, node_id, &masks);
break;
case SLURM_DIST_ARBITRARY:
case SLURM_DIST_BLOCK:
case SLURM_DIST_CYCLIC:
case SLURM_DIST_UNKNOWN:
if (slurm_get_select_type_param()
& CR_CORE_DEFAULT_DIST_BLOCK) {
rc = _task_layout_lllp_block(req, node_id, &masks);
break;
}
/* We want to fall through here if we aren't doing a
default dist block.
*/
default:
rc = _task_layout_lllp_cyclic(req, node_id, &masks);
break;
}
/* FIXME: I'm worried about core_bitmap with CPU_BIND_TO_SOCKETS &
* max_cores - does select/cons_res plugin allocate whole
* socket??? Maybe not. Check srun man page.
*/
if (rc == SLURM_SUCCESS) {
_task_layout_display_masks(req, gtid, maxtasks, masks);
/* translate abstract masks to actual hardware layout */
_lllp_map_abstract_masks(maxtasks, masks);
_task_layout_display_masks(req, gtid, maxtasks, masks);
#ifdef HAVE_NUMA
if (req->cpu_bind_type & CPU_BIND_TO_LDOMS) {
_match_masks_to_ldom(maxtasks, masks);
_task_layout_display_masks(req, gtid, maxtasks, masks);
}
#endif
/* convert masks into cpu_bind mask string */
_lllp_generate_cpu_bind(req, maxtasks, masks);
} else {
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
if (avail_mask) {
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type &= (~bind_mode);
req->cpu_bind_type |= CPU_BIND_MASK;
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
error("lllp_distribution jobid [%u] overriding binding: %s",
req->job_id, buf_type);
error("Verify socket/core/thread counts in configuration");
}
if (masks)
_lllp_free_masks(maxtasks, masks);
}
/*
* lllp_distribution
*
* Note: lllp stands for Lowest Level of Logical Processors.
*
* When automatic binding is enabled:
* - no binding flags set >= CPU_BIND_NONE, and
* - a auto binding level selected CPU_BIND_TO_{SOCKETS,CORES,THREADS}
* Otherwise limit job step to the allocated CPUs
*
* generate the appropriate cpu_bind type and string which results in
* the specified lllp distribution.
*
* IN/OUT- job launch request (cpu_bind_type and cpu_bind updated)
* IN- global task id array
*/
void lllp_distribution(launch_tasks_request_msg_t *req, uint32_t node_id)
{
int rc = SLURM_SUCCESS;
bitstr_t **masks = NULL;
char buf_type[100];
int maxtasks = req->tasks_to_launch[(int)node_id];
int whole_nodes, whole_sockets, whole_cores, whole_threads;
int part_sockets, part_cores;
const uint32_t *gtid = req->global_task_ids[(int)node_id];
static uint16_t bind_entity = CPU_BIND_TO_THREADS | CPU_BIND_TO_CORES |
CPU_BIND_TO_SOCKETS | CPU_BIND_TO_LDOMS;
static uint16_t bind_mode = CPU_BIND_NONE | CPU_BIND_MASK |
CPU_BIND_RANK | CPU_BIND_MAP |
CPU_BIND_LDMASK | CPU_BIND_LDRANK |
CPU_BIND_LDMAP;
if (req->cpu_bind_type & bind_mode) {
/* Explicit step binding specified by user */
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
if ((whole_nodes == 0) && avail_mask) {
/* Step does NOT have access to whole node,
* bind to full mask of available processors */
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type &= (~bind_mode);
req->cpu_bind_type |= CPU_BIND_MASK;
} else {
/* Step does have access to whole node,
* bind to whatever step wants */
xfree(avail_mask);
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] manual binding: %s",
req->job_id, buf_type);
return;
}
if (!(req->cpu_bind_type & bind_entity)) {
/* No bind unit (sockets, cores) specified by user,
* pick something reasonable */
int max_tasks = req->tasks_to_launch[(int)node_id];
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
debug("binding tasks:%d to "
"nodes:%d sockets:%d:%d cores:%d:%d threads:%d",
max_tasks, whole_nodes, whole_sockets ,part_sockets,
whole_cores, part_cores, whole_threads);
if ((max_tasks == whole_sockets) && (part_sockets == 0)) {
req->cpu_bind_type |= CPU_BIND_TO_SOCKETS;
goto make_auto;
}
if ((max_tasks == whole_cores) && (part_cores == 0)) {
req->cpu_bind_type |= CPU_BIND_TO_CORES;
goto make_auto;
}
if (max_tasks == whole_threads) {
req->cpu_bind_type |= CPU_BIND_TO_THREADS;
goto make_auto;
}
if (avail_mask) {
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type |= CPU_BIND_MASK;
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] auto binding off: %s",
req->job_id, buf_type);
return;
make_auto: xfree(avail_mask);
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] implicit auto binding: "
"%s, dist %d", req->job_id, buf_type, req->task_dist);
} else {
/* Explicit bind unit (sockets, cores) specified by user */
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
info("lllp_distribution jobid [%u] binding: %s, dist %d",
req->job_id, buf_type, req->task_dist);
}
switch (req->task_dist) {
case SLURM_DIST_BLOCK_BLOCK:
case SLURM_DIST_CYCLIC_BLOCK:
case SLURM_DIST_PLANE:
/* tasks are distributed in blocks within a plane */
rc = _task_layout_lllp_block(req, node_id, &masks);
break;
case SLURM_DIST_CYCLIC:
case SLURM_DIST_BLOCK:
case SLURM_DIST_CYCLIC_CYCLIC:
case SLURM_DIST_BLOCK_CYCLIC:
rc = _task_layout_lllp_cyclic(req, node_id, &masks);
break;
default:
if (req->cpus_per_task > 1)
rc = _task_layout_lllp_multi(req, node_id, &masks);
else
rc = _task_layout_lllp_cyclic(req, node_id, &masks);
req->task_dist = SLURM_DIST_BLOCK_CYCLIC;
break;
}
/* FIXME: I'm worried about core_bitmap with CPU_BIND_TO_SOCKETS &
* max_cores - does select/cons_res plugin allocate whole
* socket??? Maybe not. Check srun man page.
*/
if (rc == SLURM_SUCCESS) {
_task_layout_display_masks(req, gtid, maxtasks, masks);
/* translate abstract masks to actual hardware layout */
_lllp_map_abstract_masks(maxtasks, masks);
_task_layout_display_masks(req, gtid, maxtasks, masks);
#ifdef HAVE_NUMA
if (req->cpu_bind_type & CPU_BIND_TO_LDOMS) {
_match_masks_to_ldom(maxtasks, masks);
_task_layout_display_masks(req, gtid, maxtasks, masks);
}
#endif
/* convert masks into cpu_bind mask string */
_lllp_generate_cpu_bind(req, maxtasks, masks);
} else {
char *avail_mask = _alloc_mask(req,
&whole_nodes, &whole_sockets,
&whole_cores, &whole_threads,
&part_sockets, &part_cores);
if (avail_mask) {
xfree(req->cpu_bind);
req->cpu_bind = avail_mask;
req->cpu_bind_type &= (~bind_mode);
req->cpu_bind_type |= CPU_BIND_MASK;
}
slurm_sprint_cpu_bind_type(buf_type, req->cpu_bind_type);
error("lllp_distribution jobid [%u] overriding binding: %s",
req->job_id, buf_type);
error("Verify socket/core/thread counts in configuration");
}
if (masks)
_lllp_free_masks(maxtasks, masks);
}
