static int ppr_mapper(orte_job_t *jdata)
{
int rc = ORTE_SUCCESS, j, n;
mca_base_component_t *c=&mca_rmaps_ppr_component.base_version;
orte_node_t *node;
orte_proc_t *proc;
orte_app_context_t *app;
orte_vpid_t total_procs, nprocs_mapped;
opal_hwloc_level_t start=OPAL_HWLOC_NODE_LEVEL;
#if OPAL_HAVE_HWLOC
hwloc_obj_t obj;
hwloc_obj_type_t lowest;
unsigned cache_level=0;
unsigned int nobjs, i;
bool pruning_reqd = false;
opal_hwloc_level_t level;
#endif
opal_list_t node_list;
opal_list_item_t *item;
orte_std_cntr_t num_slots;
orte_app_idx_t idx;
char **ppr_req, **ck;
size_t len;
bool initial_map=true;
/* only handle initial launch of loadbalanced
* or NPERxxx jobs - allow restarting of failed apps
*/
if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: job %s being restarted - ppr cannot map",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
if (NULL != jdata->map->req_mapper &&
0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
/* a mapper has been specified, and it isn't me */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: job %s not using ppr mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
if (NULL == jdata->map->ppr ||
ORTE_MAPPING_PPR != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
/* not for us */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: job %s not using ppr mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: mapping job %s with ppr %s",
ORTE_JOBID_PRINT(jdata->jobid), jdata->map->ppr);
/* flag that I did the mapping */
if (NULL != jdata->map->last_mapper) {
free(jdata->map->last_mapper);
}
jdata->map->last_mapper = strdup(c->mca_component_name);
/* initialize */
memset(ppr, 0, OPAL_HWLOC_HWTHREAD_LEVEL * sizeof(opal_hwloc_level_t));
/* parse option */
n=0;
ppr_req = opal_argv_split(jdata->map->ppr, ',');
for (j=0; NULL != ppr_req[j]; j++) {
/* split on the colon */
ck = opal_argv_split(ppr_req[j], ':');
if (2 != opal_argv_count(ck)) {
/* must provide a specification */
orte_show_help("help-orte-rmaps-ppr.txt", "invalid-ppr", true, jdata->map->ppr);
opal_argv_free(ppr_req);
opal_argv_free(ck);
return ORTE_ERR_SILENT;
}
len = strlen(ck[1]);
if (0 == strncasecmp(ck[1], "node", len)) {
ppr[OPAL_HWLOC_NODE_LEVEL] = strtol(ck[0], NULL, 10);
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYNODE);
start = OPAL_HWLOC_NODE_LEVEL;
n++;
#if OPAL_HAVE_HWLOC
} else if (0 == strncasecmp(ck[1], "hwthread", len) ||
0 == strncasecmp(ck[1], "thread", len)) {
ppr[OPAL_HWLOC_HWTHREAD_LEVEL] = strtol(ck[0], NULL, 10);
start = OPAL_HWLOC_HWTHREAD_LEVEL;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYHWTHREAD);
n++;
} else if (0 == strncasecmp(ck[1], "core", len)) {
ppr[OPAL_HWLOC_CORE_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_CORE_LEVEL) {
start = OPAL_HWLOC_CORE_LEVEL;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYCORE);
}
n++;
} else if (0 == strncasecmp(ck[1], "socket", len) ||
0 == strncasecmp(ck[1], "skt", len)) {
ppr[OPAL_HWLOC_SOCKET_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_SOCKET_LEVEL) {
start = OPAL_HWLOC_SOCKET_LEVEL;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSOCKET);
}
n++;
} else if (0 == strncasecmp(ck[1], "l1cache", len)) {
ppr[OPAL_HWLOC_L1CACHE_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_L1CACHE_LEVEL) {
start = OPAL_HWLOC_L1CACHE_LEVEL;
cache_level = 1;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL1CACHE);
}
n++;
} else if (0 == strncasecmp(ck[1], "l2cache", len)) {
ppr[OPAL_HWLOC_L2CACHE_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_L2CACHE_LEVEL) {
start = OPAL_HWLOC_L2CACHE_LEVEL;
cache_level = 2;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL2CACHE);
}
n++;
} else if (0 == strncasecmp(ck[1], "l3cache", len)) {
ppr[OPAL_HWLOC_L3CACHE_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_L3CACHE_LEVEL) {
start = OPAL_HWLOC_L3CACHE_LEVEL;
cache_level = 3;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYL3CACHE);
}
n++;
} else if (0 == strncasecmp(ck[1], "numa", len)) {
ppr[OPAL_HWLOC_NUMA_LEVEL] = strtol(ck[0], NULL, 10);
if (start < OPAL_HWLOC_NUMA_LEVEL) {
start = OPAL_HWLOC_NUMA_LEVEL;
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYNUMA);
}
n++;
#endif
} else {
/* unknown spec */
orte_show_help("help-orte-rmaps-ppr.txt", "unrecognized-ppr-option", true, ck[1], jdata->map->ppr);
opal_argv_free(ppr_req);
opal_argv_free(ck);
return ORTE_ERR_SILENT;
}
opal_argv_free(ck);
}
opal_argv_free(ppr_req);
/* if nothing was given, that's an error */
if (0 == n) {
opal_output(0, "NOTHING GIVEN");
return ORTE_ERR_SILENT;
}
#if OPAL_HAVE_HWLOC
/* if more than one level was specified, then pruning will be reqd */
if (1 < n) {
pruning_reqd = true;
}
#endif
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: job %s assigned policy %s",
ORTE_JOBID_PRINT(jdata->jobid),
orte_rmaps_base_print_mapping(jdata->map->mapping));
#if OPAL_HAVE_HWLOC
/* convenience */
level = start;
lowest = opal_hwloc_levels[start];
#endif
for (idx=0; idx < (orte_app_idx_t)jdata->apps->size; idx++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
continue;
}
/* if the number of total procs was given, set that
* limit - otherwise, set to max so we simply fill
* all the nodes with the pattern
*/
if (0 < app->num_procs) {
total_procs = app->num_procs;
} else {
total_procs = ORTE_VPID_MAX;
}
/* get the available nodes */
OBJ_CONSTRUCT(&node_list, opal_list_t);
if(ORTE_SUCCESS != (rc = orte_rmaps_base_get_target_nodes(&node_list, &num_slots, app,
jdata->map->mapping, initial_map, false))) {
ORTE_ERROR_LOG(rc);
goto error;
}
/* flag that all subsequent requests should not reset the node->mapped flag */
initial_map = false;
/* if a bookmark exists from some prior mapping, set us to start there */
jdata->bookmark = orte_rmaps_base_get_starting_point(&node_list, jdata);
/* cycle across the nodes */
nprocs_mapped = 0;
for (item = opal_list_get_first(&node_list);
item != opal_list_get_end(&node_list);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
#if OPAL_HAVE_HWLOC
/* bozo check */
if (NULL == node->topology) {
orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
true, node->name);
rc = ORTE_ERR_SILENT;
goto error;
}
#endif
/* add the node to the map, if needed */
if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
if (ORTE_SUCCESS > (rc = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
ORTE_ERROR_LOG(rc);
goto error;
}
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
OBJ_RETAIN(node); /* maintain accounting on object */
jdata->map->num_nodes++;
}
/* if we are mapping solely at the node level, just put
* that many procs on this node
*/
if (OPAL_HWLOC_NODE_LEVEL == start) {
#if OPAL_HAVE_HWLOC
obj = hwloc_get_root_obj(node->topology);
#endif
for (j=0; j < ppr[start] && nprocs_mapped < total_procs; j++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, idx))) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto error;
}
nprocs_mapped++;
#if OPAL_HAVE_HWLOC
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
#endif
}
#if OPAL_HAVE_HWLOC
} else {
/* get the number of lowest resources on this node */
nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology,
lowest, cache_level,
OPAL_HWLOC_AVAILABLE);
/* map the specified number of procs to each such resource on this node,
* recording the locale of each proc so we know its cpuset
*/
for (i=0; i < nobjs; i++) {
obj = opal_hwloc_base_get_obj_by_type(node->topology,
lowest, cache_level,
i, OPAL_HWLOC_AVAILABLE);
for (j=0; j < ppr[start] && nprocs_mapped < total_procs; j++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, idx))) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto error;
}
nprocs_mapped++;
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
}
}
if (pruning_reqd) {
/* go up the ladder and prune the procs according to
* the specification, adjusting the count of procs on the
* node as we go
*/
level--;
prune(jdata->jobid, idx, node, &level, &nprocs_mapped);
}
#endif
}
/* set the total slots used */
if ((int)node->num_procs <= node->slots) {
node->slots_inuse = (int)node->num_procs;
} else {
node->slots_inuse = node->slots;
}
/* if no-oversubscribe was specified, check to see if
* we have violated the total slot specification - regardless,
* if slots_max was given, we are not allowed to violate it!
*/
if ((node->slots < (int)node->num_procs) ||
(0 < node->slots_max && node->slots_max < (int)node->num_procs)) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, node->num_procs, app->app);
rc = ORTE_ERR_SILENT;
goto error;
}
/* flag the node as oversubscribed so that sched-yield gets
* properly set
*/
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
}
/* if we haven't mapped all the procs, continue on to the
* next node
*/
if (total_procs == nprocs_mapped) {
break;
}
}
if (0 == app->num_procs) {
app->num_procs = nprocs_mapped;
}
if (ORTE_VPID_MAX != total_procs && nprocs_mapped < total_procs) {
/* couldn't map them all */
orte_show_help("help-orte-rmaps-ppr.txt", "ppr-too-many-procs",
true, app->app, app->num_procs, jdata->map->ppr);
rc = ORTE_ERR_SILENT;
goto error;
}
/* compute vpids and add proc objects to the job */
if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_vpids(jdata, app, &node_list))) {
ORTE_ERROR_LOG(rc);
goto error;
}
/* track the total number of processes we mapped - must update
* this AFTER we compute vpids so that computation is done
* correctly
*/
jdata->num_procs += app->num_procs;
while (NULL != (item = opal_list_remove_first(&node_list))) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&node_list);
}
return ORTE_SUCCESS;
error:
while (NULL != (item = opal_list_remove_first(&node_list))) {
OBJ_RELEASE(item);
}
OBJ_DESTRUCT(&node_list);
return rc;
}
static int
sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
int32_t my_smp_rank,
int n)
{
size_t length, length_payload;
sm_fifo_t *my_fifos;
int my_mem_node, num_mem_nodes, i, rc;
mca_mpool_base_resources_t *res = NULL;
mca_btl_sm_component_t* m = &mca_btl_sm_component;
/* Assume we don't have hwloc support and fill in dummy info */
mca_btl_sm_component.mem_node = my_mem_node = 0;
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = 1;
#if OPAL_HAVE_HWLOC
/* If we have hwloc support, then get accurate information */
if (NULL != opal_hwloc_topology) {
i = opal_hwloc_base_get_nbobjs_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0,
OPAL_HWLOC_AVAILABLE);
/* If we find >0 NUMA nodes, then investigate further */
if (i > 0) {
int numa=0, w;
unsigned n_bound=0;
hwloc_cpuset_t avail;
hwloc_obj_t obj;
/* JMS This tells me how many numa nodes are *available*,
but it's not how many are being used *by this job*.
Note that this is the value we've previously used (from
the previous carto-based implementation), but it really
should be improved to be how many NUMA nodes are being
used *in this job*. */
mca_btl_sm_component.num_mem_nodes = num_mem_nodes = i;
/* if we are not bound, then there is nothing further to do */
if (NULL != ompi_process_info.cpuset) {
/* count the number of NUMA nodes to which we are bound */
for (w=0; w < i; w++) {
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(opal_hwloc_topology,
HWLOC_OBJ_NODE, 0, w,
OPAL_HWLOC_AVAILABLE))) {
continue;
}
/* get that NUMA node's available cpus */
avail = opal_hwloc_base_get_available_cpus(opal_hwloc_topology, obj);
/* see if we intersect */
if (hwloc_bitmap_intersects(avail, opal_hwloc_my_cpuset)) {
n_bound++;
numa = w;
}
}
/* if we are located on more than one NUMA, or we didn't find
* a NUMA we are on, then not much we can do
*/
if (1 == n_bound) {
mca_btl_sm_component.mem_node = my_mem_node = numa;
} else {
mca_btl_sm_component.mem_node = my_mem_node = -1;
}
}
}
}
#endif
if (NULL == (res = calloc(1, sizeof(*res)))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* lookup shared memory pool */
mca_btl_sm_component.sm_mpools =
(mca_mpool_base_module_t **)calloc(num_mem_nodes,
sizeof(mca_mpool_base_module_t *));
/* Disable memory binding, because each MPI process will claim pages in the
* mpool for their local NUMA node */
res->mem_node = -1;
if (OMPI_SUCCESS != (rc = setup_mpool_base_resources(m, res))) {
free(res);
return rc;
}
/* now that res is fully populated, create the thing */
mca_btl_sm_component.sm_mpools[0] =
mca_mpool_base_module_create(mca_btl_sm_component.sm_mpool_name,
sm_btl, res);
/* Sanity check to ensure that we found it */
if (NULL == mca_btl_sm_component.sm_mpools[0]) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
mca_btl_sm_component.sm_mpool = mca_btl_sm_component.sm_mpools[0];
mca_btl_sm_component.sm_mpool_base =
mca_btl_sm_component.sm_mpools[0]->mpool_base(mca_btl_sm_component.sm_mpools[0]);
/* create a list of peers */
mca_btl_sm_component.sm_peers = (struct mca_btl_base_endpoint_t**)
calloc(n, sizeof(struct mca_btl_base_endpoint_t*));
if (NULL == mca_btl_sm_component.sm_peers) {
free(res);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* remember that node rank zero is already attached */
if (0 != my_smp_rank) {
if (OMPI_SUCCESS != (rc = sm_segment_attach(m))) {
free(res);
return rc;
}
}
/* it is now safe to free the mpool resources */
free(res);
/* check to make sure number of local procs is within the
* specified limits */
if(mca_btl_sm_component.sm_max_procs > 0 &&
mca_btl_sm_component.num_smp_procs + n >
mca_btl_sm_component.sm_max_procs) {
return OMPI_ERROR;
}
mca_btl_sm_component.shm_fifo = (volatile sm_fifo_t **)mca_btl_sm_component.sm_seg->module_data_addr;
mca_btl_sm_component.shm_bases = (char**)(mca_btl_sm_component.shm_fifo + n);
mca_btl_sm_component.shm_mem_nodes = (uint16_t*)(mca_btl_sm_component.shm_bases + n);
/* set the base of the shared memory segment */
mca_btl_sm_component.shm_bases[mca_btl_sm_component.my_smp_rank] =
(char*)mca_btl_sm_component.sm_mpool_base;
mca_btl_sm_component.shm_mem_nodes[mca_btl_sm_component.my_smp_rank] =
(uint16_t)my_mem_node;
/* initialize the array of fifo's "owned" by this process */
if(NULL == (my_fifos = (sm_fifo_t*)mpool_calloc(FIFO_MAP_NUM(n), sizeof(sm_fifo_t))))
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.shm_fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
/* cache the pointer to the 2d fifo array. These addresses
* are valid in the current process space */
mca_btl_sm_component.fifo = (sm_fifo_t**)malloc(sizeof(sm_fifo_t*) * n);
if(NULL == mca_btl_sm_component.fifo)
return OMPI_ERR_OUT_OF_RESOURCE;
mca_btl_sm_component.fifo[mca_btl_sm_component.my_smp_rank] = my_fifos;
mca_btl_sm_component.mem_nodes = (uint16_t *) malloc(sizeof(uint16_t) * n);
if(NULL == mca_btl_sm_component.mem_nodes)
return OMPI_ERR_OUT_OF_RESOURCE;
/* initialize fragment descriptor free lists */
/* allocation will be for the fragment descriptor and payload buffer */
length = sizeof(mca_btl_sm_frag1_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.eager_limit;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_eager, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag1_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
length = sizeof(mca_btl_sm_frag2_t);
length_payload =
sizeof(mca_btl_sm_hdr_t) + mca_btl_sm_component.max_frag_size;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_max, length,
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_frag2_t),
length_payload, opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user,
sizeof(mca_btl_sm_user_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_sm_user_t),
sizeof(mca_btl_sm_hdr_t), opal_cache_line_size,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
mca_btl_sm_component.num_outstanding_frags = 0;
mca_btl_sm_component.num_pending_sends = 0;
i = opal_free_list_init(&mca_btl_sm_component.pending_send_fl,
sizeof(btl_sm_pending_send_item_t),
OBJ_CLASS(opal_free_list_item_t),
16, -1, 32);
if ( OMPI_SUCCESS != i )
return i;
/* set flag indicating btl has been inited */
sm_btl->btl_inited = true;
return OMPI_SUCCESS;
}
static int rank_span(orte_job_t *jdata,
orte_app_context_t *app,
opal_list_t *nodes,
hwloc_obj_type_t target,
unsigned cache_level)
{
hwloc_obj_t obj;
int num_objs, i, j, rc;
orte_vpid_t num_ranked=0;
orte_node_t *node;
orte_proc_t *proc;
orte_vpid_t vpid;
int cnt;
opal_list_item_t *item;
hwloc_obj_t locale;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span: for job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* if the ranking is spanned, then we perform the
* ranking as if it was one big node - i.e., we
* rank one proc on each object, step to the next object
* moving across all the nodes, then wrap around to the
* first object on the first node.
*
* Node 0 Node 1
* Obj 0 Obj 1 Obj 0 Obj 1
* 0 4 1 5 2 6 3 7
* 8 12 9 13 10 14 11 15
*/
/* In the interest of getting this committed in finite time,
* just loop across the nodes and objects until all procs
* are mapped
*/
vpid = jdata->num_procs;
cnt = 0;
while (cnt < app->num_procs) {
for (item = opal_list_get_first(nodes);
item != opal_list_get_end(nodes);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
/* get the number of objects - only consider those we can actually use */
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target,
cache_level, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span: found %d objects on node %s with %d procs",
num_objs, node->name, (int)node->num_procs);
if (0 == num_objs) {
return ORTE_ERR_NOT_SUPPORTED;
}
/* for each object */
for (i=0; i < num_objs && cnt < app->num_procs; i++) {
obj = opal_hwloc_base_get_obj_by_type(node->topology, target,
cache_level, i, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span: working object %d", i);
/* cycle thru the procs on this node */
for (j=0; j < node->procs->size && cnt < app->num_procs; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
/* ignore procs from other jobs */
if (proc->name.jobid != jdata->jobid) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span skipping proc %s - from another job, num_ranked %d",
ORTE_NAME_PRINT(&proc->name), num_ranked);
continue;
}
/* ignore procs that are already assigned */
if (ORTE_VPID_INVALID != proc->name.vpid) {
continue;
}
/* ignore procs from other apps */
if (proc->app_idx != app->idx) {
continue;
}
/* protect against bozo case */
locale = NULL;
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
ORTE_ERROR_LOG(ORTE_ERROR);
return ORTE_ERROR;
}
/* ignore procs not on this object */
if (!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span: proc at position %d is not on object %d",
j, i);
continue;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_span: assigning vpid %s", ORTE_VPID_PRINT(vpid));
proc->name.vpid = vpid++;
if (0 == cnt) {
app->first_rank = proc->name.vpid;
}
cnt++;
/* insert the proc into the jdata array - no harm if already there */
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* track where the highest vpid landed - this is our
* new bookmark
*/
jdata->bookmark = node;
/* move to next object */
break;
}
}
}
}
return ORTE_SUCCESS;
}
/* recursively climb the topology, pruning procs beyond that allowed
* by the given ppr
*/
static void prune(orte_jobid_t jobid,
orte_app_idx_t app_idx,
orte_node_t *node,
opal_hwloc_level_t *level,
orte_vpid_t *nmapped)
{
hwloc_obj_t obj, top;
unsigned int i, nobjs;
hwloc_obj_type_t lvl;
unsigned cache_level = 0, k;
int nprocs;
hwloc_cpuset_t avail, cpus, childcpus;
int n, limit, nmax, nunder, idx, idxmax = 0;
orte_proc_t *proc, *pptr, *procmax;
opal_hwloc_level_t ll;
char dang[64];
hwloc_obj_t locale;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: pruning level %d",
*level);
/* convenience */
ll = *level;
/* convenience */
lvl = opal_hwloc_levels[ll];
limit = ppr[ll];
if (0 == limit) {
/* no limit at this level, so move up if necessary */
if (0 == ll) {
/* done */
return;
}
--(*level);
prune(jobid, app_idx, node, level, nmapped);
return;
}
/* handle the darn cache thing again */
if (OPAL_HWLOC_L3CACHE_LEVEL == ll) {
cache_level = 3;
} else if (OPAL_HWLOC_L2CACHE_LEVEL == ll) {
cache_level = 2;
} else if (OPAL_HWLOC_L1CACHE_LEVEL == ll) {
cache_level = 1;
}
/* get the number of resources at this level on this node */
nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology,
lvl, cache_level,
OPAL_HWLOC_AVAILABLE);
/* for each resource, compute the number of procs sitting
* underneath it and check against the limit
*/
for (i=0; i < nobjs; i++) {
obj = opal_hwloc_base_get_obj_by_type(node->topology,
lvl, cache_level,
i, OPAL_HWLOC_AVAILABLE);
/* get the available cpuset */
avail = opal_hwloc_base_get_available_cpus(node->topology, obj);
/* look at the intersection of this object's cpuset and that
* of each proc in the job/app - if they intersect, then count this proc
* against the limit
*/
nprocs = 0;
for (n=0; n < node->procs->size; n++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, n))) {
continue;
}
if (proc->name.jobid != jobid ||
proc->app_idx != app_idx) {
continue;
}
locale = NULL;
if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
cpus = opal_hwloc_base_get_available_cpus(node->topology, locale);
if (hwloc_bitmap_intersects(avail, cpus)) {
nprocs++;
}
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: found %d procs limit %d",
nprocs, limit);
/* check against the limit */
while (limit < nprocs) {
/* need to remove procs - do this in a semi-intelligent
* manner to provide a little load balancing by cycling
* across the objects beneath this one, removing procs
* in a round-robin fashion until the limit is satisfied
*
* NOTE: I'm sure someone more knowledgeable with hwloc
* will come up with a more efficient way to do this, so
* consider this is a starting point
*/
/* find the first level that has more than
* one child beneath it - if all levels
* have only one child, then return this
* object
*/
top = find_split(node->topology, obj);
hwloc_obj_type_snprintf(dang, 64, top, 1);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: SPLIT AT LEVEL %s", dang);
/* cycle across the children of this object */
nmax = 0;
procmax = NULL;
idx = 0;
/* find the child with the most procs underneath it */
for (k=0; k < top->arity && limit < nprocs; k++) {
/* get this object's available cpuset */
childcpus = opal_hwloc_base_get_available_cpus(node->topology, top->children[k]);
nunder = 0;
pptr = NULL;
for (n=0; n < node->procs->size; n++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, n))) {
continue;
}
if (proc->name.jobid != jobid ||
proc->app_idx != app_idx) {
continue;
}
locale = NULL;
if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return;
}
cpus = opal_hwloc_base_get_available_cpus(node->topology, locale);
if (hwloc_bitmap_intersects(childcpus, cpus)) {
nunder++;
if (NULL == pptr) {
/* save the location of the first proc under this object */
pptr = proc;
idx = n;
}
}
}
if (nmax < nunder) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: PROCS UNDER CHILD %d %d MAX %d",
k, nunder, nmax);
nmax = nunder;
procmax = pptr;
idxmax = idx;
}
}
if (NULL == procmax) {
/* can't find anything to remove - error out */
goto error;
}
/* remove it */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: removing proc at posn %d",
idxmax);
opal_pointer_array_set_item(node->procs, idxmax, NULL);
node->num_procs--;
node->slots_inuse--;
if (node->slots_inuse < 0) {
node->slots_inuse = 0;
}
nprocs--;
*nmapped -= 1;
OBJ_RELEASE(procmax);
}
}
/* finished with this level - move up if necessary */
if (0 == ll) {
return;
}
--(*level);
prune(jobid, app_idx, node, level, nmapped);
return;
error:
opal_output(0, "INFINITE LOOP");
}
static int rank_by(orte_job_t *jdata,
orte_app_context_t *app,
opal_list_t *nodes,
hwloc_obj_type_t target,
unsigned cache_level)
{
hwloc_obj_t obj;
int num_objs, i, j, rc;
orte_vpid_t num_ranked=0;
orte_node_t *node;
orte_proc_t *proc;
orte_vpid_t vpid;
int cnt;
opal_pointer_array_t objs;
bool all_done;
opal_list_item_t *item;
hwloc_obj_t locale;
if (ORTE_RANKING_SPAN & ORTE_GET_RANKING_DIRECTIVE(jdata->map->ranking)) {
return rank_span(jdata, app, nodes, target, cache_level);
} else if (ORTE_RANKING_FILL & ORTE_GET_RANKING_DIRECTIVE(jdata->map->ranking)) {
return rank_fill(jdata, app, nodes, target, cache_level);
}
/* if ranking is not spanned or filled, then we
* default to assign ranks sequentially across
* target objects within a node until that node
* is fully ranked, and then move on to the next
* node
*
* Node 0 Node 1
* Obj 0 Obj 1 Obj 0 Obj 1
* 0 2 1 3 8 10 9 11
* 4 6 5 7 12 14 13 15
*/
/* setup the pointer array */
OBJ_CONSTRUCT(&objs, opal_pointer_array_t);
opal_pointer_array_init(&objs, 2, INT_MAX, 2);
vpid = jdata->num_procs;
cnt = 0;
for (item = opal_list_get_first(nodes);
item != opal_list_get_end(nodes);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
/* get the number of objects - only consider those we can actually use */
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target,
cache_level, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_by: found %d objects on node %s with %d procs",
num_objs, node->name, (int)node->num_procs);
if (0 == num_objs) {
return ORTE_ERR_NOT_SUPPORTED;
}
/* collect all the objects */
for (i=0; i < num_objs; i++) {
obj = opal_hwloc_base_get_obj_by_type(node->topology, target,
cache_level, i, OPAL_HWLOC_AVAILABLE);
opal_pointer_array_set_item(&objs, i, obj);
}
/* cycle across the objects, assigning a proc to each one,
* until all procs have been assigned - unfortunately, since
* more than this job may be mapped onto a node, the number
* of procs on the node can't be used to tell us when we
* are done. Instead, we have to just keep going until all
* procs are ranked - which means we have to make one extra
* pass thru the loop
*
* Perhaps someday someone will come up with a more efficient
* algorithm, but this works for now.
*/
all_done = false;
while (!all_done && cnt < app->num_procs) {
all_done = true;
/* cycle across the objects */
for (i=0; i < num_objs && cnt < app->num_procs; i++) {
obj = (hwloc_obj_t)opal_pointer_array_get_item(&objs, i);
/* find the next proc on this object */
for (j=0; j < node->procs->size && cnt < app->num_procs; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
/* ignore procs from other jobs */
if (proc->name.jobid != jdata->jobid) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_by skipping proc %s - from another job, num_ranked %d",
ORTE_NAME_PRINT(&proc->name), num_ranked);
continue;
}
/* ignore procs that are already ranked */
if (ORTE_VPID_INVALID != proc->name.vpid) {
continue;
}
/* ignore procs from other apps */
if (proc->app_idx != app->idx) {
continue;
}
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
continue;
}
/* ignore procs on other objects */
if (!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_by: proc at position %d is not on object %d",
j, i);
continue;
}
proc->name.vpid = vpid++;
if (0 == cnt) {
app->first_rank = proc->name.vpid;
}
cnt++;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_by: assigned rank %s", ORTE_VPID_PRINT(proc->name.vpid));
/* insert the proc into the jdata array - no harm if already there */
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* flag that one was mapped */
all_done = false;
/* track where the highest vpid landed - this is our
* new bookmark
*/
jdata->bookmark = node;
/* move to next object */
break;
}
}
}
}
/* cleanup */
OBJ_DESTRUCT(&objs);
return ORTE_SUCCESS;
}
static int rank_fill(orte_job_t *jdata,
hwloc_obj_type_t target,
unsigned cache_level)
{
orte_app_context_t *app;
hwloc_obj_t obj;
int num_objs, i, j, m, n, rc;
orte_vpid_t num_ranked=0;
orte_node_t *node;
orte_proc_t *proc, *pptr;
orte_vpid_t vpid;
int cnt;
hwloc_obj_t locale;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill: for job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* if the ranking is fill, then we rank all the procs
* within a given object before moving on to the next
*
* Node 0 Node 1
* Obj 0 Obj 1 Obj 0 Obj 1
* 0 1 4 5 8 9 12 13
* 2 3 6 7 10 11 14 15
*/
vpid = 0;
for (n=0; n < jdata->apps->size; n++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
continue;
}
cnt = 0;
for (m=0; m < jdata->map->nodes->size; m++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
continue;
}
/* get the number of objects - only consider those we can actually use */
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, target,
cache_level, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill: found %d objects on node %s with %d procs",
num_objs, node->name, (int)node->num_procs);
if (0 == num_objs) {
return ORTE_ERR_NOT_SUPPORTED;
}
/* for each object */
for (i=0; i < num_objs && cnt < app->num_procs; i++) {
obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, target,
cache_level, i, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill: working object %d", i);
/* cycle thru the procs on this node */
for (j=0; j < node->procs->size && cnt < app->num_procs; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
/* ignore procs from other jobs */
if (proc->name.jobid != jdata->jobid) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill skipping proc %s - from another job, num_ranked %d",
ORTE_NAME_PRINT(&proc->name), num_ranked);
continue;
}
/* ignore procs that are already assigned */
if (ORTE_VPID_INVALID != proc->name.vpid) {
continue;
}
/* ignore procs from other apps */
if (proc->app_idx != app->idx) {
continue;
}
/* protect against bozo case */
locale = NULL;
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
ORTE_ERROR_LOG(ORTE_ERROR);
return ORTE_ERROR;
}
/* ignore procs not on this object */
if (!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill: proc at position %d is not on object %d",
j, i);
continue;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rank_fill: assigning vpid %s", ORTE_VPID_PRINT(vpid));
proc->name.vpid = vpid++;
if (0 == cnt) {
app->first_rank = proc->name.vpid;
}
cnt++;
/* insert the proc into the jdata array */
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
OBJ_RELEASE(pptr);
}
OBJ_RETAIN(proc);
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* track where the highest vpid landed - this is our
* new bookmark
*/
jdata->bookmark = node;
}
}
}
}
return ORTE_SUCCESS;
}
static int assign_locations(orte_job_t *jdata)
{
int i, j, m, n;
mca_base_component_t *c=&mca_rmaps_ppr_component.base_version;
orte_node_t *node;
orte_proc_t *proc;
orte_app_context_t *app;
hwloc_obj_type_t level;
hwloc_obj_t obj;
unsigned int cache_level=0;
int ppr, cnt, nobjs, nprocs_mapped;
char **ppr_req, **ck;
if (NULL == jdata->map->last_mapper ||
0 != strcasecmp(jdata->map->last_mapper, c->mca_component_name)) {
/* a mapper has been specified, and it isn't me */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: job %s not using ppr assign: %s",
ORTE_JOBID_PRINT(jdata->jobid),
(NULL == jdata->map->last_mapper) ? "NULL" : jdata->map->last_mapper);
return ORTE_ERR_TAKE_NEXT_OPTION;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:ppr: assigning locations for job %s with ppr %s policy %s",
ORTE_JOBID_PRINT(jdata->jobid), jdata->map->ppr,
orte_rmaps_base_print_mapping(jdata->map->mapping));
/* pickup the object level */
if (ORTE_MAPPING_BYNODE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_MACHINE;
} else if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_PU;
} else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_CORE;
} else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_SOCKET;
} else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_L1CACHE;
cache_level = 1;
} else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_L2CACHE;
cache_level = 2;
} else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_L3CACHE;
cache_level = 3;
} else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
level = HWLOC_OBJ_NUMANODE;
} else {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* get the ppr value */
ppr_req = opal_argv_split(jdata->map->ppr, ',');
ck = opal_argv_split(ppr_req[0], ':');
ppr = strtol(ck[0], NULL, 10);
opal_argv_free(ck);
opal_argv_free(ppr_req);
/* start assigning procs to objects, filling each object as we go until
* all procs are assigned. */
for (n=0; n < jdata->apps->size; n++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
continue;
}
nprocs_mapped = 0;
for (m=0; m < jdata->map->nodes->size; m++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
continue;
}
if (NULL == node->topology || NULL == node->topology->topo) {
orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
true, node->name);
return ORTE_ERR_SILENT;
}
if (HWLOC_OBJ_MACHINE == level) {
obj = hwloc_get_root_obj(node->topology->topo);
for (j=0; j < node->procs->size; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
if (proc->name.jobid != jdata->jobid) {
continue;
}
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
}
} else {
/* get the number of resources on this node at this level */
nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo,
level, cache_level,
OPAL_HWLOC_AVAILABLE);
/* map the specified number of procs to each such resource on this node,
* recording the locale of each proc so we know its cpuset
*/
for (i=0; i < nobjs; i++) {
cnt = 0;
obj = opal_hwloc_base_get_obj_by_type(node->topology->topo,
level, cache_level,
i, OPAL_HWLOC_AVAILABLE);
for (j=0; j < node->procs->size && cnt < ppr && nprocs_mapped < app->num_procs; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
if (proc->name.jobid != jdata->jobid) {
continue;
}
/* if we already assigned it, then skip */
if (orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, NULL, OPAL_PTR)) {
continue;
}
nprocs_mapped++;
cnt++;
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
}
}
}
}
}
return ORTE_SUCCESS;
}
static int byobj_span(orte_job_t *jdata,
orte_app_context_t *app,
opal_list_t *node_list,
orte_std_cntr_t num_slots,
orte_vpid_t num_procs,
hwloc_obj_type_t target, unsigned cache_level)
{
int i, j, nprocs_mapped, lag, delta, navg;
orte_node_t *node;
orte_proc_t *proc;
opal_list_item_t *item;
int num_procs_to_assign, nperobj, nprocs, nxtra_objs=0;
int extra_procs_to_assign=0, nxtra_nodes=0, idx;
hwloc_obj_t obj=NULL;
unsigned int nobjs;
float balance;
bool add_one=false;
bool oversubscribed=false;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr: mapping span by %s for job %s slots %d num_procs %lu",
hwloc_obj_type_string(target),
ORTE_JOBID_PRINT(jdata->jobid),
(int)num_slots, (unsigned long)num_procs);
/* quick check to see if we can map all the procs - can't
* do more because we don't know how many total objects exist
* across all the nodes
*/
if (num_slots < (int)app->num_procs) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
return ORTE_ERR_SILENT;
}
oversubscribed = true;
}
/* divide the procs evenly across all nodes - this is the
* average we have to maintain as we go, but we adjust
* the number on each node to reflect its available slots.
* Obviously, if all nodes have the same number of slots,
* then the avg is what we get on each node - this is
* the most common situation.
*/
navg = app->num_procs / opal_list_get_size(node_list);
if (0 == navg) {
/* if there are less procs than nodes, we have to
* place at least one/node
*/
navg = 1;
}
/* compute how many extra procs to put on each node */
balance = (float)((jdata->num_procs + app->num_procs) - (navg * opal_list_get_size(node_list))) / (float)opal_list_get_size(node_list);
extra_procs_to_assign = (int)balance;
if (0 < (balance - (float)extra_procs_to_assign)) {
/* compute how many nodes need an extra proc */
nxtra_nodes = (jdata->num_procs + app->num_procs) - ((navg + extra_procs_to_assign) * opal_list_get_size(node_list));
/* add one so that we add an extra proc to the first nodes
* until all procs are mapped
*/
extra_procs_to_assign++;
/* flag that we added one */
add_one = true;
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr: mapping by %s navg %d extra_procs %d extra_nodes %d",
hwloc_obj_type_string(target),
navg, extra_procs_to_assign, nxtra_nodes);
nprocs_mapped = 0;
lag = 0;
for (item = opal_list_get_first(node_list);
item != opal_list_get_end(node_list);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
/* bozo check */
if (NULL == node->topology) {
orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
true, node->name);
return ORTE_ERR_SILENT;
}
/* add this node to the map, if reqd */
if (!node->mapped) {
if (ORTE_SUCCESS > (idx = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
ORTE_ERROR_LOG(idx);
return idx;
}
node->mapped = true;
OBJ_RETAIN(node); /* maintain accounting on object */
++(jdata->map->num_nodes);
}
/* compute the number of procs to go on this node */
if (add_one) {
if (0 == nxtra_nodes) {
--extra_procs_to_assign;
add_one = false;
} else {
--nxtra_nodes;
}
}
if (oversubscribed) {
/* everybody just takes their share */
num_procs_to_assign = navg + extra_procs_to_assign;
} else {
/* if we are not oversubscribed, then there are enough
* slots to handle all the procs. However, not every
* node will have the same number of slots, so we
* have to track how many procs to "shift" elsewhere
* to make up the difference
*/
if (node->slots <= node->slots_inuse) {
/* if there are no extras to take, then we can
* safely remove this node as we don't need it
*/
if (0 == extra_procs_to_assign) {
opal_pointer_array_set_item(jdata->map->nodes, node->index, NULL);
OBJ_RELEASE(node);
--(jdata->map->num_nodes);
/* update how many we are lagging behind */
lag += navg;
continue;
}
/* everybody has to take at least the extras */
num_procs_to_assign = extra_procs_to_assign;
/* update how many we are lagging behind */
lag += navg;
} else {
/* if slots < avg, then take all */
if ((node->slots - node->slots_inuse) < navg) {
num_procs_to_assign = (node->slots - node->slots_inuse) + extra_procs_to_assign;
/* update how many we are lagging behind */
lag += navg - (node->slots - node->slots_inuse);
} else {
/* take the avg plus as much of the "lag" as we can */
delta = 0;
if (0 < lag) {
delta = (node->slots - node->slots_inuse) - navg;
if (lag < delta) {
delta = lag;
}
lag -= delta;
}
num_procs_to_assign = navg + delta + extra_procs_to_assign;
}
}
}
/* get the number of objects of this type on this node */
nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target, cache_level, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: found %d objs on node %s", nobjs, node->name);
/* compute the number of procs to go on each object */
nperobj = num_procs_to_assign / nobjs;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: placing %d procs on each object", nperobj);
if ((int)(nperobj * nobjs) < num_procs_to_assign) {
/* compute how many objs need an extra proc */
nxtra_objs = num_procs_to_assign - nperobj * nobjs;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: adding 1 extra proc to the first %d objects, if needed", nxtra_objs);
}
/* loop through the number of objects */
for (i=0; i < (int)nobjs && nprocs_mapped < (int)app->num_procs; i++) {
/* get the hwloc object */
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology, target, cache_level, i, OPAL_HWLOC_AVAILABLE))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
/* map the reqd number of procs */
if (0 < nxtra_objs) {
nprocs = nperobj + 1;
--nxtra_objs;
} else {
nprocs = nperobj;
}
for (j=0; j < nprocs && nprocs_mapped < app->num_procs; j++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
nprocs_mapped++;
proc->locale = obj;
}
/* keep track of the node we last used */
jdata->bookmark = node;
}
/* not all nodes are equal, so only set oversubscribed for
* this node if it is in that state
*/
if (node->slots < (int)node->num_procs) {
/* flag the node as oversubscribed so that sched-yield gets
* properly set
*/
node->oversubscribed = true;
}
if (nprocs_mapped == app->num_procs) {
/* we are done */
break;
}
}
return ORTE_SUCCESS;
}
/* mapping by hwloc object looks a lot like mapping by node,
* but has the added complication of possibly having different
* numbers of objects on each node
*/
int orte_rmaps_rr_byobj(orte_job_t *jdata,
orte_app_context_t *app,
opal_list_t *node_list,
orte_std_cntr_t num_slots,
orte_vpid_t num_procs,
hwloc_obj_type_t target, unsigned cache_level)
{
int i, j, nprocs_mapped;
orte_node_t *node;
orte_proc_t *proc;
opal_list_item_t *item;
int num_procs_to_assign, nperobj, nprocs, nxtra_objs=0;
int extra_procs_to_assign=0, nxtra_nodes=0, idx;
hwloc_obj_t obj=NULL;
unsigned int nobjs;
float balance;
bool add_one=false;
/* there are two modes for mapping by object: span and not-span. The
* span mode essentially operates as if there was just a single
* "super-node" in the system - i.e., it balances the load across
* all objects of the indicated type regardless of their location.
* In essence, it acts as if we placed one proc on each object, cycling
* across all objects on all nodes, and then wrapped around to place
* another proc on each object, doing so until all procs were placed.
*
* In contrast, the non-span mode operates similar to byslot mapping.
* All slots on each node are filled, assigning each proc to an object
* on that node in a balanced fashion, and then the mapper moves on
* to the next node. Thus, procs tend to be "front loaded" onto the
* list of nodes, as opposed to being "load balanced" in the span mode
*/
if (ORTE_MAPPING_SPAN & jdata->map->mapping) {
return byobj_span(jdata, app, node_list, num_slots,
num_procs, target, cache_level);
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr: mapping no-span by %s for job %s slots %d num_procs %lu",
hwloc_obj_type_string(target),
ORTE_JOBID_PRINT(jdata->jobid),
(int)num_slots, (unsigned long)num_procs);
/* quick check to see if we can map all the procs - can't
* do more because we don't know how many total objects exist
* across all the nodes
*/
if (num_slots < (app->num_procs * orte_rmaps_base.cpus_per_rank)) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
return ORTE_ERR_SILENT;
}
/* compute how many extra procs to put on each node */
if (1 == opal_list_get_size(node_list)) {
/* if there is only one node, then they all have to go on it */
extra_procs_to_assign = app->num_procs;
} else {
balance = (float)(((jdata->num_procs + app->num_procs)*orte_rmaps_base.cpus_per_rank) - num_slots) / (float)opal_list_get_size(node_list);
extra_procs_to_assign = (int)balance;
if (0 < (balance - (float)extra_procs_to_assign)) {
/* compute how many nodes need an extra proc */
nxtra_nodes = ((jdata->num_procs + app->num_procs)*orte_rmaps_base.cpus_per_rank) - num_slots - (extra_procs_to_assign * opal_list_get_size(node_list));
/* add one so that we add an extra proc to the first nodes
* until all procs are mapped
*/
extra_procs_to_assign++;
/* flag that we added one */
add_one = true;
}
}
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr: mapping no-span by %s extra_procs %d extra_nodes %d",
hwloc_obj_type_string(target),
extra_procs_to_assign, nxtra_nodes);
nprocs_mapped = 0;
for (item = opal_list_get_first(node_list);
item != opal_list_get_end(node_list);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
/* bozo check */
if (NULL == node->topology) {
orte_show_help("help-orte-rmaps-ppr.txt", "ppr-topo-missing",
true, node->name);
return ORTE_ERR_SILENT;
}
/* add this node to the map, if reqd */
if (!node->mapped) {
if (ORTE_SUCCESS > (idx = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
ORTE_ERROR_LOG(idx);
return idx;
}
node->mapped = true;
OBJ_RETAIN(node); /* maintain accounting on object */
++(jdata->map->num_nodes);
}
/* compute the number of procs to go on this node */
if (add_one) {
if (0 == nxtra_nodes) {
--extra_procs_to_assign;
add_one = false;
} else {
--nxtra_nodes;
}
}
if (node->slots <= node->slots_inuse) {
/* everybody takes at least the extras */
num_procs_to_assign = extra_procs_to_assign;
} else {
num_procs_to_assign = (node->slots - node->slots_inuse)/orte_rmaps_base.cpus_per_rank + extra_procs_to_assign;
if (app->num_procs < num_procs_to_assign) {
/* might have more slots than procs */
num_procs_to_assign = app->num_procs;
}
}
/* get the number of objects of this type on this node */
nobjs = opal_hwloc_base_get_nbobjs_by_type(node->topology, target, cache_level, OPAL_HWLOC_AVAILABLE);
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: nprocs-to-assign %d for %d objs on node %s", num_procs_to_assign, nobjs, node->name);
/* if there are no objects of this type, then report the error
* and abort - this can happen, for example, on systems that
* don't report "sockets" as an independent object. However, IF
* this object is the default one - i.e., not specified by the
* user - then we can fall back to mapping by slot
*/
if (0 == nobjs) {
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:no-objects",
true, hwloc_obj_type_string(target), node->name);
return ORTE_ERR_SILENT;
} else {
/* this was the default mapping policy, so clear the map
* of any prior work and indicate that map-by slot is reqd
*/
for (i=0; i < jdata->map->nodes->size; i++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, i))) {
continue;
}
for (idx=0; idx < node->procs->size; idx++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, idx))) {
continue;
}
if (proc->name.jobid != jdata->jobid) {
continue;
}
--node->num_procs;
OBJ_RELEASE(proc);
opal_pointer_array_set_item(node->procs, idx, NULL);
}
if (0 == node->num_procs) {
node->mapped = false;
OBJ_RELEASE(node);
opal_pointer_array_set_item(jdata->map->nodes, i, NULL);
}
}
return ORTE_ERR_NOT_SUPPORTED;
}
}
/* compute the number of procs to go on each object */
nperobj = num_procs_to_assign / nobjs;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: placing %d procs on each object", nperobj);
if ((int)(nperobj * nobjs) < num_procs_to_assign) {
/* compute how many objs need an extra proc */
nxtra_objs = num_procs_to_assign - nperobj * nobjs;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:rr:byobj: adding 1 extra proc to the first %d objects, if needed", nxtra_objs);
}
/* loop through the number of objects */
for (i=0; i < (int)nobjs && nprocs_mapped < (int)app->num_procs; i++) {
/* get the hwloc object */
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology, target, cache_level, i, OPAL_HWLOC_AVAILABLE))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
/* map the reqd number of procs */
if (0 < nxtra_objs) {
nprocs = nperobj + 1;
--nxtra_objs;
} else {
nprocs = nperobj;
}
for (j=0; j < nprocs && nprocs_mapped < app->num_procs; j++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, app->idx))) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
nprocs_mapped++;
proc->locale = obj;
}
}
/* not all nodes are equal, so only set oversubscribed for
* this node if it is in that state
*/
if (node->slots < (int)node->num_procs) {
/* flag the node as oversubscribed so that sched-yield gets
* properly set
*/
node->oversubscribed = true;
}
if (nprocs_mapped == app->num_procs) {
/* we are done */
break;
}
}
return ORTE_SUCCESS;
}
