static int orte_rmaps_rank_file_open(void)
{
/* ensure we flag mapping by user */
#if OPAL_HAVE_HWLOC
if (NULL != opal_hwloc_base_slot_list || NULL != orte_rankfile) {
#else
if (NULL != orte_rankfile) {
#endif
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
/* if a non-default mapping is already specified, then we
* have an error
*/
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"RANK_FILE", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_CONFLICTED);
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYUSER);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* we are going to bind to cpuset since the user is specifying the cpus */
OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CPUSET);
/* make us first */
my_priority = 10000;
}
return ORTE_SUCCESS;
}
static int orte_rmaps_rank_file_query(mca_base_module_t **module, int *priority)
{
*priority = my_priority;
*module = (mca_base_module_t *)&orte_rmaps_rank_file_module;
return ORTE_SUCCESS;
}
static int check_modifiers(char *ck, orte_mapping_policy_t *tmp)
{
char **ck2, *ptr;
int i;
bool found = false;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base check modifiers with %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == ck) ? "NULL" : ck);
if (NULL == ck) {
return ORTE_SUCCESS;
}
ck2 = opal_argv_split(ck, ',');
for (i=0; NULL != ck2[i]; i++) {
if (0 == strncasecmp(ck2[i], "span", strlen(ck2[i]))) {
ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_SPAN);
found = true;
} else if (0 == strncasecmp(ck2[i], "pe", strlen("pe"))) {
/* break this at the = sign to get the number */
if (NULL == (ptr = strchr(ck2[i], '='))) {
/* missing the value */
orte_show_help("help-orte-rmaps-base.txt", "missing-value", true, "pe", ck2[i]);
opal_argv_free(ck2);
return ORTE_ERR_SILENT;
}
ptr++;
orte_rmaps_base.cpus_per_rank = strtol(ptr, NULL, 10);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base setting pe/rank to %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orte_rmaps_base.cpus_per_rank);
found = true;
} else if (0 == strncasecmp(ck2[i], "oversubscribe", strlen(ck2[i]))) {
ORTE_UNSET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_SUBSCRIBE_GIVEN);
found = true;
} else if (0 == strncasecmp(ck2[i], "nooversubscribe", strlen(ck2[i]))) {
ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(*tmp, ORTE_MAPPING_SUBSCRIBE_GIVEN);
found = true;
} else {
/* unrecognized modifier */
opal_argv_free(ck2);
return ORTE_ERR_BAD_PARAM;
}
}
opal_argv_free(ck2);
if (found) {
return ORTE_SUCCESS;
}
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/*
* Function for selecting one component from all those that are
* available.
*/
void orte_ras_base_allocate(int fd, short args, void *cbdata)
{
int rc;
orte_job_t *jdata;
opal_list_t nodes;
orte_node_t *node;
orte_std_cntr_t i;
orte_app_context_t *app;
orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* convenience */
jdata = caddy->jdata;
/* if we already did this, don't do it again - the pool of
* global resources is set.
*/
if (orte_ras_base.allocation_read) {
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate allocation already read",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
goto next_state;
}
orte_ras_base.allocation_read = true;
/* Otherwise, we have to create
* the initial set of resources that will delineate all
* further operations serviced by this HNP. This list will
* contain ALL nodes that can be used by any subsequent job.
*
* In other words, if a node isn't found in this step, then
* no job launched by this HNP will be able to utilize it.
*/
/* construct a list to hold the results */
OBJ_CONSTRUCT(&nodes, opal_list_t);
/* if a component was selected, then we know we are in a managed
* environment. - the active module will return a list of what it found
*/
if (NULL != orte_ras_base.active_module) {
/* read the allocation */
if (ORTE_SUCCESS != (rc = orte_ras_base.active_module->allocate(jdata, &nodes))) {
if (ORTE_ERR_ALLOCATION_PENDING == rc) {
/* an allocation request is underway, so just do nothing */
OBJ_DESTRUCT(&nodes);
OBJ_RELEASE(caddy);
return;
}
if (ORTE_ERR_SYSTEM_WILL_BOOTSTRAP == rc) {
/* this module indicates that nodes will be discovered
* on a bootstrap basis, so all we do here is add our
* own node to the list
*/
goto addlocal;
}
if (ORTE_ERR_TAKE_NEXT_OPTION == rc) {
/* we have an active module, but it is unable to
* allocate anything for this job - this indicates
* that it isn't a fatal error, but could be if
* an allocation is required
*/
if (orte_allocation_required) {
/* an allocation is required, so this is fatal */
OBJ_DESTRUCT(&nodes);
orte_show_help("help-ras-base.txt", "ras-base:no-allocation", true);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
} else {
/* an allocation is not required, so we can just
* run on the local node - go add it
*/
goto addlocal;
}
}
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
}
/* If something came back, save it and we are done */
if (!opal_list_is_empty(&nodes)) {
/* store the results in the global resource pool - this removes the
* list items
*/
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nodes, jdata))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
OBJ_DESTRUCT(&nodes);
/* default to no-oversubscribe-allowed for managed systems */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
}
/* flag that the allocation is managed */
orte_managed_allocation = true;
goto DISPLAY;
} else if (orte_allocation_required) {
/* if nothing was found, and an allocation is
* required, then error out
*/
OBJ_DESTRUCT(&nodes);
orte_show_help("help-ras-base.txt", "ras-base:no-allocation", true);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate nothing found in module - proceeding to hostfile",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* nothing was found, or no active module was alive. Our next
* option is to look for a hostfile and assign our global
* pool from there.
*
* Individual hostfile names, if given, are included
* in the app_contexts for this job. We therefore need to
* retrieve the app_contexts for the job, and then cycle
* through them to see if anything is there. The parser will
* add the nodes found in each hostfile to our list - i.e.,
* the resulting list contains the UNION of all nodes specified
* in hostfiles from across all app_contexts
*
* We then continue to add any hosts provided by dash-host and
* the default hostfile, if we have it. We will then filter out
* all the non-desired hosts (i.e., those not specified by
* -host and/or -hostfile) when we start the mapping process
*
* Note that any relative node syntax found in the hostfiles will
* generate an error in this scenario, so only non-relative syntax
* can be present
*/
if (NULL != orte_default_hostfile) {
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate parsing default hostfile %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orte_default_hostfile));
/* a default hostfile was provided - parse it */
if (ORTE_SUCCESS != (rc = orte_util_add_hostfile_nodes(&nodes,
orte_default_hostfile))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
}
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
if (NULL != app->hostfile) {
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate adding hostfile %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
app->hostfile));
/* hostfile was specified - parse it and add it to the list */
if (ORTE_SUCCESS != (rc = orte_util_add_hostfile_nodes(&nodes,
app->hostfile))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
/* set an error event */
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
} else if (!orte_soft_locations && NULL != app->dash_host) {
/* if we are using soft locations, then any dash-host would
* just include desired nodes and not required. We don't want
* to pick them up here as this would mean the request was
* always satisfied - instead, we want to allow the request
* to fail later on and use whatever nodes are actually
* available
*/
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate adding dash_hosts",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != (rc = orte_util_add_dash_host_nodes(&nodes,
app->dash_host))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
}
}
/* if something was found in the hostfile(s), we use that as our global
* pool - set it and we are done
*/
if (!opal_list_is_empty(&nodes)) {
/* store the results in the global resource pool - this removes the
* list items
*/
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nodes, jdata))) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
/* cleanup */
OBJ_DESTRUCT(&nodes);
goto DISPLAY;
}
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate nothing found in hostfiles - checking for rankfile",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* Our next option is to look for a rankfile - if one was provided, we
* will use its nodes to create a default allocation pool
*/
if (NULL != orte_rankfile) {
/* check the rankfile for node information */
if (ORTE_SUCCESS != (rc = orte_util_add_hostfile_nodes(&nodes,
orte_rankfile))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return ;
}
}
/* if something was found in rankfile, we use that as our global
* pool - set it and we are done
*/
if (!opal_list_is_empty(&nodes)) {
/* store the results in the global resource pool - this removes the
* list items
*/
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nodes, jdata))) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
/* rankfile is considered equivalent to an RM allocation */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
}
/* cleanup */
OBJ_DESTRUCT(&nodes);
goto DISPLAY;
}
OPAL_OUTPUT_VERBOSE((5, orte_ras_base_framework.framework_output,
"%s ras:base:allocate nothing found in rankfile - inserting current node",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
addlocal:
/* if nothing was found by any of the above methods, then we have no
* earthly idea what to do - so just add the local host
*/
node = OBJ_NEW(orte_node_t);
if (NULL == node) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
/* use the same name we got in orte_process_info so we avoid confusion in
* the session directories
*/
node->name = strdup(orte_process_info.nodename);
node->state = ORTE_NODE_STATE_UP;
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = 1;
opal_list_append(&nodes, &node->super);
/* store the results in the global resource pool - this removes the
* list items
*/
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nodes, jdata))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nodes);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
OBJ_DESTRUCT(&nodes);
DISPLAY:
/* shall we display the results? */
if (4 < opal_output_get_verbosity(orte_ras_base_framework.framework_output)) {
orte_ras_base_display_alloc();
}
next_state:
/* are we to report this event? */
if (orte_report_events) {
if (ORTE_SUCCESS != (rc = orte_util_comm_report_event(ORTE_COMM_EVENT_ALLOCATE))) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
}
}
/* set total slots alloc */
jdata->total_slots_alloc = orte_ras_base.total_slots_alloc;
/* set the job state to the next position */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATION_COMPLETE);
/* cleanup */
OBJ_RELEASE(caddy);
}
/*
* Sequentially map the ranks according to the placement in the
* specified hostfile
*/
static int orte_rmaps_seq_map(orte_job_t *jdata)
{
orte_job_map_t *map;
orte_app_context_t *app;
int i, n;
orte_std_cntr_t j;
opal_list_item_t *item;
orte_node_t *node, *nd;
seq_node_t *sq, *save=NULL, *seq;;
orte_vpid_t vpid;
orte_std_cntr_t num_nodes;
int rc;
opal_list_t default_seq_list;
opal_list_t node_list, *seq_list, sq_list;
orte_proc_t *proc;
mca_base_component_t *c = &mca_rmaps_seq_component.base_version;
char *hosts = NULL, *sep, *eptr;
FILE *fp;
opal_hwloc_resource_type_t rtype;
OPAL_OUTPUT_VERBOSE((1, orte_rmaps_base_framework.framework_output,
"%s rmaps:seq called on job %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jdata->jobid)));
/* this mapper can only handle initial launch
* when seq mapping is desired - allow
* restarting of failed apps
*/
if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s is being restarted - seq cannot map",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
if (NULL != jdata->map->req_mapper) {
if (0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
/* a mapper has been specified, and it isn't me */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s not using sequential mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* we need to process it */
goto process;
}
if (ORTE_MAPPING_SEQ != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
/* I don't know how to do these - defer */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s not using seq mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
process:
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: mapping job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* flag that I did the mapping */
if (NULL != jdata->map->last_mapper) {
free(jdata->map->last_mapper);
}
jdata->map->last_mapper = strdup(c->mca_component_name);
/* convenience def */
map = jdata->map;
/* if there is a default hostfile, go and get its ordered list of nodes */
OBJ_CONSTRUCT(&default_seq_list, opal_list_t);
if (NULL != orte_default_hostfile) {
char *hstname = NULL;
/* open the file */
fp = fopen(orte_default_hostfile, "r");
if (NULL == fp) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto error;
}
while (NULL != (hstname = orte_getline(fp))) {
if (0 == strlen(hstname)) {
free(hstname);
/* blank line - ignore */
continue;
}
if( '#' == hstname[0] ) {
free(hstname);
/* Comment line - ignore */
continue;
}
sq = OBJ_NEW(seq_node_t);
if (NULL != (sep = strchr(hstname, ' '))) {
*sep = '\0';
sep++;
/* remove any trailing space */
eptr = sep + strlen(sep) - 1;
while (eptr > sep && isspace(*eptr)) {
eptr--;
}
*(eptr+1) = 0;
sq->cpuset = strdup(sep);
}
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
char *ptr;
if (NULL != (ptr = strchr(hstname, '.'))) {
*ptr = '\0';
}
}
sq->hostname = hstname;
opal_list_append(&default_seq_list, &sq->super);
}
fclose(fp);
}
/* start at the beginning... */
vpid = 0;
jdata->num_procs = 0;
if (0 < opal_list_get_size(&default_seq_list)) {
save = (seq_node_t*)opal_list_get_first(&default_seq_list);
}
/* default to LOGICAL processors */
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_PHYSICAL_CPUIDS, NULL, OPAL_BOOL)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using PHYSICAL processors");
rtype = OPAL_HWLOC_PHYSICAL;
} else {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using LOGICAL processors");
rtype = OPAL_HWLOC_LOGICAL;
}
/* initialize all the nodes as not included in this job map */
for (j=0; j < orte_node_pool->size; j++) {
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
}
}
/* cycle through the app_contexts, mapping them sequentially */
for(i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
/* dash-host trumps hostfile */
if (orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using dash-host nodes on app %s", app->app);
OBJ_CONSTRUCT(&node_list, opal_list_t);
/* dash host entries cannot specify cpusets, so used the std function to retrieve the list */
if (ORTE_SUCCESS != (rc = orte_util_get_ordered_dash_host_list(&node_list, hosts))) {
ORTE_ERROR_LOG(rc);
free(hosts);
goto error;
}
free(hosts);
/* transfer the list to a seq_node_t list */
OBJ_CONSTRUCT(&sq_list, opal_list_t);
while (NULL != (nd = (orte_node_t*)opal_list_remove_first(&node_list))) {
sq = OBJ_NEW(seq_node_t);
sq->hostname = strdup(nd->name);
opal_list_append(&sq_list, &sq->super);
OBJ_RELEASE(nd);
}
OBJ_DESTRUCT(&node_list);
seq_list = &sq_list;
} else if (orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
char *hstname;
if (NULL == hosts) {
rc = ORTE_ERR_NOT_FOUND;
goto error;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using hostfile %s nodes on app %s", hosts, app->app);
OBJ_CONSTRUCT(&sq_list, opal_list_t);
/* open the file */
fp = fopen(hosts, "r");
if (NULL == fp) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
OBJ_DESTRUCT(&sq_list);
goto error;
}
while (NULL != (hstname = orte_getline(fp))) {
if (0 == strlen(hstname)) {
free(hstname);
/* blank line - ignore */
continue;
}
if( '#' == hstname[0] ) {
free(hstname);
/* Comment line - ignore */
continue;
}
sq = OBJ_NEW(seq_node_t);
if (NULL != (sep = strchr(hstname, ' '))) {
*sep = '\0';
sep++;
/* remove any trailing space */
eptr = sep + strlen(sep) - 1;
while (eptr > sep && isspace(*eptr)) {
eptr--;
}
*(eptr+1) = 0;
sq->cpuset = strdup(sep);
}
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
char *ptr;
if (NULL != (ptr = strchr(hstname, '.'))) {
(*ptr) = '\0';
}
}
sq->hostname = hstname;
opal_list_append(&sq_list, &sq->super);
}
fclose(fp);
free(hosts);
seq_list = &sq_list;
} else if (0 < opal_list_get_size(&default_seq_list)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using default hostfile nodes on app %s", app->app);
seq_list = &default_seq_list;
} else {
/* can't do anything - no nodes available! */
orte_show_help("help-orte-rmaps-base.txt",
"orte-rmaps-base:no-available-resources",
true);
return ORTE_ERR_SILENT;
}
/* check for nolocal and remove the head node, if required */
if (map->mapping & ORTE_MAPPING_NO_USE_LOCAL) {
for (item = opal_list_get_first(seq_list);
item != opal_list_get_end(seq_list);
item = opal_list_get_next(item) ) {
seq = (seq_node_t*)item;
/* need to check ifislocal because the name in the
* hostfile may not have been FQDN, while name returned
* by gethostname may have been (or vice versa)
*/
if (orte_ifislocal(seq->hostname)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: removing head node %s", seq->hostname);
opal_list_remove_item(seq_list, item);
OBJ_RELEASE(item); /* "un-retain" it */
}
}
}
if (NULL == seq_list || 0 == (num_nodes = (orte_std_cntr_t)opal_list_get_size(seq_list))) {
orte_show_help("help-orte-rmaps-base.txt",
"orte-rmaps-base:no-available-resources",
true);
return ORTE_ERR_SILENT;
}
/* if num_procs wasn't specified, set it now */
if (0 == app->num_procs) {
app->num_procs = num_nodes;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: setting num procs to %s for app %s",
ORTE_VPID_PRINT(app->num_procs), app->app);
} else if (num_nodes < app->num_procs) {
orte_show_help("help-orte-rmaps-base.txt", "seq:not-enough-resources", true,
app->num_procs, num_nodes);
return ORTE_ERR_SILENT;
}
if (seq_list == &default_seq_list) {
sq = save;
} else {
sq = (seq_node_t*)opal_list_get_first(seq_list);
}
for (n=0; n < app->num_procs; n++) {
/* find this node on the global array - this is necessary so
* that our mapping gets saved on that array as the objects
* returned by the hostfile function are -not- on the array
*/
node = NULL;
for (j=0; j < orte_node_pool->size; j++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
continue;
}
if (0 == strcmp(sq->hostname, node->name)) {
break;
}
}
if (NULL == node) {
/* wasn't found - that is an error */
orte_show_help("help-orte-rmaps-seq.txt",
"orte-rmaps-seq:resource-not-found",
true, sq->hostname);
rc = ORTE_ERR_SILENT;
goto error;
}
/* ensure the node is in the map */
if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
OBJ_RETAIN(node);
opal_pointer_array_add(map->nodes, node);
jdata->map->num_nodes++;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
}
proc = orte_rmaps_base_setup_proc(jdata, node, i);
if ((node->slots < (int)node->num_procs) ||
(0 < node->slots_max && node->slots_max < (int)node->num_procs)) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, node->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
rc = ORTE_ERR_SILENT;
goto error;
}
/* flag the node as oversubscribed so that sched-yield gets
* properly set
*/
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
/* check for permission */
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
/* if we weren't given a directive either way, then we will error out
* as the #slots were specifically given, either by the host RM or
* via hostfile/dash-host */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
return ORTE_ERR_SILENT;
} else if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
/* if we were explicitly told not to oversubscribe, then don't */
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
return ORTE_ERR_SILENT;
}
}
}
/* assign the vpid */
proc->name.vpid = vpid++;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: assign proc %s to node %s for app %s",
ORTE_VPID_PRINT(proc->name.vpid), sq->hostname, app->app);
/* record the cpuset, if given */
if (NULL != sq->cpuset) {
hwloc_cpuset_t bitmap;
char *cpu_bitmap;
if (NULL == node->topology) {
/* not allowed - for sequential cpusets, we must have
* the topology info
*/
orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology", true, node->name);
rc = ORTE_ERR_SILENT;
goto error;
}
/* if we are using hwthreads as cpus and binding to hwthreads, then
* we can just copy the cpuset across as it already specifies things
* at that level */
if (opal_hwloc_use_hwthreads_as_cpus &&
OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
cpu_bitmap = strdup(sq->cpuset);
} else {
/* setup the bitmap */
bitmap = hwloc_bitmap_alloc();
/* parse the slot_list to find the socket and core */
if (ORTE_SUCCESS != (rc = opal_hwloc_base_slot_list_parse(sq->cpuset, node->topology, rtype, bitmap))) {
ORTE_ERROR_LOG(rc);
hwloc_bitmap_free(bitmap);
goto error;
}
/* note that we cannot set the proc locale to any specific object
* as the slot list may have assigned it to more than one - so
* leave that field NULL
*/
/* set the proc to the specified map */
hwloc_bitmap_list_asprintf(&cpu_bitmap, bitmap);
hwloc_bitmap_free(bitmap);
}
orte_set_attribute(&proc->attributes, ORTE_PROC_CPU_BITMAP, ORTE_ATTR_GLOBAL, cpu_bitmap, OPAL_STRING);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: binding proc %s to cpuset %s bitmap %s",
ORTE_VPID_PRINT(proc->name.vpid), sq->cpuset, cpu_bitmap);
/* we are going to bind to cpuset since the user is specifying the cpus */
OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CPUSET);
/* note that the user specified the mapping */
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYUSER);
ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_GIVEN);
/* cleanup */
free(cpu_bitmap);
} else {
hwloc_obj_t locale;
/* assign the locale - okay for the topo to be null as
* it just means it wasn't returned
*/
if (NULL != node->topology) {
locale = hwloc_get_root_obj(node->topology);
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE,
ORTE_ATTR_LOCAL, locale, OPAL_PTR);
}
}
/* add to the jdata proc array */
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
ORTE_ERROR_LOG(rc);
goto error;
}
/* move to next node */
sq = (seq_node_t*)opal_list_get_next(&sq->super);
}
/** track the total number of processes we mapped */
jdata->num_procs += app->num_procs;
/* cleanup the node list if it came from this app_context */
if (seq_list != &default_seq_list) {
OPAL_LIST_DESTRUCT(seq_list);
} else {
save = sq;
}
}
return ORTE_SUCCESS;
error:
OPAL_LIST_DESTRUCT(&default_seq_list);
return rc;
}
int orte_rmaps_base_set_mapping_policy(orte_mapping_policy_t *policy,
char **device, char *inspec)
{
char *ck;
char *ptr;
orte_mapping_policy_t tmp;
int rc;
size_t len;
char *spec;
char *pch;
/* set defaults */
tmp = 0;
if (NULL != device) {
*device = NULL;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base set policy with %s device %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == inspec) ? "NULL" : inspec,
(NULL == device) ? "NULL" : "NONNULL");
if (NULL == inspec) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
} else {
spec = strdup(inspec); // protect the input string
/* see if a colon was included - if so, then we have a policy + modifier */
ck = strchr(spec, ':');
if (NULL != ck) {
/* if the colon is the first character of the string, then we
* just have modifiers on the default mapping policy */
if (ck == spec) {
ck++;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base only modifiers %s provided - assuming bysocket mapping",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ck);
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
if (ORTE_ERR_SILENT == (rc = check_modifiers(ck, &tmp)) &&
ORTE_ERR_BAD_PARAM != rc) {
free(spec);
return ORTE_ERR_SILENT;
}
free(spec);
goto setpolicy;
}
/* split the string */
*ck = '\0';
ck++;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base policy %s modifiers %s provided",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), spec, ck);
/* if the policy is "dist", then we set the policy to that value
* and save the second argument as the device
*/
if (0 == strncasecmp(spec, "ppr", strlen(spec))) {
/* we have to allow additional modifiers here - e.g., specifying
* #pe's/proc or oversubscribe - so check for modifiers
*/
if (NULL == (ptr = strrchr(ck, ':'))) {
/* this is an error - there had to be at least one
* colon to delimit the number from the object type
*/
orte_show_help("help-orte-rmaps-base.txt", "invalid-pattern", true, inspec);
free(spec);
return ORTE_ERR_SILENT;
}
ptr++; // move past the colon
/* check the remaining string for modifiers - may be none, so
* don't emit an error message if the modifier isn't recognized
*/
if (ORTE_ERR_SILENT == (rc = check_modifiers(ptr, &tmp)) &&
ORTE_ERR_BAD_PARAM != rc) {
free(spec);
return ORTE_ERR_SILENT;
}
/* if we found something, then we need to adjust the string */
if (ORTE_SUCCESS == rc) {
ptr--;
*ptr = '\0';
}
/* now get the pattern */
orte_rmaps_base.ppr = strdup(ck);
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_PPR);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
free(spec);
goto setpolicy;
}
if (ORTE_SUCCESS != (rc = check_modifiers(ck, &tmp)) &&
ORTE_ERR_TAKE_NEXT_OPTION != rc) {
if (ORTE_ERR_BAD_PARAM == rc) {
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-modifier", true, inspec);
}
free(spec);
return rc;
}
}
len = strlen(spec);
if (0 == strncasecmp(spec, "slot", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSLOT);
} else if (0 == strncasecmp(spec, "node", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNODE);
} else if (0 == strncasecmp(spec, "seq", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_SEQ);
} else if (0 == strncasecmp(spec, "core", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYCORE);
} else if (0 == strncasecmp(spec, "l1cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL1CACHE);
} else if (0 == strncasecmp(spec, "l2cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL2CACHE);
} else if (0 == strncasecmp(spec, "l3cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL3CACHE);
} else if (0 == strncasecmp(spec, "socket", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
} else if (0 == strncasecmp(spec, "numa", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNUMA);
} else if (0 == strncasecmp(spec, "board", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYBOARD);
} else if (0 == strncasecmp(spec, "hwthread", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYHWTHREAD);
/* if we are mapping processes to individual hwthreads, then
* we need to treat those hwthreads as separate cpus
*/
opal_hwloc_use_hwthreads_as_cpus = true;
} else if (0 == strncasecmp(spec, "dist", len)) {
if (NULL != rmaps_dist_device) {
if (NULL != (pch = strchr(rmaps_dist_device, ':'))) {
*pch = '\0';
}
if (NULL != device) {
*device = strdup(rmaps_dist_device);
}
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYDIST);
} else {
orte_show_help("help-orte-rmaps-base.txt", "device-not-specified", true);
free(spec);
return ORTE_ERR_SILENT;
}
} else {
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy", true, "mapping", spec);
free(spec);
return ORTE_ERR_SILENT;
}
free(spec);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
}
setpolicy:
*policy = tmp;
return ORTE_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
static int orte_rmaps_base_open(mca_base_open_flag_t flags)
{
int rc;
/* init the globals */
OBJ_CONSTRUCT(&orte_rmaps_base.selected_modules, opal_list_t);
orte_rmaps_base.slot_list = NULL;
orte_rmaps_base.mapping = 0;
orte_rmaps_base.ranking = 0;
orte_rmaps_base.device = NULL;
/* if a topology file was given, then set our topology
* from it. Even though our actual topology may differ,
* mpirun only needs to see the compute node topology
* for mapping purposes
*/
if (NULL != rmaps_base_topo_file) {
if (OPAL_SUCCESS != (rc = opal_hwloc_base_set_topology(rmaps_base_topo_file))) {
orte_show_help("help-orte-rmaps-base.txt", "topo-file", true, rmaps_base_topo_file);
return ORTE_ERR_SILENT;
}
}
/* check for violations that has to be detected before we parse the mapping option */
if (NULL != orte_rmaps_base.ppr) {
orte_show_help("help-orte-rmaps-base.txt", "deprecated", true,
"--ppr, -ppr", "--map-by ppr:<pattern>",
"rmaps_base_pattern, rmaps_ppr_pattern",
"rmaps_base_mapping_policy=ppr:<pattern>");
/* if the mapping policy is NULL, then we can proceed */
if (NULL == rmaps_base_mapping_policy) {
asprintf(&rmaps_base_mapping_policy, "ppr:%s", orte_rmaps_base.ppr);
} else {
return ORTE_ERR_SILENT;
}
}
if (1 < orte_rmaps_base.cpus_per_rank) {
orte_show_help("help-orte-rmaps-base.txt", "deprecated", true,
"--cpus-per-proc, -cpus-per-proc, --cpus-per-rank, -cpus-per-rank",
"--map-by <obj>:PE=N, default <obj>=NUMA",
"rmaps_base_cpus_per_proc", "rmaps_base_mapping_policy=<obj>:PE=N, default <obj>=NUMA");
}
if (ORTE_SUCCESS != (rc = orte_rmaps_base_set_mapping_policy(&orte_rmaps_base.mapping,
&orte_rmaps_base.device,
rmaps_base_mapping_policy))) {
return rc;
}
if (ORTE_SUCCESS != (rc = orte_rmaps_base_set_ranking_policy(&orte_rmaps_base.ranking,
orte_rmaps_base.mapping,
rmaps_base_ranking_policy))) {
return rc;
}
if (rmaps_base_bycore) {
orte_show_help("help-orte-rmaps-base.txt", "deprecated", true,
"--bycore, -bycore", "--map-by core",
"rmaps_base_bycore", "rmaps_base_mapping_policy=core");
/* set mapping policy to bycore - error if something else already set */
if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) != ORTE_MAPPING_BYCORE) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bycore", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYCORE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* set ranking policy to bycore - error if something else already set */
if ((ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(orte_rmaps_base.ranking)) &&
ORTE_GET_RANKING_POLICY(orte_rmaps_base.ranking) != ORTE_RANK_BY_CORE) {
/* error - cannot redefine the default ranking policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "ranking",
"bycore", orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
return ORTE_ERR_SILENT;
}
ORTE_SET_RANKING_POLICY(orte_rmaps_base.ranking, ORTE_RANK_BY_CORE);
ORTE_SET_RANKING_DIRECTIVE(orte_rmaps_base.ranking, ORTE_RANKING_GIVEN);
}
if (rmaps_base_byslot) {
orte_show_help("help-orte-rmaps-base.txt", "deprecated", true,
"--byslot, -byslot", "--map-by slot",
"rmaps_base_byslot", "rmaps_base_mapping_policy=slot");
/* set mapping policy to byslot - error if something else already set */
if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) != ORTE_MAPPING_BYSLOT) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"byslot", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYSLOT);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* set ranking policy to byslot - error if something else already set */
if ((ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(orte_rmaps_base.ranking)) &&
ORTE_GET_RANKING_POLICY(orte_rmaps_base.ranking) != ORTE_RANK_BY_SLOT) {
/* error - cannot redefine the default ranking policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "ranking",
"byslot", orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
return ORTE_ERR_SILENT;
}
ORTE_SET_RANKING_POLICY(orte_rmaps_base.ranking, ORTE_RANK_BY_SLOT);
ORTE_SET_RANKING_DIRECTIVE(orte_rmaps_base.ranking, ORTE_RANKING_GIVEN);
}
if (rmaps_base_bynode) {
orte_show_help("help-orte-rmaps-base.txt", "deprecated", true,
"--bynode, -bynode", "--map-by node",
"rmaps_base_bynode", "rmaps_base_mapping_policy=node");
/* set mapping policy to bynode - error if something else already set */
if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) != ORTE_MAPPING_BYNODE) {
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bynode", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYNODE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* set ranking policy to bynode - error if something else already set */
if ((ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(orte_rmaps_base.ranking)) &&
ORTE_GET_RANKING_POLICY(orte_rmaps_base.ranking) != ORTE_RANK_BY_NODE) {
/* error - cannot redefine the default ranking policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "ranking",
"bynode", orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
return ORTE_ERR_SILENT;
}
ORTE_SET_RANKING_POLICY(orte_rmaps_base.ranking, ORTE_RANK_BY_NODE);
ORTE_SET_RANKING_DIRECTIVE(orte_rmaps_base.ranking, ORTE_RANKING_GIVEN);
}
if (1 < orte_rmaps_base.cpus_per_rank) {
/* if we were asked for multiple cpus/proc, then we have to
* bind to those cpus - any other binding policy is an
* error
*/
if (OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy)) {
if (opal_hwloc_use_hwthreads_as_cpus) {
if (OPAL_BIND_TO_HWTHREAD != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
orte_show_help("help-orte-rmaps-base.txt", "mismatch-binding", true,
orte_rmaps_base.cpus_per_rank, "use-hwthreads-as-cpus",
opal_hwloc_base_print_binding(opal_hwloc_binding_policy),
"bind-to hwthread");
return ORTE_ERR_SILENT;
}
} else if (OPAL_BIND_TO_CORE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy) &&
OPAL_BIND_TO_NONE != OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
orte_show_help("help-orte-rmaps-base.txt", "mismatch-binding", true,
orte_rmaps_base.cpus_per_rank, "cores as cpus",
opal_hwloc_base_print_binding(opal_hwloc_binding_policy),
"bind-to core");
return ORTE_ERR_SILENT;
}
} else {
if (opal_hwloc_use_hwthreads_as_cpus) {
OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_HWTHREAD);
} else {
OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_CORE);
}
}
/* we also need to ensure we are mapping to a high-enough level to have
* multiple cpus beneath it - by default, we'll go to the NUMA level */
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
if (ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) == ORTE_MAPPING_BYHWTHREAD ||
(ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) == ORTE_MAPPING_BYCORE &&
!opal_hwloc_use_hwthreads_as_cpus)) {
orte_show_help("help-orte-rmaps-base.txt", "mapping-too-low-init", true);
return ORTE_ERR_SILENT;
}
} else {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"%s rmaps:base pe/rank set - setting mapping to BYNUMA",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYNUMA);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
}
}
if (orte_rmaps_base_pernode) {
/* there is no way to resolve this conflict, so if something else was
* given, we have no choice but to error out
*/
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bynode", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
/* ensure we set the mapping policy to ppr */
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_PPR);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* define the ppr */
orte_rmaps_base.ppr = strdup("1:node");
}
if (0 < orte_rmaps_base_n_pernode) {
/* there is no way to resolve this conflict, so if something else was
* given, we have no choice but to error out
*/
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bynode", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
/* ensure we set the mapping policy to ppr */
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_PPR);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* define the ppr */
asprintf(&orte_rmaps_base.ppr, "%d:node", orte_rmaps_base_n_pernode);
}
if (0 < orte_rmaps_base_n_persocket) {
/* there is no way to resolve this conflict, so if something else was
* given, we have no choice but to error out
*/
if (ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bynode", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
/* ensure we set the mapping policy to ppr */
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_PPR);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* define the ppr */
asprintf(&orte_rmaps_base.ppr, "%d:socket", orte_rmaps_base_n_persocket);
}
/* Should we schedule on the local node or not? */
if (rmaps_base_no_schedule_local) {
orte_rmaps_base.mapping |= ORTE_MAPPING_NO_USE_LOCAL;
}
/* Should we oversubscribe or not? */
if (rmaps_base_no_oversubscribe) {
if ((ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
!(ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"no-oversubscribe", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_SUBSCRIBE_GIVEN);
}
/** force oversubscription permission */
if (rmaps_base_oversubscribe) {
if ((ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
(ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"oversubscribe", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_UNSET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_SUBSCRIBE_GIVEN);
/* also set the overload allowed flag */
opal_hwloc_binding_policy |= OPAL_BIND_ALLOW_OVERLOAD;
}
/* should we display a detailed (developer-quality) version of the map after determining it? */
if (rmaps_base_display_devel_map) {
orte_rmaps_base.display_map = true;
orte_devel_level_output = true;
}
/* should we display a diffable report of proc locations after determining it? */
if (rmaps_base_display_diffable_map) {
orte_rmaps_base.display_map = true;
orte_display_diffable_output = true;
}
/* Open up all available components */
rc = mca_base_framework_components_open(&orte_rmaps_base_framework, flags);
/* check to see if any component indicated a problem */
if (ORTE_MAPPING_CONFLICTED & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
/* the component would have already reported the error, so
* tell the rest of the chain to shut up
*/
return ORTE_ERR_SILENT;
}
/* All done */
return rc;
}
/*
* Function for selecting one component from all those that are
* available.
*/
void orte_rmaps_base_map_job(int fd, short args, void *cbdata)
{
orte_job_t *jdata;
orte_job_map_t *map;
int rc;
bool did_map;
opal_list_item_t *item;
orte_rmaps_base_selected_module_t *mod;
orte_job_t *parent;
orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
/* convenience */
jdata = caddy->jdata;
jdata->state = ORTE_JOB_STATE_MAP;
/* NOTE: NO PROXY COMPONENT REQUIRED - REMOTE PROCS ARE NOT
* ALLOWED TO CALL RMAPS INDEPENDENTLY. ONLY THE PLM CAN
* DO SO, AND ALL PLM COMMANDS ARE RELAYED TO HNP
*/
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps: mapping job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* NOTE: CHECK FOR JDATA->MAP == NULL. IF IT IS, THEN USE
* THE VALUES THAT WERE READ BY THE LOCAL MCA PARAMS. THE
* PLM PROXY WILL SEND A JOB-OBJECT THAT WILL INCLUDE ANY
* MAPPING DIRECTIVES - OTHERWISE, THAT OBJECT WILL HAVE A
* NULL MAP FIELD
* LONE EXCEPTION - WE COPY DISPLAY MAP ACROSS IF THEY
* DIDN'T SET IT
*/
if (NULL == jdata->map) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps: creating new map for job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* create a map object where we will store the results */
map = OBJ_NEW(orte_job_map_t);
if (NULL == map) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
/* load it with the system defaults */
map->mapping = orte_rmaps_base.mapping;
map->ranking = orte_rmaps_base.ranking;
#if OPAL_HAVE_HWLOC
map->binding = opal_hwloc_binding_policy;
#endif
if (NULL != orte_rmaps_base.ppr) {
map->ppr = strdup(orte_rmaps_base.ppr);
}
map->cpus_per_rank = orte_rmaps_base.cpus_per_rank;
map->display_map = orte_rmaps_base.display_map;
/* assign the map object to this job */
jdata->map = map;
} else {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps: setting mapping policies for job %s",
ORTE_JOBID_PRINT(jdata->jobid));
if (!jdata->map->display_map) {
jdata->map->display_map = orte_rmaps_base.display_map;
}
/* set the default mapping policy IFF it wasn't provided */
if (!ORTE_MAPPING_POLICY_IS_SET(jdata->map->mapping)) {
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, orte_rmaps_base.mapping);
}
if (!ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping));
}
/* ditto for rank and bind policies */
if (!ORTE_RANKING_POLICY_IS_SET(jdata->map->ranking)) {
ORTE_SET_RANKING_POLICY(jdata->map->ranking, orte_rmaps_base.ranking);
}
#if OPAL_HAVE_HWLOC
if (!OPAL_BINDING_POLICY_IS_SET(jdata->map->binding)) {
jdata->map->binding = opal_hwloc_binding_policy;
}
#endif
}
#if OPAL_HAVE_HWLOC
/* if we are not going to launch, then we need to set any
* undefined topologies to match our own so the mapper
* can operate
*/
if (orte_do_not_launch) {
orte_node_t *node;
hwloc_topology_t t0;
int i;
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
t0 = node->topology;
for (i=1; i < orte_node_pool->size; i++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) {
continue;
}
if (NULL == node->topology) {
node->topology = t0;
}
}
}
#endif
/* cycle thru the available mappers until one agrees to map
* the job
*/
did_map = false;
if (1 == opal_list_get_size(&orte_rmaps_base.selected_modules)) {
/* forced selection */
mod = (orte_rmaps_base_selected_module_t*)opal_list_get_first(&orte_rmaps_base.selected_modules);
jdata->map->req_mapper = strdup(mod->component->mca_component_name);
}
for (item = opal_list_get_first(&orte_rmaps_base.selected_modules);
item != opal_list_get_end(&orte_rmaps_base.selected_modules);
item = opal_list_get_next(item)) {
mod = (orte_rmaps_base_selected_module_t*)item;
if (ORTE_SUCCESS == (rc = mod->module->map_job(jdata)) ||
ORTE_ERR_RESOURCE_BUSY == rc) {
did_map = true;
break;
}
/* mappers return "next option" if they didn't attempt to
* map the job. anything else is a true error.
*/
if (ORTE_ERR_TAKE_NEXT_OPTION != rc) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
}
if (did_map && ORTE_ERR_RESOURCE_BUSY == rc) {
/* the map was done but nothing could be mapped
* for launch as all the resources were busy
*/
OBJ_RELEASE(caddy);
return;
}
/* if we get here without doing the map, or with zero procs in
* the map, then that's an error
*/
if (!did_map || 0 == jdata->num_procs) {
orte_show_help("help-orte-rmaps-base.txt", "failed-map", true);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
/* compute and save local ranks */
if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_local_ranks(jdata))) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
#if OPAL_HAVE_HWLOC
/* compute and save bindings */
if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_bindings(jdata))) {
ORTE_ERROR_LOG(rc);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
OBJ_RELEASE(caddy);
return;
}
#endif
/* set the offset so shared memory components can potentially
* connect to any spawned jobs
*/
jdata->offset = orte_total_procs;
/* track the total number of procs launched by us */
orte_total_procs += jdata->num_procs;
/* if it is a dynamic spawn, save the bookmark on the parent's job too */
if (ORTE_JOBID_INVALID != jdata->originator.jobid) {
if (NULL != (parent = orte_get_job_data_object(jdata->originator.jobid))) {
parent->bookmark = jdata->bookmark;
}
}
/* if we wanted to display the map, now is the time to do it - ignore
* daemon job
*/
if (jdata->map->display_map) {
char *output=NULL;
int i, j;
orte_node_t *node;
orte_proc_t *proc;
if (orte_display_diffable_output) {
/* intended solely to test mapping methods, this output
* can become quite long when testing at scale. Rather
* than enduring all the malloc/free's required to
* create an arbitrary-length string, custom-generate
* the output a line at a time here
*/
/* display just the procs in a diffable format */
opal_output(orte_clean_output, "<map>\n\t<jobid=%s>\n\t<offset=%s>",
ORTE_JOBID_PRINT(jdata->jobid), ORTE_VPID_PRINT(jdata->offset));
fflush(stderr);
/* loop through nodes */
for (i=0; i < jdata->map->nodes->size; i++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, i))) {
continue;
}
opal_output(orte_clean_output, "\t<host name=%s>", (NULL == node->name) ? "UNKNOWN" : node->name);
fflush(stderr);
for (j=0; j < node->procs->size; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
#if OPAL_HAVE_HWLOC
{
char locale[64];
if (NULL != proc->locale) {
hwloc_bitmap_list_snprintf(locale, 64, proc->locale->cpuset);
}
opal_output(orte_clean_output, "\t\t<process rank=%s app_idx=%ld local_rank=%lu node_rank=%lu locale=%s binding=%s>",
ORTE_VPID_PRINT(proc->name.vpid), (long)proc->app_idx,
(unsigned long)proc->local_rank,
(unsigned long)proc->node_rank, locale,
(NULL == proc->cpu_bitmap) ? "NULL" : proc->cpu_bitmap);
}
#else
opal_output(orte_clean_output, "\t\t<process rank=%s app_idx=%ld local_rank=%lu node_rank=%lu>",
ORTE_VPID_PRINT(proc->name.vpid), (long)proc->app_idx,
(unsigned long)proc->local_rank,
(unsigned long)proc->node_rank);
#endif
fflush(stderr);
}
opal_output(orte_clean_output, "\t</host>");
fflush(stderr);
}
#if OPAL_HAVE_HWLOC
{
opal_hwloc_locality_t locality;
orte_proc_t *p0;
/* test locality - for the first node, print the locality of each proc relative to the first one */
node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, 0);
p0 = (orte_proc_t*)opal_pointer_array_get_item(node->procs, 0);
opal_output(orte_clean_output, "\t<locality>");
for (j=1; j < node->procs->size; j++) {
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
continue;
}
locality = opal_hwloc_base_get_relative_locality(node->topology,
p0->cpu_bitmap,
proc->cpu_bitmap);
opal_output(orte_clean_output, "\t\t<rank=%s rank=%s locality=%s>",
ORTE_VPID_PRINT(p0->name.vpid),
ORTE_VPID_PRINT(proc->name.vpid),
opal_hwloc_base_print_locality(locality));
}
opal_output(orte_clean_output, "\t</locality>\n</map>");
fflush(stderr);
}
#else
opal_output(orte_clean_output, "\n</map>");
fflush(stderr);
#endif
} else {
opal_output(orte_clean_output, " Data for JOB %s offset %s", ORTE_JOBID_PRINT(jdata->jobid), ORTE_VPID_PRINT(jdata->offset));
opal_dss.print(&output, NULL, jdata->map, ORTE_JOB_MAP);
if (orte_xml_output) {
fprintf(orte_xml_fp, "%s\n", output);
fflush(orte_xml_fp);
} else {
opal_output(orte_clean_output, "%s", output);
}
free(output);
}
}
/* set the job state to the next position */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_MAP_COMPLETE);
/* cleanup */
OBJ_RELEASE(caddy);
}
static void recv_data(int fd, short args, void *cbdata)
{
bool found;
int i, rc;
orte_node_t *nd, *nd2;
opal_list_t nds, ndtmp;
opal_list_item_t *item, *itm;
char recv_msg[8192];
int nbytes, idx, sjob;
char **alloc, *nodelist, *tpn;
local_jobtracker_t *ptr, *jtrk;
local_apptracker_t *aptrk;
orte_app_context_t *app;
orte_jobid_t jobid;
orte_job_t *jdata;
char **dash_host = NULL;
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation - data recvd",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* read the data from the socket and put it in the
* nodes field of op
*/
memset(recv_msg, 0, sizeof(recv_msg));
nbytes = read(fd, recv_msg, sizeof(recv_msg) - 1);
opal_output_verbose(2, orte_ras_base_framework.framework_output,
"%s ras:slurm: dynamic allocation msg: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), recv_msg);
/* check if we got something */
if (0 == nbytes || 0 == strlen(recv_msg) || strstr(recv_msg, "failure") != NULL) {
/* show an error here - basically, a "nothing was available"
* message
*/
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true,
(0 == strlen(recv_msg)) ? "NO MSG" : recv_msg);
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
return;
}
/* break the message into its component parts, separated by colons */
alloc = opal_argv_split(recv_msg, ':');
/* the first section contains the ORTE jobid for this allocation */
tpn = strchr(alloc[0], '=');
orte_util_convert_string_to_jobid(&jobid, tpn+1);
/* get the corresponding job object */
jdata = orte_get_job_data_object(jobid);
jtrk = NULL;
/* find the associated tracking object */
for (item = opal_list_get_first(&jobs);
item != opal_list_get_end(&jobs);
item = opal_list_get_next(item)) {
ptr = (local_jobtracker_t*)item;
if (ptr->jobid == jobid) {
jtrk = ptr;
break;
}
}
if (NULL == jtrk) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, "NO JOB TRACKER");
ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
return;
}
/* stop the timeout event */
opal_event_del(&jtrk->timeout_ev);
/* cycle across all the remaining parts - each is the allocation for
* an app in this job
*/
OBJ_CONSTRUCT(&nds, opal_list_t);
OBJ_CONSTRUCT(&ndtmp, opal_list_t);
idx = -1;
sjob = -1;
nodelist = NULL;
tpn = NULL;
for (i=1; NULL != alloc[i]; i++) {
if (ORTE_SUCCESS != parse_alloc_msg(alloc[i], &idx, &sjob, &nodelist, &tpn)) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
if (NULL != nodelist) {
free(nodelist);
}
if (NULL != tpn) {
free(tpn);
}
return;
}
if (idx < 0) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, idx))) {
orte_show_help("help-ras-slurm.txt", "slurm-dyn-alloc-failed", true, jtrk->cmd);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
/* release the current dash_host as that contained the *desired* allocation */
orte_remove_attribute(&app->attributes, ORTE_APP_DASH_HOST);
/* track the Slurm jobid */
if (NULL == (aptrk = (local_apptracker_t*)opal_pointer_array_get_item(&jtrk->apps, idx))) {
aptrk = OBJ_NEW(local_apptracker_t);
opal_pointer_array_set_item(&jtrk->apps, idx, aptrk);
}
aptrk->sjob = sjob;
/* since the nodelist/tpn may contain regular expressions, parse them */
if (ORTE_SUCCESS != (rc = orte_ras_slurm_discover(nodelist, tpn, &ndtmp))) {
ORTE_ERROR_LOG(rc);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOC_FAILED);
opal_argv_free(alloc);
free(nodelist);
free(tpn);
return;
}
/* transfer the discovered nodes to our node list, and construct
* the new dash_host entry to match what was allocated
*/
while (NULL != (item = opal_list_remove_first(&ndtmp))) {
nd = (orte_node_t*)item;
opal_argv_append_nosize(&dash_host, nd->name);
/* check for duplicates */
found = false;
for (itm = opal_list_get_first(&nds);
itm != opal_list_get_end(&nds);
itm = opal_list_get_next(itm)) {
nd2 = (orte_node_t*)itm;
if (0 == strcmp(nd->name, nd2->name)) {
found = true;
nd2->slots += nd->slots;
OBJ_RELEASE(item);
break;
}
}
if (!found) {
/* append the new node to our list */
opal_list_append(&nds, item);
}
}
/* cleanup */
free(nodelist);
free(tpn);
}
/* cleanup */
opal_argv_free(alloc);
OBJ_DESTRUCT(&ndtmp);
if (NULL != dash_host) {
tpn = opal_argv_join(dash_host, ',');
orte_set_attribute(&app->attributes, ORTE_APP_DASH_HOST, ORTE_ATTR_LOCAL, (void*)tpn, OPAL_STRING);
opal_argv_free(dash_host);
free(tpn);
}
if (opal_list_is_empty(&nds)) {
/* if we get here, then we were able to contact slurm,
* which means we are in an actively managed cluster.
* However, slurm indicated that nothing is currently
* available that meets our requirements. This is a fatal
* situation - we do NOT have the option of running on
* user-specified hosts as the cluster is managed.
*/
OBJ_DESTRUCT(&nds);
orte_show_help("help-ras-base.txt", "ras-base:no-allocation", true);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
}
/* store the found nodes */
if (ORTE_SUCCESS != (rc = orte_ras_base_node_insert(&nds, jdata))) {
ORTE_ERROR_LOG(rc);
OBJ_DESTRUCT(&nds);
ORTE_FORCED_TERMINATE(ORTE_ERROR_DEFAULT_EXIT_CODE);
return;
}
OBJ_DESTRUCT(&nds);
/* default to no-oversubscribe-allowed for managed systems */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
}
/* flag that the allocation is managed */
orte_managed_allocation = true;
/* move the job along */
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_ALLOCATION_COMPLETE);
/* all done */
return;
}
int orte_rmaps_base_set_mapping_policy(orte_mapping_policy_t *policy,
char **device, char *inspec)
{
char *ck;
#if OPAL_HAVE_HWLOC
char *ptr;
#endif
orte_mapping_policy_t tmp;
int rc;
size_t len;
char *spec;
/* set defaults */
tmp = 0;
*device = NULL;
if (NULL == inspec) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
} else {
spec = strdup(inspec); // protect the input string
/* see if a colon was included - if so, then we have a policy + modifier */
ck = strchr(spec, ':');
if (NULL != ck) {
/* split the string */
*ck = '\0';
ck++;
/* if the policy is "dist", then we set the policy to that value
* and save the second argument as the device
*/
#if OPAL_HAVE_HWLOC
if (0 == strncasecmp(spec, "dist", strlen(spec))) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYDIST);
/* the first argument after the colon *must* be the
* device we are mapping near - however, other modifiers
* could have been provided, so check for them, okay if
* none found
*/
if (NULL != (ptr = strchr(ck, ','))) {
*ptr = '\0';
ptr++; // move past the comma
/* check the remaining string for modifiers - may be none, so
* don't emit an error message if the modifier isn't recognized
*/
if (ORTE_ERR_SILENT == (rc = check_modifiers(ptr, &tmp)) &&
ORTE_ERR_BAD_PARAM != rc) {
free(spec);
return ORTE_ERR_SILENT;
}
}
*device = strdup(ck);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
free(spec);
goto setpolicy;
} else if (0 == strncasecmp(spec, "ppr", strlen(spec))) {
/* we have to allow additional modifiers here - e.g., specifying
* #pe's/proc or oversubscribe - so check for modifiers
*/
if (NULL == (ptr = strrchr(ck, ':'))) {
/* this is an error - there had to be at least one
* colon to delimit the number from the object type
*/
orte_show_help("help-orte-rmaps-base.txt", "invalid-pattern", true, inspec);
free(spec);
return ORTE_ERR_SILENT;
}
ptr++; // move past the colon
/* check the remaining string for modifiers - may be none, so
* don't emit an error message if the modifier isn't recognized
*/
if (ORTE_ERR_SILENT == (rc = check_modifiers(ptr, &tmp)) &&
ORTE_ERR_BAD_PARAM != rc) {
free(spec);
return ORTE_ERR_SILENT;
}
/* if we found something, then we need to adjust the string */
if (ORTE_SUCCESS == rc) {
ptr--;
*ptr = '\0';
}
/* now get the pattern */
orte_rmaps_base.ppr = strdup(ck);
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_PPR);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
free(spec);
goto setpolicy;
}
#endif
if (ORTE_SUCCESS != (rc = check_modifiers(ck, &tmp)) &&
ORTE_ERR_TAKE_NEXT_OPTION != rc) {
if (ORTE_ERR_BAD_PARAM == rc) {
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-modifier", true, inspec);
}
free(spec);
return rc;
}
}
len = strlen(spec);
if (0 == strncasecmp(spec, "slot", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSLOT);
} else if (0 == strncasecmp(spec, "node", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNODE);
} else if (0 == strncasecmp(spec, "seq", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_SEQ);
#if OPAL_HAVE_HWLOC
} else if (0 == strncasecmp(spec, "core", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYCORE);
} else if (0 == strncasecmp(spec, "l1cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL1CACHE);
} else if (0 == strncasecmp(spec, "l2cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL2CACHE);
} else if (0 == strncasecmp(spec, "l3cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL3CACHE);
} else if (0 == strncasecmp(spec, "socket", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
} else if (0 == strncasecmp(spec, "numa", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNUMA);
} else if (0 == strncasecmp(spec, "board", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYBOARD);
} else if (0 == strncasecmp(spec, "hwthread", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYHWTHREAD);
/* if we are mapping processes to individual hwthreads, then
* we need to treat those hwthreads as separate cpus
*/
opal_hwloc_use_hwthreads_as_cpus = true;
#endif
} else {
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy", true, "mapping", spec);
free(spec);
return ORTE_ERR_SILENT;
}
free(spec);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
}
#if OPAL_HAVE_HWLOC
setpolicy:
#endif
*policy = tmp;
return ORTE_SUCCESS;
}
int orte_rmaps_base_set_mapping_policy(orte_mapping_policy_t *policy,
char **device, char *spec)
{
char **ck, **ck2;
orte_mapping_policy_t tmp;
int i;
size_t len;
/* set defaults */
tmp = 0;
*device = NULL;
if (NULL == spec) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_SPAN);
ORTE_UNSET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
} else {
ck = opal_argv_split(spec, ':');
if (2 < opal_argv_count(ck)) {
/* incorrect format */
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy", true, "mapping", rmaps_base_mapping_policy);
opal_argv_free(ck);
return ORTE_ERR_SILENT;
}
if (2 == opal_argv_count(ck)) {
/* if the policy is "dist", then we set the policy to that value
* and save the second argument as the device
*/
#if OPAL_HAVE_HWLOC
if (0 == strncasecmp(ck[0], "dist", strlen(ck[0]))) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYDIST);
ck2 = opal_argv_split(ck[1], ',');
if (ck2[0] != NULL) {
*device = strdup(ck2[0]);
for (i=1; NULL != ck2[i]; i++) {
if (0 == strncasecmp(ck2[i], "span", strlen(ck2[i]))) {
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_SPAN);
}
}
}
opal_argv_free(ck2);
goto setpolicy;
}
#endif
ck2 = opal_argv_split(ck[1], ',');
for (i=0; NULL != ck2[i]; i++) {
if (0 == strncasecmp(ck2[i], "span", strlen(ck2[i]))) {
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_SPAN);
} else if (0 == strncasecmp(ck2[i], "oversubscribe", strlen(ck2[i]))) {
if (ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(tmp)) {
ORTE_UNSET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_SUBSCRIBE_GIVEN);
} else if (0 == strncasecmp(ck2[i], "nooversubscribe", strlen(ck2[i]))) {
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_SUBSCRIBE_GIVEN);
} else {
/* unrecognized modifier */
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-modifier", true, "mapping", ck2[i]);
opal_argv_free(ck);
opal_argv_free(ck2);
return ORTE_ERR_SILENT;
}
}
opal_argv_free(ck2);
}
}
len = strlen(ck[0]);
if (0 == strncasecmp(ck[0], "slot", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSLOT);
} else if (0 == strncasecmp(ck[0], "node", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNODE);
#if OPAL_HAVE_HWLOC
} else if (0 == strncasecmp(ck[0], "core", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYCORE);
} else if (0 == strncasecmp(ck[0], "l1cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL1CACHE);
} else if (0 == strncasecmp(ck[0], "l2cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL2CACHE);
} else if (0 == strncasecmp(ck[0], "l3cache", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYL3CACHE);
} else if (0 == strncasecmp(ck[0], "socket", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYSOCKET);
} else if (0 == strncasecmp(ck[0], "numa", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYNUMA);
} else if (0 == strncasecmp(ck[0], "board", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYBOARD);
} else if (0 == strncasecmp(ck[0], "hwthread", len)) {
ORTE_SET_MAPPING_POLICY(tmp, ORTE_MAPPING_BYHWTHREAD);
/* if we are mapping processes to individual hwthreads, then
* we need to treat those hwthreads as separate cpus
*/
opal_hwloc_use_hwthreads_as_cpus = true;
#endif
} else {
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy", true, "mapping", rmaps_base_mapping_policy);
opal_argv_free(ck);
return ORTE_ERR_SILENT;
}
opal_argv_free(ck);
ORTE_SET_MAPPING_DIRECTIVE(tmp, ORTE_MAPPING_GIVEN);
}
#if OPAL_HAVE_HWLOC
setpolicy:
#endif
*policy = tmp;
return ORTE_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
static int orte_rmaps_base_open(mca_base_open_flag_t flags)
{
int rc;
/* init the globals */
OBJ_CONSTRUCT(&orte_rmaps_base.selected_modules, opal_list_t);
orte_rmaps_base.ppr = NULL;
orte_rmaps_base.slot_list = NULL;
orte_rmaps_base.mapping = 0;
orte_rmaps_base.ranking = 0;
#if OPAL_HAVE_HWLOC
/* if a topology file was given, then set our topology
* from it. Even though our actual topology may differ,
* mpirun only needs to see the compute node topology
* for mapping purposes
*/
if (NULL != rmaps_base_topo_file) {
if (OPAL_SUCCESS != (rc = opal_hwloc_base_set_topology(rmaps_base_topo_file))) {
orte_show_help("help-orte-rmaps-base.txt", "topo-file", true, rmaps_base_topo_file);
return ORTE_ERR_SILENT;
}
}
#endif
if (ORTE_SUCCESS != (rc = orte_rmaps_base_set_mapping_policy(&orte_rmaps_base.mapping,
&orte_rmaps_base.device,
rmaps_base_mapping_policy))) {
return rc;
}
if (ORTE_SUCCESS != (rc = orte_rmaps_base_set_ranking_policy(&orte_rmaps_base.ranking,
orte_rmaps_base.mapping,
rmaps_base_ranking_policy))) {
return rc;
}
if (rmaps_base_byslot) {
/* set mapping policy to byslot - error if something else already set */
if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) != ORTE_MAPPING_BYSLOT) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"byslot", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYSLOT);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* set ranking policy to byslot - error if something else already set */
if ((ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(orte_rmaps_base.ranking)) &&
ORTE_GET_RANKING_POLICY(orte_rmaps_base.ranking) != ORTE_RANK_BY_SLOT) {
/* error - cannot redefine the default ranking policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "ranking",
"byslot", orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
return ORTE_ERR_SILENT;
}
ORTE_SET_RANKING_POLICY(orte_rmaps_base.ranking, ORTE_RANK_BY_SLOT);
ORTE_SET_RANKING_DIRECTIVE(orte_rmaps_base.ranking, ORTE_RANKING_GIVEN);
}
if (rmaps_base_bynode) {
/* set mapping policy to bynode - error if something else already set */
if ((ORTE_MAPPING_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
ORTE_GET_MAPPING_POLICY(orte_rmaps_base.mapping) != ORTE_MAPPING_BYNODE) {
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"bynode", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_POLICY(orte_rmaps_base.mapping, ORTE_MAPPING_BYNODE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_GIVEN);
/* set ranking policy to bynode - error if something else already set */
if ((ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(orte_rmaps_base.ranking)) &&
ORTE_GET_RANKING_POLICY(orte_rmaps_base.ranking) != ORTE_RANK_BY_NODE) {
/* error - cannot redefine the default ranking policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "ranking",
"bynode", orte_rmaps_base_print_ranking(orte_rmaps_base.ranking));
return ORTE_ERR_SILENT;
}
ORTE_SET_RANKING_POLICY(orte_rmaps_base.ranking, ORTE_RANK_BY_NODE);
ORTE_SET_RANKING_DIRECTIVE(orte_rmaps_base.ranking, ORTE_RANKING_GIVEN);
}
/* Should we schedule on the local node or not? */
if (rmaps_base_no_schedule_local) {
orte_rmaps_base.mapping |= ORTE_MAPPING_NO_USE_LOCAL;
}
/* Should we oversubscribe or not? */
if (rmaps_base_no_oversubscribe) {
if ((ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
!(ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"no-oversubscribe", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_SUBSCRIBE_GIVEN);
}
/** force oversubscription permission */
if (rmaps_base_oversubscribe) {
if ((ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) &&
(ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
/* error - cannot redefine the default mapping policy */
orte_show_help("help-orte-rmaps-base.txt", "redefining-policy", true, "mapping",
"oversubscribe", orte_rmaps_base_print_mapping(orte_rmaps_base.mapping));
return ORTE_ERR_SILENT;
}
ORTE_UNSET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
ORTE_SET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping, ORTE_MAPPING_SUBSCRIBE_GIVEN);
}
/* should we display a detailed (developer-quality) version of the map after determining it? */
if (rmaps_base_display_devel_map) {
orte_rmaps_base.display_map = true;
orte_devel_level_output = true;
}
/* should we display a diffable report of proc locations after determining it? */
if (rmaps_base_display_diffable_map) {
orte_rmaps_base.display_map = true;
orte_display_diffable_output = true;
}
/* Open up all available components */
rc = mca_base_framework_components_open(&orte_rmaps_base_framework, flags);
/* check to see if any component indicated a problem */
if (ORTE_MAPPING_CONFLICTED & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping)) {
/* the component would have already reported the error, so
* tell the rest of the chain to shut up
*/
return ORTE_ERR_SILENT;
}
/* All done */
return rc;
}
static void setup_job_complete(int fd, short args, void *cbdata)
{
orte_state_caddy_t *caddy = (orte_state_caddy_t*)cbdata;
orte_job_t *jdata = caddy->jdata;
int i, j;
orte_app_context_t *app;
orte_proc_t *proc;
orte_vpid_t vpid;
opal_buffer_t *buf;
/* check that the job meets our requirements */
vpid = 0;
for (i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
if (app->num_procs <= 0) {
/* must specify -np for staged_hnp execution */
orte_show_help("help-state-staged-hnp.txt", "no-np", true);
ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_SILENT_ABORT);
OBJ_RELEASE(caddy);
return;
}
/* build the proc arrays - we'll need them later */
for (j=0; j < app->num_procs; j++) {
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = jdata->jobid;
proc->name.vpid = vpid;
proc->app_idx = i;
proc->app_rank = j;
/* flag that the proc is NOT to be included
* in a pidmap message so we don't do it until
* the proc is actually scheduled for launch
*/
ORTE_FLAG_UNSET(proc, ORTE_PROC_FLAG_UPDATED);
/* procs must not barrier when executing in stages */
orte_set_attribute(&proc->attributes, ORTE_PROC_NOBARRIER, ORTE_ATTR_GLOBAL, NULL, OPAL_BOOL);
/* add it to the job */
opal_pointer_array_set_item(jdata->procs, vpid, proc);
jdata->num_procs++;
vpid++;
/* add it to the app */
OBJ_RETAIN(proc);
opal_pointer_array_set_item(&app->procs, j, proc);
}
}
/* set the job map to use the staged_hnp mapper */
if (NULL == jdata->map) {
jdata->map = OBJ_NEW(orte_job_map_t);
jdata->map->req_mapper = strdup("staged");
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_STAGED);
ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_NO_OVERSUBSCRIBE);
jdata->map->display_map = orte_rmaps_base.display_map;
}
/* if there are any file_maps attached to this job, load them */
buf = NULL;
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_FILE_MAPS, (void**)&buf, OPAL_BUFFER)) {
orte_dfs.load_file_maps(jdata->jobid, buf, NULL, NULL);
OBJ_RELEASE(buf);
}
orte_plm_base_setup_job_complete(0, 0, (void*)caddy);
}
