/*
* Sequentially map the ranks according to the placement in the
* specified hostfile
*/
static int orte_rmaps_seq_map(orte_job_t *jdata)
{
orte_job_map_t *map;
orte_app_context_t *app;
int i, n;
orte_std_cntr_t j;
opal_list_item_t *item;
orte_node_t *node, *nd;
seq_node_t *sq, *save=NULL, *seq;;
orte_vpid_t vpid;
orte_std_cntr_t num_nodes;
int rc;
opal_list_t default_seq_list;
opal_list_t node_list, *seq_list, sq_list;
orte_proc_t *proc;
mca_base_component_t *c = &mca_rmaps_seq_component.base_version;
char *hosts = NULL, *sep, *eptr;
FILE *fp;
opal_hwloc_resource_type_t rtype;
OPAL_OUTPUT_VERBOSE((1, orte_rmaps_base_framework.framework_output,
"%s rmaps:seq called on job %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jdata->jobid)));
/* this mapper can only handle initial launch
* when seq mapping is desired - allow
* restarting of failed apps
*/
if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s is being restarted - seq cannot map",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
if (NULL != jdata->map->req_mapper) {
if (0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
/* a mapper has been specified, and it isn't me */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s not using sequential mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
/* we need to process it */
goto process;
}
if (ORTE_MAPPING_SEQ != ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
/* I don't know how to do these - defer */
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: job %s not using seq mapper",
ORTE_JOBID_PRINT(jdata->jobid));
return ORTE_ERR_TAKE_NEXT_OPTION;
}
process:
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: mapping job %s",
ORTE_JOBID_PRINT(jdata->jobid));
/* flag that I did the mapping */
if (NULL != jdata->map->last_mapper) {
free(jdata->map->last_mapper);
}
jdata->map->last_mapper = strdup(c->mca_component_name);
/* convenience def */
map = jdata->map;
/* if there is a default hostfile, go and get its ordered list of nodes */
OBJ_CONSTRUCT(&default_seq_list, opal_list_t);
if (NULL != orte_default_hostfile) {
char *hstname = NULL;
/* open the file */
fp = fopen(orte_default_hostfile, "r");
if (NULL == fp) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
goto error;
}
while (NULL != (hstname = orte_getline(fp))) {
if (0 == strlen(hstname)) {
free(hstname);
/* blank line - ignore */
continue;
}
if( '#' == hstname[0] ) {
free(hstname);
/* Comment line - ignore */
continue;
}
sq = OBJ_NEW(seq_node_t);
if (NULL != (sep = strchr(hstname, ' '))) {
*sep = '\0';
sep++;
/* remove any trailing space */
eptr = sep + strlen(sep) - 1;
while (eptr > sep && isspace(*eptr)) {
eptr--;
}
*(eptr+1) = 0;
sq->cpuset = strdup(sep);
}
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
char *ptr;
if (NULL != (ptr = strchr(hstname, '.'))) {
*ptr = '\0';
}
}
sq->hostname = hstname;
opal_list_append(&default_seq_list, &sq->super);
}
fclose(fp);
}
/* start at the beginning... */
vpid = 0;
jdata->num_procs = 0;
if (0 < opal_list_get_size(&default_seq_list)) {
save = (seq_node_t*)opal_list_get_first(&default_seq_list);
}
/* default to LOGICAL processors */
if (orte_get_attribute(&jdata->attributes, ORTE_JOB_PHYSICAL_CPUIDS, NULL, OPAL_BOOL)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using PHYSICAL processors");
rtype = OPAL_HWLOC_PHYSICAL;
} else {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using LOGICAL processors");
rtype = OPAL_HWLOC_LOGICAL;
}
/* initialize all the nodes as not included in this job map */
for (j=0; j < orte_node_pool->size; j++) {
if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
}
}
/* cycle through the app_contexts, mapping them sequentially */
for(i=0; i < jdata->apps->size; i++) {
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
continue;
}
/* dash-host trumps hostfile */
if (orte_get_attribute(&app->attributes, ORTE_APP_DASH_HOST, (void**)&hosts, OPAL_STRING)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using dash-host nodes on app %s", app->app);
OBJ_CONSTRUCT(&node_list, opal_list_t);
/* dash host entries cannot specify cpusets, so used the std function to retrieve the list */
if (ORTE_SUCCESS != (rc = orte_util_get_ordered_dash_host_list(&node_list, hosts))) {
ORTE_ERROR_LOG(rc);
free(hosts);
goto error;
}
free(hosts);
/* transfer the list to a seq_node_t list */
OBJ_CONSTRUCT(&sq_list, opal_list_t);
while (NULL != (nd = (orte_node_t*)opal_list_remove_first(&node_list))) {
sq = OBJ_NEW(seq_node_t);
sq->hostname = strdup(nd->name);
opal_list_append(&sq_list, &sq->super);
OBJ_RELEASE(nd);
}
OBJ_DESTRUCT(&node_list);
seq_list = &sq_list;
} else if (orte_get_attribute(&app->attributes, ORTE_APP_HOSTFILE, (void**)&hosts, OPAL_STRING)) {
char *hstname;
if (NULL == hosts) {
rc = ORTE_ERR_NOT_FOUND;
goto error;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using hostfile %s nodes on app %s", hosts, app->app);
OBJ_CONSTRUCT(&sq_list, opal_list_t);
/* open the file */
fp = fopen(hosts, "r");
if (NULL == fp) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
rc = ORTE_ERR_NOT_FOUND;
OBJ_DESTRUCT(&sq_list);
goto error;
}
while (NULL != (hstname = orte_getline(fp))) {
if (0 == strlen(hstname)) {
free(hstname);
/* blank line - ignore */
continue;
}
if( '#' == hstname[0] ) {
free(hstname);
/* Comment line - ignore */
continue;
}
sq = OBJ_NEW(seq_node_t);
if (NULL != (sep = strchr(hstname, ' '))) {
*sep = '\0';
sep++;
/* remove any trailing space */
eptr = sep + strlen(sep) - 1;
while (eptr > sep && isspace(*eptr)) {
eptr--;
}
*(eptr+1) = 0;
sq->cpuset = strdup(sep);
}
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hstname) ) {
char *ptr;
if (NULL != (ptr = strchr(hstname, '.'))) {
(*ptr) = '\0';
}
}
sq->hostname = hstname;
opal_list_append(&sq_list, &sq->super);
}
fclose(fp);
free(hosts);
seq_list = &sq_list;
} else if (0 < opal_list_get_size(&default_seq_list)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: using default hostfile nodes on app %s", app->app);
seq_list = &default_seq_list;
} else {
/* can't do anything - no nodes available! */
orte_show_help("help-orte-rmaps-base.txt",
"orte-rmaps-base:no-available-resources",
true);
return ORTE_ERR_SILENT;
}
/* check for nolocal and remove the head node, if required */
if (map->mapping & ORTE_MAPPING_NO_USE_LOCAL) {
for (item = opal_list_get_first(seq_list);
item != opal_list_get_end(seq_list);
item = opal_list_get_next(item) ) {
seq = (seq_node_t*)item;
/* need to check ifislocal because the name in the
* hostfile may not have been FQDN, while name returned
* by gethostname may have been (or vice versa)
*/
if (orte_ifislocal(seq->hostname)) {
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: removing head node %s", seq->hostname);
opal_list_remove_item(seq_list, item);
OBJ_RELEASE(item); /* "un-retain" it */
}
}
}
if (NULL == seq_list || 0 == (num_nodes = (orte_std_cntr_t)opal_list_get_size(seq_list))) {
orte_show_help("help-orte-rmaps-base.txt",
"orte-rmaps-base:no-available-resources",
true);
return ORTE_ERR_SILENT;
}
/* if num_procs wasn't specified, set it now */
if (0 == app->num_procs) {
app->num_procs = num_nodes;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: setting num procs to %s for app %s",
ORTE_VPID_PRINT(app->num_procs), app->app);
} else if (num_nodes < app->num_procs) {
orte_show_help("help-orte-rmaps-base.txt", "seq:not-enough-resources", true,
app->num_procs, num_nodes);
return ORTE_ERR_SILENT;
}
if (seq_list == &default_seq_list) {
sq = save;
} else {
sq = (seq_node_t*)opal_list_get_first(seq_list);
}
for (n=0; n < app->num_procs; n++) {
/* find this node on the global array - this is necessary so
* that our mapping gets saved on that array as the objects
* returned by the hostfile function are -not- on the array
*/
node = NULL;
for (j=0; j < orte_node_pool->size; j++) {
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, j))) {
continue;
}
if (0 == strcmp(sq->hostname, node->name)) {
break;
}
}
if (NULL == node) {
/* wasn't found - that is an error */
orte_show_help("help-orte-rmaps-seq.txt",
"orte-rmaps-seq:resource-not-found",
true, sq->hostname);
rc = ORTE_ERR_SILENT;
goto error;
}
/* ensure the node is in the map */
if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
OBJ_RETAIN(node);
opal_pointer_array_add(map->nodes, node);
jdata->map->num_nodes++;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
}
proc = orte_rmaps_base_setup_proc(jdata, node, i);
if ((node->slots < (int)node->num_procs) ||
(0 < node->slots_max && node->slots_max < (int)node->num_procs)) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, node->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
rc = ORTE_ERR_SILENT;
goto error;
}
/* flag the node as oversubscribed so that sched-yield gets
* properly set
*/
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
/* check for permission */
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
/* if we weren't given a directive either way, then we will error out
* as the #slots were specifically given, either by the host RM or
* via hostfile/dash-host */
if (!(ORTE_MAPPING_SUBSCRIBE_GIVEN & ORTE_GET_MAPPING_DIRECTIVE(orte_rmaps_base.mapping))) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
return ORTE_ERR_SILENT;
} else if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
/* if we were explicitly told not to oversubscribe, then don't */
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
return ORTE_ERR_SILENT;
}
}
}
/* assign the vpid */
proc->name.vpid = vpid++;
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: assign proc %s to node %s for app %s",
ORTE_VPID_PRINT(proc->name.vpid), sq->hostname, app->app);
/* record the cpuset, if given */
if (NULL != sq->cpuset) {
hwloc_cpuset_t bitmap;
char *cpu_bitmap;
if (NULL == node->topology) {
/* not allowed - for sequential cpusets, we must have
* the topology info
*/
orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology", true, node->name);
rc = ORTE_ERR_SILENT;
goto error;
}
/* if we are using hwthreads as cpus and binding to hwthreads, then
* we can just copy the cpuset across as it already specifies things
* at that level */
if (opal_hwloc_use_hwthreads_as_cpus &&
OPAL_BIND_TO_HWTHREAD == OPAL_GET_BINDING_POLICY(opal_hwloc_binding_policy)) {
cpu_bitmap = strdup(sq->cpuset);
} else {
/* setup the bitmap */
bitmap = hwloc_bitmap_alloc();
/* parse the slot_list to find the socket and core */
if (ORTE_SUCCESS != (rc = opal_hwloc_base_slot_list_parse(sq->cpuset, node->topology, rtype, bitmap))) {
ORTE_ERROR_LOG(rc);
hwloc_bitmap_free(bitmap);
goto error;
}
/* note that we cannot set the proc locale to any specific object
* as the slot list may have assigned it to more than one - so
* leave that field NULL
*/
/* set the proc to the specified map */
hwloc_bitmap_list_asprintf(&cpu_bitmap, bitmap);
hwloc_bitmap_free(bitmap);
}
orte_set_attribute(&proc->attributes, ORTE_PROC_CPU_BITMAP, ORTE_ATTR_GLOBAL, cpu_bitmap, OPAL_STRING);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:seq: binding proc %s to cpuset %s bitmap %s",
ORTE_VPID_PRINT(proc->name.vpid), sq->cpuset, cpu_bitmap);
/* we are going to bind to cpuset since the user is specifying the cpus */
OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_CPUSET);
/* note that the user specified the mapping */
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYUSER);
ORTE_SET_MAPPING_DIRECTIVE(jdata->map->mapping, ORTE_MAPPING_GIVEN);
/* cleanup */
free(cpu_bitmap);
} else {
hwloc_obj_t locale;
/* assign the locale - okay for the topo to be null as
* it just means it wasn't returned
*/
if (NULL != node->topology) {
locale = hwloc_get_root_obj(node->topology);
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE,
ORTE_ATTR_LOCAL, locale, OPAL_PTR);
}
}
/* add to the jdata proc array */
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
ORTE_ERROR_LOG(rc);
goto error;
}
/* move to next node */
sq = (seq_node_t*)opal_list_get_next(&sq->super);
}
/** track the total number of processes we mapped */
jdata->num_procs += app->num_procs;
/* cleanup the node list if it came from this app_context */
if (seq_list != &default_seq_list) {
OPAL_LIST_DESTRUCT(seq_list);
} else {
save = sq;
}
}
return ORTE_SUCCESS;
error:
OPAL_LIST_DESTRUCT(&default_seq_list);
return rc;
}
static int hostfile_parse_line(int token, opal_list_t* updates,
opal_list_t* exclude, bool keep_all)
{
int rc;
orte_node_t* node;
bool got_max = false;
char* value;
char **argv;
char* node_name = NULL;
char* username = NULL;
int cnt;
int number_of_slots = 0;
char buff[64];
if (ORTE_HOSTFILE_STRING == token ||
ORTE_HOSTFILE_HOSTNAME == token ||
ORTE_HOSTFILE_INT == token ||
ORTE_HOSTFILE_IPV4 == token ||
ORTE_HOSTFILE_IPV6 == token) {
if(ORTE_HOSTFILE_INT == token) {
snprintf(buff, 64, "%d", orte_util_hostfile_value.ival);
value = buff;
} else {
value = orte_util_hostfile_value.sval;
}
argv = opal_argv_split (value, '@');
cnt = opal_argv_count (argv);
if (1 == cnt) {
node_name = strdup(argv[0]);
} else if (2 == cnt) {
username = argv[0];
node_name = strdup(argv[1]);
} else {
opal_output(0, "WARNING: Unhandled user@host-combination\n"); /* XXX */
}
opal_argv_free (argv);
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(node_name) ) {
char *ptr;
if (NULL != (ptr = strchr(node_name, '.'))) {
*ptr = '\0';
}
}
/* if the first letter of the name is '^', then this is a node
* to be excluded. Remove the ^ character so the nodename is
* usable, and put it on the exclude list
*/
if ('^' == node_name[0]) {
int i, len;
len = strlen(node_name);
for (i=1; i < len; i++) {
node_name[i-1] = node_name[i];
}
node_name[len-1] = '\0'; /* truncate */
OPAL_OUTPUT_VERBOSE((3, orte_ras_base_framework.framework_output,
"%s hostfile: node %s is being excluded",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node_name));
/* see if this is another name for us */
if (orte_ifislocal(node_name)) {
/* Nodename has been allocated, that is for sure */
free (node_name);
node_name = strdup(orte_process_info.nodename);
}
/* Do we need to make a new node object? First check to see
if it's already in the exclude list */
if (NULL == (node = hostfile_lookup(exclude, node_name))) {
node = OBJ_NEW(orte_node_t);
node->name = node_name;
if (NULL != username) {
orte_set_attribute(&node->attributes, ORTE_NODE_USERNAME, ORTE_ATTR_LOCAL, username, OPAL_STRING);
}
opal_list_append(exclude, &node->super);
} else {
free(node_name);
}
return ORTE_SUCCESS;
}
/* this is not a node to be excluded, so we need to process it and
* add it to the "include" list. See if this host is actually us.
*/
if (orte_ifislocal(node_name)) {
/* Nodename has been allocated, that is for sure */
free (node_name);
node_name = strdup(orte_process_info.nodename);
}
OPAL_OUTPUT_VERBOSE((3, orte_ras_base_framework.framework_output,
"%s hostfile: node %s is being included - keep all is %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node_name,
keep_all ? "TRUE" : "FALSE"));
/* Do we need to make a new node object? */
if (keep_all || NULL == (node = hostfile_lookup(updates, node_name))) {
node = OBJ_NEW(orte_node_t);
node->name = node_name;
node->slots = 1;
if (NULL != username) {
orte_set_attribute(&node->attributes, ORTE_NODE_USERNAME, ORTE_ATTR_LOCAL, username, OPAL_STRING);
}
opal_list_append(updates, &node->super);
} else {
/* this node was already found once - add a slot and mark slots as "given" */
node->slots++;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_SLOTS_GIVEN);
free(node_name);
}
} else if (ORTE_HOSTFILE_RELATIVE == token) {
/* store this for later processing */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_util_hostfile_value.sval);
opal_list_append(updates, &node->super);
} else if (ORTE_HOSTFILE_RANK == token) {
/* we can ignore the rank, but we need to extract the node name. we
* first need to shift over to the other side of the equal sign as
* this is where the node name will be
*/
while (!orte_util_hostfile_done &&
ORTE_HOSTFILE_EQUAL != token) {
token = orte_util_hostfile_lex();
}
if (orte_util_hostfile_done) {
/* bad syntax somewhere */
return ORTE_ERROR;
}
/* next position should be the node name */
token = orte_util_hostfile_lex();
if(ORTE_HOSTFILE_INT == token) {
snprintf(buff, 64, "%d", orte_util_hostfile_value.ival);
value = buff;
} else {
value = orte_util_hostfile_value.sval;
}
argv = opal_argv_split (value, '@');
cnt = opal_argv_count (argv);
if (1 == cnt) {
node_name = strdup(argv[0]);
} else if (2 == cnt) {
username = argv[0];
node_name = strdup(argv[1]);
} else {
opal_output(0, "WARNING: Unhandled user@host-combination\n"); /* XXX */
}
opal_argv_free (argv);
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(node_name) ) {
char *ptr;
if (NULL != (ptr = strchr(node_name, '.'))) {
*ptr = '\0';
}
}
/* Do we need to make a new node object? */
if (NULL == (node = hostfile_lookup(updates, node_name))) {
node = OBJ_NEW(orte_node_t);
node->name = node_name;
node->slots = 1;
if (NULL != username) {
orte_set_attribute(&node->attributes, ORTE_NODE_USERNAME, ORTE_ATTR_LOCAL, username, OPAL_STRING);
}
opal_list_append(updates, &node->super);
} else {
/* add a slot */
node->slots++;
free(node_name);
}
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s hostfile: node %s slots %d nodes-given %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name, node->slots,
ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN) ? "TRUE" : "FALSE"));
/* mark the slots as "given" since we take them as being the
* number specified via the rankfile
*/
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_SLOTS_GIVEN);
/* skip to end of line */
while (!orte_util_hostfile_done &&
ORTE_HOSTFILE_NEWLINE != token) {
token = orte_util_hostfile_lex();
}
return ORTE_SUCCESS;
} else {
hostfile_parse_error(token);
return ORTE_ERROR;
}
free(username);
while (!orte_util_hostfile_done) {
token = orte_util_hostfile_lex();
switch (token) {
case ORTE_HOSTFILE_DONE:
goto done;
case ORTE_HOSTFILE_NEWLINE:
goto done;
case ORTE_HOSTFILE_USERNAME:
username = hostfile_parse_string();
if (NULL != username) {
orte_set_attribute(&node->attributes, ORTE_NODE_USERNAME, ORTE_ATTR_LOCAL, username, OPAL_STRING);
free(username);
}
break;
case ORTE_HOSTFILE_PORT:
rc = hostfile_parse_int();
if (rc < 0) {
orte_show_help("help-hostfile.txt", "port",
true,
cur_hostfile_name, rc);
return ORTE_ERROR;
}
orte_set_attribute(&node->attributes, ORTE_NODE_PORT, ORTE_ATTR_LOCAL, &rc, OPAL_INT);
break;
case ORTE_HOSTFILE_COUNT:
case ORTE_HOSTFILE_CPU:
case ORTE_HOSTFILE_SLOTS:
rc = hostfile_parse_int();
if (rc < 0) {
orte_show_help("help-hostfile.txt", "slots",
true,
cur_hostfile_name, rc);
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
if (ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
/* multiple definitions were given for the
* slot count - this is not allowed
*/
orte_show_help("help-hostfile.txt", "slots-given",
true,
cur_hostfile_name, node->name);
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
node->slots = rc;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_SLOTS_GIVEN);
/* Ensure that slots_max >= slots */
if (node->slots_max != 0 && node->slots_max < node->slots) {
node->slots_max = node->slots;
}
break;
case ORTE_HOSTFILE_SLOTS_MAX:
rc = hostfile_parse_int();
if (rc < 0) {
orte_show_help("help-hostfile.txt", "max_slots",
true,
cur_hostfile_name, ((size_t) rc));
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
/* Only take this update if it puts us >= node_slots */
if (rc >= node->slots) {
if (node->slots_max != rc) {
node->slots_max = rc;
got_max = true;
}
} else {
orte_show_help("help-hostfile.txt", "max_slots_lt",
true,
cur_hostfile_name, node->slots, rc);
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
break;
case ORTE_HOSTFILE_STRING:
case ORTE_HOSTFILE_INT:
/* just ignore it */
break;
default:
hostfile_parse_error(token);
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
if (number_of_slots > node->slots) {
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
opal_list_remove_item(updates, &node->super);
OBJ_RELEASE(node);
return ORTE_ERROR;
}
}
done:
if (got_max && !ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_SLOTS_GIVEN)) {
node->slots = node->slots_max;
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_SLOTS_GIVEN);
}
return ORTE_SUCCESS;
}
int orte_proc_info(void)
{
int idx, i;
char *ptr;
char hostname[OPAL_MAXHOSTNAMELEN];
char **prefixes;
bool match;
if (init) {
return ORTE_SUCCESS;
}
init = true;
OBJ_CONSTRUCT(&orte_process_info.super, opal_proc_t);
orte_process_info.my_hnp_uri = NULL;
mca_base_var_register ("orte", "orte", NULL, "hnp_uri",
"HNP contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_hnp_uri);
if (NULL != orte_process_info.my_hnp_uri) {
ptr = orte_process_info.my_hnp_uri;
/* the uri value passed to us will have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr));
}
}
orte_process_info.my_daemon_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "local_daemon_uri",
"Daemon contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_daemon_uri);
if (NULL != orte_process_info.my_daemon_uri) {
ptr = orte_process_info.my_daemon_uri;
/* the uri value passed to us may have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr) - 1);
}
}
orte_process_info.app_num = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_num",
"Index of the app_context that defines this proc",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_num);
/* get the process id */
orte_process_info.pid = getpid();
/* get the nodename */
gethostname(hostname, sizeof(hostname));
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, hostname);
// Strip off the FQDN if present, ignore IP addresses
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hostname) ) {
if (NULL != (ptr = strchr(hostname, '.'))) {
*ptr = '\0';
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, hostname);
}
}
orte_strip_prefix = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "strip_prefix",
"Prefix(es) to match when deciding whether to strip leading characters and zeroes from "
"node names returned by daemons", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_strip_prefix);
/* we have to strip node names here, if user directs, to ensure that
* the names exchanged in the modex match the names found locally
*/
if (NULL != orte_strip_prefix) {
prefixes = opal_argv_split(orte_strip_prefix, ',');
match = false;
for (i=0; NULL != prefixes[i]; i++) {
if (0 == strncmp(hostname, prefixes[i], strlen(prefixes[i]))) {
/* remove the prefix and leading zeroes */
idx = strlen(prefixes[i]);
while (idx < (int)strlen(hostname) &&
(hostname[idx] <= '0' || '9' < hostname[idx])) {
idx++;
}
if ((int)strlen(hostname) <= idx) {
/* there were no non-zero numbers in the name */
orte_process_info.nodename = strdup(&hostname[strlen(prefixes[i])]);
} else {
orte_process_info.nodename = strdup(&hostname[idx]);
}
/* add this to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, orte_process_info.nodename);
match = true;
break;
}
}
/* if we didn't find a match, then just use the hostname as-is */
if (!match) {
orte_process_info.nodename = strdup(hostname);
}
opal_argv_free(prefixes);
} else {
orte_process_info.nodename = strdup(hostname);
}
/* add "localhost" to our list of aliases */
opal_argv_append_nosize(&orte_process_info.aliases, "localhost");
/* get the number of nodes in the job */
orte_process_info.num_nodes = 1;
(void) mca_base_var_register ("orte", "orte", NULL, "num_nodes",
"Number of nodes in the job",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_nodes);
/* get the number of times this proc has restarted */
orte_process_info.num_restarts = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "num_restarts",
"Number of times this proc has restarted",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_restarts);
orte_process_info.app_rank = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_rank",
"Rank of this proc within its app_context",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_rank);
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
orte_ess_node_rank = ORTE_NODE_RANK_INVALID;
(void) mca_base_var_register ("orte", "orte", NULL, "ess_node_rank", "Process node rank",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_ess_node_rank);
orte_process_info.my_node_rank = (orte_node_rank_t) orte_ess_node_rank;
/* setup the sync buffer */
orte_process_info.sync_buf = OBJ_NEW(opal_buffer_t);
return ORTE_SUCCESS;
}
/* init the module */
static int init(void)
{
char *slurm_host=NULL;
uint16_t port=0;
struct sockaddr_in address;
int flags;
struct hostent *h;
if (mca_ras_slurm_component.dyn_alloc_enabled) {
if (NULL == mca_ras_slurm_component.config_file) {
orte_show_help("help-ras-slurm.txt", "dyn-alloc-no-config", true);
return ORTE_ERR_SILENT;
}
/* setup the socket */
if (ORTE_SUCCESS != read_ip_port(mca_ras_slurm_component.config_file,
&slurm_host, &port) ||
NULL == slurm_host || 0 == port) {
if (NULL != slurm_host) {
free(slurm_host);
}
return ORTE_ERR_SILENT;
}
OPAL_OUTPUT_VERBOSE((2, orte_ras_base_framework.framework_output,
"ras:slurm got [ ip = %s, port = %u ] from %s\n",
slurm_host, port, mca_ras_slurm_component.config_file));
/* obtain a socket for our use */
if ((socket_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
free(slurm_host);
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* connect to the Slurm dynamic allocation port */
bzero(&address, sizeof(address));
address.sin_family = AF_INET;
if (!opal_net_isaddr(slurm_host)) {
/* if the ControlMachine was not specified as an IP address,
* we need to resolve it here
*/
if (NULL == (h = gethostbyname(slurm_host))) {
/* could not resolve it */
orte_show_help("help-ras-slurm.txt", "host-not-resolved",
true, slurm_host);
free(slurm_host);
return ORTE_ERR_SILENT;
}
free(slurm_host);
slurm_host = strdup(inet_ntoa(*(struct in_addr*)h->h_addr_list[0]));
}
address.sin_addr.s_addr = inet_addr(slurm_host);
address.sin_port = htons(port);
if (connect(socket_fd, (struct sockaddr*)&address, sizeof(address)) < 0) {
orte_show_help("help-ras-slurm.txt", "connection-failed",
true, slurm_host, (int)port);
free(slurm_host);
return ORTE_ERR_SILENT;
}
free(slurm_host);
/* set socket up to be non-blocking */
if ((flags = fcntl(socket_fd, F_GETFL, 0)) < 0) {
opal_output(0, "ras:slurm:dyn: fcntl(F_GETFL) failed: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
return ORTE_ERROR;
} else {
flags |= O_NONBLOCK;
if (fcntl(socket_fd, F_SETFL, flags) < 0) {
opal_output(0, "ras:slurm:dyn: fcntl(F_SETFL) failed: %s (%d)",
strerror(opal_socket_errno), opal_socket_errno);
return ORTE_ERROR;
}
}
/* setup to recv data */
opal_event_set(orte_event_base, &recv_ev, socket_fd,
OPAL_EV_READ, recv_data, NULL);
opal_event_add(&recv_ev, 0);
/* initialize the list of jobs for tracking dynamic allocations */
OBJ_CONSTRUCT(&jobs, opal_list_t);
}
return ORTE_SUCCESS;
}
static int allocate(orte_job_t *jdata, opal_list_t *nodes)
{
char **nodelist;
orte_node_t *node;
int i, num_nodes;
char *affinity_file;
struct stat buf;
char *ptr;
/* get the list of allocated nodes */
if ((num_nodes = lsb_getalloc(&nodelist)) < 0) {
orte_show_help("help-ras-lsf.txt", "nodelist-failed", true);
return ORTE_ERR_NOT_AVAILABLE;
}
node = NULL;
/* step through the list */
for (i = 0; i < num_nodes; i++) {
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(nodelist[i]) ) {
if (NULL != (ptr = strchr(nodelist[i], '.'))) {
*ptr = '\0';
}
}
/* is this a repeat of the current node? */
if (NULL != node && 0 == strcmp(nodelist[i], node->name)) {
/* it is a repeat - just bump the slot count */
++node->slots;
opal_output_verbose(10, orte_ras_base_framework.framework_output,
"ras/lsf: +++ Node (%s) [slots=%d]", node->name, node->slots);
continue;
}
/* not a repeat - create a node entry for it */
node = OBJ_NEW(orte_node_t);
node->name = strdup(nodelist[i]);
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = 1;
node->state = ORTE_NODE_STATE_UP;
opal_list_append(nodes, &node->super);
opal_output_verbose(10, orte_ras_base_framework.framework_output,
"ras/lsf: New Node (%s) [slots=%d]", node->name, node->slots);
}
/* release the nodelist from lsf */
opal_argv_free(nodelist);
/* check for an affinity file */
if (NULL != (affinity_file = getenv("LSB_AFFINITY_HOSTFILE"))) {
/* check to see if the file is empty - if it is,
* then affinity wasn't actually set for this job */
if (0 != stat(affinity_file, &buf)) {
orte_show_help("help-ras-lsf.txt", "affinity-file-not-found", true, affinity_file);
return ORTE_ERR_SILENT;
}
if (0 == buf.st_size) {
/* no affinity, so just return */
return ORTE_SUCCESS;
}
/* the affinity file sequentially lists rank locations, with
* cpusets given as physical cpu-ids. Setup the job object
* so it knows to process this accordingly */
if (NULL == jdata->map) {
jdata->map = OBJ_NEW(orte_job_map_t);
}
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_SEQ);
jdata->map->req_mapper = strdup("seq"); // need sequential mapper
/* tell the sequential mapper that all cpusets are to be treated as "physical" */
orte_set_attribute(&jdata->attributes, ORTE_JOB_PHYSICAL_CPUIDS, true, NULL, OPAL_BOOL);
/* LSF provides its info as hwthreads, so set the hwthread-as-cpus flag */
opal_hwloc_use_hwthreads_as_cpus = true;
/* don't override something provided by the user, but default to bind-to hwthread */
if (!OPAL_BINDING_POLICY_IS_SET(opal_hwloc_binding_policy)) {
OPAL_SET_BINDING_POLICY(opal_hwloc_binding_policy, OPAL_BIND_TO_HWTHREAD);
}
/*
* Do not set the hostfile attribute on each app_context since that
* would confuse the sequential mapper when it tries to assign bindings
* when running an MPMD job.
* Instead just overwrite the orte_default_hostfile so it will be
* general for all of the app_contexts.
*/
if( NULL != orte_default_hostfile ) {
free(orte_default_hostfile);
orte_default_hostfile = NULL;
}
orte_default_hostfile = strdup(affinity_file);
opal_output_verbose(10, orte_ras_base_framework.framework_output,
"ras/lsf: Set default_hostfile to %s",orte_default_hostfile);
return ORTE_SUCCESS;
}
return ORTE_SUCCESS;
}
/**
* Discover the available resources.
*
* In order to fully support slurm, we need to be able to handle
* node regexp/task_per_node strings such as:
* foo,bar 5,3
* foo 5
* foo[2-10,12,99-105],bar,foobar[3-11] 2(x10),5,100(x16)
*
* @param *regexp A node regular expression from SLURM (i.e. SLURM_NODELIST)
* @param *tasks_per_node A tasks per node expression from SLURM
* (i.e. SLURM_TASKS_PER_NODE)
* @param *nodelist A list which has already been constucted to return
* the found nodes in
*/
static int orte_ras_slurm_discover(char *regexp, char *tasks_per_node,
opal_list_t* nodelist)
{
int i, j, len, ret, count, reps, num_nodes;
char *base, **names = NULL;
char *begptr, *endptr, *orig;
int *slots;
bool found_range = false;
bool more_to_come = false;
char *ptr;
orig = base = strdup(regexp);
if (NULL == base) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:slurm:allocate:discover: checking nodelist: %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
regexp));
do {
/* Find the base */
len = strlen(base);
for (i = 0; i <= len; ++i) {
if (base[i] == '[') {
/* we found a range. this gets dealt with below */
base[i] = '\0';
found_range = true;
break;
}
if (base[i] == ',') {
/* we found a singleton node, and there are more to come */
base[i] = '\0';
found_range = false;
more_to_come = true;
break;
}
if (base[i] == '\0') {
/* we found a singleton node */
found_range = false;
more_to_come = false;
break;
}
}
if(i == 0) {
/* we found a special character at the beginning of the string */
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
if (found_range) {
/* If we found a range, now find the end of the range */
for (j = i; j < len; ++j) {
if (base[j] == ']') {
base[j] = '\0';
break;
}
}
if (j >= len) {
/* we didn't find the end of the range */
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
ret = orte_ras_slurm_parse_ranges(base, base + i + 1, &names);
if(ORTE_SUCCESS != ret) {
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value",
1, regexp, tasks_per_node, "SLURM_NODELIST");
ORTE_ERROR_LOG(ret);
free(orig);
return ret;
}
if(base[j + 1] == ',') {
more_to_come = true;
base = &base[j + 2];
} else {
more_to_come = false;
}
} else {
/* If we didn't find a range, just add the node */
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:slurm:allocate:discover: found node %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
base));
if(ORTE_SUCCESS != (ret = opal_argv_append_nosize(&names, base))) {
ORTE_ERROR_LOG(ret);
free(orig);
return ret;
}
/* set base equal to the (possible) next base to look at */
base = &base[i + 1];
}
} while(more_to_come);
free(orig);
num_nodes = opal_argv_count(names);
/* Find the number of slots per node */
slots = malloc(sizeof(int) * num_nodes);
if (NULL == slots) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
memset(slots, 0, sizeof(int) * num_nodes);
orig = begptr = strdup(tasks_per_node);
if (NULL == begptr) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
free(slots);
return ORTE_ERR_OUT_OF_RESOURCE;
}
j = 0;
while (begptr) {
count = strtol(begptr, &endptr, 10);
if ((endptr[0] == '(') && (endptr[1] == 'x')) {
reps = strtol((endptr+2), &endptr, 10);
if (endptr[0] == ')') {
endptr++;
}
} else {
reps = 1;
}
/**
* TBP: it seems like it would be an error to have more slot
* descriptions than nodes. Turns out that this valid, and SLURM will
* return such a thing. For instance, if I did:
* srun -A -N 30 -w odin001
* I would get SLURM_NODELIST=odin001 SLURM_TASKS_PER_NODE=4(x30)
* That is, I am allocated 30 nodes, but since I only requested
* one specific node, that's what is in the nodelist.
* I'm not sure this is what users would expect, but I think it is
* more of a SLURM issue than a orte issue, since SLURM is OK with it,
* I'm ok with it
*/
for (i = 0; i < reps && j < num_nodes; i++) {
slots[j++] = count;
}
if (*endptr == ',') {
begptr = endptr + 1;
} else if (*endptr == '\0' || j >= num_nodes) {
break;
} else {
orte_show_help("help-ras-slurm.txt", "slurm-env-var-bad-value", 1,
regexp, tasks_per_node, "SLURM_TASKS_PER_NODE");
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
free(slots);
free(orig);
return ORTE_ERR_BAD_PARAM;
}
}
free(orig);
/* Convert the argv of node names to a list of node_t's */
for (i = 0; NULL != names && NULL != names[i]; ++i) {
orte_node_t *node;
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(names[i]) ) {
if (NULL != (ptr = strchr(names[i], '.'))) {
*ptr = '\0';
}
}
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:slurm:allocate:discover: adding node %s (%d slot%s)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
names[i], slots[i], (1 == slots[i]) ? "" : "s"));
node = OBJ_NEW(orte_node_t);
if (NULL == node) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
free(slots);
return ORTE_ERR_OUT_OF_RESOURCE;
}
node->name = strdup(names[i]);
node->state = ORTE_NODE_STATE_UP;
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = slots[i];
opal_list_append(nodelist, &node->super);
}
free(slots);
opal_argv_free(names);
/* All done */
return ret;
}
/**
* Discover available (pre-allocated) nodes. Allocate the
* requested number of nodes/process slots to the job.
*
*/
static int orte_ras_gridengine_allocate(orte_job_t *jdata, opal_list_t *nodelist)
{
char *pe_hostfile = getenv("PE_HOSTFILE");
char *job_id = getenv("JOB_ID");
char buf[1024], *tok, *num, *queue, *arch, *ptr, *tmp;
int rc;
FILE *fp;
orte_node_t *node;
opal_list_item_t *item;
bool found;
/* show the Grid Engine's JOB_ID */
if (mca_ras_gridengine_component.show_jobid ||
mca_ras_gridengine_component.verbose != -1) {
opal_output(0, "ras:gridengine: JOB_ID: %s", job_id);
}
/* check the PE_HOSTFILE before continuing on */
if (!(fp = fopen(pe_hostfile, "r"))) {
orte_show_help("help-ras-gridengine.txt", "cannot-read-pe-hostfile",
true, pe_hostfile, strerror(errno));
rc = ORTE_ERROR;
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* parse the pe_hostfile for hostname, slots, etc, then compare the
* current node with a list of hosts in the nodelist, if the current
* node is not found in nodelist, add it in */
opal_output(mca_ras_gridengine_component.verbose,
"ras:gridengine: PE_HOSTFILE: %s", pe_hostfile);
while (fgets(buf, sizeof(buf), fp)) {
ptr = strtok_r(buf, " \n", &tok);
num = strtok_r(NULL, " \n", &tok);
queue = strtok_r(NULL, " \n", &tok);
arch = strtok_r(NULL, " \n", &tok);
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(ptr) ) {
if (NULL != (tmp = strchr(ptr, '.'))) {
*tmp = '\0';
}
}
/* see if we already have this node */
found = false;
for (item = opal_list_get_first(nodelist);
item != opal_list_get_end(nodelist);
item = opal_list_get_next(item)) {
node = (orte_node_t*)item;
if (0 == strcmp(ptr, node->name)) {
/* just add the slots */
node->slots += (int)strtol(num, (char **)NULL, 10);
found = true;
break;
}
}
if (!found) {
/* create a new node entry */
node = OBJ_NEW(orte_node_t);
if (NULL == node) {
fclose(fp);
return ORTE_ERR_OUT_OF_RESOURCE;
}
node->name = strdup(ptr);
node->state = ORTE_NODE_STATE_UP;
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = (int)strtol(num, (char **)NULL, 10);
opal_output(mca_ras_gridengine_component.verbose,
"ras:gridengine: %s: PE_HOSTFILE shows slots=%d",
node->name, node->slots);
opal_list_append(nodelist, &node->super);
}
} /* finished reading the $PE_HOSTFILE */
cleanup:
fclose(fp);
/* in gridengine, if we didn't find anything, then something
* is wrong. The user may not have indicated this was a parallel
* job, or may not have an allocation at all. In any case, this
* is considered an unrecoverable error and we need to report it
*/
if (opal_list_is_empty(nodelist)) {
orte_show_help("help-ras-gridengine.txt", "no-nodes-found", true);
return ORTE_ERR_NOT_FOUND;
}
return ORTE_SUCCESS;
}
static int discover(opal_list_t* nodelist, char *pbs_jobid)
{
int32_t nodeid;
orte_node_t *node;
opal_list_item_t* item;
FILE *fp;
char *hostname, *cppn;
int ppn;
char *ptr;
/* Ignore anything that the user already specified -- we're
getting nodes only from TM. */
/* TM "nodes" may actually correspond to PBS "VCPUs", which means
there may be multiple "TM nodes" that correspond to the same
physical node. This doesn't really affect what we're doing
here (we actually ignore the fact that they're duplicates --
slightly inefficient, but no big deal); just mentioned for
completeness... */
/* if we are in SMP mode, then read the environment to get the
* number of cpus for each node read in the file
*/
if (mca_ras_tm_component.smp_mode) {
if (NULL == (cppn = getenv("PBS_PPN"))) {
orte_show_help("help-ras-tm.txt", "smp-error", true);
return ORTE_ERR_NOT_FOUND;
}
ppn = strtol(cppn, NULL, 10);
} else {
ppn = 1;
}
/* setup the full path to the PBS file */
filename = opal_os_path(false, mca_ras_tm_component.nodefile_dir,
pbs_jobid, NULL);
fp = fopen(filename, "r");
if (NULL == fp) {
ORTE_ERROR_LOG(ORTE_ERR_FILE_OPEN_FAILURE);
free(filename);
return ORTE_ERR_FILE_OPEN_FAILURE;
}
/* Iterate through all the nodes and make an entry for each. TM
node ID's will never be duplicated, but they may end up
resolving to the same hostname (i.e., vcpu's on a single
host). */
nodeid=0;
while (NULL != (hostname = tm_getline(fp))) {
if( !orte_keep_fqdn_hostnames && !opal_net_isaddr(hostname) ) {
if (NULL != (ptr = strchr(hostname, '.'))) {
*ptr = '\0';
}
}
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:tm:allocate:discover: got hostname %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), hostname));
/* Remember that TM may list the same node more than once. So
we have to check for duplicates. */
for (item = opal_list_get_first(nodelist);
opal_list_get_end(nodelist) != item;
item = opal_list_get_next(item)) {
node = (orte_node_t*) item;
if (0 == strcmp(node->name, hostname)) {
if (mca_ras_tm_component.smp_mode) {
/* this cannot happen in smp mode */
orte_show_help("help-ras-tm.txt", "smp-multi", true);
return ORTE_ERR_BAD_PARAM;
}
++node->slots;
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:tm:allocate:discover: found -- bumped slots to %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->slots));
break;
}
}
/* Did we find it? */
if (opal_list_get_end(nodelist) == item) {
/* Nope -- didn't find it, so add a new item to the list */
OPAL_OUTPUT_VERBOSE((1, orte_ras_base_framework.framework_output,
"%s ras:tm:allocate:discover: not found -- added to list",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
node = OBJ_NEW(orte_node_t);
node->name = hostname;
orte_set_attribute(&node->attributes, ORTE_NODE_LAUNCH_ID, ORTE_ATTR_LOCAL, &nodeid, OPAL_INT32);
node->slots_inuse = 0;
node->slots_max = 0;
node->slots = ppn;
node->state = ORTE_NODE_STATE_UP;
opal_list_append(nodelist, &node->super);
} else {
/* Yes, so we need to free the hostname that came back */
free(hostname);
}
/* up the nodeid */
nodeid++;
}
fclose(fp);
return ORTE_SUCCESS;
}