/**
* Make a prettyprint string for a cset in a map format.
* Example: [B . . .]
* Key: [] - signifies socket boundary
* . - signifies core a process not bound to
* B - signifies core a process is bound to
*/
int opal_paffinity_base_cset2mapstr(char *str, int len,
opal_paffinity_base_cpu_set_t *cset)
{
int ret, i, j, k, num_sockets, num_cores, flag;
int phys_socket, phys_core;
char tmp[BUFSIZ];
const int stmp = sizeof(tmp) - 1;
str[0] = tmp[stmp] = '\0';
/* Loop over the number of sockets in this machine */
ret = opal_paffinity_base_get_socket_info(&num_sockets);
if (OPAL_SUCCESS != ret) {
return ret;
}
for (i = 0; i < num_sockets; ++i) {
strncat(str, "[", len - strlen(str));
/* Loop over the number of cores in this socket */
ret = opal_paffinity_base_get_core_info(i, &num_cores);
if (OPAL_SUCCESS != ret) {
return ret;
}
phys_socket = opal_paffinity_base_get_physical_socket_id(i);
for (j = 0; j < num_cores; j++) {
if (0 < j) {
/* add space after first core is printed */
strncat(str, " ", len - strlen(str));
}
phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, j);
ret = opal_paffinity_base_get_map_to_processor_id(phys_socket, phys_core, &k);
if (OPAL_SUCCESS != ret) {
return ret;
}
flag = OPAL_PAFFINITY_CPU_ISSET(k, *cset);
if (flag) {
/* mark core as bound to process */
strncat(str, "B", len - strlen(str));
} else {
/* mark core as no process bound to it */
strncat(str, ".", len - strlen(str));
}
}
strncat(str, "]", len - strlen(str));
}
return OPAL_SUCCESS;
}
/**
* Make a prettyprint string for a cset.
*/
int opal_paffinity_base_cset2str(char *str, int len,
opal_paffinity_base_cpu_set_t *cset)
{
int ret, i, j, k, num_sockets, num_cores, flag, count,
range_first=0, range_last;
int phys_socket, phys_core;
char tmp[BUFSIZ];
const int stmp = sizeof(tmp) - 1;
str[0] = tmp[stmp] = '\0';
/* Loop over the number of sockets in this machine */
ret = opal_paffinity_base_get_socket_info(&num_sockets);
if (OPAL_SUCCESS != ret) {
return ret;
}
for (i = 0; i < num_sockets; ++i) {
/* Loop over the number of cores in this socket */
ret = opal_paffinity_base_get_core_info(i, &num_cores);
if (OPAL_SUCCESS != ret) {
return ret;
}
phys_socket = opal_paffinity_base_get_physical_socket_id(i);
/* Must initially set range_last to a low number -- smaller
than -1, so that the comparisons below work out
properly. */
for (range_last = -5, count = j = 0; j < num_cores; ++j) {
phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, j);
ret = opal_paffinity_base_get_map_to_processor_id(phys_socket, phys_core, &k);
if (OPAL_SUCCESS != ret) {
return ret;
}
/* Prettyprint the cores that we're actually bound to */
flag = OPAL_PAFFINITY_CPU_ISSET(k, *cset);
if (flag) {
if (0 == count) {
snprintf(tmp, stmp, "socket %d[core %d", i, j);
strncat(str, tmp, len - strlen(str));
range_first = range_last = j;
} else {
if (j - 1 == range_last) {
range_last = j;
} else {
snprintf(tmp, stmp, "-%d,%d", range_last, j);
strncat(str, tmp, len - strlen(str));
range_first = range_last = j;
}
}
++count;
}
}
if (count > 0) {
if (range_first != range_last) {
snprintf(tmp, stmp, "-%d", range_last);
strncat(str, tmp, len - strlen(str));
}
strncat(str, "] ", len - strlen(str));
}
}
/* If the last character is a space, remove it */
if (' ' == str[strlen(str) - 1]) {
str[strlen(str) - 1] = '\0';
}
return OPAL_SUCCESS;
}
static int opal_paffinity_base_socket_core_to_cpu_set(char **socket_core_list, int socket_core_list_cnt, long rank, bool logical_map)
{
int rc, i;
char **socket_core;
int socket_core_cnt;
char **range;
int range_cnt;
int lower_range, upper_range;
int socket, core;
int num_sockets, num_cores;
int phys_socket, phys_core, phys_processor;
opal_paffinity_base_cpu_set_t cpumask;
socket_core = opal_argv_split (socket_core_list[0], ':');
socket_core_cnt = opal_argv_count(socket_core);
OPAL_PAFFINITY_CPU_ZERO(cpumask);
socket = atoi(socket_core[0]);
core = atoi(socket_core[1]);
/* get the number of LOGICAL sockets on this node */
if ( OPAL_SUCCESS != ( rc = opal_paffinity_base_get_socket_info(&num_sockets))) {
return rc;
}
if (logical_map) {
/* need to convert provided socket to a PHYSICAL socket id */
phys_socket = opal_paffinity_base_get_physical_socket_id(socket);
if (0 > phys_socket) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical socket id for logical socket %ld",
rank, (long)socket);
return OPAL_ERROR;
}
} else {
phys_socket = socket;
}
phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core);
/* get the LOGICAL core info for this socket */
if ( OPAL_SUCCESS != ( rc = opal_paffinity_base_get_core_info(phys_socket, &num_cores))) {
opal_output(0,"Rank %ld: PAFFINITY Error !!! Could not get core info for physical socket number %d (%d)",
rank, phys_socket, socket);
return rc;
}
if (0 == strcmp("*",socket_core[1])) {
/* bind to all available LOGICAL cores */
for (core = 0; core < num_cores; core++) {
/* convert to PHYSICAL core id */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
/* get the PHYSICAL processor id for the PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
}
/* tell paffinity to bind us */
if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical processor #%d ( %d : %d)",
rank, phys_processor, socket, core);
}
} else {
range = opal_argv_split(socket_core[1], '-');
range_cnt = opal_argv_count(range);
switch (range_cnt) {
case 1: /* only one core specified */
core = atoi(range[0]);
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* tell paffinity to bind us */
if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
break;
case 2: /* range of core id's was given */
lower_range = atoi(range[0]);
upper_range = atoi(range[1]);
if ( 0 > lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) {
opal_output(0,"Rank %ld: PAFFINITY Error !!! Check your boundaries lower %d upper %d num_cores %d",
rank, lower_range, upper_range, num_cores);
return OPAL_ERROR;
}
for (core=lower_range; core<=upper_range; core++) {
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0,"paffinity slot assignment: rank %ld runs on cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
}
/* tell paffinity to bind us */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
break;
default:
opal_argv_free(range);
opal_argv_free(socket_core);
return OPAL_ERROR;
}
opal_argv_free(range);
opal_argv_free(socket_core);
}
for (i=1; i<socket_core_list_cnt; i++) {
socket_core = opal_argv_split (socket_core_list[i], ':');
socket_core_cnt = opal_argv_count(socket_core);
switch (socket_core_cnt) {
case 1:
/* no colon => these cores are on the same socket as the last one specified,
* so we map them on that same physical socket
*/
range = opal_argv_split(socket_core[0], '-');
range_cnt = opal_argv_count(range);
switch (range_cnt) {
case 1: /* only one core provided */
core = atoi(range[0]);
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* tell paffinity to bind us */
if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
break;
case 2: /* range of core id's was given */
lower_range = atoi(range[0]);
upper_range = atoi(range[1]);
if ( 0 > lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) {
opal_output(0,"Rank %ld: PAFFINITY Error !!! Check your boundaries lower %d upper %d num_cores %d",
rank, lower_range, upper_range, num_cores);
return OPAL_ERROR;
}
for (core=lower_range; core<=upper_range; core++) {
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
}
/* tell paffinity to bind us */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
break;
default:
opal_argv_free(range);
opal_argv_free(socket_core);
return OPAL_ERROR;
}
opal_argv_free(range);
break;
case 2: /* colon was given => refers to a new socket! */
socket = atoi(socket_core[0]);
if (logical_map) {
/* need to convert provided socket to a PHYSICAL socket id */
phys_socket = opal_paffinity_base_get_physical_socket_id(socket);
if (0 > phys_socket) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical socket id for logical socket %ld",
rank, (long)socket);
return OPAL_ERROR;
}
} else {
phys_socket = socket;
}
/* get the LOGICAL core info for this socket */
if ( OPAL_SUCCESS != ( rc = opal_paffinity_base_get_core_info(phys_socket, &num_cores))) {
opal_output(0,"Rank %ld: PAFFINITY Error !!! Could not get core info for physical socket number %d (%d)",
rank, phys_socket, socket);
return rc;
}
if (0 == strcmp("*",socket_core[1])) {
/* bind to all available LOGICAL cores */
for (core = 0; core < num_cores; core++) {
/* convert to PHYSICAL core id */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
/* get the PHYSICAL processor id for the PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
}
/* tell paffinity to bind us */
if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
} else {
range = opal_argv_split(socket_core[1], '-');
range_cnt = opal_argv_count(range);
socket = atoi(socket_core[0]);
switch (range_cnt) {
case 1: /* only one core specified */
core = atoi(range[0]);
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* tell paffinity to bind us */
if (OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
break;
case 2: /* range of core id's was given */
lower_range = atoi(range[0]);
upper_range = atoi(range[1]);
if ( 0 > lower_range || num_cores < (upper_range - lower_range) || lower_range >= upper_range ) {
opal_output(0,"Rank %ld: PAFFINITY Error !!! Check your boundaries lower %d upper %d num_cores %d",
rank, lower_range, upper_range, num_cores);
return OPAL_ERROR;
}
for (core=lower_range; core<=upper_range; core++) {
if (logical_map) {
/* convert to physical core */
if (0 > (phys_core = opal_paffinity_base_get_physical_core_id(phys_socket, core))) {
opal_output(0, "Rank %ld: PAFFINITY cannot get physical core id for logical core %ld in physical socket %ld (%ld)",
rank, (long)core, (long)phys_socket, (long)socket);
return OPAL_ERROR;
}
} else {
phys_core = core;
}
/* get the PHYSICAL processor id for this PHYSICAL socket/core */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_get_map_to_processor_id (phys_socket, phys_core, &phys_processor))) {
return rc;
}
/* set the bit for this processor */
OPAL_PAFFINITY_CPU_SET(phys_processor, cpumask);
/* output diagnostic if requested */
if (diag_requested) {
opal_output(0, "paffinity slot assignment: rank %ld runs on physical cpu #%d ( %d[%d] : %d[%d])",
rank, phys_processor, phys_socket, socket, phys_core, core);
}
}
/* tell paffinity to bind us */
if ( OPAL_SUCCESS != (rc = opal_paffinity_base_set(cpumask))) {
return rc;
}
default:
opal_argv_free(range);
opal_argv_free(socket_core);
return OPAL_ERROR;
}
opal_argv_free(range);
}
break;
default:
opal_argv_free(socket_core);
return OPAL_ERROR;
}
opal_argv_free(socket_core);
}
return OPAL_SUCCESS;
}
static int local_setup(void)
{
int ret;
char *error = NULL;
int value;
orte_proc_t *proc;
orte_node_t *node;
orte_app_context_t *app;
/* initialize the global list of local children and job data */
OBJ_CONSTRUCT(&orte_local_children, opal_list_t);
OBJ_CONSTRUCT(&orte_local_jobdata, opal_list_t);
/* determine the topology info */
if (0 == orte_default_num_sockets_per_board) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_socket_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
orte_default_num_sockets_per_board = (uint8_t)value;
}
if (0 == orte_default_num_cores_per_socket) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_core_info(0, &value)) {
/* don't have topo info - can we at least get #processors? */
if (OPAL_SUCCESS != opal_paffinity_base_get_processor_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
}
orte_default_num_cores_per_socket = (uint8_t)value;
}
/* setup the global job and node arrays */
orte_job_data = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_job_data,
1,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
1))) {
ORTE_ERROR_LOG(ret);
error = "setup job array";
goto error;
}
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_pool,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node array";
goto error;
}
/* Setup the job data object for the daemons */
/* create and store the job data object */
daemons = OBJ_NEW(orte_job_t);
daemons->jobid = ORTE_PROC_MY_NAME->jobid;
daemons->name = strdup("ORCM DVM");
daemons->instance = strdup(ORTE_JOBID_PRINT(ORTE_PROC_MY_NAME->jobid));
/* create an app */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("orcmd");
opal_argv_append_nosize(&app->argv, "orcmd");
/* add to the daemon job - always must be an app for a job */
opal_pointer_array_add(daemons->apps, app);
/* setup the daemon map so it knows how to map them */
daemons->map = OBJ_NEW(orte_job_map_t);
daemons->map->policy = ORTE_MAPPING_BYNODE;
/* save it */
opal_pointer_array_set_item(orte_job_data, 0, daemons);
/* ensure our mapping policy will utilize any VM */
ORTE_ADD_MAPPING_POLICY(ORTE_MAPPING_USE_VM);
/* use bynode mapping by default */
ORTE_ADD_MAPPING_POLICY(ORTE_MAPPING_BYNODE);
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->slots = 1; /* min number */
node->slots_alloc = node->slots;
node->index = ORTE_PROC_MY_NAME->vpid;
opal_pointer_array_set_item(orte_node_pool, ORTE_PROC_MY_NAME->vpid, node);
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
proc->nodename = node->name;
opal_pointer_array_set_item(daemons->procs, proc->name.vpid, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* record that the daemon job is running */
daemons->num_procs = 1;
daemons->state = ORTE_JOB_STATE_RUNNING;
/* open and setup the opal_pstat framework so we can provide
* process stats if requested
*/
if (ORTE_SUCCESS != (ret = opal_pstat_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pstat_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_pstat_base_select";
goto error;
}
/* open and setup the local resource discovery framework */
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_select";
goto error;
}
/* Setup the communication infrastructure */
/* Runtime Messaging Layer - this opens/selects the OOB as well */
if (ORTE_SUCCESS != (ret = orte_rml_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/* Routed system */
if (ORTE_SUCCESS != (ret = orte_routed_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/* multicast */
if (ORTE_SUCCESS != (ret = orte_rmcast_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmcast_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = orte_odls_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
proc->rml_uri = orte_rml.get_contact_info();
/* setup the pnp framework */
if (ORCM_SUCCESS != (ret = orcm_pnp_base_open())) {
error = "pnp_open";
goto error;
}
if (ORCM_SUCCESS != (ret = orcm_pnp_base_select())) {
error = "pnp_select";
goto error;
}
/* setup the leader framework */
if (ORCM_SUCCESS != (ret = orcm_leader_base_open())) {
error = "leader_open";
goto error;
}
if (ORCM_SUCCESS != (ret = orcm_leader_base_select())) {
error = "leader_select";
goto error;
}
/* set the communication function */
orte_comm = orte_global_comm;
/* update the routing tree */
if (ORTE_SUCCESS != (ret = orte_routed.update_routing_tree())) {
ORTE_ERROR_LOG(ret);
error = "failed to update routing tree";
goto error;
}
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* open/select the errmgr - do this after the daemon job
* has been defined so that the errmgr can get that
* job object
*/
if (ORTE_SUCCESS != (ret = orte_errmgr_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* We actually do *not* want an orcm to voluntarily yield() the
processor more than necessary. orcm already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at orcm, we want the
OS to wake us up in a timely fashion (which most OS's
seem good about doing) and then we want orcm to process
the message as fast as possible. If orcm yields and lets
aggressive applications get the processor back, it may be a
long time before the OS schedules orcm to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to processes, which can be
problematic in some scenarios */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orcmd that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* setup the primary daemon command receive function */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_NON_PERSISTENT, orte_daemon_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
error = "daemon_recv";
goto error;
}
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "%s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* listen for termination cmds */
if (ORCM_SUCCESS != (ret = orcm_pnp.register_receive("orcm", "0.1", "alpha",
ORCM_PNP_SYS_CHANNEL,
ORCM_PNP_TAG_TERMINATE,
vm_term, NULL))) {
error = "orcm recv";
goto error;
}
if (ORCM_SUCCESS != (ret = orcm_pnp.register_receive("orcm-stop", "0.1", "alpha",
ORCM_PNP_SYS_CHANNEL,
ORCM_PNP_TAG_TERMINATE,
vm_term, NULL))) {
error = "orcm-stop recv";
goto error;
}
/* register to catch vm commands requests */
if (ORCM_SUCCESS != (ret = orcm_pnp.register_receive("orcm-sched", "0.1", "alpha",
ORCM_PNP_SYS_CHANNEL,
ORCM_PNP_TAG_COMMAND,
vm_cmd, NULL))) {
error = "orcm-sched recv";
goto error;
}
/* listen for state data requests */
if (ORCM_SUCCESS != (ret = orcm_pnp.register_receive("orcm-sched", "0.1", "alpha",
ORCM_PNP_SYS_CHANNEL,
ORCM_PNP_TAG_BOOTSTRAP,
recv_contact, NULL))) {
error = "contact recv";
goto error;
}
/* setup the SENSOR framework */
if (ORTE_SUCCESS != (ret = orte_sensor_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_sensor_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_sensor_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_sensor_select";
goto error;
}
/* announce our existence - this carries with it our rml uri and
* our local node system info
*/
if (ORCM_SUCCESS != (ret = orcm_pnp.announce("ORCMD", "0.1", "alpha", vm_tracker))) {
ORTE_ERROR_LOG(ret);
error = "announce";
goto error;
}
return ORTE_SUCCESS;
error:
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
return ret;
}
static int rte_init(char flags)
{
int ret;
char *error = NULL;
char *contact_path, *jobfam_dir;
orte_job_t *jdata;
orte_node_t *node;
orte_proc_t *proc;
int value;
/* initialize the global list of local children and job data */
OBJ_CONSTRUCT(&orte_local_children, opal_list_t);
OBJ_CONSTRUCT(&orte_local_jobdata, opal_list_t);
/* run the prolog */
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
error = "orte_ess_base_std_prolog";
goto error;
}
/* if we are using xml for output, put a basename start tag */
if (orte_xml_output) {
fprintf(orte_xml_fp, "<%s>\n", orte_cmd_basename);
fflush(orte_xml_fp);
}
/* determine the topology info */
if (0 == orte_default_num_sockets_per_board) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_socket_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
orte_default_num_sockets_per_board = (uint8_t)value;
}
if (0 == orte_default_num_cores_per_socket) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != (ret = opal_paffinity_base_get_core_info(0, &value))) {
/* don't have topo info - can we at least get #processors? */
if (OPAL_SUCCESS != opal_paffinity_base_get_processor_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
}
orte_default_num_cores_per_socket = (uint8_t)value;
}
/* Since we are the HNP, then responsibility for
* defining the name falls to the PLM component for our
* respective environment - hence, we have to open the PLM
* first and select that component. Note that ONLY the
* HNP ever uses a PLM component anyway
*/
if (ORTE_SUCCESS != (ret = orte_plm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
/* Setup the communication infrastructure */
/*
* Runtime Messaging Layer
*/
if (ORTE_SUCCESS != (ret = orte_rml_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/*
* Routed system
*/
if (ORTE_SUCCESS != (ret = orte_routed_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
*/
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
/*
* Setup the remaining resource
* management and errmgr frameworks - application procs
* and daemons do not open these frameworks as they only use
* the hnp proxy support in the PLM framework.
*/
if (ORTE_SUCCESS != (ret = orte_ras_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_ras_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_open())) {
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = orte_odls_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
#if !ORTE_DISABLE_FULL_SUPPORT
/* setup the orte_show_help system to recv remote output */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_SHOW_HELP,
ORTE_RML_NON_PERSISTENT, orte_show_help_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
error = "setup receive for orte_show_help";
goto error;
}
#endif
/* setup my session directory */
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output hnp file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* save my contact info in a file for others to find */
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
contact_path));
if (ORTE_SUCCESS != (ret = orte_write_hnp_contact_file(contact_path))) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file failed with error %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_ERROR_NAME(ret)));
} else {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s wrote contact file",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
free(contact_path);
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_add(orte_job_data, jdata);
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->arch = orte_process_info.arch;
node->index = opal_pointer_array_add(orte_node_pool, node);
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
proc->nodename = node->name;
opal_pointer_array_add(jdata->procs, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = orte_iof_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = orte_filem_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = orte_snapc_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(orte_process_info.hnp, !orte_process_info.daemon))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
/* For HNP, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the notifier system */
if (ORTE_SUCCESS != (ret = orte_notifier_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_notifier_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_select";
goto error;
}
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
/* cleanup the global list of local children and job data */
OBJ_DESTRUCT(&orte_local_children);
OBJ_DESTRUCT(&orte_local_jobdata);
return ret;
}
static int rte_init(void)
{
int ret;
char *error = NULL;
char *contact_path, *jobfam_dir;
orte_job_t *jdata;
orte_node_t *node;
orte_proc_t *proc;
int value;
/* initialize the global list of local children and job data */
OBJ_CONSTRUCT(&orte_local_children, opal_list_t);
OBJ_CONSTRUCT(&orte_local_jobdata, opal_list_t);
/* run the prolog */
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
error = "orte_ess_base_std_prolog";
goto error;
}
/* determine the topology info */
if (0 == orte_default_num_sockets_per_board) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != opal_paffinity_base_get_socket_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
orte_default_num_sockets_per_board = (uint8_t)value;
}
if (0 == orte_default_num_cores_per_socket) {
/* we weren't given a number, so try to determine it */
if (OPAL_SUCCESS != (ret = opal_paffinity_base_get_core_info(0, &value))) {
/* don't have topo info - can we at least get #processors? */
if (OPAL_SUCCESS != opal_paffinity_base_get_processor_info(&value)) {
/* can't get any info - default to 1 */
value = 1;
}
}
orte_default_num_cores_per_socket = (uint8_t)value;
}
/* if we are using xml for output, put an mpirun start tag */
if (orte_xml_output) {
fprintf(orte_xml_fp, "<mpirun>\n");
fflush(orte_xml_fp);
}
/* open and setup the opal_pstat framework so we can provide
* process stats if requested
*/
if (ORTE_SUCCESS != (ret = opal_pstat_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_pstat_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_pstat_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_pstat_base_select";
goto error;
}
/* open and setup the local resource discovery framework */
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_open())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = opal_sysinfo_base_select())) {
ORTE_ERROR_LOG(ret);
error = "opal_sysinfo_base_select";
goto error;
}
/* Since we are the HNP, then responsibility for
* defining the name falls to the PLM component for our
* respective environment - hence, we have to open the PLM
* first and select that component.
*/
if (ORTE_SUCCESS != (ret = orte_plm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
/* Setup the communication infrastructure */
/*
* Runtime Messaging Layer
*/
if (ORTE_SUCCESS != (ret = orte_rml_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rml_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml_base_select";
goto error;
}
/*
* Routed system
*/
if (ORTE_SUCCESS != (ret = orte_routed_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_routed_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_routed_base_select";
goto error;
}
/*
* Group communications
*/
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_grpcomm_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_grpcomm_base_select";
goto error;
}
/* multicast */
#if ORTE_ENABLE_MULTICAST
if (ORTE_SUCCESS != (ret = orte_rmcast_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmcast_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmcast_base_select";
goto error;
}
#endif
/* Now provide a chance for the PLM
* to perform any module-specific init functions. This
* needs to occur AFTER the communications are setup
* as it may involve starting a non-blocking recv
*/
if (ORTE_SUCCESS != (ret = orte_plm.init())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_init";
goto error;
}
/*
* Setup the remaining resource
* management and errmgr frameworks - application procs
* and daemons do not open these frameworks as they only use
* the hnp proxy support in the PLM framework.
*/
if (ORTE_SUCCESS != (ret = orte_ras_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_ras_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_ras_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_rmaps_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_rmaps_base_find_available";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_open())) {
error = "orte_errmgr_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_errmgr_base_select";
goto error;
}
/* Open/select the odls */
if (ORTE_SUCCESS != (ret = orte_odls_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_odls_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_odls_base_select";
goto error;
}
/* enable communication with the rml */
if (ORTE_SUCCESS != (ret = orte_rml.enable_comm())) {
ORTE_ERROR_LOG(ret);
error = "orte_rml.enable_comm";
goto error;
}
/* we are an hnp, so update the contact info field for later use */
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
/* we are also officially a daemon, so better update that field too */
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
#if !ORTE_DISABLE_FULL_SUPPORT
/* setup the orte_show_help system to recv remote output */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_SHOW_HELP,
ORTE_RML_NON_PERSISTENT, orte_show_help_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
error = "setup receive for orte_show_help";
goto error;
}
#endif
/* setup my session directory */
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s setting up session dir with\n\ttmpdir: %s\n\thost %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base,
orte_process_info.nodename));
if (ORTE_SUCCESS != (ret = orte_session_dir(true,
orte_process_info.tmpdir_base,
orte_process_info.nodename, NULL,
ORTE_PROC_MY_NAME))) {
ORTE_ERROR_LOG(ret);
error = "orte_session_dir";
goto error;
}
/* Once the session directory location has been established, set
the opal_output hnp file location to be in the
proc-specific session directory. */
opal_output_set_output_file_info(orte_process_info.proc_session_dir,
"output-", NULL, NULL);
/* save my contact info in a file for others to find */
jobfam_dir = opal_dirname(orte_process_info.job_session_dir);
contact_path = opal_os_path(false, jobfam_dir, "contact.txt", NULL);
free(jobfam_dir);
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
contact_path));
if (ORTE_SUCCESS != (ret = orte_write_hnp_contact_file(contact_path))) {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s writing contact file failed with error %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_ERROR_NAME(ret)));
} else {
OPAL_OUTPUT_VERBOSE((2, orte_debug_output,
"%s wrote contact file",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
}
free(contact_path);
/* setup the global job and node arrays */
orte_job_data = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_job_data,
1,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
1))) {
ORTE_ERROR_LOG(ret);
error = "setup job array";
goto error;
}
orte_node_pool = OBJ_NEW(opal_pointer_array_t);
if (ORTE_SUCCESS != (ret = opal_pointer_array_init(orte_node_pool,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE,
ORTE_GLOBAL_ARRAY_MAX_SIZE,
ORTE_GLOBAL_ARRAY_BLOCK_SIZE))) {
ORTE_ERROR_LOG(ret);
error = "setup node array";
goto error;
}
/* Setup the job data object for the daemons */
/* create and store the job data object */
jdata = OBJ_NEW(orte_job_t);
jdata->jobid = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_set_item(orte_job_data, 0, jdata);
/* create and store a node object where we are */
node = OBJ_NEW(orte_node_t);
node->name = strdup(orte_process_info.nodename);
node->index = opal_pointer_array_add(orte_node_pool, node);
/* create and store a proc object for us */
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = ORTE_PROC_MY_NAME->jobid;
proc->name.vpid = ORTE_PROC_MY_NAME->vpid;
proc->pid = orte_process_info.pid;
proc->rml_uri = orte_rml.get_contact_info();
proc->state = ORTE_PROC_STATE_RUNNING;
OBJ_RETAIN(node); /* keep accounting straight */
proc->node = node;
proc->nodename = node->name;
opal_pointer_array_add(jdata->procs, proc);
/* record that the daemon (i.e., us) is on this node
* NOTE: we do not add the proc object to the node's
* proc array because we are not an application proc.
* Instead, we record it in the daemon field of the
* node object
*/
OBJ_RETAIN(proc); /* keep accounting straight */
node->daemon = proc;
node->daemon_launched = true;
node->state = ORTE_NODE_STATE_UP;
/* record that the daemon job is running */
jdata->num_procs = 1;
jdata->state = ORTE_JOB_STATE_RUNNING;
/* setup the routed info - the selected routed component
* will know what to do.
*/
if (ORTE_SUCCESS != (ret = orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL))) {
ORTE_ERROR_LOG(ret);
error = "orte_routed.init_routes";
goto error;
}
/* setup I/O forwarding system - must come after we init routes */
if (ORTE_SUCCESS != (ret = orte_iof_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_iof_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_iof_base_select";
goto error;
}
/* setup the FileM */
if (ORTE_SUCCESS != (ret = orte_filem_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_filem_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_filem_base_select";
goto error;
}
#if OPAL_ENABLE_FT_CR == 1
/*
* Setup the SnapC
*/
if (ORTE_SUCCESS != (ret = orte_snapc_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_snapc_base_select(ORTE_PROC_IS_HNP, !ORTE_PROC_IS_DAEMON))) {
ORTE_ERROR_LOG(ret);
error = "orte_snapc_base_select";
goto error;
}
/* For HNP, ORTE doesn't need the OPAL CR stuff */
opal_cr_set_enabled(false);
#else
opal_cr_set_enabled(false);
#endif
/*
* Initalize the CR setup
* Note: Always do this, even in non-FT builds.
* If we don't some user level tools may hang.
*/
if (ORTE_SUCCESS != (ret = orte_cr_init())) {
ORTE_ERROR_LOG(ret);
error = "orte_cr_init";
goto error;
}
/* setup the notifier system */
if (ORTE_SUCCESS != (ret = orte_notifier_base_open())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_open";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_notifier_base_select())) {
ORTE_ERROR_LOG(ret);
error = "orte_notifer_select";
goto error;
}
/* if a tool has launched us and is requesting event reports,
* then set its contact info into the comm system
*/
if (orte_report_events) {
if (ORTE_SUCCESS != (ret = orte_util_comm_connect_tool(orte_report_events_uri))) {
error = "could not connect to tool";
goto error;
}
}
/* We actually do *not* want an HNP to voluntarily yield() the
processor more than necessary. Orterun already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at orterun, we want the
OS to wake us up in a timely fashion (which most OS's
seem good about doing) and then we want orterun to process
the message as fast as possible. If orterun yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules orterun to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
return ORTE_SUCCESS;
error:
if (ORTE_ERR_SILENT != ret) {
orte_show_help("help-orte-runtime.txt",
"orte_init:startup:internal-failure",
true, error, ORTE_ERROR_NAME(ret), ret);
}
/* cleanup the global list of local children and job data */
OBJ_DESTRUCT(&orte_local_children);
OBJ_DESTRUCT(&orte_local_jobdata);
return ret;
}
