int orte_util_setup_local_nidmap_entries(void)
{
orte_nid_t *node;
orte_jmap_t *jmap;
orte_pmap_t *pmap;
/* add a jmap entry for myself */
jmap = OBJ_NEW(orte_jmap_t);
jmap->job = ORTE_PROC_MY_NAME->jobid;
opal_pointer_array_add(&orte_jobmap, jmap);
jmap->num_procs = 1;
/* create a nidmap entry for this node */
node = OBJ_NEW(orte_nid_t);
node->name = strdup(orte_process_info.nodename);
node->daemon = ORTE_PROC_MY_DAEMON->vpid;
pmap = OBJ_NEW(orte_pmap_t);
pmap->local_rank = 0;
pmap->node_rank = 0;
node->index = opal_pointer_array_add(&orte_nidmap, node);
/* value array copies values, so everything must be set before
* calling the set_item function
*/
pmap->node = node->index;
opal_pointer_array_set_item(&jmap->pmap, ORTE_PROC_MY_NAME->vpid, pmap);
/* all done */
return ORTE_SUCCESS;
}
/**
* Function for finding and opening either all MCA components, or the one
* that was specifically requested via a MCA parameter.
*/
static int mca_spml_base_open(mca_base_open_flag_t flags)
{
/**
* Construct the send and receive request queues. There are 2 reasons to do it
* here. First, as they are globals it's better to construct them in one common
* place. Second, in order to be able to allow the external debuggers to show
* their content, they should get constructed as soon as possible once the MPI
* process is started.
*/
OBJ_CONSTRUCT(&mca_spml_base_put_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_spml_base_get_requests, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_spml_base_spml, opal_pointer_array_t);
/* Open up all available components */
if (OPAL_SUCCESS !=
mca_base_framework_components_open(&oshmem_spml_base_framework, flags)) {
return OSHMEM_ERROR;
}
/* Set a sentinel in case we don't select any components (e.g.,
ompi_info) */
mca_spml_base_selected_component.spmlm_finalize = NULL;
/**
* Right now our selection of BTLs is completely broken. If we have
* multiple SPMLs that use BTLs than we will open all BTLs several times, leading to
* undefined behaviors. The simplest solution, at least until we
* figure out the correct way to do it, is to force a default SPML that
* uses BTLs and any other SPMLs that do not in the mca_spml_base_spml array.
*/
#if MCA_ompi_pml_DIRECT_CALL
opal_pointer_array_add(&mca_spml_base_spml,
strdup(stringify(MCA_oshmem_spml_DIRECT_CALL_COMPONENT)));
#else
{
const char **default_spml = NULL;
int var_id;
var_id = mca_base_var_find("oshmem", "spml", NULL, NULL);
mca_base_var_get_value(var_id, &default_spml, NULL, NULL);
if( (NULL == default_spml || NULL == default_spml[0] ||
0 == strlen(default_spml[0])) || (default_spml[0][0] == '^') ) {
#ifdef OSHMEM_HAS_IKRIT
opal_pointer_array_add(&mca_spml_base_spml, strdup("ikrit"));
#endif
opal_pointer_array_add(&mca_spml_base_spml, strdup("yoda"));
} else {
opal_pointer_array_add(&mca_spml_base_spml, strdup(default_spml[0]));
}
}
#endif
return OSHMEM_SUCCESS;
}
int orte_rmaps_base_add_proc_to_map(orte_job_map_t *map, orte_node_t *node,
bool oversubscribed, orte_proc_t *proc)
{
orte_std_cntr_t i;
orte_node_t *node_from_map;
int rc;
/* see if this node has already been assigned to the map - if
* not, then add the pointer to the pointer array
*/
for (i=0; i < map->nodes->size; i++) {
if (NULL == (node_from_map = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
continue;
}
if (node_from_map->index == node->index) {
/* we have this node in the array */
goto PROCESS;
}
}
/* if we get here, then this node isn't already in the map - add it */
OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base.rmaps_output,
"%s rmaps:base: adding node %s to map",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == node->name) ? "NULL" : node->name));
if (ORTE_SUCCESS > (rc = opal_pointer_array_add(map->nodes, (void*)node))) {
ORTE_ERROR_LOG(rc);
return rc;
}
OBJ_RETAIN(node); /* maintain accounting on object */
++map->num_nodes;
PROCESS:
/* add the proc to this node's local processes - it is assumed
* that the proc isn't already there as this would be an error
* in the mapper
*/
OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base.rmaps_output,
"%s rmaps:base: mapping proc for job %s to node %s whose daemon is %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(proc->name.jobid),
(NULL == node->name) ? "NULL" : node->name,
(NULL == node->daemon) ? "NULL" : ORTE_NAME_PRINT(&(node->daemon->name))));
if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* retain the proc struct so that we correctly track its release */
OBJ_RETAIN(proc);
++node->num_procs;
/* update the oversubscribed state of the node */
node->oversubscribed = oversubscribed;
return ORTE_SUCCESS;
}
static void killprocs(orte_jobid_t job, orte_vpid_t vpid)
{
opal_pointer_array_t cmd;
orte_proc_t proc;
int rc;
if (ORTE_JOBID_WILDCARD == job
&& ORTE_VPID_WILDCARD == vpid) {
if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(NULL))) {
ORTE_ERROR_LOG(rc);
}
return;
}
OBJ_CONSTRUCT(&cmd, opal_pointer_array_t);
OBJ_CONSTRUCT(&proc, orte_proc_t);
proc.name.jobid = job;
proc.name.vpid = vpid;
opal_pointer_array_add(&cmd, &proc);
if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(&cmd))) {
ORTE_ERROR_LOG(rc);
}
OBJ_DESTRUCT(&cmd);
OBJ_DESTRUCT(&proc);
}
int mca_btl_ugni_init_ep (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc)
{
mca_btl_ugni_endpoint_t *endpoint;
int rc;
endpoint = OBJ_NEW(mca_btl_ugni_endpoint_t);
assert (endpoint != NULL);
endpoint->smsg_progressing = 0;
endpoint->state = MCA_BTL_UGNI_EP_STATE_INIT;
endpoint->peer_proc = peer_proc;
/* get the modex info for this endpoint and setup a ugni endpoint. this call may lead
* to re-entry through opal_progress(). */
rc = mca_btl_ugni_endpoint_get_modex (endpoint);
if (OPAL_SUCCESS != rc) {
assert (0);
return rc;
}
/* add this endpoint to the pointer array */
endpoint->index = opal_pointer_array_add (&ugni_module->endpoints, endpoint);
*ep = endpoint;
return OPAL_SUCCESS;
}
/**
* Initialization of data structures for running under a debugger
* using the MPICH/TotalView parallel debugger interface. Before the
* spawn we need to check if we are being run under a TotalView-like
* debugger; if so then inform applications via an MCA parameter.
*/
void orte_debugger_init_before_spawn(orte_job_t *jdata)
{
char *env_name;
orte_app_context_t **apps, *app;
orte_std_cntr_t i;
int32_t ljob;
if (!MPIR_being_debugged && !orte_in_parallel_debugger) {
/* not being debugged - check if we want to enable
* later attachment by debugger
*/
if (orte_enable_debug_cospawn_while_running) {
/* setup a timer to wake us up periodically
* to check for debugger attach
*/
ORTE_TIMER_EVENT(orte_debugger_check_rate, 0, check_debugger);
}
return;
}
if (orte_debug_flag) {
opal_output(0, "Info: Spawned by a debugger");
}
/* tell the procs they are being debugged */
apps = (orte_app_context_t**)jdata->apps->addr;
env_name = mca_base_param_environ_variable("orte",
"in_parallel_debugger", NULL);
for (i=0; i < jdata->num_apps; i++) {
opal_setenv(env_name, "1", true, &apps[i]->env);
}
free(env_name);
/* check if we need to co-spawn the debugger daemons */
if ('\0' != MPIR_executable_path[0]) {
/* add debugger info to launch message */
orte_debugger_daemon = OBJ_NEW(orte_job_t);
/* create a jobid for these daemons - this is done solely
* to avoid confusing the rest of the system's bookkeeping
*/
orte_plm_base_create_jobid(orte_debugger_daemon);
/* flag the job as being debugger daemons */
orte_debugger_daemon->controls |= ORTE_JOB_CONTROL_DEBUGGER_DAEMON;
/* unless directed, we do not forward output */
if (!MPIR_forward_output) {
orte_debugger_daemon->controls &= ~ORTE_JOB_CONTROL_FORWARD_OUTPUT;
}
/* add it to the global job pool */
ljob = ORTE_LOCAL_JOBID(orte_debugger_daemon->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, orte_debugger_daemon);
/* create an app_context for the debugger daemon */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup((char*)MPIR_executable_path);
opal_argv_append_nosize(&app->argv, app->app);
build_debugger_args(app);
opal_pointer_array_add(orte_debugger_daemon->apps, &app->super);
orte_debugger_daemon->num_apps = 1;
}
}
orte_proc_t* orte_rmaps_base_setup_proc(orte_job_t *jdata,
orte_node_t *node,
orte_app_idx_t idx)
{
orte_proc_t *proc;
int rc;
proc = OBJ_NEW(orte_proc_t);
/* set the jobid */
proc->name.jobid = jdata->jobid;
/* flag the proc as ready for launch */
proc->state = ORTE_PROC_STATE_INIT;
proc->app_idx = idx;
OBJ_RETAIN(node); /* maintain accounting on object */
proc->node = node;
proc->nodename = node->name;
node->num_procs++;
if (node->slots_inuse < node->slots) {
node->slots_inuse += orte_rmaps_base.cpus_per_rank;
}
if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(proc);
return NULL;
}
/* retain the proc struct so that we correctly track its release */
OBJ_RETAIN(proc);
return proc;
}
static void killprocs(orte_jobid_t job, orte_vpid_t vpid)
{
opal_pointer_array_t cmd;
orte_proc_t proc;
int rc;
/* stop local sensors for this job */
if (ORTE_VPID_WILDCARD == vpid) {
orte_sensor.stop(job);
}
if (ORTE_JOBID_WILDCARD == job
&& ORTE_VPID_WILDCARD == vpid) {
if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(NULL))) {
ORTE_ERROR_LOG(rc);
}
return;
}
OBJ_CONSTRUCT(&cmd, opal_pointer_array_t);
OBJ_CONSTRUCT(&proc, orte_proc_t);
proc.name.jobid = job;
proc.name.vpid = vpid;
ORTE_EPOCH_SET(proc.name.epoch,orte_ess.proc_get_epoch(&(proc.name)));
opal_pointer_array_add(&cmd, &proc);
if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(&cmd))) {
ORTE_ERROR_LOG(rc);
}
OBJ_DESTRUCT(&cmd);
OBJ_DESTRUCT(&proc);
}
static int mca_pml_monitoring_component_open(void)
{
if( mca_pml_monitoring_enabled ) {
opal_pointer_array_add(&mca_pml_base_pml,
strdup(mca_pml_monitoring_component.pmlm_version.mca_component_name));
}
return OMPI_SUCCESS;
}
/*
* Op constructor
*/
static void ompi_op_construct(ompi_op_t *new_op)
{
int ret_val;
/* assign entry in fortran <-> c translation array */
ret_val = opal_pointer_array_add(ompi_op_f_to_c_table, new_op);
new_op->o_f_to_c_index = ret_val;
}
static void check_installed(bool check_all)
{
int i, n;
orcm_cfgi_app_t *app;
orcm_cfgi_run_t *run;
/* run a check of the installed apps against
* the configured apps so we can start anything that was awaiting
* installation
*/
for (i=0; i < orcm_cfgi_base.installed_apps.size; i++) {
if (NULL == (app = (orcm_cfgi_app_t*)opal_pointer_array_get_item(&orcm_cfgi_base.installed_apps, i))) {
continue;
}
if (!check_all && !app->modified) {
OPAL_OUTPUT_VERBOSE((2, orcm_cfgi_base.output,
"APP %s HAS NOT BEEN MODIFIED",
app->application));
continue;
}
OPAL_OUTPUT_VERBOSE((2, orcm_cfgi_base.output,
"CHECKING INSTALL-RUNNING CONFIG FOR APP %s", app->application));
/* reset the flag */
app->modified = false;
/* search the configuration array for instances of this app */
for (n=0; n < orcm_cfgi_base.confgd_apps.size; n++) {
if (NULL == (run = (orcm_cfgi_run_t*)opal_pointer_array_get_item(&orcm_cfgi_base.confgd_apps, n))) {
continue;
}
if (NULL == run->app) {
/* still waiting for app to be defined - is this it? */
if (0 == strcmp(run->application, app->application)) {
/* yep - see if we can run it */
if (0 <= app->max_instances && app->max_instances <= app->num_instances) {
/* at our limit - can't run at this time */
continue;
}
/* add this instance */
run->app = app;
run->app_idx = opal_pointer_array_add(&app->instances, run);
app->num_instances++;
link_launch(app, run, check_all);
}
} else if (0 == strcmp(run->application, app->application)) {
OPAL_OUTPUT_VERBOSE((2, orcm_cfgi_base.output,
"%s EXISTING INSTANCE %s:%s CAN BE LAUNCHED",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
run->application, run->instance));
/* check to see what has changed, and launch it if reqd */
link_launch(app, run, check_all);
}
}
}
}
static void cons_request(allocate_request_t **request)
{
resource_request_t *res_req1, *res_req2;
/* new 'request' */
*request = (allocate_request_t *)malloc(sizeof(allocate_request_t));
/* init 'request->res_req_array' */
//OBJ_CONSTRUCT(request->res_req_array, opal_pointer_array_t);
(*request)->res_array = OBJ_NEW(opal_pointer_array_t);
opal_pointer_array_init((*request)->res_array, 1, INT_MAX, 1);
(*request)->nreq = 0;
(*request)->attempt_id = 9;
/* create 'res_req' */
/* res_req = OBJ_NEW(resource_request_t); */ /* what's the problem? */
res_req1 = (resource_request_t *)malloc(sizeof(resource_request_t));
res_req1->appid = 100;
res_req1->np = 3;
res_req1->min_nodes = 2;
res_req1->node_list = strdup("node[1-2]");
res_req1->mandatory = true;
/* add 'res_req' to 'request->res_req_array' */
opal_pointer_array_add((*request)->res_array, res_req1);
res_req2 = (resource_request_t *)malloc(sizeof(resource_request_t));
res_req2->appid = 101;
res_req2->np = 1;
res_req2->min_nodes = 1;
res_req2->node_list = strdup("node[3-4]");
res_req2->mandatory = false;
opal_pointer_array_add((*request)->res_array, res_req2);
}
/*
* This function is invoked when OBJ_NEW() is called. Here, we add this
* info pointer to the table and then store its index as the handle
*/
static void info_constructor(ompi_info_t *info)
{
info->i_f_to_c_index = opal_pointer_array_add(&ompi_info_f_to_c_table,
info);
info->i_lock = OBJ_NEW(opal_mutex_t);
info->i_freed = false;
/* If the user doesn't want us to ever free it, then add an extra
RETAIN here */
if (ompi_debug_no_free_handles) {
OBJ_RETAIN(&(info->super));
}
}
static void _terminate_job(orte_jobid_t jobid)
{
opal_pointer_array_t procs;
orte_proc_t pobj;
OBJ_CONSTRUCT(&procs, opal_pointer_array_t);
opal_pointer_array_init(&procs, 1, 1, 1);
OBJ_CONSTRUCT(&pobj, orte_proc_t);
pobj.name.jobid = jobid;
pobj.name.vpid = ORTE_VPID_WILDCARD;
opal_pointer_array_add(&procs, &pobj);
orte_plm.terminate_procs(&procs);
OBJ_DESTRUCT(&procs);
OBJ_DESTRUCT(&pobj);
}
static int store_pointer(const orte_process_name_t *proc,
opal_value_t *kv)
{
int i;
job_data_t *jtable, *jtab;
proc_data_t *proc_data;
opal_value_t *k2;
OPAL_OUTPUT_VERBOSE((5, orte_db_base.output,
"%s db:hash:store: storing pointer of key %s for proc %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
kv->key, ORTE_NAME_PRINT(proc)));
/* get the job data object for this proc */
jtable = NULL;
for (i=0; i < job_data.size; i++) {
if (NULL == (jtab = (job_data_t*)opal_pointer_array_get_item(&job_data, i))) {
continue;
}
if (jtab->jobid == proc->jobid) {
jtable = jtab;
break;
}
}
if (NULL == jtable) {
/* need to add an entry for this job */
jtable = OBJ_NEW(job_data_t);
jtable->jobid = proc->jobid;
opal_pointer_array_add(&job_data, jtable);
}
/* lookup the proc data object for this proc */
if (NULL == (proc_data = lookup_orte_proc(jtable->data, proc->vpid))) {
/* unrecoverable error */
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* see if we already have this key in the data - means we are updating
* a pre-existing value
*/
if (NULL != (k2 = lookup_keyval(proc_data, kv->key))) {
opal_list_remove_item(&proc_data->data, &k2->super);
OBJ_RELEASE(k2);
}
opal_list_append(&proc_data->data, &kv->super);
return ORTE_SUCCESS;
}
/*
* group constructor
*/
static void ompi_group_construct(ompi_group_t *new_group)
{
int ret_val;
/* Note that we do *NOT* increase the refcount on all the included
procs here because that is handled at a different level (e.g.,
the proc counts are not decreased during the desstructor,
either). */
/* assign entry in fortran <-> c translation array */
ret_val = opal_pointer_array_add(&ompi_group_f_to_c_table, new_group);
new_group->grp_f_to_c_index = ret_val;
new_group->grp_flags = 0;
/* default the sparse values for groups */
new_group->grp_parent_group_ptr = NULL;
}
/*
* Constructor
*/
static void file_constructor(ompi_file_t *file)
{
/* Initialize the MPI_FILE_OPEN params */
file->f_comm = NULL;
file->f_filename = NULL;
file->f_amode = 0;
file->f_info = NULL;
/* Initialize flags */
file->f_flags = 0;
/* Initialize the fortran <--> C translation index */
file->f_f_to_c_index = opal_pointer_array_add(&ompi_file_f_to_c_table,
file);
/* Initialize the error handler. Per MPI-2:9.7 (p265), the
default error handler on file handles is the error handler on
MPI_FILE_NULL, which starts out as MPI_ERRORS_RETURN (but can
be changed by invoking MPI_FILE_SET_ERRHANDLER on
MPI_FILE_NULL). */
file->errhandler_type = OMPI_ERRHANDLER_TYPE_FILE;
if (file != &ompi_mpi_file_null.file) {
file->error_handler = ompi_mpi_file_null.file.error_handler;
} else {
file->error_handler = &ompi_mpi_errors_return.eh;
}
OBJ_RETAIN(file->error_handler);
/* Initialize the module */
file->f_io_version = MCA_IO_BASE_V_NONE;
memset(&(file->f_io_selected_module), 0,
sizeof(file->f_io_selected_module));
file->f_io_selected_data = NULL;
/* If the user doesn't want us to ever free it, then add an extra
RETAIN here */
if (ompi_debug_no_free_handles) {
OBJ_RETAIN(&(file->super));
}
}
/*
* Op constructor
*/
static void ompi_op_construct(ompi_op_t *new_op)
{
int i;
/* assign entry in fortran <-> c translation array */
new_op->o_f_to_c_index =
opal_pointer_array_add(ompi_op_f_to_c_table, new_op);
/* Set everything to NULL so that we can intelligently free
non-NULL's in the destructor */
for (i = 0; i < OMPI_OP_BASE_TYPE_MAX; ++i) {
new_op->o_func.intrinsic.fns[i] = NULL;
new_op->o_func.intrinsic.modules[i] = NULL;
new_op->o_3buff_intrinsic.fns[i] = NULL;
new_op->o_3buff_intrinsic.modules[i] = NULL;
}
}
/**
* Errhandler constructor
*/
static void ompi_errhandler_construct(ompi_errhandler_t *new_errhandler)
{
int ret_val;
/* assign entry in fortran <-> c translation array */
ret_val = opal_pointer_array_add(&ompi_errhandler_f_to_c_table,
new_errhandler);
new_errhandler->eh_f_to_c_index = ret_val;
new_errhandler->eh_lang = OMPI_ERRHANDLER_LANG_C;
new_errhandler->eh_comm_fn = NULL;
new_errhandler->eh_win_fn = NULL;
new_errhandler->eh_file_fn = NULL;
new_errhandler->eh_fort_fn = NULL;
new_errhandler->eh_cxx_dispatch_fn = NULL;
memset (new_errhandler->eh_name, 0, MPI_MAX_OBJECT_NAME);
}
static int orte_routed_base_open(mca_base_open_flag_t flags)
{
orte_routed_jobfam_t *jfam;
orte_routed_base_wait_sync = false;
/* Initialize storage of remote hnp uris */
OBJ_CONSTRUCT(&orte_routed_jobfams, opal_pointer_array_t);
opal_pointer_array_init(&orte_routed_jobfams, 8, INT_MAX, 8);
/* prime it with our HNP uri */
jfam = OBJ_NEW(orte_routed_jobfam_t);
jfam->route.jobid = ORTE_PROC_MY_HNP->jobid;
jfam->route.vpid = ORTE_PROC_MY_HNP->vpid;
jfam->job_family = ORTE_JOB_FAMILY(ORTE_PROC_MY_NAME->jobid);
if (NULL != orte_process_info.my_hnp_uri) {
jfam->hnp_uri = strdup(orte_process_info.my_hnp_uri);
}
opal_pointer_array_add(&orte_routed_jobfams, jfam);
/* Open up all available components */
return mca_base_framework_components_open(&orte_routed_base_framework, flags);
}
/**
* This graph API returns an array of pointers of all the
* vertices in the graph.
*
*
* @param graph
* @param vertices_list an array of pointers of all the
* vertices in the graph vertices.
*
* @return int returning the graph order (the
* number of vertices in the returned array)
*/
int opal_graph_get_graph_vertices(opal_graph_t *graph, opal_pointer_array_t *vertices_list)
{
opal_adjacency_list_t *aj_list;
opal_list_item_t *item;
int i;
/**
* If the graph order is 0, return NULL.
*/
if (0 == graph->number_of_vertices) {
return 0;
}
/* Run on all the vertices of the graph */
for (item = opal_list_get_first(graph->adjacency_list), i = 0;
item != opal_list_get_end(graph->adjacency_list);
item = opal_list_get_next(item), i++) {
aj_list = (opal_adjacency_list_t *) item;
/* Add the vertex to the vertices array */
opal_pointer_array_add(vertices_list,(void *)aj_list->vertex);
}
/* return the vertices list */
return graph->number_of_vertices;
}
orte_proc_t* orte_rmaps_base_setup_proc(orte_job_t *jdata,
orte_node_t *node,
orte_app_idx_t idx)
{
orte_proc_t *proc;
int rc;
proc = OBJ_NEW(orte_proc_t);
/* set the jobid */
proc->name.jobid = jdata->jobid;
/* flag the proc as ready for launch */
proc->state = ORTE_PROC_STATE_INIT;
proc->app_idx = idx;
/* mark the proc as UPDATED so it will be included in the launch */
ORTE_FLAG_SET(proc, ORTE_PROC_FLAG_UPDATED);
if (NULL == node->daemon) {
proc->parent = ORTE_VPID_INVALID;
} else {
proc->parent = node->daemon->name.vpid;
}
OBJ_RETAIN(node); /* maintain accounting on object */
proc->node = node;
node->num_procs++;
if (node->slots_inuse < node->slots) {
node->slots_inuse += orte_rmaps_base.cpus_per_rank;
}
if (0 > (rc = opal_pointer_array_add(node->procs, (void*)proc))) {
ORTE_ERROR_LOG(rc);
OBJ_RELEASE(proc);
return NULL;
}
/* retain the proc struct so that we correctly track its release */
OBJ_RETAIN(proc);
return proc;
}
static int
config_window(void *base, size_t size, int disp_unit,
int flavor, int model, ompi_win_t *win)
{
int ret;
ret = ompi_attr_set_c(WIN_ATTR, win, &win->w_keyhash,
MPI_WIN_BASE, base, true);
if (OMPI_SUCCESS != ret) return ret;
ret = ompi_attr_set_aint(WIN_ATTR, win,
&win->w_keyhash,
MPI_WIN_SIZE, size, true);
if (OMPI_SUCCESS != ret) return ret;
ret = ompi_attr_set_int(WIN_ATTR, win,
&win->w_keyhash,
MPI_WIN_DISP_UNIT, disp_unit,
true);
if (OMPI_SUCCESS != ret) return ret;
ret = ompi_attr_set_int(WIN_ATTR, win,
&win->w_keyhash,
MPI_WIN_CREATE_FLAVOR, flavor, true);
if (OMPI_SUCCESS != ret) return ret;
ret = ompi_attr_set_int(WIN_ATTR, win,
&win->w_keyhash,
MPI_WIN_MODEL, model, true);
if (OMPI_SUCCESS != ret) return ret;
win->w_f_to_c_index = opal_pointer_array_add(&ompi_mpi_windows, win);
if (-1 == win->w_f_to_c_index) return OMPI_ERR_OUT_OF_RESOURCE;
return OMPI_SUCCESS;
}
int orte_plm_base_orted_terminate_job(orte_jobid_t jobid)
{
opal_pointer_array_t procs;
orte_proc_t proc;
int rc;
OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
"%s plm:base:orted_terminate job %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(jobid)));
OBJ_CONSTRUCT(&procs, opal_pointer_array_t);
opal_pointer_array_init(&procs, 1, 1, 1);
OBJ_CONSTRUCT(&proc, orte_proc_t);
proc.name.jobid = jobid;
proc.name.vpid = ORTE_VPID_WILDCARD;
opal_pointer_array_add(&procs, &proc);
if (ORTE_SUCCESS != (rc = orte_plm_base_orted_kill_local_procs(&procs))) {
ORTE_ERROR_LOG(rc);
}
OBJ_DESTRUCT(&procs);
OBJ_DESTRUCT(&proc);
return rc;
}
/*
* JOB
* NOTE: We do not pack all of the job object's fields as many of them have no
* value in sending them to another location. The only purpose in packing and
* sending a job object is to communicate the data required to dynamically
* spawn another job - so we only pack that limited set of required data.
* Therefore, only unpack what was packed
*/
int orte_dt_unpack_job(opal_buffer_t *buffer, void *dest,
int32_t *num_vals, opal_data_type_t type)
{
int rc;
int32_t i, k, n, count;
orte_job_t **jobs;
orte_app_idx_t j;
orte_attribute_t *kv;
/* unpack into array of orte_job_t objects */
jobs = (orte_job_t**) dest;
for (i=0; i < *num_vals; i++) {
/* create the orte_job_t object */
jobs[i] = OBJ_NEW(orte_job_t);
if (NULL == jobs[i]) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
/* unpack the jobid */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
&(jobs[i]->jobid), &n, ORTE_JOBID))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the personality */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer, &jobs[i]->personality, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the num apps */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->num_apps)), &n, ORTE_APP_IDX))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* if there are apps, unpack them */
if (0 < jobs[i]->num_apps) {
orte_app_context_t *app;
for (j=0; j < jobs[i]->num_apps; j++) {
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
&app, &n, ORTE_APP_CONTEXT))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_pointer_array_add(jobs[i]->apps, app);
}
}
/* unpack num procs and offset */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->num_procs)), &n, ORTE_VPID))) {
ORTE_ERROR_LOG(rc);
return rc;
}
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->offset)), &n, ORTE_VPID))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* and the procs, if provided */
if (0 < jobs[i]->num_procs) {
orte_proc_t *proc;
for (j=0; j < jobs[i]->num_procs; j++) {
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
&proc, &n, ORTE_PROC))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_pointer_array_add(jobs[i]->procs, proc);
}
}
/* unpack stdin target */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->stdin_target)), &n, ORTE_VPID))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the total slots allocated to the job */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->total_slots_alloc)), &n, ORTE_STD_CNTR))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* if the map is NULL, then we din't pack it as there was
* nothing to pack. Instead, we packed a flag to indicate whether or not
* the map is included */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
&j, &n, ORTE_STD_CNTR))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (0 < j) {
/* unpack the map */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->map)), &n, ORTE_JOB_MAP))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* no bookmark of oversubscribe_override flags to unpack */
/* unpack the job state */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->state)), &n, ORTE_JOB_STATE))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the flags */
n = 1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
(&(jobs[i]->flags)), &n, ORTE_JOB_FLAGS_T))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* unpack the attributes */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer, &count,
&n, ORTE_STD_CNTR))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (k=0; k < count; k++) {
n=1;
if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer, &kv,
&n, ORTE_ATTRIBUTE))) {
ORTE_ERROR_LOG(rc);
return rc;
}
kv->local = ORTE_ATTR_GLOBAL; // obviously not a local value
opal_list_append(&jobs[i]->attributes, &kv->super);
}
}
return ORTE_SUCCESS;
}
static int update_route(orte_process_name_t *target,
orte_process_name_t *route)
{
int i;
orte_routed_jobfam_t *jfam;
uint16_t jfamily;
if (target->jobid == ORTE_JOBID_INVALID ||
target->vpid == ORTE_VPID_INVALID) {
return ORTE_ERR_BAD_PARAM;
}
/* if I am an application process, we don't update the route since
* we automatically route everything through the local daemon
*/
if (ORTE_PROC_IS_APP) {
return ORTE_SUCCESS;
}
OPAL_OUTPUT_VERBOSE((1, orte_routed_base_framework.framework_output,
"%s routed_radix_update: %s --> %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(target),
ORTE_NAME_PRINT(route)));
/* if I am a daemon and the target is my HNP, then check
* the route - if it isn't direct, then we just flag that
* we have a route to the HNP
*/
if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_PROC_MY_HNP, target) &&
OPAL_EQUAL != orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_PROC_MY_HNP, route)) {
hnp_direct = false;
return ORTE_SUCCESS;
}
/* if this is from a different job family, then I need to
* track how to send messages to it
*/
if (ORTE_JOB_FAMILY(target->jobid) != ORTE_JOB_FAMILY(ORTE_PROC_MY_NAME->jobid)) {
/* if I am a daemon, then I will automatically route
* anything to this job family via my HNP - so nothing to do
* here, just return
*/
if (ORTE_PROC_IS_DAEMON) {
return ORTE_SUCCESS;
}
OPAL_OUTPUT_VERBOSE((1, orte_routed_base_framework.framework_output,
"%s routed_radix_update: diff job family routing job %s --> %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOBID_PRINT(target->jobid),
ORTE_NAME_PRINT(route)));
/* see if this target is already present */
jfamily = ORTE_JOB_FAMILY(target->jobid);
for (i=0; i < orte_routed_jobfams.size; i++) {
if (NULL == (jfam = (orte_routed_jobfam_t*)opal_pointer_array_get_item(&orte_routed_jobfams, i))) {
continue;
}
if (jfam->job_family == jfamily) {
OPAL_OUTPUT_VERBOSE((2, orte_routed_base_framework.framework_output,
"%s routed_radix: updating route to %s via %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOB_FAMILY_PRINT(target->jobid),
ORTE_NAME_PRINT(route)));
jfam->route.jobid = route->jobid;
jfam->route.vpid = route->vpid;
return ORTE_SUCCESS;
}
}
/* not there, so add the route FOR THE JOB FAMILY*/
OPAL_OUTPUT_VERBOSE((2, orte_routed_base_framework.framework_output,
"%s routed_radix: adding route to %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_JOB_FAMILY_PRINT(target->jobid)));
jfam = OBJ_NEW(orte_routed_jobfam_t);
jfam->job_family = jfamily;
jfam->route.jobid = route->jobid;
jfam->route.vpid = route->vpid;
opal_pointer_array_add(&orte_routed_jobfams, jfam);
return ORTE_SUCCESS;
}
return ORTE_SUCCESS;
}
static void test(bool thread_usage){
/* local variables */
opal_pointer_array_t *array;
value_t *test_data;
int len_test_data,i,test_len_in_array,error_cnt;
int ele_index;
int use_threads,error_code;
value_t value;
/* initialize thread levels */
use_threads=(int)opal_set_using_threads(thread_usage);
array=OBJ_NEW(opal_pointer_array_t);
assert(array);
len_test_data=5;
test_data=malloc(sizeof(value_t)*len_test_data);
assert(test_data);
for(i=0 ; i < len_test_data ; i++ ) {
test_data[i].ivalue = (i+1);
}
/* add data to table */
test_len_in_array=3;
assert(len_test_data>=test_len_in_array);
for(i=0 ; i < test_len_in_array ; i++ ) {
opal_pointer_array_add(array,test_data[i].cvalue);
}
/* check to see that test_len_in_array are in array */
if( (array->size - array->number_free) == test_len_in_array) {
test_success();
} else {
test_failure("check on number of elments in array");
}
/* check order of data */
error_cnt=0;
for(i=0 ; i < test_len_in_array ; i++ ) {
value.cvalue = array->addr[i];
if( (i+1) != value.ivalue ) {
error_cnt++;
}
}
if( 0 == error_cnt ) {
test_success();
} else {
test_failure(" data check ");
}
/* free 2nd element and make sure that value is reset correctly,
* and that the lowest_index is also reset correctly */
ele_index=1;
error_code=opal_pointer_array_set_item(array,ele_index,NULL);
if( 0 == error_code ) {
test_success();
} else {
test_failure(" opal_pointer_array_set_item ");
}
if( NULL == array->addr[ele_index]){
test_success();
} else {
test_failure(" set pointer value");
}
if( ele_index == array->lowest_free ) {
test_success();
} else {
test_failure(" lowest free ");
}
/* test opal_pointer_array_get_item */
array->number_free=array->size;
array->lowest_free=0;
for(i=0 ; i < array->size ; i++ ) {
array->addr[i] = NULL;
}
error_cnt=0;
for(i=0 ; i < array->size ; i++ ) {
value.ivalue = i + 2;
ele_index=opal_pointer_array_add(array, value.cvalue);
if( i != ele_index ) {
error_cnt++;
}
}
if( 0 == error_cnt ) {
test_success();
} else {
test_failure(" opal_pointer_array_add 2nd ");
}
error_cnt=0;
for(i=0 ; i < array->size ; i++ ) {
value.cvalue = opal_pointer_array_get_item(array,i);
if( (i+2) != value.ivalue ) {
error_cnt++;
}
}
if( 0 == error_cnt ) {
test_success();
} else {
test_failure(" data check - 2nd ");
}
free (array);
free(test_data);
}
int orte_daemon(int argc, char *argv[])
{
int ret = 0;
opal_cmd_line_t *cmd_line = NULL;
char *rml_uri;
int i;
opal_buffer_t *buffer;
char hostname[100];
char *tmp_env_var = NULL;
/* initialize the globals */
memset(&orted_globals, 0, sizeof(orted_globals));
/* initialize the singleton died pipe to an illegal value so we can detect it was set */
orted_globals.singleton_died_pipe = -1;
/* init the failure orted vpid to an invalid value */
orted_globals.fail = ORTE_VPID_INVALID;
/* setup to check common command line options that just report and die */
cmd_line = OBJ_NEW(opal_cmd_line_t);
if (OPAL_SUCCESS != opal_cmd_line_create(cmd_line, orte_cmd_line_opts)) {
OBJ_RELEASE(cmd_line);
exit(1);
}
mca_base_cmd_line_setup(cmd_line);
if (ORTE_SUCCESS != (ret = opal_cmd_line_parse(cmd_line, false,
argc, argv))) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
fprintf(stderr, "Usage: %s [OPTION]...\n%s\n", argv[0], args);
free(args);
OBJ_RELEASE(cmd_line);
return ret;
}
/*
* Since this process can now handle MCA/GMCA parameters, make sure to
* process them.
*/
mca_base_cmd_line_process_args(cmd_line, &environ, &environ);
/* Ensure that enough of OPAL is setup for us to be able to run */
/*
* NOTE: (JJH)
* We need to allow 'mca_base_cmd_line_process_args()' to process command
* line arguments *before* calling opal_init_util() since the command
* line could contain MCA parameters that affect the way opal_init_util()
* functions. AMCA parameters are one such option normally received on the
* command line that affect the way opal_init_util() behaves.
* It is "safe" to call mca_base_cmd_line_process_args() before
* opal_init_util() since mca_base_cmd_line_process_args() does *not*
* depend upon opal_init_util() functionality.
*/
if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) {
fprintf(stderr, "OPAL failed to initialize -- orted aborting\n");
exit(1);
}
/* save the environment for launch purposes. This MUST be
* done so that we can pass it to any local procs we
* spawn - otherwise, those local procs won't see any
* non-MCA envars that were set in the enviro when the
* orted was executed - e.g., by .csh
*/
orte_launch_environ = opal_argv_copy(environ);
/* purge any ess flag set in the environ when we were launched */
opal_unsetenv("OMPI_MCA_ess", &orte_launch_environ);
/* if orte_daemon_debug is set, let someone know we are alive right
* away just in case we have a problem along the way
*/
if (orted_globals.debug) {
gethostname(hostname, 100);
fprintf(stderr, "Daemon was launched on %s - beginning to initialize\n", hostname);
}
/* check for help request */
if (orted_globals.help) {
char *args = NULL;
args = opal_cmd_line_get_usage_msg(cmd_line);
orte_show_help("help-orted.txt", "orted:usage", false,
argv[0], args);
free(args);
return 1;
}
#if defined(HAVE_SETSID) && !defined(__WINDOWS__)
/* see if we were directed to separate from current session */
if (orted_globals.set_sid) {
setsid();
}
#endif /* !defined(__WINDOWS__) */
/* see if they want us to spin until they can connect a debugger to us */
i=0;
while (orted_spin_flag) {
i++;
if (1000 < i) i=0;
}
#if OPAL_ENABLE_FT_CR == 1
/* Mark as a tool program */
tmp_env_var = mca_base_param_env_var("opal_cr_is_tool");
opal_setenv(tmp_env_var,
"1",
true, &environ);
free(tmp_env_var);
#endif
tmp_env_var = NULL; /* Silence compiler warning */
/* if mapreduce set, flag it */
if (orted_globals.mapreduce) {
orte_map_reduce = true;
}
/* Set the flag telling OpenRTE that I am NOT a
* singleton, but am "infrastructure" - prevents setting
* up incorrect infrastructure that only a singleton would
* require.
*/
if (orted_globals.hnp) {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_HNP))) {
ORTE_ERROR_LOG(ret);
return ret;
}
} else {
if (ORTE_SUCCESS != (ret = orte_init(&argc, &argv, ORTE_PROC_DAEMON))) {
ORTE_ERROR_LOG(ret);
return ret;
}
}
/* finalize the OPAL utils. As they are opened again from orte_init->opal_init
* we continue to have a reference count on them. So we have to finalize them twice...
*/
opal_finalize_util();
if ((int)ORTE_VPID_INVALID != orted_globals.fail) {
orted_globals.abort=false;
/* some vpid was ordered to fail. The value can be positive
* or negative, depending upon the desired method for failure,
* so need to check both here
*/
if (0 > orted_globals.fail) {
orted_globals.fail = -1*orted_globals.fail;
orted_globals.abort = true;
}
/* are we the specified vpid? */
if ((int)ORTE_PROC_MY_NAME->vpid == orted_globals.fail) {
/* if the user specified we delay, then setup a timer
* and have it kill us
*/
if (0 < orted_globals.fail_delay) {
ORTE_TIMER_EVENT(orted_globals.fail_delay, 0, shutdown_callback, ORTE_SYS_PRI);
} else {
opal_output(0, "%s is executing clean %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
orted_globals.abort ? "abort" : "abnormal termination");
/* do -not- call finalize as this will send a message to the HNP
* indicating clean termination! Instead, just forcibly cleanup
* the local session_dir tree and exit
*/
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
/* if we were ordered to abort, do so */
if (orted_globals.abort) {
abort();
}
/* otherwise, return with non-zero status */
ret = ORTE_ERROR_DEFAULT_EXIT_CODE;
goto DONE;
}
}
}
/* detach from controlling terminal
* otherwise, remain attached so output can get to us
*/
if(!orte_debug_flag &&
!orte_debug_daemons_flag &&
orted_globals.daemonize) {
opal_daemon_init(NULL);
}
/* insert our contact info into our process_info struct so we
* have it for later use and set the local daemon field to our name
*/
orte_process_info.my_daemon_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_DAEMON->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_DAEMON->vpid = ORTE_PROC_MY_NAME->vpid;
/* if I am also the hnp, then update that contact info field too */
if (ORTE_PROC_IS_HNP) {
orte_process_info.my_hnp_uri = orte_rml.get_contact_info();
ORTE_PROC_MY_HNP->jobid = ORTE_PROC_MY_NAME->jobid;
ORTE_PROC_MY_HNP->vpid = ORTE_PROC_MY_NAME->vpid;
}
/* setup the primary daemon command receive function */
ret = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, ORTE_RML_TAG_DAEMON,
ORTE_RML_PERSISTENT, orte_daemon_recv, NULL);
if (ret != ORTE_SUCCESS && ret != ORTE_ERR_NOT_IMPLEMENTED) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* output a message indicating we are alive, our name, and our pid
* for debugging purposes
*/
if (orte_debug_daemons_flag) {
fprintf(stderr, "Daemon %s checking in as pid %ld on host %s\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (long)orte_process_info.pid,
orte_process_info.nodename);
}
/* We actually do *not* want the orted to voluntarily yield() the
processor more than necessary. The orted already blocks when
it is doing nothing, so it doesn't use any more CPU cycles than
it should; but when it *is* doing something, we do not want it
to be unnecessarily delayed because it voluntarily yielded the
processor in the middle of its work.
For example: when a message arrives at the orted, we want the
OS to wake up the orted in a timely fashion (which most OS's
seem good about doing) and then we want the orted to process
the message as fast as possible. If the orted yields and lets
aggressive MPI applications get the processor back, it may be a
long time before the OS schedules the orted to run again
(particularly if there is no IO event to wake it up). Hence,
routed OOB messages (for example) may be significantly delayed
before being delivered to MPI processes, which can be
problematic in some scenarios (e.g., COMM_SPAWN, BTL's that
require OOB messages for wireup, etc.). */
opal_progress_set_yield_when_idle(false);
/* Change the default behavior of libevent such that we want to
continually block rather than blocking for the default timeout
and then looping around the progress engine again. There
should be nothing in the orted that cannot block in libevent
until "something" happens (i.e., there's no need to keep
cycling through progress because the only things that should
happen will happen in libevent). This is a minor optimization,
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, obtain and report a new process name and my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *proc;
orte_node_t *node;
orte_app_context_t *app;
char *tmp, *nptr, *sysinfo;
int32_t ljob;
/* setup the singleton's job */
jdata = OBJ_NEW(orte_job_t);
orte_plm_base_create_jobid(jdata);
ljob = ORTE_LOCAL_JOBID(jdata->jobid);
opal_pointer_array_set_item(orte_job_data, ljob, jdata);
/* must create a map for it (even though it has no
* info in it) so that the job info will be picked
* up in subsequent pidmaps or other daemons won't
* know how to route
*/
jdata->map = OBJ_NEW(orte_job_map_t);
/* setup an app_context for the singleton */
app = OBJ_NEW(orte_app_context_t);
app->app = strdup("singleton");
app->num_procs = 1;
opal_pointer_array_add(jdata->apps, app);
#if 0
/* run our local allocator to read the available
* allocation in case this singleton decides to
* comm_spawn other procs
*/
if (ORTE_SUCCESS != (ret = orte_ras.allocate(jdata))) {
ORTE_ERROR_LOG(ret);
/* don't quit as this would cause the singleton
* to hang!
*/
}
#endif
/* setup a proc object for the singleton - since we
* -must- be the HNP, and therefore we stored our
* node on the global node pool, and since the singleton
* -must- be on the same node as us, indicate that
*/
proc = OBJ_NEW(orte_proc_t);
proc->name.jobid = jdata->jobid;
proc->name.vpid = 0;
proc->alive = true;
proc->state = ORTE_PROC_STATE_RUNNING;
proc->app_idx = 0;
/* obviously, it is on my node */
node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, 0);
proc->node = node;
OBJ_RETAIN(node); /* keep accounting straight */
opal_pointer_array_add(jdata->procs, proc);
jdata->num_procs = 1;
/* and obviously it is one of my local procs */
OBJ_RETAIN(proc);
opal_pointer_array_add(orte_local_children, proc);
jdata->num_local_procs = 1;
/* set the trivial */
proc->local_rank = 0;
proc->node_rank = 0;
proc->app_rank = 0;
proc->state = ORTE_PROC_STATE_RUNNING;
proc->alive = true;
proc->app_idx = 0;
proc->local_proc = true;
#if OPAL_HAVE_HWLOC
proc->bind_idx = 0;
#endif
/* the singleton will use the first three collectives
* for its modex/barriers
*/
orte_grpcomm_base.coll_id += 3;
/* need to setup a pidmap for it */
jdata->pmap = (opal_byte_object_t*)malloc(sizeof(opal_byte_object_t));
if (ORTE_SUCCESS != (ret = orte_util_encode_pidmap(jdata->pmap))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* if we don't yet have a daemon map, then we have to generate one
* to pass back to it
*/
if (NULL == orte_odls_globals.dmap) {
orte_odls_globals.dmap = (opal_byte_object_t*)malloc(sizeof(opal_byte_object_t));
/* construct a nodemap */
if (ORTE_SUCCESS != (ret = orte_util_encode_nodemap(orte_odls_globals.dmap))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* create a string that contains our uri + the singleton's name + sysinfo */
orte_util_convert_process_name_to_string(&nptr, &proc->name);
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s][%s]", orte_process_info.my_daemon_uri, nptr, sysinfo);
free(nptr);
free(sysinfo);
/* pass that info to the singleton */
#ifndef __WINDOWS__
write(orted_globals.uri_pipe, tmp, strlen(tmp)+1); /* need to add 1 to get the NULL */
#else
send(orted_globals.uri_pipe, tmp, strlen(tmp)+1, 0); /* need to add 1 to get the NULL */
#endif
/* cleanup */
free(tmp);
}
/* if we were given a pipe to monitor for singleton termination, set that up */
if (orted_globals.singleton_died_pipe > 0) {
/* register shutdown handler */
pipe_handler = (opal_event_t*)malloc(sizeof(opal_event_t));
opal_event_set(orte_event_base, pipe_handler,
orted_globals.singleton_died_pipe,
OPAL_EV_READ,
pipe_closed,
pipe_handler);
opal_event_add(pipe_handler, NULL);
}
/* If I have a parent, then save his contact info so
* any messages we send can flow thru him.
*/
mca_base_param_reg_string_name("orte", "parent_uri",
"URI for the parent if tree launch is enabled.",
true, false, NULL, &rml_uri);
if (NULL != rml_uri) {
orte_process_name_t parent;
/* set the contact info into the hash table */
if (ORTE_SUCCESS != (ret = orte_rml.set_contact_info(rml_uri))) {
ORTE_ERROR_LOG(ret);
free(rml_uri);
goto DONE;
}
ret = orte_rml_base_parse_uris(rml_uri, &parent, NULL );
if( ORTE_SUCCESS != ret ) {
ORTE_ERROR_LOG(ret);
free(rml_uri);
goto DONE;
}
free(rml_uri);
/* tell the routed module that we have a path
* back to the HNP
*/
if (ORTE_SUCCESS != (ret = orte_routed.update_route(ORTE_PROC_MY_HNP, &parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
/* set the lifeline to point to our parent so that we
* can handle the situation if that lifeline goes away
*/
if (ORTE_SUCCESS != (ret = orte_routed.set_lifeline(&parent))) {
ORTE_ERROR_LOG(ret);
goto DONE;
}
}
/* if we are not the HNP...the only time we will be an HNP
* is if we are launched by a singleton to provide support
* for it
*/
if (!ORTE_PROC_IS_HNP) {
/* send the information to the orted report-back point - this function
* will process the data, but also counts the number of
* orteds that reported back so the launch procedure can continue.
* We need to do this at the last possible second as the HNP
* can turn right around and begin issuing orders to us
*/
buffer = OBJ_NEW(opal_buffer_t);
/* insert our name for rollup purposes */
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, ORTE_PROC_MY_NAME, 1, ORTE_NAME))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* for now, always include our contact info, even if we are using
* static ports. Eventually, this will be removed
*/
rml_uri = orte_rml.get_contact_info();
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &rml_uri, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
/* include our node name */
opal_dss.pack(buffer, &orte_process_info.nodename, 1, OPAL_STRING);
#if OPAL_HAVE_HWLOC
/* add the local topology */
if (NULL != opal_hwloc_topology &&
(1 == ORTE_PROC_MY_NAME->vpid || orte_hetero_nodes)) {
if (ORTE_SUCCESS != (ret = opal_dss.pack(buffer, &opal_hwloc_topology, 1, OPAL_HWLOC_TOPO))) {
ORTE_ERROR_LOG(ret);
}
}
#endif
if ((orte_static_ports || orte_use_common_port) && !orted_globals.tree_spawn) {
/* use the rollup collective to send our data to the HNP
* so we minimize the HNP bottleneck
*/
orte_grpcomm_collective_t *coll;
coll = OBJ_NEW(orte_grpcomm_collective_t);
/* get the list of contributors we need from the routed module */
orte_routed.get_routing_list(ORTE_GRPCOMM_COLL_PEERS, coll);
/* add the collective to our list */
opal_list_append(&orte_grpcomm_base.active_colls, &coll->super);
/* send the buffer to ourselves to start the collective */
if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_NAME, buffer,
ORTE_RML_TAG_ROLLUP, 0,
rml_cbfunc, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
} else {
/* send directly to the HNP's callback */
if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, buffer,
ORTE_RML_TAG_ORTED_CALLBACK, 0,
rml_cbfunc, NULL))) {
ORTE_ERROR_LOG(ret);
OBJ_RELEASE(buffer);
goto DONE;
}
}
}
if (orte_debug_daemons_flag) {
opal_output(0, "%s orted: up and running - waiting for commands!", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
}
/* loop the event lib until an exit event is detected */
while (orte_event_base_active) {
opal_event_loop(orte_event_base, OPAL_EVLOOP_ONCE);
}
/* ensure all local procs are dead */
orte_odls.kill_local_procs(NULL);
DONE:
/* update the exit status, in case it wasn't done */
ORTE_UPDATE_EXIT_STATUS(orte_exit_status);
/* cleanup and leave */
orte_finalize();
if (orte_debug_flag) {
fprintf(stderr, "exiting with status %d\n", orte_exit_status);
}
exit(orte_exit_status);
}
int ompi_request_init(void)
{
OBJ_CONSTRUCT(&ompi_request_lock, opal_mutex_t);
OBJ_CONSTRUCT(&ompi_request_cond, opal_condition_t);
OBJ_CONSTRUCT(&ompi_request_null, ompi_request_t);
OBJ_CONSTRUCT(&ompi_request_f_to_c_table, opal_pointer_array_t);
if( OPAL_SUCCESS != opal_pointer_array_init(&ompi_request_f_to_c_table,
0, OMPI_FORTRAN_HANDLE_MAX, 64) ) {
return OMPI_ERROR;
}
ompi_request_null.request.req_type = OMPI_REQUEST_NULL;
ompi_request_null.request.req_status.MPI_SOURCE = MPI_ANY_SOURCE;
ompi_request_null.request.req_status.MPI_TAG = MPI_ANY_TAG;
ompi_request_null.request.req_status.MPI_ERROR = MPI_SUCCESS;
ompi_request_null.request.req_status._ucount = 0;
ompi_request_null.request.req_status._cancelled = 0;
ompi_request_null.request.req_complete = true;
ompi_request_null.request.req_state = OMPI_REQUEST_INACTIVE;
ompi_request_null.request.req_persistent = false;
ompi_request_null.request.req_f_to_c_index =
opal_pointer_array_add(&ompi_request_f_to_c_table, &ompi_request_null);
ompi_request_null.request.req_free = ompi_request_null_free;
ompi_request_null.request.req_cancel = ompi_request_null_cancel;
ompi_request_null.request.req_mpi_object.comm = &ompi_mpi_comm_world.comm;
if (0 != ompi_request_null.request.req_f_to_c_index) {
return OMPI_ERR_REQUEST;
}
/* We need a way to distinguish between the user provided
* MPI_REQUEST_NULL to MPI_Wait* and a non-active (MPI_PROC_NULL)
* request passed to any P2P non-blocking function.
*
* The main difference to ompi_request_null is
* req_state being OMPI_REQUEST_ACTIVE, so that MPI_Waitall
* does not set the status to ompi_status_empty and the different
* req_free function, which resets the
* request to MPI_REQUEST_NULL.
* The req_cancel function need not be changed.
*/
OBJ_CONSTRUCT(&ompi_request_empty, ompi_request_t);
ompi_request_empty.req_type = OMPI_REQUEST_NULL;
ompi_request_empty.req_status.MPI_SOURCE = MPI_PROC_NULL;
ompi_request_empty.req_status.MPI_TAG = MPI_ANY_TAG;
ompi_request_empty.req_status.MPI_ERROR = MPI_SUCCESS;
ompi_request_empty.req_status._ucount = 0;
ompi_request_empty.req_status._cancelled = 0;
ompi_request_empty.req_complete = true;
ompi_request_empty.req_state = OMPI_REQUEST_ACTIVE;
ompi_request_empty.req_persistent = false;
ompi_request_empty.req_f_to_c_index =
opal_pointer_array_add(&ompi_request_f_to_c_table, &ompi_request_empty);
ompi_request_empty.req_free = ompi_request_empty_free;
ompi_request_empty.req_cancel = ompi_request_null_cancel;
ompi_request_empty.req_mpi_object.comm = &ompi_mpi_comm_world.comm;
if (1 != ompi_request_empty.req_f_to_c_index) {
return OMPI_ERR_REQUEST;
}
ompi_status_empty.MPI_SOURCE = MPI_ANY_SOURCE;
ompi_status_empty.MPI_TAG = MPI_ANY_TAG;
ompi_status_empty.MPI_ERROR = MPI_SUCCESS;
ompi_status_empty._ucount = 0;
ompi_status_empty._cancelled = 0;
return OMPI_SUCCESS;
}
int main(int argc, char *argv[])
{
int ret = 0;
bool want_help = false;
bool cmd_error = false;
bool acted = false;
bool want_all = false;
char **app_env = NULL, **global_env = NULL;
int i, len;
char *str;
/* Initialize the argv parsing handle */
if (OMPI_SUCCESS != opal_init_util(&argc, &argv)) {
orte_show_help("help-ompi_info.txt", "lib-call-fail", true,
"opal_init_util", __FILE__, __LINE__, NULL);
exit(ret);
}
ompi_info_cmd_line = OBJ_NEW(opal_cmd_line_t);
if (NULL == ompi_info_cmd_line) {
ret = errno;
orte_show_help("help-ompi_info.txt", "lib-call-fail", true,
"opal_cmd_line_create", __FILE__, __LINE__, NULL);
opal_finalize_util();
exit(ret);
}
opal_cmd_line_make_opt3(ompi_info_cmd_line, 'v', NULL, "version", 2,
"Show version of Open MPI or a component. The first parameter can be the keywords \"ompi\" or \"all\", a framework name (indicating all components in a framework), or a framework:component string (indicating a specific component). The second parameter can be one of: full, major, minor, release, greek, svn.");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "param", 2,
"Show MCA parameters. The first parameter is the framework (or the keyword \"all\"); the second parameter is the specific component name (or the keyword \"all\").");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "internal", 0,
"Show internal MCA parameters (not meant to be modified by users)");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "path", 1,
"Show paths that Open MPI was configured with. Accepts the following parameters: prefix, bindir, libdir, incdir, mandir, pkglibdir, sysconfdir");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "arch", 0,
"Show architecture Open MPI was compiled on");
opal_cmd_line_make_opt3(ompi_info_cmd_line, 'c', NULL, "config", 0,
"Show configuration options");
opal_cmd_line_make_opt3(ompi_info_cmd_line, 'h', NULL, "help", 0,
"Show this help message");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "ompi_info_pretty", 0,
"When used in conjunction with other parameters, the output is displayed in 'ompi_info_prettyprint' format (default)");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "parsable", 0,
"When used in conjunction with other parameters, the output is displayed in a machine-parsable format");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "parseable", 0,
"Synonym for --parsable");
opal_cmd_line_make_opt3(ompi_info_cmd_line, '\0', NULL, "hostname", 0,
"Show the hostname that Open MPI was configured "
"and built on");
opal_cmd_line_make_opt3(ompi_info_cmd_line, 'a', NULL, "all", 0,
"Show all configuration options and MCA parameters");
/* Call some useless functions in order to guarantee to link in some
* global variables. Only check the return value so that the
* compiler doesn't optimize out the useless function.
*/
if (OMPI_SUCCESS != ompi_comm_link_function()) {
/* Stop .. or I'll say stop again! */
++ret;
} else {
--ret;
}
/* set our threading level */
opal_set_using_threads(false);
/* Get MCA parameters, if any */
if( OMPI_SUCCESS != mca_base_open() ) {
orte_show_help("help-ompi_info.txt", "lib-call-fail", true, "mca_base_open", __FILE__, __LINE__ );
OBJ_RELEASE(ompi_info_cmd_line);
opal_finalize_util();
exit(1);
}
mca_base_cmd_line_setup(ompi_info_cmd_line);
/* Do the parsing */
if (OMPI_SUCCESS != opal_cmd_line_parse(ompi_info_cmd_line, false, argc, argv)) {
cmd_error = true;
}
if (!cmd_error &&
(opal_cmd_line_is_taken(ompi_info_cmd_line, "help") ||
opal_cmd_line_is_taken(ompi_info_cmd_line, "h"))) {
want_help = true;
}
if (cmd_error || want_help) {
char *usage = opal_cmd_line_get_usage_msg(ompi_info_cmd_line);
orte_show_help("help-ompi_info.txt", "usage", true, usage);
free(usage);
mca_base_close();
OBJ_RELEASE(ompi_info_cmd_line);
opal_finalize_util();
exit(cmd_error ? 1 : 0);
}
mca_base_cmd_line_process_args(ompi_info_cmd_line, &app_env, &global_env);
/* putenv() all the stuff that we got back from env (in case the
* user specified some --mca params on the command line). This
* creates a memory leak, but that's unfortunately how putenv()
* works. :-(
*/
len = opal_argv_count(app_env);
for (i = 0; i < len; ++i) {
putenv(app_env[i]);
}
len = opal_argv_count(global_env);
for (i = 0; i < len; ++i) {
putenv(global_env[i]);
}
/* setup the mca_types array */
OBJ_CONSTRUCT(&mca_types, opal_pointer_array_t);
opal_pointer_array_init(&mca_types, 256, INT_MAX, 128);
opal_pointer_array_add(&mca_types, "mca");
opal_pointer_array_add(&mca_types, "mpi");
opal_pointer_array_add(&mca_types, "orte");
opal_pointer_array_add(&mca_types, "opal");
opal_pointer_array_add(&mca_types, "filter");
opal_pointer_array_add(&mca_types, "backtrace");
opal_pointer_array_add(&mca_types, "memchecker");
opal_pointer_array_add(&mca_types, "memory");
opal_pointer_array_add(&mca_types, "paffinity");
opal_pointer_array_add(&mca_types, "carto");
opal_pointer_array_add(&mca_types, "shmem");
opal_pointer_array_add(&mca_types, "maffinity");
opal_pointer_array_add(&mca_types, "timer");
opal_pointer_array_add(&mca_types, "installdirs");
opal_pointer_array_add(&mca_types, "sysinfo");
opal_pointer_array_add(&mca_types, "hwloc");
#if OPAL_ENABLE_FT_CR == 1
opal_pointer_array_add(&mca_types, "crs");
#endif
opal_pointer_array_add(&mca_types, "dpm");
opal_pointer_array_add(&mca_types, "pubsub");
opal_pointer_array_add(&mca_types, "allocator");
opal_pointer_array_add(&mca_types, "coll");
opal_pointer_array_add(&mca_types, "io");
opal_pointer_array_add(&mca_types, "mpool");
opal_pointer_array_add(&mca_types, "pml");
opal_pointer_array_add(&mca_types, "bml");
opal_pointer_array_add(&mca_types, "rcache");
opal_pointer_array_add(&mca_types, "btl");
opal_pointer_array_add(&mca_types, "mtl");
opal_pointer_array_add(&mca_types, "topo");
opal_pointer_array_add(&mca_types, "osc");
opal_pointer_array_add(&mca_types, "op");
opal_pointer_array_add(&mca_types, "common");
#if OPAL_ENABLE_FT_CR == 1
opal_pointer_array_add(&mca_types, "crcp");
#endif
#if !ORTE_DISABLE_FULL_SUPPORT
opal_pointer_array_add(&mca_types, "iof");
opal_pointer_array_add(&mca_types, "oob");
opal_pointer_array_add(&mca_types, "odls");
opal_pointer_array_add(&mca_types, "ras");
opal_pointer_array_add(&mca_types, "rmaps");
opal_pointer_array_add(&mca_types, "rml");
opal_pointer_array_add(&mca_types, "routed");
opal_pointer_array_add(&mca_types, "plm");
#if OPAL_ENABLE_FT_CR == 1
opal_pointer_array_add(&mca_types, "snapc");
#endif
opal_pointer_array_add(&mca_types, "filem");
#endif
/* these are always included */
opal_pointer_array_add(&mca_types, "errmgr");
opal_pointer_array_add(&mca_types, "ess");
opal_pointer_array_add(&mca_types, "grpcomm");
opal_pointer_array_add(&mca_types, "notifier");
/* Execute the desired action(s) */
if (opal_cmd_line_is_taken(ompi_info_cmd_line, "ompi_info_pretty")) {
ompi_info_pretty = true;
} else if (opal_cmd_line_is_taken(ompi_info_cmd_line, "parsable") || opal_cmd_line_is_taken(ompi_info_cmd_line, "parseable")) {
ompi_info_pretty = false;
}
want_all = opal_cmd_line_is_taken(ompi_info_cmd_line, "all");
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "version")) {
ompi_info_do_version(want_all, ompi_info_cmd_line);
acted = true;
}
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "path")) {
ompi_info_do_path(want_all, ompi_info_cmd_line);
acted = true;
}
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "arch")) {
ompi_info_do_arch();
acted = true;
}
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "hostname")) {
ompi_info_do_hostname();
acted = true;
}
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "config")) {
ompi_info_do_config(true);
acted = true;
}
if (want_all || opal_cmd_line_is_taken(ompi_info_cmd_line, "param")) {
ompi_info_do_params(want_all, opal_cmd_line_is_taken(ompi_info_cmd_line, "internal"));
acted = true;
}
/* If no command line args are specified, show default set */
if (!acted) {
ompi_info_show_ompi_version(ompi_info_ver_full);
ompi_info_show_path(ompi_info_path_prefix, opal_install_dirs.prefix);
ompi_info_do_arch();
ompi_info_do_hostname();
ompi_info_do_config(false);
ompi_info_open_components();
for (i = 0; i < mca_types.size; ++i) {
if (NULL == (str = (char*)opal_pointer_array_get_item(&mca_types, i))) {
continue;
}
if (0 != strcmp("mpi", str)) {
ompi_info_show_component_version(str, ompi_info_component_all,
ompi_info_ver_full, ompi_info_type_all);
}
}
}
/* All done */
if (NULL != app_env) {
opal_argv_free(app_env);
}
if (NULL != global_env) {
opal_argv_free(global_env);
}
ompi_info_close_components();
OBJ_RELEASE(ompi_info_cmd_line);
OBJ_DESTRUCT(&mca_types);
mca_base_close();
opal_finalize_util();
return 0;
}
