int mca_common_pmi_id(char **pmi_id_ret, char **error){ char *pmi_id = NULL; int rc; // Default values *pmi_id_ret = pmi_id; *error = NULL; #if WANT_PMI2_SUPPORT if( mca_common_pmi_version == 2 ){ // TODO: add proper error handling pmi_id = (char*)malloc(PMI2_MAX_VALLEN); if( pmi_id == NULL ){ *error = "mca_common_pmi_id: could not get memory for PMIv2 ID"; return OPAL_ERR_OUT_OF_RESOURCE; } strncpy(pmi_id, pmi_kvs_name, pmi_kvslen_max); } else #endif { int pmi_maxlen; /* get our PMI id length */ if (PMI_SUCCESS != (rc = PMI_Get_id_length_max(&pmi_maxlen))) { *error = "PMI_Get_id_length_max"; return OPAL_ERROR; } // TODO: add proper error handling pmi_id = (char*)malloc(pmi_maxlen); if( pmi_id == NULL ){ *error = "mca_common_pmi_id: could not get memory for PMIv1 ID"; return OPAL_ERR_OUT_OF_RESOURCE; } /* Get domain id */ if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(pmi_id, pmi_maxlen))) { free(pmi_id); *error = "PMI_Get_kvs_domain_id"; return OPAL_ERROR; } } *pmi_id_ret = pmi_id; return OPAL_SUCCESS; }
static int s1_init(void) { PMI_BOOL initialized; int spawned; int rc, ret = OPAL_ERROR; int i, rank, lrank, nrank; char *pmix_id, tmp[64]; opal_value_t kv; char *str; uint32_t ui32; opal_process_name_t ldr; char **localranks=NULL; if (PMI_SUCCESS != (rc = PMI_Initialized(&initialized))) { OPAL_PMI_ERROR(rc, "PMI_Initialized"); return OPAL_ERROR; } if (PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned))) { OPAL_PMI_ERROR(rc, "PMI_Init"); return OPAL_ERROR; } // setup hash table opal_pmix_base_hash_init(); // Initialize space demands rc = PMI_KVS_Get_value_length_max(&pmix_vallen_max); if (PMI_SUCCESS != rc) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_value_length_max"); goto err_exit; } pmix_vallen_threshold = pmix_vallen_max * 3; pmix_vallen_threshold >>= 2; rc = PMI_KVS_Get_name_length_max(&pmix_kvslen_max); if (PMI_SUCCESS != rc) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_name_length_max"); goto err_exit; } rc = PMI_KVS_Get_key_length_max(&pmix_keylen_max); if (PMI_SUCCESS != rc) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_key_length_max"); goto err_exit; } // Initialize job environment information pmix_id = (char*)malloc(pmix_vallen_max); if (pmix_id == NULL) { ret = OPAL_ERR_OUT_OF_RESOURCE; goto err_exit; } /* Get domain id */ if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(pmix_id, pmix_vallen_max))) { free(pmix_id); goto err_exit; } /* get our rank */ ret = PMI_Get_rank(&rank); if( PMI_SUCCESS != ret ) { OPAL_PMI_ERROR(ret, "PMI_Get_rank"); free(pmix_id); goto err_exit; } /* Slurm PMI provides the job id as an integer followed * by a '.', followed by essentially a stepid. The first integer * defines an overall job number. The second integer is the number of * individual jobs we have run within that allocation. */ s1_pname.jobid = strtoul(pmix_id, &str, 10); s1_pname.jobid = (s1_pname.jobid << 16) & 0xffff0000; if (NULL != str) { ui32 = strtoul(str, NULL, 10); s1_pname.jobid |= (ui32 & 0x0000ffff); } ldr.jobid = s1_pname.jobid; s1_pname.vpid = rank; /* store our name in the opal_proc_t so that * debug messages will make sense - an upper * layer will eventually overwrite it, but that * won't do any harm */ opal_proc_set_name(&s1_pname); opal_output_verbose(2, opal_pmix_base_framework.framework_output, "%s pmix:s1: assigned tmp name", OPAL_NAME_PRINT(s1_pname)); OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_JOBID); kv.type = OPAL_UINT32; kv.data.uint32 = s1_pname.jobid; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* save it */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_RANK); kv.type = OPAL_UINT32; kv.data.uint32 = rank; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); pmix_kvs_name = (char*)malloc(pmix_kvslen_max); if (pmix_kvs_name == NULL) { ret = OPAL_ERR_OUT_OF_RESOURCE; goto err_exit; } rc = PMI_KVS_Get_my_name(pmix_kvs_name, pmix_kvslen_max); if (PMI_SUCCESS != rc) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_my_name"); goto err_exit; } /* get our local proc info to find our local rank */ if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&nlranks))) { OPAL_PMI_ERROR(rc, "PMI_Get_clique_size"); return rc; } /* save the local size */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_LOCAL_SIZE); kv.type = OPAL_UINT32; kv.data.uint32 = nlranks; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); lrank = 0; nrank = 0; ldr.vpid = rank; if (0 < nlranks) { /* now get the specific ranks */ lranks = (int*)calloc(nlranks, sizeof(int)); if (NULL == lranks) { rc = OPAL_ERR_OUT_OF_RESOURCE; OPAL_ERROR_LOG(rc); return rc; } if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(lranks, nlranks))) { OPAL_PMI_ERROR(rc, "PMI_Get_clique_ranks"); free(lranks); return rc; } /* note the local ldr */ ldr.vpid = lranks[0]; /* save this */ memset(tmp, 0, 64); for (i=0; i < nlranks; i++) { (void)snprintf(tmp, 64, "%d", lranks[i]); opal_argv_append_nosize(&localranks, tmp); if (rank == lranks[i]) { lrank = i; nrank = i; } } str = opal_argv_join(localranks, ','); opal_argv_free(localranks); OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_LOCAL_PEERS); kv.type = OPAL_STRING; kv.data.string = str; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); } /* save the local leader */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_LOCALLDR); kv.type = OPAL_UINT64; kv.data.uint64 = *(uint64_t*)&ldr; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* save our local rank */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_LOCAL_RANK); kv.type = OPAL_UINT16; kv.data.uint16 = lrank; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* and our node rank */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_NODE_RANK); kv.type = OPAL_UINT16; kv.data.uint16 = nrank; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* get universe size */ ret = PMI_Get_universe_size(&i); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_universe_size"); goto err_exit; } /* push this into the dstore for subsequent fetches */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_UNIV_SIZE); kv.type = OPAL_UINT32; kv.data.uint32 = i; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* get job size */ ret = PMI_Get_size(&i); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_size"); goto err_exit; } OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_JOB_SIZE); kv.type = OPAL_UINT32; kv.data.uint32 = i; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* get appnum */ ret = PMI_Get_appnum(&i); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_appnum"); goto err_exit; } OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_PMIX_APPNUM); kv.type = OPAL_UINT32; kv.data.uint32 = i; if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* increment the init count */ ++pmix_init_count; return OPAL_SUCCESS; err_exit: PMI_Finalize(); return ret; }
static int test_item1(void) { int rc = 0; int val = 0; log_assert(spawned == PMI_FALSE || spawned == PMI_TRUE, ""); if (PMI_SUCCESS != (rc = PMI_Get_size(&size))) { log_fatal("PMI_Get_Size failed: %d\n", rc); return rc; } log_assert(size >= 0, ""); if (PMI_SUCCESS != (rc = PMI_Get_rank(&rank))) { log_fatal("PMI_Get_Rank failed: %d\n", rc); return rc; } log_assert(rank >= 0, ""); log_assert(rank < size, ""); if (PMI_SUCCESS != (rc = PMI_Get_appnum(&appnum))) { log_fatal("PMI_Get_appnum failed: %d\n", rc); return rc; } log_info("spawned=%d size=%d rank=%d appnum=%d\n", spawned, size, rank, appnum); val = random_value(10, 100); if (PMI_SUCCESS != (rc = PMI_Get_universe_size(&val))) { log_fatal("PMI_Get_universe_size failed: %d\n", rc); return rc; } log_assert(size == val, ""); val = random_value(10, 100); if (PMI_SUCCESS != (rc = PMI_Get_id_length_max(&val))) { log_fatal("PMI_Get_id_length_max failed: %d\n", rc); return rc; } log_info("PMI_Get_id_length_max=%d\n", val); if (!_legacy) { log_assert(sizeof(jobid) == val, "Check PMIX_MAX_NSLEN value in pmix_common.h"); } sprintf(jobid, "%s", __func__); if (PMI_SUCCESS != (rc = PMI_Get_id(jobid, sizeof(jobid)))) { log_fatal("PMI_Get_id failed: %d\n", rc); return rc; } log_info("jobid=%s\n", jobid); log_assert(memcmp(jobid, __func__, sizeof(__func__)), ""); sprintf(jobid, "%s", __func__); if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(jobid, sizeof(jobid)))) { log_fatal("PMI_Get_kvs_domain_id failed: %d\n", rc); return rc; } log_info("PMI_Get_kvs_domain_id=%s\n", jobid); log_assert(memcmp(jobid, __func__, sizeof(__func__)), ""); sprintf(jobid, "%s", __func__); if (PMI_SUCCESS != (rc = PMI_KVS_Get_my_name(jobid, sizeof(jobid)))) { log_fatal("PMI_KVS_Get_my_name failed: %d\n", rc); return rc; } log_info("PMI_KVS_Get_my_name=%s\n", jobid); log_assert(memcmp(jobid, __func__, sizeof(__func__)), ""); return rc; }
static int s1_init(void) { PMI_BOOL initialized; int spawned; int rc, ret = OPAL_ERROR; int i; char *pmix_id, *tmp; uint32_t jobfam, stepid; opal_value_t kv; if (PMI_SUCCESS != (rc = PMI_Initialized(&initialized))) { OPAL_PMI_ERROR(rc, "PMI_Initialized"); return OPAL_ERROR; } if( PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned)) ) { OPAL_PMI_ERROR(rc, "PMI_Init"); return OPAL_ERROR; } // Initialize space demands rc = PMI_KVS_Get_value_length_max(&pmix_vallen_max); if( PMI_SUCCESS != rc ) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_value_length_max"); goto err_exit; } rc = PMI_KVS_Get_name_length_max(&pmix_kvslen_max); if (PMI_SUCCESS != rc ) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_name_length_max"); goto err_exit; } rc = PMI_KVS_Get_key_length_max(&pmix_keylen_max); if( PMI_SUCCESS != rc ) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_key_length_max"); goto err_exit; } // Initialize job environment information pmix_id = (char*)malloc(pmix_vallen_max); if( pmix_id == NULL ){ ret = OPAL_ERR_OUT_OF_RESOURCE; goto err_exit; } /* Get domain id */ if (PMI_SUCCESS != (rc = PMI_Get_kvs_domain_id(pmix_id, pmix_vallen_max))) { free(pmix_id); goto err_exit; } /* Slurm PMI provides the job id as an integer followed * by a '.', followed by essentially a stepid. The first integer * defines an overall job number. The second integer is the number of * individual jobs we have run within that allocation. So we translate * this as the overall job number equating to our job family, and * the individual number equating to our local jobid */ jobfam = strtoul(pmix_id, &tmp, 10); if (NULL == tmp) { /* hmmm - no '.', so let's just use zero */ stepid = 0; } else { tmp++; /* step over the '.' */ stepid = strtoul(tmp, NULL, 10); } /* now build the jobid */ s1_jobid = (jobfam << 16) | stepid; free(pmix_id); /* get our rank */ ret = PMI_Get_rank(&s1_rank); if( PMI_SUCCESS != ret ) { OPAL_PMI_ERROR(ret, "PMI_Get_rank"); goto err_exit; } /* store our name in the opal_proc_t so that * debug messages will make sense - an upper * layer will eventually overwrite it, but that * won't do any harm */ s1_pname.jid = s1_jobid; s1_pname.vid = s1_rank; opal_proc_set_name((opal_process_name_t*)&s1_pname); opal_output_verbose(2, opal_pmix_base_framework.framework_output, "%s pmix:s1: assigned tmp name", OPAL_NAME_PRINT(*(opal_process_name_t*)&s1_pname)); pmix_kvs_name = (char*)malloc(pmix_kvslen_max); if( pmix_kvs_name == NULL ){ ret = OPAL_ERR_OUT_OF_RESOURCE; goto err_exit; } rc = PMI_KVS_Get_my_name(pmix_kvs_name, pmix_kvslen_max); if( PMI_SUCCESS != rc ) { OPAL_PMI_ERROR(rc, "PMI_KVS_Get_my_name"); goto err_exit; } /* get our local proc info to find our local rank */ if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&s1_nlranks))) { OPAL_PMI_ERROR(rc, "PMI_Get_clique_size"); return rc; } /* now get the specific ranks */ s1_lranks = (int*)calloc(s1_nlranks, sizeof(int)); if (NULL == s1_lranks) { rc = OPAL_ERR_OUT_OF_RESOURCE; OPAL_ERROR_LOG(rc); return rc; } if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(s1_lranks, s1_nlranks))) { OPAL_PMI_ERROR(rc, "PMI_Get_clique_ranks"); free(s1_lranks); return rc; } /* find ourselves */ for (i=0; i < s1_nlranks; i++) { if (s1_rank == s1_lranks[i]) { s1_lrank = i; s1_nrank = i; break; } } /* get universe size */ ret = PMI_Get_universe_size(&s1_usize); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_universe_size"); goto err_exit; } /* push this into the dstore for subsequent fetches */ OBJ_CONSTRUCT(&kv, opal_value_t); kv.key = strdup(OPAL_DSTORE_UNIV_SIZE); kv.type = OPAL_UINT32; kv.data.uint32 = s1_usize; if (OPAL_SUCCESS != (ret = opal_dstore.store(opal_dstore_internal, &OPAL_PROC_MY_NAME, &kv))) { OPAL_ERROR_LOG(ret); OBJ_DESTRUCT(&kv); goto err_exit; } OBJ_DESTRUCT(&kv); /* get job size */ ret = PMI_Get_size(&s1_jsize); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_size"); goto err_exit; } /* get appnum */ ret = PMI_Get_appnum(&s1_appnum); if (PMI_SUCCESS != ret) { OPAL_PMI_ERROR(ret, "PMI_Get_appnum"); goto err_exit; } return OPAL_SUCCESS; err_exit: PMI_Finalize(); return ret; }
static int rte_init(void) { int ret, i, j; char *error = NULL, *localj; int32_t jobfam, stepid; char *envar, *ev1, *ev2; uint64_t unique_key[2]; char *cs_env, *string_key; char *pmi_id=NULL; int *ranks; char *tmp; orte_jobid_t jobid; orte_process_name_t proc; orte_local_rank_t local_rank; orte_node_rank_t node_rank; /* run the prolog */ if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) { error = "orte_ess_base_std_prolog"; goto error; } #if OPAL_HAVE_HWLOC /* get the topology */ if (NULL == opal_hwloc_topology) { if (OPAL_SUCCESS != opal_hwloc_base_get_topology()) { error = "topology discovery"; goto error; } } #endif if (ORTE_PROC_IS_DAEMON) { /* I am a daemon, launched by mpirun */ /* we had to be given a jobid */ mca_base_param_reg_string_name("orte", "ess_jobid", "Process jobid", true, false, NULL, &tmp); if (NULL == tmp) { error = "missing jobid"; ret = ORTE_ERR_FATAL; goto error; } if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_jobid(&jobid, tmp))) { ORTE_ERROR_LOG(ret); error = "convert jobid"; goto error; } free(tmp); ORTE_PROC_MY_NAME->jobid = jobid; /* get our rank from PMI */ if (PMI_SUCCESS != (ret = PMI_Get_rank(&i))) { ORTE_PMI_ERROR(ret, "PMI_Get_rank"); error = "could not get PMI rank"; goto error; } ORTE_PROC_MY_NAME->vpid = i + 1; /* compensate for orterun */ /* get the number of procs from PMI */ if (PMI_SUCCESS != (ret = PMI_Get_universe_size(&i))) { ORTE_PMI_ERROR(ret, "PMI_Get_universe_size"); error = "could not get PMI universe size"; goto error; } orte_process_info.num_procs = i + 1; /* compensate for orterun */ /* complete setup */ if (ORTE_SUCCESS != (ret = orte_ess_base_orted_setup(NULL))) { ORTE_ERROR_LOG(ret); error = "orte_ess_base_orted_setup"; goto error; } } else { /* we are a direct-launched MPI process */ /* get our PMI id length */ if (PMI_SUCCESS != (ret = PMI_Get_id_length_max(&pmi_maxlen))) { error = "PMI_Get_id_length_max"; goto error; } pmi_id = malloc(pmi_maxlen); if (PMI_SUCCESS != (ret = PMI_Get_kvs_domain_id(pmi_id, pmi_maxlen))) { free(pmi_id); error = "PMI_Get_kvs_domain_id"; goto error; } /* PMI is very nice to us - the domain id is an integer followed * by a '.', followed by essentially a stepid. The first integer * defines an overall job number. The second integer is the number of * individual jobs we have run within that allocation. So we translate * this as the overall job number equating to our job family, and * the individual number equating to our local jobid */ jobfam = strtol(pmi_id, &localj, 10); if (NULL == localj) { /* hmmm - no '.', so let's just use zero */ stepid = 0; } else { localj++; /* step over the '.' */ stepid = strtol(localj, NULL, 10) + 1; /* add one to avoid looking like a daemon */ } free(pmi_id); /* now build the jobid */ ORTE_PROC_MY_NAME->jobid = ORTE_CONSTRUCT_LOCAL_JOBID(jobfam << 16, stepid); /* get our rank */ if (PMI_SUCCESS != (ret = PMI_Get_rank(&i))) { ORTE_PMI_ERROR(ret, "PMI_Get_rank"); error = "could not get PMI rank"; goto error; } ORTE_PROC_MY_NAME->vpid = i; /* get the number of procs from PMI */ if (PMI_SUCCESS != (ret = PMI_Get_universe_size(&i))) { ORTE_PMI_ERROR(ret, "PMI_Get_universe_size"); error = "could not get PMI universe size"; goto error; } orte_process_info.num_procs = i; /* push into the environ for pickup in MPI layer for * MPI-3 required info key */ asprintf(&ev1, "OMPI_MCA_orte_ess_num_procs=%d", i); putenv(ev1); asprintf(&ev2, "OMPI_APP_CTX_NUM_PROCS=%d", i); putenv(ev2); /* setup transport keys in case the MPI layer needs them - * we can use the jobfam and stepid as unique keys * because they are unique values assigned by the RM */ unique_key[0] = (uint64_t)jobfam; unique_key[1] = (uint64_t)stepid; if (NULL == (string_key = orte_pre_condition_transports_print(unique_key))) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); return ORTE_ERR_OUT_OF_RESOURCE; } if (NULL == (cs_env = mca_base_param_environ_variable("orte_precondition_transports",NULL,NULL))) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); return ORTE_ERR_OUT_OF_RESOURCE; } asprintf(&envar, "%s=%s", cs_env, string_key); putenv(envar); /* cannot free the envar as that messes up our environ */ free(cs_env); free(string_key); /* our app_context number can only be 0 as we don't support * dynamic spawns */ orte_process_info.app_num = 0; /* setup my daemon's name - arbitrary, since we don't route * messages */ ORTE_PROC_MY_DAEMON->jobid = 0; ORTE_PROC_MY_DAEMON->vpid = 0; /* ensure we pick the correct critical components */ putenv("OMPI_MCA_grpcomm=pmi"); putenv("OMPI_MCA_routed=direct"); /* now use the default procedure to finish my setup */ if (ORTE_SUCCESS != (ret = orte_ess_base_app_setup())) { ORTE_ERROR_LOG(ret); error = "orte_ess_base_app_setup"; goto error; } /* store our info into the database */ if (ORTE_SUCCESS != (ret = orte_db.store(ORTE_PROC_MY_NAME, ORTE_DB_HOSTNAME, orte_process_info.nodename, OPAL_STRING))) { error = "db store daemon vpid"; goto error; } /* get our local proc info to find our local rank */ if (PMI_SUCCESS != (ret = PMI_Get_clique_size(&i))) { ORTE_PMI_ERROR(ret, "PMI_Get_clique_size"); error = "could not get PMI clique size"; goto error; } /* store that info - remember, we want the number of peers that * share the node WITH ME, so we have to subtract ourselves from * that number */ orte_process_info.num_local_peers = i - 1; /* now get the specific ranks */ ranks = (int*)malloc(i * sizeof(int)); if (PMI_SUCCESS != (ret = PMI_Get_clique_ranks(ranks, i))) { ORTE_PMI_ERROR(ret, "PMI_Get_clique_ranks"); error = "could not get clique ranks"; goto error; } /* The clique ranks are returned in rank order, so * cycle thru the array and update the local/node * rank info */ proc.jobid = ORTE_PROC_MY_NAME->jobid; for (j=0; j < i; j++) { proc.vpid = ranks[j]; local_rank = j; node_rank = j; if (ranks[j] == (int)ORTE_PROC_MY_NAME->vpid) { orte_process_info.my_local_rank = local_rank; orte_process_info.my_node_rank = node_rank; } if (ORTE_SUCCESS != (ret = orte_db.store(&proc, ORTE_DB_LOCALRANK, &local_rank, ORTE_LOCAL_RANK))) { error = "db store local rank"; goto error; } if (ORTE_SUCCESS != (ret = orte_db.store(&proc, ORTE_DB_NODERANK, &node_rank, ORTE_NODE_RANK))) { error = "db store node rank"; goto error; } } free(ranks); /* setup process binding */ if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) { error = "proc_binding"; goto error; } } /* set max procs */ if (orte_process_info.max_procs < orte_process_info.num_procs) { orte_process_info.max_procs = orte_process_info.num_procs; } /* flag that we completed init */ app_init_complete = true; return ORTE_SUCCESS; error: if (ORTE_ERR_SILENT != ret && !orte_report_silent_errors) { orte_show_help("help-orte-runtime.txt", "orte_init:startup:internal-failure", true, error, ORTE_ERROR_NAME(ret), ret); } return ret; }