static int cray_local_info(int vpid, int **ranks_ret,
int *procs_ret, char **error)
{
int *ranks;
int procs = -1;
int rc;
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
*error = "mca_common_pmix_local_info: could not get memory for PMIv2 process mapping";
return OPAL_ERR_OUT_OF_RESOURCE;
}
int found;
int my_node;
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI_SUCCESS != rc ) {
/* can't check PMI_SUCCESS as some folks (i.e., Cray) don't define it */
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
*error = "mca_common_pmix_local_info: could not get PMI_process_mapping";
return OPAL_ERROR;
}
ranks = pmix_cray_parse_pmap(pmapping, vpid, &my_node, &procs);
if (NULL == ranks) {
*error = "mca_common_pmix_local_info: could not get memory for PMIv2 local ranks";
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(pmapping);
*ranks_ret = ranks;
*procs_ret = procs;
return OPAL_SUCCESS;
}
static int test_item3(void)
{
int rc = 0;
char val[PMI2_MAX_VALLEN];
int found = 0;
/* Predefined Job attributes */
const char *tkeys[] = {
"universeSize",
"hasNameServ",
"physTopology",
"physTopologyLevels",
"cartDims",
"isHeterogeneous",
NULL
};
const char **ptr = tkeys;
while (*ptr) {
if (PMI2_SUCCESS != (rc = PMI2_Info_GetJobAttr(*ptr, val, sizeof(val), &found))) {
log_fatal("PMI2_Info_GetJobAttr: [%s] %d\n", *ptr, rc);
return rc;
}
log_info("key=%s value=%s found=%d\n", *ptr, (found ? val : "N/A"), found);
if (!_legacy) {
log_assert(!found, "Check test. Probably PMIx has a new functionality");
}
ptr++;
}
return rc;
}
void pmi_get_local_ranks(int **local_ranks, int *local_cnt)
{
int rc, found;
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
fprintf(stderr,"pmi_get_local_ranks (pmi2): cannot mallic for pmappig\n");
abort();
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI2_SUCCESS != rc ) {
fprintf(stderr,"pmi_get_local_ranks (pmi2): PMI2_Info_GetJobAttr rc = %d\n", rc);
abort();
}
*local_ranks = NULL;
*local_ranks = mca_common_pmi2_parse_pmap(pmapping, my_rank, &my_node, local_cnt);
if (NULL == *local_ranks) {
fprintf(stderr,"mca_common_pmi2_parse_pmap: error\n");
abort();
}
free(pmapping);
}
/*
* MPID_Get_universe_size - Get the universe size from the process manager
*
* Notes: The ch3 device requires that the PMI routines are used to
* communicate with the process manager. If a channel wishes to
* bypass the standard PMI implementations, it is the responsibility of the
* channel to provide an implementation of the PMI routines.
*/
int MPID_Get_universe_size(int * universe_size)
{
int mpi_errno = MPI_SUCCESS;
#ifdef USE_PMI2_API
char val[PMI2_MAX_VALLEN];
int found = 0;
char *endptr;
mpi_errno = PMI2_Info_GetJobAttr("universeSize", val, sizeof(val), &found);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (!found)
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
else {
*universe_size = strtol(val, &endptr, 0);
MPIR_ERR_CHKINTERNAL(endptr - val != strlen(val), mpi_errno, "can't parse universe size");
}
#else
int pmi_errno = PMI_SUCCESS;
pmi_errno = PMI_Get_universe_size(universe_size);
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_get_universe_size",
"**pmi_get_universe_size %d", pmi_errno);
}
if (*universe_size < 0)
{
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
}
#endif
fn_exit:
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*** MODEX SECTION ***/
static int modex(orte_grpcomm_collective_t *coll)
{
int *local_ranks, local_rank_count;
opal_hwloc_locality_t locality;
const char *cpuset;
orte_process_name_t name;
orte_vpid_t v;
bool local;
int rc, i;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_framework.framework_output,
"%s grpcomm:pmi: modex entered",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* discover the local ranks */
#if WANT_PMI2_SUPPORT
{
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
int found, sid, nodes, k;
orte_vpid_t n;
char *p;
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if (!found || PMI_SUCCESS != rc) { /* can't check PMI2_SUCCESS as some folks (i.e., Cray) don't define it */
opal_output(0, "%s could not get PMI_process_mapping (PMI2_Info_GetJobAttr() failed)",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_ERROR;
}
i = 0; n = 0; local_rank_count = 0;
if (NULL != (p = strstr(pmapping, "(vector"))) {
while (NULL != (p = strstr(p+1, ",("))) {
if (3 == sscanf(p, ",(%d,%d,%d)", &sid, &nodes, &local_rank_count)) {
for (k = 0; k < nodes; k++) {
if ((ORTE_PROC_MY_NAME->vpid >= n) &&
(ORTE_PROC_MY_NAME->vpid < (n + local_rank_count))) {
break;
}
n += local_rank_count;
}
}
}
}
free(pmapping);
if (local_rank_count > 0) {
local_ranks = (int*)malloc(local_rank_count * sizeof(int));
for (i=0; i < local_rank_count; i++) {
local_ranks[i] = n + i;
}
}
if (NULL == local_ranks) {
opal_output(0, "%s could not get PMI_process_mapping",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_ERROR;
}
}
#else
rc = PMI_Get_clique_size (&local_rank_count);
if (PMI_SUCCESS != rc) {
ORTE_ERROR_LOG(ORTE_ERROR);
return ORTE_ERROR;
}
local_ranks = calloc (local_rank_count, sizeof (int));
if (NULL == local_ranks) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
return ORTE_ERR_OUT_OF_RESOURCE;
}
rc = PMI_Get_clique_ranks (local_ranks, local_rank_count);
if (PMI_SUCCESS != rc) {
ORTE_ERROR_LOG(ORTE_ERROR);
return ORTE_ERROR;
}
#endif
/* our RTE data was constructed and pushed in the ESS pmi component */
/* commit our modex info */
opal_db.commit((opal_identifier_t *)ORTE_PROC_MY_NAME);
/* cycle thru all my peers and collect their RTE info */
name.jobid = ORTE_PROC_MY_NAME->jobid;
for (v=0; v < orte_process_info.num_procs; v++) {
if (v == ORTE_PROC_MY_NAME->vpid) {
continue;
}
name.vpid = v;
/* check if this is a local process */
for (i = 0, local = false ; i < local_rank_count ; ++i) {
if ((orte_vpid_t) local_ranks[i] == v) {
local = true;
break;
}
}
/* compute and store the locality as it isn't something that gets pushed to PMI - doing
* this here will prevent the MPI layer from fetching data for all ranks
*/
if (local) {
if (ORTE_SUCCESS != (rc = opal_db.fetch_pointer((opal_identifier_t*)&name, OPAL_DB_CPUSET,
(void **)&cpuset, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (NULL == cpuset) {
/* if we share a node, but we don't know anything more, then
* mark us as on the node as this is all we know
*/
locality = OPAL_PROC_ON_NODE;
} else {
/* determine relative location on our node */
locality = opal_hwloc_base_get_relative_locality(opal_hwloc_topology,
orte_process_info.cpuset,
(char *) cpuset);
}
} else {
/* this is on a different node, then mark as non-local */
locality = OPAL_PROC_NON_LOCAL;
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_framework.framework_output,
"%s grpcomm:pmi proc %s locality %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), opal_hwloc_base_print_locality(locality)));
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)&name, OPAL_SCOPE_INTERNAL,
OPAL_DB_LOCALITY, &locality, OPAL_HWLOC_LOCALITY_T))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* execute the callback */
coll->active = false;
if (NULL != coll->cbfunc) {
coll->cbfunc(NULL, coll->cbdata);
}
return rc;
}
int
main(int argc, char **argv)
{
int rank;
int size;
int appnum;
int spawned;
int flag;
int len;
int i;
struct timeval tv;
struct timeval tv2;
char jobid[128];
char key[128];
char val[128];
char buf[128];
{
int x = 1;
while (x == 0) {
sleep(2);
}
}
gettimeofday(&tv, NULL);
srand(tv.tv_sec);
PMI2_Init(&spawned, &size, &rank, &appnum);
PMI2_Job_GetId(jobid, sizeof(buf));
memset(val, 0, sizeof(val));
PMI2_Info_GetJobAttr("mpi_reserved_ports",
val,
PMI2_MAX_ATTRVALUE,
&flag);
sprintf(key, "mpi_reserved_ports");
PMI2_KVS_Put(key, val);
memset(val, 0, sizeof(val));
sprintf(buf, "PMI_netinfo_of_task");
PMI2_Info_GetJobAttr(buf,
val,
PMI2_MAX_ATTRVALUE,
&flag);
sprintf(key, buf);
PMI2_KVS_Put(key, val);
memset(val, 0, sizeof(val));
sprintf(key, "david@%d", rank);
sprintf(val, "%s", mrand(97, 122));
PMI2_KVS_Put(key, val);
PMI2_KVS_Fence();
for (i = 0; i < size; i++) {
memset(val, 0, sizeof(val));
sprintf(key, "PMI_netinfo_of_task");
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
memset(val, 0, sizeof(val));
sprintf(key, "david@%d", rank);
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
memset(val, 0, sizeof(val));
sprintf(key, "mpi_reserved_ports");
PMI2_KVS_Get(jobid,
PMI2_ID_NULL,
key,
val,
sizeof(val),
&len);
printf("rank: %d key:%s val:%s\n", rank, key, val);
}
PMI2_Finalize();
gettimeofday(&tv2, NULL);
printf("%f\n",
((tv2.tv_sec - tv.tv_sec) * 1000.0
+ (tv2.tv_usec - tv.tv_usec) / 1000.0));
return 0;
}
static int cray_init(void)
{
int i, spawned, size, rank, appnum, my_node;
int rc, ret = OPAL_ERROR;
char *pmapping = NULL;
char buf[PMI2_MAX_ATTRVALUE];
int found;
uint32_t jobfam;
++pmix_init_count;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
if (PMI_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
pmix_size = size;
pmix_rank = rank;
pmix_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
pmix_usize = atoi(buf);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
rc = sscanf(pmix_kvs_name,"kvs_%u",&jobfam);
if (rc != 1) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
rc = OPAL_ERROR;
goto err_exit;
}
pmix_jobid = jobfam << 16;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
pmix_pname.jid = pmix_jobid;
pmix_pname.vid = pmix_rank;
opal_proc_set_name((opal_process_name_t*)&pmix_pname);
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: assigned tmp name %d %d pmix_kvs_name %s",
OPAL_NAME_PRINT(*(opal_process_name_t*)&pmix_pname),pmix_pname.jid,pmix_pname.vid,pmix_kvs_name);
pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
pmix_lranks = pmix_cray_parse_pmap(pmapping, pmix_rank, &my_node, &pmix_nlranks);
if (NULL == pmix_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
/* find ourselves */
for (i=0; i < pmix_nlranks; i++) {
if (pmix_rank == pmix_lranks[i]) {
pmix_lrank = i;
pmix_nrank = my_node;
break;
}
}
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
static int cray_init(void)
{
int i, spawned, size, rank, appnum, my_node;
int rc, ret = OPAL_ERROR;
char *pmapping = NULL;
char buf[PMI2_MAX_ATTRVALUE];
int found;
int major, minor, revision;
uint32_t jobfam;
opal_value_t kv;
opal_process_name_t ldr;
char nmtmp[64];
char *str, **localranks = NULL;
++pmix_init_count;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: pmi already initialized",
OPAL_NAME_PRINT(pmix_pname));
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
if (PMI_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
pmix_size = size;
pmix_rank = rank;
pmix_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
/*
* get the version info
*/
if (PMI_SUCCESS != PMI_Get_version_info(&major,&minor,&revision)) {
return OPAL_ERROR;
}
snprintf(cray_pmi_version, sizeof(cray_pmi_version),
"%d.%d.%d", major, minor, revision);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
rc = sscanf(pmix_kvs_name,"kvs_%u",&jobfam);
if (rc != 1) {
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: pmix_kvs_name %s",
OPAL_NAME_PRINT(pmix_pname), pmix_kvs_name);
rc = OPAL_ERROR;
goto err_exit;
}
pmix_jobid = jobfam << 16;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
pmix_pname.jobid = pmix_jobid;
pmix_pname.vpid = pmix_rank;
opal_proc_set_name(&pmix_pname);
opal_output_verbose(10, opal_pmix_base_framework.framework_output,
"%s pmix:cray: assigned tmp name %d %d pmix_kvs_name %s",
OPAL_NAME_PRINT(pmix_pname),pmix_pname.jobid,pmix_pname.vpid,pmix_kvs_name);
pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
pmix_lranks = pmix_cray_parse_pmap(pmapping, pmix_rank, &my_node, &pmix_nlranks);
if (NULL == pmix_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
// setup hash table
opal_pmix_base_hash_init();
/* save the job size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_size;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the appnum */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_appnum;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
pmix_usize = atoi(buf);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_usize;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_UINT32;
kv.data.uint32 = pmix_jobid;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT16;
kv.data.uint16 = pmix_nlranks;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
ldr.vpid = pmix_lranks[0];
ldr.jobid = pmix_pname.jobid;
/* find ourselves and build up a string for local peer info */
memset(nmtmp, 0, 64);
for (i=0; i < pmix_nlranks; i++) {
ret = snprintf(nmtmp, 64, "%d", pmix_lranks[i]);
opal_argv_append_nosize(&localranks, nmtmp);
if (pmix_rank == pmix_lranks[i]) {
pmix_lrank = i;
pmix_nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = pmix_lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = pmix_nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
static int s2_init(void)
{
int spawned, size, rank, appnum;
int rc, ret = OPAL_ERROR;
char buf[16];
int found;
int my_node;
char *tmp;
uint32_t jobfam, stepid;
int i;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
s2_jsize = size;
s2_rank = rank;
s2_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
s2_usize = atoi(buf);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation. So we translate
* this as the overall job number equating to our job family, and
* the individual number equating to our local jobid
*/
jobfam = strtoul(pmix_kvs_name, &tmp, 10);
if (NULL == tmp) {
/* hmmm - no '.', so let's just use zero */
stepid = 0;
} else {
tmp++; /* step over the '.' */
stepid = strtoul(tmp, NULL, 10);
}
/* now build the jobid */
s2_jobid = (jobfam << 16) | stepid;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s2_pname.jobid = s2_jobid;
s2_pname.vpid = s2_rank;
opal_proc_set_name(&s2_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s2: assigned tmp name",
OPAL_NAME_PRINT(s2_pname));
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
s2_lranks = mca_common_pmi2_parse_pmap(pmapping, s2_pname.vpid, &my_node, &s2_nlranks);
if (NULL == s2_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
/* find ourselves */
for (i=0; i < s2_nlranks; i++) {
if (s2_rank == s2_lranks[i]) {
s2_lrank = i;
s2_nrank = i;
break;
}
}
/* increment the init count */
++pmix_init_count;
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
static int mca_initialize_pmi_v2(void)
{
int spawned, size, rank, appnum;
int rc, ret = OPAL_ERROR;
/* deal with a Slurm bug by first checking if we were
* even launched by a PMI server before attempting
* to use PMI */
if (NULL == getenv("PMI_FD")) {
return OPAL_ERROR;
}
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-common-pmi.txt", "pmi2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_output(0, "SIZE %d RANK %d", size, rank);
opal_show_help("help-common-pmi.txt", "pmi2-init-returned-bad-values", true);
goto err_exit;
}
pmi_size = size;
pmi_rank = rank;
pmi_appnum = appnum;
pmi_vallen_max = PMI2_MAX_VALLEN;
pmi_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmi_keylen_max = PMI2_MAX_KEYLEN;
char buf[16];
int found;
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
pmi_usize = atoi(buf);
pmi_kvs_name = (char*)malloc(pmi_kvslen_max);
if( pmi_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmi_kvs_name, pmi_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
int mca_common_pmi_local_info(int vpid, int **ranks_ret,
int *procs_ret, char **error)
{
int *ranks;
int procs = -1;
int rc;
#if WANT_PMI2_SUPPORT
if(mca_common_pmi_version == 2){
{
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
*error = "mca_common_pmi_local_info: could not get memory for PMIv2 process mapping";
return OPAL_ERR_OUT_OF_RESOURCE;
}
int found;
int my_node;
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI2_SUCCESS != rc ) {
/* can't check PMI2_SUCCESS as some folks (i.e., Cray) don't define it */
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
*error = "mca_common_pmi_local_info: could not get PMI_process_mapping";
return OPAL_ERROR;
}
ranks = mca_common_pmi2_parse_pmap(pmapping, vpid, &my_node, &procs);
if (NULL == ranks) {
*error = "mca_common_pmi_local_info: could not get memory for PMIv2 local ranks";
return OPAL_ERR_OUT_OF_RESOURCE;
}
free(pmapping);
}
}
else
#endif
{
/* get our local proc info to find our local rank */
if (PMI_SUCCESS != (rc = PMI_Get_clique_size(&procs))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_size");
*error = "mca_common_pmi_local_info: could not get PMI clique size";
return OPAL_ERROR;
}
/* now get the specific ranks */
ranks = (int*)calloc(procs, sizeof(int));
if (NULL == ranks) {
*error = "mca_common_pmi_local_info: could not get memory for local ranks";
return OPAL_ERR_OUT_OF_RESOURCE;
}
if (PMI_SUCCESS != (rc = PMI_Get_clique_ranks(ranks, procs))) {
OPAL_PMI_ERROR(rc, "PMI_Get_clique_ranks");
*error = "mca_common_pmi_local_info: could not get clique ranks";
return OPAL_ERROR;
}
}
*ranks_ret = ranks;
*procs_ret = procs;
return OPAL_SUCCESS;
}
static int s2_init(void)
{
int spawned, size, rank, appnum;
int rc, ret = OPAL_ERROR;
char buf[16];
int found;
int my_node;
uint32_t stepid;
int i;
opal_process_name_t ldr;
opal_value_t kv;
char **localranks;
char *str;
char nmtmp[64];
opal_process_name_t wildcard_rank;
/* if we can't startup PMI, we can't be used */
if ( PMI2_Initialized () ) {
return OPAL_SUCCESS;
}
size = -1;
rank = -1;
appnum = -1;
// setup hash table so we always can finalize it
opal_pmix_base_hash_init();
if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
}
if( size < 0 || rank < 0 ){
opal_show_help("help-pmix-base.txt", "pmix2-init-returned-bad-values", true);
goto err_exit;
}
s2_jsize = size;
s2_rank = rank;
s2_appnum = appnum;
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
free(pmix_kvs_name);
goto err_exit;
}
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s2_pname.jobid = strtoul(pmix_kvs_name, &str, 10);
s2_pname.jobid = (s2_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
stepid = strtoul(str, NULL, 10);
s2_pname.jobid |= (stepid & 0x0000ffff);
}
s2_pname.vpid = s2_rank;
opal_proc_set_name(&s2_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s2: assigned tmp name",
OPAL_NAME_PRINT(s2_pname));
/* setup wildcard rank*/
wildcard_rank = OPAL_PROC_MY_NAME;
wildcard_rank.vpid = OPAL_VPID_WILDCARD;
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_UINT32;
kv.data.uint32 = s2_pname.jobid;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv); // frees pmix_kvs_name
/* save the job size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOB_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = size;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save the appnum */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_APPNUM);
kv.type = OPAL_UINT32;
kv.data.uint32 = appnum;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
goto err_exit;
}
/* save it */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_UNIV_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = atoi(buf);
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* push this into the dstore for subsequent fetches */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_MAX_PROCS);
kv.type = OPAL_UINT32;
kv.data.uint32 = atoi(buf);
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
rc = PMI2_Info_GetJobAttr("PMI_process_mapping", pmapping, PMI2_MAX_VALLEN, &found);
if( !found || PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc,"PMI2_Info_GetJobAttr");
return OPAL_ERROR;
}
s2_lranks = mca_common_pmi2_parse_pmap(pmapping, s2_pname.vpid, &my_node, &s2_nlranks);
if (NULL == s2_lranks) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
return rc;
}
free(pmapping);
/* save the local size */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_SIZE);
kv.type = OPAL_UINT32;
kv.data.uint32 = s2_nlranks;
if (OPAL_SUCCESS != (rc = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
s2_lrank = 0;
s2_nrank = 0;
ldr.vpid = rank;
localranks = NULL;
if (0 < s2_nlranks && NULL != s2_lranks) {
/* note the local ldr */
ldr.vpid = s2_lranks[0];
/* find ourselves */
ldr.jobid = s2_pname.jobid;
ldr.vpid = s2_pname.vpid;
memset(nmtmp, 0, 64);
for (i=0; i < s2_nlranks; i++) {
(void)snprintf(nmtmp, 64, "%d", s2_lranks[i]);
opal_argv_append_nosize(&localranks, nmtmp);
if (s2_rank == s2_lranks[i]) {
s2_lrank = i;
s2_nrank = i;
}
}
str = opal_argv_join(localranks, ',');
opal_argv_free(localranks);
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_PEERS);
kv.type = OPAL_STRING;
kv.data.string = str;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&wildcard_rank, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
}
/* save the local leader */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCALLDR);
kv.type = OPAL_UINT64;
kv.data.uint64 = *(uint64_t*)&ldr;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* save our local rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_LOCAL_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = s2_lrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* and our node rank */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_NODE_RANK);
kv.type = OPAL_UINT16;
kv.data.uint16 = s2_nrank;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* increment the init count */
++pmix_init_count;
return OPAL_SUCCESS;
err_exit:
PMI2_Finalize();
return ret;
}
