static int pmi_get_proc_attr(const orte_process_name_t name,
const char* attr_name,
void **buffer, size_t *size)
{
char *attrval, *attr;
int rc;
if (NULL == pmi_kvs_name) {
return ORTE_ERR_UNREACH;
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:pmi: get attr %s for proc %s in KVS %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), attr_name,
ORTE_NAME_PRINT(&name), pmi_kvs_name));
attrval = malloc(pmi_vallen_max);
if (NULL == attrval) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
if (0 > asprintf(&attr, "%s-%s", ORTE_NAME_PRINT(&name), attr_name)) {
free(attrval);
return ORTE_ERR_OUT_OF_RESOURCE;
}
rc = PMI_KVS_Get(pmi_kvs_name, attr, attrval, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
free(attrval);
free(attr);
return ORTE_ERROR;
}
*buffer = pmi_decode((unsigned char *)attrval, size);
free(attrval);
free(attr);
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base.output,
"%s grpcomm:pmi: got attr %s of size %lu",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
attr_name, (unsigned long)(*size)));
if (NULL == buffer) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
return ORTE_SUCCESS;
}
static int pmi_get_proc_attr(const orte_process_name_t name,
const char* attr_name,
void **buffer, size_t *size)
{
int rc;
/* set default */
*size = 0;
*buffer = NULL;
if (NULL == pmi_kvs_name) {
if (ORTE_SUCCESS != (rc = setup_pmi())) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: get attr %s for proc %s in KVS %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), attr_name,
ORTE_NAME_PRINT(&name), pmi_kvs_name));
if (ORTE_SUCCESS != (rc = setup_key(&name, attr_name))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
*buffer = pmi_decode(size);
if (NULL == *buffer) {
return ORTE_ERR_OUT_OF_RESOURCE;
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: got attr %s of size %lu",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
attr_name, (unsigned long)(*size)));
return ORTE_SUCCESS;
}
void pmi_get_key_rem(int rank, char *key_name, int **key_val, int *key_size)
{
int len, rc;
size_t tmp_size;
char *tmp = calloc(PMI2_MAX_VALLEN, sizeof(char));
if( (rc = PMI2_KVS_Get(kvs_name, PMI2_ID_NULL, key_name, tmp, PMI2_MAX_VALLEN, &len) ) ){
fprintf(stderr,"PMI2_Info_GetNodeAttr: error rc = %d\n", rc);
abort();
}
*key_val = (int*)pmi_decode(tmp, &tmp_size);
*key_size = tmp_size / sizeof(int);
if( NULL == *key_val ){
fprintf(stderr,"pmi_decode: cannot decode key %s\n", key_name);
abort();
}
free(tmp);
}
void pmi_get_key_loc(int rank, char *key_name, int **key_val, int *key_size)
{
int found, rc;
size_t tmp_size;
char *tmp = calloc(PMI2_MAX_VALLEN, sizeof(char));
if( (rc = PMI2_Info_GetNodeAttr(key_name, tmp, PMI2_MAX_VALLEN, &found, 1) ) ){
fprintf(stderr,"PMI2_Info_GetNodeAttr: error rc = %d\n", rc);
abort();
}
if( !found ){
fprintf(stderr,"pmi_get_key_loc: key %s not found\n", key_name);
abort();
}
*key_val = (int*)pmi_decode(tmp, &tmp_size);
*key_size = tmp_size / sizeof(int);
if( NULL == *key_val ){
fprintf(stderr,"pmi_decode: cannot decode key %s\n", key_name);
abort();
}
free(tmp);
}
int opal_pmix_base_get_packed(const opal_process_name_t* proc, char **packed_data,
size_t *len, int vallen, kvs_get_fn fn)
{
char *tmp_encoded = NULL, *pmikey, *pmi_tmp;
int remote_key, size;
size_t bytes_read;
int rc = OPAL_ERR_NOT_FOUND;
/* set default */
*packed_data = NULL;
*len = 0;
pmi_tmp = calloc (vallen, 1);
if (NULL == pmi_tmp) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
/* read all of the packed data from this proc */
for (remote_key = 0, bytes_read = 0 ; ; ++remote_key) {
char tmp_key[32];
sprintf (tmp_key, "key%d", remote_key);
if (NULL == (pmikey = setup_key(proc, tmp_key, vallen))) {
rc = OPAL_ERR_OUT_OF_RESOURCE;
OPAL_ERROR_LOG(rc);
free(pmi_tmp);
if (NULL != tmp_encoded) {
free(tmp_encoded);
}
return rc;
}
OPAL_OUTPUT_VERBOSE((10, opal_pmix_base_framework.framework_output,
"GETTING KEY %s", pmikey));
rc = fn(pmikey, pmi_tmp, vallen);
free (pmikey);
if (OPAL_SUCCESS != rc) {
break;
}
size = strlen (pmi_tmp);
if (NULL == tmp_encoded) {
tmp_encoded = malloc (size + 1);
} else {
tmp_encoded = realloc (tmp_encoded, bytes_read + size + 1);
}
strcpy (tmp_encoded + bytes_read, pmi_tmp);
bytes_read += size;
/* is the string terminator present? */
if ('-' == tmp_encoded[bytes_read-1]) {
break;
}
}
free (pmi_tmp);
OPAL_OUTPUT_VERBOSE((10, opal_pmix_base_framework.framework_output,
"Read data %s\n",
(NULL == tmp_encoded) ? "NULL" : tmp_encoded));
if (NULL != tmp_encoded) {
*packed_data = (char *) pmi_decode (tmp_encoded, len);
free (tmp_encoded);
if (NULL == *packed_data) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
}
return rc;
}
/*** MODEX SECTION ***/
static int modex(opal_list_t *procs)
{
int rc, i;
size_t len;
char *rml_uri, val[64];
orte_vpid_t v;
orte_process_name_t name;
orte_jmap_t *jmap;
orte_nid_t *nid, *loc;
orte_pmap_t *pmap;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: modex entered",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (NULL == pmi_kvs_name) {
if (ORTE_SUCCESS != (rc = setup_pmi())) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* provide our hostname so others can know our location */
if (strlen(orte_process_info.nodename) > (size_t)pmi_vallen_max) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
return ORTE_ERR_VALUE_OUT_OF_BOUNDS;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "HOSTNAME"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_put(pmi_kvs_key, orte_process_info.nodename);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
/* add our oob endpoint info so that oob communications
* can be supported
*/
rml_uri = orte_rml.get_contact_info();
if (strlen(rml_uri) > (size_t)pmi_vallen_max) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
return ORTE_ERROR;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "RMLURI"))) {
ORTE_ERROR_LOG(rc);
free(rml_uri);
return rc;
}
/* NTH: some characters are not allowed in pmi2 land so we need to encode */
if (ORTE_SUCCESS != (rc = pmi_encode(rml_uri, strlen(rml_uri)))) {
ORTE_ERROR_LOG(rc);
free(rml_uri);
return rc;
}
/* encoding puts the encoded value in pmi_attr_val */
rc = kvs_put(pmi_kvs_key, pmi_attr_val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
free(rml_uri);
return ORTE_ERROR;
}
free(rml_uri);
/* get the job map for this job */
jmap = (orte_jmap_t*)opal_pointer_array_get_item(&orte_jobmap, 0);
/* get my pidmap entry */
pmap = (orte_pmap_t*)opal_pointer_array_get_item(&jmap->pmap, ORTE_PROC_MY_NAME->vpid);
/* add our local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
snprintf(val, 64, "%lu", (unsigned long)pmap->local_rank);
rc = kvs_put(pmi_kvs_key, val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "NODERANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
snprintf(val, 64, "%lu", (unsigned long)pmap->node_rank);
rc = kvs_put(pmi_kvs_key, val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
/* commit our modex info */
if (PMI_SUCCESS != (rc = kvs_commit())) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Commit failed");
return ORTE_ERROR;
}
/* harvest the oob endpoint info and hostname for all other procs
* in our job so oob wireup can be completed and we
* can setup their nidmap/pidmap
*/
name.jobid = ORTE_PROC_MY_NAME->jobid;
orte_process_info.num_nodes = 1; /* have to account for mine! */
for (v=0; v < orte_process_info.num_procs; v++) {
if (v == ORTE_PROC_MY_NAME->vpid) {
continue;
}
name.vpid = v;
if (ORTE_SUCCESS != (rc = setup_key(&name, "RMLURI"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
/* Had to encode to protect against pmi2-prohibited chars */
rml_uri = pmi_decode(&len);
if (NULL == rml_uri) {
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s oob endpoint %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), rml_uri));
/* set the contact info into the hash table */
if (ORTE_SUCCESS != (rc = orte_rml.set_contact_info(rml_uri))) {
free(rml_uri);
return rc;
}
free(rml_uri);
if (ORTE_SUCCESS != (rc = setup_key(&name, "HOSTNAME"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s location %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), pmi_attr_val));
/* see if this node is already in nidmap */
loc = NULL;
for (i=0; i < orte_nidmap.size; i++) {
if (NULL == (nid = (orte_nid_t*)opal_pointer_array_get_item(&orte_nidmap, i))) {
continue;
}
if (0 == strcmp(pmi_attr_val, nid->name)) {
/* found it */
loc = nid;
break;
}
}
if (NULL == loc) {
/* new node - save it */
loc = OBJ_NEW(orte_nid_t);
loc->name = strdup(pmi_attr_val);
loc->index = opal_pointer_array_add(&orte_nidmap, loc);
loc->daemon = loc->index;
/* keep track */
orte_process_info.num_nodes++;
}
/* see if this proc is already in the pidmap */
if (NULL == (pmap = (orte_pmap_t*)opal_pointer_array_get_item(&jmap->pmap, v))) {
/* nope - add it */
pmap = OBJ_NEW(orte_pmap_t);
pmap->node = loc->index;
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(&jmap->pmap, v, pmap))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* get the proc's local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(&name, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
pmap->local_rank = (orte_local_rank_t)strtoul(pmi_attr_val, NULL, 10);
if (ORTE_SUCCESS != (rc = setup_key(&name, "NODERANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
pmap->node_rank = (orte_node_rank_t)strtoul(pmi_attr_val, NULL, 10);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s lrank %u nrank %u",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name),
(unsigned int)pmap->local_rank,
(unsigned int)pmap->node_rank));
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: modex completed",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return rc;
}
static int fetch(const opal_identifier_t *uid,
const char *key, void **data, opal_data_type_t type)
{
opal_byte_object_t *boptr;
uint16_t ui16;
uint32_t ui32;
int ival;
unsigned int uival;
char *pmikey;
char tmp_val[1024];
size_t sval;
opal_identifier_t proc;
/* to protect alignment, copy the data across */
memcpy(&proc, uid, sizeof(opal_identifier_t));
OPAL_OUTPUT_VERBOSE((5, opal_db_base_framework.framework_output,
"db:pmi:fetch: searching for key %s[%s] on proc %" PRIu64 "",
(NULL == key) ? "NULL" : key,
opal_dss.lookup_data_type(type), proc));
/* if the key is NULL, that is an error */
if (NULL == key) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
/* setup the key */
if (NULL == (pmikey = setup_key(proc, key))) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
/* check to see if they are looking for a string */
if (OPAL_STRING == type) {
/* might have been passed in multiple sections */
*data = fetch_string(pmikey);
free(pmikey);
return OPAL_SUCCESS;
}
/* otherwise, retrieve the pmi keyval */
if (NULL == (pmikey = setup_key(proc, key))) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
if (PMI_SUCCESS != kvs_get(pmikey, tmp_val, pmi_vallen_max)) {
OPAL_ERROR_LOG(OPAL_ERR_NOT_FOUND);
free(pmikey);
return OPAL_ERR_NOT_FOUND;
}
free(pmikey);
/* return the value according to the provided type */
switch (type) {
case OPAL_UINT32:
ui32 = (uint32_t)strtoul(tmp_val, NULL, 10);
memcpy(*data, &ui32, sizeof(uint32_t));
break;
case OPAL_UINT16:
ui16 = (uint16_t)strtoul(tmp_val, NULL, 10);
memcpy(*data, &ui16, sizeof(uint16_t));
break;
case OPAL_INT:
ival = (int)strtol(tmp_val, NULL, 10);
memcpy(*data, &ival, sizeof(int));
break;
case OPAL_UINT:
uival = (unsigned int)strtoul(tmp_val, NULL, 10);
memcpy(*data, &uival, sizeof(unsigned int));
break;
case OPAL_BYTE_OBJECT:
sval = 0;
boptr = (opal_byte_object_t*)malloc(sizeof(opal_byte_object_t));
boptr->bytes = (uint8_t*)pmi_decode(tmp_val, &sval);
boptr->size = sval;
*data = boptr;
break;
default:
OPAL_ERROR_LOG(OPAL_ERR_NOT_SUPPORTED);
return OPAL_ERR_NOT_SUPPORTED;
}
return OPAL_SUCCESS;
}
