static int pmi_set_proc_attr(const char* attr_name,
const void *buffer, size_t size)
{
int rc;
if (NULL == pmi_kvs_name) {
if (ORTE_SUCCESS != (rc = setup_pmi())) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, attr_name))) {
ORTE_ERROR_LOG(rc);
return rc;
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: set attr %s of size %lu in KVS %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), attr_name,
(unsigned long)size, pmi_kvs_name));
if (ORTE_SUCCESS != (rc = pmi_encode(buffer, size))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_put(pmi_kvs_key, pmi_attr_val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
return ORTE_SUCCESS;
}
void cmd_copy_tokvs (flux_t h, int argc, char **argv)
{
char *file, *key;
int fd, len;
uint8_t *buf;
JSON o;
if (argc != 2)
msg_exit ("copy-tokvs: specify key and filename");
key = argv[0];
file = argv[1];
if (!strcmp (file, "-")) {
if ((len = read_all (STDIN_FILENO, &buf)) < 0)
err_exit ("stdin");
} else {
if ((fd = open (file, O_RDONLY)) < 0)
err_exit ("%s", file);
if ((len = read_all (fd, &buf)) < 0)
err_exit ("%s", file);
(void)close (fd);
}
o = Jnew ();
util_json_object_add_data (o, "data", buf, len);
if (kvs_put (h, key, Jtostr (o)) < 0)
err_exit ("%s", key);
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
Jput (o);
free (buf);
}
int kz_close (kz_t *kz)
{
int rc = -1;
char *json_str = NULL;
char *key = NULL;
if ((kz->flags & KZ_FLAGS_WRITE)) {
if (!(kz->flags & KZ_FLAGS_RAW)) {
if (asprintf (&key, "%s.%.6d", kz->name, kz->seq++) < 0)
oom ();
if (!(json_str = zio_json_encode (NULL, 0, true))) { /* EOF */
errno = EPROTO;
goto done;
}
if (kvs_put (kz->h, key, json_str) < 0)
goto done;
}
if (!(kz->flags & KZ_FLAGS_NOCOMMIT_CLOSE)) {
if (kvs_commit (kz->h) < 0)
goto done;
}
if (kz->nprocs > 0 && kz->grpname) {
if (kz_fence (kz) < 0)
goto done;
}
}
rc = 0;
done:
if (json_str)
free (json_str);
if (key)
free (key);
kz_destroy (kz);
return rc;
}
static int
_setup_stepd_kvs(const stepd_step_rec_t *job, char ***env)
{
int rc = SLURM_SUCCESS, i = 0, pp_cnt = 0;
char *p, env_key[32], *ppkey, *ppval;
kvs_seq = 1;
rc = temp_kvs_init();
if (rc != SLURM_SUCCESS)
return rc;
rc = kvs_init();
if (rc != SLURM_SUCCESS)
return rc;
/* preput */
p = getenvp(*env, PMI2_PREPUT_CNT_ENV);
if (p) {
pp_cnt = atoi(p);
}
for (i = 0; i < pp_cnt; i ++) {
snprintf(env_key, 32, PMI2_PPKEY_ENV"%d", i);
p = getenvp(*env, env_key);
ppkey = p; /* getenvp will not modify p */
snprintf(env_key, 32, PMI2_PPVAL_ENV"%d", i);
p = getenvp(*env, env_key);
ppval = p;
kvs_put(ppkey, ppval);
}
/*
* For PMI11.
* A better logic would be to put PMI_process_mapping in KVS only if
* the task distribution method is not "arbitrary", because in
* "arbitrary" distribution the process mapping varible is not correct.
* MPICH2 may deduce the clique info from the hostnames. But that
* is rather costly.
*/
kvs_put("PMI_process_mapping", job_info.proc_mapping);
return SLURM_SUCCESS;
}
static int update_1pdesc (flux_t h, int r, int64_t j, JSON o, JSON ha, JSON ea)
{
int rc = -1;
JSON d = NULL;
char *key;
char *json_str = NULL;
const char *hn = NULL, *en = NULL;
int64_t pid = 0, hindx = 0, eindx = 0, hrank = 0;
if (!Jget_int64 (o, JSC_PDESC_RANK_PDARRAY_PID, &pid)) return -1;
if (!Jget_int64 (o, JSC_PDESC_RANK_PDARRAY_HINDX, &hindx)) return -1;
if (!Jget_int64 (o, JSC_PDESC_RANK_PDARRAY_EINDX, &eindx)) return -1;
if (!Jget_ar_str (ha, (int)hindx, &hn)) return -1;
if (!Jget_ar_str (ea, (int)eindx, &en)) return -1;
key = xasprintf ("lwj.%"PRId64".%"PRId32".procdesc", j, r);
if (kvs_get (h, key, &json_str) < 0
|| !(d = Jfromstr (json_str))) {
flux_log_error (h, "extract %s", key);
goto done;
}
Jadd_str (d, "command", en);
Jadd_int64 (d, "pid", pid);
errno = 0;
if ( (hrank = strtoul (hn, NULL, 10)) && errno != 0) {
flux_log (h, LOG_ERR, "invalid hostname %s", hn);
goto done;
}
Jadd_int64 (d, "nodeid", (int64_t)hrank);
if (kvs_put (h, key, Jtostr (d)) < 0) {
flux_log_error (h, "put %s", key);
goto done;
}
rc = 0;
done:
free (key);
if (d)
Jput (d);
if (json_str)
free (json_str);
return rc;
}
void cmd_put (flux_t h, int argc, char **argv)
{
int i;
if (argc == 0)
msg_exit ("put: specify one or more key=value pairs");
for (i = 0; i < argc; i++) {
char *key = xstrdup (argv[i]);
char *val = strchr (key, '=');
if (!val)
msg_exit ("put: you must specify a value as key=value");
*val++ = '\0';
if (kvs_put (h, key, val) < 0) {
if (errno != EINVAL || kvs_put_string (h, key, val) < 0)
err_exit ("%s", key);
}
free (key);
}
if (kvs_commit (h) < 0)
err_exit ("kvs_commit");
}
static int putnext (kz_t *kz, const char *json_str)
{
char *key = NULL;
int rc = -1;
if (!(kz->flags & KZ_FLAGS_WRITE)) {
errno = EINVAL;
goto done;
}
if (asprintf (&key, "%s.%.6d", kz->name, kz->seq++) < 0)
oom ();
if (kvs_put (kz->h, key, json_str) < 0)
goto done;
if (!(kz->flags & KZ_FLAGS_NOCOMMIT_PUT)) {
if (kvs_commit (kz->h) < 0)
goto done;
}
rc = 0;
done:
if (key)
free (key);
return rc;
}
/*** MODEX SECTION ***/
static int modex(opal_list_t *procs)
{
int rc, i;
size_t len;
char *rml_uri, val[64];
orte_vpid_t v;
orte_process_name_t name;
orte_jmap_t *jmap;
orte_nid_t *nid, *loc;
orte_pmap_t *pmap;
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: modex entered",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (NULL == pmi_kvs_name) {
if (ORTE_SUCCESS != (rc = setup_pmi())) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* provide our hostname so others can know our location */
if (strlen(orte_process_info.nodename) > (size_t)pmi_vallen_max) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
return ORTE_ERR_VALUE_OUT_OF_BOUNDS;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "HOSTNAME"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_put(pmi_kvs_key, orte_process_info.nodename);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
/* add our oob endpoint info so that oob communications
* can be supported
*/
rml_uri = orte_rml.get_contact_info();
if (strlen(rml_uri) > (size_t)pmi_vallen_max) {
ORTE_ERROR_LOG(ORTE_ERR_VALUE_OUT_OF_BOUNDS);
return ORTE_ERROR;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "RMLURI"))) {
ORTE_ERROR_LOG(rc);
free(rml_uri);
return rc;
}
/* NTH: some characters are not allowed in pmi2 land so we need to encode */
if (ORTE_SUCCESS != (rc = pmi_encode(rml_uri, strlen(rml_uri)))) {
ORTE_ERROR_LOG(rc);
free(rml_uri);
return rc;
}
/* encoding puts the encoded value in pmi_attr_val */
rc = kvs_put(pmi_kvs_key, pmi_attr_val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
free(rml_uri);
return ORTE_ERROR;
}
free(rml_uri);
/* get the job map for this job */
jmap = (orte_jmap_t*)opal_pointer_array_get_item(&orte_jobmap, 0);
/* get my pidmap entry */
pmap = (orte_pmap_t*)opal_pointer_array_get_item(&jmap->pmap, ORTE_PROC_MY_NAME->vpid);
/* add our local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
snprintf(val, 64, "%lu", (unsigned long)pmap->local_rank);
rc = kvs_put(pmi_kvs_key, val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
if (ORTE_SUCCESS != (rc = setup_key(ORTE_PROC_MY_NAME, "NODERANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
snprintf(val, 64, "%lu", (unsigned long)pmap->node_rank);
rc = kvs_put(pmi_kvs_key, val);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Put");
return ORTE_ERROR;
}
/* commit our modex info */
if (PMI_SUCCESS != (rc = kvs_commit())) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Commit failed");
return ORTE_ERROR;
}
/* harvest the oob endpoint info and hostname for all other procs
* in our job so oob wireup can be completed and we
* can setup their nidmap/pidmap
*/
name.jobid = ORTE_PROC_MY_NAME->jobid;
orte_process_info.num_nodes = 1; /* have to account for mine! */
for (v=0; v < orte_process_info.num_procs; v++) {
if (v == ORTE_PROC_MY_NAME->vpid) {
continue;
}
name.vpid = v;
if (ORTE_SUCCESS != (rc = setup_key(&name, "RMLURI"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
/* Had to encode to protect against pmi2-prohibited chars */
rml_uri = pmi_decode(&len);
if (NULL == rml_uri) {
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s oob endpoint %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), rml_uri));
/* set the contact info into the hash table */
if (ORTE_SUCCESS != (rc = orte_rml.set_contact_info(rml_uri))) {
free(rml_uri);
return rc;
}
free(rml_uri);
if (ORTE_SUCCESS != (rc = setup_key(&name, "HOSTNAME"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s location %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name), pmi_attr_val));
/* see if this node is already in nidmap */
loc = NULL;
for (i=0; i < orte_nidmap.size; i++) {
if (NULL == (nid = (orte_nid_t*)opal_pointer_array_get_item(&orte_nidmap, i))) {
continue;
}
if (0 == strcmp(pmi_attr_val, nid->name)) {
/* found it */
loc = nid;
break;
}
}
if (NULL == loc) {
/* new node - save it */
loc = OBJ_NEW(orte_nid_t);
loc->name = strdup(pmi_attr_val);
loc->index = opal_pointer_array_add(&orte_nidmap, loc);
loc->daemon = loc->index;
/* keep track */
orte_process_info.num_nodes++;
}
/* see if this proc is already in the pidmap */
if (NULL == (pmap = (orte_pmap_t*)opal_pointer_array_get_item(&jmap->pmap, v))) {
/* nope - add it */
pmap = OBJ_NEW(orte_pmap_t);
pmap->node = loc->index;
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(&jmap->pmap, v, pmap))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* get the proc's local/node rank info */
if (ORTE_SUCCESS != (rc = setup_key(&name, "LOCALRANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
pmap->local_rank = (orte_local_rank_t)strtoul(pmi_attr_val, NULL, 10);
if (ORTE_SUCCESS != (rc = setup_key(&name, "NODERANK"))) {
ORTE_ERROR_LOG(rc);
return rc;
}
rc = kvs_get(pmi_kvs_key, pmi_attr_val, pmi_vallen_max);
if (PMI_SUCCESS != rc) {
ORTE_PMI_ERROR(rc, "PMI_KVS_Get");
return ORTE_ERROR;
}
pmap->node_rank = (orte_node_rank_t)strtoul(pmi_attr_val, NULL, 10);
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_output,
"%s grpcomm:pmi: proc %s lrank %u nrank %u",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&name),
(unsigned int)pmap->local_rank,
(unsigned int)pmap->node_rank));
}
OPAL_OUTPUT_VERBOSE((1, orte_grpcomm_base_output,
"%s grpcomm:pmi: modex completed",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
return rc;
}
static int store(const opal_identifier_t *uid,
opal_db_locality_t locality,
const char *key, const void *data, opal_data_type_t type)
{
int i, rc;
char *pmidata, *str, *localdata;
int64_t i64;
uint64_t ui64;
opal_byte_object_t *bo;
char *pmikey, *tmpkey, *tmp, sav;
char **strdata=NULL;
opal_identifier_t proc;
/* to protect alignment, copy the data across */
memcpy(&proc, uid, sizeof(opal_identifier_t));
/* pass internal stores down to someone else */
if (OPAL_DB_INTERNAL == locality) {
return OPAL_ERR_TAKE_NEXT_OPTION;
}
OPAL_OUTPUT_VERBOSE((5, opal_db_base_framework.framework_output,
"db:pmi:store: storing key %s[%s] for proc %" PRIu64 "",
key, opal_dss.lookup_data_type(type), proc));
if (NULL == (pmikey = setup_key(proc, key))) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
switch (type) {
case OPAL_STRING:
#if WANT_PMI2_SUPPORT
{
/* the blasted Cray PMI implementation marked a number of common
* ASCII characters as "illegal", so if we are on one of those
* machines, then we have to replace those characters with something
* else
*/
size_t n, k;
bool subbed;
char *ptr;
str = (char*)data;
/* first, count how many characters need to be replaced - since Cray
* is the source of the trouble, we only make this slow for them!
*/
ptr = str;
i=0;
for (n=0; n < strlen(illegal); n++) {
while (NULL != (tmp = strchr(ptr, illegal[n]))) {
i++;
ptr = tmp;
ptr++;
}
}
/* stretch the string */
ptr = (char*)malloc(sizeof(char) * (1 + strlen(str) + 2*i));
/* now construct it */
k=0;
for (n=0; n < strlen(str); n++) {
subbed = false;
for (i=0; i < (int)strlen(illegal); i++) {
if (str[n] == illegal[i]) {
/* escape the character */
ptr[k++] = escape_char;
ptr[k++] = sub[i];
subbed = true;
break;
}
}
if (!subbed) {
ptr[k++] = str[n];
}
}
/* pass the result */
localdata = ptr;
}
#else
localdata = strdup((char*)data);
#endif
str = localdata;
while (pmi_vallen_max < (int)(OPAL_PMI_PAD + strlen(str))) {
/* the string is too long, so we need to break it into
* multiple sections
*/
tmp = str + pmi_vallen_max - OPAL_PMI_PAD;
sav = *tmp;
*tmp = '\0';
opal_argv_append_nosize(&strdata, str);
*tmp = sav;
str = tmp;
}
/* put whatever remains on the stack */
opal_argv_append_nosize(&strdata, str);
/* cleanup */
free(localdata);
/* the first value we put uses the original key, but
* the data is prepended with the number of sections
* required to hold the entire string
*/
asprintf(&pmidata, "%d:%s", opal_argv_count(strdata), strdata[0]);
OPAL_OUTPUT_VERBOSE((5, opal_db_base_framework.framework_output,
"db:pmi:store: storing key %s data %s",
pmikey, pmidata));
if (PMI_SUCCESS != (rc = kvs_put(pmikey, pmidata))) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Put");
free(pmidata);
free(pmikey);
opal_argv_free(strdata);
return OPAL_ERROR;
}
free(pmidata);
/* for each remaining segment, augment the key with the index */
for (i=1; NULL != strdata[i]; i++) {
asprintf(&tmpkey, "%s:%d", pmikey, i);
OPAL_OUTPUT_VERBOSE((5, opal_db_base_framework.framework_output,
"db:pmi:store: storing key %s data %s",
pmikey, strdata[i]));
if (PMI_SUCCESS != (rc = kvs_put(tmpkey, strdata[i]))) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Put");
free(pmikey);
opal_argv_free(strdata);
return OPAL_ERROR;
}
free(tmpkey);
}
free(pmikey);
opal_argv_free(strdata);
return OPAL_SUCCESS;
case OPAL_INT:
i64 = (int64_t)(*((int*)data));
asprintf(&pmidata, "%ld", (long)i64);
break;
case OPAL_INT32:
i64 = (int64_t)(*((int32_t*)data));
asprintf(&pmidata, "%ld", (long)i64);
break;
case OPAL_INT64:
i64 = (int64_t)(*((int*)data));
asprintf(&pmidata, "%ld", (long)i64);
break;
case OPAL_UINT64:
ui64 = *((uint64_t*)data);
asprintf(&pmidata, "%lu", (unsigned long)ui64);
break;
case OPAL_UINT32:
ui64 = (uint64_t)(*((uint32_t*)data));
asprintf(&pmidata, "%lu", (unsigned long)ui64);
break;
case OPAL_UINT16:
ui64 = (uint64_t)(*((uint16_t*)data));
asprintf(&pmidata, "%lu", (unsigned long)ui64);
break;
case OPAL_BYTE_OBJECT:
bo = (opal_byte_object_t*)data;
pmidata = (char*)malloc(pmi_vallen_max*sizeof(char));
if (OPAL_SUCCESS != (rc = pmi_encode(pmidata, bo->bytes, bo->size))) {
OPAL_ERROR_LOG(rc);
free(pmidata);
return rc;
}
break;
default:
OPAL_ERROR_LOG(OPAL_ERR_NOT_SUPPORTED);
return OPAL_ERR_NOT_SUPPORTED;
}
OPAL_OUTPUT_VERBOSE((10, opal_db_base_framework.framework_output,
"PUTTING KEY %s DATA %s",
pmikey, pmidata));
rc = kvs_put(pmikey, pmidata);
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Put");
return OPAL_ERROR;
}
free(pmidata);
free(pmikey);
return OPAL_SUCCESS;
}