static int native_fence_nb(opal_process_name_t *procs, size_t nprocs,
opal_pmix_cbfunc_t cbfunc, void *cbdata)
{
opal_buffer_t *msg;
pmix_cmd_t cmd = PMIX_FENCENB_CMD;
int rc;
pmix_cb_t *cb;
opal_pmix_scope_t scope;
if (NULL == mca_pmix_native_component.uri) {
/* no server available, so just execute the callback */
if (NULL != cbfunc) {
cbfunc(OPAL_SUCCESS, NULL, cbdata);
}
return OPAL_SUCCESS;
}
msg = OBJ_NEW(opal_buffer_t);
/* pack the fence cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
/* pack the number of procs */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &nprocs, 1, OPAL_SIZE))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (0 < nprocs) {
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, procs, nprocs, OPAL_NAME))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
}
/* if we haven't already done it, ensure we have committed our values */
if (NULL != mca_pmix_native_component.cache_local) {
scope = PMIX_LOCAL;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_local, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_local);
}
if (NULL != mca_pmix_native_component.cache_remote) {
scope = PMIX_REMOTE;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_remote, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_remote);
}
if (NULL != mca_pmix_native_component.cache_global) {
scope = PMIX_GLOBAL;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_global, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_global);
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(msg, fencenb_cbfunc, cb);
return OPAL_SUCCESS;
}
static int native_abort(int flag, const char msg[])
{
opal_buffer_t *bfr;
pmix_cmd_t cmd = PMIX_ABORT_CMD;
int rc;
pmix_cb_t *cb;
opal_event_t ev;
struct timeval tv = {1, 0};
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native abort called",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
if (NULL == mca_pmix_native_component.uri) {
/* no server available, so just return */
return OPAL_SUCCESS;
}
if (PMIX_USOCK_CONNECTED == mca_pmix_native_component.state) {
/* create a buffer to hold the message */
bfr = OBJ_NEW(opal_buffer_t);
/* pack the cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(bfr, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(bfr);
return rc;
}
/* pack the status flag */
if (OPAL_SUCCESS != (rc = opal_dss.pack(bfr, &flag, 1, OPAL_INT))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(bfr);
return rc;
}
/* pack the string message - a NULL is okay */
if (OPAL_SUCCESS != (rc = opal_dss.pack(bfr, &msg, 1, OPAL_STRING))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(bfr);
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->active = true;
/* push a timeout event to wake us up just in case this
* message cannot get thru - e.g., someone else may have
* detected the failure of the server and ordered an abort */
opal_event_evtimer_set(mca_pmix_native_component.evbase,
&ev, timeout, cb);
opal_event_evtimer_add(&ev, &tv);
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(bfr, wait_cbfunc, cb);
/* wait for the release */
PMIX_WAIT_FOR_COMPLETION(cb->active);
OBJ_RELEASE(cb);
}
return OPAL_SUCCESS;
}
static int native_fence(opal_process_name_t *procs, size_t nprocs)
{
opal_buffer_t *msg, *bptr;
pmix_cmd_t cmd = PMIX_FENCE_CMD;
pmix_cb_t *cb;
int rc, ret;
opal_pmix_scope_t scope;
int32_t cnt;
opal_value_t *kp;
opal_process_name_t id;
size_t i;
uint64_t np;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native executing fence on %u procs",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), (unsigned int)nprocs);
if (NULL == mca_pmix_native_component.uri) {
/* no server available, so just return */
return OPAL_SUCCESS;
}
msg = OBJ_NEW(opal_buffer_t);
/* pack the fence cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
/* pack the number of procs */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &nprocs, 1, OPAL_SIZE))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (0 < nprocs) {
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, procs, nprocs, OPAL_NAME))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
}
/* if we haven't already done it, ensure we have committed our values */
if (NULL != mca_pmix_native_component.cache_local) {
scope = PMIX_LOCAL;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_local, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_local);
}
if (NULL != mca_pmix_native_component.cache_remote) {
scope = PMIX_REMOTE;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_remote, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_remote);
}
if (NULL != mca_pmix_native_component.cache_global) {
scope = PMIX_GLOBAL;
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &scope, 1, PMIX_SCOPE_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &mca_pmix_native_component.cache_global, 1, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
OBJ_RELEASE(mca_pmix_native_component.cache_global);
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(msg, wait_cbfunc, cb);
/* wait for the fence to complete */
PMIX_WAIT_FOR_COMPLETION(cb->active);
/* get the number of contributors */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&cb->data, &np, &cnt, OPAL_UINT64))) {
OPAL_ERROR_LOG(rc);
return rc;
}
/* if data was returned, unpack and store it */
for (i=0; i < np; i++) {
/* get the buffer that contains the data for the next proc */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&cb->data, &msg, &cnt, OPAL_BUFFER))) {
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) {
break;
}
OPAL_ERROR_LOG(rc);
return rc;
}
/* extract the id of the contributor from the blob */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(msg, &id, &cnt, OPAL_NAME))) {
OPAL_ERROR_LOG(rc);
return rc;
}
/* extract all blobs from this proc, starting with the scope */
cnt = 1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(msg, &scope, &cnt, PMIX_SCOPE_T))) {
/* extract the blob for this scope */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(msg, &bptr, &cnt, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
return rc;
}
/* now unpack and store the values - everything goes into our internal store */
cnt = 1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(bptr, &kp, &cnt, OPAL_VALUE))) {
if (OPAL_SUCCESS != (ret = opal_dstore.store(opal_dstore_internal, &id, kp))) {
OPAL_ERROR_LOG(ret);
}
OBJ_RELEASE(kp);
cnt = 1;
}
OBJ_RELEASE(bptr);
cnt = 1;
}
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
}
OBJ_RELEASE(msg);
if (OPAL_SUCCESS != rc && OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
} else {
rc = OPAL_SUCCESS;
}
}
OBJ_RELEASE(cb);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native fence released",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
return OPAL_SUCCESS;
}
static bool native_get_attr(const char *attr, opal_value_t **kv)
{
opal_buffer_t *msg, *bptr;
opal_list_t vals;
opal_value_t *kp, *lclpeers=NULL, kvn;
pmix_cmd_t cmd = PMIX_GETATTR_CMD;
char **ranks;
int rc, ret;
int32_t cnt;
bool found=false;
opal_hwloc_locality_t locality;
pmix_cb_t *cb;
uint32_t i, myrank;
opal_process_name_t id;
char *cpuset;
opal_buffer_t buf, buf2;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native get_attr called",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* try to retrieve the requested value from the dstore */
OBJ_CONSTRUCT(&vals, opal_list_t);
if (OPAL_SUCCESS == opal_dstore.fetch(opal_dstore_internal, &OPAL_PROC_MY_NAME, attr, &vals)) {
*kv = (opal_value_t*)opal_list_remove_first(&vals);
OPAL_LIST_DESTRUCT(&vals);
return true;
}
if (NULL == mca_pmix_native_component.uri) {
/* no server available, so just return */
return false;
}
/* if the value isn't yet available, then we should try to retrieve
* all the available attributes and store them for future use */
msg = OBJ_NEW(opal_buffer_t);
/* pack the cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return false;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(msg, wait_cbfunc, cb);
/* wait for the data to return */
PMIX_WAIT_FOR_COMPLETION(cb->active);
/* we have received the entire data blob for this process - unpack
* and cache all values, keeping the one we requested to return
* to the caller */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&cb->data, &ret, &cnt, OPAL_INT))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(cb);
return false;
}
if (OPAL_SUCCESS == ret) {
/* unpack the buffer containing the values */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&cb->data, &bptr, &cnt, OPAL_BUFFER))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(cb);
return false;
}
cnt = 1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(bptr, &kp, &cnt, OPAL_VALUE))) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s unpacked attr %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), kp->key);
/* if this is the local topology, we need to save it in a special way */
#if OPAL_HAVE_HWLOC
{
hwloc_topology_t topo;
if (0 == strcmp(PMIX_LOCAL_TOPO, kp->key)) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s saving topology",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* transfer the byte object for unpacking */
OBJ_CONSTRUCT(&buf, opal_buffer_t);
opal_dss.load(&buf, kp->data.bo.bytes, kp->data.bo.size);
kp->data.bo.bytes = NULL; // protect the data region
kp->data.bo.size = 0;
OBJ_RELEASE(kp);
/* extract the topology */
cnt=1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&buf, &topo, &cnt, OPAL_HWLOC_TOPO))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
continue;
}
OBJ_DESTRUCT(&buf);
if (NULL == opal_hwloc_topology) {
opal_hwloc_topology = topo;
} else {
hwloc_topology_destroy(topo);
}
cnt = 1;
continue;
}
}
#endif
/* if this is the local cpuset blob, then unpack and store its contents */
if (0 == strcmp(PMIX_LOCAL_CPUSETS, kp->key)) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s received local cpusets",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* transfer the byte object for unpacking */
OBJ_CONSTRUCT(&buf, opal_buffer_t);
opal_dss.load(&buf, kp->data.bo.bytes, kp->data.bo.size);
kp->data.bo.bytes = NULL; // protect the data region
kp->data.bo.size = 0;
OBJ_RELEASE(kp);
cnt=1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&buf, &id, &cnt, OPAL_NAME))) {
cnt=1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&buf, &cpuset, &cnt, OPAL_STRING))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
cnt = 1;
continue;
}
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s saving cpuset %s for local peer %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(NULL == cpuset) ? "NULL" : cpuset,
OPAL_NAME_PRINT(id));
OBJ_CONSTRUCT(&kvn, opal_value_t);
kvn.key = strdup(OPAL_DSTORE_CPUSET);
kvn.type = OPAL_STRING;
kvn.data.string = cpuset;
if (OPAL_SUCCESS != (rc = opal_dstore.store(opal_dstore_internal, &id, &kvn))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&kvn);
cnt = 1;
continue;
}
OBJ_DESTRUCT(&kvn);
}
OBJ_DESTRUCT(&buf);
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
return false;
}
cnt=1;
continue;
} else if (0 == strcmp(PMIX_PROC_MAP, kp->key)) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s received proc map",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* transfer the byte object for unpacking */
OBJ_CONSTRUCT(&buf, opal_buffer_t);
opal_dss.load(&buf, kp->data.bo.bytes, kp->data.bo.size);
kp->data.bo.bytes = NULL; // protect the data region
kp->data.bo.size = 0;
OBJ_RELEASE(kp);
/* get the jobid */
cnt=1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&buf, &kp, &cnt, OPAL_VALUE))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
cnt = 1;
return false;
}
if (0 != strcmp(PMIX_JOBID, kp->key)) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
OBJ_DESTRUCT(&buf);
OBJ_RELEASE(kp);
cnt = 1;
return false;
}
id.jobid = kp->data.uint32;
OBJ_RELEASE(kp);
/* unpack the data for each rank */
cnt=1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&buf, &kp, &cnt, OPAL_VALUE))) {
if (0 != strcmp(PMIX_RANK, kp->key)) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
OBJ_DESTRUCT(&buf);
OBJ_RELEASE(kp);
cnt = 1;
return false;
}
id.vpid = kp->data.uint32;
/* unpack the blob for this rank */
cnt=1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&buf, &kp, &cnt, OPAL_VALUE))) {
OPAL_ERROR_LOG(rc);
OBJ_DESTRUCT(&buf);
cnt = 1;
return false;
}
if (0 != strcmp(PMIX_PROC_MAP, kp->key)) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
OBJ_DESTRUCT(&buf);
OBJ_RELEASE(kp);
cnt = 1;
return false;
}
/* transfer the byte object for unpacking */
OBJ_CONSTRUCT(&buf2, opal_buffer_t);
opal_dss.load(&buf2, kp->data.bo.bytes, kp->data.bo.size);
kp->data.bo.bytes = NULL; // protect the data region
kp->data.bo.size = 0;
OBJ_RELEASE(kp);
/* unpack and store the map */
cnt=1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&buf2, &kp, &cnt, OPAL_VALUE))) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s storing key %s for peer %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
kp->key, OPAL_NAME_PRINT(id));
if (OPAL_SUCCESS != (rc = opal_dstore.store(opal_dstore_internal, &id, kp))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(kp);
OBJ_DESTRUCT(&buf2);
return false;
}
}
OBJ_DESTRUCT(&buf2);
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
return false;
}
cnt=1;
}
OBJ_DESTRUCT(&buf);
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
return false;
}
cnt=1;
continue;
}
/* otherwise, it is a single piece of info, so store it */
if (OPAL_SUCCESS != (rc = opal_dstore.store(opal_dstore_internal, &OPAL_PROC_MY_NAME, kp))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(kp);
cnt = 1;
continue;
}
/* save the list of local peers */
if (0 == strcmp(PMIX_LOCAL_PEERS, kp->key)) {
OBJ_RETAIN(kp);
lclpeers = kp;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s saving local peers %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), lclpeers->data.string);
} else if (0 == strcmp(PMIX_JOBID, kp->key)) {
native_pname.jobid = kp->data.uint32;
} else if (0 == strcmp(PMIX_RANK, kp->key)) {
native_pname.vpid = kp->data.uint32;
}
if (0 == strcmp(attr, kp->key)) {
OBJ_RETAIN(kp);
*kv = kp;
found = true;
}
OBJ_RELEASE(kp);
cnt = 1;
}
OBJ_RELEASE(bptr);
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
return false;
}
} else {
OPAL_ERROR_LOG(ret);
OBJ_RELEASE(cb);
return false;
}
OBJ_RELEASE(cb);
opal_proc_set_name(&native_pname);
/* if the list of local peers wasn't included, then we are done */
if (NULL == lclpeers) {
opal_output_verbose(0, opal_pmix_base_framework.framework_output,
"%s no local peers reported",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
return found;
}
/* baseline all the procs as nonlocal */
myrank = native_pname.vpid;
id.jobid = native_pname.jobid;
#if OPAL_HAVE_HWLOC
/* fetch my cpuset */
OBJ_CONSTRUCT(&vals, opal_list_t);
if (OPAL_SUCCESS == (rc = opal_dstore.fetch(opal_dstore_internal, &native_pname,
OPAL_DSTORE_CPUSET, &vals))) {
kp = (opal_value_t*)opal_list_get_first(&vals);
cpuset = strdup(kp->data.string);
} else {
cpuset = NULL;
}
OPAL_LIST_DESTRUCT(&vals);
#endif
/* we only need to set locality for each local rank as "not found"
* equates to "non local" */
ranks = opal_argv_split(lclpeers->data.string, ',');
for (i=0; NULL != ranks[i]; i++) {
uint32_t vid = strtoul(ranks[i], NULL, 10);
if (myrank == vid) {
continue;
}
id.vpid = vid;
#if OPAL_HAVE_HWLOC
OBJ_CONSTRUCT(&vals, opal_list_t);
if (OPAL_SUCCESS != (rc = opal_dstore.fetch(opal_dstore_internal, &id,
OPAL_DSTORE_CPUSET, &vals))) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s cpuset for local proc %s not found",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
OPAL_NAME_PRINT(id));
OPAL_LIST_DESTRUCT(&vals);
/* even though the cpuset wasn't found, we at least know it is
* on the same node with us */
locality = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
} else {
kp = (opal_value_t*)opal_list_get_first(&vals);
if (NULL == kp->data.string) {
/* if we share a node, but we don't know anything more, then
* mark us as on the node as this is all we know
*/
locality = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
} else {
/* determine relative location on our node */
locality = opal_hwloc_base_get_relative_locality(opal_hwloc_topology,
cpuset,
kp->data.string);
}
OPAL_LIST_DESTRUCT(&vals);
}
#else
/* all we know is we share a node */
locality = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
#endif
OPAL_OUTPUT_VERBOSE((1, opal_pmix_base_framework.framework_output,
"%s pmix:native proc %s locality %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
OPAL_NAME_PRINT(id),
opal_hwloc_base_print_locality(locality)));
OBJ_CONSTRUCT(&kvn, opal_value_t);
kvn.key = strdup(OPAL_DSTORE_LOCALITY);
kvn.type = OPAL_UINT16;
kvn.data.uint16 = locality;
(void)opal_dstore.store(opal_dstore_internal, &id, &kvn);
OBJ_DESTRUCT(&kvn);
}
#if OPAL_HAVE_HWLOC
if (NULL != cpuset) {
free(cpuset);
}
#endif
opal_argv_free(ranks);
return found;
}
static int native_fini(void)
{
opal_buffer_t *msg;
pmix_cb_t *cb;
pmix_cmd_t cmd = PMIX_FINALIZE_CMD;
int rc;
if (1 != init_cntr) {
--init_cntr;
return OPAL_SUCCESS;
}
init_cntr = 0;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native finalize called",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
if (NULL == mca_pmix_native_component.uri) {
/* nothing was setup, so return */
return OPAL_SUCCESS;
}
if (PMIX_USOCK_CONNECTED == mca_pmix_native_component.state) {
/* setup a cmd message to notify the PMIx
* server that we are normally terminating */
msg = OBJ_NEW(opal_buffer_t);
/* pack the cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->active = true;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native sending finalize sync to server",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(msg, wait_cbfunc, cb);
/* wait for the ack to return */
PMIX_WAIT_FOR_COMPLETION(cb->active);
OBJ_RELEASE(cb);
}
if (NULL != mca_pmix_native_component.evbase) {
opal_stop_progress_thread("opal_async", true);
mca_pmix_native_component.evbase = NULL;
}
if (0 <= mca_pmix_native_component.sd) {
CLOSE_THE_SOCKET(mca_pmix_native_component.sd);
}
return OPAL_SUCCESS;
}
PMIX_EXPORT pmix_status_t PMIx_Unpublish_nb(char **keys,
const pmix_info_t info[], size_t ninfo,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_UNPUBLISHNB_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
size_t i, j;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: unpublish called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* create the unpublish cmd */
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack our effective userid - will be used to constrain lookup */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &pmix_globals.uid, 1, PMIX_UINT32))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the number of keys */
i = pmix_argv_count(keys);
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &i, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < i) {
for (j=0; j < i; j++) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &keys[j], 1, PMIX_STRING))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
}
/* pass the number of info structs - needed on remote end so
* space can be malloc'd for the values */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the info structs */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* create a callback object */
cb = PMIX_NEW(pmix_cb_t);
cb->op_cbfunc = cbfunc;
cb->cbdata = cbdata;
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
return PMIX_SUCCESS;
}
static int native_get(const opal_process_name_t *id,
const char *key,
opal_value_t **kv)
{
opal_buffer_t *msg, *bptr;
pmix_cmd_t cmd = PMIX_GET_CMD;
pmix_cb_t *cb;
int rc, ret;
int32_t cnt;
opal_list_t vals;
opal_value_t *kp;
bool found;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native getting value for proc %s key %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
OPAL_NAME_PRINT(*id), key);
/* first see if we already have the info in our dstore */
OBJ_CONSTRUCT(&vals, opal_list_t);
if (OPAL_SUCCESS == opal_dstore.fetch(opal_dstore_internal, id,
key, &vals)) {
*kv = (opal_value_t*)opal_list_remove_first(&vals);
OPAL_LIST_DESTRUCT(&vals);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native value retrieved from dstore",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
return OPAL_SUCCESS;
}
if (NULL == mca_pmix_native_component.uri) {
/* no server available, so just return */
return OPAL_ERR_NOT_FOUND;
}
/* nope - see if we can get it */
msg = OBJ_NEW(opal_buffer_t);
/* pack the get cmd */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, &cmd, 1, PMIX_CMD_T))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
/* pack the request information - we'll get the entire blob
* for this proc, so we don't need to pass the key */
if (OPAL_SUCCESS != (rc = opal_dss.pack(msg, id, 1, OPAL_NAME))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(msg);
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = OBJ_NEW(pmix_cb_t);
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(msg, wait_cbfunc, cb);
/* wait for the data to return */
PMIX_WAIT_FOR_COMPLETION(cb->active);
/* we have received the entire data blob for this process - unpack
* and cache all values, keeping the one we requested to return
* to the caller */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(&cb->data, &ret, &cnt, OPAL_INT))) {
OPAL_ERROR_LOG(rc);
OBJ_RELEASE(cb);
return rc;
}
found = false;
cnt = 1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&cb->data, &bptr, &cnt, OPAL_BUFFER))) {
while (OPAL_SUCCESS == (rc = opal_dss.unpack(bptr, &kp, &cnt, OPAL_VALUE))) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native retrieved %s (%s) from server for proc %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), kp->key,
(OPAL_STRING == kp->type) ? kp->data.string : "NS",
OPAL_NAME_PRINT(*id));
if (OPAL_SUCCESS != (ret = opal_dstore.store(opal_dstore_internal, id, kp))) {
OPAL_ERROR_LOG(ret);
}
if (0 == strcmp(key, kp->key)) {
*kv = kp;
found = true;
} else {
OBJ_RELEASE(kp);
}
}
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
}
OBJ_RELEASE(bptr);
cnt = 1;
}
if (OPAL_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
OPAL_ERROR_LOG(rc);
} else {
rc = OPAL_SUCCESS;
}
OBJ_RELEASE(cb);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:native get completed",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
if (found) {
return OPAL_SUCCESS;
}
/* we didn't find the requested data - pass back a
* status that indicates the source of the problem,
* either during the data fetch, message unpacking,
* or not found */
*kv = NULL;
if (OPAL_SUCCESS == rc) {
if (OPAL_SUCCESS == ret) {
rc = OPAL_ERR_NOT_FOUND;
} else {
rc = ret;
}
}
return rc;
}
PMIX_EXPORT pmix_status_t PMIx_Lookup_nb(char **keys,
const pmix_info_t info[], size_t ninfo,
pmix_lookup_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_LOOKUPNB_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
size_t nkeys, n;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: lookup called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* check for bozo cases */
if (NULL == keys) {
return PMIX_ERR_BAD_PARAM;
}
/* create the lookup cmd */
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack our effective userid - will be used to constrain lookup */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &pmix_globals.uid, 1, PMIX_UINT32))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the keys */
nkeys = pmix_argv_count(keys);
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &nkeys, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < nkeys) {
for (n=0; n < nkeys; n++) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &keys[n], 1, PMIX_STRING))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
}
/* pass the number of info structs - needed on remote end so
* space can be malloc'd for the values */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the info structs */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->lookup_cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_lookup_cbfunc, cb);
return PMIX_SUCCESS;
}
PMIX_EXPORT pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_PUBLISHNB_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: publish called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
return PMIX_ERR_UNREACH;
}
/* check for bozo cases */
if (NULL == info) {
/* nothing to publish */
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
/* create the publish cmd */
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack our effective userid - will be used to constrain lookup */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &pmix_globals.uid, 1, PMIX_UINT32))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pass the number of info structs - needed on remote end so
* space can be malloc'd for the values */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the info structs */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->op_cbfunc = cbfunc;
cb->cbdata = cbdata;
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
return PMIX_SUCCESS;
}
pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t procs[], size_t nprocs,
const pmix_info_t info[], size_t ninfo,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_DISCONNECTNB_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: disconnect called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
return PMIX_ERR_UNREACH;
}
/* check for bozo input */
if (NULL == procs || 0 >= nprocs) {
return PMIX_ERR_BAD_PARAM;
}
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
return rc;
}
/* pack the number of procs */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &nprocs, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
return rc;
}
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, procs, nprocs, PMIX_PROC))) {
PMIX_ERROR_LOG(rc);
return rc;
}
/* pack the info structs */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < ninfo) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->op_cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: disconnect completed");
return PMIX_SUCCESS;
}
PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t ninfo,
const pmix_app_t apps[], size_t napps,
pmix_spawn_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_SPAWNNB_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: spawn called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
return PMIX_ERR_UNREACH;
}
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the job-level directives */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &ninfo, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < ninfo) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, job_info, ninfo, PMIX_INFO))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
/* pack the apps */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &napps, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < napps) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, apps, napps, PMIX_APP))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->spawn_cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
return PMIX_SUCCESS;
}
int PMIx_Unpublish_nb(pmix_data_range_t range, char **keys,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_UNPUBLISHNB_CMD;
int rc;
pmix_cb_t *cb;
size_t i, j;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: unpublish called");
if (pmix_globals.init_cntr <= 0) {
return PMIX_ERR_INIT;
}
/* create the unpublish cmd */
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack our effective userid - will be used to constrain lookup */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &pmix_globals.uid, 1, PMIX_UINT32))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the range */
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &range, 1, PMIX_DATA_RANGE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the number of keys */
i = pmix_argv_count(keys);
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &i, 1, PMIX_SIZE))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < i) {
for (j=0; j < i; j++) {
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(msg, &keys[j], 1, PMIX_STRING))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
}
/* create a callback object */
cb = PMIX_NEW(pmix_cb_t);
cb->op_cbfunc = cbfunc;
cb->cbdata = cbdata;
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
return PMIX_SUCCESS;
}
