static void _dmdx_pmix_cb(pmix_status_t status, char *data, size_t sz,
void *cbdata)
{
dmdx_caddy_t *caddy = (dmdx_caddy_t *)cbdata;
Buf buf = pmixp_server_new_buf();
char *addr;
int rc;
/* setup response header */
_setup_header(buf, DMDX_RESPONSE, caddy->proc.nspace, caddy->proc.rank,
status);
/* pack the response */
packmem(data, sz, buf);
/* setup response address */
addr = pmixp_info_nspace_usock(caddy->sender_ns);
/* send the request */
rc = pmixp_server_send(caddy->sender_host, PMIXP_MSG_DMDX,
caddy->seq_num, addr, get_buf_data(buf),
get_buf_offset(buf), 1);
if (SLURM_SUCCESS != rc) {
/* not much we can do here. Caller will react by timeout */
PMIXP_ERROR("Cannot send direct modex response to %s",
caddy->sender_host);
}
xfree(addr);
free_buf(buf);
_dmdx_free_caddy(caddy);
}
int pmixp_stepd_finalize(void)
{
char *path;
if (!_was_initialized) {
/* nothing to do */
return 0;
}
pmixp_libpmix_finalize();
pmixp_dmdx_finalize();
pmixp_conn_fini();
pmixp_dconn_fini();
pmixp_state_finalize();
pmixp_nspaces_finalize();
/* cleanup the UNIX socket */
PMIXP_DEBUG("Remove PMIx plugin usock");
close(pmixp_info_srv_usock_fd());
path = pmixp_info_nspace_usock(pmixp_info_namespace());
unlink(path);
xfree(path);
/* free the information */
pmixp_info_free();
return SLURM_SUCCESS;
}
int pmixp_dmdx_get(const char *nspace, int rank,
pmix_modex_cbfunc_t cbfunc, void *cbdata)
{
dmdx_req_info_t *req;
char *addr, *host;
Buf buf;
int rc;
uint32_t seq;
/* need to send the request */
host = pmixp_nspace_resolve(nspace, rank);
xassert(NULL != host);
if (NULL == host) {
return SLURM_ERROR;
}
buf = pmixp_server_new_buf();
/* setup message header */
_setup_header(buf, DMDX_REQUEST, nspace, rank, SLURM_SUCCESS);
/* generate namespace usocket name */
addr = pmixp_info_nspace_usock(nspace);
/* store cur seq. num and move to the next request */
seq = _dmdx_seq_num++;
/* track this request */
req = xmalloc(sizeof(dmdx_req_info_t));
req->seq_num = seq;
req->cbfunc = cbfunc;
req->cbdata = cbdata;
req->ts = time(NULL);
#ifndef NDEBUG
strncpy(req->nspace, nspace, PMIX_MAX_NSLEN);
req->rank = rank;
#endif
list_append(_dmdx_requests, req);
/* send the request */
rc = pmixp_server_send(host, PMIXP_MSG_DMDX, seq, addr,
get_buf_data(buf), get_buf_offset(buf), 1);
/* cleanup the resources */
xfree(addr);
free_buf(buf);
/* check the return status */
if (SLURM_SUCCESS != rc) {
PMIXP_ERROR("Cannot send direct modex request to %s", host);
cbfunc(PMIX_ERROR, NULL, 0, cbdata, NULL, NULL);
return SLURM_ERROR;
}
return rc;
}
static void _respond_with_error(int seq_num, char *sender_host,
char *sender_ns, int status)
{
Buf buf = create_buf(NULL, 0);
char *addr;
int rc;
/* rank doesn't matter here, don't send it */
_setup_header(buf, DMDX_RESPONSE, pmixp_info_namespace(), -1, status);
/* generate namespace usocket name */
addr = pmixp_info_nspace_usock(sender_ns);
/* send response */
rc = pmixp_server_send(sender_host, PMIXP_MSG_DMDX, seq_num, addr,
get_buf_data(buf), get_buf_offset(buf), 1);
if (SLURM_SUCCESS != rc) {
PMIXP_ERROR("Cannot send direct modex error" " response to %s",
sender_host);
}
xfree(addr);
free_buf(buf);
}
int pmixp_stepd_init(const stepd_step_rec_t *job, char ***env)
{
char *path;
int fd, rc;
if (SLURM_SUCCESS != (rc = pmixp_info_set(job, env))) {
PMIXP_ERROR("pmixp_info_set(job, env) failed");
goto err_info;
}
/* Create UNIX socket for slurmd communication */
path = pmixp_info_nspace_usock(pmixp_info_namespace());
if (NULL == path) {
PMIXP_ERROR("pmixp_info_nspace_usock: out-of-memory");
rc = SLURM_ERROR;
goto err_path;
}
if ((fd = pmixp_usock_create_srv(path)) < 0) {
PMIXP_ERROR("pmixp_usock_create_srv");
rc = SLURM_ERROR;
goto err_usock;
}
fd_set_close_on_exec(fd);
pmixp_info_srv_usock_set(path, fd);
if (!pmixp_info_same_arch()){
_direct_proto.hdr_unpack_cb = _direct_hdr_unpack_portable;
_direct_hdr_pack = _direct_hdr_pack_portable;
}
pmixp_conn_init(_slurm_proto, _direct_proto);
if((rc = pmixp_dconn_init(pmixp_info_nodes_uni(), _direct_proto)) ){
PMIXP_ERROR("pmixp_dconn_init() failed");
goto err_dconn;
}
if ((rc = pmixp_nspaces_init())) {
PMIXP_ERROR("pmixp_nspaces_init() failed");
goto err_nspaces;
}
if (SLURM_SUCCESS != (rc = pmixp_state_init())) {
PMIXP_ERROR("pmixp_state_init() failed");
goto err_state;
}
if (SLURM_SUCCESS != (rc = pmixp_dmdx_init())) {
PMIXP_ERROR("pmixp_dmdx_init() failed");
goto err_dmdx;
}
if (SLURM_SUCCESS != (rc = pmixp_libpmix_init())) {
PMIXP_ERROR("pmixp_libpmix_init() failed");
goto err_lib;
}
if (SLURM_SUCCESS != (rc = pmixp_libpmix_job_set())) {
PMIXP_ERROR("pmixp_libpmix_job_set() failed");
goto err_job;
}
pmixp_server_init_pp(env);
pmixp_server_init_cperf(env);
xfree(path);
_was_initialized = 1;
return SLURM_SUCCESS;
err_job:
pmixp_libpmix_finalize();
err_lib:
pmixp_dmdx_finalize();
err_dmdx:
pmixp_state_finalize();
err_state:
pmixp_nspaces_finalize();
err_nspaces:
pmixp_dconn_fini();
err_dconn:
pmixp_conn_fini();
close(pmixp_info_srv_usock_fd());
err_usock:
xfree(path);
err_path:
pmixp_info_free();
err_info:
return rc;
}
int pmixp_stepd_init(const stepd_step_rec_t *job, char ***env)
{
char *path;
int fd, rc;
if (SLURM_SUCCESS != (rc = pmixp_info_set(job, env))) {
PMIXP_ERROR("pmixp_info_set(job, env) failed");
return rc;
}
/* Create UNIX socket for slurmd communication */
path = pmixp_info_nspace_usock(pmixp_info_namespace());
if (NULL == path) {
PMIXP_ERROR("Out-of-memory");
rc = SLURM_ERROR;
goto err_path;
}
if ((fd = pmixp_usock_create_srv(path)) < 0) {
rc = SLURM_ERROR;
goto err_usock;
}
fd_set_close_on_exec(fd);
pmixp_info_srv_contacts(path, fd);
if (SLURM_SUCCESS != (rc = pmixp_nspaces_init())) {
PMIXP_ERROR("pmixp_nspaces_init() failed");
goto err_usock;
}
if (SLURM_SUCCESS != (rc = pmixp_state_init())) {
PMIXP_ERROR("pmixp_state_init() failed");
goto err_state;
}
if (SLURM_SUCCESS != (rc = pmixp_dmdx_init())) {
PMIXP_ERROR("pmixp_dmdx_init() failed");
goto err_dmdx;
}
if (SLURM_SUCCESS != (rc = pmixp_libpmix_init())) {
PMIXP_ERROR("pmixp_libpmix_init() failed");
goto err_lib;
}
if (SLURM_SUCCESS != (rc = pmixp_libpmix_job_set())) {
PMIXP_ERROR("pmixp_libpmix_job_set() failed");
goto err_job;
}
xfree(path);
_was_initialized = 1;
return SLURM_SUCCESS;
err_job:
pmixp_libpmix_finalize();
err_lib:
pmixp_dmdx_finalize();
err_dmdx:
pmixp_state_finalize();
err_state:
pmixp_nspaces_finalize();
err_usock:
xfree(path);
err_path:
pmixp_info_free();
return rc;
}
