int pmixp_libpmix_init(void)
{
int rc;
mode_t rights = (S_IRUSR | S_IWUSR | S_IXUSR) | (S_IRGRP | S_IWGRP | S_IXGRP);
pmix_info_t *kvp;
/* NOTE: we need user who owns the job to access PMIx usock
* file. According to 'man 7 unix':
* "... In the Linux implementation, sockets which are visible in the file system
* honor the permissions of the directory they are in... "
* Our case is the following: slurmstepd is usually running as root, user application will
* be "sudo'ed". To provide both of them with acces to the unix socket we do the following:
* 1. Owner ID is set to the job owner.
* 2. Group ID corresponds to slurmstepd.
* 3. Set 0770 access mode */
if (0 != mkdir(pmixp_info_tmpdir_lib(), rights) ) {
PMIXP_ERROR_STD("Cannot create directory \"%s\"", pmixp_info_tmpdir_lib());
return errno;
}
/* There might be umask that will drop essential rights. Fix it explicitly.
* TODO: is there more elegant solution? */
if (chmod(pmixp_info_tmpdir_lib(), rights) < 0) {
error("chown(%s): %m", pmixp_info_tmpdir_lib());
return errno;
}
if (chown(pmixp_info_tmpdir_lib(), (uid_t) pmixp_info_jobuid(), (gid_t) -1) < 0) {
error("chown(%s): %m", pmixp_info_tmpdir_lib());
return errno;
}
setenv(PMIXP_PMIXLIB_TMPDIR, pmixp_info_tmpdir_lib(), 1);
PMIXP_ALLOC_KEY(kvp, PMIX_USERID);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_jobuid());
/* setup the server library */
if (PMIX_SUCCESS != (rc = PMIx_server_init(&_slurm_pmix_cb, kvp, 1))) {
PMIXP_ERROR_STD("PMIx_server_init failed with error %d\n", rc);
return SLURM_ERROR;
}
PMIXP_FREE_KEY(kvp);
/*
if( pmixp_fixrights(pmixp_info_tmpdir_lib(),
(uid_t) pmixp_info_jobuid(), rights) ){
}
*/
/* register the errhandler */
PMIx_Register_errhandler(NULL, 0, errhandler, errhandler_reg_callbk,
NULL);
return 0;
}
int pmixp_mkdir(char *path, mode_t rights)
{
/* NOTE: we need user who owns the job to access PMIx usock
* file. According to 'man 7 unix':
* "... In the Linux implementation, sockets which are visible in the
* file system honor the permissions of the directory they are in... "
* Our case is the following: slurmstepd is usually running as root,
* user application will be "sudo'ed". To provide both of them with
* access to the unix socket we do the following:
* 1. Owner ID is set to the job owner.
* 2. Group ID corresponds to slurmstepd.
* 3. Set 0770 access mode
*/
if (0 != mkdir(path, rights) ) {
PMIXP_ERROR_STD("Cannot create directory \"%s\"",
path);
return errno;
}
/* There might be umask that will drop essential rights.
* Fix it explicitly.
* TODO: is there more elegant solution? */
if (chmod(path, rights) < 0) {
error("%s: chown(%s): %m", __func__, path);
return errno;
}
if (chown(path, (uid_t) pmixp_info_jobuid(), (gid_t) -1) < 0) {
error("%s: chown(%s): %m", __func__, path);
return errno;
}
return 0;
}
int pmixp_lib_init(void)
{
pmix_info_t *kvp = NULL;
pmix_status_t rc;
PMIXP_INFO_ADD(kvp, PMIX_USERID, uint32_t, pmixp_info_jobuid());
#ifdef PMIX_SERVER_TMPDIR
PMIXP_INFO_ADD(kvp, PMIX_SERVER_TMPDIR, string,
pmixp_info_tmpdir_lib());
#endif
/* setup the server library */
if (PMIX_SUCCESS != (rc = PMIx_server_init(&slurm_pmix_cb, kvp,
PMIXP_INFO_SIZE(kvp)))) {
PMIXP_ERROR_STD("PMIx_server_init failed with error %d\n", rc);
return SLURM_ERROR;
}
PMIXP_FREE_KEY(kvp);
/* register the errhandler */
PMIx_Register_event_handler(NULL, 0, NULL, 0, _errhandler,
_errhandler_reg_callbk, NULL);
return SLURM_SUCCESS;
}
int pmixp_libpmix_job_set(void)
{
List lresp;
pmix_info_t *info;
int ninfo;
ListIterator it;
pmix_info_t *kvp;
int i, rc;
uid_t uid = pmixp_info_jobuid();
gid_t gid = pmixp_info_jobgid();
_register_caddy_t *register_caddy;
register_caddy = xmalloc(sizeof(_register_caddy_t)*(pmixp_info_tasks_loc()+1));
pmixp_debug_hang(0);
/* Use list to safely expand/reduce key-value pairs. */
lresp = list_create(pmixp_xfree_xmalloced);
_general_proc_info(lresp);
_set_tmpdirs(lresp);
_set_procdatas(lresp);
_set_sizeinfo(lresp);
if (SLURM_SUCCESS != _set_mapsinfo(lresp)) {
list_destroy(lresp);
PMIXP_ERROR("Can't build nodemap");
return SLURM_ERROR;
}
_set_localinfo(lresp);
ninfo = list_count(lresp);
PMIX_INFO_CREATE(info, ninfo);
it = list_iterator_create(lresp);
i = 0;
while (NULL != (kvp = list_next(it))) {
info[i] = *kvp;
i++;
}
list_destroy(lresp);
register_caddy[0].active = 1;
rc = PMIx_server_register_nspace(pmixp_info_namespace(),
pmixp_info_tasks_loc(), info, ninfo, _release_cb,
®ister_caddy[0]);
if (PMIX_SUCCESS != rc) {
PMIXP_ERROR("Cannot register namespace %s, nlocalproc=%d, "
"ninfo = %d", pmixp_info_namespace(),
pmixp_info_tasks_loc(), ninfo);
return SLURM_ERROR;
}
PMIXP_DEBUG("task initialization");
for (i = 0; i < pmixp_info_tasks_loc(); i++) {
pmix_proc_t proc;
register_caddy[i+1].active = 1;
strncpy(proc.nspace, pmixp_info_namespace(), PMIX_MAX_NSLEN);
proc.rank = pmixp_info_taskid(i);
rc = PMIx_server_register_client(&proc, uid, gid, NULL,
_release_cb, ®ister_caddy[i + 1]);
if (PMIX_SUCCESS != rc) {
PMIXP_ERROR("Cannot register client %d(%d) in namespace %s",
pmixp_info_taskid(i), i,
pmixp_info_namespace());
return SLURM_ERROR;
}
}
/* wait for all registration actions to finish */
while( 1 ){
int exit_flag = 1;
struct timespec ts;
ts.tv_sec = 0;
ts.tv_nsec = 100;
for(i=0; i < pmixp_info_tasks_loc() + 1; i++){
if( register_caddy[i].active ){
exit_flag = 0;
}
}
if( exit_flag ){
break;
}
nanosleep(&ts, NULL);
}
PMIX_INFO_FREE(info, ninfo);
xfree(register_caddy);
return SLURM_SUCCESS;
}
