int pmixp_libpmix_job_set(void)
{
List lresp;
pmix_info_t *info;
int ninfo;
ListIterator it;
pmix_info_t *kvp;
int i, rc;
uid_t uid = pmixp_info_jobuid();
gid_t gid = pmixp_info_jobgid();
_register_caddy_t *register_caddy;
register_caddy = xmalloc(sizeof(_register_caddy_t)*(pmixp_info_tasks_loc()+1));
pmixp_debug_hang(0);
/* Use list to safely expand/reduce key-value pairs. */
lresp = list_create(pmixp_xfree_xmalloced);
_general_proc_info(lresp);
_set_tmpdirs(lresp);
_set_procdatas(lresp);
_set_sizeinfo(lresp);
if (SLURM_SUCCESS != _set_mapsinfo(lresp)) {
list_destroy(lresp);
PMIXP_ERROR("Can't build nodemap");
return SLURM_ERROR;
}
_set_localinfo(lresp);
ninfo = list_count(lresp);
PMIX_INFO_CREATE(info, ninfo);
it = list_iterator_create(lresp);
i = 0;
while (NULL != (kvp = list_next(it))) {
info[i] = *kvp;
i++;
}
list_destroy(lresp);
register_caddy[0].active = 1;
rc = PMIx_server_register_nspace(pmixp_info_namespace(),
pmixp_info_tasks_loc(), info, ninfo, _release_cb,
®ister_caddy[0]);
if (PMIX_SUCCESS != rc) {
PMIXP_ERROR("Cannot register namespace %s, nlocalproc=%d, "
"ninfo = %d", pmixp_info_namespace(),
pmixp_info_tasks_loc(), ninfo);
return SLURM_ERROR;
}
PMIXP_DEBUG("task initialization");
for (i = 0; i < pmixp_info_tasks_loc(); i++) {
pmix_proc_t proc;
register_caddy[i+1].active = 1;
strncpy(proc.nspace, pmixp_info_namespace(), PMIX_MAX_NSLEN);
proc.rank = pmixp_info_taskid(i);
rc = PMIx_server_register_client(&proc, uid, gid, NULL,
_release_cb, ®ister_caddy[i + 1]);
if (PMIX_SUCCESS != rc) {
PMIXP_ERROR("Cannot register client %d(%d) in namespace %s",
pmixp_info_taskid(i), i,
pmixp_info_namespace());
return SLURM_ERROR;
}
}
/* wait for all registration actions to finish */
while( 1 ){
int exit_flag = 1;
struct timespec ts;
ts.tv_sec = 0;
ts.tv_nsec = 100;
for(i=0; i < pmixp_info_tasks_loc() + 1; i++){
if( register_caddy[i].active ){
exit_flag = 0;
}
}
if( exit_flag ){
break;
}
nanosleep(&ts, NULL);
}
PMIX_INFO_FREE(info, ninfo);
xfree(register_caddy);
return SLURM_SUCCESS;
}
int launch_clients(int num_procs, char *binary, char *** client_env, char ***base_argv)
{
int n;
uid_t myuid;
gid_t mygid;
char *ranks = NULL;
char digit[MAX_DIGIT_LEN];
int rc;
static int counter = 0;
static int num_ns = 0;
pmix_proc_t proc;
TEST_VERBOSE(("Setting job info"));
fill_seq_ranks_array(num_procs, counter, &ranks);
if (NULL == ranks) {
PMIx_server_finalize();
TEST_ERROR(("fill_seq_ranks_array failed"));
return PMIX_ERROR;
}
(void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, num_ns);
set_namespace(num_procs, ranks, proc.nspace);
if (NULL != ranks) {
free(ranks);
}
myuid = getuid();
mygid = getgid();
/* fork/exec the test */
for (n = 0; n < num_procs; n++) {
proc.rank = counter;
if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n
TEST_ERROR(("Server fork setup failed with error %d", rc));
PMIx_server_finalize();
cli_kill_all();
return rc;
}
if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL))) {//n
TEST_ERROR(("Server fork setup failed with error %d", rc));
PMIx_server_finalize();
cli_kill_all();
return rc;
}
cli_info[counter].pid = fork();
if (cli_info[counter].pid < 0) {
TEST_ERROR(("Fork failed"));
PMIx_server_finalize();
cli_kill_all();
return -1;
}
cli_info[counter].rank = counter;//n
cli_info[counter].ns = strdup(proc.nspace);
char **client_argv = pmix_argv_copy(*base_argv);
/* add two last arguments: -r <rank> */
sprintf(digit, "%d", counter);//n
pmix_argv_append_nosize(&client_argv, "-r");
pmix_argv_append_nosize(&client_argv, digit);
pmix_argv_append_nosize(&client_argv, "-s");
pmix_argv_append_nosize(&client_argv, proc.nspace);
sprintf(digit, "%d", num_procs);
pmix_argv_append_nosize(&client_argv, "--ns-size");
pmix_argv_append_nosize(&client_argv, digit);
sprintf(digit, "%d", num_ns);
pmix_argv_append_nosize(&client_argv, "--ns-id");
pmix_argv_append_nosize(&client_argv, digit);
sprintf(digit, "%d", (counter-n));
pmix_argv_append_nosize(&client_argv, "--base-rank");
pmix_argv_append_nosize(&client_argv, digit);
if (cli_info[counter].pid == 0) {
if( !TEST_VERBOSE_GET() ){
// Hide clients stdout
// TODO: on some systems stdout is a constant, address this
fclose(stdout);
stdout = fopen("/dev/null","w");
}
execve(binary, client_argv, *client_env);
/* Does not return */
exit(0);
}
cli_info[counter].state = CLI_FORKED;
pmix_argv_free(client_argv);
counter++;
}
num_ns++;
return PMIX_SUCCESS;
}