static void _set_localinfo(List lresp)
{
pmix_info_t *kvp;
uint32_t tmp;
char *p = NULL;
int i;
xstrfmtcat(p, "%u", pmixp_info_taskid(0));
tmp = pmixp_info_taskid(0);
for (i = 1; i < pmixp_info_tasks_loc(); i++) {
uint32_t rank = pmixp_info_taskid(i);
xstrfmtcat(p, ",%u", rank);
if (tmp > rank) {
tmp = rank;
}
}
PMIXP_ALLOC_KEY(kvp, PMIX_LOCAL_PEERS);
PMIX_VAL_SET(&kvp->value, string, p);
xfree(p);
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_LOCALLDR);
PMIX_VAL_SET(&kvp->value, uint32_t, tmp);
list_append(lresp, kvp);
}
static void _set_sizeinfo(List lresp)
{
pmix_info_t *kvp;
/* size information */
PMIXP_ALLOC_KEY(kvp, PMIX_UNIV_SIZE);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_tasks_uni());
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_JOB_SIZE);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_tasks());
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_LOCAL_SIZE);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_tasks_loc());
list_append(lresp, kvp);
/* TODO: fix it in future */
PMIXP_ALLOC_KEY(kvp, PMIX_NODE_SIZE);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_tasks_loc());
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_MAX_PROCS);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_tasks_uni());
list_append(lresp, kvp);
}
static int _set_mapsinfo(List lresp)
{
pmix_info_t *kvp;
char *regexp, *input;
pmixp_namespace_t *nsptr = pmixp_nspaces_local();
hostlist_t hl = nsptr->hl;
int rc, i, j;
int count = hostlist_count(hl);
input = hostlist_deranged_string_malloc(hl);
rc = PMIx_generate_regex(input, ®exp);
free(input);
if (PMIX_SUCCESS != rc) {
return SLURM_ERROR;
}
PMIXP_ALLOC_KEY(kvp, PMIX_NODE_MAP);
PMIX_VAL_SET(&kvp->value, string, regexp);
regexp = NULL;
list_append(lresp, kvp);
input = NULL;
for (i = 0; i < count; i++) {
/* for each node - run through all tasks and
* record taskid's that reside on this node
*/
int first = 1;
for (j = 0; j < nsptr->ntasks; j++) {
if (nsptr->task_map[j] == i) {
if (first) {
first = 0;
} else {
xstrfmtcat(input, ",");
}
xstrfmtcat(input, "%u", j);
}
}
if (i < (count - 1)) {
xstrfmtcat(input, ";");
}
}
rc = PMIx_generate_ppn(input, ®exp);
xfree(input);
if (PMIX_SUCCESS != rc) {
return SLURM_ERROR;
}
PMIXP_ALLOC_KEY(kvp, PMIX_PROC_MAP);
PMIX_VAL_SET(&kvp->value, string, regexp);
regexp = NULL;
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_ANL_MAP);
PMIX_VAL_SET(&kvp->value, string, pmixp_info_task_map());
regexp = NULL;
list_append(lresp, kvp);
return SLURM_SUCCESS;
}
int pmixp_libpmix_init(void)
{
int rc;
mode_t rights = (S_IRUSR | S_IWUSR | S_IXUSR) | (S_IRGRP | S_IWGRP | S_IXGRP);
pmix_info_t *kvp;
/* NOTE: we need user who owns the job to access PMIx usock
* file. According to 'man 7 unix':
* "... In the Linux implementation, sockets which are visible in the file system
* honor the permissions of the directory they are in... "
* Our case is the following: slurmstepd is usually running as root, user application will
* be "sudo'ed". To provide both of them with acces to the unix socket we do the following:
* 1. Owner ID is set to the job owner.
* 2. Group ID corresponds to slurmstepd.
* 3. Set 0770 access mode */
if (0 != mkdir(pmixp_info_tmpdir_lib(), rights) ) {
PMIXP_ERROR_STD("Cannot create directory \"%s\"", pmixp_info_tmpdir_lib());
return errno;
}
/* There might be umask that will drop essential rights. Fix it explicitly.
* TODO: is there more elegant solution? */
if (chmod(pmixp_info_tmpdir_lib(), rights) < 0) {
error("chown(%s): %m", pmixp_info_tmpdir_lib());
return errno;
}
if (chown(pmixp_info_tmpdir_lib(), (uid_t) pmixp_info_jobuid(), (gid_t) -1) < 0) {
error("chown(%s): %m", pmixp_info_tmpdir_lib());
return errno;
}
setenv(PMIXP_PMIXLIB_TMPDIR, pmixp_info_tmpdir_lib(), 1);
PMIXP_ALLOC_KEY(kvp, PMIX_USERID);
PMIX_VAL_SET(&kvp->value, uint32_t, pmixp_info_jobuid());
/* setup the server library */
if (PMIX_SUCCESS != (rc = PMIx_server_init(&_slurm_pmix_cb, kvp, 1))) {
PMIXP_ERROR_STD("PMIx_server_init failed with error %d\n", rc);
return SLURM_ERROR;
}
PMIXP_FREE_KEY(kvp);
/*
if( pmixp_fixrights(pmixp_info_tmpdir_lib(),
(uid_t) pmixp_info_jobuid(), rights) ){
}
*/
/* register the errhandler */
PMIx_Register_errhandler(NULL, 0, errhandler, errhandler_reg_callbk,
NULL);
return 0;
}
/*
* scratch directory locations for use by applications
*/
static void _set_tmpdirs(List lresp)
{
pmix_info_t *kvp;
char *p = NULL;
/* We consider two sources of the tempdir:
* - SLURM's slurm.conf TmpFS option;
* - env var SLURM_PMIX_TMPDIR;
* do we need to do anything else?
*/
p = pmixp_info_tmpdir_cli();
if (NULL == p) {
p = PMIXP_TMPDIR_DEFAULT;
}
PMIXP_ALLOC_KEY(kvp, PMIX_TMPDIR);
PMIX_VAL_SET(&kvp->value, string, p);
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_NSDIR);
PMIX_VAL_SET(&kvp->value, string, p);
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_PROCDIR);
PMIX_VAL_SET(&kvp->value, string, p);
list_append(lresp, kvp);
}
int main(int argc, char **argv)
{
int spawned;
int rc;
/* init us */
if (PMI_SUCCESS != (rc = PMI_Init(&spawned))) {
fprintf(stderr, "PMI_Init failed: %d\n", rc);
return rc;
}
#if 0
key = "local-key";
PMIX_VAL_SET(&value, int, 12345, rc, kvp_error );
if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, key, &value))) {
fprintf(stderr, "PMIx_Put failed: %d\n", rc);
}
key = "remote-key";
char *ptr = "Test string";
PMIX_VAL_SET(&value, string, ptr, rc, kvp_error );
if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, key, &value))) {
fprintf(stderr, "PMIx_Put failed: %d\n", rc);
}
key = "global-key";
PMIX_VAL_SET(&value, float, 10.15, rc, kvp_error );
if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, key, &value))) {
fprintf(stderr, "PMIx_Put failed: %d\n", rc);
}
/* Submit the data */
pmix_range_t range;
range.ranks = NULL;
range.nranks = 0;
if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0))) {
fprintf(stderr, "PMIx_Fence failed: %d\n", rc);
return rc;
}
#endif
/* finalize us */
if (PMI_SUCCESS != (rc = PMI_Finalize())) {
fprintf(stderr, "PMI_Finalize failed: %d\n", rc);
}
return rc;
}
/*
* general proc-level attributes
*/
static void _general_proc_info(List lresp)
{
pmix_info_t *kvp;
/* TODO: how can we get this information in SLURM?
* PMIXP_ALLOC_KEY(kvp, PMIX_CPUSET);
* PMIX_VAL_SET(&kvp->value, string, "");
* list_append(lresp, kvp);
* TODO: what should we provide for credentials?
* #define PMIX_CREDENTIAL "pmix.cred"
* TODO: Once spawn will be implemented we'll need to check here
*/
PMIXP_ALLOC_KEY(kvp, PMIX_SPAWNED);
PMIX_VAL_SET(&kvp->value, flag, 0);
list_append(lresp, kvp);
/*
* TODO: what is the portable way to get arch string?
* #define PMIX_ARCH "pmix.arch"
*/
}
/*
* information about relative ranks as assigned by the RM
*/
static void _set_procdatas(List lresp)
{
pmixp_namespace_t *nsptr = pmixp_nspaces_local();
pmix_info_t *kvp, *tkvp;
char *p = NULL;
int i;
/* (char*) jobid assigned by scheduler */
xstrfmtcat(p, "%d.%d", pmixp_info_jobid(), pmixp_info_stepid());
PMIXP_ALLOC_KEY(kvp, PMIX_JOBID);
PMIX_VAL_SET(&kvp->value, string, p);
xfree(p);
list_append(lresp, kvp);
PMIXP_ALLOC_KEY(kvp, PMIX_NODEID);
PMIX_VAL_SET(&kvp->value, uint32_t, nsptr->node_id);
list_append(lresp, kvp);
/* store information about local processes */
for (i = 0; i < pmixp_info_tasks(); i++) {
List rankinfo;
ListIterator it;
int count, j, localid, nodeid;
char *nodename;
pmix_info_t *info;
rankinfo = list_create(pmixp_xfree_xmalloced);
PMIXP_ALLOC_KEY(kvp, PMIX_RANK);
PMIX_VAL_SET(&kvp->value, int, i);
list_append(rankinfo, kvp);
/* TODO: always use 0 so far. this is not the general case though
* (see SLURM MIMD: man srun, section MULTIPLE PROGRAM CONFIGURATION)
*/
PMIXP_ALLOC_KEY(kvp, PMIX_APPNUM);
PMIX_VAL_SET(&kvp->value, int, 0);
list_append(rankinfo, kvp);
/* TODO: the same as for previous here */
PMIXP_ALLOC_KEY(kvp, PMIX_APPLDR);
PMIX_VAL_SET(&kvp->value, int, 0);
list_append(rankinfo, kvp);
/* TODO: fix when several apps will appear */
PMIXP_ALLOC_KEY(kvp, PMIX_GLOBAL_RANK);
PMIX_VAL_SET(&kvp->value, uint32_t, i);
list_append(rankinfo, kvp);
/* TODO: fix when several apps will appear */
PMIXP_ALLOC_KEY(kvp, PMIX_APP_RANK);
PMIX_VAL_SET(&kvp->value, uint32_t, i);
list_append(rankinfo, kvp);
localid = pmixp_info_taskid2localid(i);
/* this rank is local, store local info ab't it! */
if (0 <= localid) {
PMIXP_ALLOC_KEY(kvp, PMIX_LOCAL_RANK);
PMIX_VAL_SET(&kvp->value, uint16_t, localid);
list_append(rankinfo, kvp);
/* TODO: fix when several apps will appear */
PMIXP_ALLOC_KEY(kvp, PMIX_NODE_RANK);
PMIX_VAL_SET(&kvp->value, uint16_t, localid);
list_append(rankinfo, kvp);
}
nodeid = nsptr->task_map[i];
nodename = hostlist_nth(nsptr->hl, nodeid);
PMIXP_ALLOC_KEY(kvp, PMIX_HOSTNAME);
PMIX_VAL_SET(&kvp->value, string, nodename);
list_append(rankinfo, kvp);
free(nodename);
/* merge rankinfo into one PMIX_PROC_DATA key */
count = list_count(rankinfo);
PMIXP_ALLOC_KEY(kvp, PMIX_PROC_DATA);
kvp->value.type = PMIX_INFO_ARRAY;
kvp->value.data.array.size = count;
PMIX_INFO_CREATE(info, count);
it = list_iterator_create(rankinfo);
j = 0;
while (NULL != (tkvp = list_next(it))) {
/* Just copy all the fields here. We will free original kvp's
* using list_destroy without free'ing their fields so it is
* safe to do so.
*/
info[j] = *tkvp;
j++;
}
list_destroy(rankinfo);
kvp->value.data.array.array = (pmix_info_t *)info;
info = NULL;
/* put the complex key to the list */
list_append(lresp, kvp);
}
}
int main(int argc, char **argv, const char **environ)
{
pmix_status_t rc;
pmix_info_t *info = NULL;
bool flag;
pmix_status_t retval;
pmix_app_t *spawned_app = NULL;
pmix_info_t *job_info = NULL;
pmix_info_t *proc_info = NULL;
int job_info_count = 0;
int job_info_index = 0;
int proc_info_count = 0;
int proc_info_index = 0;
char spawned_nsp[PMIX_MAX_NSLEN+1];
char *path_to_app = NULL;
char *host_to_use = NULL;
int number_of_clients = 0;
int temp_counter = 0;
done_flag = false;
gethostname(hostn, 500);
int spawned_app_argc = 0;
char **scr_environ = NULL;
int proc_count = 1;
int node_count = 0;
bool blocking_mode = true;
char *node_list = NULL;
bool forward_all_scr_envs = false;
bool pmix_mode = false;
const char *optstring = "+n:N:L:x:bB:pPvhe";
int temp_slen=0;
/* todo: add arg parsing with ompi schizo */
verbose_print = false;
int sleep_max = 30;
const int fixed_sleep = 5;
int c;
while((c = getopt(argc, argv, optstring)) != -1){
switch(c){
case 'h':
print_usage(argv[0]);
exit(0);
break;
case 'n':
proc_count = atoi(optarg);
if(proc_count <= 0 || proc_count > 100){
printf("outside the range of allowable instances to spawn [1-100]\n");
exit(1);
}
if(verbose_print) {
printf("proc_count = %d\n", proc_count);
}
break;
case 'N':
/* node_count = atoi(optarg); */
node_count = 1;
if(verbose_print) {
printf("node_count = %d\n", node_count);
}
break;
case 'B':
blocking_mode = true;
sleep_max = atoi(optarg);
if(sleep_max < 0){
printf("can't sleep for less than 0 seconds\n");
exit(1);
}
if(verbose_print){
printf("blocking mode = %x\n", blocking_mode);
}
break;
case 'b':
blocking_mode = false;
if(verbose_print){
printf("blocking mode = %x\n", blocking_mode);
}
break;
case 'L':
node_list = optarg;
host_to_use = node_list;
if(verbose_print){
printf("node_list = '%s'\n", node_list);
}
break;
case 'x':
temp_slen = strlen(optarg);
/* check if the string is the same length as 'SCR', if so compare them */
if(temp_slen == strlen(SCR_STRING)){
if(strncmp(optarg, SCR_STRING, strlen(SCR_STRING)) == 0){
/* if the string is SCR, then forward all SCR related env vars */
if(verbose_print) printf("all scr envs will be forwarded\n");
forward_all_scr_envs = true;
}
else{
/* handled like a normal env var */
handle_standard_env_var(optarg, &scr_environ);
}
}
else{
/*handled like a normal env var */
handle_standard_env_var(optarg, &scr_environ);
}
break;
case 'v':
verbose_print = true;
break;
case 'p':
pmix_mode = true;
if(verbose_print){
printf("pmix_mode = %x\n", pmix_mode);
}
break;
case 'P':
pmix_mode = false;
if(verbose_print){
printf("pmix_mode = %x\n", pmix_mode);
}
break;
case 'e':
experimental = true;
break;
case '?':
printf("missing a required argument or invalid option: %x\n", optopt);
print_usage(argv[0]);
exit(1);
break;
default:
printf("Unrecognized argument: %c\n", c);
print_usage(argv[0]);
exit(1);
break;
}
}
/* number of instances to spawn */
number_of_clients = proc_count;
/* check to make sure an application was specified to launch */
if( optind < argc ){
/* if optind is < argc, it means there is at least one more arg
* beyond the args for this program */
path_to_app = argv[optind];
spawned_app_argc = argc - optind;
if(verbose_print) {
printf("app to launch: %s @ %s:%d\n",
path_to_app, __FILE__, __LINE__);
}
}
else{
printf("program_to_spawn option was not provded\n");
print_usage(argv[0]);
exit(1);
}
if(verbose_print){
printf("master process will spawn %d instances; app to run: %s\n\n",
number_of_clients, path_to_app);
printf("pmix version: %s (host: %s)\n", PMIx_Get_version(), hostn);
}
/* init pmix */
retval = PMIx_Init(&main_proc, NULL, 0);
if(retval != PMIX_SUCCESS){
error_helper(retval, hostn, "error initializing pmix");
exit(0);
}
if(verbose_print){
printf("rank %d, host '%s', nspace: '%s' init'd pmix succesfully\n\n",
main_proc.rank, hostn, main_proc.nspace);
}
/* we need to attach to a "system" PMIx server so we
* can ask it to spawn applications for us. There can
* only be one such connection on a node, so we will
* instruct the tool library to only look for it */
int ninfo = 1;
PMIX_INFO_CREATE(info, ninfo);
flag = true;
PMIX_INFO_LOAD(&info[0], PMIX_CONNECT_TO_SYSTEM, &flag, PMIX_BOOL);
/* initialize the library and make the connection */
if (PMIX_SUCCESS != (rc = PMIx_tool_init(&tool_proc, NULL, 0 ))) {
fprintf(stderr, "PMIx_tool_init failed: %d\n", rc );
exit(rc);
}
if (0 < ninfo) {
PMIX_INFO_FREE(info, ninfo);
}
/* first call fence to sync all processes */
retval = fence_helper();
if(retval != PMIX_SUCCESS)
{
error_helper(retval, hostn, "error fencing");
exit(retval);
}
/* Process SCR env vars if needed */
if(forward_all_scr_envs){
parse_all_scr_envs(&scr_environ, environ);
}
/* finalize the env array so a NULL is in place */
finalize_array(scr_environ);
/* Setup info structs to pass to this: */
/* pmix_info_t *error_info = NULL; */
/* PMIX_INFO_CREATE(error_info, 1); */
/*
strncpy(error_info[0].key, PMIX_ERROR_GROUP_ABORT, PMIX_MAX_KEYLEN);
error_info[0].value.type = PMIX_BOOL;
error_info[0].value.data.flag = true;
*/
/* strncpy(error_info[0].key, PMIX_ERROR_GROUP_SPAWN, PMIX_MAX_KEYLEN);
int t_val = 1;
pmix_value_load(&error_info[1].value, &t_val, PMIX_BOOL);
*/
/*error_info[1].value.type = PMIX_BOOL;
error_info[1].value.data.flag = true; */
/* strncpy(error_info[2].key, PMIX_ERROR_GROUP_GENERAL, PMIX_MAX_KEYLEN);
error_info[2].value.type = PMIX_BOOL;
error_info[2].value.data.flag = true;
*/
/* TODO: setup error handling when implemented in pmix with the
* following error codes: */
/*
pmix_status_t registered_codes[5];
registered_codes[0] = PMIX_ERR_JOB_TERMINATED;
registered_codes[1] = PMIX_ERR_PROC_ABORTED;
registered_codes[2] = PMIX_ERR_PROC_ABORTING;
*/
PMIx_Register_event_handler(NULL, 0,
NULL, 0,
errhandler_cb,
errhandler_reg_callbk,
(void *) NULL);
/* PMIX_INFO_DESTRUCT(error_info); */
/* allocate memory to hold the spawend app struct */
PMIX_APP_CREATE(spawned_app, 1);
/* maxprocs isn't documented very well, but it appears to control
* how many instances of the spanwed app are created */
spawned_app->maxprocs = number_of_clients;
/* set the app to run */
(void)asprintf(&spawned_app->cmd, "%s", path_to_app);
/* set argv for spawned app starting with remaining argv */
spawned_app->argv = &argv[optind];
/* set the environment pointer */
spawned_app->env = scr_environ;
/*--START: add all proc level infos */
/* add things to the proc level info */
if(!pmix_mode){
job_info_count++;
}
if(host_to_use != NULL){
proc_info_count++;
}
if(verbose_print){
printf("enabling debug feature for forwarding stdout/stderr\n");
proc_info_count+=2;
/* add PMIX_FWD_STDOUT and PMIX_FWD_STDERR later*/
}
if(experimental){
job_info_count++;
}
if(node_count == 1){
job_info_count++;
}
/*--END: add all proc level infos */
/*--START: append actual proc level info */
PMIX_INFO_CREATE(job_info, job_info_count);
PMIX_INFO_CREATE(proc_info, proc_info_count);
/* PMIX_VAL_set_assign(_v, _field, _val ) */
/* PMIX_VAL_set_strdup(_v, _field, _val ) */
if(host_to_use != NULL){
/* add info struct to the spawned app itself for the host */
/* old way */
strncpy(proc_info[proc_info_index].key, PMIX_HOST, PMIX_MAX_KEYLEN);
//proc_info[proc_info_index].value.type = PMIX_STRING;
/* set the data for host list to use */
//proc_info[proc_info_index].value.data.string = host_to_use;
/* end old way */
if(verbose_print) printf("about to set host val\n");
PMIX_VAL_SET(&(proc_info[proc_info_index].value), string,
host_to_use );
proc_info_index++;
}
if(!pmix_mode){
strncpy(job_info[job_info_index].key, PMIX_NON_PMI,
PMIX_MAX_KEYLEN);
if(verbose_print) printf("about to set non pmix flag\n");
PMIX_VAL_SET(&(job_info[job_info_index].value), flag, true);
job_info_index++;
}
if(verbose_print){
strncpy(proc_info[proc_info_index].key, PMIX_FWD_STDOUT,
PMIX_MAX_KEYLEN);
if(verbose_print) printf("about to set stdout flag\n");
PMIX_VAL_SET(&(proc_info[proc_info_index].value), flag, true );
proc_info_index++;
strncpy(proc_info[proc_info_index].key, PMIX_FWD_STDERR,
PMIX_MAX_KEYLEN);
if(verbose_print) printf("about to set stderr flag\n");
PMIX_VAL_SET(&(proc_info[proc_info_index].value), flag, true );
proc_info_index++;
}
if(experimental){
printf("attempting to perform experiment\n");
bool local_flag = true;
PMIX_INFO_LOAD(&job_info[job_info_index], PMIX_NOTIFY_COMPLETION, &local_flag, PMIX_BOOL);
job_info_index++;
}
if(node_count == 1){
strncpy(job_info[job_info_index].key, PMIX_PPR,
PMIX_MAX_KEYLEN);
PMIX_VAL_SET(&(job_info[job_info_index].value), string,
"1:n");
job_info_index++;
}
/*--END: append actual proc level info */
/* sanity check to make sure we covered all the info structs */
if(proc_info_index != proc_info_count ){
printf("bug: mismatch with appending proc info\n");
exit(1);
}
if(job_info_index != job_info_count){
printf("bug: mismatch with appending job info\n");
exit(1);
}
/* TODO: TEST PMIX_NOTIFY_COMPLETION WHEN IT'S IMPLEMENTED IN PMIX */
/* fill in job_info */
/*
strncpy(job_info[0].key, PMIX_TIMEOUT, PMIX_MAX_KEYLEN);
job_info[0].value.type = PMIX_INT;
job_info[0].value.data.integer = 10; */
/* strncpy(job_info[0].key, PMIX_NOTIFY_COMPLETION, PMIX_MAX_KEYLEN);
job_info[0].value.type = PMIX_BOOL;
job_info[0].value.data.flag = true; */
/*strncpy(spawned_app->info[0].key, PMIX_DISPLAY_MAP, PMIX_MAX_KEYLEN);
job_info[0].value.type = PMIX_BOOL;
job_info[0].value.data.flag = true;*/
/* TODO: TEST PMIX_NOTIFY_COMPLETION WHEN IT'S IMPLEMENTED IN PMIX */
spawned_app->info = proc_info;
spawned_app->ninfo = proc_info_count;
if(verbose_print){
printf("proc level info count: %d\n", proc_info_count);
}
/* call spawn */
retval = PMIx_Spawn(job_info, job_info_count,
spawned_app, 1, spawned_nsp);
if(verbose_print) {
printf("rank %d (host %s) just called spawn; spawned nspace: %s, retval:%d\n",
main_proc.rank,
hostn,
spawned_nsp,
retval);
}
if(retval != PMIX_SUCCESS){
error_helper(retval, hostn, "error with spawn");
goto done;
}
/* TODO: TEMPORARY WORKAROUND TO WAIT FOR A SPAWNED PROCESS */
if(blocking_mode){
sleep(fixed_sleep);
/* wait until app completes: */
while(!done_flag){
sleep(fixed_sleep);
temp_counter++;
if(temp_counter*fixed_sleep >= sleep_max) {
if(verbose_print) printf("broke out early\n");
break;
}
}
if(verbose_print){
if(done_flag == true) {
printf("done_flag was set to true!\n");
}
}
}
done:
/* fence first */
retval = fence_helper();
if(retval != PMIX_SUCCESS){
if(verbose_print) printf("error fencing, finalize may fail ! \n");
}
/* finalize */
PMIx_Deregister_event_handler(_g_errhandler_ref, NULL, NULL);
if(verbose_print){
fprintf(stdout,
"spawn master process (rank %d) (host %s) finalizing\n",
main_proc.rank,
hostn);
}
/* clean up pmix */
retval = PMIx_tool_finalize();
if(retval == PMIX_SUCCESS)
{
if(verbose_print){
printf("spawn master process %d finalize success\n\n",
main_proc.rank);
}
}
else
{
printf("spawn master process %d pmix_finalize FAILURE: %d\n\n",
main_proc.rank,
retval);
}
retval = PMIx_Finalize(NULL, 0);
fflush(stdout);
/* cleanup before returning */
PMIX_INFO_FREE(job_info, job_info_count);
spawned_app->argv = NULL;
PMIX_APP_FREE(spawned_app, 1);
if(verbose_print) printf("%s exiting cleanly :)\n", argv[0]);
return 0;
}