static pmix_status_t spawn_debugger(char *appspace, myrel_t *myrel)
{
pmix_status_t rc;
pmix_info_t *dinfo;
pmix_app_t *debugger;
size_t dninfo;
char cwd[1024];
char dspace[PMIX_MAX_NSLEN+1];
mylock_t mylock;
pmix_status_t code = PMIX_ERR_JOB_TERMINATED;
/* setup the debugger */
PMIX_APP_CREATE(debugger, 1);
debugger[0].cmd = strdup("./debuggerd");
PMIX_ARGV_APPEND(rc, debugger[0].argv, "./debuggerd");
getcwd(cwd, 1024); // point us to our current directory
debugger[0].cwd = strdup(cwd);
/* provide directives so the daemons go where we want, and
* let the RM know these are debugger daemons */
dninfo = 6;
PMIX_INFO_CREATE(dinfo, dninfo);
PMIX_INFO_LOAD(&dinfo[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING); // instruct the RM to launch one copy of the executable on each node
PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUGGER_DAEMONS, NULL, PMIX_BOOL); // these are debugger daemons
PMIX_INFO_LOAD(&dinfo[1], PMIX_DEBUG_JOB, appspace, PMIX_STRING); // the nspace being debugged
PMIX_INFO_LOAD(&dinfo[2], PMIX_NOTIFY_COMPLETION, NULL, PMIX_BOOL); // notify us when the debugger job completes
PMIX_INFO_LOAD(&dinfo[3], PMIX_DEBUG_WAITING_FOR_NOTIFY, NULL, PMIX_BOOL); // tell the daemon that the proc is waiting to be released
PMIX_INFO_LOAD(&dinfo[4], PMIX_FWD_STDOUT, NULL, PMIX_BOOL); // forward stdout to me
PMIX_INFO_LOAD(&dinfo[5], PMIX_FWD_STDERR, NULL, PMIX_BOOL); // forward stderr to me
/* spawn the daemons */
fprintf(stderr, "Debugger: spawning %s\n", debugger[0].cmd);
if (PMIX_SUCCESS != (rc = PMIx_Spawn(dinfo, dninfo, debugger, 1, dspace))) {
fprintf(stderr, "Debugger daemons failed to launch with error: %s\n", PMIx_Error_string(rc));
PMIX_INFO_FREE(dinfo, dninfo);
PMIX_APP_FREE(debugger, 1);
return rc;
}
/* cleanup */
PMIX_INFO_FREE(dinfo, dninfo);
PMIX_APP_FREE(debugger, 1);
/* register callback for when this job terminates */
myrel->nspace = strdup(dspace);
PMIX_INFO_CREATE(dinfo, 2);
PMIX_INFO_LOAD(&dinfo[0], PMIX_EVENT_RETURN_OBJECT, myrel, PMIX_POINTER);
/* only call me back when this specific job terminates */
PMIX_INFO_LOAD(&dinfo[1], PMIX_NSPACE, dspace, PMIX_STRING);
DEBUG_CONSTRUCT_LOCK(&mylock);
PMIx_Register_event_handler(&code, 1, dinfo, 2,
release_fn, evhandler_reg_callbk, (void*)&mylock);
DEBUG_WAIT_THREAD(&mylock);
rc = mylock.status;
DEBUG_DESTRUCT_LOCK(&mylock);
PMIX_INFO_FREE(dinfo, 2);
return rc;
}
int main(int argc, char **argv)
{
pmix_status_t rc;
pmix_value_t *val;
pmix_proc_t proc;
pmix_info_t *info;
size_t ninfo;
volatile int active;
pmix_query_t *query;
size_t nq, n;
myquery_data_t myquery_data;
fprintf(stderr, "I AM HERE\n");
fflush(stderr);
sleep(10);
exit(0);
/* init us - since we were launched by the RM, our connection info
* will have been provided at startup. */
if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) {
fprintf(stderr, "Debugger daemon ns %s rank %d: PMIx_tool_init failed: %d\n", myproc.nspace, myproc.rank, rc);
exit(0);
}
fprintf(stderr, "Debugger daemon ns %s rank %d: Running\n", myproc.nspace, myproc.rank);
/* register our default event handler */
active = -1;
PMIx_Register_event_handler(NULL, 0, NULL, 0,
notification_fn, evhandler_reg_callbk, (void*)&active);
while (-1 == active) {
usleep(10);
}
if (0 != active) {
exit(active);
}
/* get the nspace of the job we are to debug */
(void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN);
proc.rank = PMIX_RANK_WILDCARD;
if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_DEBUG_JOB, NULL, 0, &val))) {
fprintf(stderr, "[%s:%d] Failed to get job being debugged - error %d\n", myproc.nspace, myproc.rank, rc);
goto done;
}
if (NULL == val) {
fprintf(stderr, "Got NULL return\n");
goto done;
}
fprintf(stderr, "[%s:%d] Debugging %s\n", myproc.nspace, myproc.rank, val->data.string);
/* get our local proctable - for scalability reasons, we don't want to
* have our "root" debugger process get the proctable for everybody and
* send it out to us. So ask the local PMIx server for the pid's of
* our local target processes */
nq = 1;
PMIX_QUERY_CREATE(query, nq);
PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_LOCAL_PROC_TABLE);
query[0].nqual = 1;
PMIX_INFO_CREATE(query[0].qualifiers, 1);
PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_NSPACE, val->data.string, PMIX_STRING); // the nspace we are enquiring about
/* setup the caddy to retrieve the data */
myquery_data.info = NULL;
myquery_data.ninfo = 0;
myquery_data.active = true;
/* execute the query */
if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&myquery_data))) {
fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
goto done;
}
while (myquery_data.active) {
usleep(10);
}
fprintf(stderr, "[%s:%d] Local proctable received\n", myproc.nspace, myproc.rank);
/* now that we have the proctable for our local processes, we can do our
* magic debugger stuff and attach to them. We then send a "release" event
* to them - i.e., it's the equivalent to setting the MPIR breakpoint. We
* do this with the event notification system */
(void)strncpy(proc.nspace, val->data.string, PMIX_MAX_NSLEN);
proc.rank = PMIX_RANK_WILDCARD;
/* we send the notification to just the local procs of the job being debugged */
ninfo = 1;
PMIX_INFO_CREATE(info, ninfo);
PMIX_INFO_LOAD(&info[0], PMIX_EVENT_CUSTOM_RANGE, &proc, PMIX_PROC); // deliver to the target nspace
fprintf(stderr, "[%s:%u] Sending release\n", myproc.nspace, myproc.rank);
PMIx_Notify_event(PMIX_ERR_DEBUGGER_RELEASE,
NULL, PMIX_RANGE_LOCAL,
info, ninfo, NULL, NULL);
/* do some debugger magic */
n = 0;
fprintf(stderr, "[%s:%u] Hanging around awhile, doing debugger magic\n", myproc.nspace, myproc.rank);
while (n < 5) {
usleep(1000);
++n;
}
done:
/* finalize us */
fprintf(stderr, "Debugger daemon ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank);
if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
fprintf(stderr, "Debugger daemon ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
} else {
fprintf(stderr, "Debugger daemon ns %s rank %d:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank);
}
fflush(stderr);
return(0);
}
static int attach_to_running_job(char *nspace)
{
pmix_status_t rc;
pmix_proc_t myproc;
pmix_query_t *query;
size_t nq;
myquery_data_t *q;
/* query the active nspaces so we can verify that the
* specified one exists */
nq = 1;
PMIX_QUERY_CREATE(query, nq);
PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_NAMESPACES);
q = (myquery_data_t*)malloc(sizeof(myquery_data_t));
DEBUG_CONSTRUCT_LOCK(&q->lock);
if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)q))) {
fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc);
return -1;
}
DEBUG_WAIT_THREAD(&q->lock);
DEBUG_DESTRUCT_LOCK(&q->lock);
if (NULL == q->info) {
fprintf(stderr, "Query returned no info\n");
return -1;
}
/* the query should have returned a comma-delimited list of nspaces */
if (PMIX_STRING != q->info[0].value.type) {
fprintf(stderr, "Query returned incorrect data type: %d\n", q->info[0].value.type);
return -1;
}
if (NULL == q->info[0].value.data.string) {
fprintf(stderr, "Query returned no active nspaces\n");
return -1;
}
fprintf(stderr, "Query returned %s\n", q->info[0].value.data.string);
return 0;
#if 0
/* split the returned string and look for the given nspace */
/* if not found, then we have an error */
PMIX_INFO_FREE(info, ninfo);
/* get the proctable for this nspace */
ninfo = 1;
PMIX_INFO_CREATE(info, ninfo);
(void)strncpy(info[0].key, PMIX_QUERY_PROC_TABLE, PMIX_MAX_KEYLEN);
(void)strncpy(info[0].qualifier, nspace, PMIX_MAX_KEYLEN);
if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(info, ninfo, infocbfunc, (void*)&active))) {
fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info_nb failed: %d\n", myproc.nspace, myproc.rank, rc);
return -1;
}
/* wait to get a response */
/* the query should have returned a data_array */
if (PMIX_DATA_ARRAY != info[0].type) {
fprintf(stderr, "Query returned incorrect data type: %d\n", info[0].type);
return -1;
}
if (NULL == info[0].data.darray.array) {
fprintf(stderr, "Query returned no proctable info\n");
return -1;
}
/* the data array consists of a struct:
* size_t size;
* void* array;
*
* In this case, the array is composed of pmix_proc_info_t structs:
* pmix_proc_t proc; // contains the nspace,rank of this proc
* char* hostname;
* char* executable_name;
* pid_t pid;
* int exit_code;
* pmix_proc_state_t state;
*/
/* this is where a debugger tool would process the proctable to
* create whatever blob it needs to provide to its daemons */
PMIX_INFO_FREE(info, ninfo);
/* setup the debugger daemon spawn request */
napps = 1;
PMIX_APP_CREATE(app, napps);
/* setup the name of the daemon executable to launch */
app[0].cmd = strdup("debuggerdaemon");
app[0].argc = 1;
app[0].argv = (char**)malloc(2*sizeof(char*));
app[0].argv[0] = strdup("debuggerdaemon");
app[0].argv[1] = NULL;
/* provide directives so the daemons go where we want, and
* let the RM know these are debugger daemons */
ninfo = 3;
PMIX_INFO_CREATE(app[0].info, ninfo);
PMIX_INFO_LOAD(&app[0].info[0], PMIX_MAPBY, "ppr:1:node", PMIX_STRING); // instruct the RM to launch one copy of the executable on each node
PMIX_INFO_LOAD(&app[0].info[1], PMIX_DEBUGGER_DAEMONS, true, PMIX_BOOL); // these are debugger daemons
PMIX_INFO_LOAD(&app[0].info[2], PMIX_DEBUG_TARGET, nspace, PMIX_STRING); // the "jobid" of the application to be debugged
/* spawn the daemons */
PMIx_Spawn(NULL, 0, app, napps, dspace);
/* cleanup */
PMIX_APP_FREE(app, napps);
/* this is where a debugger tool would wait until the debug operation is complete */
return 0;
#endif
}
int main(int argc, char **argv)
{
pmix_status_t rc;
pmix_value_t *val;
pmix_proc_t proc;
pmix_info_t *info;
size_t ninfo;
pmix_query_t *query;
size_t nq, n;
myquery_data_t myquery_data;
pid_t pid;
pmix_status_t code = PMIX_ERR_JOB_TERMINATED;
mylock_t mylock;
myrel_t myrel;
uint16_t localrank;
char *target = NULL;
pid = getpid();
/* init us - since we were launched by the RM, our connection info
* will have been provided at startup. */
if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) {
fprintf(stderr, "Debugger daemon: PMIx_tool_init failed: %d\n", rc);
exit(0);
}
fprintf(stderr, "Debugger daemon ns %s rank %d pid %lu: Running\n", myproc.nspace, myproc.rank, (unsigned long)pid);
/* register our default event handler */
DEBUG_CONSTRUCT_LOCK(&mylock);
PMIx_Register_event_handler(NULL, 0, NULL, 0,
notification_fn, evhandler_reg_callbk, (void*)&mylock);
DEBUG_WAIT_THREAD(&mylock);
if (PMIX_SUCCESS != mylock.status) {
rc = mylock.status;
DEBUG_DESTRUCT_LOCK(&mylock);
goto done;
}
DEBUG_DESTRUCT_LOCK(&mylock);
/* get the nspace of the job we are to debug - it will be in our JOB info */
#ifdef PMIX_LOAD_PROCID
PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD);
#else
PMIX_PROC_CONSTRUCT(&proc);
(void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_KEYLEN);
proc.rank = PMIX_RANK_WILDCARD;
#endif
if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_DEBUG_JOB, NULL, 0, &val))) {
fprintf(stderr, "[%s:%d:%lu] Failed to get job being debugged - error %s\n",
myproc.nspace, myproc.rank,
(unsigned long)pid, PMIx_Error_string(rc));
goto done;
}
if (NULL == val || PMIX_STRING != val->type || NULL == val->data.string) {
fprintf(stderr, "[%s:%d:%lu] Failed to get job being debugged - NULL data returned\n",
myproc.nspace, myproc.rank, (unsigned long)pid);
goto done;
}
/* save it for later */
target = strdup(val->data.string);
PMIX_VALUE_RELEASE(val);
fprintf(stderr, "[%s:%d:%lu] Debugging %s\n", myproc.nspace, myproc.rank,
(unsigned long)pid, target);
/* get my local rank so I can determine which local proc is "mine"
* to debug */
val = NULL;
if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val))) {
fprintf(stderr, "[%s:%d:%lu] Failed to get my local rank - error %s\n",
myproc.nspace, myproc.rank,
(unsigned long)pid, PMIx_Error_string(rc));
goto done;
}
if (NULL == val) {
fprintf(stderr, "[%s:%d:%lu] Failed to get my local rank - NULL data returned\n",
myproc.nspace, myproc.rank, (unsigned long)pid);
goto done;
}
if (PMIX_UINT16 != val->type) {
fprintf(stderr, "[%s:%d:%lu] Failed to get my local rank - returned wrong type %s\n",
myproc.nspace, myproc.rank, (unsigned long)pid, PMIx_Data_type_string(val->type));
goto done;
}
/* save the data */
localrank = val->data.uint16;
PMIX_VALUE_RELEASE(val);
fprintf(stderr, "[%s:%d:%lu] my local rank %d\n", myproc.nspace, myproc.rank,
(unsigned long)pid, (int)localrank);
/* register another handler specifically for when the target
* job completes */
DEBUG_CONSTRUCT_LOCK(&myrel.lock);
myrel.nspace = strdup(proc.nspace);
PMIX_INFO_CREATE(info, 2);
PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER);
/* only call me back when this specific job terminates */
PMIX_LOAD_PROCID(&proc, target, PMIX_RANK_WILDCARD);
PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &proc, PMIX_PROC);
fprintf(stderr, "[%s:%d:%lu] registering for termination of %s\n", myproc.nspace, myproc.rank,
(unsigned long)pid, proc.nspace);
DEBUG_CONSTRUCT_LOCK(&mylock);
PMIx_Register_event_handler(&code, 1, info, 2,
release_fn, evhandler_reg_callbk, (void*)&mylock);
DEBUG_WAIT_THREAD(&mylock);
if (PMIX_SUCCESS != mylock.status) {
rc = mylock.status;
DEBUG_DESTRUCT_LOCK(&mylock);
PMIX_INFO_FREE(info, 2);
goto done;
}
DEBUG_DESTRUCT_LOCK(&mylock);
PMIX_INFO_FREE(info, 2);
/* get our local proctable - for scalability reasons, we don't want to
* have our "root" debugger process get the proctable for everybody and
* send it out to us. So ask the local PMIx server for the pid's of
* our local target processes */
nq = 1;
PMIX_QUERY_CREATE(query, nq);
PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_LOCAL_PROC_TABLE);
query[0].nqual = 1;
PMIX_INFO_CREATE(query[0].qualifiers, 1);
PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_NSPACE, target, PMIX_STRING); // the nspace we are enquiring about
/* setup the caddy to retrieve the data */
DEBUG_CONSTRUCT_LOCK(&myquery_data.lock);
myquery_data.info = NULL;
myquery_data.ninfo = 0;
/* execute the query */
if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&myquery_data))) {
fprintf(stderr, "PMIx_Query_info failed: %d\n", rc);
goto done;
}
DEBUG_WAIT_THREAD(&myquery_data.lock);
DEBUG_DESTRUCT_LOCK(&myquery_data.lock);
PMIX_QUERY_FREE(query, nq);
if (PMIX_SUCCESS != myquery_data.status) {
rc = myquery_data.status;
goto done;
}
fprintf(stderr, "[%s:%d:%lu] Local proctable received\n", myproc.nspace, myproc.rank, (unsigned long)pid);
/* now that we have the proctable for our local processes, we can do our
* magic debugger stuff and attach to them. We then send a "release" event
* to them - i.e., it's the equivalent to setting the MPIR breakpoint. We
* do this with the event notification system. For this example, we just
* send it to all local procs of the job being debugged */
(void)strncpy(proc.nspace, target, PMIX_MAX_NSLEN);
proc.rank = PMIX_RANK_WILDCARD;
ninfo = 2;
PMIX_INFO_CREATE(info, ninfo);
PMIX_INFO_LOAD(&info[0], PMIX_EVENT_CUSTOM_RANGE, &proc, PMIX_PROC); // deliver to the target nspace
PMIX_INFO_LOAD(&info[1], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); // deliver to the target nspace
fprintf(stderr, "[%s:%u:%lu] Sending release\n", myproc.nspace, myproc.rank, (unsigned long)pid);
rc = PMIx_Notify_event(PMIX_ERR_DEBUGGER_RELEASE,
NULL, PMIX_RANGE_CUSTOM,
info, ninfo, NULL, NULL);
if (PMIX_SUCCESS != rc) {
fprintf(stderr, "%s[%s:%u:%lu] Sending release failed with error %s(%d)\n",
myproc.nspace, myproc.rank, (unsigned long)pid, PMIx_Error_string(rc), rc);
goto done;
}
/* do some debugger magic while waiting for the job to terminate */
DEBUG_WAIT_THREAD(&myrel.lock);
done:
if (NULL != target) {
free(target);
}
/* finalize us */
fprintf(stderr, "Debugger daemon ns %s rank %d pid %lu: Finalizing\n", myproc.nspace, myproc.rank, (unsigned long)pid);
if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
fprintf(stderr, "Debugger daemon ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc);
} else {
fprintf(stderr, "Debugger daemon ns %s rank %d pid %lu:PMIx_Finalize successfully completed\n", myproc.nspace, myproc.rank, (unsigned long)pid);
}
fflush(stderr);
return(0);
}