int pmixp_stepd_send(char *nodelist, const char *address, char *data, uint32_t len, unsigned int start_delay, unsigned int retry_cnt, int silent) { int retry = 0, rc; unsigned int delay = start_delay; /* in milliseconds */ char *copy_of_nodelist = xstrdup(nodelist); while (1) { if (!silent && retry >= 1) { PMIXP_ERROR("send failed, rc=%d, try #%d", rc, retry); } rc = slurm_forward_data( ©_of_nodelist, (char *)address, len, data); if (rc == SLURM_SUCCESS) break; retry++; if (retry >= retry_cnt) break; /* wait with constantly increasing delay */ struct timespec ts = {(delay / 1000), ((delay % 1000) * 1000000)}; nanosleep(&ts, NULL); delay *= 2; } xfree(copy_of_nodelist); return rc; }
extern int temp_kvs_send(void) { int rc = SLURM_ERROR, retry = 0; unsigned int delay = 1; char *nodelist = NULL; if (!in_stepd()) /* srun */ nodelist = xstrdup(job_info.step_nodelist); else if (tree_info.parent_node) nodelist = xstrdup(tree_info.parent_node); /* cmd included in temp_kvs_buf */ kvs_seq++; /* expecting new kvs after now */ while (1) { if (retry == 1) verbose("failed to send temp kvs, rc=%d, retrying", rc); if (nodelist) /* srun or non-first-level stepds */ rc = slurm_forward_data(&nodelist, tree_sock_addr, temp_kvs_cnt, temp_kvs_buf); else /* first level stepds */ rc = tree_msg_to_srun(temp_kvs_cnt, temp_kvs_buf); if (rc == SLURM_SUCCESS) break; if (++retry >= MAX_RETRIES) break; /* wait, in case parent stepd / srun not ready */ sleep(delay); delay *= 2; } temp_kvs_init(); /* clear old temp kvs */ xfree(nodelist); return rc; }
/* send message defined by buf and size to given rank stepd */ static int pmix_stepd_send(const char* buf, uint32_t size, int rank) { int rc = SLURM_SUCCESS; /* map rank to host name */ char* host = hostlist_nth(pmix_stepd_hostlist, rank); /* strdup-ed */ /* delay to sleep between retries in seconds, * if there are multiple retires, we'll grow this delay * using exponential backoff, doubling it each time */ unsigned int delay = 1; /* we'll try multiple times to send message to stepd, * we retry in case stepd is just slow to get started */ int retries = 0; while (1) { /* attempt to send message */ rc = slurm_forward_data(&host, tree_sock_addr, size, buf); if (rc == SLURM_SUCCESS) { /* message sent successfully, we're done */ break; } /* check whether we've exceeded our retry count */ retries++; if (retries >= MAX_RETRIES) { /* cancel the step to avoid tasks hang */ slurm_kill_job_step(job_info.jobid, job_info.stepid, SIGKILL); } /* didn't succeeded, but we'll retry again, * sleep for a bit first */ sleep(delay); delay *= 2; } /* free host name */ free(host); /* strdup-ed */ return rc; }