static int _pmix_p2p_send_core(const char *nodename, const char *address,
const char *data, uint32_t len)
{
int rc, timeout;
slurm_msg_t msg;
forward_data_msg_t req;
List ret_list;
ret_data_info_t *ret_data_info = NULL;
pmixp_debug_hang(0);
slurm_msg_t_init(&msg);
PMIXP_DEBUG("nodelist=%s, address=%s, len=%u", nodename, address, len);
req.address = (char *)address;
req.len = len;
/* there is not much we can do - just cast) */
req.data = (char*)data;
msg.msg_type = REQUEST_FORWARD_DATA;
msg.data = &req;
if (slurm_conf_get_addr(nodename, &msg.address) == SLURM_ERROR) {
PMIXP_ERROR("Can't find address for host "
"%s, check slurm.conf", nodename);
return SLURM_ERROR;
}
timeout = slurm_get_msg_timeout() * 1000;
msg.forward.timeout = timeout;
msg.forward.cnt = 0;
msg.forward.nodelist = NULL;
ret_list = slurm_send_addr_recv_msgs(&msg, (char*)nodename, timeout);
if (!ret_list) {
/* This should never happen (when this was
* written slurm_send_addr_recv_msgs always
* returned a list */
PMIXP_ERROR("No return list given from "
"slurm_send_addr_recv_msgs spawned for %s",
nodename);
return SLURM_ERROR;
} else if ((errno != SLURM_COMMUNICATIONS_CONNECTION_ERROR) &&
!list_count(ret_list)) {
PMIXP_ERROR("failed to send to %s, errno=%d", nodename, errno);
return SLURM_ERROR;
}
rc = SLURM_SUCCESS;
while ((ret_data_info = list_pop(ret_list))) {
int temp_rc = slurm_get_return_code(ret_data_info->type,
ret_data_info->data);
if (temp_rc != SLURM_SUCCESS)
rc = temp_rc;
destroy_data_info(ret_data_info);
}
FREE_NULL_LIST(ret_list);
return rc;
}
void *_fwd_tree_thread(void *arg)
{
fwd_tree_t *fwd_tree = (fwd_tree_t *)arg;
List ret_list = NULL;
char *name = NULL;
char *buf = NULL;
slurm_msg_t send_msg;
slurm_msg_t_init(&send_msg);
send_msg.msg_type = fwd_tree->orig_msg->msg_type;
send_msg.data = fwd_tree->orig_msg->data;
send_msg.protocol_version = fwd_tree->orig_msg->protocol_version;
/* repeat until we are sure the message was sent */
while ((name = hostlist_shift(fwd_tree->tree_hl))) {
if (slurm_conf_get_addr(name, &send_msg.address)
== SLURM_ERROR) {
error("fwd_tree_thread: can't find address for host "
"%s, check slurm.conf", name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(&fwd_tree->ret_list, name,
SLURM_UNKNOWN_FORWARD_ADDR);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
send_msg.forward.timeout = fwd_tree->timeout;
if ((send_msg.forward.cnt = hostlist_count(fwd_tree->tree_hl))){
buf = hostlist_ranged_string_xmalloc(
fwd_tree->tree_hl);
send_msg.forward.nodelist = buf;
} else
send_msg.forward.nodelist = NULL;
if (send_msg.forward.nodelist && send_msg.forward.nodelist[0]) {
debug3("Tree sending to %s along with %s",
name, send_msg.forward.nodelist);
} else
debug3("Tree sending to %s", name);
ret_list = slurm_send_addr_recv_msgs(&send_msg, name,
fwd_tree->timeout);
xfree(send_msg.forward.nodelist);
if (ret_list) {
int ret_cnt = list_count(ret_list);
/* This is most common if a slurmd is running
an older version of Slurm than the
originator of the message.
*/
if ((ret_cnt <= send_msg.forward.cnt) &&
(errno != SLURM_COMMUNICATIONS_CONNECTION_ERROR)) {
error("fwd_tree_thread: %s failed to forward "
"the message, expecting %d ret got only "
"%d",
name, send_msg.forward.cnt + 1, ret_cnt);
if (ret_cnt > 1) { /* not likely */
ret_data_info_t *ret_data_info = NULL;
ListIterator itr =
list_iterator_create(ret_list);
while ((ret_data_info =
list_next(itr))) {
if (xstrcmp(ret_data_info->
node_name, name))
hostlist_delete_host(
fwd_tree->
tree_hl,
ret_data_info->
node_name);
}
list_iterator_destroy(itr);
}
}
slurm_mutex_lock(fwd_tree->tree_mutex);
list_transfer(fwd_tree->ret_list, ret_list);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
FREE_NULL_LIST(ret_list);
/* try next node */
if (ret_cnt <= send_msg.forward.cnt) {
free(name);
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_start_msg_tree_internal(
fwd_tree->tree_hl, NULL,
fwd_tree,
hostlist_count(fwd_tree->tree_hl));
continue;
}
} else {
/* This should never happen (when this was
* written slurm_send_addr_recv_msgs always
* returned a list */
error("fwd_tree_thread: no return list given from "
"slurm_send_addr_recv_msgs spawned for %s",
name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(
&fwd_tree->ret_list, name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
slurm_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
free(name);
/* check for error and try again */
if (errno == SLURM_COMMUNICATIONS_CONNECTION_ERROR)
continue;
break;
}
_destroy_tree_fwd(fwd_tree);
return NULL;
}
/*
* _thread_per_group_rpc - thread to issue an RPC for a group of nodes
* sending message out to one and forwarding it to
* others if necessary.
* IN/OUT args - pointer to task_info_t, xfree'd on completion
*/
static void *_thread_per_group_rpc(void *args)
{
int rc = SLURM_SUCCESS;
slurm_msg_t msg;
task_info_t *task_ptr = (task_info_t *) args;
/* we cache some pointers from task_info_t because we need
* to xfree args before being finished with their use. xfree
* is required for timely termination of this pthread because
* xfree could lock it at the end, preventing a timely
* thread_exit */
pthread_mutex_t *thread_mutex_ptr = task_ptr->thread_mutex_ptr;
pthread_cond_t *thread_cond_ptr = task_ptr->thread_cond_ptr;
uint32_t *threads_active_ptr = task_ptr->threads_active_ptr;
thd_t *thread_ptr = task_ptr->thread_struct_ptr;
state_t thread_state = DSH_NO_RESP;
slurm_msg_type_t msg_type = task_ptr->msg_type;
bool is_kill_msg, srun_agent;
List ret_list = NULL;
ListIterator itr;
ret_data_info_t *ret_data_info = NULL;
int found = 0;
int sig_array[2] = {SIGUSR1, 0};
/* Locks: Write job, write node */
slurmctld_lock_t job_write_lock = {
NO_LOCK, WRITE_LOCK, WRITE_LOCK, NO_LOCK };
xassert(args != NULL);
xsignal(SIGUSR1, _sig_handler);
xsignal_unblock(sig_array);
is_kill_msg = ( (msg_type == REQUEST_KILL_TIMELIMIT) ||
(msg_type == REQUEST_TERMINATE_JOB) );
srun_agent = ( (msg_type == SRUN_PING) ||
(msg_type == SRUN_EXEC) ||
(msg_type == SRUN_JOB_COMPLETE) ||
(msg_type == SRUN_STEP_MISSING) ||
(msg_type == SRUN_TIMEOUT) ||
(msg_type == SRUN_USER_MSG) ||
(msg_type == RESPONSE_RESOURCE_ALLOCATION) ||
(msg_type == SRUN_NODE_FAIL) );
thread_ptr->start_time = time(NULL);
slurm_mutex_lock(thread_mutex_ptr);
thread_ptr->state = DSH_ACTIVE;
thread_ptr->end_time = thread_ptr->start_time + COMMAND_TIMEOUT;
slurm_mutex_unlock(thread_mutex_ptr);
/* send request message */
slurm_msg_t_init(&msg);
msg.msg_type = msg_type;
msg.data = task_ptr->msg_args_ptr;
#if 0
info("sending message type %u to %s", msg_type, thread_ptr->nodelist);
#endif
if (task_ptr->get_reply) {
if(thread_ptr->addr) {
msg.address = *thread_ptr->addr;
if(!(ret_list = slurm_send_addr_recv_msgs(
&msg, thread_ptr->nodelist, 0))) {
error("_thread_per_group_rpc: "
"no ret_list given");
goto cleanup;
}
} else {
if(!(ret_list = slurm_send_recv_msgs(
thread_ptr->nodelist,
&msg, 0, true))) {
error("_thread_per_group_rpc: "
"no ret_list given");
goto cleanup;
}
}
} else {
if(thread_ptr->addr) {
//info("got the address");
msg.address = *thread_ptr->addr;
} else {
//info("no address given");
if(slurm_conf_get_addr(thread_ptr->nodelist,
&msg.address) == SLURM_ERROR) {
error("_thread_per_group_rpc: "
"can't find address for host %s, "
"check slurm.conf",
thread_ptr->nodelist);
goto cleanup;
}
}
//info("sending %u to %s", msg_type, thread_ptr->nodelist);
if (slurm_send_only_node_msg(&msg) == SLURM_SUCCESS) {
thread_state = DSH_DONE;
} else {
if (!srun_agent)
_comm_err(thread_ptr->nodelist, msg_type);
}
goto cleanup;
}
//info("got %d messages back", list_count(ret_list));
found = 0;
itr = list_iterator_create(ret_list);
while ((ret_data_info = list_next(itr)) != NULL) {
rc = slurm_get_return_code(ret_data_info->type,
ret_data_info->data);
/* SPECIAL CASE: Mark node as IDLE if job already
complete */
if (is_kill_msg &&
(rc == ESLURMD_KILL_JOB_ALREADY_COMPLETE)) {
kill_job_msg_t *kill_job;
kill_job = (kill_job_msg_t *)
task_ptr->msg_args_ptr;
rc = SLURM_SUCCESS;
lock_slurmctld(job_write_lock);
if (job_epilog_complete(kill_job->job_id,
ret_data_info->
node_name,
rc))
run_scheduler = true;
unlock_slurmctld(job_write_lock);
}
/* SPECIAL CASE: Kill non-startable batch job,
* Requeue the job on ESLURMD_PROLOG_FAILED */
if ((msg_type == REQUEST_BATCH_JOB_LAUNCH) &&
(rc != SLURM_SUCCESS) && (rc != ESLURMD_PROLOG_FAILED) &&
(ret_data_info->type != RESPONSE_FORWARD_FAILED)) {
batch_job_launch_msg_t *launch_msg_ptr =
task_ptr->msg_args_ptr;
uint32_t job_id = launch_msg_ptr->job_id;
info("Killing non-startable batch job %u: %s",
job_id, slurm_strerror(rc));
thread_state = DSH_DONE;
ret_data_info->err = thread_state;
lock_slurmctld(job_write_lock);
job_complete(job_id, 0, false, false, _wif_status());
unlock_slurmctld(job_write_lock);
continue;
}
if (((msg_type == REQUEST_SIGNAL_TASKS) ||
(msg_type == REQUEST_TERMINATE_TASKS)) &&
(rc == ESRCH)) {
/* process is already dead, not a real error */
rc = SLURM_SUCCESS;
}
switch (rc) {
case SLURM_SUCCESS:
/* debug("agent processed RPC to node %s", */
/* ret_data_info->node_name); */
thread_state = DSH_DONE;
break;
case SLURM_UNKNOWN_FORWARD_ADDR:
error("We were unable to forward message to '%s'. "
"Make sure the slurm.conf for each slurmd "
"contain all other nodes in your system.",
ret_data_info->node_name);
thread_state = DSH_NO_RESP;
break;
case ESLURMD_EPILOG_FAILED:
error("Epilog failure on host %s, "
"setting DOWN",
ret_data_info->node_name);
thread_state = DSH_FAILED;
break;
case ESLURMD_PROLOG_FAILED:
thread_state = DSH_FAILED;
break;
case ESLURM_INVALID_JOB_ID:
/* Not indicative of a real error */
case ESLURMD_JOB_NOTRUNNING:
/* Not indicative of a real error */
debug2("agent processed RPC to node %s: %s",
ret_data_info->node_name,
slurm_strerror(rc));
thread_state = DSH_DONE;
break;
default:
if (!srun_agent) {
if (ret_data_info->err)
errno = ret_data_info->err;
else
errno = rc;
rc = _comm_err(ret_data_info->node_name,
msg_type);
}
if (srun_agent)
thread_state = DSH_FAILED;
else if(ret_data_info->type == RESPONSE_FORWARD_FAILED)
/* check if a forward failed */
thread_state = DSH_NO_RESP;
else { /* some will fail that don't mean anything went
* bad like a job term request on a job that is
* already finished, we will just exit on those
* cases */
thread_state = DSH_DONE;
}
}
ret_data_info->err = thread_state;
}
list_iterator_destroy(itr);
cleanup:
xfree(args);
/* handled at end of thread just in case resend is needed */
destroy_forward(&msg.forward);
slurm_mutex_lock(thread_mutex_ptr);
thread_ptr->ret_list = ret_list;
thread_ptr->state = thread_state;
thread_ptr->end_time = (time_t) difftime(time(NULL),
thread_ptr->start_time);
/* Signal completion so another thread can replace us */
(*threads_active_ptr)--;
pthread_cond_signal(thread_cond_ptr);
slurm_mutex_unlock(thread_mutex_ptr);
return (void *) NULL;
}
void *_fwd_tree_thread(void *arg)
{
fwd_tree_t *fwd_tree = (fwd_tree_t *)arg;
List ret_list = NULL;
char *name = NULL;
char *buf = NULL;
slurm_msg_t send_msg;
slurm_msg_t_init(&send_msg);
send_msg.msg_type = fwd_tree->orig_msg->msg_type;
send_msg.data = fwd_tree->orig_msg->data;
/* repeat until we are sure the message was sent */
while ((name = hostlist_shift(fwd_tree->tree_hl))) {
if (slurm_conf_get_addr(name, &send_msg.address)
== SLURM_ERROR) {
error("fwd_tree_thread: can't find address for host "
"%s, check slurm.conf", name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(&fwd_tree->ret_list, name,
SLURM_UNKNOWN_FORWARD_ADDR);
pthread_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
send_msg.forward.timeout = fwd_tree->timeout;
if((send_msg.forward.cnt = hostlist_count(fwd_tree->tree_hl))) {
buf = hostlist_ranged_string_xmalloc(
fwd_tree->tree_hl);
send_msg.forward.nodelist = buf;
} else
send_msg.forward.nodelist = NULL;
if (send_msg.forward.nodelist && send_msg.forward.nodelist[0]) {
debug3("Tree sending to %s along with %s",
name, send_msg.forward.nodelist);
} else
debug3("Tree sending to %s", name);
ret_list = slurm_send_addr_recv_msgs(&send_msg, name,
fwd_tree->timeout);
xfree(send_msg.forward.nodelist);
if(ret_list) {
slurm_mutex_lock(fwd_tree->tree_mutex);
list_transfer(fwd_tree->ret_list, ret_list);
pthread_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
list_destroy(ret_list);
} else {
/* This should never happen (when this was
written slurm_send_addr_recv_msgs always
returned a list */
error("fwd_tree_thread: no return list given from "
"slurm_send_addr_recv_msgs spawned for %s",
name);
slurm_mutex_lock(fwd_tree->tree_mutex);
mark_as_failed_forward(
&fwd_tree->ret_list, name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
pthread_cond_signal(fwd_tree->notify);
slurm_mutex_unlock(fwd_tree->tree_mutex);
free(name);
continue;
}
free(name);
/* check for error and try again */
if(errno == SLURM_COMMUNICATIONS_CONNECTION_ERROR)
continue;
break;
}
_destroy_tree_fwd(fwd_tree);
return NULL;
}
