void *_forward_thread(void *arg)
{
forward_msg_t *fwd_msg = (forward_msg_t *)arg;
forward_struct_t *fwd_struct = fwd_msg->fwd_struct;
Buf buffer = init_buf(BUF_SIZE); /* probably enough for header */
List ret_list = NULL;
int fd = -1;
ret_data_info_t *ret_data_info = NULL;
char *name = NULL;
hostlist_t hl = hostlist_create(fwd_msg->header.forward.nodelist);
slurm_addr_t addr;
char *buf = NULL;
int steps = 0;
int start_timeout = fwd_msg->timeout;
/* repeat until we are sure the message was sent */
while ((name = hostlist_shift(hl))) {
if (slurm_conf_get_addr(name, &addr) == SLURM_ERROR) {
error("forward_thread: can't find address for host "
"%s, check slurm.conf", name);
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
SLURM_UNKNOWN_FORWARD_ADDR);
free(name);
if (hostlist_count(hl) > 0) {
slurm_mutex_unlock(&fwd_struct->forward_mutex);
continue;
}
goto cleanup;
}
if ((fd = slurm_open_msg_conn(&addr)) < 0) {
error("forward_thread to %s: %m", name);
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(
&fwd_struct->ret_list, name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
free(name);
if (hostlist_count(hl) > 0) {
slurm_mutex_unlock(&fwd_struct->forward_mutex);
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_forward_msg_internal(hl, NULL, fwd_struct,
&fwd_msg->header, 0,
hostlist_count(hl));
continue;
}
goto cleanup;
}
buf = hostlist_ranged_string_xmalloc(hl);
xfree(fwd_msg->header.forward.nodelist);
fwd_msg->header.forward.nodelist = buf;
fwd_msg->header.forward.cnt = hostlist_count(hl);
#if 0
info("sending %d forwards (%s) to %s",
fwd_msg->header.forward.cnt,
fwd_msg->header.forward.nodelist, name);
#endif
if (fwd_msg->header.forward.nodelist[0]) {
debug3("forward: send to %s along with %s",
name, fwd_msg->header.forward.nodelist);
} else
debug3("forward: send to %s ", name);
pack_header(&fwd_msg->header, buffer);
/* add forward data to buffer */
if (remaining_buf(buffer) < fwd_struct->buf_len) {
int new_size = buffer->processed + fwd_struct->buf_len;
new_size += 1024; /* padded for paranoia */
xrealloc_nz(buffer->head, new_size);
buffer->size = new_size;
}
if (fwd_struct->buf_len) {
memcpy(&buffer->head[buffer->processed],
fwd_struct->buf, fwd_struct->buf_len);
buffer->processed += fwd_struct->buf_len;
}
/*
* forward message
*/
if (slurm_msg_sendto(fd,
get_buf_data(buffer),
get_buf_offset(buffer),
SLURM_PROTOCOL_NO_SEND_RECV_FLAGS ) < 0) {
error("forward_thread: slurm_msg_sendto: %m");
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
errno);
free(name);
if (hostlist_count(hl) > 0) {
free_buf(buffer);
buffer = init_buf(fwd_struct->buf_len);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
slurm_close(fd);
fd = -1;
/* Abandon tree. This way if all the
* nodes in the branch are down we
* don't have to time out for each
* node serially.
*/
_forward_msg_internal(hl, NULL, fwd_struct,
&fwd_msg->header, 0,
hostlist_count(hl));
continue;
}
goto cleanup;
}
/* These messages don't have a return message, but if
* we got here things worked out so make note of the
* list of nodes as success.
*/
if ((fwd_msg->header.msg_type == REQUEST_SHUTDOWN) ||
(fwd_msg->header.msg_type == REQUEST_RECONFIGURE) ||
(fwd_msg->header.msg_type == REQUEST_REBOOT_NODES)) {
slurm_mutex_lock(&fwd_struct->forward_mutex);
ret_data_info = xmalloc(sizeof(ret_data_info_t));
list_push(fwd_struct->ret_list, ret_data_info);
ret_data_info->node_name = xstrdup(name);
free(name);
while ((name = hostlist_shift(hl))) {
ret_data_info =
xmalloc(sizeof(ret_data_info_t));
list_push(fwd_struct->ret_list, ret_data_info);
ret_data_info->node_name = xstrdup(name);
free(name);
}
goto cleanup;
}
if (fwd_msg->header.forward.cnt > 0) {
static int message_timeout = -1;
if (message_timeout < 0)
message_timeout =
slurm_get_msg_timeout() * 1000;
if (!fwd_msg->header.forward.tree_width)
fwd_msg->header.forward.tree_width =
slurm_get_tree_width();
steps = (fwd_msg->header.forward.cnt+1) /
fwd_msg->header.forward.tree_width;
fwd_msg->timeout = (message_timeout*steps);
/* info("got %d * %d = %d", message_timeout, */
/* steps, fwd_msg->timeout); */
steps++;
fwd_msg->timeout += (start_timeout*steps);
/* info("now + %d*%d = %d", start_timeout, */
/* steps, fwd_msg->timeout); */
}
ret_list = slurm_receive_msgs(fd, steps, fwd_msg->timeout);
/* info("sent %d forwards got %d back", */
/* fwd_msg->header.forward.cnt, list_count(ret_list)); */
if (!ret_list || (fwd_msg->header.forward.cnt != 0
&& list_count(ret_list) <= 1)) {
slurm_mutex_lock(&fwd_struct->forward_mutex);
mark_as_failed_forward(&fwd_struct->ret_list, name,
errno);
free(name);
FREE_NULL_LIST(ret_list);
if (hostlist_count(hl) > 0) {
free_buf(buffer);
buffer = init_buf(fwd_struct->buf_len);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
slurm_close(fd);
fd = -1;
continue;
}
goto cleanup;
} else if ((fwd_msg->header.forward.cnt+1)
!= list_count(ret_list)) {
/* this should never be called since the above
should catch the failed forwards and pipe
them back down, but this is here so we
never have to worry about a locked
mutex */
ListIterator itr = NULL;
char *tmp = NULL;
int first_node_found = 0;
hostlist_iterator_t host_itr
= hostlist_iterator_create(hl);
error("We shouldn't be here. We forwarded to %d "
"but only got %d back",
(fwd_msg->header.forward.cnt+1),
list_count(ret_list));
while ((tmp = hostlist_next(host_itr))) {
int node_found = 0;
itr = list_iterator_create(ret_list);
while ((ret_data_info = list_next(itr))) {
if (!ret_data_info->node_name) {
first_node_found = 1;
ret_data_info->node_name =
xstrdup(name);
}
if (!xstrcmp(tmp,
ret_data_info->node_name)) {
node_found = 1;
break;
}
}
list_iterator_destroy(itr);
if (!node_found) {
mark_as_failed_forward(
&fwd_struct->ret_list,
tmp,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
}
free(tmp);
}
hostlist_iterator_destroy(host_itr);
if (!first_node_found) {
mark_as_failed_forward(
&fwd_struct->ret_list,
name,
SLURM_COMMUNICATIONS_CONNECTION_ERROR);
}
}
break;
}
slurm_mutex_lock(&fwd_struct->forward_mutex);
if (ret_list) {
while ((ret_data_info = list_pop(ret_list)) != NULL) {
if (!ret_data_info->node_name) {
ret_data_info->node_name = xstrdup(name);
}
list_push(fwd_struct->ret_list, ret_data_info);
debug3("got response from %s",
ret_data_info->node_name);
}
FREE_NULL_LIST(ret_list);
}
free(name);
cleanup:
if ((fd >= 0) && slurm_close(fd) < 0)
error ("close(%d): %m", fd);
hostlist_destroy(hl);
destroy_forward(&fwd_msg->header.forward);
free_buf(buffer);
slurm_cond_signal(&fwd_struct->notify);
slurm_mutex_unlock(&fwd_struct->forward_mutex);
xfree(fwd_msg);
return (NULL);
}
/*
* slurm_allocate_resources_blocking
* allocate resources for a job request. This call will block until
* the allocation is granted, or the specified timeout limit is reached.
* IN req - description of resource allocation request
* IN timeout - amount of time, in seconds, to wait for a response before
* giving up.
* A timeout of zero will wait indefinitely.
* IN pending_callback - If the allocation cannot be granted immediately,
* the controller will put the job in the PENDING state. If
* pending callback is not NULL, it will be called with the job_id
* of the pending job as the sole parameter.
*
* RET allocation structure on success, NULL on error set errno to
* indicate the error (errno will be ETIMEDOUT if the timeout is reached
* with no allocation granted)
* NOTE: free the response using slurm_free_resource_allocation_response_msg()
*/
resource_allocation_response_msg_t *
slurm_allocate_resources_blocking (const job_desc_msg_t *user_req,
time_t timeout,
void(*pending_callback)(uint32_t job_id))
{
int rc;
slurm_msg_t req_msg;
slurm_msg_t resp_msg;
resource_allocation_response_msg_t *resp = NULL;
char *hostname = NULL;
uint32_t job_id;
job_desc_msg_t *req;
listen_t *listen = NULL;
int errnum = SLURM_SUCCESS;
slurm_msg_t_init(&req_msg);
slurm_msg_t_init(&resp_msg);
/* make a copy of the user's job description struct so that we
* can make changes before contacting the controller */
req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t));
if (req == NULL)
return NULL;
memcpy(req, user_req, sizeof(job_desc_msg_t));
/*
* set Node and session id for this request
*/
if (req->alloc_sid == NO_VAL)
req->alloc_sid = getsid(0);
if (user_req->alloc_node != NULL) {
req->alloc_node = xstrdup(user_req->alloc_node);
} else if ((hostname = xshort_hostname()) != NULL) {
req->alloc_node = hostname;
} else {
error("Could not get local hostname,"
" forcing immediate allocation mode.");
req->immediate = 1;
}
if (!req->immediate) {
listen = _create_allocation_response_socket(hostname);
if (listen == NULL) {
xfree(req);
return NULL;
}
req->alloc_resp_port = listen->port;
}
req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION;
req_msg.data = req;
rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg);
if (rc == SLURM_SOCKET_ERROR) {
int errnum = errno;
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return NULL;
}
switch (resp_msg.msg_type) {
case RESPONSE_SLURM_RC:
if (_handle_rc_msg(&resp_msg) < 0) {
/* will reach this when the allocation fails */
errnum = errno;
} else {
/* shouldn't get here */
errnum = -1;
}
break;
case RESPONSE_RESOURCE_ALLOCATION:
/* Yay, the controller has acknowledged our request! But did
we really get an allocation yet? */
resp = (resource_allocation_response_msg_t *) resp_msg.data;
if (resp->node_cnt > 0) {
/* yes, allocation has been granted */
errno = SLURM_PROTOCOL_SUCCESS;
} else if (!req->immediate) {
if (resp->error_code != SLURM_SUCCESS)
info("%s", slurm_strerror(resp->error_code));
/* no, we need to wait for a response */
job_id = resp->job_id;
slurm_free_resource_allocation_response_msg(resp);
if (pending_callback != NULL)
pending_callback(job_id);
resp = _wait_for_allocation_response(job_id, listen,
timeout);
/* If NULL, we didn't get the allocation in
the time desired, so just free the job id */
if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) {
errnum = errno;
slurm_complete_job(job_id, -1);
}
}
break;
default:
errnum = SLURM_UNEXPECTED_MSG_ERROR;
resp = NULL;
}
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return resp;
}
/*
* slurm_allocate_resources_blocking
* allocate resources for a job request. This call will block until
* the allocation is granted, or the specified timeout limit is reached.
* IN req - description of resource allocation request
* IN timeout - amount of time, in seconds, to wait for a response before
* giving up.
* A timeout of zero will wait indefinitely.
* IN pending_callback - If the allocation cannot be granted immediately,
* the controller will put the job in the PENDING state. If
* pending callback is not NULL, it will be called with the job_id
* of the pending job as the sole parameter.
*
* RET allocation structure on success, NULL on error set errno to
* indicate the error (errno will be ETIMEDOUT if the timeout is reached
* with no allocation granted)
* NOTE: free the response using slurm_free_resource_allocation_response_msg()
*/
resource_allocation_response_msg_t *
slurm_allocate_resources_blocking (const job_desc_msg_t *user_req,
time_t timeout,
void(*pending_callback)(uint32_t job_id))
{
int rc;
slurm_msg_t req_msg;
slurm_msg_t resp_msg;
resource_allocation_response_msg_t *resp = NULL;
uint32_t job_id;
job_desc_msg_t *req;
listen_t *listen = NULL;
int errnum = SLURM_SUCCESS;
bool already_done = false;
slurm_msg_t_init(&req_msg);
slurm_msg_t_init(&resp_msg);
/* make a copy of the user's job description struct so that we
* can make changes before contacting the controller */
req = (job_desc_msg_t *)xmalloc(sizeof(job_desc_msg_t));
if (req == NULL)
return NULL;
memcpy(req, user_req, sizeof(job_desc_msg_t));
/*
* set Node and session id for this request
*/
if (req->alloc_sid == NO_VAL)
req->alloc_sid = getsid(0);
if (!req->immediate) {
listen = _create_allocation_response_socket();
if (listen == NULL) {
xfree(req);
return NULL;
}
req->alloc_resp_port = listen->port;
}
req_msg.msg_type = REQUEST_RESOURCE_ALLOCATION;
req_msg.data = req;
rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg,
working_cluster_rec);
if (rc == SLURM_ERROR) {
int errnum = errno;
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
errno = errnum;
return NULL;
}
switch (resp_msg.msg_type) {
case RESPONSE_SLURM_RC:
if (_handle_rc_msg(&resp_msg) < 0) {
/* will reach this when the allocation fails */
errnum = errno;
} else {
/* shouldn't get here */
errnum = SLURM_ERROR;
}
break;
case RESPONSE_RESOURCE_ALLOCATION:
/* Yay, the controller has acknowledged our request!
* Test if we have an allocation yet? */
resp = (resource_allocation_response_msg_t *) resp_msg.data;
if (resp->node_cnt > 0) {
/* yes, allocation has been granted */
errno = SLURM_SUCCESS;
} else if (!req->immediate) {
if (resp->error_code != SLURM_SUCCESS)
info("%s", slurm_strerror(resp->error_code));
/* no, we need to wait for a response */
/* print out any user messages before we wait. */
print_multi_line_string(resp->job_submit_user_msg,
-1, LOG_LEVEL_INFO);
job_id = resp->job_id;
slurm_free_resource_allocation_response_msg(resp);
if (pending_callback != NULL)
pending_callback(job_id);
_wait_for_allocation_response(job_id, listen,
RESPONSE_RESOURCE_ALLOCATION,
timeout, (void **) &resp);
/* If NULL, we didn't get the allocation in
the time desired, so just free the job id */
if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) {
errnum = errno;
slurm_complete_job(job_id, -1);
}
if ((resp == NULL) && (errno == ESLURM_ALREADY_DONE))
already_done = true;
}
break;
default:
errnum = SLURM_UNEXPECTED_MSG_ERROR;
resp = NULL;
}
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (!req->immediate)
_destroy_allocation_response_socket(listen);
xfree(req);
if (!resp && already_done && (errnum == SLURM_SUCCESS))
errnum = ESLURM_ALREADY_DONE;
errno = errnum;
return resp;
}
/*
* slurm_allocate_pack_job_blocking
* allocate resources for a list of job requests. This call will block
* until the entire allocation is granted, or the specified timeout limit
* is reached.
* IN req - List of resource allocation requests
* IN timeout - amount of time, in seconds, to wait for a response before
* giving up.
* A timeout of zero will wait indefinitely.
* IN pending_callback - If the allocation cannot be granted immediately,
* the controller will put the job in the PENDING state. If
* pending callback is not NULL, it will be called with the job_id
* of the pending job as the sole parameter.
*
* RET List of allocation structures on success, NULL on error set errno to
* indicate the error (errno will be ETIMEDOUT if the timeout is reached
* with no allocation granted)
* NOTE: free the response using list_destroy()
*/
List slurm_allocate_pack_job_blocking(List job_req_list, time_t timeout,
void(*pending_callback)(uint32_t job_id))
{
int rc;
slurm_msg_t req_msg;
slurm_msg_t resp_msg;
List resp = NULL;
char *local_hostname = NULL;
job_desc_msg_t *req;
listen_t *listen = NULL;
int errnum = SLURM_SUCCESS;
ListIterator iter;
bool immediate_flag = false;
bool immediate_logged = false;
uint32_t node_cnt = 0, job_id = 0;
slurm_msg_t_init(&req_msg);
slurm_msg_t_init(&resp_msg);
/*
* set node name and session ID for this request
*/
if (!immediate_flag) {
listen = _create_allocation_response_socket(local_hostname);
if (listen == NULL)
return NULL;
}
local_hostname = xshort_hostname();
iter = list_iterator_create(job_req_list);
while ((req = (job_desc_msg_t *) list_next(iter))) {
if (req->alloc_sid == NO_VAL)
req->alloc_sid = getsid(0);
if (listen)
req->alloc_resp_port = listen->port;
if (!req->alloc_node) {
if (local_hostname) {
req->alloc_node = local_hostname;
} else if (immediate_logged) {
req->immediate = 1;
} else {
req->immediate = 1;
error("Could not get local hostname, forcing "
"immediate allocation mode");
immediate_logged = true;
}
}
if (req->immediate)
immediate_flag = true;
}
list_iterator_destroy(iter);
req_msg.msg_type = REQUEST_JOB_PACK_ALLOCATION;
req_msg.data = job_req_list;
rc = slurm_send_recv_controller_msg(&req_msg, &resp_msg,
working_cluster_rec);
if (rc == SLURM_SOCKET_ERROR) {
int errnum = errno;
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (listen)
_destroy_allocation_response_socket(listen);
iter = list_iterator_create(job_req_list);
while ((req = (job_desc_msg_t *)list_next(iter))) {
if (req->alloc_node == local_hostname)
req->alloc_node = NULL;
}
list_iterator_destroy(iter);
xfree(local_hostname);
errno = errnum;
return NULL;
}
switch (resp_msg.msg_type) {
case RESPONSE_SLURM_RC:
if (_handle_rc_msg(&resp_msg) < 0) {
/* will reach this when the allocation fails */
errnum = errno;
} else {
/* shouldn't get here */
errnum = SLURM_ERROR;
}
break;
case RESPONSE_JOB_PACK_ALLOCATION:
/* Yay, the controller has acknowledged our request!
* Test if we have an allocation yet? */
resp = (List) resp_msg.data;
_pack_alloc_test(resp, &node_cnt, &job_id);
if (node_cnt > 0) {
/* yes, allocation has been granted */
errno = SLURM_PROTOCOL_SUCCESS;
} else if (immediate_flag) {
debug("Immediate allocation not granted");
} else {
/* no, we need to wait for a response */
FREE_NULL_LIST(resp);
if (pending_callback != NULL)
pending_callback(job_id);
_wait_for_allocation_response(job_id, listen,
RESPONSE_JOB_PACK_ALLOCATION,
timeout, (void **) &resp);
/* If NULL, we didn't get the allocation in
* the time desired, so just free the job id */
if ((resp == NULL) && (errno != ESLURM_ALREADY_DONE)) {
errnum = errno;
slurm_complete_job(job_id, -1);
}
}
break;
default:
errnum = SLURM_UNEXPECTED_MSG_ERROR;
}
destroy_forward(&req_msg.forward);
destroy_forward(&resp_msg.forward);
if (listen)
_destroy_allocation_response_socket(listen);
iter = list_iterator_create(job_req_list);
while ((req = (job_desc_msg_t *)list_next(iter))) {
if (req->alloc_node == local_hostname)
req->alloc_node = NULL;
}
list_iterator_destroy(iter);
xfree(local_hostname);
errno = errnum;
return resp;
}
/*
* _thread_per_group_rpc - thread to issue an RPC for a group of nodes
* sending message out to one and forwarding it to
* others if necessary.
* IN/OUT args - pointer to task_info_t, xfree'd on completion
*/
static void *_thread_per_group_rpc(void *args)
{
int rc = SLURM_SUCCESS;
slurm_msg_t msg;
task_info_t *task_ptr = (task_info_t *) args;
/* we cache some pointers from task_info_t because we need
* to xfree args before being finished with their use. xfree
* is required for timely termination of this pthread because
* xfree could lock it at the end, preventing a timely
* thread_exit */
pthread_mutex_t *thread_mutex_ptr = task_ptr->thread_mutex_ptr;
pthread_cond_t *thread_cond_ptr = task_ptr->thread_cond_ptr;
uint32_t *threads_active_ptr = task_ptr->threads_active_ptr;
thd_t *thread_ptr = task_ptr->thread_struct_ptr;
state_t thread_state = DSH_NO_RESP;
slurm_msg_type_t msg_type = task_ptr->msg_type;
bool is_kill_msg, srun_agent;
List ret_list = NULL;
ListIterator itr;
ret_data_info_t *ret_data_info = NULL;
int found = 0;
int sig_array[2] = {SIGUSR1, 0};
/* Locks: Write job, write node */
slurmctld_lock_t job_write_lock = {
NO_LOCK, WRITE_LOCK, WRITE_LOCK, NO_LOCK };
xassert(args != NULL);
xsignal(SIGUSR1, _sig_handler);
xsignal_unblock(sig_array);
is_kill_msg = ( (msg_type == REQUEST_KILL_TIMELIMIT) ||
(msg_type == REQUEST_TERMINATE_JOB) );
srun_agent = ( (msg_type == SRUN_PING) ||
(msg_type == SRUN_EXEC) ||
(msg_type == SRUN_JOB_COMPLETE) ||
(msg_type == SRUN_STEP_MISSING) ||
(msg_type == SRUN_TIMEOUT) ||
(msg_type == SRUN_USER_MSG) ||
(msg_type == RESPONSE_RESOURCE_ALLOCATION) ||
(msg_type == SRUN_NODE_FAIL) );
thread_ptr->start_time = time(NULL);
slurm_mutex_lock(thread_mutex_ptr);
thread_ptr->state = DSH_ACTIVE;
thread_ptr->end_time = thread_ptr->start_time + COMMAND_TIMEOUT;
slurm_mutex_unlock(thread_mutex_ptr);
/* send request message */
slurm_msg_t_init(&msg);
msg.msg_type = msg_type;
msg.data = task_ptr->msg_args_ptr;
#if 0
info("sending message type %u to %s", msg_type, thread_ptr->nodelist);
#endif
if (task_ptr->get_reply) {
if(thread_ptr->addr) {
msg.address = *thread_ptr->addr;
if(!(ret_list = slurm_send_addr_recv_msgs(
&msg, thread_ptr->nodelist, 0))) {
error("_thread_per_group_rpc: "
"no ret_list given");
goto cleanup;
}
} else {
if(!(ret_list = slurm_send_recv_msgs(
thread_ptr->nodelist,
&msg, 0, true))) {
error("_thread_per_group_rpc: "
"no ret_list given");
goto cleanup;
}
}
} else {
if(thread_ptr->addr) {
//info("got the address");
msg.address = *thread_ptr->addr;
} else {
//info("no address given");
if(slurm_conf_get_addr(thread_ptr->nodelist,
&msg.address) == SLURM_ERROR) {
error("_thread_per_group_rpc: "
"can't find address for host %s, "
"check slurm.conf",
thread_ptr->nodelist);
goto cleanup;
}
}
//info("sending %u to %s", msg_type, thread_ptr->nodelist);
if (slurm_send_only_node_msg(&msg) == SLURM_SUCCESS) {
thread_state = DSH_DONE;
} else {
if (!srun_agent)
_comm_err(thread_ptr->nodelist, msg_type);
}
goto cleanup;
}
//info("got %d messages back", list_count(ret_list));
found = 0;
itr = list_iterator_create(ret_list);
while ((ret_data_info = list_next(itr)) != NULL) {
rc = slurm_get_return_code(ret_data_info->type,
ret_data_info->data);
/* SPECIAL CASE: Mark node as IDLE if job already
complete */
if (is_kill_msg &&
(rc == ESLURMD_KILL_JOB_ALREADY_COMPLETE)) {
kill_job_msg_t *kill_job;
kill_job = (kill_job_msg_t *)
task_ptr->msg_args_ptr;
rc = SLURM_SUCCESS;
lock_slurmctld(job_write_lock);
if (job_epilog_complete(kill_job->job_id,
ret_data_info->
node_name,
rc))
run_scheduler = true;
unlock_slurmctld(job_write_lock);
}
/* SPECIAL CASE: Kill non-startable batch job,
* Requeue the job on ESLURMD_PROLOG_FAILED */
if ((msg_type == REQUEST_BATCH_JOB_LAUNCH) &&
(rc != SLURM_SUCCESS) && (rc != ESLURMD_PROLOG_FAILED) &&
(ret_data_info->type != RESPONSE_FORWARD_FAILED)) {
batch_job_launch_msg_t *launch_msg_ptr =
task_ptr->msg_args_ptr;
uint32_t job_id = launch_msg_ptr->job_id;
info("Killing non-startable batch job %u: %s",
job_id, slurm_strerror(rc));
thread_state = DSH_DONE;
ret_data_info->err = thread_state;
lock_slurmctld(job_write_lock);
job_complete(job_id, 0, false, false, _wif_status());
unlock_slurmctld(job_write_lock);
continue;
}
if (((msg_type == REQUEST_SIGNAL_TASKS) ||
(msg_type == REQUEST_TERMINATE_TASKS)) &&
(rc == ESRCH)) {
/* process is already dead, not a real error */
rc = SLURM_SUCCESS;
}
switch (rc) {
case SLURM_SUCCESS:
/* debug("agent processed RPC to node %s", */
/* ret_data_info->node_name); */
thread_state = DSH_DONE;
break;
case SLURM_UNKNOWN_FORWARD_ADDR:
error("We were unable to forward message to '%s'. "
"Make sure the slurm.conf for each slurmd "
"contain all other nodes in your system.",
ret_data_info->node_name);
thread_state = DSH_NO_RESP;
break;
case ESLURMD_EPILOG_FAILED:
error("Epilog failure on host %s, "
"setting DOWN",
ret_data_info->node_name);
thread_state = DSH_FAILED;
break;
case ESLURMD_PROLOG_FAILED:
thread_state = DSH_FAILED;
break;
case ESLURM_INVALID_JOB_ID:
/* Not indicative of a real error */
case ESLURMD_JOB_NOTRUNNING:
/* Not indicative of a real error */
debug2("agent processed RPC to node %s: %s",
ret_data_info->node_name,
slurm_strerror(rc));
thread_state = DSH_DONE;
break;
default:
if (!srun_agent) {
if (ret_data_info->err)
errno = ret_data_info->err;
else
errno = rc;
rc = _comm_err(ret_data_info->node_name,
msg_type);
}
if (srun_agent)
thread_state = DSH_FAILED;
else if(ret_data_info->type == RESPONSE_FORWARD_FAILED)
/* check if a forward failed */
thread_state = DSH_NO_RESP;
else { /* some will fail that don't mean anything went
* bad like a job term request on a job that is
* already finished, we will just exit on those
* cases */
thread_state = DSH_DONE;
}
}
ret_data_info->err = thread_state;
}
list_iterator_destroy(itr);
cleanup:
xfree(args);
/* handled at end of thread just in case resend is needed */
destroy_forward(&msg.forward);
slurm_mutex_lock(thread_mutex_ptr);
thread_ptr->ret_list = ret_list;
thread_ptr->state = thread_state;
thread_ptr->end_time = (time_t) difftime(time(NULL),
thread_ptr->start_time);
/* Signal completion so another thread can replace us */
(*threads_active_ptr)--;
pthread_cond_signal(thread_cond_ptr);
slurm_mutex_unlock(thread_mutex_ptr);
return (void *) NULL;
}
