/* Process incoming RPCs. Meant to execute as a pthread */
extern void *rpc_mgr(void *no_data)
{
int sockfd, newsockfd;
int i;
uint16_t port;
slurm_addr_t cli_addr;
slurmdbd_conn_t *conn_arg = NULL;
master_thread_id = pthread_self();
/* initialize port for RPCs */
if ((sockfd = slurm_init_msg_engine_port(get_dbd_port()))
== SLURM_SOCKET_ERROR)
fatal("slurm_init_msg_engine_port error %m");
slurm_persist_conn_recv_server_init();
/*
* Process incoming RPCs until told to shutdown
*/
while (!shutdown_time &&
(i = slurm_persist_conn_wait_for_thread_loc()) >= 0) {
/*
* accept needed for stream implementation is a no-op in
* message implementation that just passes sockfd to newsockfd
*/
if ((newsockfd = slurm_accept_msg_conn(sockfd,
&cli_addr)) ==
SLURM_SOCKET_ERROR) {
slurm_persist_conn_free_thread_loc(i);
if (errno != EINTR)
error("slurm_accept_msg_conn: %m");
continue;
}
fd_set_nonblocking(newsockfd);
conn_arg = xmalloc(sizeof(slurmdbd_conn_t));
conn_arg->conn = xmalloc(sizeof(slurm_persist_conn_t));
conn_arg->conn->fd = newsockfd;
conn_arg->conn->flags = PERSIST_FLAG_DBD;
conn_arg->conn->callback_proc = proc_req;
conn_arg->conn->callback_fini = _connection_fini_callback;
conn_arg->conn->shutdown = &shutdown_time;
conn_arg->conn->version = SLURM_MIN_PROTOCOL_VERSION;
conn_arg->conn->rem_host = xmalloc_nz(sizeof(char) * 16);
/* Don't fill in the rem_port here. It will be filled in
* later if it is a slurmctld connection. */
slurm_get_ip_str(&cli_addr, &port,
conn_arg->conn->rem_host, sizeof(char) * 16);
slurm_persist_conn_recv_thread_init(
conn_arg->conn, i, conn_arg);
}
debug("rpc_mgr shutting down");
(void) slurm_shutdown_msg_engine(sockfd);
pthread_exit((void *) 0);
return NULL;
}
slurm_fd_t _slurm_open_stream(slurm_addr_t *addr, bool retry)
{
int retry_cnt;
slurm_fd_t fd;
uint16_t port;
char ip[32];
if ( (addr->sin_family == 0) || (addr->sin_port == 0) ) {
error("Error connecting, bad data: family = %u, port = %u",
addr->sin_family, addr->sin_port);
return SLURM_SOCKET_ERROR;
}
for (retry_cnt=0; ; retry_cnt++) {
int rc;
if ((fd =_slurm_create_socket(SLURM_STREAM)) < 0) {
error("Error creating slurm stream socket: %m");
slurm_seterrno(errno);
return SLURM_SOCKET_ERROR;
}
if (retry_cnt) {
if (retry_cnt == 1) {
debug3("Error connecting, "
"picking new stream port");
}
_sock_bind_wild(fd);
}
rc = _slurm_connect(fd, (struct sockaddr const *)addr,
sizeof(*addr));
if (rc >= 0) /* success */
break;
if (((errno != ECONNREFUSED) && (errno != ETIMEDOUT)) ||
(!retry) || (retry_cnt >= PORT_RETRIES)) {
slurm_seterrno(errno);
goto error;
}
if ((_slurm_close_stream(fd) < 0) && (errno == EINTR))
_slurm_close_stream(fd); /* try again */
}
return fd;
error:
slurm_get_ip_str(addr, &port, ip, sizeof(ip));
debug2("Error connecting slurm stream socket at %s:%d: %m",
ip, ntohs(port));
if ((_slurm_close_stream(fd) < 0) && (errno == EINTR))
_slurm_close_stream(fd); /* try again */
return SLURM_SOCKET_ERROR;
}
/* Process incoming RPCs. Meant to execute as a pthread */
extern void *rpc_mgr(void *no_data)
{
pthread_attr_t thread_attr_rpc_req;
slurm_fd_t sockfd, newsockfd;
int i, retry_cnt, sigarray[] = {SIGUSR1, 0};
slurm_addr_t cli_addr;
slurmdbd_conn_t *conn_arg = NULL;
slurm_mutex_lock(&thread_count_lock);
master_thread_id = pthread_self();
slurm_mutex_unlock(&thread_count_lock);
(void) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
(void) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
/* threads to process individual RPC's are detached */
slurm_attr_init(&thread_attr_rpc_req);
if (pthread_attr_setdetachstate
(&thread_attr_rpc_req, PTHREAD_CREATE_DETACHED))
fatal("pthread_attr_setdetachstate %m");
/* initialize port for RPCs */
if ((sockfd = slurm_init_msg_engine_port(get_dbd_port()))
== SLURM_SOCKET_ERROR)
fatal("slurm_init_msg_engine_port error %m");
/* Prepare to catch SIGUSR1 to interrupt accept().
* This signal is generated by the slurmdbd signal
* handler thread upon receipt of SIGABRT, SIGINT,
* or SIGTERM. That thread does all processing of
* all signals. */
xsignal(SIGUSR1, _sig_handler);
xsignal_unblock(sigarray);
/*
* Process incoming RPCs until told to shutdown
*/
while ((i = _wait_for_server_thread()) >= 0) {
/*
* accept needed for stream implementation is a no-op in
* message implementation that just passes sockfd to newsockfd
*/
if ((newsockfd = slurm_accept_msg_conn(sockfd,
&cli_addr)) ==
SLURM_SOCKET_ERROR) {
_free_server_thread((pthread_t) 0);
if (errno != EINTR)
error("slurm_accept_msg_conn: %m");
continue;
}
fd_set_nonblocking(newsockfd);
conn_arg = xmalloc(sizeof(slurmdbd_conn_t));
conn_arg->newsockfd = newsockfd;
slurm_get_ip_str(&cli_addr, &conn_arg->orig_port,
conn_arg->ip, sizeof(conn_arg->ip));
retry_cnt = 0;
while (pthread_create(&slave_thread_id[i],
&thread_attr_rpc_req,
_service_connection,
(void *) conn_arg)) {
if (retry_cnt > 0) {
error("pthread_create failure, "
"aborting RPC: %m");
close(newsockfd);
break;
}
error("pthread_create failure: %m");
retry_cnt++;
usleep(1000); /* retry in 1 msec */
}
}
debug3("rpc_mgr shutting down");
slurm_attr_destroy(&thread_attr_rpc_req);
(void) slurm_shutdown_msg_engine(sockfd);
_wait_for_thread_fini();
pthread_exit((void *) 0);
return NULL;
}
/*
* This function handles the initialization information from slurmd
* sent by _send_slurmstepd_init() in src/slurmd/slurmd/req.c.
*/
static int
_init_from_slurmd(int sock, char **argv,
slurm_addr_t **_cli, slurm_addr_t **_self, slurm_msg_t **_msg,
int *_ngids, gid_t **_gids)
{
char *incoming_buffer = NULL;
Buf buffer;
int step_type;
int len;
slurm_addr_t *cli = NULL;
slurm_addr_t *self = NULL;
slurm_msg_t *msg = NULL;
int ngids = 0;
gid_t *gids = NULL;
uint16_t port;
char buf[16];
log_options_t lopts = LOG_OPTS_INITIALIZER;
log_init(argv[0], lopts, LOG_DAEMON, NULL);
/* receive job type from slurmd */
safe_read(sock, &step_type, sizeof(int));
debug3("step_type = %d", step_type);
/* receive reverse-tree info from slurmd */
pthread_mutex_lock(&step_complete.lock);
safe_read(sock, &step_complete.rank, sizeof(int));
safe_read(sock, &step_complete.parent_rank, sizeof(int));
safe_read(sock, &step_complete.children, sizeof(int));
safe_read(sock, &step_complete.depth, sizeof(int));
safe_read(sock, &step_complete.max_depth, sizeof(int));
safe_read(sock, &step_complete.parent_addr, sizeof(slurm_addr_t));
step_complete.bits = bit_alloc(step_complete.children);
step_complete.jobacct = jobacct_gather_g_create(NULL);
pthread_mutex_unlock(&step_complete.lock);
/* receive conf from slurmd */
if ((conf = read_slurmd_conf_lite (sock)) == NULL)
fatal("Failed to read conf from slurmd");
log_alter(conf->log_opts, 0, conf->logfile);
debug2("debug level is %d.", conf->debug_level);
/* acct info */
jobacct_gather_g_startpoll(conf->job_acct_gather_freq);
switch_g_slurmd_step_init();
slurm_get_ip_str(&step_complete.parent_addr, &port, buf, 16);
debug3("slurmstepd rank %d, parent address = %s, port = %u",
step_complete.rank, buf, port);
/* receive cli from slurmd */
safe_read(sock, &len, sizeof(int));
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
cli = xmalloc(sizeof(slurm_addr_t));
if(slurm_unpack_slurm_addr_no_alloc(cli, buffer) == SLURM_ERROR)
fatal("slurmstepd: problem with unpack of slurmd_conf");
free_buf(buffer);
/* receive self from slurmd */
safe_read(sock, &len, sizeof(int));
if (len > 0) {
/* receive packed self from main slurmd */
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
self = xmalloc(sizeof(slurm_addr_t));
if (slurm_unpack_slurm_addr_no_alloc(self, buffer)
== SLURM_ERROR) {
fatal("slurmstepd: problem with unpack of "
"slurmd_conf");
}
free_buf(buffer);
}
/* Receive GRES information from slurmd */
gres_plugin_recv_stepd(sock);
/* receive req from slurmd */
safe_read(sock, &len, sizeof(int));
incoming_buffer = xmalloc(sizeof(char) * len);
safe_read(sock, incoming_buffer, len);
buffer = create_buf(incoming_buffer,len);
msg = xmalloc(sizeof(slurm_msg_t));
slurm_msg_t_init(msg);
switch(step_type) {
case LAUNCH_BATCH_JOB:
msg->msg_type = REQUEST_BATCH_JOB_LAUNCH;
break;
case LAUNCH_TASKS:
msg->msg_type = REQUEST_LAUNCH_TASKS;
break;
default:
fatal("Unrecognized launch RPC");
break;
}
if(unpack_msg(msg, buffer) == SLURM_ERROR)
fatal("slurmstepd: we didn't unpack the request correctly");
free_buf(buffer);
/* receive cached group ids array for the relevant uid */
safe_read(sock, &ngids, sizeof(int));
if (ngids > 0) {
int i;
uint32_t tmp32;
gids = (gid_t *)xmalloc(sizeof(gid_t) * ngids);
for (i = 0; i < ngids; i++) {
safe_read(sock, &tmp32, sizeof(uint32_t));
gids[i] = (gid_t)tmp32;
debug2("got gid %d", gids[i]);
}
}
*_cli = cli;
*_self = self;
*_msg = msg;
*_ngids = ngids;
*_gids = gids;
return 1;
rwfail:
fatal("Error reading initialization data from slurmd");
exit(1);
}
