/* Process incoming RPCs. Meant to execute as a pthread */ extern void *rpc_mgr(void *no_data) { int sockfd, newsockfd; int i; uint16_t port; slurm_addr_t cli_addr; slurmdbd_conn_t *conn_arg = NULL; master_thread_id = pthread_self(); /* initialize port for RPCs */ if ((sockfd = slurm_init_msg_engine_port(get_dbd_port())) == SLURM_SOCKET_ERROR) fatal("slurm_init_msg_engine_port error %m"); slurm_persist_conn_recv_server_init(); /* * Process incoming RPCs until told to shutdown */ while (!shutdown_time && (i = slurm_persist_conn_wait_for_thread_loc()) >= 0) { /* * accept needed for stream implementation is a no-op in * message implementation that just passes sockfd to newsockfd */ if ((newsockfd = slurm_accept_msg_conn(sockfd, &cli_addr)) == SLURM_SOCKET_ERROR) { slurm_persist_conn_free_thread_loc(i); if (errno != EINTR) error("slurm_accept_msg_conn: %m"); continue; } fd_set_nonblocking(newsockfd); conn_arg = xmalloc(sizeof(slurmdbd_conn_t)); conn_arg->conn = xmalloc(sizeof(slurm_persist_conn_t)); conn_arg->conn->fd = newsockfd; conn_arg->conn->flags = PERSIST_FLAG_DBD; conn_arg->conn->callback_proc = proc_req; conn_arg->conn->callback_fini = _connection_fini_callback; conn_arg->conn->shutdown = &shutdown_time; conn_arg->conn->version = SLURM_MIN_PROTOCOL_VERSION; conn_arg->conn->rem_host = xmalloc_nz(sizeof(char) * 16); /* Don't fill in the rem_port here. It will be filled in * later if it is a slurmctld connection. */ slurm_get_ip_str(&cli_addr, &port, conn_arg->conn->rem_host, sizeof(char) * 16); slurm_persist_conn_recv_thread_init( conn_arg->conn, i, conn_arg); } debug("rpc_mgr shutting down"); (void) slurm_shutdown_msg_engine(sockfd); pthread_exit((void *) 0); return NULL; }
slurm_fd_t _slurm_open_stream(slurm_addr_t *addr, bool retry) { int retry_cnt; slurm_fd_t fd; uint16_t port; char ip[32]; if ( (addr->sin_family == 0) || (addr->sin_port == 0) ) { error("Error connecting, bad data: family = %u, port = %u", addr->sin_family, addr->sin_port); return SLURM_SOCKET_ERROR; } for (retry_cnt=0; ; retry_cnt++) { int rc; if ((fd =_slurm_create_socket(SLURM_STREAM)) < 0) { error("Error creating slurm stream socket: %m"); slurm_seterrno(errno); return SLURM_SOCKET_ERROR; } if (retry_cnt) { if (retry_cnt == 1) { debug3("Error connecting, " "picking new stream port"); } _sock_bind_wild(fd); } rc = _slurm_connect(fd, (struct sockaddr const *)addr, sizeof(*addr)); if (rc >= 0) /* success */ break; if (((errno != ECONNREFUSED) && (errno != ETIMEDOUT)) || (!retry) || (retry_cnt >= PORT_RETRIES)) { slurm_seterrno(errno); goto error; } if ((_slurm_close_stream(fd) < 0) && (errno == EINTR)) _slurm_close_stream(fd); /* try again */ } return fd; error: slurm_get_ip_str(addr, &port, ip, sizeof(ip)); debug2("Error connecting slurm stream socket at %s:%d: %m", ip, ntohs(port)); if ((_slurm_close_stream(fd) < 0) && (errno == EINTR)) _slurm_close_stream(fd); /* try again */ return SLURM_SOCKET_ERROR; }
/* Process incoming RPCs. Meant to execute as a pthread */ extern void *rpc_mgr(void *no_data) { pthread_attr_t thread_attr_rpc_req; slurm_fd_t sockfd, newsockfd; int i, retry_cnt, sigarray[] = {SIGUSR1, 0}; slurm_addr_t cli_addr; slurmdbd_conn_t *conn_arg = NULL; slurm_mutex_lock(&thread_count_lock); master_thread_id = pthread_self(); slurm_mutex_unlock(&thread_count_lock); (void) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); (void) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); /* threads to process individual RPC's are detached */ slurm_attr_init(&thread_attr_rpc_req); if (pthread_attr_setdetachstate (&thread_attr_rpc_req, PTHREAD_CREATE_DETACHED)) fatal("pthread_attr_setdetachstate %m"); /* initialize port for RPCs */ if ((sockfd = slurm_init_msg_engine_port(get_dbd_port())) == SLURM_SOCKET_ERROR) fatal("slurm_init_msg_engine_port error %m"); /* Prepare to catch SIGUSR1 to interrupt accept(). * This signal is generated by the slurmdbd signal * handler thread upon receipt of SIGABRT, SIGINT, * or SIGTERM. That thread does all processing of * all signals. */ xsignal(SIGUSR1, _sig_handler); xsignal_unblock(sigarray); /* * Process incoming RPCs until told to shutdown */ while ((i = _wait_for_server_thread()) >= 0) { /* * accept needed for stream implementation is a no-op in * message implementation that just passes sockfd to newsockfd */ if ((newsockfd = slurm_accept_msg_conn(sockfd, &cli_addr)) == SLURM_SOCKET_ERROR) { _free_server_thread((pthread_t) 0); if (errno != EINTR) error("slurm_accept_msg_conn: %m"); continue; } fd_set_nonblocking(newsockfd); conn_arg = xmalloc(sizeof(slurmdbd_conn_t)); conn_arg->newsockfd = newsockfd; slurm_get_ip_str(&cli_addr, &conn_arg->orig_port, conn_arg->ip, sizeof(conn_arg->ip)); retry_cnt = 0; while (pthread_create(&slave_thread_id[i], &thread_attr_rpc_req, _service_connection, (void *) conn_arg)) { if (retry_cnt > 0) { error("pthread_create failure, " "aborting RPC: %m"); close(newsockfd); break; } error("pthread_create failure: %m"); retry_cnt++; usleep(1000); /* retry in 1 msec */ } } debug3("rpc_mgr shutting down"); slurm_attr_destroy(&thread_attr_rpc_req); (void) slurm_shutdown_msg_engine(sockfd); _wait_for_thread_fini(); pthread_exit((void *) 0); return NULL; }
/* * This function handles the initialization information from slurmd * sent by _send_slurmstepd_init() in src/slurmd/slurmd/req.c. */ static int _init_from_slurmd(int sock, char **argv, slurm_addr_t **_cli, slurm_addr_t **_self, slurm_msg_t **_msg, int *_ngids, gid_t **_gids) { char *incoming_buffer = NULL; Buf buffer; int step_type; int len; slurm_addr_t *cli = NULL; slurm_addr_t *self = NULL; slurm_msg_t *msg = NULL; int ngids = 0; gid_t *gids = NULL; uint16_t port; char buf[16]; log_options_t lopts = LOG_OPTS_INITIALIZER; log_init(argv[0], lopts, LOG_DAEMON, NULL); /* receive job type from slurmd */ safe_read(sock, &step_type, sizeof(int)); debug3("step_type = %d", step_type); /* receive reverse-tree info from slurmd */ pthread_mutex_lock(&step_complete.lock); safe_read(sock, &step_complete.rank, sizeof(int)); safe_read(sock, &step_complete.parent_rank, sizeof(int)); safe_read(sock, &step_complete.children, sizeof(int)); safe_read(sock, &step_complete.depth, sizeof(int)); safe_read(sock, &step_complete.max_depth, sizeof(int)); safe_read(sock, &step_complete.parent_addr, sizeof(slurm_addr_t)); step_complete.bits = bit_alloc(step_complete.children); step_complete.jobacct = jobacct_gather_g_create(NULL); pthread_mutex_unlock(&step_complete.lock); /* receive conf from slurmd */ if ((conf = read_slurmd_conf_lite (sock)) == NULL) fatal("Failed to read conf from slurmd"); log_alter(conf->log_opts, 0, conf->logfile); debug2("debug level is %d.", conf->debug_level); /* acct info */ jobacct_gather_g_startpoll(conf->job_acct_gather_freq); switch_g_slurmd_step_init(); slurm_get_ip_str(&step_complete.parent_addr, &port, buf, 16); debug3("slurmstepd rank %d, parent address = %s, port = %u", step_complete.rank, buf, port); /* receive cli from slurmd */ safe_read(sock, &len, sizeof(int)); incoming_buffer = xmalloc(sizeof(char) * len); safe_read(sock, incoming_buffer, len); buffer = create_buf(incoming_buffer,len); cli = xmalloc(sizeof(slurm_addr_t)); if(slurm_unpack_slurm_addr_no_alloc(cli, buffer) == SLURM_ERROR) fatal("slurmstepd: problem with unpack of slurmd_conf"); free_buf(buffer); /* receive self from slurmd */ safe_read(sock, &len, sizeof(int)); if (len > 0) { /* receive packed self from main slurmd */ incoming_buffer = xmalloc(sizeof(char) * len); safe_read(sock, incoming_buffer, len); buffer = create_buf(incoming_buffer,len); self = xmalloc(sizeof(slurm_addr_t)); if (slurm_unpack_slurm_addr_no_alloc(self, buffer) == SLURM_ERROR) { fatal("slurmstepd: problem with unpack of " "slurmd_conf"); } free_buf(buffer); } /* Receive GRES information from slurmd */ gres_plugin_recv_stepd(sock); /* receive req from slurmd */ safe_read(sock, &len, sizeof(int)); incoming_buffer = xmalloc(sizeof(char) * len); safe_read(sock, incoming_buffer, len); buffer = create_buf(incoming_buffer,len); msg = xmalloc(sizeof(slurm_msg_t)); slurm_msg_t_init(msg); switch(step_type) { case LAUNCH_BATCH_JOB: msg->msg_type = REQUEST_BATCH_JOB_LAUNCH; break; case LAUNCH_TASKS: msg->msg_type = REQUEST_LAUNCH_TASKS; break; default: fatal("Unrecognized launch RPC"); break; } if(unpack_msg(msg, buffer) == SLURM_ERROR) fatal("slurmstepd: we didn't unpack the request correctly"); free_buf(buffer); /* receive cached group ids array for the relevant uid */ safe_read(sock, &ngids, sizeof(int)); if (ngids > 0) { int i; uint32_t tmp32; gids = (gid_t *)xmalloc(sizeof(gid_t) * ngids); for (i = 0; i < ngids; i++) { safe_read(sock, &tmp32, sizeof(uint32_t)); gids[i] = (gid_t)tmp32; debug2("got gid %d", gids[i]); } } *_cli = cli; *_self = self; *_msg = msg; *_ngids = ngids; *_gids = gids; return 1; rwfail: fatal("Error reading initialization data from slurmd"); exit(1); }