/*
* TODO: we need to keep track of the "me"
* structures created here, because we need to
* free them in "pmixp_stepd_finalize"
*/
void pmixp_server_slurm_conn(int fd)
{
eio_obj_t *obj;
pmixp_conn_t *conn = NULL;
PMIXP_DEBUG("Request from fd = %d", fd);
pmixp_debug_hang(0);
/* Set nonblocking */
fd_set_nonblocking(fd);
fd_set_close_on_exec(fd);
conn = pmixp_conn_new_temp(PMIXP_PROTO_SLURM, fd, _slurm_new_msg);
/* try to process right here */
pmixp_conn_progress_rcv(conn);
if (!pmixp_conn_is_alive(conn)) {
/* success, don't need this connection anymore */
pmixp_conn_return(conn);
return;
}
/* If it is a blocking operation: create AIO object to
* handle it */
obj = eio_obj_create(fd, &slurm_peer_ops, (void *)conn);
eio_new_obj(pmixp_info_io(), obj);
}
eio_handle_t *eio_handle_create(uint16_t shutdown_wait)
{
eio_handle_t *eio = xmalloc(sizeof(*eio));
if (pipe(eio->fds) < 0) {
error ("eio_create: pipe: %m");
eio_handle_destroy(eio);
return (NULL);
}
fd_set_nonblocking(eio->fds[0]);
fd_set_close_on_exec(eio->fds[0]);
fd_set_close_on_exec(eio->fds[1]);
xassert(eio->magic = EIO_MAGIC);
eio->obj_list = list_create(eio_obj_destroy);
eio->new_objs = list_create(eio_obj_destroy);
slurm_mutex_init(&eio->shutdown_mutex);
eio->shutdown_wait = DEFAULT_EIO_SHUTDOWN_WAIT;
if (shutdown_wait > 0)
eio->shutdown_wait = shutdown_wait;
return eio;
}
void pmixp_server_direct_conn(int fd)
{
eio_obj_t *obj;
pmixp_conn_t *conn;
PMIXP_DEBUG("Request from fd = %d", fd);
/* Set nonblocking */
fd_set_nonblocking(fd);
fd_set_close_on_exec(fd);
pmixp_fd_set_nodelay(fd);
conn = pmixp_conn_new_temp(PMIXP_PROTO_DIRECT, fd,
_direct_conn_establish);
/* try to process right here */
pmixp_conn_progress_rcv(conn);
if (!pmixp_conn_is_alive(conn)) {
/* success, don't need this connection anymore */
pmixp_conn_return(conn);
return;
}
/* If it is a blocking operation: create AIO object to
* handle it */
obj = eio_obj_create(fd, &direct_peer_ops, (void *)conn);
eio_new_obj(pmixp_info_io(), obj);
/* wakeup this connection to get processed */
eio_signal_wakeup(pmixp_info_io());
}
/**
* Initialize RX dumping.
*
* @return TRUE if initialized.
*/
static gboolean
dump_initialize(struct dump *dump)
{
char *pathname;
if (dump->initialized)
return TRUE;
pathname = make_pathname(settings_config_dir(), dump->filename);
dump->fd = file_open_missing(pathname, O_WRONLY | O_APPEND | O_NONBLOCK);
HFREE_NULL(pathname);
/*
* If the dump "file" is actually a named pipe, we'd block quickly
* if there was no reader. So set the file as non-blocking and
* we'll disable dumping as soon as we can't write all the data
* we want.
*/
if (dump->fd < 0) {
g_warning("can't open %s -- disabling dumping", dump->filename);
dump_disable(dump);
return FALSE;
}
fd_set_nonblocking(dump->fd);
dump->slist = slist_new();
dump->fill = 0;
dump->initialized = TRUE;
return TRUE;
}
/******************************************************************************
**函数名称: udp_listen
**功 能: 侦听指定端口
**输入参数:
** port: 端口号
**输出参数: NONE
**返 回: 套接字ID(<0:失败)
**实现描述:
** 1. 创建套接字
** 2. 绑定指定端口
** 3. 侦听指定端口
** 4. 设置套接字属性(可重用、非阻塞)
**注意事项:
**作 者: # Qifeng.zou # 2014.03.24 #
******************************************************************************/
int udp_listen(int port)
{
int fd, opt = 1;
struct sockaddr_in svraddr;
/* 1. 创建套接字 */
fd = socket(AF_INET, SOCK_DGRAM, 0);
if (fd < 0) {
return -1;
}
opt = 1;
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(int));
/* 2. 绑定指定端口 */
bzero(&svraddr, sizeof(svraddr));
svraddr.sin_family = AF_INET;
svraddr.sin_addr.s_addr = htonl(INADDR_ANY);
svraddr.sin_port = htons(port);
if (bind(fd, (struct sockaddr *)&svraddr, sizeof(svraddr)) < 0) {
close(fd);
return -1;
}
/* 3. 设置非阻塞属性 */
fd_set_nonblocking(fd);
return fd;
}
/*
* TODO: we need to keep track of the "me"
* structures created here, because we need to
* free them in "pmixp_stepd_finalize"
*/
void pmix_server_new_conn(int fd)
{
eio_obj_t *obj;
PMIXP_DEBUG("Request from fd = %d", fd);
/* Set nonblocking */
fd_set_nonblocking(fd);
fd_set_close_on_exec(fd);
pmixp_io_engine_t *me = xmalloc(sizeof(pmixp_io_engine_t));
pmix_io_init(me, fd, srv_rcvd_header);
/* We use slurm_forward_data to send message to stepd's
* SLURM will put user ID there. We need to skip it.
*/
pmix_io_rcvd_padding(me, sizeof(uint32_t));
if( 2 == _process_message(me) ){
/* connection was fully processed here */
xfree(me);
return;
}
/* If it is a blocking operation: create AIO object to
* handle it */
obj = eio_obj_create(fd, &peer_ops, (void *)me);
eio_new_obj(pmixp_info_io(), obj);
}
/* Set up port to handle messages from slurmctld */
slurm_fd_t
slurmctld_msg_init(void)
{
slurm_addr_t slurm_address;
uint16_t port;
static slurm_fd_t slurmctld_fd = (slurm_fd_t) 0;
if (slurmctld_fd) /* May set early for queued job allocation */
return slurmctld_fd;
slurmctld_fd = -1;
slurmctld_comm_addr.port = 0;
if ((slurmctld_fd = slurm_init_msg_engine_port(0)) < 0) {
error("slurm_init_msg_engine_port error %m");
exit(error_exit);
}
if (slurm_get_stream_addr(slurmctld_fd, &slurm_address) < 0) {
error("slurm_get_stream_addr error %m");
exit(error_exit);
}
fd_set_nonblocking(slurmctld_fd);
/* hostname is not set, so slurm_get_addr fails
slurm_get_addr(&slurm_address, &port, hostname, sizeof(hostname)); */
port = ntohs(slurm_address.sin_port);
slurmctld_comm_addr.port = port;
debug2("srun PMI messages to port=%u", slurmctld_comm_addr.port);
return slurmctld_fd;
}
/***************************************************************************
* Support functions for slurm_allocate_resources_blocking()
***************************************************************************/
static listen_t *_create_allocation_response_socket(char *interface_hostname)
{
listen_t *listen = NULL;
uint16_t *ports;
listen = xmalloc(sizeof(listen_t));
if ((ports = slurm_get_srun_port_range()))
listen->fd = slurm_init_msg_engine_ports(ports);
else
listen->fd = slurm_init_msg_engine_port(0);
if (listen->fd < 0) {
error("slurm_init_msg_engine_port error %m");
return NULL;
}
if (slurm_get_stream_addr(listen->fd, &listen->address) < 0) {
error("slurm_get_stream_addr error %m");
slurm_shutdown_msg_engine(listen->fd);
return NULL;
}
listen->hostname = xstrdup(interface_hostname);
/* FIXME - screw it! I can't seem to get the port number through
slurm_* functions */
listen->port = ntohs(listen->address.sin_port);
fd_set_nonblocking(listen->fd);
return listen;
}
/* Process incoming RPCs. Meant to execute as a pthread */
extern void *rpc_mgr(void *no_data)
{
int sockfd, newsockfd;
int i;
uint16_t port;
slurm_addr_t cli_addr;
slurmdbd_conn_t *conn_arg = NULL;
master_thread_id = pthread_self();
/* initialize port for RPCs */
if ((sockfd = slurm_init_msg_engine_port(get_dbd_port()))
== SLURM_SOCKET_ERROR)
fatal("slurm_init_msg_engine_port error %m");
slurm_persist_conn_recv_server_init();
/*
* Process incoming RPCs until told to shutdown
*/
while (!shutdown_time &&
(i = slurm_persist_conn_wait_for_thread_loc()) >= 0) {
/*
* accept needed for stream implementation is a no-op in
* message implementation that just passes sockfd to newsockfd
*/
if ((newsockfd = slurm_accept_msg_conn(sockfd,
&cli_addr)) ==
SLURM_SOCKET_ERROR) {
slurm_persist_conn_free_thread_loc(i);
if (errno != EINTR)
error("slurm_accept_msg_conn: %m");
continue;
}
fd_set_nonblocking(newsockfd);
conn_arg = xmalloc(sizeof(slurmdbd_conn_t));
conn_arg->conn = xmalloc(sizeof(slurm_persist_conn_t));
conn_arg->conn->fd = newsockfd;
conn_arg->conn->flags = PERSIST_FLAG_DBD;
conn_arg->conn->callback_proc = proc_req;
conn_arg->conn->callback_fini = _connection_fini_callback;
conn_arg->conn->shutdown = &shutdown_time;
conn_arg->conn->version = SLURM_MIN_PROTOCOL_VERSION;
conn_arg->conn->rem_host = xmalloc_nz(sizeof(char) * 16);
/* Don't fill in the rem_port here. It will be filled in
* later if it is a slurmctld connection. */
slurm_get_ip_str(&cli_addr, &port,
conn_arg->conn->rem_host, sizeof(char) * 16);
slurm_persist_conn_recv_thread_init(
conn_arg->conn, i, conn_arg);
}
debug("rpc_mgr shutting down");
(void) slurm_shutdown_msg_engine(sockfd);
pthread_exit((void *) 0);
return NULL;
}
zio_t *zio_reader_create (const char *name, int srcfd, void *arg)
{
zio_t *zio = zio_allocate (name, 1, arg);
zio->srcfd = srcfd;
fd_set_nonblocking (zio->srcfd);
zio->send = NULL;
return (zio);
}
zio_t *zio_writer_create (const char *name, int dstfd, void *arg)
{
zio_t *zio = zio_allocate (name, 0, arg);
zio->dstfd = dstfd;
fd_set_nonblocking (zio->dstfd);
/* Return zio object and wait for data via zio_write() operations...
*/
return (zio);
}
extern int slurm_persist_conn_open_without_init(
slurm_persist_conn_t *persist_conn)
{
slurm_addr_t addr;
xassert(persist_conn);
xassert(persist_conn->rem_host);
xassert(persist_conn->rem_port);
xassert(persist_conn->cluster_name);
if (persist_conn->fd > 0)
_close_fd(&persist_conn->fd);
else
persist_conn->fd = -1;
if (!persist_conn->inited)
persist_conn->inited = true;
if (!persist_conn->version) {
/* Set to MIN_PROTOCOL so that a higher version controller can
* talk to a lower protocol version controller. When talking to
* the DBD, the protocol version should be set to the current
* protocol version prior to calling this. */
persist_conn->version = SLURM_MIN_PROTOCOL_VERSION;
}
if (persist_conn->timeout < 0)
persist_conn->timeout = slurm_get_msg_timeout() * 1000;
slurm_set_addr_char(&addr, persist_conn->rem_port,
persist_conn->rem_host);
if ((persist_conn->fd = slurm_open_msg_conn(&addr)) < 0) {
if (_comm_fail_log(persist_conn)) {
char *s = xstrdup_printf("%s: failed to open persistent connection to %s:%d: %m",
__func__,
persist_conn->rem_host,
persist_conn->rem_port);
if (persist_conn->flags & PERSIST_FLAG_SUPPRESS_ERR)
debug2("%s", s);
else
error("%s", s);
xfree(s);
}
return SLURM_ERROR;
}
fd_set_nonblocking(persist_conn->fd);
fd_set_close_on_exec(persist_conn->fd);
return SLURM_SUCCESS;
}
/******************************************************************************
**函数名称: rtmq_listen_accept
**功 能: 接收连接请求
**输入参数:
** ctx: 全局对象
** lsn: 侦听对象
**输出参数: NONE
**返 回: 0:成功 !0:失败
**实现描述:
** 1. 接收连接请求
** 2. 发送至接收端
**注意事项:
**作 者: # Qifeng.zou # 2014.12.30 #
******************************************************************************/
static int rtrd_lsn_accept(rtrd_cntx_t *ctx, rtrd_listen_t *lsn)
{
int sckid;
socklen_t len;
rtmq_cmd_t cmd;
struct sockaddr_in cliaddr;
rtmq_cmd_add_sck_t *param = (rtmq_cmd_add_sck_t *)&cmd.param;
/* 1. 接收连接请求 */
for (;;) {
memset(&cliaddr, 0, sizeof(cliaddr));
len = sizeof(struct sockaddr_in);
sckid = accept(lsn->lsn_sck_id, (struct sockaddr *)&cliaddr, &len);
if (sckid >= 0) {
break;
}
else if (EINTR == errno) {
continue;
}
log_error(lsn->log, "errmsg:[%d] %s", errno, strerror(errno));
return RTMQ_ERR;
}
fd_set_nonblocking(sckid);
/* 2. 发送至接收端 */
memset(&cmd, 0, sizeof(cmd));
cmd.type = RTMQ_CMD_ADD_SCK;
param->sckid = sckid;
param->sid = ++lsn->sid; /* 设置套接字序列号 */
snprintf(param->ipaddr, sizeof(param->ipaddr), "%s", inet_ntoa(cliaddr.sin_addr));
log_trace(lsn->log, "New connection! serial:%lu sckid:%d ip:%s",
lsn->sid, sckid, inet_ntoa(cliaddr.sin_addr));
if (rtrd_cmd_to_rsvr(ctx, lsn->cmd_sck_id, &cmd, rtrd_rand_rsvr(ctx)) < 0) {
CLOSE(sckid);
log_error(lsn->log, "Send command failed! serial:%lu sckid:[%d]", lsn->sid, sckid);
return RTMQ_ERR;
}
return RTMQ_OK;
}
/*
* create_listen_socket - create a listening UNIX domain socket
* for srun to connect
* RETURN: the socket fd on success, -1 on error
*/
static int
create_listen_socket(void)
{
struct sockaddr_un sa;
unsigned int sa_len;
int re_use_addr = 1;
close (listen_fd); /* close possible old socket */
sprintf(cr_sock_addr, "/tmp/sock.srun_cr.%u", (unsigned int)getpid());
listen_fd = socket(AF_UNIX, SOCK_STREAM, 0);
if (listen_fd < 0) {
error("failed to create listen socket: %m");
return -1;
}
sa.sun_family = AF_UNIX;
strcpy(sa.sun_path, cr_sock_addr);
sa_len = strlen(sa.sun_path) + sizeof(sa.sun_family);
unlink(sa.sun_path); /* remove possible old socket */
setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR,
(void*)&re_use_addr, sizeof(int));
if (bind(listen_fd, (struct sockaddr *)&sa, sa_len) < 0) {
error("failed to bind listen socket: %m");
unlink(sa.sun_path);
return -1;
}
if (listen(listen_fd, 2) < 0) {
error("failed to listen: %m");
unlink(sa.sun_path);
return -1;
}
fd_set_nonblocking(listen_fd);
return listen_fd;
}
size_t pmixp_read_buf(int sd, void *buf, size_t count, int *shutdown,
bool blocking)
{
ssize_t ret, offs = 0;
*shutdown = 0;
if (!blocking && !pmixp_fd_read_ready(sd, shutdown)) {
return 0;
}
if (blocking) {
fd_set_blocking(sd);
}
while (count - offs > 0) {
ret = read(sd, (char *)buf + offs, count - offs);
if (ret > 0) {
offs += ret;
continue;
} else if (ret == 0) {
/* connection closed. */
*shutdown = 1;
return offs;
}
switch (errno) {
case EINTR:
continue;
case EWOULDBLOCK:
/* we can get here in non-blocking mode only */
return offs;
default:
PMIXP_ERROR_STD("blocking=%d", blocking);
*shutdown = -errno;
return offs;
}
}
if (blocking) {
fd_set_nonblocking(sd);
}
return offs;
}
/* Open event_fd as needed
* RET 0 on success, -1 on failure */
static int _open_fd(time_t now)
{
if (event_fd != -1)
return 0;
/* Identify address for socket connection.
* Done only on first call, then cached. */
if (event_addr_set == 0) {
slurm_set_addr(&moab_event_addr, e_port, e_host);
event_addr_set = 1;
if (e_host_bu[0] != '\0') {
slurm_set_addr(&moab_event_addr_bu, e_port,
e_host_bu);
event_addr_set = 2;
}
}
/* Open the event port on moab as needed */
if (event_fd == -1) {
event_fd = slurm_open_msg_conn(&moab_event_addr);
if (event_fd == -1) {
error("Unable to open primary wiki "
"event port %s:%u: %m",
e_host, e_port);
}
}
if ((event_fd == -1) && (event_addr_set == 2)) {
event_fd = slurm_open_msg_conn(&moab_event_addr_bu);
if (event_fd == -1) {
error("Unable to open backup wiki "
"event port %s:%u: %m",
e_host_bu, e_port);
}
}
if (event_fd == -1)
return -1;
/* We can't have the controller block on the following write() */
fd_set_nonblocking(event_fd);
return 0;
}
eio_handle_t *eio_handle_create(void)
{
eio_handle_t *eio = xmalloc(sizeof(*eio));
if (pipe(eio->fds) < 0) {
error ("eio_create: pipe: %m");
eio_handle_destroy(eio);
return (NULL);
}
fd_set_nonblocking(eio->fds[0]);
fd_set_close_on_exec(eio->fds[0]);
fd_set_close_on_exec(eio->fds[1]);
xassert(eio->magic = EIO_MAGIC);
eio->obj_list = list_create(eio_obj_destroy);
eio->new_objs = list_create(eio_obj_destroy);
return eio;
}
size_t pmixp_write_buf(int sd, void *buf, size_t count, int *shutdown,
bool blocking)
{
ssize_t ret, offs = 0;
*shutdown = 0;
if (!blocking && !pmixp_fd_write_ready(sd, shutdown)) {
return 0;
}
if (blocking) {
fd_set_blocking(sd);
}
while (count - offs > 0) {
ret = write(sd, (char *)buf + offs, count - offs);
if (ret > 0) {
offs += ret;
continue;
}
switch (errno) {
case EINTR:
continue;
case EWOULDBLOCK:
return offs;
default:
*shutdown = -errno;
return offs;
}
}
if (blocking) {
fd_set_nonblocking(sd);
}
return offs;
}
int
msg_thr_create(slurmd_job_t *job)
{
int fd;
eio_obj_t *eio_obj;
pthread_attr_t attr;
int rc = SLURM_SUCCESS, retries = 0;
errno = 0;
fd = _domain_socket_create(conf->spooldir, conf->node_name,
job->jobid, job->stepid);
if (fd == -1)
return SLURM_ERROR;
fd_set_nonblocking(fd);
eio_obj = eio_obj_create(fd, &msg_socket_ops, (void *)job);
job->msg_handle = eio_handle_create();
eio_new_initial_obj(job->msg_handle, eio_obj);
slurm_attr_init(&attr);
while (pthread_create(&job->msgid, &attr,
&_msg_thr_internal, (void *)job)) {
error("msg_thr_create: pthread_create error %m");
if (++retries > MAX_RETRIES) {
error("msg_thr_create: Can't create pthread");
rc = SLURM_ERROR;
break;
}
usleep(10); /* sleep and again */
}
slurm_attr_destroy(&attr);
return rc;
}
/***************************************************************************
* Support functions for slurm_allocate_resources_blocking()
***************************************************************************/
static listen_t *_create_allocation_response_socket(char *interface_hostname)
{
listen_t *listen = NULL;
listen = xmalloc(sizeof(listen_t));
/* port "0" lets the operating system pick any port */
if ((listen->fd = slurm_init_msg_engine_port(0)) < 0) {
error("slurm_init_msg_engine_port error %m");
return NULL;
}
if (slurm_get_stream_addr(listen->fd, &listen->address) < 0) {
error("slurm_get_stream_addr error %m");
slurm_shutdown_msg_engine(listen->fd);
return NULL;
}
listen->hostname = xstrdup(interface_hostname);
/* FIXME - screw it! I can't seem to get the port number through
slurm_* functions */
listen->port = ntohs(listen->address.sin_port);
fd_set_nonblocking(listen->fd);
return listen;
}
void
job_accept (conf_t conf)
{
work_p w;
m_msg_t m;
int sd;
assert (conf != NULL);
assert (conf->ld >= 0);
if (!(w = work_init ((work_func_t) _job_exec, conf->nthreads))) {
log_errno (EMUNGE_SNAFU, LOG_ERR,
"Failed to create %d work thread%s", conf->nthreads,
((conf->nthreads > 1) ? "s" : ""));
}
log_msg (LOG_INFO, "Created %d work thread%s", conf->nthreads,
((conf->nthreads > 1) ? "s" : ""));
while (!done) {
if ((sd = accept (conf->ld, NULL, NULL)) < 0) {
switch (errno) {
case ECONNABORTED:
case EINTR:
continue;
case EMFILE:
case ENFILE:
case ENOBUFS:
case ENOMEM:
log_msg (LOG_INFO,
"Suspended new connections while processing backlog");
work_wait (w);
continue;
default:
log_errno (EMUNGE_SNAFU, LOG_ERR,
"Failed to accept connection");
break;
}
}
/* With fd_timed_read_n(), a poll() is performed before any read()
* in order to provide timeouts and ensure the read() won't block.
* As such, it shouldn't be necessary to set the client socket as
* non-blocking. However according to the Linux poll(2) and
* select(2) manpages, spurious readiness notifications can occur.
* poll()/select() may report a socket as ready for reading while
* the subsequent read() blocks. This could happen when data has
* arrived, but upon examination is discarded due to an invalid
* checksum. To protect against this, the client socket is set
* non-blocking and EAGAIN is handled appropriately.
*/
if (fd_set_nonblocking (sd) < 0) {
close (sd);
log_msg (LOG_WARNING,
"Failed to set nonblocking client socket: %s",
strerror (errno));
}
else if (m_msg_create (&m) != EMUNGE_SUCCESS) {
close (sd);
log_msg (LOG_WARNING, "Failed to create client request");
}
else if (m_msg_bind (m, sd) != EMUNGE_SUCCESS) {
m_msg_destroy (m);
log_msg (LOG_WARNING, "Failed to bind socket for client request");
}
else if (work_queue (w, m) < 0) {
m_msg_destroy (m);
log_msg (LOG_WARNING, "Failed to queue client request");
}
}
log_msg (LOG_NOTICE, "Exiting on signal=%d", done);
work_fini (w, 1);
return;
}
/* Process incoming RPCs. Meant to execute as a pthread */
extern void *rpc_mgr(void *no_data)
{
pthread_attr_t thread_attr_rpc_req;
slurm_fd_t sockfd, newsockfd;
int i, retry_cnt, sigarray[] = {SIGUSR1, 0};
slurm_addr_t cli_addr;
slurmdbd_conn_t *conn_arg = NULL;
slurm_mutex_lock(&thread_count_lock);
master_thread_id = pthread_self();
slurm_mutex_unlock(&thread_count_lock);
(void) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
(void) pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
/* threads to process individual RPC's are detached */
slurm_attr_init(&thread_attr_rpc_req);
if (pthread_attr_setdetachstate
(&thread_attr_rpc_req, PTHREAD_CREATE_DETACHED))
fatal("pthread_attr_setdetachstate %m");
/* initialize port for RPCs */
if ((sockfd = slurm_init_msg_engine_port(get_dbd_port()))
== SLURM_SOCKET_ERROR)
fatal("slurm_init_msg_engine_port error %m");
/* Prepare to catch SIGUSR1 to interrupt accept().
* This signal is generated by the slurmdbd signal
* handler thread upon receipt of SIGABRT, SIGINT,
* or SIGTERM. That thread does all processing of
* all signals. */
xsignal(SIGUSR1, _sig_handler);
xsignal_unblock(sigarray);
/*
* Process incoming RPCs until told to shutdown
*/
while ((i = _wait_for_server_thread()) >= 0) {
/*
* accept needed for stream implementation is a no-op in
* message implementation that just passes sockfd to newsockfd
*/
if ((newsockfd = slurm_accept_msg_conn(sockfd,
&cli_addr)) ==
SLURM_SOCKET_ERROR) {
_free_server_thread((pthread_t) 0);
if (errno != EINTR)
error("slurm_accept_msg_conn: %m");
continue;
}
fd_set_nonblocking(newsockfd);
conn_arg = xmalloc(sizeof(slurmdbd_conn_t));
conn_arg->newsockfd = newsockfd;
slurm_get_ip_str(&cli_addr, &conn_arg->orig_port,
conn_arg->ip, sizeof(conn_arg->ip));
retry_cnt = 0;
while (pthread_create(&slave_thread_id[i],
&thread_attr_rpc_req,
_service_connection,
(void *) conn_arg)) {
if (retry_cnt > 0) {
error("pthread_create failure, "
"aborting RPC: %m");
close(newsockfd);
break;
}
error("pthread_create failure: %m");
retry_cnt++;
usleep(1000); /* retry in 1 msec */
}
}
debug3("rpc_mgr shutting down");
slurm_attr_destroy(&thread_attr_rpc_req);
(void) slurm_shutdown_msg_engine(sockfd);
_wait_for_thread_fini();
pthread_exit((void *) 0);
return NULL;
}
/* Get slurm message with timeout
* RET message size (as specified in argument) or SLURM_ERROR on error */
int _slurm_recv_timeout(slurm_fd_t fd, char *buffer, size_t size,
uint32_t flags, int timeout )
{
int rc;
int recvlen = 0;
int fd_flags;
struct pollfd ufds;
struct timeval tstart;
int timeleft = timeout;
ufds.fd = fd;
ufds.events = POLLIN;
fd_flags = _slurm_fcntl(fd, F_GETFL);
fd_set_nonblocking(fd);
gettimeofday(&tstart, NULL);
while (recvlen < size) {
timeleft = timeout - _tot_wait(&tstart);
if (timeleft <= 0) {
debug("_slurm_recv_timeout at %d of %zd, timeout",
recvlen, size);
slurm_seterrno(SLURM_PROTOCOL_SOCKET_IMPL_TIMEOUT);
recvlen = SLURM_ERROR;
goto done;
}
if ((rc = poll(&ufds, 1, timeleft)) <= 0) {
if ((errno == EINTR) || (errno == EAGAIN) || (rc == 0))
continue;
else {
debug("_slurm_recv_timeout at %d of %zd, "
"poll error: %s",
recvlen, size, strerror(errno));
slurm_seterrno(
SLURM_COMMUNICATIONS_RECEIVE_ERROR);
recvlen = SLURM_ERROR;
goto done;
}
}
if (ufds.revents & POLLERR) {
debug("_slurm_recv_timeout: Socket POLLERR");
slurm_seterrno(ENOTCONN);
recvlen = SLURM_ERROR;
goto done;
}
if ((ufds.revents & POLLNVAL) ||
((ufds.revents & POLLHUP) &&
((ufds.revents & POLLIN) == 0))) {
debug2("_slurm_recv_timeout: Socket no longer there");
slurm_seterrno(ENOTCONN);
recvlen = SLURM_ERROR;
goto done;
}
if ((ufds.revents & POLLIN) != POLLIN) {
error("_slurm_recv_timeout: Poll failure, revents:%d",
ufds.revents);
continue;
}
rc = _slurm_recv(fd, &buffer[recvlen], (size - recvlen), flags);
if (rc < 0) {
if (errno == EINTR)
continue;
else {
debug("_slurm_recv_timeout at %d of %zd, "
"recv error: %s",
recvlen, size, strerror(errno));
slurm_seterrno(
SLURM_COMMUNICATIONS_RECEIVE_ERROR);
recvlen = SLURM_ERROR;
goto done;
}
}
if (rc == 0) {
debug("_slurm_recv_timeout at %d of %zd, "
"recv zero bytes", recvlen, size);
slurm_seterrno(SLURM_PROTOCOL_SOCKET_ZERO_BYTES_SENT);
recvlen = SLURM_ERROR;
goto done;
}
recvlen += rc;
}
done:
/* Reset fd flags to prior state, preserve errno */
if (fd_flags != SLURM_PROTOCOL_ERROR) {
int slurm_err = slurm_get_errno();
_slurm_fcntl(fd , F_SETFL , fd_flags);
slurm_seterrno(slurm_err);
}
return recvlen;
}
int _slurm_set_stream_non_blocking(slurm_fd_t fd)
{
fd_set_nonblocking(fd);
return SLURM_SUCCESS;
}
static munge_err_t
_m_msg_client_connect (m_msg_t m, char *path)
{
struct stat st;
struct sockaddr_un addr;
int sd;
int n;
int i;
unsigned long delay_msecs;
assert (m != NULL);
assert (m->sd < 0);
if ((path == NULL) || (*path == '\0')) {
m_msg_set_err (m, EMUNGE_SOCKET,
strdup ("MUNGE socket name is undefined"));
return (EMUNGE_SOCKET);
}
if (stat (path, &st) < 0) {
m_msg_set_err (m, EMUNGE_SOCKET,
strdupf ("Failed to access \"%s\": %s", path, strerror (errno)));
return (EMUNGE_SOCKET);
}
if (!S_ISSOCK (st.st_mode)) {
m_msg_set_err (m, EMUNGE_SOCKET,
strdupf ("Invalid file type for socket \"%s\"", path));
return (EMUNGE_SOCKET);
}
if ((sd = socket (PF_UNIX, SOCK_STREAM, 0)) < 0) {
m_msg_set_err (m, EMUNGE_SOCKET,
strdupf ("Failed to create socket: %s", strerror (errno)));
return (EMUNGE_SOCKET);
}
if (fd_set_nonblocking (sd) < 0) {
close (sd);
m_msg_set_err (m, EMUNGE_SOCKET,
strdupf ("Failed to set nonblocking socket: %s",
strerror (errno)));
return (EMUNGE_SOCKET);
}
memset (&addr, 0, sizeof (addr));
addr.sun_family = AF_UNIX;
addr.sun_path[ sizeof (addr.sun_path) - 1 ] = '\0';
strncpy (addr.sun_path, path, sizeof (addr.sun_path));
if (addr.sun_path[ sizeof (addr.sun_path) - 1 ] != '\0') {
close (sd);
m_msg_set_err (m, EMUNGE_OVERFLOW,
strdup ("Exceeded maximum length of socket pathname"));
return (EMUNGE_OVERFLOW);
}
i = 1;
while (1) {
/*
* If a call to connect() for a Unix domain stream socket finds that
* the listening socket's queue is full, ECONNREFUSED is returned
* immediately. (cf, Stevens UNPv1, s14.4, p378)
* If ECONNREFUSED, try again up to MUNGE_SOCKET_CONNECT_ATTEMPTS.
*/
n = connect (sd, (struct sockaddr *) &addr, sizeof (addr));
if (n == 0) {
break;
}
if (errno == EINTR) {
continue;
}
if (errno != ECONNREFUSED) {
break;
}
if (i >= MUNGE_SOCKET_CONNECT_ATTEMPTS) {
break;
}
delay_msecs = i * MUNGE_SOCKET_CONNECT_RETRY_MSECS;
if (_m_msg_client_millisleep (m, delay_msecs) != EMUNGE_SUCCESS) {
break;
}
i++;
}
if (n < 0) {
close (sd);
m_msg_set_err (m, EMUNGE_SOCKET,
strdupf ("Failed to connect to \"%s\": %s", path,
strerror (errno)));
return (EMUNGE_SOCKET);
}
m->sd = sd;
return (EMUNGE_SUCCESS);
}
static void attach (flux_t *h, const char *key, bool rawtty, int kzoutflags,
int blocksize)
{
t_kzutil_ctx_t *ctx = xzmalloc (sizeof (*ctx));
char *name;
int fdin = dup (STDIN_FILENO);
struct termios saved_tio;
flux_reactor_t *r = flux_get_reactor (h);
flux_watcher_t *w = NULL;
log_msg ("process attached to %s", key);
ctx->h = h;
ctx->blocksize = blocksize;
/* FIXME: need a ~. style escape sequence to terminate stdin
* in raw mode.
*/
if (rawtty) {
if (fd_set_raw (fdin, &saved_tio, true) < 0)
log_err_exit ("fd_set_raw stdin");
}
if (fd_set_nonblocking (fdin, true) < 0)
log_err_exit ("fd_set_nonblocking stdin");
if (asprintf (&name, "%s.stdin", key) < 0)
oom ();
if (!(ctx->kz[0] = kz_open (h, name, kzoutflags)))
if (errno == EEXIST)
log_err ("disabling stdin");
else
log_err_exit ("%s", name);
else {
if (!(w = flux_fd_watcher_create (r, fdin, FLUX_POLLIN,
attach_stdin_ready_cb, ctx)))
log_err_exit ("flux_fd_watcher_create %s", name);
flux_watcher_start (w);
}
free (name);
if (asprintf (&name, "%s.stdout", key) < 0)
oom ();
if (!(ctx->kz[1] = kz_open (h, name, KZ_FLAGS_READ | KZ_FLAGS_NONBLOCK)))
log_err_exit ("kz_open %s", name);
if (kz_set_ready_cb (ctx->kz[1], attach_stdout_ready_cb, ctx) < 0)
log_err_exit ("kz_set_ready_cb %s", name);
free (name);
ctx->readers++;
if (asprintf (&name, "%s.stderr", key) < 0)
oom ();
if (!(ctx->kz[2] = kz_open (h, name, KZ_FLAGS_READ | KZ_FLAGS_NONBLOCK)))
log_err_exit ("kz_open %s", name);
if (kz_set_ready_cb (ctx->kz[2], attach_stderr_ready_cb, ctx) < 0)
log_err_exit ("kz_set_ready_cb %s", name);
free (name);
ctx->readers++;
/* Reactor terminates when ctx->readers reaches zero, i.e.
* when EOF is read from remote stdout and stderr.
* (Note: if they are already at eof, we will have already terminated
* before the reactor is started, since kvs_watch callbacks make one
* call to the callback in the context of the caller).
*/
if (ctx->readers > 0) {
if (flux_reactor_run (r, 0) < 0)
log_err_exit ("flux_reactor_run");
}
(void)kz_close (ctx->kz[1]);
(void)kz_close (ctx->kz[2]);
/* FIXME: tty state needs to be restored on all exit paths.
*/
if (rawtty) {
if (fd_set_raw (fdin, &saved_tio, false) < 0)
log_err_exit ("fd_set_raw stdin");
}
flux_watcher_destroy (w);
free (ctx);
}
/* Send slurm message with timeout
* RET message size (as specified in argument) or SLURM_ERROR on error */
int _slurm_send_timeout(slurm_fd_t fd, char *buf, size_t size,
uint32_t flags, int timeout)
{
int rc;
int sent = 0;
int fd_flags;
struct pollfd ufds;
struct timeval tstart;
int timeleft = timeout;
char temp[2];
ufds.fd = fd;
ufds.events = POLLOUT;
fd_flags = _slurm_fcntl(fd, F_GETFL);
fd_set_nonblocking(fd);
gettimeofday(&tstart, NULL);
while (sent < size) {
timeleft = timeout - _tot_wait(&tstart);
if (timeleft <= 0) {
debug("_slurm_send_timeout at %d of %zd, timeout",
sent, size);
slurm_seterrno(SLURM_PROTOCOL_SOCKET_IMPL_TIMEOUT);
sent = SLURM_ERROR;
goto done;
}
if ((rc = poll(&ufds, 1, timeleft)) <= 0) {
if ((rc == 0) || (errno == EINTR) || (errno == EAGAIN))
continue;
else {
debug("_slurm_send_timeout at %d of %zd, "
"poll error: %s",
sent, size, strerror(errno));
slurm_seterrno(SLURM_COMMUNICATIONS_SEND_ERROR);
sent = SLURM_ERROR;
goto done;
}
}
/*
* Check here to make sure the socket really is there.
* If not then exit out and notify the sender. This
* is here since a write doesn't always tell you the
* socket is gone, but getting 0 back from a
* nonblocking read means just that.
*/
if (ufds.revents & POLLERR) {
debug("_slurm_send_timeout: Socket POLLERR");
slurm_seterrno(ENOTCONN);
sent = SLURM_ERROR;
goto done;
}
if ((ufds.revents & POLLHUP) || (ufds.revents & POLLNVAL) ||
(_slurm_recv(fd, &temp, 1, flags) == 0)) {
debug2("_slurm_send_timeout: Socket no longer there");
slurm_seterrno(ENOTCONN);
sent = SLURM_ERROR;
goto done;
}
if ((ufds.revents & POLLOUT) != POLLOUT) {
error("_slurm_send_timeout: Poll failure, revents:%d",
ufds.revents);
}
rc = _slurm_send(fd, &buf[sent], (size - sent), flags);
if (rc < 0) {
if (errno == EINTR)
continue;
debug("_slurm_send_timeout at %d of %zd, "
"send error: %s",
sent, size, strerror(errno));
if (errno == EAGAIN) { /* poll() lied to us */
usleep(10000);
continue;
}
slurm_seterrno(SLURM_COMMUNICATIONS_SEND_ERROR);
sent = SLURM_ERROR;
goto done;
}
if (rc == 0) {
debug("_slurm_send_timeout at %d of %zd, "
"sent zero bytes", sent, size);
slurm_seterrno(SLURM_PROTOCOL_SOCKET_ZERO_BYTES_SENT);
sent = SLURM_ERROR;
goto done;
}
sent += rc;
}
done:
/* Reset fd flags to prior state, preserve errno */
if (fd_flags != SLURM_PROTOCOL_ERROR) {
int slurm_err = slurm_get_errno();
_slurm_fcntl(fd , F_SETFL , fd_flags);
slurm_seterrno(slurm_err);
}
return sent;
}
int main(int argc,char **argv)
{
server *srv=NULL;
//step 1 : initialize server
if( (srv= server_init()) ==NULL){
fprintf(stderr,"failed to initialize server in [%s|%d|%s]\n",__FILE__,__LINE__,__FUNCTION__);
return -1;
}
//step 2 : parameter parse
char opt_chr;
while( (opt_chr=getopt(argc,argv,"f:hvD"))!=-1 ){
switch(opt_chr){
/*configuration file path */
case 'f':{
buffer_copy_string(srv->config->minihttpd_global_config_filepath,optarg);
break;
}
/* show help */
case 'h':{
print_help();
server_free(srv);
return 0;
}
case 'v':{
fprintf(stdout,"%s-%s",srv->config->service_name->ptr,srv->config->version_info->ptr);
server_free(srv);
return 0;
}
case 'D':{
srv->dont_daemonize=1;
break;
}
default:{
print_help();
server_free(srv);
return -1;
}
}
}
//step 3 :check if all configuraiton is legal
if(buffer_is_empty(srv->config->service_root_dir)){
fprintf(stderr,"[%s|%d|%s]:please specify minihttp root dir in configuration file\n",
__FILE__,__LINE__,__FUNCTION__);
server_free(srv);
return -1;
}
/*parse the mime configuration file */
if(buffer_is_empty(srv->config->mimetype_filepath)
|| (srv->config->table= mime_table_initialize( (const char*)srv->config->mimetype_filepath->ptr) )==NULL){
fprintf(stderr,"invalid mime configuration file is specified,pls check it..\n");
server_free(srv);
return -1;
}
//step4 :server started
srv->uid=getuid();
srv->gid=getgid();
if(srv->uid==0) { //we are root
struct rlimit res_limit;
if(getrlimit(RLIMIT_NOFILE,&res_limit)!=0){
fprintf(stderr,"[%s|%d|%s]: failed to get file descriptor max number for current process!\n",
__FILE__,__LINE__,__FUNCTION__);
server_free(srv);
return -1;
}
res_limit.rlim_cur=srv->config->max_fd;
res_limit.rlim_max=srv->config->max_fd;
if(setrlimit(RLIMIT_NOFILE,&res_limit)!=0){
fprintf(stderr,"[%s|%d|%s]: failed call setrlimit(RLIMIT_NOFILE,) for current process!\n",
__FILE__,__LINE__,__FUNCTION__);
server_free(srv);
return -1;
}
}else{
struct rlimit res_limit;
if(getrlimit(RLIMIT_NOFILE,&res_limit)!=0){
fprintf(stderr,"[%s|%d|%s]: failed to get file descriptor max number for current process!\n",
__FILE__,__LINE__,__FUNCTION__);
server_free(srv);
return -1;
}
if(srv->config->max_fd< res_limit.rlim_cur)
res_limit.rlim_cur=srv->config->max_fd;
else if(srv->config->max_fd<=res_limit.rlim_max)
res_limit.rlim_cur=srv->config->max_fd;
if(setrlimit(RLIMIT_NOFILE,&res_limit)!=0){
fprintf(stderr,"[%s|%d|%s]: failed call setrlimit(RLIMIT_NOFILE,) for current process!\n",
__FILE__,__LINE__,__FUNCTION__);
server_free(srv);
return -1;
}
}
//step 5: become a daemon process if dont_daemonize=0;
if(!srv->dont_daemonize){
daemonize((const char*)srv->config->service_name->ptr);
}
//step 6: open log file for error log, by default we use syslog.
// if the minihttpd log filepath is specified manually or server dont_daemonize=1,
// we set mode = LOG_MODE_FILE;
if(!buffer_is_empty(srv->config->log_filename) || srv->dont_daemonize ){
if(buffer_is_empty(srv->config->log_filename))
buffer_copy_string(srv->config->log_filename,MINIHTTPD_DEFAULT_LOG_FILEPATH);
srv->log_fd= open((const char*)srv->config->log_filename->ptr,O_WRONLY|O_CREAT|O_TRUNC,
S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
if(srv->log_fd<0){
server_free(srv);
return -1;
}
fd_close_on_exec(srv->log_fd);
srv->mode=server::LOG_MODE_FILE;
}
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_INFO,"%s is start now...",(const char*)srv->config->service_name->ptr);
//step 7 : create listening tcp socket(we only support ipv4 now)
struct sockaddr_in * addr= (struct sockaddr_in*)&srv->server_addr;
memset(addr,0,sizeof(*addr));
addr->sin_family=AF_INET;
addr->sin_addr.s_addr=htonl(INADDR_ANY);
addr->sin_port=htons(srv->config->listenint_port);
srv->listening_socket_fd= create_tcp_socket((struct sockaddr*)addr,srv->config->max_listening_number);
if(srv->listening_socket_fd<0){
log_to_backend(srv, MINIHTTPD_LOG_LEVEL_ERROR,"failed to create listening tcp socket on port:%d",
srv->config->listenint_port);
server_free(srv);
return -1;
}
/*
step 8: setup signal handler
signo: SIGCHLD
signo: SIGPIPE: unix domain socket pipe is broken
signo: SIGINT: user intend to shutdown the minihttpd server
if SIGINT is kill to the server proces for twice, the minihtpd server is going to shutdown
signo: SIGTERM: exit minihttpd service now
*/
struct sigaction act;
sigemptyset(&act.sa_mask);
act.sa_flags=0;
act.sa_handler=signal_handler;
sigaction(SIGPIPE,&act,NULL);
sigaction(SIGCHLD,&act,NULL);
sigaction(SIGINT,&act,NULL);
sigaction(SIGTERM,&act,NULL);
/*
step 9: fork worker child process and transfter accept socket file descriptor to worker process
the only tasks for main processis :
1) to call accept to wait for connection and pick one worker process to handle the connection
2) wait all worker process to finish before exit.
*/
for(uint32_t worker_process_id=0;worker_process_id< srv->worker_number ;worker_process_id++) {
server_child * child= &srv->child[worker_process_id];
//create unix domain socket
if(socketpair(AF_UNIX,SOCK_STREAM,0,child->unix_domain_socket_fd)<0){
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"failed to create unix domain socket for worker%d",
worker_process_id);
close(srv->listening_socket_fd);
server_free(srv);
return -1;
}
child->sent_connection_number=0;
int unix_domain_socket_child_fd=child->unix_domain_socket_fd[1];
fd_set_nonblocking(unix_domain_socket_child_fd);
child->pid=fork();
if(child->pid <0){ //we can not fork worker process, this should not be happened
close(srv->listening_socket_fd);
server_free(srv) ;
return -1;
}
else if(child->pid ==0) { /* worker process */
/*we should use p_worker only in the child worker process */
#if 0
minihttpd_running_log(srv->log_fd,MINIHTTPD_LOG_LEVEL_INFO,__FILE__,__LINE__,__FUNCTION__,
"worker(pid=%d) is starting.....",getpid());
#endif
worker * server_worker = (worker*)malloc(sizeof(worker));
memset(server_worker,0,sizeof(worker));
server_worker->worker_id= worker_process_id;
server_worker->unix_domain_socekt_fd=unix_domain_socket_child_fd;
server_worker->log_filepath=buffer_init();
server_worker->global_config= srv->config;
/*step1 : get current file descriptor max number (it should be same as parent process
which we have set the resouces)*/
struct rlimit limit;
if(getrlimit(RLIMIT_NOFILE,&limit)<0){
exit(-1); // terminated the worker
}
//close unnecessary file descriptor
for(uint32_t file_descriptor_index=0;file_descriptor_index< limit.rlim_cur;file_descriptor_index++){
if(file_descriptor_index> STDERR_FILENO && file_descriptor_index != unix_domain_socket_child_fd){
close(file_descriptor_index);
}
}
//step 2: set event handler
server_worker->ev= fdevent_initialize(limit.rlim_cur);
/*support max connection number */
uint32_t worker_support_max_connections=limit.rlim_cur/2;
worker_connection_initialize(server_worker, worker_support_max_connections);
//step 3 : register unix domain socket event
fdevents_register_fd(server_worker->ev,server_worker->unix_domain_socekt_fd,
unix_domain_socket_handle,server_worker);
//EPOLLHUP |EPOLLERR events is set by default
fdevents_set_events(server_worker->ev,server_worker->unix_domain_socekt_fd,EPOLLIN);
//step 4 : open log file for worker to log debug/info/warning/error
if(buffer_is_empty(server_worker->log_filepath)){
char worker_log_filepath[255];
snprintf(worker_log_filepath,sizeof(worker_log_filepath),
MINIHTTPD_WORKER_CONFIG_PATH"%u.log", server_worker->worker_id );
buffer_append_string(server_worker->log_filepath,worker_log_filepath);
}
server_worker->log_fd= open((const char*)server_worker->log_filepath->ptr, O_WRONLY|O_CREAT|O_TRUNC,
S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
if(server_worker->log_fd<0){
exit(-2);
}
//step 5 : setup timer and expect timer will expire with internal 1 seconds
time(&server_worker->cur_ts);
int timer_fd=timerfd_create(CLOCK_REALTIME,TFD_NONBLOCK);
struct itimerspec timer_spec;
timer_spec.it_value.tv_sec=1;
timer_spec.it_value.tv_nsec=0;
timer_spec.it_interval.tv_sec=1;
timer_spec.it_interval.tv_nsec=0;
timerfd_settime(timer_fd,0,&timer_spec,NULL);
// setup timer experation events handler
fdevents_register_fd(server_worker->ev, timer_fd,worker_timer_expire_handler,
server_worker);
fdevents_set_events(server_worker->ev,timer_fd,EPOLLIN);
/* main loop for worker: epoll event loop for unix domain socket and connections */
while(server_shutdown==0 || server_worker->cur_connection_number >0 ) {
int n=epoll_wait(server_worker->ev->epoll_fd, server_worker->ev->epoll_events,
server_worker->ev->max_epoll_events,-1);
if(n<0 ){
if(errno!=EINTR){
minihttpd_running_log(server_worker->log_fd,MINIHTTPD_LOG_LEVEL_ERROR
,__FILE__,__LINE__,__FUNCTION__,
"failed to call epoll with errno=%d",errno);
}
continue;
}
else if(n==0){
//we should not get to here
continue;
}else {
for(uint32_t event_index=0;event_index<n;event_index++){
struct epoll_event * event= &server_worker->ev->epoll_events[event_index];
assert(event!=NULL);
int connection_socket_fd= event->data.fd;
event_handle handler=fdevents_get_handle(server_worker->ev,connection_socket_fd);
void * event_ctx=fdevents_get_context(server_worker->ev, connection_socket_fd);
assert(handler!=NULL);
int handle_status= handler(connection_socket_fd,event_ctx,event->events);
minihttpd_running_log(server_worker->log_fd,handle_status==0?
MINIHTTPD_LOG_LEVEL_INFO:MINIHTTPD_LOG_LEVEL_ERROR,
__FILE__,__LINE__,__FUNCTION__,"the epoll event is already handled!");
}
}
}
minihttpd_running_log(server_worker->log_fd,MINIHTTPD_LOG_LEVEL_INFO,
__FILE__,__LINE__,__FUNCTION__,"child worker process has finished all client requests!\n");
/*free all connections */
worker_free_connectons(server_worker);
/* unregister timer file descriptor */
fdevents_unset_event(server_worker->ev,timer_fd);
fdevents_unregister_fd(server_worker->ev,timer_fd);
close(timer_fd);
/* unregister unix domain socket hanlde events and handler context */
fdevents_unset_event(server_worker->ev,server_worker->unix_domain_socekt_fd);
fdevents_unregister_fd(server_worker->ev,server_worker->unix_domain_socekt_fd);
close(server_worker->unix_domain_socekt_fd);
/* free fevents resources */
close(server_worker->ev->epoll_fd);
fdevent_free(server_worker->ev);
//close the log file
close(server_worker->log_fd);
buffer_free(server_worker->log_filepath);
/* free worker */
free( (void*) server_worker);
exit(0); //termianted the worker
}
//close the unix domain socket worker file descriptor;
close(child->unix_domain_socket_fd[1]);
// child worker is running
child->worker_running=1;
//parent process
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_INFO,"worker process %d is already created!",worker_process_id);
}
//main loop to accept client connection and re-transfter to worker process
while(!server_shutdown) {
/*
log signal events after signal si handled by signal handler
*/
if(signal_pipe_handled){
/* if unix domain socket pipe is broken and we still write data to the pipe */
}
if(signal_child_handled){
/*a child worker process has terminated */
int worker_exit_status;
pid_t exit_worker_pid;
while( (exit_worker_pid= waitpid(-1,&worker_exit_status,WNOHANG))>0){
if(WIFEXITED(worker_exit_status)){
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"worker child process(pid=%d) has exited normally with exit" \
"status=%d",exit_worker_pid,WEXITSTATUS(worker_exit_status));
}
else if(WIFSIGNALED(worker_exit_status)){
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"worker child process(pid=%d) is killed by signal(%d)",
exit_worker_pid,WTERMSIG(worker_exit_status))
}
else{
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"worker child process(pid=%d) has exited unexpected", exit_worker_pid);
}
//remove the worker from available worker list and do not send socket file descriptor to it
for(uint32_t child_worker_id=0;child_worker_id< srv->worker_number;child_worker_id++){
if(srv->child[child_worker_id].pid==exit_worker_pid)
srv->child[child_worker_id].worker_running=0;
}
}
signal_child_handled=0;
}
//we block here to wait connection(only IPV4 is supported now )
struct sockaddr_in client_addr;
socklen_t client_addr_length=sizeof(client_addr);
int connection_fd =accept(srv->listening_socket_fd,(struct sockaddr*)&client_addr,(socklen_t*)& client_addr_length);
if(connection_fd<0){
switch(errno){
case EINTR:
// the connection is reset by client
case ECONNABORTED:
continue;
case EMFILE: //file descriptor is all used now, need to send file descriptor to worker soon
break;
default: {
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"failed to call accept() with errno=%d\n",errno);
break;
}
}
}
else{
/*
pick up a worker process and send the @conneciton_fd to it
the pick algorithm is round-robin,;
but for the draft version, we just pick a worker that we has sent the min connections
*/
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_INFO,"client connection is accepted,pick a worker to handle it.");
uint32_t pick_worker_index= srv->worker_number;
uint32_t min_sent_connections=0xFFFFFFFF;
for(uint32_t worker_process_id=0; worker_process_id<srv->worker_number;worker_process_id++){
if(srv->child[worker_process_id].sent_connection_number < min_sent_connections
&& srv->child[worker_process_id].worker_running) {
min_sent_connections= srv->child[worker_process_id].sent_connection_number;
pick_worker_index= worker_process_id;
}
}
if(pick_worker_index>= srv->worker_number){
/* we can not handle it as all child worker has exited...*/
close(connection_fd);
continue;
}
/*set file descriptor to nonblocking and set close_on_exec flag*/
fd_set_nonblocking(connection_fd);
fd_close_on_exec(connection_fd);
if(unix_domain_socket_sendfd(srv->child[pick_worker_index].unix_domain_socket_fd[0],
connection_fd)<0){
log_to_backend(srv,MINIHTTPD_LOG_LEVEL_ERROR,"failed to send the connection file descriptor to worker!");
close(connection_fd); //just close it to tell the client,we can not handle it now.
continue;
}
srv->child[pick_worker_index].sent_connection_number++;
//close the file descriptor as it is already marked in flight
close(connection_fd);
}
}
