static void install_sighandlers(void)
{
sigset_t mask;
int p[2];
/* Create signal pipe */
if (pipe(p) < 0)
err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create"
" signal pipe: pipe", my_index, my_pid);
if ((sig_pipe[0] = st_netfd_open(p[0])) == NULL ||
(sig_pipe[1] = st_netfd_open(p[1])) == NULL)
err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create"
" signal pipe: st_netfd_open", my_index, my_pid);
/* Install signal handlers */
Signal(SIGTERM, child_sighandler); /* terminate */
Signal(SIGHUP, child_sighandler); /* restart */
Signal(SIGUSR1, child_sighandler); /* dump info */
/* Unblock signals */
sigemptyset(&mask);
sigaddset(&mask, SIGTERM);
sigaddset(&mask, SIGHUP);
sigaddset(&mask, SIGUSR1);
sigprocmask(SIG_UNBLOCK, &mask, NULL);
}
static void open_log_files(void)
{
int fd;
char str[32];
if (interactive_mode)
return;
/* Open access log */
if (log_access)
logbuf_open();
/* Open and write pid to pid file */
if ((fd = open(PID_FILE, O_CREAT | O_WRONLY | O_TRUNC, 0644)) < 0)
err_sys_quit(errfd, "ERROR: can't open pid file: open");
sprintf(str, "%d\n", (int)getpid());
if (write(fd, str, strlen(str)) != strlen(str))
err_sys_quit(errfd, "ERROR: can't write to pid file: write");
close(fd);
/* Open error log file */
if ((fd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0)
err_sys_quit(errfd, "ERROR: can't open error log file: open");
errfd = fd;
err_report(errfd, "INFO: starting the server...");
}
/* ARGSUSED */
static void *process_signals(void *arg)
{
int signo;
for ( ; ; ) {
/* Read the next signal from the signal pipe */
if (st_read(sig_pipe[0], &signo, sizeof(int),
ST_UTIME_NO_TIMEOUT) != sizeof(int))
err_sys_quit(errfd, "ERROR: process %d (pid %d): signal processor:"
" st_read", my_index, my_pid);
switch (signo) {
case SIGHUP:
err_report(errfd, "INFO: process %d (pid %d): caught SIGHUP,"
" reloading configuration", my_index, my_pid);
if (interactive_mode) {
load_configs();
break;
}
/* Reopen log files - needed for log rotation */
if (log_access) {
logbuf_flush();
logbuf_close();
logbuf_open();
}
close(errfd);
if ((errfd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0)
err_sys_quit(STDERR_FILENO, "ERROR: process %d (pid %d): signal"
" processor: open", my_index, my_pid);
/* Reload configuration */
load_configs();
break;
case SIGTERM:
/*
* Terminate ungracefully since it is generally not known how long
* it will take to gracefully complete all client sessions.
*/
err_report(errfd, "INFO: process %d (pid %d): caught SIGTERM,"
" terminating", my_index, my_pid);
if (log_access)
logbuf_flush();
exit(0);
case SIGUSR1:
err_report(errfd, "INFO: process %d (pid %d): caught SIGUSR1",
my_index, my_pid);
/* Print server info to stderr */
dump_server_info();
break;
default:
err_report(errfd, "INFO: process %d (pid %d): caught signal %d",
my_index, my_pid, signo);
}
}
/* NOTREACHED */
return NULL;
}
/*
* General server example: accept a client connection and do something.
* This program just outputs a short HTML page, but can be easily adapted
* to do other things.
*
* This server creates a constant number of processes ("virtual processors"
* or VPs) and replaces them when they die. Each virtual processor manages
* its own independent set of state threads (STs), the number of which varies
* with load against the server. Each state thread listens to exactly one
* listening socket. The initial process becomes the watchdog, waiting for
* children (VPs) to die or for a signal requesting termination or restart.
* Upon receiving a restart signal (SIGHUP), all VPs close and then reopen
* log files and reload configuration. All currently active connections remain
* active. It is assumed that new configuration affects only request
* processing and not the general server parameters such as number of VPs,
* thread limits, bind addresses, etc. Those are specified as command line
* arguments, so the server has to be stopped and then started again in order
* to change them.
*
* Each state thread loops processing connections from a single listening
* socket. Only one ST runs on a VP at a time, and VPs do not share memory,
* so no mutual exclusion locking is necessary on any data, and the entire
* server is free to use all the static variables and non-reentrant library
* functions it wants, greatly simplifying programming and debugging and
* increasing performance (for example, it is safe to ++ and -- all global
* counters or call inet_ntoa(3) without any mutexes). The current thread on
* each VP maintains equilibrium on that VP, starting a new thread or
* terminating itself if the number of spare threads exceeds the lower or
* upper limit.
*
* All I/O operations on sockets must use the State Thread library's I/O
* functions because only those functions prevent blocking of the entire VP
* process and perform state thread scheduling.
*/
int main(int argc, char *argv[])
{
/* Parse command-line options */
parse_arguments(argc, argv);
/* Allocate array of server pids */
if ((vp_pids = calloc(vp_count, sizeof(pid_t))) == NULL)
err_sys_quit(errfd, "ERROR: calloc failed");
/* Start the daemon */
if (!interactive_mode)
start_daemon();
/* Initialize the ST library */
if (st_init() < 0)
err_sys_quit(errfd, "ERROR: initialization failed: st_init");
/* Set thread throttling parameters */
set_thread_throttling();
/* Create listening sockets */
create_listeners();
/* Change the user */
if (username)
change_user();
/* Open log files */
open_log_files();
/* Start server processes (VPs) */
start_processes();
/* Turn time caching on */
st_timecache_set(1);
/* Install signal handlers */
install_sighandlers();
/* Load configuration from config files */
load_configs();
/* Start all threads */
start_threads();
/* Become a signal processing thread */
process_signals(NULL);
/* NOTREACHED */
return 1;
}
static void change_user(void)
{
struct passwd *pw;
if ((pw = getpwnam(username)) == NULL)
err_quit(errfd, "ERROR: can't find user '%s': getpwnam failed", username);
if (setgid(pw->pw_gid) < 0)
err_sys_quit(errfd, "ERROR: can't change group id: setgid");
if (setuid(pw->pw_uid) < 0)
err_sys_quit(errfd, "ERROR: can't change user id: setuid");
err_report(errfd, "INFO: changed process user id to '%s'", username);
}
/**
* 启动工作进程
*/
void start_processes(const int procnum)
{
int i, status;
pid_t pid;
sigset_t mask, omask;
my_pid = getpid();
vp_count = procnum + 1;
for (i = 1; i <= procnum; i++) {
if ((pid = fork()) < 0) {
err_sys_quit(1, "fork");
break;
} else if (pid == 0) {
// child
my_index = i;
my_pid = getpid();
break;
} else {
// parent, or master
my_index = 0;
}
}
}
static void parse_arguments( int argc, char *argv[] )
{
extern char *optarg;
int opt;
char* c = NULL;
while (( opt = getopt( argc, argv, "b:p:l:h" ) ) != EOF )
{
switch ( opt )
{
case 'b':
if (( c = strdup( optarg ) ) == NULL )
err_sys_quit( s_errfd, "ERROR: strdup" );
s_serverIP = c;
break;
case 'p':
s_nListenPort = atoi( optarg );
if ( s_nListenPort < 1024 )
err_quit( s_errfd, "ERROR: invalid listening port: %s", optarg );
break;
case 'l':
s_logdir = optarg;
break;
case 'h':
case '?':
usage( argv[0] );
}
}
if ( s_logdir == NULL )
{
err_report( s_errfd, "ERROR: logging directory is required\n" );
usage( argv[0] );
}
}
static void start_threads(void)
{
long i, n;
/* Create access log flushing thread */
if (log_access && st_thread_create(flush_acclog_buffer, NULL, 0, 0) == NULL)
err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create"
" log flushing thread", my_index, my_pid);
/* Create connections handling threads */
for (i = 0; i < sk_count; i++) {
err_report(errfd, "INFO: process %d (pid %d): starting %d threads"
" on %s:%u", my_index, my_pid, max_wait_threads,
srv_socket[i].addr, srv_socket[i].port);
WAIT_THREADS(i) = 0;
BUSY_THREADS(i) = 0;
RQST_COUNT(i) = 0;
for (n = 0; n < max_wait_threads; n++) {
if (st_thread_create(handle_connections, (void *)i, 0, 0) != NULL)
WAIT_THREADS(i)++;
else
err_sys_report(errfd, "ERROR: process %d (pid %d): can't create"
" thread", my_index, my_pid);
}
if (WAIT_THREADS(i) == 0)
exit(1);
}
}
/**
* Open error log file.
*/
int
open_log_file( const std::string& sLogDir )
{
int fd = -1;
if (( fd = open( sLogDir.c_str(), O_CREAT | O_WRONLY | O_APPEND, 0644 ) ) < 0 )
{
err_sys_quit( s_errfd, "ERROR: can't open error log file: [%s]", sLogDir.c_str() );
}
return fd;
}
static void child_sighandler(int signo)
{
int err, fd;
err = errno;
fd = st_netfd_fileno(sig_pipe[1]);
/* write() is async-safe */
if (write(fd, &signo, sizeof(int)) != sizeof(int))
err_sys_quit(errfd, "ERROR: process %d (pid %d): child's signal"
" handler: write", my_index, my_pid);
errno = err;
}
static void open_log_files( void )
{
int fd = -1;
char str[32];
std::string sFilename;
/* Open and write pid to pid file */
sFilename = std::string(s_logdir) + PID_FILE;
if (( fd = open( sFilename.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644 ) ) < 0 )
err_sys_quit( s_errfd, "ERROR: can't open pid file: [%s]", sFilename.c_str() );
sprintf( str, "%d\n", ( int )getpid() );
if ( write( fd, str, strlen( str ) ) != (int) strlen( str ) )
err_sys_quit( s_errfd, "ERROR: can't write to pid file: write" );
close( fd );
/* Open error log file */
sFilename = std::string(s_logdir) + ERRORS_FILE;
s_errfd = ::open_log_file( sFilename );
/* Open server log file */
s_serverfd = ::open_log_file( SERVER_FILE );
}
static void start_daemon(void)
{
pid_t pid;
/* Start forking */
if ((pid = fork()) < 0)
err_sys_quit(errfd, "ERROR: fork");
if (pid > 0)
exit(0); /* parent */
/* First child process */
setsid(); /* become session leader */
if ((pid = fork()) < 0)
err_sys_quit(errfd, "ERROR: fork");
if (pid > 0) /* first child */
exit(0);
umask(022);
if (chdir(logdir) < 0)
err_sys_quit(errfd, "ERROR: can't change directory to %s: chdir", logdir);
}
static void create_listeners(void)
{
int i, n, sock;
char *c;
struct sockaddr_in serv_addr;
struct hostent *hp;
unsigned short port;
for (i = 0; i < sk_count; i++) {
port = 0;
if ((c = strchr(srv_socket[i].addr, ':')) != NULL) {
*c++ = '\0';
port = (unsigned short) atoi(c);
}
if (srv_socket[i].addr[0] == '\0')
srv_socket[i].addr = "0.0.0.0";
if (port == 0)
port = SERV_PORT_DEFAULT;
/* Create server socket */
if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0)
err_sys_quit(errfd, "ERROR: can't create socket: socket");
n = 1;
if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&n, sizeof(n)) < 0)
err_sys_quit(errfd, "ERROR: can't set SO_REUSEADDR: setsockopt");
memset(&serv_addr, 0, sizeof(serv_addr));
serv_addr.sin_family = AF_INET;
serv_addr.sin_port = htons(port);
serv_addr.sin_addr.s_addr = inet_addr(srv_socket[i].addr);
if (serv_addr.sin_addr.s_addr == INADDR_NONE) {
/* not dotted-decimal */
if ((hp = gethostbyname(srv_socket[i].addr)) == NULL)
err_quit(errfd, "ERROR: can't resolve address: %s",
srv_socket[i].addr);
memcpy(&serv_addr.sin_addr, hp->h_addr, hp->h_length);
}
srv_socket[i].port = port;
/* Do bind and listen */
if (bind(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0)
err_sys_quit(errfd, "ERROR: can't bind to address %s, port %hu",
srv_socket[i].addr, port);
if (listen(sock, listenq_size) < 0)
err_sys_quit(errfd, "ERROR: listen");
/* Create file descriptor object from OS socket */
if ((srv_socket[i].nfd = st_netfd_open_socket(sock)) == NULL)
err_sys_quit(errfd, "ERROR: st_netfd_open_socket");
/*
* On some platforms (e.g. IRIX, Linux) accept() serialization is never
* needed for any OS version. In that case st_netfd_serialize_accept()
* is just a no-op. Also see the comment above.
*/
if (serialize_accept && st_netfd_serialize_accept(srv_socket[i].nfd) < 0)
err_sys_quit(errfd, "ERROR: st_netfd_serialize_accept");
}
}
static void wdog_sighandler(int signo)
{
int i, err;
/* Save errno */
err = errno;
/* Forward the signal to all children */
for (i = 0; i < vp_count; i++) {
if (vp_pids[i] > 0)
kill(vp_pids[i], signo);
}
/*
* It is safe to do pretty much everything here because process is
* sleeping in wait() which is async-safe.
*/
switch (signo) {
case SIGHUP:
err_report(errfd, "INFO: watchdog: caught SIGHUP");
/* Reopen log files - needed for log rotation */
if (log_access) {
logbuf_close();
logbuf_open();
}
close(errfd);
if ((errfd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0)
err_sys_quit(STDERR_FILENO, "ERROR: watchdog: open");
break;
case SIGTERM:
/* Non-graceful termination */
err_report(errfd, "INFO: watchdog: caught SIGTERM, terminating");
unlink(PID_FILE);
exit(0);
case SIGUSR1:
err_report(errfd, "INFO: watchdog: caught SIGUSR1");
break;
default:
err_report(errfd, "INFO: watchdog: caught signal %d", signo);
}
/* Restore errno */
errno = err;
}
/**
* 创建用于 rpc 通讯的 TCP/IP 内容
*/
void create_rpc_listeners(void) {
int fd;
struct addrinfo hints, *ai, *p;
int reuseaddr = 1; // for SO_REUSEADDR
int rv;
memset(&hints, 0, sizeof(hints));
hints.ai_family = PF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_flags = AI_PASSIVE;
int rpc_port = RPC_PORT + my_index;
char port[16];
sprintf(port, "%d", rpc_port);
fprintf(stdout, "[%d] want to getaddrinfo to: 0.0.0.0:%d\n", my_index, rpc_port);
if ((rv = getaddrinfo("0.0.0.0", port, &hints, &ai)) != 0) {
err_sys_quit(1, "[%d] getaddrinfo: %s\n", my_index, gai_strerror(rv));
}
for (p = ai; p != NULL; p = p->ai_next) {
fd = socket(p->ai_family, p->ai_socktype, p->ai_protocol);
if (fd < 0) {
fprintf(stderr, "[%d] socket: %s\n", my_index, strerror(errno));
continue;
}
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr, sizeof(int));
if (bind(fd, p->ai_addr, p->ai_addrlen) < 0) {
close(fd);
fprintf(stderr, "[%d] bind: %s\n", my_index, strerror(errno));
continue;
}
fprintf(stdout, "[%d] success to bind.\n", my_index);
break;
}
if (p == NULL) {
err_sys_quit(1, "[%d] failed to bind at 0.0.0.0\n", my_index);
}
freeaddrinfo(ai); // all done with this
// 防止野指针
ai = NULL;
p = NULL;
if (listen(fd, 10) == -1) {
err_sys_quit(1, "[%d] failed to listen in %d\n", my_index, rpc_port);
}
// 初始化 st 线程
if (st_set_eventsys(ST_EVENTSYS_ALT) == -1)
fprintf(stderr, "[%d] Can't set event system to alt: %s\n", my_index, strerror(errno));
if (st_init() < 0) {
err_sys_quit(1, "[%d] failed to init st\n", my_index);
}
fprintf(stdout, "[%d] the event system is: %s\n", my_index, st_get_eventsys_name());
// 转换 fd
if ((rpc_fd = st_netfd_open(fd)) == NULL)
err_sys_quit(1, "[%d] failed to convert fd into st_fd: %s\n", my_index, strerror(errno));
fprintf(stdout, "[%d] vp_count is %d.\n", my_index, vp_count);
fd_list = (st_netfd_t*)calloc(vp_count, sizeof(st_netfd_t));
fd_list[my_index] = rpc_fd;
}
static void parse_arguments(int argc, char *argv[])
{
extern char *optarg;
int opt;
char *c;
while ((opt = getopt(argc, argv, "b:p:l:t:u:q:aiSh")) != EOF) {
switch (opt) {
case 'b':
if (sk_count >= MAX_BIND_ADDRS)
err_quit(errfd, "ERROR: max number of bind addresses (%d) exceeded",
MAX_BIND_ADDRS);
if ((c = strdup(optarg)) == NULL)
err_sys_quit(errfd, "ERROR: strdup");
srv_socket[sk_count++].addr = c;
break;
case 'p':
vp_count = atoi(optarg);
if (vp_count < 1)
err_quit(errfd, "ERROR: invalid number of processes: %s", optarg);
break;
case 'l':
logdir = optarg;
break;
case 't':
max_wait_threads = (int) strtol(optarg, &c, 10);
if (*c++ == ':')
max_threads = atoi(c);
if (max_wait_threads < 0 || max_threads < 0)
err_quit(errfd, "ERROR: invalid number of threads: %s", optarg);
break;
case 'u':
username = optarg;
break;
case 'q':
listenq_size = atoi(optarg);
if (listenq_size < 1)
err_quit(errfd, "ERROR: invalid listen queue size: %s", optarg);
break;
case 'a':
log_access = 1;
break;
case 'i':
interactive_mode = 1;
break;
case 'S':
/*
* Serialization decision is tricky on some platforms. For example,
* Solaris 2.6 and above has kernel sockets implementation, so supposedly
* there is no need for serialization. The ST library may be compiled
* on one OS version, but used on another, so the need for serialization
* should be determined at run time by the application. Since it's just
* an example, the serialization decision is left up to user.
* Only on platforms where the serialization is never needed on any OS
* version st_netfd_serialize_accept() is a no-op.
*/
serialize_accept = 1;
break;
case 'h':
case '?':
usage(argv[0]);
}
}
if (logdir == NULL && !interactive_mode) {
err_report(errfd, "ERROR: logging directory is required\n");
usage(argv[0]);
}
if (getuid() == 0 && username == NULL)
err_report(errfd, "WARNING: running as super-user!");
if (vp_count == 0 && (vp_count = cpu_count()) < 1)
vp_count = 1;
if (sk_count == 0) {
sk_count = 1;
srv_socket[0].addr = "0.0.0.0";
}
}
static void start_processes(void)
{
int i, status;
pid_t pid;
sigset_t mask, omask;
if (interactive_mode) {
my_index = 0;
my_pid = getpid();
return;
}
for (i = 0; i < vp_count; i++) {
if ((pid = fork()) < 0) {
err_sys_report(errfd, "ERROR: can't create process: fork");
if (i == 0)
exit(1);
err_report(errfd, "WARN: started only %d processes out of %d", i,
vp_count);
vp_count = i;
break;
}
if (pid == 0) {
my_index = i;
my_pid = getpid();
/* Child returns to continue in main() */
return;
}
vp_pids[i] = pid;
}
/*
* Parent process becomes a "watchdog" and never returns to main().
*/
/* Install signal handlers */
Signal(SIGTERM, wdog_sighandler); /* terminate */
Signal(SIGHUP, wdog_sighandler); /* restart */
Signal(SIGUSR1, wdog_sighandler); /* dump info */
/* Now go to sleep waiting for a child termination or a signal */
for ( ; ; ) {
if ((pid = wait(&status)) < 0) {
if (errno == EINTR)
continue;
err_sys_quit(errfd, "ERROR: watchdog: wait");
}
/* Find index of the exited child */
for (i = 0; i < vp_count; i++) {
if (vp_pids[i] == pid)
break;
}
/* Block signals while printing and forking */
sigemptyset(&mask);
sigaddset(&mask, SIGTERM);
sigaddset(&mask, SIGHUP);
sigaddset(&mask, SIGUSR1);
sigprocmask(SIG_BLOCK, &mask, &omask);
if (WIFEXITED(status))
err_report(errfd, "WARN: watchdog: process %d (pid %d) exited"
" with status %d", i, pid, WEXITSTATUS(status));
else if (WIFSIGNALED(status))
err_report(errfd, "WARN: watchdog: process %d (pid %d) terminated"
" by signal %d", i, pid, WTERMSIG(status));
else if (WIFSTOPPED(status))
err_report(errfd, "WARN: watchdog: process %d (pid %d) stopped"
" by signal %d", i, pid, WSTOPSIG(status));
else
err_report(errfd, "WARN: watchdog: process %d (pid %d) terminated:"
" unknown termination reason", i, pid);
/* Fork another VP */
if ((pid = fork()) < 0) {
err_sys_report(errfd, "ERROR: watchdog: can't create process: fork");
} else if (pid == 0) {
my_index = i;
my_pid = getpid();
/* Child returns to continue in main() */
return;
}
vp_pids[i] = pid;
/* Restore the signal mask */
sigprocmask(SIG_SETMASK, &omask, NULL);
}
}
