static void install_sighandlers(void) { sigset_t mask; int p[2]; /* Create signal pipe */ if (pipe(p) < 0) err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create" " signal pipe: pipe", my_index, my_pid); if ((sig_pipe[0] = st_netfd_open(p[0])) == NULL || (sig_pipe[1] = st_netfd_open(p[1])) == NULL) err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create" " signal pipe: st_netfd_open", my_index, my_pid); /* Install signal handlers */ Signal(SIGTERM, child_sighandler); /* terminate */ Signal(SIGHUP, child_sighandler); /* restart */ Signal(SIGUSR1, child_sighandler); /* dump info */ /* Unblock signals */ sigemptyset(&mask); sigaddset(&mask, SIGTERM); sigaddset(&mask, SIGHUP); sigaddset(&mask, SIGUSR1); sigprocmask(SIG_UNBLOCK, &mask, NULL); }
static void open_log_files(void) { int fd; char str[32]; if (interactive_mode) return; /* Open access log */ if (log_access) logbuf_open(); /* Open and write pid to pid file */ if ((fd = open(PID_FILE, O_CREAT | O_WRONLY | O_TRUNC, 0644)) < 0) err_sys_quit(errfd, "ERROR: can't open pid file: open"); sprintf(str, "%d\n", (int)getpid()); if (write(fd, str, strlen(str)) != strlen(str)) err_sys_quit(errfd, "ERROR: can't write to pid file: write"); close(fd); /* Open error log file */ if ((fd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0) err_sys_quit(errfd, "ERROR: can't open error log file: open"); errfd = fd; err_report(errfd, "INFO: starting the server..."); }
/* ARGSUSED */ static void *process_signals(void *arg) { int signo; for ( ; ; ) { /* Read the next signal from the signal pipe */ if (st_read(sig_pipe[0], &signo, sizeof(int), ST_UTIME_NO_TIMEOUT) != sizeof(int)) err_sys_quit(errfd, "ERROR: process %d (pid %d): signal processor:" " st_read", my_index, my_pid); switch (signo) { case SIGHUP: err_report(errfd, "INFO: process %d (pid %d): caught SIGHUP," " reloading configuration", my_index, my_pid); if (interactive_mode) { load_configs(); break; } /* Reopen log files - needed for log rotation */ if (log_access) { logbuf_flush(); logbuf_close(); logbuf_open(); } close(errfd); if ((errfd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0) err_sys_quit(STDERR_FILENO, "ERROR: process %d (pid %d): signal" " processor: open", my_index, my_pid); /* Reload configuration */ load_configs(); break; case SIGTERM: /* * Terminate ungracefully since it is generally not known how long * it will take to gracefully complete all client sessions. */ err_report(errfd, "INFO: process %d (pid %d): caught SIGTERM," " terminating", my_index, my_pid); if (log_access) logbuf_flush(); exit(0); case SIGUSR1: err_report(errfd, "INFO: process %d (pid %d): caught SIGUSR1", my_index, my_pid); /* Print server info to stderr */ dump_server_info(); break; default: err_report(errfd, "INFO: process %d (pid %d): caught signal %d", my_index, my_pid, signo); } } /* NOTREACHED */ return NULL; }
/* * General server example: accept a client connection and do something. * This program just outputs a short HTML page, but can be easily adapted * to do other things. * * This server creates a constant number of processes ("virtual processors" * or VPs) and replaces them when they die. Each virtual processor manages * its own independent set of state threads (STs), the number of which varies * with load against the server. Each state thread listens to exactly one * listening socket. The initial process becomes the watchdog, waiting for * children (VPs) to die or for a signal requesting termination or restart. * Upon receiving a restart signal (SIGHUP), all VPs close and then reopen * log files and reload configuration. All currently active connections remain * active. It is assumed that new configuration affects only request * processing and not the general server parameters such as number of VPs, * thread limits, bind addresses, etc. Those are specified as command line * arguments, so the server has to be stopped and then started again in order * to change them. * * Each state thread loops processing connections from a single listening * socket. Only one ST runs on a VP at a time, and VPs do not share memory, * so no mutual exclusion locking is necessary on any data, and the entire * server is free to use all the static variables and non-reentrant library * functions it wants, greatly simplifying programming and debugging and * increasing performance (for example, it is safe to ++ and -- all global * counters or call inet_ntoa(3) without any mutexes). The current thread on * each VP maintains equilibrium on that VP, starting a new thread or * terminating itself if the number of spare threads exceeds the lower or * upper limit. * * All I/O operations on sockets must use the State Thread library's I/O * functions because only those functions prevent blocking of the entire VP * process and perform state thread scheduling. */ int main(int argc, char *argv[]) { /* Parse command-line options */ parse_arguments(argc, argv); /* Allocate array of server pids */ if ((vp_pids = calloc(vp_count, sizeof(pid_t))) == NULL) err_sys_quit(errfd, "ERROR: calloc failed"); /* Start the daemon */ if (!interactive_mode) start_daemon(); /* Initialize the ST library */ if (st_init() < 0) err_sys_quit(errfd, "ERROR: initialization failed: st_init"); /* Set thread throttling parameters */ set_thread_throttling(); /* Create listening sockets */ create_listeners(); /* Change the user */ if (username) change_user(); /* Open log files */ open_log_files(); /* Start server processes (VPs) */ start_processes(); /* Turn time caching on */ st_timecache_set(1); /* Install signal handlers */ install_sighandlers(); /* Load configuration from config files */ load_configs(); /* Start all threads */ start_threads(); /* Become a signal processing thread */ process_signals(NULL); /* NOTREACHED */ return 1; }
static void change_user(void) { struct passwd *pw; if ((pw = getpwnam(username)) == NULL) err_quit(errfd, "ERROR: can't find user '%s': getpwnam failed", username); if (setgid(pw->pw_gid) < 0) err_sys_quit(errfd, "ERROR: can't change group id: setgid"); if (setuid(pw->pw_uid) < 0) err_sys_quit(errfd, "ERROR: can't change user id: setuid"); err_report(errfd, "INFO: changed process user id to '%s'", username); }
/** * 启动工作进程 */ void start_processes(const int procnum) { int i, status; pid_t pid; sigset_t mask, omask; my_pid = getpid(); vp_count = procnum + 1; for (i = 1; i <= procnum; i++) { if ((pid = fork()) < 0) { err_sys_quit(1, "fork"); break; } else if (pid == 0) { // child my_index = i; my_pid = getpid(); break; } else { // parent, or master my_index = 0; } } }
static void parse_arguments( int argc, char *argv[] ) { extern char *optarg; int opt; char* c = NULL; while (( opt = getopt( argc, argv, "b:p:l:h" ) ) != EOF ) { switch ( opt ) { case 'b': if (( c = strdup( optarg ) ) == NULL ) err_sys_quit( s_errfd, "ERROR: strdup" ); s_serverIP = c; break; case 'p': s_nListenPort = atoi( optarg ); if ( s_nListenPort < 1024 ) err_quit( s_errfd, "ERROR: invalid listening port: %s", optarg ); break; case 'l': s_logdir = optarg; break; case 'h': case '?': usage( argv[0] ); } } if ( s_logdir == NULL ) { err_report( s_errfd, "ERROR: logging directory is required\n" ); usage( argv[0] ); } }
static void start_threads(void) { long i, n; /* Create access log flushing thread */ if (log_access && st_thread_create(flush_acclog_buffer, NULL, 0, 0) == NULL) err_sys_quit(errfd, "ERROR: process %d (pid %d): can't create" " log flushing thread", my_index, my_pid); /* Create connections handling threads */ for (i = 0; i < sk_count; i++) { err_report(errfd, "INFO: process %d (pid %d): starting %d threads" " on %s:%u", my_index, my_pid, max_wait_threads, srv_socket[i].addr, srv_socket[i].port); WAIT_THREADS(i) = 0; BUSY_THREADS(i) = 0; RQST_COUNT(i) = 0; for (n = 0; n < max_wait_threads; n++) { if (st_thread_create(handle_connections, (void *)i, 0, 0) != NULL) WAIT_THREADS(i)++; else err_sys_report(errfd, "ERROR: process %d (pid %d): can't create" " thread", my_index, my_pid); } if (WAIT_THREADS(i) == 0) exit(1); } }
/** * Open error log file. */ int open_log_file( const std::string& sLogDir ) { int fd = -1; if (( fd = open( sLogDir.c_str(), O_CREAT | O_WRONLY | O_APPEND, 0644 ) ) < 0 ) { err_sys_quit( s_errfd, "ERROR: can't open error log file: [%s]", sLogDir.c_str() ); } return fd; }
static void child_sighandler(int signo) { int err, fd; err = errno; fd = st_netfd_fileno(sig_pipe[1]); /* write() is async-safe */ if (write(fd, &signo, sizeof(int)) != sizeof(int)) err_sys_quit(errfd, "ERROR: process %d (pid %d): child's signal" " handler: write", my_index, my_pid); errno = err; }
static void open_log_files( void ) { int fd = -1; char str[32]; std::string sFilename; /* Open and write pid to pid file */ sFilename = std::string(s_logdir) + PID_FILE; if (( fd = open( sFilename.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0644 ) ) < 0 ) err_sys_quit( s_errfd, "ERROR: can't open pid file: [%s]", sFilename.c_str() ); sprintf( str, "%d\n", ( int )getpid() ); if ( write( fd, str, strlen( str ) ) != (int) strlen( str ) ) err_sys_quit( s_errfd, "ERROR: can't write to pid file: write" ); close( fd ); /* Open error log file */ sFilename = std::string(s_logdir) + ERRORS_FILE; s_errfd = ::open_log_file( sFilename ); /* Open server log file */ s_serverfd = ::open_log_file( SERVER_FILE ); }
static void start_daemon(void) { pid_t pid; /* Start forking */ if ((pid = fork()) < 0) err_sys_quit(errfd, "ERROR: fork"); if (pid > 0) exit(0); /* parent */ /* First child process */ setsid(); /* become session leader */ if ((pid = fork()) < 0) err_sys_quit(errfd, "ERROR: fork"); if (pid > 0) /* first child */ exit(0); umask(022); if (chdir(logdir) < 0) err_sys_quit(errfd, "ERROR: can't change directory to %s: chdir", logdir); }
static void create_listeners(void) { int i, n, sock; char *c; struct sockaddr_in serv_addr; struct hostent *hp; unsigned short port; for (i = 0; i < sk_count; i++) { port = 0; if ((c = strchr(srv_socket[i].addr, ':')) != NULL) { *c++ = '\0'; port = (unsigned short) atoi(c); } if (srv_socket[i].addr[0] == '\0') srv_socket[i].addr = "0.0.0.0"; if (port == 0) port = SERV_PORT_DEFAULT; /* Create server socket */ if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) err_sys_quit(errfd, "ERROR: can't create socket: socket"); n = 1; if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (char *)&n, sizeof(n)) < 0) err_sys_quit(errfd, "ERROR: can't set SO_REUSEADDR: setsockopt"); memset(&serv_addr, 0, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; serv_addr.sin_port = htons(port); serv_addr.sin_addr.s_addr = inet_addr(srv_socket[i].addr); if (serv_addr.sin_addr.s_addr == INADDR_NONE) { /* not dotted-decimal */ if ((hp = gethostbyname(srv_socket[i].addr)) == NULL) err_quit(errfd, "ERROR: can't resolve address: %s", srv_socket[i].addr); memcpy(&serv_addr.sin_addr, hp->h_addr, hp->h_length); } srv_socket[i].port = port; /* Do bind and listen */ if (bind(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) err_sys_quit(errfd, "ERROR: can't bind to address %s, port %hu", srv_socket[i].addr, port); if (listen(sock, listenq_size) < 0) err_sys_quit(errfd, "ERROR: listen"); /* Create file descriptor object from OS socket */ if ((srv_socket[i].nfd = st_netfd_open_socket(sock)) == NULL) err_sys_quit(errfd, "ERROR: st_netfd_open_socket"); /* * On some platforms (e.g. IRIX, Linux) accept() serialization is never * needed for any OS version. In that case st_netfd_serialize_accept() * is just a no-op. Also see the comment above. */ if (serialize_accept && st_netfd_serialize_accept(srv_socket[i].nfd) < 0) err_sys_quit(errfd, "ERROR: st_netfd_serialize_accept"); } }
static void wdog_sighandler(int signo) { int i, err; /* Save errno */ err = errno; /* Forward the signal to all children */ for (i = 0; i < vp_count; i++) { if (vp_pids[i] > 0) kill(vp_pids[i], signo); } /* * It is safe to do pretty much everything here because process is * sleeping in wait() which is async-safe. */ switch (signo) { case SIGHUP: err_report(errfd, "INFO: watchdog: caught SIGHUP"); /* Reopen log files - needed for log rotation */ if (log_access) { logbuf_close(); logbuf_open(); } close(errfd); if ((errfd = open(ERRORS_FILE, O_CREAT | O_WRONLY | O_APPEND, 0644)) < 0) err_sys_quit(STDERR_FILENO, "ERROR: watchdog: open"); break; case SIGTERM: /* Non-graceful termination */ err_report(errfd, "INFO: watchdog: caught SIGTERM, terminating"); unlink(PID_FILE); exit(0); case SIGUSR1: err_report(errfd, "INFO: watchdog: caught SIGUSR1"); break; default: err_report(errfd, "INFO: watchdog: caught signal %d", signo); } /* Restore errno */ errno = err; }
/** * 创建用于 rpc 通讯的 TCP/IP 内容 */ void create_rpc_listeners(void) { int fd; struct addrinfo hints, *ai, *p; int reuseaddr = 1; // for SO_REUSEADDR int rv; memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; int rpc_port = RPC_PORT + my_index; char port[16]; sprintf(port, "%d", rpc_port); fprintf(stdout, "[%d] want to getaddrinfo to: 0.0.0.0:%d\n", my_index, rpc_port); if ((rv = getaddrinfo("0.0.0.0", port, &hints, &ai)) != 0) { err_sys_quit(1, "[%d] getaddrinfo: %s\n", my_index, gai_strerror(rv)); } for (p = ai; p != NULL; p = p->ai_next) { fd = socket(p->ai_family, p->ai_socktype, p->ai_protocol); if (fd < 0) { fprintf(stderr, "[%d] socket: %s\n", my_index, strerror(errno)); continue; } setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr, sizeof(int)); if (bind(fd, p->ai_addr, p->ai_addrlen) < 0) { close(fd); fprintf(stderr, "[%d] bind: %s\n", my_index, strerror(errno)); continue; } fprintf(stdout, "[%d] success to bind.\n", my_index); break; } if (p == NULL) { err_sys_quit(1, "[%d] failed to bind at 0.0.0.0\n", my_index); } freeaddrinfo(ai); // all done with this // 防止野指针 ai = NULL; p = NULL; if (listen(fd, 10) == -1) { err_sys_quit(1, "[%d] failed to listen in %d\n", my_index, rpc_port); } // 初始化 st 线程 if (st_set_eventsys(ST_EVENTSYS_ALT) == -1) fprintf(stderr, "[%d] Can't set event system to alt: %s\n", my_index, strerror(errno)); if (st_init() < 0) { err_sys_quit(1, "[%d] failed to init st\n", my_index); } fprintf(stdout, "[%d] the event system is: %s\n", my_index, st_get_eventsys_name()); // 转换 fd if ((rpc_fd = st_netfd_open(fd)) == NULL) err_sys_quit(1, "[%d] failed to convert fd into st_fd: %s\n", my_index, strerror(errno)); fprintf(stdout, "[%d] vp_count is %d.\n", my_index, vp_count); fd_list = (st_netfd_t*)calloc(vp_count, sizeof(st_netfd_t)); fd_list[my_index] = rpc_fd; }
static void parse_arguments(int argc, char *argv[]) { extern char *optarg; int opt; char *c; while ((opt = getopt(argc, argv, "b:p:l:t:u:q:aiSh")) != EOF) { switch (opt) { case 'b': if (sk_count >= MAX_BIND_ADDRS) err_quit(errfd, "ERROR: max number of bind addresses (%d) exceeded", MAX_BIND_ADDRS); if ((c = strdup(optarg)) == NULL) err_sys_quit(errfd, "ERROR: strdup"); srv_socket[sk_count++].addr = c; break; case 'p': vp_count = atoi(optarg); if (vp_count < 1) err_quit(errfd, "ERROR: invalid number of processes: %s", optarg); break; case 'l': logdir = optarg; break; case 't': max_wait_threads = (int) strtol(optarg, &c, 10); if (*c++ == ':') max_threads = atoi(c); if (max_wait_threads < 0 || max_threads < 0) err_quit(errfd, "ERROR: invalid number of threads: %s", optarg); break; case 'u': username = optarg; break; case 'q': listenq_size = atoi(optarg); if (listenq_size < 1) err_quit(errfd, "ERROR: invalid listen queue size: %s", optarg); break; case 'a': log_access = 1; break; case 'i': interactive_mode = 1; break; case 'S': /* * Serialization decision is tricky on some platforms. For example, * Solaris 2.6 and above has kernel sockets implementation, so supposedly * there is no need for serialization. The ST library may be compiled * on one OS version, but used on another, so the need for serialization * should be determined at run time by the application. Since it's just * an example, the serialization decision is left up to user. * Only on platforms where the serialization is never needed on any OS * version st_netfd_serialize_accept() is a no-op. */ serialize_accept = 1; break; case 'h': case '?': usage(argv[0]); } } if (logdir == NULL && !interactive_mode) { err_report(errfd, "ERROR: logging directory is required\n"); usage(argv[0]); } if (getuid() == 0 && username == NULL) err_report(errfd, "WARNING: running as super-user!"); if (vp_count == 0 && (vp_count = cpu_count()) < 1) vp_count = 1; if (sk_count == 0) { sk_count = 1; srv_socket[0].addr = "0.0.0.0"; } }
static void start_processes(void) { int i, status; pid_t pid; sigset_t mask, omask; if (interactive_mode) { my_index = 0; my_pid = getpid(); return; } for (i = 0; i < vp_count; i++) { if ((pid = fork()) < 0) { err_sys_report(errfd, "ERROR: can't create process: fork"); if (i == 0) exit(1); err_report(errfd, "WARN: started only %d processes out of %d", i, vp_count); vp_count = i; break; } if (pid == 0) { my_index = i; my_pid = getpid(); /* Child returns to continue in main() */ return; } vp_pids[i] = pid; } /* * Parent process becomes a "watchdog" and never returns to main(). */ /* Install signal handlers */ Signal(SIGTERM, wdog_sighandler); /* terminate */ Signal(SIGHUP, wdog_sighandler); /* restart */ Signal(SIGUSR1, wdog_sighandler); /* dump info */ /* Now go to sleep waiting for a child termination or a signal */ for ( ; ; ) { if ((pid = wait(&status)) < 0) { if (errno == EINTR) continue; err_sys_quit(errfd, "ERROR: watchdog: wait"); } /* Find index of the exited child */ for (i = 0; i < vp_count; i++) { if (vp_pids[i] == pid) break; } /* Block signals while printing and forking */ sigemptyset(&mask); sigaddset(&mask, SIGTERM); sigaddset(&mask, SIGHUP); sigaddset(&mask, SIGUSR1); sigprocmask(SIG_BLOCK, &mask, &omask); if (WIFEXITED(status)) err_report(errfd, "WARN: watchdog: process %d (pid %d) exited" " with status %d", i, pid, WEXITSTATUS(status)); else if (WIFSIGNALED(status)) err_report(errfd, "WARN: watchdog: process %d (pid %d) terminated" " by signal %d", i, pid, WTERMSIG(status)); else if (WIFSTOPPED(status)) err_report(errfd, "WARN: watchdog: process %d (pid %d) stopped" " by signal %d", i, pid, WSTOPSIG(status)); else err_report(errfd, "WARN: watchdog: process %d (pid %d) terminated:" " unknown termination reason", i, pid); /* Fork another VP */ if ((pid = fork()) < 0) { err_sys_report(errfd, "ERROR: watchdog: can't create process: fork"); } else if (pid == 0) { my_index = i; my_pid = getpid(); /* Child returns to continue in main() */ return; } vp_pids[i] = pid; /* Restore the signal mask */ sigprocmask(SIG_SETMASK, &omask, NULL); } }