ngx_int_t ngx_http_push_shutdown_ipc(ngx_cycle_t *cycle) { int i; for (i=0; i<NGX_MAX_PROCESSES; i++) { if(ngx_http_push_socketpairs[i][0] != NGX_INVALID_FILE || ngx_http_push_socketpairs[i][1] != NGX_INVALID_FILE) { ngx_close_channel(ngx_http_push_socketpairs[i], cycle->log); } } return NGX_OK; }
ngx_int_t ngx_http_push_init_ipc(ngx_cycle_t *cycle, ngx_int_t workers) { //initialize socketpairs for workers in advance. static int invalid_sockets_initialized = 0; int i, s = 0, on = 1; ngx_int_t last_expected_process = ngx_last_process; if(!invalid_sockets_initialized) { for(i=0; i< NGX_MAX_PROCESSES; i++) { ngx_http_push_socketpairs[i][0]=NGX_INVALID_FILE; ngx_http_push_socketpairs[i][1]=NGX_INVALID_FILE; } invalid_sockets_initialized=1; } /* here's the deal: we have no control over fork()ing, nginx's internal * socketpairs are unusable for our purposes (as of nginx 0.8 -- check the * code to see why), and the module initialization callbacks occur before * any workers are spawned. Rather than futzing around with existing * socketpairs, we populate our own socketpairs array. * Trouble is, ngx_spawn_process() creates them one-by-one, and we need to * do it all at once. So we must guess all the workers' ngx_process_slots in * advance. Meaning the spawning logic must be copied to the T. * ... with some allowances for already-opened sockets... */ for(i=0; i < workers; i++) { //copypasta from os/unix/ngx_process.c (ngx_spawn_process) while (s < last_expected_process && ngx_processes[s].pid != -1) { //find empty existing slot s++; } ngx_socket_t *socks = ngx_http_push_socketpairs[s]; if(socks[0] == NGX_INVALID_FILE || socks[1] == NGX_INVALID_FILE) { //copypasta from os/unix/ngx_process.c (ngx_spawn_process) if (socketpair(AF_UNIX, SOCK_STREAM, 0, socks) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed on socketpair while initializing push module"); return NGX_ERROR; } if (ngx_nonblocking(socks[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (ngx_nonblocking(socks[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (ioctl(socks[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } } s++; //NEXT!! } return NGX_OK; }
static ngx_uint_t ngx_reap_children(ngx_cycle_t *cycle) { ngx_int_t i, n; ngx_uint_t live; ngx_channel_t ch; ngx_core_conf_t *ccf; ngx_memzero(&ch, sizeof(ngx_channel_t)); ch.command = NGX_CMD_CLOSE_CHANNEL; ch.fd = -1; live = 0; for (i = 0; i < ngx_last_process; i++) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "child: %d %P e:%d t:%d d:%d r:%d j:%d", i, ngx_processes[i].pid, ngx_processes[i].exiting, ngx_processes[i].exited, ngx_processes[i].detached, ngx_processes[i].respawn, ngx_processes[i].just_spawn); if (ngx_processes[i].pid == -1) { continue; } if (ngx_processes[i].exited) { if (!ngx_processes[i].detached) { ngx_close_channel(ngx_processes[i].channel, cycle->log); ngx_processes[i].channel[0] = -1; ngx_processes[i].channel[1] = -1; ch.pid = ngx_processes[i].pid; ch.slot = i; for (n = 0; n < ngx_last_process; n++) { if (ngx_processes[n].exited || ngx_processes[n].pid == -1 || ngx_processes[n].channel[0] == -1) { continue; } ngx_log_debug3(NGX_LOG_DEBUG_CORE, cycle->log, 0, "pass close channel s:%i pid:%P to:%P", ch.slot, ch.pid, ngx_processes[n].pid); /* TODO: NGX_AGAIN */ ngx_write_channel(ngx_processes[n].channel[0], &ch, sizeof(ngx_channel_t), cycle->log); } } if (ngx_processes[i].respawn && !ngx_processes[i].exiting && !ngx_terminate && !ngx_quit) { if (ngx_spawn_process(cycle, ngx_processes[i].proc, ngx_processes[i].data, ngx_processes[i].name, i) == NGX_INVALID_PID) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "could not respawn %s", ngx_processes[i].name); continue; } ch.command = NGX_CMD_OPEN_CHANNEL; ch.pid = ngx_processes[ngx_process_slot].pid; ch.slot = ngx_process_slot; ch.fd = ngx_processes[ngx_process_slot].channel[0]; ngx_pass_open_channel(cycle, &ch); live = 1; continue; } if (ngx_processes[i].pid == ngx_new_binary) { ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); if (ngx_rename_file((char *) ccf->oldpid.data, (char *) ccf->pid.data) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_rename_file_n " %s back to %s failed " "after the new binary process \"%s\" exited", ccf->oldpid.data, ccf->pid.data, ngx_argv[0]); } ngx_new_binary = 0; if (ngx_noaccepting) { ngx_restart = 1; ngx_noaccepting = 0; } } if (i == ngx_last_process - 1) { ngx_last_process--; } else { ngx_processes[i].pid = -1; } } else if (ngx_processes[i].exiting || !ngx_processes[i].detached) { live = 1; } } return live; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) {//ngx_start_worker_processes-> NGX_PROCESS_RESPAWN ,表示要重新创建进程 s = respawn; } else {//得到当前有多少工作进程创建了ngx_last_process会不断增加的,当然可能有人死了 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) {//NGX_PROCESS_DETACHED说明是热代码替换 //不是热代码替换 /* Solaris 9 still has no AF_LOCAL */ //创建socketpair,进程之间通信 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1){ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //将socket对设置为非阻塞模式 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1;//FIOASYNC 改变O_ASYNC 标志来打开或者关闭套接字的异步 IO 模式。 if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //close on exec, not on-fork, 意为如果对描述符设置了FD_CLOEXEC,使用execl执行的程序里, //此描述符被关闭,不能再使用它,但是在使用fork调用的子进程中,此描述符并不关闭,仍可使用。 if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置fork对socket对是在子进程依然可用的 if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else {//如果是热代码替换,不跟其他进程通信 ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s;//当前在处理这个 pid = fork();//fork子进程 switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0://子进程调用proc ngx_pid = ngx_getpid();//k getpid proc(cycle, data);//等于ngx_worker_process_cycle //子进程永远也从这回不来了,里面有exit break; default://父进程默认不干活 break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } //设置ngx_processes数组该项的信息 ngx_processes[s].proc = proc;//设置工作进程的处理函数ngx_worker_process_cycle,会fork之后调用的 ngx_processes[s].data = data; ngx_processes[s].name = name;//进程名字 ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++;//递增总进程数 } return pid; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s; pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: ngx_pid = ngx_getpid(); proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }
//创建子进程 ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { //初始化ngx_processes数组 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } //进程个数不能大于NGX_MAX_PROCESSES if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ //创建用于进程间通信的UNIX域套结字 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //将套结字的读写端都设置为非阻塞 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; //以下设置套结字读端为信号驱动,注意我们还没有fork子进程,设置是在master进程中 //设置FIOASYNC标志(我们必须要这样做),fcntl函数也可以设置,但大多数系统不支持, //因此用ioctl来实现 if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置F_SETOWN标志,将套结字读端和master进程关联起来,当管道有数据的时侯会向该进程 //发送SIGIO信号 if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置管道的FD_CLOEXEC标志 if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s; pid = fork(); switch (pid) { case -1: //error ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //child ngx_pid = ngx_getpid(); //执行我们传递的子进程执行函数 proc(cycle, data); break; default: //father break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; //设置子进程的执行函数 ngx_processes[s].data = data; //设置子进程的参数 ngx_processes[s].name = name; //设置子进程的name ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; //last_proces可以视为子进程个数 } return pid; }
/* * 创建进程 */ ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { /*在全局进程表 ngx_processes 中查找第一个空的结构,用来存放新创建的进程的信息*/ for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ /*调用 socketpair 创建一对互相连接的 unix域socket,用于进程间的全双工通信*/ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); /*将用于进程通信的 socket 设置成非阻塞的*/ if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; /*设置成信号驱动异步IO,有IO时内核发送SIGIO信号*/ if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*设置将接受 SIGIO 和 SIGURG 信号的进程id*/ if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*当执行 exec 时关闭描述符*/ if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*当前子进程的描述符*/ ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } /*新创建进的进程在 ngx_processes 数组中的下标*/ ngx_process_slot = s; pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //子进程 ngx_pid = ngx_getpid(); proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); /*新创建的进程放入 master 主进程的 ngx_processes 数组中*/ ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }
/*proc:是子进程的执行函数ngx_worker_process_cycle,data是参数,name是子进程的名字*/ ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; //将要创建的子进程在进程表中的位置 ngx_int_t s; //替换进程ngx_processes[respawn],可安全重用该进程表项 if (respawn >= 0) { s = respawn; } else { //先找到一个被回收的进程表项 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } //进程表已经满 if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,"no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } //不是分离的子进程 if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ //ngx_processes[s].channel数组是用于父子间通信的UNIX域套接字对 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,"channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //设置socket为非阻塞模式 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //开启channel[0]的消息驱动IO on = 1; /*FIOASYNC:设置/清除信号驱动异步I/O标志.根据iocl 的第三个参数指向一个0 值或非0 值分别清除或设置针对本套接口的信号驱动异步I/O * 标志,它决定是否收取针对本套接口的异步I/O 信号(SIGIO )。本请求和O_ASYNC 文件状态标志等效,而该标志可以通过fcntl 的F_SETFL命令清除或设置。*/ if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置channel[0]的宿主,控制channel[0]的SIGIO信号只发给这个进程 if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //若进程执行了exec后,关闭socket if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //同上 if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"",name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //用于监听可读事件的socket ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } //设置当前子进程的进程表索引值 ngx_process_slot = s; //创建子进程 pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //设置当前子进程的父进程id ngx_pid = ngx_getpid(); //子进程运行执行函数 proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); //设置一些进程表项字段 ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; //如果是重复创建,即为替换进程,不用设置其他进程表字段,直接返回。 if (respawn >= 0) { return pid; } //设置其他进程表字段 ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; //设置进程表项的一些状态字 switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; /* 如果respawn参数大于0,则相当于创建某个指定的进程, * 也就是直接使用respawn作为数组中的索引 */ if (respawn >= 0) { s = respawn; } else { /* 获取一个合适的进程数组索引,新建的子进程信息就放在全局进程结构的s索引处, * */ for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ /* 创建AF_UNIX套接字用于两个进程通信, * 注意创建的文件描述符存放在对应ngx_process_t结构当中, * 注意当前的情景,是master进程创建worker进程,而master进程并不在ngx_processes * 数组当中, */ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); /* 设置相应的非阻塞标记 */ if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /* 父进程创建一对通道,父进程使用的是0通道,子进程使用的是1通道, * 在worker进程的初始化过程当中会将该文件描述符添加到读监听集里面, * 主要目的是两个子进程之间通信 */ ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } /* 设置创建子进程在全局进程结构中的索引, * 执行fork之后,该全局变量就被拷贝到子进程当中了, * 而且子进程可以通过该变量知道自己在ngx_processes * 数组中的位置 */ ngx_process_slot = s; /* 错误返回-1 * 子进程返回0 * 父进程返回子进程的pid */ pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: ngx_pid = ngx_getpid(); /* 开始执行子进程的函数,注意子进程是一个服务进程,所以在proc函数在执行之后 * 就会一直循环 */ proc(cycle, data); break; default: break; } /* 此处之下就都是父进程执行的代码 */ ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); /* 设置子进程的相关信息 */ ngx_processes[s].pid = pid; /* 表示进程没有退出 */ ngx_processes[s].exited = 0; /* 如果启动的是指定进程,则直接返回 */ if (respawn >= 0) { return pid; } /* 设置刚刚创建子进程的信息 */ ngx_processes[s].proc = proc; ngx_processes[s].data = data; /* 数据其实就是1,2,3 */ ngx_processes[s].name = name; ngx_processes[s].exiting = 0; /* 根据respawn参数来确定 respawn,just_spawn,detached的值 */ switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: /* 注意这个标记对应了两个值 */ ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } /* 创建完一个子进程时,会相应的增加ngx_last_process变量 */ if (s == ngx_last_process) { ngx_last_process++; } return pid; }
static void ngx_http_push_stream_ipc_exit_worker(ngx_cycle_t *cycle) { ngx_close_channel((ngx_socket_t *) ngx_http_push_stream_socketpairs[ngx_process_slot], cycle->log); }
static ngx_uint_t ngx_reap_childs(ngx_cycle_t *cycle) { ngx_int_t i, n; ngx_uint_t live; ngx_channel_t ch; ch.command = NGX_CMD_CLOSE_CHANNEL; ch.fd = -1; live = 0; for (i = 0; i < ngx_last_process; i++) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "child: %d " PID_T_FMT " e:%d t:%d d:%d r:%d j:%d", i, ngx_processes[i].pid, ngx_processes[i].exiting, ngx_processes[i].exited, ngx_processes[i].detached, ngx_processes[i].respawn, ngx_processes[i].just_respawn); if (ngx_processes[i].pid == -1) { continue; } if (ngx_processes[i].exited) { if (!ngx_processes[i].detached) { ngx_close_channel(ngx_processes[i].channel, cycle->log); ngx_processes[i].channel[0] = -1; ngx_processes[i].channel[1] = -1; ch.pid = ngx_processes[i].pid; ch.slot = i; for (n = 0; n < ngx_last_process; n++) { if (ngx_processes[n].exited || ngx_processes[n].pid == -1 || ngx_processes[n].channel[0] == -1) { continue; } ngx_log_debug3(NGX_LOG_DEBUG_CORE, cycle->log, 0, "pass close channel s:%d pid:" PID_T_FMT " to:" PID_T_FMT, ch.slot, ch.pid, ngx_processes[n].pid); /* TODO: NGX_AGAIN */ ngx_write_channel(ngx_processes[n].channel[0], &ch, sizeof(ngx_channel_t), cycle->log); } } if (ngx_processes[i].respawn && !ngx_processes[i].exiting && !ngx_terminate && !ngx_quit) { if (ngx_spawn_process(cycle, ngx_processes[i].proc, ngx_processes[i].data, ngx_processes[i].name, i) == NGX_ERROR) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "can not respawn %s", ngx_processes[i].name); continue; } live = 1; continue; } if (ngx_processes[i].pid == ngx_new_binary) { ngx_new_binary = 0; if (ngx_noaccepting) { ngx_restart = 1; ngx_noaccepting = 0; } } if (i == ngx_last_process - 1) { ngx_last_process--; } else { ngx_processes[i].pid = -1; } } else if (ngx_processes[i].exiting || !ngx_processes[i].detached) { live = 1; } } return live; }