ngx_int_t ngx_http_push_shutdown_ipc(ngx_cycle_t *cycle) {
int i;
for (i=0; i<NGX_MAX_PROCESSES; i++) {
if(ngx_http_push_socketpairs[i][0] != NGX_INVALID_FILE || ngx_http_push_socketpairs[i][1] != NGX_INVALID_FILE) {
ngx_close_channel(ngx_http_push_socketpairs[i], cycle->log);
}
}
return NGX_OK;
}
ngx_int_t ngx_http_push_init_ipc(ngx_cycle_t *cycle, ngx_int_t workers) {
//initialize socketpairs for workers in advance.
static int invalid_sockets_initialized = 0;
int i, s = 0, on = 1;
ngx_int_t last_expected_process = ngx_last_process;
if(!invalid_sockets_initialized) {
for(i=0; i< NGX_MAX_PROCESSES; i++) {
ngx_http_push_socketpairs[i][0]=NGX_INVALID_FILE;
ngx_http_push_socketpairs[i][1]=NGX_INVALID_FILE;
}
invalid_sockets_initialized=1;
}
/* here's the deal: we have no control over fork()ing, nginx's internal
* socketpairs are unusable for our purposes (as of nginx 0.8 -- check the
* code to see why), and the module initialization callbacks occur before
* any workers are spawned. Rather than futzing around with existing
* socketpairs, we populate our own socketpairs array.
* Trouble is, ngx_spawn_process() creates them one-by-one, and we need to
* do it all at once. So we must guess all the workers' ngx_process_slots in
* advance. Meaning the spawning logic must be copied to the T.
* ... with some allowances for already-opened sockets...
*/
for(i=0; i < workers; i++) {
//copypasta from os/unix/ngx_process.c (ngx_spawn_process)
while (s < last_expected_process && ngx_processes[s].pid != -1) {
//find empty existing slot
s++;
}
ngx_socket_t *socks = ngx_http_push_socketpairs[s];
if(socks[0] == NGX_INVALID_FILE || socks[1] == NGX_INVALID_FILE) {
//copypasta from os/unix/ngx_process.c (ngx_spawn_process)
if (socketpair(AF_UNIX, SOCK_STREAM, 0, socks) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed on socketpair while initializing push module");
return NGX_ERROR;
}
if (ngx_nonblocking(socks[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
if (ngx_nonblocking(socks[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
if (ioctl(socks[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed on socketpair while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
if (fcntl(socks[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed on socketpair while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
if (fcntl(socks[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed on socketpair while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
if (fcntl(socks[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while initializing push module");
ngx_close_channel(socks, cycle->log);
return NGX_ERROR;
}
}
s++; //NEXT!!
}
return NGX_OK;
}
static ngx_uint_t
ngx_reap_children(ngx_cycle_t *cycle)
{
ngx_int_t i, n;
ngx_uint_t live;
ngx_channel_t ch;
ngx_core_conf_t *ccf;
ngx_memzero(&ch, sizeof(ngx_channel_t));
ch.command = NGX_CMD_CLOSE_CHANNEL;
ch.fd = -1;
live = 0;
for (i = 0; i < ngx_last_process; i++) {
ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"child: %d %P e:%d t:%d d:%d r:%d j:%d",
i,
ngx_processes[i].pid,
ngx_processes[i].exiting,
ngx_processes[i].exited,
ngx_processes[i].detached,
ngx_processes[i].respawn,
ngx_processes[i].just_spawn);
if (ngx_processes[i].pid == -1) {
continue;
}
if (ngx_processes[i].exited) {
if (!ngx_processes[i].detached) {
ngx_close_channel(ngx_processes[i].channel, cycle->log);
ngx_processes[i].channel[0] = -1;
ngx_processes[i].channel[1] = -1;
ch.pid = ngx_processes[i].pid;
ch.slot = i;
for (n = 0; n < ngx_last_process; n++) {
if (ngx_processes[n].exited
|| ngx_processes[n].pid == -1
|| ngx_processes[n].channel[0] == -1)
{
continue;
}
ngx_log_debug3(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"pass close channel s:%i pid:%P to:%P",
ch.slot, ch.pid, ngx_processes[n].pid);
/* TODO: NGX_AGAIN */
ngx_write_channel(ngx_processes[n].channel[0],
&ch, sizeof(ngx_channel_t), cycle->log);
}
}
if (ngx_processes[i].respawn
&& !ngx_processes[i].exiting
&& !ngx_terminate
&& !ngx_quit)
{
if (ngx_spawn_process(cycle, ngx_processes[i].proc,
ngx_processes[i].data,
ngx_processes[i].name, i)
== NGX_INVALID_PID)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"could not respawn %s",
ngx_processes[i].name);
continue;
}
ch.command = NGX_CMD_OPEN_CHANNEL;
ch.pid = ngx_processes[ngx_process_slot].pid;
ch.slot = ngx_process_slot;
ch.fd = ngx_processes[ngx_process_slot].channel[0];
ngx_pass_open_channel(cycle, &ch);
live = 1;
continue;
}
if (ngx_processes[i].pid == ngx_new_binary) {
ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx,
ngx_core_module);
if (ngx_rename_file((char *) ccf->oldpid.data,
(char *) ccf->pid.data)
== NGX_FILE_ERROR)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_rename_file_n " %s back to %s failed "
"after the new binary process \"%s\" exited",
ccf->oldpid.data, ccf->pid.data, ngx_argv[0]);
}
ngx_new_binary = 0;
if (ngx_noaccepting) {
ngx_restart = 1;
ngx_noaccepting = 0;
}
}
if (i == ngx_last_process - 1) {
ngx_last_process--;
} else {
ngx_processes[i].pid = -1;
}
} else if (ngx_processes[i].exiting || !ngx_processes[i].detached) {
live = 1;
}
}
return live;
}
ngx_pid_t
ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
ngx_int_t s;
if (respawn >= 0) {//ngx_start_worker_processes-> NGX_PROCESS_RESPAWN ,表示要重新创建进程
s = respawn;
} else {//得到当前有多少工作进程创建了ngx_last_process会不断增加的，当然可能有人死了
for (s = 0; s < ngx_last_process; s++) {
if (ngx_processes[s].pid == -1) {
break;
}
}
if (s == NGX_MAX_PROCESSES) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
if (respawn != NGX_PROCESS_DETACHED) {//NGX_PROCESS_DETACHED说明是热代码替换
//不是热代码替换
/* Solaris 9 still has no AF_LOCAL */
//创建socketpair，进程之间通信
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1){
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]);
//将socket对设置为非阻塞模式
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
on = 1;//FIOASYNC 改变O_ASYNC 标志来打开或者关闭套接字的异步 IO 模式。
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//close on exec, not on-fork, 意为如果对描述符设置了FD_CLOEXEC，使用execl执行的程序里，
//此描述符被关闭，不能再使用它，但是在使用fork调用的子进程中，此描述符并不关闭，仍可使用。
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//设置fork对socket对是在子进程依然可用的
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
ngx_channel = ngx_processes[s].channel[1];
} else {//如果是热代码替换，不跟其他进程通信
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
ngx_process_slot = s;//当前在处理这个
pid = fork();//fork子进程
switch (pid) {
case -1:
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0://子进程调用proc
ngx_pid = ngx_getpid();//k getpid
proc(cycle, data);//等于ngx_worker_process_cycle
//子进程永远也从这回不来了，里面有exit
break;
default://父进程默认不干活
break;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
ngx_processes[s].pid = pid;
ngx_processes[s].exited = 0;
if (respawn >= 0) {
return pid;
}
//设置ngx_processes数组该项的信息
ngx_processes[s].proc = proc;//设置工作进程的处理函数ngx_worker_process_cycle，会fork之后调用的
ngx_processes[s].data = data;
ngx_processes[s].name = name;//进程名字
ngx_processes[s].exiting = 0;
switch (respawn) {
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
if (s == ngx_last_process) {
ngx_last_process++;//递增总进程数
}
return pid;
}
ngx_pid_t
ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
ngx_int_t s;
if (respawn >= 0) {
s = respawn;
} else {
for (s = 0; s < ngx_last_process; s++) {
if (ngx_processes[s].pid == -1) {
break;
}
}
if (s == NGX_MAX_PROCESSES) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"no more than %d processes can be spawned",
NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
if (respawn != NGX_PROCESS_DETACHED) {
/* Solaris 9 still has no AF_LOCAL */
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"channel %d:%d",
ngx_processes[s].channel[0],
ngx_processes[s].channel[1]);
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
on = 1;
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
ngx_channel = ngx_processes[s].channel[1];
} else {
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
ngx_process_slot = s;
pid = fork();
switch (pid) {
case -1:
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0:
ngx_pid = ngx_getpid();
proc(cycle, data);
break;
default:
break;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
ngx_processes[s].pid = pid;
ngx_processes[s].exited = 0;
if (respawn >= 0) {
return pid;
}
ngx_processes[s].proc = proc;
ngx_processes[s].data = data;
ngx_processes[s].name = name;
ngx_processes[s].exiting = 0;
switch (respawn) {
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
if (s == ngx_last_process) {
ngx_last_process++;
}
return pid;
}
//创建子进程
ngx_pid_t
ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
ngx_int_t s;
if (respawn >= 0) {
s = respawn;
} else {
//初始化ngx_processes数组
for (s = 0; s < ngx_last_process; s++) {
if (ngx_processes[s].pid == -1) {
break;
}
}
//进程个数不能大于NGX_MAX_PROCESSES
if (s == NGX_MAX_PROCESSES) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"no more than %d processes can be spawned",
NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
if (respawn != NGX_PROCESS_DETACHED) {
/* Solaris 9 still has no AF_LOCAL */
//创建用于进程间通信的UNIX域套结字
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"channel %d:%d",
ngx_processes[s].channel[0],
ngx_processes[s].channel[1]);
//将套结字的读写端都设置为非阻塞
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
on = 1;
//以下设置套结字读端为信号驱动,注意我们还没有fork子进程，设置是在master进程中
//设置FIOASYNC标志（我们必须要这样做），fcntl函数也可以设置，但大多数系统不支持，
//因此用ioctl来实现
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//设置F_SETOWN标志，将套结字读端和master进程关联起来，当管道有数据的时侯会向该进程
//发送SIGIO信号
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//设置管道的FD_CLOEXEC标志
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
ngx_channel = ngx_processes[s].channel[1];
} else {
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
ngx_process_slot = s;
pid = fork();
switch (pid) {
case -1: //error
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0: //child
ngx_pid = ngx_getpid();
//执行我们传递的子进程执行函数
proc(cycle, data);
break;
default: //father
break;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
ngx_processes[s].pid = pid;
ngx_processes[s].exited = 0;
if (respawn >= 0) {
return pid;
}
ngx_processes[s].proc = proc; //设置子进程的执行函数
ngx_processes[s].data = data; //设置子进程的参数
ngx_processes[s].name = name; //设置子进程的name
ngx_processes[s].exiting = 0;
switch (respawn) {
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
if (s == ngx_last_process) {
ngx_last_process++; //last_proces可以视为子进程个数
}
return pid;
}
/*
* 创建进程
*/
ngx_pid_t
ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
ngx_int_t s;
if (respawn >= 0) {
s = respawn;
} else {
/*在全局进程表 ngx_processes 中查找第一个空的结构，用来存放新创建的进程的信息*/
for (s = 0; s < ngx_last_process; s++) {
if (ngx_processes[s].pid == -1) {
break;
}
}
if (s == NGX_MAX_PROCESSES) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"no more than %d processes can be spawned",
NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
if (respawn != NGX_PROCESS_DETACHED) {
/* Solaris 9 still has no AF_LOCAL */
/*调用 socketpair 创建一对互相连接的 unix域socket，用于进程间的全双工通信*/
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"channel %d:%d",
ngx_processes[s].channel[0],
ngx_processes[s].channel[1]);
/*将用于进程通信的 socket 设置成非阻塞的*/
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
on = 1;
/*设置成信号驱动异步IO，有IO时内核发送SIGIO信号*/
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
/*设置将接受 SIGIO 和 SIGURG 信号的进程id*/
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
/*当执行 exec 时关闭描述符*/
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
/*当前子进程的描述符*/
ngx_channel = ngx_processes[s].channel[1];
} else {
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
/*新创建进的进程在 ngx_processes 数组中的下标*/
ngx_process_slot = s;
pid = fork();
switch (pid) {
case -1:
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0: //子进程
ngx_pid = ngx_getpid();
proc(cycle, data);
break;
default:
break;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
/*新创建的进程放入 master 主进程的 ngx_processes 数组中*/
ngx_processes[s].pid = pid;
ngx_processes[s].exited = 0;
if (respawn >= 0) {
return pid;
}
ngx_processes[s].proc = proc;
ngx_processes[s].data = data;
ngx_processes[s].name = name;
ngx_processes[s].exiting = 0;
switch (respawn) {
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
if (s == ngx_last_process) {
ngx_last_process++;
}
return pid;
}
/*proc:是子进程的执行函数ngx_worker_process_cycle,data是参数，name是子进程的名字*/
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
//将要创建的子进程在进程表中的位置
ngx_int_t s;
//替换进程ngx_processes[respawn],可安全重用该进程表项
if (respawn >= 0)
{
s = respawn;
}
else
{
//先找到一个被回收的进程表项
for (s = 0; s < ngx_last_process; s++)
{
if (ngx_processes[s].pid == -1) {
break;
}
}
//进程表已经满
if (s == NGX_MAX_PROCESSES)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,"no more than %d processes can be spawned", NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
//不是分离的子进程
if (respawn != NGX_PROCESS_DETACHED)
{
/* Solaris 9 still has no AF_LOCAL */
//ngx_processes[s].channel数组是用于父子间通信的UNIX域套接字对
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,"channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]);
//设置socket为非阻塞模式
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//开启channel[0]的消息驱动IO
on = 1;
/*FIOASYNC:设置/清除信号驱动异步I/O标志.根据iocl 的第三个参数指向一个0 值或非0 值分别清除或设置针对本套接口的信号驱动异步I/O
* 标志，它决定是否收取针对本套接口的异步I/O 信号（SIGIO ）。本请求和O_ASYNC 文件状态标志等效，而该标志可以通过fcntl 的F_SETFL命令清除或设置。*/
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//设置channel[0]的宿主，控制channel[0]的SIGIO信号只发给这个进程
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//若进程执行了exec后，关闭socket
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//同上
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"",name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
//用于监听可读事件的socket
ngx_channel = ngx_processes[s].channel[1];
}
else
{
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
//设置当前子进程的进程表索引值
ngx_process_slot = s;
//创建子进程
pid = fork();
switch (pid)
{
case -1:
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0:
//设置当前子进程的父进程id
ngx_pid = ngx_getpid();
//子进程运行执行函数
proc(cycle, data);
break;
default:
break;
}
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
//设置一些进程表项字段
ngx_processes[s].pid = pid;
ngx_processes[s].exited = 0;
//如果是重复创建，即为替换进程，不用设置其他进程表字段，直接返回。
if (respawn >= 0)
{
return pid;
}
//设置其他进程表字段
ngx_processes[s].proc = proc;
ngx_processes[s].data = data;
ngx_processes[s].name = name;
ngx_processes[s].exiting = 0;
//设置进程表项的一些状态字
switch (respawn)
{
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
if (s == ngx_last_process) {
ngx_last_process++;
}
return pid;
}
ngx_pid_t
ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,
char *name, ngx_int_t respawn)
{
u_long on;
ngx_pid_t pid;
ngx_int_t s;
/* 如果respawn参数大于0，则相当于创建某个指定的进程，
* 也就是直接使用respawn作为数组中的索引
*/
if (respawn >= 0) {
s = respawn;
} else {
/* 获取一个合适的进程数组索引，新建的子进程信息就放在全局进程结构的s索引处，
*
*/
for (s = 0; s < ngx_last_process; s++) {
if (ngx_processes[s].pid == -1) {
break;
}
}
if (s == NGX_MAX_PROCESSES) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"no more than %d processes can be spawned",
NGX_MAX_PROCESSES);
return NGX_INVALID_PID;
}
}
if (respawn != NGX_PROCESS_DETACHED) {
/* Solaris 9 still has no AF_LOCAL */
/* 创建AF_UNIX套接字用于两个进程通信，
* 注意创建的文件描述符存放在对应ngx_process_t结构当中，
* 注意当前的情景，是master进程创建worker进程，而master进程并不在ngx_processes
* 数组当中，
*/
if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"socketpair() failed while spawning \"%s\"", name);
return NGX_INVALID_PID;
}
ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"channel %d:%d",
ngx_processes[s].channel[0],
ngx_processes[s].channel[1]);
/* 设置相应的非阻塞标记 */
if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
ngx_nonblocking_n " failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
on = 1;
if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"ioctl(FIOASYNC) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(F_SETOWN) failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) {
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fcntl(FD_CLOEXEC) failed while spawning \"%s\"",
name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
}
/* 父进程创建一对通道，父进程使用的是0通道，子进程使用的是1通道，
* 在worker进程的初始化过程当中会将该文件描述符添加到读监听集里面，
* 主要目的是两个子进程之间通信
*/
ngx_channel = ngx_processes[s].channel[1];
} else {
ngx_processes[s].channel[0] = -1;
ngx_processes[s].channel[1] = -1;
}
/* 设置创建子进程在全局进程结构中的索引，
* 执行fork之后，该全局变量就被拷贝到子进程当中了，
* 而且子进程可以通过该变量知道自己在ngx_processes
* 数组中的位置
*/
ngx_process_slot = s;
/* 错误返回-1
* 子进程返回0
* 父进程返回子进程的pid
*/
pid = fork();
switch (pid) {
case -1:
ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
"fork() failed while spawning \"%s\"", name);
ngx_close_channel(ngx_processes[s].channel, cycle->log);
return NGX_INVALID_PID;
case 0:
ngx_pid = ngx_getpid();
/* 开始执行子进程的函数，注意子进程是一个服务进程，所以在proc函数在执行之后
* 就会一直循环
*/
proc(cycle, data);
break;
default:
break;
}
/* 此处之下就都是父进程执行的代码 */
ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid);
/* 设置子进程的相关信息 */
ngx_processes[s].pid = pid;
/* 表示进程没有退出 */
ngx_processes[s].exited = 0;
/* 如果启动的是指定进程，则直接返回 */
if (respawn >= 0) {
return pid;
}
/* 设置刚刚创建子进程的信息 */
ngx_processes[s].proc = proc;
ngx_processes[s].data = data; /* 数据其实就是1，2，3 */
ngx_processes[s].name = name;
ngx_processes[s].exiting = 0;
/* 根据respawn参数来确定 respawn，just_spawn,detached的值 */
switch (respawn) {
case NGX_PROCESS_NORESPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_SPAWN:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_RESPAWN:
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_JUST_RESPAWN: /* 注意这个标记对应了两个值 */
ngx_processes[s].respawn = 1;
ngx_processes[s].just_spawn = 1;
ngx_processes[s].detached = 0;
break;
case NGX_PROCESS_DETACHED:
ngx_processes[s].respawn = 0;
ngx_processes[s].just_spawn = 0;
ngx_processes[s].detached = 1;
break;
}
/* 创建完一个子进程时，会相应的增加ngx_last_process变量 */
if (s == ngx_last_process) {
ngx_last_process++;
}
return pid;
}
static void
ngx_http_push_stream_ipc_exit_worker(ngx_cycle_t *cycle)
{
ngx_close_channel((ngx_socket_t *) ngx_http_push_stream_socketpairs[ngx_process_slot], cycle->log);
}
static ngx_uint_t ngx_reap_childs(ngx_cycle_t *cycle)
{
ngx_int_t i, n;
ngx_uint_t live;
ngx_channel_t ch;
ch.command = NGX_CMD_CLOSE_CHANNEL;
ch.fd = -1;
live = 0;
for (i = 0; i < ngx_last_process; i++) {
ngx_log_debug7(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
"child: %d " PID_T_FMT " e:%d t:%d d:%d r:%d j:%d",
i,
ngx_processes[i].pid,
ngx_processes[i].exiting,
ngx_processes[i].exited,
ngx_processes[i].detached,
ngx_processes[i].respawn,
ngx_processes[i].just_respawn);
if (ngx_processes[i].pid == -1) {
continue;
}
if (ngx_processes[i].exited) {
if (!ngx_processes[i].detached) {
ngx_close_channel(ngx_processes[i].channel, cycle->log);
ngx_processes[i].channel[0] = -1;
ngx_processes[i].channel[1] = -1;
ch.pid = ngx_processes[i].pid;
ch.slot = i;
for (n = 0; n < ngx_last_process; n++) {
if (ngx_processes[n].exited
|| ngx_processes[n].pid == -1
|| ngx_processes[n].channel[0] == -1)
{
continue;
}
ngx_log_debug3(NGX_LOG_DEBUG_CORE, cycle->log, 0,
"pass close channel s:%d pid:" PID_T_FMT
" to:" PID_T_FMT, ch.slot, ch.pid, ngx_processes[n].pid);
/* TODO: NGX_AGAIN */
ngx_write_channel(ngx_processes[n].channel[0],
&ch, sizeof(ngx_channel_t), cycle->log);
}
}
if (ngx_processes[i].respawn
&& !ngx_processes[i].exiting
&& !ngx_terminate
&& !ngx_quit)
{
if (ngx_spawn_process(cycle, ngx_processes[i].proc,
ngx_processes[i].data,
ngx_processes[i].name, i)
== NGX_ERROR)
{
ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
"can not respawn %s", ngx_processes[i].name);
continue;
}
live = 1;
continue;
}
if (ngx_processes[i].pid == ngx_new_binary) {
ngx_new_binary = 0;
if (ngx_noaccepting) {
ngx_restart = 1;
ngx_noaccepting = 0;
}
}
if (i == ngx_last_process - 1) {
ngx_last_process--;
} else {
ngx_processes[i].pid = -1;
}
} else if (ngx_processes[i].exiting || !ngx_processes[i].detached) {
live = 1;
}
}
return live;
}
