static ngx_int_t ngx_http_clojure_jvm_worker_pipe_init(ngx_log_t *log) { ngx_connection_t *c; ngx_int_t rc; rc = ngx_http_clojure_pipe(nc_jvm_worker_pipe_fds); if (rc != 0) { ngx_log_error(NGX_LOG_ERR, log, 0, "ngx clojure: create worker_pipe failed"); return NGX_ERROR; } if (ngx_nonblocking(nc_jvm_worker_pipe_fds[0]) == -1) { ngx_log_error(NGX_LOG_ERR, log, 0, "ngx clojure create worker_pipe at ngx_nonblocking(fds[0]) failed"); return NGX_ERROR; } if (ngx_nonblocking(nc_jvm_worker_pipe_fds[1]) == -1) { ngx_log_error(NGX_LOG_ERR, log, 0, "ngx clojure: create worker_pipe at ngx_nonblocking(fds[1]) failed"); return NGX_ERROR; } ngx_log_error(NGX_LOG_NOTICE, log, 0, "ngx clojure: init pipe for jvm worker, fds: %d, %d", nc_jvm_worker_pipe_fds[0], nc_jvm_worker_pipe_fds[1]); c = ngx_get_connection(nc_jvm_worker_pipe_fds[0], log); if (c == NULL) { ngx_http_clojure_pipe_close(nc_jvm_worker_pipe_fds[1]); ngx_http_clojure_pipe_close(nc_jvm_worker_pipe_fds[0]); ngx_log_error(NGX_LOG_ERR, log, 0, "ngx clojure: create worker_pipe at ngx_get_connection failed"); return NGX_ERROR; } c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->log = log; c->read->log = log; c->write->log = log; c->data = NULL; c->read->handler = ngx_http_clojure_jvm_worker_result_handler; rc = ngx_handle_read_event(c->read, 0); if (rc != NGX_OK) { ngx_log_error(NGX_LOG_ERR, log, 0, "ngx clojure: create worker_pipe at ngx_handle_read_event failed"); } return rc; }
ngx_int_t ngx_open_pipes(ngx_cycle_t *cycle) { ngx_int_t stat; ngx_uint_t i; ngx_core_conf_t *ccf; ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); for (i = 0; i < ngx_last_pipe; i++) { if (!ngx_pipes[i].configured) { continue; } if (ngx_pipes[i].generation != ngx_pipe_generation) { continue; } ngx_pipes[i].backup = ngx_pipes[i].open_fd->name; ngx_pipes[i].user = ccf->user; stat = ngx_open_pipe(cycle, &ngx_pipes[i]); ngx_log_debug4(NGX_LOG_DEBUG_CORE, cycle->log, 0, "pipe: %ui(%d, %d) \"%s\"", i, ngx_pipes[i].pfd[0], ngx_pipes[i].pfd[1], ngx_pipes[i].cmd); if (stat == NGX_ERROR) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "open pipe \"%s\" failed", ngx_pipes[i].cmd); return NGX_ERROR; } if (fcntl(ngx_pipes[i].open_fd->fd, F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) \"%s\" failed", ngx_pipes[i].cmd); return NGX_ERROR; } if (ngx_nonblocking(ngx_pipes[i].open_fd->fd) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "nonblock \"%s\" failed", ngx_pipes[i].cmd); return NGX_ERROR; } ngx_pipes[i].open_fd->name.len = 0; ngx_pipes[i].open_fd->name.data = NULL; } return NGX_OK; }
static ngx_connection_t* init_dummy_conn(void *data, int fd, ngx_event_handler_pt readhandler) { ngx_connection_t *c = calloc(1,sizeof(ngx_connection_t)); if (c == NULL) { ngx_log_error(NGX_LOG_ERR, pcycle->log, 0, "calloc ngx_connection_t failed"); return NULL; } c->read = calloc(1,sizeof(ngx_event_t)); if (c->read == NULL) { ngx_log_error(NGX_LOG_ERR, pcycle->log, 0, "calloc ngx_event_t failed"); free(c); return NULL; } c->write = calloc(1,sizeof(ngx_event_t)); if (c->write == NULL) { ngx_log_error(NGX_LOG_ERR, pcycle->log, 0, "calloc ngx_event_t failed"); free(c->read); free(c); return NULL; } if (ngx_nonblocking(fd) == -1) { ngx_log_error(NGX_LOG_ALERT, pcycle->log, errno, "init_dummy_conn ngx_nonblocking failed"); } c->fd = fd; c->data = data; c->log = pcycle->log; c->read->data = c; c->read->log = pcycle->log; c->read->handler = readhandler; c->write->data = c; c->write->log = pcycle->log; c->write->handler = dummy_write_handler; if (ngx_add_event(c->read, NGX_READ_EVENT, 0) == NGX_ERROR) { ngx_log_error(NGX_LOG_INFO, pcycle->log, 0, "add dummy conn read event failed"); free(c->read); free(c->write); free(c); return NULL; } return c; }
static void ngx_proc_send_accept(ngx_event_t *ev) { u_char sa[NGX_SOCKADDRLEN]; socklen_t socklen; ngx_socket_t s; ngx_connection_t *lc; lc = ev->data; s = accept(lc->fd, (struct sockaddr *) sa, &socklen); if (s == -1) { return; } if (ngx_nonblocking(s) == -1) { goto finish; } finish: ngx_close_socket(s); }
static ngx_int_t ngx_open_log_connection(ngx_syslog_t *task) { size_t len; ngx_socket_t fd; fd = ngx_socket(task->addr.sockaddr->sa_family, SOCK_DGRAM, 0); if (fd == -1) { goto err; } len = 0; setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &len, sizeof(len)); if (ngx_nonblocking(fd) == -1) { goto err; } if (connect(fd, task->addr.sockaddr, task->addr.socklen) == -1) { goto err; } shutdown(fd, SHUT_RD); task->fd = fd; return NGX_OK; err: if (fd != -1) { ngx_close_socket(fd); } task->next_try = ngx_cached_time->sec + ngx_syslog_retry_interval; return NGX_DECLINED; }
//创建子进程 ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { //初始化ngx_processes数组 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } //进程个数不能大于NGX_MAX_PROCESSES if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ //创建用于进程间通信的UNIX域套结字 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //将套结字的读写端都设置为非阻塞 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; //以下设置套结字读端为信号驱动,注意我们还没有fork子进程,设置是在master进程中 //设置FIOASYNC标志(我们必须要这样做),fcntl函数也可以设置,但大多数系统不支持, //因此用ioctl来实现 if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置F_SETOWN标志,将套结字读端和master进程关联起来,当管道有数据的时侯会向该进程 //发送SIGIO信号 if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置管道的FD_CLOEXEC标志 if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s; pid = fork(); switch (pid) { case -1: //error ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //child ngx_pid = ngx_getpid(); //执行我们传递的子进程执行函数 proc(cycle, data); break; default: //father break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; //设置子进程的执行函数 ngx_processes[s].data = data; //设置子进程的参数 ngx_processes[s].name = name; //设置子进程的name ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; //last_proces可以视为子进程个数 } return pid; }
/* * 创建进程 */ ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { /*在全局进程表 ngx_processes 中查找第一个空的结构,用来存放新创建的进程的信息*/ for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ /*调用 socketpair 创建一对互相连接的 unix域socket,用于进程间的全双工通信*/ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); /*将用于进程通信的 socket 设置成非阻塞的*/ if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; /*设置成信号驱动异步IO,有IO时内核发送SIGIO信号*/ if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*设置将接受 SIGIO 和 SIGURG 信号的进程id*/ if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*当执行 exec 时关闭描述符*/ if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /*当前子进程的描述符*/ ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } /*新创建进的进程在 ngx_processes 数组中的下标*/ ngx_process_slot = s; pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //子进程 ngx_pid = ngx_getpid(); proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); /*新创建的进程放入 master 主进程的 ngx_processes 数组中*/ ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }
//监听,绑定cycle中listening动态数组指定的相应端口 //cycle是当前进程的ngx_cycle_t结构体指针 ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle) { int reuseaddr; ngx_uint_t i, tries, failed; ngx_err_t err; ngx_log_t *log; ngx_socket_t s; ngx_listening_t *ls; reuseaddr = 1; #if (NGX_SUPPRESS_WARN) failed = 0; #endif log = cycle->log; /* TODO: configurable try number */ for (tries = 5; tries; tries--) { failed = 0; /* for each listening socket */ ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { if (ls[i].ignore) { continue; } if (ls[i].fd != (ngx_socket_t) -1) { continue; } if (ls[i].inherited) { /* TODO: close on exit */ /* TODO: nonblocking */ /* TODO: deferred accept */ continue; } s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_socket_n " %V failed", &ls[i].addr_text); return NGX_ERROR; } if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(SO_REUSEADDR) %V failed", &ls[i].addr_text); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) if (ls[i].sockaddr->sa_family == AF_INET6) { int ipv6only; ipv6only = ls[i].ipv6only; if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, (const void *) &ipv6only, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(IPV6_V6ONLY) %V failed, ignored", &ls[i].addr_text); } } #endif /* TODO: close on exit */ if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_nonblocking_n " %V failed", &ls[i].addr_text); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } } ngx_log_debug2(NGX_LOG_DEBUG_CORE, log, 0, "bind() %V #%d ", &ls[i].addr_text, s); if (bind(s, ls[i].sockaddr, ls[i].socklen) == -1) { err = ngx_socket_errno; if (err != NGX_EADDRINUSE || !ngx_test_config) { ngx_log_error(NGX_LOG_EMERG, log, err, "bind() to %V failed", &ls[i].addr_text); } if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } if (err != NGX_EADDRINUSE) { return NGX_ERROR; } if (!ngx_test_config) { failed = 1; } continue; } #if (NGX_HAVE_UNIX_DOMAIN) if (ls[i].sockaddr->sa_family == AF_UNIX) { mode_t mode; u_char *name; name = ls[i].addr_text.data + sizeof("unix:") - 1; mode = (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); if (chmod((char *) name, mode) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "chmod() \"%s\" failed", name); } if (ngx_test_config) { if (ngx_delete_file(name) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, ngx_delete_file_n " %s failed", name); } } } #endif if (listen(s, ls[i].backlog) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "listen() to %V, backlog %d failed", &ls[i].addr_text, ls[i].backlog); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } ls[i].listen = 1; ls[i].fd = s; } if (!failed) { break; } /* TODO: delay configurable */ ngx_log_error(NGX_LOG_NOTICE, log, 0, "try again to bind() after 500ms"); ngx_msleep(500); } if (failed) { ngx_log_error(NGX_LOG_EMERG, log, 0, "still could not bind()"); return NGX_ERROR; } return NGX_OK; }
ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc) { int rc; ngx_int_t event; ngx_err_t err; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_connection_t *c; // 取得我们将要发送的upstream对端 rc = pc->get(pc, pc->data); if (rc != NGX_OK) { return rc; } // 新建socket s = ngx_socket(pc->sockaddr->sa_family, SOCK_STREAM, 0); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, 0, "socket %d", s); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_socket_n " failed"); return NGX_ERROR; } // 取得连接 c = ngx_get_connection(s, pc->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_close_socket_n "failed"); } return NGX_ERROR; } // 设置rcvbuf的大小 if (pc->rcvbuf) { if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (const void *) &pc->rcvbuf, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, "setsockopt(SO_RCVBUF) failed"); goto failed; } } // 设置非阻塞 if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_nonblocking_n " failed"); goto failed; } if (pc->local) { if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) { ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno, "bind(%V) failed", &pc->local->name); goto failed; } } c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->sendfile = 1; c->log_error = pc->log_error; if (pc->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } rev = c->read; wev = c->write; rev->log = pc->log; wev->log = pc->log; pc->connection = c; c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); if (ngx_add_conn) { if (ngx_add_conn(c) == NGX_ERROR) { goto failed; } } ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connect to %V, fd:%d #%uA", pc->name, s, c->number); // 非阻塞connect rc = connect(s, pc->sockaddr, pc->socklen); if (rc == -1) { err = ngx_socket_errno; if (err != NGX_EINPROGRESS #if (NGX_WIN32) /* Winsock returns WSAEWOULDBLOCK (NGX_EAGAIN) */ && err != NGX_EAGAIN #endif ) { if (err == NGX_ECONNREFUSED #if (NGX_LINUX) /* * Linux returns EAGAIN instead of ECONNREFUSED * for unix sockets if listen queue is full */ || err == NGX_EAGAIN #endif || err == NGX_ECONNRESET || err == NGX_ENETDOWN || err == NGX_ENETUNREACH || err == NGX_EHOSTDOWN || err == NGX_EHOSTUNREACH) { level = NGX_LOG_ERR; } else { level = NGX_LOG_CRIT; } ngx_log_error(level, c->log, err, "connect() to %V failed", pc->name); ngx_close_connection(c); pc->connection = NULL; return NGX_DECLINED; } } if (ngx_add_conn) { if (rc == -1) { /* NGX_EINPROGRESS */ return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_AIO_EVENT) { ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, ngx_socket_errno, "connect(): %d", rc); /* aio, iocp */ if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_blocking_n " failed"); goto failed; } /* * FreeBSD's aio allows to post an operation on non-connected socket. * NT does not support it. * * TODO: check in Win32, etc. As workaround we can use NGX_ONESHOT_EVENT */ rev->ready = 1; wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { /* kqueue */ event = NGX_CLEAR_EVENT; } else { /* select, poll, /dev/poll */ event = NGX_LEVEL_EVENT; } if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) { goto failed; } if (rc == -1) { /* NGX_EINPROGRESS */ if (ngx_add_event(wev, NGX_WRITE_EVENT, event) != NGX_OK) { goto failed; } return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; failed: ngx_close_connection(c); pc->connection = NULL; return NGX_ERROR; }
static ngx_int_t ngx_rtmp_auto_push_init_process(ngx_cycle_t *cycle) { #if (NGX_HAVE_UNIX_DOMAIN) ngx_rtmp_auto_push_conf_t *apcf; ngx_listening_t *ls, *lss; struct sockaddr_un *saun; int reuseaddr; ngx_socket_t s; size_t n; ngx_file_info_t fi; if (ngx_process != NGX_PROCESS_WORKER) { return NGX_OK; } ngx_rtmp_auto_push_module.ctx_index = ngx_rtmp_core_module.ctx_index; apcf = (ngx_rtmp_auto_push_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_rtmp_auto_push_module); if (apcf->auto_push == 0) { return NGX_OK; } next_publish = ngx_rtmp_publish; ngx_rtmp_publish = ngx_rtmp_auto_push_publish; next_delete_stream = ngx_rtmp_delete_stream; ngx_rtmp_delete_stream = ngx_rtmp_auto_push_delete_stream; reuseaddr = 1; s = (ngx_socket_t) -1; ngx_log_debug0(NGX_LOG_DEBUG_RTMP, cycle->log, 0, "auto_push: creating sockets"); /*TODO: clone all RTMP listenings? */ ls = cycle->listening.elts; lss = NULL; for (n = 0; n < cycle->listening.nelts; ++n, ++ls) { if (ls->handler == ngx_stream_init_connection) { lss = ls; break; } } if (lss == NULL) { return NGX_OK; } ls = ngx_array_push(&cycle->listening); if (ls == NULL) { return NGX_ERROR; } *ls = *lss; /* Disable unix socket client address extraction * from accept call * Nginx generates bad addr_text with this enabled */ ls->addr_ntop = 0; ls->socklen = sizeof(struct sockaddr_un); saun = ngx_pcalloc(cycle->pool, ls->socklen); ls->sockaddr = (struct sockaddr *) saun; if (ls->sockaddr == NULL) { return NGX_ERROR; } saun->sun_family = AF_UNIX; *ngx_snprintf((u_char *) saun->sun_path, sizeof(saun->sun_path), "%V/" NGX_RTMP_AUTO_PUSH_SOCKNAME ".%i", &apcf->socket_dir, ngx_process_slot) = 0; ngx_log_debug1(NGX_LOG_DEBUG_RTMP, cycle->log, 0, "auto_push: create socket '%s'", saun->sun_path); if (ngx_file_info(saun->sun_path, &fi) != ENOENT) { ngx_log_debug1(NGX_LOG_DEBUG_RTMP, cycle->log, 0, "auto_push: delete existing socket '%s'", saun->sun_path); ngx_delete_file(saun->sun_path); } ngx_str_set(&ls->addr_text, "worker_socket"); s = ngx_socket(AF_UNIX, SOCK_STREAM, 0); if (s == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, ngx_socket_n " worker_socket failed"); return NGX_ERROR; } if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, "setsockopt(SO_REUSEADDR) worker_socket failed"); goto sock_error; } if (!(ngx_event_flags & NGX_USE_AIO_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, ngx_nonblocking_n " worker_socket failed"); return NGX_ERROR; } } if (bind(s, (struct sockaddr *) saun, sizeof(*saun)) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, ngx_nonblocking_n " worker_socket bind failed"); goto sock_error; } if (listen(s, NGX_LISTEN_BACKLOG) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, "listen() to worker_socket, backlog %d failed", NGX_LISTEN_BACKLOG); goto sock_error; } ls->fd = s; ls->listen = 1; return NGX_OK; sock_error: if (s != (ngx_socket_t) -1 && ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, ngx_close_socket_n " worker_socket failed"); } ngx_delete_file(saun->sun_path); return NGX_ERROR; #else /* NGX_HAVE_UNIX_DOMAIN */ return NGX_OK; #endif /* NGX_HAVE_UNIX_DOMAIN */ }
ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc) { int rc, type; #if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT || NGX_LINUX) in_port_t port; #endif ngx_int_t event; ngx_err_t err; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_connection_t *c; rc = pc->get(pc, pc->data); if (rc != NGX_OK) { return rc; } type = (pc->type ? pc->type : SOCK_STREAM); s = ngx_socket(pc->sockaddr->sa_family, type, 0); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, pc->log, 0, "%s socket %d", (type == SOCK_STREAM) ? "stream" : "dgram", s); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_socket_n " failed"); return NGX_ERROR; } c = ngx_get_connection(s, pc->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_close_socket_n "failed"); } return NGX_ERROR; } c->type = type; if (pc->rcvbuf) { if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (const void *) &pc->rcvbuf, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, "setsockopt(SO_RCVBUF) failed"); goto failed; } } if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_nonblocking_n " failed"); goto failed; } if (pc->local) { #if (NGX_HAVE_TRANSPARENT_PROXY) if (pc->transparent) { if (ngx_event_connect_set_transparent(pc, s) != NGX_OK) { goto failed; } } #endif #if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT || NGX_LINUX) port = ngx_inet_get_port(pc->local->sockaddr); #endif #if (NGX_HAVE_IP_BIND_ADDRESS_NO_PORT) if (pc->sockaddr->sa_family != AF_UNIX && port == 0) { static int bind_address_no_port = 1; if (bind_address_no_port) { if (setsockopt(s, IPPROTO_IP, IP_BIND_ADDRESS_NO_PORT, (const void *) &bind_address_no_port, sizeof(int)) == -1) { err = ngx_socket_errno; if (err != NGX_EOPNOTSUPP && err != NGX_ENOPROTOOPT) { ngx_log_error(NGX_LOG_ALERT, pc->log, err, "setsockopt(IP_BIND_ADDRESS_NO_PORT) " "failed, ignored"); } else { bind_address_no_port = 0; } } } } #endif #if (NGX_LINUX) if (pc->type == SOCK_DGRAM && port != 0) { int reuse_addr = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuse_addr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, "setsockopt(SO_REUSEADDR) failed"); goto failed; } } #endif if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) { ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno, "bind(%V) failed", &pc->local->name); goto failed; } } if (type == SOCK_STREAM) { c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->sendfile = 1; if (pc->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } } else { /* type == SOCK_DGRAM */ c->recv = ngx_udp_recv; c->send = ngx_send; c->send_chain = ngx_udp_send_chain; } c->log_error = pc->log_error; rev = c->read; wev = c->write; rev->log = pc->log; wev->log = pc->log; pc->connection = c; c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); if (ngx_add_conn) { if (ngx_add_conn(c) == NGX_ERROR) { goto failed; } } ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connect to %V, fd:%d #%uA", pc->name, s, c->number); rc = connect(s, pc->sockaddr, pc->socklen); if (rc == -1) { err = ngx_socket_errno; if (err != NGX_EINPROGRESS #if (NGX_WIN32) /* Winsock returns WSAEWOULDBLOCK (NGX_EAGAIN) */ && err != NGX_EAGAIN #endif ) { if (err == NGX_ECONNREFUSED #if (NGX_LINUX) /* * Linux returns EAGAIN instead of ECONNREFUSED * for unix sockets if listen queue is full */ || err == NGX_EAGAIN #endif || err == NGX_ECONNRESET || err == NGX_ENETDOWN || err == NGX_ENETUNREACH || err == NGX_EHOSTDOWN || err == NGX_EHOSTUNREACH) { level = NGX_LOG_ERR; } else { level = NGX_LOG_CRIT; } ngx_log_error(level, c->log, err, "connect() to %V failed", pc->name); ngx_close_connection(c); pc->connection = NULL; return NGX_DECLINED; } } if (ngx_add_conn) { if (rc == -1) { /* NGX_EINPROGRESS */ return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_IOCP_EVENT) { ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, ngx_socket_errno, "connect(): %d", rc); if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_blocking_n " failed"); goto failed; } /* * FreeBSD's aio allows to post an operation on non-connected socket. * NT does not support it. * * TODO: check in Win32, etc. As workaround we can use NGX_ONESHOT_EVENT */ rev->ready = 1; wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { /* kqueue */ event = NGX_CLEAR_EVENT; } else { /* select, poll, /dev/poll */ event = NGX_LEVEL_EVENT; } if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) { goto failed; } if (rc == -1) { /* NGX_EINPROGRESS */ if (ngx_add_event(wev, NGX_WRITE_EVENT, event) != NGX_OK) { goto failed; } return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; failed: ngx_close_connection(c); pc->connection = NULL; return NGX_ERROR; }
static ngx_int_t ngx_send_radius_request( ngx_http_request_t *r, radius_req_queue_node_t* prev_req ) { ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request 0x%xl", r ); ngx_http_auth_radius_main_conf_t* conf = ngx_http_get_module_main_conf( r, ngx_http_auth_radius_module ); ngx_http_core_loc_conf_t *clcf; clcf = ngx_http_get_module_loc_conf( r, ngx_http_core_module ); ngx_http_auth_radius_ctx_t* ctx = ngx_http_get_module_ctx( r, ngx_http_auth_radius_module ); if ( ctx == NULL ) abort(); // TODO radius_str_t user = { RADIUS_STR_FROM_NGX_STR_INITIALIZER( r->headers_in.user ) }; radius_str_t passwd = { RADIUS_STR_FROM_NGX_STR_INITIALIZER( r->headers_in.passwd ) }; radius_req_queue_node_t* n; n = radius_send_request( prev_req, &user, &passwd, clcf->error_log ); if ( n == NULL ) { abort(); // TODO } ngx_http_auth_radius_main_conf_t* lconf = ngx_http_get_module_loc_conf( r, ngx_http_auth_radius_module ); ngx_add_timer( r->connection->read, lconf->radius_timeout ); radius_server_t* rs; rs = get_server_by_req( n ); ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request rs: %d, assign 0x%xl to 0x%xl, id: %d", rs->id, r, n, n->ident ); n->data = r; ctx->n = n; ngx_connection_t* c = rs->data; ngx_event_t* rev; if ( c == NULL ) { c = ngx_get_connection( rs->s, conf->log ); if ( c == NULL ) { ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request: ngx_get_connection" ); if (ngx_close_socket( rs->s ) == -1) ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request: ngx_close_socket" ); return NGX_ERROR; } if ( ngx_nonblocking( rs->s ) == -1 ) { ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request: ngx_nonblocking" ); ngx_free_connection( c ); if (ngx_close_socket( rs->s ) == -1) ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request: ngx_close_socket" ); return NGX_ERROR; } rs->data = c; c->data = rs; rev = c->read; rev->handler = radius_read_handler; rev->log = clcf->error_log; rs->log = clcf->error_log; if ( ngx_add_event( rev, NGX_READ_EVENT, NGX_LEVEL_EVENT ) != NGX_OK ) { ngx_log_error( NGX_LOG_ERR, r->connection->log, 0, "ngx_send_radius_request: ngx_add_event" ); return NGX_ERROR; } c->number = ngx_atomic_fetch_add( ngx_connection_counter, 1 ); } return NGX_OK; }
/*----------------------------------------------------------------------------*/ ngx_int_t rp_module_send_response(ngx_http_request_t *r, cJSON **json_root) { ngx_buf_t *b; ngx_chain_t out; ngx_int_t rc; char *out_buffer; int out_buffer_len; cJSON *j_params; cJSON *j_status; b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t)); if(b == NULL) { rp_error(r->connection->log, "Can not allocate memory"); return NGX_HTTP_INTERNAL_SERVER_ERROR; } out.buf = b; out.next = NULL; r->headers_out.content_type_len = strlen(json_content_str); r->headers_out.content_type.len = strlen(json_content_str); r->headers_out.content_type.data = (u_char *)json_content_str; out_buffer = cJSON_PrintUnformatted(*json_root, r->pool); if(out_buffer == NULL) { rp_error(r->connection->log, "Creating output buffer failed"); return NGX_HTTP_INTERNAL_SERVER_ERROR; } out_buffer_len = strlen(out_buffer); b->pos = (u_char *)out_buffer; b->last = (u_char *)out_buffer + out_buffer_len; b->memory = 1; b->last_buf = b->last_in_chain = 1; b->sync = b->flush = 1; r->headers_out.status = NGX_HTTP_OK; r->headers_out.content_length_n = out_buffer_len; /* Debug purpopses - output params & status */ j_params = cJSON_GetObjectItem(*json_root, "params"); if(j_params != NULL) { int params_cnt = -1; int i; params_cnt = cJSON_GetArraySize(j_params); rp_debug(r->connection->log, "Output parameters: "); for(i = 0; i < params_cnt; i++) { rp_debug(r->connection->log, "\t%d - %f", i, (float)cJSON_GetArrayItem(j_params, i)->valuedouble); } } j_status = cJSON_GetObjectItem(*json_root, "status"); //if(j_status != NULL) { // rp_debug(r->connection->log, "Output status (req :%d): %s", r->method, // j_status->valuestring); //} cJSON_Delete(*json_root, r->pool); rc = ngx_http_send_header(r); /* If error while sending OK output we re-send it */ if((rc == NGX_ERROR) && (r->method == NGX_HTTP_GET) && j_status && (j_status->valuestring[0] = 'O') && (j_status->valuestring[1] == 'K')) { rp_data_clear_signals_dirty(); } if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) { return rc; } /* TODO: Be sure that outputting is always flushed! Had some problems * at this part. */ /* send the buffer chain of your response */ /* Temp, got from ruby-forum.com - put socket to blocking */ ngx_blocking(r->connection->fd); rc = ngx_http_output_filter(r, &out); while(rc == NGX_AGAIN) { r->connection->write->ready = 1; rc = ngx_http_output_filter(r, &out); if(rc == NGX_ERROR) break; } ngx_nonblocking(r->connection->fd); rc = ngx_http_output_filter(r, NULL); return NGX_DONE; }
ngx_int_t ngx_open_listening_sockets(ngx_cycle_t *cycle) { int reuseaddr; ngx_uint_t i, tries, failed; ngx_err_t err; ngx_log_t *log; ngx_socket_t s; ngx_listening_t *ls; reuseaddr = 1; #if (NGX_SUPPRESS_WARN) failed = 0; #endif log = cycle->log; /* TODO: configurable try number */ for (tries = 5; tries; tries--) { //bind和listen最多从实5次 failed = 0; /* for each listening socket */ ls = cycle->listening.elts; for (i = 0; i < cycle->listening.nelts; i++) { if (ls[i].ignore) { continue; } #if (NGX_HAVE_REUSEPORT) if (ls[i].add_reuseport) { /* * to allow transition from a socket without SO_REUSEPORT * to multiple sockets with SO_REUSEPORT, we have to set * SO_REUSEPORT on the old socket before opening new ones */ int reuseport = 1; if (setsockopt(ls[i].fd, SOL_SOCKET, SO_REUSEPORT, (const void *) &reuseport, sizeof(int)) //端口复用需要内核支持 == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, "setsockopt(SO_REUSEPORT) %V failed, ignored", &ls[i].addr_text); } ls[i].add_reuseport = 0; } #endif if (ls[i].fd != (ngx_socket_t) -1) { continue; } if (ls[i].inherited) { /* TODO: close on exit */ /* TODO: nonblocking */ /* TODO: deferred accept */ continue; } s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_socket_n " %V failed", &ls[i].addr_text); return NGX_ERROR; } /* 默认情况下,server重启,调用socket,bind,然后listen,会失败.因为该端口正在被使用.如果设定SO_REUSEADDR,那么server重启才会成功.因此, 所有的TCP server都必须设定此选项,用以应对server重启的现象. */ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(SO_REUSEADDR) %V failed, close sock:%d", &ls[i].addr_text, s); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } #if (NGX_HAVE_REUSEPORT) if (ls[i].reuseport) { int reuseport; reuseport = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, (const void *) &reuseport, sizeof(int)) //端口复用需要内核支持 == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(SO_REUSEPORT) %V failed, ignored, close sock:%d", &ls[i].addr_text, s); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } } #endif #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) if (ls[i].sockaddr->sa_family == AF_INET6) { int ipv6only; ipv6only = ls[i].ipv6only; if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, (const void *) &ipv6only, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(IPV6_V6ONLY) %V failed, ignored", &ls[i].addr_text); } } #endif /* TODO: close on exit */ if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_nonblocking_n " %V failed, close sock:%d", &ls[i].addr_text,s); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } } ngx_log_debug2(NGX_LOG_DEBUG_CORE, log, 0, "bind() %V #%d ", &ls[i].addr_text, s); if (bind(s, ls[i].sockaddr, ls[i].socklen) == -1) { err = ngx_socket_errno; if (err != NGX_EADDRINUSE || !ngx_test_config) { ngx_log_error(NGX_LOG_EMERG, log, err, "bind() to %V failed", &ls[i].addr_text); } ngx_log_error(NGX_LOG_EMERG, log, err, "close socket:%d", s); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } if (err != NGX_EADDRINUSE) { return NGX_ERROR; } if (!ngx_test_config) { failed = 1; } continue; } #if (NGX_HAVE_UNIX_DOMAIN) if (ls[i].sockaddr->sa_family == AF_UNIX) { mode_t mode; u_char *name; name = ls[i].addr_text.data + sizeof("unix:") - 1; mode = (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); if (chmod((char *) name, mode) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, "chmod() \"%s\" failed", name); } if (ngx_test_config) { if (ngx_delete_file(name) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, ngx_delete_file_n " %s failed", name); } } } #endif if (listen(s, ls[i].backlog) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "listen() to %V, backlog %d failed", &ls[i].addr_text, ls[i].backlog); ngx_log_error(NGX_LOG_EMERG, log, err, "close socket:%d", s); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls[i].addr_text); } return NGX_ERROR; } ls[i].listen = 1; ls[i].fd = s; } if (!failed) { break; } /* TODO: delay configurable */ ngx_log_error(NGX_LOG_NOTICE, log, 0, "try again to bind() after 500ms"); ngx_msleep(500); } if (failed) { ngx_log_error(NGX_LOG_EMERG, log, 0, "still could not bind()"); return NGX_ERROR; } return NGX_OK; }
//处理新连接的回调函数 void ngx_event_accept(ngx_event_t *ev) { socklen_t socklen; ngx_err_t err; ngx_log_t *log; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_listening_t *ls; ngx_connection_t *c, *lc; ngx_event_conf_t *ecf; u_char sa[NGX_SOCKADDRLEN]; #if (NGX_HAVE_ACCEPT4) static ngx_uint_t use_accept4 = 1; #endif if (ev->timedout) { if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { return; } ev->timedout = 0; } ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { ev->available = ecf->multi_accept; } lc = ev->data; ls = lc->listening; ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "accept on %V, ready: %d", &ls->addr_text, ev->available); do { socklen = NGX_SOCKADDRLEN; //首先尝试调用accept方法试图建立新连接 #if (NGX_HAVE_ACCEPT4) if (use_accept4) { s = accept4(lc->fd, (struct sockaddr *) sa, &socklen, SOCK_NONBLOCK); } else { s = accept(lc->fd, (struct sockaddr *) sa, &socklen); } #else s = accept(lc->fd, (struct sockaddr *) sa, &socklen); #endif if (s == (ngx_socket_t) -1) { err = ngx_socket_errno; if (err == NGX_EAGAIN) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "accept() not ready"); return; } level = NGX_LOG_ALERT; if (err == NGX_ECONNABORTED) { level = NGX_LOG_ERR; } else if (err == NGX_EMFILE || err == NGX_ENFILE) { level = NGX_LOG_CRIT; } #if (NGX_HAVE_ACCEPT4) ngx_log_error(level, ev->log, err, use_accept4 ? "accept4() failed" : "accept() failed"); if (use_accept4 && err == NGX_ENOSYS) { use_accept4 = 0; ngx_inherited_nonblocking = 0; continue; } #else ngx_log_error(level, ev->log, err, "accept() failed"); #endif if (err == NGX_ECONNABORTED) { if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } if (ev->available) { continue; } } if (err == NGX_EMFILE || err == NGX_ENFILE) { if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { return; } if (ngx_use_accept_mutex) { if (ngx_accept_mutex_held) { ngx_shmtx_unlock(&ngx_accept_mutex); ngx_accept_mutex_held = 0; } ngx_accept_disabled = 1; } else { ngx_add_timer(ev, ecf->accept_mutex_delay); } } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif /* * ngx_accept_disabled是负载均衡机制实现的关键阈值,这个阈值与连接池中连接的使用情况密切相关 * 初始时这个值为负数,为负数时不会触发负载均衡操作,而当为正数时触发负载均衡操作 * 当ngx_accept_disabled为正数时当前进程将不再处理新连接事件,而仅仅是ngx_accept_disabled减1 * * 如果当前活动连接数超过最大可承受连接数的7/8(ngx_accept_disabled为负),则表示发生过载 */ ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; //由连接池中获取一个ngx_connection_t连接对象 c = ngx_get_connection(s, ev->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif c->pool = ngx_create_pool(ls->pool_size, ev->log); if (c->pool == NULL) { ngx_close_accepted_connection(c); return; } c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); return; } ngx_memcpy(c->sockaddr, sa, socklen); log = ngx_palloc(c->pool, sizeof(ngx_log_t)); if (log == NULL) { ngx_close_accepted_connection(c); return; } /* set a blocking mode for iocp and non-blocking mode for others */ if (ngx_inherited_nonblocking) { if (ngx_event_flags & NGX_USE_IOCP_EVENT) { if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_blocking_n " failed"); ngx_close_accepted_connection(c); return; } } } else { if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_nonblocking_n " failed"); ngx_close_accepted_connection(c); return; } } } *log = ls->log; c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->log = log; c->pool->log = log; c->socklen = socklen; c->listening = ls; c->local_sockaddr = ls->sockaddr; c->local_socklen = ls->socklen; c->unexpected_eof = 1; #if (NGX_HAVE_UNIX_DOMAIN) if (c->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } #endif rev = c->read; wev = c->write; wev->ready = 1; if (ngx_event_flags & NGX_USE_IOCP_EVENT) { rev->ready = 1; } if (ev->deferred_accept) { rev->ready = 1; #if (NGX_HAVE_KQUEUE) rev->available = 1; #endif } rev->log = log; wev->log = log; /* * TODO: MT: - ngx_atomic_fetch_add() * or protection by critical section or light mutex * * TODO: MP: - allocated in a shared memory * - ngx_atomic_fetch_add() * or protection by critical section or light mutex */ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_handled, 1); #endif if (ls->addr_ntop) { c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len); if (c->addr_text.data == NULL) { ngx_close_accepted_connection(c); return; } c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen, c->addr_text.data, ls->addr_text_max_len, 0); if (c->addr_text.len == 0) { ngx_close_accepted_connection(c); return; } } #if (NGX_DEBUG) { ngx_str_t addr; struct sockaddr_in *sin; ngx_cidr_t *cidr; ngx_uint_t i; u_char text[NGX_SOCKADDR_STRLEN]; #if (NGX_HAVE_INET6) struct sockaddr_in6 *sin6; ngx_uint_t n; #endif cidr = ecf->debug_connection.elts; for (i = 0; i < ecf->debug_connection.nelts; i++) { if (cidr[i].family != (ngx_uint_t) c->sockaddr->sa_family) { goto next; } switch (cidr[i].family) { #if (NGX_HAVE_INET6) case AF_INET6: sin6 = (struct sockaddr_in6 *) c->sockaddr; for (n = 0; n < 16; n++) { if ((sin6->sin6_addr.s6_addr[n] & cidr[i].u.in6.mask.s6_addr[n]) != cidr[i].u.in6.addr.s6_addr[n]) { goto next; } } break; #endif #if (NGX_HAVE_UNIX_DOMAIN) case AF_UNIX: break; #endif default: /* AF_INET */ sin = (struct sockaddr_in *) c->sockaddr; if ((sin->sin_addr.s_addr & cidr[i].u.in.mask) != cidr[i].u.in.addr) { goto next; } break; } log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL; break; next: continue; } if (log->log_level & NGX_LOG_DEBUG_EVENT) { addr.data = text; addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text, NGX_SOCKADDR_STRLEN, 1); ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA accept: %V fd:%d", c->number, &addr, s); } } #endif if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) { if (ngx_add_conn(c) == NGX_ERROR) { ngx_close_accepted_connection(c); return; } } log->data = NULL; log->handler = NULL; ls->handler(c); if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } //如果监听事件的available标志位为1,则再次循环,对应着multi_accept配置项,告诉nginx一次性尽量多的建立新连接 } while (ev->available); }
/*proc:是子进程的执行函数ngx_worker_process_cycle,data是参数,name是子进程的名字*/ ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data,char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; //将要创建的子进程在进程表中的位置 ngx_int_t s; //替换进程ngx_processes[respawn],可安全重用该进程表项 if (respawn >= 0) { s = respawn; } else { //先找到一个被回收的进程表项 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } //进程表已经满 if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,"no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } //不是分离的子进程 if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ //ngx_processes[s].channel数组是用于父子间通信的UNIX域套接字对 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0,"channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //设置socket为非阻塞模式 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //开启channel[0]的消息驱动IO on = 1; /*FIOASYNC:设置/清除信号驱动异步I/O标志.根据iocl 的第三个参数指向一个0 值或非0 值分别清除或设置针对本套接口的信号驱动异步I/O * 标志,它决定是否收取针对本套接口的异步I/O 信号(SIGIO )。本请求和O_ASYNC 文件状态标志等效,而该标志可以通过fcntl 的F_SETFL命令清除或设置。*/ if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置channel[0]的宿主,控制channel[0]的SIGIO信号只发给这个进程 if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //若进程执行了exec后,关闭socket if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,"fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //同上 if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"",name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //用于监听可读事件的socket ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } //设置当前子进程的进程表索引值 ngx_process_slot = s; //创建子进程 pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: //设置当前子进程的父进程id ngx_pid = ngx_getpid(); //子进程运行执行函数 proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); //设置一些进程表项字段 ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; //如果是重复创建,即为替换进程,不用设置其他进程表字段,直接返回。 if (respawn >= 0) { return pid; } //设置其他进程表字段 ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; //设置进程表项的一些状态字 switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; /* 如果respawn参数大于0,则相当于创建某个指定的进程, * 也就是直接使用respawn作为数组中的索引 */ if (respawn >= 0) { s = respawn; } else { /* 获取一个合适的进程数组索引,新建的子进程信息就放在全局进程结构的s索引处, * */ for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ /* 创建AF_UNIX套接字用于两个进程通信, * 注意创建的文件描述符存放在对应ngx_process_t结构当中, * 注意当前的情景,是master进程创建worker进程,而master进程并不在ngx_processes * 数组当中, */ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); /* 设置相应的非阻塞标记 */ if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } /* 父进程创建一对通道,父进程使用的是0通道,子进程使用的是1通道, * 在worker进程的初始化过程当中会将该文件描述符添加到读监听集里面, * 主要目的是两个子进程之间通信 */ ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } /* 设置创建子进程在全局进程结构中的索引, * 执行fork之后,该全局变量就被拷贝到子进程当中了, * 而且子进程可以通过该变量知道自己在ngx_processes * 数组中的位置 */ ngx_process_slot = s; /* 错误返回-1 * 子进程返回0 * 父进程返回子进程的pid */ pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: ngx_pid = ngx_getpid(); /* 开始执行子进程的函数,注意子进程是一个服务进程,所以在proc函数在执行之后 * 就会一直循环 */ proc(cycle, data); break; default: break; } /* 此处之下就都是父进程执行的代码 */ ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); /* 设置子进程的相关信息 */ ngx_processes[s].pid = pid; /* 表示进程没有退出 */ ngx_processes[s].exited = 0; /* 如果启动的是指定进程,则直接返回 */ if (respawn >= 0) { return pid; } /* 设置刚刚创建子进程的信息 */ ngx_processes[s].proc = proc; ngx_processes[s].data = data; /* 数据其实就是1,2,3 */ ngx_processes[s].name = name; ngx_processes[s].exiting = 0; /* 根据respawn参数来确定 respawn,just_spawn,detached的值 */ switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: /* 注意这个标记对应了两个值 */ ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } /* 创建完一个子进程时,会相应的增加ngx_last_process变量 */ if (s == ngx_last_process) { ngx_last_process++; } return pid; }
static ngx_int_t ngx_proc_send_process_init(ngx_cycle_t *cycle) { int reuseaddr; ngx_event_t *rev; ngx_socket_t fd; ngx_connection_t *c; struct sockaddr_in sin; ngx_proc_send_conf_t *pbcf; pbcf = ngx_proc_get_conf(cycle->conf_ctx, ngx_proc_send_module); fd = ngx_socket(AF_INET, SOCK_STREAM, 0); if (fd == -1) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "send socket error"); return NGX_ERROR; } reuseaddr = 1; if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, "send setsockopt(SO_REUSEADDR) failed"); ngx_close_socket(fd); return NGX_ERROR; } if (ngx_nonblocking(fd) == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_socket_errno, "send nonblocking failed"); ngx_close_socket(fd); return NGX_ERROR; } sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_ANY); sin.sin_port = htons(pbcf->port); if (bind(fd, (struct sockaddr *) &sin, sizeof(sin)) == -1) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "send bind error"); return NGX_ERROR; } if (listen(fd, 20) == -1) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "send listen error"); return NGX_ERROR; } c = ngx_get_connection(fd, cycle->log); if (c == NULL) { ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "send no connection"); return NGX_ERROR; } c->log = cycle->log; rev = c->read; rev->log = c->log; rev->accept = 1; rev->handler = ngx_proc_send_accept; if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) { return NGX_ERROR; } pbcf->fd = fd; return NGX_OK; }
ngx_int_t ngx_tcp_open_listening_socket(ngx_listening_t *ls) { int reuseaddr; ngx_uint_t tries, failed; ngx_err_t err; ngx_log_t *log; ngx_socket_t s; reuseaddr = 1; log = ls->logp; for (tries = 5; tries; tries--) { failed = 0; if (ls->ignore) { continue; } if (ls->fd != -1) { break; } s = ngx_socket(ls->sockaddr->sa_family, ls->type, 0); if (s == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_socket_n " %V failed", &ls->addr_text); return NGX_ERROR; } if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(SO_REUSEADDR) %V failed", &ls->addr_text); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls->addr_text); } return NGX_ERROR; } #if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) if (ls->sockaddr->sa_family == AF_INET6) { int ipv6only; ipv6only = ls->ipv6only; if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, (const void *) &ipv6only, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "setsockopt(IPV6_V6ONLY) %V failed, ignored", &ls->addr_text); } } #endif /* TODO: close on exit */ if (!(ngx_event_flags & NGX_USE_AIO_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_nonblocking_n " %V failed", &ls->addr_text); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls->addr_text); } return NGX_ERROR; } } ngx_log_debug2(NGX_LOG_DEBUG_CORE, log, 0, "bind() %V #%d ", &ls->addr_text, s); #if (NGX_HAVE_UNIX_DOMAIN) { u_char *name; name = ls->addr_text.data + sizeof("unix:") - 1; unlink((const char *)name); } #endif if (bind(s, ls->sockaddr, ls->socklen) == -1) { err = ngx_socket_errno; if (err == NGX_EADDRINUSE && ngx_test_config) { continue; } ngx_log_error(NGX_LOG_EMERG, log, err, "bind() to %V failed", &ls->addr_text); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls->addr_text); } if (err != NGX_EADDRINUSE) { return NGX_ERROR; } failed = 1; break; } #if (NGX_HAVE_UNIX_DOMAIN) if (ls->sockaddr->sa_family == AF_UNIX) { mode_t mode; u_char *name; name = ls->addr_text.data + sizeof("unix:") - 1; mode = (S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH); if (chmod((char *) name, mode) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "chmod() \"%s\" failed", name); } if (ngx_test_config) { if (ngx_delete_file(name) == NGX_FILE_ERROR) { ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, ngx_delete_file_n " %s failed", name); } } } #endif if (listen(s, ls->backlog) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, "listen() to %V, backlog %d failed", &ls->addr_text, ls->backlog); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, ngx_close_socket_n " %V failed", &ls->addr_text); } return NGX_ERROR; } ls->listen = 1; ls->fd = s; if (!failed) { break; } /* TODO: delay configurable */ ngx_log_error(NGX_LOG_NOTICE, log, 0, "try again to bind() after 500ms"); ngx_msleep(500); } if (failed) { ngx_log_error(NGX_LOG_EMERG, log, 0, "still could not bind()"); return NGX_ERROR; } return NGX_OK; }
char *ngx_nlog(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { ngx_str_t *value; ngx_url_t u_l; ngx_url_t u_r; ngx_socket_t s; int reuseaddr; ngx_pool_cleanup_t *cln; ngx_tcp_core_srv_conf_t *cscf = conf; //need set file first if (NULL == cscf->error_log) { return "need set error_log first"; } if (cscf->error_log->fd != -1) { return "is duplicate"; } value = cf->args->elts; ngx_memzero(&u_l, sizeof(ngx_url_t)); u_l.url = value[1]; u_l.default_port = (in_port_t) 0; u_l.no_resolve = 1; if (ngx_parse_url(cf->pool, &u_l) != NGX_OK) { if (u_l.err) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "%s in \"%V\" of the \"nlog\" directive", u_l.err, &u_l.url); } return NGX_CONF_ERROR; } if (u_l.no_port || u_l.family != AF_INET) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "no valid port or no valid ipv4 address"); return NGX_CONF_ERROR; } ngx_memzero(&u_r, sizeof(ngx_url_t)); u_r.url = value[2]; u_r.default_port = (in_port_t) 0; u_r.no_resolve = 1; if (ngx_parse_url(cf->pool, &u_r) != NGX_OK) { if (u_r.err) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "%s in \"%V\" of the \"nlog\" directive", u_r.err, &u_r.url); } return NGX_CONF_ERROR; } if (u_r.no_port || u_r.family != AF_INET) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "no valid port or no valid ipv4 address"); return NGX_CONF_ERROR; } s = ngx_socket(AF_INET, SOCK_DGRAM, 0); ngx_conf_log_error(NGX_LOG_DEBUG, cf, 0,"nlog create udp socket %d",s); if (s == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "socket() failed, %d", ngx_socket_errno); return NGX_CONF_ERROR; } if (ngx_nonblocking(s) == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "ngx_nonblocking() failed, %d", ngx_socket_errno); goto failed; } reuseaddr = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "setsockopt() failed, %d", ngx_socket_errno); goto failed; } if (bind(s, (struct sockaddr_in*) &u_l.sockaddr, u_l.socklen) == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "bind() failed, %d", ngx_socket_errno); goto failed; } if (connect(s, (struct sockaddr_in*) &u_r.sockaddr, u_r.socklen) == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "connect() failed, %d", ngx_socket_errno); goto failed; } cscf->error_log->fd = s; cln = ngx_pool_cleanup_add(cf->pool, 0); cln->data = cscf->error_log; cln->handler = ngx_clean_nlog_sock; return NGX_CONF_OK; failed: if (ngx_close_socket(s) == -1) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "ngx_close_socket() failed, %d", ngx_socket_errno); } return NGX_CONF_ERROR; }
ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc) { //Here,we initilize a connection!!! int rc; ngx_int_t event; ngx_err_t err; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_connection_t *c; //Here,very important, this handler is use to get a select handler!! /* For ip_hash module,will set the get handler!!! r->upstream->peer.get = ngx_http_upstream_get_round_robin_peer; r->upstream->peer.free = ngx_http_upstream_free_round_robin_peer; */ //will stored the really backend ip_addr in the pc!! rc = pc->get(pc, pc->data); if (rc != NGX_OK) { return rc; } //Here,Do the socket operation! alloc a socket_fd s = ngx_socket(pc->sockaddr->sa_family, SOCK_STREAM, 0); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, 0, "socket %d", s); if (s == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_socket_n " failed"); return NGX_ERROR; } c = ngx_get_connection(s, pc->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_close_socket_n "failed"); } return NGX_ERROR; } if (pc->rcvbuf) { if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (const void *) &pc->rcvbuf, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, "setsockopt(SO_RCVBUF) failed"); goto failed; } } if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_nonblocking_n " failed"); goto failed; } if (pc->local) { if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) { ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno, "bind(%V) failed", &pc->local->name); goto failed; } } //########################### Here set the connection data's read write handler!!! c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->sendfile = 1; c->log_error = pc->log_error; if (pc->sockaddr->sa_family != AF_INET) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } rev = c->read; wev = c->write; rev->log = pc->log; wev->log = pc->log; pc->connection = c; c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_THREADS) /* TODO: lock event when call completion handler */ rev->lock = pc->lock; wev->lock = pc->lock; rev->own_lock = &c->lock; wev->own_lock = &c->lock; #endif if (ngx_add_conn) { if (ngx_add_conn(c) == NGX_ERROR) { goto failed; } } ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connect to %V, fd:%d #%d", pc->name, s, c->number); //########################## Here,doing the really connect operation rc = connect(s, pc->sockaddr, pc->socklen); if (rc == -1) { err = ngx_socket_errno; if (err != NGX_EINPROGRESS #if (NGX_WIN32) /* Winsock returns WSAEWOULDBLOCK (NGX_EAGAIN) */ && err != NGX_EAGAIN #endif ) { if (err == NGX_ECONNREFUSED #if (NGX_LINUX) /* * Linux returns EAGAIN instead of ECONNREFUSED * for unix sockets if listen queue is full */ || err == NGX_EAGAIN #endif || err == NGX_ECONNRESET || err == NGX_ENETDOWN || err == NGX_ENETUNREACH || err == NGX_EHOSTDOWN || err == NGX_EHOSTUNREACH) { level = NGX_LOG_ERR; } else { level = NGX_LOG_CRIT; } ngx_log_error(level, c->log, err, "connect() to %V failed", pc->name); return NGX_DECLINED; } } if (ngx_add_conn) { if (rc == -1) { /* NGX_EINPROGRESS */ return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1;// return NGX_OK; } if (ngx_event_flags & NGX_USE_AIO_EVENT) { ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, ngx_socket_errno, "connect(): %d", rc); /* aio, iocp */ if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_blocking_n " failed"); goto failed; } /* * FreeBSD's aio allows to post an operation on non-connected socket. * NT does not support it. * * TODO: check in Win32, etc. As workaround we can use NGX_ONESHOT_EVENT */ rev->ready = 1; wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { /* kqueue */ event = NGX_CLEAR_EVENT; } else { /* select, poll, /dev/poll */ event = NGX_LEVEL_EVENT; } //##########################################Very important! //##########################################add the connection read event!! if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) { goto failed; } if (rc == -1) { /* NGX_EINPROGRESS */ if (ngx_add_event(wev, NGX_WRITE_EVENT, event) != NGX_OK) { goto failed; } return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; //Means the connection is ready to write! return NGX_OK; failed: ngx_free_connection(c); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return NGX_ERROR; }
static int ngx_tcp_lua_ngx_nlog(lua_State *L) { ngx_tcp_session_t *ps; ngx_socket_t *u; ngx_url_t u_l; ngx_url_t u_r; ngx_socket_t s; u_char *p; size_t len; int reuseaddr; ps = ngx_tcp_lua_get_session(L); if (ps != NULL) { //only in init_by_lua return luaL_error(L, "only use in init_by_lua"); } if (lua_gettop(L) != 2) { return luaL_error(L, "expecting 2 arguments, but got %d", lua_gettop(L)); } //check and parse args p = (u_char *) luaL_checklstring(L, 1, &len); if (p == NULL) { return luaL_argerror(L, 1, "bad argument, string expected"); } ngx_memzero(&u_l, sizeof(ngx_url_t)); u_l.url.data = p; u_l.url.len = len; u_l.default_port = (in_port_t) 0; u_l.no_resolve = 1; //use ngx_cycle pool if (ngx_parse_url(ngx_cycle->pool, &u_l) != NGX_OK) { return luaL_argerror(L, 1, "bad argument, wrong format of ip:port"); } if (u_l.no_port || u_l.family != AF_INET) { return luaL_argerror(L, 1, "bad argument, wrong format of ip:port"); } p = (u_char *) luaL_checklstring(L, 2, &len); if (p == NULL) { return luaL_argerror(L, 2, "bad argument, string expected"); } ngx_memzero(&u_r, sizeof(ngx_url_t)); u_r.url.data = p; u_r.url.len = len; u_r.default_port = (in_port_t) 0; u_r.no_resolve = 1; if (ngx_parse_url(ngx_cycle->pool, &u_r) != NGX_OK) { return luaL_argerror(L, 2, "bad argument, wrong format of ip:port"); } if (u_r.no_port || u_r.family != AF_INET) { return luaL_argerror(L, 2, "bad argument, wrong format of ip:port"); } //create socket s = ngx_socket(AF_INET, SOCK_DGRAM, 0); if (s == -1) { return luaL_error(L, "ngx_socket failed"); } if (ngx_nonblocking(s) == -1) { ngx_close_socket(s); return luaL_error(L, "ngx_nonblocking failed"); } reuseaddr = 1; if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (const void *) &reuseaddr, sizeof(int)) == -1) { ngx_close_socket(s); return luaL_error(L, "setsockopt SO_REUSEADDR failed"); } if (bind(s, (struct sockaddr_in*) &u_l.sockaddr, u_l.socklen) == -1) { ngx_close_socket(s); return luaL_error(L, "bind failed"); } if (connect(s, (struct sockaddr_in*) &u_r.sockaddr, u_r.socklen) == -1) { ngx_close_socket(s); return luaL_error(L, "connect failed"); } u = lua_newuserdata(L, sizeof(ngx_socket_t)); if (u == NULL) { ngx_close_socket(s); return luaL_error(L, "out of memory"); } *u = s; lua_pushlightuserdata(L, &ngx_tcp_lua_nlog_udata_metatable_key); lua_rawget(L, LUA_REGISTRYINDEX); lua_setmetatable(L, -2); return 1; }
static ngx_int_t ngx_http_lua_udp_connect(ngx_udp_connection_t *uc) { int rc; ngx_int_t event; ngx_event_t *rev, *wev; ngx_socket_t s; ngx_connection_t *c; s = ngx_socket(uc->sockaddr->sa_family, SOCK_DGRAM, 0); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, &uc->log, 0, "UDP socket %d", s); if (s == -1) { ngx_log_error(NGX_LOG_ALERT, &uc->log, ngx_socket_errno, ngx_socket_n " failed"); return NGX_ERROR; } c = ngx_get_connection(s, &uc->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, &uc->log, ngx_socket_errno, ngx_close_socket_n "failed"); } return NGX_ERROR; } if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, &uc->log, ngx_socket_errno, ngx_nonblocking_n " failed"); ngx_free_connection(c); if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, &uc->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return NGX_ERROR; } rev = c->read; wev = c->write; rev->log = &uc->log; wev->log = &uc->log; uc->connection = c; c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_THREADS) /* TODO: lock event when call completion handler */ rev->lock = &c->lock; wev->lock = &c->lock; rev->own_lock = &c->lock; wev->own_lock = &c->lock; #endif #if (NGX_HTTP_LUA_HAVE_SO_PASSCRED) if (uc->sockaddr->sa_family == AF_UNIX) { struct sockaddr addr; addr.sa_family = AF_UNIX; /* just to make valgrind happy */ ngx_memzero(addr.sa_data, sizeof(addr.sa_data)); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, &uc->log, 0, "datagram unix " "domain socket autobind"); if (bind(uc->connection->fd, &addr, sizeof(sa_family_t)) != 0) { ngx_log_error(NGX_LOG_CRIT, &uc->log, ngx_socket_errno, "bind() failed"); return NGX_ERROR; } } #endif ngx_log_debug3(NGX_LOG_DEBUG_EVENT, &uc->log, 0, "connect to %V, fd:%d #%d", &uc->server, s, c->number); rc = connect(s, uc->sockaddr, uc->socklen); /* TODO: aio, iocp */ if (rc == -1) { ngx_log_error(NGX_LOG_CRIT, &uc->log, ngx_socket_errno, "connect() failed"); return NGX_ERROR; } /* UDP sockets are always ready to write */ wev->ready = 1; if (ngx_add_event) { event = (ngx_event_flags & NGX_USE_CLEAR_EVENT) ? /* kqueue, epoll */ NGX_CLEAR_EVENT: /* select, poll, /dev/poll */ NGX_LEVEL_EVENT; /* eventport event type has no meaning: oneshot only */ if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) { return NGX_ERROR; } } else { /* rtsig */ if (ngx_add_conn(c) == NGX_ERROR) { return NGX_ERROR; } } return NGX_OK; }
//这里的event是在ngx_event_process_init中从连接池中获取的 ngx_connection_t中的->read读事件 //accept是在ngx_event_process_init(但进程或者不配置负载均衡的时候)或者(多进程,配置负载均衡)的时候把accept事件添加到epoll中 void //该形参中的ngx_connection_t(ngx_event_t)是为accept事件连接准备的空间,当accept返回成功后,会重新获取一个ngx_connection_t(ngx_event_t)用来读写该连接 ngx_event_accept(ngx_event_t *ev) //在ngx_process_events_and_timers中执行 { //一个accept事件对应一个ev,如当前一次有4个客户端accept,应该对应4个ev事件,一次来多个accept的处理在下面的do {}while中实现 socklen_t socklen; ngx_err_t err; ngx_log_t *log; ngx_uint_t level; ngx_socket_t s; //如果是文件异步i/o中的ngx_event_aio_t,则它来自ngx_event_aio_t->ngx_event_t(只有读),如果是网络事件中的event,则为ngx_connection_s中的event(包括读和写) ngx_event_t *rev, *wev; ngx_listening_t *ls; ngx_connection_t *c, *lc; ngx_event_conf_t *ecf; u_char sa[NGX_SOCKADDRLEN]; #if (NGX_HAVE_ACCEPT4) static ngx_uint_t use_accept4 = 1; #endif if (ev->timedout) { if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { return; } ev->timedout = 0; } ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { ev->available = ecf->multi_accept; } lc = ev->data; ls = lc->listening; ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "accept on %V, ready: %d", &ls->addr_text, ev->available); do { socklen = NGX_SOCKADDRLEN; #if (NGX_HAVE_ACCEPT4) //ngx_close_socket可以关闭套接字 if (use_accept4) { s = accept4(lc->fd, (struct sockaddr *) sa, &socklen, SOCK_NONBLOCK); } else { s = accept(lc->fd, (struct sockaddr *) sa, &socklen); } #else /* 针对非阻塞I/O执行的系统调用则总是立即返回,而不管事件足否已经发生。如果事件没有眭即发生,这些系统调用就 返回—1.和出错的情况一样。此时我们必须根据errno来区分这两种情况。对accept、send和recv而言,事件未发牛时errno 通常被设置成EAGAIN(意为“再来一次”)或者EWOULDBLOCK(意为“期待阻塞”):对conncct而言,errno则被 设置成EINPROGRESS(意为“在处理中")。 */ s = accept(lc->fd, (struct sockaddr *) sa, &socklen); #endif if (s == (ngx_socket_t) -1) { err = ngx_socket_errno; if (err == NGX_EAGAIN) { //如果event{}开启multi_accept,则在accept完该listen ip:port对应的ip和端口连接后,会通过这里返回 ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "accept() not ready"); return; } level = NGX_LOG_ALERT; if (err == NGX_ECONNABORTED) { level = NGX_LOG_ERR; } else if (err == NGX_EMFILE || err == NGX_ENFILE) { level = NGX_LOG_CRIT; } #if (NGX_HAVE_ACCEPT4) ngx_log_error(level, ev->log, err, use_accept4 ? "accept4() failed" : "accept() failed"); if (use_accept4 && err == NGX_ENOSYS) { use_accept4 = 0; ngx_inherited_nonblocking = 0; continue; } #else ngx_log_error(level, ev->log, err, "accept() failed"); #endif if (err == NGX_ECONNABORTED) { if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } if (ev->available) { continue; } } if (err == NGX_EMFILE || err == NGX_ENFILE) { if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle, 1) != NGX_OK) { return; } if (ngx_use_accept_mutex) { if (ngx_accept_mutex_held) { ngx_shmtx_unlock(&ngx_accept_mutex); ngx_accept_mutex_held = 0; } //当前进程连接accpet失败,则可以暂时设置为1,下次来的时候由其他进程竞争accpet锁,下下次该进程继续竞争该accept,因为在下次的时候ngx_process_events_and_timers //ngx_accept_disabled = 1; 减去1后为0,可以继续竞争 ngx_accept_disabled = 1; } else { ////如果是不需要实现负载均衡,则扫尾延时下继续在ngx_process_events_and_timers中accept ngx_add_timer(ev, ecf->accept_mutex_delay, NGX_FUNC_LINE); } } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif //设置负载均衡阀值 ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; //判断可用连接的数目和总数目的八分之一大小,如果可用的小于八分之一,为正 //在服务器端accept客户端连接成功(ngx_event_accept)后,会通过ngx_get_connection从连接池获取一个ngx_connection_t结构,也就是每个客户端连接对于一个ngx_connection_t结构, //并且为其分配一个ngx_http_connection_t结构,ngx_connection_t->data = ngx_http_connection_t,见ngx_http_init_connection //从连接池中获取一个空闲ngx_connection_t,用于客户端连接建立成功后向该连接读写数据,函数形参中的ngx_event_t对应的是为accept事件对应的 //ngx_connection_t中对应的event c = ngx_get_connection(s, ev->log); //ngx_get_connection中c->fd = s; //注意,这里的ngx_connection_t是从连接池中从新获取的,和ngx_epoll_process_events中的ngx_connection_t是两个不同的。 if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif c->pool = ngx_create_pool(ls->pool_size, ev->log); if (c->pool == NULL) { ngx_close_accepted_connection(c); return; } c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); return; } ngx_memcpy(c->sockaddr, sa, socklen); log = ngx_palloc(c->pool, sizeof(ngx_log_t)); if (log == NULL) { ngx_close_accepted_connection(c); return; } /* set a blocking mode for iocp and non-blocking mode for others */ if (ngx_inherited_nonblocking) { if (ngx_event_flags & NGX_USE_IOCP_EVENT) { if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_blocking_n " failed"); ngx_close_accepted_connection(c); return; } } } else { if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_nonblocking_n " failed"); ngx_close_accepted_connection(c); return; } } } *log = ls->log; c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->log = log; c->pool->log = log; c->socklen = socklen; c->listening = ls; c->local_sockaddr = ls->sockaddr; c->local_socklen = ls->socklen; c->unexpected_eof = 1; #if (NGX_HAVE_UNIX_DOMAIN) if (c->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } #endif //注意,这里的ngx_connection_t是从连接池中从新获取的,和ngx_epoll_process_events中的ngx_connection_t是两个不同的。 rev = c->read; wev = c->write; wev->ready = 1; if (ngx_event_flags & NGX_USE_IOCP_EVENT) { rev->ready = 1; } if (ev->deferred_accept) { rev->ready = 1; #if (NGX_HAVE_KQUEUE) rev->available = 1; #endif } rev->log = log; wev->log = log; /* * TODO: MT: - ngx_atomic_fetch_add() * or protection by critical section or light mutex * * TODO: MP: - allocated in a shared memory * - ngx_atomic_fetch_add() * or protection by critical section or light mutex */ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_handled, 1); #endif if (ls->addr_ntop) { c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len); if (c->addr_text.data == NULL) { ngx_close_accepted_connection(c); return; } c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen, c->addr_text.data, ls->addr_text_max_len, 0); if (c->addr_text.len == 0) { ngx_close_accepted_connection(c); return; } } #if (NGX_DEBUG) { ngx_str_t addr; struct sockaddr_in *sin; ngx_cidr_t *cidr; ngx_uint_t i; u_char text[NGX_SOCKADDR_STRLEN]; #if (NGX_HAVE_INET6) struct sockaddr_in6 *sin6; ngx_uint_t n; #endif cidr = ecf->debug_connection.elts; for (i = 0; i < ecf->debug_connection.nelts; i++) { if (cidr[i].family != (ngx_uint_t) c->sockaddr->sa_family) { goto next; } switch (cidr[i].family) { #if (NGX_HAVE_INET6) case AF_INET6: sin6 = (struct sockaddr_in6 *) c->sockaddr; for (n = 0; n < 16; n++) { if ((sin6->sin6_addr.s6_addr[n] & cidr[i].u.in6.mask.s6_addr[n]) != cidr[i].u.in6.addr.s6_addr[n]) { goto next; } } break; #endif #if (NGX_HAVE_UNIX_DOMAIN) case AF_UNIX: break; #endif default: /* AF_INET */ sin = (struct sockaddr_in *) c->sockaddr; if ((sin->sin_addr.s_addr & cidr[i].u.in.mask) != cidr[i].u.in.addr) { goto next; } break; } log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL; break; next: continue; } if (log->log_level & NGX_LOG_DEBUG_EVENT) { addr.data = text; addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text, NGX_SOCKADDR_STRLEN, 1); ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA accept: %V fd:%d", c->number, &addr, s); } } #endif if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) { //如果是epoll,不会走到这里面去 if (ngx_add_conn(c) == NGX_ERROR) { ngx_close_accepted_connection(c); return; } } log->data = NULL; log->handler = NULL; ls->handler(c);//ngx_http_init_connection if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } } while (ev->available); //一次性读取所有当前的accept,直到accept返回NGX_EAGAIN,然后退出 }
void ngx_event_accept(ngx_event_t *ev) { socklen_t socklen; ngx_err_t err; ngx_log_t *log; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_listening_t *ls; ngx_connection_t *c, *lc; ngx_event_conf_t *ecf; u_char sa[NGX_SOCKADDRLEN]; ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (ngx_event_flags & NGX_USE_RTSIG_EVENT) { ev->available = 1; } else if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { ev->available = ecf->multi_accept; } lc = ev->data; ls = lc->listening; ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "accept on %V, ready: %d", &ls->addr_text, ev->available); do { socklen = NGX_SOCKADDRLEN; s = accept(lc->fd, (struct sockaddr *) sa, &socklen); if (s == -1) { err = ngx_socket_errno; if (err == NGX_EAGAIN) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "accept() not ready"); return; } ngx_log_error((ngx_uint_t) ((err == NGX_ECONNABORTED) ? NGX_LOG_ERR : NGX_LOG_ALERT), ev->log, err, "accept() failed"); if (err == NGX_ECONNABORTED) { if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } if (ev->available) { continue; } } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; c = ngx_get_connection(s, ev->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif c->pool = ngx_create_pool(ls->pool_size, ev->log); if (c->pool == NULL) { ngx_close_accepted_connection(c); return; } c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); return; } ngx_memcpy(c->sockaddr, sa, socklen); log = ngx_palloc(c->pool, sizeof(ngx_log_t)); if (log == NULL) { ngx_close_accepted_connection(c); return; } /* set a blocking mode for aio and non-blocking mode for others */ if (ngx_inherited_nonblocking) { if (ngx_event_flags & NGX_USE_AIO_EVENT) { if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_blocking_n " failed"); ngx_close_accepted_connection(c); return; } } } else { if (!(ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_RTSIG_EVENT))) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_nonblocking_n " failed"); ngx_close_accepted_connection(c); return; } } } *log = ls->log; c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->log = log; c->pool->log = log; c->socklen = socklen; c->listening = ls; c->local_sockaddr = ls->sockaddr; c->unexpected_eof = 1; #if (NGX_HAVE_UNIX_DOMAIN) if (c->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } #endif rev = c->read; wev = c->write; wev->ready = 1; if (ngx_event_flags & (NGX_USE_AIO_EVENT|NGX_USE_RTSIG_EVENT)) { /* rtsig, aio, iocp */ rev->ready = 1; } if (ev->deferred_accept) { rev->ready = 1; #if (NGX_HAVE_KQUEUE) rev->available = 1; #endif } rev->log = log; wev->log = log; /* * TODO: MT: - ngx_atomic_fetch_add() * or protection by critical section or light mutex * * TODO: MP: - allocated in a shared memory * - ngx_atomic_fetch_add() * or protection by critical section or light mutex */ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_handled, 1); #endif #if (NGX_THREADS) rev->lock = &c->lock; wev->lock = &c->lock; rev->own_lock = &c->lock; wev->own_lock = &c->lock; #endif if (ls->addr_ntop) { c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len); if (c->addr_text.data == NULL) { ngx_close_accepted_connection(c); return; } c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->addr_text.data, ls->addr_text_max_len, 0); if (c->addr_text.len == 0) { ngx_close_accepted_connection(c); return; } } #if (NGX_DEBUG) { in_addr_t i; ngx_event_debug_t *dc; struct sockaddr_in *sin; sin = (struct sockaddr_in *) sa; dc = ecf->debug_connection.elts; for (i = 0; i < ecf->debug_connection.nelts; i++) { if ((sin->sin_addr.s_addr & dc[i].mask) == dc[i].addr) { log->log_level = NGX_LOG_DEBUG_CONNECTION|NGX_LOG_DEBUG_ALL; break; } } } #endif ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0, "*%d accept: %V fd:%d", c->number, &c->addr_text, s); if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) { if (ngx_add_conn(c) == NGX_ERROR) { ngx_close_accepted_connection(c); return; } } log->data = NULL; log->handler = NULL; ls->handler(c); if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } } while (ev->available); }
ngx_int_t ngx_event_connect_peer(ngx_peer_connection_t *pc) { int rc; ngx_int_t event; ngx_err_t err; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_connection_t *c; rc = pc->get(pc, pc->data);//ngx_http_upstream_get_round_robin_peer获取一个peer if (rc != NGX_OK) { return rc; } s = ngx_socket(pc->sockaddr->sa_family, SOCK_STREAM, 0); ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, 0, "socket %d", s); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_socket_n " failed"); return NGX_ERROR; } /* 由于Nginx的事件框架要求每个连接都由一个ngx_connection-t结构体来承载,因此这一步将调用ngx_get_connection方法,由ngx_cycle_t 核心结构体中free_connections指向的空闲连接池处获取到一个ngx_connection_t结构体,作为承载Nginx与上游服务器间的TCP连接 */ c = ngx_get_connection(s, pc->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_close_socket_n "failed"); } return NGX_ERROR; } if (pc->rcvbuf) { if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, (const void *) &pc->rcvbuf, sizeof(int)) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, "setsockopt(SO_RCVBUF) failed"); goto failed; } } if (ngx_nonblocking(s) == -1) { //建立一个TCP套接字,同时,这个套接字需要设置为非阻塞模式。 ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_nonblocking_n " failed"); goto failed; } if (pc->local) { if (bind(s, pc->local->sockaddr, pc->local->socklen) == -1) { ngx_log_error(NGX_LOG_CRIT, pc->log, ngx_socket_errno, "bind(%V) failed", &pc->local->name); goto failed; } } c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; /* 和后端的ngx_connection_t在ngx_event_connect_peer这里置为1,但在ngx_http_upstream_connect中c->sendfile &= r->connection->sendfile;, 和客户端浏览器的ngx_connextion_t的sendfile需要在ngx_http_update_location_config中判断,因此最终是由是否在configure的时候是否有加 sendfile选项来决定是置1还是置0 */ c->sendfile = 1; c->log_error = pc->log_error; if (pc->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } rev = c->read; wev = c->write; rev->log = pc->log; wev->log = pc->log; pc->connection = c; c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); /* 事件模块的ngx_event_actions接口,其中的add_conn方法可以将TCP套接字以期待可读、可写事件的方式添加到事件搜集器中。对于 epoll事件模块来说,add_conn方法就是把套接字以期待EPOLLIN EPOLLOUT事件的方式加入epoll中,这一步即调用add_conn方法把刚刚 建立的套接字添加到epoll中,表示如果这个套接字上出现了预期的网络事件,则希望epoll能够回调它的handler方法。 */ if (ngx_add_conn) { /* 将这个连接ngx_connection t上的读/写事件的handler回调方法都设置为ngx_http_upstream_handler。,见函数外层的ngx_http_upstream_connect */ if (ngx_add_conn(c) == NGX_ERROR) { goto failed; } } ngx_log_debug3(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connect to %V, fd:%d #%uA", pc->name, s, c->number); /* 调用connect方法向上游服务器发起TCP连接,作为非阻塞套接字,connect方法可能立刻返回连接建立成功,也可能告诉用户继续等待上游服务器的响应 对connect连接是否建立成功的检查会在函数外面的u->read_event_handler = ngx_http_upstream_process_header;见函数外层的ngx_http_upstream_connect */ /* 针对非阻塞I/O执行的系统调用则总是立即返回,而不管事件足否已经发生。如果事件没有眭即发生,这些系统调用就 返回—1.和出错的情况一样。此时我们必须根据errno来区分这两种情况。对accept、send和recv而言,事件未发牛时errno 通常被设置成EAGAIN(意为“再来一次”)或者EWOULDBLOCK(意为“期待阻塞”):对conncct而言,errno则被 设置成EINPROGRESS(意为“在处理中")。 */ //connect的时候返回成功后使用的sock就是socket创建的sock,这和服务器端accept成功返回一个新的sock不一样 //上面的ngx_add_conn已经把读写事件一起添加到了epoll中 rc = connect(s, pc->sockaddr, pc->socklen); //connect返回值可以参考<linux高性能服务器开发> 9.5节 if (rc == -1) { err = ngx_socket_errno; if (err != NGX_EINPROGRESS #if (NGX_WIN32) /* Winsock returns WSAEWOULDBLOCK (NGX_EAGAIN) */ && err != NGX_EAGAIN #endif ) { if (err == NGX_ECONNREFUSED #if (NGX_LINUX) /* * Linux returns EAGAIN instead of ECONNREFUSED * for unix sockets if listen queue is full */ || err == NGX_EAGAIN #endif || err == NGX_ECONNRESET || err == NGX_ENETDOWN || err == NGX_ENETUNREACH || err == NGX_EHOSTDOWN || err == NGX_EHOSTUNREACH) { level = NGX_LOG_ERR; } else { level = NGX_LOG_CRIT; } ngx_log_error(level, c->log, err, "connect() to %V failed", pc->name); ngx_close_connection(c); pc->connection = NULL; return NGX_DECLINED; } } if (ngx_add_conn) { if (rc == -1) { //这个表示发出了连接三步握手中的SYN,单还没有等待对方完全应答回来表示连接成功通过外层的 //c->write->handler = ngx_http_upstream_handler; u->write_event_handler = ngx_http_upstream_send_request_handler促发返回成功 /* NGX_EINPROGRESS */ return NGX_AGAIN; } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_IOCP_EVENT) { ngx_log_debug1(NGX_LOG_DEBUG_EVENT, pc->log, ngx_socket_errno, "connect(): %d", rc); if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, pc->log, ngx_socket_errno, ngx_blocking_n " failed"); goto failed; } /* * FreeBSD's aio allows to post an operation on non-connected socket. * NT does not support it. * * TODO: check in Win32, etc. As workaround we can use NGX_ONESHOT_EVENT */ rev->ready = 1; wev->ready = 1; return NGX_OK; } if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { /* kqueue */ event = NGX_CLEAR_EVENT; } else { /* select, poll, /dev/poll */ event = NGX_LEVEL_EVENT; } char tmpbuf[256]; snprintf(tmpbuf, sizeof(tmpbuf), "<%25s, %5d> epoll NGX_READ_EVENT(et) read add", NGX_FUNC_LINE); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, tmpbuf); if (ngx_add_event(rev, NGX_READ_EVENT, event) != NGX_OK) { goto failed; } if (rc == -1) { /* NGX_EINPROGRESS */ char tmpbuf[256]; snprintf(tmpbuf, sizeof(tmpbuf), "<%25s, %5d> epoll NGX_WRITE_EVENT(et) read add", NGX_FUNC_LINE); ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, tmpbuf); if (ngx_add_event(wev, NGX_WRITE_EVENT, event) != NGX_OK) { goto failed; } return NGX_AGAIN; //这个表示发出了连接三步握手中的SYN,单还没有等待对方完全应答回来表示连接成功 //通过外层的 c->write->handler = ngx_http_upstream_handler; u->write_event_handler = ngx_http_upstream_send_request_handler促发返回成功 } ngx_log_debug0(NGX_LOG_DEBUG_EVENT, pc->log, 0, "connected"); wev->ready = 1; return NGX_OK; failed: ngx_close_connection(c); pc->connection = NULL; return NGX_ERROR; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) {//ngx_start_worker_processes-> NGX_PROCESS_RESPAWN ,表示要重新创建进程 s = respawn; } else {//得到当前有多少工作进程创建了ngx_last_process会不断增加的,当然可能有人死了 for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) {//NGX_PROCESS_DETACHED说明是热代码替换 //不是热代码替换 /* Solaris 9 still has no AF_LOCAL */ //创建socketpair,进程之间通信 if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1){ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); //将socket对设置为非阻塞模式 if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1;//FIOASYNC 改变O_ASYNC 标志来打开或者关闭套接字的异步 IO 模式。 if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //close on exec, not on-fork, 意为如果对描述符设置了FD_CLOEXEC,使用execl执行的程序里, //此描述符被关闭,不能再使用它,但是在使用fork调用的子进程中,此描述符并不关闭,仍可使用。 if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } //设置fork对socket对是在子进程依然可用的 if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else {//如果是热代码替换,不跟其他进程通信 ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s;//当前在处理这个 pid = fork();//fork子进程 switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0://子进程调用proc ngx_pid = ngx_getpid();//k getpid proc(cycle, data);//等于ngx_worker_process_cycle //子进程永远也从这回不来了,里面有exit break; default://父进程默认不干活 break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } //设置ngx_processes数组该项的信息 ngx_processes[s].proc = proc;//设置工作进程的处理函数ngx_worker_process_cycle,会fork之后调用的 ngx_processes[s].data = data; ngx_processes[s].name = name;//进程名字 ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++;//递增总进程数 } return pid; }
void ngx_event_accept(ngx_event_t *ev) { socklen_t socklen; ngx_err_t err; ngx_log_t *log; ngx_uint_t level; ngx_socket_t s; ngx_event_t *rev, *wev; ngx_listening_t *ls; ngx_connection_t *c, *lc; ngx_event_conf_t *ecf; u_char sa[NGX_SOCKADDRLEN]; #if (NGX_HAVE_ACCEPT4) static ngx_uint_t use_accept4 = 1; #endif if (ev->timedout) { if (ngx_enable_accept_events((ngx_cycle_t *) ngx_cycle) != NGX_OK) { return; } ev->timedout = 0; } ecf = ngx_event_get_conf(ngx_cycle->conf_ctx, ngx_event_core_module); if (!(ngx_event_flags & NGX_USE_KQUEUE_EVENT)) { ev->available = ecf->multi_accept; } lc = ev->data; ls = lc->listening; ev->ready = 0; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, "accept on %V, ready: %d", &ls->addr_text, ev->available); do { socklen = NGX_SOCKADDRLEN; #if (NGX_HAVE_ACCEPT4) if (use_accept4) { s = accept4(lc->fd, (struct sockaddr *) sa, &socklen, SOCK_NONBLOCK); } else { s = accept(lc->fd, (struct sockaddr *) sa, &socklen); } #else s = accept(lc->fd, (struct sockaddr *) sa, &socklen); #endif if (s == (ngx_socket_t) -1) { err = ngx_socket_errno; if (err == NGX_EAGAIN) { ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, err, "accept() not ready"); return; } level = NGX_LOG_ALERT; if (err == NGX_ECONNABORTED) { level = NGX_LOG_ERR; } else if (err == NGX_EMFILE || err == NGX_ENFILE) { level = NGX_LOG_CRIT; } #if (NGX_HAVE_ACCEPT4) ngx_log_error(level, ev->log, err, use_accept4 ? "accept4() failed" : "accept() failed"); if (use_accept4 && err == NGX_ENOSYS) { use_accept4 = 0; ngx_inherited_nonblocking = 0; continue; } #else ngx_log_error(level, ev->log, err, "accept() failed"); #endif if (err == NGX_ECONNABORTED) { if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } if (ev->available) { continue; } } if (err == NGX_EMFILE || err == NGX_ENFILE) { if (ngx_disable_accept_events((ngx_cycle_t *) ngx_cycle, 1) != NGX_OK) { return; } if (ngx_use_accept_mutex) { if (ngx_accept_mutex_held) { ngx_shmtx_unlock(&ngx_accept_mutex); ngx_accept_mutex_held = 0; } ngx_accept_disabled = 1; } else { ngx_add_timer(ev, ecf->accept_mutex_delay); } } return; } #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; c = ngx_get_connection(s, ev->log); if (c == NULL) { if (ngx_close_socket(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_close_socket_n " failed"); } return; } c->type = SOCK_STREAM; #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_active, 1); #endif c->pool = ngx_create_pool(ls->pool_size, ev->log); if (c->pool == NULL) { ngx_close_accepted_connection(c); return; } c->sockaddr = ngx_palloc(c->pool, socklen); if (c->sockaddr == NULL) { ngx_close_accepted_connection(c); return; } ngx_memcpy(c->sockaddr, sa, socklen); log = ngx_palloc(c->pool, sizeof(ngx_log_t)); if (log == NULL) { ngx_close_accepted_connection(c); return; } /* set a blocking mode for iocp and non-blocking mode for others */ if (ngx_inherited_nonblocking) { if (ngx_event_flags & NGX_USE_IOCP_EVENT) { if (ngx_blocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_blocking_n " failed"); ngx_close_accepted_connection(c); return; } } } else { if (!(ngx_event_flags & NGX_USE_IOCP_EVENT)) { if (ngx_nonblocking(s) == -1) { ngx_log_error(NGX_LOG_ALERT, ev->log, ngx_socket_errno, ngx_nonblocking_n " failed"); ngx_close_accepted_connection(c); return; } } } *log = ls->log; c->recv = ngx_recv; c->send = ngx_send; c->recv_chain = ngx_recv_chain; c->send_chain = ngx_send_chain; c->log = log; c->pool->log = log; c->socklen = socklen; c->listening = ls; c->local_sockaddr = ls->sockaddr; c->local_socklen = ls->socklen; c->unexpected_eof = 1; #if (NGX_HAVE_UNIX_DOMAIN) if (c->sockaddr->sa_family == AF_UNIX) { c->tcp_nopush = NGX_TCP_NOPUSH_DISABLED; c->tcp_nodelay = NGX_TCP_NODELAY_DISABLED; #if (NGX_SOLARIS) /* Solaris's sendfilev() supports AF_NCA, AF_INET, and AF_INET6 */ c->sendfile = 0; #endif } #endif rev = c->read; wev = c->write; wev->ready = 1; if (ngx_event_flags & NGX_USE_IOCP_EVENT) { rev->ready = 1; } if (ev->deferred_accept) { rev->ready = 1; #if (NGX_HAVE_KQUEUE) rev->available = 1; #endif } rev->log = log; wev->log = log; /* * TODO: MT: - ngx_atomic_fetch_add() * or protection by critical section or light mutex * * TODO: MP: - allocated in a shared memory * - ngx_atomic_fetch_add() * or protection by critical section or light mutex */ c->number = ngx_atomic_fetch_add(ngx_connection_counter, 1); #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_handled, 1); #endif if (ls->addr_ntop) { c->addr_text.data = ngx_pnalloc(c->pool, ls->addr_text_max_len); if (c->addr_text.data == NULL) { ngx_close_accepted_connection(c); return; } c->addr_text.len = ngx_sock_ntop(c->sockaddr, c->socklen, c->addr_text.data, ls->addr_text_max_len, 0); if (c->addr_text.len == 0) { ngx_close_accepted_connection(c); return; } } #if (NGX_DEBUG) { ngx_str_t addr; u_char text[NGX_SOCKADDR_STRLEN]; ngx_debug_accepted_connection(ecf, c); if (log->log_level & NGX_LOG_DEBUG_EVENT) { addr.data = text; addr.len = ngx_sock_ntop(c->sockaddr, c->socklen, text, NGX_SOCKADDR_STRLEN, 1); ngx_log_debug3(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA accept: %V fd:%d", c->number, &addr, s); } } #endif if (ngx_add_conn && (ngx_event_flags & NGX_USE_EPOLL_EVENT) == 0) { if (ngx_add_conn(c) == NGX_ERROR) { ngx_close_accepted_connection(c); return; } } log->data = NULL; log->handler = NULL; ls->handler(c); if (ngx_event_flags & NGX_USE_KQUEUE_EVENT) { ev->available--; } } while (ev->available); }
ngx_int_t ngx_http_push_init_ipc(ngx_cycle_t *cycle, ngx_int_t workers) { //initialize socketpairs for workers in advance. static int invalid_sockets_initialized = 0; int i, s = 0, on = 1; ngx_int_t last_expected_process = ngx_last_process; if(!invalid_sockets_initialized) { for(i=0; i< NGX_MAX_PROCESSES; i++) { ngx_http_push_socketpairs[i][0]=NGX_INVALID_FILE; ngx_http_push_socketpairs[i][1]=NGX_INVALID_FILE; } invalid_sockets_initialized=1; } /* here's the deal: we have no control over fork()ing, nginx's internal * socketpairs are unusable for our purposes (as of nginx 0.8 -- check the * code to see why), and the module initialization callbacks occur before * any workers are spawned. Rather than futzing around with existing * socketpairs, we populate our own socketpairs array. * Trouble is, ngx_spawn_process() creates them one-by-one, and we need to * do it all at once. So we must guess all the workers' ngx_process_slots in * advance. Meaning the spawning logic must be copied to the T. * ... with some allowances for already-opened sockets... */ for(i=0; i < workers; i++) { //copypasta from os/unix/ngx_process.c (ngx_spawn_process) while (s < last_expected_process && ngx_processes[s].pid != -1) { //find empty existing slot s++; } ngx_socket_t *socks = ngx_http_push_socketpairs[s]; if(socks[0] == NGX_INVALID_FILE || socks[1] == NGX_INVALID_FILE) { //copypasta from os/unix/ngx_process.c (ngx_spawn_process) if (socketpair(AF_UNIX, SOCK_STREAM, 0, socks) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed on socketpair while initializing push module"); return NGX_ERROR; } if (ngx_nonblocking(socks[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (ngx_nonblocking(socks[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (ioctl(socks[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed on socketpair while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } if (fcntl(socks[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while initializing push module"); ngx_close_channel(socks, cycle->log); return NGX_ERROR; } } s++; //NEXT!! } return NGX_OK; }
ngx_pid_t ngx_spawn_process(ngx_cycle_t *cycle, ngx_spawn_proc_pt proc, void *data, char *name, ngx_int_t respawn) { u_long on; ngx_pid_t pid; ngx_int_t s; if (respawn >= 0) { s = respawn; } else { for (s = 0; s < ngx_last_process; s++) { if (ngx_processes[s].pid == -1) { break; } } if (s == NGX_MAX_PROCESSES) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "no more than %d processes can be spawned", NGX_MAX_PROCESSES); return NGX_INVALID_PID; } } if (respawn != NGX_PROCESS_DETACHED) { /* Solaris 9 still has no AF_LOCAL */ if (socketpair(AF_UNIX, SOCK_STREAM, 0, ngx_processes[s].channel) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "socketpair() failed while spawning \"%s\"", name); return NGX_INVALID_PID; } ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, "channel %d:%d", ngx_processes[s].channel[0], ngx_processes[s].channel[1]); if (ngx_nonblocking(ngx_processes[s].channel[0]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (ngx_nonblocking(ngx_processes[s].channel[1]) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, ngx_nonblocking_n " failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } on = 1; if (ioctl(ngx_processes[s].channel[0], FIOASYNC, &on) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "ioctl(FIOASYNC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETOWN, ngx_pid) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(F_SETOWN) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[0], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } if (fcntl(ngx_processes[s].channel[1], F_SETFD, FD_CLOEXEC) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fcntl(FD_CLOEXEC) failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; } ngx_channel = ngx_processes[s].channel[1]; } else { ngx_processes[s].channel[0] = -1; ngx_processes[s].channel[1] = -1; } ngx_process_slot = s; pid = fork(); switch (pid) { case -1: ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "fork() failed while spawning \"%s\"", name); ngx_close_channel(ngx_processes[s].channel, cycle->log); return NGX_INVALID_PID; case 0: ngx_pid = ngx_getpid(); proc(cycle, data); break; default: break; } ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "start %s %P", name, pid); ngx_processes[s].pid = pid; ngx_processes[s].exited = 0; if (respawn >= 0) { return pid; } ngx_processes[s].proc = proc; ngx_processes[s].data = data; ngx_processes[s].name = name; ngx_processes[s].exiting = 0; switch (respawn) { case NGX_PROCESS_NORESPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_SPAWN: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 0; break; case NGX_PROCESS_JUST_RESPAWN: ngx_processes[s].respawn = 1; ngx_processes[s].just_spawn = 1; ngx_processes[s].detached = 0; break; case NGX_PROCESS_DETACHED: ngx_processes[s].respawn = 0; ngx_processes[s].just_spawn = 0; ngx_processes[s].detached = 1; break; } if (s == ngx_last_process) { ngx_last_process++; } return pid; }