static void rwl_lock_mutex(ngx_rwlock_t *lock) { #if (NGX_HAVE_ATOMIC_OPS) ngx_atomic_t *mutex = &lock->mutex; ngx_uint_t i, n; for(;;) { if(*mutex == 0 && ngx_atomic_cmp_set(mutex, 0, ngx_pid)) { return; } if(ngx_ncpu > 1) { for(n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) { for(i = 0; i < n; i++) { ngx_cpu_pause(); } #if (DEBUG_NGX_RWLOCK) ngx_log_error(NGX_LOG_WARN, ngx_cycle->log, 0, "rwlock %p mutex wait", lock); #endif if(*mutex == 0 && ngx_atomic_cmp_set(mutex, 0, ngx_pid)) { return; } } } ngx_sched_yield(); } #else #if (NGX_THREADS) #error ngx_spinlock() or ngx_atomic_cmp_set() are not defined ! #endif #endif }
void ngx_rwlock_wlock(ngx_atomic_t *lock) { ngx_uint_t i, n; for ( ;; ) { if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK)) { return; } if (ngx_ncpu > 1) { for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) { for (i = 0; i < n; i++) { ngx_cpu_pause(); } if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, NGX_RWLOCK_WLOCK)) { return; } } } ngx_sched_yield(); } }
void ngx_spinlock(ngx_atomic_t *lock, ngx_uint_t spin) { #if (NGX_HAVE_ATOMIC_OPS) ngx_uint_t tries; tries = 0; for ( ;; ) { if (*lock) { if (ngx_ncpu > 1 && tries++ < spin) { continue; } ngx_sched_yield(); tries = 0; } else { if (ngx_atomic_cmp_set(lock, 0, 1)) { return; } } } #else #endif }
static void ngx_shmtx_wakeup(ngx_shmtx_t *mtx) { #if (NGX_HAVE_POSIX_SEM) ngx_atomic_uint_t wait; if (!mtx->semaphore) { return; } for ( ;; ) { wait = *mtx->wait; if ((ngx_atomic_int_t) wait <= 0) { return; } if (ngx_atomic_cmp_set(mtx->wait, wait, wait - 1)) { break; } } ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx wake %uA", wait); if (sem_post(&mtx->sem) == -1) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, ngx_errno, "sem_post() failed while wake shmtx"); } #endif }
static void rwl_unlock_mutex(ngx_rwlock_t *lock) { #if (NGX_HAVE_ATOMIC_OPS) ngx_atomic_cmp_set(&lock->mutex, ngx_pid, 0); #else #if (NGX_THREADS) #error ngx_spinlock() or ngx_atomic_cmp_set() are not defined ! #endif #endif }
void ngx_spinlock(ngx_atomic_t *lock, ngx_atomic_int_t value, ngx_uint_t spin) { #if (NGX_HAVE_ATOMIC_OPS) ngx_uint_t i, n; for ( ;; ) { if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, value)) { return; } if (ngx_ncpu > 1) { for (n = 1; n < spin; n <<= 1) { for (i = 0; i < n; i++) { ngx_cpu_pause(); } if (*lock == 0 && ngx_atomic_cmp_set(lock, 0, value)) { return; } } } ngx_sched_yield(); } #else #if (NGX_THREADS) #error ngx_spinlock() or ngx_atomic_cmp_set() are not defined ! #endif #endif }
void ngx_shmtx_unlock(ngx_shmtx_t *mtx) { if (mtx->spin != (ngx_uint_t) -1) { ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx unlock"); } if (ngx_atomic_cmp_set(mtx->lock, ngx_pid, 0)) { ngx_shmtx_wakeup(mtx); } }
ngx_uint_t ngx_shmtx_force_unlock(ngx_shmtx_t *mtx, ngx_pid_t pid) { ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx forced unlock"); if (ngx_atomic_cmp_set(mtx->lock, pid, 0)) { ngx_shmtx_wakeup(mtx); return 1; } return 0; }
void ngx_rwlock_rlock(ngx_atomic_t *lock) { ngx_uint_t i, n; ngx_atomic_uint_t readers; for ( ;; ) { readers = *lock; if (readers != NGX_RWLOCK_WLOCK && ngx_atomic_cmp_set(lock, readers, readers + 1)) { return; } if (ngx_ncpu > 1) { for (n = 1; n < NGX_RWLOCK_SPIN; n <<= 1) { for (i = 0; i < n; i++) { ngx_cpu_pause(); } readers = *lock; if (readers != NGX_RWLOCK_WLOCK && ngx_atomic_cmp_set(lock, readers, readers + 1)) { return; } } } ngx_sched_yield(); } }
static void ngx_http_file_cache_loader(void *data) { ngx_http_file_cache_t *cache = data; ngx_tree_ctx_t tree; // ??? if (!cache->sh->cold || cache->sh->loading) { return; } if (!ngx_atomic_cmp_set(&cache->sh->loading, 0, ngx_pid)) { return; } ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ngx_cycle->log, 0, "http file cache loader"); tree.init_handler = NULL; tree.file_handler = ngx_http_file_cache_manage_file; tree.pre_tree_handler = ngx_http_file_cache_noop; tree.post_tree_handler = ngx_http_file_cache_noop; tree.spec_handler = ngx_http_file_cache_delete_file; tree.data = cache; tree.alloc = 0; tree.log = ngx_cycle->log; cache->last = ngx_current_msec; // 更新缓存最后被访问的时间 cache->files = 0; if (ngx_walk_tree(&tree, &cache->path->name) == NGX_ABORT) { cache->sh->loading = 0; return; } cache->sh->cold = 0; cache->sh->loading = 0; ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "http file cache: %V %.3fM, bsize: %uz", &cache->path->name, ((double) cache->sh->size * cache->bsize) / (1024 * 1024), cache->bsize); }
void ngx_rwlock_unlock(ngx_atomic_t *lock) { ngx_atomic_uint_t readers; readers = *lock; if (readers == NGX_RWLOCK_WLOCK) { *lock = 0; return; } for ( ;; ) { if (ngx_atomic_cmp_set(lock, readers, readers - 1)) { return; } readers = *lock; } }
static void ngx_http_healthcheck_try_for_ownership(ngx_event_t *event) { ngx_http_healthcheck_status_t * stat; ngx_int_t i_own_it; stat = event->data; if (ngx_terminate || ngx_exiting || ngx_quit) { ngx_http_healthcheck_clear_events(stat->health_ev.log); return; } i_own_it = 0; // nxg_time_update(0, 0); // Spinlock. So don't own for a long time! // Use spinlock so two worker processes don't try to healthcheck the same // peer ngx_spinlock(&stat->shm->lock, ngx_pid, 1024); if (stat->shm->owner == ngx_pid) { i_own_it = 1; } else if (ngx_current_msec - stat->shm->action_time >= (stat->conf->health_delay + stat->conf->health_timeout) * 3) { stat->shm->owner = ngx_pid; stat->shm->action_time = ngx_current_msec; stat->state = NGX_HEALTH_WAITING; ngx_http_healthcheck_begin_healthcheck(&stat->health_ev); i_own_it = 1; } if (!ngx_atomic_cmp_set(&stat->shm->lock, ngx_pid, 0)) { ngx_log_error(NGX_LOG_CRIT, event->log, 0, "healthcheck: spinlock didn't work. Should be %P, but isn't", ngx_pid); stat->shm->lock = 0; } if (!i_own_it) { // Try again for ownership later in case the guy that DOES own it dies or // something ngx_add_timer(&stat->ownership_ev, 5000); } }
void _spinlock(ngx_atomic_t *lock) { ngx_int_t tries; tries = 0; for ( ;; ) { if (*lock) { if (ngx_ncpu > 1 && tries++ < 1000) { continue; } sched_yield(); tries = 0; } else { if (ngx_atomic_cmp_set(lock, 0, 1)) { return; } } } }
static void ngx_process_get_status(void) { int status; char *process; ngx_pid_t pid; ngx_err_t err; ngx_int_t i; ngx_uint_t one; one = 0; for ( ;; ) { pid = waitpid(-1, &status, WNOHANG); if (pid == 0) { return; } if (pid == -1) { err = ngx_errno; if (err == NGX_EINTR) { continue; } if (err == NGX_ECHILD && one) { return; } #if (NGX_SOLARIS || NGX_FREEBSD) /* * Solaris always calls the signal handler for each exited process * despite waitpid() may be already called for this process. * * When several processes exit at the same time FreeBSD may * erroneously call the signal handler for exited process * despite waitpid() may be already called for this process. */ if (err == NGX_ECHILD) { ngx_log_error(NGX_LOG_INFO, ngx_cycle->log, 0, "waitpid() failed (%d: %s)", err, ngx_sigsafe_strerror(err)); return; } #endif ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "waitpid() failed (%d: %s)", err, ngx_sigsafe_strerror(err)); return; } if (ngx_accept_mutex_ptr) { /* * unlock the accept mutex if the abnormally exited process * held it */ ngx_atomic_cmp_set(ngx_accept_mutex_ptr, pid, 0); } one = 1; process = "unknown process"; for (i = 0; i < ngx_last_process; i++) { if (ngx_processes[i].pid == pid) { ngx_processes[i].status = status; ngx_processes[i].exited = 1; process = ngx_processes[i].name; break; } } if (WTERMSIG(status)) { #ifdef WCOREDUMP ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "%s %P exited on signal %d%s", process, pid, WTERMSIG(status), WCOREDUMP(status) ? " (core dumped)" : ""); #else ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "%s %P exited on signal %d", process, pid, WTERMSIG(status)); #endif } else { ngx_log_error(NGX_LOG_NOTICE, ngx_cycle->log, 0, "%s %P exited with code %d", process, pid, WEXITSTATUS(status)); } if (WEXITSTATUS(status) == 2 && ngx_processes[i].respawn) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "%s %P exited with fatal code %d " "and can not be respawn", process, pid, WEXITSTATUS(status)); ngx_processes[i].respawn = 0; } } }
static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle) { void ***cf; u_char *shared; size_t size, cl; ngx_shm_t shm; ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; cf = ngx_get_conf(cycle->conf_ctx, ngx_events_module); ecf = (*cf)[ngx_event_core_module.ctx_index]; if (!ngx_test_config && ngx_process <= NGX_PROCESS_MASTER) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "using the \"%s\" event method", ecf->name); } ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); ngx_timer_resolution = ccf->timer_resolution; #if !(NGX_WIN32) { ngx_int_t limit; struct rlimit rlmt; if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "getrlimit(RLIMIT_NOFILE) failed, ignored"); } else { if (ecf->connections > (ngx_uint_t) rlmt.rlim_cur && (ccf->rlimit_nofile == NGX_CONF_UNSET || ecf->connections > (ngx_uint_t) ccf->rlimit_nofile)) { limit = (ccf->rlimit_nofile == NGX_CONF_UNSET) ? (ngx_int_t) rlmt.rlim_cur : ccf->rlimit_nofile; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "%ui worker_connections exceed " "open file resource limit: %i", ecf->connections, limit); } } } #endif /* !(NGX_WIN32) */ if (ccf->master == 0) { return NGX_OK; } if (ngx_accept_mutex_ptr) { return NGX_OK; } /* cl should be equal to or greater than cache line size */ cl = 128; size = cl /* ngx_accept_mutex */ + cl /* ngx_connection_counter */ + cl; /* ngx_temp_number */ #if (NGX_STAT_STUB) size += cl /* ngx_stat_accepted */ + cl /* ngx_stat_handled */ + cl /* ngx_stat_requests */ + cl /* ngx_stat_active */ + cl /* ngx_stat_reading */ + cl /* ngx_stat_writing */ + cl; /* ngx_stat_waiting */ #endif shm.size = size; shm.name.len = sizeof("nginx_shared_zone"); shm.name.data = (u_char *) "nginx_shared_zone"; shm.log = cycle->log; if (ngx_shm_alloc(&shm) != NGX_OK) { return NGX_ERROR; } shared = shm.addr; ngx_accept_mutex_ptr = (ngx_atomic_t *) shared; ngx_accept_mutex.spin = (ngx_uint_t) -1; if (ngx_shmtx_create(&ngx_accept_mutex, (ngx_shmtx_sh_t *) shared, cycle->lock_file.data) != NGX_OK) { return NGX_ERROR; } ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %d", ngx_connection_counter, *ngx_connection_counter); ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); tp = ngx_timeofday(); ngx_random_number = (tp->msec << 16) + ngx_pid; #if (NGX_STAT_STUB) ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); ngx_stat_waiting = (ngx_atomic_t *) (shared + 9 * cl); #endif return NGX_OK; }
static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle) { void ***cf; u_char *shared; size_t size, cl; ngx_shm_t shm; ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; cf = ngx_get_conf(cycle->conf_ctx, ngx_events_module); if (cf == NULL) { ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "no \"events\" section in configuration"); return NGX_ERROR; } ecf = (*cf)[ngx_event_core_module.ctx_index]; if (!ngx_test_config && ngx_process <= NGX_PROCESS_MASTER) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "using the \"%s\" event method", ecf->name); } ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); ngx_timer_resolution = ccf->timer_resolution; #if !(NGX_WIN32) { ngx_int_t limit; struct rlimit rlmt; if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) { /* [analy] 获取进程能打开的最大文件数,内核默认是1024 */ ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "getrlimit(RLIMIT_NOFILE) failed, ignored"); } else { /* [analy] 1. 当worker_connections指定的连接数量 > 进程可以打开的文件个数 并且 指令worker_rlimit_nofile没有显示指定 2. 当worker_connections指定的连接数量 > 指令worker_rlimit_nofile指定的打开文件数 */ if (ecf->connections > (ngx_uint_t) rlmt.rlim_cur && (ccf->rlimit_nofile == NGX_CONF_UNSET || ecf->connections > (ngx_uint_t) ccf->rlimit_nofile)) { limit = (ccf->rlimit_nofile == NGX_CONF_UNSET) ? (ngx_int_t) rlmt.rlim_cur : ccf->rlimit_nofile; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "%ui worker_connections are more than " "open file resource limit: %i", ecf->connections, limit); } } } #endif /* !(NGX_WIN32) */ if (ccf->master == 0) { /* [analy] 单进程工作模式,开发和调试时使用;master_process = off */ return NGX_OK; } if (ngx_accept_mutex_ptr) { return NGX_OK; } /* cl should be equal or bigger than cache line size */ cl = 128; size = cl /* ngx_accept_mutex */ + cl /* ngx_connection_counter */ + cl; /* ngx_temp_number */ #if (NGX_STAT_STUB) size += cl /* ngx_stat_accepted */ + cl /* ngx_stat_handled */ + cl /* ngx_stat_requests */ + cl /* ngx_stat_active */ + cl /* ngx_stat_reading */ + cl; /* ngx_stat_writing */ #endif shm.size = size; shm.name.len = sizeof("nginx_shared_zone"); shm.name.data = (u_char *) "nginx_shared_zone"; shm.log = cycle->log; if (ngx_shm_alloc(&shm) != NGX_OK) { return NGX_ERROR; } shared = shm.addr; ngx_accept_mutex_ptr = (ngx_atomic_t *) shared; ngx_accept_mutex.spin = (ngx_uint_t) -1; /* 设置ngx_accept_mutex="nginx.lock.accept", cycle->lock_file被unlink后在目录下已经被移除, 但还存在磁盘中当调用close()时才会被真正删掉 */ if (ngx_shmtx_create(&ngx_accept_mutex, (ngx_shmtx_sh_t *) shared, cycle->lock_file.data) != NGX_OK) { return NGX_ERROR; } ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %d", ngx_connection_counter, *ngx_connection_counter); ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); tp = ngx_timeofday(); ngx_random_number = (tp->msec << 16) + ngx_pid; #if (NGX_STAT_STUB) ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); #endif return NGX_OK; }
// 在ngx_init_cycle里调用,fork子进程之前 // 创建共享内存,存放负载均衡锁和统计用的原子变量 static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle) { void ***cf; u_char *shared; size_t size, cl; ngx_shm_t shm; ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; // events模块的配置结构体 cf = ngx_get_conf(cycle->conf_ctx, ngx_events_module); // event_core模块的配置结构体 ecf = (*cf)[ngx_event_core_module.ctx_index]; if (!ngx_test_config && ngx_process <= NGX_PROCESS_MASTER) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "using the \"%s\" event method", ecf->name); } // core模块的配置结构体 ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); // 获取核心配置的时间精度,用在epoll里更新缓存时间 ngx_timer_resolution = ccf->timer_resolution; // unix专用代码, 可打开的最多文件描述符 #if !(NGX_WIN32) { ngx_int_t limit; struct rlimit rlmt; // 系统调用getrlimit,Linux内核对进程的限制 // RLIMIT_NOFILE,进程可打开的最大文件描述符数量,超出将产生EMFILE错误 if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) { // 系统调用失败则记录alert级别日志 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "getrlimit(RLIMIT_NOFILE) failed, ignored"); } else { // 成功获取内核参数 // // rlmt.rlim_cur是系统的软限制 // event里配置的连接数不能超过系统内核限制 // 或者是配置的rlimit_nofile限制 if (ecf->connections > (ngx_uint_t) rlmt.rlim_cur && (ccf->rlimit_nofile == NGX_CONF_UNSET || ecf->connections > (ngx_uint_t) ccf->rlimit_nofile)) { // 如果超过了报警告级别日志 // limit就是上限 limit = (ccf->rlimit_nofile == NGX_CONF_UNSET) ? (ngx_int_t) rlmt.rlim_cur : ccf->rlimit_nofile; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "%ui worker_connections exceed " "open file resource limit: %i", ecf->connections, limit); } } } #endif /* !(NGX_WIN32) */ // 如果非master/worker进程,即只启动一个进程,那么就没必要使用负载均衡锁 if (ccf->master == 0) { return NGX_OK; } // 已经有了负载均衡锁,已经初始化过了,就没必要再做操作 if (ngx_accept_mutex_ptr) { return NGX_OK; } /* cl should be equal to or greater than cache line size */ // cl是一个基本长度,可以容纳原子变量 cl = 128; // 最基本的三个:负载均衡锁,连接计数器, size = cl /* ngx_accept_mutex */ + cl /* ngx_connection_counter */ + cl; /* ngx_temp_number */ // 其他统计用的原子变量 #if (NGX_STAT_STUB) size += cl /* ngx_stat_accepted */ + cl /* ngx_stat_handled */ + cl /* ngx_stat_requests */ + cl /* ngx_stat_active */ + cl /* ngx_stat_reading */ + cl /* ngx_stat_writing */ + cl; /* ngx_stat_waiting */ #endif // 创建共享内存,存放负载均衡锁和统计用的原子变量 shm.size = size; shm.name.len = sizeof("nginx_shared_zone") - 1; shm.name.data = (u_char *) "nginx_shared_zone"; shm.log = cycle->log; if (ngx_shm_alloc(&shm) != NGX_OK) { return NGX_ERROR; } // shared是共享内存的地址指针 shared = shm.addr; // 第一个就是负载均衡锁 ngx_accept_mutex_ptr = (ngx_atomic_t *) shared; ngx_accept_mutex.spin = (ngx_uint_t) -1; if (ngx_shmtx_create(&ngx_accept_mutex, (ngx_shmtx_sh_t *) shared, cycle->lock_file.data) != NGX_OK) { return NGX_ERROR; } // 连接计数器 ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %uA", ngx_connection_counter, *ngx_connection_counter); // 临时文件用 ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); tp = ngx_timeofday(); // 随机数 ngx_random_number = (tp->msec << 16) + ngx_pid; #if (NGX_STAT_STUB) ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); ngx_stat_waiting = (ngx_atomic_t *) (shared + 9 * cl); #endif return NGX_OK; }
static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle) { void ***cf; u_char *shared; size_t size, cl; ngx_shm_t shm; //nginx多进程间的共享内存 ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; // 获取使用的事件模块,如IOCP,epool cf = ngx_get_conf(cycle->conf_ctx, ngx_events_module); if (cf == NULL) { ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "no \"events\" section in configuration"); return NGX_ERROR; } // 获取使用的事件模块ngx_event_core_module的配置 ecf = (*cf)[ngx_event_core_module.ctx_index]; if (!ngx_test_config && ngx_process <= NGX_PROCESS_MASTER) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "using the \"%s\" event method", ecf->name); } // 获取ngx_core_module模块的配置 ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); // 设置时间精度 ngx_timer_resolution = ccf->timer_resolution; #if !(NGX_WIN32) { ngx_int_t limit; struct rlimit rlmt; if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) { ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "getrlimit(RLIMIT_NOFILE) failed, ignored"); } else { if (ecf->connections > (ngx_uint_t) rlmt.rlim_cur // 检查连接上否已经超过上限 && (ccf->rlimit_nofile == NGX_CONF_UNSET || ecf->connections > (ngx_uint_t) ccf->rlimit_nofile)) { limit = (ccf->rlimit_nofile == NGX_CONF_UNSET) ? (ngx_int_t) rlmt.rlim_cur : ccf->rlimit_nofile; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "%ui worker_connections are more than " "open file resource limit: %i", ecf->connections, limit); } } } #endif /* !(NGX_WIN32) */ // 检查master进程是否关闭。如果master进程关闭了,就是单进程方式了。下面步骤是实现内存实现锁等,单进程不需要。 if (ccf->master == 0) { return NGX_OK; } if (ngx_accept_mutex_ptr) { // 已经存在accept互斥体了,不需要再重复创建 return NGX_OK; } /* cl should be equal or bigger than cache line size */ cl = 128; // 创建共享内存,用于accept mutex,connection counter和ngx_temp_number size = cl /* ngx_accept_mutex */ + cl /* ngx_connection_counter */ + cl; /* ngx_temp_number */ #if (NGX_STAT_STUB) size += cl /* ngx_stat_accepted */ + cl /* ngx_stat_handled */ + cl /* ngx_stat_requests */ + cl /* ngx_stat_active */ + cl /* ngx_stat_reading */ + cl; /* ngx_stat_writing */ #endif shm.size = size; shm.name.len = sizeof("nginx_shared_zone"); shm.name.data = (u_char *) "nginx_shared_zone"; shm.log = cycle->log; // 创建共享内存,共享内存的起始地址保存在shm.addr if (ngx_shm_alloc(&shm) != NGX_OK) { return NGX_ERROR; } // 获得共享内存的起始地址 shared = shm.addr; // accept互斥体取得共享内存的第一段cl大小内存 ngx_accept_mutex_ptr = (ngx_atomic_t *) shared; ngx_accept_mutex.spin = (ngx_uint_t) -1; // 系统支持原子数据则使用原子数据实现accept mutex,否则使用文件上锁实现 if (ngx_shmtx_create(&ngx_accept_mutex, shared, cycle->lock_file.data) != NGX_OK) { return NGX_ERROR; } // ngx_connection_counter指到共享内存那 ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %d", ngx_connection_counter, *ngx_connection_counter); ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); tp = ngx_timeofday(); //根据当前时间和进程ID,获取一个随机数 ngx_random_number = (tp->msec << 16) + ngx_pid; #if (NGX_STAT_STUB) ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); #endif return NGX_OK; }
int msg_refcount_invalidate_if_zero(nchan_msg_t *msg) { return ngx_atomic_cmp_set((ngx_atomic_uint_t *)&msg->refcount, 0, MSG_REFCOUNT_INVALID); }
/** * @param [in] mtx 锁对象 * @return 0|1 * 非阻塞尝试获取mtx锁 */ ngx_uint_t ngx_shmtx_trylock(ngx_shmtx_t *mtx) { return (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)); }
void ngx_shmtx_lock(ngx_shmtx_t *mtx) { ngx_uint_t i, n; ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx lock"); for ( ;; ) { if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) { return; } if (ngx_ncpu > 1) { for (n = 1; n < mtx->spin; n <<= 1) { for (i = 0; i < n; i++) { ngx_cpu_pause(); } if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) { return; } } } #if (NGX_HAVE_POSIX_SEM) if (mtx->semaphore) { (void) ngx_atomic_fetch_add(mtx->wait, 1); if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) { (void) ngx_atomic_fetch_add(mtx->wait, -1); return; } ngx_log_debug1(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx wait %uA", *mtx->wait); while (sem_wait(&mtx->sem) == -1) { ngx_err_t err; err = ngx_errno; if (err != NGX_EINTR) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, err, "sem_wait() failed while waiting on shmtx"); break; } } ngx_log_debug0(NGX_LOG_DEBUG_CORE, ngx_cycle->log, 0, "shmtx awoke"); continue; } #endif ngx_sched_yield(); } }
/* ngx_event_module_init方法其实很简单,它主要初始化了一些变量,尤其是ngx_http_stub_status_module统计模块使用的一些原子性的统计变量 */ static ngx_int_t ngx_event_module_init(ngx_cycle_t *cycle) { void ***cf; u_char *shared; size_t size, cl; ngx_shm_t shm; ngx_time_t *tp; ngx_core_conf_t *ccf; ngx_event_conf_t *ecf; cf = ngx_get_conf(cycle->conf_ctx, ngx_events_module); ecf = (*cf)[ngx_event_core_module.ctx_index]; if (!ngx_test_config && ngx_process <= NGX_PROCESS_MASTER) { ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, "using the \"%s\" event method", ecf->name); } ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); ngx_timer_resolution = ccf->timer_resolution; #if !(NGX_WIN32) { ngx_int_t limit; struct rlimit rlmt; if (getrlimit(RLIMIT_NOFILE, &rlmt) == -1) { // 每个进程能打开的最多文件数。 ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, "getrlimit(RLIMIT_NOFILE) failed, ignored"); } else { if (ecf->connections > (ngx_uint_t) rlmt.rlim_cur && (ccf->rlimit_nofile == NGX_CONF_UNSET || ecf->connections > (ngx_uint_t) ccf->rlimit_nofile)) { limit = (ccf->rlimit_nofile == NGX_CONF_UNSET) ? (ngx_int_t) rlmt.rlim_cur : ccf->rlimit_nofile; ngx_log_error(NGX_LOG_WARN, cycle->log, 0, "%ui worker_connections exceed " "open file resource limit: %i", ecf->connections, limit); } } } #endif /* !(NGX_WIN32) */ if (ccf->master == 0) { return NGX_OK; } if (ngx_accept_mutex_ptr) { return NGX_OK; } /* cl should be equal to or greater than cache line size */ /* 计算出需要使用的共享内存的大小。为什么每个统计成员需要使用128字节呢?这似乎太大了,看上去,每个ngx_atomic_t原子变量最多需要8字 节而已。其实是因为Nginx充分考虑了CPU的二级缓存。在目前许多CPU架构下缓存行的大小都是128字节,而下面需要统计的变量都是访问非常频 繁的成员,同时它们占用的内存又非常少,所以采用了每个成员都使用128字节存放的形式,这样速度更快 */ cl = 128; size = cl /* ngx_accept_mutex */ + cl /* ngx_connection_counter */ + cl; /* ngx_temp_number */ #if (NGX_STAT_STUB) size += cl /* ngx_stat_accepted */ + cl /* ngx_stat_handled */ + cl /* ngx_stat_requests */ + cl /* ngx_stat_active */ + cl /* ngx_stat_reading */ + cl /* ngx_stat_writing */ + cl; /* ngx_stat_waiting */ #endif shm.size = size; shm.name.len = sizeof("nginx_shared_zone") - 1; shm.name.data = (u_char *) "nginx_shared_zone"; shm.log = cycle->log; //开辟一块共享内存,共享内存的大小为shm.size if (ngx_shm_alloc(&shm) != NGX_OK) { return NGX_ERROR; } //共享内存的首地址就在shm.addr成员中 shared = shm.addr; //原子变量类型的accept锁使用了128字节的共享内存 ngx_accept_mutex_ptr = (ngx_atomic_t *) shared; /* ngx_accept_mutex就是负载均衡锁,spin值为-1则是告诉Nginx这把锁不可以使进程进入睡眠状态,详见14.8节 */ ngx_accept_mutex.spin = (ngx_uint_t) -1; if (ngx_shmtx_create(&ngx_accept_mutex, (ngx_shmtx_sh_t *) shared, cycle->lock_file.data) != NGX_OK) { return NGX_ERROR; } //原予变量类型的ngx_connection counter将统计所有建立过的连接数(包括主动发起的连接) ngx_connection_counter = (ngx_atomic_t *) (shared + 1 * cl); (void) ngx_atomic_cmp_set(ngx_connection_counter, 0, 1); ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "counter: %p, %d", ngx_connection_counter, *ngx_connection_counter); ngx_temp_number = (ngx_atomic_t *) (shared + 2 * cl); tp = ngx_timeofday(); ngx_random_number = (tp->msec << 16) + ngx_pid; #if (NGX_STAT_STUB) //依次初始化需要统计的6个原子变量,也就是使用共享内存作为原子变量 ngx_stat_accepted = (ngx_atomic_t *) (shared + 3 * cl); ngx_stat_handled = (ngx_atomic_t *) (shared + 4 * cl); ngx_stat_requests = (ngx_atomic_t *) (shared + 5 * cl); ngx_stat_active = (ngx_atomic_t *) (shared + 6 * cl); ngx_stat_reading = (ngx_atomic_t *) (shared + 7 * cl); ngx_stat_writing = (ngx_atomic_t *) (shared + 8 * cl); ngx_stat_waiting = (ngx_atomic_t *) (shared + 9 * cl); #endif return NGX_OK; }
void ngx_process_get_status() { int status; char *process; ngx_pid_t pid; ngx_err_t err; ngx_int_t i; ngx_uint_t one; struct timeval tv; one = 0; for ( ;; ) { pid = waitpid(-1, &status, WNOHANG); if (pid == 0) { return; } if (pid == -1) { err = ngx_errno; if (err == NGX_EINTR) { continue; } if (err == NGX_ECHILD && one) { return; } ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, errno, "waitpid() failed"); return; } if (ngx_accept_mutex_ptr) { /* * unlock the accept mutex if the abnormally exited process * held it */ ngx_atomic_cmp_set(ngx_accept_mutex_ptr, pid, 0); } one = 1; process = "unknown process"; for (i = 0; i < ngx_last_process; i++) { if (ngx_processes[i].pid == pid) { ngx_processes[i].status = status; ngx_processes[i].exited = 1; process = ngx_processes[i].name; break; } } if (WTERMSIG(status)) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "%s " PID_T_FMT " exited on signal %d%s", process, pid, WTERMSIG(status), WCOREDUMP(status) ? " (core dumped)" : ""); } else { ngx_log_error(NGX_LOG_INFO, ngx_cycle->log, 0, "%s " PID_T_FMT " exited with code %d", process, pid, WEXITSTATUS(status)); } if (WEXITSTATUS(status) == 2 && ngx_processes[i].respawn) { ngx_log_error(NGX_LOG_ALERT, ngx_cycle->log, 0, "%s " PID_T_FMT " exited with fatal code %d and could not respawn", process, pid, WEXITSTATUS(status)); ngx_processes[i].respawn = 0; } } }