static void
ngx_tcp_check_clean_event(ngx_tcp_check_peer_conf_t *peer_conf)
{
ngx_connection_t *c;
c = peer_conf->pc.connection;
ngx_log_debug2(NGX_LOG_DEBUG_TCP, c->log, 0,
"tcp check clean event: index:%d, fd: %d",
peer_conf->index, c->fd);
ngx_close_connection(c);
if (peer_conf->check_timeout_ev.timer_set) {
ngx_del_timer(&peer_conf->check_timeout_ev);
}
peer_conf->state = NGX_TCP_CHECK_ALL_DONE;
if (peer_conf->check_data != NULL && peer_conf->reinit) {
peer_conf->reinit(peer_conf);
}
ngx_spinlock(&peer_conf->shm->lock, ngx_pid, 1024);
peer_conf->shm->owner = NGX_INVALID_PID;
ngx_spinlock_unlock(&peer_conf->shm->lock);
}
void
ngx_tcp_check_get_peer(ngx_uint_t index)
{
ngx_tcp_check_peer_conf_t *peer_conf;
if (check_peers_ctx == NULL || index >= check_peers_ctx->peers.nelts) {
return;
}
peer_conf = check_peers_ctx->peers.elts;
ngx_spinlock(&peer_conf[index].shm->lock, ngx_pid, 1024);
peer_conf[index].shm->business++;
peer_conf[index].shm->access_count++;
ngx_spinlock_unlock(&peer_conf[index].shm->lock);
}
void
ngx_tcp_check_free_peer(ngx_uint_t index)
{
ngx_tcp_check_peer_conf_t *peer_conf;
if (check_peers_ctx == NULL || index >= check_peers_ctx->peers.nelts) {
return;
}
peer_conf = check_peers_ctx->peers.elts;
ngx_spinlock(&peer_conf[index].shm->lock, ngx_pid, 1024);
if (peer_conf[index].shm->business > 0) {
peer_conf[index].shm->business--;
}
ngx_spinlock_unlock(&peer_conf[index].shm->lock);
}
//任务处理完后，epoll的通知读事件会调用该函数
////ngx_notify通告主线程，该任务处理完毕，ngx_thread_pool_handler由主线程执行，也就是进程cycle{}通过epoll_wait返回执行，而不是由线程池中的线程执行
static void
ngx_thread_pool_handler(ngx_event_t *ev)
{
ngx_event_t *event;
ngx_thread_task_t *task;
ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "thread pool handler");
ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
/* 这里是不是有问题?
如果线程池中的线程执行任务比较快，而主进程在执行epoll_wait过程中有点阻塞，那么就检测不到ngx_notify中的epoll事件，有可能下次检测到该事件的时候
ngx_thread_pool_done上已经积累了很多执行完的任务事件，见ngx_thread_pool_cycle。
单这里好像只取了队列首部的任务啊?????? 队首外的任务丢弃了???????????不对吧
答案是，这里面所有的任务都在下面的while{}中得到了执行
*/
task = ngx_thread_pool_done.first;
ngx_thread_pool_done.first = NULL;
//尾部指向头，但是头已经变为空，即不执行任务
ngx_thread_pool_done.last = &ngx_thread_pool_done.first;
ngx_unlock(&ngx_thread_pool_done_lock);
while (task) {//遍历执行前面队列ngx_thread_pool_done中的每一个任务
ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
"run completion handler for task #%ui", task->id);
event = &task->event;
task = task->next;
event->complete = 1;
event->active = 0;
/*如果是小文件，则一次可以读完，函数指向可以参考ngx_http_cache_thread_handler ngx_http_copy_thread_handler ngx_thread_read
如果是大文件下载，则第一次走这里函数式上面的几个函数，但是由于一次最多获取32768字节，因此需要多次读取文件，就是由一次tread执行完任务后
触发ngx_notify通道epoll，然后走到这里继续读
*/
event->handler(event);//这里是否应该检查event->handler是否为空，例如参考ngx_thread_pool_destroy
}
}
static void
ngx_tcp_check_begin_handler(ngx_event_t *event)
{
ngx_tcp_check_peer_conf_t *peer_conf;
ngx_tcp_upstream_srv_conf_t *uscf;
if (ngx_tcp_check_need_exit()) {
return;
}
peer_conf = event->data;
uscf = peer_conf->conf;
ngx_add_timer(event, uscf->check_interval/2);
/* This process are processing the event now. */
if (peer_conf->shm->owner == ngx_pid) {
return;
}
ngx_log_debug4(NGX_LOG_DEBUG_TCP, event->log, 0,
"tcp check begin handler index:%ud, owner: %d, "
"ngx_pid: %ud, time:%d",
peer_conf->index, peer_conf->shm->owner, ngx_pid,
(ngx_current_msec - peer_conf->shm->access_time));
ngx_spinlock(&peer_conf->shm->lock, ngx_pid, 1024);
if (((ngx_current_msec - peer_conf->shm->access_time) >= uscf->check_interval) &&
peer_conf->shm->owner == NGX_INVALID_PID)
{
peer_conf->shm->owner = ngx_pid;
}
ngx_spinlock_unlock(&peer_conf->shm->lock);
if (peer_conf->shm->owner == ngx_pid) {
ngx_tcp_check_connect_handler(event);
}
}
static void ngx_http_healthcheck_try_for_ownership(ngx_event_t *event) {
ngx_http_healthcheck_status_t * stat;
ngx_int_t i_own_it;
stat = event->data;
if (ngx_terminate || ngx_exiting || ngx_quit) {
ngx_http_healthcheck_clear_events(stat->health_ev.log);
return;
}
i_own_it = 0;
// nxg_time_update(0, 0);
// Spinlock. So don't own for a long time!
// Use spinlock so two worker processes don't try to healthcheck the same
// peer
ngx_spinlock(&stat->shm->lock, ngx_pid, 1024);
if (stat->shm->owner == ngx_pid) {
i_own_it = 1;
} else if (ngx_current_msec - stat->shm->action_time >=
(stat->conf->health_delay + stat->conf->health_timeout) * 3) {
stat->shm->owner = ngx_pid;
stat->shm->action_time = ngx_current_msec;
stat->state = NGX_HEALTH_WAITING;
ngx_http_healthcheck_begin_healthcheck(&stat->health_ev);
i_own_it = 1;
}
if (!ngx_atomic_cmp_set(&stat->shm->lock, ngx_pid, 0)) {
ngx_log_error(NGX_LOG_CRIT, event->log, 0,
"healthcheck: spinlock didn't work. Should be %P, but isn't",
ngx_pid);
stat->shm->lock = 0;
}
if (!i_own_it) {
// Try again for ownership later in case the guy that DOES own it dies or
// something
ngx_add_timer(&stat->ownership_ev, 5000);
}
}
static void
ngx_thread_pool_handler(ngx_event_t *ev)
{
ngx_event_t *event;
ngx_thread_task_t *task;
ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "thread pool handler");
ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
task = ngx_thread_pool_done.first;
ngx_thread_pool_done.first = NULL;
ngx_thread_pool_done.last = &ngx_thread_pool_done.first;
ngx_unlock(&ngx_thread_pool_done_lock);
while (task)
{
ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0,
"run completion handler for task #%ui", task->id);
event = &task->event;
task = task->next;
event->complete = 1;
event->active = 0;
event->handler(event);
}
}
static void
ngx_http_check_begin_handler(ngx_event_t *event)
{
ngx_msec_t interval;
ngx_http_check_peer_t *peer;
ngx_http_check_peers_t *peers;
ngx_http_check_peers_shm_t *peers_shm;
ngx_http_upstream_check_srv_conf_t *ucscf;
if (ngx_http_check_need_exit()) {
return;
}
peers = check_peers_ctx;
if (peers == NULL) {
return;
}
peers_shm = peers->peers_shm;
if (peers_shm == NULL) {
return;
}
peer = event->data;
ucscf = peer->conf;
ngx_add_timer(event, ucscf->check_interval/2);
/* This process is processing this peer now. */
if ((peer->shm->owner == ngx_pid) ||
(peer->pc.connection != NULL) ||
(peer->check_timeout_ev.timer_set)) {
return;
}
interval = ngx_current_msec - peer->shm->access_time;
ngx_log_debug5(NGX_LOG_DEBUG_HTTP, event->log, 0,
"http check begin handler index: %ud, owner: %P, "
"ngx_pid: %P, interval: %M, check_interval: %M",
peer->index, peer->shm->owner,
ngx_pid, interval,
ucscf->check_interval);
ngx_spinlock(&peer->shm->lock, ngx_pid, 1024);
if (peers_shm->generation != ngx_http_check_shm_generation) {
ngx_spinlock_unlock(&peer->shm->lock);
return;
}
if ((interval >= ucscf->check_interval)
&& peer->shm->owner == NGX_INVALID_PID)
{
peer->shm->owner = ngx_pid;
}
else if (interval >= (ucscf->check_interval << 4)) {
/* If the check peer has been untouched for 4 times of
* the check interval, activates current timer.
* The checking process may be disappeared
* in some circumstance, and the clean event will never
* be triggered. */
peer->shm->owner = ngx_pid;
peer->shm->access_time = ngx_current_msec;
}
ngx_spinlock_unlock(&peer->shm->lock);
if (peer->shm->owner == ngx_pid) {
ngx_http_check_connect_handler(event);
}
}
static void *
ngx_thread_pool_cycle(void *data)
{
ngx_thread_pool_t *tp = data;
int err;
sigset_t set;
ngx_thread_task_t *task;
#if 0
ngx_time_update();
#endif
ngx_log_debug1(NGX_LOG_DEBUG_CORE, tp->log, 0,
"thread in pool \"%V\" started", &tp->name);
sigfillset(&set);
sigdelset(&set, SIGILL);
sigdelset(&set, SIGFPE);
sigdelset(&set, SIGSEGV);
sigdelset(&set, SIGBUS);
err = pthread_sigmask(SIG_BLOCK, &set, NULL);
if (err)
{
ngx_log_error(NGX_LOG_ALERT, tp->log, err, "pthread_sigmask() failed");
return NULL;
}
for (;;)
{
if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK)
{
return NULL;
}
/* the number may become negative */
tp->waiting--;
while (tp->queue.first == NULL)
{
if (ngx_thread_cond_wait(&tp->cond, &tp->mtx, tp->log)
!= NGX_OK)
{
(void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
return NULL;
}
}
task = tp->queue.first;
tp->queue.first = task->next;
if (tp->queue.first == NULL)
{
tp->queue.last = &tp->queue.first;
}
if (ngx_thread_mutex_unlock(&tp->mtx, tp->log) != NGX_OK)
{
return NULL;
}
#if 0
ngx_time_update();
#endif
ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
"run task #%ui in thread pool \"%V\"",
task->id, &tp->name);
task->handler(task->ctx, tp->log);
ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
"complete task #%ui in thread pool \"%V\"",
task->id, &tp->name);
task->next = NULL;
ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
*ngx_thread_pool_done.last = task;
ngx_thread_pool_done.last = &task->next;
ngx_unlock(&ngx_thread_pool_done_lock);
(void) ngx_notify(ngx_thread_pool_handler);
}
}
//ngx_thread_pool_cycle和ngx_thread_task_post配合阅读
static void *
ngx_thread_pool_cycle(void *data)
{
ngx_thread_pool_t *tp = data; //一个该结构对应一个threads_pool配置
int err;
sigset_t set;
ngx_thread_task_t *task;
#if 0
ngx_time_update();
#endif
ngx_log_debug1(NGX_LOG_DEBUG_CORE, tp->log, 0,
"thread in pool \"%V\" started", &tp->name);
sigfillset(&set);
sigdelset(&set, SIGILL);
sigdelset(&set, SIGFPE);
sigdelset(&set, SIGSEGV);
sigdelset(&set, SIGBUS);
err = pthread_sigmask(SIG_BLOCK, &set, NULL); //
if (err) {
ngx_log_error(NGX_LOG_ALERT, tp->log, err, "pthread_sigmask() failed");
return NULL;
}
/*
一般主线程通过ngx_thread_task_post添加任务，线程队列中的线程执行任务，主线程号和进程号一样，线程队列中的线程和进程号不一样，例如
2016/01/07 12:38:01[ ngx_thread_task_post, 280][yangya [debug] 20090#20090: ngx add task to thread, task id:183
20090#20090前面是进程号，后面是主线程号，他们相同
*/
for ( ;; ) {//一次任务执行完后又会走到这里，循环
if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) {
return NULL;
}
/* the number may become negative */
tp->waiting--;
//如果队列中有任务，则直接执行任务，不会在while中等待conf signal
while (tp->queue.first == NULL) { //此时任务队列为空，在条件变量上等待 配合ngx_thread_task_post阅读
//在添加任务的时候唤醒ngx_thread_task_post -> ngx_thread_cond_signal
if (ngx_thread_cond_wait(&tp->cond, &tp->mtx, tp->log) //等待ngx_thread_cond_signal后才会返回
!= NGX_OK)
{
(void) ngx_thread_mutex_unlock(&tp->mtx, tp->log);
return NULL;
}
}
//取出队首任务，然后执行
task = tp->queue.first;
tp->queue.first = task->next;
if (tp->queue.first == NULL) {
tp->queue.last = &tp->queue.first;
}
//这一段加锁是因为线程池是共享资源，多个线程都从队列中取线程，并且主线程会添加任务到队列中
if (ngx_thread_mutex_unlock(&tp->mtx, tp->log) != NGX_OK) {
return NULL;
}
#if 0
ngx_time_update();
#endif
ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
"run task #%ui in thread pool name:\"%V\"",
task->id, &tp->name);
task->handler(task->ctx, tp->log); //每个任务有各自的ctx,因此这里不需要加锁
ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0,
"complete task #%ui in thread pool name: \"%V\"",
task->id, &tp->name);
task->next = NULL;
ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048);
//task添加到队列尾部，同时可以保证多次添加的时候，让新task和以前的task形成一个还，first执行第一个task，last指向最后一个task
*ngx_thread_pool_done.last = task;
ngx_thread_pool_done.last = &task->next;
ngx_unlock(&ngx_thread_pool_done_lock);
//ngx_notify通告主线程，该任务处理完毕，ngx_thread_pool_handler由主线程执行，也就是进程cycle{}通过epoll_wait返回执行，而不是由线程池中的线程执行
(void) ngx_notify(ngx_thread_pool_handler);
}
}