ngx_int_t
ngx_http_vhost_traffic_status_display_get_upstream_nelts(ngx_http_request_t *r)
{
ngx_uint_t i, j, n;
ngx_http_upstream_server_t *us;
#if (NGX_HTTP_UPSTREAM_ZONE)
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
#endif
ngx_http_upstream_srv_conf_t *uscf, **uscfp;
ngx_http_upstream_main_conf_t *umcf;
umcf = ngx_http_get_module_main_conf(r, ngx_http_upstream_module);
uscfp = umcf->upstreams.elts;
for (i = 0, j = 0, n = 0; i < umcf->upstreams.nelts; i++) {
uscf = uscfp[i];
/* groups */
if (uscf->servers && !uscf->port) {
us = uscf->servers->elts;
#if (NGX_HTTP_UPSTREAM_ZONE)
if (uscf->shm_zone == NULL) {
goto not_supported;
}
peers = uscf->peer.data;
ngx_http_upstream_rr_peers_rlock(peers);
for (peer = peers->peer; peer; peer = peer->next) {
n++;
}
ngx_http_upstream_rr_peers_unlock(peers);
not_supported:
#endif
for (j = 0; j < uscf->servers->nelts; j++) {
n += us[j].naddrs;
}
}
}
return n;
}
void
ngx_http_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
ngx_ssl_session_t *old_ssl_session, *ssl_session;
ngx_http_upstream_rr_peer_t *peer;
#if (NGX_HTTP_UPSTREAM_ZONE)
int len;
u_char *p;
ngx_http_upstream_rr_peers_t *peers;
u_char buf[NGX_SSL_MAX_SESSION_SIZE];
#endif
#if (NGX_HTTP_UPSTREAM_ZONE)
peers = rrp->peers;
if (peers->shpool) {
ssl_session = SSL_get0_session(pc->connection->ssl->connection);
if (ssl_session == NULL) {
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"save session: %p", ssl_session);
len = i2d_SSL_SESSION(ssl_session, NULL);
/* do not cache too big session */
if (len > NGX_SSL_MAX_SESSION_SIZE) {
return;
}
p = buf;
(void) i2d_SSL_SESSION(ssl_session, &p);
peer = rrp->current;
ngx_http_upstream_rr_peers_rlock(peers);
ngx_http_upstream_rr_peer_lock(peers, peer);
if (len > peer->ssl_session_len) {
ngx_shmtx_lock(&peers->shpool->mutex);
if (peer->ssl_session) {
ngx_slab_free_locked(peers->shpool, peer->ssl_session);
}
peer->ssl_session = ngx_slab_alloc_locked(peers->shpool, len);
ngx_shmtx_unlock(&peers->shpool->mutex);
if (peer->ssl_session == NULL) {
peer->ssl_session_len = 0;
ngx_http_upstream_rr_peer_unlock(peers, peer);
ngx_http_upstream_rr_peers_unlock(peers);
return;
}
peer->ssl_session_len = len;
}
ngx_memcpy(peer->ssl_session, buf, len);
ngx_http_upstream_rr_peer_unlock(peers, peer);
ngx_http_upstream_rr_peers_unlock(peers);
return;
}
#endif
ssl_session = ngx_ssl_get_session(pc->connection);
if (ssl_session == NULL) {
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"save session: %p", ssl_session);
peer = rrp->current;
old_ssl_session = peer->ssl_session;
peer->ssl_session = ssl_session;
if (old_ssl_session) {
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"old session: %p", old_ssl_session);
/* TODO: may block */
ngx_ssl_free_session(old_ssl_session);
}
}
ngx_int_t
ngx_http_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
ngx_int_t rc;
ngx_ssl_session_t *ssl_session;
ngx_http_upstream_rr_peer_t *peer;
#if (NGX_HTTP_UPSTREAM_ZONE)
int len;
#if OPENSSL_VERSION_NUMBER >= 0x0090707fL
const
#endif
u_char *p;
ngx_http_upstream_rr_peers_t *peers;
u_char buf[NGX_SSL_MAX_SESSION_SIZE];
#endif
peer = rrp->current;
#if (NGX_HTTP_UPSTREAM_ZONE)
peers = rrp->peers;
if (peers->shpool) {
ngx_http_upstream_rr_peers_rlock(peers);
ngx_http_upstream_rr_peer_lock(peers, peer);
if (peer->ssl_session == NULL) {
ngx_http_upstream_rr_peer_unlock(peers, peer);
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
}
len = peer->ssl_session_len;
ngx_memcpy(buf, peer->ssl_session, len);
ngx_http_upstream_rr_peer_unlock(peers, peer);
ngx_http_upstream_rr_peers_unlock(peers);
p = buf;
ssl_session = d2i_SSL_SESSION(NULL, &p, len);
rc = ngx_ssl_set_session(pc->connection, ssl_session);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"set session: %p", ssl_session);
ngx_ssl_free_session(ssl_session);
return rc;
}
#endif
ssl_session = peer->ssl_session;
rc = ngx_ssl_set_session(pc->connection, ssl_session);
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"set session: %p", ssl_session);
return rc;
}
void
ngx_http_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
ngx_uint_t state)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
time_t now;
ngx_http_upstream_rr_peer_t *peer;
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"free rr peer %ui %ui", pc->tries, state);
/* TODO: NGX_PEER_KEEPALIVE */
peer = rrp->current;
ngx_http_upstream_rr_peers_rlock(rrp->peers);
ngx_http_upstream_rr_peer_lock(rrp->peers, peer);
if (rrp->peers->single) {
peer->conns--;
ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
ngx_http_upstream_rr_peers_unlock(rrp->peers);
pc->tries = 0;
return;
}
if (state & NGX_PEER_FAILED) {
now = ngx_time();
peer->fails++;
peer->accessed = now;
peer->checked = now;
if (peer->max_fails) {
peer->effective_weight -= peer->weight / peer->max_fails;
if (peer->fails >= peer->max_fails) {
ngx_log_error(NGX_LOG_WARN, pc->log, 0,
"upstream server temporarily disabled");
}
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"free rr peer failed: %p %i",
peer, peer->effective_weight);
if (peer->effective_weight < 0) {
peer->effective_weight = 0;
}
} else {
/* mark peer live if check passed */
if (peer->accessed < peer->checked) {
peer->fails = 0;
}
}
peer->conns--;
ngx_http_upstream_rr_peer_unlock(rrp->peers, peer);
ngx_http_upstream_rr_peers_unlock(rrp->peers);
if (pc->tries) {
pc->tries--;
}
}
ngx_int_t
ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
ngx_int_t rc;
ngx_uint_t i, n;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get rr peer, try: %ui", pc->tries);
pc->cached = 0;
pc->connection = NULL;
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
if (peers->single) {
peer = peers->peer;
if (peer->down) {
goto failed;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
goto failed;
}
rrp->current = peer;
} else {
/* there are several peers */
peer = ngx_http_upstream_get_peer(rrp);
if (peer == NULL) {
goto failed;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get rr peer, current: %p %i",
peer, peer->current_weight);
}
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
if (rc != NGX_BUSY) {
return rc;
}
ngx_http_upstream_rr_peers_wlock(peers);
}
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
static ngx_int_t
ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
//本来data指向的是ngx_http_upstream_ip_hash_peer_data_t->rrp,因为rrp是该结构中的第一个成员，因此也就直接可以获取该结构，所以rrp必须是第一个成员
time_t now;
ngx_int_t w;
uintptr_t m;
ngx_uint_t i, n, p, hash;
ngx_http_upstream_rr_peer_t *peer;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, try: %ui", pc->tries);
/* TODO: cached */
ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers);
//如果失败次数太多，或者只有一个后端服务，那么直接做RR选择
if (iphp->tries > 20 || iphp->rrp.peers->single) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
now = ngx_time();
pc->cached = 0;
pc->connection = NULL;
hash = iphp->hash;
for ( ;; ) {
//计算IP的hash值
/*
1）由IP计算哈希值的算法如下， 其中公式中hash初始值为89，iphp->addr[i]表示客户端的IP， 通过三次哈希计算得出一个IP的哈希值：
for (i = 0; i < 3; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
2）在选择下一个server时，ip_hash的选择策略是这样的：
它在上一次哈希值的基础上，再次哈希，就会得到一个全新的哈希值，再根据哈希值选择另外一个后台的服务器。
哈希算法仍然是
for (i = 0; i < 3; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
*/
for (i = 0; i < (ngx_uint_t) iphp->addrlen; i++) { //iphp->hash默认89，如果是同一个客户端来的请求，则下面计算出的hash肯定相同
//113质数，可以让哈希结果更散列
hash = (hash * 113 + iphp->addr[i]) % 6271; //根据IP地址的前三位和
}
w = hash % iphp->rrp.peers->total_weight;
peer = iphp->rrp.peers->peer;
p = 0;
//根据哈希结果得到被选中的后端服务器
while (w >= peer->weight) {
w -= peer->weight;
peer = peer->next;
p++;
}
//服务器对应在位图中的位置计算
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
if (iphp->rrp.tried[n] & m) {
goto next;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, hash: %ui %04XA", p, m);
if (peer->down) {
goto next;
}
/*
fail_timeout时间内访问后端出现错误的次数大于等于max_fails，则认为该服务器不可用，那么如果不可用了，后端该服务器有恢复了怎么判断检测呢?
答:当这个fail_timeout时间段过了后，会重置peer->checked，那么有可以试探该服务器了，参考ngx_http_upstream_get_peer
//checked用来检测时间，例如某个时间段fail_timeout这段时间后端失效了，那么这个fail_timeout过了后，也可以试探使用该服务器
*/
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout) //失败次数已达上限
{
goto next;
}
break;
next:
/*
在这种ip_hash策略，如果一个后台服务器不能提供提服务（连接超时或读超时），该服务器的失败次数就会加一，当一个服务器的失败次
数达到max_fails所设置的值，就会在fail_timeout所设置的时间段内不能对外提供服务，这点和RR是一致的。
如果当前server不能提供服务，就会根据当前的哈希值再哈希出一个新哈希值，选择另一个服务器继续尝试，尝试的最大次是upstream中
server的个数，如果server的个数超过20，也就是要最大尝试次数在20次以上，当尝试次数达到20次，仍然找不到一个合适的服务器，
ip_hah策略不再尝试ip哈希值来选择server, 而在剩余的尝试中，它会转而使用RR的策略，使用轮循的方法，选择新的server。
*/
if (++iphp->tries > 20) {//已经尝试了20个后端服务器都还没找到一个可用的服务器，则直接在剩余的服务器中采用轮询算法
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
}
//当前服务索引
iphp->rrp.current = peer;
//服务器地址及名字保存
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
}
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
iphp->rrp.tried[n] |= m; //位图更新
iphp->hash = hash;//保留种子，使下次get_ip_hash_peer的时候能够选到同一个peer上
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
time_t now;
ngx_int_t w;
uintptr_t m;
ngx_uint_t i, n, p, hash;
ngx_http_upstream_rr_peer_t *peer;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, try: %ui", pc->tries);
/* TODO: cached */
ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers);
if (iphp->tries > 20 || iphp->rrp.peers->single) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
now = ngx_time();
pc->cached = 0;
pc->connection = NULL;
hash = iphp->hash;
for ( ;; ) {
for (i = 0; i < (ngx_uint_t) iphp->addrlen; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
w = hash % iphp->rrp.peers->total_weight;
peer = iphp->rrp.peers->peer;
p = 0;
while (w >= peer->weight) {
w -= peer->weight;
peer = peer->next;
p++;
}
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
if (iphp->rrp.tried[n] & m) {
goto next;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, hash: %ui %04XL", p, (uint64_t) m);
if (peer->down) {
goto next;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
goto next;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
goto next;
}
break;
next:
if (++iphp->tries > 20) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
}
iphp->rrp.current = peer;
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
}
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
iphp->rrp.tried[n] |= m;
iphp->hash = hash;
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
time_t now;
uintptr_t m;
ngx_int_t rc, total;
ngx_uint_t i, n, p, many;
ngx_http_upstream_rr_peer_t *peer, *best;
ngx_http_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, try: %ui", pc->tries);
if (rrp->peers->single) {
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
pc->cached = 0;
pc->connection = NULL;
now = ngx_time();
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
best = NULL;
total = 0;
#if (NGX_SUPPRESS_WARN)
many = 0;
p = 0;
#endif
for (peer = peers->peer, i = 0;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m) {
continue;
}
if (peer->down) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
/*
* select peer with least number of connections; if there are
* multiple peers with the same number of connections, select
* based on round-robin
*/
if (best == NULL
|| peer->conns * best->weight < best->conns * peer->weight)
{
best = peer;
many = 0;
p = i;
} else if (peer->conns * best->weight == best->conns * peer->weight) {
many = 1;
}
}
if (best == NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, no peer found");
goto failed;
}
if (many) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, many");
for (peer = best, i = p;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m) {
continue;
}
if (peer->down) {
continue;
}
if (peer->conns * best->weight != best->conns * peer->weight) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
peer->current_weight += peer->effective_weight;
total += peer->effective_weight;
if (peer->effective_weight < peer->weight) {
peer->effective_weight++;
}
if (peer->current_weight > best->current_weight) {
best = peer;
p = i;
}
}
}
best->current_weight -= total;
if (now - best->checked > best->fail_timeout) {
best->checked = now;
}
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
pc->name = &best->name;
best->conns++;
rrp->current = best;
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
rrp->tried[n] |= m;
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
rc = ngx_http_upstream_get_least_conn_peer(pc, rrp);
if (rc != NGX_BUSY) {
return rc;
}
ngx_http_upstream_rr_peers_wlock(peers);
}
/* all peers failed, mark them as live for quick recovery */
for (peer = peers->peer; peer; peer = peer->next) {
peer->fails = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
