ngx_int_t
ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
ngx_int_t rc;
ngx_uint_t i, n;
ngx_http_upstream_rr_peer_t *peer;
ngx_http_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get rr peer, try: %ui", pc->tries);
pc->cached = 0;
pc->connection = NULL;
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
if (peers->single) {
peer = peers->peer;
if (peer->down) {
goto failed;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
goto failed;
}
rrp->current = peer;
} else {
/* there are several peers */
peer = ngx_http_upstream_get_peer(rrp);
if (peer == NULL) {
goto failed;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get rr peer, current: %p %i",
peer, peer->current_weight);
}
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
rc = ngx_http_upstream_get_round_robin_peer(pc, rrp);
if (rc != NGX_BUSY) {
return rc;
}
ngx_http_upstream_rr_peers_wlock(peers);
}
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
static ngx_int_t
ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
time_t now;
ngx_int_t w;
uintptr_t m;
ngx_uint_t i, n, p, hash;
ngx_http_upstream_rr_peer_t *peer;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, try: %ui", pc->tries);
/* TODO: cached */
ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers);
if (iphp->tries > 20 || iphp->rrp.peers->single) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
now = ngx_time();
pc->cached = 0;
pc->connection = NULL;
hash = iphp->hash;
for ( ;; ) {
for (i = 0; i < (ngx_uint_t) iphp->addrlen; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
w = hash % iphp->rrp.peers->total_weight;
peer = iphp->rrp.peers->peer;
p = 0;
while (w >= peer->weight) {
w -= peer->weight;
peer = peer->next;
p++;
}
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
if (iphp->rrp.tried[n] & m) {
goto next;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, hash: %ui %04XL", p, (uint64_t) m);
if (peer->down) {
goto next;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
goto next;
}
if (peer->max_conns && peer->conns >= peer->max_conns) {
goto next;
}
break;
next:
if (++iphp->tries > 20) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
}
iphp->rrp.current = peer;
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
}
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
iphp->rrp.tried[n] |= m;
iphp->hash = hash;
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_ip_hash_peer_data_t *iphp = data;
//本来data指向的是ngx_http_upstream_ip_hash_peer_data_t->rrp,因为rrp是该结构中的第一个成员，因此也就直接可以获取该结构，所以rrp必须是第一个成员
time_t now;
ngx_int_t w;
uintptr_t m;
ngx_uint_t i, n, p, hash;
ngx_http_upstream_rr_peer_t *peer;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, try: %ui", pc->tries);
/* TODO: cached */
ngx_http_upstream_rr_peers_wlock(iphp->rrp.peers);
//如果失败次数太多，或者只有一个后端服务，那么直接做RR选择
if (iphp->tries > 20 || iphp->rrp.peers->single) {
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
now = ngx_time();
pc->cached = 0;
pc->connection = NULL;
hash = iphp->hash;
for ( ;; ) {
//计算IP的hash值
/*
1）由IP计算哈希值的算法如下， 其中公式中hash初始值为89，iphp->addr[i]表示客户端的IP， 通过三次哈希计算得出一个IP的哈希值：
for (i = 0; i < 3; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
2）在选择下一个server时，ip_hash的选择策略是这样的：
它在上一次哈希值的基础上，再次哈希，就会得到一个全新的哈希值，再根据哈希值选择另外一个后台的服务器。
哈希算法仍然是
for (i = 0; i < 3; i++) {
hash = (hash * 113 + iphp->addr[i]) % 6271;
}
*/
for (i = 0; i < (ngx_uint_t) iphp->addrlen; i++) { //iphp->hash默认89，如果是同一个客户端来的请求，则下面计算出的hash肯定相同
//113质数，可以让哈希结果更散列
hash = (hash * 113 + iphp->addr[i]) % 6271; //根据IP地址的前三位和
}
w = hash % iphp->rrp.peers->total_weight;
peer = iphp->rrp.peers->peer;
p = 0;
//根据哈希结果得到被选中的后端服务器
while (w >= peer->weight) {
w -= peer->weight;
peer = peer->next;
p++;
}
//服务器对应在位图中的位置计算
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
if (iphp->rrp.tried[n] & m) {
goto next;
}
ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get ip hash peer, hash: %ui %04XA", p, m);
if (peer->down) {
goto next;
}
/*
fail_timeout时间内访问后端出现错误的次数大于等于max_fails，则认为该服务器不可用，那么如果不可用了，后端该服务器有恢复了怎么判断检测呢?
答:当这个fail_timeout时间段过了后，会重置peer->checked，那么有可以试探该服务器了，参考ngx_http_upstream_get_peer
//checked用来检测时间，例如某个时间段fail_timeout这段时间后端失效了，那么这个fail_timeout过了后，也可以试探使用该服务器
*/
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout) //失败次数已达上限
{
goto next;
}
break;
next:
/*
在这种ip_hash策略，如果一个后台服务器不能提供提服务（连接超时或读超时），该服务器的失败次数就会加一，当一个服务器的失败次
数达到max_fails所设置的值，就会在fail_timeout所设置的时间段内不能对外提供服务，这点和RR是一致的。
如果当前server不能提供服务，就会根据当前的哈希值再哈希出一个新哈希值，选择另一个服务器继续尝试，尝试的最大次是upstream中
server的个数，如果server的个数超过20，也就是要最大尝试次数在20次以上，当尝试次数达到20次，仍然找不到一个合适的服务器，
ip_hah策略不再尝试ip哈希值来选择server, 而在剩余的尝试中，它会转而使用RR的策略，使用轮循的方法，选择新的server。
*/
if (++iphp->tries > 20) {//已经尝试了20个后端服务器都还没找到一个可用的服务器，则直接在剩余的服务器中采用轮询算法
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
return iphp->get_rr_peer(pc, &iphp->rrp);
}
}
//当前服务索引
iphp->rrp.current = peer;
//服务器地址及名字保存
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
if (now - peer->checked > peer->fail_timeout) {
peer->checked = now;
}
ngx_http_upstream_rr_peers_unlock(iphp->rrp.peers);
iphp->rrp.tried[n] |= m; //位图更新
iphp->hash = hash;//保留种子，使下次get_ip_hash_peer的时候能够选到同一个peer上
return NGX_OK;
}
static ngx_int_t
ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_http_upstream_rr_peer_data_t *rrp = data;
time_t now;
uintptr_t m;
ngx_int_t rc, total;
ngx_uint_t i, n, p, many;
ngx_http_upstream_rr_peer_t *peer, *best;
ngx_http_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, try: %ui", pc->tries);
if (rrp->peers->single) {
return ngx_http_upstream_get_round_robin_peer(pc, rrp);
}
pc->cached = 0;
pc->connection = NULL;
now = ngx_time();
peers = rrp->peers;
ngx_http_upstream_rr_peers_wlock(peers);
best = NULL;
total = 0;
#if (NGX_SUPPRESS_WARN)
many = 0;
p = 0;
#endif
for (peer = peers->peer, i = 0;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m) {
continue;
}
if (peer->down) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
/*
* select peer with least number of connections; if there are
* multiple peers with the same number of connections, select
* based on round-robin
*/
if (best == NULL
|| peer->conns * best->weight < best->conns * peer->weight)
{
best = peer;
many = 0;
p = i;
} else if (peer->conns * best->weight == best->conns * peer->weight) {
many = 1;
}
}
if (best == NULL) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, no peer found");
goto failed;
}
if (many) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, many");
for (peer = best, i = p;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m) {
continue;
}
if (peer->down) {
continue;
}
if (peer->conns * best->weight != best->conns * peer->weight) {
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
peer->current_weight += peer->effective_weight;
total += peer->effective_weight;
if (peer->effective_weight < peer->weight) {
peer->effective_weight++;
}
if (peer->current_weight > best->current_weight) {
best = peer;
p = i;
}
}
}
best->current_weight -= total;
if (now - best->checked > best->fail_timeout) {
best->checked = now;
}
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
pc->name = &best->name;
best->conns++;
rrp->current = best;
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
rrp->tried[n] |= m;
ngx_http_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0,
"get least conn peer, backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
rc = ngx_http_upstream_get_least_conn_peer(pc, rrp);
if (rc != NGX_BUSY) {
return rc;
}
ngx_http_upstream_rr_peers_wlock(peers);
}
/* all peers failed, mark them as live for quick recovery */
for (peer = peers->peer; peer; peer = peer->next) {
peer->fails = 0;
}
ngx_http_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
