static ngx_int_t ngx_http_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data) { ngx_http_upstream_lc_peer_data_t *lcp = data; time_t now; uintptr_t m; ngx_int_t rc, total; ngx_uint_t i, n, p, many; ngx_http_upstream_rr_peer_t *peer, *best; ngx_http_upstream_rr_peers_t *peers; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, try: %ui", pc->tries); if (lcp->rrp.peers->single) { return lcp->get_rr_peer(pc, &lcp->rrp); } pc->cached = 0; pc->connection = NULL; now = ngx_time(); peers = lcp->rrp.peers; best = NULL; total = 0; #if (NGX_SUPPRESS_WARN) many = 0; p = 0; #endif for (i = 0; i < peers->number; i++) { n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); if (lcp->rrp.tried[n] & m) { continue; } peer = &peers->peer[i]; if (peer->down) { continue; } #if (NGX_HTTP_UPSTREAM_CHECK) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least_conn peer, check_index: %ui", peer->check_index); if (ngx_http_upstream_check_peer_down(peer->check_index)) { continue; } #endif if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) { continue; } /* * select peer with least number of connections; if there are * multiple peers with the same number of connections, select * based on round-robin */ if (best == NULL || lcp->conns[i] * best->weight < lcp->conns[p] * peer->weight) { best = peer; many = 0; p = i; } else if (lcp->conns[i] * best->weight == lcp->conns[p] * peer->weight) { many = 1; } } if (best == NULL) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, no peer found"); goto failed; } if (many) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, many"); for (i = p; i < peers->number; i++) { n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); if (lcp->rrp.tried[n] & m) { continue; } peer = &peers->peer[i]; if (peer->down) { continue; } #if (NGX_HTTP_UPSTREAM_CHECK) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least_conn peer, check_index: %ui", peer->check_index); if (ngx_http_upstream_check_peer_down(peer->check_index)) { continue; } #endif if (lcp->conns[i] * best->weight != lcp->conns[p] * peer->weight) { continue; } if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) { continue; } peer->current_weight += peer->effective_weight; total += peer->effective_weight; if (peer->effective_weight < peer->weight) { peer->effective_weight++; } if (peer->current_weight > best->current_weight) { best = peer; p = i; } } } best->current_weight -= total; best->checked = now; pc->sockaddr = best->sockaddr; pc->socklen = best->socklen; pc->name = &best->name; lcp->rrp.current = p; n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); lcp->rrp.tried[n] |= m; lcp->conns[p]++; if (pc->tries == 1 && peers->next) { pc->tries += peers->next->number; } return NGX_OK; failed: if (peers->next) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get least conn peer, backup servers"); lcp->conns += peers->number; lcp->rrp.peers = peers->next; pc->tries = lcp->rrp.peers->number; n = lcp->rrp.peers->number / (8 * sizeof(uintptr_t)) + 1; for (i = 0; i < n; i++) { lcp->rrp.tried[i] = 0; } rc = ngx_http_upstream_get_least_conn_peer(pc, lcp); if (rc != NGX_BUSY) { return rc; } } /* all peers failed, mark them as live for quick recovery */ for (i = 0; i < peers->number; i++) { peers->peer[i].fails = 0; } pc->name = peers->name; return NGX_BUSY; }
static ngx_int_t ngx_http_upstream_get_ip_hash_peer(ngx_peer_connection_t *pc, void *data) { ngx_http_upstream_ip_hash_peer_data_t *iphp = data; time_t now; ngx_int_t w; uintptr_t m; ngx_uint_t i, n, p, hash; ngx_http_upstream_rr_peer_t *peer; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get ip hash peer, try: %ui", pc->tries); /* TODO: cached */ if (iphp->tries > 20 || iphp->rrp.peers->single) { return iphp->get_rr_peer(pc, &iphp->rrp); } now = ngx_time(); pc->cached = 0; pc->connection = NULL; hash = iphp->hash; for ( ;; ) { for (i = 0; i < iphp->addrlen; i++) { hash = (hash * 113 + iphp->addr[i]) % 6271; } if (!iphp->rrp.peers->weighted) { p = hash % iphp->rrp.peers->number; } else { w = hash % iphp->rrp.peers->total_weight; for (i = 0; i < iphp->rrp.peers->number; i++) { w -= iphp->rrp.peers->peer[i].weight; if (w < 0) { break; } } p = i; } n = p / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t)); if (!(iphp->rrp.tried[n] & m)) { ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get ip hash peer, hash: %ui %04XA", p, m); peer = &iphp->rrp.peers->peer[p]; /* ngx_lock_mutex(iphp->rrp.peers->mutex); */ if (!peer->down) { #if (NGX_HTTP_UPSTREAM_CHECK) ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get ip_hash peer, check_index: %ui", peer->check_index); if (!ngx_http_upstream_check_peer_down(peer->check_index)) { #endif if (peer->max_fails == 0 || peer->fails < peer->max_fails) { break; } if (now - peer->checked > peer->fail_timeout) { peer->checked = now; break; } #if (NGX_HTTP_UPSTREAM_CHECK) } #endif } iphp->rrp.tried[n] |= m; /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ pc->tries--; } if (++iphp->tries >= 20) { return iphp->get_rr_peer(pc, &iphp->rrp); } } iphp->rrp.current = p; pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; pc->name = &peer->name; /* ngx_unlock_mutex(iphp->rrp.peers->mutex); */ iphp->rrp.tried[n] |= m; iphp->hash = hash; return NGX_OK; }
static ngx_int_t ngx_http_upstream_session_sticky_get_peer(ngx_peer_connection_t *pc, void *data) { ngx_int_t rc; ngx_uint_t i, n; ngx_http_ss_ctx_t *ctx; ngx_http_request_t *r; ngx_http_ss_server_t *server; ngx_http_upstream_ss_srv_conf_t *sscf; ngx_http_upstream_ss_peer_data_t *sspd = data; sscf = sspd->sscf; r = sspd->r; n = sscf->number; server = sscf->server; ctx = ngx_http_get_module_ctx(r, ngx_http_upstream_session_sticky_module); if (ctx->frist == 1 || ctx->sid.len == 0) { goto failed; } if (ctx->tries == 0 && !(ctx->sscf->flag & NGX_HTTP_SESSION_STICKY_FALLBACK_OFF)) { goto failed; } for (i = 0; i < n; i++) { if (ctx->sid.len == server[i].sid.len && ngx_strncmp(ctx->sid.data, server[i].sid.data, ctx->sid.len) == 0) { #if (NGX_HTTP_UPSTREAM_CHECK) if (ngx_http_upstream_check_peer_down(server[i].check_index)) { if (ctx->sscf->flag & NGX_HTTP_SESSION_STICKY_FALLBACK_OFF) { return NGX_BUSY; } else { goto failed; } } #endif pc->name = server[i].name; pc->socklen = server[i].socklen; pc->sockaddr = server[i].sockaddr; ctx->sid.len = server[i].sid.len; ctx->sid.data = server[i].sid.data; sspd->rrp.current = i; ctx->tries--; return NGX_OK; } } failed: if (ctx->frist != 1 && (ctx->sscf->flag & NGX_HTTP_SESSION_STICKY_FALLBACK_OFF)) { return NGX_BUSY; } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "session sticky failed, sid[%V]", &ctx->sid); rc = sspd->get_rr_peer(pc, &sspd->rrp); if (rc != NGX_OK) { return rc; } for (i = 0; i < n; i++) { if (server[i].name->len == pc->name->len && ngx_strncmp(server[i].name->data, pc->name->data, pc->name->len) == 0) { ctx->sid.len = server[i].sid.len; ctx->sid.data = server[i].sid.data; break; } } ctx->frist = 1; return rc; }
static ngx_int_t ngx_http_upstream_get_chash_peer(ngx_peer_connection_t *pc, void *data) { time_t now; uint32_t index, index1, index2; uint32_t diff1, diff2; ngx_queue_t *q, *temp; ngx_segment_node_t node, *p; ngx_http_upstream_rr_peer_t *peer; ngx_http_upstream_chash_server_t *server; ngx_http_upstream_chash_srv_conf_t *ucscf; ngx_http_upstream_chash_peer_data_t *uchpd = data; ngx_http_upstream_chash_down_server_t *down_server; ucscf = uchpd->ucscf; if (!ngx_queue_empty(&ucscf->down_servers)) { q = ngx_queue_head(&ucscf->down_servers); while(q != ngx_queue_sentinel(&ucscf->down_servers)) { temp = ngx_queue_next(q); down_server = ngx_queue_data(q, ngx_http_upstream_chash_down_server_t, queue); now = ngx_time(); if (now >= down_server->timeout) { peer = ucscf->servers[down_server->id].peer; #if (NGX_HTTP_UPSTREAM_CHECK) if (!ngx_http_upstream_check_peer_down(peer->check_index)) { #endif peer->fails = 0; peer->down = 0; ucscf->servers[down_server->id].down = 0; ngx_queue_remove(&down_server->queue); node.key = down_server->id; ucscf->tree->insert(ucscf->tree, 1, 1, ucscf->number, down_server->id, &node); #if (NGX_HTTP_UPSTREAM_CHECK) } #endif } q = temp; } } pc->cached = 0; pc->connection = NULL; index = ngx_http_upstream_chash_get_server_index(ucscf->servers, ucscf->number, uchpd->hash); server = &ucscf->servers[index]; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "consistent hash [peer name]:%V %ud", &server->peer->name, server->hash); if ( #if (NGX_HTTP_UPSTREAM_CHECK) ngx_http_upstream_check_peer_down(server->peer->check_index) || #endif server->peer->fails > server->peer->max_fails || server->peer->down ) { ngx_http_upstream_chash_delete_node(ucscf, server); while (1) { p = ucscf->tree->query(ucscf->tree, 1, 1, ucscf->number, 1, index - 1); index1 = p->key; p = ucscf->tree->query(ucscf->tree, 1, 1, ucscf->number, index + 1, ucscf->number); index2 = p->key; if (index1 == ucscf->tree->extreme) { if (index2 == ucscf->tree->extreme) { ngx_log_error(NGX_LOG_ERR, pc->log, 0, "all servers are down!"); return NGX_BUSY; } else { index1 = index2; server = &ucscf->servers[index2]; } } else if (index2 == ucscf->tree->extreme) { server = &ucscf->servers[index1]; } else { if (ucscf->servers[index1].hash > uchpd->hash) { diff1 = ucscf->servers[index1].hash - uchpd->hash; } else { diff1 = uchpd->hash - ucscf->servers[index1].hash; } if (uchpd->hash > ucscf->servers[index2].hash) { diff2 = uchpd->hash - ucscf->servers[index2].hash; } else { diff2 = ucscf->servers[index2].hash - uchpd->hash; } index1 = diff1 > diff2 ? index2 : index1; server = &ucscf->servers[index1]; } if ( #if (NGX_HTTP_UPSTREAM_CHECK) ngx_http_upstream_check_peer_down(server->peer->check_index) || #endif server->peer->fails > server->peer->max_fails || server->peer->down) { ngx_http_upstream_chash_delete_node(ucscf, server); } else { break; } index = index1; } } if (server->down) { ngx_log_error(NGX_LOG_ERR, pc->log, 0, "all servers are down"); return NGX_BUSY; } uchpd->server = server; peer = server->peer; pc->name = &peer->name; pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; return NGX_OK; }
static ngx_http_upstream_rr_peer_t * ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) { time_t now; uintptr_t m; ngx_int_t total; ngx_uint_t i, n; ngx_http_upstream_rr_peer_t *peer, *best; #if (NGX_HTTP_PERSISTENCE) ngx_int_t persist_index; persist_index = ngx_http_upstream_ps_get(rrp->request, rrp->peers->number, rrp->group); #endif now = ngx_time(); best = NULL; total = 0; for (i = 0; i < rrp->peers->number; i++) { #if (NGX_HTTP_PERSISTENCE) if(persist_index >= 0) { i = persist_index; } #endif n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); if (rrp->tried[n] & m) { #if (NGX_HTTP_PERSISTENCE) if(persist_index >= 0) { persist_index = -1; i = 0; } #endif continue; } peer = &rrp->peers->peer[i]; if (peer->down) { #if (NGX_HTTP_PERSISTENCE) if(persist_index >= 0) { persist_index = -1; i = 0; } #endif continue; } #if (NGX_HTTP_UPSTREAM_CHECK) if (ngx_http_upstream_check_peer_down(peer->check_index)) { continue; } #endif if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) { #if (NGX_HTTP_PERSISTENCE) if(persist_index >= 0) { persist_index = -1; i = 0; } #endif continue; } #if (NGX_HTTP_PERSISTENCE) if(persist_index >= 0) { best = peer; break; } #endif peer->current_weight += peer->effective_weight; total += peer->effective_weight; if (peer->effective_weight < peer->weight) { peer->effective_weight++; } if (best == NULL || peer->current_weight > best->current_weight) { best = peer; } } if (best == NULL) { return NULL; } i = best - &rrp->peers->peer[0]; rrp->current = i; n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); rrp->tried[n] |= m; best->current_weight -= total; if (now - best->checked > best->fail_timeout) { best->checked = now; } #if (NGX_HTTP_PERSISTENCE) ngx_http_upstream_ps_set(rrp->request, rrp->current, rrp->group); #endif return best; }
ngx_int_t ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) { ngx_http_upstream_rr_peer_data_t *rrp = data; ngx_int_t rc; ngx_uint_t i, n; ngx_http_upstream_rr_peer_t *peer; ngx_http_upstream_rr_peers_t *peers; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get rr peer, try: %ui", pc->tries); /* ngx_lock_mutex(rrp->peers->mutex); */ pc->cached = 0; pc->connection = NULL; if (rrp->peers->single) { peer = &rrp->peers->peer[0]; if (peer->down) { goto failed; } #if (NGX_HTTP_UPSTREAM_CHECK) if (ngx_http_upstream_check_peer_down(peer->check_index)) { goto failed; } #endif } else { /* there are several peers */ peer = ngx_http_upstream_get_peer(rrp); if (peer == NULL) { goto failed; } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get rr peer, current: %ui %i", rrp->current, peer->current_weight); } pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; pc->name = &peer->name; pc->host = &peer->host; pc->dyn_resolve = peer->dyn_resolve; #if (NGX_DEBUG) struct sockaddr_in *in; in = (struct sockaddr_in *)peer->sockaddr; ngx_log_debug4(NGX_LOG_DEBUG_HTTP, pc->log, 0, "peer: %xd, %d, %V, %V", in->sin_addr.s_addr, peer->socklen, &peer->name, &peer->host); #endif /* ngx_unlock_mutex(rrp->peers->mutex); */ if (pc->tries == 1 && rrp->peers->next) { pc->tries += rrp->peers->next->number; } return NGX_OK; failed: peers = rrp->peers; if (peers->next) { /* ngx_unlock_mutex(peers->mutex); */ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers"); rrp->peers = peers->next; pc->tries = rrp->peers->number; n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t)); for (i = 0; i < n; i++) { rrp->tried[i] = 0; } rc = ngx_http_upstream_get_round_robin_peer(pc, rrp); if (rc != NGX_BUSY) { return rc; } /* ngx_lock_mutex(peers->mutex); */ } /* all peers failed, mark them as live for quick recovery */ for (i = 0; i < peers->number; i++) { peers->peer[i].fails = 0; } /* ngx_unlock_mutex(peers->mutex); */ pc->name = peers->name; return NGX_BUSY; }
ngx_int_t ngx_http_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data) { ngx_http_upstream_rr_peer_data_t *rrp = data; ngx_int_t rc; ngx_uint_t i, n; ngx_connection_t *c; ngx_http_upstream_rr_peer_t *peer; ngx_http_upstream_rr_peers_t *peers; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get rr peer, try: %ui", pc->tries); /* ngx_lock_mutex(rrp->peers->mutex); */ if (rrp->peers->last_cached) { /* cached connection */ c = rrp->peers->cached[rrp->peers->last_cached]; rrp->peers->last_cached--; /* ngx_unlock_mutex(ppr->peers->mutex); */ #if (NGX_THREADS) c->read->lock = c->read->own_lock; c->write->lock = c->write->own_lock; #endif pc->connection = c; pc->cached = 1; return NGX_OK; } pc->cached = 0; pc->connection = NULL; if (rrp->peers->single) { peer = &rrp->peers->peer[0]; if (peer->down) { goto failed; } #if (NGX_HTTP_UPSTREAM_CHECK) if (ngx_http_upstream_check_peer_down(peer->check_index)) { goto failed; } #endif } else { /* there are several peers */ peer = ngx_http_upstream_get_peer(rrp); if (peer == NULL) { goto failed; } ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pc->log, 0, "get rr peer, current: %ui %i", rrp->current, peer->current_weight); } pc->sockaddr = peer->sockaddr; pc->socklen = peer->socklen; pc->name = &peer->name; /* ngx_unlock_mutex(rrp->peers->mutex); */ if (pc->tries == 1 && rrp->peers->next) { pc->tries += rrp->peers->next->number; } return NGX_OK; failed: peers = rrp->peers; if (peers->next) { /* ngx_unlock_mutex(peers->mutex); */ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, "backup servers"); rrp->peers = peers->next; pc->tries = rrp->peers->number; n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1)) / (8 * sizeof(uintptr_t)); for (i = 0; i < n; i++) { rrp->tried[i] = 0; } rc = ngx_http_upstream_get_round_robin_peer(pc, rrp); if (rc != NGX_BUSY) { return rc; } /* ngx_lock_mutex(peers->mutex); */ } /* all peers failed, mark them as live for quick recovery */ for (i = 0; i < peers->number; i++) { peers->peer[i].fails = 0; } /* ngx_unlock_mutex(peers->mutex); */ pc->name = peers->name; return NGX_BUSY; }
static ngx_http_upstream_rr_peer_t * ngx_http_upstream_get_peer(ngx_http_upstream_rr_peer_data_t *rrp) { time_t now; uintptr_t m; ngx_int_t total, flag; ngx_uint_t i, n; ngx_http_upstream_rr_peer_t *peer, *best; now = ngx_time(); best = NULL; total = 0; flag = 1; for (i = rrp->peers->init_number; i != rrp->peers->init_number || flag; i = (i + 1) % rrp->peers->number) { flag = 0; n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); if (rrp->tried[n] & m) { continue; } peer = &rrp->peers->peer[i]; if (peer->down) { continue; } #if (NGX_HTTP_UPSTREAM_CHECK) if (ngx_http_upstream_check_peer_down(peer->check_index)) { continue; } #endif if (peer->max_fails && peer->fails >= peer->max_fails && now - peer->checked <= peer->fail_timeout) { continue; } peer->current_weight += peer->effective_weight; total += peer->effective_weight; if (peer->effective_weight < peer->weight) { peer->effective_weight++; } if (best == NULL || peer->current_weight > best->current_weight) { best = peer; } } if (best == NULL) { return NULL; } i = best - &rrp->peers->peer[0]; rrp->current = i; n = i / (8 * sizeof(uintptr_t)); m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t)); rrp->tried[n] |= m; best->current_weight -= total; if (now - best->checked > best->fail_timeout) { best->checked = now; } return best; }