static void
ngx_stream_upstream_save_round_robin_peer_session(ngx_peer_connection_t *pc,
void *data)
{
ngx_stream_upstream_rr_peer_data_t *rrp = data;
ngx_ssl_session_t *old_ssl_session, *ssl_session;
ngx_stream_upstream_rr_peer_t *peer;
#if (NGX_STREAM_UPSTREAM_ZONE)
int len;
u_char *p;
ngx_stream_upstream_rr_peers_t *peers;
u_char buf[NGX_SSL_MAX_SESSION_SIZE];
#endif
#if (NGX_STREAM_UPSTREAM_ZONE)
peers = rrp->peers;
if (peers->shpool) {
ssl_session = SSL_get0_session(pc->connection->ssl->connection);
if (ssl_session == NULL) {
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"save session: %p", ssl_session);
len = i2d_SSL_SESSION(ssl_session, NULL);
/* do not cache too big session */
if (len > NGX_SSL_MAX_SESSION_SIZE) {
return;
}
p = buf;
(void) i2d_SSL_SESSION(ssl_session, &p);
peer = rrp->current;
ngx_stream_upstream_rr_peers_rlock(peers);
ngx_stream_upstream_rr_peer_lock(peers, peer);
if (len > peer->ssl_session_len) {
ngx_shmtx_lock(&peers->shpool->mutex);
if (peer->ssl_session) {
ngx_slab_free_locked(peers->shpool, peer->ssl_session);
}
peer->ssl_session = ngx_slab_alloc_locked(peers->shpool, len);
ngx_shmtx_unlock(&peers->shpool->mutex);
if (peer->ssl_session == NULL) {
peer->ssl_session_len = 0;
ngx_stream_upstream_rr_peer_unlock(peers, peer);
ngx_stream_upstream_rr_peers_unlock(peers);
return;
}
peer->ssl_session_len = len;
}
ngx_memcpy(peer->ssl_session, buf, len);
ngx_stream_upstream_rr_peer_unlock(peers, peer);
ngx_stream_upstream_rr_peers_unlock(peers);
return;
}
#endif
ssl_session = ngx_ssl_get_session(pc->connection);
if (ssl_session == NULL) {
return;
}
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"save session: %p", ssl_session);
peer = rrp->current;
old_ssl_session = peer->ssl_session;
peer->ssl_session = ssl_session;
if (old_ssl_session) {
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"old session: %p", old_ssl_session);
/* TODO: may block */
ngx_ssl_free_session(old_ssl_session);
}
}
static ngx_int_t
ngx_stream_upstream_set_round_robin_peer_session(ngx_peer_connection_t *pc,
void *data)
{
ngx_stream_upstream_rr_peer_data_t *rrp = data;
ngx_int_t rc;
ngx_ssl_session_t *ssl_session;
ngx_stream_upstream_rr_peer_t *peer;
#if (NGX_STREAM_UPSTREAM_ZONE)
int len;
#if OPENSSL_VERSION_NUMBER >= 0x0090707fL
const
#endif
u_char *p;
ngx_stream_upstream_rr_peers_t *peers;
u_char buf[NGX_SSL_MAX_SESSION_SIZE];
#endif
peer = rrp->current;
#if (NGX_STREAM_UPSTREAM_ZONE)
peers = rrp->peers;
if (peers->shpool) {
ngx_stream_upstream_rr_peers_rlock(peers);
ngx_stream_upstream_rr_peer_lock(peers, peer);
if (peer->ssl_session == NULL) {
ngx_stream_upstream_rr_peer_unlock(peers, peer);
ngx_stream_upstream_rr_peers_unlock(peers);
return NGX_OK;
}
len = peer->ssl_session_len;
ngx_memcpy(buf, peer->ssl_session, len);
ngx_stream_upstream_rr_peer_unlock(peers, peer);
ngx_stream_upstream_rr_peers_unlock(peers);
p = buf;
ssl_session = d2i_SSL_SESSION(NULL, &p, len);
rc = ngx_ssl_set_session(pc->connection, ssl_session);
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"set session: %p", ssl_session);
ngx_ssl_free_session(ssl_session);
return rc;
}
#endif
ssl_session = peer->ssl_session;
rc = ngx_ssl_set_session(pc->connection, ssl_session);
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"set session: %p", ssl_session);
return rc;
}
ngx_int_t
ngx_stream_upstream_get_round_robin_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_stream_upstream_rr_peer_data_t *rrp = data;
ngx_int_t rc;
ngx_uint_t i, n;
ngx_stream_upstream_rr_peer_t *peer;
ngx_stream_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get rr peer, try: %ui", pc->tries);
pc->connection = NULL;
peers = rrp->peers;
ngx_stream_upstream_rr_peers_wlock(peers);
if (peers->single) {
peer = peers->peer;
if (peer->down) {
goto failed;
}
rrp->current = peer;
} else {
/* there are several peers */
peer = ngx_stream_upstream_get_peer(rrp);
if (peer == NULL) {
goto failed;
}
ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get rr peer, current: %p %i",
peer, peer->current_weight);
}
pc->sockaddr = peer->sockaddr;
pc->socklen = peer->socklen;
pc->name = &peer->name;
peer->conns++;
ngx_stream_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next) {
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0, "backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++) {
rrp->tried[i] = 0;
}
ngx_stream_upstream_rr_peers_unlock(peers);
rc = ngx_stream_upstream_get_round_robin_peer(pc, rrp);
if (rc != NGX_BUSY) {
return rc;
}
ngx_stream_upstream_rr_peers_wlock(peers);
}
/* all peers failed, mark them as live for quick recovery */
for (peer = peers->peer; peer; peer = peer->next) {
peer->fails = 0;
}
ngx_stream_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
void
ngx_stream_upstream_free_round_robin_peer(ngx_peer_connection_t *pc, void *data,
ngx_uint_t state)
{
ngx_stream_upstream_rr_peer_data_t *rrp = data;
time_t now;
ngx_stream_upstream_rr_peer_t *peer;
ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"free rr peer %ui %ui", pc->tries, state);
peer = rrp->current;
ngx_stream_upstream_rr_peers_rlock(rrp->peers);
ngx_stream_upstream_rr_peer_lock(rrp->peers, peer);
if (rrp->peers->single) {
peer->conns--;
ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
ngx_stream_upstream_rr_peers_unlock(rrp->peers);
pc->tries = 0;
return;
}
if (state & NGX_PEER_FAILED) {
now = ngx_time();
peer->fails++;
peer->accessed = now;
peer->checked = now;
if (peer->max_fails) {
peer->effective_weight -= peer->weight / peer->max_fails;
if (peer->fails >= peer->max_fails) {
ngx_log_error(NGX_LOG_WARN, pc->log, 0,
"upstream server temporarily disabled");
}
}
ngx_log_debug2(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"free rr peer failed: %p %i",
peer, peer->effective_weight);
if (peer->effective_weight < 0) {
peer->effective_weight = 0;
}
} else {
/* mark peer live if check passed */
if (peer->accessed < peer->checked) {
peer->fails = 0;
}
}
peer->conns--;
ngx_stream_upstream_rr_peer_unlock(rrp->peers, peer);
ngx_stream_upstream_rr_peers_unlock(rrp->peers);
if (pc->tries) {
pc->tries--;
}
}
static ngx_int_t
ngx_stream_upstream_get_least_conn_peer(ngx_peer_connection_t *pc, void *data)
{
ngx_stream_upstream_rr_peer_data_t *rrp = data;
time_t now;
uintptr_t m;
ngx_int_t rc, total;
ngx_uint_t i, n, p, many;
ngx_stream_upstream_rr_peer_t *peer, *best;
ngx_stream_upstream_rr_peers_t *peers;
ngx_log_debug1(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get least conn peer, try: %ui", pc->tries);
if (rrp->peers->single)
{
return ngx_stream_upstream_get_round_robin_peer(pc, rrp);
}
pc->connection = NULL;
now = ngx_time();
peers = rrp->peers;
ngx_stream_upstream_rr_peers_wlock(peers);
best = NULL;
total = 0;
#if (NGX_SUPPRESS_WARN)
many = 0;
p = 0;
#endif
for (peer = peers->peer, i = 0;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m)
{
continue;
}
if (peer->down)
{
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
/*
* select peer with least number of connections; if there are
* multiple peers with the same number of connections, select
* based on round-robin
*/
if (best == NULL
|| peer->conns * best->weight < best->conns * peer->weight)
{
best = peer;
many = 0;
p = i;
}
else if (peer->conns * best->weight == best->conns * peer->weight)
{
many = 1;
}
}
if (best == NULL)
{
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get least conn peer, no peer found");
goto failed;
}
if (many)
{
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get least conn peer, many");
for (peer = best, i = p;
peer;
peer = peer->next, i++)
{
n = i / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << i % (8 * sizeof(uintptr_t));
if (rrp->tried[n] & m)
{
continue;
}
if (peer->down)
{
continue;
}
if (peer->conns * best->weight != best->conns * peer->weight)
{
continue;
}
if (peer->max_fails
&& peer->fails >= peer->max_fails
&& now - peer->checked <= peer->fail_timeout)
{
continue;
}
peer->current_weight += peer->effective_weight;
total += peer->effective_weight;
if (peer->effective_weight < peer->weight)
{
peer->effective_weight++;
}
if (peer->current_weight > best->current_weight)
{
best = peer;
p = i;
}
}
}
best->current_weight -= total;
if (now - best->checked > best->fail_timeout)
{
best->checked = now;
}
pc->sockaddr = best->sockaddr;
pc->socklen = best->socklen;
pc->name = &best->name;
best->conns++;
rrp->current = best;
n = p / (8 * sizeof(uintptr_t));
m = (uintptr_t) 1 << p % (8 * sizeof(uintptr_t));
rrp->tried[n] |= m;
ngx_stream_upstream_rr_peers_unlock(peers);
return NGX_OK;
failed:
if (peers->next)
{
ngx_log_debug0(NGX_LOG_DEBUG_STREAM, pc->log, 0,
"get least conn peer, backup servers");
rrp->peers = peers->next;
n = (rrp->peers->number + (8 * sizeof(uintptr_t) - 1))
/ (8 * sizeof(uintptr_t));
for (i = 0; i < n; i++)
{
rrp->tried[i] = 0;
}
ngx_stream_upstream_rr_peers_unlock(peers);
rc = ngx_stream_upstream_get_least_conn_peer(pc, rrp);
if (rc != NGX_BUSY)
{
return rc;
}
ngx_stream_upstream_rr_peers_wlock(peers);
}
/* all peers failed, mark them as live for quick recovery */
for (peer = peers->peer; peer; peer = peer->next)
{
peer->fails = 0;
}
ngx_stream_upstream_rr_peers_unlock(peers);
pc->name = peers->name;
return NGX_BUSY;
}
