static ngx_int_t ngx_http_upstream_init_ip_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_ip_hash_peer; return NGX_OK; }
static ngx_int_t ngx_http_lua_balancer_init(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } /* this callback is called upon individual requests */ us->peer.init = ngx_http_lua_balancer_init_peer; return NGX_OK; }
static ngx_int_t ngx_http_upstream_init_available_capacity(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init available_capacity"); // initialize us->peer.data if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_available_capacity_peer; ngx_http_upstream_available_capacity_srv_conf_t *conf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_available_capacity_module); available_capacity_server_conf = conf; return NGX_OK; }
static ngx_int_t ngx_http_upstream_init_least_conn(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init least conn"); if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_least_conn_peer; return NGX_OK; }
static ngx_int_t ngx_http_upstream_init_least_conn(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { ngx_uint_t n; ngx_http_upstream_rr_peers_t *peers; ngx_http_upstream_least_conn_conf_t *lcf; ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, "init least conn"); if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } peers = us->peer.data; n = peers->number; if (peers->next) { n += peers->next->number; } lcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_least_conn_module); lcf->conns = ngx_pcalloc(cf->pool, sizeof(ngx_uint_t) * n); if (lcf->conns == NULL) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_least_conn_peer; us->peer.reinit_upstream = ngx_http_upstream_reinit_least_conn; return NGX_OK; }
static ngx_int_t ngx_http_upstream_init_q_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { unsigned char hash_data[HASH_DATA_LENGTH] = {}; ngx_http_upstream_q_chash_ring *q_chash_ring; ngx_http_upstream_q_chash_srv_conf_t *uchscf; ngx_http_upstream_rr_peers_t *peers; ngx_uint_t vnode_num, i, j, k, fill_next; ngx_int_t si; uint32_t point; uchscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_q_chash_module); if (uchscf == NULL) { return NGX_ERROR; } q_chash_ring = ngx_pcalloc(cf->pool, sizeof(*q_chash_ring)); if(q_chash_ring == NULL) return NGX_ERROR; if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_q_chash_peer; uchscf->q_chash_ring = q_chash_ring; peers = us->peer.data; for(i = 0; i < peers->number; i++) { if(!peers->peer[i].down) q_chash_ring->nr_valid_peers ++; } // old_cycle's log_level is NGX_LOG_NOTICE //ngx_log_debug(NGX_LOG_DEBUG_HTTP, cf->log, 0, "upstream %V nr_valid_peers %ui", peers->name, q_chash_ring->nr_valid_peers); // no need to hash if(q_chash_ring->nr_valid_peers <= 1) { return NGX_OK; } // create vnodes, peer_index field, sort q_chash_ring->vnodes = ngx_palloc(cf->pool, sizeof(q_chash_vnode_t) * peers->total_weight * NR_VNODE); if(q_chash_ring->vnodes == NULL) { return NGX_ERROR; } for(i = 0; i < peers->number; i++) { if(peers->peer[i].down) { continue; } vnode_num = peers->peer[i].weight * NR_VNODE; for(j = 0; j < vnode_num / 4; j++) { ngx_snprintf(hash_data, HASH_DATA_LENGTH, "%V-%ui%Z", &peers->peer[i].name, j); u_char md5[16]; ngx_md5_t ctx; ngx_md5_init(&ctx); ngx_md5_update(&ctx, hash_data, ngx_strlen(hash_data)); ngx_md5_final(md5, &ctx); for(k = 0; k < 4; k++) { point = *(uint32_t *)&md5[k * 4]; q_chash_ring->vnodes[q_chash_ring->nr_vnodes].peer_index = i; q_chash_ring->vnodes[q_chash_ring->nr_vnodes].point = point; q_chash_ring->nr_vnodes ++; } } } // old_cycle's log_level is NGX_LOG_NOTICE //ngx_log_debug(NGX_LOG_DEBUG_HTTP, cf->log, 0, "upstream %V nr_vnodes %ui", peers->name, q_chash_ring->nr_vnodes); ngx_qsort(q_chash_ring->vnodes, q_chash_ring->nr_vnodes, sizeof(q_chash_vnode_t), (const void *)compare_vnodes_point); // fill vnode's next field for(i = 1; ; i ++) { if(q_chash_ring->vnodes[0].peer_index == q_chash_ring->vnodes[i].peer_index) continue; q_chash_ring->vnodes[0].next = i; break; } fill_next = 0; for(si = q_chash_ring->nr_vnodes - 1; si >= 0; si--) { if(q_chash_ring->vnodes[si].peer_index == q_chash_ring->vnodes[fill_next].peer_index) { q_chash_ring->vnodes[si].next = q_chash_ring->vnodes[fill_next].next; } else { q_chash_ring->vnodes[si].next = fill_next; fill_next = si; } } // old_cycle's log_level is NGX_LOG_NOTICE /* for(i = 0; i < q_chash_ring->nr_vnodes; i++) { ngx_log_debug(NGX_LOG_DEBUG_HTTP, cf->log, 0, "%ui, next %ui peer_index %ui point %uD", i, q_chash_ring->vnodes[i].next, q_chash_ring->vnodes[i].peer_index, q_chash_ring->vnodes[i].point); } */ // calculate peer ratio for debug ~ ngx_uint_t *statistic_array = ngx_pcalloc(cf->pool, sizeof(uint32_t) * peers->number); if(statistic_array == NULL) return NGX_OK; uint32_t before_point = 0; for(i = 1; i < q_chash_ring->nr_vnodes; i++) { statistic_array[q_chash_ring->vnodes[i].peer_index] += q_chash_ring->vnodes[i].point - before_point; before_point = q_chash_ring->vnodes[i].point; } statistic_array[q_chash_ring->vnodes[0].peer_index] += 0xFFFFFFFF - before_point; for(i = 0; i < peers->number; i++) { if(peers->peer[i].down) continue; ngx_log_error(NGX_LOG_NOTICE, cf->log, 0, "upstream %V %V weight %ui actually ratio %.2f%%", peers->name, &peers->peer[i].name, peers->peer[i].weight, 100 * (double)statistic_array[i] / 0xFFFFFFFF); } ngx_pfree(cf->pool, statistic_array); return NGX_OK; }
/* * function called by the upstream module to init itself * it's called once per instance */ ngx_int_t ngx_http_init_upstream_sticky(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { ngx_http_upstream_rr_peers_t *rr_peers; ngx_http_sticky_srv_conf_t *conf; ngx_uint_t i; /* call the rr module on wich the sticky module is based on */ if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } /* calculate each peer digest once and save */ rr_peers = us->peer.data; /* do nothing there's only one peer */ if (rr_peers->number <= 1 || rr_peers->single) { return NGX_OK; } /* tell the upstream module to call ngx_http_init_sticky_peer when it inits peer */ us->peer.init = ngx_http_init_sticky_peer; conf = ngx_http_conf_upstream_srv_conf(us, ngx_http_sticky_module); /* if 'index', no need to alloc and generate digest */ if (!conf->hash && !conf->hmac && !conf->text) { conf->peers = NULL; return NGX_OK; } /* create our own upstream indexes */ conf->peers = ngx_pcalloc(cf->pool, sizeof(ngx_http_sticky_peer_t) * rr_peers->number); if (conf->peers == NULL) { return NGX_ERROR; } /* parse each peer and generate digest if necessary */ for (i = 0; i < rr_peers->number; i++) { conf->peers[i].rr_peer = &rr_peers->peer[i]; if (conf->hmac) { /* generate hmac */ conf->hmac(cf->pool, rr_peers->peer[i].sockaddr, rr_peers->peer[i].socklen, &conf->hmac_key, &conf->peers[i].digest); } else if (conf->text) { /* generate text */ conf->text(cf->pool, rr_peers->peer[i].sockaddr, &conf->peers[i].digest); } else { /* generate hash */ conf->hash(cf->pool, rr_peers->peer[i].sockaddr, rr_peers->peer[i].socklen, &conf->peers[i].digest); } #if 0 /* FIXME: is it possible to log to debug level when at configuration stage ? */ ngx_conf_log_error(NGX_LOG_WARN, cf, 0, "[sticky/ngx_http_init_upstream_sticky] generated digest \"%V\" for upstream at index %d", &conf->peers[i].digest, i); #endif } return NGX_OK; }
static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { u_char hash_buf[256]; ngx_int_t j, weight; ngx_uint_t sid, id, hash_len; ngx_uint_t i, n, *number, rnindex; ngx_http_upstream_rr_peer_t *peer; ngx_http_upstream_rr_peers_t *peers; ngx_http_upstream_chash_server_t *server; ngx_http_upstream_chash_srv_conf_t *ucscf; if (ngx_http_upstream_init_round_robin(cf, us) == NGX_ERROR) { return NGX_ERROR; } ucscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_consistent_hash_module); if (ucscf == NULL) { return NGX_ERROR; } us->peer.init = ngx_http_upstream_init_chash_peer; peers = (ngx_http_upstream_rr_peers_t *) us->peer.data; if (peers == NULL) { return NGX_ERROR; } n = peers->number; ucscf->number = 0; ucscf->real_node = ngx_pcalloc(cf->pool, n * sizeof(ngx_http_upstream_chash_server_t**)); if (ucscf->real_node == NULL) { return NGX_ERROR; } for (i = 0; i < n; i++) { ucscf->number += peers->peer[i].weight * NGX_CHASH_VIRTUAL_NODE_NUMBER; ucscf->real_node[i] = ngx_pcalloc(cf->pool, (peers->peer[i].weight * NGX_CHASH_VIRTUAL_NODE_NUMBER + 1) * sizeof(ngx_http_upstream_chash_server_t*)); if (ucscf->real_node[i] == NULL) { return NGX_ERROR; } } ucscf->servers = ngx_pcalloc(cf->pool, (ucscf->number + 1) * sizeof(ngx_http_upstream_chash_server_t)); if (ucscf->servers == NULL) { return NGX_ERROR; } ucscf->d_servers = ngx_pcalloc(cf->pool, (ucscf->number + 1) * sizeof(ngx_http_upstream_chash_down_server_t)); if (ucscf->d_servers == NULL) { return NGX_ERROR; } ucscf->number = 0; for (i = 0; i < n; i++) { peer = &peers->peer[i]; sid = (ngx_uint_t) ngx_atoi(peer->id.data, peer->id.len); if (sid == (ngx_uint_t) NGX_ERROR || sid > 65535) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, cf->log, 0, "server id %d", sid); ngx_snprintf(hash_buf, 256, "%V%Z", &peer->name); hash_len = ngx_strlen(hash_buf); sid = ngx_murmur_hash2(hash_buf, hash_len); } weight = peer->weight * NGX_CHASH_VIRTUAL_NODE_NUMBER; if (weight >= 1 << 14) { ngx_log_error(NGX_LOG_WARN, cf->log, 0, "weigth[%d] is too large, is must be less than %d", weight / NGX_CHASH_VIRTUAL_NODE_NUMBER, (1 << 14) / NGX_CHASH_VIRTUAL_NODE_NUMBER); weight = 1 << 14; } for (j = 0; j < weight; j++) { server = &ucscf->servers[++ucscf->number]; server->peer = peer; server->rnindex = i; id = sid * 256 * 16 + j; server->hash = ngx_murmur_hash2((u_char *) (&id), 4); } } ngx_qsort(ucscf->servers + 1, ucscf->number, sizeof(ngx_http_upstream_chash_server_t), (const void *)ngx_http_upstream_chash_cmp); number = ngx_calloc(n * sizeof(ngx_uint_t), cf->log); if (number == NULL) { return NGX_ERROR; } for (i = 1; i <= ucscf->number; i++) { ucscf->servers[i].index = i; ucscf->d_servers[i].id = i; rnindex = ucscf->servers[i].rnindex; ucscf->real_node[rnindex][number[rnindex]] = &ucscf->servers[i]; number[rnindex]++; } ngx_free(number); ucscf->tree = ngx_pcalloc(cf->pool, sizeof(ngx_segment_tree_t)); if (ucscf->tree == NULL) { return NGX_ERROR; } ngx_segment_tree_init(ucscf->tree, ucscf->number, cf->pool); ucscf->tree->build(ucscf->tree, 1, 1, ucscf->number); ngx_queue_init(&ucscf->down_servers); return NGX_OK; }
ngx_int_t ngx_http_upstream_init_consistent_hash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us) { /* ip max 15, :port max 6, maxweight is highest number of uchar */ u_char *last, hash_data[28]; uint32_t step; ngx_uint_t i, j, k, n; ngx_uint_t real_nodes, points_per_node; ngx_http_upstream_server_t *server; ngx_http_upstream_rr_peer_t *rr_peer; ngx_http_upstream_rr_peers_t *prr_peers; ngx_http_upstream_consistent_hash_buckets *buckets; ngx_http_upstream_consistent_hash_continuum *continuum; ngx_http_upstream_consistent_hash_srv_conf_t *uchscf; uchscf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_consistent_hash_module); if (uchscf == NULL) { return NGX_ERROR; } if (ngx_http_upstream_init_round_robin(cf, us) != NGX_OK) { return NGX_ERROR; } prr_peers = us->peer.data; us->peer.init = ngx_http_upstream_init_consistent_hash_peer; buckets = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_consistent_hash_buckets)); if (!us->servers) { return NGX_ERROR; } server = us->servers->elts; n = real_nodes = 0; for (i = 0; i < us->servers->nelts; i++) { n += server[i].naddrs; real_nodes += server[i].weight * server[i].naddrs; } /* * The optimal points number is Q/S * See the section 6.2 from the paper 'Dynamo: Amazon's Highly Available * Key-value Store' */ points_per_node = (ngx_uint_t) MMC_CONSISTENT_BUCKETS / real_nodes; if (points_per_node == 0) { points_per_node = 1; } continuum = ngx_pcalloc(cf->pool, sizeof(ngx_http_upstream_consistent_hash_continuum)); continuum->nodes = ngx_palloc(cf->pool, sizeof(ngx_http_upstream_consistent_hash_node) * MMC_CONSISTENT_BUCKETS); for (i = 0; i < us->servers->nelts; i++) { for (j = 0; j < server[i].naddrs; j++) { rr_peer = ngx_http_upstream_consistent_hash_find_rr_peer(prr_peers, server[i].addrs[j].sockaddr); if (rr_peer == NULL) { return NGX_ERROR; } for (k = 0; k < (points_per_node * server[i].weight); k++) { last = ngx_snprintf(hash_data, 28, "%V-%ui", &server[i].addrs[j].name, k); continuum->nodes[continuum->nnodes].point = ngx_http_upstream_consistent_hash_node_point(hash_data, (last - hash_data)); continuum->nodes[continuum->nnodes].rr_peer = rr_peer; continuum->nnodes++; } } } ngx_qsort(continuum->nodes, continuum->nnodes, sizeof(ngx_http_upstream_consistent_hash_node), (const void*) ngx_http_upstream_consistent_hash_compare_continuum_nodes); step = (uint32_t) (0xffffffff / MMC_CONSISTENT_BUCKETS); for (i = 0; i < MMC_CONSISTENT_BUCKETS; i++) { buckets->bucket[i] = ngx_http_upstream_consistent_hash_find(continuum, step * i); } #if (CONSISTENT_DEBUG) ngx_http_upstream_consistent_hash_print_continuum(cf, continuum); ngx_http_upstream_consistent_hash_print_buckets(cf, buckets); #endif buckets->continuum = continuum; uchscf->data = buckets; return NGX_OK; }