void
rack_init(struct rack *rack)
{
rack->continuum = NULL;
rack->ncontinuum = 0;
rack->nserver_continuum = 0;
rack->name = dn_alloc(sizeof(struct string));
string_init(rack->name);
rack->dc = dn_alloc(sizeof(struct string));
string_init(rack->dc);
}
static struct string *token_to_string(struct dyn_token *token) {
uint32_t num = token->mag[0];
int len = num_len(num);
struct string *result = dn_alloc(sizeof(*result));
string_init(result);
result->data = dn_alloc(sizeof(uint8_t) * len);
result->len = len;
string_write_uint32(result, num, result->len - 1);
return result;
}
rstatus_t
rack_init(struct rack *rack)
{
rack->continuum = dn_alloc(sizeof(struct continuum));
rack->ncontinuum = 0;
rack->nserver_continuum = 0;
rack->name = dn_alloc(sizeof(struct string));
string_init(rack->name);
rack->dc = dn_alloc(sizeof(struct string));
string_init(rack->dc);
return DN_OK;
}
struct dmsg *
dmsg_get(void)
{
struct dmsg *dmsg;
if (!TAILQ_EMPTY(&free_dmsgq)) {
ASSERT(nfree_dmsgq > 0);
dmsg = TAILQ_FIRST(&free_dmsgq);
nfree_dmsgq--;
TAILQ_REMOVE(&free_dmsgq, dmsg, m_tqe);
goto done;
}
dmsg = dn_alloc(sizeof(*dmsg));
if (dmsg == NULL) {
return NULL;
}
done:
// STAILQ_INIT(&dmsg->mhdr);
dmsg->mlen = 0;
dmsg->data = NULL;
dmsg->plen = 0;
dmsg->payload = NULL;
dmsg->type = DMSG_UNKNOWN;
dmsg->version = VERSION_10;
dmsg->id = 0;
dmsg->source_address = NULL;
dmsg->owner = NULL;
return dmsg;
}
static char *hostname_to_ip(char * hostname)
{
struct hostent *he;
struct in_addr **addr_list;
int i;
if ((he = gethostbyname(hostname)) == NULL)
{
return NULL;
}
addr_list = (struct in_addr **) he->h_addr_list;
for(i = 0; addr_list[i] != NULL; i++);
char *ip = dn_alloc(i);
for(i = 0; addr_list[i] != NULL; i++)
{
//Return the first one;
strcpy(ip , inet_ntoa(*(0 + addr_list[i])) );
return ip;
}
return NULL;
}
static rstatus_t
dnode_peer_add_node(struct server_pool *sp, struct node *node)
{
rstatus_t status;
struct array *peers = &sp->peers;
struct server *s = array_push(peers);
s->owner = sp;
uint32_t i,nelem;
s->idx = array_idx(peers, s);
//log_debug(LOG_VERB, "node rack_name : '%.*s'", node->rack.len, node->rack.data);
//log_debug(LOG_VERB, "node dc_name : '%.*s'", node->dc.len, node->dc.data);
//log_debug(LOG_VERB, "node address : '%.*s'", node->pname.len, node->pname.data);
//log_debug(LOG_VERB, "node ip : '%.*s'", node->name.len, node->name.data);
string_copy(&s->pname, node->pname.data, node->pname.len);
string_copy(&s->name, node->name.data, node->name.len);
string_copy(&s->rack, node->rack.data, node->rack.len);
string_copy(&s->dc, node->dc.data, node->dc.len);
s->port = (uint16_t) node->port;
s->is_local = node->is_local;
s->state = node->state;
s->processed = 0;
array_init(&s->tokens, 1, sizeof(struct dyn_token));
struct dyn_token *src_token = &node->token;
struct dyn_token *dst_token = array_push(&s->tokens);
copy_dyn_token(src_token, dst_token);
struct sockinfo *info = dn_alloc(sizeof(*info)); //need to free this
dn_resolve(&s->name, s->port, info);
s->family = info->family;
s->addrlen = info->addrlen;
s->addr = (struct sockaddr *)&info->addr; //TODOs: fix this by copying, not reference
s->ns_conn_q = 0;
TAILQ_INIT(&s->s_conn_q);
s->next_retry = 0LL;
s->failure_count = 0;
s->is_seed = node->is_seed;
log_debug(LOG_VERB, "add a node to peer %"PRIu32" '%.*s'",
s->idx, s->pname.len, s->pname.data);
dnode_peer_relink_conn_owner(sp);
status = dnode_peer_pool_run(sp);
if (status != DN_OK)
return status;
status = dnode_peer_each_preconnect(s, NULL);
return status;
}
rstatus_t
dnode_peer_replace(void *rmsg)
{
//rstatus_t status;
struct ring_msg *msg = rmsg;
struct server_pool *sp = msg->sp;
struct node *node = array_get(&msg->nodes, 0);
log_debug(LOG_VVERB, "dyn: peer has a replaced message '%.*s'", node->name.len, node->name.data);
struct array *peers = &sp->peers;
struct server *s = NULL;
uint32_t i,nelem;
//bool node_exist = false;
//TODOs: use hash table here
for (i=1, nelem = array_n(peers); i< nelem; i++) {
struct server * peer = (struct server *) array_get(peers, i);
if (string_compare(&peer->rack, &node->rack) == 0) {
//TODOs: now only compare 1st token and support vnode later - use hash string on a tokens for comparison
struct dyn_token *ptoken = (struct dyn_token *) array_get(&peer->tokens, 0);
struct dyn_token *ntoken = &node->token;
if (cmp_dyn_token(ptoken, ntoken) == 0) {
s = peer; //found a node to replace
}
}
}
if (s != NULL) {
log_debug(LOG_INFO, "Found an old node to replace '%.*s'", s->name.len, s->name.data);
log_debug(LOG_INFO, "Replace with address '%.*s'", node->name.len, node->name.data);
string_deinit(&s->pname);
string_deinit(&s->name);
string_copy(&s->pname, node->pname.data, node->pname.len);
string_copy(&s->name, node->name.data, node->name.len);
//TODOs: need to free the previous s->addr?
//if (s->addr != NULL) {
// dn_free(s->addr);
//}
struct sockinfo *info = dn_alloc(sizeof(*info)); //need to free this
dn_resolve(&s->name, s->port, info);
s->family = info->family;
s->addrlen = info->addrlen;
s->addr = (struct sockaddr *)&info->addr; //TODOs: fix this by copying, not reference
dnode_peer_each_disconnect(s, NULL);
dnode_peer_each_preconnect(s, NULL);
} else {
log_debug(LOG_INFO, "Unable to find any node matched the token");
}
return DN_OK;
}
static rstatus_t
stats_msg_to_core(stats_cmd_t cmd, void *cb, void *post_cb)
{
struct stat_msg *msg = dn_alloc(sizeof(*msg));
msg->cmd = cmd;
msg->data = NULL;
msg->cb = cb;
msg->post_cb = post_cb;
CBUF_Push(C2S_OutQ, msg);
return DN_OK;
}
static struct mbuf *
_mbuf_get(void)
{
struct mbuf *mbuf;
uint8_t *buf;
//loga("_mbuf_get, nfree_mbufq = %d", nfree_mbufq);
if (!STAILQ_EMPTY(&free_mbufq)) {
ASSERT(nfree_mbufq > 0);
mbuf = STAILQ_FIRST(&free_mbufq);
nfree_mbufq--;
STAILQ_REMOVE_HEAD(&free_mbufq, next);
ASSERT(mbuf->magic == MBUF_MAGIC);
goto done;
}
buf = dn_alloc(mbuf_chunk_size);
if (buf == NULL) {
return NULL;
}
mbuf_alloc_count++;
/*
* mbuf header is at the tail end of the mbuf. This enables us to catch
* buffer overrun early by asserting on the magic value during get or
* put operations
*
* <------------- mbuf_chunk_size ------------------------->
* +-------------------------------------------------------+
* | mbuf data | mbuf header |
* | (mbuf_offset) | (struct mbuf) |
* +-------------------------------------------------------+
* ^ ^ ^ ^ ^^
* | | | | ||
* | | | | \ \mbuf->end_extra (one byte past valid bound)
* \ | | \ \
* mbuf->start \ | mbuf->end mbuf
* mbuf->pos |
* \
* mbuf->last (one byte past valid byte)
*
*/
mbuf = (struct mbuf *)(buf + mbuf_offset);
mbuf->magic = MBUF_MAGIC;
mbuf->chunk_size = mbuf_chunk_size;
done:
STAILQ_NEXT(mbuf, next) = NULL;
return mbuf;
}
rstatus_t dc_init(struct datacenter *dc)
{
rstatus_t status;
dc->dict_rack = dictCreate(&dc_string_dict_type, NULL);
dc->name = dn_alloc(sizeof(struct string));
string_init(dc->name);
status = array_init(&dc->racks, 3, sizeof(struct rack));
return status;
}
struct event_base *
event_base_create(int nevent, event_cb_t cb)
{
struct event_base *evb;
int status, ep;
struct epoll_event *event;
ASSERT(nevent > 0);
ep = epoll_create(nevent);
if (ep < 0) {
log_error("epoll create of size %d failed: %s", nevent, strerror(errno));
return NULL;
}
event = dn_calloc(nevent, sizeof(*event));
if (event == NULL) {
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb = dn_alloc(sizeof(*evb));
if (evb == NULL) {
dn_free(event);
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb->ep = ep;
evb->event = event;
evb->nevent = nevent;
evb->cb = cb;
log_debug(LOG_INFO, "e %d with nevent %d", evb->ep, evb->nevent);
return evb;
}
static struct dyn_token *
dnode_peer_pool_hash(struct server_pool *pool, uint8_t *key, uint32_t keylen)
{
ASSERT(array_n(&pool->peers) != 0);
ASSERT(key != NULL && keylen != 0);
struct dyn_token *token = dn_alloc(sizeof(struct dyn_token));
if (token == NULL) {
return NULL;
}
init_dyn_token(token);
rstatus_t status = pool->key_hash((char *)key, keylen, token);
if (status != DN_OK) {
dn_free(token);
return NULL;
}
return token;
}
//allocate an arbitrary size mbuf for a general purpose operation
struct mbuf *
mbuf_alloc(size_t size)
{
uint8_t *buf = dn_alloc(size + MBUF_HSIZE);
if (buf == NULL) {
return NULL;
}
size_t mbuf_offset = size;
struct mbuf *mbuf = (struct mbuf *)(buf + mbuf_offset);
mbuf->magic = MBUF_MAGIC;
mbuf->chunk_size = size;
STAILQ_NEXT(mbuf, next) = NULL;
mbuf->start = buf;
mbuf->end = buf + mbuf_offset - MBUF_ESIZE;
mbuf->end_extra = buf + mbuf_offset;
mbuf->pos = mbuf->start;
mbuf->last = mbuf->start;
return mbuf;
}
static struct context *
core_ctx_create(struct instance *nci)
{
rstatus_t status;
struct context *ctx;
srand((unsigned) time(NULL));
ctx = dn_alloc(sizeof(*ctx));
if (ctx == NULL) {
return NULL;
}
ctx->id = ++ctx_id;
ctx->cf = NULL;
ctx->stats = NULL;
ctx->evb = NULL;
array_null(&ctx->pool);
ctx->max_timeout = nci->stats_interval;
ctx->timeout = ctx->max_timeout;
ctx->dyn_state = INIT;
/* parse and create configuration */
ctx->cf = conf_create(nci->conf_filename);
if (ctx->cf == NULL) {
loga("Failed to create context!!!");
dn_free(ctx);
return NULL;
}
/* initialize server pool from configuration */
status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
if (status != DN_OK) {
loga("Failed to initialize server pool!!!");
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* crypto init */
status = crypto_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize crypto!!!");
dn_free(ctx);
return NULL;
}
/* create stats per server pool */
ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
nci->hostname, &ctx->pool, ctx);
if (ctx->stats == NULL) {
loga("Failed to create stats!!!");
crypto_deinit();
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize event handling for client, proxy and server */
ctx->evb = event_base_create(EVENT_SIZE, &core_core);
if (ctx->evb == NULL) {
loga("Failed to create socket event handling!!!");
crypto_deinit();
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* preconnect? servers in server pool */
status = server_pool_preconnect(ctx);
if (status != DN_OK) {
loga("Failed to preconnect for server pool!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize proxy per server pool */
status = proxy_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize proxy!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
/* initialize dnode listener per server pool */
status = dnode_init(ctx);
if (status != DN_OK) {
loga("Failed to initialize dnode!!!");
crypto_deinit();
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
ctx->dyn_state = JOINING; //TODOS: change this to JOINING
/* initialize peers */
status = dnode_peer_init(&ctx->pool, ctx);
if (status != DN_OK) {
loga("Failed to initialize dnode peers!!!");
crypto_deinit();
dnode_deinit(ctx);
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
core_debug(ctx);
/* preconntect peers - probably start gossip here */
status = dnode_peer_pool_preconnect(ctx);
if (status != DN_OK) {
loga("Failed to preconnect dnode peers!!!");
crypto_deinit();
dnode_peer_deinit(&ctx->pool);
dnode_deinit(ctx);
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
dn_free(ctx);
return NULL;
}
//init ring msg queue
CBUF_Init(C2G_InQ);
CBUF_Init(C2G_OutQ);
//init stats msg queue
CBUF_Init(C2S_InQ);
CBUF_Init(C2S_OutQ);
gossip_pool_init(ctx);
log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
return ctx;
}
static rstatus_t
stats_create_buf(struct stats *st)
{
uint32_t int64_max_digits = 20; /* INT64_MAX = 9223372036854775807 */
uint32_t int32_max_digits = 10; /* INT32_MAX = 4294967294 */
uint32_t key_value_extra = 8; /* "key": "value", */
uint32_t pool_extra = 8; /* '"pool_name": { ' + ' }' */
uint32_t server_extra = 8; /* '"server_name": { ' + ' }' */
size_t size = 0;
uint32_t i;
ASSERT(st->buf.data == NULL && st->buf.size == 0);
/* header */
size += 1;
size += st->service_str.len;
size += st->service.len;
size += key_value_extra;
size += st->source_str.len;
size += st->source.len;
size += key_value_extra;
size += st->version_str.len;
size += st->version.len;
size += key_value_extra;
size += st->uptime_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->timestamp_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->rack_str.len;
size += st->rack.len;
size += key_value_extra;
size += st->dc_str.len;
size += st->dc.len;
size += key_value_extra;
size += st->latency_999th_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->latency_99th_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->latency_95th_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->latency_mean_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->latency_max_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->payload_size_999th_str.len;
size += int32_max_digits;
size += key_value_extra;
size += st->payload_size_99th_str.len;
size += int32_max_digits;
size += key_value_extra;
size += st->payload_size_95th_str.len;
size += int32_max_digits;
size += key_value_extra;
size += st->payload_size_mean_str.len;
size += int32_max_digits;
size += key_value_extra;
size += st->payload_size_max_str.len;
size += int32_max_digits;
size += key_value_extra;
size += st->alloc_msgs_str.len;
size += int32_max_digits;
size += key_value_extra;
/* server pools */
for (i = 0; i < array_n(&st->sum); i++) {
struct stats_pool *stp = array_get(&st->sum, i);
uint32_t j;
size += stp->name.len;
size += pool_extra;
for (j = 0; j < array_n(&stp->metric); j++) {
struct stats_metric *stm = array_get(&stp->metric, j);
size += stm->name.len;
size += int64_max_digits;
size += key_value_extra;
}
/* servers per pool */
for (j = 0; j < array_n(&stp->server); j++) {
struct stats_server *sts = array_get(&stp->server, j);
uint32_t k;
size += sts->name.len;
size += server_extra;
for (k = 0; k < array_n(&sts->metric); k++) {
struct stats_metric *stm = array_get(&sts->metric, k);
size += stm->name.len;
size += int64_max_digits;
size += key_value_extra;
}
}
}
/* footer */
size += 2;
size = DN_ALIGN(size, DN_ALIGNMENT);
st->buf.data = dn_alloc(size);
if (st->buf.data == NULL) {
log_error("create stats buffer of size %zu failed: %s", size,
strerror(errno));
return DN_ENOMEM;
}
st->buf.size = size;
log_debug(LOG_DEBUG, "stats buffer size %zu", size);
return DN_OK;
}
struct stats *
stats_create(uint16_t stats_port, char *stats_ip, int stats_interval,
char *source, struct array *server_pool, struct context *ctx)
{
rstatus_t status;
struct stats *st;
st = dn_alloc(sizeof(*st));
if (st == NULL) {
return NULL;
}
st->port = stats_port;
st->interval = stats_interval;
string_set_raw(&st->addr, stats_ip);
st->start_ts = (int64_t)time(NULL);
st->buf.len = 0;
st->buf.data = NULL;
st->buf.size = 0;
array_null(&st->current);
array_null(&st->shadow);
array_null(&st->sum);
st->tid = (pthread_t) -1;
st->sd = -1;
string_set_text(&st->service_str, "service");
string_set_text(&st->service, "dynomite");
string_set_text(&st->source_str, "source");
string_set_raw(&st->source, source);
string_set_text(&st->version_str, "version");
string_set_text(&st->version, DN_VERSION_STRING);
string_set_text(&st->uptime_str, "uptime");
string_set_text(&st->timestamp_str, "timestamp");
//for latency histo
string_set_text(&st->latency_999th_str, "latency_999th");
string_set_text(&st->latency_99th_str, "latency_99th");
string_set_text(&st->latency_95th_str, "latency_95th");
string_set_text(&st->latency_mean_str, "latency_mean");
string_set_text(&st->latency_max_str, "latency_max");
//for payload size histo
string_set_text(&st->payload_size_999th_str, "payload_size_999th");
string_set_text(&st->payload_size_99th_str, "payload_size_99th");
string_set_text(&st->payload_size_95th_str, "payload_size_95th");
string_set_text(&st->payload_size_mean_str, "payload_size_mean");
string_set_text(&st->payload_size_max_str, "payload_size_max");
string_set_text(&st->alloc_msgs_str, "alloc_msgs");
//only display the first pool
struct server_pool *sp = (struct server_pool*) array_get(server_pool, 0);
string_set_text(&st->rack_str, "rack");
string_copy(&st->rack, sp->rack.data, sp->rack.len);
string_set_text(&st->dc_str, "dc");
string_copy(&st->dc, sp->dc.data, sp->dc.len);
st->updated = 0;
st->aggregate = 0;
histo_init(&st->latency_histo);
histo_init(&st->payload_size_histo);
st->alloc_msgs = 0;
/* map server pool to current (a), shadow (b) and sum (c) */
status = stats_pool_map(&st->current, server_pool);
if (status != DN_OK) {
goto error;
}
status = stats_pool_map(&st->shadow, server_pool);
if (status != DN_OK) {
goto error;
}
status = stats_pool_map(&st->sum, server_pool);
if (status != DN_OK) {
goto error;
}
status = stats_create_buf(st);
if (status != DN_OK) {
goto error;
}
status = stats_start_aggregator(st);
if (status != DN_OK) {
goto error;
}
st->ctx = ctx;
return st;
error:
stats_destroy(st);
return NULL;
}
