static int
nc_event_init(struct nc_event *event)
{
struct nc_event_state *e_state;
e_state = nc_alloc(sizeof(*e_state));
if (e_state == NULL) {
return -1;
}
e_state->events = nc_calloc(event->nevent, sizeof(struct epoll_event));
if (e_state->events == NULL) {
nc_free(e_state);
return -1;
}
e_state->ep = epoll_create(event->nevent);
if (e_state->ep < 0) {
nc_free(e_state->events);
nc_free(e_state);
log_error("epoll create of size %d failed: %s", event->nevent,
strerror(errno));
return -1;
}
log_debug(LOG_INFO, "e %d with nevent %d", e_state->ep, event->nevent);
event->event_data = e_state;
return 0;
}
cursor_t *
cursor_create(store_t *store)
{
cursor_t *cursor;
cursor = nc_alloc(sizeof(*cursor));
if (cursor == NULL) {
return NULL;
}
cursor->iter = leveldb_create_iterator(store->db, store->roptions);
if (cursor->iter == NULL) {
nc_free(cursor);
return NULL;
}
leveldb_iter_seek_to_first(cursor->iter);
cursor->id = g_cursor_id++;
cursor->owner = store;
ncursor++;
STAILQ_INSERT_HEAD(&cursorq, cursor, next);
return cursor;
}
/**
* eval a statement, leave value in the stack
* like luaL_xxx functions, return 0 on success.
*/
static int
_lua_eval(lua_State *L, const char *expr)
{
char *buf;
int status;
buf = nc_alloc(strlen(expr) + sizeof("return "));
strcpy(buf, "return ");
strcat(buf, expr);
status = luaL_loadbuffer(L, buf, strlen(buf), "eval");
if (status) {
log_stderr("nc_conf: error on loadbuffer: %s", lua_tostring(L, -1));
lua_pop(L, 1);
goto out;
}
status = lua_pcall(L, 0, 1, 0);
if (status) {
log_stderr("nc_conf: error on lua_pcall: %s", lua_tostring(L, -1));
lua_pop(L, 1);
goto out;
}
if (lua_isnil(L, -1)) {
log_stderr("nc_conf: value is nil: %s", expr);
status = NC_ERROR;
}
status = NC_OK;
out:
nc_free(buf);
return status;
}
static struct mbuf *
_mbuf_get_proxy_adm(void)
{
struct mbuf *mbuf;
uint8_t *buf;
if (!STAILQ_EMPTY(&free_mbufq_proxy_adm)) {
ASSERT(nfree_mbufq_proxy_adm > 0);
mbuf = STAILQ_FIRST(&free_mbufq_proxy_adm);
nfree_mbufq_proxy_adm--;
STAILQ_REMOVE_HEAD(&free_mbufq_proxy_adm, next);
ASSERT(mbuf->magic == MBUF_MAGIC);
goto done;
}
buf = nc_alloc(mbuf_chunk_size);
if (buf == NULL) {
return NULL;
}
#if 1 //shenzheng 2015-7-9 proxy administer
#ifdef NC_DEBUG_LOG
ntotal_mbuf_proxy_adm ++;
#endif
#endif //shenzheng 2015-7-9 proxy administer
mbuf = (struct mbuf *)(buf + mbuf_offset);
mbuf->magic = MBUF_MAGIC;
done:
STAILQ_NEXT(mbuf, next) = NULL;
return mbuf;
}
struct event_base *
event_base_create(int nevent, event_cb_t cb)
{
struct event_base *evb;
int status, kq;
struct kevent *change, *event;
ASSERT(nevent > 0);
kq = kqueue();
if (kq < 0) {
log_error("kqueue failed: %s", strerror(errno));
return NULL;
}
change = nc_calloc(nevent, sizeof(*change));
if (change == NULL) {
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
event = nc_calloc(nevent, sizeof(*event));
if (event == NULL) {
nc_free(change);
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
evb = nc_alloc(sizeof(*evb));
if (evb == NULL) {
nc_free(change);
nc_free(event);
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
evb->kq = kq;
evb->change = change;
evb->nchange = 0;
evb->event = event;
evb->nevent = nevent;
evb->nreturned = 0;
evb->nprocessed = 0;
evb->cb = cb;
log_debug(LOG_INFO, "kq %d with nevent %d", evb->kq, evb->nevent);
return evb;
}
static struct conn *
_conn_get(void)
{
struct conn *conn;
if (!TAILQ_EMPTY(&free_connq)) {
ASSERT(nfree_connq > 0);
conn = TAILQ_FIRST(&free_connq);
nfree_connq--;
TAILQ_REMOVE(&free_connq, conn, conn_tqe);
} else {
conn = nc_alloc(sizeof(*conn));
if (conn == NULL) {
return NULL;
}
}
conn->owner = NULL;
conn->sd = -1;
/* {family, addrlen, addr} are initialized in enqueue handler */
TAILQ_INIT(&conn->imsg_q);
TAILQ_INIT(&conn->omsg_q);
conn->rmsg = NULL;
conn->smsg = NULL;
/*
* Callbacks {recv, recv_next, recv_done}, {send, send_next, send_done},
* {close, active}, parse, {ref, unref}, {enqueue_inq, dequeue_inq} and
* {enqueue_outq, dequeue_outq} are initialized by the wrapper.
*/
conn->send_bytes = 0;
conn->recv_bytes = 0;
conn->events = 0;
conn->err = 0;
conn->recv_active = 0;
conn->recv_ready = 0;
conn->send_active = 0;
conn->send_ready = 0;
conn->client = 0;
conn->proxy = 0;
conn->connecting = 0;
conn->connected = 0;
conn->eof = 0;
conn->done = 0;
conn->redis = 0;
conn->need_auth = 0;
ntotal_conn++;
ncurr_conn++;
return conn;
}
int
nc_connect_init(struct context *ctx)
{
ctx->cb = nc_alloc(sizeof(struct conn_base));
if (ctx->cb == NULL) {
return NC_ENOMEM;
}
conn_init(ctx->cb);
return NC_OK;
}
static struct mbuf *
_mbuf_get(void)
{
struct mbuf *mbuf;
uint8_t *buf;
if (!STAILQ_EMPTY(&free_mbufq)) {
ASSERT(nfree_mbufq > 0);
mbuf = STAILQ_FIRST(&free_mbufq);
nfree_mbufq--;
STAILQ_REMOVE_HEAD(&free_mbufq, next);
ASSERT(mbuf->magic == MBUF_MAGIC);
goto done;
}
buf = nc_alloc(mbuf_chunk_size);
if (buf == NULL) {
return NULL;
}
#if 1 //shenzheng 2015-3-23 common
#ifdef NC_DEBUG_LOG
ntotal_mbuf ++;
#endif
#endif //shenzheng 2015-3-23 common
/*
* mbuf header is at the tail end of the mbuf. This enables us to catch
* buffer overrun early by asserting on the magic value during get or
* put operations
*
* <------------- mbuf_chunk_size ------------->
* +-------------------------------------------+
* | mbuf data | mbuf header |
* | (mbuf_offset) | (struct mbuf) |
* +-------------------------------------------+
* ^ ^ ^ ^^
* | | | ||
* \ | | |\
* mbuf->start \ | | mbuf->end (one byte past valid bound)
* mbuf->pos \
* \ mbuf
* mbuf->last (one byte past valid byte)
*
*/
mbuf = (struct mbuf *)(buf + mbuf_offset);
mbuf->magic = MBUF_MAGIC;
done:
STAILQ_NEXT(mbuf, next) = NULL;
return mbuf;
}
static struct conf *
conf_open(char *filename)
{
rstatus_t status;
struct conf *cf;
FILE *fh;
fh = fopen(filename, "r");
if (fh == NULL) {
log_error("conf: failed to open configuration '%s': %s", filename,
strerror(errno));
return NULL;
}
cf = nc_alloc(sizeof(*cf));
if (cf == NULL) {
fclose(fh);
return NULL;
}
status = array_init(&cf->arg, CONF_DEFAULT_ARGS, sizeof(struct string));
if (status != NC_OK) {
nc_free(cf);
fclose(fh);
return NULL;
}
status = array_init(&cf->pool, CONF_DEFAULT_POOL, sizeof(struct conf_pool));
if (status != NC_OK) {
array_deinit(&cf->arg);
nc_free(cf);
fclose(fh);
return NULL;
}
cf->fname = filename;
cf->fh = fh;
cf->depth = 0;
/* parser, event, and token are initialized later */
cf->seq = 0;
cf->valid_parser = 0;
cf->valid_event = 0;
cf->valid_token = 0;
cf->sound = 0;
cf->parsed = 0;
cf->valid = 0;
log_debug(LOG_VVERB, "opened conf '%s'", filename);
return cf;
}
struct evbase *
evbase_create(int nevent, void (*callback_fp)(void *, uint32_t))
{
struct evbase *evb;
int status, ep;
struct epoll_event *event;
if (nevent <= 0) {
log_error("nevent has to be positive %d", nevent);
return NULL;
}
ep = epoll_create(nevent);
if (ep < 0) {
log_error("epoll create of size %d failed: %s", nevent, strerror(errno));
return NULL;
}
event = nc_calloc(nevent, sizeof(*event));
if (event == NULL) {
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb = nc_alloc(sizeof(*evb));
if (evb == NULL) {
nc_free(event);
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb->nevent = nevent;
evb->ep = ep;
evb->event = event;
evb->callback_fp = callback_fp;
log_debug(LOG_INFO, "e %d with nevent %d", evb->ep,
evb->nevent);
return evb;
}
static struct conn *
_conn_get(void)
{
struct conn *conn;
if (!TAILQ_EMPTY(&free_connq)) {
ASSERT(nfree_connq > 0);
conn = TAILQ_FIRST(&free_connq);
nfree_connq--;
TAILQ_REMOVE(&free_connq, conn, conn_tqe);
} else {
conn = nc_alloc(sizeof(*conn));
if (conn == NULL) {
return NULL;
}
}
STAILQ_INIT(&conn->recv_queue);
STAILQ_INIT(&conn->send_queue);
conn->recv_queue_bytes = 0;
conn->send_queue_bytes = 0;
conn->owner = NULL;
conn->data = NULL;
conn->fd = -1;
conn->send_bytes = 0;
conn->recv_bytes = 0;
conn->events = 0;
conn->err = 0;
conn->recv_active = 0;
conn->recv_ready = 0;
conn->send_active = 0;
conn->send_ready = 0;
conn->eof = 0;
conn->done = 0;
/* for client conn */
conn->recv = conn_recv;
conn->send = conn_send;
conn->close = conn_close;
return conn;
}
/* Returns a new netconnection (Input and Output buffers) for a
socket connection on fd */
netconnection_t *net_new_connection(int fd)
{
netconnection_t *nc;
if( (nc = nc_alloc()) ) {
net_connection_reset(nc);
nc->fd = fd;
nc->record_fd = 0;
nc->incoming = 0;
nc->fake = 0;
nc->remote_address = net_remote_address(fd);
}
return nc;
}
struct replicaset*
ffi_replicaset_new(void)
{
int i;
struct replicaset *rs;
rs = nc_alloc(sizeof(struct replicaset));
if (rs == NULL) {
log_error("failed to allocate memory");
return NULL;
}
for (i = 0; i < NC_MAXTAGNUM; i++) {
array_init(&rs->tagged_servers[i], 2, sizeof(struct server *));
}
return rs;
}
/* Create a new slowlog entry.
* Incrementing the ref count of all the objects retained is up to
* this function. */
static slowlog_entry *slowlog_create_entry(struct msg *r, long long duration) {
slowlog_entry *se = nc_alloc(sizeof(*se));
uint32_t j, keys_count, keys_count_input;
se->cmdtype = r->type;
keys_count_input = keys_count = r->keys==NULL?0:array_n(r->keys);
if (keys_count_input > SLOWLOG_ENTRY_MAX_KEYS) keys_count_input = SLOWLOG_ENTRY_MAX_KEYS;
se->keys_count = (int)keys_count;
if (keys_count_input > 0) {
se->keys = array_create(keys_count_input, sizeof(struct string));
for (j = 0; j < keys_count_input; j ++) {
struct keypos *kp = array_get(r->keys, j);
struct string *key = array_push(se->keys);
uint32_t key_len = (uint32_t)(kp->end-kp->start);
string_init(key);
if (key_len > SLOWLOG_ENTRY_MAX_STRING) {
int len;
uint8_t buf[SLOWLOG_ENTRY_MAX_STRING+50];
memcpy(buf,kp->start,SLOWLOG_ENTRY_MAX_STRING);
len = nc_scnprintf(buf+SLOWLOG_ENTRY_MAX_STRING,
50,"... (%lu more bytes)",
key_len-SLOWLOG_ENTRY_MAX_STRING);
if (len > 0) {
string_copy(key,buf,SLOWLOG_ENTRY_MAX_STRING+(uint32_t)len);
} else {
string_copy(key,kp->start,SLOWLOG_ENTRY_MAX_STRING);
}
} else {
string_copy(key,kp->start,key_len);
}
}
}else {
se->keys = NULL;
}
se->time = time(NULL);
se->duration = duration;
STAILQ_NEXT(se, next) = NULL;
return se;
}
struct server*
ffi_server_new(struct server_pool *pool, char *name, char *id, char *ip, int port)
{
struct server *s;
struct string address;
rstatus_t status;
s = nc_alloc(sizeof(struct server));
if (s == NULL) {
log_error("failed to allocate memory");
return NULL;
}
s->owner = pool;
s->idx = 0;
s->weight = 1;
/* set name */
string_init(&s->name);
string_copy(&s->name, (uint8_t*)name, (uint32_t)nc_strlen(name));
string_init(&s->pname);
string_copy(&s->pname, (uint8_t*)name, (uint32_t)nc_strlen(name));
string_init(&address);
string_copy(&address, (uint8_t*)ip, (uint32_t)nc_strlen(ip));
/* set port */
s->port = (uint16_t)port;
status = nc_resolve(&address, s->port, &s->sockinfo);
if (status != NC_OK) {
log_error("conf: failed to resolve %.*s:%d", address.len, address.data, s->port);
return NULL;
}
s->family = s->sockinfo.family;
s->addrlen = s->sockinfo.addrlen;
s->addr = (struct sockaddr *)&s->sockinfo.addr;
s->ns_conn_q = 0;
TAILQ_INIT(&s->s_conn_q);
s->next_retry = 0LL;
s->failure_count = 0;
return s;
}
struct event_base *
event_base_create(int nevent, event_cb_t cb)
{
struct event_base *evb;
int status, ep;
struct epoll_event *event;
ASSERT(nevent > 0);
ep = epoll_create(nevent);
if (ep < 0) {
log_error("epoll create of size %d failed: %s", nevent, strerror(errno));
return NULL;
}
event = nc_calloc(nevent, sizeof(*event));
if (event == NULL) {
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb = nc_alloc(sizeof(*evb));
if (evb == NULL) {
nc_free(event);
status = close(ep);
if (status < 0) {
log_error("close e %d failed, ignored: %s", ep, strerror(errno));
}
return NULL;
}
evb->ep = ep;
evb->event = event;
evb->nevent = nevent;
evb->cb = cb;
log_debug(LOG_INFO, "e %d with nevent %d", evb->ep, evb->nevent);
return evb;
}
struct event_base *
event_base_create(int nevent, event_cb_t cb)
{
struct event_base *evb;
int status, evp;
port_event_t *event;
ASSERT(nevent > 0);
evp = port_create();
if (evp < 0) {
log_error("port create failed: %s", strerror(errno));
return NULL;
}
event = nc_calloc(nevent, sizeof(*event));
if (event == NULL) {
status = close(evp);
if (status < 0) {
log_error("close evp %d failed, ignored: %s", evp, strerror(errno));
}
return NULL;
}
evb = nc_alloc(sizeof(*evb));
if (evb == NULL) {
nc_free(event);
status = close(evp);
if (status < 0) {
log_error("close evp %d failed, ignored: %s", evp, strerror(errno));
}
return NULL;
}
evb->evp = evp;
evb->event = event;
evb->nevent = nevent;
evb->cb = cb;
log_info("evp %d with nevent %d", evb->evp, evb->nevent);
return evb;
}
/* Start a fake net connection (used for demos on CLIENT) */
netconnection_t *net_fake_connection(char *demoname, demo_header *header)
{
int fd;
ssize_t size;
netconnection_t *nc;
if( !(fd = open(demoname, O_RDONLY)) ) {
perror("open");
return NULL;
}
size = read(fd, header, sizeof(demo_header));
if( size != sizeof(demo_header) ) {
printf("Couldn't Read header information from \"%s\"\n", demoname);
if( size < 0 )
perror("read");
else
printf("Read only %d bytes of %d.\n", size, sizeof(demo_header));
close(fd);
return NULL;
}
NTOHS( header->x_tiles );
NTOHS( header->y_tiles );
NTOHS( header->fps );
if( (nc = nc_alloc()) ) {
net_connection_reset(nc);
nc->fd = fd;
nc->record_fd = 0;
nc->fake = 1;
nc->remote_address = fake_address;
}
return nc;
}
struct stats *
stats_create(uint16_t stats_port, char *stats_ip, int stats_interval,
char *source, struct array *server_pool)
{
rstatus_t status;
struct stats *st;
char *server_name;
st = nc_alloc(sizeof(*st));
if (st == NULL) {
return NULL;
}
st->port = stats_port;
st->interval = stats_interval;
string_set_raw(&st->addr, stats_ip);
st->start_ts = (int64_t)time(NULL);
st->buf.len = 0;
st->buf.data = NULL;
st->buf.size = 0;
array_null(&st->current);
array_null(&st->shadow);
array_null(&st->sum);
st->tid = (pthread_t) -1;
st->sd = -1;
string_set_text(&st->service_str, "service");
//string_set_text(&st->service, "bilitw");
server_name = nc_alloc(100);
memset(server_name, 0, 100);
sprintf(server_name, "bilitw worker %d", nc_worker_index);
string_set_raw(&st->service, server_name);
//&st->service->len = strlen(server_name) + 1;
//&st->service->data = (uint8_t *)server_name;
string_set_text(&st->source_str, "source");
string_set_raw(&st->source, source);
string_set_text(&st->version_str, "version");
string_set_text(&st->version, NC_VERSION_STRING);
string_set_text(&st->uptime_str, "uptime");
string_set_text(&st->timestamp_str, "timestamp");
string_set_text(&st->ntotal_conn_str, "total_connections");
string_set_text(&st->ncurr_conn_str, "curr_connections");
st->updated = 0;
st->aggregate = 0;
/* map server pool to current (a), shadow (b) and sum (c) */
status = stats_pool_map(&st->current, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_pool_map(&st->shadow, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_pool_map(&st->sum, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_create_buf(st);
if (status != NC_OK) {
goto error;
}
status = stats_start_aggregator(st);
if (status != NC_OK) {
goto error;
}
return st;
error:
stats_destroy(st);
return NULL;
}
static rstatus_t
stats_create_buf(struct stats *st)
{
uint32_t int64_max_digits = 20; /* INT64_MAX = 9223372036854775807 */
uint32_t key_value_extra = 8; /* "key": "value", */
uint32_t pool_extra = 8; /* '"pool_name": { ' + ' }' */
uint32_t server_extra = 8; /* '"server_name": { ' + ' }' */
size_t size = 0;
uint32_t i;
ASSERT(st->buf.data == NULL && st->buf.size == 0);
/* header */
size += 1;
size += st->service_str.len;
size += st->service.len;
size += key_value_extra;
size += st->source_str.len;
size += st->source.len;
size += key_value_extra;
size += st->version_str.len;
size += st->version.len;
size += key_value_extra;
size += st->uptime_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->timestamp_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->ntotal_conn_str.len;
size += int64_max_digits;
size += key_value_extra;
size += st->ncurr_conn_str.len;
size += int64_max_digits;
size += key_value_extra;
/* server pools */
for (i = 0; i < array_n(&st->sum); i++) {
struct stats_pool *stp = array_get(&st->sum, i);
uint32_t j;
size += stp->name.len;
size += pool_extra;
for (j = 0; j < array_n(&stp->metric); j++) {
struct stats_metric *stm = array_get(&stp->metric, j);
size += stm->name.len;
size += int64_max_digits;
size += key_value_extra;
}
/* servers per pool */
for (j = 0; j < array_n(&stp->server); j++) {
struct stats_server *sts = array_get(&stp->server, j);
uint32_t k;
size += sts->name.len;
size += server_extra;
for (k = 0; k < array_n(&sts->metric); k++) {
struct stats_metric *stm = array_get(&sts->metric, k);
size += stm->name.len;
size += int64_max_digits;
size += key_value_extra;
}
}
}
/* footer */
size += 2;
size = NC_ALIGN(size, NC_ALIGNMENT);
st->buf.data = nc_alloc(size);
if (st->buf.data == NULL) {
log_error("create stats buffer of size %zu failed: %s", size,
strerror(errno));
return NC_ENOMEM;
}
st->buf.size = size;
log_debug(LOG_DEBUG, "stats buffer size %zu", size);
return NC_OK;
}
rstatus_t
stats_master_server(uint16_t stats_port, char *stats_ip)
{
rstatus_t status;
//struct stats *st;
struct sockinfo si;
struct string addr;
//int sd;
string_set_raw(&addr, stats_ip);
status = nc_resolve(&addr, stats_port, &si);
if (status < 0) {
return status;
}
nc_stats_listen_sd = socket(si.family, SOCK_STREAM, 0);
if (nc_stats_listen_sd < 0) {
log_error("socket failed: %s", strerror(errno));
return NC_ERROR;
}
status = nc_set_reuseaddr(nc_stats_listen_sd);
if (status < 0) {
log_error("set reuseaddr on m %d failed: %s", nc_stats_listen_sd, strerror(errno));
return NC_ERROR;
}
pthread_t *ptid = nc_alloc(sizeof(pthread_t));
*ptid = (pthread_t) -1;
log_error("master sd %d", nc_stats_listen_sd);
status = pthread_create(ptid, NULL, stats_master_loop, (void*)(&nc_stats_listen_sd));
if (status < 0) {
log_error("stats aggregator create failed: %s", strerror(status));
return NC_ERROR;
}
log_debug(LOG_NOTICE, "pthread stats master create success %d'", status);
status = bind(nc_stats_listen_sd, (struct sockaddr *)&si.addr, si.addrlen);
if (status < 0) {
log_error("bind on m %d to addr '%.*s:%u' failed: %s", nc_stats_listen_sd,
addr.len, addr.data, stats_port, strerror(errno));
return NC_ERROR;
}
status = listen(nc_stats_listen_sd, SOMAXCONN);
if (status < 0) {
log_error("listen on m %d failed: %s", nc_stats_listen_sd, strerror(errno));
return NC_ERROR;
}
log_debug(LOG_NOTICE, "m %d listening on '%.*s:%u'",nc_stats_listen_sd,
addr.len, addr.data, stats_port);
return NC_OK;
}
static struct context *
core_ctx_create(struct instance *nci)
{
rstatus_t status;
struct context *ctx;
struct conn *sentinel_conn;
ctx = nc_alloc(sizeof(*ctx));
if (ctx == NULL) {
return NULL;
}
ctx->id = ++ctx_id;
ctx->cf = NULL;
ctx->stats = NULL;
array_null(&ctx->pool);
ctx->ep = -1;
ctx->nevent = EVENT_SIZE_HINT;
ctx->max_timeout = nci->stats_interval;
ctx->timeout = ctx->max_timeout;
ctx->event = NULL;
ctx->server_reconnect_interval = nci->server_reconnect_interval;
ctx->whitelist = nci->whitelist;
/* parse and create configuration */
ctx->cf = conf_create(nci->conf_filename);
if (ctx->cf == NULL) {
nc_free(ctx);
return NULL;
}
/* initialize server pool from configuration */
status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
if (status != NC_OK) {
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* initialize sentinel server */
ctx->sentinel = sentinel_init(nci->sentinel_port, nci->sentinel_addr);
if (ctx->sentinel == NULL) {
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* create stats per server pool */
ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
nci->hostname, &ctx->pool);
if (ctx->stats == NULL) {
sentinel_deinit(ctx->sentinel);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* initialize event handling for client, proxy and server */
status = event_init(ctx, EVENT_SIZE_HINT);
if (status != NC_OK) {
stats_destroy(ctx->stats);
sentinel_deinit(ctx->sentinel);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* preconnect? servers in server pool */
status = server_pool_preconnect(ctx);
if (status != NC_OK) {
server_pool_disconnect(ctx);
event_deinit(ctx);
stats_destroy(ctx->stats);
sentinel_deinit(ctx->sentinel);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* initialize sentinel server conn */
sentinel_conn = sentinel_connect(ctx);
if (sentinel_conn == NULL) {
server_pool_disconnect(ctx);
event_deinit(ctx);
stats_destroy(ctx->stats);
sentinel_deinit(ctx->sentinel);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
}
/* initialize proxy per server pool */
status = proxy_init(ctx);
if (status != NC_OK) {
sentinel_conn->close(ctx, sentinel_conn);
server_pool_disconnect(ctx);
event_deinit(ctx);
stats_destroy(ctx->stats);
sentinel_deinit(ctx->sentinel);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
return ctx;
}
static struct context *
core_ctx_create(struct instance *nci)
{
rstatus_t status;
struct context *ctx;
int64_t now;
uint32_t npool;
ctx = nc_alloc(sizeof(*ctx));
if (ctx == NULL) {
return NULL;
}
now = nc_msec_now();
if (now < 0) {
nc_free(ctx);
return NULL;
}
ctx->id = ++ctx_id;
ctx->cf = NULL;
ctx->stats = NULL;
ctx->evb = NULL;
array_null(&ctx->pool);
ctx->max_timeout = nci->stats_interval;
ctx->timeout = ctx->max_timeout;
ctx->next_tick = now + NC_TICK_INTERVAL;
/* parse and create configuration */
ctx->cf = conf_create(nci->conf_filename);
if (ctx->cf == NULL) {
nc_free(ctx);
return NULL;
}
npool = array_n(&ctx->cf->pool);
/* initialize server pool from configuration */
if (npool != 0) {
status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
if (status != NC_OK) {
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
}
/* create stats per server pool */
if (npool != 0) {
ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
nci->hostname, &ctx->pool);
if (ctx->stats == NULL) {
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
}
/* initialize event handling for client, proxy and server */
ctx->evb = evbase_create(NC_EVENT_SIZE, &core_core);
if (ctx->evb == NULL) {
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* preconnect? servers in server pool */
if (npool != 0) {
status = server_pool_preconnect(ctx);
if (status != NC_OK) {
server_pool_disconnect(ctx);
evbase_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
}
/* initialize proxy per server pool */
if (npool != 0) {
status = proxy_init(ctx);
if (status != NC_OK) {
server_pool_disconnect(ctx);
evbase_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
}
log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
return ctx;
}
struct evbase *
evbase_create(int nevent, void (*callback_fp)(void *, uint32_t))
{
struct evbase *evb;
int status, kq;
struct kevent *changes, *kevents;
if (nevent <= 0) {
log_error("nevent has to be positive %d", nevent);
return NULL;
}
/* Initialize the kernel queue */
if ((kq = kqueue()) == -1) {
log_error("kernel queue create failed: %s", kq, strerror(errno));
return NULL;
}
changes = nc_calloc(nevent, sizeof(*changes));
if (changes == NULL) {
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
kevents = nc_calloc(nevent, sizeof(*kevents));
if (kevents == NULL) {
nc_free(changes);
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
evb = (struct evbase *) nc_alloc(sizeof(*evb));
if (evb == NULL) {
nc_free(changes);
nc_free(kevents);
status = close(kq);
if (status < 0) {
log_error("close kq %d failed, ignored: %s", kq, strerror(errno));
}
return NULL;
}
evb->kq = kq;
evb->changes = changes;
evb->kevents = kevents;
evb->nevent = nevent;
evb->callback_fp = callback_fp;
evb->n_changes = 0;
log_debug(LOG_INFO, "kq %d with nevent %d", evb->kq,
evb->nevent);
return evb;
}
struct stats *
stats_create(uint16_t stats_port, char *stats_ip, int stats_interval,
char *source, struct array *server_pool)
{
rstatus_t status;
struct stats *st;
st = nc_alloc(sizeof(*st));
if (st == NULL) {
return NULL;
}
st->port = stats_port;
st->interval = stats_interval;
string_set_raw(&st->addr, stats_ip);
st->start_ts = (int64_t)time(NULL);
st->buf.len = 0;
st->buf.data = NULL;
st->buf.size = 0;
array_null(&st->current);
array_null(&st->shadow);
array_null(&st->sum);
st->tid = (pthread_t) -1;
st->sd = -1;
string_set_text(&st->service_str, "service");
string_set_text(&st->service, "nutcracker");
string_set_text(&st->source_str, "source");
string_set_raw(&st->source, source);
string_set_text(&st->version_str, "version");
string_set_text(&st->version, NC_VERSION_STRING);
string_set_text(&st->uptime_str, "uptime");
string_set_text(&st->timestamp_str, "timestamp");
string_set_text(&st->ntotal_conn_str, "total_connections");
string_set_text(&st->ncurr_conn_str, "curr_connections");
st->updated = 0;
st->aggregate = 0;
/* map server pool to current (a), shadow (b) and sum (c) */
status = stats_pool_map(&st->current, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_pool_map(&st->shadow, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_pool_map(&st->sum, server_pool);
if (status != NC_OK) {
goto error;
}
status = stats_create_buf(st); //为stats格式信息分配buf空间
if (status != NC_OK) {
goto error;
}
//stats状态信息专门起一个线程来处理
status = stats_start_aggregator(st);
if (status != NC_OK) {
goto error;
}
return st;
error:
stats_destroy(st);
return NULL;
}
/*
* read the comment in proto/nc_redis.c
*/
static rstatus_t
memcache_fragment_retrieval(struct msg *r, uint32_t ncontinuum,
struct msg_tqh *frag_msgq,
uint32_t key_step)
{
struct mbuf *mbuf;
struct msg **sub_msgs;
uint32_t i;
rstatus_t status;
sub_msgs = nc_zalloc(ncontinuum * sizeof(*sub_msgs));
if (sub_msgs == NULL) {
return NC_ENOMEM;
}
ASSERT(r->frag_seq == NULL);
r->frag_seq = nc_alloc(array_n(r->keys) * sizeof(*r->frag_seq));
if (r->frag_seq == NULL) {
nc_free(sub_msgs);
return NC_ENOMEM;
}
mbuf = STAILQ_FIRST(&r->mhdr);
mbuf->pos = mbuf->start;
/*
* This code is based on the assumption that 'gets ' is located
* in a contiguous location.
* This is always true because we have capped our MBUF_MIN_SIZE at 512 and
* whenever we have multiple messages, we copy the tail message into a new mbuf
*/
for (; *(mbuf->pos) != ' ';) { /* eat get/gets */
mbuf->pos++;
}
mbuf->pos++;
r->frag_id = msg_gen_frag_id();
r->nfrag = 0;
r->frag_owner = r;
for (i = 0; i < array_n(r->keys); i++) { /* for each key */
struct msg *sub_msg;
struct keypos *kpos = array_get(r->keys, i);
uint32_t idx = msg_backend_idx(r, kpos->start, kpos->end - kpos->start);
if (sub_msgs[idx] == NULL) {
sub_msgs[idx] = msg_get(r->owner, r->request, r->redis);
if (sub_msgs[idx] == NULL) {
nc_free(sub_msgs);
return NC_ENOMEM;
}
}
r->frag_seq[i] = sub_msg = sub_msgs[idx];
sub_msg->narg++;
status = memcache_append_key(sub_msg, kpos->start, kpos->end - kpos->start);
if (status != NC_OK) {
nc_free(sub_msgs);
return status;
}
}
for (i = 0; i < ncontinuum; i++) { /* prepend mget header, and forward it */
struct msg *sub_msg = sub_msgs[i];
if (sub_msg == NULL) {
continue;
}
/* prepend get/gets */
if (r->type == MSG_REQ_MC_GET) {
status = msg_prepend(sub_msg, (uint8_t *)"get ", 4);
} else if (r->type == MSG_REQ_MC_GETS) {
status = msg_prepend(sub_msg, (uint8_t *)"gets ", 5);
}
if (status != NC_OK) {
nc_free(sub_msgs);
return status;
}
/* append \r\n */
status = msg_append(sub_msg, (uint8_t *)CRLF, CRLF_LEN);
if (status != NC_OK) {
nc_free(sub_msgs);
return status;
}
sub_msg->type = r->type;
sub_msg->frag_id = r->frag_id;
sub_msg->frag_owner = r->frag_owner;
TAILQ_INSERT_TAIL(frag_msgq, sub_msg, m_tqe);
r->nfrag++;
}
nc_free(sub_msgs);
return NC_OK;
}
static struct context *
core_ctx_create(struct instance *nci)
{
rstatus_t status;
struct context *ctx;
ctx = nc_alloc(sizeof(*ctx));
if (ctx == NULL) {
return NULL;
}
ctx->id = ++ctx_id;
ctx->cf = NULL;
ctx->stats = NULL;
ctx->evb = NULL;
array_null(&ctx->pool);
ctx->max_timeout = nci->stats_interval;
ctx->timeout = ctx->max_timeout;
ctx->max_nfd = 0;
ctx->max_ncconn = 0;
ctx->max_nsconn = 0;
/* parse and create configuration */
ctx->cf = conf_create(nci->conf_filename);
if (ctx->cf == NULL) {
nc_free(ctx);
return NULL;
}
/* initialize server pool from configuration */
status = server_pool_init(&ctx->pool, &ctx->cf->pool, ctx);
if (status != NC_OK) {
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/*
* Get rlimit and calculate max client connections after we have
* calculated max server connections
*/
status = core_calc_connections(ctx);
if (status != NC_OK) {
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* create stats per server pool */
ctx->stats = stats_create(nci->stats_port, nci->stats_addr, nci->stats_interval,
nci->hostname, &ctx->pool);
if (ctx->stats == NULL) {
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* initialize event handling for client, proxy and server */
ctx->evb = event_base_create(EVENT_SIZE, &core_core);
if (ctx->evb == NULL) {
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* preconnect? servers in server pool */
status = server_pool_preconnect(ctx);
if (status != NC_OK) {
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
/* initialize proxy per server pool */
status = proxy_init(ctx);
if (status != NC_OK) {
server_pool_disconnect(ctx);
event_base_destroy(ctx->evb);
stats_destroy(ctx->stats);
server_pool_deinit(&ctx->pool);
conf_destroy(ctx->cf);
nc_free(ctx);
return NULL;
}
log_debug(LOG_VVERB, "created ctx %p id %"PRIu32"", ctx, ctx->id);
return ctx;
}
struct server *
sentinel_init(uint16_t sentinel_port, char *sentinel_ip)
{
rstatus_t status;
struct server *sentinel;
struct string address;
struct sockinfo info;
char pname[NC_PNAME_MAXLEN];
string_init(&address);
sentinel_status = SENTINEL_CONN_DISCONNECTED;
sentinel = (struct server *)nc_alloc(sizeof(*sentinel));
if(sentinel == NULL) {
goto error;
}
/* sentinel server don't have owner server pool */
sentinel->owner = NULL;
sentinel->ns_conn_q = 0;
TAILQ_INIT(&sentinel->s_conn_q);
sentinel->addr = NULL;
string_init(&sentinel->pname);
string_init(&sentinel->name);
nc_snprintf(pname, NC_PNAME_MAXLEN, "%s:%d:0", sentinel_ip, sentinel_port);
status = string_copy(&sentinel->pname, pname, (uint32_t)(nc_strlen(pname)));
if (status != NC_OK) {
goto error;
}
string_copy(&sentinel->name, pname, (uint32_t)(nc_strlen(pname)) - 2);
if (status != NC_OK) {
goto error;
}
sentinel->port = sentinel_port;
status = string_copy(&address, sentinel_ip, (uint32_t)(nc_strlen(sentinel_ip)));
if (status != NC_OK) {
goto error;
}
status = nc_resolve(&address, sentinel_port, &info);
if (status != NC_OK) {
goto error;
}
sentinel->family = info.family;
sentinel->addrlen = info.addrlen;
sentinel->addr = (struct sockaddr*)nc_alloc(info.addrlen);
if (sentinel->addr == NULL) {
goto error;
}
nc_memcpy(sentinel->addr, &info.addr, info.addrlen);
done:
string_deinit(&address);
return sentinel;
error:
sentinel_deinit(sentinel);
sentinel = NULL;
goto done;
}
static rstatus_t
stats_master_send_rsp(int *psd)
{
int n;
ssize_t len;
int sd;
int i;
nc_channel_msg_t message;
char buf[1000];
memset(buf, 0, sizeof(char)*1000);
char *snd_buf = NULL;
memset(&message, 0, sizeof(nc_channel_msg_t));
message.command = NC_CMD_GET_STATS;
if (*psd) {
sd = accept(*psd, NULL, NULL);
if (sd < 0) {
log_error("accept on m %d failed: %s", sd, strerror(errno));
return NC_ERROR;
}
}
// still in reconfiguration process
if (nc_reload_start) {
sprintf(buf, "%s", "no stats get due to still during reconfiguration period.");
if (*psd) {
len = nc_strlen(buf)+1;
len = nc_sendn(sd, buf, len);
if (len < 0) {
log_error("send stats on sd %d failed: %s", sd, strerror(errno));
}
close(sd);
}
return NC_OK;
}
//broadcast
for (i = 0; i < nc_last_process; i++)
{
if (nc_processes[i].pid == -1 || nc_processes[i].pid == 0)
{
continue;
}
if (nc_write_channel(nc_processes[i].channel[0],
&message, sizeof(nc_channel_msg_t))
== NC_OK)
{
if (nc_set_blocking(nc_processes[i].channel[0]) < 0) {
log_error("set channel %d block failed while core timeout %s",
nc_processes[i].channel[0] , strerror(errno));
continue;
}
n = nc_read_channel(nc_processes[i].channel[0], &env_global.ctrl_msg, sizeof(nc_channel_msg_t));
if (env_global.ctrl_msg.command != NC_CMD_GET_STATS || n < 0) {
log_error("failure: get stats from worker %d receive length, %s",
nc_processes[i].channel[0] , strerror(errno));
} else {
log_error("success: get stats from worker %d receive length, %d",
nc_processes[i].channel[0] , n);
snd_buf = nc_alloc(n + n);
len = nc_recvn(nc_processes[i].channel[0], snd_buf, n);
if (len < 0) {
if (*psd) {
log_error("recv stats on sd %d failed: %s", sd, strerror(errno));
}
nc_free(snd_buf);
snd_buf = NULL;
continue;
}
if (*psd) {
len = nc_sendn(sd, snd_buf, len);
if (len < 0) {
log_error("send stats on sd %d failed: %s", sd, strerror(errno));
nc_free(snd_buf);
snd_buf = NULL;
continue;
}
}
len = nc_write(env_global.stats_fd, snd_buf, len);
if (len < 0) {
log_error("nc_write %d failed: %s", env_global.stats_fd, strerror(errno));
continue;
}
nc_free(snd_buf);
snd_buf = NULL;
}
if (nc_set_nonblocking(nc_processes[i].channel[0]) < 0) {
log_error("set channel %d nonblock failed while core timeout %s",
nc_processes[i].channel[0] , strerror(errno));
}
}
}
if (*psd) {
shutdown(sd, SHUT_RDWR);
close(sd);
}
return NC_OK;
}
