static bool
_get_key(struct response *rsp, struct bstring *key)
{
struct item *it;
struct val val;
it = cuckoo_get(key);
if (it != NULL) {
rsp->type = RSP_VALUE;
rsp->key = *key;
rsp->flag = item_flag(it);
rsp->vcas = item_cas(it);
item_val(&val, it);
if (val.type == VAL_TYPE_INT) {
rsp->num = 1;
rsp->vint = val.vint;
} else {
rsp->vstr = val.vstr;
}
log_verb("found key at %p, location %p", key, it);
return true;
} else {
log_verb("key at %p not found", key);
return false;
}
}
static void
_server_event(void *arg, uint32_t events)
{
struct buf_sock *s = arg;
log_verb("server event %06"PRIX32" on buf_sock %p", events, s);
if (events & EVENT_ERR) {
INCR(server_metrics, server_event_error);
_server_close(s);
return;
}
if (events & EVENT_READ) {
log_verb("processing server read event on buf_sock %p", s);
INCR(server_metrics, server_event_read);
_server_event_read(s);
}
if (events & EVENT_WRITE) {
/* the only server write event is write on pipe */
log_verb("processing server write event");
_server_pipe_write();
INCR(server_metrics, server_event_write);
}
}
/**
* Return an item if it hasn't been marked as expired, lazily expiring
* item as-and-when needed
*/
struct item *
item_get(const struct bstring *key)
{
struct item *it;
it = hashtable_get(key->data, key->len, hash_table);
if (it == NULL) {
log_verb("get it '%.*s' not found", key->len, key->data);
return NULL;
}
log_verb("get it key %.*s val %.*s", key->len, key->data, it->vlen,
item_data(it));
if (_item_expired(it)) {
log_verb("get it '%.*s' expired and nuked", key->len, key->data);
_item_unlink(it);
_item_dealloc(&it);
return NULL;
}
log_verb("get it %p of id %"PRIu8, it, it->id);
return it;
}
/*
* Allocate an item. We allocate an item by consuming the next free item
* from slab of the item's slab class.
*
* On success we return the pointer to the allocated item.
*/
static item_rstatus_e
_item_alloc(struct item **it_p, uint8_t klen, uint32_t vlen, uint8_t olen)
{
uint8_t id = slab_id(item_ntotal(klen, vlen, olen));
struct item *it;
log_verb("allocate item with klen %u vlen %u", klen, vlen);
*it_p = NULL;
if (id == SLABCLASS_INVALID_ID) {
return ITEM_EOVERSIZED;
}
it = slab_get_item(id);
*it_p = it;
if (it != NULL) {
_item_reset(it);
slab_ref(item_to_slab(it)); /* slab to be deref'ed in _item_link */
INCR(slab_metrics, item_curr);
INCR(slab_metrics, item_alloc);
PERSLAB_INCR(id, item_curr);
log_verb("alloc it %p of id %"PRIu8" at offset %"PRIu32, it, it->id,
it->offset);
return ITEM_OK;
} else {
INCR(slab_metrics, item_alloc_ex);
log_warn("server error on allocating item in slab %"PRIu8, id);
return ITEM_ENOMEM;
}
}
/*
* TODO: return size_t , but actually we return NC_ERROR
* */
static ssize_t
conn_recv_buf(struct conn *conn, void *buf, size_t size)
{
ssize_t n;
ASSERT(buf != NULL);
ASSERT(size > 0);
ASSERT(conn->recv_ready);
for (;;) {
n = nc_read(conn->fd, buf, size);
log_verb("recv on conn:%p, fd:%d got %zd/%zu", conn, conn->fd, n, size);
if (n > 0) {
if (n < (ssize_t)size) {
conn->recv_ready = 0;
}
conn->recv_bytes += (size_t)n;
conn->recv_queue_bytes += (size_t)n;
return n;
}
if (n == 0) {
conn->recv_ready = 0;
conn->eof = 1;
log_info("recv on conn:%p fd:%d eof rb %zu sb %zu", conn, conn->fd,
conn->recv_bytes, conn->send_bytes);
return n;
}
if (errno == EINTR) {
log_verb("recv on conn:%p fd:%d not ready - eintr", conn, conn->fd);
continue;
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
conn->recv_ready = 0;
log_verb("recv on conn:%p fd:%d not ready - eagain", conn, conn->fd);
return NC_EAGAIN;
} else {
conn->recv_ready = 0;
conn->err = errno;
log_error("recv on conn:%p fd:%d failed: %s", conn, conn->fd, strerror(errno));
return NC_ERROR;
}
}
NOT_REACHED();
return NC_ERROR;
}
static ssize_t
conn_send_buf(struct conn *conn, void *buf, size_t size)
{
ssize_t n;
ASSERT(buf != NULL);
ASSERT(size > 0);
ASSERT(conn->send_ready);
for (;;) {
n = nc_write(conn->fd, buf, size);
log_verb("sendv on fd %d %zd of %zu",
conn->fd, n, size);
if (n > 0) {
if (n < (ssize_t)size) {
conn->send_ready = 0;
}
conn->send_bytes += (size_t)n;
return n;
}
if (n == 0) {
log_warn("sendv on fd %d returned zero", conn->fd);
conn->send_ready = 0;
return 0;
}
if (errno == EINTR) {
log_verb("sendv on fd %d not ready - eintr", conn->fd);
continue;
} else if (errno == EAGAIN || errno == EWOULDBLOCK) {
conn->send_ready = 0;
log_verb("sendv on fd %d not ready - eagain", conn->fd);
return NC_EAGAIN;
} else {
conn->send_ready = 0;
conn->err = errno;
log_error("sendv on fd %d failed: %s", conn->fd, strerror(errno));
return NC_ERROR;
}
}
NOT_REACHED();
return NC_ERROR;
}
static void
_process_get(struct response *rsp, struct request *req)
{
struct bstring *key;
struct response *r = rsp;
uint32_t i;
INCR(process_metrics, get);
/* use chained responses, move to the next response if key is found. */
for (i = 0; i < array_nelem(req->keys); ++i) {
INCR(process_metrics, get_key);
key = array_get(req->keys, i);
if (_get_key(r, key)) {
req->nfound++;
r->cas = false;
r = STAILQ_NEXT(r, next);
if (r == NULL) {
INCR(process_metrics, get_ex);
log_warn("get response incomplete due to lack of rsp objects");
return;
}
INCR(process_metrics, get_key_hit);
} else {
INCR(process_metrics, get_key_miss);
}
}
r->type = RSP_END;
log_verb("get req %p processed, %d out of %d keys found", req, req->nfound, i);
}
/*
* Get an item from the item free q of the given slab with id.
*/
static struct item *
_slab_get_item_from_freeq(uint8_t id)
{
struct slabclass *p; /* parent slabclass */
struct item *it;
if (!use_freeq) {
return NULL;
}
p = &slabclass[id];
if (p->nfree_itemq == 0) {
return NULL;
}
it = SLIST_FIRST(&p->free_itemq);
ASSERT(it->magic == ITEM_MAGIC);
ASSERT(it->in_freeq);
ASSERT(!(it->is_linked));
it->in_freeq = 0;
ASSERT(p->nfree_itemq > 0);
p->nfree_itemq--;
SLIST_REMOVE(&p->free_itemq, it, item, i_sle);
PERSLAB_DECR(id, item_free);
log_verb("get free q it %p at offset %"PRIu32" with id %"PRIu8, it,
it->offset, it->id);
return it;
}
/*
* Get an item from the slab with a given id. We get an item either from:
* 1. item free Q of given slab with id. or,
* 2. current slab.
* If the current slab is empty, we get a new slab from the slab allocator
* and return the next item from this new slab.
*/
static struct item *
_slab_get_item(uint8_t id)
{
struct slabclass *p;
struct item *it;
p = &slabclass[id];
it = _slab_get_item_from_freeq(id);
if (it != NULL) {
return it;
}
if (p->next_item_in_slab == NULL && (_slab_get(id) != CC_OK)) {
return NULL;
}
/* return item from current slab */
it = p->next_item_in_slab;
if (--p->nfree_item != 0) {
p->next_item_in_slab = (struct item *)((char *)p->next_item_in_slab + p->size);
} else {
p->next_item_in_slab = NULL;
}
log_verb("get new it at offset %"PRIu32" with id %"PRIu8"",
it->offset, it->id);
return it;
}
void
cmd_list_create(struct response *rsp, struct request *req, struct command *cmd)
{
struct item *it;
struct bstring *key = _get_key(req);
struct element *reply = (struct element *)array_push(rsp->token);
INCR(process_metrics, list_create);
it = _add_key(rsp, key);
if (it == NULL) {
log_debug("command '%.*s' '%.*s' failed: cannot store", cmd->bstr.len,
cmd->bstr.data, key->len, key->data);
return;
}
/* initialize data structure */
ziplist_reset((ziplist_p)item_data(it));
it->vlen = ZIPLIST_HEADER_SIZE;
/* link into index */
item_insert(it, key);
rsp->type = reply->type = ELEM_STR;
reply->bstr = str2bstr(RSP_OK);
log_verb("command '%.*s' '%.*s' succeeded", cmd->bstr.len, cmd->bstr.data,
key->len, key->data);
}
void
_nc_free(void *ptr, const char *name, int line)
{
ASSERT(ptr != NULL);
log_verb("free(%p) @ %s:%d", ptr, name, line);
free(ptr);
}
static void
_item_delete(struct item **it)
{
log_verb("delete it %p of id %"PRIu8, *it, (*it)->id);
_item_unlink(*it);
_item_dealloc(it);
}
void
item_update(struct item *it, const struct bstring *val)
{
ASSERT(item_slabid(it->klen, val->len, it->olen) == it->id);
it->vlen = val->len;
cc_memcpy(item_data(it), val->data, val->len);
item_set_cas(it);
log_verb("update it %p of id %"PRIu8, it, it->id);
}
static void
_slab_table_update(struct slab *slab)
{
ASSERT(heapinfo.nslab < heapinfo.max_nslab);
heapinfo.slab_table[heapinfo.nslab] = slab;
heapinfo.nslab++;
log_verb("new slab %p allocated at pos %u", slab,
heapinfo.nslab - 1);
}
void
item_insert(struct item *it, const struct bstring *key)
{
ASSERT(it != NULL && key != NULL);
item_delete(key);
_item_link(it);
log_verb("insert it %p of id %"PRIu8" for key %.*s", it, it->id, key->len,
key->data);
}
void
item_backfill(struct item *it, const struct bstring *val)
{
ASSERT(it != NULL);
cc_memcpy(item_data(it) + it->vlen, val->data, val->len);
it->vlen += val->len;
log_verb("backfill it %p with %"PRIu32" bytes, now %"PRIu32" bytes total",
it, val->len, it->vlen);
}
void
cmd_list_delete(struct response *rsp, struct request *req, struct command *cmd)
{
struct bstring *key = _get_key(req);
struct element *reply = (struct element *)array_push(rsp->token);
INCR(process_metrics, list_delete);
if (item_delete(key)) {
reply->bstr = str2bstr(RSP_OK);
INCR(process_metrics, list_delete_deleted);
log_verb("command '%.*s' '%.*s' succeeded", cmd->bstr.len,
cmd->bstr.data, key->len, key->data);
} else {
reply->bstr = str2bstr(RSP_NOTFOUND);
INCR(process_metrics, list_delete_notfound);
log_verb("command '%.*s' '%.*s' completed as no-op, key not found",
cmd->bstr.len, cmd->bstr.data, key->len, key->data);
}
}
static bool
_get_key(struct response *rsp, struct bstring *key)
{
struct item *it;
it = item_get(key);
if (it != NULL) {
rsp->type = RSP_VALUE;
rsp->key = *key;
rsp->flag = item_flag(it);
rsp->vcas = item_get_cas(it);
rsp->vstr.len = it->vlen;
rsp->vstr.data = item_data(it);
log_verb("found key at %p, location %p", key, it);
return true;
} else {
log_verb("key at %p not found", key);
return false;
}
}
void
ring_array_destroy(struct ring_array **arr)
{
log_verb("destroying ring array %p and freeing memory", *arr);
if ((arr == NULL) || (*arr == NULL)) {
log_warn("destroying NULL ring_array pointer");
return;
}
cc_free(*arr);
*arr = NULL;
}
void
conn_put(struct conn *conn)
{
ASSERT(conn->fd < 0);
ASSERT(STAILQ_EMPTY(&conn->recv_queue));
ASSERT(STAILQ_EMPTY(&conn->send_queue));
log_verb("put conn %p", conn);
/* TODO: free mbuf here */
nfree_connq++;
TAILQ_INSERT_HEAD(&free_connq, conn, conn_tqe);
}
struct conn *
conn_get(void *owner)
{
struct conn *conn;
conn = _conn_get();
if (conn == NULL) {
return NULL;
}
conn->owner = owner;
log_verb("get conn %p", conn);
return conn;
}
void *
_nc_realloc(void *ptr, size_t size, const char *name, int line)
{
void *p;
ASSERT(size != 0);
p = realloc(ptr, size);
if (p == NULL) {
log_error("realloc(%zu) failed @ %s:%d", size, name, line);
} else {
log_verb("realloc(%zu) at %p @ %s:%d", size, p, name, line);
}
return p;
}
static inline void
_server_pipe_write(void)
{
ASSERT(pipe_c != NULL);
ssize_t status = pipe_send(pipe_c, "", 1);
if (status == 0 || status == CC_EAGAIN) {
/* retry write */
log_verb("server core: retry send on pipe");
event_add_write(ctx->evb, pipe_write_id(pipe_c), NULL);
} else if (status == CC_ERROR) {
/* other reasn write can't be done */
log_error("could not write to pipe - %s", strerror(pipe_c->err));
}
/* else, pipe write succeeded and no action needs to be taken */
}
/*
* Unlinks an item from the hash table.
*/
static void
_item_unlink(struct item *it)
{
ASSERT(it->magic == ITEM_MAGIC);
log_verb("unlink it %p of id %"PRIu8" at offset %"PRIu32, it, it->id,
it->offset);
if (it->is_linked) {
it->is_linked = 0;
hashtable_delete(item_key(it), it->klen, hash_table);
}
DECR(slab_metrics, item_linked_curr);
INCR(slab_metrics, item_unlink);
DECR_N(slab_metrics, item_keyval_byte, it->klen + it->vlen);
DECR_N(slab_metrics, item_val_byte, it->vlen);
PERSLAB_DECR_N(it->id, item_keyval_byte, it->klen + it->vlen);
PERSLAB_DECR_N(it->id, item_val_byte, it->vlen);
}
item_rstatus_e
item_reserve(struct item **it_p, const struct bstring *key, const struct bstring
*val, uint32_t vlen, uint8_t olen, proc_time_i expire_at)
{
item_rstatus_e status;
struct item *it;
if ((status = _item_alloc(it_p, key->len, vlen, olen)) != ITEM_OK) {
log_debug("item reservation failed");
return status;
}
it = *it_p;
_item_define(it, key, val, olen, expire_at);
log_verb("reserve it %p of id %"PRIu8" for key '%.*s' optional len %"PRIu8,
it, it->id,key->len, key->data, olen);
return ITEM_OK;
}
/*
* Put an item back into the slab by inserting into the item free Q.
*/
static void
_slab_put_item_into_freeq(struct item *it, uint8_t id)
{
struct slabclass *p = &slabclass[id];
ASSERT(id >= SLABCLASS_MIN_ID && id <= profile_last_id);
ASSERT(item_to_slab(it)->id == id);
ASSERT(!(it->is_linked));
ASSERT(!(it->in_freeq));
ASSERT(it->offset != 0);
log_verb("put free q it %p at offset %"PRIu32" with id %"PRIu8, it,
it->offset, it->id);
it->in_freeq = 1;
p->nfree_itemq++;
SLIST_INSERT_HEAD(&p->free_itemq, it, i_sle);
PERSLAB_INCR(id, item_free);
}
/*
* Link an item into the hash table
*/
static void
_item_link(struct item *it)
{
ASSERT(it->magic == ITEM_MAGIC);
ASSERT(!(it->is_linked));
ASSERT(!(it->in_freeq));
log_verb("link it %p of id %"PRIu8" at offset %"PRIu32, it, it->id,
it->offset);
it->is_linked = 1;
slab_deref(item_to_slab(it)); /* slab ref'ed in _item_alloc */
hashtable_put(it, hash_table);
INCR(slab_metrics, item_linked_curr);
INCR(slab_metrics, item_link);
/* TODO(yao): how do we track optional storage? Separate or treat as val? */
INCR_N(slab_metrics, item_keyval_byte, it->klen + it->vlen);
INCR_N(slab_metrics, item_val_byte, it->vlen);
PERSLAB_INCR_N(it->id, item_keyval_byte, it->klen + it->vlen);
PERSLAB_INCR_N(it->id, item_val_byte, it->vlen);
}
/*
* Evict by looking into least recently used queue of all slabs.
*/
static struct slab *
_slab_evict_lru(int id)
{
struct slab *slab = _slab_lruq_head();
int i = 0;
while (slab != NULL && ++i < TRIES_MAX && !_slab_evict_ok(slab)) {
slab = TAILQ_NEXT(slab, s_tqe);
};
if (slab == NULL) {
/* warning here because eviction failure should be rare. This can
* indicate there are dead/idle connections hanging onto items and
* slab refcounts.
*/
log_warn("can't find a slab for lru-evicting slab with %d tries", i);
} else {
log_verb("lru-evicting slab %p with id %u", slab, slab->id);
_slab_evict_one(slab);
}
return slab;
}
/*
* Get a random slab from all active slabs and evict it for new allocation.
*
* Note that the slab_table enables us to have O(1) lookup for every slab in
* the system. The inserts into the table are just appends - O(1) and there
* are no deletes from the slab_table. These two constraints allows us to keep
* our random choice uniform.
*/
static struct slab *
_slab_evict_rand(void)
{
struct slab *slab;
int i = 0;
do {
slab = _slab_table_rand();
} while (slab != NULL && ++i < TRIES_MAX && !_slab_evict_ok(slab));
if (slab == NULL) {
/* warning here because eviction failure should be rare. This can
* indicate there are dead/idle connections hanging onto items and
* slab refcounts.
*/
log_warn("can't find a slab for random-evicting slab with %d tries", i);
} else {
log_verb("random-evicting slab %p with id %u", slab, slab->id);
_slab_evict_one(slab);
}
return slab;
}
void
slab_setup(slab_options_st *options, slab_metrics_st *metrics)
{
char *profile_str = SLAB_PROFILE;
log_info("set up the %s module", SLAB_MODULE_NAME);
if (slab_init) {
log_warn("%s has already been set up, re-creating", SLAB_MODULE_NAME);
slab_teardown();
}
log_verb("Slab header size: %d, item header size: %d", SLAB_HDR_SIZE,
ITEM_HDR_SIZE);
slab_metrics = metrics;
if (options != NULL) {
slab_size = option_uint(&options->slab_size);
slab_mem = option_uint(&options->slab_mem);
prealloc = option_bool(&options->slab_prealloc);
evict_opt = option_uint(&options->slab_evict_opt);
use_freeq = option_bool(&options->slab_use_freeq);
profile_str = option_str(&options->slab_profile);
item_min = option_uint(&options->slab_item_min);
item_max = option_uint(&options->slab_item_max);
item_growth = option_fpn(&options->slab_item_growth);
max_ttl = option_uint(&options->slab_item_max_ttl);
use_cas = option_bool(&options->slab_use_cas);
hash_power = option_uint(&options->slab_hash_power);
}
hash_table = hashtable_create(hash_power);
if (hash_table == NULL) {
log_crit("Could not create hash table");
goto error;
}
if (_slab_heapinfo_setup() != CC_OK) {
log_crit("Could not setup slab heap info");
goto error;
}
if (_slab_profile_setup(profile_str) != CC_OK) {
log_crit("Could not setup slab profile");
goto error;
}
if (_slab_slabclass_setup() != CC_OK) {
log_crit("Could not setup slabclasses");
goto error;
}
slab_init = true;
return;
error:
slab_teardown();
exit(EX_CONFIG);
}