static struct item* _item_alloc(uint8_t id, char *key, uint16_t nkey, int exptime, char *value, uint32_t nbyte) {
struct item *it;
struct item *uit;
assert(id >= SLABCLASS_MIN_ID && id <= SLABCLASS_MAX_ID);
it = item_get_from_lruq(id);
if (it != NULL && item_expired(it)) {
item_reuse(it);
goto done;
}
uit = (settings.evict_opt & EVICT_LRU)? it : NULL;
it = slab_get_item(id);
if (it != NULL) {
goto done;
}
if (uit != NULL) {
it = uit;
item_reuse(it);
goto done;
}
return NULL;
done:
assert(it->id == id);
assert(!item_is_linked(it));
assert(!item_is_slabbed(it));
assert(it->offset != 0);
assert(it->refcount == 0);
it->flags = 0;
it->nbyte = nbyte;
it->exptime = exptime + time_now();
it->nkey = nkey;
memcpy(item_key(it), key, nkey);
memcpy(item_key(it) + nkey, value, nbyte);
return it;
}
static inline void
_copy_key_item(struct item *nit, struct item *oit)
{
nit->olen = oit->olen;
cc_memcpy(item_key(nit), item_key(oit), oit->klen);
nit->klen = oit->klen;
}
/*
* Build the response. Each hit adds three elements to the outgoing
* reponse vector, viz:
* "VALUE "
* key
* " " + flags + " " + data length + "\r\n" + data (with \r\n)
*/
static rstatus_t
asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
bool return_cas)
{
rstatus_t status;
char *cas_suffix = NULL;
int cas_suffix_len = 0;
int total_len = it->nkey + it->nsuffix + it->nbyte;
if (return_cas) {
status = asc_create_cas_suffix(c, valid_key_iter, &cas_suffix);
if (status != MC_OK) {
return status;
}
cas_suffix_len = snprintf(cas_suffix, CAS_SUFFIX_SIZE, " %"PRIu64,
item_cas(it));
}
status = conn_add_iov(c, "VALUE ", sizeof("VALUE ") - 1);
if (status != MC_OK) {
return status;
}
status = conn_add_iov(c, item_key(it), it->nkey);
if (status != MC_OK) {
return status;
}
status = conn_add_iov(c, item_suffix(it), it->nsuffix - CRLF_LEN);
if (status != MC_OK) {
return status;
}
if (return_cas) {
status = conn_add_iov(c, cas_suffix, cas_suffix_len);
if (status != MC_OK) {
return status;
}
total_len += cas_suffix_len;
}
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
status = conn_add_iov(c, item_data(it), it->nbyte);
if (status != MC_OK) {
return status;
}
klog_write(c->peer, c->req_type, item_key(it), it->nkey, 0, total_len);
return MC_OK;
}
void
hashtable_put(struct item *it, struct hash_table *ht)
{
struct item_slh *bucket;
ASSERT(hashtable_get(item_key(it), it->klen, ht) == NULL);
bucket = _get_bucket(item_key(it), it->klen, ht);
SLIST_INSERT_HEAD(bucket, it, i_sle);
++(ht->nhash_item);
}
void
hashtable_delete(const char *key, uint32_t klen, struct hash_table *ht)
{
struct item_slh *bucket;
struct item *it, *prev;
ASSERT(hashtable_get(key, klen, ht) != NULL);
bucket = _get_bucket(key, klen, ht);
for (prev = NULL, it = SLIST_FIRST(bucket); it != NULL;
prev = it, it = SLIST_NEXT(it, i_sle)) {
/* iterate through bucket to find item to be removed */
if ((klen == it->klen) && cc_memcmp(key, item_key(it), klen) == 0) {
/* found item */
break;
}
}
if (prev == NULL) {
SLIST_REMOVE_HEAD(bucket, i_sle);
} else {
SLIST_REMOVE_AFTER(prev, i_sle);
}
--(ht->nhash_item);
}
void item_reuse(struct item *it) {
assert(pthread_mutex_trylock(&cache_lock) != 0);
assert(it->magic == ITEM_MAGIC);
assert(!item_is_slabbed(it));
assert(item_is_linked(it));
assert(it->refcount == 0);
it->flags &= ~ITEM_LINKED;
assoc_delete(item_key(it), it->nkey);
item_unlink_q(it);
}
static void _item_unlink(struct item *it) {
assert(it->magic == ITEM_MAGIC);
assert(item_is_linked(it));
if (item_is_linked(it)) {
it->flags &= ~ITEM_LINKED;
assoc_delete(item_key(it), it->nkey);
item_unlink_q(it);
if (it->refcount == 0) {
item_free(it);
}
}
}
/* TODO(yao): move this to memcache-specific location */
static void
_item_define(struct item *it, const struct bstring *key, const struct bstring
*val, uint8_t olen, proc_time_i expire_at)
{
proc_time_i expire_cap = time_delta2proc_sec(max_ttl);
it->create_at = time_proc_sec();
it->expire_at = expire_at < expire_cap ? expire_at : expire_cap;
item_set_cas(it);
it->olen = olen;
cc_memcpy(item_key(it), key->data, key->len);
it->klen = key->len;
if (val != NULL) {
cc_memcpy(item_data(it), val->data, val->len);
}
it->vlen = (val == NULL) ? 0 : val->len;
}
/*
* Unlinks an item from the hash table.
*/
static void
_item_unlink(struct item *it)
{
ASSERT(it->magic == ITEM_MAGIC);
log_verb("unlink it %p of id %"PRIu8" at offset %"PRIu32, it, it->id,
it->offset);
if (it->is_linked) {
it->is_linked = 0;
hashtable_delete(item_key(it), it->klen, hash_table);
}
DECR(slab_metrics, item_linked_curr);
INCR(slab_metrics, item_unlink);
DECR_N(slab_metrics, item_keyval_byte, it->klen + it->vlen);
DECR_N(slab_metrics, item_val_byte, it->vlen);
PERSLAB_DECR_N(it->id, item_keyval_byte, it->klen + it->vlen);
PERSLAB_DECR_N(it->id, item_val_byte, it->vlen);
}
struct item *
hashtable_get(const char *key, uint32_t klen, struct hash_table *ht)
{
struct item_slh *bucket;
struct item *it;
ASSERT(key != NULL);
ASSERT(klen != 0);
bucket = _get_bucket(key, klen, ht);
/* iterate through bucket looking for item */
for (it = SLIST_FIRST(bucket); it != NULL; it = SLIST_NEXT(it, i_sle)) {
if ((klen == it->klen) && cc_memcmp(key, item_key(it), klen) == 0) {
/* found item */
return it;
}
}
return NULL;
}
static void *
assoc_maintenance_thread(void *arg)
{
uint32_t i, hv;
struct item_slh *old_bucket, *new_bucket;
struct item *it, *next;
while (run_maintenance_thread) {
/*
* Lock the cache, and bulk move multiple buckets to the new
* hash table
*/
pthread_mutex_lock(&cache_lock);
for (i = 0; i < nhash_move_size && expanding == 1; i++) {
old_bucket = &old_hashtable[expand_bucket];
SLIST_FOREACH_SAFE(it, old_bucket, h_sle, next) {
hv = hash(item_key(it), it->nkey, 0);
new_bucket = &primary_hashtable[hv & HASHMASK(hash_power)];
SLIST_REMOVE(old_bucket, it, item, h_sle);
SLIST_INSERT_HEAD(new_bucket, it, h_sle);
}
expand_bucket++;
if (expand_bucket == HASHSIZE(hash_power - 1)) {
expanding = 0;
mc_free(old_hashtable);
nbyte_old = 0;
}
}
if (expanding == 0) {
/* we are done expanding, just wait for the next invocation */
pthread_cond_wait(&maintenance_cond, &cache_lock);
}
pthread_mutex_unlock(&cache_lock);
}
/*
* Evict a slab by evicting all the items within it. This means that the
* items that are carved out of the slab must either be deleted from their
* a) hash + lru Q, or b) free Q. The candidate slab itself must also be
* delinked from its respective slab pool so that it is available for reuse.
*
* Eviction complexity is O(#items/slab).
*/
static void
_slab_evict_one(struct slab *slab)
{
struct slabclass *p;
struct item *it;
uint32_t i;
p = &slabclass[slab->id];
INCR(slab_metrics, slab_evict);
/* candidate slab is also the current slab */
if (p->next_item_in_slab != NULL && slab == item_to_slab(p->next_item_in_slab)) {
p->nfree_item = 0;
p->next_item_in_slab = NULL;
}
/* delete slab items either from hash or free Q */
for (i = 0; i < p->nitem; i++) {
it = _slab_to_item(slab, i, p->size);
if (it->is_linked) {
it->is_linked = 0;
hashtable_delete(item_key(it), it->klen, hash_table);
} else if (it->in_freeq) {
ASSERT(slab == item_to_slab(it));
ASSERT(!SLIST_EMPTY(&p->free_itemq));
ASSERT(p->nfree_itemq > 0);
it->in_freeq = 0;
p->nfree_itemq--;
SLIST_REMOVE(&p->free_itemq, it, item, i_sle);
}
}
/* unlink the slab from its class */
_slab_lruq_remove(slab);
}
/* gets the inode of a directory entry, or 0 if errno.
* (inode 0 is the root dir, which is not an entry in any dir)
*/
PUBLIC uint32_t get_dir_ent_inode(struct fs_info *fsi, uint32_t dir_inode,
char *name) {
struct path p;
struct key key;
struct dir_ent_metadata *demd;
int ret;
set_dir_ent_key(dir_inode, name, &key);
ret = search_slot(&fsi->fs_root, &key, &p, 0);
if (ret == KEY_NOT_FOUND) {
free_path(&p); /* only for search_slot(), not step_to_next_slot() */
}
while (TRUE) {
if (ret == KEY_NOT_FOUND) {
errno = ENOENT;
}
if (ret < 0) {
errno = -ret;
}
if (ret != KEY_FOUND) {
return 0; /* failure, with errno */
}
demd = (struct dir_ent_metadata *) metadata_for(&p);
if (!strcmp(name, demd->name)) { /* name matches */
ret = demd->inode;
free_path(&p);
return ret;
}
ret = step_to_next_slot(&p);
if (ret == KEY_FOUND && compare_keys(&key, item_key(&p))) {
errno = ENOENT;
free_path(&p);
return 0; /* failure, with errno */
}
}
}
static inline void
asc_process_read(struct conn *c, struct token *token, int ntoken)
{
rstatus_t status;
char *key;
size_t nkey;
unsigned valid_key_iter = 0;
struct item *it;
struct token *key_token;
bool return_cas;
if (!asc_ntoken_valid(c, ntoken)) {
log_hexdump(LOG_NOTICE, c->req, c->req_len, "client error on c %d for "
"req of type %d with %d invalid tokens", c->sd,
c->req_type, ntoken);
asc_write_client_error(c);
return;
}
return_cas = (c->req_type == REQ_GETS) ? true : false;
key_token = &token[TOKEN_KEY];
do {
while (key_token->len != 0) {
key = key_token->val;
nkey = key_token->len;
if (nkey > KEY_MAX_LEN) {
log_debug(LOG_NOTICE, "client error on c %d for req of type %d "
"and %d length key", c->sd, c->req_type, nkey);
asc_write_client_error(c);
return;
}
if (return_cas) {
stats_thread_incr(gets);
} else {
stats_thread_incr(get);
}
it = item_get(key, nkey);
if (it != NULL) {
/* item found */
if (return_cas) {
stats_slab_incr(it->id, gets_hit);
} else {
stats_slab_incr(it->id, get_hit);
}
if (valid_key_iter >= c->isize) {
struct item **new_list;
new_list = mc_realloc(c->ilist, sizeof(struct item *) * c->isize * 2);
if (new_list != NULL) {
c->isize *= 2;
c->ilist = new_list;
} else {
item_remove(it);
break;
}
}
status = asc_respond_get(c, valid_key_iter, it, return_cas);
if (status != MC_OK) {
log_debug(LOG_NOTICE, "client error on c %d for req of type "
"%d with %d tokens", c->sd, c->req_type, ntoken);
stats_thread_incr(cmd_error);
item_remove(it);
break;
}
log_debug(LOG_VVERB, ">%d sending key %.*s", c->sd, it->nkey,
item_key(it));
item_touch(it);
*(c->ilist + valid_key_iter) = it;
valid_key_iter++;
} else {
/* item not found */
if (return_cas) {
stats_thread_incr(gets_miss);
} else {
stats_thread_incr(get_miss);
}
klog_write(c->peer, c->req_type, key, nkey, 1, 0);
}
key_token++;
}
/*
* If the command string hasn't been fully processed, get the next set
* of token.
*/
if (key_token->val != NULL) {
ntoken = asc_tokenize(key_token->val, token, TOKEN_MAX);
/* ntoken is unused */
key_token = token;
}
} while (key_token->val != NULL);
c->icurr = c->ilist;
c->ileft = valid_key_iter;
if (return_cas) {
c->scurr = c->slist;
c->sleft = valid_key_iter;
}
log_debug(LOG_VVERB, ">%d END", c->sd);
/*
* If the loop was terminated because of out-of-memory, it is not
* reliable to add END\r\n to the buffer, because it might not end
* in \r\n. So we send SERVER_ERROR instead.
*/
if (key_token->val != NULL || conn_add_iov(c, "END\r\n", 5) != MC_OK ||
(c->udp && conn_build_udp_headers(c) != MC_OK)) {
log_warn("server error on c %d for req of type %d with enomem", c->sd,
c->req_type);
asc_write_server_error(c);
} else {
conn_set_state(c, CONN_MWRITE);
c->msg_curr = 0;
}
}
/*
* Build the response. Each hit adds three elements to the outgoing
* reponse vector, viz:
* "VALUE "
* key
* " " + flags + " " + data length + "\r\n" + data (with \r\n)
*/
static rstatus_t
asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
bool return_cas)
{
rstatus_t status;
char *cas_suffix = NULL;
char suffix[SUFFIX_MAX_LEN];
int sz;
int total_len = 0;
status = conn_add_iov(c, "VALUE ", sizeof("VALUE ") - 1);
if (status != MC_OK) {
return status;
}
status = conn_add_iov(c, item_key(it), it->nkey);
if (status != MC_OK) {
return status;
}
total_len += it->nkey;
sz = snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32, it->dataflags,
it->nbyte);
if (sz < 0) {
return MC_ERROR;
}
ASSERT(sz < SUFFIX_MAX_LEN); /* or we have a corrupted item */
status = conn_add_iov(c, suffix, sz);
if (status != MC_OK) {
return status;
}
total_len += sz;
if (return_cas) {
status = asc_create_cas_suffix(c, valid_key_iter, &cas_suffix);
if (status != MC_OK) {
return status;
}
sz = snprintf(cas_suffix, CAS_SUFFIX_SIZE, " %"PRIu64, item_cas(it));
if (sz < 0) {
return MC_ERROR;
}
ASSERT(sz < CAS_SUFFIX_SIZE);
status = conn_add_iov(c, cas_suffix, sz);
if (status != MC_OK) {
return status;
}
total_len += sz;
}
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
total_len += CRLF_LEN;
status = conn_add_iov(c, item_data(it), it->nbyte);
if (status != MC_OK) {
return status;
}
total_len += it->nbyte;
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
total_len += CRLF_LEN;
klog_write(c->peer, c->req_type, item_key(it), it->nkey, 0, total_len);
return MC_OK;
}
/*
* Build the response. Each hit adds three elements to the outgoing
* reponse vector, viz:
* "VALUE "
* key
* " " + flags + " " + data length + "\r\n" + data (with \r\n)
*/
static rstatus_t
asc_respond_get(struct conn *c, unsigned valid_key_iter, struct item *it,
bool return_cas)
{
rstatus_t status;
char *suffix = NULL;
int sz;
int total_len = 0;
uint32_t nbyte = it->nbyte;
char *data = item_data(it);
status = conn_add_iov(c, VALUE, VALUE_LEN);
if (status != MC_OK) {
return status;
}
total_len += VALUE_LEN;
status = conn_add_iov(c, item_key(it), it->nkey);
if (status != MC_OK) {
return status;
}
total_len += it->nkey;
status = asc_create_suffix(c, valid_key_iter, &suffix);
if (status != MC_OK) {
return status;
}
if (return_cas) {
sz = mc_snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32" %"PRIu64,
it->dataflags, nbyte, item_cas(it));
ASSERT(sz <= SUFFIX_SIZE + CAS_SUFFIX_SIZE);
} else {
sz = mc_snprintf(suffix, SUFFIX_MAX_LEN, " %"PRIu32" %"PRIu32,
it->dataflags, nbyte);
ASSERT(sz <= SUFFIX_SIZE);
}
if (sz < 0) {
return MC_ERROR;
}
status = conn_add_iov(c, suffix, sz);
if (status != MC_OK) {
return status;
}
total_len += sz;
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
total_len += CRLF_LEN;
status = conn_add_iov(c, data, nbyte);
if (status != MC_OK) {
return status;
}
total_len += nbyte;
status = conn_add_iov(c, CRLF, CRLF_LEN);
if (status != MC_OK) {
return status;
}
total_len += CRLF_LEN;
klog_write(c->peer, c->req_type, item_key(it), it->nkey, 0, total_len);
return MC_OK;
}
void
slab_put_item(struct item *it)
{
log_debug(LOG_INFO, "put it '%.*s' at offset %"PRIu32" with cid %"PRIu8,
it->nkey, item_key(it), it->offset, it->cid);
}
