//将item插入到哈希表和LRU队列中,hv为哈希值 int do_item_link(item *it, const uint32_t hv) { MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); //确保这个item已经从slab分配出去并且还没插入到LRU队列中 assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); mutex_lock(&cache_lock); //加入link标记 it->it_flags |= ITEM_LINKED; it->time = current_time; STATS_LOCK(); stats.curr_bytes += ITEM_ntotal(it); stats.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); /* Allocate a new CAS ID on link. */ ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); //插入到hash表中 assoc_insert(it, hv); //item插入到链表中 item_link_q(it); //引用计数加1 refcount_incr(&it->refcount); mutex_unlock(&cache_lock); return 1; }
/* 形成了一个完成的 item 后, 就要把它放入两个数据结构中, 一是 memcached 的哈希表, memcached 运行过程中只有一个哈希表, 二是 item 所在的 slabclass 的 LRU 队列. */ int do_item_link(item *it, const uint32_t hv) { MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); mutex_lock(&cache_lock); it->it_flags |= ITEM_LINKED; it->time = current_time; STATS_LOCK(); stats.curr_bytes += ITEM_ntotal(it); stats.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); /* Allocate a new CAS ID on link. */ ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); /* 把 item 放入哈希表 */ assoc_insert(it, hv); /* 把 item 放入 LRU 队列*/ item_link_q(it); refcount_incr(&it->refcount); mutex_unlock(&cache_lock); return 1; }
int do_item_link(item *it) { MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); assert(it->nbytes < (1024 * 1024)); /* 1MB max size */ it->it_flags |= ITEM_LINKED; it->time = current_time; assoc_insert(it); #ifdef MOXI_ITEM_MALLOC it->refcount++; #endif STATS_LOCK(); stats.curr_bytes += ITEM_ntotal(it); stats.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); /* Allocate a new CAS ID on link. */ ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); item_link_q(it); return 1; }
//将item加入到hashtable和LRU链中 int do_item_link(item *it, const uint32_t hv) { syslog(LOG_INFO, "[%s:%s:%d]", __FILE__, __func__, __LINE__); MEMCACHED_ITEM_LINK(ITEM_key(it), it->nkey, it->nbytes); assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0); //判断状态,即没有在hash表LRU链中或被释放 mutex_lock(&cache_lock); it->it_flags |= ITEM_LINKED; //设置linked状态 it->time = current_time;//设置最近访问的时间 STATS_LOCK(); stats.curr_bytes += ITEM_ntotal(it); //增加每个item所需要的字节大小,包括item结构体和item内容大小 stats.curr_items += 1; stats.total_items += 1; STATS_UNLOCK(); /* Allocate a new CAS ID on link. */ ITEM_set_cas(it, (settings.use_cas) ? get_cas_id() : 0); //设置新CAS,CAS是memcache用来处理并发请求的一种机制 assoc_insert(it, hv);//插入hashtable item_link_q(it); //加入LRU链 refcount_incr(&it->refcount); mutex_unlock(&cache_lock); return 1; }
void cproxy_process_a2a_downstream(conn *c, char *line) { assert(c != NULL); assert(c->next == NULL); assert(c->extra != NULL); assert(c->cmd == -1); assert(c->item == NULL); assert(line != NULL); assert(line == c->rcurr); assert(IS_ASCII(c->protocol)); assert(IS_PROXY(c->protocol)); if (settings.verbose > 1) fprintf(stderr, "<%d cproxy_process_a2a_downstream %s\n", c->sfd, line); downstream *d = c->extra; assert(d != NULL); assert(d->ptd != NULL); assert(d->ptd->proxy != NULL); if (strncmp(line, "VALUE ", 6) == 0) { token_t tokens[MAX_TOKENS]; size_t ntokens; unsigned int flags; int clen = 0; int vlen; uint64_t cas = CPROXY_NOT_CAS; ntokens = scan_tokens(line, tokens, MAX_TOKENS, &clen); if (ntokens >= 5 && // Accounts for extra termimation token. ntokens <= 6 && tokens[KEY_TOKEN].length <= KEY_MAX_LENGTH && safe_strtoul(tokens[2].value, (uint32_t *) &flags) && safe_strtoul(tokens[3].value, (uint32_t *) &vlen)) { char *key = tokens[KEY_TOKEN].value; size_t nkey = tokens[KEY_TOKEN].length; item *it = item_alloc(key, nkey, flags, 0, vlen + 2); if (it != NULL) { if (ntokens == 5 || safe_strtoull(tokens[4].value, &cas)) { ITEM_set_cas(it, cas); c->item = it; c->ritem = ITEM_data(it); c->rlbytes = it->nbytes; c->cmd = -1; conn_set_state(c, conn_nread); return; // Success. } else { if (settings.verbose > 1) fprintf(stderr, "cproxy could not parse cas\n"); } } else { if (settings.verbose > 1) fprintf(stderr, "cproxy could not item_alloc size %u\n", vlen + 2); } if (it != NULL) item_remove(it); it = NULL; c->sbytes = vlen + 2; // Number of bytes to swallow. conn_set_state(c, conn_swallow); // Note, eventually, we'll see an END later. } else { // We don't know how much to swallow, so close the downstream. // The conn_closing should release the downstream, // which should write a suffix/error to the upstream. // conn_set_state(c, conn_closing); } } else if (strncmp(line, "END", 3) == 0) { conn_set_state(c, conn_pause); } else if (strncmp(line, "OK", 2) == 0) { conn_set_state(c, conn_pause); // TODO: Handle flush_all's expiration parameter against // the front_cache. // // TODO: We flush the front_cache too often, inefficiently // on every downstream flush_all OK response, rather than // on just the last flush_all OK response. // conn *uc = d->upstream_conn; if (uc != NULL && uc->cmd_curr == PROTOCOL_BINARY_CMD_FLUSH) { mcache_flush_all(&d->ptd->proxy->front_cache, 0); } } else if (strncmp(line, "STAT ", 5) == 0 || strncmp(line, "ITEM ", 5) == 0 || strncmp(line, "PREFIX ", 7) == 0) { assert(d->merger != NULL); conn *uc = d->upstream_conn; if (uc != NULL) { assert(uc->next == NULL); if (protocol_stats_merge_line(d->merger, line) == false) { // Forward the line as-is if we couldn't merge it. // int nline = strlen(line); item *it = item_alloc("s", 1, 0, 0, nline + 2); if (it != NULL) { strncpy(ITEM_data(it), line, nline); strncpy(ITEM_data(it) + nline, "\r\n", 2); if (add_conn_item(uc, it)) { add_iov(uc, ITEM_data(it), nline + 2); it = NULL; } if (it != NULL) item_remove(it); } } } conn_set_state(c, conn_new_cmd); } else { conn_set_state(c, conn_pause); // The upstream conn might be NULL when closed already // or while handling a noreply. // conn *uc = d->upstream_conn; if (uc != NULL) { assert(uc->next == NULL); out_string(uc, line); if (!update_event(uc, EV_WRITE | EV_PERSIST)) { if (settings.verbose > 1) fprintf(stderr, "Can't update upstream write event\n"); d->ptd->stats.stats.err_oom++; cproxy_close_conn(uc); } cproxy_del_front_cache_key_ascii_response(d, line, uc->cmd_start); } } }
static int storage_write(void *storage, const int clsid, const int item_age) { int did_moves = 0; struct lru_pull_tail_return it_info; it_info.it = NULL; lru_pull_tail(clsid, COLD_LRU, 0, LRU_PULL_RETURN_ITEM, 0, &it_info); /* Item is locked, and we have a reference to it. */ if (it_info.it == NULL) { return did_moves; } obj_io io; item *it = it_info.it; /* First, storage for the header object */ size_t orig_ntotal = ITEM_ntotal(it); uint32_t flags; if ((it->it_flags & ITEM_HDR) == 0 && (item_age == 0 || current_time - it->time > item_age)) { FLAGS_CONV(it, flags); item *hdr_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, sizeof(item_hdr)); /* Run the storage write understanding the start of the item is dirty. * We will fill it (time/exptime/etc) from the header item on read. */ if (hdr_it != NULL) { int bucket = (it->it_flags & ITEM_CHUNKED) ? PAGE_BUCKET_CHUNKED : PAGE_BUCKET_DEFAULT; // Compress soon to expire items into similar pages. if (it->exptime - current_time < settings.ext_low_ttl) { bucket = PAGE_BUCKET_LOWTTL; } hdr_it->it_flags |= ITEM_HDR; io.len = orig_ntotal; io.mode = OBJ_IO_WRITE; // NOTE: when the item is read back in, the slab mover // may see it. Important to have refcount>=2 or ~ITEM_LINKED assert(it->refcount >= 2); // NOTE: write bucket vs free page bucket will disambiguate once // lowttl feature is better understood. if (extstore_write_request(storage, bucket, bucket, &io) == 0) { // cuddle the hash value into the time field so we don't have // to recalculate it. item *buf_it = (item *) io.buf; buf_it->time = it_info.hv; // copy from past the headers + time headers. // TODO: should be in items.c if (it->it_flags & ITEM_CHUNKED) { // Need to loop through the item and copy item_chunk *sch = (item_chunk *) ITEM_schunk(it); int remain = orig_ntotal; int copied = 0; // copy original header int hdrtotal = ITEM_ntotal(it) - it->nbytes; memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, hdrtotal - STORE_OFFSET); copied = hdrtotal; // copy data in like it were one large object. while (sch && remain) { assert(remain >= sch->used); memcpy((char *)io.buf+copied, sch->data, sch->used); // FIXME: use one variable? remain -= sch->used; copied += sch->used; sch = sch->next; } } else { memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, io.len-STORE_OFFSET); } // crc what we copied so we can do it sequentially. buf_it->it_flags &= ~ITEM_LINKED; buf_it->exptime = crc32c(0, (char*)io.buf+STORE_OFFSET, orig_ntotal-STORE_OFFSET); extstore_write(storage, &io); item_hdr *hdr = (item_hdr *) ITEM_data(hdr_it); hdr->page_version = io.page_version; hdr->page_id = io.page_id; hdr->offset = io.offset; // overload nbytes for the header it hdr_it->nbytes = it->nbytes; /* success! Now we need to fill relevant data into the new * header and replace. Most of this requires the item lock */ /* CAS gets set while linking. Copy post-replace */ item_replace(it, hdr_it, it_info.hv); ITEM_set_cas(hdr_it, ITEM_get_cas(it)); do_item_remove(hdr_it); did_moves = 1; LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EXTSTORE_WRITE, it, bucket); } else { /* Failed to write for some reason, can't continue. */ slabs_free(hdr_it, ITEM_ntotal(hdr_it), ITEM_clsid(hdr_it)); } } } do_item_remove(it); item_unlock(it_info.hv); return did_moves; }