/* * Returns a fresh connection queue item. */ static CQ_ITEM *cqi_new(void) { CQ_ITEM *item = NULL; cb_mutex_enter(&cqi_freelist_lock); if (cqi_freelist) { item = cqi_freelist; cqi_freelist = item->next; } cb_mutex_exit(&cqi_freelist_lock); if (NULL == item) { int i; /* Allocate a bunch of items at once to reduce fragmentation */ item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC); if (NULL == item) return NULL; /* * Link together all the new items except the first one * (which we'll return to the caller) for placement on * the freelist. */ for (i = 2; i < ITEMS_PER_ALLOC; i++) item[i - 1].next = &item[i]; cb_mutex_enter(&cqi_freelist_lock); item[ITEMS_PER_ALLOC - 1].next = cqi_freelist; cqi_freelist = &item[1]; cb_mutex_exit(&cqi_freelist_lock); } return item; }
void item_scrubber_main(struct default_engine *engine) { hash_item cursor; int ii; memset(&cursor, 0, sizeof(cursor)); cursor.refcount = 1; for (ii = 0; ii < POWER_LARGEST; ++ii) { bool skip = false; cb_mutex_enter(&engine->items.lock); if (engine->items.heads[ii] == NULL) { skip = true; } else { /* add the item at the tail */ do_item_link_cursor(engine, &cursor, ii); } cb_mutex_exit(&engine->items.lock); if (!skip) { item_scrub_class(engine, &cursor); } } cb_mutex_enter(&engine->scrubber.lock); engine->scrubber.stopped = time(NULL); engine->scrubber.running = false; cb_mutex_exit(&engine->scrubber.lock); }
/* * Flushes expired items after a flush_all call */ void item_flush_expired(struct default_engine *engine) { cb_mutex_enter(&engine->items.lock); rel_time_t now = engine->server.core->get_current_time(); if (now > engine->config.oldest_live) { engine->config.oldest_live = now - 1; } for (int ii = 0; ii < POWER_LARGEST; ii++) { hash_item *iter, *next; /* * The LRU is sorted in decreasing time order, and an item's * timestamp is never newer than its last access time, so we * only need to walk back until we hit an item older than the * oldest_live time. * The oldest_live checking will auto-expire the remaining items. */ for (iter = engine->items.heads[ii]; iter != NULL; iter = next) { if (iter->time >= engine->config.oldest_live) { next = iter->next; if ((iter->iflag & ITEM_SLABBED) == 0) { do_item_unlink(engine, iter); } } else { /* We've hit the first old item. Continue to the next queue. */ break; } } } cb_mutex_exit(&engine->items.lock); }
void item_stats_sizes(struct default_engine *engine, ADD_STAT add_stat, const void *cookie) { cb_mutex_enter(&engine->items.lock); do_item_stats_sizes(engine, add_stat, cookie); cb_mutex_exit(&engine->items.lock); }
bool initialize_item_tap_walker(struct default_engine *engine, const void* cookie) { bool linked = false; int ii; struct tap_client *client = calloc(1, sizeof(*client)); if (client == NULL) { return false; } client->cursor.refcount = 1; /* Link the cursor! */ for (ii = 0; ii < POWER_LARGEST && !linked; ++ii) { cb_mutex_enter(&engine->items.lock); if (engine->items.heads[ii] != NULL) { /* add the item at the tail */ do_item_link_cursor(engine, &client->cursor, ii); linked = true; } cb_mutex_exit(&engine->items.lock); } engine->server.cookie->store_engine_specific(cookie, client); return true; }
void mcache_delete(mcache *m, char *key, int key_len) { (void)key_len; (void)key; assert(key); assert(key_len > 0); assert(key[key_len] == '\0' || key[key_len] == ' '); if (m == NULL) { return; } if (m->lock) { cb_mutex_enter(m->lock); } if (m->map != NULL) { void *existing = genhash_find(m->map, key); if (existing != NULL) { mcache_item_unlink(m, existing); genhash_delete(m->map, key); m->tot_deletes++; if (settings.verbose > 1) { moxi_log_write("mcache delete: %s\n", key); } } } if (m->lock) { cb_mutex_exit(m->lock); } }
void mcache_stop(mcache *m) { assert(m); if (m->lock) { cb_mutex_enter(m->lock); } genhash_t *x = m->map; m->map = NULL; m->max = 0; m->lru_head = NULL; m->lru_tail = NULL; m->oldest_live = 0; if (m->lock) { cb_mutex_exit(m->lock); } /* Destroying hash table outside the lock. */ if (x != NULL) { genhash_free(x); } }
void mcache_start(mcache *m, uint32_t max) { assert(m); if (m->lock) { cb_mutex_enter(m->lock); } assert(m->funcs); assert(m->map == NULL); assert(m->max == 0); assert(m->lru_head == NULL); assert(m->lru_tail == NULL); assert(m->oldest_live == 0); struct hash_ops hops = skeyhash_ops; hops.freeKey = m->key_alloc ? free : noop_free; hops.freeValue = m->funcs->item_dec_ref; m->map = genhash_init(128, hops); if (m->map != NULL) { m->max = max; m->lru_head = NULL; m->lru_tail = NULL; m->oldest_live = 0; } if (m->lock) { cb_mutex_exit(m->lock); } }
void* cache_alloc(cache_t *cache) { void *ret; void *object; cb_mutex_enter(&cache->mutex); if (cache->freecurr > 0) { ret = cache->ptr[--cache->freecurr]; object = get_object(ret); } else { object = ret = malloc(cache->bufsize); if (ret != NULL) { object = get_object(ret); if (cache->constructor != NULL && cache->constructor(object, NULL, 0) != 0) { free(ret); object = NULL; } } } cb_mutex_exit(&cache->mutex); #ifndef NDEBUG if (object != NULL) { /* add a simple form of buffer-check */ uint64_t *pre = ret; *pre = redzone_pattern; ret = pre+1; memcpy(((char*)ret) + cache->bufsize - (2 * sizeof(redzone_pattern)), &redzone_pattern, sizeof(redzone_pattern)); } #endif return object; }
ENGINE_ERROR_CODE arithmetic(struct default_engine *engine, const void* cookie, const void* key, const int nkey, const bool increment, const bool create, const uint64_t delta, const uint64_t initial, const rel_time_t exptime, item **item, uint8_t datatype, uint64_t *result) { ENGINE_ERROR_CODE ret; hash_key hkey; if (!hash_key_create(&hkey, key, nkey, engine, cookie)) { return ENGINE_ENOMEM; } cb_mutex_enter(&engine->items.lock); ret = do_arithmetic(engine, cookie, &hkey, increment, create, delta, initial, exptime, item, datatype, result); cb_mutex_exit(&engine->items.lock); hash_key_destroy(&hkey); return ret; }
void cache_free(cache_t *cache, void *ptr) { #ifndef NDEBUG uint64_t *pre; #endif cb_mutex_enter(&cache->mutex); #ifndef NDEBUG /* validate redzone... */ if (memcmp(((char*)ptr) + cache->bufsize - (2 * sizeof(redzone_pattern)), &redzone_pattern, sizeof(redzone_pattern)) != 0) { raise(SIGABRT); cache_error = 1; cb_mutex_exit(&cache->mutex); return; } pre = ptr; --pre; if (*pre != redzone_pattern) { raise(SIGABRT); cache_error = -1; cb_mutex_exit(&cache->mutex); return; } ptr = pre; #endif if (cache->freecurr < cache->freetotal) { cache->ptr[cache->freecurr++] = ptr; } else { /* try to enlarge free connections array */ size_t newtotal = cache->freetotal * 2; void **new_free = realloc(cache->ptr, sizeof(char *) * newtotal); if (new_free) { cache->freetotal = (int)newtotal; cache->ptr = new_free; cache->ptr[cache->freecurr++] = ptr; } else { if (cache->destructor) { cache->destructor(ptr, NULL); } free(ptr); } } cb_mutex_exit(&cache->mutex); }
void threadlocal_stats_reset(struct thread_stats *thread_stats) { int ii; for (ii = 0; ii < settings.num_threads; ++ii) { cb_mutex_enter(&thread_stats[ii].mutex); threadlocal_stats_clear(&thread_stats[ii]); cb_mutex_exit(&thread_stats[ii].mutex); } }
void *slabs_alloc(struct default_engine *engine, size_t size, unsigned int id) { void *ret; cb_mutex_enter(&engine->slabs.lock); ret = do_slabs_alloc(engine, size, id); cb_mutex_exit(&engine->slabs.lock); return ret; }
int work_collect_wait(work_collect *c) { int rv = 0; cb_mutex_enter(&c->collect_lock); while (c->count != 0 && rv == 0) { /* Can't test for > 0, due to -1 on init race. */ cb_cond_wait(&c->collect_cond, &c->collect_lock); } cb_mutex_exit(&c->collect_lock); return rv; }
void stop_assoc_maintenance_thread() { cb_mutex_enter(&cache_lock); do_run_maintenance_thread = 0; cb_cond_signal(&maintenance_cond); cb_mutex_exit(&cache_lock); /* Wait for the maintenance thread to stop */ cb_join_thread(maintenance_tid); }
int work_collect_count(work_collect *c, int count) { int rv = 0; cb_mutex_enter(&c->collect_lock); c->count = count; if (c->count <= 0) { cb_cond_signal(&c->collect_cond); } cb_mutex_exit(&c->collect_lock); return rv; }
int work_collect_one(work_collect *c) { int rv = 0; cb_mutex_enter(&c->collect_lock); cb_assert(c->count >= 1); c->count--; if (c->count <= 0) { cb_cond_signal(&c->collect_cond); } cb_mutex_exit(&c->collect_lock); return rv; }
ENGINE_ERROR_CODE item_dcp_step(struct default_engine *engine, struct dcp_connection *connection, const void *cookie, struct dcp_message_producers *producers) { ENGINE_ERROR_CODE ret; cb_mutex_enter(&engine->items.lock); ret = do_item_dcp_step(engine, connection, cookie, producers); cb_mutex_exit(&engine->items.lock); return ret; }
/* * Adds an item to a connection queue. */ static void cq_push(CQ *cq, CQ_ITEM *item) { item->next = NULL; cb_mutex_enter(&cq->lock); if (NULL == cq->tail) cq->head = item; else cq->tail->next = item; cq->tail = item; cb_cond_signal(&cq->cond); cb_mutex_exit(&cq->lock); }
/** Use work_send() to place work on another thread's work queue. * The receiving thread will invoke the given function with * the given callback data. * * Returns true on success. */ bool work_send(work_queue *m, void (*func)(void *data0, void *data1), void *data0, void *data1) { cb_assert(m != NULL); cb_assert(m->recv_fd >= 0); cb_assert(m->send_fd >= 0); cb_assert(m->event_base != NULL); cb_assert(func != NULL); bool rv = false; /* TODO: Add a free-list of work_items. */ work_item *w = calloc(1, sizeof(work_item)); if (w != NULL) { w->func = func; w->data0 = data0; w->data1 = data1; w->next = NULL; cb_mutex_enter(&m->work_lock); if (m->work_tail != NULL) m->work_tail->next = w; m->work_tail = w; if (m->work_head == NULL) m->work_head = w; if (send(m->send_fd, "", 1, 0) == 1) { m->num_items++; m->tot_sends++; #ifdef WORK_DEBUG moxi_log_write("work_send %x %x %x %d %d %d %llu %llu\n", (int) cb_thread_self(), (int) m, (int) m->event_base, m->send_fd, m->recv_fd, m->work_head != NULL, m->num_items, m->tot_sends); #endif rv = true; } cb_mutex_exit(&m->work_lock); } return rv; }
/* * Initializes the thread subsystem, creating various worker threads. * * nthreads Number of worker event handler threads to spawn * main_base Event base for main thread */ void thread_init(int nthr, struct event_base *main_base, void (*dispatcher_callback)(evutil_socket_t, short, void *)) { int i; nthreads = nthr + 1; cqi_freelist = NULL; cb_mutex_initialize(&conn_lock); cb_mutex_initialize(&cqi_freelist_lock); cb_mutex_initialize(&init_lock); cb_cond_initialize(&init_cond); threads = calloc(nthreads, sizeof(LIBEVENT_THREAD)); if (! threads) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't allocate thread descriptors: %s", strerror(errno)); exit(1); } thread_ids = calloc(nthreads, sizeof(cb_thread_t)); if (! thread_ids) { settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL, "Can't allocate thread descriptors: %s", strerror(errno)); exit(1); } setup_dispatcher(main_base, dispatcher_callback); for (i = 0; i < nthreads; i++) { if (!create_notification_pipe(&threads[i])) { exit(1); } threads[i].index = i; setup_thread(&threads[i]); } /* Create threads after we've done all the libevent setup. */ for (i = 0; i < nthreads; i++) { create_worker(worker_libevent, &threads[i], &thread_ids[i]); threads[i].thread_id = thread_ids[i]; } /* Wait for all the threads to set themselves up before returning. */ cb_mutex_enter(&init_lock); while (init_count < nthreads) { cb_cond_wait(&init_cond, &init_lock); } cb_mutex_exit(&init_lock); }
/* * Looks for an item on a connection queue, but doesn't block if there isn't * one. * Returns the item, or NULL if no item is available */ static CQ_ITEM *cq_pop(CQ *cq) { CQ_ITEM *item; cb_mutex_enter(&cq->lock); item = cq->head; if (NULL != item) { cq->head = item->next; if (NULL == cq->head) cq->tail = NULL; } cb_mutex_exit(&cq->lock); return item; }
static void item_scrub_class(struct default_engine *engine, hash_item *cursor) { ENGINE_ERROR_CODE ret; bool more; do { cb_mutex_enter(&engine->items.lock); more = do_item_walk_cursor(engine, cursor, 200, item_scrub, NULL, &ret); cb_mutex_exit(&engine->items.lock); if (ret != ENGINE_SUCCESS) { break; } } while (more); }
/* * Worker thread: main event loop */ static void worker_libevent(void *arg) { LIBEVENT_THREAD *me = arg; /* Any per-thread setup can happen here; thread_init() will block until * all threads have finished initializing. */ cb_mutex_enter(&init_lock); init_count++; cb_cond_signal(&init_cond); cb_mutex_exit(&init_lock); event_base_loop(me->base, 0); }
tap_event_t item_tap_walker(ENGINE_HANDLE* handle, const void *cookie, item **itm, void **es, uint16_t *nes, uint8_t *ttl, uint16_t *flags, uint32_t *seqno, uint16_t *vbucket) { tap_event_t ret; struct default_engine *engine = (struct default_engine*)handle; cb_mutex_enter(&engine->items.lock); ret = do_item_tap_walker(engine, cookie, itm, es, nes, ttl, flags, seqno, vbucket); cb_mutex_exit(&engine->items.lock); return ret; }
/* * Stores an item in the cache (high level, obeys set/add/replace semantics) */ ENGINE_ERROR_CODE store_item(struct default_engine *engine, hash_item *item, uint64_t *cas, ENGINE_STORE_OPERATION operation, const void *cookie) { ENGINE_ERROR_CODE ret; hash_item* stored_item = NULL; cb_mutex_enter(&engine->items.lock); ret = do_store_item(engine, item, operation, cookie, &stored_item); if (ret == ENGINE_SUCCESS) { *cas = item_get_cas(stored_item); } cb_mutex_exit(&engine->items.lock); return ret; }
bool mcache_started(mcache *m) { assert(m); if (m->lock) { cb_mutex_enter(m->lock); } bool rv = m->map != NULL; if (m->lock) { cb_mutex_exit(m->lock); } return rv; }
/* * Allocates a new item. */ hash_item *item_alloc(struct default_engine *engine, const void *key, size_t nkey, int flags, rel_time_t exptime, int nbytes, const void *cookie, uint8_t datatype) { hash_item *it; hash_key hkey; if (!hash_key_create(&hkey, key, nkey, engine, cookie)) { return NULL; } cb_mutex_enter(&engine->items.lock); it = do_item_alloc(engine, &hkey, flags, exptime, nbytes, cookie, datatype); cb_mutex_exit(&engine->items.lock); hash_key_destroy(&hkey); return it; }
/* * Returns an item if it hasn't been marked as expired, * lazy-expiring as needed. */ hash_item *item_get(struct default_engine *engine, const void *cookie, const void *key, const size_t nkey) { hash_item *it; hash_key hkey; if (!hash_key_create(&hkey, key, nkey, engine, cookie)) { return NULL; } cb_mutex_enter(&engine->items.lock); it = do_item_get(engine, &hkey); cb_mutex_exit(&engine->items.lock); hash_key_destroy(&hkey); return it; }
void threadlocal_stats_aggregate(struct thread_stats *thread_stats, struct thread_stats *stats) { int ii, sid; for (ii = 0; ii < settings.num_threads; ++ii) { cb_mutex_enter(&thread_stats[ii].mutex); stats->cmd_get += thread_stats[ii].cmd_get; stats->get_misses += thread_stats[ii].get_misses; stats->delete_misses += thread_stats[ii].delete_misses; stats->decr_misses += thread_stats[ii].decr_misses; stats->incr_misses += thread_stats[ii].incr_misses; stats->decr_hits += thread_stats[ii].decr_hits; stats->incr_hits += thread_stats[ii].incr_hits; stats->cas_misses += thread_stats[ii].cas_misses; stats->bytes_read += thread_stats[ii].bytes_read; stats->bytes_written += thread_stats[ii].bytes_written; stats->cmd_flush += thread_stats[ii].cmd_flush; stats->conn_yields += thread_stats[ii].conn_yields; stats->auth_cmds += thread_stats[ii].auth_cmds; stats->auth_errors += thread_stats[ii].auth_errors; stats->rbufs_allocated += thread_stats[ii].rbufs_allocated; stats->rbufs_loaned += thread_stats[ii].rbufs_loaned; stats->rbufs_existing += thread_stats[ii].rbufs_existing; stats->wbufs_allocated += thread_stats[ii].wbufs_allocated; stats->wbufs_loaned += thread_stats[ii].wbufs_loaned; if (thread_stats[ii].iovused_high_watermark > stats->iovused_high_watermark) { stats->iovused_high_watermark = thread_stats[ii].iovused_high_watermark; } if (thread_stats[ii].msgused_high_watermark > stats->msgused_high_watermark) { stats->msgused_high_watermark = thread_stats[ii].msgused_high_watermark; } for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) { stats->slab_stats[sid].cmd_set += thread_stats[ii].slab_stats[sid].cmd_set; stats->slab_stats[sid].get_hits += thread_stats[ii].slab_stats[sid].get_hits; stats->slab_stats[sid].delete_hits += thread_stats[ii].slab_stats[sid].delete_hits; stats->slab_stats[sid].cas_hits += thread_stats[ii].slab_stats[sid].cas_hits; stats->slab_stats[sid].cas_badval += thread_stats[ii].slab_stats[sid].cas_badval; } cb_mutex_exit(&thread_stats[ii].mutex); } }