void item_scrubber_main(struct default_engine *engine)
{
hash_item cursor;
int ii;
memset(&cursor, 0, sizeof(cursor));
cursor.refcount = 1;
for (ii = 0; ii < POWER_LARGEST; ++ii) {
bool skip = false;
cb_mutex_enter(&engine->items.lock);
if (engine->items.heads[ii] == NULL) {
skip = true;
} else {
/* add the item at the tail */
do_item_link_cursor(engine, &cursor, ii);
}
cb_mutex_exit(&engine->items.lock);
if (!skip) {
item_scrub_class(engine, &cursor);
}
}
cb_mutex_enter(&engine->scrubber.lock);
engine->scrubber.stopped = time(NULL);
engine->scrubber.running = false;
cb_mutex_exit(&engine->scrubber.lock);
}
/*
* Returns a fresh connection queue item.
*/
static CQ_ITEM *cqi_new(void) {
CQ_ITEM *item = NULL;
cb_mutex_enter(&cqi_freelist_lock);
if (cqi_freelist) {
item = cqi_freelist;
cqi_freelist = item->next;
}
cb_mutex_exit(&cqi_freelist_lock);
if (NULL == item) {
int i;
/* Allocate a bunch of items at once to reduce fragmentation */
item = malloc(sizeof(CQ_ITEM) * ITEMS_PER_ALLOC);
if (NULL == item)
return NULL;
/*
* Link together all the new items except the first one
* (which we'll return to the caller) for placement on
* the freelist.
*/
for (i = 2; i < ITEMS_PER_ALLOC; i++)
item[i - 1].next = &item[i];
cb_mutex_enter(&cqi_freelist_lock);
item[ITEMS_PER_ALLOC - 1].next = cqi_freelist;
cqi_freelist = &item[1];
cb_mutex_exit(&cqi_freelist_lock);
}
return item;
}
void* cache_alloc(cache_t *cache) {
void *ret;
void *object;
cb_mutex_enter(&cache->mutex);
if (cache->freecurr > 0) {
ret = cache->ptr[--cache->freecurr];
object = get_object(ret);
} else {
object = ret = malloc(cache->bufsize);
if (ret != NULL) {
object = get_object(ret);
if (cache->constructor != NULL &&
cache->constructor(object, NULL, 0) != 0) {
free(ret);
object = NULL;
}
}
}
cb_mutex_exit(&cache->mutex);
#ifndef NDEBUG
if (object != NULL) {
/* add a simple form of buffer-check */
uint64_t *pre = ret;
*pre = redzone_pattern;
ret = pre+1;
memcpy(((char*)ret) + cache->bufsize - (2 * sizeof(redzone_pattern)),
&redzone_pattern, sizeof(redzone_pattern));
}
#endif
return object;
}
void item_stats_sizes(struct default_engine *engine,
ADD_STAT add_stat, const void *cookie)
{
cb_mutex_enter(&engine->items.lock);
do_item_stats_sizes(engine, add_stat, cookie);
cb_mutex_exit(&engine->items.lock);
}
bool initialize_item_tap_walker(struct default_engine *engine,
const void* cookie)
{
bool linked = false;
int ii;
struct tap_client *client = calloc(1, sizeof(*client));
if (client == NULL) {
return false;
}
client->cursor.refcount = 1;
/* Link the cursor! */
for (ii = 0; ii < POWER_LARGEST && !linked; ++ii) {
cb_mutex_enter(&engine->items.lock);
if (engine->items.heads[ii] != NULL) {
/* add the item at the tail */
do_item_link_cursor(engine, &client->cursor, ii);
linked = true;
}
cb_mutex_exit(&engine->items.lock);
}
engine->server.cookie->store_engine_specific(cookie, client);
return true;
}
void mcache_delete(mcache *m, char *key, int key_len) {
(void)key_len;
(void)key;
assert(key);
assert(key_len > 0);
assert(key[key_len] == '\0' ||
key[key_len] == ' ');
if (m == NULL) {
return;
}
if (m->lock) {
cb_mutex_enter(m->lock);
}
if (m->map != NULL) {
void *existing = genhash_find(m->map, key);
if (existing != NULL) {
mcache_item_unlink(m, existing);
genhash_delete(m->map, key);
m->tot_deletes++;
if (settings.verbose > 1) {
moxi_log_write("mcache delete: %s\n", key);
}
}
}
if (m->lock) {
cb_mutex_exit(m->lock);
}
}
void mcache_stop(mcache *m) {
assert(m);
if (m->lock) {
cb_mutex_enter(m->lock);
}
genhash_t *x = m->map;
m->map = NULL;
m->max = 0;
m->lru_head = NULL;
m->lru_tail = NULL;
m->oldest_live = 0;
if (m->lock) {
cb_mutex_exit(m->lock);
}
/* Destroying hash table outside the lock. */
if (x != NULL) {
genhash_free(x);
}
}
void mcache_start(mcache *m, uint32_t max) {
assert(m);
if (m->lock) {
cb_mutex_enter(m->lock);
}
assert(m->funcs);
assert(m->map == NULL);
assert(m->max == 0);
assert(m->lru_head == NULL);
assert(m->lru_tail == NULL);
assert(m->oldest_live == 0);
struct hash_ops hops = skeyhash_ops;
hops.freeKey = m->key_alloc ? free : noop_free;
hops.freeValue = m->funcs->item_dec_ref;
m->map = genhash_init(128, hops);
if (m->map != NULL) {
m->max = max;
m->lru_head = NULL;
m->lru_tail = NULL;
m->oldest_live = 0;
}
if (m->lock) {
cb_mutex_exit(m->lock);
}
}
ENGINE_ERROR_CODE arithmetic(struct default_engine *engine,
const void* cookie,
const void* key,
const int nkey,
const bool increment,
const bool create,
const uint64_t delta,
const uint64_t initial,
const rel_time_t exptime,
item **item,
uint8_t datatype,
uint64_t *result)
{
ENGINE_ERROR_CODE ret;
hash_key hkey;
if (!hash_key_create(&hkey, key, nkey, engine, cookie)) {
return ENGINE_ENOMEM;
}
cb_mutex_enter(&engine->items.lock);
ret = do_arithmetic(engine, cookie, &hkey, increment,
create, delta, initial, exptime, item,
datatype, result);
cb_mutex_exit(&engine->items.lock);
hash_key_destroy(&hkey);
return ret;
}
/*
* Flushes expired items after a flush_all call
*/
void item_flush_expired(struct default_engine *engine) {
cb_mutex_enter(&engine->items.lock);
rel_time_t now = engine->server.core->get_current_time();
if (now > engine->config.oldest_live) {
engine->config.oldest_live = now - 1;
}
for (int ii = 0; ii < POWER_LARGEST; ii++) {
hash_item *iter, *next;
/*
* The LRU is sorted in decreasing time order, and an item's
* timestamp is never newer than its last access time, so we
* only need to walk back until we hit an item older than the
* oldest_live time.
* The oldest_live checking will auto-expire the remaining items.
*/
for (iter = engine->items.heads[ii]; iter != NULL; iter = next) {
if (iter->time >= engine->config.oldest_live) {
next = iter->next;
if ((iter->iflag & ITEM_SLABBED) == 0) {
do_item_unlink(engine, iter);
}
} else {
/* We've hit the first old item. Continue to the next queue. */
break;
}
}
}
cb_mutex_exit(&engine->items.lock);
}
void threadlocal_stats_reset(struct thread_stats *thread_stats) {
int ii;
for (ii = 0; ii < settings.num_threads; ++ii) {
cb_mutex_enter(&thread_stats[ii].mutex);
threadlocal_stats_clear(&thread_stats[ii]);
cb_mutex_exit(&thread_stats[ii].mutex);
}
}
void *slabs_alloc(struct default_engine *engine, size_t size, unsigned int id) {
void *ret;
cb_mutex_enter(&engine->slabs.lock);
ret = do_slabs_alloc(engine, size, id);
cb_mutex_exit(&engine->slabs.lock);
return ret;
}
int work_collect_wait(work_collect *c) {
int rv = 0;
cb_mutex_enter(&c->collect_lock);
while (c->count != 0 && rv == 0) { /* Can't test for > 0, due to -1 on init race. */
cb_cond_wait(&c->collect_cond, &c->collect_lock);
}
cb_mutex_exit(&c->collect_lock);
return rv;
}
void stop_assoc_maintenance_thread() {
cb_mutex_enter(&cache_lock);
do_run_maintenance_thread = 0;
cb_cond_signal(&maintenance_cond);
cb_mutex_exit(&cache_lock);
/* Wait for the maintenance thread to stop */
cb_join_thread(maintenance_tid);
}
int work_collect_count(work_collect *c, int count) {
int rv = 0;
cb_mutex_enter(&c->collect_lock);
c->count = count;
if (c->count <= 0) {
cb_cond_signal(&c->collect_cond);
}
cb_mutex_exit(&c->collect_lock);
return rv;
}
ENGINE_ERROR_CODE item_dcp_step(struct default_engine *engine,
struct dcp_connection *connection,
const void *cookie,
struct dcp_message_producers *producers)
{
ENGINE_ERROR_CODE ret;
cb_mutex_enter(&engine->items.lock);
ret = do_item_dcp_step(engine, connection, cookie, producers);
cb_mutex_exit(&engine->items.lock);
return ret;
}
int work_collect_one(work_collect *c) {
int rv = 0;
cb_mutex_enter(&c->collect_lock);
cb_assert(c->count >= 1);
c->count--;
if (c->count <= 0) {
cb_cond_signal(&c->collect_cond);
}
cb_mutex_exit(&c->collect_lock);
return rv;
}
/*
* Adds an item to a connection queue.
*/
static void cq_push(CQ *cq, CQ_ITEM *item) {
item->next = NULL;
cb_mutex_enter(&cq->lock);
if (NULL == cq->tail)
cq->head = item;
else
cq->tail->next = item;
cq->tail = item;
cb_cond_signal(&cq->cond);
cb_mutex_exit(&cq->lock);
}
/** Use work_send() to place work on another thread's work queue.
* The receiving thread will invoke the given function with
* the given callback data.
*
* Returns true on success.
*/
bool work_send(work_queue *m,
void (*func)(void *data0, void *data1),
void *data0, void *data1) {
cb_assert(m != NULL);
cb_assert(m->recv_fd >= 0);
cb_assert(m->send_fd >= 0);
cb_assert(m->event_base != NULL);
cb_assert(func != NULL);
bool rv = false;
/* TODO: Add a free-list of work_items. */
work_item *w = calloc(1, sizeof(work_item));
if (w != NULL) {
w->func = func;
w->data0 = data0;
w->data1 = data1;
w->next = NULL;
cb_mutex_enter(&m->work_lock);
if (m->work_tail != NULL)
m->work_tail->next = w;
m->work_tail = w;
if (m->work_head == NULL)
m->work_head = w;
if (send(m->send_fd, "", 1, 0) == 1) {
m->num_items++;
m->tot_sends++;
#ifdef WORK_DEBUG
moxi_log_write("work_send %x %x %x %d %d %d %llu %llu\n",
(int) cb_thread_self(),
(int) m,
(int) m->event_base,
m->send_fd, m->recv_fd,
m->work_head != NULL,
m->num_items,
m->tot_sends);
#endif
rv = true;
}
cb_mutex_exit(&m->work_lock);
}
return rv;
}
/*
* Initializes the thread subsystem, creating various worker threads.
*
* nthreads Number of worker event handler threads to spawn
* main_base Event base for main thread
*/
void thread_init(int nthr, struct event_base *main_base,
void (*dispatcher_callback)(evutil_socket_t, short, void *)) {
int i;
nthreads = nthr + 1;
cqi_freelist = NULL;
cb_mutex_initialize(&conn_lock);
cb_mutex_initialize(&cqi_freelist_lock);
cb_mutex_initialize(&init_lock);
cb_cond_initialize(&init_cond);
threads = calloc(nthreads, sizeof(LIBEVENT_THREAD));
if (! threads) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't allocate thread descriptors: %s",
strerror(errno));
exit(1);
}
thread_ids = calloc(nthreads, sizeof(cb_thread_t));
if (! thread_ids) {
settings.extensions.logger->log(EXTENSION_LOG_WARNING, NULL,
"Can't allocate thread descriptors: %s",
strerror(errno));
exit(1);
}
setup_dispatcher(main_base, dispatcher_callback);
for (i = 0; i < nthreads; i++) {
if (!create_notification_pipe(&threads[i])) {
exit(1);
}
threads[i].index = i;
setup_thread(&threads[i]);
}
/* Create threads after we've done all the libevent setup. */
for (i = 0; i < nthreads; i++) {
create_worker(worker_libevent, &threads[i], &thread_ids[i]);
threads[i].thread_id = thread_ids[i];
}
/* Wait for all the threads to set themselves up before returning. */
cb_mutex_enter(&init_lock);
while (init_count < nthreads) {
cb_cond_wait(&init_cond, &init_lock);
}
cb_mutex_exit(&init_lock);
}
static void item_scrub_class(struct default_engine *engine,
hash_item *cursor) {
ENGINE_ERROR_CODE ret;
bool more;
do {
cb_mutex_enter(&engine->items.lock);
more = do_item_walk_cursor(engine, cursor, 200, item_scrub, NULL, &ret);
cb_mutex_exit(&engine->items.lock);
if (ret != ENGINE_SUCCESS) {
break;
}
} while (more);
}
/*
* Looks for an item on a connection queue, but doesn't block if there isn't
* one.
* Returns the item, or NULL if no item is available
*/
static CQ_ITEM *cq_pop(CQ *cq) {
CQ_ITEM *item;
cb_mutex_enter(&cq->lock);
item = cq->head;
if (NULL != item) {
cq->head = item->next;
if (NULL == cq->head)
cq->tail = NULL;
}
cb_mutex_exit(&cq->lock);
return item;
}
tap_event_t item_tap_walker(ENGINE_HANDLE* handle,
const void *cookie, item **itm,
void **es, uint16_t *nes, uint8_t *ttl,
uint16_t *flags, uint32_t *seqno,
uint16_t *vbucket)
{
tap_event_t ret;
struct default_engine *engine = (struct default_engine*)handle;
cb_mutex_enter(&engine->items.lock);
ret = do_item_tap_walker(engine, cookie, itm, es, nes, ttl, flags, seqno, vbucket);
cb_mutex_exit(&engine->items.lock);
return ret;
}
/*
* Worker thread: main event loop
*/
static void worker_libevent(void *arg) {
LIBEVENT_THREAD *me = arg;
/* Any per-thread setup can happen here; thread_init() will block until
* all threads have finished initializing.
*/
cb_mutex_enter(&init_lock);
init_count++;
cb_cond_signal(&init_cond);
cb_mutex_exit(&init_lock);
event_base_loop(me->base, 0);
}
bool mcache_started(mcache *m) {
assert(m);
if (m->lock) {
cb_mutex_enter(m->lock);
}
bool rv = m->map != NULL;
if (m->lock) {
cb_mutex_exit(m->lock);
}
return rv;
}
/*
* Allocates a new item.
*/
hash_item *item_alloc(struct default_engine *engine,
const void *key, size_t nkey, int flags,
rel_time_t exptime, int nbytes, const void *cookie,
uint8_t datatype) {
hash_item *it;
hash_key hkey;
if (!hash_key_create(&hkey, key, nkey, engine, cookie)) {
return NULL;
}
cb_mutex_enter(&engine->items.lock);
it = do_item_alloc(engine, &hkey, flags, exptime, nbytes, cookie, datatype);
cb_mutex_exit(&engine->items.lock);
hash_key_destroy(&hkey);
return it;
}
/*
* Returns an item if it hasn't been marked as expired,
* lazy-expiring as needed.
*/
hash_item *item_get(struct default_engine *engine,
const void *cookie,
const void *key,
const size_t nkey) {
hash_item *it;
hash_key hkey;
if (!hash_key_create(&hkey, key, nkey, engine, cookie)) {
return NULL;
}
cb_mutex_enter(&engine->items.lock);
it = do_item_get(engine, &hkey);
cb_mutex_exit(&engine->items.lock);
hash_key_destroy(&hkey);
return it;
}
/*
* Stores an item in the cache (high level, obeys set/add/replace semantics)
*/
ENGINE_ERROR_CODE store_item(struct default_engine *engine,
hash_item *item, uint64_t *cas,
ENGINE_STORE_OPERATION operation,
const void *cookie) {
ENGINE_ERROR_CODE ret;
hash_item* stored_item = NULL;
cb_mutex_enter(&engine->items.lock);
ret = do_store_item(engine, item, operation, cookie, &stored_item);
if (ret == ENGINE_SUCCESS) {
*cas = item_get_cas(stored_item);
}
cb_mutex_exit(&engine->items.lock);
return ret;
}
void threadlocal_stats_aggregate(struct thread_stats *thread_stats, struct thread_stats *stats) {
int ii, sid;
for (ii = 0; ii < settings.num_threads; ++ii) {
cb_mutex_enter(&thread_stats[ii].mutex);
stats->cmd_get += thread_stats[ii].cmd_get;
stats->get_misses += thread_stats[ii].get_misses;
stats->delete_misses += thread_stats[ii].delete_misses;
stats->decr_misses += thread_stats[ii].decr_misses;
stats->incr_misses += thread_stats[ii].incr_misses;
stats->decr_hits += thread_stats[ii].decr_hits;
stats->incr_hits += thread_stats[ii].incr_hits;
stats->cas_misses += thread_stats[ii].cas_misses;
stats->bytes_read += thread_stats[ii].bytes_read;
stats->bytes_written += thread_stats[ii].bytes_written;
stats->cmd_flush += thread_stats[ii].cmd_flush;
stats->conn_yields += thread_stats[ii].conn_yields;
stats->auth_cmds += thread_stats[ii].auth_cmds;
stats->auth_errors += thread_stats[ii].auth_errors;
stats->rbufs_allocated += thread_stats[ii].rbufs_allocated;
stats->rbufs_loaned += thread_stats[ii].rbufs_loaned;
stats->rbufs_existing += thread_stats[ii].rbufs_existing;
stats->wbufs_allocated += thread_stats[ii].wbufs_allocated;
stats->wbufs_loaned += thread_stats[ii].wbufs_loaned;
if (thread_stats[ii].iovused_high_watermark > stats->iovused_high_watermark) {
stats->iovused_high_watermark = thread_stats[ii].iovused_high_watermark;
}
if (thread_stats[ii].msgused_high_watermark > stats->msgused_high_watermark) {
stats->msgused_high_watermark = thread_stats[ii].msgused_high_watermark;
}
for (sid = 0; sid < MAX_NUMBER_OF_SLAB_CLASSES; sid++) {
stats->slab_stats[sid].cmd_set +=
thread_stats[ii].slab_stats[sid].cmd_set;
stats->slab_stats[sid].get_hits +=
thread_stats[ii].slab_stats[sid].get_hits;
stats->slab_stats[sid].delete_hits +=
thread_stats[ii].slab_stats[sid].delete_hits;
stats->slab_stats[sid].cas_hits +=
thread_stats[ii].slab_stats[sid].cas_hits;
stats->slab_stats[sid].cas_badval +=
thread_stats[ii].slab_stats[sid].cas_badval;
}
cb_mutex_exit(&thread_stats[ii].mutex);
}
}
void slabs_adjust_mem_requested(struct default_engine *engine, unsigned int id, size_t old, size_t ntotal)
{
slabclass_t *p;
cb_mutex_enter(&engine->slabs.lock);
if (id < POWER_SMALLEST || id > engine->slabs.power_largest) {
EXTENSION_LOGGER_DESCRIPTOR *logger;
logger = (void*)engine->server.extension->get_extension(EXTENSION_LOGGER);
logger->log(EXTENSION_LOG_WARNING, NULL,
"Internal error! Invalid slab class\n");
abort();
}
p = &engine->slabs.slabclass[id];
p->requested = p->requested - old + ntotal;
cb_mutex_exit(&engine->slabs.lock);
}
