/* Returns number of items remove, expired, or evicted.
* Callable from worker threads or the LRU maintainer thread */
static int lru_pull_tail(const int orig_id, const int cur_lru,
const uint64_t total_bytes, uint8_t flags) {
item *it = NULL;
int id = orig_id;
int removed = 0;
if (id == 0)
return 0;
int tries = 5;
item *search;
item *next_it;
void *hold_lock = NULL;
unsigned int move_to_lru = 0;
uint64_t limit = 0;
id |= cur_lru;
pthread_mutex_lock(&lru_locks[id]);
search = tails[id];
/* We walk up *only* for locked items, and if bottom is expired. */
for (; tries > 0 && search != NULL; tries--, search=next_it) {
/* we might relink search mid-loop, so search->prev isn't reliable */
next_it = search->prev;
if (search->nbytes == 0 && search->nkey == 0 && search->it_flags == 1) {
/* We are a crawler, ignore it. */
if (flags & LRU_PULL_CRAWL_BLOCKS) {
pthread_mutex_unlock(&lru_locks[id]);
return 0;
}
tries++;
continue;
}
uint32_t hv = hash(ITEM_key(search), search->nkey);
/* Attempt to hash item lock the "search" item. If locked, no
* other callers can incr the refcount. Also skip ourselves. */
if ((hold_lock = item_trylock(hv)) == NULL)
continue;
/* Now see if the item is refcount locked */
if (refcount_incr(&search->refcount) != 2) {
/* Note pathological case with ref'ed items in tail.
* Can still unlink the item, but it won't be reusable yet */
itemstats[id].lrutail_reflocked++;
/* In case of refcount leaks, enable for quick workaround. */
/* WARNING: This can cause terrible corruption */
if (settings.tail_repair_time &&
search->time + settings.tail_repair_time < current_time) {
itemstats[id].tailrepairs++;
search->refcount = 1;
/* This will call item_remove -> item_free since refcnt is 1 */
do_item_unlink_nolock(search, hv);
item_trylock_unlock(hold_lock);
continue;
}
}
/* Expired or flushed */
if ((search->exptime != 0 && search->exptime < current_time)
|| item_is_flushed(search)) {
itemstats[id].reclaimed++;
if ((search->it_flags & ITEM_FETCHED) == 0) {
itemstats[id].expired_unfetched++;
}
/* refcnt 2 -> 1 */
do_item_unlink_nolock(search, hv);
/* refcnt 1 -> 0 -> item_free */
do_item_remove(search);
item_trylock_unlock(hold_lock);
removed++;
/* If all we're finding are expired, can keep going */
continue;
}
/* If we're HOT_LRU or WARM_LRU and over size limit, send to COLD_LRU.
* If we're COLD_LRU, send to WARM_LRU unless we need to evict
*/
switch (cur_lru) {
case HOT_LRU:
limit = total_bytes * settings.hot_lru_pct / 100;
case WARM_LRU:
if (limit == 0)
limit = total_bytes * settings.warm_lru_pct / 100;
if (sizes_bytes[id] > limit) {
itemstats[id].moves_to_cold++;
move_to_lru = COLD_LRU;
do_item_unlink_q(search);
it = search;
removed++;
break;
} else if ((search->it_flags & ITEM_ACTIVE) != 0) {
/* Only allow ACTIVE relinking if we're not too large. */
itemstats[id].moves_within_lru++;
search->it_flags &= ~ITEM_ACTIVE;
do_item_update_nolock(search);
do_item_remove(search);
item_trylock_unlock(hold_lock);
} else {
/* Don't want to move to COLD, not active, bail out */
it = search;
}
break;
case COLD_LRU:
it = search; /* No matter what, we're stopping */
if (flags & LRU_PULL_EVICT) {
if (settings.evict_to_free == 0) {
/* Don't think we need a counter for this. It'll OOM. */
break;
}
itemstats[id].evicted++;
itemstats[id].evicted_time = current_time - search->time;
if (search->exptime != 0)
itemstats[id].evicted_nonzero++;
if ((search->it_flags & ITEM_FETCHED) == 0) {
itemstats[id].evicted_unfetched++;
}
LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EVICTION, search);
do_item_unlink_nolock(search, hv);
removed++;
if (settings.slab_automove == 2) {
slabs_reassign(-1, orig_id);
}
} else if ((search->it_flags & ITEM_ACTIVE) != 0
&& settings.lru_maintainer_thread) {
itemstats[id].moves_to_warm++;
search->it_flags &= ~ITEM_ACTIVE;
move_to_lru = WARM_LRU;
do_item_unlink_q(search);
removed++;
}
break;
}
if (it != NULL)
break;
}
pthread_mutex_unlock(&lru_locks[id]);
if (it != NULL) {
if (move_to_lru) {
it->slabs_clsid = ITEM_clsid(it);
it->slabs_clsid |= move_to_lru;
item_link_q(it);
}
do_item_remove(it);
item_trylock_unlock(hold_lock);
}
return removed;
}
/** wrapper around assoc_find which does the lazy expiration logic */
item *do_item_get(const char *key, const size_t nkey, const uint32_t hv, conn *c) {
item *it = assoc_find(key, nkey, hv);
if (it != NULL) {
refcount_incr(&it->refcount);
/* Optimization for slab reassignment. prevents popular items from
* jamming in busy wait. Can only do this here to satisfy lock order
* of item_lock, slabs_lock. */
/* This was made unsafe by removal of the cache_lock:
* slab_rebalance_signal and slab_rebal.* are modified in a separate
* thread under slabs_lock. If slab_rebalance_signal = 1, slab_start =
* NULL (0), but slab_end is still equal to some value, this would end
* up unlinking every item fetched.
* This is either an acceptable loss, or if slab_rebalance_signal is
* true, slab_start/slab_end should be put behind the slabs_lock.
* Which would cause a huge potential slowdown.
* Could also use a specific lock for slab_rebal.* and
* slab_rebalance_signal (shorter lock?)
*/
/*if (slab_rebalance_signal &&
((void *)it >= slab_rebal.slab_start && (void *)it < slab_rebal.slab_end)) {
do_item_unlink(it, hv);
do_item_remove(it);
it = NULL;
}*/
}
int was_found = 0;
if (settings.verbose > 2) {
int ii;
if (it == NULL) {
fprintf(stderr, "> NOT FOUND ");
} else {
fprintf(stderr, "> FOUND KEY ");
}
for (ii = 0; ii < nkey; ++ii) {
fprintf(stderr, "%c", key[ii]);
}
}
if (it != NULL) {
was_found = 1;
if (item_is_flushed(it)) {
do_item_unlink(it, hv);
do_item_remove(it);
it = NULL;
pthread_mutex_lock(&c->thread->stats.mutex);
c->thread->stats.get_flushed++;
pthread_mutex_unlock(&c->thread->stats.mutex);
if (settings.verbose > 2) {
fprintf(stderr, " -nuked by flush");
}
was_found = 2;
} else if (it->exptime != 0 && it->exptime <= current_time) {
do_item_unlink(it, hv);
do_item_remove(it);
it = NULL;
pthread_mutex_lock(&c->thread->stats.mutex);
c->thread->stats.get_expired++;
pthread_mutex_unlock(&c->thread->stats.mutex);
if (settings.verbose > 2) {
fprintf(stderr, " -nuked by expire");
}
was_found = 3;
} else {
it->it_flags |= ITEM_FETCHED|ITEM_ACTIVE;
DEBUG_REFCNT(it, '+');
}
}
if (settings.verbose > 2)
fprintf(stderr, "\n");
/* For now this is in addition to the above verbose logging. */
LOGGER_LOG(c->thread->l, LOG_FETCHERS, LOGGER_ITEM_GET, NULL, was_found, key, nkey);
return it;
}
// TODO: hoist the storage bits from lru_maintainer_thread in here.
// would be nice if they could avoid hammering the same locks though?
// I guess it's only COLD. that's probably fine.
static void *storage_compact_thread(void *arg) {
void *storage = arg;
useconds_t to_sleep = MAX_STORAGE_COMPACT_SLEEP;
bool compacting = false;
uint64_t page_version = 0;
uint64_t page_size = 0;
uint64_t page_offset = 0;
uint32_t page_id = 0;
bool drop_unread = false;
char *readback_buf = NULL;
struct storage_compact_wrap wrap;
logger *l = logger_create();
if (l == NULL) {
fprintf(stderr, "Failed to allocate logger for storage compaction thread\n");
abort();
}
readback_buf = malloc(settings.ext_wbuf_size);
if (readback_buf == NULL) {
fprintf(stderr, "Failed to allocate readback buffer for storage compaction thread\n");
abort();
}
pthread_mutex_init(&wrap.lock, NULL);
wrap.done = false;
wrap.submitted = false;
wrap.io.data = &wrap;
wrap.io.buf = (void *)readback_buf;
wrap.io.len = settings.ext_wbuf_size;
wrap.io.mode = OBJ_IO_READ;
wrap.io.cb = _storage_compact_cb;
pthread_mutex_lock(&storage_compact_plock);
while (1) {
pthread_mutex_unlock(&storage_compact_plock);
if (to_sleep) {
extstore_run_maint(storage);
usleep(to_sleep);
}
pthread_mutex_lock(&storage_compact_plock);
if (!compacting && storage_compact_check(storage, l,
&page_id, &page_version, &page_size, &drop_unread)) {
page_offset = 0;
compacting = true;
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_START,
NULL, page_id, page_version);
}
if (compacting) {
pthread_mutex_lock(&wrap.lock);
if (page_offset < page_size && !wrap.done && !wrap.submitted) {
wrap.io.page_version = page_version;
wrap.io.page_id = page_id;
wrap.io.offset = page_offset;
// FIXME: should be smarter about io->next (unlink at use?)
wrap.io.next = NULL;
wrap.submitted = true;
wrap.miss = false;
extstore_submit(storage, &wrap.io);
} else if (wrap.miss) {
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_ABORT,
NULL, page_id);
wrap.done = false;
wrap.submitted = false;
compacting = false;
} else if (wrap.submitted && wrap.done) {
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_START,
NULL, page_id, page_offset);
storage_compact_readback(storage, l, drop_unread,
readback_buf, page_id, page_version, settings.ext_wbuf_size);
page_offset += settings.ext_wbuf_size;
wrap.done = false;
wrap.submitted = false;
} else if (page_offset >= page_size) {
compacting = false;
wrap.done = false;
wrap.submitted = false;
extstore_close_page(storage, page_id, page_version);
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_END,
NULL, page_id);
}
pthread_mutex_unlock(&wrap.lock);
if (to_sleep > MIN_STORAGE_COMPACT_SLEEP)
to_sleep /= 2;
} else {
if (to_sleep < MAX_STORAGE_COMPACT_SLEEP)
to_sleep += MIN_STORAGE_COMPACT_SLEEP;
}
}
free(readback_buf);
return NULL;
}
static void storage_compact_readback(void *storage, logger *l,
bool drop_unread, char *readback_buf,
uint32_t page_id, uint64_t page_version, uint64_t read_size) {
uint64_t offset = 0;
unsigned int rescues = 0;
unsigned int lost = 0;
unsigned int skipped = 0;
while (offset < read_size) {
item *hdr_it = NULL;
item_hdr *hdr = NULL;
item *it = (item *)(readback_buf+offset);
unsigned int ntotal;
// probably zeroed out junk at the end of the wbuf
if (it->nkey == 0) {
break;
}
ntotal = ITEM_ntotal(it);
uint32_t hv = (uint32_t)it->time;
item_lock(hv);
// We don't have a conn and don't need to do most of do_item_get
hdr_it = assoc_find(ITEM_key(it), it->nkey, hv);
if (hdr_it != NULL) {
bool do_write = false;
refcount_incr(hdr_it);
// Check validity but don't bother removing it.
if ((hdr_it->it_flags & ITEM_HDR) && !item_is_flushed(hdr_it) &&
(hdr_it->exptime == 0 || hdr_it->exptime > current_time)) {
hdr = (item_hdr *)ITEM_data(hdr_it);
if (hdr->page_id == page_id && hdr->page_version == page_version) {
// Item header is still completely valid.
extstore_delete(storage, page_id, page_version, 1, ntotal);
// drop inactive items.
if (drop_unread && GET_LRU(hdr_it->slabs_clsid) == COLD_LRU) {
do_write = false;
skipped++;
} else {
do_write = true;
}
}
}
if (do_write) {
bool do_update = false;
int tries;
obj_io io;
io.len = ntotal;
io.mode = OBJ_IO_WRITE;
for (tries = 10; tries > 0; tries--) {
if (extstore_write_request(storage, PAGE_BUCKET_COMPACT, PAGE_BUCKET_COMPACT, &io) == 0) {
memcpy(io.buf, it, io.len);
extstore_write(storage, &io);
do_update = true;
break;
} else {
usleep(1000);
}
}
if (do_update) {
if (it->refcount == 2) {
hdr->page_version = io.page_version;
hdr->page_id = io.page_id;
hdr->offset = io.offset;
rescues++;
} else {
lost++;
// TODO: re-alloc and replace header.
}
} else {
lost++;
}
}
do_item_remove(hdr_it);
}
item_unlock(hv);
offset += ntotal;
if (read_size - offset < sizeof(struct _stritem))
break;
}
STATS_LOCK();
stats.extstore_compact_lost += lost;
stats.extstore_compact_rescues += rescues;
stats.extstore_compact_skipped += skipped;
STATS_UNLOCK();
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_READ_END,
NULL, page_id, offset, rescues, lost, skipped);
}
/* Fetch stats from the external storage system and decide to compact.
* If we're more than half full, start skewing how aggressively to run
* compaction, up to a desired target when all pages are full.
*/
static int storage_compact_check(void *storage, logger *l,
uint32_t *page_id, uint64_t *page_version,
uint64_t *page_size, bool *drop_unread) {
struct extstore_stats st;
int x;
double rate;
uint64_t frag_limit;
uint64_t low_version = ULLONG_MAX;
uint64_t lowest_version = ULLONG_MAX;
unsigned int low_page = 0;
unsigned int lowest_page = 0;
extstore_get_stats(storage, &st);
if (st.pages_used == 0)
return 0;
// lets pick a target "wasted" value and slew.
if (st.pages_free > settings.ext_compact_under)
return 0;
*drop_unread = false;
// the number of free pages reduces the configured frag limit
// this allows us to defrag early if pages are very empty.
rate = 1.0 - ((double)st.pages_free / st.page_count);
rate *= settings.ext_max_frag;
frag_limit = st.page_size * rate;
LOGGER_LOG(l, LOG_SYSEVENTS, LOGGER_COMPACT_FRAGINFO,
NULL, rate, frag_limit);
st.page_data = calloc(st.page_count, sizeof(struct extstore_page_data));
extstore_get_page_data(storage, &st);
// find oldest page by version that violates the constraint
for (x = 0; x < st.page_count; x++) {
if (st.page_data[x].version == 0 ||
st.page_data[x].bucket == PAGE_BUCKET_LOWTTL)
continue;
if (st.page_data[x].version < lowest_version) {
lowest_page = x;
lowest_version = st.page_data[x].version;
}
if (st.page_data[x].bytes_used < frag_limit) {
if (st.page_data[x].version < low_version) {
low_page = x;
low_version = st.page_data[x].version;
}
}
}
*page_size = st.page_size;
free(st.page_data);
// we have a page + version to attempt to reclaim.
if (low_version != ULLONG_MAX) {
*page_id = low_page;
*page_version = low_version;
return 1;
} else if (lowest_version != ULLONG_MAX && settings.ext_drop_unread
&& st.pages_free <= settings.ext_drop_under) {
// nothing matched the frag rate barrier, so pick the absolute oldest
// version if we're configured to drop items.
*page_id = lowest_page;
*page_version = lowest_version;
*drop_unread = true;
return 1;
}
return 0;
}
static int storage_write(void *storage, const int clsid, const int item_age) {
int did_moves = 0;
struct lru_pull_tail_return it_info;
it_info.it = NULL;
lru_pull_tail(clsid, COLD_LRU, 0, LRU_PULL_RETURN_ITEM, 0, &it_info);
/* Item is locked, and we have a reference to it. */
if (it_info.it == NULL) {
return did_moves;
}
obj_io io;
item *it = it_info.it;
/* First, storage for the header object */
size_t orig_ntotal = ITEM_ntotal(it);
uint32_t flags;
if ((it->it_flags & ITEM_HDR) == 0 &&
(item_age == 0 || current_time - it->time > item_age)) {
FLAGS_CONV(it, flags);
item *hdr_it = do_item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, sizeof(item_hdr));
/* Run the storage write understanding the start of the item is dirty.
* We will fill it (time/exptime/etc) from the header item on read.
*/
if (hdr_it != NULL) {
int bucket = (it->it_flags & ITEM_CHUNKED) ?
PAGE_BUCKET_CHUNKED : PAGE_BUCKET_DEFAULT;
// Compress soon to expire items into similar pages.
if (it->exptime - current_time < settings.ext_low_ttl) {
bucket = PAGE_BUCKET_LOWTTL;
}
hdr_it->it_flags |= ITEM_HDR;
io.len = orig_ntotal;
io.mode = OBJ_IO_WRITE;
// NOTE: when the item is read back in, the slab mover
// may see it. Important to have refcount>=2 or ~ITEM_LINKED
assert(it->refcount >= 2);
// NOTE: write bucket vs free page bucket will disambiguate once
// lowttl feature is better understood.
if (extstore_write_request(storage, bucket, bucket, &io) == 0) {
// cuddle the hash value into the time field so we don't have
// to recalculate it.
item *buf_it = (item *) io.buf;
buf_it->time = it_info.hv;
// copy from past the headers + time headers.
// TODO: should be in items.c
if (it->it_flags & ITEM_CHUNKED) {
// Need to loop through the item and copy
item_chunk *sch = (item_chunk *) ITEM_schunk(it);
int remain = orig_ntotal;
int copied = 0;
// copy original header
int hdrtotal = ITEM_ntotal(it) - it->nbytes;
memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, hdrtotal - STORE_OFFSET);
copied = hdrtotal;
// copy data in like it were one large object.
while (sch && remain) {
assert(remain >= sch->used);
memcpy((char *)io.buf+copied, sch->data, sch->used);
// FIXME: use one variable?
remain -= sch->used;
copied += sch->used;
sch = sch->next;
}
} else {
memcpy((char *)io.buf+STORE_OFFSET, (char *)it+STORE_OFFSET, io.len-STORE_OFFSET);
}
// crc what we copied so we can do it sequentially.
buf_it->it_flags &= ~ITEM_LINKED;
buf_it->exptime = crc32c(0, (char*)io.buf+STORE_OFFSET, orig_ntotal-STORE_OFFSET);
extstore_write(storage, &io);
item_hdr *hdr = (item_hdr *) ITEM_data(hdr_it);
hdr->page_version = io.page_version;
hdr->page_id = io.page_id;
hdr->offset = io.offset;
// overload nbytes for the header it
hdr_it->nbytes = it->nbytes;
/* success! Now we need to fill relevant data into the new
* header and replace. Most of this requires the item lock
*/
/* CAS gets set while linking. Copy post-replace */
item_replace(it, hdr_it, it_info.hv);
ITEM_set_cas(hdr_it, ITEM_get_cas(it));
do_item_remove(hdr_it);
did_moves = 1;
LOGGER_LOG(NULL, LOG_EVICTIONS, LOGGER_EXTSTORE_WRITE, it, bucket);
} else {
/* Failed to write for some reason, can't continue. */
slabs_free(hdr_it, ITEM_ntotal(hdr_it), ITEM_clsid(hdr_it));
}
}
}
do_item_remove(it);
item_unlock(it_info.hv);
return did_moves;
}
