void *
obuf_reserve_slow(struct obuf *buf, size_t size)
{
struct iovec *iov = &buf->iov[buf->pos];
size_t capacity = buf->capacity[buf->pos];
if (iov->iov_len > 0) {
/* Move to the next buffer. */
if (buf->pos + 1 >= SMALL_OBUF_IOV_MAX)
return NULL;
buf->pos++;
iov = &buf->iov[buf->pos];
capacity = buf->capacity[buf->pos];
}
assert(iov->iov_len == 0);
/* Make sure the next buffer can store size. */
if (size > capacity) {
if (capacity > 0) {
/* Simply realloc. */
while (capacity < size)
capacity = capacity * 2;
struct slab *slab = slab_get(buf->slabc, capacity);
if (slab == NULL)
return NULL;
struct slab *old =
slab_from_data(buf->iov[buf->pos].iov_base);
slab_put(buf->slabc, old);
buf->iov[buf->pos].iov_base = slab_data(slab);
buf->capacity[buf->pos] = slab_capacity(slab);
} else if (obuf_alloc_pos(buf, size) == NULL) {
return NULL;
}
}
assert(buf->iov[buf->pos].iov_len + size <= buf->capacity[buf->pos]);
return (char*) buf->iov[buf->pos].iov_base + buf->iov[buf->pos].iov_len;
}
void
slab_print(void)
{
uint8_t id;
struct slabclass *p;
loga("slab size %zu, slab hdr size %zu, item hdr size %zu, item chunk size"
"%zu, total memory %zu", slab_size, SLAB_HDR_SIZE, ITEM_HDR_SIZE,
item_min, slab_mem);
for (id = SLABCLASS_MIN_ID; id <= profile_last_id; id++) {
p = &slabclass[id];
loga("class %3"PRId8": items %7"PRIu32" size %7zu data %7zu "
"slack %7zu", id, p->nitem, p->size, p->size - ITEM_HDR_SIZE,
slab_capacity() - p->nitem * p->size);
}
}
/** Allocate memory for a single iovec buffer. */
static inline void *
obuf_alloc_pos(struct obuf *buf, size_t size)
{
int pos = buf->pos;
assert(buf->capacity[pos] == 0 && buf->iov[pos].iov_len == 0);
assert(pos < SMALL_OBUF_IOV_MAX);
assert(buf->n_iov == pos);
/** Initialize the next pos. */
buf->iov[pos+1] = buf->iov[pos];
buf->capacity[pos+1] = buf->capacity[pos];
size_t capacity = buf->start_capacity << pos;
while (capacity < size) {
capacity = capacity == 0 ? buf->start_capacity: capacity * 2;
}
struct slab *slab = slab_get(buf->slabc, capacity);
if (slab == NULL)
return NULL;
buf->iov[pos].iov_base = slab_data(slab);
buf->capacity[pos] = slab_capacity(slab);
buf->n_iov++;
return buf->iov[pos].iov_base;
}
/*
* Initialize all slabclasses.
*
* Every slabclass is a collection of slabs of fixed size specified by
* --slab-size. A single slab is a collection of contiguous, equal sized
* item chunks of a given size specified by the profile array
*/
static rstatus_i
_slab_slabclass_setup(void)
{
uint8_t id; /* slabclass id */
ASSERT(profile_last_id <= SLABCLASS_MAX_ID);
for (id = SLABCLASS_MIN_ID; id <= profile_last_id; id++) {
struct slabclass *p; /* slabclass */
uint32_t nitem; /* # item per slabclass */
size_t item_sz; /* item size */
nitem = slab_capacity() / profile[id];
if (nitem == 0) {
log_error("Invalid slab class size %u; too large to fit in slab!",
profile[id]);
return CC_ERROR;
}
item_sz = profile[id];
p = &slabclass[id];
p->nitem = nitem;
p->size = item_sz;
/* chunk_size is static */
perslab[id] = (perslab_metrics_st){PERSLAB_METRIC(METRIC_INIT)};
UPDATE_VAL(&perslab[id], chunk_size, item_sz);
p->nfree_itemq = 0;
SLIST_INIT(&p->free_itemq);
p->nfree_item = 0;
p->next_item_in_slab = NULL;
}
return CC_OK;
}
static rstatus_i
_slab_profile_setup(char *profile_str)
{
int i;
/* TODO(yao): check alignment (with machine word length) in the user config,
* reject ones that don't align
*/
if (profile_str != NULL) {
/* slab profile specified */
char *profile_entry;
i = SLABCLASS_MIN_ID - 1;
do {
/*TODO(yao): strsep alters the original slab profile string, which
* is probably not desirable (we don't want to modify options once
* they are loaded. Do another memcpy to some local variable.
*/
profile_entry = strsep(&profile_str, " \n\r\t");
profile[++i] = atol(profile_entry);
if (profile[i] <= profile[i - 1]) {
log_error("Invalid setup profile configuration provided");
return CC_ERROR;
}
} while (profile_str != NULL);
profile_last_id = i;
} else {
/* generate slab profile using chunk size, slab size, and factor */
size_t nbyte, nitem, linear_nitem;
if (item_min <= ITEM_HDR_SIZE) {
log_error("invalid min chunk size - too small for item overhead");
return CC_ERROR;
}
if (item_max + SLAB_HDR_SIZE > slab_size) {
log_error("invalid max chunk size - too large to fit in one slab");
return CC_ERROR;
}
if (item_min > item_max) {
log_error("Could not setup slab profile; invalid min/max chunk size");
return CC_ERROR;
}
if (item_growth <= 1.0) {
log_error("Could not setup slab profile; invalid growth factor");
return CC_ERROR;
}
/*
* Slab profile is determined as follows:
*
* Exponential growth of item size based on gf^n, from which we determine
* the # items that can fit in the slab. At the point that the # items
* change is <= 1, we form each slab profile entry as 1 item less than
* the last until we hit 1 item.
*
* Example (assuming max chunk size == slab size):
*
* Suppose gf == 1.2, so we determine 1.2 * x >= x + 1 --> x >= 5
* Thus, once we hit nitem == 5, we reduce nitem linearly by 1 until we
* reach nitem == 1:
*
* Exponential growth while nitem increase still > 1
* +-----------------------------------------------------------+
* | | | | | | | | | | | | | | | |
* +-----------------------------------------------------------+
*
* +-----------------------------------------------------------+
* | | | | | | | | | | | | |
* +-----------------------------------------------------------+
*
* +-----------------------------------------------------------+
* | | | | | | | | | | |
* +-----------------------------------------------------------+
*
*
* .
* .
* .
*
*
* Transition to linear growth
* +-----------------------------------------------------------+
* | | | | |
* +-----------------------------------------------------------+
*
* +-----------------------------------------------------------+
* | | | |
* +-----------------------------------------------------------+
*
* +-----------------------------------------------------------+
* | | |
* +-----------------------------------------------------------+
*
* +-----------------------------------------------------------+
* | |
* +-----------------------------------------------------------+
*/
linear_nitem = 1.0 / (item_growth - 1.0);
i = SLABCLASS_MIN_ID;
nitem = slab_capacity() / (CC_ALIGN(item_min, CC_ALIGNMENT));
nbyte = slab_capacity() / nitem;
/* exponential growth phase */
while (nbyte <= item_max && nitem > linear_nitem) {
if (i > SLABCLASS_MAX_ID) {
log_error("Slab profile improperly configured - max chunk size "
"too large or growth factor too small");
return CC_ERROR;
}
if (profile[i - 1] == nbyte) {
nbyte += CC_ALIGNMENT;
}
profile[i++] = nbyte;
nitem = slab_capacity() / nbyte / item_growth;
nbyte = SLAB_ALIGN_DOWN(slab_capacity() / nitem, CC_ALIGNMENT);
}
/* linear growth phase */
nitem = linear_nitem;
nbyte = SLAB_ALIGN_DOWN(slab_capacity() / nitem, CC_ALIGNMENT);
while (nbyte <= item_max && nitem > 0) {
if (i > SLABCLASS_MAX_ID) {
log_error("Slab profile improperly configured - max chunk size "
"too large or growth factor too small");
return CC_ERROR;
}
profile[i++] = nbyte;
if (--nitem > 0) {
nbyte = SLAB_ALIGN_DOWN(slab_capacity() / nitem, CC_ALIGNMENT);
}
}
profile_last_id = i - 1;
}
log_verb("setup slab profile profile_last_id: %u", profile_last_id);
log_verb("slab profile:");
for (i = SLABCLASS_MIN_ID; i <= profile_last_id; ++i) {
log_verb("%u", profile[i]);
}
return CC_OK;
}
