/*
* memblock_run_bitmap -- calculate bitmap parameters for given arguments
*/
void
memblock_run_bitmap(uint32_t *size_idx, uint16_t flags,
uint64_t unit_size, uint64_t alignment, void *content,
struct run_bitmap *b)
{
ASSERTne(*size_idx, 0);
/*
* Flexible bitmaps have a variably sized values array. The size varies
* depending on:
* alignment - initial run alignment might require up-to a unit
* size idx - the larger the run, the more units it carries
* unit_size - the smaller the unit size, the more units per run
*
* The size of the bitmap also has to be calculated in such a way that
* the beginning of allocations data is cacheline aligned. This is
* required to perform many optimizations throughout the codebase.
* This alignment requirement means that some of the bitmap values might
* remain unused and will serve only as a padding for data.
*/
if (flags & CHUNK_FLAG_FLEX_BITMAP) {
/*
* First calculate the number of values without accounting for
* the bitmap size.
*/
size_t content_size = RUN_CONTENT_SIZE_BYTES(*size_idx);
b->nbits = (unsigned)(content_size / unit_size);
b->nvalues = util_div_ceil(b->nbits, RUN_BITS_PER_VALUE);
/*
* Then, align the number of values up, so that the cacheline
* alignment is preserved.
*/
b->nvalues = ALIGN_UP(b->nvalues + RUN_BASE_METADATA_VALUES, 8U)
- RUN_BASE_METADATA_VALUES;
/*
* This is the total number of bytes needed for the bitmap AND
* padding.
*/
b->size = b->nvalues * sizeof(*b->values);
/*
* Calculate the number of allocations again, but this time
* accounting for the bitmap/padding.
*/
b->nbits = (unsigned)((content_size - b->size) / unit_size)
- (alignment ? 1U : 0U);
/*
* The last step is to calculate how much of the padding
* is left at the end of the bitmap.
*/
unsigned unused_bits = (b->nvalues * RUN_BITS_PER_VALUE)
- b->nbits;
unsigned unused_values = unused_bits / RUN_BITS_PER_VALUE;
b->nvalues -= unused_values;
b->values = (uint64_t *)content;
return;
}
b->size = RUN_DEFAULT_BITMAP_SIZE;
b->nbits = memblock_run_default_nallocs(size_idx, flags,
unit_size, alignment);
unsigned unused_bits = RUN_DEFAULT_BITMAP_NBITS - b->nbits;
unsigned unused_values = unused_bits / RUN_BITS_PER_VALUE;
b->nvalues = RUN_DEFAULT_BITMAP_VALUES - unused_values;
b->values = (uint64_t *)content;
}
/*
* alloc_class_find_or_create -- (internal) searches for the
* biggest allocation class for which unit_size is evenly divisible by n.
* If no such class exists, create one.
*/
static struct alloc_class *
alloc_class_find_or_create(struct alloc_class_collection *ac, size_t n)
{
LOG(10, NULL);
COMPILE_ERROR_ON(MAX_ALLOCATION_CLASSES > UINT8_MAX);
uint64_t required_size_bytes = n * RUN_MIN_NALLOCS;
uint32_t required_size_idx = 1;
if (required_size_bytes > RUN_DEFAULT_SIZE) {
required_size_bytes -= RUN_DEFAULT_SIZE;
required_size_idx +=
CALC_SIZE_IDX(CHUNKSIZE, required_size_bytes);
if (required_size_idx > RUN_SIZE_IDX_CAP)
required_size_idx = RUN_SIZE_IDX_CAP;
}
for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
struct alloc_class *c = ac->aclasses[i];
if (c == NULL || c->type == CLASS_HUGE ||
c->run.size_idx < required_size_idx)
continue;
if (n % c->unit_size == 0 &&
n / c->unit_size <= RUN_UNIT_MAX_ALLOC)
return c;
}
/*
* In order to minimize the wasted space at the end of the run the
* run data size must be divisible by the allocation class unit size
* with the smallest possible remainder, preferably 0.
*/
struct run_bitmap b;
size_t runsize_bytes = 0;
do {
if (runsize_bytes != 0) /* don't increase on first iteration */
n += ALLOC_BLOCK_SIZE_GEN;
uint32_t size_idx = required_size_idx;
memblock_run_bitmap(&size_idx, ALLOC_CLASS_DEFAULT_FLAGS, n, 0,
NULL, &b);
runsize_bytes = RUN_CONTENT_SIZE_BYTES(size_idx) - b.size;
} while ((runsize_bytes % n) > MAX_RUN_WASTED_BYTES);
/*
* Now that the desired unit size is found the existing classes need
* to be searched for possible duplicates. If a class that can handle
* the calculated size already exists, simply return that.
*/
for (int i = 1; i < MAX_ALLOCATION_CLASSES; ++i) {
struct alloc_class *c = ac->aclasses[i];
if (c == NULL || c->type == CLASS_HUGE)
continue;
if (n / c->unit_size <= RUN_UNIT_MAX_ALLOC &&
n % c->unit_size == 0)
return c;
if (c->unit_size == n)
return c;
}
return alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, n, 0,
required_size_idx);
}