/* * memblock_run_bitmap -- calculate bitmap parameters for given arguments */ void memblock_run_bitmap(uint32_t *size_idx, uint16_t flags, uint64_t unit_size, uint64_t alignment, void *content, struct run_bitmap *b) { ASSERTne(*size_idx, 0); /* * Flexible bitmaps have a variably sized values array. The size varies * depending on: * alignment - initial run alignment might require up-to a unit * size idx - the larger the run, the more units it carries * unit_size - the smaller the unit size, the more units per run * * The size of the bitmap also has to be calculated in such a way that * the beginning of allocations data is cacheline aligned. This is * required to perform many optimizations throughout the codebase. * This alignment requirement means that some of the bitmap values might * remain unused and will serve only as a padding for data. */ if (flags & CHUNK_FLAG_FLEX_BITMAP) { /* * First calculate the number of values without accounting for * the bitmap size. */ size_t content_size = RUN_CONTENT_SIZE_BYTES(*size_idx); b->nbits = (unsigned)(content_size / unit_size); b->nvalues = util_div_ceil(b->nbits, RUN_BITS_PER_VALUE); /* * Then, align the number of values up, so that the cacheline * alignment is preserved. */ b->nvalues = ALIGN_UP(b->nvalues + RUN_BASE_METADATA_VALUES, 8U) - RUN_BASE_METADATA_VALUES; /* * This is the total number of bytes needed for the bitmap AND * padding. */ b->size = b->nvalues * sizeof(*b->values); /* * Calculate the number of allocations again, but this time * accounting for the bitmap/padding. */ b->nbits = (unsigned)((content_size - b->size) / unit_size) - (alignment ? 1U : 0U); /* * The last step is to calculate how much of the padding * is left at the end of the bitmap. */ unsigned unused_bits = (b->nvalues * RUN_BITS_PER_VALUE) - b->nbits; unsigned unused_values = unused_bits / RUN_BITS_PER_VALUE; b->nvalues -= unused_values; b->values = (uint64_t *)content; return; } b->size = RUN_DEFAULT_BITMAP_SIZE; b->nbits = memblock_run_default_nallocs(size_idx, flags, unit_size, alignment); unsigned unused_bits = RUN_DEFAULT_BITMAP_NBITS - b->nbits; unsigned unused_values = unused_bits / RUN_BITS_PER_VALUE; b->nvalues = RUN_DEFAULT_BITMAP_VALUES - unused_values; b->values = (uint64_t *)content; }
/* * alloc_class_find_or_create -- (internal) searches for the * biggest allocation class for which unit_size is evenly divisible by n. * If no such class exists, create one. */ static struct alloc_class * alloc_class_find_or_create(struct alloc_class_collection *ac, size_t n) { LOG(10, NULL); COMPILE_ERROR_ON(MAX_ALLOCATION_CLASSES > UINT8_MAX); uint64_t required_size_bytes = n * RUN_MIN_NALLOCS; uint32_t required_size_idx = 1; if (required_size_bytes > RUN_DEFAULT_SIZE) { required_size_bytes -= RUN_DEFAULT_SIZE; required_size_idx += CALC_SIZE_IDX(CHUNKSIZE, required_size_bytes); if (required_size_idx > RUN_SIZE_IDX_CAP) required_size_idx = RUN_SIZE_IDX_CAP; } for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE || c->run.size_idx < required_size_idx) continue; if (n % c->unit_size == 0 && n / c->unit_size <= RUN_UNIT_MAX_ALLOC) return c; } /* * In order to minimize the wasted space at the end of the run the * run data size must be divisible by the allocation class unit size * with the smallest possible remainder, preferably 0. */ struct run_bitmap b; size_t runsize_bytes = 0; do { if (runsize_bytes != 0) /* don't increase on first iteration */ n += ALLOC_BLOCK_SIZE_GEN; uint32_t size_idx = required_size_idx; memblock_run_bitmap(&size_idx, ALLOC_CLASS_DEFAULT_FLAGS, n, 0, NULL, &b); runsize_bytes = RUN_CONTENT_SIZE_BYTES(size_idx) - b.size; } while ((runsize_bytes % n) > MAX_RUN_WASTED_BYTES); /* * Now that the desired unit size is found the existing classes need * to be searched for possible duplicates. If a class that can handle * the calculated size already exists, simply return that. */ for (int i = 1; i < MAX_ALLOCATION_CLASSES; ++i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE) continue; if (n / c->unit_size <= RUN_UNIT_MAX_ALLOC && n % c->unit_size == 0) return c; if (c->unit_size == n) return c; } return alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, n, 0, required_size_idx); }