/*
* alloc_class_find_min_frag -- searches for an existing allocation
* class that will provide the smallest internal fragmentation for the given
* size.
*/
static struct alloc_class *
alloc_class_find_min_frag(struct alloc_class_collection *ac, size_t n)
{
LOG(10, NULL);
struct alloc_class *best_c = NULL;
size_t lowest_waste = SIZE_MAX;
ASSERTne(n, 0);
/*
* Start from the largest buckets in order to minimize unit size of
* allocated memory blocks.
*/
for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
struct alloc_class *c = ac->aclasses[i];
/* can't use alloc classes /w no headers by default */
if (c == NULL || c->header_type == HEADER_NONE)
continue;
size_t real_size = n + header_type_to_size[c->header_type];
size_t units = CALC_SIZE_IDX(c->unit_size, real_size);
/* can't exceed the maximum allowed run unit max */
if (c->type == CLASS_RUN && units > RUN_UNIT_MAX_ALLOC)
continue;
if (c->unit_size * units == real_size)
return c;
size_t waste = (c->unit_size * units) - real_size;
/*
* If we assume that the allocation class is only ever going to
* be used with exactly one size, the effective internal
* fragmentation would be increased by the leftover
* memory at the end of the run.
*/
if (c->type == CLASS_RUN) {
size_t wasted_units = c->run.nallocs % units;
size_t wasted_bytes = wasted_units * c->unit_size;
size_t waste_avg_per_unit = wasted_bytes /
c->run.nallocs;
waste += waste_avg_per_unit;
}
if (best_c == NULL || lowest_waste > waste) {
best_c = c;
lowest_waste = waste;
}
}
ASSERTne(best_c, NULL);
return best_c;
}
/*
* alloc_class_find_or_create -- (internal) searches for the
* biggest allocation class for which unit_size is evenly divisible by n.
* If no such class exists, create one.
*/
static struct alloc_class *
alloc_class_find_or_create(struct alloc_class_collection *ac, size_t n)
{
LOG(10, NULL);
COMPILE_ERROR_ON(MAX_ALLOCATION_CLASSES > UINT8_MAX);
uint64_t required_size_bytes = n * RUN_MIN_NALLOCS;
uint32_t required_size_idx = 1;
if (required_size_bytes > RUNSIZE) {
required_size_bytes -= RUNSIZE;
required_size_idx +=
CALC_SIZE_IDX(CHUNKSIZE, required_size_bytes);
if (required_size_idx > RUN_SIZE_IDX_CAP)
required_size_idx = RUN_SIZE_IDX_CAP;
}
for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
struct alloc_class *c = ac->aclasses[i];
if (c == NULL || c->type == CLASS_HUGE ||
c->run.size_idx < required_size_idx)
continue;
if (n % c->unit_size == 0 &&
n / c->unit_size <= RUN_UNIT_MAX_ALLOC)
return c;
}
/*
* In order to minimize the wasted space at the end of the run the
* run data size must be divisible by the allocation class unit size
* with the smallest possible remainder, preferably 0.
*/
size_t runsize_bytes = RUN_SIZE_BYTES(required_size_idx);
while ((runsize_bytes % n) > MAX_RUN_WASTED_BYTES) {
n += ALLOC_BLOCK_SIZE_GEN;
}
/*
* Now that the desired unit size is found the existing classes need
* to be searched for possible duplicates. If a class that can handle
* the calculated size already exists, simply return that.
*/
for (int i = 1; i < MAX_ALLOCATION_CLASSES; ++i) {
struct alloc_class *c = ac->aclasses[i];
if (c == NULL || c->type == CLASS_HUGE)
continue;
if (n / c->unit_size <= RUN_UNIT_MAX_ALLOC &&
n % c->unit_size == 0)
return c;
if (c->unit_size == n)
return c;
}
return alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, n, 0,
required_size_idx);
}
/*
* alloc_class_find_min_frag -- searches for an existing allocation
* class that will provide the smallest internal fragmentation for the given
* size.
*/
static struct alloc_class *
alloc_class_find_min_frag(struct alloc_class_collection *ac, size_t n)
{
LOG(10, NULL);
struct alloc_class *best_c = NULL;
size_t best_frag_d = SIZE_MAX;
size_t best_frag_r = SIZE_MAX;
ASSERTne(n, 0);
/*
* Start from the largest buckets in order to minimize unit size of
* allocated memory blocks.
*/
for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
struct alloc_class *c = ac->aclasses[i];
/* can't use alloc classes /w no headers by default */
if (c == NULL || c->header_type == HEADER_NONE)
continue;
size_t real_size = n + header_type_to_size[c->header_type];
size_t units = CALC_SIZE_IDX(c->unit_size, real_size);
/* can't exceed the maximum allowed run unit max */
if (units > RUN_UNIT_MAX_ALLOC)
continue;
if (c->unit_size * units == real_size)
return c;
ASSERT(c->unit_size * units > real_size);
size_t frag_d = (c->unit_size * units) / real_size;
size_t frag_r = (c->unit_size * units) % real_size;
if (best_c == NULL || frag_d < best_frag_d ||
(frag_d == best_frag_d && frag_r < best_frag_r)) {
best_c = c;
best_frag_d = frag_d;
best_frag_r = frag_r;
}
}
ASSERTne(best_c, NULL);
return best_c;
}
/*
* heap_run_foreach_object -- (internal) iterates through objects in a run
*/
int
heap_run_foreach_object(struct palloc_heap *heap, object_callback cb,
void *arg, struct memory_block *m, struct alloc_class *c)
{
if (c == NULL)
return -1;
uint16_t i = m->block_off / BITS_PER_VALUE;
uint16_t block_start = m->block_off % BITS_PER_VALUE;
uint16_t block_off;
struct chunk_run *run = (struct chunk_run *)
&ZID_TO_ZONE(heap->layout, m->zone_id)->chunks[m->chunk_id];
for (; i < c->run.bitmap_nval; ++i) {
uint64_t v = run->bitmap[i];
block_off = (uint16_t)(BITS_PER_VALUE * i);
for (uint16_t j = block_start; j < BITS_PER_VALUE; ) {
if (block_off + j >= (uint16_t)c->run.bitmap_nallocs)
break;
if (!BIT_IS_CLR(v, j)) {
m->block_off = (uint16_t)(block_off + j);
/*
* The size index of this memory block cannot be
* retrieved at this time because the header
* might not be initialized in valgrind yet.
*/
m->size_idx = 0;
if (cb(m, arg)
!= 0)
return 1;
m->size_idx = CALC_SIZE_IDX(c->unit_size,
m->m_ops->get_real_size(m));
j = (uint16_t)(j + m->size_idx);
} else {
++j;
}
}
block_start = 0;
}
return 0;
}
/*
* run_iterate_used -- iterates over used blocks in a run
*/
static int
run_iterate_used(const struct memory_block *m, object_callback cb, void *arg)
{
uint32_t i = m->block_off / RUN_BITS_PER_VALUE;
uint32_t block_start = m->block_off % RUN_BITS_PER_VALUE;
uint32_t block_off;
struct chunk_run *run = heap_get_chunk_run(m->heap, m);
struct memory_block iter = *m;
struct run_bitmap b;
run_get_bitmap(m, &b);
for (; i < b.nvalues; ++i) {
uint64_t v = b.values[i];
block_off = (uint32_t)(RUN_BITS_PER_VALUE * i);
for (uint32_t j = block_start; j < RUN_BITS_PER_VALUE; ) {
if (block_off + j >= (uint32_t)b.nbits)
break;
if (!BIT_IS_CLR(v, j)) {
iter.block_off = (uint32_t)(block_off + j);
/*
* The size index of this memory block cannot be
* retrieved at this time because the header
* might not be initialized in valgrind yet.
*/
iter.size_idx = 0;
if (cb(&iter, arg) != 0)
return 1;
iter.size_idx = CALC_SIZE_IDX(
run->hdr.block_size,
iter.m_ops->get_real_size(&iter));
j = (uint32_t)(j + iter.size_idx);
} else {
++j;
}
}
block_start = 0;
}
return 0;
}
/*
* get_mblock_from_alloc -- (internal) returns allocation memory block
*/
static struct memory_block
get_mblock_from_alloc(PMEMobjpool *pop, struct allocation_header *alloc)
{
struct memory_block mblock = {
alloc->chunk_id,
alloc->zone_id,
0,
0
};
uint64_t unit_size = heap_get_chunk_block_size(pop, mblock);
mblock.block_off = calc_block_offset(pop, alloc, unit_size);
mblock.size_idx = CALC_SIZE_IDX(unit_size, alloc->size);
return mblock;
}
/*
* alloc_class_calc_size_idx -- calculates how many units does the size require
*/
ssize_t
alloc_class_calc_size_idx(struct alloc_class *c, size_t size)
{
uint32_t size_idx = CALC_SIZE_IDX(c->unit_size,
size + header_type_to_size[c->header_type]);
if (c->type == CLASS_RUN) {
if (c->header_type == HEADER_NONE && size_idx != 1)
return -1;
else if (size_idx > RUN_UNIT_MAX)
return -1;
else if (size_idx > c->run.bitmap_nallocs)
return -1;
}
return size_idx;
}
/*
* memblock_from_offset -- resolves a memory block data from an offset that
* originates from the heap
*/
struct memory_block
memblock_from_offset_opt(struct palloc_heap *heap, uint64_t off, int size)
{
struct memory_block m = MEMORY_BLOCK_NONE;
m.heap = heap;
off -= HEAP_PTR_TO_OFF(heap, &heap->layout->zone0);
m.zone_id = (uint32_t)(off / ZONE_MAX_SIZE);
off -= (ZONE_MAX_SIZE * m.zone_id) + sizeof(struct zone);
m.chunk_id = (uint32_t)(off / CHUNKSIZE);
struct chunk_header *hdr = &ZID_TO_ZONE(heap->layout, m.zone_id)
->chunk_headers[m.chunk_id];
if (hdr->type == CHUNK_TYPE_RUN_DATA)
m.chunk_id -= hdr->size_idx;
off -= CHUNKSIZE * m.chunk_id;
m.header_type = memblock_header_type(&m);
off -= header_type_to_size[m.header_type];
m.type = off != 0 ? MEMORY_BLOCK_RUN : MEMORY_BLOCK_HUGE;
#ifdef DEBUG
enum memory_block_type t = memblock_detect_type(&m, heap->layout);
ASSERTeq(t, m.type);
#endif
m.m_ops = &mb_ops[m.type];
uint64_t unit_size = m.m_ops->block_size(&m);
if (off != 0) { /* run */
off -= RUN_METASIZE;
m.block_off = (uint16_t)(off / unit_size);
off -= m.block_off * unit_size;
}
m.size_idx = !size ? 0 : CALC_SIZE_IDX(unit_size,
memblock_header_ops[m.header_type].get_size(&m));
ASSERTeq(off, 0);
return m;
}
/*
* memblock_from_offset -- resolves a memory block data from an offset that
* originates from the heap
*/
struct memory_block
memblock_from_offset_opt(struct palloc_heap *heap, uint64_t off, int size)
{
struct memory_block m = MEMORY_BLOCK_NONE;
m.heap = heap;
off -= HEAP_PTR_TO_OFF(heap, &heap->layout->zone0);
m.zone_id = (uint32_t)(off / ZONE_MAX_SIZE);
off -= (ZONE_MAX_SIZE * m.zone_id) + sizeof(struct zone);
m.chunk_id = (uint32_t)(off / CHUNKSIZE);
struct chunk_header *hdr = heap_get_chunk_hdr(heap, &m);
if (hdr->type == CHUNK_TYPE_RUN_DATA)
m.chunk_id -= hdr->size_idx;
off -= CHUNKSIZE * m.chunk_id;
m.header_type = memblock_header_type(&m);
off -= header_type_to_size[m.header_type];
m.type = off != 0 ? MEMORY_BLOCK_RUN : MEMORY_BLOCK_HUGE;
ASSERTeq(memblock_detect_type(heap, &m), m.type);
m.m_ops = &mb_ops[m.type];
uint64_t unit_size = m.m_ops->block_size(&m);
if (off != 0) { /* run */
struct chunk_run *run = heap_get_chunk_run(heap, &m);
off -= run_get_alignment_padding(hdr, run, m.header_type);
off -= RUN_METASIZE;
m.block_off = (uint16_t)(off / unit_size);
off -= m.block_off * unit_size;
}
m.size_idx = !size ? 0 : CALC_SIZE_IDX(unit_size,
memblock_header_ops[m.header_type].get_size(&m));
ASSERTeq(off, 0);
return m;
}
/*
* get_mblock_from_alloc -- (internal) returns allocation memory block
*/
static struct memory_block
get_mblock_from_alloc(struct palloc_heap *heap,
struct allocation_header *alloc)
{
struct memory_block m = {
alloc->chunk_id,
alloc->zone_id,
0,
0
};
uint64_t unit_size = MEMBLOCK_OPS(AUTO, &m)->
block_size(&m, heap->layout);
m.block_off = MEMBLOCK_OPS(AUTO, &m)->block_offset(&m, heap, alloc);
m.size_idx = CALC_SIZE_IDX(unit_size, alloc->size);
return m;
}
static int
pfree(uint64_t *off)
{
uint64_t offset = *off;
if (offset == 0)
return 0;
PMEMoid oid;
oid.off = offset;
struct allocation_header *hdr = &D_RW_OBJ(oid)->alloch;
struct zone *z = ZID_TO_ZONE(heap, hdr->zone_id);
struct chunk_header *chdr = &z->chunk_headers[hdr->chunk_id];
if (chdr->type == CHUNK_TYPE_USED) {
chdr->type = CHUNK_TYPE_FREE;
pmempool_convert_persist(poolset, &chdr->type,
sizeof(chdr->type));
*off = 0;
pmempool_convert_persist(poolset, off, sizeof(*off));
return 0;
} else if (chdr->type != CHUNK_TYPE_RUN) {
assert(0);
}
struct chunk_run *run =
(struct chunk_run *)&z->chunks[hdr->chunk_id].data;
uintptr_t diff = (uintptr_t)hdr - (uintptr_t)&run->data;
uint64_t block_off = (uint16_t)((size_t)diff / run->block_size);
uint64_t size_idx = CALC_SIZE_IDX(run->block_size, hdr->size);
uint64_t bmask = ((1ULL << size_idx) - 1ULL) <<
(block_off % BITS_PER_VALUE);
uint64_t bpos = block_off / BITS_PER_VALUE;
run->bitmap[bpos] &= ~bmask;
pmempool_convert_persist(poolset, &run->bitmap[bpos],
sizeof(run->bitmap[bpos]));
*off = 0;
pmempool_convert_persist(poolset, off, sizeof(*off));
return 0;
}
/*
* alloc_class_find_min_frag -- searches for an existing allocation
* class that will provide the smallest internal fragmentation for the given
* size.
*/
static struct alloc_class *
alloc_class_find_min_frag(struct alloc_class_collection *ac, size_t n)
{
struct alloc_class *best_c = NULL;
float best_frag = FLT_MAX;
ASSERTne(n, 0);
/*
* Start from the largest buckets in order to minimize unit size of
* allocated memory blocks.
*/
for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) {
struct alloc_class *c = ac->aclasses[i];
/* can't use alloc classes /w no headers by default */
if (c == NULL || c->header_type == HEADER_NONE)
continue;
size_t real_size = n + header_type_to_size[c->header_type];
size_t units = CALC_SIZE_IDX(c->unit_size, real_size);
/* can't exceed the maximum allowed run unit max */
if (units > RUN_UNIT_MAX_ALLOC)
continue;
float frag = (float)(c->unit_size * units) / (float)real_size;
if (frag == 1.f)
return c;
ASSERT(frag >= 1.f);
if (frag < best_frag || best_c == NULL) {
best_c = c;
best_frag = frag;
}
}
ASSERTne(best_c, NULL);
return best_c;
}
/*
* memblock_validate_offset -- checks the state of any arbtirary offset within
* the heap.
*
* This function traverses an entire zone, so use with caution.
*/
enum memblock_state
memblock_validate_offset(struct palloc_heap *heap, uint64_t off)
{
struct memory_block m = MEMORY_BLOCK_NONE;
m.heap = heap;
off -= HEAP_PTR_TO_OFF(heap, &heap->layout->zone0);
m.zone_id = (uint32_t)(off / ZONE_MAX_SIZE);
off -= (ZONE_MAX_SIZE * m.zone_id) + sizeof(struct zone);
m.chunk_id = (uint32_t)(off / CHUNKSIZE);
struct zone *z = ZID_TO_ZONE(heap->layout, m.zone_id);
struct chunk_header *hdr = &z->chunk_headers[m.chunk_id];
if (hdr->type == CHUNK_TYPE_RUN_DATA)
m.chunk_id -= hdr->size_idx;
off -= CHUNKSIZE * m.chunk_id;
for (uint32_t i = 0; i < z->header.size_idx; ) {
hdr = &z->chunk_headers[i];
if (i + hdr->size_idx > m.chunk_id && i < m.chunk_id) {
return MEMBLOCK_STATE_UNKNOWN; /* invalid chunk */
} else if (m.chunk_id == i) {
break;
}
i += hdr->size_idx;
}
ASSERTne(hdr, NULL);
m.header_type = memblock_header_type(&m);
if (hdr->type != CHUNK_TYPE_RUN) {
if (header_type_to_size[m.header_type] != off)
return MEMBLOCK_STATE_UNKNOWN;
else if (hdr->type == CHUNK_TYPE_USED)
return MEMBLOCK_ALLOCATED;
else if (hdr->type == CHUNK_TYPE_FREE)
return MEMBLOCK_FREE;
else
return MEMBLOCK_STATE_UNKNOWN;
}
if (header_type_to_size[m.header_type] > off)
return MEMBLOCK_STATE_UNKNOWN;
off -= header_type_to_size[m.header_type];
m.type = off != 0 ? MEMORY_BLOCK_RUN : MEMORY_BLOCK_HUGE;
#ifdef DEBUG
enum memory_block_type t = memblock_detect_type(&m, heap->layout);
ASSERTeq(t, m.type);
#endif
m.m_ops = &mb_ops[m.type];
uint64_t unit_size = m.m_ops->block_size(&m);
if (off != 0) { /* run */
off -= RUN_METASIZE;
m.block_off = (uint16_t)(off / unit_size);
off -= m.block_off * unit_size;
}
m.size_idx = CALC_SIZE_IDX(unit_size,
memblock_header_ops[m.header_type].get_size(&m));
ASSERTeq(off, 0);
return m.m_ops->get_state(&m);
}
