/*
* calc_block_offset -- (internal) calculates the block offset of allocation
*/
static uint16_t
calc_block_offset(PMEMobjpool *pop, struct bucket *b,
struct allocation_header *alloc)
{
uint16_t block_off = 0;
if (bucket_is_small(b)) {
struct memory_block m = {alloc->chunk_id, alloc->zone_id, 0, 0};
void *data = heap_get_block_data(pop, m);
uintptr_t diff = (uintptr_t)alloc - (uintptr_t)data;
block_off = diff / bucket_unit_size(b);
ASSERT(diff % bucket_unit_size(b) == 0);
}
return block_off;
}
/*
* heap_populate_run_bucket -- (internal) split bitmap into memory blocks
*/
static void
heap_populate_run_bucket(PMEMobjpool *pop, struct bucket *b,
uint32_t chunk_id, uint32_t zone_id)
{
struct pmalloc_heap *h = pop->heap;
struct zone *z = &h->layout->zones[zone_id];
struct chunk_header *hdr = &z->chunk_headers[chunk_id];
struct chunk_run *run = (struct chunk_run *)&z->chunks[chunk_id];
if (hdr->type != CHUNK_TYPE_RUN)
heap_init_run(pop, b, hdr, run);
ASSERT(hdr->size_idx == 1);
ASSERT(bucket_unit_size(b) == run->block_size);
uint16_t run_bits = RUNSIZE / run->block_size;
ASSERT(run_bits < (MAX_BITMAP_VALUES * BITS_PER_VALUE));
uint16_t block_off = 0;
uint16_t block_size_idx = 0;
for (int i = 0; i < bucket_bitmap_nval(b); ++i) {
uint64_t v = run->bitmap[i];
block_off = BITS_PER_VALUE * i;
if (v == 0) {
heap_run_insert(b, chunk_id, zone_id,
BITS_PER_VALUE, block_off);
continue;
} else if (v == ~0L) {
continue;
}
for (int j = 0; j < BITS_PER_VALUE; ++j) {
if (BIT_IS_CLR(v, j)) {
block_size_idx++;
} else if (block_size_idx != 0) {
heap_run_insert(b, chunk_id, zone_id,
block_size_idx,
block_off - block_size_idx);
block_size_idx = 0;
}
if ((block_off++) == run_bits) {
i = MAX_BITMAP_VALUES;
break;
}
}
if (block_size_idx != 0) {
heap_run_insert(b, chunk_id, zone_id, block_size_idx,
block_off - block_size_idx);
block_size_idx = 0;
}
}
}
/*
* bucket_insert_block -- inserts a new memory block into the container
*/
int
bucket_insert_block(PMEMobjpool *pop, struct bucket *b, struct memory_block m)
{
ASSERT(m.chunk_id < MAX_CHUNK);
ASSERT(m.zone_id < UINT16_MAX);
ASSERT(m.size_idx != 0);
#ifdef USE_VG_MEMCHECK
if (On_valgrind) {
size_t rsize = m.size_idx * bucket_unit_size(b);
void *block_data = heap_get_block_data(pop, m);
VALGRIND_DO_MAKE_MEM_NOACCESS(pop, block_data, rsize);
}
#endif
uint64_t key = CHUNK_KEY_PACK(m.zone_id, m.chunk_id, m.block_off,
m.size_idx);
return ctree_insert(b->tree, key, 0);
}
/*
* prealloc_construct -- resizes an existing memory block with a constructor
*
* The block offset is written persistently into the off variable, but only
* after the constructor function has been called.
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
prealloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg), void *arg,
uint64_t data_off)
{
if (size <= pmalloc_usable_size(pop, *off))
return 0;
size_t sizeh = size + sizeof (struct allocation_header);
int err = 0;
struct allocation_header *alloc = alloc_get_header(pop, *off);
struct bucket *b = heap_get_best_bucket(pop, alloc->size);
uint32_t add_size_idx = bucket_calc_units(b, sizeh - alloc->size);
uint32_t new_size_idx = bucket_calc_units(b, sizeh);
uint64_t real_size = new_size_idx * bucket_unit_size(b);
struct memory_block cnt = get_mblock_from_alloc(pop, b, alloc);
if ((err = heap_lock_if_run(pop, cnt)) != 0)
return err;
struct memory_block next = {0};
if ((err = heap_get_adjacent_free_block(pop, &next, cnt, 0)) != 0)
goto error;
if (next.size_idx < add_size_idx) {
err = ENOMEM;
goto error;
}
if ((err = heap_get_exact_block(pop, b, &next,
add_size_idx)) != 0)
goto error;
struct memory_block *blocks[2] = {&cnt, &next};
uint64_t op_result;
void *hdr;
struct memory_block m =
heap_coalesce(pop, blocks, 2, HEAP_OP_ALLOC, &hdr, &op_result);
void *block_data = heap_get_block_data(pop, m);
void *datap = block_data + sizeof (struct allocation_header);
if (constructor != NULL)
constructor(pop, datap + data_off, arg);
struct lane_section *lane;
if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0)
goto error;
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, &alloc->size), real_size);
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
if (lane_release(pop) != 0) {
ERR("Failed to release the lane");
ASSERT(0);
}
if (heap_unlock_if_run(pop, cnt) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return 0;
error:
if (heap_unlock_if_run(pop, cnt) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return err;
}
/*
* pmalloc_construct -- allocates a new block of memory with a constructor
*
* The block offset is written persistently into the off variable, but only
* after the constructor function has been called.
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
pmalloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg),
void *arg, uint64_t data_off)
{
size_t sizeh = size + sizeof (struct allocation_header);
struct bucket *b = heap_get_best_bucket(pop, sizeh);
int err = 0;
uint32_t units = bucket_calc_units(b, sizeh);
struct memory_block m = {0, 0, units, 0};
if ((err = heap_get_bestfit_block(pop, b, &m)) != 0)
return err;
uint64_t op_result = 0;
void *block_data = heap_get_block_data(pop, m);
void *datap = block_data + sizeof (struct allocation_header);
ASSERT((uint64_t)block_data % _POBJ_CL_ALIGNMENT == 0);
uint64_t real_size = bucket_unit_size(b) * units;
alloc_write_header(pop, block_data, m.chunk_id, m.zone_id, real_size);
if (constructor != NULL)
constructor(pop, datap + data_off, arg);
if ((err = heap_lock_if_run(pop, m)) != 0)
return err;
void *hdr = heap_get_block_header(pop, m, HEAP_OP_ALLOC, &op_result);
struct lane_section *lane;
if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0)
goto err_lane_hold;
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, off), pop_offset(pop, datap));
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
if (lane_release(pop) != 0) {
ERR("Failed to release the lane");
ASSERT(0);
}
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return 0;
err_lane_hold:
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
if (bucket_insert_block(b, m) != 0) {
ERR("Failed to recover heap volatile state");
ASSERT(0);
}
return err;
}
/*
* pmalloc_construct -- allocates a new block of memory with a constructor
*
* The block offset is written persistently into the off variable, but only
* after the constructor function has been called.
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
pmalloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size,
void (*constructor)(PMEMobjpool *pop, void *ptr,
size_t usable_size, void *arg), void *arg, uint64_t data_off)
{
int err = 0;
struct lane_section *lane;
if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0)
return err;
size_t sizeh = size + sizeof (struct allocation_header);
struct bucket *b = heap_get_best_bucket(pop, sizeh);
struct memory_block m = {0, 0, 0, 0};
m.size_idx = bucket_calc_units(b, sizeh);
err = heap_get_bestfit_block(pop, b, &m);
if (err == ENOMEM && !bucket_is_small(b))
goto out; /* there's only one huge bucket */
if (err == ENOMEM) {
/*
* There's no more available memory in the common heap and in
* this lane cache, fallback to the auxiliary (shared) bucket.
*/
b = heap_get_auxiliary_bucket(pop, sizeh);
err = heap_get_bestfit_block(pop, b, &m);
}
if (err == ENOMEM) {
/*
* The auxiliary bucket cannot satisfy our request, borrow
* memory from other caches.
*/
heap_drain_to_auxiliary(pop, b, m.size_idx);
err = heap_get_bestfit_block(pop, b, &m);
}
if (err == ENOMEM) {
/* we are completely out of memory */
goto out;
}
/*
* Now that the memory is reserved we can go ahead with making the
* allocation persistent.
*/
uint64_t real_size = bucket_unit_size(b) * m.size_idx;
err = persist_alloc(pop, lane, m, real_size, off,
constructor, arg, data_off);
out:
if (lane_release(pop) != 0) {
ERR("Failed to release the lane");
ASSERT(0);
}
return err;
}
/*
* heap_chunk_init -- (internal) writes chunk header
*/
static void
heap_chunk_init(PMEMobjpool *pop, struct chunk_header *hdr,
uint16_t type, uint32_t size_idx)
{
struct chunk_header nhdr = {
.type = type,
.flags = 0,
.size_idx = size_idx
};
*hdr = nhdr; /* write the entire header (8 bytes) at once */
pop->persist(hdr, sizeof (*hdr));
heap_chunk_write_footer(hdr, size_idx);
}
/*
* heap_zone_init -- (internal) writes zone's first chunk and header
*/
static void
heap_zone_init(PMEMobjpool *pop, uint32_t zone_id)
{
struct zone *z = &pop->heap->layout->zones[zone_id];
uint32_t size_idx = get_zone_size_idx(zone_id, pop->heap->max_zone,
pop->heap_size);
heap_chunk_init(pop, &z->chunk_headers[0], CHUNK_TYPE_FREE, size_idx);
struct zone_header nhdr = {
.size_idx = size_idx,
.magic = ZONE_HEADER_MAGIC,
};
z->header = nhdr; /* write the entire header (8 bytes) at once */
pop->persist(&z->header, sizeof (z->header));
}
/*
* heap_init_run -- (internal) creates a run based on a chunk
*/
static void
heap_init_run(PMEMobjpool *pop, struct bucket *b, struct chunk_header *hdr,
struct chunk_run *run)
{
/* add/remove chunk_run and chunk_header to valgrind transaction */
VALGRIND_ADD_TO_TX(run, sizeof (*run));
run->block_size = bucket_unit_size(b);
pop->persist(&run->block_size, sizeof (run->block_size));
ASSERT(hdr->type == CHUNK_TYPE_FREE);
/* set all the bits */
memset(run->bitmap, 0xFF, sizeof (run->bitmap));
/* clear only the bits available for allocations from this bucket */
memset(run->bitmap, 0, sizeof (uint64_t) * (bucket_bitmap_nval(b) - 1));
run->bitmap[bucket_bitmap_nval(b) - 1] = bucket_bitmap_lastval(b);
VALGRIND_REMOVE_FROM_TX(run, sizeof (*run));
pop->persist(run->bitmap, sizeof (run->bitmap));
VALGRIND_ADD_TO_TX(hdr, sizeof (*hdr));
hdr->type = CHUNK_TYPE_RUN;
VALGRIND_REMOVE_FROM_TX(hdr, sizeof (*hdr));
pop->persist(hdr, sizeof (*hdr));
}
