/*
* heap_init -- initializes the heap
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_init(void *heap_start, uint64_t heap_size, struct pmem_ops *p_ops)
{
if (heap_size < HEAP_MIN_SIZE)
return EINVAL;
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap_start, heap_size);
struct heap_layout *layout = heap_start;
heap_write_header(&layout->header, heap_size);
pmemops_persist(p_ops, &layout->header, sizeof(struct heap_header));
unsigned zones = heap_max_zone(heap_size);
for (unsigned i = 0; i < zones; ++i) {
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->header,
0, sizeof(struct zone_header));
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->chunk_headers,
0, sizeof(struct chunk_header));
/* only explicitly allocated chunks should be accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(
&ZID_TO_ZONE(layout, i)->chunk_headers,
sizeof(struct chunk_header));
}
return 0;
}
/*
* alloc_prep_block -- (internal) prepares a memory block for allocation
*
* Once the block is fully reserved and it's guaranteed that no one else will
* be able to write to this memory region it is safe to write the allocation
* header and call the object construction function.
*
* Because the memory block at this stage is only reserved in transient state
* there's no need to worry about fail-safety of this method because in case
* of a crash the memory will be back in the free blocks collection.
*/
static int
alloc_prep_block(struct palloc_heap *heap, struct memory_block m,
palloc_constr constructor, void *arg, uint64_t *offset_value)
{
void *block_data = heap_get_block_data(heap, m);
void *userdatap = (char *)block_data + ALLOC_OFF;
uint64_t unit_size = MEMBLOCK_OPS(AUTO, &m)->
block_size(&m, heap->layout);
uint64_t real_size = unit_size * m.size_idx;
ASSERT((uint64_t)block_data % ALLOC_BLOCK_SIZE == 0);
ASSERT((uint64_t)userdatap % ALLOC_BLOCK_SIZE == 0);
/* mark everything (including headers) as accessible */
VALGRIND_DO_MAKE_MEM_UNDEFINED(block_data, real_size);
/* mark space as allocated */
VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, userdatap,
real_size - ALLOC_OFF);
alloc_write_header(heap, block_data, m, real_size);
int ret;
if (constructor != NULL &&
(ret = constructor(heap->base, userdatap,
real_size - ALLOC_OFF, arg)) != 0) {
/*
* If canceled, revert the block back to the free state in vg
* machinery. Because the free operation is only performed on
* the user data, the allocation header is made inaccessible
* in a separate call.
*/
VALGRIND_DO_MEMPOOL_FREE(heap->layout, userdatap);
VALGRIND_DO_MAKE_MEM_NOACCESS(block_data, ALLOC_OFF);
/*
* During this method there are several stores to pmem that are
* not immediately flushed and in case of a cancellation those
* stores are no longer relevant anyway.
*/
VALGRIND_SET_CLEAN(block_data, ALLOC_OFF);
return ret;
}
/* flushes both the alloc and oob headers */
pmemops_persist(&heap->p_ops, block_data, ALLOC_OFF);
/*
* To avoid determining the user data pointer twice this method is also
* responsible for calculating the offset of the object in the pool that
* will be used to set the offset destination pointer provided by the
* caller.
*/
*offset_value = PMALLOC_PTR_TO_OFF(heap, userdatap);
return 0;
}
/*
* persist_alloc -- (internal) performs a persistent allocation of the
* memory block previously reserved by volatile bucket
*/
static int
persist_alloc(PMEMobjpool *pop, struct lane_section *lane,
struct memory_block m, uint64_t real_size, uint64_t *off,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg),
void *arg, uint64_t data_off)
{
int err;
#ifdef DEBUG
if (heap_block_is_allocated(pop, m)) {
ERR("heap corruption");
ASSERT(0);
}
#endif /* DEBUG */
uint64_t op_result = 0;
void *block_data = heap_get_block_data(pop, m);
void *datap = (char *)block_data + sizeof (struct allocation_header);
void *userdatap = (char *)datap + data_off;
ASSERT((uint64_t)block_data % _POBJ_CL_ALIGNMENT == 0);
/* mark everything (including headers) as accessible */
VALGRIND_DO_MAKE_MEM_UNDEFINED(pop, block_data, real_size);
/* mark space as allocated */
VALGRIND_DO_MEMPOOL_ALLOC(pop, userdatap,
real_size -
sizeof (struct allocation_header) - data_off);
alloc_write_header(pop, block_data, m.chunk_id, m.zone_id, real_size);
if (constructor != NULL)
constructor(pop, userdatap, arg);
if ((err = heap_lock_if_run(pop, m)) != 0) {
VALGRIND_DO_MEMPOOL_FREE(pop, userdatap);
return err;
}
void *hdr = heap_get_block_header(pop, m, HEAP_OP_ALLOC, &op_result);
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, off), pop_offset(pop, datap));
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return err;
}
/*
* huge_prep_operation_hdr -- prepares the new value of a chunk header that will
* be set after the operation concludes.
*/
static void
huge_prep_operation_hdr(const struct memory_block *m, enum memblock_state op,
struct operation_context *ctx)
{
struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m);
/*
* Depending on the operation that needs to be performed a new chunk
* header needs to be prepared with the new chunk state.
*/
uint64_t val = chunk_get_chunk_hdr_value(
op == MEMBLOCK_ALLOCATED ? CHUNK_TYPE_USED : CHUNK_TYPE_FREE,
hdr->flags,
m->size_idx);
if (ctx == NULL) {
util_atomic_store_explicit64((uint64_t *)hdr, val,
memory_order_relaxed);
pmemops_persist(&m->heap->p_ops, hdr, sizeof(*hdr));
} else {
operation_add_entry(ctx, hdr, val, ULOG_OPERATION_SET);
}
VALGRIND_DO_MAKE_MEM_NOACCESS(hdr + 1,
(hdr->size_idx - 1) * sizeof(struct chunk_header));
/*
* In the case of chunks larger than one unit the footer must be
* created immediately AFTER the persistent state is safely updated.
*/
if (m->size_idx == 1)
return;
struct chunk_header *footer = hdr + m->size_idx - 1;
VALGRIND_DO_MAKE_MEM_UNDEFINED(footer, sizeof(*footer));
val = chunk_get_chunk_hdr_value(CHUNK_TYPE_FOOTER, 0, m->size_idx);
/*
* It's only safe to write the footer AFTER the persistent part of
* the operation have been successfully processed because the footer
* pointer might point to a currently valid persistent state
* of a different chunk.
* The footer entry change is updated as transient because it will
* be recreated at heap boot regardless - it's just needed for runtime
* operations.
*/
if (ctx == NULL) {
util_atomic_store_explicit64((uint64_t *)footer, val,
memory_order_relaxed);
VALGRIND_SET_CLEAN(footer, sizeof(*footer));
} else {
operation_add_typed_entry(ctx,
footer, val, ULOG_OPERATION_SET, LOG_TRANSIENT);
}
}
/*
* memblock_header_compact_write --
* (internal) writes a compact header of an object
*/
static void
memblock_header_compact_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
struct allocation_header_compact *hdr = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr));
VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr));
hdr->size = size | ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
hdr->extra = extra;
VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr));
}
/*
* memblock_header_compact_write --
* (internal) writes a compact header of an object
*/
static void
memblock_header_compact_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
COMPILE_ERROR_ON(ALLOC_HDR_COMPACT_SIZE > CACHELINE_SIZE);
struct {
struct allocation_header_compact hdr;
uint8_t padding[CACHELINE_SIZE - ALLOC_HDR_COMPACT_SIZE];
} padded;
padded.hdr.size = size | ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
padded.hdr.extra = extra;
struct allocation_header_compact *hdrp = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp));
/*
* If possible write the entire header with a single memcpy, this allows
* the copy implementation to avoid a cache miss on a partial cache line
* write.
*/
size_t hdr_size = ALLOC_HDR_COMPACT_SIZE;
if ((uintptr_t)hdrp % CACHELINE_SIZE == 0 && size >= sizeof(padded))
hdr_size = sizeof(padded);
VALGRIND_ADD_TO_TX(hdrp, hdr_size);
pmemops_memcpy(&m->heap->p_ops, hdrp, &padded, hdr_size,
PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED);
VALGRIND_DO_MAKE_MEM_UNDEFINED((char *)hdrp + ALLOC_HDR_COMPACT_SIZE,
hdr_size - ALLOC_HDR_COMPACT_SIZE);
VALGRIND_REMOVE_FROM_TX(hdrp, hdr_size);
}
/*
* heap_chunk_write_footer -- writes a chunk footer
*/
static void
heap_chunk_write_footer(struct chunk_header *hdr, uint32_t size_idx)
{
if (size_idx == 1) /* that would overwrite the header */
return;
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr + size_idx - 1, sizeof(*hdr));
struct chunk_header f = *hdr;
f.type = CHUNK_TYPE_FOOTER;
f.size_idx = size_idx;
*(hdr + size_idx - 1) = f;
/* no need to persist, footers are recreated in heap_populate_buckets */
VALGRIND_SET_CLEAN(hdr + size_idx - 1, sizeof(f));
}
/*
* memblock_header_legacy_write --
* (internal) writes a legacy header of an object
*/
static void
memblock_header_legacy_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
struct allocation_header_legacy *hdr = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr));
VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr));
hdr->size = size;
hdr->type_num = extra;
hdr->root_size = ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr));
/* unused fields of the legacy headers are used as a red zone */
VALGRIND_DO_MAKE_MEM_NOACCESS(hdr->unused, sizeof(hdr->unused));
}
/*
* huge_prep_operation_hdr -- prepares the new value of a chunk header that will
* be set after the operation concludes.
*/
static void
huge_prep_operation_hdr(const struct memory_block *m, enum memblock_state op,
struct operation_context *ctx)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
/*
* Depending on the operation that needs to be performed a new chunk
* header needs to be prepared with the new chunk state.
*/
uint64_t val = chunk_get_chunk_hdr_value(
op == MEMBLOCK_ALLOCATED ? CHUNK_TYPE_USED : CHUNK_TYPE_FREE,
hdr->flags,
m->size_idx);
operation_add_entry(ctx, hdr, val, OPERATION_SET);
VALGRIND_DO_MAKE_MEM_NOACCESS(hdr + 1,
(hdr->size_idx - 1) * sizeof(struct chunk_header));
/*
* In the case of chunks larger than one unit the footer must be
* created immediately AFTER the persistent state is safely updated.
*/
if (m->size_idx == 1)
return;
struct chunk_header *footer = hdr + m->size_idx - 1;
VALGRIND_DO_MAKE_MEM_UNDEFINED(footer, sizeof(*footer));
val = chunk_get_chunk_hdr_value(CHUNK_TYPE_FOOTER, 0, m->size_idx);
/*
* It's only safe to write the footer AFTER the persistent part of
* the operation have been successfully processed because the footer
* pointer might point to a currently valid persistent state
* of a different chunk.
* The footer entry change is updated as transient because it will
* be recreated at heap boot regardless - it's just needed for runtime
* operations.
*/
operation_add_typed_entry(ctx,
footer, val, OPERATION_SET, ENTRY_TRANSIENT);
}
static void
init_run_with_max_block(struct heap_layout *l, uint32_t chunk_id)
{
l->zone0.chunk_headers[chunk_id].size_idx = 1;
l->zone0.chunk_headers[chunk_id].type = CHUNK_TYPE_RUN;
l->zone0.chunk_headers[chunk_id].flags = 0;
struct chunk_run *run = (struct chunk_run *)
&l->zone0.chunks[chunk_id];
VALGRIND_DO_MAKE_MEM_UNDEFINED(run, sizeof(*run));
run->block_size = 1024;
memset(run->bitmap, 0xFF, sizeof(run->bitmap));
/* the biggest block is 10 bits */
run->bitmap[3] =
0b1000001110111000111111110000111111000000000011111111110000000011;
}
static void
init_run_with_score(struct heap_layout *l, uint32_t chunk_id, int score)
{
l->zone0.chunk_headers[chunk_id].size_idx = 1;
l->zone0.chunk_headers[chunk_id].type = CHUNK_TYPE_RUN;
l->zone0.chunk_headers[chunk_id].flags = 0;
struct chunk_run *run = (struct chunk_run *)
&l->zone0.chunks[chunk_id];
VALGRIND_DO_MAKE_MEM_UNDEFINED(run, sizeof(*run));
run->block_size = 1024;
memset(run->bitmap, 0xFF, sizeof(run->bitmap));
UT_ASSERTeq(score % 64, 0);
score /= 64;
for (; score > 0; --score) {
run->bitmap[score] = 0;
}
}
/*
* memblock_header_legacy_write --
* (internal) writes a legacy header of an object
*/
static void
memblock_header_legacy_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
struct allocation_header_legacy hdr;
hdr.size = size;
hdr.type_num = extra;
hdr.root_size = ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
struct allocation_header_legacy *hdrp = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp));
VALGRIND_ADD_TO_TX(hdrp, sizeof(*hdrp));
pmemops_memcpy(&m->heap->p_ops, hdrp, &hdr,
sizeof(hdr), /* legacy header is 64 bytes in size */
PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED);
VALGRIND_REMOVE_FROM_TX(hdrp, sizeof(*hdrp));
/* unused fields of the legacy headers are used as a red zone */
VALGRIND_DO_MAKE_MEM_NOACCESS(hdrp->unused, sizeof(hdrp->unused));
}
/*
* alloc_prep_block -- (internal) prepares a memory block for allocation
*
* Once the block is fully reserved and it's guaranteed that no one else will
* be able to write to this memory region it is safe to write the allocation
* header and call the object construction function.
*
* Because the memory block at this stage is only reserved in transient state
* there's no need to worry about fail-safety of this method because in case
* of a crash the memory will be back in the free blocks collection.
*/
static int
alloc_prep_block(struct palloc_heap *heap, const struct memory_block *m,
palloc_constr constructor, void *arg,
uint64_t extra_field, uint16_t object_flags,
uint64_t *offset_value)
{
void *uptr = m->m_ops->get_user_data(m);
size_t usize = m->m_ops->get_user_size(m);
VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, uptr, usize);
VALGRIND_DO_MAKE_MEM_UNDEFINED(uptr, usize);
VALGRIND_ANNOTATE_NEW_MEMORY(uptr, usize);
int ret;
if (constructor != NULL &&
(ret = constructor(heap->base, uptr, usize, arg)) != 0) {
/*
* If canceled, revert the block back to the free state in vg
* machinery.
*/
VALGRIND_DO_MEMPOOL_FREE(heap->layout, uptr);
return ret;
}
m->m_ops->write_header(m, extra_field, object_flags);
/*
* To avoid determining the user data pointer twice this method is also
* responsible for calculating the offset of the object in the pool that
* will be used to set the offset destination pointer provided by the
* caller.
*/
*offset_value = HEAP_PTR_TO_OFF(heap, uptr);
return 0;
}
/*
* prealloc_construct -- resizes an existing memory block with a constructor
*
* The block offset is written persistently into the off variable, but only
* after the constructor function has been called.
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
prealloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size,
void (*constructor)(PMEMobjpool *pop, void *ptr,
size_t usable_size, void *arg), void *arg, uint64_t data_off)
{
if (size <= pmalloc_usable_size(pop, *off))
return 0;
size_t sizeh = size + sizeof (struct allocation_header);
int err = 0;
struct allocation_header *alloc = alloc_get_header(pop, *off);
struct lane_section *lane;
if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0)
return err;
struct bucket *b = heap_get_best_bucket(pop, alloc->size);
uint32_t add_size_idx = bucket_calc_units(b, sizeh - alloc->size);
uint32_t new_size_idx = bucket_calc_units(b, sizeh);
uint64_t real_size = new_size_idx * bucket_unit_size(b);
struct memory_block cnt = get_mblock_from_alloc(pop, b, alloc);
if ((err = heap_lock_if_run(pop, cnt)) != 0)
goto out_lane;
struct memory_block next = {0, 0, 0, 0};
if ((err = heap_get_adjacent_free_block(pop, &next, cnt, 0)) != 0)
goto out;
if (next.size_idx < add_size_idx) {
err = ENOMEM;
goto out;
}
if ((err = heap_get_exact_block(pop, b, &next,
add_size_idx)) != 0)
goto out;
struct memory_block *blocks[2] = {&cnt, &next};
uint64_t op_result;
void *hdr;
struct memory_block m =
heap_coalesce(pop, blocks, 2, HEAP_OP_ALLOC, &hdr, &op_result);
void *block_data = heap_get_block_data(pop, m);
void *datap = (char *)block_data + sizeof (struct allocation_header);
void *userdatap = (char *)datap + data_off;
/* mark new part as accessible and undefined */
VALGRIND_DO_MAKE_MEM_UNDEFINED(pop, (char *)block_data + alloc->size,
real_size - alloc->size);
/* resize allocated space */
VALGRIND_DO_MEMPOOL_CHANGE(pop, userdatap, userdatap,
real_size - sizeof (struct allocation_header) - data_off);
if (constructor != NULL)
constructor(pop, userdatap,
real_size - sizeof (struct allocation_header) -
data_off, arg);
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, &alloc->size), real_size);
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
out:
if (heap_unlock_if_run(pop, cnt) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
out_lane:
if (lane_release(pop) != 0) {
ERR("Failed to release the lane");
ASSERT(0);
}
return err;
}
/*
* heap_chunk_init -- (internal) writes chunk header
*/
static void
heap_chunk_init(struct palloc_heap *heap, struct chunk_header *hdr,
uint16_t type, uint32_t size_idx)
{
struct chunk_header nhdr = {
.type = type,
.flags = 0,
.size_idx = size_idx
};
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr));
*hdr = nhdr; /* write the entire header (8 bytes) at once */
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
heap_chunk_write_footer(hdr, size_idx);
}
/*
* heap_zone_init -- (internal) writes zone's first chunk and header
*/
static void
heap_zone_init(struct palloc_heap *heap, uint32_t zone_id)
{
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
uint32_t size_idx = get_zone_size_idx(zone_id, heap->rt->max_zone,
heap->size);
heap_chunk_init(heap, &z->chunk_headers[0], CHUNK_TYPE_FREE, size_idx);
struct zone_header nhdr = {
.size_idx = size_idx,
.magic = ZONE_HEADER_MAGIC,
};
z->header = nhdr; /* write the entire header (8 bytes) at once */
pmemops_persist(&heap->p_ops, &z->header, sizeof(z->header));
}
/*
* heap_run_init -- (internal) creates a run based on a chunk
*/
static void
heap_run_init(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
ASSERTne(m->size_idx, 0);
size_t runsize = SIZEOF_RUN(run, m->size_idx);
VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize);
/* add/remove chunk_run and chunk_header to valgrind transaction */
VALGRIND_ADD_TO_TX(run, runsize);
run->block_size = c->unit_size;
pmemops_persist(&heap->p_ops, &run->block_size,
sizeof(run->block_size));
/* set all the bits */
memset(run->bitmap, 0xFF, sizeof(run->bitmap));
unsigned nval = c->run.bitmap_nval;
ASSERT(nval > 0);
/* clear only the bits available for allocations from this bucket */
memset(run->bitmap, 0, sizeof(uint64_t) * (nval - 1));
run->bitmap[nval - 1] = c->run.bitmap_lastval;
run->incarnation_claim = heap->run_id;
VALGRIND_SET_CLEAN(&run->incarnation_claim,
sizeof(run->incarnation_claim));
VALGRIND_REMOVE_FROM_TX(run, runsize);
pmemops_persist(&heap->p_ops, run->bitmap, sizeof(run->bitmap));
struct chunk_header run_data_hdr;
run_data_hdr.type = CHUNK_TYPE_RUN_DATA;
run_data_hdr.flags = 0;
struct chunk_header *data_hdr;
for (unsigned i = 1; i < m->size_idx; ++i) {
data_hdr = &z->chunk_headers[m->chunk_id + i];
VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr));
VALGRIND_ADD_TO_TX(data_hdr, sizeof(*data_hdr));
run_data_hdr.size_idx = i;
*data_hdr = run_data_hdr;
VALGRIND_REMOVE_FROM_TX(data_hdr, sizeof(*data_hdr));
}
pmemops_persist(&heap->p_ops,
&z->chunk_headers[m->chunk_id + 1],
sizeof(struct chunk_header) * (m->size_idx - 1));
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
ASSERT(hdr->type == CHUNK_TYPE_FREE);
VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr));
struct chunk_header run_hdr;
run_hdr.size_idx = hdr->size_idx;
run_hdr.type = CHUNK_TYPE_RUN;
run_hdr.flags = header_type_to_flag[c->header_type];
*hdr = run_hdr;
VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr));
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
}
/*
* heap_run_insert -- (internal) inserts and splits a block of memory into a run
*/
static void
heap_run_insert(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m, uint32_t size_idx, uint16_t block_off)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
ASSERT(size_idx <= BITS_PER_VALUE);
ASSERT(block_off + size_idx <= c->run.bitmap_nallocs);
uint32_t unit_max = c->run.unit_max;
struct memory_block nm = *m;
nm.size_idx = unit_max - (block_off % unit_max);
nm.block_off = block_off;
if (nm.size_idx > size_idx)
nm.size_idx = size_idx;
do {
bucket_insert_block(b, &nm);
ASSERT(nm.size_idx <= UINT16_MAX);
ASSERT(nm.block_off + nm.size_idx <= UINT16_MAX);
nm.block_off = (uint16_t)(nm.block_off + (uint16_t)nm.size_idx);
size_idx -= nm.size_idx;
nm.size_idx = size_idx > unit_max ? unit_max : size_idx;
} while (size_idx != 0);
}
/*
* heap_process_run_metadata -- (internal) parses the run bitmap
*/
static uint32_t
heap_process_run_metadata(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
uint16_t block_off = 0;
uint16_t block_size_idx = 0;
uint32_t inserted_blocks = 0;
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
for (unsigned i = 0; i < c->run.bitmap_nval; ++i) {
ASSERT(i < MAX_BITMAP_VALUES);
uint64_t v = run->bitmap[i];
ASSERT(BITS_PER_VALUE * i <= UINT16_MAX);
block_off = (uint16_t)(BITS_PER_VALUE * i);
if (v == 0) {
heap_run_insert(heap, b, m, BITS_PER_VALUE, block_off);
inserted_blocks += BITS_PER_VALUE;
continue;
} else if (v == UINT64_MAX) {
continue;
}
for (unsigned j = 0; j < BITS_PER_VALUE; ++j) {
if (BIT_IS_CLR(v, j)) {
block_size_idx++;
} else if (block_size_idx != 0) {
ASSERT(block_off >= block_size_idx);
heap_run_insert(heap, b, m,
block_size_idx,
(uint16_t)(block_off - block_size_idx));
inserted_blocks += block_size_idx;
block_size_idx = 0;
}
if ((block_off++) == c->run.bitmap_nallocs) {
i = MAX_BITMAP_VALUES;
break;
}
}
if (block_size_idx != 0) {
ASSERT(block_off >= block_size_idx);
heap_run_insert(heap, b, m,
block_size_idx,
(uint16_t)(block_off - block_size_idx));
inserted_blocks += block_size_idx;
block_size_idx = 0;
}
}
return inserted_blocks;
}
/*
* heap_create_run -- (internal) initializes a new run on an existing free chunk
*/
static void
heap_create_run(struct palloc_heap *heap, struct bucket *b,
struct memory_block *m)
{
heap_run_init(heap, b, m);
memblock_rebuild_state(heap, m);
heap_process_run_metadata(heap, b, m);
}
/*
* heap_vg_open -- notifies Valgrind about heap layout
*/
void
heap_vg_open(struct palloc_heap *heap, object_callback cb,
void *arg, int objects)
{
ASSERTne(cb, NULL);
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap->layout, heap->size);
struct heap_layout *layout = heap->layout;
VALGRIND_DO_MAKE_MEM_DEFINED(&layout->header, sizeof(layout->header));
unsigned zones = heap_max_zone(heap->size);
struct memory_block m = MEMORY_BLOCK_NONE;
for (unsigned i = 0; i < zones; ++i) {
struct zone *z = ZID_TO_ZONE(layout, i);
uint32_t chunks;
m.zone_id = i;
m.chunk_id = 0;
VALGRIND_DO_MAKE_MEM_DEFINED(&z->header, sizeof(z->header));
if (z->header.magic != ZONE_HEADER_MAGIC)
continue;
chunks = z->header.size_idx;
for (uint32_t c = 0; c < chunks; ) {
struct chunk_header *hdr = &z->chunk_headers[c];
m.chunk_id = c;
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
m.size_idx = hdr->size_idx;
heap_vg_open_chunk(heap, cb, arg, objects, &m);
m.block_off = 0;
ASSERT(hdr->size_idx > 0);
if (hdr->type == CHUNK_TYPE_RUN) {
/*
* Mark run data headers as defined.
*/
for (unsigned j = 1; j < hdr->size_idx; ++j) {
struct chunk_header *data_hdr =
&z->chunk_headers[c + j];
VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr,
sizeof(struct chunk_header));
ASSERTeq(data_hdr->type,
CHUNK_TYPE_RUN_DATA);
}
} else {
/*
* Mark unused chunk headers as not accessible.
*/
VALGRIND_DO_MAKE_MEM_NOACCESS(
&z->chunk_headers[c + 1],
(hdr->size_idx - 1) *
sizeof(struct chunk_header));
}
c += hdr->size_idx;
}
/* mark all unused chunk headers after last as not accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(&z->chunk_headers[chunks],
(MAX_CHUNK - chunks) * sizeof(struct chunk_header));
}
}
/*
* memblock_huge_init -- initializes a new huge memory block
*/
struct memory_block
memblock_huge_init(struct palloc_heap *heap,
uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx)
{
struct memory_block m = MEMORY_BLOCK_NONE;
m.chunk_id = chunk_id;
m.zone_id = zone_id;
m.size_idx = size_idx;
m.heap = heap;
struct chunk_header nhdr = {
.type = CHUNK_TYPE_FREE,
.flags = 0,
.size_idx = size_idx
};
struct chunk_header *hdr = heap_get_chunk_hdr(heap, &m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr));
VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr));
*hdr = nhdr; /* write the entire header (8 bytes) at once */
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
huge_write_footer(hdr, size_idx);
memblock_rebuild_state(heap, &m);
return m;
}
/*
* memblock_run_init -- initializes a new run memory block
*/
struct memory_block
memblock_run_init(struct palloc_heap *heap,
uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx, uint16_t flags,
uint64_t unit_size, uint64_t alignment)
{
ASSERTne(size_idx, 0);
struct memory_block m = MEMORY_BLOCK_NONE;
m.chunk_id = chunk_id;
m.zone_id = zone_id;
m.size_idx = size_idx;
m.heap = heap;
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
struct chunk_run *run = heap_get_chunk_run(heap, &m);
size_t runsize = SIZEOF_RUN(run, size_idx);
VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize);
/* add/remove chunk_run and chunk_header to valgrind transaction */
VALGRIND_ADD_TO_TX(run, runsize);
run->hdr.block_size = unit_size;
run->hdr.alignment = alignment;
struct run_bitmap b;
memblock_run_bitmap(&size_idx, flags, unit_size, alignment,
run->content, &b);
size_t bitmap_size = b.size;
/* set all the bits */
memset(b.values, 0xFF, bitmap_size);
/* clear only the bits available for allocations from this bucket */
memset(b.values, 0, sizeof(*b.values) * (b.nvalues - 1));
unsigned trailing_bits = b.nbits % RUN_BITS_PER_VALUE;
uint64_t last_value = UINT64_MAX << trailing_bits;
b.values[b.nvalues - 1] = last_value;
VALGRIND_REMOVE_FROM_TX(run, runsize);
pmemops_flush(&heap->p_ops, run,
sizeof(struct chunk_run_header) +
bitmap_size);
struct chunk_header run_data_hdr;
run_data_hdr.type = CHUNK_TYPE_RUN_DATA;
run_data_hdr.flags = 0;
VALGRIND_ADD_TO_TX(&z->chunk_headers[chunk_id],
sizeof(struct chunk_header) * size_idx);
struct chunk_header *data_hdr;
for (unsigned i = 1; i < size_idx; ++i) {
data_hdr = &z->chunk_headers[chunk_id + i];
VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr));
VALGRIND_ANNOTATE_NEW_MEMORY(data_hdr, sizeof(*data_hdr));
run_data_hdr.size_idx = i;
*data_hdr = run_data_hdr;
}
pmemops_persist(&heap->p_ops,
&z->chunk_headers[chunk_id + 1],
sizeof(struct chunk_header) * (size_idx - 1));
struct chunk_header *hdr = &z->chunk_headers[chunk_id];
ASSERT(hdr->type == CHUNK_TYPE_FREE);
VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr));
struct chunk_header run_hdr;
run_hdr.size_idx = hdr->size_idx;
run_hdr.type = CHUNK_TYPE_RUN;
run_hdr.flags = flags;
*hdr = run_hdr;
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
VALGRIND_REMOVE_FROM_TX(&z->chunk_headers[chunk_id],
sizeof(struct chunk_header) * size_idx);
memblock_rebuild_state(heap, &m);
return m;
}
