/*
* heap_init -- initializes the heap
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_init(void *heap_start, uint64_t heap_size, struct pmem_ops *p_ops)
{
if (heap_size < HEAP_MIN_SIZE)
return EINVAL;
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap_start, heap_size);
struct heap_layout *layout = heap_start;
heap_write_header(&layout->header, heap_size);
pmemops_persist(p_ops, &layout->header, sizeof(struct heap_header));
unsigned zones = heap_max_zone(heap_size);
for (unsigned i = 0; i < zones; ++i) {
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->header,
0, sizeof(struct zone_header));
pmemops_memset_persist(p_ops,
&ZID_TO_ZONE(layout, i)->chunk_headers,
0, sizeof(struct chunk_header));
/* only explicitly allocated chunks should be accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(
&ZID_TO_ZONE(layout, i)->chunk_headers,
sizeof(struct chunk_header));
}
return 0;
}
/*
* heap_check -- verifies if the heap is consistent and can be opened properly
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_check(void *heap_start, uint64_t heap_size)
{
if (heap_size < HEAP_MIN_SIZE) {
ERR("heap: invalid heap size");
return -1;
}
struct heap_layout *layout = heap_start;
if (heap_size != layout->header.size) {
ERR("heap: heap size missmatch");
return -1;
}
if (heap_verify_header(&layout->header))
return -1;
for (unsigned i = 0; i < heap_max_zone(layout->header.size); ++i) {
if (heap_verify_zone(ZID_TO_ZONE(layout, i)))
return -1;
}
return 0;
}
/*
* heap_chunk_foreach_object -- (internal) iterates through objects in a chunk
*/
static int
heap_chunk_foreach_object(struct palloc_heap *heap, object_callback cb,
void *arg, struct memory_block *m)
{
struct zone *zone = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_header *hdr = &zone->chunk_headers[m->chunk_id];
memblock_rebuild_state(heap, m);
switch (hdr->type) {
case CHUNK_TYPE_FREE:
return 0;
case CHUNK_TYPE_USED:
m->size_idx = hdr->size_idx;
return cb(m, arg);
case CHUNK_TYPE_RUN:
return heap_run_foreach_object(heap, cb, arg, m,
alloc_class_get_create_by_unit_size(
heap->rt->alloc_classes,
m->m_ops->block_size(m)));
default:
ASSERT(0);
}
return 0;
}
/*
* heap_get_adjacent_free_block -- locates adjacent free memory block in heap
*/
static int
heap_get_adjacent_free_block(struct palloc_heap *heap,
const struct memory_block *in, struct memory_block *out, int prev)
{
struct zone *z = ZID_TO_ZONE(heap->layout, in->zone_id);
struct chunk_header *hdr = &z->chunk_headers[in->chunk_id];
out->zone_id = in->zone_id;
if (prev) {
if (in->chunk_id == 0)
return ENOENT;
struct chunk_header *prev_hdr =
&z->chunk_headers[in->chunk_id - 1];
out->chunk_id = in->chunk_id - prev_hdr->size_idx;
if (z->chunk_headers[out->chunk_id].type != CHUNK_TYPE_FREE)
return ENOENT;
out->size_idx = z->chunk_headers[out->chunk_id].size_idx;
} else { /* next */
if (in->chunk_id + hdr->size_idx == z->header.size_idx)
return ENOENT;
out->chunk_id = in->chunk_id + hdr->size_idx;
if (z->chunk_headers[out->chunk_id].type != CHUNK_TYPE_FREE)
return ENOENT;
out->size_idx = z->chunk_headers[out->chunk_id].size_idx;
}
memblock_rebuild_state(heap, out);
return 0;
}
/*
* huge_vg_init -- initalizes chunk metadata in memcheck state
*/
static void
huge_vg_init(const struct memory_block *m, int objects,
object_callback cb, void *arg)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m);
struct chunk *chunk = heap_get_chunk(m->heap, m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
/*
* Mark unused chunk headers as not accessible.
*/
VALGRIND_DO_MAKE_MEM_NOACCESS(
&z->chunk_headers[m->chunk_id + 1],
(m->size_idx - 1) *
sizeof(struct chunk_header));
size_t size = block_get_real_size(m);
VALGRIND_DO_MAKE_MEM_NOACCESS(chunk, size);
if (objects && huge_get_state(m) == MEMBLOCK_ALLOCATED) {
if (cb(m, arg) != 0)
FATAL("failed to initialize valgrind state");
}
}
/*
* run_block_size -- looks for the right chunk and returns the block size
* information that is attached to the run block metadata.
*/
static size_t
run_block_size(const struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
return run->block_size;
}
/*
* huge_get_real_data -- returns pointer to the beginning data of a huge block
*/
static void *
huge_get_real_data(const struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
void *data = &z->chunks[m->chunk_id].data;
return (char *)data;
}
/*
* run_block_size -- looks for the right chunk and returns the block size
* information that is attached to the run block metadata.
*/
static size_t
run_block_size(struct memory_block *m, struct heap_layout *h)
{
struct zone *z = ZID_TO_ZONE(h, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
return run->block_size;
}
/*
* heap_reclaim_zone_garbage -- (internal) creates volatile state of unused runs
*/
static int
heap_reclaim_zone_garbage(struct palloc_heap *heap, uint32_t zone_id, int init)
{
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
struct chunk_run *run = NULL;
int rchunks = 0;
/*
* If this is the first time this zone is processed, recreate all
* footers BEFORE any other operation takes place. For example, the
* heap_init_free_chunk call expects the footers to be created.
*/
if (init) {
for (uint32_t i = 0; i < z->header.size_idx; ) {
struct chunk_header *hdr = &z->chunk_headers[i];
switch (hdr->type) {
case CHUNK_TYPE_USED:
heap_chunk_write_footer(hdr,
hdr->size_idx);
break;
}
i += hdr->size_idx;
}
}
for (uint32_t i = 0; i < z->header.size_idx; ) {
struct chunk_header *hdr = &z->chunk_headers[i];
ASSERT(hdr->size_idx != 0);
struct memory_block m = MEMORY_BLOCK_NONE;
m.zone_id = zone_id;
m.chunk_id = i;
m.size_idx = hdr->size_idx;
memblock_rebuild_state(heap, &m);
switch (hdr->type) {
case CHUNK_TYPE_RUN:
run = (struct chunk_run *)&z->chunks[i];
rchunks += heap_reclaim_run(heap, run, &m);
break;
case CHUNK_TYPE_FREE:
if (init)
heap_init_free_chunk(heap, hdr, &m);
break;
case CHUNK_TYPE_USED:
break;
default:
ASSERT(0);
}
i = m.chunk_id + m.size_idx; /* hdr might have changed */
}
return rchunks == 0 ? ENOMEM : 0;
}
/*
* heap_end -- returns first address after heap
*/
void *
heap_end(struct palloc_heap *h)
{
ASSERT(h->rt->max_zone > 0);
struct zone *last_zone = ZID_TO_ZONE(h->layout, h->rt->max_zone - 1);
return &last_zone->chunks[last_zone->header.size_idx];
}
/*
* run_get_real_data -- returns pointer to the beginning data of a run block
*/
static void *
run_get_real_data(const struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_run *run =
(struct chunk_run *)&z->chunks[m->chunk_id].data;
ASSERT(run->block_size != 0);
return (char *)&run->data + (run->block_size * m->block_off);
}
/*
* run_ensure_header_type -- runs must be created with appropriate header type.
*/
static void
run_ensure_header_type(const struct memory_block *m,
enum header_type t)
{
#ifdef DEBUG
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
ASSERTeq(hdr->type, CHUNK_TYPE_RUN);
ASSERT((hdr->flags & header_type_to_flag[t]) == header_type_to_flag[t]);
#endif
}
/*
* memblock_header_type -- determines the memory block's header type
*/
static enum header_type
memblock_header_type(const struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
if (hdr->flags & CHUNK_FLAG_COMPACT_HEADER)
return HEADER_COMPACT;
if (hdr->flags & CHUNK_FLAG_HEADER_NONE)
return HEADER_NONE;
return HEADER_LEGACY;
}
/*
* huge_get_state -- returns whether a huge block is allocated or not
*/
static enum memblock_state
huge_get_state(struct memory_block *m, struct palloc_heap *heap)
{
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
if (hdr->type == CHUNK_TYPE_USED)
return MEMBLOCK_ALLOCATED;
if (hdr->type == CHUNK_TYPE_FREE)
return MEMBLOCK_FREE;
return MEMBLOCK_STATE_UNKNOWN;
}
/*
* heap_run_foreach_object -- (internal) iterates through objects in a run
*/
int
heap_run_foreach_object(struct palloc_heap *heap, object_callback cb,
void *arg, struct memory_block *m, struct alloc_class *c)
{
if (c == NULL)
return -1;
uint16_t i = m->block_off / BITS_PER_VALUE;
uint16_t block_start = m->block_off % BITS_PER_VALUE;
uint16_t block_off;
struct chunk_run *run = (struct chunk_run *)
&ZID_TO_ZONE(heap->layout, m->zone_id)->chunks[m->chunk_id];
for (; i < c->run.bitmap_nval; ++i) {
uint64_t v = run->bitmap[i];
block_off = (uint16_t)(BITS_PER_VALUE * i);
for (uint16_t j = block_start; j < BITS_PER_VALUE; ) {
if (block_off + j >= (uint16_t)c->run.bitmap_nallocs)
break;
if (!BIT_IS_CLR(v, j)) {
m->block_off = (uint16_t)(block_off + j);
/*
* The size index of this memory block cannot be
* retrieved at this time because the header
* might not be initialized in valgrind yet.
*/
m->size_idx = 0;
if (cb(m, arg)
!= 0)
return 1;
m->size_idx = CALC_SIZE_IDX(c->unit_size,
m->m_ops->get_real_size(m));
j = (uint16_t)(j + m->size_idx);
} else {
++j;
}
}
block_start = 0;
}
return 0;
}
/*
* heap_check_remote -- verifies if the heap of a remote pool is consistent
* and can be opened properly
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
heap_check_remote(void *heap_start, uint64_t heap_size, struct remote_ops *ops)
{
if (heap_size < HEAP_MIN_SIZE) {
ERR("heap: invalid heap size");
return -1;
}
struct heap_layout *layout = heap_start;
struct heap_header header;
if (ops->read(ops->ctx, ops->base, &header, &layout->header,
sizeof(struct heap_header))) {
ERR("heap: obj_read_remote error");
return -1;
}
if (heap_size != header.size) {
ERR("heap: heap size mismatch");
return -1;
}
if (heap_verify_header(&header))
return -1;
struct zone *zone_buff = (struct zone *)Malloc(sizeof(struct zone));
if (zone_buff == NULL) {
ERR("heap: zone_buff malloc error");
return -1;
}
for (unsigned i = 0; i < heap_max_zone(header.size); ++i) {
if (ops->read(ops->ctx, ops->base, zone_buff,
ZID_TO_ZONE(layout, i), sizeof(struct zone))) {
ERR("heap: obj_read_remote error");
goto out;
}
if (heap_verify_zone(zone_buff)) {
goto out;
}
}
Free(zone_buff);
return 0;
out:
Free(zone_buff);
return -1;
}
/*
* huge_ensure_header_type -- checks the header type of a chunk and modifies
* it if necessery. This is fail-safe atomic.
*/
static void
huge_ensure_header_type(const struct memory_block *m,
enum header_type t)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
ASSERTeq(hdr->type, CHUNK_TYPE_FREE);
if ((hdr->flags & header_type_to_flag[t]) == 0) {
VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr));
uint16_t f = ((uint16_t)header_type_to_flag[t]);
hdr->flags |= f;
pmemops_persist(&m->heap->p_ops, hdr, sizeof(*hdr));
VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr));
}
}
/*
* memblock_from_offset -- resolves a memory block data from an offset that
* originates from the heap
*/
struct memory_block
memblock_from_offset_opt(struct palloc_heap *heap, uint64_t off, int size)
{
struct memory_block m = MEMORY_BLOCK_NONE;
m.heap = heap;
off -= HEAP_PTR_TO_OFF(heap, &heap->layout->zone0);
m.zone_id = (uint32_t)(off / ZONE_MAX_SIZE);
off -= (ZONE_MAX_SIZE * m.zone_id) + sizeof(struct zone);
m.chunk_id = (uint32_t)(off / CHUNKSIZE);
struct chunk_header *hdr = &ZID_TO_ZONE(heap->layout, m.zone_id)
->chunk_headers[m.chunk_id];
if (hdr->type == CHUNK_TYPE_RUN_DATA)
m.chunk_id -= hdr->size_idx;
off -= CHUNKSIZE * m.chunk_id;
m.header_type = memblock_header_type(&m);
off -= header_type_to_size[m.header_type];
m.type = off != 0 ? MEMORY_BLOCK_RUN : MEMORY_BLOCK_HUGE;
#ifdef DEBUG
enum memory_block_type t = memblock_detect_type(&m, heap->layout);
ASSERTeq(t, m.type);
#endif
m.m_ops = &mb_ops[m.type];
uint64_t unit_size = m.m_ops->block_size(&m);
if (off != 0) { /* run */
off -= RUN_METASIZE;
m.block_off = (uint16_t)(off / unit_size);
off -= m.block_off * unit_size;
}
m.size_idx = !size ? 0 : CALC_SIZE_IDX(unit_size,
memblock_header_ops[m.header_type].get_size(&m));
ASSERTeq(off, 0);
return m;
}
/*
* huge_prep_operation_hdr -- prepares the new value of a chunk header that will
* be set after the operation concludes.
*/
static void
huge_prep_operation_hdr(const struct memory_block *m, enum memblock_state op,
struct operation_context *ctx)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
/*
* Depending on the operation that needs to be performed a new chunk
* header needs to be prepared with the new chunk state.
*/
uint64_t val = chunk_get_chunk_hdr_value(
op == MEMBLOCK_ALLOCATED ? CHUNK_TYPE_USED : CHUNK_TYPE_FREE,
hdr->flags,
m->size_idx);
operation_add_entry(ctx, hdr, val, OPERATION_SET);
VALGRIND_DO_MAKE_MEM_NOACCESS(hdr + 1,
(hdr->size_idx - 1) * sizeof(struct chunk_header));
/*
* In the case of chunks larger than one unit the footer must be
* created immediately AFTER the persistent state is safely updated.
*/
if (m->size_idx == 1)
return;
struct chunk_header *footer = hdr + m->size_idx - 1;
VALGRIND_DO_MAKE_MEM_UNDEFINED(footer, sizeof(*footer));
val = chunk_get_chunk_hdr_value(CHUNK_TYPE_FOOTER, 0, m->size_idx);
/*
* It's only safe to write the footer AFTER the persistent part of
* the operation have been successfully processed because the footer
* pointer might point to a currently valid persistent state
* of a different chunk.
* The footer entry change is updated as transient because it will
* be recreated at heap boot regardless - it's just needed for runtime
* operations.
*/
operation_add_typed_entry(ctx,
footer, val, OPERATION_SET, ENTRY_TRANSIENT);
}
/*
* run_prep_operation_hdr -- prepares the new value for a select few bytes of
* a run bitmap that will be set after the operation concludes.
*
* It's VERY important to keep in mind that the particular value of the
* bitmap this method is modifying must not be changed after this function
* is called and before the operation is processed.
*/
static void
run_prep_operation_hdr(const struct memory_block *m, enum memblock_state op,
struct operation_context *ctx)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_run *r = (struct chunk_run *)&z->chunks[m->chunk_id];
ASSERT(m->size_idx <= BITS_PER_VALUE);
/*
* Free blocks are represented by clear bits and used blocks by set
* bits - which is the reverse of the commonly used scheme.
*
* Here a bit mask is prepared that flips the bits that represent the
* memory block provided by the caller - because both the size index and
* the block offset are tied 1:1 to the bitmap this operation is
* relatively simple.
*/
uint64_t bmask;
if (m->size_idx == BITS_PER_VALUE) {
ASSERTeq(m->block_off % BITS_PER_VALUE, 0);
bmask = UINT64_MAX;
} else {
bmask = ((1ULL << m->size_idx) - 1ULL) <<
(m->block_off % BITS_PER_VALUE);
}
/*
* The run bitmap is composed of several 8 byte values, so a proper
* element of the bitmap array must be selected.
*/
int bpos = m->block_off / BITS_PER_VALUE;
/* the bit mask is applied immediately by the add entry operations */
if (op == MEMBLOCK_ALLOCATED) {
operation_add_entry(ctx, &r->bitmap[bpos],
bmask, OPERATION_OR);
} else if (op == MEMBLOCK_FREE) {
operation_add_entry(ctx, &r->bitmap[bpos],
~bmask, OPERATION_AND);
} else {
ASSERT(0);
}
}
static int
pfree(uint64_t *off)
{
uint64_t offset = *off;
if (offset == 0)
return 0;
PMEMoid oid;
oid.off = offset;
struct allocation_header *hdr = &D_RW_OBJ(oid)->alloch;
struct zone *z = ZID_TO_ZONE(heap, hdr->zone_id);
struct chunk_header *chdr = &z->chunk_headers[hdr->chunk_id];
if (chdr->type == CHUNK_TYPE_USED) {
chdr->type = CHUNK_TYPE_FREE;
pmempool_convert_persist(poolset, &chdr->type,
sizeof(chdr->type));
*off = 0;
pmempool_convert_persist(poolset, off, sizeof(*off));
return 0;
} else if (chdr->type != CHUNK_TYPE_RUN) {
assert(0);
}
struct chunk_run *run =
(struct chunk_run *)&z->chunks[hdr->chunk_id].data;
uintptr_t diff = (uintptr_t)hdr - (uintptr_t)&run->data;
uint64_t block_off = (uint16_t)((size_t)diff / run->block_size);
uint64_t size_idx = CALC_SIZE_IDX(run->block_size, hdr->size);
uint64_t bmask = ((1ULL << size_idx) - 1ULL) <<
(block_off % BITS_PER_VALUE);
uint64_t bpos = block_off / BITS_PER_VALUE;
run->bitmap[bpos] &= ~bmask;
pmempool_convert_persist(poolset, &run->bitmap[bpos],
sizeof(run->bitmap[bpos]));
*off = 0;
pmempool_convert_persist(poolset, off, sizeof(*off));
return 0;
}
/*
* heap_resize_chunk -- (internal) splits the chunk into two smaller ones
*/
static void
heap_resize_chunk(struct palloc_heap *heap,
uint32_t chunk_id, uint32_t zone_id, uint32_t new_size_idx)
{
uint32_t new_chunk_id = chunk_id + new_size_idx;
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
struct chunk_header *old_hdr = &z->chunk_headers[chunk_id];
struct chunk_header *new_hdr = &z->chunk_headers[new_chunk_id];
uint32_t rem_size_idx = old_hdr->size_idx - new_size_idx;
heap_chunk_init(heap, new_hdr, CHUNK_TYPE_FREE, rem_size_idx);
heap_chunk_init(heap, old_hdr, CHUNK_TYPE_FREE, new_size_idx);
struct bucket *def_bucket = heap->rt->default_bucket;
struct memory_block m = {new_chunk_id, zone_id, rem_size_idx, 0,
0, 0, NULL, NULL};
memblock_rebuild_state(heap, &m);
bucket_insert_block(def_bucket, &m);
}
/*
* heap_populate_buckets -- (internal) creates volatile state of memory blocks
*/
static int
heap_populate_buckets(struct palloc_heap *heap)
{
struct heap_rt *h = heap->rt;
if (h->zones_exhausted == h->max_zone)
return ENOMEM;
uint32_t zone_id = h->zones_exhausted++;
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
/* ignore zone and chunk headers */
VALGRIND_ADD_TO_GLOBAL_TX_IGNORE(z, sizeof(z->header) +
sizeof(z->chunk_headers));
if (z->header.magic != ZONE_HEADER_MAGIC)
heap_zone_init(heap, zone_id);
return heap_reclaim_zone_garbage(heap, zone_id, 1 /* init */);
}
/*
* memblock_autodetect_type -- looks for the corresponding chunk header and
* depending on the chunks type returns the right memory block type.
*/
enum memory_block_type
memblock_autodetect_type(struct memory_block *m, struct heap_layout *h)
{
enum memory_block_type ret;
switch (ZID_TO_ZONE(h, m->zone_id)->chunk_headers[m->chunk_id].type) {
case CHUNK_TYPE_RUN:
ret = MEMORY_BLOCK_RUN;
break;
case CHUNK_TYPE_FREE:
case CHUNK_TYPE_USED:
case CHUNK_TYPE_FOOTER:
ret = MEMORY_BLOCK_HUGE;
break;
default:
/* unreachable */
FATAL("possible zone chunks metadata corruption");
}
return ret;
}
/*
* heap_zone_foreach_object -- (internal) iterates through objects in a zone
*/
static int
heap_zone_foreach_object(struct palloc_heap *heap, object_callback cb,
void *arg, struct memory_block *m)
{
struct zone *zone = ZID_TO_ZONE(heap->layout, m->zone_id);
if (zone->header.magic == 0)
return 0;
for (; m->chunk_id < zone->header.size_idx; ) {
if (heap_chunk_foreach_object(heap, cb, arg, m) != 0)
return 1;
m->chunk_id += zone->chunk_headers[m->chunk_id].size_idx;
/* reset the starting position of memblock */
m->block_off = 0;
m->size_idx = 0;
}
return 0;
}
/*
* heap_vg_open_chunk -- (internal) notifies Valgrind about chunk layout
*/
static void
heap_vg_open_chunk(struct palloc_heap *heap,
object_callback cb, void *arg, int objects,
struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
void *chunk = &z->chunks[m->chunk_id];
memblock_rebuild_state(heap, m);
if (m->type == MEMORY_BLOCK_RUN) {
struct chunk_run *run = chunk;
ASSERTne(m->size_idx, 0);
VALGRIND_DO_MAKE_MEM_NOACCESS(run,
SIZEOF_RUN(run, m->size_idx));
/* set the run metadata as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run,
sizeof(*run) - sizeof(run->data));
if (objects) {
int ret = heap_run_foreach_object(heap, cb, arg, m,
alloc_class_get_create_by_unit_size(
heap->rt->alloc_classes,
m->m_ops->block_size(m)));
ASSERTeq(ret, 0);
}
} else {
size_t size = m->m_ops->get_real_size(m);
VALGRIND_DO_MAKE_MEM_NOACCESS(chunk, size);
if (objects && m->m_ops->get_state(m) == MEMBLOCK_ALLOCATED) {
int ret = cb(m, arg);
ASSERTeq(ret, 0);
}
}
}
/*
* run_vg_init -- initalizes run metadata in memcheck state
*/
static void
run_vg_init(const struct memory_block *m, int objects,
object_callback cb, void *arg)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m);
struct chunk_run *run = heap_get_chunk_run(m->heap, m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
/* set the run metadata as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run, RUN_BASE_METADATA_SIZE);
struct run_bitmap b;
run_get_bitmap(m, &b);
/*
* Mark run data headers as defined.
*/
for (unsigned j = 1; j < m->size_idx; ++j) {
struct chunk_header *data_hdr =
&z->chunk_headers[m->chunk_id + j];
VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr,
sizeof(struct chunk_header));
ASSERTeq(data_hdr->type, CHUNK_TYPE_RUN_DATA);
}
VALGRIND_DO_MAKE_MEM_NOACCESS(run, SIZEOF_RUN(run, m->size_idx));
/* set the run bitmap as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run, b.size + RUN_BASE_METADATA_SIZE);
if (objects) {
if (run_iterate_used(m, cb, arg) != 0)
FATAL("failed to initialize valgrind state");
}
}
/*
* huge_get_state -- returns whether a block from a run is allocated or not
*/
static enum memblock_state
run_get_state(struct memory_block *m, struct palloc_heap *heap)
{
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
ASSERTeq(hdr->type, CHUNK_TYPE_RUN);
struct chunk_run *r = (struct chunk_run *)&z->chunks[m->chunk_id];
unsigned v = m->block_off / BITS_PER_VALUE;
uint64_t bitmap = r->bitmap[v];
unsigned b = m->block_off % BITS_PER_VALUE;
unsigned b_last = b + m->size_idx;
ASSERT(b_last <= BITS_PER_VALUE);
for (unsigned i = b; i < b_last; ++i) {
if (!BIT_IS_CLR(bitmap, i)) {
return MEMBLOCK_ALLOCATED;
}
}
return MEMBLOCK_FREE;
}
/*
* heap_reclaim_run -- checks the run for available memory if unclaimed.
*
* Returns 1 if reclaimed chunk, 0 otherwise.
*/
static int
heap_reclaim_run(struct palloc_heap *heap, struct chunk_run *run,
struct memory_block *m)
{
if (m->m_ops->claim(m) != 0)
return 0; /* this run already has an owner */
struct alloc_class *c = alloc_class_get_create_by_unit_size(
heap->rt->alloc_classes, run->block_size);
if (c == NULL)
return 0;
ASSERTeq(c->type, CLASS_RUN);
pthread_mutex_t *lock = m->m_ops->get_lock(m);
util_mutex_lock(lock);
unsigned i;
unsigned nval = c->run.bitmap_nval;
for (i = 0; nval > 0 && i < nval - 1; ++i)
if (run->bitmap[i] != 0)
break;
int empty = (i == (nval - 1)) &&
(run->bitmap[i] == c->run.bitmap_lastval);
if (empty) {
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
struct bucket *defb = heap_get_default_bucket(heap);
/*
* The redo log ptr can be NULL if we are sure that there's only
* one persistent value modification in the entire operation
* context.
*/
struct operation_context ctx;
operation_init(&ctx, heap->base, NULL, NULL);
ctx.p_ops = &heap->p_ops;
struct memory_block nb = MEMORY_BLOCK_NONE;
nb.chunk_id = m->chunk_id;
nb.zone_id = m->zone_id;
nb.block_off = 0;
nb.size_idx = m->size_idx;
heap_chunk_init(heap, hdr, CHUNK_TYPE_FREE, nb.size_idx);
memblock_rebuild_state(heap, &nb);
nb = heap_coalesce_huge(heap, &nb);
nb.m_ops->prep_hdr(&nb, MEMBLOCK_FREE, &ctx);
operation_process(&ctx);
bucket_insert_block(defb, &nb);
*m = nb;
} else {
recycler_put(heap->rt->recyclers[c->id], m);
}
util_mutex_unlock(lock);
return empty;
}
/*
* heap_chunk_init -- (internal) writes chunk header
*/
static void
heap_chunk_init(struct palloc_heap *heap, struct chunk_header *hdr,
uint16_t type, uint32_t size_idx)
{
struct chunk_header nhdr = {
.type = type,
.flags = 0,
.size_idx = size_idx
};
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr));
*hdr = nhdr; /* write the entire header (8 bytes) at once */
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
heap_chunk_write_footer(hdr, size_idx);
}
/*
* heap_zone_init -- (internal) writes zone's first chunk and header
*/
static void
heap_zone_init(struct palloc_heap *heap, uint32_t zone_id)
{
struct zone *z = ZID_TO_ZONE(heap->layout, zone_id);
uint32_t size_idx = get_zone_size_idx(zone_id, heap->rt->max_zone,
heap->size);
heap_chunk_init(heap, &z->chunk_headers[0], CHUNK_TYPE_FREE, size_idx);
struct zone_header nhdr = {
.size_idx = size_idx,
.magic = ZONE_HEADER_MAGIC,
};
z->header = nhdr; /* write the entire header (8 bytes) at once */
pmemops_persist(&heap->p_ops, &z->header, sizeof(z->header));
}
/*
* heap_run_init -- (internal) creates a run based on a chunk
*/
static void
heap_run_init(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
ASSERTne(m->size_idx, 0);
size_t runsize = SIZEOF_RUN(run, m->size_idx);
VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize);
/* add/remove chunk_run and chunk_header to valgrind transaction */
VALGRIND_ADD_TO_TX(run, runsize);
run->block_size = c->unit_size;
pmemops_persist(&heap->p_ops, &run->block_size,
sizeof(run->block_size));
/* set all the bits */
memset(run->bitmap, 0xFF, sizeof(run->bitmap));
unsigned nval = c->run.bitmap_nval;
ASSERT(nval > 0);
/* clear only the bits available for allocations from this bucket */
memset(run->bitmap, 0, sizeof(uint64_t) * (nval - 1));
run->bitmap[nval - 1] = c->run.bitmap_lastval;
run->incarnation_claim = heap->run_id;
VALGRIND_SET_CLEAN(&run->incarnation_claim,
sizeof(run->incarnation_claim));
VALGRIND_REMOVE_FROM_TX(run, runsize);
pmemops_persist(&heap->p_ops, run->bitmap, sizeof(run->bitmap));
struct chunk_header run_data_hdr;
run_data_hdr.type = CHUNK_TYPE_RUN_DATA;
run_data_hdr.flags = 0;
struct chunk_header *data_hdr;
for (unsigned i = 1; i < m->size_idx; ++i) {
data_hdr = &z->chunk_headers[m->chunk_id + i];
VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr));
VALGRIND_ADD_TO_TX(data_hdr, sizeof(*data_hdr));
run_data_hdr.size_idx = i;
*data_hdr = run_data_hdr;
VALGRIND_REMOVE_FROM_TX(data_hdr, sizeof(*data_hdr));
}
pmemops_persist(&heap->p_ops,
&z->chunk_headers[m->chunk_id + 1],
sizeof(struct chunk_header) * (m->size_idx - 1));
struct chunk_header *hdr = &z->chunk_headers[m->chunk_id];
ASSERT(hdr->type == CHUNK_TYPE_FREE);
VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr));
struct chunk_header run_hdr;
run_hdr.size_idx = hdr->size_idx;
run_hdr.type = CHUNK_TYPE_RUN;
run_hdr.flags = header_type_to_flag[c->header_type];
*hdr = run_hdr;
VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr));
pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr));
}
/*
* heap_run_insert -- (internal) inserts and splits a block of memory into a run
*/
static void
heap_run_insert(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m, uint32_t size_idx, uint16_t block_off)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
ASSERT(size_idx <= BITS_PER_VALUE);
ASSERT(block_off + size_idx <= c->run.bitmap_nallocs);
uint32_t unit_max = c->run.unit_max;
struct memory_block nm = *m;
nm.size_idx = unit_max - (block_off % unit_max);
nm.block_off = block_off;
if (nm.size_idx > size_idx)
nm.size_idx = size_idx;
do {
bucket_insert_block(b, &nm);
ASSERT(nm.size_idx <= UINT16_MAX);
ASSERT(nm.block_off + nm.size_idx <= UINT16_MAX);
nm.block_off = (uint16_t)(nm.block_off + (uint16_t)nm.size_idx);
size_idx -= nm.size_idx;
nm.size_idx = size_idx > unit_max ? unit_max : size_idx;
} while (size_idx != 0);
}
/*
* heap_process_run_metadata -- (internal) parses the run bitmap
*/
static uint32_t
heap_process_run_metadata(struct palloc_heap *heap, struct bucket *b,
const struct memory_block *m)
{
struct alloc_class *c = b->aclass;
ASSERTeq(c->type, CLASS_RUN);
uint16_t block_off = 0;
uint16_t block_size_idx = 0;
uint32_t inserted_blocks = 0;
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id];
for (unsigned i = 0; i < c->run.bitmap_nval; ++i) {
ASSERT(i < MAX_BITMAP_VALUES);
uint64_t v = run->bitmap[i];
ASSERT(BITS_PER_VALUE * i <= UINT16_MAX);
block_off = (uint16_t)(BITS_PER_VALUE * i);
if (v == 0) {
heap_run_insert(heap, b, m, BITS_PER_VALUE, block_off);
inserted_blocks += BITS_PER_VALUE;
continue;
} else if (v == UINT64_MAX) {
continue;
}
for (unsigned j = 0; j < BITS_PER_VALUE; ++j) {
if (BIT_IS_CLR(v, j)) {
block_size_idx++;
} else if (block_size_idx != 0) {
ASSERT(block_off >= block_size_idx);
heap_run_insert(heap, b, m,
block_size_idx,
(uint16_t)(block_off - block_size_idx));
inserted_blocks += block_size_idx;
block_size_idx = 0;
}
if ((block_off++) == c->run.bitmap_nallocs) {
i = MAX_BITMAP_VALUES;
break;
}
}
if (block_size_idx != 0) {
ASSERT(block_off >= block_size_idx);
heap_run_insert(heap, b, m,
block_size_idx,
(uint16_t)(block_off - block_size_idx));
inserted_blocks += block_size_idx;
block_size_idx = 0;
}
}
return inserted_blocks;
}
/*
* heap_create_run -- (internal) initializes a new run on an existing free chunk
*/
static void
heap_create_run(struct palloc_heap *heap, struct bucket *b,
struct memory_block *m)
{
heap_run_init(heap, b, m);
memblock_rebuild_state(heap, m);
heap_process_run_metadata(heap, b, m);
}
