/*
* alloc_prep_block -- (internal) prepares a memory block for allocation
*
* Once the block is fully reserved and it's guaranteed that no one else will
* be able to write to this memory region it is safe to write the allocation
* header and call the object construction function.
*
* Because the memory block at this stage is only reserved in transient state
* there's no need to worry about fail-safety of this method because in case
* of a crash the memory will be back in the free blocks collection.
*/
static int
alloc_prep_block(struct palloc_heap *heap, struct memory_block m,
palloc_constr constructor, void *arg, uint64_t *offset_value)
{
void *block_data = heap_get_block_data(heap, m);
void *userdatap = (char *)block_data + ALLOC_OFF;
uint64_t unit_size = MEMBLOCK_OPS(AUTO, &m)->
block_size(&m, heap->layout);
uint64_t real_size = unit_size * m.size_idx;
ASSERT((uint64_t)block_data % ALLOC_BLOCK_SIZE == 0);
ASSERT((uint64_t)userdatap % ALLOC_BLOCK_SIZE == 0);
/* mark everything (including headers) as accessible */
VALGRIND_DO_MAKE_MEM_UNDEFINED(block_data, real_size);
/* mark space as allocated */
VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, userdatap,
real_size - ALLOC_OFF);
alloc_write_header(heap, block_data, m, real_size);
int ret;
if (constructor != NULL &&
(ret = constructor(heap->base, userdatap,
real_size - ALLOC_OFF, arg)) != 0) {
/*
* If canceled, revert the block back to the free state in vg
* machinery. Because the free operation is only performed on
* the user data, the allocation header is made inaccessible
* in a separate call.
*/
VALGRIND_DO_MEMPOOL_FREE(heap->layout, userdatap);
VALGRIND_DO_MAKE_MEM_NOACCESS(block_data, ALLOC_OFF);
/*
* During this method there are several stores to pmem that are
* not immediately flushed and in case of a cancellation those
* stores are no longer relevant anyway.
*/
VALGRIND_SET_CLEAN(block_data, ALLOC_OFF);
return ret;
}
/* flushes both the alloc and oob headers */
pmemops_persist(&heap->p_ops, block_data, ALLOC_OFF);
/*
* To avoid determining the user data pointer twice this method is also
* responsible for calculating the offset of the object in the pool that
* will be used to set the offset destination pointer provided by the
* caller.
*/
*offset_value = PMALLOC_PTR_TO_OFF(heap, userdatap);
return 0;
}
/*
* persist_alloc -- (internal) performs a persistent allocation of the
* memory block previously reserved by volatile bucket
*/
static int
persist_alloc(PMEMobjpool *pop, struct lane_section *lane,
struct memory_block m, uint64_t real_size, uint64_t *off,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg),
void *arg, uint64_t data_off)
{
int err;
#ifdef DEBUG
if (heap_block_is_allocated(pop, m)) {
ERR("heap corruption");
ASSERT(0);
}
#endif /* DEBUG */
uint64_t op_result = 0;
void *block_data = heap_get_block_data(pop, m);
void *datap = (char *)block_data + sizeof (struct allocation_header);
void *userdatap = (char *)datap + data_off;
ASSERT((uint64_t)block_data % _POBJ_CL_ALIGNMENT == 0);
/* mark everything (including headers) as accessible */
VALGRIND_DO_MAKE_MEM_UNDEFINED(pop, block_data, real_size);
/* mark space as allocated */
VALGRIND_DO_MEMPOOL_ALLOC(pop, userdatap,
real_size -
sizeof (struct allocation_header) - data_off);
alloc_write_header(pop, block_data, m.chunk_id, m.zone_id, real_size);
if (constructor != NULL)
constructor(pop, userdatap, arg);
if ((err = heap_lock_if_run(pop, m)) != 0) {
VALGRIND_DO_MEMPOOL_FREE(pop, userdatap);
return err;
}
void *hdr = heap_get_block_header(pop, m, HEAP_OP_ALLOC, &op_result);
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, off), pop_offset(pop, datap));
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return err;
}
/*
* pmalloc_construct -- allocates a new block of memory with a constructor
*
* The block offset is written persistently into the off variable, but only
* after the constructor function has been called.
*
* If successful function returns zero. Otherwise an error number is returned.
*/
int
pmalloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg),
void *arg, uint64_t data_off)
{
size_t sizeh = size + sizeof (struct allocation_header);
struct bucket *b = heap_get_best_bucket(pop, sizeh);
int err = 0;
uint32_t units = bucket_calc_units(b, sizeh);
struct memory_block m = {0, 0, units, 0};
if ((err = heap_get_bestfit_block(pop, b, &m)) != 0)
return err;
uint64_t op_result = 0;
void *block_data = heap_get_block_data(pop, m);
void *datap = block_data + sizeof (struct allocation_header);
ASSERT((uint64_t)block_data % _POBJ_CL_ALIGNMENT == 0);
uint64_t real_size = bucket_unit_size(b) * units;
alloc_write_header(pop, block_data, m.chunk_id, m.zone_id, real_size);
if (constructor != NULL)
constructor(pop, datap + data_off, arg);
if ((err = heap_lock_if_run(pop, m)) != 0)
return err;
void *hdr = heap_get_block_header(pop, m, HEAP_OP_ALLOC, &op_result);
struct lane_section *lane;
if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0)
goto err_lane_hold;
struct allocator_lane_section *sec =
(struct allocator_lane_section *)lane->layout;
redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET,
pop_offset(pop, off), pop_offset(pop, datap));
redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER,
pop_offset(pop, hdr), op_result);
redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO);
if (lane_release(pop) != 0) {
ERR("Failed to release the lane");
ASSERT(0);
}
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
return 0;
err_lane_hold:
if (heap_unlock_if_run(pop, m) != 0) {
ERR("Failed to release run lock");
ASSERT(0);
}
if (bucket_insert_block(b, m) != 0) {
ERR("Failed to recover heap volatile state");
ASSERT(0);
}
return err;
}
