/*
* memblock_header_legacy_write --
* (internal) writes a legacy header of an object
*/
static void
memblock_header_legacy_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
struct allocation_header_legacy hdr;
hdr.size = size;
hdr.type_num = extra;
hdr.root_size = ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
struct allocation_header_legacy *hdrp = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp));
VALGRIND_ADD_TO_TX(hdrp, sizeof(*hdrp));
pmemops_memcpy(&m->heap->p_ops, hdrp, &hdr,
sizeof(hdr), /* legacy header is 64 bytes in size */
PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED);
VALGRIND_REMOVE_FROM_TX(hdrp, sizeof(*hdrp));
/* unused fields of the legacy headers are used as a red zone */
VALGRIND_DO_MAKE_MEM_NOACCESS(hdrp->unused, sizeof(hdrp->unused));
}
/*
* memblock_header_compact_write --
* (internal) writes a compact header of an object
*/
static void
memblock_header_compact_write(const struct memory_block *m,
size_t size, uint64_t extra, uint16_t flags)
{
COMPILE_ERROR_ON(ALLOC_HDR_COMPACT_SIZE > CACHELINE_SIZE);
struct {
struct allocation_header_compact hdr;
uint8_t padding[CACHELINE_SIZE - ALLOC_HDR_COMPACT_SIZE];
} padded;
padded.hdr.size = size | ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT);
padded.hdr.extra = extra;
struct allocation_header_compact *hdrp = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp));
/*
* If possible write the entire header with a single memcpy, this allows
* the copy implementation to avoid a cache miss on a partial cache line
* write.
*/
size_t hdr_size = ALLOC_HDR_COMPACT_SIZE;
if ((uintptr_t)hdrp % CACHELINE_SIZE == 0 && size >= sizeof(padded))
hdr_size = sizeof(padded);
VALGRIND_ADD_TO_TX(hdrp, hdr_size);
pmemops_memcpy(&m->heap->p_ops, hdrp, &padded, hdr_size,
PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED);
VALGRIND_DO_MAKE_MEM_UNDEFINED((char *)hdrp + ALLOC_HDR_COMPACT_SIZE,
hdr_size - ALLOC_HDR_COMPACT_SIZE);
VALGRIND_REMOVE_FROM_TX(hdrp, hdr_size);
}
/*
* palloc_operation -- persistent memory operation. Takes a NULL pointer
* or an existing memory block and modifies it to occupy, at least, 'size'
* number of bytes.
*
* The malloc, free and realloc routines are implemented in the context of this
* common operation which encompasses all of the functionality usually done
* separately in those methods.
*
* The first thing that needs to be done is determining which memory blocks
* will be affected by the operation - this varies depending on the whether the
* operation will need to modify or free an existing block and/or allocate
* a new one.
*
* Simplified allocation process flow is as follows:
* - reserve a new block in the transient heap
* - prepare the new block
* - create redo log of required modifications
* - chunk metadata
* - offset of the new object
* - commit and process the redo log
*
* And similarly, the deallocation process:
* - create redo log of required modifications
* - reverse the chunk metadata back to the 'free' state
* - set the destination of the object offset to zero
* - commit and process the redo log
* There's an important distinction in the deallocation process - it does not
* return the memory block to the transient container. That is done once no more
* memory is available.
*
* Reallocation is a combination of the above, with one additional step
* of copying the old content.
*/
int
palloc_operation(struct palloc_heap *heap,
uint64_t off, uint64_t *dest_off, size_t size,
palloc_constr constructor, void *arg,
uint64_t extra_field, uint16_t object_flags, uint16_t class_id,
struct operation_context *ctx)
{
struct pobj_action_internal alloc =
OBJ_HEAP_ACTION_INITIALIZER(0, MEMBLOCK_ALLOCATED);
struct pobj_action_internal dealloc =
OBJ_HEAP_ACTION_INITIALIZER(off, MEMBLOCK_FREE);
size_t user_size = 0;
int nops = 0;
struct pobj_action_internal ops[2];
if (dealloc.offset != 0) {
dealloc.m = memblock_from_offset(heap, dealloc.offset);
user_size = dealloc.m.m_ops->get_user_size(&dealloc.m);
if (user_size == size)
return 0;
}
if (size != 0) {
if (palloc_reservation_create(heap, size, constructor, arg,
extra_field, object_flags, class_id, &alloc) != 0)
return -1;
ops[nops++] = alloc;
}
/*
* The offset of an existing block can be nonzero which means this
* operation is either free or a realloc - either way the offset of the
* object needs to be translated into memory block, which is a structure
* that all of the heap methods expect.
*/
if (dealloc.offset != 0) {
/* realloc */
if (!MEMORY_BLOCK_IS_NONE(alloc.m)) {
size_t old_size = user_size;
size_t to_cpy = old_size > size ? size : old_size;
VALGRIND_ADD_TO_TX(
HEAP_OFF_TO_PTR(heap, alloc.offset),
to_cpy);
pmemops_memcpy(&heap->p_ops,
HEAP_OFF_TO_PTR(heap, alloc.offset),
HEAP_OFF_TO_PTR(heap, off),
to_cpy,
0);
VALGRIND_REMOVE_FROM_TX(
HEAP_OFF_TO_PTR(heap, alloc.offset),
to_cpy);
}
dealloc.lock = dealloc.m.m_ops->get_lock(&dealloc.m);
ops[nops++] = dealloc;
}
/*
* If the caller provided a destination value to update, it needs to be
* modified atomically alongside the heap metadata, and so the operation
* context must be used.
* The actual offset value depends on the operation type, but
* alloc.offset variable is used because it's 0 in the case of free,
* and valid otherwise.
*/
if (dest_off)
operation_add_entry(ctx, dest_off, alloc.offset, OPERATION_SET);
palloc_exec_actions(heap, ctx, ops, nops);
return 0;
}
