/*
* pvector_pop_back -- decreases the number of values and executes
* a user-defined callback in which the caller must zero the value.
*/
uint64_t
pvector_pop_back(struct pvector_context *ctx, entry_op_callback cb)
{
if (ctx->nvalues == 0)
return 0;
uint64_t idx = ctx->nvalues - 1;
struct array_spec s = pvector_get_array_spec(idx);
uint64_t *arrp = OBJ_OFF_TO_PTR(ctx->pop, ctx->vec->arrays[s.idx]);
uint64_t ret = arrp[s.pos];
if (cb)
cb(ctx->pop, &arrp[s.pos]);
if (s.pos == 0 && s.idx != 0 /* the array 0 is embedded */) {
#ifdef USE_VG_PMEMCHECK
if (On_valgrind) {
size_t usable_size = sizeof(uint64_t) *
(1ULL << (s.idx + PVECTOR_INIT_SHIFT));
VALGRIND_REMOVE_FROM_TX(arrp, usable_size);
}
#endif
pfree(ctx->pop, &ctx->vec->arrays[s.idx]);
}
ctx->nvalues--;
return ret;
}
/*
* pvector_get -- returns the vector value at the index.
*/
static uint64_t
pvector_get(PMEMobjpool *pop, struct pvector *vec, uint64_t idx)
{
struct array_spec s = pvector_get_array_spec(idx);
uint64_t *arrp = OBJ_OFF_TO_PTR(pop, vec->arrays[s.idx]);
return arrp[s.pos];
}
/*
* pvector_new -- allocates and initializes persistent vector runtime context.
*
* To make sure the runtime information is correct (the number of values) the
* persistent vector is iterated through and appropriate metrics are measured.
*/
struct pvector_context *
pvector_new(PMEMobjpool *pop, struct pvector *vec)
{
struct pvector_context *ctx = Malloc(sizeof(*ctx));
if (ctx == NULL) {
ERR("!failed to create pvector context");
return NULL;
}
ctx->nvalues = 0;
ctx->pop = pop;
ctx->vec = vec;
ctx->iter = 0;
/*
* First the arrays are traversed to find position of the last element.
* To save some time calculating the sum of the sequence at each step
* the number of values from the array is added to the global nvalues
* counter.
*/
size_t narrays;
for (narrays = 0; narrays < PVECTOR_MAX_ARRAYS; ++narrays) {
if (vec->arrays[narrays] == 0)
break;
if (narrays != PVECTOR_MAX_ARRAYS - 1 &&
vec->arrays[narrays + 1] != 0)
ctx->nvalues += 1ULL << (narrays + PVECTOR_INIT_SHIFT);
}
if (narrays != 0) {
/*
* But if the last array is found and non-null it needs to be
* iterated over to count the number of values present.
*/
size_t last_array = narrays - 1;
size_t arr_size = 1ULL << (last_array + PVECTOR_INIT_SHIFT);
uint64_t *arrp = OBJ_OFF_TO_PTR(pop, vec->arrays[last_array]);
size_t nvalues;
for (nvalues = 0; nvalues < arr_size; ++nvalues) {
if (arrp[nvalues] == 0)
break;
}
/*
* If the last array is not the embedded one and is empty
* it means that the application was interrupted in either
* the push_back or pop_back methods. Either way there's really
* no point in keeping the array.
*/
if (nvalues == 0 && last_array != 0 /* 0 array is embedded*/)
pfree(pop, &vec->arrays[last_array]);
else
ctx->nvalues += nvalues;
}
return ctx;
}
/*
* pvector_reinit -- reinitializes the pvector runtime data
*/
void
pvector_reinit(struct pvector_context *ctx)
{
VALGRIND_ANNOTATE_NEW_MEMORY(ctx, sizeof(*ctx));
for (size_t n = 1; n < PVECTOR_MAX_ARRAYS; ++n) {
if (ctx->vec->arrays[n] == 0)
break;
size_t arr_size = 1ULL << (n + PVECTOR_INIT_SHIFT);
uint64_t *arrp = OBJ_OFF_TO_PTR(ctx->pop, ctx->vec->arrays[n]);
VALGRIND_ANNOTATE_NEW_MEMORY(arrp, sizeof(*arrp) * arr_size);
}
}
/*
* list_get_dest -- (internal) return destination object ID
*
* If the input dest is not OID_NULL returns dest.
* If the input dest is OID_NULL and before is set returns first element.
* If the input dest is OID_NULL and before is no set returns last element.
*/
static inline PMEMoid
list_get_dest(PMEMobjpool *pop, struct list_head *head, PMEMoid dest,
ssize_t pe_offset, int before)
{
if (dest.off)
return dest;
if (head->pe_first.off == 0 || !!before == POBJ_LIST_DEST_HEAD)
return head->pe_first;
struct list_entry *first_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop,
(uintptr_t)((ssize_t)head->pe_first.off + pe_offset));
return first_ptr->pe_prev;
}
/*
* pvector_push_back -- bumps the number of values in the vector and returns
* the pointer to the value position to which the caller must set the
* value. Calling this method without actually setting the value will
* result in an inconsistent vector state.
*/
uint64_t *
pvector_push_back(struct pvector_context *ctx)
{
uint64_t idx = ctx->nvalues;
struct array_spec s = pvector_get_array_spec(idx);
if (s.idx >= PVECTOR_MAX_ARRAYS) {
ERR("Exceeded maximum number of entries in persistent vector");
return NULL;
}
PMEMobjpool *pop = ctx->pop;
/*
* If the destination array does not exist, calculate its size
* and allocate it.
*/
if (ctx->vec->arrays[s.idx] == 0) {
if (s.idx == 0) {
/*
* In the case the vector is completely empty the
* initial embedded array must be assigned as the first
* element of the sequence.
*/
ASSERTeq(util_is_zeroed(ctx->vec,
sizeof(*ctx->vec)), 1);
ctx->vec->arrays[0] = OBJ_PTR_TO_OFF(pop,
&ctx->vec->embedded);
pmemops_persist(&pop->p_ops, &ctx->vec->arrays[0],
sizeof(ctx->vec->arrays[0]));
} else {
size_t arr_size = sizeof(uint64_t) *
(1ULL << (s.idx + PVECTOR_INIT_SHIFT));
if (pmalloc_construct(pop,
&ctx->vec->arrays[s.idx],
arr_size, pvector_array_constr, NULL,
0, OBJ_INTERNAL_OBJECT_MASK, 0) != 0)
return NULL;
}
}
ctx->nvalues++;
uint64_t *arrp = OBJ_OFF_TO_PTR(pop, ctx->vec->arrays[s.idx]);
return &arrp[s.pos];
}
/*
* obj_realloc_root -- (internal) reallocate root object
*/
static int
obj_realloc_root(PMEMobjpool *pop, struct object_store *store, size_t size,
size_t old_size,
void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg), void *arg)
{
LOG(3, "pop %p store %p size %zu old_size %zu",
pop, store, size, old_size);
struct list_head *lhead = &store->root.head;
uint64_t size_offset = OOB_OFFSET_OF(lhead->pe_first, size);
struct carg_realloc carg;
carg.ptr = OBJ_OFF_TO_PTR(pop, lhead->pe_first.off);
carg.old_size = old_size;
carg.new_size = size;
carg.user_type = POBJ_ROOT_TYPE_NUM;
carg.constructor = constructor;
carg.arg = arg;
return list_realloc(pop, lhead, 0, NULL, size,
constructor_zrealloc_root, &carg,
size_offset, size, &lhead->pe_first);
}
/*
* obj_realloc_common -- (internal) common routine for resizing
* existing objects
*/
static int
obj_realloc_common(PMEMobjpool *pop, struct object_store *store,
PMEMoid *oidp, size_t size, unsigned int type_num,
void (*constr_alloc)(PMEMobjpool *pop, void *ptr, void *arg),
void (*constr_realloc)(PMEMobjpool *pop, void *ptr, void *arg))
{
/* if OID is NULL just allocate memory */
if (OBJ_OID_IS_NULL(*oidp)) {
struct carg_alloc carg;
carg.size = size;
/* if size is 0 - do nothing */
if (size == 0)
return 0;
return obj_alloc_construct(pop, oidp, size, type_num,
constr_alloc, &carg);
}
if (size > PMEMOBJ_MAX_ALLOC_SIZE) {
ERR("requested size too large");
errno = ENOMEM;
return -1;
}
/* if size is 0 just free */
if (size == 0) {
obj_free(pop, oidp);
return 0;
}
struct carg_realloc carg;
carg.ptr = OBJ_OFF_TO_PTR(pop, oidp->off);
carg.new_size = size;
carg.old_size = pmemobj_alloc_usable_size(*oidp);
carg.user_type = type_num;
carg.constructor = NULL;
carg.arg = NULL;
struct oob_header *pobj = OOB_HEADER_FROM_OID(pop, *oidp);
uint16_t user_type_old = pobj->data.user_type;
ASSERT(user_type_old < PMEMOBJ_NUM_OID_TYPES);
if (type_num >= PMEMOBJ_NUM_OID_TYPES) {
errno = EINVAL;
ERR("!obj_realloc_construct");
LOG(2, "type_num has to be in range [0, %u]",
PMEMOBJ_NUM_OID_TYPES - 1);
return -1;
}
struct list_head *lhead_old = &store->bytype[user_type_old].head;
if (type_num == user_type_old) {
int ret = list_realloc(pop, lhead_old, 0, NULL, size,
constr_realloc, &carg, 0, 0, oidp);
if (ret)
LOG(2, "list_realloc failed");
return ret;
} else {
struct list_head *lhead_new = &store->bytype[type_num].head;
/*
* Redo log updates 8 byte entries, so we have to prepare
* full 8-byte value even if we want to update smaller field
* (here: user_type).
*/
struct oob_header_data d = pobj->data;
d.user_type = type_num;
uint64_t data_offset = OOB_OFFSET_OF(*oidp, data);
int ret = list_realloc_move(pop, lhead_old, lhead_new, 0, NULL,
size, constr_realloc, &carg, data_offset,
*((uint64_t *)&d), oidp);
if (ret)
LOG(2, "list_realloc_move failed");
return ret;
}
}
/*
* list_remove -- remove object from list
*
* pop - pmemobj handle
* pe_offset - offset to list entry on user list relative to user data
* head - list head
* oid - target object ID
*/
int
list_remove(PMEMobjpool *pop,
ssize_t pe_offset, struct list_head *head,
PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head, NULL);
int ret;
struct lane_section *lane_section;
lane_hold(pop, &lane_section, LANE_SECTION_LIST);
ASSERTne(lane_section, NULL);
ASSERTne(lane_section->layout, NULL);
if ((ret = pmemobj_mutex_lock(pop, &head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
ret = -1;
goto err;
}
struct lane_list_layout *section =
(struct lane_list_layout *)lane_section->layout;
struct redo_log *redo = section->redo;
size_t redo_index = 0;
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + (size_t)pe_offset);
struct list_args_remove args = {
.pe_offset = (ssize_t)pe_offset,
.head = head,
.entry_ptr = entry_ptr,
.obj_doffset = oid.off,
};
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
/* remove element from user list */
redo_index = list_remove_single(pop, redo, redo_index, &args);
/* clear next and prev offsets in removing element using redo log */
redo_index = list_fill_entry_redo_log(pop, redo, redo_index,
&args_common, 0, 0, 0);
redo_log_set_last(pop->redo, redo, redo_index - 1);
redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES);
pmemobj_mutex_unlock_nofail(pop, &head->lock);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_move -- move object between two lists
*
* pop - pmemobj handle
* pe_offset_old - offset to old list entry relative to user data
* head_old - old list head
* pe_offset_new - offset to new list entry relative to user data
* head_new - new list head
* dest - destination object ID
* before - before/after destination
* oid - target object ID
*/
int
list_move(PMEMobjpool *pop,
size_t pe_offset_old, struct list_head *head_old,
size_t pe_offset_new, struct list_head *head_new,
PMEMoid dest, int before, PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head_old, NULL);
ASSERTne(head_new, NULL);
int ret;
struct lane_section *lane_section;
lane_hold(pop, &lane_section, LANE_SECTION_LIST);
ASSERTne(lane_section, NULL);
ASSERTne(lane_section->layout, NULL);
/*
* Grab locks in specified order to avoid dead-locks.
*
* XXX performance improvement: initialize oob locks at pool opening
*/
if ((ret = list_mutexes_lock(pop, head_new, head_old))) {
errno = ret;
LOG(2, "list_mutexes_lock failed");
ret = -1;
goto err;
}
struct lane_list_layout *section =
(struct lane_list_layout *)lane_section->layout;
struct redo_log *redo = section->redo;
size_t redo_index = 0;
dest = list_get_dest(pop, head_new, dest,
(ssize_t)pe_offset_new, before);
struct list_entry *entry_ptr_old =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + pe_offset_old);
struct list_entry *entry_ptr_new =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + pe_offset_new);
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
dest.off + pe_offset_new);
if (head_old == head_new) {
/* moving within the same list */
if (dest.off == oid.off)
goto unlock;
if (before && dest_entry_ptr->pe_prev.off == oid.off) {
if (head_old->pe_first.off != dest.off)
goto unlock;
redo_index = list_update_head(pop, redo, redo_index,
head_old, oid.off);
goto redo_last;
}
if (!before && dest_entry_ptr->pe_next.off == oid.off) {
if (head_old->pe_first.off != oid.off)
goto unlock;
redo_index = list_update_head(pop, redo, redo_index,
head_old, entry_ptr_old->pe_next.off);
goto redo_last;
}
}
ASSERT((ssize_t)pe_offset_old >= 0);
struct list_args_remove args_remove = {
.pe_offset = (ssize_t)pe_offset_old,
.head = head_old,
.entry_ptr = entry_ptr_old,
.obj_doffset = oid.off,
};
struct list_args_insert args_insert = {
.head = head_new,
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.before = before,
};
ASSERT((ssize_t)pe_offset_new >= 0);
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr_new,
.pe_offset = (ssize_t)pe_offset_new,
};
uint64_t next_offset;
uint64_t prev_offset;
/* remove element from user list */
redo_index = list_remove_single(pop, redo, redo_index, &args_remove);
/* insert element to user list */
redo_index = list_insert_user(pop, redo, redo_index, &args_insert,
&args_common, &next_offset, &prev_offset);
/* offsets differ, move is between different list entries - set uuid */
int set_uuid = pe_offset_new != pe_offset_old ? 1 : 0;
/* fill next and prev offsets of moving element using redo log */
redo_index = list_fill_entry_redo_log(pop, redo, redo_index,
&args_common, next_offset, prev_offset, set_uuid);
redo_last:
redo_log_set_last(pop->redo, redo, redo_index - 1);
redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES);
unlock:
list_mutexes_unlock(pop, head_new, head_old);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* lane_list_recovery -- (internal) recover the list section of the lane
*/
static int
lane_list_recovery(PMEMobjpool *pop, void *data, unsigned length)
{
LOG(7, "list lane %p", data);
struct lane_list_layout *section = data;
ASSERT(sizeof(*section) <= length);
redo_log_recover(pop->redo, section->redo, REDO_NUM_ENTRIES);
if (section->obj_offset) {
/* alloc or free recovery */
pfree(pop, §ion->obj_offset);
}
return 0;
}
/*
* lane_list_check -- (internal) check consistency of lane
*/
static int
lane_list_check(PMEMobjpool *pop, void *data, unsigned length)
{
LOG(3, "list lane %p", data);
struct lane_list_layout *section = data;
int ret = 0;
if ((ret = redo_log_check(pop->redo,
section->redo, REDO_NUM_ENTRIES)) != 0) {
ERR("list lane: redo log check failed");
ASSERT(ret == 0 || ret == -1);
return ret;
}
if (section->obj_offset &&
!OBJ_OFF_FROM_HEAP(pop, section->obj_offset)) {
ERR("list lane: invalid offset 0x%" PRIx64,
section->obj_offset);
return -1;
}
return 0;
}
/*
* lane_list_construct_rt -- (internal) construct runtime part of list section
*/
static void *
lane_list_construct_rt(PMEMobjpool *pop)
{
return NULL;
}
/*
* lane_list_destroy_rt -- (internal) destroy runtime part of list section
*/
static void
lane_list_destroy_rt(PMEMobjpool *pop, void *rt)
{
/* NOP */
}
/*
* lane_list_boot -- global runtime init routine of list section
*/
static int
lane_list_boot(PMEMobjpool *pop)
{
/* NOP */
return 0;
}
/*
* lane_list_cleanup -- global runtime cleanup routine of list section
*/
static int
lane_list_cleanup(PMEMobjpool *pop)
{
/* NOP */
return 0;
}
static struct section_operations list_ops = {
.construct_rt = lane_list_construct_rt,
.destroy_rt = lane_list_destroy_rt,
.recover = lane_list_recovery,
.check = lane_list_check,
.boot = lane_list_boot,
.cleanup = lane_list_cleanup,
};
SECTION_PARM(LANE_SECTION_LIST, &list_ops);
/*
* list_insert -- insert object to a single list
*
* pop - pmemobj handle
* pe_offset - offset to list entry on user list relative to user data
* head - list head
* dest - destination object ID
* before - before/after destination
* oid - target object ID
*/
int
list_insert(PMEMobjpool *pop,
ssize_t pe_offset, struct list_head *head,
PMEMoid dest, int before,
PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head, NULL);
int ret;
struct lane_section *lane_section;
lane_hold(pop, &lane_section, LANE_SECTION_LIST);
if ((ret = pmemobj_mutex_lock(pop, &head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
ret = -1;
goto err;
}
ASSERTne(lane_section, NULL);
ASSERTne(lane_section->layout, NULL);
struct lane_list_layout *section =
(struct lane_list_layout *)lane_section->layout;
struct redo_log *redo = section->redo;
size_t redo_index = 0;
dest = list_get_dest(pop, head, dest, pe_offset, before);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
(uintptr_t)((ssize_t)oid.off + pe_offset));
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
(uintptr_t)((ssize_t)dest.off + pe_offset));
struct list_args_insert args = {
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.head = head,
.before = before,
};
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
uint64_t next_offset;
uint64_t prev_offset;
/* insert element to user list */
redo_index = list_insert_user(pop, redo, redo_index,
&args, &args_common, &next_offset, &prev_offset);
/* fill entry of existing element using redo log */
redo_index = list_fill_entry_redo_log(pop, redo, redo_index,
&args_common, next_offset, prev_offset, 1);
redo_log_set_last(pop->redo, redo, redo_index - 1);
redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES);
pmemobj_mutex_unlock_nofail(pop, &head->lock);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_remove_free -- remove from two lists and free an object
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head, *must* be locked if not NULL
* oidp - pointer to target object ID
*/
static void
list_remove_free(PMEMobjpool *pop, size_t pe_offset,
struct list_head *user_head, PMEMoid *oidp)
{
LOG(3, NULL);
ASSERT(user_head != NULL);
#ifdef DEBUG
int r = pmemobj_mutex_assert_locked(pop, &user_head->lock);
ASSERTeq(r, 0);
#endif
struct lane_section *lane_section;
lane_hold(pop, &lane_section, LANE_SECTION_LIST);
ASSERTne(lane_section, NULL);
ASSERTne(lane_section->layout, NULL);
struct lane_list_layout *section =
(struct lane_list_layout *)lane_section->layout;
uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset);
struct redo_log *redo = section->redo;
size_t redo_index = 0;
uint64_t obj_doffset = oidp->off;
ASSERT((ssize_t)pe_offset >= 0);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
obj_doffset + pe_offset);
struct list_args_remove args = {
.pe_offset = (ssize_t)pe_offset,
.head = user_head,
.entry_ptr = entry_ptr,
.obj_doffset = obj_doffset
};
/* remove from user list */
redo_index = list_remove_single(pop, redo, redo_index, &args);
/* clear the oid */
if (OBJ_PTR_IS_VALID(pop, oidp))
redo_index = list_set_oid_redo_log(pop, redo, redo_index,
oidp, 0, 1);
else
oidp->off = 0;
redo_log_store_last(pop->redo, redo, redo_index, sec_off_off,
obj_doffset);
redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES);
/*
* Don't need to fill next and prev offsets of removing element
* because the element is freed.
*/
pfree(pop, §ion->obj_offset);
lane_release(pop);
}
/*
* list_remove_free_user -- remove from two lists and free an object
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head
* oidp - pointer to target object ID
*/
int
list_remove_free_user(PMEMobjpool *pop, size_t pe_offset,
struct list_head *user_head, PMEMoid *oidp)
{
LOG(3, NULL);
int ret;
if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
return -1;
}
list_remove_free(pop, pe_offset, user_head, oidp);
pmemobj_mutex_unlock_nofail(pop, &user_head->lock);
return 0;
}
/*
* list_insert_new -- allocate and insert element to oob and user lists
*
* pop - pmemobj pool handle
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head, must be locked if not NULL
* dest - destination on user list
* before - insert before/after destination on user list
* size - size of allocation, will be increased by OBJ_OOB_SIZE
* constructor - object's constructor
* arg - argument for object's constructor
* oidp - pointer to target object ID
*/
static int
list_insert_new(PMEMobjpool *pop,
size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before,
size_t size, int (*constructor)(void *ctx, void *ptr,
size_t usable_size, void *arg), void *arg, PMEMoid *oidp)
{
LOG(3, NULL);
ASSERT(user_head != NULL);
int ret;
struct lane_section *lane_section;
#ifdef DEBUG
int r = pmemobj_mutex_assert_locked(pop, &user_head->lock);
ASSERTeq(r, 0);
#endif
lane_hold(pop, &lane_section, LANE_SECTION_LIST);
ASSERTne(lane_section, NULL);
ASSERTne(lane_section->layout, NULL);
struct lane_list_layout *section =
(struct lane_list_layout *)lane_section->layout;
struct redo_log *redo = section->redo;
size_t redo_index = 0;
uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset);
if (constructor) {
if ((ret = pmalloc_construct(pop,
§ion->obj_offset, size,
constructor, arg, 0, 0, 0))) {
ERR("!pmalloc_construct");
goto err_pmalloc;
}
} else {
ret = pmalloc(pop, §ion->obj_offset, size, 0, 0);
if (ret) {
ERR("!pmalloc");
goto err_pmalloc;
}
}
uint64_t obj_doffset = section->obj_offset;
ASSERT((ssize_t)pe_offset >= 0);
dest = list_get_dest(pop, user_head, dest,
(ssize_t)pe_offset, before);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
obj_doffset + pe_offset);
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
dest.off + pe_offset);
struct list_args_insert args = {
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.head = user_head,
.before = before,
};
struct list_args_common args_common = {
.obj_doffset = obj_doffset,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
uint64_t next_offset;
uint64_t prev_offset;
/* insert element to user list */
redo_index = list_insert_user(pop,
redo, redo_index, &args, &args_common,
&next_offset, &prev_offset);
/* don't need to use redo log for filling new element */
list_fill_entry_persist(pop, entry_ptr,
next_offset, prev_offset);
if (oidp != NULL) {
if (OBJ_PTR_IS_VALID(pop, oidp)) {
redo_index = list_set_oid_redo_log(pop, redo,
redo_index, oidp, obj_doffset, 0);
} else {
oidp->off = obj_doffset;
oidp->pool_uuid_lo = pop->uuid_lo;
}
}
/* clear the obj_offset in lane section */
redo_log_store_last(pop->redo, redo, redo_index, sec_off_off, 0);
redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES);
ret = 0;
err_pmalloc:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_insert_new_user -- allocate and insert element to oob and user lists
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head
* dest - destination on user list
* before - insert before/after destination on user list
* size - size of allocation, will be increased by OBJ_OOB_SIZE
* constructor - object's constructor
* arg - argument for object's constructor
* oidp - pointer to target object ID
*/
int
list_insert_new_user(PMEMobjpool *pop,
size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before,
size_t size, int (*constructor)(void *ctx, void *ptr,
size_t usable_size, void *arg), void *arg, PMEMoid *oidp)
{
int ret;
if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
return -1;
}
ret = list_insert_new(pop, pe_offset, user_head,
dest, before, size, constructor, arg, oidp);
pmemobj_mutex_unlock_nofail(pop, &user_head->lock);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* obj_realloc_common -- (internal) common routine for resizing
* existing objects
*/
static int
obj_realloc_common(PMEMobjpool *pop, struct object_store *store,
PMEMoid *oidp, size_t size, type_num_t type_num, int zero_init)
{
/* if OID is NULL just allocate memory */
if (OBJ_OID_IS_NULL(*oidp)) {
/* if size is 0 - do nothing */
if (size == 0)
return 0;
return obj_alloc_construct(pop, oidp, size, type_num,
zero_init, NULL, NULL);
}
if (size > PMEMOBJ_MAX_ALLOC_SIZE) {
ERR("requested size too large");
errno = ENOMEM;
return -1;
}
/* if size is 0 just free */
if (size == 0) {
obj_free(pop, oidp);
return 0;
}
struct carg_realloc carg;
carg.ptr = OBJ_OFF_TO_PTR(pop, oidp->off);
carg.new_size = size;
carg.old_size = pmemobj_alloc_usable_size(*oidp);
carg.user_type = type_num;
carg.constructor = NULL;
carg.arg = NULL;
carg.zero_init = zero_init;
struct oob_header *pobj = OOB_HEADER_FROM_OID(pop, *oidp);
type_num_t user_type_old = pobj->data.user_type;
/* callers should have checked this */
ASSERT(type_num < PMEMOBJ_NUM_OID_TYPES);
ASSERT(user_type_old < PMEMOBJ_NUM_OID_TYPES);
struct list_head *lhead_old = &store->bytype[user_type_old].head;
if (type_num == user_type_old) {
int ret = list_realloc(pop, lhead_old, 0, NULL, size,
constructor_realloc, &carg, 0, 0, oidp);
if (ret)
LOG(2, "list_realloc failed");
/* oidp could be different, so we need to get the ptr again */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
return ret;
} else {
struct list_head *lhead_new = &store->bytype[type_num].head;
/*
* Header padding doubles as a red zone to check for header
* overwrites. Disable it temporarily so we can modify the type
* number.
*/
VALGRIND_DO_MAKE_MEM_DEFINED(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
/*
* Redo log updates 8 byte entries, so we have to prepare
* full 8-byte value even if we want to update smaller field
* (here: user_type).
*/
struct oob_header_data d = pobj->data;
d.user_type = type_num;
uint64_t data_offset = OOB_OFFSET_OF(*oidp, data);
int ret = list_realloc_move(pop, lhead_old, lhead_new, 0, NULL,
size, constructor_realloc, &carg, data_offset,
*((uint64_t *)&d), oidp);
if (ret)
LOG(2, "list_realloc_move failed");
/* oidp could be different, so we need to get the ptr again */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
return ret;
}
}
/*
* list_remove -- remove object from list
*
* pop - pmemobj handle
* pe_offset - offset to list entry on user list relative to user data
* head - list head
* oid - target object ID
*/
int
list_remove(PMEMobjpool *pop,
ssize_t pe_offset, struct list_head *head,
PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head, NULL);
int ret;
struct lane *lane;
lane_hold(pop, &lane);
if ((ret = pmemobj_mutex_lock(pop, &head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
ret = -1;
goto err;
}
struct operation_context *ctx = lane->external;
operation_start(ctx);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + (size_t)pe_offset);
struct list_args_remove args = {
.pe_offset = (ssize_t)pe_offset,
.head = head,
.entry_ptr = entry_ptr,
.obj_doffset = oid.off,
};
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
/* remove element from user list */
list_remove_single(pop, ctx, &args);
/* clear next and prev offsets in removing element using redo log */
list_fill_entry_redo_log(pop, ctx,
&args_common, 0, 0, 0);
operation_finish(ctx);
pmemobj_mutex_unlock_nofail(pop, &head->lock);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_move -- move object between two lists
*
* pop - pmemobj handle
* pe_offset_old - offset to old list entry relative to user data
* head_old - old list head
* pe_offset_new - offset to new list entry relative to user data
* head_new - new list head
* dest - destination object ID
* before - before/after destination
* oid - target object ID
*/
int
list_move(PMEMobjpool *pop,
size_t pe_offset_old, struct list_head *head_old,
size_t pe_offset_new, struct list_head *head_new,
PMEMoid dest, int before, PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head_old, NULL);
ASSERTne(head_new, NULL);
int ret;
struct lane *lane;
lane_hold(pop, &lane);
/*
* Grab locks in specified order to avoid dead-locks.
*
* XXX performance improvement: initialize oob locks at pool opening
*/
if ((ret = list_mutexes_lock(pop, head_new, head_old))) {
errno = ret;
LOG(2, "list_mutexes_lock failed");
ret = -1;
goto err;
}
struct operation_context *ctx = lane->external;
operation_start(ctx);
dest = list_get_dest(pop, head_new, dest,
(ssize_t)pe_offset_new, before);
struct list_entry *entry_ptr_old =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + pe_offset_old);
struct list_entry *entry_ptr_new =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
oid.off + pe_offset_new);
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
dest.off + pe_offset_new);
if (head_old == head_new) {
/* moving within the same list */
if (dest.off == oid.off)
goto unlock;
if (before && dest_entry_ptr->pe_prev.off == oid.off) {
if (head_old->pe_first.off != dest.off)
goto unlock;
list_update_head(pop, ctx,
head_old, oid.off);
goto redo_last;
}
if (!before && dest_entry_ptr->pe_next.off == oid.off) {
if (head_old->pe_first.off != oid.off)
goto unlock;
list_update_head(pop, ctx,
head_old, entry_ptr_old->pe_next.off);
goto redo_last;
}
}
ASSERT((ssize_t)pe_offset_old >= 0);
struct list_args_remove args_remove = {
.pe_offset = (ssize_t)pe_offset_old,
.head = head_old,
.entry_ptr = entry_ptr_old,
.obj_doffset = oid.off,
};
struct list_args_insert args_insert = {
.head = head_new,
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.before = before,
};
ASSERT((ssize_t)pe_offset_new >= 0);
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr_new,
.pe_offset = (ssize_t)pe_offset_new,
};
uint64_t next_offset;
uint64_t prev_offset;
/* remove element from user list */
list_remove_single(pop, ctx, &args_remove);
/* insert element to user list */
list_insert_user(pop, ctx, &args_insert,
&args_common, &next_offset, &prev_offset);
/* offsets differ, move is between different list entries - set uuid */
int set_uuid = pe_offset_new != pe_offset_old ? 1 : 0;
/* fill next and prev offsets of moving element using redo log */
list_fill_entry_redo_log(pop, ctx,
&args_common, next_offset, prev_offset, set_uuid);
redo_last:
unlock:
operation_finish(ctx);
list_mutexes_unlock(pop, head_new, head_old);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_insert -- insert object to a single list
*
* pop - pmemobj handle
* pe_offset - offset to list entry on user list relative to user data
* head - list head
* dest - destination object ID
* before - before/after destination
* oid - target object ID
*/
int
list_insert(PMEMobjpool *pop,
ssize_t pe_offset, struct list_head *head,
PMEMoid dest, int before,
PMEMoid oid)
{
LOG(3, NULL);
ASSERTne(head, NULL);
struct lane *lane;
lane_hold(pop, &lane);
int ret;
if ((ret = pmemobj_mutex_lock(pop, &head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
ret = -1;
goto err;
}
struct operation_context *ctx = lane->external;
operation_start(ctx);
dest = list_get_dest(pop, head, dest, pe_offset, before);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
(uintptr_t)((ssize_t)oid.off + pe_offset));
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
(uintptr_t)((ssize_t)dest.off + pe_offset));
struct list_args_insert args = {
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.head = head,
.before = before,
};
struct list_args_common args_common = {
.obj_doffset = oid.off,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
uint64_t next_offset;
uint64_t prev_offset;
/* insert element to user list */
list_insert_user(pop, ctx,
&args, &args_common, &next_offset, &prev_offset);
/* fill entry of existing element using redo log */
list_fill_entry_redo_log(pop, ctx,
&args_common, next_offset, prev_offset, 1);
operation_finish(ctx);
pmemobj_mutex_unlock_nofail(pop, &head->lock);
err:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_remove_free -- remove from two lists and free an object
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head, *must* be locked if not NULL
* oidp - pointer to target object ID
*/
static void
list_remove_free(PMEMobjpool *pop, size_t pe_offset,
struct list_head *user_head, PMEMoid *oidp)
{
LOG(3, NULL);
ASSERT(user_head != NULL);
#ifdef DEBUG
int r = pmemobj_mutex_assert_locked(pop, &user_head->lock);
ASSERTeq(r, 0);
#endif
struct lane *lane;
lane_hold(pop, &lane);
struct operation_context *ctx = lane->external;
operation_start(ctx);
struct pobj_action deferred;
palloc_defer_free(&pop->heap, oidp->off, &deferred);
uint64_t obj_doffset = oidp->off;
ASSERT((ssize_t)pe_offset >= 0);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
obj_doffset + pe_offset);
struct list_args_remove args = {
.pe_offset = (ssize_t)pe_offset,
.head = user_head,
.entry_ptr = entry_ptr,
.obj_doffset = obj_doffset
};
/* remove from user list */
list_remove_single(pop, ctx, &args);
/* clear the oid */
if (OBJ_PTR_IS_VALID(pop, oidp))
list_set_oid_redo_log(pop, ctx, oidp, 0, 1);
else
oidp->off = 0;
palloc_publish(&pop->heap, &deferred, 1, ctx);
lane_release(pop);
}
/*
* list_remove_free_user -- remove from two lists and free an object
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head
* oidp - pointer to target object ID
*/
int
list_remove_free_user(PMEMobjpool *pop, size_t pe_offset,
struct list_head *user_head, PMEMoid *oidp)
{
LOG(3, NULL);
int ret;
if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
return -1;
}
list_remove_free(pop, pe_offset, user_head, oidp);
pmemobj_mutex_unlock_nofail(pop, &user_head->lock);
return 0;
}
/*
* list_insert_new -- allocate and insert element to oob and user lists
*
* pop - pmemobj pool handle
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head, must be locked if not NULL
* dest - destination on user list
* before - insert before/after destination on user list
* size - size of allocation, will be increased by OBJ_OOB_SIZE
* constructor - object's constructor
* arg - argument for object's constructor
* oidp - pointer to target object ID
*/
static int
list_insert_new(PMEMobjpool *pop,
size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before,
size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr,
size_t usable_size, void *arg), void *arg, PMEMoid *oidp)
{
LOG(3, NULL);
ASSERT(user_head != NULL);
int ret;
#ifdef DEBUG
int r = pmemobj_mutex_assert_locked(pop, &user_head->lock);
ASSERTeq(r, 0);
#endif
struct lane *lane;
lane_hold(pop, &lane);
struct pobj_action reserved;
if (palloc_reserve(&pop->heap, size, constructor, arg,
type_num, 0, 0, &reserved) != 0) {
ERR("!palloc_reserve");
ret = -1;
goto err_pmalloc;
}
uint64_t obj_doffset = reserved.heap.offset;
struct operation_context *ctx = lane->external;
operation_start(ctx);
ASSERT((ssize_t)pe_offset >= 0);
dest = list_get_dest(pop, user_head, dest,
(ssize_t)pe_offset, before);
struct list_entry *entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
obj_doffset + pe_offset);
struct list_entry *dest_entry_ptr =
(struct list_entry *)OBJ_OFF_TO_PTR(pop,
dest.off + pe_offset);
struct list_args_insert args = {
.dest = dest,
.dest_entry_ptr = dest_entry_ptr,
.head = user_head,
.before = before,
};
struct list_args_common args_common = {
.obj_doffset = obj_doffset,
.entry_ptr = entry_ptr,
.pe_offset = (ssize_t)pe_offset,
};
uint64_t next_offset;
uint64_t prev_offset;
/* insert element to user list */
list_insert_user(pop,
ctx, &args, &args_common,
&next_offset, &prev_offset);
/* don't need to use redo log for filling new element */
list_fill_entry_persist(pop, entry_ptr,
next_offset, prev_offset);
if (oidp != NULL) {
if (OBJ_PTR_IS_VALID(pop, oidp)) {
list_set_oid_redo_log(pop, ctx,
oidp, obj_doffset, 0);
} else {
oidp->off = obj_doffset;
oidp->pool_uuid_lo = pop->uuid_lo;
}
}
palloc_publish(&pop->heap, &reserved, 1, ctx);
ret = 0;
err_pmalloc:
lane_release(pop);
ASSERT(ret == 0 || ret == -1);
return ret;
}
/*
* list_insert_new_user -- allocate and insert element to oob and user lists
*
* pop - pmemobj pool handle
* oob_head - oob list head
* pe_offset - offset to list entry on user list relative to user data
* user_head - user list head
* dest - destination on user list
* before - insert before/after destination on user list
* size - size of allocation, will be increased by OBJ_OOB_SIZE
* constructor - object's constructor
* arg - argument for object's constructor
* oidp - pointer to target object ID
*/
int
list_insert_new_user(PMEMobjpool *pop,
size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before,
size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr,
size_t usable_size, void *arg), void *arg, PMEMoid *oidp)
{
int ret;
if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) {
errno = ret;
LOG(2, "pmemobj_mutex_lock failed");
return -1;
}
ret = list_insert_new(pop, pe_offset, user_head,
dest, before, size, type_num, constructor, arg, oidp);
pmemobj_mutex_unlock_nofail(pop, &user_head->lock);
ASSERT(ret == 0 || ret == -1);
return ret;
}