/* * pvector_pop_back -- decreases the number of values and executes * a user-defined callback in which the caller must zero the value. */ uint64_t pvector_pop_back(struct pvector_context *ctx, entry_op_callback cb) { if (ctx->nvalues == 0) return 0; uint64_t idx = ctx->nvalues - 1; struct array_spec s = pvector_get_array_spec(idx); uint64_t *arrp = OBJ_OFF_TO_PTR(ctx->pop, ctx->vec->arrays[s.idx]); uint64_t ret = arrp[s.pos]; if (cb) cb(ctx->pop, &arrp[s.pos]); if (s.pos == 0 && s.idx != 0 /* the array 0 is embedded */) { #ifdef USE_VG_PMEMCHECK if (On_valgrind) { size_t usable_size = sizeof(uint64_t) * (1ULL << (s.idx + PVECTOR_INIT_SHIFT)); VALGRIND_REMOVE_FROM_TX(arrp, usable_size); } #endif pfree(ctx->pop, &ctx->vec->arrays[s.idx]); } ctx->nvalues--; return ret; }
/* * pvector_get -- returns the vector value at the index. */ static uint64_t pvector_get(PMEMobjpool *pop, struct pvector *vec, uint64_t idx) { struct array_spec s = pvector_get_array_spec(idx); uint64_t *arrp = OBJ_OFF_TO_PTR(pop, vec->arrays[s.idx]); return arrp[s.pos]; }
/* * pvector_new -- allocates and initializes persistent vector runtime context. * * To make sure the runtime information is correct (the number of values) the * persistent vector is iterated through and appropriate metrics are measured. */ struct pvector_context * pvector_new(PMEMobjpool *pop, struct pvector *vec) { struct pvector_context *ctx = Malloc(sizeof(*ctx)); if (ctx == NULL) { ERR("!failed to create pvector context"); return NULL; } ctx->nvalues = 0; ctx->pop = pop; ctx->vec = vec; ctx->iter = 0; /* * First the arrays are traversed to find position of the last element. * To save some time calculating the sum of the sequence at each step * the number of values from the array is added to the global nvalues * counter. */ size_t narrays; for (narrays = 0; narrays < PVECTOR_MAX_ARRAYS; ++narrays) { if (vec->arrays[narrays] == 0) break; if (narrays != PVECTOR_MAX_ARRAYS - 1 && vec->arrays[narrays + 1] != 0) ctx->nvalues += 1ULL << (narrays + PVECTOR_INIT_SHIFT); } if (narrays != 0) { /* * But if the last array is found and non-null it needs to be * iterated over to count the number of values present. */ size_t last_array = narrays - 1; size_t arr_size = 1ULL << (last_array + PVECTOR_INIT_SHIFT); uint64_t *arrp = OBJ_OFF_TO_PTR(pop, vec->arrays[last_array]); size_t nvalues; for (nvalues = 0; nvalues < arr_size; ++nvalues) { if (arrp[nvalues] == 0) break; } /* * If the last array is not the embedded one and is empty * it means that the application was interrupted in either * the push_back or pop_back methods. Either way there's really * no point in keeping the array. */ if (nvalues == 0 && last_array != 0 /* 0 array is embedded*/) pfree(pop, &vec->arrays[last_array]); else ctx->nvalues += nvalues; } return ctx; }
/* * pvector_reinit -- reinitializes the pvector runtime data */ void pvector_reinit(struct pvector_context *ctx) { VALGRIND_ANNOTATE_NEW_MEMORY(ctx, sizeof(*ctx)); for (size_t n = 1; n < PVECTOR_MAX_ARRAYS; ++n) { if (ctx->vec->arrays[n] == 0) break; size_t arr_size = 1ULL << (n + PVECTOR_INIT_SHIFT); uint64_t *arrp = OBJ_OFF_TO_PTR(ctx->pop, ctx->vec->arrays[n]); VALGRIND_ANNOTATE_NEW_MEMORY(arrp, sizeof(*arrp) * arr_size); } }
/* * list_get_dest -- (internal) return destination object ID * * If the input dest is not OID_NULL returns dest. * If the input dest is OID_NULL and before is set returns first element. * If the input dest is OID_NULL and before is no set returns last element. */ static inline PMEMoid list_get_dest(PMEMobjpool *pop, struct list_head *head, PMEMoid dest, ssize_t pe_offset, int before) { if (dest.off) return dest; if (head->pe_first.off == 0 || !!before == POBJ_LIST_DEST_HEAD) return head->pe_first; struct list_entry *first_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)head->pe_first.off + pe_offset)); return first_ptr->pe_prev; }
/* * pvector_push_back -- bumps the number of values in the vector and returns * the pointer to the value position to which the caller must set the * value. Calling this method without actually setting the value will * result in an inconsistent vector state. */ uint64_t * pvector_push_back(struct pvector_context *ctx) { uint64_t idx = ctx->nvalues; struct array_spec s = pvector_get_array_spec(idx); if (s.idx >= PVECTOR_MAX_ARRAYS) { ERR("Exceeded maximum number of entries in persistent vector"); return NULL; } PMEMobjpool *pop = ctx->pop; /* * If the destination array does not exist, calculate its size * and allocate it. */ if (ctx->vec->arrays[s.idx] == 0) { if (s.idx == 0) { /* * In the case the vector is completely empty the * initial embedded array must be assigned as the first * element of the sequence. */ ASSERTeq(util_is_zeroed(ctx->vec, sizeof(*ctx->vec)), 1); ctx->vec->arrays[0] = OBJ_PTR_TO_OFF(pop, &ctx->vec->embedded); pmemops_persist(&pop->p_ops, &ctx->vec->arrays[0], sizeof(ctx->vec->arrays[0])); } else { size_t arr_size = sizeof(uint64_t) * (1ULL << (s.idx + PVECTOR_INIT_SHIFT)); if (pmalloc_construct(pop, &ctx->vec->arrays[s.idx], arr_size, pvector_array_constr, NULL, 0, OBJ_INTERNAL_OBJECT_MASK, 0) != 0) return NULL; } } ctx->nvalues++; uint64_t *arrp = OBJ_OFF_TO_PTR(pop, ctx->vec->arrays[s.idx]); return &arrp[s.pos]; }
/* * obj_realloc_root -- (internal) reallocate root object */ static int obj_realloc_root(PMEMobjpool *pop, struct object_store *store, size_t size, size_t old_size, void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg), void *arg) { LOG(3, "pop %p store %p size %zu old_size %zu", pop, store, size, old_size); struct list_head *lhead = &store->root.head; uint64_t size_offset = OOB_OFFSET_OF(lhead->pe_first, size); struct carg_realloc carg; carg.ptr = OBJ_OFF_TO_PTR(pop, lhead->pe_first.off); carg.old_size = old_size; carg.new_size = size; carg.user_type = POBJ_ROOT_TYPE_NUM; carg.constructor = constructor; carg.arg = arg; return list_realloc(pop, lhead, 0, NULL, size, constructor_zrealloc_root, &carg, size_offset, size, &lhead->pe_first); }
/* * obj_realloc_common -- (internal) common routine for resizing * existing objects */ static int obj_realloc_common(PMEMobjpool *pop, struct object_store *store, PMEMoid *oidp, size_t size, unsigned int type_num, void (*constr_alloc)(PMEMobjpool *pop, void *ptr, void *arg), void (*constr_realloc)(PMEMobjpool *pop, void *ptr, void *arg)) { /* if OID is NULL just allocate memory */ if (OBJ_OID_IS_NULL(*oidp)) { struct carg_alloc carg; carg.size = size; /* if size is 0 - do nothing */ if (size == 0) return 0; return obj_alloc_construct(pop, oidp, size, type_num, constr_alloc, &carg); } if (size > PMEMOBJ_MAX_ALLOC_SIZE) { ERR("requested size too large"); errno = ENOMEM; return -1; } /* if size is 0 just free */ if (size == 0) { obj_free(pop, oidp); return 0; } struct carg_realloc carg; carg.ptr = OBJ_OFF_TO_PTR(pop, oidp->off); carg.new_size = size; carg.old_size = pmemobj_alloc_usable_size(*oidp); carg.user_type = type_num; carg.constructor = NULL; carg.arg = NULL; struct oob_header *pobj = OOB_HEADER_FROM_OID(pop, *oidp); uint16_t user_type_old = pobj->data.user_type; ASSERT(user_type_old < PMEMOBJ_NUM_OID_TYPES); if (type_num >= PMEMOBJ_NUM_OID_TYPES) { errno = EINVAL; ERR("!obj_realloc_construct"); LOG(2, "type_num has to be in range [0, %u]", PMEMOBJ_NUM_OID_TYPES - 1); return -1; } struct list_head *lhead_old = &store->bytype[user_type_old].head; if (type_num == user_type_old) { int ret = list_realloc(pop, lhead_old, 0, NULL, size, constr_realloc, &carg, 0, 0, oidp); if (ret) LOG(2, "list_realloc failed"); return ret; } else { struct list_head *lhead_new = &store->bytype[type_num].head; /* * Redo log updates 8 byte entries, so we have to prepare * full 8-byte value even if we want to update smaller field * (here: user_type). */ struct oob_header_data d = pobj->data; d.user_type = type_num; uint64_t data_offset = OOB_OFFSET_OF(*oidp, data); int ret = list_realloc_move(pop, lhead_old, lhead_new, 0, NULL, size, constr_realloc, &carg, data_offset, *((uint64_t *)&d), oidp); if (ret) LOG(2, "list_realloc_move failed"); return ret; } }
/* * list_remove -- remove object from list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * oid - target object ID */ int list_remove(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); int ret; struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + (size_t)pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = head, .entry_ptr = entry_ptr, .obj_doffset = oid.off, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; /* remove element from user list */ redo_index = list_remove_single(pop, redo, redo_index, &args); /* clear next and prev offsets in removing element using redo log */ redo_index = list_fill_entry_redo_log(pop, redo, redo_index, &args_common, 0, 0, 0); redo_log_set_last(pop->redo, redo, redo_index - 1); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_move -- move object between two lists * * pop - pmemobj handle * pe_offset_old - offset to old list entry relative to user data * head_old - old list head * pe_offset_new - offset to new list entry relative to user data * head_new - new list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_move(PMEMobjpool *pop, size_t pe_offset_old, struct list_head *head_old, size_t pe_offset_new, struct list_head *head_new, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head_old, NULL); ASSERTne(head_new, NULL); int ret; struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); /* * Grab locks in specified order to avoid dead-locks. * * XXX performance improvement: initialize oob locks at pool opening */ if ((ret = list_mutexes_lock(pop, head_new, head_old))) { errno = ret; LOG(2, "list_mutexes_lock failed"); ret = -1; goto err; } struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; dest = list_get_dest(pop, head_new, dest, (ssize_t)pe_offset_new, before); struct list_entry *entry_ptr_old = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + pe_offset_old); struct list_entry *entry_ptr_new = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + pe_offset_new); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset_new); if (head_old == head_new) { /* moving within the same list */ if (dest.off == oid.off) goto unlock; if (before && dest_entry_ptr->pe_prev.off == oid.off) { if (head_old->pe_first.off != dest.off) goto unlock; redo_index = list_update_head(pop, redo, redo_index, head_old, oid.off); goto redo_last; } if (!before && dest_entry_ptr->pe_next.off == oid.off) { if (head_old->pe_first.off != oid.off) goto unlock; redo_index = list_update_head(pop, redo, redo_index, head_old, entry_ptr_old->pe_next.off); goto redo_last; } } ASSERT((ssize_t)pe_offset_old >= 0); struct list_args_remove args_remove = { .pe_offset = (ssize_t)pe_offset_old, .head = head_old, .entry_ptr = entry_ptr_old, .obj_doffset = oid.off, }; struct list_args_insert args_insert = { .head = head_new, .dest = dest, .dest_entry_ptr = dest_entry_ptr, .before = before, }; ASSERT((ssize_t)pe_offset_new >= 0); struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr_new, .pe_offset = (ssize_t)pe_offset_new, }; uint64_t next_offset; uint64_t prev_offset; /* remove element from user list */ redo_index = list_remove_single(pop, redo, redo_index, &args_remove); /* insert element to user list */ redo_index = list_insert_user(pop, redo, redo_index, &args_insert, &args_common, &next_offset, &prev_offset); /* offsets differ, move is between different list entries - set uuid */ int set_uuid = pe_offset_new != pe_offset_old ? 1 : 0; /* fill next and prev offsets of moving element using redo log */ redo_index = list_fill_entry_redo_log(pop, redo, redo_index, &args_common, next_offset, prev_offset, set_uuid); redo_last: redo_log_set_last(pop->redo, redo, redo_index - 1); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); unlock: list_mutexes_unlock(pop, head_new, head_old); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * lane_list_recovery -- (internal) recover the list section of the lane */ static int lane_list_recovery(PMEMobjpool *pop, void *data, unsigned length) { LOG(7, "list lane %p", data); struct lane_list_layout *section = data; ASSERT(sizeof(*section) <= length); redo_log_recover(pop->redo, section->redo, REDO_NUM_ENTRIES); if (section->obj_offset) { /* alloc or free recovery */ pfree(pop, §ion->obj_offset); } return 0; } /* * lane_list_check -- (internal) check consistency of lane */ static int lane_list_check(PMEMobjpool *pop, void *data, unsigned length) { LOG(3, "list lane %p", data); struct lane_list_layout *section = data; int ret = 0; if ((ret = redo_log_check(pop->redo, section->redo, REDO_NUM_ENTRIES)) != 0) { ERR("list lane: redo log check failed"); ASSERT(ret == 0 || ret == -1); return ret; } if (section->obj_offset && !OBJ_OFF_FROM_HEAP(pop, section->obj_offset)) { ERR("list lane: invalid offset 0x%" PRIx64, section->obj_offset); return -1; } return 0; } /* * lane_list_construct_rt -- (internal) construct runtime part of list section */ static void * lane_list_construct_rt(PMEMobjpool *pop) { return NULL; } /* * lane_list_destroy_rt -- (internal) destroy runtime part of list section */ static void lane_list_destroy_rt(PMEMobjpool *pop, void *rt) { /* NOP */ } /* * lane_list_boot -- global runtime init routine of list section */ static int lane_list_boot(PMEMobjpool *pop) { /* NOP */ return 0; } /* * lane_list_cleanup -- global runtime cleanup routine of list section */ static int lane_list_cleanup(PMEMobjpool *pop) { /* NOP */ return 0; } static struct section_operations list_ops = { .construct_rt = lane_list_construct_rt, .destroy_rt = lane_list_destroy_rt, .recover = lane_list_recovery, .check = lane_list_check, .boot = lane_list_boot, .cleanup = lane_list_cleanup, }; SECTION_PARM(LANE_SECTION_LIST, &list_ops);
/* * list_insert -- insert object to a single list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_insert(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); int ret; struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; dest = list_get_dest(pop, head, dest, pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)oid.off + pe_offset)); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)dest.off + pe_offset)); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = head, .before = before, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ redo_index = list_insert_user(pop, redo, redo_index, &args, &args_common, &next_offset, &prev_offset); /* fill entry of existing element using redo log */ redo_index = list_fill_entry_redo_log(pop, redo, redo_index, &args_common, next_offset, prev_offset, 1); redo_log_set_last(pop->redo, redo, redo_index - 1); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_remove_free -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, *must* be locked if not NULL * oidp - pointer to target object ID */ static void list_remove_free(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset); struct redo_log *redo = section->redo; size_t redo_index = 0; uint64_t obj_doffset = oidp->off; ASSERT((ssize_t)pe_offset >= 0); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = user_head, .entry_ptr = entry_ptr, .obj_doffset = obj_doffset }; /* remove from user list */ redo_index = list_remove_single(pop, redo, redo_index, &args); /* clear the oid */ if (OBJ_PTR_IS_VALID(pop, oidp)) redo_index = list_set_oid_redo_log(pop, redo, redo_index, oidp, 0, 1); else oidp->off = 0; redo_log_store_last(pop->redo, redo, redo_index, sec_off_off, obj_doffset); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); /* * Don't need to fill next and prev offsets of removing element * because the element is freed. */ pfree(pop, §ion->obj_offset); lane_release(pop); } /* * list_remove_free_user -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * oidp - pointer to target object ID */ int list_remove_free_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } list_remove_free(pop, pe_offset, user_head, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); return 0; }
/* * list_insert_new -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, must be locked if not NULL * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ static int list_insert_new(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); int ret; struct lane_section *lane_section; #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset); if (constructor) { if ((ret = pmalloc_construct(pop, §ion->obj_offset, size, constructor, arg, 0, 0, 0))) { ERR("!pmalloc_construct"); goto err_pmalloc; } } else { ret = pmalloc(pop, §ion->obj_offset, size, 0, 0); if (ret) { ERR("!pmalloc"); goto err_pmalloc; } } uint64_t obj_doffset = section->obj_offset; ASSERT((ssize_t)pe_offset >= 0); dest = list_get_dest(pop, user_head, dest, (ssize_t)pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = user_head, .before = before, }; struct list_args_common args_common = { .obj_doffset = obj_doffset, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ redo_index = list_insert_user(pop, redo, redo_index, &args, &args_common, &next_offset, &prev_offset); /* don't need to use redo log for filling new element */ list_fill_entry_persist(pop, entry_ptr, next_offset, prev_offset); if (oidp != NULL) { if (OBJ_PTR_IS_VALID(pop, oidp)) { redo_index = list_set_oid_redo_log(pop, redo, redo_index, oidp, obj_doffset, 0); } else { oidp->off = obj_doffset; oidp->pool_uuid_lo = pop->uuid_lo; } } /* clear the obj_offset in lane section */ redo_log_store_last(pop->redo, redo, redo_index, sec_off_off, 0); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); ret = 0; err_pmalloc: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_insert_new_user -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ int list_insert_new_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } ret = list_insert_new(pop, pe_offset, user_head, dest, before, size, constructor, arg, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); ASSERT(ret == 0 || ret == -1); return ret; }
/* * obj_realloc_common -- (internal) common routine for resizing * existing objects */ static int obj_realloc_common(PMEMobjpool *pop, struct object_store *store, PMEMoid *oidp, size_t size, type_num_t type_num, int zero_init) { /* if OID is NULL just allocate memory */ if (OBJ_OID_IS_NULL(*oidp)) { /* if size is 0 - do nothing */ if (size == 0) return 0; return obj_alloc_construct(pop, oidp, size, type_num, zero_init, NULL, NULL); } if (size > PMEMOBJ_MAX_ALLOC_SIZE) { ERR("requested size too large"); errno = ENOMEM; return -1; } /* if size is 0 just free */ if (size == 0) { obj_free(pop, oidp); return 0; } struct carg_realloc carg; carg.ptr = OBJ_OFF_TO_PTR(pop, oidp->off); carg.new_size = size; carg.old_size = pmemobj_alloc_usable_size(*oidp); carg.user_type = type_num; carg.constructor = NULL; carg.arg = NULL; carg.zero_init = zero_init; struct oob_header *pobj = OOB_HEADER_FROM_OID(pop, *oidp); type_num_t user_type_old = pobj->data.user_type; /* callers should have checked this */ ASSERT(type_num < PMEMOBJ_NUM_OID_TYPES); ASSERT(user_type_old < PMEMOBJ_NUM_OID_TYPES); struct list_head *lhead_old = &store->bytype[user_type_old].head; if (type_num == user_type_old) { int ret = list_realloc(pop, lhead_old, 0, NULL, size, constructor_realloc, &carg, 0, 0, oidp); if (ret) LOG(2, "list_realloc failed"); /* oidp could be different, so we need to get the ptr again */ VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &OOB_HEADER_FROM_OID(pop, *oidp)->data.padding, sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding)); return ret; } else { struct list_head *lhead_new = &store->bytype[type_num].head; /* * Header padding doubles as a red zone to check for header * overwrites. Disable it temporarily so we can modify the type * number. */ VALGRIND_DO_MAKE_MEM_DEFINED(pop, &OOB_HEADER_FROM_OID(pop, *oidp)->data.padding, sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding)); /* * Redo log updates 8 byte entries, so we have to prepare * full 8-byte value even if we want to update smaller field * (here: user_type). */ struct oob_header_data d = pobj->data; d.user_type = type_num; uint64_t data_offset = OOB_OFFSET_OF(*oidp, data); int ret = list_realloc_move(pop, lhead_old, lhead_new, 0, NULL, size, constructor_realloc, &carg, data_offset, *((uint64_t *)&d), oidp); if (ret) LOG(2, "list_realloc_move failed"); /* oidp could be different, so we need to get the ptr again */ VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &OOB_HEADER_FROM_OID(pop, *oidp)->data.padding, sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding)); return ret; } }
/* * list_remove -- remove object from list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * oid - target object ID */ int list_remove(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); int ret; struct lane *lane; lane_hold(pop, &lane); if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } struct operation_context *ctx = lane->external; operation_start(ctx); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + (size_t)pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = head, .entry_ptr = entry_ptr, .obj_doffset = oid.off, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; /* remove element from user list */ list_remove_single(pop, ctx, &args); /* clear next and prev offsets in removing element using redo log */ list_fill_entry_redo_log(pop, ctx, &args_common, 0, 0, 0); operation_finish(ctx); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_move -- move object between two lists * * pop - pmemobj handle * pe_offset_old - offset to old list entry relative to user data * head_old - old list head * pe_offset_new - offset to new list entry relative to user data * head_new - new list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_move(PMEMobjpool *pop, size_t pe_offset_old, struct list_head *head_old, size_t pe_offset_new, struct list_head *head_new, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head_old, NULL); ASSERTne(head_new, NULL); int ret; struct lane *lane; lane_hold(pop, &lane); /* * Grab locks in specified order to avoid dead-locks. * * XXX performance improvement: initialize oob locks at pool opening */ if ((ret = list_mutexes_lock(pop, head_new, head_old))) { errno = ret; LOG(2, "list_mutexes_lock failed"); ret = -1; goto err; } struct operation_context *ctx = lane->external; operation_start(ctx); dest = list_get_dest(pop, head_new, dest, (ssize_t)pe_offset_new, before); struct list_entry *entry_ptr_old = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + pe_offset_old); struct list_entry *entry_ptr_new = (struct list_entry *)OBJ_OFF_TO_PTR(pop, oid.off + pe_offset_new); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset_new); if (head_old == head_new) { /* moving within the same list */ if (dest.off == oid.off) goto unlock; if (before && dest_entry_ptr->pe_prev.off == oid.off) { if (head_old->pe_first.off != dest.off) goto unlock; list_update_head(pop, ctx, head_old, oid.off); goto redo_last; } if (!before && dest_entry_ptr->pe_next.off == oid.off) { if (head_old->pe_first.off != oid.off) goto unlock; list_update_head(pop, ctx, head_old, entry_ptr_old->pe_next.off); goto redo_last; } } ASSERT((ssize_t)pe_offset_old >= 0); struct list_args_remove args_remove = { .pe_offset = (ssize_t)pe_offset_old, .head = head_old, .entry_ptr = entry_ptr_old, .obj_doffset = oid.off, }; struct list_args_insert args_insert = { .head = head_new, .dest = dest, .dest_entry_ptr = dest_entry_ptr, .before = before, }; ASSERT((ssize_t)pe_offset_new >= 0); struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr_new, .pe_offset = (ssize_t)pe_offset_new, }; uint64_t next_offset; uint64_t prev_offset; /* remove element from user list */ list_remove_single(pop, ctx, &args_remove); /* insert element to user list */ list_insert_user(pop, ctx, &args_insert, &args_common, &next_offset, &prev_offset); /* offsets differ, move is between different list entries - set uuid */ int set_uuid = pe_offset_new != pe_offset_old ? 1 : 0; /* fill next and prev offsets of moving element using redo log */ list_fill_entry_redo_log(pop, ctx, &args_common, next_offset, prev_offset, set_uuid); redo_last: unlock: operation_finish(ctx); list_mutexes_unlock(pop, head_new, head_old); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; }
/* * list_insert -- insert object to a single list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_insert(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); struct lane *lane; lane_hold(pop, &lane); int ret; if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } struct operation_context *ctx = lane->external; operation_start(ctx); dest = list_get_dest(pop, head, dest, pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)oid.off + pe_offset)); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)dest.off + pe_offset)); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = head, .before = before, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ list_insert_user(pop, ctx, &args, &args_common, &next_offset, &prev_offset); /* fill entry of existing element using redo log */ list_fill_entry_redo_log(pop, ctx, &args_common, next_offset, prev_offset, 1); operation_finish(ctx); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_remove_free -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, *must* be locked if not NULL * oidp - pointer to target object ID */ static void list_remove_free(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane *lane; lane_hold(pop, &lane); struct operation_context *ctx = lane->external; operation_start(ctx); struct pobj_action deferred; palloc_defer_free(&pop->heap, oidp->off, &deferred); uint64_t obj_doffset = oidp->off; ASSERT((ssize_t)pe_offset >= 0); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = user_head, .entry_ptr = entry_ptr, .obj_doffset = obj_doffset }; /* remove from user list */ list_remove_single(pop, ctx, &args); /* clear the oid */ if (OBJ_PTR_IS_VALID(pop, oidp)) list_set_oid_redo_log(pop, ctx, oidp, 0, 1); else oidp->off = 0; palloc_publish(&pop->heap, &deferred, 1, ctx); lane_release(pop); } /* * list_remove_free_user -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * oidp - pointer to target object ID */ int list_remove_free_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } list_remove_free(pop, pe_offset, user_head, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); return 0; }
/* * list_insert_new -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, must be locked if not NULL * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ static int list_insert_new(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); int ret; #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane *lane; lane_hold(pop, &lane); struct pobj_action reserved; if (palloc_reserve(&pop->heap, size, constructor, arg, type_num, 0, 0, &reserved) != 0) { ERR("!palloc_reserve"); ret = -1; goto err_pmalloc; } uint64_t obj_doffset = reserved.heap.offset; struct operation_context *ctx = lane->external; operation_start(ctx); ASSERT((ssize_t)pe_offset >= 0); dest = list_get_dest(pop, user_head, dest, (ssize_t)pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = user_head, .before = before, }; struct list_args_common args_common = { .obj_doffset = obj_doffset, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ list_insert_user(pop, ctx, &args, &args_common, &next_offset, &prev_offset); /* don't need to use redo log for filling new element */ list_fill_entry_persist(pop, entry_ptr, next_offset, prev_offset); if (oidp != NULL) { if (OBJ_PTR_IS_VALID(pop, oidp)) { list_set_oid_redo_log(pop, ctx, oidp, obj_doffset, 0); } else { oidp->off = obj_doffset; oidp->pool_uuid_lo = pop->uuid_lo; } } palloc_publish(&pop->heap, &reserved, 1, ctx); ret = 0; err_pmalloc: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_insert_new_user -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ int list_insert_new_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } ret = list_insert_new(pop, pe_offset, user_head, dest, before, size, type_num, constructor, arg, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); ASSERT(ret == 0 || ret == -1); return ret; }