/* * operation_add_entry -- adds new entry to the current operation with * entry type autodetected based on the memory location */ void operation_add_entry(struct operation_context *ctx, void *ptr, uint64_t value, enum operation_type type) { operation_add_typed_entry(ctx, ptr, value, type, OBJ_PTR_IS_VALID(ctx->pop, ptr) ? ENTRY_PERSISTENT : ENTRY_TRANSIENT); }
/* * list_set_oid_redo_log -- (internal) set PMEMoid value using redo log */ static size_t list_set_oid_redo_log(PMEMobjpool *pop, struct operation_context *ctx, PMEMoid *oidp, uint64_t obj_doffset, int oidp_inited) { ASSERT(OBJ_PTR_IS_VALID(pop, oidp)); if (!oidp_inited || oidp->pool_uuid_lo != pop->uuid_lo) { if (oidp_inited) ASSERTeq(oidp->pool_uuid_lo, 0); operation_add_entry(ctx, &oidp->pool_uuid_lo, pop->uuid_lo, ULOG_OPERATION_SET); } operation_add_entry(ctx, &oidp->off, obj_doffset, ULOG_OPERATION_SET); return 0; }
/* * list_set_oid_redo_log -- (internal) set PMEMoid value using redo log */ static size_t list_set_oid_redo_log(PMEMobjpool *pop, struct redo_log *redo, size_t redo_index, PMEMoid *oidp, uint64_t obj_doffset, int oidp_inited) { ASSERT(OBJ_PTR_IS_VALID(pop, oidp)); if (!oidp_inited || oidp->pool_uuid_lo != pop->uuid_lo) { if (oidp_inited) ASSERTeq(oidp->pool_uuid_lo, 0); uint64_t oid_uuid_off = OBJ_PTR_TO_OFF(pop, &oidp->pool_uuid_lo); redo_log_store(pop->redo, redo, redo_index, oid_uuid_off, pop->uuid_lo); redo_index += 1; } uint64_t oid_off_off = OBJ_PTR_TO_OFF(pop, &oidp->off); redo_log_store(pop->redo, redo, redo_index, oid_off_off, obj_doffset); return redo_index + 1; }
/* * list_insert -- insert object to a single list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_insert(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); int ret; struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; dest = list_get_dest(pop, head, dest, pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)oid.off + pe_offset)); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)dest.off + pe_offset)); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = head, .before = before, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ redo_index = list_insert_user(pop, redo, redo_index, &args, &args_common, &next_offset, &prev_offset); /* fill entry of existing element using redo log */ redo_index = list_fill_entry_redo_log(pop, redo, redo_index, &args_common, next_offset, prev_offset, 1); redo_log_set_last(pop->redo, redo, redo_index - 1); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_remove_free -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, *must* be locked if not NULL * oidp - pointer to target object ID */ static void list_remove_free(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane_section *lane_section; lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset); struct redo_log *redo = section->redo; size_t redo_index = 0; uint64_t obj_doffset = oidp->off; ASSERT((ssize_t)pe_offset >= 0); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = user_head, .entry_ptr = entry_ptr, .obj_doffset = obj_doffset }; /* remove from user list */ redo_index = list_remove_single(pop, redo, redo_index, &args); /* clear the oid */ if (OBJ_PTR_IS_VALID(pop, oidp)) redo_index = list_set_oid_redo_log(pop, redo, redo_index, oidp, 0, 1); else oidp->off = 0; redo_log_store_last(pop->redo, redo, redo_index, sec_off_off, obj_doffset); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); /* * Don't need to fill next and prev offsets of removing element * because the element is freed. */ pfree(pop, §ion->obj_offset); lane_release(pop); } /* * list_remove_free_user -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * oidp - pointer to target object ID */ int list_remove_free_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } list_remove_free(pop, pe_offset, user_head, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); return 0; }
/* * list_insert_new -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, must be locked if not NULL * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ static int list_insert_new(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); int ret; struct lane_section *lane_section; #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif lane_hold(pop, &lane_section, LANE_SECTION_LIST); ASSERTne(lane_section, NULL); ASSERTne(lane_section->layout, NULL); struct lane_list_layout *section = (struct lane_list_layout *)lane_section->layout; struct redo_log *redo = section->redo; size_t redo_index = 0; uint64_t sec_off_off = OBJ_PTR_TO_OFF(pop, §ion->obj_offset); if (constructor) { if ((ret = pmalloc_construct(pop, §ion->obj_offset, size, constructor, arg, 0, 0, 0))) { ERR("!pmalloc_construct"); goto err_pmalloc; } } else { ret = pmalloc(pop, §ion->obj_offset, size, 0, 0); if (ret) { ERR("!pmalloc"); goto err_pmalloc; } } uint64_t obj_doffset = section->obj_offset; ASSERT((ssize_t)pe_offset >= 0); dest = list_get_dest(pop, user_head, dest, (ssize_t)pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = user_head, .before = before, }; struct list_args_common args_common = { .obj_doffset = obj_doffset, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ redo_index = list_insert_user(pop, redo, redo_index, &args, &args_common, &next_offset, &prev_offset); /* don't need to use redo log for filling new element */ list_fill_entry_persist(pop, entry_ptr, next_offset, prev_offset); if (oidp != NULL) { if (OBJ_PTR_IS_VALID(pop, oidp)) { redo_index = list_set_oid_redo_log(pop, redo, redo_index, oidp, obj_doffset, 0); } else { oidp->off = obj_doffset; oidp->pool_uuid_lo = pop->uuid_lo; } } /* clear the obj_offset in lane section */ redo_log_store_last(pop->redo, redo, redo_index, sec_off_off, 0); redo_log_process(pop->redo, redo, REDO_NUM_ENTRIES); ret = 0; err_pmalloc: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_insert_new_user -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ int list_insert_new_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } ret = list_insert_new(pop, pe_offset, user_head, dest, before, size, constructor, arg, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); ASSERT(ret == 0 || ret == -1); return ret; }
/* * list_insert -- insert object to a single list * * pop - pmemobj handle * pe_offset - offset to list entry on user list relative to user data * head - list head * dest - destination object ID * before - before/after destination * oid - target object ID */ int list_insert(PMEMobjpool *pop, ssize_t pe_offset, struct list_head *head, PMEMoid dest, int before, PMEMoid oid) { LOG(3, NULL); ASSERTne(head, NULL); struct lane *lane; lane_hold(pop, &lane); int ret; if ((ret = pmemobj_mutex_lock(pop, &head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); ret = -1; goto err; } struct operation_context *ctx = lane->external; operation_start(ctx); dest = list_get_dest(pop, head, dest, pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)oid.off + pe_offset)); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, (uintptr_t)((ssize_t)dest.off + pe_offset)); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = head, .before = before, }; struct list_args_common args_common = { .obj_doffset = oid.off, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ list_insert_user(pop, ctx, &args, &args_common, &next_offset, &prev_offset); /* fill entry of existing element using redo log */ list_fill_entry_redo_log(pop, ctx, &args_common, next_offset, prev_offset, 1); operation_finish(ctx); pmemobj_mutex_unlock_nofail(pop, &head->lock); err: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_remove_free -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, *must* be locked if not NULL * oidp - pointer to target object ID */ static void list_remove_free(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane *lane; lane_hold(pop, &lane); struct operation_context *ctx = lane->external; operation_start(ctx); struct pobj_action deferred; palloc_defer_free(&pop->heap, oidp->off, &deferred); uint64_t obj_doffset = oidp->off; ASSERT((ssize_t)pe_offset >= 0); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_args_remove args = { .pe_offset = (ssize_t)pe_offset, .head = user_head, .entry_ptr = entry_ptr, .obj_doffset = obj_doffset }; /* remove from user list */ list_remove_single(pop, ctx, &args); /* clear the oid */ if (OBJ_PTR_IS_VALID(pop, oidp)) list_set_oid_redo_log(pop, ctx, oidp, 0, 1); else oidp->off = 0; palloc_publish(&pop->heap, &deferred, 1, ctx); lane_release(pop); } /* * list_remove_free_user -- remove from two lists and free an object * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * oidp - pointer to target object ID */ int list_remove_free_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid *oidp) { LOG(3, NULL); int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } list_remove_free(pop, pe_offset, user_head, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); return 0; }
/* * list_insert_new -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * pe_offset - offset to list entry on user list relative to user data * user_head - user list head, must be locked if not NULL * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ static int list_insert_new(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { LOG(3, NULL); ASSERT(user_head != NULL); int ret; #ifdef DEBUG int r = pmemobj_mutex_assert_locked(pop, &user_head->lock); ASSERTeq(r, 0); #endif struct lane *lane; lane_hold(pop, &lane); struct pobj_action reserved; if (palloc_reserve(&pop->heap, size, constructor, arg, type_num, 0, 0, &reserved) != 0) { ERR("!palloc_reserve"); ret = -1; goto err_pmalloc; } uint64_t obj_doffset = reserved.heap.offset; struct operation_context *ctx = lane->external; operation_start(ctx); ASSERT((ssize_t)pe_offset >= 0); dest = list_get_dest(pop, user_head, dest, (ssize_t)pe_offset, before); struct list_entry *entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, obj_doffset + pe_offset); struct list_entry *dest_entry_ptr = (struct list_entry *)OBJ_OFF_TO_PTR(pop, dest.off + pe_offset); struct list_args_insert args = { .dest = dest, .dest_entry_ptr = dest_entry_ptr, .head = user_head, .before = before, }; struct list_args_common args_common = { .obj_doffset = obj_doffset, .entry_ptr = entry_ptr, .pe_offset = (ssize_t)pe_offset, }; uint64_t next_offset; uint64_t prev_offset; /* insert element to user list */ list_insert_user(pop, ctx, &args, &args_common, &next_offset, &prev_offset); /* don't need to use redo log for filling new element */ list_fill_entry_persist(pop, entry_ptr, next_offset, prev_offset); if (oidp != NULL) { if (OBJ_PTR_IS_VALID(pop, oidp)) { list_set_oid_redo_log(pop, ctx, oidp, obj_doffset, 0); } else { oidp->off = obj_doffset; oidp->pool_uuid_lo = pop->uuid_lo; } } palloc_publish(&pop->heap, &reserved, 1, ctx); ret = 0; err_pmalloc: lane_release(pop); ASSERT(ret == 0 || ret == -1); return ret; } /* * list_insert_new_user -- allocate and insert element to oob and user lists * * pop - pmemobj pool handle * oob_head - oob list head * pe_offset - offset to list entry on user list relative to user data * user_head - user list head * dest - destination on user list * before - insert before/after destination on user list * size - size of allocation, will be increased by OBJ_OOB_SIZE * constructor - object's constructor * arg - argument for object's constructor * oidp - pointer to target object ID */ int list_insert_new_user(PMEMobjpool *pop, size_t pe_offset, struct list_head *user_head, PMEMoid dest, int before, size_t size, uint64_t type_num, int (*constructor)(void *ctx, void *ptr, size_t usable_size, void *arg), void *arg, PMEMoid *oidp) { int ret; if ((ret = pmemobj_mutex_lock(pop, &user_head->lock))) { errno = ret; LOG(2, "pmemobj_mutex_lock failed"); return -1; } ret = list_insert_new(pop, pe_offset, user_head, dest, before, size, type_num, constructor, arg, oidp); pmemobj_mutex_unlock_nofail(pop, &user_head->lock); ASSERT(ret == 0 || ret == -1); return ret; }