/*
* huge_vg_init -- initalizes chunk metadata in memcheck state
*/
static void
huge_vg_init(const struct memory_block *m, int objects,
object_callback cb, void *arg)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m);
struct chunk *chunk = heap_get_chunk(m->heap, m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
/*
* Mark unused chunk headers as not accessible.
*/
VALGRIND_DO_MAKE_MEM_NOACCESS(
&z->chunk_headers[m->chunk_id + 1],
(m->size_idx - 1) *
sizeof(struct chunk_header));
size_t size = block_get_real_size(m);
VALGRIND_DO_MAKE_MEM_NOACCESS(chunk, size);
if (objects && huge_get_state(m) == MEMBLOCK_ALLOCATED) {
if (cb(m, arg) != 0)
FATAL("failed to initialize valgrind state");
}
}
/*
* pmemobj_vg_register_object -- (internal) notify Valgrind about object
*/
static void
pmemobj_vg_register_object(struct pmemobjpool *pop, PMEMoid oid, int is_root)
{
LOG(4, "pop %p oid.off 0x%016jx is_root %d", pop, oid.off, is_root);
void *addr = pmemobj_direct(oid);
size_t sz;
if (is_root)
sz = pmemobj_root_size(pop);
else
sz = pmemobj_alloc_usable_size(oid);
size_t headers = sizeof (struct allocation_header) + OBJ_OOB_SIZE;
VALGRIND_DO_MEMPOOL_ALLOC(pop, addr, sz);
VALGRIND_DO_MAKE_MEM_DEFINED(pop, addr - headers, sz + headers);
struct oob_header *oob = OOB_HEADER_FROM_PTR(addr);
if (!is_root)
/* no one should touch it */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &oob->size,
sizeof (oob->size));
/* no one should touch it */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop, &oob->data.padding,
sizeof (oob->data.padding));
}
/*
* memblock_header_compact_reinit --
* (internal) reinitializes a compact header after a heap restart
*/
static void
memblock_header_compact_reinit(const struct memory_block *m)
{
struct allocation_header_compact *hdr = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
}
/*
* rpmem_fip_read -- perform read operation
*/
int
rpmem_fip_read(struct rpmem_fip *fip, void *buff, size_t len, size_t off)
{
RPMEM_ASSERT(!rpmem_fip_lane_busy(&fip->rd_lane.lane));
int ret;
size_t rd = 0;
uint8_t *cbuff = buff;
while (rd < len) {
rpmem_fip_lane_begin(&fip->rd_lane.lane, FI_READ);
size_t rd_len = len - rd < RPMEM_RD_BUFF_SIZE ?
len - rd : RPMEM_RD_BUFF_SIZE;
size_t rd_off = off + rd;
uint64_t raddr = fip->raddr + rd_off;
ret = rpmem_fip_readmsg(fip->ep, &fip->rd_lane.read,
fip->rd_buff, rd_len, raddr);
VALGRIND_DO_MAKE_MEM_DEFINED(fip->rd_buff, rd_len);
ret = rpmem_fip_lane_wait(&fip->rd_lane.lane, FI_READ);
if (ret) {
ERR("error when processing read request");
errno = ret;
return -1;
}
memcpy(&cbuff[rd], fip->rd_buff, rd_len);
rd += rd_len;
}
return 0;
}
/*
* palloc_vg_register_object -- registers object in Valgrind
*/
void
palloc_vg_register_object(struct palloc_heap *heap, void *addr, size_t size)
{
size_t headers = sizeof(struct allocation_header) + PALLOC_DATA_OFF;
VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, addr, size);
VALGRIND_DO_MAKE_MEM_DEFINED((char *)addr - headers, size + headers);
}
/*
* memblock_header_legacy_reinit --
* (internal) reinitializes a legacy header after a heap restart
*/
static void
memblock_header_legacy_reinit(const struct memory_block *m)
{
struct allocation_header_legacy *hdr = m->m_ops->get_real_data(m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
/* unused fields of the legacy headers are used as a red zone */
VALGRIND_DO_MAKE_MEM_NOACCESS(hdr->unused, sizeof(hdr->unused));
}
/*
* palloc_vg_register_alloc -- (internal) registers allocation header
* in Valgrind
*/
static int
palloc_vg_register_alloc(const struct memory_block *m, void *arg)
{
struct palloc_heap *heap = arg;
m->m_ops->reinit_header(m);
void *uptr = m->m_ops->get_user_data(m);
size_t usize = m->m_ops->get_user_size(m);
VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, uptr, usize);
VALGRIND_DO_MAKE_MEM_DEFINED(uptr, usize);
return 0;
}
/*
* run_vg_init -- initalizes run metadata in memcheck state
*/
static void
run_vg_init(const struct memory_block *m, int objects,
object_callback cb, void *arg)
{
struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id);
struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m);
struct chunk_run *run = heap_get_chunk_run(m->heap, m);
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
/* set the run metadata as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run, RUN_BASE_METADATA_SIZE);
struct run_bitmap b;
run_get_bitmap(m, &b);
/*
* Mark run data headers as defined.
*/
for (unsigned j = 1; j < m->size_idx; ++j) {
struct chunk_header *data_hdr =
&z->chunk_headers[m->chunk_id + j];
VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr,
sizeof(struct chunk_header));
ASSERTeq(data_hdr->type, CHUNK_TYPE_RUN_DATA);
}
VALGRIND_DO_MAKE_MEM_NOACCESS(run, SIZEOF_RUN(run, m->size_idx));
/* set the run bitmap as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run, b.size + RUN_BASE_METADATA_SIZE);
if (objects) {
if (run_iterate_used(m, cb, arg) != 0)
FATAL("failed to initialize valgrind state");
}
}
/*
* container_ravl_insert_block -- (internal) inserts a new memory block
* into the container
*/
static int
container_ravl_insert_block(struct block_container *bc,
const struct memory_block *m)
{
struct block_container_ravl *c =
(struct block_container_ravl *)bc;
struct memory_block *e = m->m_ops->get_user_data(m);
VALGRIND_DO_MAKE_MEM_DEFINED(e, sizeof(*e));
VALGRIND_ADD_TO_TX(e, sizeof(*e));
*e = *m;
VALGRIND_SET_CLEAN(e, sizeof(*e));
VALGRIND_REMOVE_FROM_TX(e, sizeof(*e));
return ravl_insert(c->tree, e);
}
/*
* heap_vg_open_chunk -- (internal) notifies Valgrind about chunk layout
*/
static void
heap_vg_open_chunk(struct palloc_heap *heap,
object_callback cb, void *arg, int objects,
struct memory_block *m)
{
struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id);
void *chunk = &z->chunks[m->chunk_id];
memblock_rebuild_state(heap, m);
if (m->type == MEMORY_BLOCK_RUN) {
struct chunk_run *run = chunk;
ASSERTne(m->size_idx, 0);
VALGRIND_DO_MAKE_MEM_NOACCESS(run,
SIZEOF_RUN(run, m->size_idx));
/* set the run metadata as defined */
VALGRIND_DO_MAKE_MEM_DEFINED(run,
sizeof(*run) - sizeof(run->data));
if (objects) {
int ret = heap_run_foreach_object(heap, cb, arg, m,
alloc_class_get_create_by_unit_size(
heap->rt->alloc_classes,
m->m_ops->block_size(m)));
ASSERTeq(ret, 0);
}
} else {
size_t size = m->m_ops->get_real_size(m);
VALGRIND_DO_MAKE_MEM_NOACCESS(chunk, size);
if (objects && m->m_ops->get_state(m) == MEMBLOCK_ALLOCATED) {
int ret = cb(m, arg);
ASSERTeq(ret, 0);
}
}
}
/*
* rpmem_fip_process_gpspm -- (internal) process completion queue entry for
* GPSPM
*/
static int
rpmem_fip_process_gpspm(struct rpmem_fip *fip, void *context, uint64_t flags)
{
if (flags & FI_RECV) {
/* RECV completion */
struct rpmem_fip_msg *resp = context;
struct rpmem_msg_persist_resp *msg_resp =
rpmem_fip_msg_get_pres(resp);
VALGRIND_DO_MAKE_MEM_DEFINED(msg_resp, sizeof(*msg_resp));
if (unlikely(msg_resp->lane >= fip->nlanes)) {
RPMEM_LOG(ERR, "lane number received (%lu) is greater "
"than maximum lane number (%u)",
msg_resp->lane, fip->nlanes - 1);
return -1;
}
struct rpmem_fip_lane *lanep =
&fip->lanes.gpspm[msg_resp->lane].lane;
/* post RECV buffer immediately */
int ret = rpmem_fip_gpspm_post_resp(fip, resp);
if (unlikely(ret))
RPMEM_FI_ERR((int)ret, "MSG send");
rpmem_fip_lane_sigret(lanep, flags, ret);
return ret;
}
struct rpmem_fip_lane *lanep = context;
/* SEND completion */
rpmem_fip_lane_signal(lanep, flags);
return 0;
}
/*
* heap_vg_open -- notifies Valgrind about heap layout
*/
void
heap_vg_open(struct palloc_heap *heap, object_callback cb,
void *arg, int objects)
{
ASSERTne(cb, NULL);
VALGRIND_DO_MAKE_MEM_UNDEFINED(heap->layout, heap->size);
struct heap_layout *layout = heap->layout;
VALGRIND_DO_MAKE_MEM_DEFINED(&layout->header, sizeof(layout->header));
unsigned zones = heap_max_zone(heap->size);
struct memory_block m = MEMORY_BLOCK_NONE;
for (unsigned i = 0; i < zones; ++i) {
struct zone *z = ZID_TO_ZONE(layout, i);
uint32_t chunks;
m.zone_id = i;
m.chunk_id = 0;
VALGRIND_DO_MAKE_MEM_DEFINED(&z->header, sizeof(z->header));
if (z->header.magic != ZONE_HEADER_MAGIC)
continue;
chunks = z->header.size_idx;
for (uint32_t c = 0; c < chunks; ) {
struct chunk_header *hdr = &z->chunk_headers[c];
m.chunk_id = c;
VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr));
m.size_idx = hdr->size_idx;
heap_vg_open_chunk(heap, cb, arg, objects, &m);
m.block_off = 0;
ASSERT(hdr->size_idx > 0);
if (hdr->type == CHUNK_TYPE_RUN) {
/*
* Mark run data headers as defined.
*/
for (unsigned j = 1; j < hdr->size_idx; ++j) {
struct chunk_header *data_hdr =
&z->chunk_headers[c + j];
VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr,
sizeof(struct chunk_header));
ASSERTeq(data_hdr->type,
CHUNK_TYPE_RUN_DATA);
}
} else {
/*
* Mark unused chunk headers as not accessible.
*/
VALGRIND_DO_MAKE_MEM_NOACCESS(
&z->chunk_headers[c + 1],
(hdr->size_idx - 1) *
sizeof(struct chunk_header));
}
c += hdr->size_idx;
}
/* mark all unused chunk headers after last as not accessible */
VALGRIND_DO_MAKE_MEM_NOACCESS(&z->chunk_headers[chunks],
(MAX_CHUNK - chunks) * sizeof(struct chunk_header));
}
}
/*
* obj_realloc_common -- (internal) common routine for resizing
* existing objects
*/
static int
obj_realloc_common(PMEMobjpool *pop, struct object_store *store,
PMEMoid *oidp, size_t size, type_num_t type_num, int zero_init)
{
/* if OID is NULL just allocate memory */
if (OBJ_OID_IS_NULL(*oidp)) {
/* if size is 0 - do nothing */
if (size == 0)
return 0;
return obj_alloc_construct(pop, oidp, size, type_num,
zero_init, NULL, NULL);
}
if (size > PMEMOBJ_MAX_ALLOC_SIZE) {
ERR("requested size too large");
errno = ENOMEM;
return -1;
}
/* if size is 0 just free */
if (size == 0) {
obj_free(pop, oidp);
return 0;
}
struct carg_realloc carg;
carg.ptr = OBJ_OFF_TO_PTR(pop, oidp->off);
carg.new_size = size;
carg.old_size = pmemobj_alloc_usable_size(*oidp);
carg.user_type = type_num;
carg.constructor = NULL;
carg.arg = NULL;
carg.zero_init = zero_init;
struct oob_header *pobj = OOB_HEADER_FROM_OID(pop, *oidp);
type_num_t user_type_old = pobj->data.user_type;
/* callers should have checked this */
ASSERT(type_num < PMEMOBJ_NUM_OID_TYPES);
ASSERT(user_type_old < PMEMOBJ_NUM_OID_TYPES);
struct list_head *lhead_old = &store->bytype[user_type_old].head;
if (type_num == user_type_old) {
int ret = list_realloc(pop, lhead_old, 0, NULL, size,
constructor_realloc, &carg, 0, 0, oidp);
if (ret)
LOG(2, "list_realloc failed");
/* oidp could be different, so we need to get the ptr again */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
return ret;
} else {
struct list_head *lhead_new = &store->bytype[type_num].head;
/*
* Header padding doubles as a red zone to check for header
* overwrites. Disable it temporarily so we can modify the type
* number.
*/
VALGRIND_DO_MAKE_MEM_DEFINED(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
/*
* Redo log updates 8 byte entries, so we have to prepare
* full 8-byte value even if we want to update smaller field
* (here: user_type).
*/
struct oob_header_data d = pobj->data;
d.user_type = type_num;
uint64_t data_offset = OOB_OFFSET_OF(*oidp, data);
int ret = list_realloc_move(pop, lhead_old, lhead_new, 0, NULL,
size, constructor_realloc, &carg, data_offset,
*((uint64_t *)&d), oidp);
if (ret)
LOG(2, "list_realloc_move failed");
/* oidp could be different, so we need to get the ptr again */
VALGRIND_DO_MAKE_MEM_NOACCESS(pop,
&OOB_HEADER_FROM_OID(pop, *oidp)->data.padding,
sizeof (OOB_HEADER_FROM_OID(pop, *oidp)->data.padding));
return ret;
}
}
