/* * heap_init -- initializes the heap * * If successful function returns zero. Otherwise an error number is returned. */ int heap_init(void *heap_start, uint64_t heap_size, struct pmem_ops *p_ops) { if (heap_size < HEAP_MIN_SIZE) return EINVAL; VALGRIND_DO_MAKE_MEM_UNDEFINED(heap_start, heap_size); struct heap_layout *layout = heap_start; heap_write_header(&layout->header, heap_size); pmemops_persist(p_ops, &layout->header, sizeof(struct heap_header)); unsigned zones = heap_max_zone(heap_size); for (unsigned i = 0; i < zones; ++i) { pmemops_memset_persist(p_ops, &ZID_TO_ZONE(layout, i)->header, 0, sizeof(struct zone_header)); pmemops_memset_persist(p_ops, &ZID_TO_ZONE(layout, i)->chunk_headers, 0, sizeof(struct chunk_header)); /* only explicitly allocated chunks should be accessible */ VALGRIND_DO_MAKE_MEM_NOACCESS( &ZID_TO_ZONE(layout, i)->chunk_headers, sizeof(struct chunk_header)); } return 0; }
/* * alloc_prep_block -- (internal) prepares a memory block for allocation * * Once the block is fully reserved and it's guaranteed that no one else will * be able to write to this memory region it is safe to write the allocation * header and call the object construction function. * * Because the memory block at this stage is only reserved in transient state * there's no need to worry about fail-safety of this method because in case * of a crash the memory will be back in the free blocks collection. */ static int alloc_prep_block(struct palloc_heap *heap, struct memory_block m, palloc_constr constructor, void *arg, uint64_t *offset_value) { void *block_data = heap_get_block_data(heap, m); void *userdatap = (char *)block_data + ALLOC_OFF; uint64_t unit_size = MEMBLOCK_OPS(AUTO, &m)-> block_size(&m, heap->layout); uint64_t real_size = unit_size * m.size_idx; ASSERT((uint64_t)block_data % ALLOC_BLOCK_SIZE == 0); ASSERT((uint64_t)userdatap % ALLOC_BLOCK_SIZE == 0); /* mark everything (including headers) as accessible */ VALGRIND_DO_MAKE_MEM_UNDEFINED(block_data, real_size); /* mark space as allocated */ VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, userdatap, real_size - ALLOC_OFF); alloc_write_header(heap, block_data, m, real_size); int ret; if (constructor != NULL && (ret = constructor(heap->base, userdatap, real_size - ALLOC_OFF, arg)) != 0) { /* * If canceled, revert the block back to the free state in vg * machinery. Because the free operation is only performed on * the user data, the allocation header is made inaccessible * in a separate call. */ VALGRIND_DO_MEMPOOL_FREE(heap->layout, userdatap); VALGRIND_DO_MAKE_MEM_NOACCESS(block_data, ALLOC_OFF); /* * During this method there are several stores to pmem that are * not immediately flushed and in case of a cancellation those * stores are no longer relevant anyway. */ VALGRIND_SET_CLEAN(block_data, ALLOC_OFF); return ret; } /* flushes both the alloc and oob headers */ pmemops_persist(&heap->p_ops, block_data, ALLOC_OFF); /* * To avoid determining the user data pointer twice this method is also * responsible for calculating the offset of the object in the pool that * will be used to set the offset destination pointer provided by the * caller. */ *offset_value = PMALLOC_PTR_TO_OFF(heap, userdatap); return 0; }
/* * persist_alloc -- (internal) performs a persistent allocation of the * memory block previously reserved by volatile bucket */ static int persist_alloc(PMEMobjpool *pop, struct lane_section *lane, struct memory_block m, uint64_t real_size, uint64_t *off, void (*constructor)(PMEMobjpool *pop, void *ptr, void *arg), void *arg, uint64_t data_off) { int err; #ifdef DEBUG if (heap_block_is_allocated(pop, m)) { ERR("heap corruption"); ASSERT(0); } #endif /* DEBUG */ uint64_t op_result = 0; void *block_data = heap_get_block_data(pop, m); void *datap = (char *)block_data + sizeof (struct allocation_header); void *userdatap = (char *)datap + data_off; ASSERT((uint64_t)block_data % _POBJ_CL_ALIGNMENT == 0); /* mark everything (including headers) as accessible */ VALGRIND_DO_MAKE_MEM_UNDEFINED(pop, block_data, real_size); /* mark space as allocated */ VALGRIND_DO_MEMPOOL_ALLOC(pop, userdatap, real_size - sizeof (struct allocation_header) - data_off); alloc_write_header(pop, block_data, m.chunk_id, m.zone_id, real_size); if (constructor != NULL) constructor(pop, userdatap, arg); if ((err = heap_lock_if_run(pop, m)) != 0) { VALGRIND_DO_MEMPOOL_FREE(pop, userdatap); return err; } void *hdr = heap_get_block_header(pop, m, HEAP_OP_ALLOC, &op_result); struct allocator_lane_section *sec = (struct allocator_lane_section *)lane->layout; redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET, pop_offset(pop, off), pop_offset(pop, datap)); redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER, pop_offset(pop, hdr), op_result); redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO); if (heap_unlock_if_run(pop, m) != 0) { ERR("Failed to release run lock"); ASSERT(0); } return err; }
/* * huge_prep_operation_hdr -- prepares the new value of a chunk header that will * be set after the operation concludes. */ static void huge_prep_operation_hdr(const struct memory_block *m, enum memblock_state op, struct operation_context *ctx) { struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m); /* * Depending on the operation that needs to be performed a new chunk * header needs to be prepared with the new chunk state. */ uint64_t val = chunk_get_chunk_hdr_value( op == MEMBLOCK_ALLOCATED ? CHUNK_TYPE_USED : CHUNK_TYPE_FREE, hdr->flags, m->size_idx); if (ctx == NULL) { util_atomic_store_explicit64((uint64_t *)hdr, val, memory_order_relaxed); pmemops_persist(&m->heap->p_ops, hdr, sizeof(*hdr)); } else { operation_add_entry(ctx, hdr, val, ULOG_OPERATION_SET); } VALGRIND_DO_MAKE_MEM_NOACCESS(hdr + 1, (hdr->size_idx - 1) * sizeof(struct chunk_header)); /* * In the case of chunks larger than one unit the footer must be * created immediately AFTER the persistent state is safely updated. */ if (m->size_idx == 1) return; struct chunk_header *footer = hdr + m->size_idx - 1; VALGRIND_DO_MAKE_MEM_UNDEFINED(footer, sizeof(*footer)); val = chunk_get_chunk_hdr_value(CHUNK_TYPE_FOOTER, 0, m->size_idx); /* * It's only safe to write the footer AFTER the persistent part of * the operation have been successfully processed because the footer * pointer might point to a currently valid persistent state * of a different chunk. * The footer entry change is updated as transient because it will * be recreated at heap boot regardless - it's just needed for runtime * operations. */ if (ctx == NULL) { util_atomic_store_explicit64((uint64_t *)footer, val, memory_order_relaxed); VALGRIND_SET_CLEAN(footer, sizeof(*footer)); } else { operation_add_typed_entry(ctx, footer, val, ULOG_OPERATION_SET, LOG_TRANSIENT); } }
/* * memblock_header_compact_write -- * (internal) writes a compact header of an object */ static void memblock_header_compact_write(const struct memory_block *m, size_t size, uint64_t extra, uint16_t flags) { struct allocation_header_compact *hdr = m->m_ops->get_real_data(m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr)); VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr)); hdr->size = size | ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT); hdr->extra = extra; VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr)); }
/* * memblock_header_compact_write -- * (internal) writes a compact header of an object */ static void memblock_header_compact_write(const struct memory_block *m, size_t size, uint64_t extra, uint16_t flags) { COMPILE_ERROR_ON(ALLOC_HDR_COMPACT_SIZE > CACHELINE_SIZE); struct { struct allocation_header_compact hdr; uint8_t padding[CACHELINE_SIZE - ALLOC_HDR_COMPACT_SIZE]; } padded; padded.hdr.size = size | ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT); padded.hdr.extra = extra; struct allocation_header_compact *hdrp = m->m_ops->get_real_data(m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp)); /* * If possible write the entire header with a single memcpy, this allows * the copy implementation to avoid a cache miss on a partial cache line * write. */ size_t hdr_size = ALLOC_HDR_COMPACT_SIZE; if ((uintptr_t)hdrp % CACHELINE_SIZE == 0 && size >= sizeof(padded)) hdr_size = sizeof(padded); VALGRIND_ADD_TO_TX(hdrp, hdr_size); pmemops_memcpy(&m->heap->p_ops, hdrp, &padded, hdr_size, PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED); VALGRIND_DO_MAKE_MEM_UNDEFINED((char *)hdrp + ALLOC_HDR_COMPACT_SIZE, hdr_size - ALLOC_HDR_COMPACT_SIZE); VALGRIND_REMOVE_FROM_TX(hdrp, hdr_size); }
/* * heap_chunk_write_footer -- writes a chunk footer */ static void heap_chunk_write_footer(struct chunk_header *hdr, uint32_t size_idx) { if (size_idx == 1) /* that would overwrite the header */ return; VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr + size_idx - 1, sizeof(*hdr)); struct chunk_header f = *hdr; f.type = CHUNK_TYPE_FOOTER; f.size_idx = size_idx; *(hdr + size_idx - 1) = f; /* no need to persist, footers are recreated in heap_populate_buckets */ VALGRIND_SET_CLEAN(hdr + size_idx - 1, sizeof(f)); }
/* * memblock_header_legacy_write -- * (internal) writes a legacy header of an object */ static void memblock_header_legacy_write(const struct memory_block *m, size_t size, uint64_t extra, uint16_t flags) { struct allocation_header_legacy *hdr = m->m_ops->get_real_data(m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr)); VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr)); hdr->size = size; hdr->type_num = extra; hdr->root_size = ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT); VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr)); /* unused fields of the legacy headers are used as a red zone */ VALGRIND_DO_MAKE_MEM_NOACCESS(hdr->unused, sizeof(hdr->unused)); }
/* * huge_prep_operation_hdr -- prepares the new value of a chunk header that will * be set after the operation concludes. */ static void huge_prep_operation_hdr(const struct memory_block *m, enum memblock_state op, struct operation_context *ctx) { struct zone *z = ZID_TO_ZONE(m->heap->layout, m->zone_id); struct chunk_header *hdr = &z->chunk_headers[m->chunk_id]; /* * Depending on the operation that needs to be performed a new chunk * header needs to be prepared with the new chunk state. */ uint64_t val = chunk_get_chunk_hdr_value( op == MEMBLOCK_ALLOCATED ? CHUNK_TYPE_USED : CHUNK_TYPE_FREE, hdr->flags, m->size_idx); operation_add_entry(ctx, hdr, val, OPERATION_SET); VALGRIND_DO_MAKE_MEM_NOACCESS(hdr + 1, (hdr->size_idx - 1) * sizeof(struct chunk_header)); /* * In the case of chunks larger than one unit the footer must be * created immediately AFTER the persistent state is safely updated. */ if (m->size_idx == 1) return; struct chunk_header *footer = hdr + m->size_idx - 1; VALGRIND_DO_MAKE_MEM_UNDEFINED(footer, sizeof(*footer)); val = chunk_get_chunk_hdr_value(CHUNK_TYPE_FOOTER, 0, m->size_idx); /* * It's only safe to write the footer AFTER the persistent part of * the operation have been successfully processed because the footer * pointer might point to a currently valid persistent state * of a different chunk. * The footer entry change is updated as transient because it will * be recreated at heap boot regardless - it's just needed for runtime * operations. */ operation_add_typed_entry(ctx, footer, val, OPERATION_SET, ENTRY_TRANSIENT); }
static void init_run_with_max_block(struct heap_layout *l, uint32_t chunk_id) { l->zone0.chunk_headers[chunk_id].size_idx = 1; l->zone0.chunk_headers[chunk_id].type = CHUNK_TYPE_RUN; l->zone0.chunk_headers[chunk_id].flags = 0; struct chunk_run *run = (struct chunk_run *) &l->zone0.chunks[chunk_id]; VALGRIND_DO_MAKE_MEM_UNDEFINED(run, sizeof(*run)); run->block_size = 1024; memset(run->bitmap, 0xFF, sizeof(run->bitmap)); /* the biggest block is 10 bits */ run->bitmap[3] = 0b1000001110111000111111110000111111000000000011111111110000000011; }
static void init_run_with_score(struct heap_layout *l, uint32_t chunk_id, int score) { l->zone0.chunk_headers[chunk_id].size_idx = 1; l->zone0.chunk_headers[chunk_id].type = CHUNK_TYPE_RUN; l->zone0.chunk_headers[chunk_id].flags = 0; struct chunk_run *run = (struct chunk_run *) &l->zone0.chunks[chunk_id]; VALGRIND_DO_MAKE_MEM_UNDEFINED(run, sizeof(*run)); run->block_size = 1024; memset(run->bitmap, 0xFF, sizeof(run->bitmap)); UT_ASSERTeq(score % 64, 0); score /= 64; for (; score > 0; --score) { run->bitmap[score] = 0; } }
/* * memblock_header_legacy_write -- * (internal) writes a legacy header of an object */ static void memblock_header_legacy_write(const struct memory_block *m, size_t size, uint64_t extra, uint16_t flags) { struct allocation_header_legacy hdr; hdr.size = size; hdr.type_num = extra; hdr.root_size = ((uint64_t)flags << ALLOC_HDR_SIZE_SHIFT); struct allocation_header_legacy *hdrp = m->m_ops->get_real_data(m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdrp, sizeof(*hdrp)); VALGRIND_ADD_TO_TX(hdrp, sizeof(*hdrp)); pmemops_memcpy(&m->heap->p_ops, hdrp, &hdr, sizeof(hdr), /* legacy header is 64 bytes in size */ PMEMOBJ_F_MEM_WC | PMEMOBJ_F_MEM_NODRAIN | PMEMOBJ_F_RELAXED); VALGRIND_REMOVE_FROM_TX(hdrp, sizeof(*hdrp)); /* unused fields of the legacy headers are used as a red zone */ VALGRIND_DO_MAKE_MEM_NOACCESS(hdrp->unused, sizeof(hdrp->unused)); }
/* * alloc_prep_block -- (internal) prepares a memory block for allocation * * Once the block is fully reserved and it's guaranteed that no one else will * be able to write to this memory region it is safe to write the allocation * header and call the object construction function. * * Because the memory block at this stage is only reserved in transient state * there's no need to worry about fail-safety of this method because in case * of a crash the memory will be back in the free blocks collection. */ static int alloc_prep_block(struct palloc_heap *heap, const struct memory_block *m, palloc_constr constructor, void *arg, uint64_t extra_field, uint16_t object_flags, uint64_t *offset_value) { void *uptr = m->m_ops->get_user_data(m); size_t usize = m->m_ops->get_user_size(m); VALGRIND_DO_MEMPOOL_ALLOC(heap->layout, uptr, usize); VALGRIND_DO_MAKE_MEM_UNDEFINED(uptr, usize); VALGRIND_ANNOTATE_NEW_MEMORY(uptr, usize); int ret; if (constructor != NULL && (ret = constructor(heap->base, uptr, usize, arg)) != 0) { /* * If canceled, revert the block back to the free state in vg * machinery. */ VALGRIND_DO_MEMPOOL_FREE(heap->layout, uptr); return ret; } m->m_ops->write_header(m, extra_field, object_flags); /* * To avoid determining the user data pointer twice this method is also * responsible for calculating the offset of the object in the pool that * will be used to set the offset destination pointer provided by the * caller. */ *offset_value = HEAP_PTR_TO_OFF(heap, uptr); return 0; }
/* * prealloc_construct -- resizes an existing memory block with a constructor * * The block offset is written persistently into the off variable, but only * after the constructor function has been called. * * If successful function returns zero. Otherwise an error number is returned. */ int prealloc_construct(PMEMobjpool *pop, uint64_t *off, size_t size, void (*constructor)(PMEMobjpool *pop, void *ptr, size_t usable_size, void *arg), void *arg, uint64_t data_off) { if (size <= pmalloc_usable_size(pop, *off)) return 0; size_t sizeh = size + sizeof (struct allocation_header); int err = 0; struct allocation_header *alloc = alloc_get_header(pop, *off); struct lane_section *lane; if ((err = lane_hold(pop, &lane, LANE_SECTION_ALLOCATOR)) != 0) return err; struct bucket *b = heap_get_best_bucket(pop, alloc->size); uint32_t add_size_idx = bucket_calc_units(b, sizeh - alloc->size); uint32_t new_size_idx = bucket_calc_units(b, sizeh); uint64_t real_size = new_size_idx * bucket_unit_size(b); struct memory_block cnt = get_mblock_from_alloc(pop, b, alloc); if ((err = heap_lock_if_run(pop, cnt)) != 0) goto out_lane; struct memory_block next = {0, 0, 0, 0}; if ((err = heap_get_adjacent_free_block(pop, &next, cnt, 0)) != 0) goto out; if (next.size_idx < add_size_idx) { err = ENOMEM; goto out; } if ((err = heap_get_exact_block(pop, b, &next, add_size_idx)) != 0) goto out; struct memory_block *blocks[2] = {&cnt, &next}; uint64_t op_result; void *hdr; struct memory_block m = heap_coalesce(pop, blocks, 2, HEAP_OP_ALLOC, &hdr, &op_result); void *block_data = heap_get_block_data(pop, m); void *datap = (char *)block_data + sizeof (struct allocation_header); void *userdatap = (char *)datap + data_off; /* mark new part as accessible and undefined */ VALGRIND_DO_MAKE_MEM_UNDEFINED(pop, (char *)block_data + alloc->size, real_size - alloc->size); /* resize allocated space */ VALGRIND_DO_MEMPOOL_CHANGE(pop, userdatap, userdatap, real_size - sizeof (struct allocation_header) - data_off); if (constructor != NULL) constructor(pop, userdatap, real_size - sizeof (struct allocation_header) - data_off, arg); struct allocator_lane_section *sec = (struct allocator_lane_section *)lane->layout; redo_log_store(pop, sec->redo, ALLOC_OP_REDO_PTR_OFFSET, pop_offset(pop, &alloc->size), real_size); redo_log_store_last(pop, sec->redo, ALLOC_OP_REDO_HEADER, pop_offset(pop, hdr), op_result); redo_log_process(pop, sec->redo, MAX_ALLOC_OP_REDO); out: if (heap_unlock_if_run(pop, cnt) != 0) { ERR("Failed to release run lock"); ASSERT(0); } out_lane: if (lane_release(pop) != 0) { ERR("Failed to release the lane"); ASSERT(0); } return err; }
/* * heap_chunk_init -- (internal) writes chunk header */ static void heap_chunk_init(struct palloc_heap *heap, struct chunk_header *hdr, uint16_t type, uint32_t size_idx) { struct chunk_header nhdr = { .type = type, .flags = 0, .size_idx = size_idx }; VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr)); *hdr = nhdr; /* write the entire header (8 bytes) at once */ pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); heap_chunk_write_footer(hdr, size_idx); } /* * heap_zone_init -- (internal) writes zone's first chunk and header */ static void heap_zone_init(struct palloc_heap *heap, uint32_t zone_id) { struct zone *z = ZID_TO_ZONE(heap->layout, zone_id); uint32_t size_idx = get_zone_size_idx(zone_id, heap->rt->max_zone, heap->size); heap_chunk_init(heap, &z->chunk_headers[0], CHUNK_TYPE_FREE, size_idx); struct zone_header nhdr = { .size_idx = size_idx, .magic = ZONE_HEADER_MAGIC, }; z->header = nhdr; /* write the entire header (8 bytes) at once */ pmemops_persist(&heap->p_ops, &z->header, sizeof(z->header)); } /* * heap_run_init -- (internal) creates a run based on a chunk */ static void heap_run_init(struct palloc_heap *heap, struct bucket *b, const struct memory_block *m) { struct alloc_class *c = b->aclass; ASSERTeq(c->type, CLASS_RUN); struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id); struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id]; ASSERTne(m->size_idx, 0); size_t runsize = SIZEOF_RUN(run, m->size_idx); VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize); /* add/remove chunk_run and chunk_header to valgrind transaction */ VALGRIND_ADD_TO_TX(run, runsize); run->block_size = c->unit_size; pmemops_persist(&heap->p_ops, &run->block_size, sizeof(run->block_size)); /* set all the bits */ memset(run->bitmap, 0xFF, sizeof(run->bitmap)); unsigned nval = c->run.bitmap_nval; ASSERT(nval > 0); /* clear only the bits available for allocations from this bucket */ memset(run->bitmap, 0, sizeof(uint64_t) * (nval - 1)); run->bitmap[nval - 1] = c->run.bitmap_lastval; run->incarnation_claim = heap->run_id; VALGRIND_SET_CLEAN(&run->incarnation_claim, sizeof(run->incarnation_claim)); VALGRIND_REMOVE_FROM_TX(run, runsize); pmemops_persist(&heap->p_ops, run->bitmap, sizeof(run->bitmap)); struct chunk_header run_data_hdr; run_data_hdr.type = CHUNK_TYPE_RUN_DATA; run_data_hdr.flags = 0; struct chunk_header *data_hdr; for (unsigned i = 1; i < m->size_idx; ++i) { data_hdr = &z->chunk_headers[m->chunk_id + i]; VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr)); VALGRIND_ADD_TO_TX(data_hdr, sizeof(*data_hdr)); run_data_hdr.size_idx = i; *data_hdr = run_data_hdr; VALGRIND_REMOVE_FROM_TX(data_hdr, sizeof(*data_hdr)); } pmemops_persist(&heap->p_ops, &z->chunk_headers[m->chunk_id + 1], sizeof(struct chunk_header) * (m->size_idx - 1)); struct chunk_header *hdr = &z->chunk_headers[m->chunk_id]; ASSERT(hdr->type == CHUNK_TYPE_FREE); VALGRIND_ADD_TO_TX(hdr, sizeof(*hdr)); struct chunk_header run_hdr; run_hdr.size_idx = hdr->size_idx; run_hdr.type = CHUNK_TYPE_RUN; run_hdr.flags = header_type_to_flag[c->header_type]; *hdr = run_hdr; VALGRIND_REMOVE_FROM_TX(hdr, sizeof(*hdr)); pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); } /* * heap_run_insert -- (internal) inserts and splits a block of memory into a run */ static void heap_run_insert(struct palloc_heap *heap, struct bucket *b, const struct memory_block *m, uint32_t size_idx, uint16_t block_off) { struct alloc_class *c = b->aclass; ASSERTeq(c->type, CLASS_RUN); ASSERT(size_idx <= BITS_PER_VALUE); ASSERT(block_off + size_idx <= c->run.bitmap_nallocs); uint32_t unit_max = c->run.unit_max; struct memory_block nm = *m; nm.size_idx = unit_max - (block_off % unit_max); nm.block_off = block_off; if (nm.size_idx > size_idx) nm.size_idx = size_idx; do { bucket_insert_block(b, &nm); ASSERT(nm.size_idx <= UINT16_MAX); ASSERT(nm.block_off + nm.size_idx <= UINT16_MAX); nm.block_off = (uint16_t)(nm.block_off + (uint16_t)nm.size_idx); size_idx -= nm.size_idx; nm.size_idx = size_idx > unit_max ? unit_max : size_idx; } while (size_idx != 0); } /* * heap_process_run_metadata -- (internal) parses the run bitmap */ static uint32_t heap_process_run_metadata(struct palloc_heap *heap, struct bucket *b, const struct memory_block *m) { struct alloc_class *c = b->aclass; ASSERTeq(c->type, CLASS_RUN); uint16_t block_off = 0; uint16_t block_size_idx = 0; uint32_t inserted_blocks = 0; struct zone *z = ZID_TO_ZONE(heap->layout, m->zone_id); struct chunk_run *run = (struct chunk_run *)&z->chunks[m->chunk_id]; for (unsigned i = 0; i < c->run.bitmap_nval; ++i) { ASSERT(i < MAX_BITMAP_VALUES); uint64_t v = run->bitmap[i]; ASSERT(BITS_PER_VALUE * i <= UINT16_MAX); block_off = (uint16_t)(BITS_PER_VALUE * i); if (v == 0) { heap_run_insert(heap, b, m, BITS_PER_VALUE, block_off); inserted_blocks += BITS_PER_VALUE; continue; } else if (v == UINT64_MAX) { continue; } for (unsigned j = 0; j < BITS_PER_VALUE; ++j) { if (BIT_IS_CLR(v, j)) { block_size_idx++; } else if (block_size_idx != 0) { ASSERT(block_off >= block_size_idx); heap_run_insert(heap, b, m, block_size_idx, (uint16_t)(block_off - block_size_idx)); inserted_blocks += block_size_idx; block_size_idx = 0; } if ((block_off++) == c->run.bitmap_nallocs) { i = MAX_BITMAP_VALUES; break; } } if (block_size_idx != 0) { ASSERT(block_off >= block_size_idx); heap_run_insert(heap, b, m, block_size_idx, (uint16_t)(block_off - block_size_idx)); inserted_blocks += block_size_idx; block_size_idx = 0; } } return inserted_blocks; } /* * heap_create_run -- (internal) initializes a new run on an existing free chunk */ static void heap_create_run(struct palloc_heap *heap, struct bucket *b, struct memory_block *m) { heap_run_init(heap, b, m); memblock_rebuild_state(heap, m); heap_process_run_metadata(heap, b, m); }
/* * heap_vg_open -- notifies Valgrind about heap layout */ void heap_vg_open(struct palloc_heap *heap, object_callback cb, void *arg, int objects) { ASSERTne(cb, NULL); VALGRIND_DO_MAKE_MEM_UNDEFINED(heap->layout, heap->size); struct heap_layout *layout = heap->layout; VALGRIND_DO_MAKE_MEM_DEFINED(&layout->header, sizeof(layout->header)); unsigned zones = heap_max_zone(heap->size); struct memory_block m = MEMORY_BLOCK_NONE; for (unsigned i = 0; i < zones; ++i) { struct zone *z = ZID_TO_ZONE(layout, i); uint32_t chunks; m.zone_id = i; m.chunk_id = 0; VALGRIND_DO_MAKE_MEM_DEFINED(&z->header, sizeof(z->header)); if (z->header.magic != ZONE_HEADER_MAGIC) continue; chunks = z->header.size_idx; for (uint32_t c = 0; c < chunks; ) { struct chunk_header *hdr = &z->chunk_headers[c]; m.chunk_id = c; VALGRIND_DO_MAKE_MEM_DEFINED(hdr, sizeof(*hdr)); m.size_idx = hdr->size_idx; heap_vg_open_chunk(heap, cb, arg, objects, &m); m.block_off = 0; ASSERT(hdr->size_idx > 0); if (hdr->type == CHUNK_TYPE_RUN) { /* * Mark run data headers as defined. */ for (unsigned j = 1; j < hdr->size_idx; ++j) { struct chunk_header *data_hdr = &z->chunk_headers[c + j]; VALGRIND_DO_MAKE_MEM_DEFINED(data_hdr, sizeof(struct chunk_header)); ASSERTeq(data_hdr->type, CHUNK_TYPE_RUN_DATA); } } else { /* * Mark unused chunk headers as not accessible. */ VALGRIND_DO_MAKE_MEM_NOACCESS( &z->chunk_headers[c + 1], (hdr->size_idx - 1) * sizeof(struct chunk_header)); } c += hdr->size_idx; } /* mark all unused chunk headers after last as not accessible */ VALGRIND_DO_MAKE_MEM_NOACCESS(&z->chunk_headers[chunks], (MAX_CHUNK - chunks) * sizeof(struct chunk_header)); } }
/* * memblock_huge_init -- initializes a new huge memory block */ struct memory_block memblock_huge_init(struct palloc_heap *heap, uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx) { struct memory_block m = MEMORY_BLOCK_NONE; m.chunk_id = chunk_id; m.zone_id = zone_id; m.size_idx = size_idx; m.heap = heap; struct chunk_header nhdr = { .type = CHUNK_TYPE_FREE, .flags = 0, .size_idx = size_idx }; struct chunk_header *hdr = heap_get_chunk_hdr(heap, &m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr)); VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr)); *hdr = nhdr; /* write the entire header (8 bytes) at once */ pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); huge_write_footer(hdr, size_idx); memblock_rebuild_state(heap, &m); return m; } /* * memblock_run_init -- initializes a new run memory block */ struct memory_block memblock_run_init(struct palloc_heap *heap, uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx, uint16_t flags, uint64_t unit_size, uint64_t alignment) { ASSERTne(size_idx, 0); struct memory_block m = MEMORY_BLOCK_NONE; m.chunk_id = chunk_id; m.zone_id = zone_id; m.size_idx = size_idx; m.heap = heap; struct zone *z = ZID_TO_ZONE(heap->layout, zone_id); struct chunk_run *run = heap_get_chunk_run(heap, &m); size_t runsize = SIZEOF_RUN(run, size_idx); VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize); /* add/remove chunk_run and chunk_header to valgrind transaction */ VALGRIND_ADD_TO_TX(run, runsize); run->hdr.block_size = unit_size; run->hdr.alignment = alignment; struct run_bitmap b; memblock_run_bitmap(&size_idx, flags, unit_size, alignment, run->content, &b); size_t bitmap_size = b.size; /* set all the bits */ memset(b.values, 0xFF, bitmap_size); /* clear only the bits available for allocations from this bucket */ memset(b.values, 0, sizeof(*b.values) * (b.nvalues - 1)); unsigned trailing_bits = b.nbits % RUN_BITS_PER_VALUE; uint64_t last_value = UINT64_MAX << trailing_bits; b.values[b.nvalues - 1] = last_value; VALGRIND_REMOVE_FROM_TX(run, runsize); pmemops_flush(&heap->p_ops, run, sizeof(struct chunk_run_header) + bitmap_size); struct chunk_header run_data_hdr; run_data_hdr.type = CHUNK_TYPE_RUN_DATA; run_data_hdr.flags = 0; VALGRIND_ADD_TO_TX(&z->chunk_headers[chunk_id], sizeof(struct chunk_header) * size_idx); struct chunk_header *data_hdr; for (unsigned i = 1; i < size_idx; ++i) { data_hdr = &z->chunk_headers[chunk_id + i]; VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr)); VALGRIND_ANNOTATE_NEW_MEMORY(data_hdr, sizeof(*data_hdr)); run_data_hdr.size_idx = i; *data_hdr = run_data_hdr; } pmemops_persist(&heap->p_ops, &z->chunk_headers[chunk_id + 1], sizeof(struct chunk_header) * (size_idx - 1)); struct chunk_header *hdr = &z->chunk_headers[chunk_id]; ASSERT(hdr->type == CHUNK_TYPE_FREE); VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr)); struct chunk_header run_hdr; run_hdr.size_idx = hdr->size_idx; run_hdr.type = CHUNK_TYPE_RUN; run_hdr.flags = flags; *hdr = run_hdr; pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); VALGRIND_REMOVE_FROM_TX(&z->chunk_headers[chunk_id], sizeof(struct chunk_header) * size_idx); memblock_rebuild_state(heap, &m); return m; }