/* * run_get_bitmap -- initializes run bitmap information */ static void run_get_bitmap(const struct memory_block *m, struct run_bitmap *b) { struct chunk_header *hdr = heap_get_chunk_hdr(m->heap, m); struct chunk_run *run = heap_get_chunk_run(m->heap, m); uint32_t size_idx = hdr->size_idx; memblock_run_bitmap(&size_idx, hdr->flags, run->hdr.block_size, run->hdr.alignment, run->content, b); ASSERTeq(size_idx, hdr->size_idx); }
/* * alloc_class_find_or_create -- (internal) searches for the * biggest allocation class for which unit_size is evenly divisible by n. * If no such class exists, create one. */ static struct alloc_class * alloc_class_find_or_create(struct alloc_class_collection *ac, size_t n) { LOG(10, NULL); COMPILE_ERROR_ON(MAX_ALLOCATION_CLASSES > UINT8_MAX); uint64_t required_size_bytes = n * RUN_MIN_NALLOCS; uint32_t required_size_idx = 1; if (required_size_bytes > RUN_DEFAULT_SIZE) { required_size_bytes -= RUN_DEFAULT_SIZE; required_size_idx += CALC_SIZE_IDX(CHUNKSIZE, required_size_bytes); if (required_size_idx > RUN_SIZE_IDX_CAP) required_size_idx = RUN_SIZE_IDX_CAP; } for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE || c->run.size_idx < required_size_idx) continue; if (n % c->unit_size == 0 && n / c->unit_size <= RUN_UNIT_MAX_ALLOC) return c; } /* * In order to minimize the wasted space at the end of the run the * run data size must be divisible by the allocation class unit size * with the smallest possible remainder, preferably 0. */ struct run_bitmap b; size_t runsize_bytes = 0; do { if (runsize_bytes != 0) /* don't increase on first iteration */ n += ALLOC_BLOCK_SIZE_GEN; uint32_t size_idx = required_size_idx; memblock_run_bitmap(&size_idx, ALLOC_CLASS_DEFAULT_FLAGS, n, 0, NULL, &b); runsize_bytes = RUN_CONTENT_SIZE_BYTES(size_idx) - b.size; } while ((runsize_bytes % n) > MAX_RUN_WASTED_BYTES); /* * Now that the desired unit size is found the existing classes need * to be searched for possible duplicates. If a class that can handle * the calculated size already exists, simply return that. */ for (int i = 1; i < MAX_ALLOCATION_CLASSES; ++i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE) continue; if (n / c->unit_size <= RUN_UNIT_MAX_ALLOC && n % c->unit_size == 0) return c; if (c->unit_size == n) return c; } return alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, n, 0, required_size_idx); }
/* * memblock_huge_init -- initializes a new huge memory block */ struct memory_block memblock_huge_init(struct palloc_heap *heap, uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx) { struct memory_block m = MEMORY_BLOCK_NONE; m.chunk_id = chunk_id; m.zone_id = zone_id; m.size_idx = size_idx; m.heap = heap; struct chunk_header nhdr = { .type = CHUNK_TYPE_FREE, .flags = 0, .size_idx = size_idx }; struct chunk_header *hdr = heap_get_chunk_hdr(heap, &m); VALGRIND_DO_MAKE_MEM_UNDEFINED(hdr, sizeof(*hdr)); VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr)); *hdr = nhdr; /* write the entire header (8 bytes) at once */ pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); huge_write_footer(hdr, size_idx); memblock_rebuild_state(heap, &m); return m; } /* * memblock_run_init -- initializes a new run memory block */ struct memory_block memblock_run_init(struct palloc_heap *heap, uint32_t chunk_id, uint32_t zone_id, uint32_t size_idx, uint16_t flags, uint64_t unit_size, uint64_t alignment) { ASSERTne(size_idx, 0); struct memory_block m = MEMORY_BLOCK_NONE; m.chunk_id = chunk_id; m.zone_id = zone_id; m.size_idx = size_idx; m.heap = heap; struct zone *z = ZID_TO_ZONE(heap->layout, zone_id); struct chunk_run *run = heap_get_chunk_run(heap, &m); size_t runsize = SIZEOF_RUN(run, size_idx); VALGRIND_DO_MAKE_MEM_UNDEFINED(run, runsize); /* add/remove chunk_run and chunk_header to valgrind transaction */ VALGRIND_ADD_TO_TX(run, runsize); run->hdr.block_size = unit_size; run->hdr.alignment = alignment; struct run_bitmap b; memblock_run_bitmap(&size_idx, flags, unit_size, alignment, run->content, &b); size_t bitmap_size = b.size; /* set all the bits */ memset(b.values, 0xFF, bitmap_size); /* clear only the bits available for allocations from this bucket */ memset(b.values, 0, sizeof(*b.values) * (b.nvalues - 1)); unsigned trailing_bits = b.nbits % RUN_BITS_PER_VALUE; uint64_t last_value = UINT64_MAX << trailing_bits; b.values[b.nvalues - 1] = last_value; VALGRIND_REMOVE_FROM_TX(run, runsize); pmemops_flush(&heap->p_ops, run, sizeof(struct chunk_run_header) + bitmap_size); struct chunk_header run_data_hdr; run_data_hdr.type = CHUNK_TYPE_RUN_DATA; run_data_hdr.flags = 0; VALGRIND_ADD_TO_TX(&z->chunk_headers[chunk_id], sizeof(struct chunk_header) * size_idx); struct chunk_header *data_hdr; for (unsigned i = 1; i < size_idx; ++i) { data_hdr = &z->chunk_headers[chunk_id + i]; VALGRIND_DO_MAKE_MEM_UNDEFINED(data_hdr, sizeof(*data_hdr)); VALGRIND_ANNOTATE_NEW_MEMORY(data_hdr, sizeof(*data_hdr)); run_data_hdr.size_idx = i; *data_hdr = run_data_hdr; } pmemops_persist(&heap->p_ops, &z->chunk_headers[chunk_id + 1], sizeof(struct chunk_header) * (size_idx - 1)); struct chunk_header *hdr = &z->chunk_headers[chunk_id]; ASSERT(hdr->type == CHUNK_TYPE_FREE); VALGRIND_ANNOTATE_NEW_MEMORY(hdr, sizeof(*hdr)); struct chunk_header run_hdr; run_hdr.size_idx = hdr->size_idx; run_hdr.type = CHUNK_TYPE_RUN; run_hdr.flags = flags; *hdr = run_hdr; pmemops_persist(&heap->p_ops, hdr, sizeof(*hdr)); VALGRIND_REMOVE_FROM_TX(&z->chunk_headers[chunk_id], sizeof(struct chunk_header) * size_idx); memblock_rebuild_state(heap, &m); return m; }
/* * alloc_class_new -- creates a new allocation class */ struct alloc_class * alloc_class_new(int id, struct alloc_class_collection *ac, enum alloc_class_type type, enum header_type htype, size_t unit_size, size_t alignment, uint32_t size_idx) { LOG(10, NULL); struct alloc_class *c = Malloc(sizeof(*c)); if (c == NULL) goto error_class_alloc; c->unit_size = unit_size; c->header_type = htype; c->type = type; c->flags = (uint16_t) (header_type_to_flag[c->header_type] | (alignment ? CHUNK_FLAG_ALIGNED : 0)) | ALLOC_CLASS_DEFAULT_FLAGS; switch (type) { case CLASS_HUGE: id = DEFAULT_ALLOC_CLASS_ID; break; case CLASS_RUN: c->run.alignment = alignment; struct run_bitmap b; memblock_run_bitmap(&size_idx, c->flags, unit_size, alignment, NULL, &b); c->run.nallocs = b.nbits; c->run.size_idx = size_idx; uint8_t slot = (uint8_t)id; if (id < 0 && alloc_class_find_first_free_slot(ac, &slot) != 0) goto error_class_alloc; id = slot; size_t map_idx = SIZE_TO_CLASS_MAP_INDEX(c->unit_size, ac->granularity); ASSERT(map_idx <= UINT32_MAX); uint32_t map_idx_s = (uint32_t)map_idx; uint16_t size_idx_s = (uint16_t)size_idx; uint16_t flags_s = (uint16_t)c->flags; uint64_t k = RUN_CLASS_KEY_PACK(map_idx_s, flags_s, size_idx_s); if (critnib_insert(ac->class_map_by_unit_size, k, c) != 0) { ERR("unable to register allocation class"); goto error_map_insert; } break; default: ASSERT(0); } c->id = (uint8_t)id; ac->aclasses[c->id] = c; return c; error_map_insert: Free(c); error_class_alloc: if (id >= 0) alloc_class_reservation_clear(ac, id); return NULL; }