/* * alloc_class_find_or_create -- (internal) searches for the * biggest allocation class for which unit_size is evenly divisible by n. * If no such class exists, create one. */ static struct alloc_class * alloc_class_find_or_create(struct alloc_class_collection *ac, size_t n) { LOG(10, NULL); COMPILE_ERROR_ON(MAX_ALLOCATION_CLASSES > UINT8_MAX); uint64_t required_size_bytes = n * RUN_MIN_NALLOCS; uint32_t required_size_idx = 1; if (required_size_bytes > RUNSIZE) { required_size_bytes -= RUNSIZE; required_size_idx += CALC_SIZE_IDX(CHUNKSIZE, required_size_bytes); if (required_size_idx > RUN_SIZE_IDX_CAP) required_size_idx = RUN_SIZE_IDX_CAP; } for (int i = MAX_ALLOCATION_CLASSES - 1; i >= 0; --i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE || c->run.size_idx < required_size_idx) continue; if (n % c->unit_size == 0 && n / c->unit_size <= RUN_UNIT_MAX_ALLOC) return c; } /* * In order to minimize the wasted space at the end of the run the * run data size must be divisible by the allocation class unit size * with the smallest possible remainder, preferably 0. */ size_t runsize_bytes = RUN_SIZE_BYTES(required_size_idx); while ((runsize_bytes % n) > MAX_RUN_WASTED_BYTES) { n += ALLOC_BLOCK_SIZE_GEN; } /* * Now that the desired unit size is found the existing classes need * to be searched for possible duplicates. If a class that can handle * the calculated size already exists, simply return that. */ for (int i = 1; i < MAX_ALLOCATION_CLASSES; ++i) { struct alloc_class *c = ac->aclasses[i]; if (c == NULL || c->type == CLASS_HUGE) continue; if (n / c->unit_size <= RUN_UNIT_MAX_ALLOC && n % c->unit_size == 0) return c; if (c->unit_size == n) return c; } return alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, n, 0, required_size_idx); }
/* * alloc_class_collection_new -- creates a new collection of allocation classes */ struct alloc_class_collection * alloc_class_collection_new() { LOG(10, NULL); struct alloc_class_collection *ac = Zalloc(sizeof(*ac)); if (ac == NULL) return NULL; memset(ac->aclasses, 0, sizeof(ac->aclasses)); ac->granularity = ALLOC_BLOCK_SIZE; ac->last_run_max_size = MAX_RUN_SIZE; ac->fail_on_missing_class = 0; ac->autogenerate_on_missing_class = 1; size_t maps_size = (MAX_RUN_SIZE / ac->granularity) + 1; if ((ac->class_map_by_alloc_size = Malloc(maps_size)) == NULL) goto error; if ((ac->class_map_by_unit_size = cuckoo_new()) == NULL) goto error; memset(ac->class_map_by_alloc_size, 0xFF, maps_size); if (alloc_class_new(-1, ac, CLASS_HUGE, HEADER_COMPACT, CHUNKSIZE, 0, 1) == NULL) goto error; struct alloc_class *predefined_class = alloc_class_new(-1, ac, CLASS_RUN, HEADER_COMPACT, MIN_RUN_SIZE, 0, 1); if (predefined_class == NULL) goto error; for (size_t i = 0; i < FIRST_GENERATED_CLASS_SIZE / ac->granularity; ++i) { ac->class_map_by_alloc_size[i] = predefined_class->id; } /* * Based on the defined categories, a set of allocation classes is * created. The unit size of those classes is depended on the category * initial size and step. */ size_t granularity_mask = ALLOC_BLOCK_SIZE_GEN - 1; for (int c = 1; c < MAX_ALLOC_CATEGORIES; ++c) { size_t n = categories[c - 1].size + ALLOC_BLOCK_SIZE_GEN; do { if (alloc_class_find_or_create(ac, n) == NULL) goto error; float stepf = (float)n * categories[c].step; size_t stepi = (size_t)stepf; stepi = (stepf - (float)stepi < FLT_EPSILON) ? stepi : stepi + 1; n += (stepi + (granularity_mask)) & ~granularity_mask; } while (n <= categories[c].size); } /* * Find the largest alloc class and use it's unit size as run allocation * threshold. */ uint8_t largest_aclass_slot; for (largest_aclass_slot = MAX_ALLOCATION_CLASSES - 1; largest_aclass_slot > 0 && ac->aclasses[largest_aclass_slot] == NULL; --largest_aclass_slot) { /* intentional NOP */ } struct alloc_class *c = ac->aclasses[largest_aclass_slot]; /* * The actual run might contain less unit blocks than the theoretical * unit max variable. This may be the case for very large unit sizes. */ size_t real_unit_max = c->run.bitmap_nallocs < RUN_UNIT_MAX_ALLOC ? c->run.bitmap_nallocs : RUN_UNIT_MAX_ALLOC; size_t theoretical_run_max_size = c->unit_size * real_unit_max; ac->last_run_max_size = MAX_RUN_SIZE > theoretical_run_max_size ? theoretical_run_max_size : MAX_RUN_SIZE; #ifdef DEBUG /* * Verify that each bucket's unit size points back to the bucket by the * bucket map. This must be true for the default allocation classes, * otherwise duplicate buckets will be created. */ for (size_t i = 0; i < MAX_ALLOCATION_CLASSES; ++i) { struct alloc_class *c = ac->aclasses[i]; if (c != NULL && c->type == CLASS_RUN) { ASSERTeq(i, c->id); ASSERTeq(alloc_class_by_run(ac, c->unit_size, c->flags, c->run.size_idx), c); } } #endif return ac; error: alloc_class_collection_delete(ac); return NULL; }
/* * CTL_WRITE_HANDLER(proto) -- creates a new allocation class */ static int CTL_WRITE_HANDLER(desc)(void *ctx, enum ctl_query_source source, void *arg, struct ctl_indexes *indexes) { PMEMobjpool *pop = ctx; uint8_t id; struct alloc_class_collection *ac = heap_alloc_classes(&pop->heap); struct pobj_alloc_class_desc *p = arg; if (p->unit_size <= 0 || p->unit_size > PMEMOBJ_MAX_ALLOC_SIZE || p->units_per_block <= 0) { errno = EINVAL; return -1; } if (p->alignment != 0 && p->unit_size % p->alignment != 0) { ERR("unit size must be evenly divisible by alignment"); errno = EINVAL; return -1; } if (p->alignment > (MEGABYTE * 2)) { ERR("alignment cannot be larger than 2 megabytes"); errno = EINVAL; return -1; } enum header_type lib_htype = MAX_HEADER_TYPES; switch (p->header_type) { case POBJ_HEADER_LEGACY: lib_htype = HEADER_LEGACY; break; case POBJ_HEADER_COMPACT: lib_htype = HEADER_COMPACT; break; case POBJ_HEADER_NONE: lib_htype = HEADER_NONE; break; case MAX_POBJ_HEADER_TYPES: default: ERR("invalid header type"); errno = EINVAL; return -1; } if (SLIST_EMPTY(indexes)) { if (alloc_class_find_first_free_slot(ac, &id) != 0) { ERR("no available free allocation class identifier"); errno = EINVAL; return -1; } } else { struct ctl_index *idx = SLIST_FIRST(indexes); ASSERTeq(strcmp(idx->name, "class_id"), 0); if (idx->value < 0 || idx->value >= MAX_ALLOCATION_CLASSES) { ERR("class id outside of the allowed range"); errno = ERANGE; return -1; } id = (uint8_t)idx->value; if (alloc_class_reserve(ac, id) != 0) { ERR("attempted to overwrite an allocation class"); errno = EEXIST; return -1; } } size_t runsize_bytes = CHUNK_ALIGN_UP((p->units_per_block * p->unit_size) + RUN_BASE_METADATA_SIZE); /* aligning the buffer might require up-to to 'alignment' bytes */ if (p->alignment != 0) runsize_bytes += p->alignment; uint32_t size_idx = (uint32_t)(runsize_bytes / CHUNKSIZE); if (size_idx > UINT16_MAX) size_idx = UINT16_MAX; struct alloc_class *c = alloc_class_new(id, heap_alloc_classes(&pop->heap), CLASS_RUN, lib_htype, p->unit_size, p->alignment, size_idx); if (c == NULL) { errno = EINVAL; return -1; } if (heap_create_alloc_class_buckets(&pop->heap, c) != 0) { alloc_class_delete(ac, c); return -1; } p->class_id = c->id; p->units_per_block = c->run.nallocs; return 0; }