ucs_status_t ucs_mpool_hugetlb_malloc(ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
ucs_hugetlb_mpool_chunk_hdr_t *chunk;
void *ptr;
ucs_status_t status;
size_t real_size;
int shmid;
/* First, try hugetlb */
real_size = *size_p;
status = ucs_sysv_alloc(&real_size, (void**)&ptr, SHM_HUGETLB, &shmid
UCS_MEMTRACK_NAME(ucs_mpool_name(mp)));
if (status == UCS_OK) {
chunk = ptr;
chunk->hugetlb = 1;
goto out_ok;
}
/* Fallback to glibc */
real_size = *size_p;
chunk = ucs_malloc(real_size, ucs_mpool_name(mp));
if (chunk != NULL) {
chunk->hugetlb = 0;
goto out_ok;
}
return UCS_ERR_NO_MEMORY;
out_ok:
*size_p = real_size - sizeof(*chunk);
*chunk_p = chunk + 1;
return UCS_OK;
}
static ucs_status_t uct_iface_mp_chunk_alloc(ucs_mpool_t *mp, size_t *size_p,
void **chunk_p)
{
uct_base_iface_t *iface = uct_iface_mp_priv(mp)->iface;
uct_iface_mp_chunk_hdr_t *hdr;
uct_allocated_memory_t mem;
ucs_status_t status;
size_t length;
length = sizeof(*hdr) + *size_p;
status = uct_iface_mem_alloc(&iface->super, length, ucs_mpool_name(mp), &mem);
if (status != UCS_OK) {
return status;
}
ucs_assert(mem.memh != UCT_INVALID_MEM_HANDLE);
ucs_assert(mem.md == iface->md);
hdr = mem.address;
hdr->method = mem.method;
hdr->length = mem.length;
hdr->memh = mem.memh;
*size_p = mem.length - sizeof(*hdr);
*chunk_p = hdr + 1;
return UCS_OK;
}
static void ucs_mpool_chunk_leak_check(ucs_mpool_t *mp, ucs_mpool_chunk_t *chunk)
{
ucs_mpool_elem_t *elem;
unsigned i;
for (i = 0; i < chunk->num_elems; ++i) {
elem = ucs_mpool_chunk_elem(mp->data, chunk, i);
if (elem->mpool != NULL) {
ucs_warn("object %p was not returned to mpool %s", elem + 1,
ucs_mpool_name(mp));
}
}
}
void ucs_mpool_cleanup(ucs_mpool_t *mp, int leak_check)
{
ucs_mpool_chunk_t *chunk, *next_chunk;
ucs_mpool_elem_t *elem, *next_elem;
ucs_mpool_data_t *data = mp->data;
void *obj;
/* Cleanup all elements in the freelist and set their header to NULL to mark
* them as released for the leak check.
*/
next_elem = mp->freelist;
while (next_elem != NULL) {
elem = next_elem;
VALGRIND_MAKE_MEM_DEFINED(elem, sizeof *elem);
next_elem = elem->next;
if (data->ops->obj_cleanup != NULL) {
obj = elem + 1;
VALGRIND_MEMPOOL_ALLOC(mp, obj, mp->data->elem_size - sizeof(ucs_mpool_elem_t));
VALGRIND_MAKE_MEM_DEFINED(obj, mp->data->elem_size - sizeof(ucs_mpool_elem_t));
data->ops->obj_cleanup(mp, obj);
VALGRIND_MEMPOOL_FREE(mp, obj);
}
elem->mpool = NULL;
}
/*
* Go over all elements in the chunks and make sure they were on the freelist.
* Then, release the chunk.
*/
next_chunk = data->chunks;
while (next_chunk != NULL) {
chunk = next_chunk;
next_chunk = chunk->next;
if (leak_check) {
ucs_mpool_chunk_leak_check(mp, chunk);
}
data->ops->chunk_release(mp, chunk);
}
VALGRIND_DESTROY_MEMPOOL(mp);
ucs_debug("mpool %s destroyed", ucs_mpool_name(mp));
free(data->name);
ucs_free(data);
}
ucs_status_t ucs_mpool_chunk_mmap(ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
ucs_mmap_mpool_chunk_hdr_t *chunk;
size_t real_size;
real_size = ucs_align_up(*size_p + sizeof(*chunk), ucs_get_page_size());
chunk = ucs_mmap(NULL, real_size, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0, ucs_mpool_name(mp));
if (chunk == MAP_FAILED) {
return UCS_ERR_NO_MEMORY;
}
chunk->size = real_size;
*size_p = real_size - sizeof(*chunk);
*chunk_p = chunk + 1;
return UCS_OK;
}
ucs_status_t ucs_mpool_init(ucs_mpool_t *mp, size_t priv_size,
size_t elem_size, size_t align_offset, size_t alignment,
unsigned elems_per_chunk, unsigned max_elems,
ucs_mpool_ops_t *ops, const char *name)
{
/* Check input values */
if ((elem_size == 0) || (align_offset > elem_size) ||
(alignment == 0) || !ucs_is_pow2(alignment) ||
(elems_per_chunk == 0) || (max_elems < elems_per_chunk))
{
ucs_error("Invalid memory pool parameter(s)");
return UCS_ERR_INVALID_PARAM;
}
mp->data = ucs_malloc(sizeof(*mp->data) + priv_size, "mpool_data");
if (mp->data == NULL) {
ucs_error("Failed to allocate memory pool slow-path area");
return UCS_ERR_NO_MEMORY;
}
mp->freelist = NULL;
mp->data->elem_size = sizeof(ucs_mpool_elem_t) + elem_size;
mp->data->alignment = alignment;
mp->data->align_offset = sizeof(ucs_mpool_elem_t) + align_offset;
mp->data->quota = max_elems;
mp->data->tail = NULL;
mp->data->chunk_size = sizeof(ucs_mpool_chunk_t) + alignment +
elems_per_chunk * ucs_mpool_elem_total_size(mp->data);
mp->data->chunks = NULL;
mp->data->ops = ops;
mp->data->name = strdup(name);
VALGRIND_CREATE_MEMPOOL(mp, 0, 0);
ucs_debug("mpool %s: align %u, maxelems %u, elemsize %u",
ucs_mpool_name(mp), mp->data->alignment, max_elems, mp->data->elem_size);
return UCS_OK;
}
ucs_status_t ucs_mpool_chunk_malloc(ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
*chunk_p = ucs_malloc(*size_p, ucs_mpool_name(mp));
return (*chunk_p == NULL) ? UCS_ERR_NO_MEMORY : UCS_OK;
}
void *ucs_mpool_get_grow(ucs_mpool_t *mp)
{
size_t chunk_size, chunk_padding;
ucs_mpool_data_t *data = mp->data;
ucs_mpool_chunk_t *chunk;
ucs_mpool_elem_t *elem;
ucs_status_t status;
unsigned i;
void *ptr;
if (data->quota == 0) {
return NULL;
}
chunk_size = data->chunk_size;
status = data->ops->chunk_alloc(mp, &chunk_size, &ptr);
if (status != UCS_OK) {
ucs_error("Failed to allocate memory pool chunk: %s", ucs_status_string(status));
return NULL;
}
/* Calculate padding, and update element count according to allocated size */
chunk = ptr;
chunk_padding = ucs_padding((uintptr_t)(chunk + 1) + data->align_offset,
data->alignment);
chunk->elems = (void*)(chunk + 1) + chunk_padding;
chunk->num_elems = (chunk_size - chunk_padding - sizeof(*chunk)) /
ucs_mpool_elem_total_size(data);
ucs_debug("mpool %s: allocated chunk %p of %lu bytes with %u elements",
ucs_mpool_name(mp), chunk, chunk_size, chunk->num_elems);
for (i = 0; i < chunk->num_elems; ++i) {
elem = ucs_mpool_chunk_elem(data, chunk, i);
if (data->ops->obj_init != NULL) {
data->ops->obj_init(mp, elem + 1, chunk);
}
ucs_mpool_add_to_freelist(mp, elem, 0);
if (mp->data->tail == NULL) {
mp->data->tail = elem;
}
}
chunk->next = data->chunks;
data->chunks = chunk;
if (data->quota == UINT_MAX) {
/* Infinite memory pool */
} else if (data->quota >= chunk->num_elems) {
data->quota -= chunk->num_elems;
} else {
data->quota = 0;
}
VALGRIND_MAKE_MEM_NOACCESS(chunk + 1, chunk_size - sizeof(*chunk));
ucs_assert(mp->freelist != NULL); /* Should not recurse */
return ucs_mpool_get(mp);
}
static inline uct_iface_mp_priv_t* uct_iface_mp_priv(ucs_mpool_t *mp)
{
return (uct_iface_mp_priv_t*)ucs_mpool_priv(mp);
}
UCS_PROFILE_FUNC(ucs_status_t, uct_iface_mp_chunk_alloc, (mp, size_p, chunk_p),
ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
uct_base_iface_t *iface = uct_iface_mp_priv(mp)->iface;
uct_iface_mp_chunk_hdr_t *hdr;
uct_allocated_memory_t mem;
ucs_status_t status;
size_t length;
length = sizeof(*hdr) + *size_p;
status = uct_iface_mem_alloc(&iface->super, length, 0, ucs_mpool_name(mp),
&mem);
if (status != UCS_OK) {
return status;
}
ucs_assert(mem.memh != UCT_MEM_HANDLE_NULL);
ucs_assert(mem.md == iface->md);
hdr = mem.address;
hdr->method = mem.method;
hdr->length = mem.length;
hdr->memh = mem.memh;
*size_p = mem.length - sizeof(*hdr);
*chunk_p = hdr + 1;
return UCS_OK;
return (uct_iface_mp_priv_t*)ucs_mpool_priv(mp);
}
UCS_PROFILE_FUNC(ucs_status_t, uct_iface_mp_chunk_alloc, (mp, size_p, chunk_p),
ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
uct_base_iface_t *iface = uct_iface_mp_priv(mp)->iface;
uct_iface_mp_chunk_hdr_t *hdr;
uct_allocated_memory_t mem;
ucs_status_t status;
size_t length;
length = sizeof(*hdr) + *size_p;
status = uct_iface_mem_alloc(&iface->super, length,
UCT_MD_MEM_ACCESS_ALL | UCT_MD_MEM_FLAG_LOCK,
ucs_mpool_name(mp), &mem);
if (status != UCS_OK) {
return status;
}
ucs_assert(mem.memh != UCT_MEM_HANDLE_NULL);
ucs_assert(mem.md == iface->md);
hdr = mem.address;
hdr->method = mem.method;
hdr->length = mem.length;
hdr->memh = mem.memh;
*size_p = mem.length - sizeof(*hdr);
*chunk_p = hdr + 1;
return UCS_OK;
}
