ucs_status_t ucs_mpool_hugetlb_malloc(ucs_mpool_t *mp, size_t *size_p, void **chunk_p)
{
ucs_hugetlb_mpool_chunk_hdr_t *chunk;
void *ptr;
ucs_status_t status;
size_t real_size;
int shmid;
/* First, try hugetlb */
real_size = *size_p;
status = ucs_sysv_alloc(&real_size, (void**)&ptr, SHM_HUGETLB, &shmid
UCS_MEMTRACK_NAME(ucs_mpool_name(mp)));
if (status == UCS_OK) {
chunk = ptr;
chunk->hugetlb = 1;
goto out_ok;
}
/* Fallback to glibc */
real_size = *size_p;
chunk = ucs_malloc(real_size, ucs_mpool_name(mp));
if (chunk != NULL) {
chunk->hugetlb = 0;
goto out_ok;
}
return UCS_ERR_NO_MEMORY;
out_ok:
*size_p = real_size - sizeof(*chunk);
*chunk_p = chunk + 1;
return UCS_OK;
}
ucs_status_t uct_mem_alloc(size_t min_length, uct_alloc_method_t *methods,
unsigned num_methods, uct_md_h *mds, unsigned num_mds,
const char *alloc_name, uct_allocated_memory_t *mem)
{
uct_alloc_method_t *method;
uct_md_attr_t md_attr;
ucs_status_t status;
size_t alloc_length;
unsigned md_index;
uct_mem_h memh;
uct_md_h md;
void *address;
int shmid;
if (min_length == 0) {
ucs_error("Allocation length cannot be 0");
return UCS_ERR_INVALID_PARAM;
}
if (num_methods == 0) {
ucs_error("No allocation methods provided");
return UCS_ERR_INVALID_PARAM;
}
for (method = methods; method < methods + num_methods; ++method) {
ucs_debug("trying allocation method %s", uct_alloc_method_names[*method]);
switch (*method) {
case UCT_ALLOC_METHOD_MD:
/* Allocate with one of the specified memory domains */
for (md_index = 0; md_index < num_mds; ++md_index) {
md = mds[md_index];
status = uct_md_query(md, &md_attr);
if (status != UCS_OK) {
ucs_error("Failed to query MD");
return status;
}
/* Check if MD supports allocation */
if (!(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) {
continue;
}
/* Allocate memory using the MD.
* If the allocation fails, it's considered an error and we don't
* fall-back, because this MD already exposed support for memory
* allocation.
*/
alloc_length = min_length;
status = uct_md_mem_alloc(md, &alloc_length, &address,
alloc_name, &memh);
if (status != UCS_OK) {
ucs_error("failed to allocate %zu bytes using md %s: %s",
alloc_length, md->component->name,
ucs_status_string(status));
return status;
}
ucs_assert(memh != UCT_INVALID_MEM_HANDLE);
mem->md = md;
mem->memh = memh;
goto allocated;
}
break;
case UCT_ALLOC_METHOD_HEAP:
/* Allocate aligned memory using libc allocator */
alloc_length = min_length;
address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length
UCS_MEMTRACK_VAL);
if (address != NULL) {
goto allocated_without_md;
}
ucs_debug("failed to allocate %zu bytes from the heap", alloc_length);
break;
case UCT_ALLOC_METHOD_MMAP:
/* Request memory from operating system using mmap() */
alloc_length = ucs_align_up_pow2(min_length, ucs_get_page_size());
address = ucs_mmap(NULL, alloc_length, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANON, -1, 0 UCS_MEMTRACK_VAL);
if (address != MAP_FAILED) {
goto allocated_without_md;
}
ucs_debug("failed to mmap %zu bytes: %m", alloc_length);
break;
case UCT_ALLOC_METHOD_HUGE:
/* Allocate huge pages */
alloc_length = min_length;
status = ucs_sysv_alloc(&alloc_length, &address, SHM_HUGETLB, &shmid
UCS_MEMTRACK_VAL);
if (status == UCS_OK) {
goto allocated_without_md;
}
ucs_debug("failed to allocate %zu bytes from hugetlb: %s",
min_length, ucs_status_string(status));
break;
default:
ucs_error("Invalid allocation method %d", *method);
return UCS_ERR_INVALID_PARAM;
}
}
ucs_debug("Could not allocate memory with any of the provided methods");
return UCS_ERR_NO_MEMORY;
allocated_without_md:
mem->md = NULL;
mem->memh = UCT_INVALID_MEM_HANDLE;
allocated:
ucs_debug("allocated %zu bytes at %p using %s", alloc_length, address,
(mem->md == NULL) ? uct_alloc_method_names[*method]
: mem->md->component->name);
mem->address = address;
mem->length = alloc_length;
mem->method = *method;
return UCS_OK;
}
ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags,
uct_alloc_method_t *methods, unsigned num_methods,
uct_md_h *mds, unsigned num_mds,
const char *alloc_name, uct_allocated_memory_t *mem)
{
uct_alloc_method_t *method;
uct_md_attr_t md_attr;
ucs_status_t status;
size_t alloc_length;
unsigned md_index;
uct_mem_h memh;
uct_md_h md;
void *address;
int shmid;
unsigned map_flags;
if (min_length == 0) {
ucs_error("Allocation length cannot be 0");
return UCS_ERR_INVALID_PARAM;
}
if (num_methods == 0) {
ucs_error("No allocation methods provided");
return UCS_ERR_INVALID_PARAM;
}
if ((flags & UCT_MD_MEM_FLAG_FIXED) &&
(!addr || ((uintptr_t)addr % ucs_get_page_size()))) {
ucs_debug("UCT_MD_MEM_FLAG_FIXED requires valid page size aligned address");
return UCS_ERR_INVALID_PARAM;
}
for (method = methods; method < methods + num_methods; ++method) {
ucs_debug("trying allocation method %s", uct_alloc_method_names[*method]);
switch (*method) {
case UCT_ALLOC_METHOD_MD:
/* Allocate with one of the specified memory domains */
for (md_index = 0; md_index < num_mds; ++md_index) {
md = mds[md_index];
status = uct_md_query(md, &md_attr);
if (status != UCS_OK) {
ucs_error("Failed to query MD");
return status;
}
/* Check if MD supports allocation */
if (!(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) {
continue;
}
/* Check if MD supports allocation with fixed address
* if it's requested */
if ((flags & UCT_MD_MEM_FLAG_FIXED) &&
!(md_attr.cap.flags & UCT_MD_FLAG_FIXED)) {
continue;
}
/* Allocate memory using the MD.
* If the allocation fails, it's considered an error and we don't
* fall-back, because this MD already exposed support for memory
* allocation.
*/
alloc_length = min_length;
address = addr;
status = uct_md_mem_alloc(md, &alloc_length, &address, flags,
alloc_name, &memh);
if (status != UCS_OK) {
ucs_error("failed to allocate %zu bytes using md %s: %s",
alloc_length, md->component->name,
ucs_status_string(status));
return status;
}
ucs_assert(memh != UCT_MEM_HANDLE_NULL);
mem->md = md;
mem->memh = memh;
goto allocated;
}
break;
case UCT_ALLOC_METHOD_THP:
#ifdef MADV_HUGEPAGE
if (!ucs_is_thp_enabled()) {
break;
}
/* Fixed option is not supported for thp allocation*/
if (flags & UCT_MD_MEM_FLAG_FIXED) {
break;
}
alloc_length = ucs_align_up(min_length, ucs_get_huge_page_size());
address = ucs_memalign(ucs_get_huge_page_size(), alloc_length
UCS_MEMTRACK_VAL);
if (address != NULL) {
status = madvise(address, alloc_length, MADV_HUGEPAGE);
if (status != UCS_OK) {
ucs_error("madvise failure status (%d) address(%p) len(%zu):"
" %m", status, address, alloc_length);
ucs_free(address);
break;
} else {
goto allocated_without_md;
}
}
ucs_debug("failed to allocate by thp %zu bytes: %m", alloc_length);
#endif
break;
case UCT_ALLOC_METHOD_HEAP:
/* Allocate aligned memory using libc allocator */
/* Fixed option is not supported for heap allocation*/
if (flags & UCT_MD_MEM_FLAG_FIXED) {
break;
}
alloc_length = min_length;
address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length
UCS_MEMTRACK_VAL);
if (address != NULL) {
goto allocated_without_md;
}
ucs_debug("failed to allocate %zu bytes from the heap", alloc_length);
break;
case UCT_ALLOC_METHOD_MMAP:
map_flags = uct_mem_get_mmap_flags(flags);
/* Request memory from operating system using mmap() */
alloc_length = ucs_align_up_pow2(min_length, ucs_get_page_size());
address = ucs_mmap(addr, alloc_length, PROT_READ | PROT_WRITE,
map_flags, -1, 0 UCS_MEMTRACK_VAL);
if (address != MAP_FAILED) {
goto allocated_without_md;
}
ucs_debug("failed to mmap %zu bytes: %m", alloc_length);
break;
case UCT_ALLOC_METHOD_HUGE:
/* Allocate huge pages */
alloc_length = min_length;
address = (flags & UCT_MD_MEM_FLAG_FIXED) ? addr : NULL;
status = ucs_sysv_alloc(&alloc_length, &address, SHM_HUGETLB, &shmid
UCS_MEMTRACK_VAL);
if (status == UCS_OK) {
goto allocated_without_md;
}
ucs_debug("failed to allocate %zu bytes from hugetlb: %s",
min_length, ucs_status_string(status));
break;
default:
ucs_error("Invalid allocation method %d", *method);
return UCS_ERR_INVALID_PARAM;
}
}
ucs_debug("Could not allocate memory with any of the provided methods");
return UCS_ERR_NO_MEMORY;
allocated_without_md:
mem->md = NULL;
mem->memh = UCT_MEM_HANDLE_NULL;
allocated:
ucs_debug("allocated %zu bytes at %p using %s", alloc_length, address,
(mem->md == NULL) ? uct_alloc_method_names[*method]
: mem->md->component->name);
mem->address = address;
mem->length = alloc_length;
mem->method = *method;
return UCS_OK;
}
