ucs_status_t ucs_mpool_hugetlb_malloc(ucs_mpool_t *mp, size_t *size_p, void **chunk_p) { ucs_hugetlb_mpool_chunk_hdr_t *chunk; void *ptr; ucs_status_t status; size_t real_size; int shmid; /* First, try hugetlb */ real_size = *size_p; status = ucs_sysv_alloc(&real_size, (void**)&ptr, SHM_HUGETLB, &shmid UCS_MEMTRACK_NAME(ucs_mpool_name(mp))); if (status == UCS_OK) { chunk = ptr; chunk->hugetlb = 1; goto out_ok; } /* Fallback to glibc */ real_size = *size_p; chunk = ucs_malloc(real_size, ucs_mpool_name(mp)); if (chunk != NULL) { chunk->hugetlb = 0; goto out_ok; } return UCS_ERR_NO_MEMORY; out_ok: *size_p = real_size - sizeof(*chunk); *chunk_p = chunk + 1; return UCS_OK; }
ucs_status_t uct_mem_alloc(size_t min_length, uct_alloc_method_t *methods, unsigned num_methods, uct_md_h *mds, unsigned num_mds, const char *alloc_name, uct_allocated_memory_t *mem) { uct_alloc_method_t *method; uct_md_attr_t md_attr; ucs_status_t status; size_t alloc_length; unsigned md_index; uct_mem_h memh; uct_md_h md; void *address; int shmid; if (min_length == 0) { ucs_error("Allocation length cannot be 0"); return UCS_ERR_INVALID_PARAM; } if (num_methods == 0) { ucs_error("No allocation methods provided"); return UCS_ERR_INVALID_PARAM; } for (method = methods; method < methods + num_methods; ++method) { ucs_debug("trying allocation method %s", uct_alloc_method_names[*method]); switch (*method) { case UCT_ALLOC_METHOD_MD: /* Allocate with one of the specified memory domains */ for (md_index = 0; md_index < num_mds; ++md_index) { md = mds[md_index]; status = uct_md_query(md, &md_attr); if (status != UCS_OK) { ucs_error("Failed to query MD"); return status; } /* Check if MD supports allocation */ if (!(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { continue; } /* Allocate memory using the MD. * If the allocation fails, it's considered an error and we don't * fall-back, because this MD already exposed support for memory * allocation. */ alloc_length = min_length; status = uct_md_mem_alloc(md, &alloc_length, &address, alloc_name, &memh); if (status != UCS_OK) { ucs_error("failed to allocate %zu bytes using md %s: %s", alloc_length, md->component->name, ucs_status_string(status)); return status; } ucs_assert(memh != UCT_INVALID_MEM_HANDLE); mem->md = md; mem->memh = memh; goto allocated; } break; case UCT_ALLOC_METHOD_HEAP: /* Allocate aligned memory using libc allocator */ alloc_length = min_length; address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length UCS_MEMTRACK_VAL); if (address != NULL) { goto allocated_without_md; } ucs_debug("failed to allocate %zu bytes from the heap", alloc_length); break; case UCT_ALLOC_METHOD_MMAP: /* Request memory from operating system using mmap() */ alloc_length = ucs_align_up_pow2(min_length, ucs_get_page_size()); address = ucs_mmap(NULL, alloc_length, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0 UCS_MEMTRACK_VAL); if (address != MAP_FAILED) { goto allocated_without_md; } ucs_debug("failed to mmap %zu bytes: %m", alloc_length); break; case UCT_ALLOC_METHOD_HUGE: /* Allocate huge pages */ alloc_length = min_length; status = ucs_sysv_alloc(&alloc_length, &address, SHM_HUGETLB, &shmid UCS_MEMTRACK_VAL); if (status == UCS_OK) { goto allocated_without_md; } ucs_debug("failed to allocate %zu bytes from hugetlb: %s", min_length, ucs_status_string(status)); break; default: ucs_error("Invalid allocation method %d", *method); return UCS_ERR_INVALID_PARAM; } } ucs_debug("Could not allocate memory with any of the provided methods"); return UCS_ERR_NO_MEMORY; allocated_without_md: mem->md = NULL; mem->memh = UCT_INVALID_MEM_HANDLE; allocated: ucs_debug("allocated %zu bytes at %p using %s", alloc_length, address, (mem->md == NULL) ? uct_alloc_method_names[*method] : mem->md->component->name); mem->address = address; mem->length = alloc_length; mem->method = *method; return UCS_OK; }
ucs_status_t uct_mem_alloc(void *addr, size_t min_length, unsigned flags, uct_alloc_method_t *methods, unsigned num_methods, uct_md_h *mds, unsigned num_mds, const char *alloc_name, uct_allocated_memory_t *mem) { uct_alloc_method_t *method; uct_md_attr_t md_attr; ucs_status_t status; size_t alloc_length; unsigned md_index; uct_mem_h memh; uct_md_h md; void *address; int shmid; unsigned map_flags; if (min_length == 0) { ucs_error("Allocation length cannot be 0"); return UCS_ERR_INVALID_PARAM; } if (num_methods == 0) { ucs_error("No allocation methods provided"); return UCS_ERR_INVALID_PARAM; } if ((flags & UCT_MD_MEM_FLAG_FIXED) && (!addr || ((uintptr_t)addr % ucs_get_page_size()))) { ucs_debug("UCT_MD_MEM_FLAG_FIXED requires valid page size aligned address"); return UCS_ERR_INVALID_PARAM; } for (method = methods; method < methods + num_methods; ++method) { ucs_debug("trying allocation method %s", uct_alloc_method_names[*method]); switch (*method) { case UCT_ALLOC_METHOD_MD: /* Allocate with one of the specified memory domains */ for (md_index = 0; md_index < num_mds; ++md_index) { md = mds[md_index]; status = uct_md_query(md, &md_attr); if (status != UCS_OK) { ucs_error("Failed to query MD"); return status; } /* Check if MD supports allocation */ if (!(md_attr.cap.flags & UCT_MD_FLAG_ALLOC)) { continue; } /* Check if MD supports allocation with fixed address * if it's requested */ if ((flags & UCT_MD_MEM_FLAG_FIXED) && !(md_attr.cap.flags & UCT_MD_FLAG_FIXED)) { continue; } /* Allocate memory using the MD. * If the allocation fails, it's considered an error and we don't * fall-back, because this MD already exposed support for memory * allocation. */ alloc_length = min_length; address = addr; status = uct_md_mem_alloc(md, &alloc_length, &address, flags, alloc_name, &memh); if (status != UCS_OK) { ucs_error("failed to allocate %zu bytes using md %s: %s", alloc_length, md->component->name, ucs_status_string(status)); return status; } ucs_assert(memh != UCT_MEM_HANDLE_NULL); mem->md = md; mem->memh = memh; goto allocated; } break; case UCT_ALLOC_METHOD_THP: #ifdef MADV_HUGEPAGE if (!ucs_is_thp_enabled()) { break; } /* Fixed option is not supported for thp allocation*/ if (flags & UCT_MD_MEM_FLAG_FIXED) { break; } alloc_length = ucs_align_up(min_length, ucs_get_huge_page_size()); address = ucs_memalign(ucs_get_huge_page_size(), alloc_length UCS_MEMTRACK_VAL); if (address != NULL) { status = madvise(address, alloc_length, MADV_HUGEPAGE); if (status != UCS_OK) { ucs_error("madvise failure status (%d) address(%p) len(%zu):" " %m", status, address, alloc_length); ucs_free(address); break; } else { goto allocated_without_md; } } ucs_debug("failed to allocate by thp %zu bytes: %m", alloc_length); #endif break; case UCT_ALLOC_METHOD_HEAP: /* Allocate aligned memory using libc allocator */ /* Fixed option is not supported for heap allocation*/ if (flags & UCT_MD_MEM_FLAG_FIXED) { break; } alloc_length = min_length; address = ucs_memalign(UCS_SYS_CACHE_LINE_SIZE, alloc_length UCS_MEMTRACK_VAL); if (address != NULL) { goto allocated_without_md; } ucs_debug("failed to allocate %zu bytes from the heap", alloc_length); break; case UCT_ALLOC_METHOD_MMAP: map_flags = uct_mem_get_mmap_flags(flags); /* Request memory from operating system using mmap() */ alloc_length = ucs_align_up_pow2(min_length, ucs_get_page_size()); address = ucs_mmap(addr, alloc_length, PROT_READ | PROT_WRITE, map_flags, -1, 0 UCS_MEMTRACK_VAL); if (address != MAP_FAILED) { goto allocated_without_md; } ucs_debug("failed to mmap %zu bytes: %m", alloc_length); break; case UCT_ALLOC_METHOD_HUGE: /* Allocate huge pages */ alloc_length = min_length; address = (flags & UCT_MD_MEM_FLAG_FIXED) ? addr : NULL; status = ucs_sysv_alloc(&alloc_length, &address, SHM_HUGETLB, &shmid UCS_MEMTRACK_VAL); if (status == UCS_OK) { goto allocated_without_md; } ucs_debug("failed to allocate %zu bytes from hugetlb: %s", min_length, ucs_status_string(status)); break; default: ucs_error("Invalid allocation method %d", *method); return UCS_ERR_INVALID_PARAM; } } ucs_debug("Could not allocate memory with any of the provided methods"); return UCS_ERR_NO_MEMORY; allocated_without_md: mem->md = NULL; mem->memh = UCT_MEM_HANDLE_NULL; allocated: ucs_debug("allocated %zu bytes at %p using %s", alloc_length, address, (mem->md == NULL) ? uct_alloc_method_names[*method] : mem->md->component->name); mem->address = address; mem->length = alloc_length; mem->method = *method; return UCS_OK; }