static int metal_init_page_sizes(void)
{
const int max_sizes = MAX_PAGE_SIZES - 1;
long sizes[max_sizes];
/* Determine system page size. */
sizes[0] = getpagesize();
if (sizes[0] <= 0) {
metal_log(METAL_LOG_ERROR, "failed to get page size\n");
return -ENOSYS;
}
_metal.page_size = sizes[0];
_metal.page_shift = metal_log2(sizes[0]);
metal_add_page_size(_metal.tmp_path, _metal.page_shift, 0);
#ifdef HAVE_HUGETLBFS_H
#ifndef MAP_HUGE_SHIFT
/* System does not support multiple huge page sizes. */
sizes[0] = gethugepagesize();
if (sizes[0] > 0) {
metal_add_page_size(hugetlbfs_find_path(),
metal_log2(sizes[0]),
MAP_HUGETLB);
}
#else
if (gethugepagesize() >= 0) {
int i, count;
/* System supports multiple huge page sizes. */
count = gethugepagesizes(sizes, max_sizes);
for (i = 0; i < count; i++) {
int shift = metal_log2(sizes[i]);
if ((shift & MAP_HUGE_MASK) != shift)
continue;
metal_add_page_size(
hugetlbfs_find_path_for_size(sizes[i]),
shift, (MAP_HUGETLB |
(shift << MAP_HUGE_SHIFT)));
}
}
#endif
#endif
/* Finally sort the resulting array by size. */
qsort(_metal.page_sizes, _metal.num_page_sizes,
sizeof(struct metal_page_size), metal_pagesize_compare);
return 0;
}
void check_free_huge_pages(int nr_pages_needed)
{
long hpage_size = gethugepagesize();
int freepages = get_huge_page_counter(hpage_size, HUGEPAGES_FREE);
if (freepages < nr_pages_needed)
CONFIG("Must have at least %i free hugepages", nr_pages_needed);
}
/**
* get_hugepage_region - Allocate an amount of memory backed by huge pages
*
* len: Size of the region to allocate
* flags: Flags specifying the behaviour of the function
*
* This function allocates a region of memory backed by huge pages. Care should
* be taken when using this function as a drop-in replacement for malloc() as
* memory can be wasted if the length is not hugepage-aligned. This function
* is more relaxed than get_huge_pages() in that it allows fallback to small
* pages when requested.
*/
void *get_hugepage_region(size_t len, ghr_t flags)
{
size_t aligned_len, wastage;
void *buf;
/* Catch an altogether-too easy typo */
if (flags & GHP_MASK)
ERROR("Improper use of GHP_* in get_hugepage_region()\n");
/* Align the len parameter to a hugepage boundary and allocate */
aligned_len = ALIGN(len, gethugepagesize());
buf = get_huge_pages(aligned_len, GHP_DEFAULT);
if (buf == NULL && (flags & GHR_FALLBACK)) {
aligned_len = ALIGN(len, getpagesize());
buf = fallback_base_pages(len, flags);
}
/* Calculate wastage for coloring */
wastage = aligned_len - len;
if (wastage != 0 && !(flags & GHR_COLOR))
DEBUG("get_hugepage_region: Wasted %zd bytes due to alignment\n",
wastage);
/* Only colour if requested */
if (flags & GHR_COLOR)
buf = cachecolor(buf, len, wastage);
return buf;
}
int main(void)
{
PS = getpagesize();
HPS = gethugepagesize();
unsigned count = hardware_corrupted();
if (!hugetlbfs_root(hugetlbfsdir))
err("hugetlbfs_root");
anonymous("anonymous", 0);
check(&count, "anonymous", 1);
anonymous("anonymous mlock", MAP_LOCKED);
check(&count, "anonymous mlock", 1);
disk_backed("disk backed", 0);
check(&count, "disk backed", 1);
disk_backed("disk backed mlock", 0);
check(&count, "disk backed mlock", 1);
shm_hugepage("shm hugepage", 0);
check(&count, "shm hugepage", HPS / PS);
anonymous_hugepage("anonymous hugepage", 0);
check(&count, "anonymous hugepage", HPS / PS);
filebacked_hugepage("file backed hugepage", 0);
check(&count, "file backed hugepage", HPS / PS);
// add more test cases here
return exitcode;
}
/* WARNING: This function relies on the hugetlb pool counters in a way that
* is known to be racy. Due to the expected usage of hugetlbfs test cases, the
* risk of a race is acceptible. This function should NOT be used for real
* applications.
*/
int kernel_has_private_reservations(void)
{
int fd;
long t, f, r, s;
long nt, nf, nr, ns;
long hpage_size = gethugepagesize();
void *map;
/* Read pool counters */
t = get_huge_page_counter(hpage_size, HUGEPAGES_TOTAL);
f = get_huge_page_counter(hpage_size, HUGEPAGES_FREE);
r = get_huge_page_counter(hpage_size, HUGEPAGES_RSVD);
s = get_huge_page_counter(hpage_size, HUGEPAGES_SURP);
fd = hugetlbfs_unlinked_fd();
if (fd < 0) {
ERROR("kernel_has_private_reservations: hugetlbfs_unlinked_fd: "
"%s\n", strerror(errno));
return -1;
}
map = mmap(NULL, hpage_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
if (map == MAP_FAILED) {
ERROR("kernel_has_private_reservations: mmap: %s\n",
strerror(errno));
return -1;
}
/* Recheck the counters */
nt = get_huge_page_counter(hpage_size, HUGEPAGES_TOTAL);
nf = get_huge_page_counter(hpage_size, HUGEPAGES_FREE);
nr = get_huge_page_counter(hpage_size, HUGEPAGES_RSVD);
ns = get_huge_page_counter(hpage_size, HUGEPAGES_SURP);
munmap(map, hpage_size);
close(fd);
/*
* There are only three valid cases:
* 1) If a surplus page was allocated to create a reservation, all
* four pool counters increment
* 2) All counters remain the same except for Hugepages_Rsvd, then
* a reservation was created using an existing pool page.
* 3) All counters remain the same, indicates that no reservation has
* been created
*/
if ((nt == t + 1) && (nf == f + 1) && (ns == s + 1) && (nr == r + 1)) {
return 1;
} else if ((nt == t) && (nf == f) && (ns == s)) {
if (nr == r + 1)
return 1;
else if (nr == r)
return 0;
} else {
ERROR("kernel_has_private_reservations: bad counter state - "
"T:%li F:%li R:%li S:%li -> T:%li F:%li R:%li S:%li\n",
t, f, r, s, nt, nf, nr, ns);
}
return -1;
}
static void HPX_CONSTRUCTOR _system_init(void) {
_hugepage_fd = hugetlbfs_unlinked_fd();
if (_hugepage_fd < 0) {
log_error("could not get huge tlb file descriptor.\n");
}
_hugepage_size = gethugepagesize();
_hugepage_mask = _hugepage_size - 1;
}
int main(int argc, char *argv[])
{
long hpage_size;
test_init(argc, argv);
hpage_size = gethugepagesize();
if (hpage_size == -1)
PASS();
FAIL("Mysteriously found a hugepage size of %ld\n", hpage_size);
}
/**
* check_swap shouldn't change the behavior of any of its
* callers, it only prints a message to the user if something
* is being done that might fail without swap available. i.e.
* resizing a huge page pool
*/
void check_swap()
{
long swap_sz;
long swap_total;
swap_total = read_meminfo(SWAP_TOTAL);
if (swap_total <= 0) {
WARNING("There is no swap space configured, resizing hugepage pool may fail\n");
WARNING("Use --add-temp-swap option to temporarily add swap during the resize\n");
return;
}
swap_sz = read_meminfo(SWAP_FREE);
/* meminfo keeps values in kb, but we use bytes for hpage sizes */
swap_sz *= 1024;
if (swap_sz <= gethugepagesize()) {
WARNING("There is very little swap space free, resizing hugepage pool may fail\n");
WARNING("Use --add-temp-swap option to temporarily add swap during the resize\n");
}
}
/**
* get_huge_pages - Allocate an amount of memory backed by huge pages
* len: Size of the region to allocate, must be hugepage-aligned
* flags: Flags specifying the behaviour of the function
*
* This function allocates a region of memory that is backed by huge pages
* and hugepage-aligned. This is not a suitable drop-in for malloc() but a
* a malloc library could use this function to create a new fixed-size heap
* similar in principal to what morecore does for glibc malloc.
*/
void *get_huge_pages(size_t len, ghp_t flags)
{
void *buf;
int buf_fd = -1;
int mmap_reserve = __hugetlb_opts.no_reserve ? MAP_NORESERVE : 0;
int mmap_hugetlb = 0;
int ret;
/* Catch an altogether-too easy typo */
if (flags & GHR_MASK)
ERROR("Improper use of GHR_* in get_huge_pages()\n");
#ifdef MAP_HUGETLB
mmap_hugetlb = MAP_HUGETLB;
#endif
if (__hugetlb_opts.map_hugetlb &&
gethugepagesize() == kernel_default_hugepage_size()) {
/* Because we can use MAP_HUGETLB, we simply mmap the region */
buf = mmap(NULL, len, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS|mmap_hugetlb|mmap_reserve,
0, 0);
} else {
/* Create a file descriptor for the new region */
buf_fd = hugetlbfs_unlinked_fd();
if (buf_fd < 0) {
WARNING("Couldn't open hugetlbfs file for %zd-sized buffer\n",
len);
return NULL;
}
/* Map the requested region */
buf = mmap(NULL, len, PROT_READ|PROT_WRITE,
MAP_PRIVATE|mmap_reserve, buf_fd, 0);
}
if (buf == MAP_FAILED) {
if (buf_fd >= 0)
close(buf_fd);
WARNING("get_huge_pages: New region mapping failed (flags: 0x%lX): %s\n",
flags, strerror(errno));
return NULL;
}
/* Fault the region to ensure accesses succeed */
ret = hugetlbfs_prefault(buf, len);
if (ret != 0) {
munmap(buf, len);
if (buf_fd >= 0)
close(buf_fd);
WARNING("get_huge_pages: Prefaulting failed (flags: 0x%lX): %s\n",
flags, strerror(ret));
return NULL;
}
/* Close the file so we do not have to track the descriptor */
if (buf_fd >= 0 && close(buf_fd) != 0) {
WARNING("Failed to close new buffer fd: %s\n", strerror(errno));
munmap(buf, len);
return NULL;
}
/* woo, new buffer of shiny */
return buf;
}
static void __free_huge_pages(void *ptr, int aligned)
{
FILE *fd;
char line[MAPS_BUF_SZ];
unsigned long start = 0, end = 0;
unsigned long palign = 0, hpalign = 0;
unsigned long hpalign_end = 0;
/*
* /proc/self/maps is used to determine the length of the original
* allocation. As mappings are based on different files, we can
* assume that maps will not merge. If the hugepages were truly
* anonymous, this assumption would be broken.
*/
fd = fopen("/proc/self/maps", "r");
if (!fd) {
ERROR("Failed to open /proc/self/maps\n");
return;
}
/*
* An unaligned address allocated by get_hugepage_region()
* could be either page or hugepage aligned
*/
if (!aligned) {
palign = ALIGN_DOWN((unsigned long)ptr, getpagesize());
hpalign = ALIGN_DOWN((unsigned long)ptr, gethugepagesize());
}
/* Parse /proc/maps for address ranges line by line */
while (!feof(fd)) {
char *bufptr;
char *saveptr = NULL;
/* Read a line of input */
if (fgets(line, MAPS_BUF_SZ, fd) == NULL)
break;
/* Parse the line to get the start and end of each mapping */
bufptr = strtok_r(line, " ", &saveptr);
bufptr = strtok_r(bufptr, "-", &saveptr);
start = strtoull(bufptr, NULL, 16);
bufptr = strtok_r(NULL, "-", &saveptr);
/* If the correct mapping is found, remove it */
if (start == (unsigned long)ptr) {
end = strtoull(bufptr, NULL, 16);
munmap(ptr, end - start);
break;
}
/* If the passed address is aligned, just move along */
if (aligned)
continue;
/*
* If an address is hpage-aligned, record it but keep looking.
* We might find a page-aligned or exact address later
*/
if (start == hpalign) {
hpalign_end = strtoull(bufptr, NULL, 16);
continue;
}
/* If an address is page-aligned, free it */
if (start == palign) {
end = strtoull(bufptr, NULL, 16);
munmap((void *)start, end - start);
break;
}
}
/*
* If no exact or page-aligned address was found, check for a
* hpage-aligned address. If found, free it, otherwise warn that
* the ptr pointed nowhere
*/
if (end == 0) {
if (hpalign_end == 0)
ERROR("hugepages_free using invalid or double free\n");
else
munmap((void *)hpalign, hpalign_end - hpalign);
}
fclose(fd);
}
void hugetlbfs_setup_morecore(void)
{
char *ep;
unsigned long heapaddr;
if (! __hugetlb_opts.morecore)
return;
if (strcasecmp(__hugetlb_opts.morecore, "no") == 0) {
INFO("HUGETLB_MORECORE=%s, not setting up morecore\n",
__hugetlb_opts.morecore);
return;
}
/*
* Determine the page size that will be used for the heap.
* This can be set explicitly by setting HUGETLB_MORECORE to a valid
* page size string or by setting HUGETLB_DEFAULT_PAGE_SIZE.
*/
if (strncasecmp(__hugetlb_opts.morecore, "y", 1) == 0)
hpage_size = gethugepagesize();
else if (__hugetlb_opts.thp_morecore)
hpage_size = kernel_default_hugepage_size();
else
hpage_size = parse_page_size(__hugetlb_opts.morecore);
if (hpage_size <= 0) {
if (errno == ENOSYS)
WARNING("Hugepages unavailable\n");
else if (errno == EOVERFLOW || errno == ERANGE)
WARNING("Hugepage size too large\n");
else if (errno == EINVAL)
WARNING("Invalid huge page size\n");
else
WARNING("Hugepage size (%s)\n", strerror(errno));
return;
}
/*
* We won't need an fd for the heap mmaps if we are using MAP_HUGETLB
* or we are depending on transparent huge pages
*/
if(__hugetlb_opts.thp_morecore || (__hugetlb_opts.map_hugetlb &&
hpage_size == kernel_default_hugepage_size())) {
heap_fd = -1;
} else {
if (!hugetlbfs_find_path_for_size(hpage_size)) {
WARNING("Hugepage size %li unavailable", hpage_size);
return;
}
heap_fd = hugetlbfs_unlinked_fd_for_size(hpage_size);
if (heap_fd < 0) {
WARNING("Couldn't open hugetlbfs file for morecore\n");
return;
}
}
/*
* THP morecore uses sbrk to allocate more heap space, counting on the
* kernel to back the area with THP. So setting heapbase is
* meaningless if thp_morecore is used.
*/
if (!__hugetlb_opts.thp_morecore && __hugetlb_opts.heapbase) {
heapaddr = strtoul(__hugetlb_opts.heapbase, &ep, 16);
if (*ep != '\0') {
WARNING("Can't parse HUGETLB_MORECORE_HEAPBASE: %s\n",
__hugetlb_opts.heapbase);
return;
}
} else {
heapaddr = (unsigned long)sbrk(0);
if (!__hugetlb_opts.thp_morecore)
heapaddr = hugetlbfs_next_addr(heapaddr);
}
INFO("setup_morecore(): heapaddr = 0x%lx\n", heapaddr);
heaptop = heapbase = (void *)heapaddr;
if (__hugetlb_opts.thp_morecore)
__morecore = &thp_morecore;
else
__morecore = &hugetlbfs_morecore;
/* Set some allocator options more appropriate for hugepages */
if (__hugetlb_opts.shrink_ok)
mallopt(M_TRIM_THRESHOLD, hpage_size / 2);
else
mallopt(M_TRIM_THRESHOLD, -1);
mallopt(M_TOP_PAD, hpage_size / 2);
/* we always want to use our morecore, not ordinary mmap().
* This doesn't appear to prohibit malloc() from falling back
* to mmap() if we run out of hugepages. */
mallopt(M_MMAP_MAX, 0);
}
size_t chpl_comm_ofi_hp_gethugepagesize(void) {
return gethugepagesize();
}