// Map a new HugeTLB page to an appropriate virtual address. // // The HugeTLB page is allocated and mapped using the shm (shared // memory) API. This API makes it easy to remap the page to a new // virtual address after we resolve the physical address. // // There are two other APIs for allocating HugeTLB pages but they do // not work as well: // // mmap() anonymous page with MAP_HUGETLB: cannot remap the address // after allocation because Linux mremap() does not seem to work on // HugeTLB pages. // // mmap() with file-backed MAP_HUGETLB: the file has to be on a // hugetlbfs mounted filesystem and that is not necessarily // available. // // Happily the shm API is happy to remap a HugeTLB page with an // additional call to shmat() and does not depend on hugetlbfs. // // Further reading: // https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt // http://stackoverflow.com/questions/27997934/mremap2-with-hugetlb-to-change-virtual-address void *allocate_huge_page(int size) { int shmid = -1; uint64_t physical_address, virtual_address; void *tmpptr = MAP_FAILED; // initial kernel assigned virtual address void *realptr = MAP_FAILED; // remapped virtual address shmid = shmget(IPC_PRIVATE, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W); tmpptr = shmat(shmid, NULL, 0); if (tmpptr == MAP_FAILED) { goto fail; } if (mlock(tmpptr, size) != 0) { goto fail; } physical_address = virtual_to_physical(tmpptr); if (physical_address == 0) { goto fail; } virtual_address = physical_address | 0x500000000000ULL; realptr = shmat(shmid, (void*)virtual_address, 0); if (realptr == MAP_FAILED) { goto fail; } if (mlock(realptr, size) != 0) { goto fail; } memset(realptr, 0, size); // zero memory to avoid potential surprises shmdt(tmpptr); shmctl(shmid, IPC_RMID, 0); return realptr; fail: if (tmpptr != MAP_FAILED) { shmdt(tmpptr); } if (realptr != MAP_FAILED) { shmdt(realptr); } if (shmid != -1) { shmctl(shmid, IPC_RMID, 0); } return NULL; }
void free_pages(void *page, unsigned int count) { if (count == 0 || count > ram_pages - pages_reserved) { printf("free_pages: sorry, can't free %d pages (only %d RAM pages available)\n", count, ram_pages - pages_reserved); shutdown(); } void *end = page + count*PAGE_SIZE - 1; if (page < (void *)0xC0000000 || end < (void *)0xC0000000) { printf("free_pages: virtual address %p through %p is too low to have come from alloc_pages\n", page, end); shutdown(); } unsigned int paddr = virtual_to_physical(page); if (paddr & 0xfff) { printf("free_pages: virtual address %p is not aligned properly\n", page); shutdown(); } unsigned int ppn = paddr / PAGE_SIZE; if (ppn < ram_start_page || ppn + count > ram_end_page) { printf("free_pages: virtual address %p is not mapped to RAM\n", page); shutdown(); } if (ppn < ram_start_page + pages_reserved) { printf("free_pages: virtual address %p is reserved and should never be freed\n", page); shutdown(); } while (count > 0) { int i = (ppn - ram_start_page - pages_reserved); if (bitmap_get(page_alloc_bitmap, i) == 0) { printf("free_pages: virtual address %p is already free\n", page); shutdown(); } bitmap_set(page_alloc_bitmap, i, 0); count--; page += PAGE_SIZE; ppn++; } }