// Map a new HugeTLB page to an appropriate virtual address.
//
// The HugeTLB page is allocated and mapped using the shm (shared
// memory) API. This API makes it easy to remap the page to a new
// virtual address after we resolve the physical address.
//
// There are two other APIs for allocating HugeTLB pages but they do
// not work as well:
//
// mmap() anonymous page with MAP_HUGETLB: cannot remap the address
// after allocation because Linux mremap() does not seem to work on
// HugeTLB pages.
//
// mmap() with file-backed MAP_HUGETLB: the file has to be on a
// hugetlbfs mounted filesystem and that is not necessarily
// available.
//
// Happily the shm API is happy to remap a HugeTLB page with an
// additional call to shmat() and does not depend on hugetlbfs.
//
// Further reading:
// https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
// http://stackoverflow.com/questions/27997934/mremap2-with-hugetlb-to-change-virtual-address
void *allocate_huge_page(int size)
{
int shmid = -1;
uint64_t physical_address, virtual_address;
void *tmpptr = MAP_FAILED; // initial kernel assigned virtual address
void *realptr = MAP_FAILED; // remapped virtual address
shmid = shmget(IPC_PRIVATE, size, SHM_HUGETLB | IPC_CREAT | SHM_R | SHM_W);
tmpptr = shmat(shmid, NULL, 0);
if (tmpptr == MAP_FAILED) { goto fail; }
if (mlock(tmpptr, size) != 0) { goto fail; }
physical_address = virtual_to_physical(tmpptr);
if (physical_address == 0) { goto fail; }
virtual_address = physical_address | 0x500000000000ULL;
realptr = shmat(shmid, (void*)virtual_address, 0);
if (realptr == MAP_FAILED) { goto fail; }
if (mlock(realptr, size) != 0) { goto fail; }
memset(realptr, 0, size); // zero memory to avoid potential surprises
shmdt(tmpptr);
shmctl(shmid, IPC_RMID, 0);
return realptr;
fail:
if (tmpptr != MAP_FAILED) { shmdt(tmpptr); }
if (realptr != MAP_FAILED) { shmdt(realptr); }
if (shmid != -1) { shmctl(shmid, IPC_RMID, 0); }
return NULL;
}
void free_pages(void *page, unsigned int count)
{
if (count == 0 || count > ram_pages - pages_reserved) {
printf("free_pages: sorry, can't free %d pages (only %d RAM pages available)\n",
count, ram_pages - pages_reserved);
shutdown();
}
void *end = page + count*PAGE_SIZE - 1;
if (page < (void *)0xC0000000 || end < (void *)0xC0000000) {
printf("free_pages: virtual address %p through %p is too low to have come from alloc_pages\n", page, end);
shutdown();
}
unsigned int paddr = virtual_to_physical(page);
if (paddr & 0xfff) {
printf("free_pages: virtual address %p is not aligned properly\n", page);
shutdown();
}
unsigned int ppn = paddr / PAGE_SIZE;
if (ppn < ram_start_page || ppn + count > ram_end_page) {
printf("free_pages: virtual address %p is not mapped to RAM\n", page);
shutdown();
}
if (ppn < ram_start_page + pages_reserved) {
printf("free_pages: virtual address %p is reserved and should never be freed\n", page);
shutdown();
}
while (count > 0) {
int i = (ppn - ram_start_page - pages_reserved);
if (bitmap_get(page_alloc_bitmap, i) == 0) {
printf("free_pages: virtual address %p is already free\n", page);
shutdown();
}
bitmap_set(page_alloc_bitmap, i, 0);
count--;
page += PAGE_SIZE;
ppn++;
}
}
